From 6f35d2c61d6f59e9adb322228c4a095f2d6120e3 Mon Sep 17 00:00:00 2001 From: larryro <371767072@qq.com> Date: Sun, 6 Sep 2026 10:55:56 +0800 Subject: [PATCH 1/8] fix(platform): retire the unread deployment dataStores section Settings > Data residency wrote a deployment-wide `dataStores` section (knowledgePostgres / appPostgres / convexStorage) plus a SOPS secrets sidecar into deployment.yml, and the docs promised the backend reads it at boot and derives its connections. No boot path did: the reader was the Convex/rag entrypoint retired in the Postgres move, and main.ts builds every pool from DATABASE_URL / KNOWLEDGE_DATABASE_URL / OBJECT_STORE_*. A compliance control that reported success while inert. Lane chosen: retire, not wire. Per-organization knowledge and object storage connections already deliver residency (no restart, mixed blob references, backfill), the deployment defaults are environment-driven and owned by the CLI/compose, and convexStorage is a Convex-era five- bucket shape with nothing behind it. - deployment schema: drop dataStores, pgConnectionSchema, convexStorageSchema, DEPLOYMENT_SECRET_KEYS and the secrets schema; keep version + sandboxRuntime (read by the sandbox spawner); fix the header that still claimed boot consumption - pgConnectionSchema moves to schemas/knowledge.ts, its only consumer (the per-org connection file), tests moved with it - parseDeploymentConfig drops a leftover dataStores section with a warning so an operator's older file keeps parsing; the next save rewrites it without the section; any other unknown key still fails closed - deployment service/routes: remove the secrets and connection-test doors and the secret masks from the view; delete core/deployment/ secret_io.ts - integration check: replace the secrets/probe assertions with the legacy-file tolerance + strict-save refusal Finding: lib-shared-schemas-1. --- .../core/deployment/file_utils.test.ts | 63 ++++ .../backend/core/deployment/file_utils.ts | 100 +++--- .../backend/core/deployment/secret_io.ts | 142 --------- .../backend/domains/deployment/routes.ts | 46 --- .../backend/domains/deployment/service.ts | 296 +----------------- .../platform/backend/integration-check.ts | 81 ++--- .../lib/shared/schemas/deployment.test.ts | 235 +------------- .../platform/lib/shared/schemas/deployment.ts | 166 ++-------- .../lib/shared/schemas/knowledge.test.ts | 65 ++++ .../platform/lib/shared/schemas/knowledge.ts | 35 ++- .../lib/shared/schemas/object_storage.ts | 12 +- 11 files changed, 294 insertions(+), 947 deletions(-) create mode 100644 services/platform/backend/core/deployment/file_utils.test.ts delete mode 100644 services/platform/backend/core/deployment/secret_io.ts create mode 100644 services/platform/lib/shared/schemas/knowledge.test.ts diff --git a/services/platform/backend/core/deployment/file_utils.test.ts b/services/platform/backend/core/deployment/file_utils.test.ts new file mode 100644 index 0000000000..90759db993 --- /dev/null +++ b/services/platform/backend/core/deployment/file_utils.test.ts @@ -0,0 +1,63 @@ +import { describe, expect, it, vi } from 'vitest'; + +import { parseDeploymentConfig, serializeDeploymentConfig } from './file_utils'; + +describe('parseDeploymentConfig', () => { + it('reads the current YAML form and the retired JSON form alike', () => { + expect( + parseDeploymentConfig('version: 1\nsandboxRuntime:\n tier: kata\n'), + ).toEqual({ + version: 1, + sandboxRuntime: { tier: 'kata' }, + }); + expect( + parseDeploymentConfig( + JSON.stringify({ version: 1, sandboxRuntime: { tier: 'runc' } }), + ), + ).toEqual({ version: 1, sandboxRuntime: { tier: 'runc' } }); + }); + + it('drops the retired dataStores section with a warning instead of failing the read', () => { + const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined); + try { + const parsed = parseDeploymentConfig( + JSON.stringify({ + version: 1, + dataStores: { + knowledgePostgres: { + host: 'pg.acme.internal', + database: 'k', + user: 'u', + }, + convexStorage: { mode: 'local' }, + }, + sandboxRuntime: { tier: 'sysbox', dockerInContainer: true }, + }), + ); + expect(parsed).toEqual({ + version: 1, + sandboxRuntime: { tier: 'sysbox', dockerInContainer: true }, + }); + expect('dataStores' in parsed).toBe(false); + expect(warn).toHaveBeenCalledTimes(1); + expect(warn.mock.calls[0]?.[0]).toContain('retired "dataStores" section'); + // The next save writes the file without the section. + expect(serializeDeploymentConfig(parsed)).not.toContain('dataStores'); + } finally { + warn.mockRestore(); + } + }); + + it('still fails closed on any other unknown key', () => { + expect(() => parseDeploymentConfig('version: 1\nbogus: true\n')).toThrow( + /Invalid deployment config/, + ); + }); + + it('fails closed on a wrong version and on unparseable content', () => { + expect(() => parseDeploymentConfig('version: 2\n')).toThrow( + /Invalid deployment config/, + ); + expect(() => parseDeploymentConfig('{ not yaml')).toThrow(); + }); +}); diff --git a/services/platform/backend/core/deployment/file_utils.ts b/services/platform/backend/core/deployment/file_utils.ts index ba82b171e7..1a075ff820 100644 --- a/services/platform/backend/core/deployment/file_utils.ts +++ b/services/platform/backend/core/deployment/file_utils.ts @@ -4,47 +4,30 @@ * Deployment-config file I/O helpers (deployment-SCOPED — no org slug). * * The single deployment config lives at the config ROOT as - * `/deployment.yml` (+ SOPS sidecar `deployment.secrets.json`; - * the retired `deployment.json` stays readable until the next save converts - * it). - * Because the path is one segment (no `/` prefix) it is intentionally - * ignored by the per-org config-watcher — this config is consumed by the - * rag/convex/platform entrypoints AT BOOT, not hot-reloaded. + * `/deployment.yml` (the retired `deployment.json` stays readable + * until the next save converts it). Because the path is one segment (no + * `/` prefix) it is intentionally ignored by the per-org + * config-watcher — this config is consumed by the sandbox spawner AT BOOT, + * not hot-reloaded. */ import { parseYamlOrThrow, stringifyYaml, } from '../../../lib/shared/config/yaml'; -import type { - DeploymentConfig, - DeploymentSecrets, -} from '../../../lib/shared/schemas/deployment'; +import type { DeploymentConfig } from '../../../lib/shared/schemas/deployment'; import { + RETIRED_DEPLOYMENT_SECTIONS, deploymentConfigSchema, - deploymentSecretsSchema, } from '../../../lib/shared/schemas/deployment'; import { getConfigRoot, safeJoinWithinDir, sha256 } from '../lib/file_io'; export { sha256 }; -export type { DeploymentConfig, DeploymentSecrets }; +export type { DeploymentConfig }; /** Deployment config is tiny; cap well below the per-org file caps. */ export const MAX_FILE_SIZE_BYTES = 64 * 1024; -export type DeploymentReadResult = - | { ok: true; config: DeploymentConfig; hash: string } - | { - ok: false; - error: - | 'not_found' - | 'corrupted' - | 'too_large' - | 'symlink' - | 'inaccessible'; - message: string; - }; - export function resolveDeploymentConfigPath(): string { return safeJoinWithinDir(getConfigRoot('deployment'), 'deployment.yml'); } @@ -54,51 +37,42 @@ export function resolveLegacyDeploymentConfigPath(): string { return safeJoinWithinDir(getConfigRoot('deployment'), 'deployment.json'); } -export function resolveDeploymentSecretsPath(): string { - return safeJoinWithinDir( - getConfigRoot('deployment'), - 'deployment.secrets.json', - ); -} - export function serializeDeploymentConfig(config: DeploymentConfig): string { return stringifyYaml(config); } -export function parseDeploymentConfig(content: string): DeploymentConfig { - // YAML is a superset of JSON, so one parser reads both the current .yml - // form and the retired .json fallback. - const parsed = parseYamlOrThrow(content); - const result = deploymentConfigSchema.safeParse(parsed); - if (!result.success) { - throw new Error(`Invalid deployment config: ${result.error.message}`); - } - return result.data; +/** + * Parse a deployment config file. YAML is a superset of JSON, so one parser + * reads both the current .yml form and the retired .json fallback. + * + * A retired section (`dataStores`, saved by the Convex-era Data residency + * page and read by nothing since) is DROPPED with a warning rather than + * failing the read: an operator's older file must keep parsing, and the next + * save rewrites it without the section. Any other unknown key still fails + * closed — the schema is strict. + */ +function isPlainRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); } -export function parseDeploymentSecrets( - data: Record, -): DeploymentSecrets { - const result = deploymentSecretsSchema.safeParse(data); +export function parseDeploymentConfig(content: string): DeploymentConfig { + const parsed: unknown = parseYamlOrThrow(content); + let candidate: unknown = parsed; + if (isPlainRecord(parsed)) { + const stripped: Record = { ...parsed }; + for (const section of RETIRED_DEPLOYMENT_SECTIONS) { + if (section in stripped) { + console.warn( + `[deployment] ignoring the retired "${section}" section of the deployment config — where data lives is set by the environment (DATABASE_URL, KNOWLEDGE_DATABASE_URL, OBJECT_STORE_*) and per organization under Settings > Data residency; the section is dropped on the next save.`, + ); + delete stripped[section]; + } + } + candidate = stripped; + } + const result = deploymentConfigSchema.safeParse(candidate); if (!result.success) { - throw new Error(`Invalid deployment secrets: ${result.error.message}`); + throw new Error(`Invalid deployment config: ${result.error.message}`); } return result.data; } - -/** Mask an IDENTIFIER for "configured?" display: first 6 + last 4. */ -export function maskDeploymentSecret(value: string): string { - if (value.length <= 10) return '••••••••••'; - return `${value.slice(0, 6)} … ${value.slice(-4)}`; -} - -/** - * Keys whose value is an IDENTIFIER (not a credential) and may show a short - * first6/last4 preview. Everything else (passwords, secretAccessKey) returns - * presence-only — a partial preview of a lower-entropy DB password would leak - * usable material to any read-only instance-admin (the read path is NOT gated - * by the editor allowlist). - */ -export const PREVIEWABLE_DEPLOYMENT_SECRET_KEYS = new Set([ - 'dataStores.convexStorage.accessKeyId', -]); diff --git a/services/platform/backend/core/deployment/secret_io.ts b/services/platform/backend/core/deployment/secret_io.ts deleted file mode 100644 index 4c5131bd44..0000000000 --- a/services/platform/backend/core/deployment/secret_io.ts +++ /dev/null @@ -1,142 +0,0 @@ -'use node'; - -/** - * Deployment-secrets I/O helper. - * - * Reads the existing SOPS-encrypted `deployment.secrets.json` (if any), - * merges incoming values over it, and hands back plaintext ready to - * persist. Secrets are a FLAT, dotted-key map - * (`dataStores.knowledgePostgres.password`, …) validated against the - * allowlist in `deploymentSecretsSchema`, so a new config section's - * secrets merge in independently — no deep-merge needed. - * - * The refusal-to-overwrite-an-unreadable-file guard, and the - * `UndecryptableExistingSecretError` / `ForceOverwriteReason` types that - * carry it, mirror the same primitive used for provider secrets so the - * Convex action + UI layers can handle both the same way. - */ - -import type { - DeploymentSecretKey, - DeploymentSecrets, -} from '../../../lib/shared/schemas/deployment'; -import { EncryptedFileWithoutKeyError, decryptSecretsFile } from '../lib/sops'; -import { parseDeploymentSecrets } from './file_utils'; - -/** - * Thrown when an existing secrets file can't be read — decrypt failure, - * JSON parse failure, or a shape that fails `deploymentSecretsSchema` — - * and the caller didn't pass `force: true`. The Convex action layer turns - * this into a `AppError` with `data.kind = 'undecryptable_existing'` so - * the UI can offer a confirm dialog and retry with `force: true`. - * - * `reason` is the inner cause's message, unwrapped. It's what the UI's - * confirm dialog interpolates into its translated copy, so it must stay - * free of this wrapper's own path + remediation text. - */ -export class UndecryptableExistingSecretError extends Error { - readonly path: string; - readonly reason: string; - constructor(path: string, cause: unknown) { - const reason = cause instanceof Error ? cause.message : String(cause); - super( - `Existing secrets file ${path} could not be read (${reason}). ` + - 'Save again with the "overwrite anyway" option to discard it, or remove the file manually first.', - ); - // Object.assign bolts `cause` onto the Error: convex/tsconfig.json's - // "lib" predates the ES2022 two-argument Error constructor overload, - // even though the runtime itself supports it. - Object.assign(this, { cause }); - this.name = 'UndecryptableExistingSecretError'; - this.path = path; - this.reason = reason; - } -} - -/** Why a force-overwrite happened; populated only when `forced` is true. */ -export type ForceOverwriteReason = - | 'encrypted_no_key' - | 'undecryptable_existing'; - -export interface PreparedDeploymentSecrets { - /** Plaintext JSON ready to encrypt or write directly (trailing newline). */ - plaintext: string; - /** True when an existing readable file was successfully merged. */ - existed: boolean; - /** True when force-overwrite skipped an unreadable existing file. */ - forced: boolean; - /** Why the force-overwrite happened; populated only when `forced` is true. */ - forceReason: ForceOverwriteReason | null; -} - -/** - * Read the existing deployment-secrets file (if any), merge `incoming` over - * it, and return the plaintext to write. Incoming values override existing; - * an explicit empty-string value DELETES that key (so the UI can clear a - * secret). The merged result is re-validated against the secret-key - * allowlist. - * - * Refuses to overwrite an existing-but-undecryptable file unless - * `options.force` is true (the ciphertext may be the only recoverable copy). - * - * @throws {EncryptedFileWithoutKeyError} SOPS-encrypted but no key, no force. - * @throws {UndecryptableExistingSecretError} exists but decrypt/parse fails, no force. - */ -export async function prepareMergedDeploymentSecrets( - secretsPath: string, - incoming: Partial>, - options: { force?: boolean } = {}, -): Promise { - let existing: DeploymentSecrets | null = null; - let existed = false; - let forced = false; - let forceReason: ForceOverwriteReason | null = null; - - try { - const raw = await decryptSecretsFile(secretsPath); - existing = parseDeploymentSecrets(raw); - existed = true; - } catch (err) { - if ( - err instanceof Error && - (err as NodeJS.ErrnoException).code === 'ENOENT' - ) { - // No file yet — this is a fresh write; existing stays null. - } else if (options.force) { - console.warn( - `[deployment/secret_io] force-overwriting ${secretsPath}: ${ - err instanceof Error ? err.message : String(err) - }`, - ); - forced = true; - forceReason = - err instanceof EncryptedFileWithoutKeyError - ? 'encrypted_no_key' - : 'undecryptable_existing'; - } else if (err instanceof EncryptedFileWithoutKeyError) { - throw err; - } else { - throw new UndecryptableExistingSecretError(secretsPath, err); - } - } - - const merged: Record = { ...existing }; - for (const [key, value] of Object.entries(incoming)) { - if (value === undefined) continue; // Not provided this save — keep existing. - if (value === '') { - delete merged[key]; // Explicit clear. - } else { - merged[key] = value; - } - } - - // Re-validate the merged set against the allowlist before persisting. - const validated = parseDeploymentSecrets(merged); - - return { - plaintext: JSON.stringify(validated, null, 2) + '\n', - existed, - forced, - forceReason, - }; -} diff --git a/services/platform/backend/domains/deployment/routes.ts b/services/platform/backend/domains/deployment/routes.ts index b81bf09590..091c12e430 100644 --- a/services/platform/backend/domains/deployment/routes.ts +++ b/services/platform/backend/domains/deployment/routes.ts @@ -9,8 +9,6 @@ import { requireInstanceAdmin, readDeploymentConfigView, saveDeploymentConfig, - saveDeploymentSecret, - testDeploymentConnection, } from './service.ts'; /** @@ -77,49 +75,5 @@ export function createDeploymentRoutes(deps: { } }); - app.post('/secrets', async (c) => { - const body = z - .object({ - secrets: z.record(z.string().max(200), z.string().max(10_000)), - force: z.boolean().optional(), - }) - .safeParse(await c.req.json()); - if (!body.success) return c.json({ error: 'invalid body' }, 400); - try { - const auth = await requireInstanceAdmin(deps.sql, caller(c), { - write: true, - }); - await saveDeploymentSecret(deps.sql, auth, body.data); - return c.json({ ok: true }); - } catch (error) { - return handleError(c, error); - } - }); - - app.post('/test', async (c) => { - const body = z - .object({ - target: z.enum(['knowledgePostgres', 'appPostgres', 'convexStorage']), - config: z.unknown(), - password: z.string().max(2_000).optional(), - }) - .safeParse(await c.req.json()); - if (!body.success) return c.json({ error: 'invalid body' }, 400); - try { - await requireInstanceAdmin(deps.sql, caller(c), { write: true }); - return c.json( - await testDeploymentConnection({ - target: body.data.target, - config: body.data.config, - ...(body.data.password !== undefined - ? { password: body.data.password } - : {}), - }), - ); - } catch (error) { - return handleError(c, error); - } - }); - return app; } diff --git a/services/platform/backend/domains/deployment/service.ts b/services/platform/backend/domains/deployment/service.ts index 1eebf21665..18169b4f69 100644 --- a/services/platform/backend/domains/deployment/service.ts +++ b/services/platform/backend/domains/deployment/service.ts @@ -2,62 +2,42 @@ import { unlink } from 'node:fs/promises'; import type { Sql } from 'postgres'; -import { checkProviderHostPolicy } from '../../../lib/net/host-policy.ts'; -import { SafeFetchError, safeFetch } from '../../../lib/net/safe-fetch.ts'; -import type { - DeploymentConfig, - DeploymentSecretKey, -} from '../../../lib/shared/schemas/deployment.ts'; +import type { DeploymentConfig } from '../../../lib/shared/schemas/deployment.ts'; import { DEPLOYMENT_CONFIG_VERSION, - DEPLOYMENT_SECRET_KEYS, - convexStorageSchema, deploymentConfigSchema, - pgConnectionSchema, } from '../../../lib/shared/schemas/deployment.ts'; import { decideInstanceAdmin } from '../../core/deployment/auth_policy.ts'; import { isDeploymentEditor } from '../../core/deployment/editors.ts'; import { MAX_FILE_SIZE_BYTES, - PREVIEWABLE_DEPLOYMENT_SECRET_KEYS, - maskDeploymentSecret, parseDeploymentConfig, - parseDeploymentSecrets, resolveDeploymentConfigPath, - resolveDeploymentSecretsPath, resolveLegacyDeploymentConfigPath, serializeDeploymentConfig, } from '../../core/deployment/file_utils.ts'; -import { - UndecryptableExistingSecretError, - prepareMergedDeploymentSecrets, -} from '../../core/deployment/secret_io.ts'; -import { testDatastoreConnection } from '../../core/deployment/test_datastore_connection.ts'; import { atomicWrite, - atomicWriteSecret, errnoCode, readJsonFile, sha256, } from '../../core/lib/file_io.ts'; -import { - EncryptedFileWithoutKeyError, - decryptSecretsFile, - encryptJsonWithSops, - hasSopsKey, - invalidateSecretsCache, -} from '../../core/lib/sops.ts'; import { sanitizeError } from '../../core/lib/utils/sanitize_secrets.ts'; import { createAuditLog } from '../audit_logs/service.ts'; /** - * INSTANCE-level deployment settings — the pg port of the 0.4 - * `deployment/file_actions` handlers, re-orchestrated over the SAME pure - * helpers (file_utils/secret_io/sops/test_datastore_connection reused - * whole). Not org-scoped: one `/deployment.yml` (+ SOPS - * secrets sidecar) per deployment. Reads need any org-settings admin; - * writes additionally require the caller's email in the - * `TALE_DEPLOYMENT_CONFIG_ADMINS` allowlist (the 0.4 editor gate). + * INSTANCE-level deployment settings — the one `/deployment.yml` + * per deployment (today: the `sandboxRuntime` section the sandbox spawner + * reads at boot). Not org-scoped. Reads need any org-settings admin; writes + * additionally require the caller's email in the + * `TALE_DEPLOYMENT_CONFIG_ADMINS` allowlist (the editor gate). + * + * Where data lives is NOT configured here: the deployment-default stores are + * environment-driven and per-organization residency is its own config lane + * (`domains/knowledge`, `domains/object_storage`). The Convex-era + * `dataStores` section, its secrets sidecar and its connection probe were + * saved by the Data residency page but read by no boot path; they are gone + * (`parseDeploymentConfig` drops a leftover section on read). */ export class DeploymentError extends Error { readonly code: string; @@ -117,29 +97,6 @@ export async function requireInstanceAdmin( }; } -function isErrnoCode(err: unknown, code: string): boolean { - return err instanceof Error && 'code' in err && err.code === code; -} - -/** SSRF-gate every host/endpoint a config persists. */ -function gateHosts(config: DeploymentConfig): void { - const ds = config.dataStores; - if (!ds) return; - if (ds.knowledgePostgres) { - checkProviderHostPolicy( - `http://${ds.knowledgePostgres.host}:${ds.knowledgePostgres.port}`, - ); - } - if (ds.appPostgres) { - checkProviderHostPolicy( - `http://${ds.appPostgres.host}:${ds.appPostgres.port}`, - ); - } - if (ds.convexStorage?.mode === 's3' && ds.convexStorage.endpoint) { - checkProviderHostPolicy(ds.convexStorage.endpoint); - } -} - async function auditBestEffort( sql: Sql, auth: InstanceAdminAuth, @@ -188,8 +145,6 @@ async function readDeploymentConfigFile() { export interface DeploymentConfigView { config: DeploymentConfig; hash: string | null; - secrets: Record; - secretsError?: 'encrypted_no_key' | 'unreadable'; canEdit: boolean; email: string; } @@ -210,38 +165,9 @@ export async function readDeploymentConfigView( throw new DeploymentError('DEPLOYMENT_CONFIG_UNREADABLE', res.message, 500); } - const secrets: Record = {}; - let secretsError: 'encrypted_no_key' | 'unreadable' | undefined; - try { - const raw = await decryptSecretsFile(resolveDeploymentSecretsPath()); - const parsed = parseDeploymentSecrets(raw); - for (const key of DEPLOYMENT_SECRET_KEYS) { - const val = parsed[key]; - if (!val) { - secrets[key] = { present: false }; - } else if (PREVIEWABLE_DEPLOYMENT_SECRET_KEYS.has(key)) { - secrets[key] = { present: true, masked: maskDeploymentSecret(val) }; - } else { - secrets[key] = { present: true }; - } - } - } catch (err) { - if (!isErrnoCode(err, 'ENOENT')) { - secretsError = - err instanceof EncryptedFileWithoutKeyError - ? 'encrypted_no_key' - : 'unreadable'; - } - for (const key of DEPLOYMENT_SECRET_KEYS) { - secrets[key] ??= { present: false }; - } - } - return { config, hash, - secrets, - ...(secretsError !== undefined ? { secretsError } : {}), canEdit: isDeploymentEditor(auth.email), email: auth.email, }; @@ -267,7 +193,6 @@ export async function saveDeploymentConfig( ); } const config = parsed.data; - gateHosts(config); const configPath = resolveDeploymentConfigPath(); if (args.expectedHash !== undefined) { @@ -297,198 +222,3 @@ export async function saveDeploymentConfig( await auditBestEffort(sql, auth, 'deployment_config_saved'); return { hash: sha256(content) }; } - -// One in-process advisory lock — the secrets file is read-modify-write. -let secretWriteLock: Promise = Promise.resolve(); - -export async function saveDeploymentSecret( - sql: Sql, - auth: InstanceAdminAuth, - args: { secrets: Record; force?: boolean }, -): Promise { - const allowed = new Set(DEPLOYMENT_SECRET_KEYS); - for (const key of Object.keys(args.secrets)) { - if (!allowed.has(key)) { - throw new DeploymentError( - 'INVALID_DEPLOYMENT_SECRET_KEY', - `Unknown deployment secret key: ${key}`, - ); - } - } - - const secretsPath = resolveDeploymentSecretsPath(); - const prev = secretWriteLock; - let release!: () => void; - const next = new Promise((resolve) => { - release = resolve; - }); - secretWriteLock = prev.then(() => next); - await prev; - - try { - let prepared: Awaited>; - try { - prepared = await prepareMergedDeploymentSecrets( - secretsPath, - args.secrets, - { force: args.force }, - ); - } catch (err) { - if (err instanceof EncryptedFileWithoutKeyError) { - throw new DeploymentError( - 'DEPLOYMENT_SECRET_REFUSED_OVERWRITE', - 'The existing secrets file is encrypted and no key is available.', - 409, - { kind: 'encrypted_no_key', path: secretsPath }, - ); - } - if (err instanceof UndecryptableExistingSecretError) { - throw new DeploymentError( - 'DEPLOYMENT_SECRET_REFUSED_OVERWRITE', - 'The existing secrets file cannot be decrypted.', - 409, - { - kind: 'undecryptable_existing', - path: secretsPath, - reason: err.reason, - }, - ); - } - throw err; - } - - const content = hasSopsKey() - ? await encryptJsonWithSops(prepared.plaintext) - : prepared.plaintext; - await atomicWriteSecret(secretsPath, content); - invalidateSecretsCache(secretsPath); - await auditBestEffort( - sql, - auth, - prepared.forced - ? 'force_overwrite_deployment_secret' - : 'deployment_secret_saved', - ); - } finally { - release(); - } -} - -async function readStoredSecret( - key: DeploymentSecretKey, -): Promise { - try { - const raw = await decryptSecretsFile(resolveDeploymentSecretsPath()); - return parseDeploymentSecrets(raw)[key]; - } catch (err) { - if (!isErrnoCode(err, 'ENOENT')) { - console.warn( - `[deployment] could not read stored secret ${key}`, - sanitizeError(err), - ); - } - return undefined; - } -} - -export type DeploymentTestTarget = - | 'knowledgePostgres' - | 'appPostgres' - | 'convexStorage'; - -/** The 0.4 pre-save connection probe, re-orchestrated (same semantics). */ -export async function testDeploymentConnection(args: { - target: DeploymentTestTarget; - config: unknown; - password?: string; -}): Promise> { - if (args.target === 'convexStorage') { - const parsed = convexStorageSchema.safeParse(args.config); - if (!parsed.success) { - return { ok: false, error: 'Invalid storage config' }; - } - const storage = parsed.data; - if (storage.mode === 'local') { - return { ok: true, hint: 'Local storage needs no connection test.' }; - } - const base = storage.endpoint - ? storage.endpoint.replace(/\/+$/, '') - : `https://s3.${storage.region}.amazonaws.com`; - const url = - storage.endpoint || storage.forcePathStyle - ? `${base}/${encodeURIComponent(storage.buckets.files)}` - : `https://${encodeURIComponent(storage.buckets.files)}.s3.${storage.region}.amazonaws.com`; - checkProviderHostPolicy(url); - const t0 = Date.now(); - try { - const res = await safeFetch(url, { method: 'HEAD', timeoutMs: 8_000 }); - return { - ok: true, - latencyMs: Date.now() - t0, - httpStatus: res.status, - hint: 'Reachability + TLS only. Credentials, bucket access, and the other buckets are verified when the deployment restarts.', - }; - } catch (err) { - return { - ok: false, - error: - err instanceof SafeFetchError || err instanceof Error - ? err.message - : String(err), - }; - } - } - - const parsed = pgConnectionSchema.safeParse(args.config); - if (!parsed.success) { - return { ok: false, error: 'Invalid Postgres connection config' }; - } - const pg = parsed.data; - checkProviderHostPolicy(`http://${pg.host}:${pg.port}`); - const testSslmode = args.target === 'appPostgres' ? 'prefer' : pg.sslmode; - const password = - args.password || - (await readStoredSecret(`dataStores.${args.target}.password`)); - - let data: Awaited>; - try { - data = await testDatastoreConnection({ - host: pg.host, - port: pg.port, - database: pg.database, - user: pg.user, - password, - sslmode: testSslmode, - }); - } catch (err) { - return { - ok: false, - error: `Could not run the datastore connection test: ${ - err instanceof Error ? err.message : String(err) - }`, - }; - } - - let hint: string | undefined; - if (data.ok && data.vector_available === false) { - hint = - 'The `vector` (pgvector) extension is not available on this database — vector search will not work. Install it before switching.'; - } else if ( - args.target === 'knowledgePostgres' && - data.ok && - data.paradedb_available === false - ) { - hint = - 'ParadeDB (`pg_search`) is not available — full-text/BM25 hybrid search will degrade to vector-only. Install ParadeDB for full search quality.'; - } - - return { - ok: data.ok, - latencyMs: data.latency_ms ?? undefined, - version: data.version ?? undefined, - vectorAvailable: data.vector_available ?? undefined, - paradedbAvailable: data.paradedb_available ?? undefined, - error: data.error ?? undefined, - ...(hint !== undefined ? { hint } : {}), - }; -} diff --git a/services/platform/backend/integration-check.ts b/services/platform/backend/integration-check.ts index 99ed82771b..8300f315fe 100644 --- a/services/platform/backend/integration-check.ts +++ b/services/platform/backend/integration-check.ts @@ -35699,13 +35699,12 @@ async function checkDataResidency( }); process.env.TALE_ALLOW_PRIVATE_PROVIDER_HOSTS = '1'; - // --- Deployment: read view, editor gate, hash OCC, secrets ------------ + // --- Deployment: read view, editor gate, hash OCC, retired section ---- const readFresh = z .object({ config: z.object({ version: z.number() }).loose(), hash: z.null(), canEdit: z.boolean(), - secrets: z.record(z.string(), z.object({ present: z.boolean() }).loose()), }) .loose() .safeParse(await (await get('/api/app/deployment/config')).json()); @@ -35729,40 +35728,46 @@ async function checkDataResidency( config: { version: 1 }, expectedHash: 'not-the-hash', }); - const secretSaved = z.object({ ok: z.boolean() }).safeParse( - await ( - await post('/api/app/deployment/secrets', { - secrets: { - 'dataStores.convexStorage.accessKeyId': 'AKITEST1234567890', - }, - }) - ).json(), - ); - const badSecret = await post('/api/app/deployment/secrets', { - secrets: { 'not.a.known.key': 'x' }, - }); - const readSecrets = z + // The Convex-era `dataStores` section (saved by the old Data residency + // page, read by no boot path) is dropped on read — an operator's older + // file keeps parsing — and the save refuses to write it back. Only the + // save is allowed to touch the file, so the read must not rewrite it. + const { resolveDeploymentConfigPath } = + await import('./core/deployment/file_utils.ts'); + const deploymentConfigPath = resolveDeploymentConfigPath(); + const legacyFile = + 'version: 1\n' + + 'dataStores:\n' + + ' knowledgePostgres: { host: pg.acme.internal, database: k, user: u }\n' + + ' convexStorage: { mode: local }\n' + + 'sandboxRuntime: { tier: sysbox }\n'; + await writeFile(deploymentConfigPath, legacyFile); + const legacyRead = z .object({ - secrets: z.record( - z.string(), - z.object({ present: z.boolean(), masked: z.string().optional() }), - ), + config: z + .object({ + version: z.number(), + sandboxRuntime: z.object({ tier: z.string() }).loose(), + }) + .strict(), + hash: z.string(), }) .loose() .safeParse(await (await get('/api/app/deployment/config')).json()); - const maskedPreview = readSecrets.success - ? readSecrets.data.secrets['dataStores.convexStorage.accessKeyId'] - : undefined; - const localProbe = z.object({ ok: z.boolean(), hint: z.string() }).safeParse( - await ( - await post('/api/app/deployment/test', { - target: 'convexStorage', - config: { mode: 'local' }, + const legacyUntouched = + (await readFile(deploymentConfigPath, 'utf8')) === legacyFile; + const rejectedLegacySave = await post('/api/app/deployment/config', { + config: { version: 1, dataStores: { convexStorage: { mode: 'local' } } }, + }); + const resaved = legacyRead.success + ? await post('/api/app/deployment/config', { + config: legacyRead.data.config, + expectedHash: legacyRead.data.hash, }) - ).json(), - ); + : null; + const rewritten = await readFile(deploymentConfigPath, 'utf8'); record( - 'data residency: deployment config view, editor gate, OCC, secrets', + 'data residency: deployment config view, editor gate, OCC, retired dataStores dropped', readFresh.success && !readFresh.data.canEdit && writeDenied.status === 403 && @@ -35771,14 +35776,14 @@ async function checkDataResidency( readBack.data.hash === saved.data.hash && readBack.data.canEdit && staleSave.status === 409 && - secretSaved.success && - badSecret.status === 400 && - maskedPreview !== undefined && - maskedPreview.present && - maskedPreview.masked?.startsWith('AKITES') === true && - localProbe.success && - localProbe.data.ok, - `fresh=${readFresh.success ? `edit=${readFresh.data.canEdit}` : 'ERR'}, denied=${writeDenied.status} (want 403), saved=${saved.success}, hashMatch=${readBack.success && saved.success ? readBack.data.hash === saved.data.hash : '?'}, stale=${staleSave.status} (want 409), secret=${secretSaved.success}/${badSecret.status}/${maskedPreview?.masked?.slice(0, 6) ?? '?'}, probe=${localProbe.success ? localProbe.data.ok : 'ERR'}`, + legacyRead.success && + legacyRead.data.config.sandboxRuntime.tier === 'sysbox' && + legacyUntouched && + rejectedLegacySave.status === 400 && + resaved?.status === 200 && + !rewritten.includes('dataStores') && + rewritten.includes('sysbox'), + `fresh=${readFresh.success ? `edit=${readFresh.data.canEdit}` : 'ERR'}, denied=${writeDenied.status} (want 403), saved=${saved.success}, hashMatch=${readBack.success && saved.success ? readBack.data.hash === saved.data.hash : 'n/a'}, stale=${staleSave.status} (want 409), legacyRead=${legacyRead.success ? `tier=${legacyRead.data.config.sandboxRuntime.tier}` : JSON.stringify(legacyRead.error?.issues)}, legacyUntouched=${legacyUntouched}, legacySaveRejected=${rejectedLegacySave.status} (want 400), resaved=${resaved?.status}, rewrittenDroppedSection=${!rewritten.includes('dataStores')}`, ); // --- Object storage: connection files + probe + blob backfill --------- diff --git a/services/platform/lib/shared/schemas/deployment.test.ts b/services/platform/lib/shared/schemas/deployment.test.ts index 24656dadad..739d5d0de4 100644 --- a/services/platform/lib/shared/schemas/deployment.test.ts +++ b/services/platform/lib/shared/schemas/deployment.test.ts @@ -2,60 +2,22 @@ import { describe, expect, it } from 'vitest'; import { DEPLOYMENT_CONFIG_VERSION, - DEPLOYMENT_SECRET_KEYS, - convexStorageSchema, + RETIRED_DEPLOYMENT_SECTIONS, deploymentConfigSchema, - deploymentSecretsSchema, - pgConnectionSchema, } from './deployment'; describe('deploymentConfigSchema', () => { - it('accepts a minimal config (version only — all stores fall back to .env)', () => { + it('accepts a minimal config (version only — every section falls back to .env)', () => { const r = deploymentConfigSchema.safeParse({ version: DEPLOYMENT_CONFIG_VERSION, }); expect(r.success).toBe(true); }); - it('accepts a full external config (knowledge PG + S3 storage + app PG)', () => { + it('accepts the sandboxRuntime section', () => { const r = deploymentConfigSchema.safeParse({ version: 1, - dataStores: { - knowledgePostgres: { - host: 'pg.acme.internal', - database: 'tale_knowledge', - user: 'tale_rw', - }, - convexStorage: { - mode: 's3', - region: 'eu-central-1', - buckets: { - files: 'tale-files', - exports: 'tale-exports', - snapshotImports: 'tale-snap-imports', - modules: 'tale-modules', - search: 'tale-search', - }, - }, - appPostgres: { - host: 'pg.acme.internal', - database: 'tale', - user: 'tale_rw', - }, - }, - }); - expect(r.success).toBe(true); - if (r.success) { - // defaults applied - expect(r.data.dataStores?.knowledgePostgres?.port).toBe(5432); - expect(r.data.dataStores?.knowledgePostgres?.sslmode).toBe('require'); - } - }); - - it('accepts local storage mode with no extra fields', () => { - const r = deploymentConfigSchema.safeParse({ - version: 1, - dataStores: { convexStorage: { mode: 'local' } }, + sandboxRuntime: { tier: 'sysbox', dockerInContainer: true }, }); expect(r.success).toBe(true); }); @@ -68,196 +30,27 @@ describe('deploymentConfigSchema', () => { expect(r.success).toBe(false); }); - it('rejects an unknown dataStores section (strict — protects against typos)', () => { + it('rejects the retired dataStores section — nothing reads it, so nothing may save it', () => { + expect(RETIRED_DEPLOYMENT_SECTIONS).toContain('dataStores'); const r = deploymentConfigSchema.safeParse({ version: 1, - dataStores: { knowledgePostgre: { host: 'x', database: 'y', user: 'z' } }, - }); - expect(r.success).toBe(false); - }); - - it('rejects a wrong version', () => { - const r = deploymentConfigSchema.safeParse({ version: 2 }); - expect(r.success).toBe(false); - }); -}); - -describe('convexStorageSchema', () => { - it('rejects s3 mode without all buckets (all-or-nothing)', () => { - const r = convexStorageSchema.safeParse({ - mode: 's3', - region: 'eu-central-1', - buckets: { files: 'only-files' }, - }); - expect(r.success).toBe(false); - }); - - it('accepts an S3-compatible endpoint + forcePathStyle (MinIO/R2)', () => { - const r = convexStorageSchema.safeParse({ - mode: 's3', - region: 'auto', - endpoint: 'https://minio.acme.internal', - forcePathStyle: true, - buckets: { - files: 'f', - exports: 'e', - snapshotImports: 's', - modules: 'm', - search: 'se', - }, - }); - expect(r.success).toBe(true); - }); - - it('rejects a non-URL endpoint', () => { - const r = convexStorageSchema.safeParse({ - mode: 's3', - region: 'auto', - endpoint: 'not-a-url', - buckets: { - files: 'f', - exports: 'e', - snapshotImports: 's', - modules: 'm', - search: 'se', + dataStores: { + knowledgePostgres: { host: 'x', database: 'y', user: 'z' }, }, }); expect(r.success).toBe(false); }); - it('rejects non-http(s) endpoint schemes (SSRF / scheme smuggling)', () => { - const buckets = { - files: 'f', - exports: 'e', - snapshotImports: 's', - modules: 'm', - search: 'se', - }; - for (const endpoint of [ - 'file:///etc/passwd', - 'javascript:alert(1)', - 'ftp://example.com', - 'gopher://example.com', - ]) { - const r = convexStorageSchema.safeParse({ - mode: 's3', - region: 'auto', - endpoint, - buckets, - }); - expect(r.success, `${endpoint} should be rejected`).toBe(false); - } - }); - - it('accepts plain http(s) endpoints', () => { - const buckets = { - files: 'f', - exports: 'e', - snapshotImports: 's', - modules: 'm', - search: 'se', - }; - for (const endpoint of [ - 'http://minio.internal:9000', - 'https://s3.example.com', - ]) { - const r = convexStorageSchema.safeParse({ - mode: 's3', - region: 'auto', - endpoint, - buckets, - }); - expect(r.success, `${endpoint} should be accepted`).toBe(true); - } - }); -}); - -describe('pgConnectionSchema', () => { - it('applies port/sslmode defaults', () => { - const r = pgConnectionSchema.safeParse({ - host: 'h', - database: 'd', - user: 'u', - }); - expect(r.success).toBe(true); - if (r.success) { - expect(r.data.port).toBe(5432); - expect(r.data.sslmode).toBe('require'); - } - }); - - it('rejects an invalid sslmode', () => { - const r = pgConnectionSchema.safeParse({ - host: 'h', - database: 'd', - user: 'u', - sslmode: 'totally', - }); - expect(r.success).toBe(false); - }); - - it('accepts hostnames, IPv4, and bracketed IPv6 hosts', () => { - for (const host of [ - 'db.internal', - 'pg-1.example.com', - '10.0.0.5', - '[::1]', - ]) { - const r = pgConnectionSchema.safeParse({ - host, - database: 'd', - user: 'u', - }); - expect(r.success).toBe(true); - } - }); - - it('rejects hosts carrying URL metacharacters (DSN-smuggle guard)', () => { - for (const host of [ - 'good.com/?sslmode=disable&x=1', // path + query smuggle - 'a.com,169.254.169.254', // multi-host - 'evil@host', // userinfo split - 'host name', // whitespace - 'h%2f', // percent escape - ]) { - const r = pgConnectionSchema.safeParse({ - host, - database: 'd', - user: 'u', - }); - expect(r.success).toBe(false); - } - }); -}); - -describe('deploymentSecretsSchema', () => { - it('accepts allowlisted secret keys', () => { - const r = deploymentSecretsSchema.safeParse({ - 'dataStores.knowledgePostgres.password': 'pw', - 'dataStores.convexStorage.accessKeyId': 'AKIA...', - 'dataStores.convexStorage.secretAccessKey': 'secret', - }); - expect(r.success).toBe(true); - }); - - it('rejects an unknown secret key', () => { - const r = deploymentSecretsSchema.safeParse({ - 'dataStores.unknown.password': 'pw', + it('rejects an unknown sandboxRuntime key (strict — protects against typos)', () => { + const r = deploymentConfigSchema.safeParse({ + version: 1, + sandboxRuntime: { teir: 'kata' }, }); expect(r.success).toBe(false); }); - it('rejects an empty secret value', () => { - const r = deploymentSecretsSchema.safeParse({ - 'dataStores.knowledgePostgres.password': '', - }); + it('rejects a wrong version', () => { + const r = deploymentConfigSchema.safeParse({ version: 2 }); expect(r.success).toBe(false); }); - - it('every secret key is namespaced under a config section', () => { - for (const key of DEPLOYMENT_SECRET_KEYS) { - expect(key.split('.').length).toBeGreaterThanOrEqual(3); - expect(key.startsWith('dataStores.')).toBe(true); - } - }); }); diff --git a/services/platform/lib/shared/schemas/deployment.ts b/services/platform/lib/shared/schemas/deployment.ts index 074213fd84..2f94341211 100644 --- a/services/platform/lib/shared/schemas/deployment.ts +++ b/services/platform/lib/shared/schemas/deployment.ts @@ -5,127 +5,38 @@ import { z } from 'zod/v4'; * * Unlike the per-org config files (`/providers.json`, * `/retention.json`, …), this is a SINGLE deployment-scoped file at - * the config root (`/deployment.json` + a SOPS-encrypted - * `deployment.secrets.json` sidecar). It is written by instance-admin Convex - * actions and CONSUMED BY THE rag/convex/platform ENTRYPOINTS AT BOOT — none - * of it is hot-reloaded (changing where data physically lives requires a - * service restart). A top-level (one-path-segment) file is intentionally - * ignored by the per-org config-watcher. + * the config root (`/deployment.yml`; the retired + * `deployment.json` stays readable until the next save converts it). It is + * written by the instance-admin deployment routes and CONSUMED AT BOOT by the + * sandbox spawner (`services/sandbox/src/config.ts` reads `sandboxRuntime`) + * — none of it is hot-reloaded. A top-level (one-path-segment) file is + * intentionally ignored by the per-org config-watcher. + * + * WHERE DATA LIVES IS NOT CONFIGURED HERE. The deployment-default stores are + * environment-driven (`DATABASE_URL`, `KNOWLEDGE_DATABASE_URL`, + * `OBJECT_STORE_*`), and per-organization residency is its own config lane + * (`/knowledge/connection.json`, + * `/object-storage/connection.json`). The Convex-era `dataStores` + * section (knowledgePostgres / appPostgres / convexStorage + a secrets + * sidecar) was saved but never read by any boot path once the Convex + * entrypoints retired; it is gone. A file that still carries it is tolerated + * on read (the section is dropped with a warning) and rewritten without it on + * the next save — see `parseDeploymentConfig`. * * The shape is a SECTIONED REGISTRY: `version` + a set of optional sections. - * Adding a future deployment section (SMTP, telemetry, …) is purely additive - * — add an optional field here and its secret keys to - * `DEPLOYMENT_SECRET_KEYS`; the Convex read/save/test actions stay - * section-agnostic (they read/write the whole file). + * Adding a future deployment section is purely additive — add an optional + * field here; the read/save routes stay section-agnostic (they read/write the + * whole file). */ export const DEPLOYMENT_CONFIG_VERSION = 1 as const; /** - * Reusable external-Postgres connection shape (no `table`/`schema` — the RAG - * service owns the whole `private_knowledge` schema on the target DB). Models - * the same fields the per-org external-pgvector connection used. Secrets - * (password) are NEVER stored here — they live in the SOPS secrets sidecar - * keyed by `DEPLOYMENT_SECRET_KEYS`. - */ -export const pgConnectionSchema = z - .object({ - // Restrict to hostname / IPv4 / IPv6 characters. Rejecting URL - // metacharacters (`/ ? & @ , space % #`) keeps a crafted host from - // smuggling libpq params / downgrading TLS once it is interpolated into a - // connection URL or DSN downstream (the SSRF-gate URL parser and the pg - // driver's DSN parser must not be able to disagree on the host). - host: z - .string() - .min(1) - .regex( - /^[A-Za-z0-9._:[\]-]+$/, - 'Host may only contain letters, digits, and . _ - : [ ] (no URL metacharacters).', - ), - port: z.number().int().min(1).max(65535).default(5432), - database: z.string().min(1), - user: z.string().min(1), - sslmode: z - .enum(['disable', 'prefer', 'require', 'verify-ca', 'verify-full']) - .default('require'), - }) - .strict(); - -/** - * The five Convex storage use-cases. When storage mode is `s3`, the - * self-hosted `convex-local-backend` puts ALL of them in S3 (it is - * all-or-nothing — there is no per-use-case local/S3 split), each in its own - * bucket via `S3_STORAGE_{FILES,EXPORTS,SNAPSHOT_IMPORTS,MODULES,SEARCH}_BUCKET`. - * `files` holds the user-uploaded `_storage` blobs — the one that matters for - * document residency. - */ -const s3BucketsSchema = z - .object({ - files: z.string().min(1), - exports: z.string().min(1), - snapshotImports: z.string().min(1), - modules: z.string().min(1), - search: z.string().min(1), - }) - .strict(); - -/** - * Convex file-storage backend. `local` (default) keeps `_storage` blobs on the - * local volume — today's behavior. `s3` points them at an external / - * S3-compatible object store (AWS S3, MinIO, Cloudflare R2); `endpoint` + - * `forcePathStyle` cover the S3-compatible cases. Credentials - * (accessKeyId/secretAccessKey) live in the secrets sidecar. - * - * NOTE: switching local→S3 on an existing deployment does NOT migrate the - * already-stored local blobs — S3 mode is greenfield (set at initial deploy) - * or requires a separate offline copy. The UI/docs must warn. - */ -export const convexStorageSchema = z.discriminatedUnion('mode', [ - z.object({ mode: z.literal('local') }).strict(), - z - .object({ - mode: z.literal('s3'), - region: z.string().min(1), - // Restrict to http(s):// — `.url()` alone also admits file:/javascript:/ - // ftp:/gopher:, and the SSRF host gate only checks the hostname, so a - // non-http scheme with a public host would otherwise flow to - // S3_ENDPOINT_URL. Mirrors moderationEndpointSchema in governance.ts. - endpoint: z - .string() - .url() - .refine((u) => { - try { - const p = new URL(u).protocol; - return p === 'https:' || p === 'http:'; - } catch { - return false; - } - }, 'Endpoint must be http(s)://') - .optional(), - forcePathStyle: z.boolean().default(false), - buckets: s3BucketsSchema, - }) - .strict(), -]); - -/** - * `dataStores` section — where the deployment's data physically lives. - * - `knowledgePostgres`: the RAG knowledge DB (documents + chunk text + - * embeddings + BM25 + semantic cache). Must be ParadeDB (pgvector + - * pg_search) or hybrid search degrades to vector-only. - * - `convexStorage`: where Convex `_storage` blobs (original uploaded files) - * live. - * - `appPostgres`: optional override for the Convex/app metadata DB. - * All optional — an absent section means "use the `.env` default" (today's - * built-in stores). + * Sections no boot path consumes any more (the Convex-era `dataStores` key). + * Tolerated by the reader so an operator's older file keeps parsing; never + * accepted by the strict schema below. */ -const dataStoresSchema = z - .object({ - knowledgePostgres: pgConnectionSchema.optional(), - convexStorage: convexStorageSchema.optional(), - appPostgres: pgConnectionSchema.optional(), - }) - .strict(); +export const RETIRED_DEPLOYMENT_SECTIONS = ['dataStores'] as const; /** * `sandboxRuntime` section — the deployment-wide container runtime tier for the @@ -158,37 +69,8 @@ const sandboxRuntimeSchema = z export const deploymentConfigSchema = z .object({ version: z.literal(DEPLOYMENT_CONFIG_VERSION), - dataStores: dataStoresSchema.optional(), sandboxRuntime: sandboxRuntimeSchema.optional(), }) .strict(); export type DeploymentConfig = z.infer; - -/** - * Allowlist of secret keys for the SOPS-encrypted `deployment.secrets.json`. - * Keys are FLAT, DOTTED, and namespaced by section so a new section's secrets - * never collide and merge independently. The secrets file validates against - * this enum — an unknown key is rejected. Adding a section = add its keys - * here. - */ -export const DEPLOYMENT_SECRET_KEYS = [ - 'dataStores.knowledgePostgres.password', - 'dataStores.convexStorage.accessKeyId', - 'dataStores.convexStorage.secretAccessKey', - 'dataStores.appPostgres.password', -] as const; - -export type DeploymentSecretKey = (typeof DEPLOYMENT_SECRET_KEYS)[number]; - -/** - * Secrets sidecar shape: a partial map from an allowlisted secret key to its - * (non-empty) string value. Stored SOPS-encrypted; never returned to the - * browser in full (masked on read). - */ -export const deploymentSecretsSchema = z.partialRecord( - z.enum(DEPLOYMENT_SECRET_KEYS), - z.string().min(1), -); - -export type DeploymentSecrets = z.infer; diff --git a/services/platform/lib/shared/schemas/knowledge.test.ts b/services/platform/lib/shared/schemas/knowledge.test.ts new file mode 100644 index 0000000000..494fb82218 --- /dev/null +++ b/services/platform/lib/shared/schemas/knowledge.test.ts @@ -0,0 +1,65 @@ +import { describe, expect, it } from 'vitest'; + +import { knowledgeConnectionSchema, pgConnectionSchema } from './knowledge'; + +describe('pgConnectionSchema', () => { + it('is the shape the per-org knowledge connection file validates against', () => { + expect(knowledgeConnectionSchema).toBe(pgConnectionSchema); + }); + + it('applies port/sslmode defaults', () => { + const r = pgConnectionSchema.safeParse({ + host: 'h', + database: 'd', + user: 'u', + }); + expect(r.success).toBe(true); + if (r.success) { + expect(r.data.port).toBe(5432); + expect(r.data.sslmode).toBe('require'); + } + }); + + it('rejects an invalid sslmode', () => { + const r = pgConnectionSchema.safeParse({ + host: 'h', + database: 'd', + user: 'u', + sslmode: 'totally', + }); + expect(r.success).toBe(false); + }); + + it('accepts hostnames, IPv4, and bracketed IPv6 hosts', () => { + for (const host of [ + 'db.internal', + 'pg-1.example.com', + '10.0.0.5', + '[::1]', + ]) { + const r = pgConnectionSchema.safeParse({ + host, + database: 'd', + user: 'u', + }); + expect(r.success).toBe(true); + } + }); + + it('rejects hosts carrying URL metacharacters (DSN-smuggle guard)', () => { + for (const host of [ + 'good.com/?sslmode=disable&x=1', // path + query smuggle + 'a.com,169.254.169.254', // multi-host + 'evil@host', // userinfo split + 'host name', // whitespace + 'h%2f', // percent escape + ]) { + const r = pgConnectionSchema.safeParse({ + host, + database: 'd', + user: 'u', + }); + expect(r.success).toBe(false); + } + }); +}); diff --git a/services/platform/lib/shared/schemas/knowledge.ts b/services/platform/lib/shared/schemas/knowledge.ts index 6a091963c4..d585241c69 100644 --- a/services/platform/lib/shared/schemas/knowledge.ts +++ b/services/platform/lib/shared/schemas/knowledge.ts @@ -10,15 +10,40 @@ * connection.secrets.json — the database password (SOPS-encrypted at rest). * embedding.json — the embedding model, stated in full. * - * `connection.json` reuses `pgConnectionSchema` verbatim rather than declaring - * a second connection shape — the deployment-wide external-Postgres setting and - * an organization's own database are the same kind of thing, and two schemas - * for it would drift. + * `pgConnectionSchema` is THE external-Postgres connection shape: every + * config lane that points at a Postgres an operator brings (today: this one) + * reuses it verbatim rather than declaring a second shape that would drift. */ import { z } from 'zod/v4'; -import { pgConnectionSchema } from './deployment'; +/** + * External-Postgres connection shape (no `table`/`schema` — the corpus owns + * whole schemas on the target DB). Secrets (password) are NEVER stored here — + * they live in the SOPS-encrypted secrets sidecar next to the file. + */ +export const pgConnectionSchema = z + .object({ + // Restrict to hostname / IPv4 / IPv6 characters. Rejecting URL + // metacharacters (`/ ? & @ , space % #`) keeps a crafted host from + // smuggling libpq params / downgrading TLS once it is interpolated into a + // connection URL or DSN downstream (the SSRF-gate URL parser and the pg + // driver's DSN parser must not be able to disagree on the host). + host: z + .string() + .min(1) + .regex( + /^[A-Za-z0-9._:[\]-]+$/, + 'Host may only contain letters, digits, and . _ - : [ ] (no URL metacharacters).', + ), + port: z.number().int().min(1).max(65535).default(5432), + database: z.string().min(1), + user: z.string().min(1), + sslmode: z + .enum(['disable', 'prefer', 'require', 'verify-ca', 'verify-full']) + .default('require'), + }) + .strict(); export const KNOWLEDGE_CONFIG_DOMAIN = 'knowledge'; export const KNOWLEDGE_CONNECTION_KEY = 'connection'; diff --git a/services/platform/lib/shared/schemas/object_storage.ts b/services/platform/lib/shared/schemas/object_storage.ts index 88fe060c25..53f9e78559 100644 --- a/services/platform/lib/shared/schemas/object_storage.ts +++ b/services/platform/lib/shared/schemas/object_storage.ts @@ -23,9 +23,8 @@ import { z } from 'zod/v4'; * {TALE_CONFIG_DIR}//object-storage/connection.json (config) * {TALE_CONFIG_DIR}//object-storage/connection.secrets.json (SOPS) * - * The bucket SHAPE reuses the S3 fields the deployment-wide - * `dataStores.convexStorage` already uses (region/endpoint/forcePathStyle) — a - * single bucket rather than Convex's five, since one org owns the whole bucket. + * The bucket SHAPE is the plain S3 coordinate set (region/endpoint/ + * forcePathStyle) — a single bucket, since one org owns the whole bucket. * Credentials (`accessKeyId`/`secretAccessKey`) live ONLY in the SOPS-encrypted * secrets sidecar, exactly like `providers/*.secrets.json`. */ @@ -34,9 +33,8 @@ export const OBJECT_STORAGE_CONFIG_DOMAIN = 'object-storage'; export const OBJECT_STORAGE_CONNECTION_KEY = 'connection'; /** - * `connection.json` — the org's S3-compatible bucket coordinates. Reuses the - * `dataStores.convexStorage` S3 shape (region + optional endpoint + - * forcePathStyle), narrowed to ONE bucket the org owns. + * `connection.json` — the org's S3-compatible bucket coordinates (region + + * optional endpoint + forcePathStyle) for ONE bucket the org owns. */ export const objectStorageConnectionFileSchema = z .object({ @@ -46,7 +44,7 @@ export const objectStorageConnectionFileSchema = z * Restricted to http(s):// — `.url()` alone also admits * file:/javascript:/ftp:, and the SSRF host gate only checks the hostname, * so a non-http scheme with a public host would otherwise reach the signer. - * Mirrors `convexStorageSchema` in deployment.ts. + * Mirrors `moderationEndpointSchema` in governance.ts. */ endpoint: z .string() From d6617b74d20f7241513284e789b5ebe9324204f2 Mon Sep 17 00:00:00 2001 From: larryro <371767072@qq.com> Date: Sun, 6 Sep 2026 10:59:59 +0800 Subject: [PATCH 2/8] fix(platform): drop the deployment-wide stores from Data residency The deployment-wide section of Settings > Data residency (knowledge database, file storage, application database) saved a config nothing read and told the operator to restart to apply it. With the backend section retired, the page keeps only the per-organization sections that actually route data: knowledge connection, embedding model, object storage. - delete deployment-stores.tsx, deployment-errors.ts and the app-side deployment contract/adapter rows; drop the deployment read/save/ secret/test hooks - component test: assert the page renders exactly the three org sections and none of the retired store headings or the operator allowlist hint - locale catalogs (en/de/fr): remove the deployment-only keys; the shared connection vocabulary the org sections use stays Finding: lib-shared-schemas-1. --- .../data-residency-settings.test.tsx | 177 +-- .../components/data-residency-settings.tsx | 29 +- .../components/deployment-stores.tsx | 1009 ----------------- .../data-residency/deployment-errors.ts | 106 -- .../data-residency/hooks/mutations.ts | 36 +- .../settings/data-residency/hooks/queries.ts | 24 +- .../data-residency/org-residency-errors.ts | 6 +- services/platform/app/lib/backend/admin.ts | 50 - .../app/lib/backend/contract/deployment.ts | 130 --- .../app/lib/backend/contract/index.ts | 2 - services/platform/messages/de.yml | 78 -- services/platform/messages/en.yml | 71 -- services/platform/messages/fr.yml | 74 -- 13 files changed, 38 insertions(+), 1754 deletions(-) delete mode 100644 services/platform/app/features/settings/data-residency/components/deployment-stores.tsx delete mode 100644 services/platform/app/features/settings/data-residency/deployment-errors.ts delete mode 100644 services/platform/app/lib/backend/contract/deployment.ts diff --git a/services/platform/app/features/settings/data-residency/components/data-residency-settings.test.tsx b/services/platform/app/features/settings/data-residency/components/data-residency-settings.test.tsx index b297c29d4f..1ad86dcd8f 100644 --- a/services/platform/app/features/settings/data-residency/components/data-residency-settings.test.tsx +++ b/services/platform/app/features/settings/data-residency/components/data-residency-settings.test.tsx @@ -12,22 +12,20 @@ import { render, screen, waitFor, within } from '@/tests/utils/render'; import { DataResidencySettings } from './data-residency-settings'; /** - * Component coverage for the unified data-residency page — the two access - * levels of the SAME surface: - * - * - A deployment operator (`canEdit` from the read) edits the deployment - * stores; a non-operator admin sees them read-only with a stated reason. - * - An org admin (`write orgSettings`) edits this organization's knowledge - * database, embedding model, and object storage; a member without it sees - * them read-only with a stated reason. + * Component coverage for the unified data-residency page — one surface, two + * access levels: an org admin (`write orgSettings`) edits this organization's + * knowledge database, embedding model, and object storage; a member without it + * sees them read-only with a stated reason. There is no deployment-wide store + * section any more (where the deployment default lives is environment-driven), + * and the page must not grow one back. * * Backend behaviour (config validation, SOPS sidecars, the real probes) is - * covered by the convex action tests — here the hooks are stubbed at the - * module boundary. The org sections save through the settings header's shared + * covered by the backend tests — here the hooks are stubbed at the module + * boundary. The org sections save through the settings header's shared * Save/Discard cluster; its slot is absent in this harness, so saves are * driven through the composed controller captured from `useActiveEditor` - * (exactly what the cluster does). Deployment editing is asserted via the - * rendered controls; Test/backfill via their own inline buttons. + * (exactly what the cluster does). Test/backfill run via their own inline + * buttons. */ const saveStorage = vi.hoisted(() => vi.fn()); @@ -71,8 +69,6 @@ interface EmbeddingFixture { } const fixtures = vi.hoisted(() => ({ - deployment: undefined as unknown, - deploymentError: false, storage: { configured: false } as unknown, knowledge: { configured: false } as unknown, embedding: { configured: false } as unknown, @@ -90,52 +86,6 @@ const fixtures = vi.hoisted(() => ({ }>, })); -/** A deployment config with all three stores populated. */ -function deploymentConfig(canEdit: boolean) { - return { - config: { - version: 1, - dataStores: { - knowledgePostgres: { - host: 'kb.example.org', - port: 5432, - database: 'knowledge', - user: 'tale', - sslmode: 'require', - }, - convexStorage: { - mode: 's3', - region: 'eu-central-1', - endpoint: 'https://minio.example.org', - forcePathStyle: true, - buckets: { - files: 'files-b', - exports: 'exports-b', - snapshotImports: 'snap-b', - modules: 'mods-b', - search: 'search-b', - }, - }, - appPostgres: { - host: 'app.example.org', - port: 5432, - database: 'appdb', - user: 'tale', - }, - }, - }, - hash: 'h1', - secrets: {}, - canEdit, - email: canEdit ? 'op@example.org' : 'viewer@example.org', - }; -} - -function setDeployment(canEdit: boolean) { - fixtures.deployment = deploymentConfig(canEdit); - fixtures.deploymentError = false; -} - function setStorageFixture(view: StorageFixture) { fixtures.storage = view; } @@ -149,14 +99,6 @@ function setEmbeddingFixture(view: EmbeddingFixture) { } vi.mock('../hooks/queries', () => ({ - useReadDeploymentConfig: () => ({ - data: fixtures.deployment, - isPending: false, - isError: fixtures.deploymentError, - error: fixtures.deploymentError - ? { data: { code: 'DEPLOYMENT_CONFIG_UNREADABLE', message: 'boom' } } - : null, - }), useOrgObjectStorageConnection: () => ({ data: fixtures.storage, isPending: false, @@ -190,12 +132,6 @@ vi.mock('../hooks/queries', () => ({ })); vi.mock('../hooks/mutations', () => ({ - useSaveDeploymentConfig: () => ({ mutateAsync: vi.fn(), isPending: false }), - useSaveDeploymentSecret: () => ({ mutateAsync: vi.fn(), isPending: false }), - useTestDeploymentConnection: () => ({ - mutateAsync: vi.fn(), - isPending: false, - }), useSaveOrgObjectStorageConnection: () => ({ mutateAsync: saveStorage, isPending: false, @@ -297,7 +233,6 @@ describe('DataResidencySettings', () => { vi.clearAllMocks(); abilityState.canRead = true; abilityState.canWrite = true; - setDeployment(false); setStorageFixture({ configured: false }); setKnowledgeFixture({ configured: false }); setEmbeddingFixture({ configured: false }); @@ -330,8 +265,7 @@ describe('DataResidencySettings', () => { await waitFor(() => expect(capture.current?.isValid).toBe(true)); }); - it('renders the org sections first and the deployment stores after', () => { - setDeployment(true); + it('renders exactly the three org sections — no deployment-wide store section', () => { render(); const headings = screen @@ -341,65 +275,20 @@ describe('DataResidencySettings', () => { 'Knowledge database', 'Embedding model', 'Object storage', - 'Knowledge database (RAG)', - 'File storage (uploaded documents)', - 'Application database (advanced)', ].map((name) => headings.findIndex((text) => text === name)); expect(order.every((index) => index >= 0)).toBe(true); expect([...order].sort((a, b) => a - b)).toEqual(order); - }); - - it('lets a deployment operator edit the deployment stores (editable state)', async () => { - setDeployment(true); - const { container } = render( - , - ); - - // Operator sees the enable switches (state as an interactive control) and - // editable, non-readonly inputs. Three "External Postgres" switches: the - // two deployment Postgres stores plus the org knowledge section's toggle. - expect( - screen.getAllByRole('switch', { name: 'External Postgres' }), - ).toHaveLength(3); - const hosts = screen.getAllByRole('textbox', { name: 'Host' }); - expect(hosts).toHaveLength(2); - expect(hosts[0]).toHaveValue('kb.example.org'); - expect(hosts[0]).not.toHaveAttribute('readonly'); - - await waitFor(() => checkAccessibility(container)); - }); - - it('shows the deployment stores read-only, with the operator-allowlist reason', async () => { - setDeployment(false); // caller is not in TALE_DEPLOYMENT_CONFIG_ADMINS - const { container } = render( - , - ); - - // State is conveyed as text (a status pill), never a bare disabled switch — - // there are no enable switches inside the deployment sections at all. - // (The org knowledge section keeps its own toggle: this caller IS an org - // admin, just not a deployment operator.) - expect( - within(sectionByHeading('Knowledge database (RAG)')).queryByRole( - 'switch', - ), - ).toBeNull(); - expect( - within(sectionByHeading('Application database (advanced)')).queryByRole( - 'switch', - ), - ).toBeNull(); - // The stored coordinates render as native read-only fields. - const hosts = screen.getAllByRole('textbox', { name: 'Host' }); - expect(hosts[0]).toHaveValue('kb.example.org'); - expect(hosts[0]).toHaveAttribute('readonly'); - // The reason is stated, not left as a silent disabled control. - expect(screen.getAllByText('Read-only access').length).toBeGreaterThan(0); - expect( - screen.getByText(/TALE_DEPLOYMENT_CONFIG_ADMINS/), - ).toBeInTheDocument(); - - await waitFor(() => checkAccessibility(container)); + // The retired deployment-wide stores (saved but never read at boot) are + // gone for good: no store may claim a restart applies it. + for (const retired of [ + 'Knowledge database (RAG)', + 'File storage (uploaded documents)', + 'Application database (advanced)', + 'Save deployment', + ]) { + expect(screen.queryByText(retired)).toBeNull(); + } + expect(screen.queryByText(/TALE_DEPLOYMENT_CONFIG_ADMINS/)).toBeNull(); }); it('renders the org knowledge section from a loaded config with its stored values', async () => { @@ -905,26 +794,4 @@ describe('DataResidencySettings', () => { screen.queryByRole('button', { name: 'Move existing files' }), ).toBeNull(); }); - - it('surfaces a deployment read failure without hiding the org sections', () => { - fixtures.deployment = undefined; - fixtures.deploymentError = true; - - render(); - - // The deployment group reports its own failure inline... - expect( - screen.getByText(/Couldn't load the deployment configuration/), - ).toBeInTheDocument(); - // ...while the org sections still render. - expect( - screen.getByRole('heading', { name: 'Object storage' }), - ).toBeInTheDocument(); - expect( - screen.getByRole('heading', { name: 'Knowledge database' }), - ).toBeInTheDocument(); - expect( - screen.getByRole('heading', { name: 'Embedding model' }), - ).toBeInTheDocument(); - }); }); diff --git a/services/platform/app/features/settings/data-residency/components/data-residency-settings.tsx b/services/platform/app/features/settings/data-residency/components/data-residency-settings.tsx index 6e9a645b6d..412aa8741b 100644 --- a/services/platform/app/features/settings/data-residency/components/data-residency-settings.tsx +++ b/services/platform/app/features/settings/data-residency/components/data-residency-settings.tsx @@ -12,17 +12,16 @@ * (the unified editor contract); Test/Remove/backfill stay instant * actions. The per-org configs live under `$TALE_CONFIG_DIR// * {knowledge,object-storage}/`, which stays the source of truth on disk. - * - Deployment stores after (knowledge database, file storage, and the - * advanced application database): viewable by any org admin, editable only - * by an operator in the `TALE_DEPLOYMENT_CONFIG_ADMINS` allowlist, with - * its own "Save deployment" header action (a save only takes effect on - * the next restart/deploy, which the editor contract cannot express). + * + * There is no deployment-wide store section: where the deployment's default + * data lives is environment-driven (`DATABASE_URL`, `KNOWLEDGE_DATABASE_URL`, + * `OBJECT_STORE_*`) and set at deploy time, never from this page. * * Read-only sections render the stored coordinates as native read-only fields * (conveyed to assistive tech, not by disabled/color alone) and the on/off * state as a status badge; write-only credentials are never shown to a viewer. * - * Strings live under `settings.dataResidency.*` (deployment vocabulary + + * Strings live under `settings.dataResidency.*` (shared connection vocabulary + * `orgKnowledge.*` / `orgEmbedding.*` / `orgStorage.*` for the org sections), * plus `navigation.dataResidency`, `metadata.dataResidency`, and * `accessDenied.dataResidency` across en/de/fr (de-CH inherits de). Code @@ -41,15 +40,12 @@ import { SettingsPage } from '@/app/features/settings/components/settings-page'; import { useAbility, useAbilityLoading } from '@/app/hooks/use-ability'; import { useT } from '@/lib/i18n/client'; -import { mapDeploymentError } from '../deployment-errors'; import { useOrgKnowledgeConnection, useOrgKnowledgeEmbedding, useOrgObjectStorageConnection, - useReadDeploymentConfig, } from '../hooks/queries'; import { mapOrgResidencyError } from '../org-residency-errors'; -import { DeploymentStoresView } from './deployment-stores'; import { OrgEmbeddingSection } from './org-embedding-section'; import { OrgKnowledgeSection } from './org-knowledge-section'; import { OrgStorageSection } from './org-storage-section'; @@ -63,15 +59,12 @@ export function DataResidencySettings({ const { t: tAccessDenied } = useT('accessDenied'); const ability = useAbility(); const abilityLoading = useAbilityLoading(); - const deploymentQuery = useReadDeploymentConfig(); const knowledgeQuery = useOrgKnowledgeConnection(organizationId); const embeddingQuery = useOrgKnowledgeEmbedding(organizationId); const storageQuery = useOrgObjectStorageConnection(organizationId); - // Viewing is open to any organization admin (`read orgSettings`) — the same - // gate the deployment read enforces server-side. Editing each store is a - // finer capability resolved per section: deployment stores need the operator - // allowlist (`canEdit` from the read); org sections need `write orgSettings`. + // Viewing is open to any organization admin (`read orgSettings`); editing + // needs `write orgSettings`. if (!abilityLoading && ability.cannot('read', 'orgSettings')) { return ; } @@ -81,9 +74,6 @@ export function DataResidencySettings({ // A failed read must not fall through to a blank, default-looking form — that // would imply "nothing configured" when the truth is unknown. Each section // reports its own read failure inline. - const deploymentReadError = deploymentQuery.isError - ? mapDeploymentError(deploymentQuery.error, t).message - : undefined; const knowledgeReadError = knowledgeQuery.isError ? mapOrgResidencyError(knowledgeQuery.error, t) : undefined; @@ -98,7 +88,6 @@ export function DataResidencySettings({ - ); diff --git a/services/platform/app/features/settings/data-residency/components/deployment-stores.tsx b/services/platform/app/features/settings/data-residency/components/deployment-stores.tsx deleted file mode 100644 index d27bc4ecd6..0000000000 --- a/services/platform/app/features/settings/data-residency/components/deployment-stores.tsx +++ /dev/null @@ -1,1009 +0,0 @@ -'use client'; - -/** - * The deployment-level stores group of the data-residency page: knowledge - * database, file storage, and the advanced application database. Open to any - * organization admin to VIEW where the deployment keeps its data; editable - * only by an operator whose email is in the `TALE_DEPLOYMENT_CONFIG_ADMINS` - * allowlist (the read action returns `canEdit`). A non-operator admin sees - * these stores read-only with a stated reason — never a bare disabled control. - * - * Deliberately NOT on the page's editor contract: its Save must be followed by - * an explicit "Apply & restart" (the config lands in containers, not in the - * app), so it registers its own header actions — labelled "Save deployment" - * and placed `leading` so they never collide with the org sections' plain - * Discard/Save cluster to their right. - */ - -import { Alert } from '@tale/ui/alert'; -import { Button } from '@tale/ui/button'; -import { HStack, Stack } from '@tale/ui/layout'; -import { - type Dispatch, - type ReactNode, - type SetStateAction, - useEffect, - useState, -} from 'react'; - -import { ConfirmDialog } from '@/app/components/ui/dialog/confirm-dialog'; -import { FormSection } from '@/app/components/ui/forms/form-section'; -import { Input } from '@/app/components/ui/forms/input'; -import { Select } from '@/app/components/ui/forms/select'; -import { Switch } from '@/app/components/ui/forms/switch'; -import { useRegisterSettingsSecondaryAction } from '@/app/features/settings/components/settings-secondary-action-context'; -import { SettingsSection } from '@/app/features/settings/components/settings-section'; -import { TestResultLine } from '@/app/features/settings/components/test-result-line'; -import { useT } from '@/lib/i18n/client'; -import { structuralEqual } from '@/lib/utils/structural-equal'; - -import { mapDeploymentError } from '../deployment-errors'; -import { - useSaveDeploymentConfig, - useSaveDeploymentSecret, - useTestDeploymentConnection, -} from '../hooks/mutations'; -import { ReadOnlyField, StatusBadge } from './residency-chrome'; - -const SSL_MODES = ['disable', 'prefer', 'require', 'verify-ca', 'verify-full']; - -type PgForm = { - enabled: boolean; - host: string; - port: string; - database: string; - user: string; - sslmode: string; - password: string; // write-only; blank = keep stored -}; - -type StorageForm = { - s3: boolean; - region: string; - endpoint: string; - forcePathStyle: boolean; - files: string; - exports: string; - snapshotImports: string; - modules: string; - search: string; - accessKeyId: string; // write-only - secretAccessKey: string; // write-only -}; - -/** Loose shape of the JSON the read action returns (deployment.json is v.any()). */ -type PgConfigJson = { - host?: string; - port?: number; - database?: string; - user?: string; - sslmode?: string; -}; - -export type DeploymentReadData = { - config?: { - version?: number; - dataStores?: { - knowledgePostgres?: PgConfigJson; - appPostgres?: PgConfigJson; - convexStorage?: { - mode?: string; - region?: string; - endpoint?: string; - forcePathStyle?: boolean; - buckets?: Record; - }; - }; - }; - hash?: string | null; - secrets?: Record; - secretsError?: string; - /** Whether THIS caller may edit (their email is in the editor allowlist). */ - canEdit?: boolean; - /** The caller's own email — surfaced in the read-only banner. */ - email?: string; -}; - -type ConnTestResult = { - ok?: boolean; - error?: string; - hint?: string; - latencyMs?: number; -}; - -const emptyPg = (): PgForm => ({ - enabled: false, - host: '', - port: '5432', - database: '', - user: '', - sslmode: 'require', - password: '', -}); - -function pgFromConfig(pg: PgConfigJson | undefined): PgForm { - if (!pg) return emptyPg(); - return { - enabled: true, - host: pg.host ?? '', - port: String(pg.port ?? 5432), - database: pg.database ?? '', - user: pg.user ?? '', - sslmode: pg.sslmode ?? 'require', - password: '', - }; -} - -/** Loose shape of the stored Convex-storage config the read action returns. */ -type StorageConfigJson = { - mode?: string; - region?: string; - endpoint?: string; - forcePathStyle?: boolean; - buckets?: Record; -}; - -function storageFromConfig(cs: StorageConfigJson | undefined): StorageForm { - return { - s3: cs?.mode === 's3', - region: cs?.region ?? '', - endpoint: cs?.endpoint ?? '', - forcePathStyle: Boolean(cs?.forcePathStyle), - files: cs?.buckets?.files ?? '', - exports: cs?.buckets?.exports ?? '', - snapshotImports: cs?.buckets?.snapshotImports ?? '', - modules: cs?.buckets?.modules ?? '', - search: cs?.buckets?.search ?? '', - accessKeyId: '', - secretAccessKey: '', - }; -} - -/** - * The deployment group's header actions, mounted ONLY for allowlisted - * operators — a non-operator never sees buttons they can never use (the same - * doctrine that hides the org sections' Save cluster from read-only viewers). - * Registration lives in its own conditionally-mounted component because the - * registrar hook requires a render-stable action count; unmounting clears the - * slot via the hook's own cleanup. - * - * The action sits `leading` (before the org sections' Discard/Save cluster) - * and is labelled "Save deployment" — two plain "Save" buttons side by side - * would be indistinguishable. - */ -function DeploymentHeaderActions({ - onSave, - saving, - isDirty, -}: { - onSave: () => void; - saving: boolean; - isDirty: boolean; -}) { - const { t } = useT('settings'); - useRegisterSettingsSecondaryAction([ - { - label: t('dataResidency.saveDeployment'), - loadingLabel: t('dataResidency.saving'), - onClick: onSave, - disabled: saving || !isDirty, - loading: saving, - placement: 'leading', - }, - ]); - return null; -} - -function PgSection({ - title, - description, - state, - setState, - secretMasked, - secretPresent, - onTest, - testing, - testResult, - readOnly, - note, - showSslMode = true, - className, -}: { - title: string; - description?: string; - state: PgForm; - setState: (next: PgForm) => void; - secretMasked?: string; - /** A stored password exists (no preview shown for credential-class secrets). */ - secretPresent?: boolean; - onTest: () => void; - testing: boolean; - testResult?: { ok: boolean; message?: string }; - /** The caller may only view this store (not in the deployment-editor allowlist). */ - readOnly: boolean; - /** Contextual footnote shown below the fields while the section is enabled. */ - note?: ReactNode; - /** - * Whether to render the SSL-mode control. Off for the app DB: the backend - * pins that target's connection test to `sslmode=prefer` - * (domains/deployment/service.ts) instead of honoring a chosen mode — - * offering the control would promise a guarantee we can't deliver. - */ - showSslMode?: boolean; - /** Forwarded to the underlying section root (e.g. a divider border). */ - className?: string; -}) { - const { t } = useT('settings'); - const onLabel = t('dataResidency.externalPostgres'); - const offLabel = t('dataResidency.status.builtIn'); - return ( - - ) : ( - - - - setState({ ...state, enabled: checked }) - } - /> - - ) - } - > - {state.enabled ? ( - readOnly ? ( - // Viewer: the stored coordinates as read-only fields. Write-only - // credentials are never shown, so no password field appears. - -
- - - - - {showSslMode ? ( - - ) : null} -
- {note} - - ) : ( - -
- setState({ ...state, host: e.target.value })} - /> - setState({ ...state, port: e.target.value })} - /> - - setState({ ...state, database: e.target.value }) - } - /> - setState({ ...state, user: e.target.value })} - /> - {showSslMode ? ( - - setState({ ...state, password: e.target.value }) - } - description={ - secretMasked - ? t('dataResidency.password.storedHint', { - masked: secretMasked, - }) - : secretPresent - ? t('dataResidency.password.storedNoPreviewHint') - : t('dataResidency.password.writeOnlyHint') - } - /> -
- - - - - {note} - - ) - ) : // Off = built-in: the status pill in the header already says so, so - // the body stays empty rather than repeating it as a grey sentence. - null} - - ); -} - -/** - * The deployment-level stores group: knowledge database, file storage, and the - * advanced application database. `data.canEdit` gates editing; a viewer sees - * every store read-only with the allowlist reason stated up top. When the read - * itself fails, the group shows a single warning in place of the stores. - */ -export function DeploymentStoresView({ - data, - readError, -}: { - data: DeploymentReadData | undefined; - readError?: string; -}) { - const { t } = useT('settings'); - const cfg = data?.config ?? { version: 1 }; - const ds = cfg.dataStores ?? {}; - const secretState = data?.secrets ?? {}; - const canEdit: boolean = Boolean(data?.canEdit); - const readOnly = !canEdit; - - const [knowledge, setKnowledgeRaw] = useState(() => - pgFromConfig(ds.knowledgePostgres), - ); - const [appPg, setAppPgRaw] = useState(() => - pgFromConfig(ds.appPostgres), - ); - const [storage, setStorageRaw] = useState(() => - storageFromConfig(ds.convexStorage), - ); - - const [saving, setSaving] = useState(false); - const [savedOk, setSavedOk] = useState(false); - const [error, setError] = useState(null); - const [forceOverwriteOpen, setForceOverwriteOpen] = useState(false); - const [testing, setTesting] = useState(null); - const [testResults, setTestResults] = useState< - Record - >({}); - - // Editing a section clears the stale failed-save banner and that section's - // now-outdated test result so the operator never sees feedback that no longer - // matches the form. (The "Saved" banner is gated on !isDirty in render, so it - // hides on edit without being cleared here.) - function clearStaleFeedback(section: string) { - setError(null); - setTestResults((prev) => { - if (!(section in prev)) return prev; - const next = { ...prev }; - delete next[section]; - return next; - }); - } - const setKnowledge: Dispatch> = (a) => { - clearStaleFeedback('knowledgePostgres'); - setKnowledgeRaw(a); - }; - const setAppPg: Dispatch> = (a) => { - clearStaleFeedback('appPostgres'); - setAppPgRaw(a); - }; - const setStorage: Dispatch> = (a) => { - clearStaleFeedback('convexStorage'); - setStorageRaw(a); - }; - - const saveConfig = useSaveDeploymentConfig(); - const saveSecret = useSaveDeploymentSecret(); - const testConn = useTestDeploymentConnection(); - - // Reset local form when a fresh read lands (e.g. after a save invalidates - // the query). Resetting all three sections to the freshly-read baseline is - // also what clears the dirty state after a successful save. - useEffect(() => { - setKnowledge(pgFromConfig(ds.knowledgePostgres)); - setAppPg(pgFromConfig(ds.appPostgres)); - setStorage(storageFromConfig(ds.convexStorage)); - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [data?.hash]); - - // Dirty = the form differs from the loaded config, OR a write-only secret - // field was entered (secret fields are blank in the baseline, so a non-empty - // value naturally shows up as a diff). Drives whether Save is enabled. - const isDirty = - !structuralEqual(knowledge, pgFromConfig(ds.knowledgePostgres)) || - !structuralEqual(appPg, pgFromConfig(ds.appPostgres)) || - !structuralEqual(storage, storageFromConfig(ds.convexStorage)); - - function buildPg(form: PgForm) { - return { - host: form.host, - port: Number(form.port) || 5432, - database: form.database, - user: form.user, - sslmode: form.sslmode, - }; - } - - function buildConfig() { - const dataStores: Record = {}; - if (knowledge.enabled) dataStores.knowledgePostgres = buildPg(knowledge); - if (appPg.enabled) dataStores.appPostgres = buildPg(appPg); - dataStores.convexStorage = storage.s3 - ? { - mode: 's3', - region: storage.region, - ...(storage.endpoint ? { endpoint: storage.endpoint } : {}), - forcePathStyle: storage.forcePathStyle, - buckets: { - files: storage.files, - exports: storage.exports, - snapshotImports: storage.snapshotImports, - modules: storage.modules, - search: storage.search, - }, - } - : { mode: 'local' }; - return { version: 1, dataStores }; - } - - function buildSecrets() { - const out: Record = {}; - if (knowledge.enabled && knowledge.password) - out['dataStores.knowledgePostgres.password'] = knowledge.password; - if (appPg.enabled && appPg.password) - out['dataStores.appPostgres.password'] = appPg.password; - if (storage.s3) { - if (storage.accessKeyId) - out['dataStores.convexStorage.accessKeyId'] = storage.accessKeyId; - if (storage.secretAccessKey) - out['dataStores.convexStorage.secretAccessKey'] = - storage.secretAccessKey; - } - return out; - } - - // Persist the hash-guarded config. Config-first (saved before any secret) is - // load-bearing: on a concurrent change the stale `expectedHash` aborts HERE — - // before any secret is written — so a version conflict can neither orphan a - // secret on disk nor let the secret-save's query invalidation re-baseline - // (and wipe) the operator's unsaved edits. - async function persistConfig() { - await saveConfig.mutateAsync({ - config: buildConfig(), - expectedHash: data?.hash ?? undefined, - }); - } - - // Persist secrets (optionally force-overwriting an undecryptable sidecar). - async function persistSecrets(force: boolean) { - const secrets = buildSecrets(); - if (Object.keys(secrets).length === 0) return; - await saveSecret.mutateAsync({ - secrets, - ...(force ? { force: true } : {}), - }); - } - - // Clear the write-only secret inputs + mark saved. A config change refetches - // and re-baselines the form, but a secret-only save leaves the config hash - // unchanged, so clearing here is what drops the form back to a clean - // (non-dirty) state in that case. - function finishSave() { - setKnowledgeRaw((k) => ({ ...k, password: '' })); - setAppPgRaw((a) => ({ ...a, password: '' })); - setStorageRaw((s) => ({ ...s, accessKeyId: '', secretAccessKey: '' })); - setSavedOk(true); - } - - async function onSave() { - setSaving(true); - setError(null); - setSavedOk(false); - try { - await persistConfig(); - await persistSecrets(false); - finishSave(); - } catch (err) { - const mapped = mapDeploymentError(err, t); - setError(mapped.message); - // An undecryptable existing secrets sidecar can only be recovered by an - // explicit force-overwrite — offer it via a confirm dialog. The config is - // already persisted (config-first), so the retry re-saves ONLY the secret. - if (mapped.canForceOverwrite) setForceOverwriteOpen(true); - } finally { - setSaving(false); - } - } - - async function onForceOverwrite() { - setSaving(true); - setError(null); - try { - // Config is already saved; only the secret sidecar needs the force - // overwrite. Re-saving config here could spuriously version-conflict - // against its own just-written hash before the read query refetches. - await persistSecrets(true); - finishSave(); - } catch (err) { - setError(mapDeploymentError(err, t).message); - } finally { - setSaving(false); - setForceOverwriteOpen(false); - } - } - - async function runTest( - target: 'knowledgePostgres' | 'appPostgres' | 'convexStorage', - ) { - setTesting(target); - try { - const form = - target === 'convexStorage' - ? null - : target === 'knowledgePostgres' - ? knowledge - : appPg; - const config = - target === 'convexStorage' - ? storage.s3 - ? { - mode: 's3', - region: storage.region, - ...(storage.endpoint ? { endpoint: storage.endpoint } : {}), - forcePathStyle: storage.forcePathStyle, - buckets: { - files: storage.files, - exports: storage.exports, - snapshotImports: storage.snapshotImports, - modules: storage.modules, - search: storage.search, - }, - } - : { mode: 'local' } - : buildPg(form ?? emptyPg()); - const res: ConnTestResult = await testConn.mutateAsync({ - target, - config, - ...(form?.password ? { password: form.password } : {}), - }); - setTestResults((prev) => ({ - ...prev, - [target]: { - ok: Boolean(res?.ok), - message: res?.error || res?.hint || undefined, - }, - })); - } catch (err) { - setTestResults((prev) => ({ - ...prev, - [target]: { - ok: false, - message: mapDeploymentError(err, t).message, - }, - })); - } finally { - setTesting(null); - } - } - - // A failed read must not fall through to a blank, editable-looking default - // form — that would imply "no overrides configured" when the truth is unknown. - if (readError) { - return ( - - ); - } - - return ( - // One wrapper (not a fragment) so the page's section-divider rule draws - // the group's leading hairline ABOVE its banners, and the internal - // section-sibling arm of the same rule separates the stores within. - - {canEdit ? ( - void onSave()} - saving={saving} - isDirty={isDirty} - /> - ) : null} - {/* Save status — inline at the top of the group so it's visible next to - the sections it concerns. */} - {error || (savedOk && !isDirty) ? ( - - {error ? : null} - {savedOk && !isDirty ? ( - - {t('dataResidency.saved.title')}{' '} - {t('dataResidency.saved.runPrefix')}{' '} - docker compose restart backend-api backend-worker{' '} - {t('dataResidency.saved.orPrefix')} tale deploy{' '} - {t('dataResidency.saved.tail')} - - } - /> - ) : null} - - ) : null} - - {readOnly || - data?.secretsError === 'encrypted_no_key' || - data?.secretsError === 'unreadable' ? ( - - {readOnly ? ( - // The reason lives in the description (a `` lead), not the - // Alert's title slot — that renders a fixed
and would skip - // heading levels under the section

s below. - - {t('dataResidency.readOnly.title')}{' '} - {t('dataResidency.readOnly.before')}{' '} - TALE_DEPLOYMENT_CONFIG_ADMINS{' '} - {t('dataResidency.readOnly.after')} - {data?.email ? ( - <> - {' '} - {t('dataResidency.readOnly.yourEmail', { - email: data.email, - })} - - ) : null} - - } - /> - ) : null} - {data?.secretsError === 'encrypted_no_key' ? ( - - ) : null} - {data?.secretsError === 'unreadable' ? ( - - ) : null} - - ) : null} - - void runTest('knowledgePostgres')} - testing={testing === 'knowledgePostgres'} - testResult={testResults.knowledgePostgres} - readOnly={readOnly} - note={} - /> - - void runTest('convexStorage')} - testing={testing === 'convexStorage'} - testResult={testResults.convexStorage} - readOnly={readOnly} - /> - - {/* Advanced Convex metadata DB — reuses the Postgres section chrome; its - own header switch toggles `enabled`. Titled "(advanced)" rather than - hidden behind a disclosure so it shares the rhythm of the sections - above. */} - void runTest('appPostgres')} - testing={testing === 'appPostgres'} - testResult={testResults.appPostgres} - readOnly={readOnly} - showSslMode={false} - note={ -

- {t('dataResidency.appDb.databaseNameNote')}{' '} - {t('dataResidency.appDb.sslModeNote')} -

- } - /> - - void onForceOverwrite()} - /> - - ); -} - -/** Deployment-level object storage (Convex blob store). */ -function DeploymentStorageSection({ - storage, - setStorage, - secretState, - onTest, - testing, - testResult, - readOnly, - className, -}: { - storage: StorageForm; - setStorage: (next: StorageForm) => void; - secretState: Record; - onTest: () => void; - testing: boolean; - testResult?: { ok: boolean; message?: string }; - readOnly: boolean; - className?: string; -}) { - const { t } = useT('settings'); - const { t: tCommon } = useT('common'); - const onLabel = t('dataResidency.storage.externalS3'); - const offLabel = t('dataResidency.storage.localLabel'); - return ( - - ) : ( - - - - setStorage({ ...storage, s3: checked }) - } - /> - - ) - } - > - {!storage.s3 ? null : readOnly ? ( // Off = local volume: the header status pill already says so. - -
- - - -
- -
- - - - - -
- - - ) : ( - -
- - setStorage({ ...storage, region: e.target.value }) - } - /> - - setStorage({ ...storage, endpoint: e.target.value }) - } - /> -
- - setStorage({ ...storage, forcePathStyle: checked }) - } - /> - -
- - setStorage({ ...storage, files: e.target.value }) - } - /> - - setStorage({ ...storage, exports: e.target.value }) - } - /> - - setStorage({ ...storage, snapshotImports: e.target.value }) - } - /> - - setStorage({ ...storage, modules: e.target.value }) - } - /> - - setStorage({ ...storage, search: e.target.value }) - } - /> -
- - -
- - setStorage({ ...storage, accessKeyId: e.target.value }) - } - description={ - secretState['dataStores.convexStorage.accessKeyId']?.masked - ? t('dataResidency.storage.accessKeyIdStoredHint', { - masked: - secretState['dataStores.convexStorage.accessKeyId'] - .masked, - }) - : t('dataResidency.storage.writeOnly') - } - /> - - setStorage({ ...storage, secretAccessKey: e.target.value }) - } - description={t('dataResidency.storage.writeOnly')} - /> -
- - - - - - - - )} - - ); -} diff --git a/services/platform/app/features/settings/data-residency/deployment-errors.ts b/services/platform/app/features/settings/data-residency/deployment-errors.ts deleted file mode 100644 index 51c02dfa5c..0000000000 --- a/services/platform/app/features/settings/data-residency/deployment-errors.ts +++ /dev/null @@ -1,106 +0,0 @@ -/** - * Map a thrown error from a deployment-config action into an operator-facing - * message + the structured code. Duck-types `AppError.data` because Vite - * chunk splitting can produce multiple `AppError` class copies that break - * `instanceof` (same rationale as the org-residency mapper it sits beside). - */ - -type Translator = (key: string, options?: Record) => string; - -export interface DeploymentErrorMapping { - /** Structured error code when present (e.g. DEPLOYMENT_VERSION_CONFLICT). */ - code?: string; - /** Operator-facing message, already localized. */ - message: string; - /** - * True when the failure is an undecryptable secrets sidecar that a - * force-overwrite can recover (DEPLOYMENT_SECRET_REFUSED_OVERWRITE). - */ - canForceOverwrite: boolean; -} - -function readBackendErrorData( - err: unknown, -): Record | undefined { - if (err == null || typeof err !== 'object') return undefined; - if (!('data' in err)) return undefined; - const data = err.data; - if (data == null || typeof data !== 'object') return undefined; - // oxlint-disable-next-line typescript/no-unsafe-type-assertion -- runtime-checked above - return data as Record; -} - -function pickString(data: unknown, key: string): string | undefined { - if (data == null || typeof data !== 'object') return undefined; - // oxlint-disable-next-line typescript/no-unsafe-type-assertion -- runtime-checked above - const v = (data as Record)[key]; - return typeof v === 'string' ? v : undefined; -} - -/** Format the `issues` array of an INVALID_DEPLOYMENT_CONFIG error, if present. */ -function formatIssues( - data: Record | undefined, -): string | undefined { - const issues = data?.issues; - if (!Array.isArray(issues) || issues.length === 0) return undefined; - return issues - .map((i) => { - const path = pickString(i, 'path'); - const message = pickString(i, 'message'); - return path ? `${path}: ${message ?? ''}`.trim() : (message ?? ''); - }) - .filter(Boolean) - .join('; '); -} - -export function mapDeploymentError( - err: unknown, - t: Translator, -): DeploymentErrorMapping { - const data = readBackendErrorData(err); - const code = pickString(data, 'code'); - const serverMessage = pickString(data, 'message'); - const fallback = - serverMessage ?? (err instanceof Error ? err.message : String(err)); - - switch (code) { - case 'DEPLOYMENT_VERSION_CONFLICT': - return { - code, - message: t('dataResidency.errors.versionConflict'), - canForceOverwrite: false, - }; - case 'INVALID_DEPLOYMENT_CONFIG': { - const issues = formatIssues(data); - return { - code, - message: issues - ? t('dataResidency.errors.invalidConfig', { issues }) - : t('dataResidency.errors.invalidConfigGeneric'), - canForceOverwrite: false, - }; - } - case 'UNAUTHENTICATED': - case 'FORBIDDEN_INSTANCE_ADMIN': - case 'FORBIDDEN_DEPLOYMENT_EDITOR': - return { - code, - message: serverMessage ?? t('dataResidency.errors.forbidden'), - canForceOverwrite: false, - }; - case 'DEPLOYMENT_SECRET_REFUSED_OVERWRITE': - return { - code, - message: t('dataResidency.errors.secretUnreadable'), - canForceOverwrite: true, - }; - case 'DEPLOYMENT_CONFIG_UNREADABLE': - return { - code, - message: serverMessage ?? t('dataResidency.errors.configUnreadable'), - canForceOverwrite: false, - }; - default: - return { code, message: fallback, canForceOverwrite: false }; - } -} diff --git a/services/platform/app/features/settings/data-residency/hooks/mutations.ts b/services/platform/app/features/settings/data-residency/hooks/mutations.ts index c6358e4f85..674f77a83a 100644 --- a/services/platform/app/features/settings/data-residency/hooks/mutations.ts +++ b/services/platform/app/features/settings/data-residency/hooks/mutations.ts @@ -5,40 +5,12 @@ import { useBackendAction } from '@/app/hooks/use-backend-action'; /** * Write hooks for the unified data-residency page. * - * The deployment-level mutations persist the single deployment config file (one - * atomic save, guarded by an optimistic hash) and its SOPS-encrypted secrets; - * the org-level mutations save / test / remove THIS organization's - * object-storage connection. Each save/delete invalidates its matching read so - * the form re-baselines from disk truth. + * Every mutation is org-level: save / test / remove THIS organization's + * knowledge connection, embedding model, and object-storage connection. Each + * save/delete invalidates its matching read so the form re-baselines from disk + * truth. */ -function useInvalidateDeployment() { - const queryClient = useQueryClient(); - return () => - queryClient.invalidateQueries({ queryKey: ['config', 'deployment'] }); -} - -/** Persist the deployment config (validated + optimistic-hash on the server). */ -export function useSaveDeploymentConfig() { - const invalidate = useInvalidateDeployment(); - return useBackendAction('deployment/file_actions:saveDeploymentConfig', { - onSuccess: () => invalidate(), - }); -} - -/** Merge/persist deployment secrets (SOPS-encrypted server-side). */ -export function useSaveDeploymentSecret() { - const invalidate = useInvalidateDeployment(); - return useBackendAction('deployment/file_actions:saveDeploymentSecret', { - onSuccess: () => invalidate(), - }); -} - -/** Probe a candidate data-store connection before saving. */ -export function useTestDeploymentConnection() { - return useBackendAction('deployment/file_actions:testDeploymentConnection'); -} - function useInvalidateOrgObjectStorage(organizationId: string) { const queryClient = useQueryClient(); return () => diff --git a/services/platform/app/features/settings/data-residency/hooks/queries.ts b/services/platform/app/features/settings/data-residency/hooks/queries.ts index 3a48872b48..cd651b5d4e 100644 --- a/services/platform/app/features/settings/data-residency/hooks/queries.ts +++ b/services/platform/app/features/settings/data-residency/hooks/queries.ts @@ -2,28 +2,10 @@ import { useActionQuery } from '@/app/hooks/use-action-query'; import { useBackendQuery } from '@/app/hooks/use-backend-query'; /** - * Read hooks for the unified data-residency page. Both reads are Convex - * ACTIONS (they read config off disk — the deployment config file and the - * per-org JSON connection files), so they go through `useActionQuery` rather - * than `useBackendQuery`. - * - * Read the deployment-level config + masked secret presence + the per-caller - * `canEdit` flag. Deployment-scoped (no org arg). The read is open to any - * organization admin (`read orgSettings`) so they can VIEW where deployment - * data lives; `canEdit` (caller's email ∈ the `TALE_DEPLOYMENT_CONFIG_ADMINS` - * allowlist) is what drives edit-vs-read-only in the UI. - * - * NOTE: `api.deployment.*` is populated by `convex codegen` — run dev/deploy - * after pulling this branch so the generated API includes the deployment module. + * Read hooks for the unified data-residency page. The reads are ACTIONS (they + * read the per-org JSON connection files off disk), so they go through + * `useActionQuery` rather than `useBackendQuery`. */ -export function useReadDeploymentConfig(options?: { enabled?: boolean }) { - return useActionQuery( - ['config', 'deployment'], - 'deployment/file_actions:readDeploymentConfig', - {}, - options, - ); -} /** The org's object-storage connection (masked — never carries credentials). */ export function useOrgObjectStorageConnection(organizationId: string) { diff --git a/services/platform/app/features/settings/data-residency/org-residency-errors.ts b/services/platform/app/features/settings/data-residency/org-residency-errors.ts index 620df9acc0..63d5a794b4 100644 --- a/services/platform/app/features/settings/data-residency/org-residency-errors.ts +++ b/services/platform/app/features/settings/data-residency/org-residency-errors.ts @@ -2,10 +2,8 @@ * Map a thrown error from a per-org data-residency action (knowledge DB, * embedding model, object storage, blob backfill) into an admin-facing * message. Duck-types `AppError.data` because Vite chunk splitting can - * produce multiple `AppError` class copies that break `instanceof` — same - * rationale as `deployment-errors.ts`, which this sits beside (the code set - * differs: these actions gate on org membership and validate a single - * connection, not the deployment file). + * produce multiple `AppError` class copies that break `instanceof`. These + * actions gate on org membership and validate a single connection. */ import { diff --git a/services/platform/app/lib/backend/admin.ts b/services/platform/app/lib/backend/admin.ts index 8e7b77c4b6..63127eedc6 100644 --- a/services/platform/app/lib/backend/admin.ts +++ b/services/platform/app/lib/backend/admin.ts @@ -32,12 +32,6 @@ type TestSsoResult = ReturnsOf<'enterprise_sso/config/actions:testConnection'>; type ParseIdpMetadataResult = ReturnsOf<'enterprise_sso/config/actions:parseIdpMetadata'>; type RegenerateScimResult = ReturnsOf<'scim/mutations:regenerateToken'>; -type DeploymentConfigViewResult = - ReturnsOf<'deployment/file_actions:readDeploymentConfig'>; -type SaveDeploymentConfigResult = - ReturnsOf<'deployment/file_actions:saveDeploymentConfig'>; -type DeploymentTestResult = - ReturnsOf<'deployment/file_actions:testDeploymentConnection'>; type ObjectStorageViewResult = ReturnsOf<'object_storage/actions:getObjectStorageConnection'>; type ObjectStorageProbeResult = @@ -258,10 +252,6 @@ export const adminDataResidencyActionQueries: Record< string, ActionQueryAdapter > = { - 'deployment/file_actions:readDeploymentConfig': () => { - return () => - backendFetch('/deployment/config', {}); - }, 'object_storage/actions:getObjectStorageConnection': (args, ctx) => { const orgId = orgOf(args, ctx); if (orgId === undefined) return null; @@ -517,40 +507,6 @@ export const adminWriteAdapters: Record = { }).then(() => null), invalidate: invalidateSso, }, - 'deployment/file_actions:saveDeploymentConfig': { - run: (args) => - backendFetch('/deployment/config', { - body: { - config: args.config, - ...(typeof args.expectedHash === 'string' - ? { expectedHash: args.expectedHash } - : {}), - }, - }), - invalidate: invalidateDeployment, - }, - 'deployment/file_actions:saveDeploymentSecret': { - run: (args) => - backendFetch<{ ok: boolean }>('/deployment/secrets', { - body: { - secrets: args.secrets, - ...(args.force === true ? { force: true } : {}), - }, - }).then(() => null), - invalidate: invalidateDeployment, - }, - 'deployment/file_actions:testDeploymentConnection': { - run: (args) => - backendFetch('/deployment/test', { - body: { - target: stringArg(args, 'target'), - config: args.config, - ...(typeof args.password === 'string' - ? { password: args.password } - : {}), - }, - }), - }, 'object_storage/actions:saveObjectStorageConnection': { run: (args, ctx) => backendFetch<{ ok: boolean }>('/object-storage/connection', { @@ -626,12 +582,6 @@ export const adminWriteAdapters: Record = { }, }; -function invalidateDeployment( - client: Parameters>[0], -): void { - void client.invalidateQueries({ queryKey: ['config', 'deployment'] }); -} - function invalidateObjectStorage( client: Parameters>[0], args: Record, diff --git a/services/platform/app/lib/backend/contract/deployment.ts b/services/platform/app/lib/backend/contract/deployment.ts deleted file mode 100644 index 80642d5366..0000000000 --- a/services/platform/app/lib/backend/contract/deployment.ts +++ /dev/null @@ -1,130 +0,0 @@ -/** - * `deployment` — the wire contract for the backend calls the app makes into this - * family: one entry per function name, carrying its argument and response - * shapes. Materialized from the shapes the app consumed at the Convex - * retirement, so the hook wrappers stay fully typed with no generated - * `_generated/api` behind them; the adapter rows in `../deployment.ts` are what - * actually serve them. - */ - -export interface DeploymentContract { - 'deployment/file_actions:readDeploymentConfig': { - kind: 'action'; - args: Record; - returns: { - config: { - version: 1; - dataStores?: { - knowledgePostgres?: { - host: string; - port: number; - database: string; - user: string; - sslmode: - | 'disable' - | 'prefer' - | 'require' - | 'verify-ca' - | 'verify-full'; - }; - convexStorage?: - | { mode: 'local' } - | { - mode: 's3'; - region: string; - forcePathStyle: boolean; - buckets: { - files: string; - exports: string; - snapshotImports: string; - modules: string; - search: string; - }; - endpoint?: string; - }; - appPostgres?: { - host: string; - port: number; - database: string; - user: string; - sslmode: - | 'disable' - | 'prefer' - | 'require' - | 'verify-ca' - | 'verify-full'; - }; - }; - sandboxRuntime?: { - tier?: 'runc' | 'gvisor' | 'sysbox' | 'kata'; - dockerInContainer?: boolean; - dockerBuildCache?: boolean; - }; - }; - hash: null | string; - secrets: Record; - secretsError: undefined | 'encrypted_no_key' | 'unreadable'; - canEdit: boolean; - email: undefined | string; - }; - }; - 'deployment/file_actions:saveDeploymentConfig': { - kind: 'action'; - args: { expectedHash?: string; config: unknown }; - returns: { hash: string }; - }; - 'deployment/file_actions:saveDeploymentSecret': { - kind: 'action'; - args: { force?: boolean; secrets: Record }; - returns: null; - }; - 'deployment/file_actions:testDeploymentConnection': { - kind: 'action'; - args: { - password?: string; - config: unknown; - target: 'knowledgePostgres' | 'convexStorage' | 'appPostgres'; - }; - returns: - | { - ok: boolean; - error: string; - hint?: undefined; - latencyMs?: undefined; - httpStatus?: undefined; - version?: undefined; - vectorAvailable?: undefined; - paradedbAvailable?: undefined; - } - | { - ok: boolean; - hint: string; - error?: undefined; - latencyMs?: undefined; - httpStatus?: undefined; - version?: undefined; - vectorAvailable?: undefined; - paradedbAvailable?: undefined; - } - | { - ok: boolean; - latencyMs: number; - httpStatus: number; - hint: string; - error?: undefined; - version?: undefined; - vectorAvailable?: undefined; - paradedbAvailable?: undefined; - } - | { - ok: boolean; - latencyMs: undefined | number; - version: undefined | string; - vectorAvailable: undefined | boolean; - paradedbAvailable: undefined | boolean; - error: undefined | string; - hint: undefined | string; - httpStatus?: undefined; - }; - }; -} diff --git a/services/platform/app/lib/backend/contract/index.ts b/services/platform/app/lib/backend/contract/index.ts index 0a8b229c07..8943933a28 100644 --- a/services/platform/app/lib/backend/contract/index.ts +++ b/services/platform/app/lib/backend/contract/index.ts @@ -25,7 +25,6 @@ import type { CollabContract } from './collab'; import type { ConnectorCredentialsContract } from './connector-credentials'; import type { ContactsContract } from './contacts'; import type { ConversationsContract } from './conversations'; -import type { DeploymentContract } from './deployment'; import type { DocumentsContract } from './documents'; import type { EnterpriseSsoContract } from './enterprise-sso'; import type { FeedbackContract } from './feedback'; @@ -77,7 +76,6 @@ export interface BackendContract ConnectorCredentialsContract, ContactsContract, ConversationsContract, - DeploymentContract, DocumentsContract, EnterpriseSsoContract, FeedbackContract, diff --git a/services/platform/messages/de.yml b/services/platform/messages/de.yml index 5d88b9828d..489d92786e 100644 --- a/services/platform/messages/de.yml +++ b/services/platform/messages/de.yml @@ -6105,24 +6105,7 @@ settings: dataResidency: readOnly: title: Nur Lesezugriff - before: Bearbeitung ist auf die in - after: - (in der Deployment-.env) aufgeführten Operatoren beschränkt. Bitte einen - Operator, deine E-Mail dort hinzuzufügen, und starte neu, um die - Bearbeitung zu aktivieren. - yourEmail: Deine Anmelde-E-Mail ist {email}. - secretsEncryptedNoKey: - Die gespeicherten Secrets sind SOPS-verschlüsselt, aber - es ist kein age-Schlüssel konfiguriert — bestehende Secrets lassen sich - nicht lesen. - secretsUnreadable: - Die gespeicherten Secrets sind vorhanden, lassen sich aber - nicht entschlüsseln (beschädigter Chiffretext oder ein fehlender/falscher - age-Schlüssel). Gib die Werte erneut ein und speichere, um sie zu - überschreiben. externalPostgres: Externes Postgres - status: - builtIn: Eingebaut field: host: Host port: Port @@ -6131,7 +6114,6 @@ settings: sslMode: SSL-Modus password: Passwort password: - storedHint: 'Gespeichert: {masked} — leer lassen, um zu behalten' storedNoPreviewHint: Ein Wert ist gespeichert — leer lassen, um ihn zu behalten, oder einen neuen eingeben, um ihn zu ersetzen. @@ -6142,78 +6124,18 @@ settings: ok: OK failed: Fehlgeschlagen knowledge: - title: Wissensdatenbank (RAG) - description: Die Postgres-Datenbank hinter Wissenssuche und -abruf. paradeDbNote: Die externe Wissensdatenbank muss ParadeDB (pgvector + pg_search) verwenden, damit die volle Hybrid-Suche funktioniert; reines pgvector fällt auf reine Vektor-Suche zurück. storage: - title: Dateispeicher (hochgeladene Dokumente) - description: Wo hochgeladene Dokumente und erzeugte Exporte gespeichert werden. externalS3: Externes S3 - localLabel: Lokales Volume region: Region endpoint: Endpunkt (MinIO/R2; leer für AWS) forcePathStyle: Path-Style erzwingen (MinIO/R2) - bucketsLabel: Buckets - credentialsLabel: Zugangsdaten - bucket: - files: Files-Bucket - exports: Exports-Bucket - snapshotImports: Snapshot-Imports-Bucket - modules: Modules-Bucket - search: Search-Bucket accessKeyId: Access Key ID - accessKeyIdStoredHint: 'Gespeichert: {masked} — leer lassen, um zu behalten' writeOnly: Nur Schreiben secretAccessKey: Secret Access Key - testReachability: Erreichbarkeit testen - reachable: Erreichbar - greenfieldWarning: - 'S3-Speicher ist Greenfield: Der Wechsel von lokal migriert - bestehende hochgeladene Dateien NICHT. Lege das beim ersten Deploy fest, - oder kopiere die lokalen Blobs separat in den Bucket.' - appDb: - summary: Anwendungsdatenbank (erweitert) - description: - Convex' interner Metadatenspeicher. Die meisten Deployments sollten - hier die mitgelieferte Datenbank belassen. - databaseNameNote: Das Convex-Backend leitet seinen Datenbanknamen aus der - Instanzkonfiguration ab, daher muss das externe Postgres diese Datenbank - bereits enthalten. Das Feld „Datenbank“ oben wird nur für den - Verbindungstest verwendet. - sslModeNote: Der TLS-Modus für diese Datenbank wird vom Convex-Treiber - vorgegeben und kann hier nicht konfiguriert werden. - saved: - title: Gespeichert — zum Anwenden neu starten. - runPrefix: 'Befehl:' - orPrefix: (oder - tail: für Zero-Downtime). - saving: Speichern… - saveDeployment: Deployment speichern - forceOverwrite: - title: Unlesbare Secrets überschreiben? - description: - Die vorhandene Secrets-Datei lässt sich nicht entschlüsseln. Beim - Überschreiben wird sie durch die von dir eingegebenen Werte ersetzt — - alle anderen darin enthaltenen Secrets gehen verloren. Fortfahren? - confirm: Überschreiben - errors: - versionConflict: - Die Deployment-Konfiguration wurde in einer anderen Sitzung - geändert. Lade neu, um den aktuellen Stand zu sehen, und wende deine - Änderungen dann erneut an. - invalidConfig: 'Die Konfiguration ist ungültig: {issues}' - invalidConfigGeneric: - Die Konfiguration ist ungültig. Überprüfe die Felder und - versuche es erneut. - forbidden: Du bist nicht berechtigt, die Deployment-Konfiguration zu bearbeiten. - secretUnreadable: - Die vorhandene Secrets-Datei lässt sich nicht entschlüsseln. - Bestätige das Überschreiben, um sie zu ersetzen. - configUnreadable: Die Deployment-Konfiguration auf dem Datenträger lässt sich nicht lesen. - readFailed: 'Die Deployment-Konfiguration konnte nicht geladen werden: {error}' orgKnowledge: title: Wissensdatenbank description: diff --git a/services/platform/messages/en.yml b/services/platform/messages/en.yml index 22bcf817f8..70a8458aa2 100644 --- a/services/platform/messages/en.yml +++ b/services/platform/messages/en.yml @@ -6207,20 +6207,7 @@ settings: dataResidency: readOnly: title: Read-only access - before: Editing is restricted to the operators listed in - after: - (in the deployment .env). Ask an operator to add your email there and - restart to enable editing. - yourEmail: Your sign-in email is {email}. - secretsEncryptedNoKey: - The stored secrets are SOPS-encrypted but no age key is - configured — existing secrets can't be read. - secretsUnreadable: The stored secrets exist but can't be decrypted (corrupt - ciphertext, or a missing/incorrect age key). Re-enter the values and save - to overwrite them. externalPostgres: External Postgres - status: - builtIn: Built-in field: host: Host port: Port @@ -6229,7 +6216,6 @@ settings: sslMode: SSL mode password: Password password: - storedHint: 'Stored: {masked} — leave blank to keep' storedNoPreviewHint: A value is stored — leave blank to keep it, or enter a new one to replace it. @@ -6240,75 +6226,18 @@ settings: ok: OK failed: Failed knowledge: - title: Knowledge database (RAG) - description: The Postgres database backing knowledge search and retrieval. paradeDbNote: The external knowledge database must run ParadeDB (pgvector + pg_search) for full hybrid search; plain pgvector degrades to vector-only. storage: - title: File storage (uploaded documents) - description: Where uploaded documents and generated exports are kept. externalS3: External S3 - localLabel: Local volume region: Region endpoint: Endpoint (MinIO/R2; blank for AWS) forcePathStyle: Force path-style (MinIO/R2) - bucketsLabel: Buckets - credentialsLabel: Credentials - bucket: - files: Files bucket - exports: Exports bucket - snapshotImports: Snapshot-imports bucket - modules: Modules bucket - search: Search bucket accessKeyId: Access key ID - accessKeyIdStoredHint: 'Stored: {masked} — leave blank to keep' writeOnly: Write-only secretAccessKey: Secret access key - testReachability: Test reachability - reachable: Reachable - greenfieldWarning: - 'S3 storage is greenfield: switching from local does NOT - migrate existing uploaded files. Set this at initial deploy, or copy the - local blobs into the bucket separately.' - appDb: - summary: Application database (advanced) - description: - Convex's internal metadata store. Most deployments should leave - this on the bundled database. - databaseNameNote: - The Convex backend derives its database name from the instance - configuration, so the external Postgres must already contain that - database. The Database field above is used only for the connection test. - sslModeNote: - TLS mode for this database is fixed by the Convex driver and cannot - be configured here. - saved: - title: Saved — restart to apply. - runPrefix: Run - orPrefix: (or - tail: for zero-downtime). - saving: Saving… - saveDeployment: Save deployment - forceOverwrite: - title: Overwrite unreadable secrets? - description: - The existing secrets file can't be decrypted. Overwriting replaces - it with the values you entered — any other secrets it held will be lost. - Continue? - confirm: Overwrite - errors: - versionConflict: - The deployment config changed in another session. Reload to see - the latest, then re-apply your changes. - invalidConfig: 'The configuration is invalid: {issues}' - invalidConfigGeneric: The configuration is invalid. Check the fields and try again. - forbidden: You're not authorized to edit deployment configuration. - secretUnreadable: The existing secrets file can't be decrypted. Confirm - overwrite to replace it. - configUnreadable: The deployment configuration on disk can't be read. - readFailed: "Couldn't load the deployment configuration: {error}" orgKnowledge: title: Knowledge database description: diff --git a/services/platform/messages/fr.yml b/services/platform/messages/fr.yml index 567d08974d..b217d67820 100644 --- a/services/platform/messages/fr.yml +++ b/services/platform/messages/fr.yml @@ -6222,20 +6222,7 @@ settings: dataResidency: readOnly: title: Accès en lecture seule - before: La modification est réservée aux opérateurs listés dans - after: - (dans le .env du déploiement). Demande à un opérateur d'y ajouter ton - courriel, puis redémarre pour activer la modification. - yourEmail: Ton courriel de connexion est {email}. - secretsEncryptedNoKey: - Les secrets stockés sont chiffrés avec SOPS, mais aucune - clé age n'est configurée — les secrets existants ne peuvent pas être lus. - secretsUnreadable: Les secrets stockés existent mais ne peuvent pas être - déchiffrés (texte chiffré corrompu, ou clé age manquante/incorrecte). - Saisis à nouveau les valeurs et enregistre pour les écraser. externalPostgres: Postgres externe - status: - builtIn: Intégrée field: host: Hôte port: Port @@ -6247,7 +6234,6 @@ settings: storedNoPreviewHint: Une valeur est stockée — laisse vide pour la conserver, ou saisis-en une nouvelle pour la remplacer. - storedHint: 'Stocké : {masked} — laisse vide pour conserver' writeOnlyHint: Écriture seule ; laisse vide pour conserver la valeur stockée testConnection: Tester la connexion testing: Test… @@ -6255,78 +6241,18 @@ settings: ok: OK failed: Échec knowledge: - title: Base de connaissances (RAG) - description: - La base Postgres qui alimente la recherche et la récupération de - connaissances. paradeDbNote: La base de connaissances externe doit utiliser ParadeDB (pgvector + pg_search) pour la recherche hybride complète ; pgvector seul se limite à la recherche vectorielle. storage: - title: Stockage de fichiers (documents téléversés) - description: L'emplacement où sont stockés les documents téléversés et les - exports générés. externalS3: S3 externe - localLabel: Volume local region: Région endpoint: Endpoint (MinIO/R2 ; vide pour AWS) forcePathStyle: Forcer le path-style (MinIO/R2) - bucketsLabel: Buckets - credentialsLabel: Identifiants - bucket: - files: Bucket Files - exports: Bucket Exports - snapshotImports: Bucket Snapshot-Imports - modules: Bucket Modules - search: Bucket Search accessKeyId: Access Key ID - accessKeyIdStoredHint: 'Stocké : {masked} — laisse vide pour conserver' writeOnly: Écriture seule secretAccessKey: Secret Access Key - testReachability: Tester l'accessibilité - reachable: Accessible - greenfieldWarning: - 'Le stockage S3 est greenfield : passer du local ne migre PAS - les fichiers déjà téléversés. Définis-le au premier déploiement, ou - copie les blobs locaux dans le bucket séparément.' - appDb: - summary: Base de données applicative (avancé) - description: Le magasin de métadonnées interne de Convex. La plupart des - déploiements devraient conserver la base fournie. - databaseNameNote: - Le backend Convex déduit le nom de sa base de données de la - configuration de l'instance ; le Postgres externe doit donc déjà - contenir cette base. Le champ « Base de données » ci-dessus sert - uniquement au test de connexion. - sslModeNote: - Le mode TLS de cette base de données est fixé par le pilote Convex - et ne peut pas être configuré ici. - saved: - title: Enregistré — redémarre pour appliquer. - runPrefix: Exécute - orPrefix: (ou - tail: pour le Zero-Downtime). - saving: Enregistrement… - saveDeployment: Enregistrer le déploiement - forceOverwrite: - title: Écraser les secrets illisibles ? - description: Le fichier de secrets existant ne peut pas être déchiffré. - L'écraser le remplace par les valeurs que tu as saisies — tout autre - secret qu'il contenait sera perdu. Continuer ? - confirm: Écraser - errors: - versionConflict: La configuration du déploiement a changé dans une autre - session. Recharge pour voir l'état le plus récent, puis réapplique tes - modifications. - invalidConfig: 'La configuration est invalide : {issues}' - invalidConfigGeneric: La configuration est invalide. Vérifie les champs et réessaie. - forbidden: Tu n'es pas autorisé à modifier la configuration du déploiement. - secretUnreadable: - Le fichier de secrets existant ne peut pas être déchiffré. - Confirme l'écrasement pour le remplacer. - configUnreadable: La configuration du déploiement sur le disque ne peut pas être lue. - readFailed: 'Impossible de charger la configuration du déploiement : {error}' orgKnowledge: title: Base de connaissances description: From 66b0d404c3a1e774dabba4e53d607f998d9574d7 Mon Sep 17 00:00:00 2001 From: larryro <371767072@qq.com> Date: Sun, 6 Sep 2026 11:05:42 +0800 Subject: [PATCH 3/8] docs(docs): describe data residency as env defaults plus per-org lanes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The self-hosted data-residency page promised that Settings > Data residency relocates the deployment's knowledge database, file storage and application database, that the backend reads deployment.json at boot and derives its connections, and that a restart applies the change. None of that was true — nothing read the saved section. Rewrite the page (en/de/fr, same outline) around what the product does: the deployment defaults are environment variables (DATABASE_URL, KNOWLEDGE_DATABASE_URL, OBJECT_STORE_*) set at deploy time, and Settings > Data residency is the per-organization surface (knowledge connection, embedding model, object storage, blob backfill) that applies live. Name the retired dataStores block and what happens to a leftover one. - environment reference: TALE_DEPLOYMENT_CONFIG_ADMINS gates writes to deployment.yml through the API, not data residency - frontmatter manifest regenerated (build:search-index) - core/deployment/editors.ts: header no longer claims the allowlist edits data-residency stores Finding: lib-shared-schemas-1. --- .../configuration/data-residency.md | 63 ++++--------- .../configuration/environment-reference.md | 2 +- .../configuration/data-residency.md | 63 ++++--------- .../configuration/environment-reference.md | 2 +- .../configuration/data-residency.md | 93 +++++++------------ .../configuration/environment-reference.md | 2 +- services/docs/app/content/frontmatter.json | 6 +- .../backend/core/deployment/editors.ts | 11 ++- 8 files changed, 81 insertions(+), 161 deletions(-) diff --git a/docs/de/self-hosted/configuration/data-residency.md b/docs/de/self-hosted/configuration/data-residency.md index 6c5f38c245..79a70027ce 100644 --- a/docs/de/self-hosted/configuration/data-residency.md +++ b/docs/de/self-hosted/configuration/data-residency.md @@ -1,43 +1,35 @@ --- title: Datenresidenz -description: Richte die Wissensdatenbank, die Anwendungsdatenbank und den Speicher für hochgeladene Dateien einer selbst gehosteten Tale-Installation auf Infrastruktur aus, die du selbst kontrollierst — von Administratoren unter Einstellungen > Datenresidenz konfiguriert und beim Neustart angewendet. +description: Wo eine selbst gehostete Tale-Installation ihre Daten hält, wie du die Deployment-Defaults beim Deploy setzt und wie eine einzelne Organisation ihren Wissens-Korpus und ihre hochgeladenen Dateien auf eigene Infrastruktur ausrichtet — live, ohne Neustart. --- -Eine selbst gehostete Tale-Installation läuft auf Infrastruktur, die du ohnehin schon kontrollierst, also liegen ihre Daten standardmäßig auf deinen Hosts. **Datenresidenz** ist für den Fall gedacht, dass du einzelne Datenspeicher auf dein eigenes verwaltetes Postgres oder deinen Objektspeicher ausrichten willst statt auf die mitgelieferten Container — etwa um Dokumenttext in einer Datenbank zu halten, die dein Team betreibt, oder hochgeladene Dateien in deinem eigenen S3-Bucket. Der Wissens-Korpus läuft genau deshalb als eigener Container (`knowledge-db`), damit er sich unabhängig von der operativen Datenbank verlagern oder ersetzen lässt — er ist der Speicher, um den sich die meisten Residenz-Anforderungen drehen. Administratoren konfigurieren das unter **Einstellungen > Datenresidenz**; die Änderung wird in eine einzige Konfigurationsdatei auf Deployment-Ebene geschrieben und **greift, sobald die betroffenen Container neu starten**. +Eine selbst gehostete Tale-Installation läuft auf Infrastruktur, die du ohnehin schon kontrollierst, also liegen ihre Daten standardmäßig auf deinen Hosts. **Datenresidenz** ist für den Fall gedacht, dass ein Speicher an einem bestimmten Ort liegen muss — Dokumenttext in einer Datenbank, die dein Team betreibt, hochgeladene Dateien in deinem eigenen S3-Bucket, der Korpus eines Mandanten isoliert von allen anderen. Tale beantwortet das auf zwei Ebenen: mit den **Deployment-Defaults**, die sich jede Organisation teilt und die du beim Deploy per Umgebungsvariablen setzt, und mit den **Verbindungen pro Organisation**, die ein Org-Admin live unter **Einstellungen > Datenresidenz** verwaltet. -Diese Seite behandelt, was sich verlagern lässt, die eine Voraussetzung, die zubeißt (ParadeDB), wie die Konfiguration abgelegt und angewendet wird, und wie du sicher neu startest. +Diese Seite behandelt, was wo liegt, wie du die Deployment-Defaults verlagerst, die eine Voraussetzung, die zubeißt (ParadeDB), und die Wissens- und Objektspeicher-Wege pro Organisation — einschließlich des Verschiebens vorhandener Dateien einer Organisation. -## Bearbeitung aktivieren +## Wo die Daten des Deployments liegen -**Einstellungen > Datenresidenz** ist eine einzige Seite mit zwei Arten von Abschnitten: den deployment-weiten Speichern, die sich alle Organisationen teilen, und den Speichern, die eine einzelne Organisation selbst mitbringt. Jeder Abschnitt erscheint lesend oder bearbeitbar, je nachdem, was die lesende Person ändern darf, und die Seite benennt den Zustand. Ansehen darf jeder Owner oder Admin einer Organisation; die **deployment-weiten Speicher bearbeiten** — einen Datenspeicher umlenken, Secrets speichern, einen Verbindungstest laufen lassen oder einen Neustart auslösen — darf nur eine benannte Allowlist von Operatoren. Trage deren Anmelde-E-Mails (kommagetrennt) in `.env` ein und starte neu: +Drei Speicher, jeder mit eigener Umgebungsvariable. Eine nicht gesetzte Variable heißt „nimm den mitgelieferten Container", eine frische Installation ohne Overrides bleibt also unverändert. -```bash -TALE_DEPLOYMENT_CONFIG_ADMINS=alice@example.com,bob@example.com -``` +- **Wissensdatenbank** — der Wissens-Korpus: Dokumentmetadaten, der extrahierte Chunk-Text, Embeddings, der BM25-Index, der semantische Cache und die gecrawlten Webseiten. Sie kommt als mitgelieferter Container `knowledge-db` (`tale_knowledge`, mit den Schemata `private_knowledge` und `public_web`) und ist der Speicher, um den sich die meisten Residenz-Anforderungen drehen, weil er deinen Dokumentinhalt hält. `KNOWLEDGE_DATABASE_URL` richtet das Backend stattdessen auf ein verwaltetes Postgres von dir aus; die Datenbank darf leer starten — das Backend legt seine Schemata beim ersten Zugriff an. +- **Dateispeicher** — wo hochgeladene Dateien (die Original-Blobs) liegen. Standardmäßig im mitgelieferten Object-Store (dem Service `object-store`, auf eigenem Volume); die `OBJECT_STORE_*`-Variablen richten das Backend stattdessen auf einen externen S3-kompatiblen Bucket aus. Der Wechsel ist Greenfield: Blobs, die schon im mitgelieferten Store liegen, werden nicht kopiert — setze ihn bei der ersten Installation, oder kopiere das Volume vorab außerhalb von Tale in den Bucket. +- **Anwendungsdatenbank** — der operative Speicher hinter Agents, Runs und dem Audit-Log (der mitgelieferte Container `db`, die Datenbank `tale_app`). `DATABASE_URL` verlagert sie; der Datenbankname ist standardmäßig `tale_app` (Override mit `APP_DB_NAME`). -Ist die Allowlist leer oder nicht gesetzt, zeigen die Deployment-Abschnitte Administratoren die aktuelle Konfiguration weiterhin an, aber nur lesend — die Kopfzeilen-Aktionen **Deployment speichern** und **Anwenden & neu starten** erscheinen nur für Operatoren auf der Allowlist. Nur ein angemeldeter Admin, dessen E-Mail auf der Liste steht, bekommt diese Abschnitte bearbeitbar; die Seite nennt dir, welche E-Mail einzutragen ist. Die Entrypoints lesen die Konfigurationsdatei unabhängig von der Allowlist, also kann ein Operator, der die Datei lieber direkt auf der Platte bearbeitet, das tun, ohne UI-Bearbeiter zu benennen. +Die Variablen stehen in der `.env` des Deployments und werden gelesen, wenn die Backend-Container starten — ändere eine und rolle dann mit `tale deploy` (Zero-Downtime, Blue-Green) oder `docker compose restart backend-api backend-worker`. Jede Variable, ihr Default und ihre genaue Form stehen in der [Umgebungsreferenz](/de/self-hosted/configuration/environment-reference). Nichts in der App schreibt diese Werte: Frühere Releases hatten unter Einstellungen > Datenresidenz einen deployment-weiten Speicher-Abschnitt, der einen `dataStores`-Block in `deployment.yml` speicherte — aber kein Boot-Pfad las ihn. Der Abschnitt ist weg; ein übrig gebliebener `dataStores`-Block in einer bestehenden `deployment.yml` wird ignoriert und beim nächsten Speichern der Datei entfernt. -## Was du verlagern kannst - -Drei Speicher, jeder unabhängig und optional. Eine fehlende Einstellung bedeutet „nimm den mitgelieferten Default" — eine frische Installation ohne Konfiguration bleibt also unverändert. - -- **Wissensdatenbank** — der Wissens-Korpus: Dokumentmetadaten, der extrahierte Chunk-Text, Embeddings, der BM25-Index, der semantische Cache und die gecrawlten Webseiten. Sie kommt als mitgelieferter `knowledge-db`-Container (`tale_knowledge`, mit den Schemata `private_knowledge` und `public_web`) und ist der Speicher, um den sich die meisten Residenz-Anforderungen drehen, weil er deinen Dokumentinhalt hält. Richte ihn auf dein eigenes verwaltetes Postgres aus, um den Korpus auf Infrastruktur zu halten, die dein Team betreibt. -- **Dateispeicher** — wo hochgeladene Dateien (die ursprünglichen Blobs) liegen. Standardmäßig liegen sie im mitgelieferten Objektspeicher des Stacks (Dienst `object-store`, auf einem eigenen Volume); du kannst sie auf einen externen S3-kompatiblen Bucket ausrichten. -- **Anwendungsdatenbank** (erweitert) — der operative Speicher hinter Agents, Runs und dem Audit-Log (der mitgelieferte `db`-Container, die `tale_app`-Datenbank). Ihn zu verlagern zeigt die `DATABASE_URL` des Backends auf dein eigenes Postgres; der Datenbankname ist standardmäßig `tale_app` (überschreib ihn mit `APP_DB_NAME`), das externe Postgres muss also eine Datenbank dieses Namens enthalten. - -> Hinweis: Die Wissensdatenbank und die Anwendungsdatenbank sind zwei separate Postgres-Instanzen — die eine zu verschieben rührt die andere nicht an. Die Wissensdatenbank zu verlagern verschiebt den extrahierten Text und die Embeddings; die ursprünglich hochgeladenen Dateien wandern erst mit, wenn du auch den **Dateispeicher** auf S3 ausrichtest. +> Hinweis: Die Wissensdatenbank und die Anwendungsdatenbank sind zwei getrennte Postgres-Instanzen — die eine zu verlagern berührt die andere nicht. Verlagerst du die Wissensdatenbank, wandern der extrahierte Text und die Embeddings; die hochgeladenen Originaldateien wandern nur, wenn du auch den **Dateispeicher** verlagerst. ## Die ParadeDB-Voraussetzung -Die Wissensdatenbank nutzt zwei Postgres-Erweiterungen: `vector` (pgvector) für Embeddings und `pg_search` (ParadeDB) für die Volltext-/BM25-Hybrid-Suche. Ein externes Wissens-Postgres **muss ParadeDB ausführen** (das beide bündelt), damit die Suchqualität voll erhalten bleibt. Richtest du es auf ein schlichtes Postgres aus, das nur `pgvector` hat, funktionieren Indexierung und Vektor-Suche weiter, aber die Hybrid-Suche fällt auf **reine Vektor-Suche** zurück — die BM25-Hälfte wird still übersprungen. Der Knopf **Verbindung testen** meldet die Verfügbarkeit von `pgvector` und `pg_search`, damit du das siehst, bevor du dich festlegst. Die externe Wissensdatenbank muss bereits existieren (sie kann jeden Namen tragen, den du einträgst — `tale_knowledge` per Konvention) mit den Schemata `private_knowledge` und `public_web`; die Baseline-Schema-Migrationen leben in [`services/db/migrations/`](https://github.com/tale-project/tale/tree/main/services/db/migrations) und werden per dbmate angewendet, wenn die Datenbank hochkommt. +Die Wissensdatenbank nutzt zwei Postgres-Erweiterungen: `vector` (pgvector) für Embeddings und `pg_search` (ParadeDB) für die hybride Volltext-/BM25-Suche. Ein externes Wissens-Postgres — der Deployment-Default oder das eigene einer Organisation — **muss ParadeDB fahren** (das beide bündelt), damit die Suche ihre volle Qualität hat. Zeigst du auf ein einfaches Postgres mit nur `pgvector`, funktionieren Indexierung und Vektorsuche weiter, die hybride Suche fällt aber auf **reine Vektorsuche** zurück: Das BM25-Bein wird stillschweigend übersprungen. Der Button **Verbindung testen** pro Organisation meldet die Verfügbarkeit von `pgvector` und `pg_search`, du siehst das also, bevor du dich festlegst; für den Deployment-Default prüfst du die Erweiterungen auf der Zieldatenbank, bevor du `KNOWLEDGE_DATABASE_URL` änderst. ## Wissensdatenbanken pro Organisation -Die Speicher oben gelten deployment-weit — jede Organisation teilt sie sich. Eine einzelne Organisation kann stattdessen **ihren eigenen** Wissens-Korpus auf ein Postgres ausrichten, das du für sie bereitstellst, während jede andere Org weiter den mitgelieferten `knowledge-db` nutzt. Greif dazu, wenn der Dokument- und Web-Crawl-Inhalt eines Mandanten auf Infrastruktur liegen muss, die vom Rest isoliert ist — eine strengere Residenz-Anforderung, als der Deployment-Default sie erfüllt. +Den Deployment-Default teilt sich jede Organisation. Eine einzelne Organisation kann stattdessen **ihren eigenen** Wissens-Korpus auf ein Postgres ausrichten, das du für sie bereitstellst, während jede andere Org weiter den mitgelieferten `knowledge-db` nutzt. Greif dazu, wenn der Dokument- und Web-Crawl-Inhalt eines Mandanten auf Infrastruktur liegen muss, die vom Rest isoliert ist — eine strengere Residenz-Anforderung, als der Deployment-Default sie erfüllt. Der **gesamte** Wissens-Korpus der Org wandert — beide Schemata: `private_knowledge` (Dokumentmetadaten, Chunk-Text, Embeddings und der semantische Cache) und `public_web` (die vom Crawler erfassten Website-Seiten, ihr Chunk-Text und die Embeddings). Nichts in der Wissensdatenbank einer Organisation wird mit einer anderen Organisation geteilt. -Die Verbindung liegt im eigenen Konfigurationsverzeichnis der Organisation, nicht in der Deployment-Datei: +Die Verbindung liegt im eigenen Konfigurationsverzeichnis der Organisation: - `$TALE_CONFIG_DIR//knowledge/connection.json` — Host, Port, Datenbank, Benutzer und sslmode. - `$TALE_CONFIG_DIR//knowledge/connection.secrets.json` — das Passwort, SOPS-verschlüsselt, sobald ein SOPS-Age-Schlüssel konfiguriert ist (siehe [Secrets mit SOPS](/de/self-hosted/configuration/secrets-with-sops)). @@ -45,9 +37,9 @@ Die Verbindung liegt im eigenen Konfigurationsverzeichnis der Organisation, nich Dieselbe ParadeDB-Voraussetzung gilt. Die Org prüft ihre Kandidaten-Datenbank mit einem organisationsweiten Verbindungstest, der die Verfügbarkeit von `pgvector` und `pg_search` meldet, bevor sie umschaltet; ein Ziel mit nur pgvector lässt die Suche dieser Org auf reine Vektor-Suche zurückfallen. Die Datenbank darf leer starten — Tale legt die Schemata `private_knowledge` und `public_web` beim ersten Zugriff an, du wendest die Baseline-Migrationen also nie von Hand an. -Dieser Weg fällt sicher zurück. Eine Organisation ohne `connection.json` nutzt weiter den Deployment-Default `knowledge-db` genau wie zuvor, das Feature ändert also nichts für Orgs, die sich nicht dafür entscheiden. Zwei Organisationen, die auf dieselbe Datenbank zeigen, teilen sich einen Verbindungs-Pool, und — anders als die deployment-weiten Speicher — braucht eine Änderung pro Org keinen Container-Neustart: die nächste Anfrage dieser Org wird auf ihre eigene Datenbank geleitet. +Dieser Weg fällt sicher zurück. Eine Organisation ohne `connection.json` nutzt weiter den Deployment-Default `knowledge-db` genau wie zuvor, das Feature ändert also nichts für Orgs, die sich nicht dafür entscheiden. Zwei Organisationen, die auf dieselbe Datenbank zeigen, teilen sich einen Verbindungs-Pool, und eine Änderung pro Org braucht keinen Container-Neustart: die nächste Anfrage dieser Org wird auf ihre eigene Datenbank geleitet. -Ein Inhaber oder Admin der Organisation kann diese Verbindung auch über die UI verwalten: die Organisations-Abschnitte von **Einstellungen > Datenresidenz** lesen und schreiben genau diese Dateien, mit demselben Verbindungstest vor dem Umschalten. Diese Abschnitte bleiben für Inhaber und Admins der Organisation bearbeitbar, ob die Operator-Allowlist sie nennt oder nicht — die Dateien dahinter gehören der Organisation, nicht dem Deployment. Die JSON-Dateien auf der Platte bleiben die Quelle der Wahrheit — ein Operator, der sie lieber von Hand bearbeitet, braucht keinen UI-Schritt. +**Einstellungen > Datenresidenz** ist genau diese Oberfläche pro Organisation: Ein Inhaber oder Admin der Organisation liest und schreibt dort exakt diese Dateien, mit demselben Verbindungstest vor dem Umschalten. Die JSON-Dateien auf der Platte bleiben die Quelle der Wahrheit — ein Operator, der sie lieber von Hand bearbeitet, braucht keinen UI-Schritt. ### Das Embedding-Modell der Organisation @@ -64,9 +56,9 @@ Die Verbindung liegt neben der Wissens-Verbindung im Konfigurationsverzeichnis d - `$TALE_CONFIG_DIR//object-storage/connection.json` — Region, optionaler Endpoint (für MinIO/R2), Path-Style-Flag, Bucket und ein optionales Key-Präfix. - `$TALE_CONFIG_DIR//object-storage/connection.secrets.json` — das Schlüsselpaar, SOPS-verschlüsselt, sobald ein SOPS-Age-Schlüssel konfiguriert ist (siehe [Secrets mit SOPS](/de/self-hosted/configuration/secrets-with-sops)). -Anders als der deployment-weite S3-Schalter oben ist dieser Weg **nicht** nur für Neuinstallationen: Sobald die Konfiguration existiert, landen neue Uploads im Bucket der Org, während zuvor gespeicherte Dateien lesbar bleiben, wo sie sind — gemischte Referenzen werden unterstützt, du kannst also jederzeit umschalten. Früher gespeicherte Dateien bleiben im Object-Store des Deployments, bis du sie mit dem Blob-Backfill unten verlagerst. Entfernst du die Konfiguration, landen neue Uploads wieder im Deployment-Default; bereits in den Bucket geschriebene Dateien bleiben dort, Tale kann sie aber erst wieder lesen, wenn die Verbindung erneut eingerichtet ist. Ein Neustart ist in keine Richtung nötig. +Dieser Weg ist **nicht** nur für Neuinstallationen: Sobald die Konfiguration existiert, landen neue Uploads im Bucket der Org, während zuvor gespeicherte Dateien lesbar bleiben, wo sie sind — gemischte Referenzen werden unterstützt, du kannst also jederzeit umschalten. Früher gespeicherte Dateien bleiben im Object-Store des Deployments, bis du sie mit dem Blob-Backfill unten verlagerst. Entfernst du die Konfiguration, landen neue Uploads wieder im Deployment-Default; bereits in den Bucket geschriebene Dateien bleiben dort, Tale kann sie aber erst wieder lesen, wenn die Verbindung erneut eingerichtet ist. Ein Neustart ist in keine Richtung nötig. -Org-Admins verwalten auch diese Verbindung in denselben Organisations-Abschnitten von **Einstellungen > Datenresidenz**; der dortige Verbindungstest führt einen echten Hochladen-Lesen-Löschen-Durchlauf gegen den Bucket aus, bevor du dich festlegst. Wie bei der Wissens-Verbindung bleiben die JSON-Dateien die Quelle der Wahrheit. +Org-Admins verwalten auch diese Verbindung unter **Einstellungen > Datenresidenz**; der dortige Verbindungstest führt einen echten Hochladen-Lesen-Löschen-Durchlauf gegen den Bucket aus, bevor du dich festlegst. Wie bei der Wissens-Verbindung bleiben die JSON-Dateien die Quelle der Wahrheit. > **Erlaube den Origin der App in der CORS-Policy des Buckets.** Uploads und Downloads laufen über vorsignierte URLs direkt zwischen Browser und Bucket, der Bucket muss Cross-Origin-Anfragen von der URL deines Deployments also akzeptieren — erlaube diesen Origin mit den Methoden `GET`, `PUT` und `HEAD` sowie allen Request-Headern (Cloudflare R2: **Settings > CORS Policy** des Buckets; AWS S3 und MinIO: die CORS-Konfiguration des Buckets). Der Verbindungstest in der App läuft auf dem Server, nicht im Browser — eine fehlende CORS-Policy zeigt sich deshalb erst später, als fehlgeschlagener Upload. @@ -78,23 +70,4 @@ Ein Org-Admin startet ihn in der UI: Ist die Bucket-Verbindung gespeichert, zeig Der Backfill ist **idempotent** und **org-gebunden**: Er verschiebt nur die Blobs dieser Organisation, überspringt alles, was schon im Bucket liegt, und lässt jede Quelle stehen, bis ihre Kopie verifiziert ist — ein erneuter Lauf nach einer Unterbrechung setzt also sicher fort und schließt jeden Umzug ab, der zwischen verifizierter Kopie und dem Löschen der Quelle abgebrochen wurde. Er geht durch jede Tabelle, die Blob-Referenzen hält: Dokumente samt Historie, hochgeladene Dateien, synthetisierte Sprachausgabe und Video-Link-Transkripte. Er braucht die zuvor konfigurierte Bucket-Verbindung und verweigert den Lauf, wenn der Bucket der Org der Store des Deployments selbst ist — dann gäbe es nichts zu verschieben, und ein abgeschlossener Umzug würde die einzige Kopie löschen. Das ist bewusst **keine** versionierte Framework-Migration — er läuft auf Abruf, pro Organisation, wenn du die Historie eines Mandanten verlagern willst, nicht an einer Release-Grenze. -## Dateispeicher auf S3 - -Externer Dateispeicher nutzt einen einzigen S3-kompatiblen Bucket — Bucket-Name, Region, Anmeldedaten und (für MinIO oder Cloudflare R2) einen Endpunkt mit aktivierter Path-Style-Adressierung. Diese entsprechen den `OBJECT_STORE_*`-Variablen in der [Umgebungsreferenz](/de/self-hosted/configuration/environment-reference). - -> **Nur Greenfield.** Den deployment-weiten Dateispeicher vom mitgelieferten Store auf einen externen Bucket umzustellen migriert die bereits auf dem lokalen Volume liegenden Blobs **nicht** — das Backend sucht sie im Bucket und findet sie nicht. Setze S3 bei der ersten Installation, oder kopiere den vorhandenen lokalen Speicher vorab in den Bucket, bevor du umstellst. - -## Wie die Konfiguration abgelegt wird - -Speichern schreibt zwei Dateien im Konfigurations-Root (nicht unter einem Org-Verzeichnis): - -- `deployment.json` — die nicht geheime Konfiguration (Hosts, Ports, Buckets, Modi). -- `deployment.secrets.json` — die Datenbank-Passwörter und S3-Schlüssel, SOPS-verschlüsselt (siehe [Secrets mit SOPS](/de/self-hosted/configuration/secrets-with-sops)). - -Beim Boot liest das Backend diese und leitet seine Verbindungen ab, bevor es startet. Wissens-Ingestion und Retrieval laufen im Backend-Worker, also ist das Backend das, was die Verbindung zur Wissensdatenbank öffnet — es gibt keinen separaten Retrieval-Dienst zu konfigurieren. Der Vertrag ist **fail-closed**: ein vorhandenes, aber unparsbares `deployment.json`, ein nicht entschlüsselbares Secret oder eine Konfiguration ohne Pflichtfelder **bricht den Start ab**, statt still auf die mitgelieferte Datenbank zurückzufallen — regulierte Daten fehlzuleiten ist schlimmer, als nicht zu starten. Eine fehlende Datei ist der normale Default-Pfad. - -## Eine Änderung anwenden: Neustart - -Die Konfiguration wird beim Boot gelesen, also greift ein Speichern erst, wenn die Backend-Container (`backend-api` und `backend-worker`) neu starten. Führe `docker compose restart backend-api backend-worker` aus, oder `tale deploy` für einen Zero-Downtime-Blue-Green-Roll — die Einstellungsseite zeigt nach dem Speichern dieselben Befehle an. - -Die relevante Umgebungsvariable ist `TALE_DEPLOYMENT_CONFIG_ADMINS` (die kommagetrennte E-Mail-Allowlist der bearbeitungsberechtigten Operatoren). Setze sie in `.env`. Siehe auch [Umgebungsvariablen-Referenz](/de/self-hosted/configuration/environment-reference) und [Secrets mit SOPS](/de/self-hosted/configuration/secrets-with-sops). +Die Deployment-Defaults und ihre Variablen stehen in der [Umgebungsreferenz](/de/self-hosted/configuration/environment-reference); die Secrets-Sidecars pro Organisation folgen [Secrets mit SOPS](/de/self-hosted/configuration/secrets-with-sops). diff --git a/docs/de/self-hosted/configuration/environment-reference.md b/docs/de/self-hosted/configuration/environment-reference.md index b541066666..8983b49750 100644 --- a/docs/de/self-hosted/configuration/environment-reference.md +++ b/docs/de/self-hosted/configuration/environment-reference.md @@ -156,7 +156,7 @@ Optionale Schalter für Features, die standardmässig nicht aktiviert sind. Jede | `TRUSTED_HEADERS_INTERNAL_SECRET` | nicht gesetzt | Shared Secret, das der authentifizierende Proxy mit jeder Trusted-Headers-Anfrage schicken muss. Pflicht, sobald der Modus an ist — ohne Secret verweigert der Endpunkt den Dienst. | | `TRUSTED_SECRET_HEADER` | `Remote-Internal-Secret` | Name des Request-Headers, der das interne Secret trägt. | | `FILE_EVENTS_ENABLED` | `false` | Aktiviert Datei-Watching-Events für die OneDrive-Sync-Connector. | -| `TALE_DEPLOYMENT_CONFIG_ADMINS` | unset | Kommagetrennte E-Mail-Allowlist der Operatoren, die die Datenresidenz bearbeiten dürfen. Leer/nicht gesetzt = nur lesend für alle Admins. | +| `TALE_DEPLOYMENT_CONFIG_ADMINS` | unset | Kommagetrennte E-Mail-Allowlist der Operatoren, die die Deployment-Konfigurationsdatei (`deployment.yml`, heute der Abschnitt zur Sandbox-Runtime) über die API schreiben dürfen. Leer/nicht gesetzt = nur lesend für alle Admins. Die Datenresidenz wird pro Organisation konfiguriert und hängt nicht an dieser Liste. | ## RAG-Retrieval-Tuning diff --git a/docs/en/self-hosted/configuration/data-residency.md b/docs/en/self-hosted/configuration/data-residency.md index d3571d0f9e..cdec7de69e 100644 --- a/docs/en/self-hosted/configuration/data-residency.md +++ b/docs/en/self-hosted/configuration/data-residency.md @@ -1,43 +1,35 @@ --- title: Data residency -description: Point a self-hosted Tale deployment's knowledge database, application database, and uploaded-file storage at infrastructure you control, configured by administrators in Settings > Data residency and applied on restart. +description: Where a self-hosted Tale deployment keeps its data, how the deployment defaults are set at deploy time, and how a single organization points its knowledge corpus and uploaded files at infrastructure of its own — live, without a restart. --- -A self-hosted Tale deployment runs on infrastructure you already control, so its data lives on your hosts by default. **Data residency** is for the case where you want individual data stores pointed at your own managed Postgres or object storage instead of the bundled containers — for example to keep document text in a database your team operates, or uploaded files in your own S3 bucket. The knowledge corpus runs as its own container (`knowledge-db`) precisely so it can be relocated or replaced independently of the operational database — it is the store most residency requirements care about. Administrators configure this in **Settings > Data residency**; the change is written to a single deployment-level config file and **takes effect when the affected containers restart**. +A self-hosted Tale deployment runs on infrastructure you already control, so its data lives on your hosts by default. **Data residency** is for the case where a store has to live somewhere specific — document text in a database your team operates, uploaded files in your own S3 bucket, one tenant's corpus isolated from every other tenant's. Tale answers that at two levels: the **deployment defaults**, which every organization shares and which you set with environment variables when you deploy, and the **per-organization connections** an org admin manages live in **Settings > Data residency**. -This page covers what can be relocated, the one prerequisite that bites (ParadeDB), how the configuration is stored and applied, and how to restart safely. +This page covers what lives where, how to relocate the deployment defaults, the one prerequisite that bites (ParadeDB), and the per-organization knowledge and object-storage lanes, including moving an organization's existing files. -## Enabling editing +## Where the deployment's data lives -**Settings > Data residency** is one page with two kinds of section: the deployment-wide stores every organization shares, and the stores a single organization brings for itself. Each section renders read-only or editable depending on what the reader may change, and the page says which state you are in. Viewing is open to any organization owner or admin; **editing the deployment-wide stores** — repointing a data store, saving secrets, running a connection test, or applying a restart — is restricted to a named allowlist of operators. List their sign-in emails (comma-separated) in `.env` and restart: +Three stores, each with its own environment variable. An unset variable means "use the bundled container", so a fresh deployment with no overrides is unchanged. -```bash -TALE_DEPLOYMENT_CONFIG_ADMINS=alice@example.com,bob@example.com -``` +- **Knowledge database** — the knowledge corpus: document metadata, the extracted chunk text, embeddings, the BM25 index, the semantic cache, and the crawled web pages. It ships as the bundled `knowledge-db` container (`tale_knowledge`, with the `private_knowledge` and `public_web` schemas) and is the store most residency requirements care about, because it holds your document content. `KNOWLEDGE_DATABASE_URL` points the backend at a managed Postgres of your own instead; the database can start empty — the backend creates its schemas on first use. +- **File storage** — where uploaded files (the original blobs) live. By default they sit in the bundled object store (the `object-store` service, on its own volume); the `OBJECT_STORE_*` variables point the backend at an external S3-compatible bucket instead. The switch is greenfield: blobs already written to the bundled store are not copied, so set it at initial deployment or copy the volume into the bucket out of band first. +- **Application database** — the operational store behind agents, runs, and the audit log (the bundled `db` container, the `tale_app` database). `DATABASE_URL` relocates it; the database name defaults to `tale_app` (override with `APP_DB_NAME`). -With the allowlist empty or unset, the deployment sections still show the current configuration to administrators, but read-only — the **Save deployment** and **Apply & restart** header actions appear only for allowlisted operators. Only a signed-in admin whose email is on the list gets those sections editable; the page tells you which email to add. The entrypoints always consume the config file regardless of the allowlist, so an operator who prefers to hand-edit the file on disk can do so without naming any UI editors. +The variables live in the deployment's `.env` and are read when the backend containers start — change one, then roll with `tale deploy` (zero-downtime blue-green) or `docker compose restart backend-api backend-worker`. Every variable, its default and its exact form is in the [Environment reference](/self-hosted/configuration/environment-reference). Nothing in the app writes these values: earlier releases carried a deployment-wide store section in Settings > Data residency that saved a `dataStores` block into `deployment.yml`, but no boot path read it — that section is gone, and a leftover `dataStores` block in an existing `deployment.yml` is ignored and dropped on the file's next save. -## What you can relocate - -Three stores, each independent and optional. An absent setting means "use the bundled default" — so a fresh deployment with no config is unchanged. - -- **Knowledge database** — the knowledge corpus: document metadata, the extracted chunk text, embeddings, the BM25 index, the semantic cache, and the crawled web pages. It ships as the bundled `knowledge-db` container (`tale_knowledge`, with the `private_knowledge` and `public_web` schemas) and is the store most residency requirements care about, because it holds your document content. Point it at your own managed Postgres to keep the corpus on infrastructure your team operates. -- **File storage** — where uploaded files (the original blobs) live. By default they sit in the bundled object store that ships with the stack (the `object-store` service, on its own volume); you can point them at an external S3-compatible bucket. -- **Application database** (advanced) — the operational store behind agents, runs, and the audit log (the bundled `db` container, the `tale_app` database). Relocating it points the backend's `DATABASE_URL` at your own Postgres; the database name defaults to `tale_app` (override with `APP_DB_NAME`), so an external Postgres must contain a database of that name. - -> Note: the knowledge database and the application database are two separate Postgres instances — moving one does not touch the other. Relocating the knowledge database moves the extracted text and embeddings; the original uploaded files move only when you also relocate **File storage** to S3. +> Note: the knowledge database and the application database are two separate Postgres instances — moving one does not touch the other. Relocating the knowledge database moves the extracted text and embeddings; the original uploaded files move only when you also relocate **File storage**. ## The ParadeDB prerequisite -The knowledge database uses two Postgres extensions: `vector` (pgvector) for embeddings and `pg_search` (ParadeDB) for full-text/BM25 hybrid search. An external knowledge Postgres **must run ParadeDB** (which bundles both) for full search quality. If you point it at a plain Postgres that has only `pgvector`, indexing and vector search still work, but hybrid search degrades to **vector-only** — the BM25 leg is silently skipped. The **Test connection** button reports both `pgvector` and `pg_search` availability so you can see this before you commit. The external knowledge database must already exist (it can have any name you enter — `tale_knowledge` by convention) with the `private_knowledge` and `public_web` schemas; the baseline schema migrations live in [`services/db/migrations/`](https://github.com/tale-project/tale/tree/main/services/db/migrations) and are applied via dbmate when the database comes up. +The knowledge database uses two Postgres extensions: `vector` (pgvector) for embeddings and `pg_search` (ParadeDB) for full-text/BM25 hybrid search. An external knowledge Postgres — the deployment default or an organization's own — **must run ParadeDB** (which bundles both) for full search quality. If you point it at a plain Postgres that has only `pgvector`, indexing and vector search still work, but hybrid search degrades to **vector-only**: the BM25 leg is silently skipped. The per-organization **Test connection** button reports both `pgvector` and `pg_search` availability so you see this before you commit; for the deployment default, check the extensions on the target database before you change `KNOWLEDGE_DATABASE_URL`. ## Per-organization knowledge databases -The stores above are deployment-wide — every organization shares them. A single organization can instead point **its own** knowledge corpus at a Postgres you provision for it, while every other org keeps using the bundled `knowledge-db`. Reach for this when one tenant's document and crawled-web content must sit on infrastructure isolated from the rest — a stricter residency requirement than the deployment default satisfies. +The deployment default is shared by every organization. A single organization can instead point **its own** knowledge corpus at a Postgres you provision for it, while every other org keeps using the default `knowledge-db`. Reach for this when one tenant's document and crawled-web content must sit on infrastructure isolated from the rest — a stricter residency requirement than the deployment default satisfies. The org's **entire** knowledge corpus moves — both schemas: `private_knowledge` (document metadata, chunk text, embeddings, and the semantic cache) and `public_web` (the crawler's website pages, their chunk text, and embeddings). Nothing in an organization's knowledge database is shared with any other organization. -The connection lives under the organization's own config directory, not the deployment file: +The connection lives under the organization's own config directory: - `$TALE_CONFIG_DIR//knowledge/connection.json` — host, port, database, user, and sslmode. - `$TALE_CONFIG_DIR//knowledge/connection.secrets.json` — the password, SOPS-encrypted when a SOPS age key is configured (see [Secrets with SOPS](/self-hosted/configuration/secrets-with-sops)). @@ -45,9 +37,9 @@ The connection lives under the organization's own config directory, not the depl The same ParadeDB requirement applies. The org validates its candidate database with an org-scoped connection test that reports `pgvector` and `pg_search` availability before switching, and a plain-pgvector target degrades that org's search to vector-only. The database can start empty — Tale creates the `private_knowledge` and `public_web` schemas on first use, so you never apply the baseline migrations by hand. -This path is fallback-safe. An organization with no `connection.json` keeps using the deployment-default `knowledge-db` exactly as before, so the feature changes nothing for orgs that don't opt in. Two organizations pointed at the same database share one connection pool, and — unlike the deployment-wide stores — a per-org change needs no container restart: the next request for that org routes to its own database. +This path is fallback-safe. An organization with no `connection.json` keeps using the deployment-default `knowledge-db` exactly as before, so the feature changes nothing for orgs that don't opt in. Two organizations pointed at the same database share one connection pool, and a per-org change needs no container restart: the next request for that org routes to its own database. -An organization owner or admin can also manage this connection from the UI: the per-organization sections of **Settings > Data residency** read and write exactly these files, with the same connection test before switching. Those sections stay editable for an org owner or admin whether or not the operator allowlist names them, because the files they touch belong to the organization rather than the deployment. The JSON files on disk stay the source of truth — an operator who prefers to edit them by hand needs no UI step. +**Settings > Data residency** is this per-organization surface: an organization owner or admin reads and writes exactly these files there, with the same connection test before switching. The JSON files on disk stay the source of truth — an operator who prefers to edit them by hand needs no UI step. ### The organization's embedding model @@ -64,9 +56,9 @@ The connection lives next to the knowledge one, under the organization's config - `$TALE_CONFIG_DIR//object-storage/connection.json` — region, optional endpoint (for MinIO/R2), path-style flag, bucket, and an optional key prefix. - `$TALE_CONFIG_DIR//object-storage/connection.secrets.json` — the access key pair, SOPS-encrypted when a SOPS age key is configured (see [Secrets with SOPS](/self-hosted/configuration/secrets-with-sops)). -Unlike the deployment-wide S3 switch above, this path is **not** greenfield-only: from the moment the config exists, new uploads go to the org's bucket, while files stored earlier stay readable where they are in the deployment's object store — mixed references are supported, so you can switch at any time and relocate the older files afterward with the blob backfill below. Removing the config sends new uploads back to the deployment default; files already written to the bucket stay there, but Tale can't read them until the connection is added again. No restart is needed in either direction. +This path is **not** greenfield-only: from the moment the config exists, new uploads go to the org's bucket, while files stored earlier stay readable where they are in the deployment's object store — mixed references are supported, so you can switch at any time and relocate the older files afterward with the blob backfill below. Removing the config sends new uploads back to the deployment default; files already written to the bucket stay there, but Tale can't read them until the connection is added again. No restart is needed in either direction. -Org admins can manage this connection from the same per-organization sections of **Settings > Data residency**; its connection test performs a real upload/read/delete round-trip against the bucket before you commit. As with the knowledge connection, the JSON files remain the source of truth. +Org admins manage this connection from the same **Settings > Data residency** page; its connection test performs a real upload/read/delete round-trip against the bucket before you commit. As with the knowledge connection, the JSON files remain the source of truth. > **Allow the app's origin in the bucket's CORS policy.** Uploads and downloads run directly between the browser and the bucket via presigned URLs, so the bucket must accept cross-origin requests from your deployment's URL — allow that origin with the methods `GET`, `PUT`, and `HEAD` and all request headers (Cloudflare R2: the bucket's **Settings > CORS Policy**; AWS S3 and MinIO: the bucket's CORS configuration). The in-app connection test runs from the server, not the browser, so a missing CORS policy surfaces only later, as a failed upload. @@ -78,23 +70,4 @@ An org admin runs it from the UI: with the bucket connection saved, the Object s The backfill is **idempotent** and **org-scoped**: it moves only that organization's blobs, skips anything already in the bucket, and leaves each source blob in place until its copy is verified — so a re-run after an interruption resumes safely, finishing any move that was cut off between the verified copy and the source delete. It walks every table that holds blob references: documents and their history, uploaded files, synthesized speech audio, and video-link transcripts. It needs the bucket connection configured first, and it refuses to run when the org's bucket is the deployment's own store — there would be nothing to move, and finishing a move would delete the only copy. This is deliberately **not** a versioned framework migration — it runs on demand, per organization, when you choose to relocate a tenant's history, not at a release boundary. -## File storage on S3 - -External file storage uses a single S3-compatible bucket — a bucket name, region, credentials, and (for MinIO or Cloudflare R2) an endpoint with path-style addressing enabled. These map to the `OBJECT_STORE_*` variables in [Environment reference](/self-hosted/configuration/environment-reference). - -> **Greenfield only.** Switching the deployment-wide file storage from the bundled store to an external bucket does **not** migrate the blobs already on the local volume — the backend will look for them in the bucket and not find them. Set S3 at initial deployment, or copy the existing local storage into the bucket out of band before switching. - -## How the configuration is stored - -Saving writes two files at the config root (not under an org directory): - -- `deployment.json` — the non-secret config (hosts, ports, buckets, modes). -- `deployment.secrets.json` — the database passwords and S3 keys, SOPS-encrypted (see [Secrets with SOPS](/self-hosted/configuration/secrets-with-sops)). - -At boot the backend reads these and derives its connections before starting. Knowledge ingestion and retrieval run inside the backend worker, so the backend is what opens the knowledge-database connection — there is no separate retrieval service to configure. The contract is **fail-closed**: a present-but-unparseable `deployment.json`, an undecryptable secret, or a config missing required fields **aborts startup** rather than silently falling back to the bundled database — mis-routing regulated data is worse than not starting. An absent file is the normal default path. - -## Applying a change: restart - -The config is read at boot, so a save does not take effect until the backend containers (`backend-api` and `backend-worker`) restart. Run `docker compose restart backend-api backend-worker`, or `tale deploy` for a zero-downtime blue-green roll — the settings page shows the same commands after a save. - -The relevant environment variable is `TALE_DEPLOYMENT_CONFIG_ADMINS` (the comma-separated email allowlist of operators allowed to edit). Set it in `.env`. See also [Environment reference](/self-hosted/configuration/environment-reference) and [Secrets with SOPS](/self-hosted/configuration/secrets-with-sops). +The deployment defaults and their variables are listed in the [Environment reference](/self-hosted/configuration/environment-reference); the per-organization secrets sidecars follow [Secrets with SOPS](/self-hosted/configuration/secrets-with-sops). diff --git a/docs/en/self-hosted/configuration/environment-reference.md b/docs/en/self-hosted/configuration/environment-reference.md index ad4f923a45..7d56fe8185 100644 --- a/docs/en/self-hosted/configuration/environment-reference.md +++ b/docs/en/self-hosted/configuration/environment-reference.md @@ -156,7 +156,7 @@ Optional toggles for features not enabled by default. Each flag turns one featur | `TRUSTED_HEADERS_INTERNAL_SECRET` | unset | Shared secret the authenticating proxy must send with every trusted-headers request. Required when the mode is enabled — the endpoint refuses to run without it. | | `TRUSTED_SECRET_HEADER` | `Remote-Internal-Secret` | Name of the request header carrying the internal secret. | | `FILE_EVENTS_ENABLED` | `false` | Enables file-watching events for the OneDrive-sync connector. | -| `TALE_DEPLOYMENT_CONFIG_ADMINS` | unset | Comma-separated email allowlist of operators allowed to edit deployment data residency. Empty/unset = read-only for all admins. | +| `TALE_DEPLOYMENT_CONFIG_ADMINS` | unset | Comma-separated email allowlist of operators allowed to write the deployment config file (`deployment.yml`, today the sandbox runtime section) through the API. Empty/unset = read-only for all admins. Data residency is configured per organization and is not gated by this list. | ## RAG retrieval tuning diff --git a/docs/fr/self-hosted/configuration/data-residency.md b/docs/fr/self-hosted/configuration/data-residency.md index 6cd712ab79..63c407f52f 100644 --- a/docs/fr/self-hosted/configuration/data-residency.md +++ b/docs/fr/self-hosted/configuration/data-residency.md @@ -1,100 +1,73 @@ --- title: Résidence des données -description: Pointe la base de connaissances, la base de données applicative et le stockage des fichiers téléversés d'une installation Tale auto-hébergée vers une infrastructure que tu contrôles — configuré par les administrateurs dans Paramètres > Résidence des données et appliqué au redémarrage. +description: Où une installation Tale auto-hébergée garde ses données, comment tu fixes les défauts du déploiement au moment du déploiement, et comment une organisation pointe son corpus de connaissances et ses fichiers téléversés vers sa propre infrastructure — à chaud, sans redémarrage. --- -Une installation Tale auto-hébergée tourne sur une infrastructure que tu contrôles déjà, donc ses données vivent sur tes hôtes par défaut. La **résidence des données** sert au cas où tu veux pointer des banques de données précises vers ton propre Postgres géré ou ton stockage objet plutôt que vers les conteneurs fournis — par exemple pour garder le texte des documents dans une base que ton équipe exploite, ou les fichiers téléversés dans ton propre bucket S3. Le corpus de connaissances tourne comme son propre conteneur (`knowledge-db`) précisément pour pouvoir être relocalisé ou remplacé indépendamment de la base opérationnelle — c'est la banque qui compte le plus pour la majorité des exigences de résidence. Les administrateurs configurent cela dans **Paramètres > Résidence des données** ; le changement est écrit dans un seul fichier de configuration au niveau du déploiement et **prend effet au redémarrage des conteneurs concernés**. +Une installation Tale auto-hébergée tourne sur une infrastructure que tu contrôles déjà : par défaut, ses données vivent donc sur tes hôtes. La **résidence des données** couvre le cas où un magasin doit vivre à un endroit précis — le texte des documents dans une base que ton équipe exploite, les fichiers téléversés dans ton propre bucket S3, le corpus d’un locataire isolé de tous les autres. Tale y répond à deux niveaux : les **défauts du déploiement**, que chaque organisation partage et que tu fixes par variables d’environnement au moment du déploiement, et les **connexions par organisation**, qu’un admin d’org gère à chaud dans **Paramètres > Résidence des données**. -Cette page couvre ce qui peut être déplacé, le seul prérequis qui mord (ParadeDB), comment la configuration est stockée et appliquée, et comment redémarrer sans risque. +Cette page couvre ce qui vit où, comment relocaliser les défauts du déploiement, le prérequis qui mord (ParadeDB), et les chemins par organisation pour les connaissances et le stockage d’objets — y compris le déplacement des fichiers existants d’une organisation. -## Activer la modification +## Où vivent les données du déploiement -**Paramètres > Résidence des données** est une seule page avec deux familles de sections : les banques à l'échelle du déploiement que toutes les organisations partagent, et celles qu'une organisation apporte pour elle seule. Chaque section s'affiche en lecture seule ou modifiable selon ce que la personne qui la lit a le droit de changer, et la page nomme l'état dans lequel tu te trouves. Voir la page est ouvert à tout owner ou admin d'une organisation ; **modifier les banques du déploiement** — repointer une banque de données, enregistrer des secrets, lancer un test de connexion ou appliquer un redémarrage — est réservé à une allowlist nommée d'opérateurs. Liste leurs courriels de connexion (séparés par des virgules) dans `.env` et redémarre : +Trois magasins, chacun avec sa variable d’environnement. Une variable non définie veut dire « utilise le conteneur fourni » : une installation neuve sans surcharge ne change donc pas. -```bash -TALE_DEPLOYMENT_CONFIG_ADMINS=alice@example.com,bob@example.com -``` +- **Base de connaissances** — le corpus de connaissances : métadonnées des documents, texte extrait des fragments, embeddings, index BM25, cache sémantique et pages web crawlées. Elle arrive sous la forme du conteneur fourni `knowledge-db` (`tale_knowledge`, avec les schémas `private_knowledge` et `public_web`) et c’est le magasin dont la plupart des exigences de résidence se soucient, parce qu’il contient le contenu de tes documents. `KNOWLEDGE_DATABASE_URL` pointe le backend vers un Postgres géré à toi ; la base peut démarrer vide — le backend crée ses schémas au premier accès. +- **Stockage de fichiers** — où vivent les fichiers téléversés (les blobs d’origine). Par défaut, ils sont dans le store d’objets fourni (le service `object-store`, sur son propre volume) ; les variables `OBJECT_STORE_*` pointent le backend vers un bucket externe compatible S3. Le basculement est greenfield : les blobs déjà écrits dans le store fourni ne sont pas copiés — fixe-le au déploiement initial, ou copie d’abord le volume dans le bucket hors bande. +- **Base de données applicative** — le magasin opérationnel derrière les agents, les runs et le log d’audit (le conteneur fourni `db`, la base `tale_app`). `DATABASE_URL` la relocalise ; le nom de base vaut `tale_app` par défaut (surcharge avec `APP_DB_NAME`). -Si l'allowlist est vide ou non définie, les sections de déploiement montrent toujours la configuration actuelle aux administrateurs, mais en lecture seule — les actions d'en-tête **Enregistrer le déploiement** et **Appliquer & redémarrer** n'apparaissent que pour les opérateurs de l'allowlist. Seul un admin connecté dont le courriel figure sur la liste rend ces sections modifiables ; la page t'indique quel courriel ajouter. Les entrypoints consomment le fichier de configuration quelle que soit l'allowlist, donc un opérateur qui préfère éditer le fichier à la main sur le disque peut le faire sans nommer d'éditeurs UI. +Les variables vivent dans le `.env` du déploiement et sont lues au démarrage des conteneurs backend — change-en une, puis déroule avec `tale deploy` (blue-green sans interruption) ou `docker compose restart backend-api backend-worker`. Chaque variable, son défaut et sa forme exacte sont dans la [référence des variables d’environnement](/fr/self-hosted/configuration/environment-reference). Rien dans l’app n’écrit ces valeurs : les versions précédentes avaient, dans Paramètres > Résidence des données, une section de magasins au niveau du déploiement qui enregistrait un bloc `dataStores` dans `deployment.yml` — mais aucun chemin de démarrage ne le lisait. Cette section a disparu ; un bloc `dataStores` resté dans un `deployment.yml` existant est ignoré et retiré au prochain enregistrement du fichier. -## Ce que tu peux relocaliser - -Trois banques de données, chacune indépendante et optionnelle. Un réglage absent signifie « utilise le défaut fourni » — une installation neuve sans configuration reste donc inchangée. - -- **Base de connaissances** — le corpus de connaissances : métadonnées des documents, texte des fragments extraits, embeddings, index BM25, cache sémantique et pages web crawlées. Elle est livrée comme le conteneur `knowledge-db` (`tale_knowledge`, avec les schémas `private_knowledge` et `public_web`) et c'est la banque qui compte le plus pour les exigences de résidence, car elle détient le contenu de tes documents. Pointe-la vers ton propre Postgres géré pour garder le corpus sur une infrastructure que ton équipe exploite. -- **Stockage de fichiers** — où vivent les fichiers téléversés (les blobs d'origine). Par défaut ils résident dans le magasin d'objets fourni avec la pile (le service `object-store`, sur son propre volume) ; tu peux les pointer vers un bucket externe compatible S3. -- **Base de données applicative** (avancé) — le magasin opérationnel derrière les agents, les runs et le log d'audit (le conteneur `db` fourni, la base `tale_app`). La relocaliser pointe la `DATABASE_URL` du backend vers ton propre Postgres ; le nom de la base est `tale_app` par défaut (override avec `APP_DB_NAME`), donc un Postgres externe doit contenir une base de ce nom. - -> Note : la base de connaissances et la base de données applicative sont deux instances Postgres séparées — déplacer l'une ne touche pas l'autre. Relocaliser la base de connaissances déplace le texte extrait et les embeddings ; les fichiers téléversés d'origine ne suivent que si tu relocalises aussi le **stockage de fichiers** vers S3. +> Note : la base de connaissances et la base applicative sont deux instances Postgres distinctes — déplacer l’une ne touche pas l’autre. Relocaliser la base de connaissances déplace le texte extrait et les embeddings ; les fichiers téléversés d’origine ne bougent que si tu relocalises aussi le **stockage de fichiers**. ## Le prérequis ParadeDB -La base de connaissances utilise deux extensions Postgres : `vector` (pgvector) pour les embeddings et `pg_search` (ParadeDB) pour la recherche hybride plein texte/BM25. Un Postgres de connaissances externe **doit faire tourner ParadeDB** (qui regroupe les deux) pour une qualité de recherche complète. Si tu le pointes vers un Postgres simple qui n'a que `pgvector`, l'indexation et la recherche vectorielle fonctionnent toujours, mais la recherche hybride se réduit à du **vectoriel seul** — la moitié BM25 est silencieusement sautée. Le bouton **Tester la connexion** signale la disponibilité de `pgvector` et de `pg_search` pour que tu le voies avant de t'engager. La base de connaissances externe doit déjà exister (elle peut porter n'importe quel nom que tu saisis — `tale_knowledge` par convention) avec les schémas `private_knowledge` et `public_web` ; les migrations de schéma de base vivent dans [`services/db/migrations/`](https://github.com/tale-project/tale/tree/main/services/db/migrations) et sont appliquées via dbmate quand la base démarre. +La base de connaissances utilise deux extensions Postgres : `vector` (pgvector) pour les embeddings et `pg_search` (ParadeDB) pour la recherche hybride plein texte/BM25. Un Postgres de connaissances externe — le défaut du déploiement ou celui d’une organisation — **doit tourner sous ParadeDB** (qui embarque les deux) pour une recherche de pleine qualité. Si tu le pointes vers un Postgres ordinaire qui n’a que `pgvector`, l’indexation et la recherche vectorielle fonctionnent toujours, mais la recherche hybride se dégrade en **vectoriel seul** : la branche BM25 est sautée en silence. Le bouton **Tester la connexion** par organisation signale la disponibilité de `pgvector` et de `pg_search`, tu le vois donc avant de t’engager ; pour le défaut du déploiement, vérifie les extensions sur la base cible avant de changer `KNOWLEDGE_DATABASE_URL`. ## Bases de connaissances par organisation -Les banques ci-dessus sont au niveau du déploiement — chaque organisation les partage. Une organisation seule peut au contraire pointer **son propre** corpus de connaissances vers un Postgres que tu provisionnes pour elle, pendant que toutes les autres orgs gardent le `knowledge-db` fourni. Réserve cela aux cas où le contenu documentaire et web-crawlé d'un locataire doit résider sur une infrastructure isolée du reste — une exigence de résidence plus stricte que ce que le défaut du déploiement satisfait. +Le défaut du déploiement est partagé par chaque organisation. Une organisation seule peut au contraire pointer **son propre** corpus de connaissances vers un Postgres que tu provisionnes pour elle, pendant que toutes les autres orgs gardent le `knowledge-db` fourni. Réserve cela aux cas où le contenu documentaire et web-crawlé d’un locataire doit résider sur une infrastructure isolée du reste — une exigence de résidence plus stricte que ce que le défaut du déploiement satisfait. -L'intégralité du corpus de connaissances de l'org se déplace — les deux schémas : `private_knowledge` (métadonnées des documents, texte des fragments, embeddings et cache sémantique) et `public_web` (les pages de sites web du crawler, leur texte de fragments et les embeddings). Rien dans la base de connaissances d'une organisation n'est partagé avec une autre organisation. +L’intégralité du corpus de connaissances de l’org se déplace — les deux schémas : `private_knowledge` (métadonnées des documents, texte des fragments, embeddings et cache sémantique) et `public_web` (les pages de sites web du crawler, leur texte de fragments et les embeddings). Rien dans la base de connaissances d’une organisation n’est partagé avec une autre organisation. -La connexion vit dans le répertoire de configuration propre à l'organisation, pas dans le fichier de déploiement : +La connexion vit dans le répertoire de configuration propre à l’organisation : - `$TALE_CONFIG_DIR//knowledge/connection.json` — hôte, port, base, utilisateur et sslmode. -- `$TALE_CONFIG_DIR//knowledge/connection.secrets.json` — le mot de passe, chiffré avec SOPS dès qu'une clé age SOPS est configurée (voir [Secrets avec SOPS](/fr/self-hosted/configuration/secrets-with-sops)). -- `$TALE_CONFIG_DIR//knowledge/embedding.json` — le modèle d'embedding de l'organisation : fournisseur, identifiants stockés optionnels, tag du modèle, largeur des vecteurs et URL de base optionnelle compatible OpenAI. +- `$TALE_CONFIG_DIR//knowledge/connection.secrets.json` — le mot de passe, chiffré avec SOPS dès qu’une clé age SOPS est configurée (voir [Secrets avec SOPS](/fr/self-hosted/configuration/secrets-with-sops)). +- `$TALE_CONFIG_DIR//knowledge/embedding.json` — le modèle d’embedding de l’organisation : fournisseur, identifiants stockés optionnels, tag du modèle, largeur des vecteurs et URL de base optionnelle compatible OpenAI. -Le même prérequis ParadeDB s'applique. L'org valide sa base candidate avec un test de connexion à l'échelle de l'organisation qui signale la disponibilité de `pgvector` et `pg_search` avant de basculer ; une cible avec seulement pgvector réduit la recherche de cette org au vectoriel seul. La base peut démarrer vide — Tale crée les schémas `private_knowledge` et `public_web` au premier accès, tu n'appliques donc jamais les migrations de base à la main. +Le même prérequis ParadeDB s’applique. L’org valide sa base candidate avec un test de connexion à l’échelle de l’organisation qui signale la disponibilité de `pgvector` et `pg_search` avant de basculer ; une cible avec seulement pgvector réduit la recherche de cette org au vectoriel seul. La base peut démarrer vide — Tale crée les schémas `private_knowledge` et `public_web` au premier accès, tu n’appliques donc jamais les migrations de base à la main. -Ce chemin retombe sans risque. Une organisation sans `connection.json` garde le `knowledge-db` par défaut du déploiement exactement comme avant, la fonctionnalité ne change donc rien pour les orgs qui n'y adhèrent pas. Deux organisations qui pointent vers la même base partagent un seul pool de connexions et — contrairement aux banques au niveau du déploiement — un changement par org ne demande aucun redémarrage de conteneur : la prochaine requête de cette org est routée vers sa propre base. +Ce chemin retombe sans risque. Une organisation sans `connection.json` garde le `knowledge-db` par défaut du déploiement exactement comme avant, la fonctionnalité ne change donc rien pour les orgs qui n’y adhèrent pas. Deux organisations qui pointent vers la même base partagent un seul pool de connexions et un changement par org ne demande aucun redémarrage de conteneur : la prochaine requête de cette org est routée vers sa propre base. -Un propriétaire ou un admin de l'organisation peut aussi gérer cette connexion depuis l'UI : les sections par organisation de **Paramètres > Résidence des données** lisent et écrivent exactement ces fichiers, avec le même test de connexion avant de basculer. Ces sections restent modifiables pour un propriétaire ou un admin d'org, que l'allowlist d'opérateurs les nomme ou non, parce que les fichiers qu'elles touchent appartiennent à l'organisation et non au déploiement. Les fichiers JSON sur le disque restent la source de vérité — un opérateur qui préfère les éditer à la main n'a besoin d'aucune étape UI. +**Paramètres > Résidence des données** est exactement cette surface par organisation : un propriétaire ou un admin de l’organisation y lit et y écrit ces fichiers, avec le même test de connexion avant de basculer. Les fichiers JSON sur le disque restent la source de vérité — un opérateur qui préfère les éditer à la main n’a besoin d’aucune étape UI. -### Le modèle d'embedding de l'organisation +### Le modèle d’embedding de l’organisation -La recherche de connaissances demande un réglage de plus par organisation avant de pouvoir tourner : le **modèle d'embedding** — quel fournisseur et quel modèle transforment documents et requêtes en vecteurs, et à quelle largeur exacte. Sans lui, l'indexation et la recherche refusent avec une erreur actionnable plutôt que de deviner un modèle. Règle-le dans la section **Modèle d'embedding** de **Paramètres > Résidence des données** (ou écris `embedding.json` à la main) : choisis un fournisseur pour lequel des identifiants sont stockés, nomme le tag du modèle comme le fournisseur l'écrit, et déclare la largeur que produit le modèle — elle n'est jamais déduite du nom du modèle, parce qu'une mauvaise supposition écrit des vecteurs que la recherche ne peut silencieusement plus exploiter. +La recherche de connaissances demande un réglage de plus par organisation avant de pouvoir tourner : le **modèle d’embedding** — quel fournisseur et quel modèle transforment documents et requêtes en vecteurs, et à quelle largeur exacte. Sans lui, l’indexation et la recherche refusent avec une erreur actionnable plutôt que de deviner un modèle. Règle-le dans la section **Modèle d’embedding** de **Paramètres > Résidence des données** (ou écris `embedding.json` à la main) : choisis un fournisseur pour lequel des identifiants sont stockés, nomme le tag du modèle comme le fournisseur l’écrit, et déclare la largeur que produit le modèle — elle n’est jamais déduite du nom du modèle, parce qu’une mauvaise supposition écrit des vecteurs que la recherche ne peut silencieusement plus exploiter. -La largeur est fixée **par base de données** à l'écriture du premier vecteur. Sur le `knowledge-db` partagé du déploiement, toutes les organisations doivent donc s'accorder sur une largeur ; une organisation qui veut un autre modèle d'embedding à une autre largeur est exactement le cas de la base de connaissances dédiée ci-dessus. +La largeur est fixée **par base de données** à l’écriture du premier vecteur. Sur le `knowledge-db` partagé du déploiement, toutes les organisations doivent donc s’accorder sur une largeur ; une organisation qui veut un autre modèle d’embedding à une autre largeur est exactement le cas de la base de connaissances dédiée ci-dessus. -## Stockage d'objets par organisation +## Stockage d’objets par organisation -Le même schéma par organisation couvre les fichiers téléversés. Une organisation seule peut pointer **ses propres** blobs de fichiers — documents du Knowledge Hub, pièces jointes de chat, audio et médias générés — vers un bucket compatible S3 que tu provisionnes pour elle (AWS S3, MinIO, Cloudflare R2, …), pendant que toutes les autres orgs gardent le défaut du déploiement. Le bucket est dédié à cette organisation ; rien de ce qu'il contient n'est partagé avec une autre. +Le même schéma par organisation couvre les fichiers téléversés. Une organisation seule peut pointer **ses propres** blobs de fichiers — documents du Knowledge Hub, pièces jointes de chat, audio et médias générés — vers un bucket compatible S3 que tu provisionnes pour elle (AWS S3, MinIO, Cloudflare R2, …), pendant que toutes les autres orgs gardent le défaut du déploiement. Le bucket est dédié à cette organisation ; rien de ce qu’il contient n’est partagé avec une autre. -La connexion vit à côté de celle des connaissances, dans le répertoire de configuration de l'organisation : +La connexion vit à côté de celle des connaissances, dans le répertoire de configuration de l’organisation : - `$TALE_CONFIG_DIR//object-storage/connection.json` — région, endpoint optionnel (pour MinIO/R2), indicateur path-style, bucket et un préfixe de clé optionnel. -- `$TALE_CONFIG_DIR//object-storage/connection.secrets.json` — la paire de clés d'accès, chiffrée avec SOPS dès qu'une clé age SOPS est configurée (voir [Secrets avec SOPS](/fr/self-hosted/configuration/secrets-with-sops)). +- `$TALE_CONFIG_DIR//object-storage/connection.secrets.json` — la paire de clés d’accès, chiffrée avec SOPS dès qu’une clé age SOPS est configurée (voir [Secrets avec SOPS](/fr/self-hosted/configuration/secrets-with-sops)). -Contrairement au basculement S3 au niveau du déploiement ci-dessus, ce chemin n'est **pas** réservé aux installations neuves : dès que la configuration existe, les nouveaux téléversements vont dans le bucket de l'org, tandis que les fichiers stockés avant restent lisibles là où ils sont — les références mixtes sont prises en charge, tu peux donc basculer à tout moment. Les fichiers stockés plus tôt restent dans le store d'objets du déploiement jusqu'à ce que tu les relocalises avec le backfill de blobs ci-dessous. Si tu supprimes la configuration, les nouveaux téléversements retournent au défaut du déploiement ; les fichiers déjà écrits dans le bucket y restent, mais Tale ne peut plus les lire tant que la connexion n'est pas rétablie. Aucun redémarrage n'est nécessaire, dans un sens comme dans l'autre. +Ce chemin n’est **pas** réservé aux installations neuves : dès que la configuration existe, les nouveaux téléversements vont dans le bucket de l’org, tandis que les fichiers stockés avant restent lisibles là où ils sont — les références mixtes sont prises en charge, tu peux donc basculer à tout moment. Les fichiers stockés plus tôt restent dans le store d’objets du déploiement jusqu’à ce que tu les relocalises avec le backfill de blobs ci-dessous. Si tu supprimes la configuration, les nouveaux téléversements retournent au défaut du déploiement ; les fichiers déjà écrits dans le bucket y restent, mais Tale ne peut plus les lire tant que la connexion n’est pas rétablie. Aucun redémarrage n’est nécessaire, dans un sens comme dans l’autre. -Les admins d'org gèrent aussi cette connexion dans les mêmes sections par organisation de **Paramètres > Résidence des données** ; son test de connexion effectue un aller-retour réel écriture-lecture-suppression contre le bucket avant que tu t'engages. Comme pour la connexion des connaissances, les fichiers JSON restent la source de vérité. +Les admins d’org gèrent aussi cette connexion dans **Paramètres > Résidence des données** ; son test de connexion effectue un aller-retour réel écriture-lecture-suppression contre le bucket avant que tu t’engages. Comme pour la connexion des connaissances, les fichiers JSON restent la source de vérité. -> **Autorise l'origine de l'app dans la politique CORS du bucket.** Les téléversements et les téléchargements passent directement du navigateur au bucket via des URL présignées : le bucket doit donc accepter les requêtes cross-origin depuis l'URL de ton déploiement — autorise cette origine avec les méthodes `GET`, `PUT` et `HEAD` et tous les en-têtes de requête (Cloudflare R2 : **Settings > CORS Policy** du bucket ; AWS S3 et MinIO : la configuration CORS du bucket). Le test de connexion dans l'app s'exécute côté serveur, pas dans le navigateur — une politique CORS manquante ne se montre donc que plus tard, sous la forme d'un téléversement échoué. +> **Autorise l’origine de l’app dans la politique CORS du bucket.** Les téléversements et les téléchargements passent directement du navigateur au bucket via des URL présignées : le bucket doit donc accepter les requêtes cross-origin depuis l’URL de ton déploiement — autorise cette origine avec les méthodes `GET`, `PUT` et `HEAD` et tous les en-têtes de requête (Cloudflare R2 : **Settings > CORS Policy** du bucket ; AWS S3 et MinIO : la configuration CORS du bucket). Le test de connexion dans l’app s’exécute côté serveur, pas dans le navigateur — une politique CORS manquante ne se montre donc que plus tard, sous la forme d’un téléversement échoué. ### Déplacer les fichiers pré-existants dans le bucket -Connecter le bucket ne réachemine que les **nouveaux** téléversements ; les blobs écrits avant la connexion restent dans le store d'objets par défaut du déploiement et continuent de fonctionner via les références mixtes ci-dessus. Pour amener aussi cet historique sur ta propre infrastructure — tout l'intérêt de la résidence des données — lance le **backfill de blobs** : il déplace chaque blob pré-existant dans le bucket de l'org — la copie arrive avec son content type d'origine, est vérifiée contre la taille de la source, et c'est seulement ensuite que la copie source est supprimée. Rien n'est réécrit : un blob garde sa clé pendant le déplacement, et les lectures le trouvent dans le store qui le détient. - -Un admin d'org le lance depuis l'UI : une fois la connexion au bucket enregistrée, la section Stockage d'objets de **Paramètres > Résidence des données** affiche **Déplacer les fichiers existants** — confirme, et le déplacement tourne en arrière-plan pendant que les téléversements continuent ; une ligne de statut dans la même section rapporte la progression et l'issue du dernier lancement. - -Le backfill est **idempotent** et **limité à l'org** : il ne déplace que les blobs de cette organisation, saute tout ce qui est déjà dans le bucket, et laisse chaque source en place tant que sa copie n'est pas vérifiée — un nouveau lancement après une interruption reprend donc sans risque et achève tout déplacement coupé entre la copie vérifiée et la suppression de la source. Il parcourt chaque table qui détient des références de blobs : les documents et leur historique, les fichiers téléversés, l'audio de synthèse vocale et les transcriptions de liens vidéo. Il exige que la connexion au bucket soit déjà configurée, et refuse de tourner quand le bucket de l'org est le store du déploiement lui-même — il n'y aurait rien à déplacer, et achever un déplacement supprimerait la seule copie. Ce n'est délibérément **pas** une migration de framework versionnée — il tourne à la demande, par organisation, quand tu choisis de relocaliser l'historique d'un locataire, pas à une frontière de version. - -## Stockage de fichiers sur S3 - -Le stockage de fichiers externe utilise un seul bucket compatible S3 — un nom de bucket, une région, des identifiants et (pour MinIO ou Cloudflare R2) un endpoint avec l'adressage path-style activé. Ils correspondent aux variables `OBJECT_STORE_*` dans la [Référence d'environnement](/fr/self-hosted/configuration/environment-reference). - -> **Greenfield uniquement.** Faire passer le stockage de fichiers au niveau du déploiement du store fourni à un bucket externe ne migre **pas** les blobs déjà sur le volume local — le backend les cherche dans le bucket et ne les trouve pas. Définis S3 au déploiement initial, ou copie le stockage local existant dans le bucket hors bande avant de basculer. - -## Comment la configuration est stockée - -Enregistrer écrit deux fichiers à la racine de configuration (pas sous un répertoire d'org) : - -- `deployment.json` — la configuration non secrète (hôtes, ports, buckets, modes). -- `deployment.secrets.json` — les mots de passe de base de données et les clés S3, chiffrés avec SOPS (voir [Secrets avec SOPS](/fr/self-hosted/configuration/secrets-with-sops)). - -Au démarrage, le backend les lit et en dérive ses connexions avant de démarrer. L'ingestion et la récupération de connaissances tournent dans le backend worker, c'est donc le backend qui ouvre la connexion à la base de connaissances — il n'y a pas de service de récupération séparé à configurer. Le contrat est **fail-closed** : un `deployment.json` présent mais impossible à parser, un secret indéchiffrable ou une configuration sans champs requis **interrompt le démarrage** au lieu de retomber silencieusement sur la base fournie — mal router des données réglementées est pire que ne pas démarrer. Un fichier absent est le chemin par défaut normal. +Connecter le bucket ne réachemine que les **nouveaux** téléversements ; les blobs écrits avant la connexion restent dans le store d’objets par défaut du déploiement et continuent de fonctionner via les références mixtes ci-dessus. Pour amener aussi cet historique sur ta propre infrastructure — tout l’intérêt de la résidence des données — lance le **backfill de blobs** : il déplace chaque blob pré-existant dans le bucket de l’org — la copie arrive avec son content type d’origine, est vérifiée contre la taille de la source, et c’est seulement ensuite que la copie source est supprimée. Rien n’est réécrit : un blob garde sa clé pendant le déplacement, et les lectures le trouvent dans le store qui le détient. -## Appliquer un changement : redémarrage +Un admin d’org le lance depuis l’UI : une fois la connexion au bucket enregistrée, la section Stockage d’objets de **Paramètres > Résidence des données** affiche **Déplacer les fichiers existants** — confirme, et le déplacement tourne en arrière-plan pendant que les téléversements continuent ; une ligne de statut dans la même section rapporte la progression et l’issue du dernier lancement. -La configuration est lue au démarrage, donc un enregistrement ne prend effet qu'au redémarrage des conteneurs backend (`backend-api` et `backend-worker`). Lance `docker compose restart backend-api backend-worker`, ou `tale deploy` pour un roulement blue-green sans interruption — la page de réglages montre les mêmes commandes après un enregistrement. +Le backfill est **idempotent** et **limité à l’org** : il ne déplace que les blobs de cette organisation, saute tout ce qui est déjà dans le bucket, et laisse chaque source en place tant que sa copie n’est pas vérifiée — un nouveau lancement après une interruption reprend donc sans risque et achève tout déplacement coupé entre la copie vérifiée et la suppression de la source. Il parcourt chaque table qui détient des références de blobs : les documents et leur historique, les fichiers téléversés, l’audio de synthèse vocale et les transcriptions de liens vidéo. Il exige que la connexion au bucket soit déjà configurée, et refuse de tourner quand le bucket de l’org est le store du déploiement lui-même — il n’y aurait rien à déplacer, et achever un déplacement supprimerait la seule copie. Ce n’est délibérément **pas** une migration de framework versionnée — il tourne à la demande, par organisation, quand tu choisis de relocaliser l’historique d’un locataire, pas à une frontière de version. -La variable d'environnement pertinente est `TALE_DEPLOYMENT_CONFIG_ADMINS` (l'allowlist de courriels, séparés par des virgules, des opérateurs autorisés à modifier). Définis-la dans `.env`. Voir aussi [Référence des variables d'environnement](/fr/self-hosted/configuration/environment-reference) et [Secrets avec SOPS](/fr/self-hosted/configuration/secrets-with-sops). +Les défauts du déploiement et leurs variables sont listés dans la [référence des variables d’environnement](/fr/self-hosted/configuration/environment-reference) ; les sidecars de secrets par organisation suivent [Secrets avec SOPS](/fr/self-hosted/configuration/secrets-with-sops). diff --git a/docs/fr/self-hosted/configuration/environment-reference.md b/docs/fr/self-hosted/configuration/environment-reference.md index f56ddb4148..12a28f62c5 100644 --- a/docs/fr/self-hosted/configuration/environment-reference.md +++ b/docs/fr/self-hosted/configuration/environment-reference.md @@ -156,7 +156,7 @@ Bascules optionnelles pour des fonctionnalités non activées par défaut. Chaqu | `TRUSTED_HEADERS_INTERNAL_SECRET` | non défini | Secret partagé que le proxy authentifiant doit envoyer avec chaque requête trusted headers. Obligatoire dès que le mode est actif — sans lui, l'endpoint refuse de fonctionner. | | `TRUSTED_SECRET_HEADER` | `Remote-Internal-Secret` | Nom de l'en-tête de requête qui porte le secret interne. | | `FILE_EVENTS_ENABLED` | `false` | Active les événements de surveillance de fichiers pour le connector OneDrive-sync. | -| `TALE_DEPLOYMENT_CONFIG_ADMINS` | non défini | Allowlist de courriels (séparés par des virgules) des opérateurs autorisés à modifier la résidence des données du déploiement. Vide/non défini = lecture seule pour tous les admins. | +| `TALE_DEPLOYMENT_CONFIG_ADMINS` | non défini | Allowlist de courriels (séparés par des virgules) des opérateurs autorisés à écrire le fichier de configuration du déploiement (`deployment.yml`, aujourd’hui la section du runtime de la sandbox) via l’API. Vide/non défini = lecture seule pour tous les admins. La résidence des données se configure par organisation et ne dépend pas de cette liste. | ## Réglage du retrieval RAG diff --git a/services/docs/app/content/frontmatter.json b/services/docs/app/content/frontmatter.json index e5a45ee1e5..073365aa4f 100644 --- a/services/docs/app/content/frontmatter.json +++ b/services/docs/app/content/frontmatter.json @@ -823,7 +823,7 @@ "locale": "fr", "frontmatter": { "title": "Résidence des données", - "description": "Pointe la base de connaissances, la base de données applicative et le stockage des fichiers téléversés d'une installation Tale auto-hébergée vers une infrastructure que tu contrôles — configuré par les administrateurs dans Paramètres > Résidence des données et appliqué au redémarrage." + "description": "Où une installation Tale auto-hébergée garde ses données, comment tu fixes les défauts du déploiement au moment du déploiement, et comment une organisation pointe son corpus de connaissances et ses fichiers téléversés vers sa propre infrastructure — à chaud, sans redémarrage." } }, "fr:self-hosted/configuration/environment-reference": { @@ -1896,7 +1896,7 @@ "locale": "en", "frontmatter": { "title": "Data residency", - "description": "Point a self-hosted Tale deployment's knowledge database, application database, and uploaded-file storage at infrastructure you control, configured by administrators in Settings > Data residency and applied on restart." + "description": "Where a self-hosted Tale deployment keeps its data, how the deployment defaults are set at deploy time, and how a single organization points its knowledge corpus and uploaded files at infrastructure of its own — live, without a restart." } }, "en:self-hosted/configuration/environment-reference": { @@ -2969,7 +2969,7 @@ "locale": "de", "frontmatter": { "title": "Datenresidenz", - "description": "Richte die Wissensdatenbank, die Anwendungsdatenbank und den Speicher für hochgeladene Dateien einer selbst gehosteten Tale-Installation auf Infrastruktur aus, die du selbst kontrollierst — von Administratoren unter Einstellungen > Datenresidenz konfiguriert und beim Neustart angewendet." + "description": "Wo eine selbst gehostete Tale-Installation ihre Daten hält, wie du die Deployment-Defaults beim Deploy setzt und wie eine einzelne Organisation ihren Wissens-Korpus und ihre hochgeladenen Dateien auf eigene Infrastruktur ausrichtet — live, ohne Neustart." } }, "de:self-hosted/configuration/environment-reference": { diff --git a/services/platform/backend/core/deployment/editors.ts b/services/platform/backend/core/deployment/editors.ts index 90ee9b85fe..93b8e85d26 100644 --- a/services/platform/backend/core/deployment/editors.ts +++ b/services/platform/backend/core/deployment/editors.ts @@ -1,11 +1,12 @@ /** * Deployment-editor allowlist — pure env parsing, no Convex/Node deps. * - * Editing deployment data-residency config (repointing a data store, saving - * secrets, testing a connection, applying a restart) is restricted to a named - * set of operators. The operator lists their sign-in emails in - * `TALE_DEPLOYMENT_CONFIG_ADMINS` at the host; viewing stays open to all - * organization owners/admins (see `auth.ts`). + * Writing the deployment config file (`deployment.yml` — today the + * `sandboxRuntime` section) and the other deployment-level operator doors that + * reuse this gate is restricted to a named set of operators. The operator lists + * their sign-in emails in `TALE_DEPLOYMENT_CONFIG_ADMINS` at the host; viewing + * stays open to all organization owners/admins (see `auth_policy.ts`). Data + * residency is per organization and is not gated here. * * Kept free of Convex component imports so the logic is unit-testable in plain * vitest (the betterAuth/rateLimiter components don't register under convexTest). From 161ffcdff23fdca451eee49ed439571c1c921f4f Mon Sep 17 00:00:00 2001 From: larryro <371767072@qq.com> Date: Sun, 6 Sep 2026 11:25:48 +0800 Subject: [PATCH 4/8] docs(docs): tell the truth about how the default blob store is set The rewritten File storage bullet and the 'change one, then roll' sentence in data-residency.md promised that editing OBJECT_STORE_* and restarting relocates the deployment-default blob store. It does not: domains/object_storage/bootstrap.ts seeds default/object-storage/connection.json (+ connection.secrets.json) from the env on the first boot only and never overwrites an existing file, and lib/object-store.ts resolves the default from that config tree, not from the env. Only DATABASE_URL and KNOWLEDGE_DATABASE_URL are read on every boot. The bullet now describes the first-boot seed, the hand-edit path for a running deployment and links Backups and restore for the repointed default; the roll sentence is restricted to the two database URLs. Same change in en/de/fr. Finding: lib-shared-schemas-1 (review blocking item 1). --- docs/de/self-hosted/configuration/data-residency.md | 4 ++-- docs/en/self-hosted/configuration/data-residency.md | 4 ++-- docs/fr/self-hosted/configuration/data-residency.md | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/de/self-hosted/configuration/data-residency.md b/docs/de/self-hosted/configuration/data-residency.md index 79a70027ce..ea994ee1a3 100644 --- a/docs/de/self-hosted/configuration/data-residency.md +++ b/docs/de/self-hosted/configuration/data-residency.md @@ -12,10 +12,10 @@ Diese Seite behandelt, was wo liegt, wie du die Deployment-Defaults verlagerst, Drei Speicher, jeder mit eigener Umgebungsvariable. Eine nicht gesetzte Variable heißt „nimm den mitgelieferten Container", eine frische Installation ohne Overrides bleibt also unverändert. - **Wissensdatenbank** — der Wissens-Korpus: Dokumentmetadaten, der extrahierte Chunk-Text, Embeddings, der BM25-Index, der semantische Cache und die gecrawlten Webseiten. Sie kommt als mitgelieferter Container `knowledge-db` (`tale_knowledge`, mit den Schemata `private_knowledge` und `public_web`) und ist der Speicher, um den sich die meisten Residenz-Anforderungen drehen, weil er deinen Dokumentinhalt hält. `KNOWLEDGE_DATABASE_URL` richtet das Backend stattdessen auf ein verwaltetes Postgres von dir aus; die Datenbank darf leer starten — das Backend legt seine Schemata beim ersten Zugriff an. -- **Dateispeicher** — wo hochgeladene Dateien (die Original-Blobs) liegen. Standardmäßig im mitgelieferten Object-Store (dem Service `object-store`, auf eigenem Volume); die `OBJECT_STORE_*`-Variablen richten das Backend stattdessen auf einen externen S3-kompatiblen Bucket aus. Der Wechsel ist Greenfield: Blobs, die schon im mitgelieferten Store liegen, werden nicht kopiert — setze ihn bei der ersten Installation, oder kopiere das Volume vorab außerhalb von Tale in den Bucket. +- **Dateispeicher** — wo hochgeladene Dateien (die Original-Blobs) liegen. Standardmäßig im mitgelieferten Object-Store (dem Service `object-store`, auf eigenem Volume). Dieser Store wird anders konfiguriert als die beiden Datenbanken: **nur beim ersten Start** schreibt das Backend den Deployment-Default aus den `OBJECT_STORE_*`-Variablen nach `$TALE_CONFIG_DIR/default/object-storage/connection.json` (plus `connection.secrets.json` für die Zugangsschlüssel, SOPS-verschlüsselt, wenn ein Schlüssel konfiguriert ist) — danach liest es nur noch diese Datei, die Variablen nie wieder; eine vorhandene Datei wird nie überschrieben. Setze `OBJECT_STORE_*` also **vor dem ersten Start** auf einen externen S3-kompatiblen Bucket, wenn du dort beginnen willst; den Default eines laufenden Deployments verlegst du, indem du `connection.json` und `connection.secrets.json` von Hand editierst und die Backend-Container rollst. In beiden Fällen ist der Wechsel Greenfield: Blobs, die schon im mitgelieferten Store liegen, werden nicht kopiert — kopiere das Volume vorab außerhalb von Tale in den Bucket, und lies [Backups und Restore](/de/self-hosted/operate/backups-and-restore), denn ein umgebogener Default nimmt die Blobs aus den Snapshots von `tale backup` heraus. - **Anwendungsdatenbank** — der operative Speicher hinter Agents, Runs und dem Audit-Log (der mitgelieferte Container `db`, die Datenbank `tale_app`). `DATABASE_URL` verlagert sie; der Datenbankname ist standardmäßig `tale_app` (Override mit `APP_DB_NAME`). -Die Variablen stehen in der `.env` des Deployments und werden gelesen, wenn die Backend-Container starten — ändere eine und rolle dann mit `tale deploy` (Zero-Downtime, Blue-Green) oder `docker compose restart backend-api backend-worker`. Jede Variable, ihr Default und ihre genaue Form stehen in der [Umgebungsreferenz](/de/self-hosted/configuration/environment-reference). Nichts in der App schreibt diese Werte: Frühere Releases hatten unter Einstellungen > Datenresidenz einen deployment-weiten Speicher-Abschnitt, der einen `dataStores`-Block in `deployment.yml` speicherte — aber kein Boot-Pfad las ihn. Der Abschnitt ist weg; ein übrig gebliebener `dataStores`-Block in einer bestehenden `deployment.yml` wird ignoriert und beim nächsten Speichern der Datei entfernt. +Die Variablen stehen in der `.env` des Deployments. `DATABASE_URL` und `KNOWLEDGE_DATABASE_URL` werden bei jedem Start der Backend-Container gelesen — ändere eine und rolle dann mit `tale deploy` (Zero-Downtime, Blue-Green) oder `docker compose restart backend-api backend-worker`; die `OBJECT_STORE_*`-Variablen zählen nur beim ersten Start, wie oben beschrieben. Jede Variable, ihr Default und ihre genaue Form stehen in der [Umgebungsreferenz](/de/self-hosted/configuration/environment-reference). Nichts in der App schreibt diese Werte: Frühere Releases hatten unter Einstellungen > Datenresidenz einen deployment-weiten Speicher-Abschnitt, der einen `dataStores`-Block in `deployment.yml` speicherte — aber kein Boot-Pfad las ihn. Der Abschnitt ist weg; ein übrig gebliebener `dataStores`-Block in einer bestehenden `deployment.yml` wird ignoriert und beim nächsten Speichern der Datei entfernt. > Hinweis: Die Wissensdatenbank und die Anwendungsdatenbank sind zwei getrennte Postgres-Instanzen — die eine zu verlagern berührt die andere nicht. Verlagerst du die Wissensdatenbank, wandern der extrahierte Text und die Embeddings; die hochgeladenen Originaldateien wandern nur, wenn du auch den **Dateispeicher** verlagerst. diff --git a/docs/en/self-hosted/configuration/data-residency.md b/docs/en/self-hosted/configuration/data-residency.md index cdec7de69e..da47713c5f 100644 --- a/docs/en/self-hosted/configuration/data-residency.md +++ b/docs/en/self-hosted/configuration/data-residency.md @@ -12,10 +12,10 @@ This page covers what lives where, how to relocate the deployment defaults, the Three stores, each with its own environment variable. An unset variable means "use the bundled container", so a fresh deployment with no overrides is unchanged. - **Knowledge database** — the knowledge corpus: document metadata, the extracted chunk text, embeddings, the BM25 index, the semantic cache, and the crawled web pages. It ships as the bundled `knowledge-db` container (`tale_knowledge`, with the `private_knowledge` and `public_web` schemas) and is the store most residency requirements care about, because it holds your document content. `KNOWLEDGE_DATABASE_URL` points the backend at a managed Postgres of your own instead; the database can start empty — the backend creates its schemas on first use. -- **File storage** — where uploaded files (the original blobs) live. By default they sit in the bundled object store (the `object-store` service, on its own volume); the `OBJECT_STORE_*` variables point the backend at an external S3-compatible bucket instead. The switch is greenfield: blobs already written to the bundled store are not copied, so set it at initial deployment or copy the volume into the bucket out of band first. +- **File storage** — where uploaded files (the original blobs) live. By default they sit in the bundled object store (the `object-store` service, on its own volume). This store is configured differently from the two databases: on the **first boot only**, the backend seeds the deployment default from the `OBJECT_STORE_*` variables into `$TALE_CONFIG_DIR/default/object-storage/connection.json` (plus `connection.secrets.json` for the access keys, SOPS-encrypted when a key is configured), and from then on it reads that file, never the variables again — a file that already exists is never overwritten. So set `OBJECT_STORE_*` to an external S3-compatible bucket **before the first boot** to start there; to relocate the default of a running deployment, edit `connection.json` and `connection.secrets.json` by hand and roll the backend containers. Either way the switch is greenfield: blobs already written to the bundled store are not copied, so copy the volume into the bucket out of band first — and read [Backups and restore](/self-hosted/operate/backups-and-restore), because a repointed default takes the blobs out of `tale backup`'s snapshots. - **Application database** — the operational store behind agents, runs, and the audit log (the bundled `db` container, the `tale_app` database). `DATABASE_URL` relocates it; the database name defaults to `tale_app` (override with `APP_DB_NAME`). -The variables live in the deployment's `.env` and are read when the backend containers start — change one, then roll with `tale deploy` (zero-downtime blue-green) or `docker compose restart backend-api backend-worker`. Every variable, its default and its exact form is in the [Environment reference](/self-hosted/configuration/environment-reference). Nothing in the app writes these values: earlier releases carried a deployment-wide store section in Settings > Data residency that saved a `dataStores` block into `deployment.yml`, but no boot path read it — that section is gone, and a leftover `dataStores` block in an existing `deployment.yml` is ignored and dropped on the file's next save. +The variables live in the deployment's `.env`. `DATABASE_URL` and `KNOWLEDGE_DATABASE_URL` are read every time the backend containers start — change one, then roll with `tale deploy` (zero-downtime blue-green) or `docker compose restart backend-api backend-worker`; the `OBJECT_STORE_*` variables only matter for the first boot, as described above. Every variable, its default and its exact form is in the [Environment reference](/self-hosted/configuration/environment-reference). Nothing in the app writes these values: earlier releases carried a deployment-wide store section in Settings > Data residency that saved a `dataStores` block into `deployment.yml`, but no boot path read it — that section is gone, and a leftover `dataStores` block in an existing `deployment.yml` is ignored and dropped on the file's next save. > Note: the knowledge database and the application database are two separate Postgres instances — moving one does not touch the other. Relocating the knowledge database moves the extracted text and embeddings; the original uploaded files move only when you also relocate **File storage**. diff --git a/docs/fr/self-hosted/configuration/data-residency.md b/docs/fr/self-hosted/configuration/data-residency.md index 63c407f52f..fa99a39d60 100644 --- a/docs/fr/self-hosted/configuration/data-residency.md +++ b/docs/fr/self-hosted/configuration/data-residency.md @@ -12,10 +12,10 @@ Cette page couvre ce qui vit où, comment relocaliser les défauts du déploieme Trois magasins, chacun avec sa variable d’environnement. Une variable non définie veut dire « utilise le conteneur fourni » : une installation neuve sans surcharge ne change donc pas. - **Base de connaissances** — le corpus de connaissances : métadonnées des documents, texte extrait des fragments, embeddings, index BM25, cache sémantique et pages web crawlées. Elle arrive sous la forme du conteneur fourni `knowledge-db` (`tale_knowledge`, avec les schémas `private_knowledge` et `public_web`) et c’est le magasin dont la plupart des exigences de résidence se soucient, parce qu’il contient le contenu de tes documents. `KNOWLEDGE_DATABASE_URL` pointe le backend vers un Postgres géré à toi ; la base peut démarrer vide — le backend crée ses schémas au premier accès. -- **Stockage de fichiers** — où vivent les fichiers téléversés (les blobs d’origine). Par défaut, ils sont dans le store d’objets fourni (le service `object-store`, sur son propre volume) ; les variables `OBJECT_STORE_*` pointent le backend vers un bucket externe compatible S3. Le basculement est greenfield : les blobs déjà écrits dans le store fourni ne sont pas copiés — fixe-le au déploiement initial, ou copie d’abord le volume dans le bucket hors bande. +- **Stockage de fichiers** — où vivent les fichiers téléversés (les blobs d’origine). Par défaut, ils sont dans le store d’objets fourni (le service `object-store`, sur son propre volume). Ce store se configure autrement que les deux bases : **au premier démarrage seulement**, le backend écrit le défaut du déploiement à partir des variables `OBJECT_STORE_*` dans `$TALE_CONFIG_DIR/default/object-storage/connection.json` (plus `connection.secrets.json` pour les clés d’accès, chiffré avec SOPS quand une clé est configurée) ; ensuite il ne lit plus que ce fichier, jamais les variables — et un fichier déjà présent n’est jamais écrasé. Fixe donc `OBJECT_STORE_*` sur un bucket externe compatible S3 **avant le premier démarrage** si tu veux commencer là ; pour déplacer le défaut d’un déploiement qui tourne, édite `connection.json` et `connection.secrets.json` à la main, puis déroule les conteneurs backend. Dans les deux cas le basculement est greenfield : les blobs déjà écrits dans le store fourni ne sont pas copiés — copie d’abord le volume dans le bucket hors bande, et lis [Backups et restauration](/fr/self-hosted/operate/backups-and-restore), car un défaut repointé sort les blobs des snapshots de `tale backup`. - **Base de données applicative** — le magasin opérationnel derrière les agents, les runs et le log d’audit (le conteneur fourni `db`, la base `tale_app`). `DATABASE_URL` la relocalise ; le nom de base vaut `tale_app` par défaut (surcharge avec `APP_DB_NAME`). -Les variables vivent dans le `.env` du déploiement et sont lues au démarrage des conteneurs backend — change-en une, puis déroule avec `tale deploy` (blue-green sans interruption) ou `docker compose restart backend-api backend-worker`. Chaque variable, son défaut et sa forme exacte sont dans la [référence des variables d’environnement](/fr/self-hosted/configuration/environment-reference). Rien dans l’app n’écrit ces valeurs : les versions précédentes avaient, dans Paramètres > Résidence des données, une section de magasins au niveau du déploiement qui enregistrait un bloc `dataStores` dans `deployment.yml` — mais aucun chemin de démarrage ne le lisait. Cette section a disparu ; un bloc `dataStores` resté dans un `deployment.yml` existant est ignoré et retiré au prochain enregistrement du fichier. +Les variables vivent dans le `.env` du déploiement. `DATABASE_URL` et `KNOWLEDGE_DATABASE_URL` sont lues à chaque démarrage des conteneurs backend — change-en une, puis déroule avec `tale deploy` (blue-green sans interruption) ou `docker compose restart backend-api backend-worker` ; les variables `OBJECT_STORE_*` ne comptent qu’au premier démarrage, comme décrit plus haut. Chaque variable, son défaut et sa forme exacte sont dans la [référence des variables d’environnement](/fr/self-hosted/configuration/environment-reference). Rien dans l’app n’écrit ces valeurs : les versions précédentes avaient, dans Paramètres > Résidence des données, une section de magasins au niveau du déploiement qui enregistrait un bloc `dataStores` dans `deployment.yml` — mais aucun chemin de démarrage ne le lisait. Cette section a disparu ; un bloc `dataStores` resté dans un `deployment.yml` existant est ignoré et retiré au prochain enregistrement du fichier. > Note : la base de connaissances et la base applicative sont deux instances Postgres distinctes — déplacer l’une ne touche pas l’autre. Relocaliser la base de connaissances déplace le texte extrait et les embeddings ; les fichiers téléversés d’origine ne bougent que si tu relocalises aussi le **stockage de fichiers**. From d872fb9188c1d446d802c043b11161541a3487b8 Mon Sep 17 00:00:00 2001 From: larryro <371767072@qq.com> Date: Sun, 6 Sep 2026 11:26:14 +0800 Subject: [PATCH 5/8] docs(docs): drop the last claims about deployment-wide residency stores environment-reference.md still said the Settings > Data residency UI writes a richer per-store config than the raw variables, and video-ingestion.md said the TALE_DEPLOYMENT_CONFIG_ADMINS allowlist guards data residency. After the deployment-store section was retired the UI writes only per-organization connection files (not gated by the allowlist), and the allowlist gates writes to deployment.yml. Both sentences now state that in en/de/fr; video-ingestion links the Environment reference instead of Data residency. Finding: lib-shared-schemas-1 (review blocking items 2 and 3). --- docs/de/self-hosted/configuration/environment-reference.md | 2 +- docs/de/self-hosted/configuration/video-ingestion.md | 2 +- docs/en/self-hosted/configuration/environment-reference.md | 2 +- docs/en/self-hosted/configuration/video-ingestion.md | 2 +- docs/fr/self-hosted/configuration/environment-reference.md | 2 +- docs/fr/self-hosted/configuration/video-ingestion.md | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/de/self-hosted/configuration/environment-reference.md b/docs/de/self-hosted/configuration/environment-reference.md index 8983b49750..07571ce795 100644 --- a/docs/de/self-hosted/configuration/environment-reference.md +++ b/docs/de/self-hosted/configuration/environment-reference.md @@ -59,7 +59,7 @@ Tale hält zwei Datenbanken: den operativen Speicher (`tale_app` — Agents, Run | `KNOWLEDGE_INDEX_REPAIR_INLINE_MAX_BYTES` | `1073741824` | **Optional.** Größter BM25-Suchindex (in Bytes), den das Backend beim Start synchron neu aufbaut, wenn es ihn beschädigt vorfindet; einen größeren baut ein Hintergrundjob neu auf, während Schreibzugriffe auf diesen Korpus abgewiesen werden. Siehe [Container-Architektur](/de/self-hosted/operate/container-architecture). | | `KNOWLEDGE_INDEX_REPAIR_DISABLED` | nicht gesetzt | **Optional.** `1` oder `true` schaltet die Prüfung und Reparatur der BM25-Suchindizes beim Start ab. Ein beschädigter Index bringt die Wissensdatenbank dann bei jedem Schreibzugriff zum Absturz, bis er von Hand neu aufgebaut wird. | -Die auto-konstruierte operative Form ist `postgresql://tale:${DB_PASSWORD}@db:5432` — ohne Datenbanknamen; die operative Datenbank wird aus der Instanz-Konfiguration abgeleitet. Das Application-Backend speichert seine Daten in der `tale_app`-Datenbank auf demselben Server (überschreib den Namen mit `APP_DB_NAME`). Der Wissens-Korpus lebt in `tale_knowledge` mit den Schemata `private_knowledge` und `public_web`; die UI unter **Einstellungen > Datenresidenz** schreibt eine reichere Per-Store-Konfiguration als diese rohen Variablen, behandelt in [Datenresidenz](/de/self-hosted/configuration/data-residency). +Die auto-konstruierte operative Form ist `postgresql://tale:${DB_PASSWORD}@db:5432` — ohne Datenbanknamen; die operative Datenbank wird aus der Instanz-Konfiguration abgeleitet. Das Application-Backend speichert seine Daten in der `tale_app`-Datenbank auf demselben Server (überschreib den Namen mit `APP_DB_NAME`). Der Wissens-Korpus lebt in `tale_knowledge` mit den Schemata `private_knowledge` und `public_web`; diese Variablen setzen die Deployment-Defaults, die alle Organisationen teilen; eine Organisation kann zusätzlich ihren eigenen Korpus und ihren eigenen Bucket unter **Einstellungen > Datenresidenz** auf eigene Infrastruktur richten (Dateien pro Organisation, live wirksam, kein Neustart), behandelt in [Datenresidenz](/de/self-hosted/configuration/data-residency). ## Object-Store diff --git a/docs/de/self-hosted/configuration/video-ingestion.md b/docs/de/self-hosted/configuration/video-ingestion.md index d6733d9a8d..e2c7f2de03 100644 --- a/docs/de/self-hosted/configuration/video-ingestion.md +++ b/docs/de/self-hosted/configuration/video-ingestion.md @@ -47,7 +47,7 @@ curl -sS -X POST "https://your-host.example.com/api/v1/browser-sessions/import" # → 201 { "sessionId": "..." } ``` -Der Import ist der heikelste Schreibzugriff der Bereitstellung und deshalb doppelt abgesichert: Der Schlüssel muss einem Administrator der Organisation gehören, und dessen E-Mail-Adresse muss auf der Allowlist `TALE_DEPLOYMENT_CONFIG_ADMINS` stehen — derselben Liste, die die [Datenresidenz](/de/self-hosted/configuration/data-residency) schützt. Alle anderen bekommen **403** mit einem `code`, der die verweigernde Hürde nennt. Gehört der Nutzer des Schlüssels mehreren Organisationen an, musst du mit `X-Organization-Slug` sagen, in welche importiert wird — ohne den Header antwortet der Schreibzugriff mit **400**; bei nur einer Mitgliedschaft kannst du den Header weglassen. `GET /api/v1/browser-sessions` listet den Pool mit Status, Ablauf und Fehlschlagzähler jeder Session — nie die Cookies selbst. Eine Session lebt 14 Tage, sofern `ttlMs` nichts anderes sagt, und nur das Einlesen von Videolinks schöpft aus dem Pool. +Der Import ist der heikelste Schreibzugriff der Bereitstellung und deshalb doppelt abgesichert: Der Schlüssel muss einem Administrator der Organisation gehören, und dessen E-Mail-Adresse muss auf der Allowlist `TALE_DEPLOYMENT_CONFIG_ADMINS` stehen — derselben Liste, die Schreibzugriffe auf die Deployment-Konfigurationsdatei (`deployment.yml`) absichert, beschrieben in der [Umgebungsreferenz](/de/self-hosted/configuration/environment-reference). Alle anderen bekommen **403** mit einem `code`, der die verweigernde Hürde nennt. Gehört der Nutzer des Schlüssels mehreren Organisationen an, musst du mit `X-Organization-Slug` sagen, in welche importiert wird — ohne den Header antwortet der Schreibzugriff mit **400**; bei nur einer Mitgliedschaft kannst du den Header weglassen. `GET /api/v1/browser-sessions` listet den Pool mit Status, Ablauf und Fehlschlagzähler jeder Session — nie die Cookies selbst. Eine Session lebt 14 Tage, sofern `ttlMs` nichts anderes sagt, und nur das Einlesen von Videolinks schöpft aus dem Pool. diff --git a/docs/en/self-hosted/configuration/environment-reference.md b/docs/en/self-hosted/configuration/environment-reference.md index 7d56fe8185..1d6ccac1ab 100644 --- a/docs/en/self-hosted/configuration/environment-reference.md +++ b/docs/en/self-hosted/configuration/environment-reference.md @@ -59,7 +59,7 @@ Tale keeps two databases: the operational store (`tale_app` — agents, runs, th | `KNOWLEDGE_INDEX_REPAIR_INLINE_MAX_BYTES` | `1073741824` | **Optional.** Largest BM25 search index (in bytes) the backend rebuilds synchronously at boot when it finds it corrupted; a larger one is rebuilt by a background job while writes to that corpus are refused. See [Container architecture](/self-hosted/operate/container-architecture). | | `KNOWLEDGE_INDEX_REPAIR_DISABLED` | unset | **Optional.** `1` or `true` switches the boot-time verification and repair of the BM25 search indexes off. A corrupted index then crashes the knowledge database on every write until it is rebuilt by hand. | -The auto-constructed operational form is `postgresql://tale:${DB_PASSWORD}@db:5432` — given without a database name; the operational database is derived from the instance configuration. The application backend stores its data in the `tale_app` database on the same server (override the name with `APP_DB_NAME`). The knowledge corpus lives in `tale_knowledge` with the `private_knowledge` and `public_web` schemas; the **Settings > Data residency** UI writes a richer per-store config than these raw variables, covered in [Data residency](/self-hosted/configuration/data-residency). +The auto-constructed operational form is `postgresql://tale:${DB_PASSWORD}@db:5432` — given without a database name; the operational database is derived from the instance configuration. The application backend stores its data in the `tale_app` database on the same server (override the name with `APP_DB_NAME`). The knowledge corpus lives in `tale_knowledge` with the `private_knowledge` and `public_web` schemas; these variables set the deployment defaults every organization shares; an organization can additionally point its own corpus and its own bucket at infrastructure of its own under **Settings > Data residency** (per-organization files, applied live, no restart), covered in [Data residency](/self-hosted/configuration/data-residency). ## Object store diff --git a/docs/en/self-hosted/configuration/video-ingestion.md b/docs/en/self-hosted/configuration/video-ingestion.md index 6ed22a10b8..6bac44d96c 100644 --- a/docs/en/self-hosted/configuration/video-ingestion.md +++ b/docs/en/self-hosted/configuration/video-ingestion.md @@ -47,7 +47,7 @@ curl -sS -X POST "https://your-host.example.com/api/v1/browser-sessions/import" # → 201 { "sessionId": "..." } ``` -The import is the deployment's most sensitive write, so it is gated twice: the key must belong to an organization administrator, and that administrator's e-mail must be on the `TALE_DEPLOYMENT_CONFIG_ADMINS` allowlist — the same list that guards [data residency](/self-hosted/configuration/data-residency). Anyone else gets **403** with a `code` naming the gate that refused. A key whose user belongs to several organizations must name the one to import into with `X-Organization-Slug` — a write without it answers **400**; a key with a single membership can drop the header. `GET /api/v1/browser-sessions` lists the pool with each session's status, expiry, and strike count — never the cookies themselves. A session lives 14 days unless `ttlMs` says otherwise, and only the video-link ingest draws from the pool. +The import is the deployment's most sensitive write, so it is gated twice: the key must belong to an organization administrator, and that administrator's e-mail must be on the `TALE_DEPLOYMENT_CONFIG_ADMINS` allowlist — the same list that gates writes to the deployment config file (`deployment.yml`), described in the [Environment reference](/self-hosted/configuration/environment-reference). Anyone else gets **403** with a `code` naming the gate that refused. A key whose user belongs to several organizations must name the one to import into with `X-Organization-Slug` — a write without it answers **400**; a key with a single membership can drop the header. `GET /api/v1/browser-sessions` lists the pool with each session's status, expiry, and strike count — never the cookies themselves. A session lives 14 days unless `ttlMs` says otherwise, and only the video-link ingest draws from the pool. diff --git a/docs/fr/self-hosted/configuration/environment-reference.md b/docs/fr/self-hosted/configuration/environment-reference.md index 12a28f62c5..bf3fe35340 100644 --- a/docs/fr/self-hosted/configuration/environment-reference.md +++ b/docs/fr/self-hosted/configuration/environment-reference.md @@ -59,7 +59,7 @@ Tale garde deux bases : le magasin opérationnel (`tale_app` — agents, runs, l | `KNOWLEDGE_INDEX_REPAIR_INLINE_MAX_BYTES` | `1073741824` | **Optionnel.** Taille maximale (en octets) d'un index de recherche BM25 que le backend reconstruit de façon synchrone au démarrage quand il le trouve corrompu ; au-delà, un job d'arrière-plan le reconstruit pendant que les écritures vers ce corpus sont refusées. Voir [Architecture des conteneurs](/fr/self-hosted/operate/container-architecture). | | `KNOWLEDGE_INDEX_REPAIR_DISABLED` | non défini | **Optionnel.** `1` ou `true` désactive la vérification et la réparation des index de recherche BM25 au démarrage. Un index corrompu fait alors planter la base de connaissances à chaque écriture jusqu'à sa reconstruction manuelle. | -La forme opérationnelle auto-construite est `postgresql://tale:${DB_PASSWORD}@db:5432` — sans nom de base ; la base opérationnelle est dérivée de la configuration d'instance. Le backend applicatif stocke ses données dans la base `tale_app` sur le même serveur (override le nom avec `APP_DB_NAME`). Le corpus de connaissances vit dans `tale_knowledge` avec les schémas `private_knowledge` et `public_web` ; l'UI **Paramètres > Résidence des données** écrit une config par banque plus riche que ces variables brutes, couverte dans [Résidence des données](/fr/self-hosted/configuration/data-residency). +La forme opérationnelle auto-construite est `postgresql://tale:${DB_PASSWORD}@db:5432` — sans nom de base ; la base opérationnelle est dérivée de la configuration d'instance. Le backend applicatif stocke ses données dans la base `tale_app` sur le même serveur (override le nom avec `APP_DB_NAME`). Le corpus de connaissances vit dans `tale_knowledge` avec les schémas `private_knowledge` et `public_web` ; ces variables fixent les défauts du déploiement que toutes les organisations partagent ; une organisation peut en plus pointer son propre corpus et son propre bucket vers sa propre infrastructure sous **Paramètres > Résidence des données** (fichiers par organisation, appliqués à chaud, sans redémarrage), couvert dans [Résidence des données](/fr/self-hosted/configuration/data-residency). ## Store d'objets diff --git a/docs/fr/self-hosted/configuration/video-ingestion.md b/docs/fr/self-hosted/configuration/video-ingestion.md index 31774aea2e..ac4cbf2d61 100644 --- a/docs/fr/self-hosted/configuration/video-ingestion.md +++ b/docs/fr/self-hosted/configuration/video-ingestion.md @@ -47,7 +47,7 @@ curl -sS -X POST "https://your-host.example.com/api/v1/browser-sessions/import" # → 201 { "sessionId": "..." } ``` -L’import est l’écriture la plus sensible du déploiement, donc il est verrouillé deux fois : la clé doit appartenir à un administrateur de l’organisation, et l’e-mail de cet administrateur doit figurer dans l’allowlist `TALE_DEPLOYMENT_CONFIG_ADMINS` — la même qui protège la [résidence des données](/fr/self-hosted/configuration/data-residency). Tous les autres reçoivent **403** avec un `code` qui nomme la barrière qui a refusé. Si l’utilisateur de la clé appartient à plusieurs organisations, indique avec `X-Organization-Slug` celle où importer — sans cet en-tête, l’écriture répond **400** ; avec une seule appartenance, tu peux l’omettre. `GET /api/v1/browser-sessions` liste le pool avec le statut, l’expiration et le compteur d’échecs de chaque session — jamais les cookies eux-mêmes. Une session vit 14 jours sauf si `ttlMs` en décide autrement, et seule l’ingestion de liens vidéo puise dans le pool. +L’import est l’écriture la plus sensible du déploiement, donc il est verrouillé deux fois : la clé doit appartenir à un administrateur de l’organisation, et l’e-mail de cet administrateur doit figurer dans l’allowlist `TALE_DEPLOYMENT_CONFIG_ADMINS` — la même qui verrouille les écritures dans le fichier de configuration du déploiement (`deployment.yml`), décrite dans la [référence des variables d’environnement](/fr/self-hosted/configuration/environment-reference). Tous les autres reçoivent **403** avec un `code` qui nomme la barrière qui a refusé. Si l’utilisateur de la clé appartient à plusieurs organisations, indique avec `X-Organization-Slug` celle où importer — sans cet en-tête, l’écriture répond **400** ; avec une seule appartenance, tu peux l’omettre. `GET /api/v1/browser-sessions` liste le pool avec le statut, l’expiration et le compteur d’échecs de chaque session — jamais les cookies eux-mêmes. Une session vit 14 jours sauf si `ttlMs` en décide autrement, et seule l’ingestion de liens vidéo puise dans le pool. From 3a6a7a01a4aace82d717e742c058c3b2f9f135f5 Mon Sep 17 00:00:00 2001 From: larryro <371767072@qq.com> Date: Sun, 6 Sep 2026 11:26:34 +0800 Subject: [PATCH 6/8] test(platform): align the manual data-residency plan with the three-section page The manual plan still told the tester to expect deployment stores under the org sections, with Save deployment + Apply & restart header actions for allowlisted operators. That surface was retired in this PR; F1 against the shipped page would fail. The scope table, the intro and the F1 expected column now describe the three org sections only. Finding: lib-shared-schemas-1 (review blocking item 4). --- services/platform/tests/manual/data-residency.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/services/platform/tests/manual/data-residency.md b/services/platform/tests/manual/data-residency.md index 72d9326747..78d46448a8 100644 --- a/services/platform/tests/manual/data-residency.md +++ b/services/platform/tests/manual/data-residency.md @@ -15,14 +15,15 @@ | Surface | Route | | ------------------------------------------------------------ | ------------------------------------------ | -| Data residency (org sections first, deployment stores after) | `/dashboard/{org}/settings/data-residency` | +| Data residency (three org sections) | `/dashboard/{org}/settings/data-residency` | | Documents (placement proof) | `/dashboard/{org}/documents` | The org sections (Knowledge database, Embedding model, Object storage) batch their edits through the settings header's shared **Discard/Save** cluster; -Test, Remove, and the backfill are instant actions inside the sections. The -deployment stores below them keep their own **Save deployment** + -**Apply & restart** header actions, mounted only for allowlisted operators. +Test, Remove, and the backfill are instant actions inside the sections. +There is no deployment-wide store section any more — the deployment +defaults are environment variables (`DATABASE_URL`, `KNOWLEDGE_DATABASE_URL`, +`OBJECT_STORE_*`), not something the page writes. ## Prerequisites @@ -60,7 +61,7 @@ To inspect the bucket during the run: | ID | Test | Steps | Expected | | --- | ----------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| F1 | Panel renders defaults | As an org admin open `/dashboard/{org}/settings/data-residency` | Three org sections first — **Knowledge database** (`settings.dataResidency.orgKnowledge.title`), **Embedding model** (`settings.dataResidency.orgEmbedding.title`), **Object storage** (`settings.dataResidency.orgStorage.title`) — then the deployment stores. Knowledge + storage carry the header badge **Deployment default** (`…orgKnowledge.statusDefault` / `…orgStorage.statusDefault`) with their toggle off; the embedding section carries **Not configured** with its toggle off, and the search-unavailable warning stays visible while it is collapsed | +| F1 | Panel renders defaults | As an org admin open `/dashboard/{org}/settings/data-residency` | Three org sections first — **Knowledge database** (`settings.dataResidency.orgKnowledge.title`), **Embedding model** (`settings.dataResidency.orgEmbedding.title`), **Object storage** (`settings.dataResidency.orgStorage.title`) and nothing below them. Knowledge + storage carry the header badge **Deployment default** (`…orgKnowledge.statusDefault` / `…orgStorage.statusDefault`) with their toggle off; the embedding section carries **Not configured** with its toggle off, and the search-unavailable warning stays visible while it is collapsed | | F2 | BYO knowledge DB: save + test | Enable **Knowledge database** → host `127.0.0.1`, port `5599`, database `tale_knowledge`, user `tale`, **SSL mode `disable`** (the throwaway ParadeDB container serves no TLS; the default `require` fails the probe with "Client network socket disconnected before secure TLS connection"), password `drtest` → **Save** in the settings header → **Test connection** inside the section. Then reload and **Test connection again with the password left blank** — it must still pass (the probe reuses the stored secret). | The header cluster flashes **Saved** (no toast). Success line **OK** appears next to Test. Reload: fields persist (password field is blank — write-only, `settings.dataResidency.password.storedNoPreviewHint`) and the badge is no longer **Deployment default**. The blank-password re-test passes (regression: it used to fail "password authentication failed"). | | F2b | Embedding model: save + search unblocks | Toggle **Embedding model** on, then pick a provider you hold a credential for (with no stored credentials the provider dropdown is disabled and explains itself in a tooltip) (add one under **Settings > AI providers** first if none — the section says so), model `text-embedding-3-small` (or your provider's tag), vector width `1536`, → **Save** in the header. Then remove it via the section's **Remove** button and re-add it. | Badge flips to **Configured** and the warning disappears; `$TALE_CONFIG_DIR/{orgSlug}/knowledge/embedding.json` exists with exactly the entered fields. Knowledge search stops refusing with "no embedding model configured" (on a stack whose indexing is live, a searchable corpus returns hits). Remove asks for confirmation, toasts (`…orgEmbedding.removed`), and the warning returns | | F3 | BYO object storage: save + test | Enable **Object storage** → region `us-east-1`, endpoint `http://127.0.0.1:9100`, path-style on, bucket `org-blobs`, both keys → **Save** in the settings header → **Test connection** inside the section. Then reload and **Test connection again with both key fields left blank** — it must still pass (the probe reuses the stored keys). | The header cluster flashes **Saved** (no toast). **Bucket verified (upload, read, delete)** (`settings.dataResidency.orgStorage.verified`). Reload: config persists, key fields blank with the stored-hint. The blank-key re-test passes (regression: it used to be un-runnable — Test stayed disabled until both keys were re-typed, so a saved connection could never be re-tested). | From 2b831ff2c5f266c3f1480a151e50b7aed878548a Mon Sep 17 00:00:00 2001 From: larryro <371767072@qq.com> Date: Sun, 6 Sep 2026 12:18:59 +0800 Subject: [PATCH 7/8] fix(platform): reword the Data residency menu card to the org lanes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Settings overview card for Data residency (`settings.menu.dataResidency.description`, rendered by use-settings-menu-groups.ts) still told every admin the page shows "the deployment-level stores behind" the organization's data — the surface this PR retires. Reword the card in en/de/fr to what the page holds today: where the organization's knowledge base and uploaded files live and which model embeds them, written natively per locale. A sweep of en/de/fr/de-CH finds no other string naming deployment stores. Refs: review of #3262 (blocking-1, round 2). --- services/platform/messages/de.yml | 4 ++-- services/platform/messages/en.yml | 4 ++-- services/platform/messages/fr.yml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/services/platform/messages/de.yml b/services/platform/messages/de.yml index 489d92786e..c0c09d90bc 100644 --- a/services/platform/messages/de.yml +++ b/services/platform/messages/de.yml @@ -4943,8 +4943,8 @@ settings: enterpriseSso: description: Single Sign-on und Verzeichnissynchronisation für deine Belegschaft. dataResidency: - description: Wo die Daten dieser Organisation liegen und welche - Deployment-Speicher dahinterstehen. + description: Wo die Wissensdatenbank und die hochgeladenen Dateien dieser + Organisation liegen und welches Embedding-Modell sie verarbeitet. mcpEndpoint: title: MCP-Endpunkt description: Verbinde einen MCP-Client mit diesem Deployment, um diff --git a/services/platform/messages/en.yml b/services/platform/messages/en.yml index 70a8458aa2..9abd9ec03e 100644 --- a/services/platform/messages/en.yml +++ b/services/platform/messages/en.yml @@ -5085,8 +5085,8 @@ settings: description: Single sign-on and directory sync for your organization. dataResidency: description: - Where this organization's data lives, and the deployment-level stores - behind it. + Where this organization's knowledge base and uploaded files live, and + which model embeds them. mcpEndpoint: title: MCP endpoint description: Point any MCP client at this deployment to author, run and diff --git a/services/platform/messages/fr.yml b/services/platform/messages/fr.yml index b217d67820..39d149b515 100644 --- a/services/platform/messages/fr.yml +++ b/services/platform/messages/fr.yml @@ -5029,8 +5029,8 @@ settings: description: Authentification unique et synchronisation d’annuaire pour ton personnel. dataResidency: description: - Où vivent les données de cette organisation, et les stockages de - déploiement qui les sous-tendent. + Où vivent la base de connaissances et les fichiers téléversés de cette + organisation, et quel modèle d'embedding les traite. mcpEndpoint: title: Endpoint MCP description: Branche n'importe quel client MCP sur ce déploiement pour From bb9b79293266b4faf6e0fdcd84a0b85a24dc1047 Mon Sep 17 00:00:00 2001 From: larryro <371767072@qq.com> Date: Sun, 6 Sep 2026 12:19:17 +0800 Subject: [PATCH 8/8] chore(platform): drop stale deployment-stores comments Three comments still described the retired deployment-stores surface: the settings header slot named [Save] [Apply & restart] as its example, the SSO form pointed at the deleted deployment-stores.tsx, and the /settings/deployment redirect claimed the stores were merged into the Data residency page. Reword them to the live state; no behaviour change. Refs: review of #3262 (non-blocking 1-3, round 2). --- .../enterprise-sso/components/enterprise-sso-form.tsx | 2 +- services/platform/app/routes/dashboard/$id/settings.tsx | 3 ++- .../app/routes/dashboard/$id/settings/deployment.tsx | 6 +++--- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/services/platform/app/features/settings/enterprise-sso/components/enterprise-sso-form.tsx b/services/platform/app/features/settings/enterprise-sso/components/enterprise-sso-form.tsx index a0bc939594..8088a4cf21 100644 --- a/services/platform/app/features/settings/enterprise-sso/components/enterprise-sso-form.tsx +++ b/services/platform/app/features/settings/enterprise-sso/components/enterprise-sso-form.tsx @@ -1359,7 +1359,7 @@ export function EnterpriseSsoForm({ organizationId, config }: Props) { {/* SCIM stays inline (its own generate/regenerate/disable lifecycle, independent of the SSO config Save). Status sits on the title row so it scans with the feature name — the far-right `action` slot - is for a control cluster (see deployment stores), not a lone pill + is for a control cluster, not a lone pill while the enable action lives below. */} { throw redirect({