diff --git a/.env.test b/.env.test index cc88364116..ae8dac0484 100644 --- a/.env.test +++ b/.env.test @@ -44,7 +44,6 @@ POSTGRES_PASSWORD=test_password_e2e # Instance identity (dev-secret derivation seeds from it) INSTANCE_SECRET=0000000000000000000000000000000000000000000000000000000000000000 -INSTANCE_NAME=tale_platform # Sandbox spawner — fixed test-only HMAC token so the smoke script can sign # the session API. The spawner refuses to boot without it (compose.test.yml diff --git a/docs/de/self-hosted/operate/upgrades.md b/docs/de/self-hosted/operate/upgrades.md index c3e580f49a..4ef604e073 100644 --- a/docs/de/self-hosted/operate/upgrades.md +++ b/docs/de/self-hosted/operate/upgrades.md @@ -144,6 +144,13 @@ tale deploy Der Experten-Override — `tale deploy --accept-data-loss` — existiert für den seltenen Fall, dass du bewusst einen Host wiederverwendest, dessen alte Volumes du bereits behandelt hast. Er tut genau, was sein Name sagt: Prä-0.5-Daten dieser Instanz werden dauerhaft unlesbar. +**Die alte Datenbank `tale_platform`.** Jeder `tale-db`-Container legte beim Start eine leere Datenbank `tale_platform` an — die Datenbank, die der mitgelieferte Convex-Dienst in 0.4 nutzte und aus der 0.5 nichts liest. Frische Installationen legen sie nicht mehr an, und nichts löscht sie für dich: Eine Instanz, die du mit einem früheren 0.5-Release aufgesetzt hast, trägt sie weiter, ebenso ein wiederverwendeter 0.4-Host. Sie stört nicht. Sobald du sicher bist, dass du nichts mehr aus der Convex-Ära brauchst, zieh einen Snapshot und lösch sie von Hand — auf `db` und, wo dein Stack eines betreibt, auf `knowledge-db`: + +```bash +tale backup +docker compose exec db psql -U tale -d tale -c 'DROP DATABASE IF EXISTS tale_platform;' +``` + ## Wo das hingehört Der Upgrade-Flow knüpft jede andere Operate-Seite an — Backups sind das, was ein gescheitertes Upgrade wiederherstellbar macht, Observability ist das, was dir sagt, dass die neue Farbe healthy ist, Hardening ist das, was du nach einer Major-Version neu durchgehst. Setzt du das CLI zum ersten Mal auf, deckt [Tale-CLI installieren](/de/self-hosted/install/cli-install) das workstationseitige Setup ab; nimmst du den Pager mitten im Rollout auf, nennt [Troubleshooting](/de/self-hosted/operate/observability/troubleshooting) die Symptome. diff --git a/docs/en/self-hosted/operate/upgrades.md b/docs/en/self-hosted/operate/upgrades.md index bf1282ab77..432b7200f0 100644 --- a/docs/en/self-hosted/operate/upgrades.md +++ b/docs/en/self-hosted/operate/upgrades.md @@ -144,6 +144,13 @@ tale deploy The expert override — `tale deploy --accept-data-loss` — exists for the rare case where you deliberately reuse a host whose old volumes you have already dealt with. It does exactly what its name says: pre-0.5 data on that instance becomes permanently unreadable. +**The legacy `tale_platform` database.** Every `tale-db` container used to create an empty `tale_platform` database on start — the database the bundled Convex service used in 0.4, which nothing in 0.5 reads. Fresh installs no longer create it, and nothing drops it for you: an instance first deployed on an earlier 0.5 release still carries it, and so does a reused 0.4 host. It is harmless. Once you are sure you need nothing from the Convex era, take a snapshot and drop it by hand — on `db`, and on `knowledge-db` where your stack runs one: + +```bash +tale backup +docker compose exec db psql -U tale -d tale -c 'DROP DATABASE IF EXISTS tale_platform;' +``` + ## Where this fits The upgrade flow ties together every other operate page — backups are what makes a failed upgrade recoverable, observability is what tells you the new colour is healthy, hardening is what you re-walk after a major version. If you are setting up the CLI for the first time, [Install the tale CLI](/self-hosted/install/cli-install) covers the workstation-side setup; if you are picking up the pager mid-rollout, [Troubleshooting](/self-hosted/operate/observability/troubleshooting) names the symptoms. diff --git a/docs/fr/self-hosted/operate/upgrades.md b/docs/fr/self-hosted/operate/upgrades.md index 94f6988c38..629e5a156c 100644 --- a/docs/fr/self-hosted/operate/upgrades.md +++ b/docs/fr/self-hosted/operate/upgrades.md @@ -144,6 +144,13 @@ tale deploy Le contournement expert — `tale deploy --accept-data-loss` — existe pour le cas rare où tu réutilises délibérément un hôte dont tu as déjà traité les anciens volumes. Il fait exactement ce que son nom dit : les données pré-0.5 de cette instance deviennent définitivement illisibles. +**L'ancienne base `tale_platform`.** Chaque conteneur `tale-db` créait au démarrage une base `tale_platform` vide — la base que le service Convex embarqué utilisait en 0.4 et que rien dans la 0.5 ne lit. Les installations neuves ne la créent plus, et rien ne la supprime pour toi : une instance déployée d'abord avec une version 0.5 antérieure la porte encore, comme un hôte 0.4 réutilisé. Elle ne gêne pas. Quand tu es sûr de n'avoir plus besoin de rien de l'ère Convex, prends un snapshot puis supprime-la à la main — sur `db`, et sur `knowledge-db` si ton déploiement en a un : + +```bash +tale backup +docker compose exec db psql -U tale -d tale -c 'DROP DATABASE IF EXISTS tale_platform;' +``` + ## Où cela s'inscrit Le flow de montée de version noue chaque autre page d'exploitation — les backups sont ce qui rend une montée de version échouée récupérable, l'observabilité est ce qui te dit que la nouvelle couleur est saine, le durcissement est ce que tu reparcours après une version majeure. Si tu mets en place la CLI pour la première fois, [Installer la CLI tale](/fr/self-hosted/install/cli-install) couvre le setup côté workstation ; si tu prends le pager en plein rollout, [Dépannage](/fr/self-hosted/operate/observability/troubleshooting) nomme les symptômes. diff --git a/services/db/Dockerfile b/services/db/Dockerfile index dadfa847c1..6f5a407268 100644 --- a/services/db/Dockerfile +++ b/services/db/Dockerfile @@ -84,7 +84,8 @@ COPY --from=ghcr.io/amacneil/dbmate:2 /usr/local/bin/dbmate /usr/local/bin/dbmat # Migration sets, grouped by container role (TALE_DB_ROLE), each schema in its # own subdirectory so it gets a schema-scoped dbmate tracking table: # migrations/knowledge-db// → applied to `tale_knowledge` (knowledge-db) -# migrations/db/ → platform DB (Convex-owned; ships empty — +# migrations/db/ → platform role; ships empty (`tale_app` is +# migrated by the platform backend at boot; # its only file is a *.md, dockerignored) # docker-entrypoint.sh applies the set matching this container's role. COPY services/db/migrations/ /etc/postgresql/migrations/ diff --git a/services/db/Dockerfile.dockerignore b/services/db/Dockerfile.dockerignore index f47b424a64..6782cf995b 100644 --- a/services/db/Dockerfile.dockerignore +++ b/services/db/Dockerfile.dockerignore @@ -117,7 +117,6 @@ compose.*.yml # ============================================================================= services/platform/ services/web/ -services/convex/ services/proxy/ packages/ tools/ diff --git a/services/db/README.md b/services/db/README.md index 6010c49c5c..75beb1f6cf 100644 --- a/services/db/README.md +++ b/services/db/README.md @@ -4,11 +4,12 @@ PostgreSQL 16 with [ParadeDB](https://www.paradedb.com/) (`pg_search` BM25 + `pg ## Overview -The shared datastore for every Tale service. Init scripts in `init-scripts/` create the databases and extensions on first boot: +The shared datastore for every Tale service. Init scripts in `init-scripts/` create the databases and extensions on every container start (idempotently), so both databases exist on every container built from this image: - `tale_app` — the 0.5 platform application database (the Hono API + pg-boss workers in `services/platform/backend`). Its schema is owned by that backend, which applies its own numbered `.sql` migrations at boot; nothing here migrates it. - `tale_knowledge` — the knowledge corpus (RAG + crawler), queried in-process by the platform backend -- `tale_platform` — the legacy Convex database, still created for backward compatibility but unused by the 0.5 backend + +These are the only two databases the platform reads. The Convex-era `tale_platform` database is no longer created; an existing deployment keeps the one it has until the operator drops it (see the upgrade docs under `docs/en/self-hosted/operate/upgrades.md`). `migrations/` holds the dbmate migration sets, grouped by container role and applied on startup by `docker-entrypoint.sh` per the `TALE_DB_ROLE` env var (set per-service in `compose.yml`): diff --git a/services/db/docker-entrypoint.sh b/services/db/docker-entrypoint.sh index 00a5b317c7..8a725c432d 100644 --- a/services/db/docker-entrypoint.sh +++ b/services/db/docker-entrypoint.sh @@ -85,7 +85,8 @@ echo "==================================================" # below. The db and knowledge-db services run the same image, so TALE_DB_ROLE (set # per-service in compose.yml), not the image, decides which set applies: # knowledge → migrations/knowledge-db// against the `tale_knowledge` DB -# platform → migrations/db/ against the platform DB (empty; Convex owns it) +# platform → migrations/db/ (empty: the platform DB `tale_app` is migrated by +# the platform backend at boot, not by dbmate) # Gating /tmp/.db_ready on this (not just pg_isready) is what lets dependents wait # for the tables, not just the socket. @@ -108,7 +109,7 @@ run_init_scripts() { # the caller never publishes /tmp/.db_ready against a half-initialized # cluster. The previous `psql ... | grep || true` swallowed both psql's # exit code (it became grep's) and every SQL error, which let a failed - # `02-create-convex-database.sql` set readiness with no `tale_platform`. + # database-creating script set readiness with that database missing. if ! psql -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" -f "$script"; then echo "ERROR: init script failed: $(basename "$script")" >&2 return 1 @@ -180,7 +181,7 @@ apply_migrations_for_role() { apply_knowledge_migrations || return 1 ;; platform) - echo "TALE_DB_ROLE=platform: platform DB schema is Convex-managed; no dbmate migrations to apply." + echo "TALE_DB_ROLE=platform: the tale_app schema is migrated by the platform backend at boot; no dbmate migrations to apply." ;; *) echo "WARN: unknown TALE_DB_ROLE='${TALE_DB_ROLE}'; defaulting to knowledge migrations." >&2 @@ -197,10 +198,10 @@ apply_migrations_for_role() { # is reachable on the local UNIX socket but NEVER on TCP. Gating on a socket probe # (`psql -d "$POSTGRES_DB"`) therefore races first-time init: the loop can latch # onto the bootstrap server and run the init scripts against it just as the -# entrypoint tears it down — `02-create-convex-database.sql` dies mid-run and -# `tale_platform` is never created, yet readiness still gets published. Probing -# TCP on 127.0.0.1 instead proves the *real* server is up (the bootstrap server -# can't answer there), eliminating the race. +# entrypoint tears it down — a database-creating script dies mid-run, its +# database (`tale_knowledge`, `tale_app`) is never created, yet readiness still +# gets published. Probing TCP on 127.0.0.1 instead proves the *real* server is +# up (the bootstrap server can't answer there), eliminating the race. # # Init + migrations are then retried as a unit, and /tmp/.db_ready is touched ONLY # after both genuinely succeed, so dependents (gated on `.db_ready`) never start diff --git a/services/db/init-scripts/02-create-convex-database.sql b/services/db/init-scripts/02-create-convex-database.sql deleted file mode 100644 index 03f092359f..0000000000 --- a/services/db/init-scripts/02-create-convex-database.sql +++ /dev/null @@ -1,13 +0,0 @@ --- Tale DB: Convex self-hosted database --- Idempotent: safe to run on every startup - -SELECT 'CREATE DATABASE tale_platform' -WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = 'tale_platform') -\gexec - -GRANT ALL PRIVILEGES ON DATABASE tale_platform TO tale; - -\c tale_platform - -CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; -CREATE EXTENSION IF NOT EXISTS "pg_trgm"; diff --git a/services/db/migrations/db/README.md b/services/db/migrations/db/README.md index f087a13dd2..342559a3c4 100644 --- a/services/db/migrations/db/README.md +++ b/services/db/migrations/db/README.md @@ -1,14 +1,14 @@ # Platform DB migrations (`db` service) -dbmate migration set for the **platform** Postgres (`db` service: the `tale` / -`tale_platform` databases). Applied by `docker-entrypoint.sh` when -`TALE_DB_ROLE=platform`. +dbmate migration set for the **platform** role of the Postgres image (the `db` +service). Applied by `docker-entrypoint.sh` when `TALE_DB_ROLE=platform`. -**This directory is intentionally empty.** The platform/auth schema in -`tale_platform` is owned and migrated by the Convex backend (`bunx convex -deploy`), not by dbmate. Put a timestamped `*.sql` here only if the platform DB -ever needs a raw-SQL migration that Convex cannot express (extensions, roles, -grants beyond `init-scripts/`). +**This directory is intentionally empty.** The 0.5 application database +(`tale_app`) is owned and migrated by the platform backend, which applies its +own numbered `.sql` migrations from `services/platform/backend/db/migrations/` +at boot — not by dbmate. Put a timestamped `*.sql` here only if the platform +Postgres ever needs a raw-SQL migration outside the backend's reach +(extensions, roles, grants beyond `init-scripts/`). The knowledge-corpus migrations live in the sibling [`../knowledge-db/`](../knowledge-db/) (applied when `TALE_DB_ROLE=knowledge`). diff --git a/services/platform/app/features/settings/data-residency/components/deployment-stores.tsx b/services/platform/app/features/settings/data-residency/components/deployment-stores.tsx index 7aab981899..d27bc4ecd6 100644 --- a/services/platform/app/features/settings/data-residency/components/deployment-stores.tsx +++ b/services/platform/app/features/settings/data-residency/components/deployment-stores.tsx @@ -224,9 +224,9 @@ function PgSection({ /** Contextual footnote shown below the fields while the section is enabled. */ note?: ReactNode; /** - * Whether to render the SSL-mode control. Off for the app (Convex metadata) - * DB: its postgres-v5 driver derives the database from INSTANCE_NAME and - * rejects a `?sslmode=` URL, so the boot path cannot honor a chosen mode — + * Whether to render the SSL-mode control. Off for the app DB: the backend + * pins that target's connection test to `sslmode=prefer` + * (domains/deployment/service.ts) instead of honoring a chosen mode — * offering the control would promise a guarantee we can't deliver. */ showSslMode?: boolean; diff --git a/services/platform/env.sh b/services/platform/env.sh index c78b4663e0..d42ebdbb7c 100644 --- a/services/platform/env.sh +++ b/services/platform/env.sh @@ -3,7 +3,8 @@ # Sourced by docker-entrypoint.sh and operator tooling; never prints secrets. set -eo pipefail -# Normalize and export the environment the platform + Convex backend expect. +# Normalize and export the environment the platform web tier and the +# application backend expect. env_normalize_common() { # Application configuration export NODE_ENV="${NODE_ENV:-production}" @@ -13,8 +14,9 @@ env_normalize_common() { # Database configuration. # POSTGRES_URL may be given explicitly; otherwise it is derived from # DB_PASSWORD (+ DB_USER/DB_HOST/DB_PORT) for the default self-hosted - # compose stack. The Convex postgres backend expects the URL WITHOUT a - # database name in the path — it manages its own database. + # compose stack. The URL carries NO database name in its path: the + # knowledge URL below appends `/tale_knowledge`, and the backend tier gets + # its own DATABASE_URL (`…/tale_app`) from compose. if [ -z "${POSTGRES_URL:-}" ]; then local db_user="${DB_USER:-tale}" if [ -z "${DB_PASSWORD:-}" ]; then @@ -37,9 +39,8 @@ env_normalize_common() { # Cross-service URLs (Docker service names by default; override in .env). export SANDBOX_URL="${SANDBOX_URL:-http://sandbox:8003}" - # Convex instance identity. INSTANCE_NAME is pinned to the database created - # by init-scripts/02-create-convex-database.sql. - export INSTANCE_NAME="tale_platform" + # Instance identity: the secret every derived key (WebDAV app passwords, + # sandbox stage tokens) is seeded from. export INSTANCE_SECRET="${INSTANCE_SECRET}" # Root config directory: per-org subtrees at $TALE_CONFIG_DIR//, @@ -47,7 +48,7 @@ env_normalize_common() { # per-domain overrides (AGENTS_DIR etc.) are purged by the entrypoint. export TALE_CONFIG_DIR="${TALE_CONFIG_DIR:-/app/data}" - # Canonical base URL; every other URL (Convex HTTP/WS) is derived in code. + # Canonical base URL; every other public URL is derived from it in code. if [ -z "${SITE_URL:-}" ]; then echo "Error: SITE_URL is required. Set it in your .env file." >&2 exit 1 diff --git a/services/platform/scripts/dev-engine.ts b/services/platform/scripts/dev-engine.ts index db511463c0..fbeceb67b6 100644 --- a/services/platform/scripts/dev-engine.ts +++ b/services/platform/scripts/dev-engine.ts @@ -148,8 +148,6 @@ function envNormalizeCommon() { const port = process.env.PORT || '3000'; const host = process.env.HOST || 'localhost'; - if (!process.env.INSTANCE_NAME) process.env.INSTANCE_NAME = 'tale_platform'; - if (!process.env.SITE_URL) { process.env.SITE_URL = `http://${host}${host === 'localhost' ? `:${port}` : ''}`; } diff --git a/services/platform/tests/guards/db-init-scripts.guard.test.ts b/services/platform/tests/guards/db-init-scripts.guard.test.ts new file mode 100644 index 0000000000..1248912795 --- /dev/null +++ b/services/platform/tests/guards/db-init-scripts.guard.test.ts @@ -0,0 +1,54 @@ +// @vitest-environment node + +import { readdirSync, readFileSync } from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import { describe, expect, it } from 'vitest'; + +/** + * The tale-db image runs every `init-scripts/*.sql` on each container start, + * for the `db` and the `knowledge-db` role alike — so a database declared + * there exists on every Postgres the stack runs. The 0.5 platform reads + * exactly two: `tale_app` (migrated by this backend at boot) and + * `tale_knowledge` (the corpus, migrated by dbmate in the db entrypoint). + * The Convex-era `tale_platform` kept being created long after nothing read + * it; this guard pins the set so a retired store cannot creep back in and a + * new one cannot appear without the platform learning to use it. + */ + +const INIT_SCRIPTS_DIR = path.join( + path.dirname(fileURLToPath(import.meta.url)), + '../../../db/init-scripts', +); + +const CREATE_DATABASE_RE = /CREATE DATABASE\s+"?([A-Za-z_][A-Za-z0-9_]*)"?/g; + +function initScripts(): { name: string; sql: string }[] { + return readdirSync(INIT_SCRIPTS_DIR) + .filter((name) => name.endsWith('.sql')) + .sort() + .map((name) => ({ + name, + sql: readFileSync(path.join(INIT_SCRIPTS_DIR, name), 'utf8'), + })); +} + +describe('tale-db init scripts', () => { + it('create exactly the two databases the 0.5 platform reads', () => { + const created = new Set(); + for (const { sql } of initScripts()) { + for (const match of sql.matchAll(CREATE_DATABASE_RE)) { + created.add(match[1] ?? ''); + } + } + expect([...created].sort()).toEqual(['tale_app', 'tale_knowledge']); + }); + + it('never mention the retired Convex-era tale_platform database', () => { + const offenders = initScripts() + .filter(({ sql }) => sql.includes('tale_platform')) + .map(({ name }) => name); + expect(offenders).toEqual([]); + }); +});