From 7c1b4a59b1c1fb3cd6bdd137985d6d476d3180e3 Mon Sep 17 00:00:00 2001 From: larryro <371767072@qq.com> Date: Fri, 4 Sep 2026 10:24:37 +0800 Subject: [PATCH 1/2] fix(platform): feedback reach, OAuth renewals, unique slots, one 429 Squash of the wt-3 batch for rebase onto main; the PR body carries the per-finding record. --- .env.example | 79 +-- .../http_connectors/token_exchange.test.ts | 4 +- .../core/http_connectors/token_exchange.ts | 7 + .../migrations/0076_messages_unique_slot.sql | 48 ++ .../domains/chat/append-message.test.ts | 89 ++++ .../platform/backend/domains/chat/store.ts | 56 ++- .../domains/connector_credentials/service.ts | 5 +- .../backend/domains/connectors/oauth.test.ts | 227 +++++++++ .../backend/domains/connectors/oauth.ts | 187 +++++-- .../domains/connectors/slack-events.ts | 9 +- .../backend/domains/documents/routes.ts | 10 + .../backend/domains/documents/service.ts | 16 +- .../backend/domains/feedback/access.test.ts | 99 ++++ .../domains/feedback/routes.access.test.ts | 70 +++ .../backend/domains/feedback/routes.ts | 16 +- .../backend/domains/feedback/service.ts | 53 ++ .../platform/backend/domains/files/routes.ts | 6 +- .../backend/domains/folders/routes.ts | 6 +- .../domains/knowledge_entries/routes.ts | 5 +- .../backend/domains/projects/routes.ts | 6 +- .../platform/backend/domains/tasks/routes.ts | 6 +- .../backend/domains/threads/store.test.ts | 85 ++++ .../platform/backend/domains/threads/store.ts | 69 +-- .../platform/backend/domains/tts/routes.ts | 10 + .../platform/backend/domains/tts/service.ts | 8 +- .../platform/backend/domains/webdav/routes.ts | 3 +- .../platform/backend/integration-check.ts | 465 +++++++++++++++++- .../backend/lib/rate-limit-response.test.ts | 44 ++ .../backend/lib/rate-limit-response.ts | 45 ++ .../backend/realtime/oracle-routes.ts | 8 +- services/platform/backend/rest/shared.ts | 5 +- services/platform/backend/rest/v1.ts | 14 +- .../generators/generate-dev-compose.test.ts | 72 ++- .../generators/generate-dev-compose.ts | 7 + 34 files changed, 1649 insertions(+), 190 deletions(-) create mode 100644 services/platform/backend/db/migrations/0076_messages_unique_slot.sql create mode 100644 services/platform/backend/domains/chat/append-message.test.ts create mode 100644 services/platform/backend/domains/connectors/oauth.test.ts create mode 100644 services/platform/backend/domains/feedback/access.test.ts create mode 100644 services/platform/backend/domains/feedback/routes.access.test.ts create mode 100644 services/platform/backend/domains/threads/store.test.ts diff --git a/.env.example b/.env.example index ca53b86276..ae8a2dbd5e 100644 --- a/.env.example +++ b/.env.example @@ -68,24 +68,29 @@ ENCRYPTION_SECRET_HEX=3143246f44def075d40141fb849faffcf409fbbeb7a282a3a7c2f4396f # Services will fail to start if this is not set. Change this password in production. DB_PASSWORD=tale_password_change_me -# Advanced: Override the auto-generated database connection URL -# If not set, POSTGRES_URL is constructed as postgresql://tale:${DB_PASSWORD}@db:5432 -# Note: Convex expects URL without database name; DB name derived from INSTANCE_NAME -# POSTGRES_URL=postgresql://user:password@host:port +# The application backend (the `backend-api` and `backend-worker` services) +# stores its state in the `tale_app` database on the `db` service. Compose +# builds its DATABASE_URL from DB_PASSWORD — nothing to configure here. +# Optional override for the application database name: +# APP_DB_NAME=tale_app # Knowledge datastore (ParadeDB: pg_search + pgvector). Holds the RAG/crawler # corpus (tale_knowledge DB, private_knowledge + public_web schemas). Split from -# the platform DB so it can be relocated/replaced independently for data -# residency. The platform's Convex node-actions connect here via postgres.js. +# the application DB so it can be relocated/replaced independently for data +# residency. The backend (api + worker) connects here via postgres.js. # If not set, defaults to postgresql://tale:${DB_PASSWORD}@knowledge-db:5432/tale_knowledge # KNOWLEDGE_DATABASE_URL=postgresql://user:password@host:port/tale_knowledge KNOWLEDGE_DB_NAME=tale_knowledge # ============================================================================ -# REQUIRED: Convex Instance Configuration +# REQUIRED: Instance secret # ============================================================================ -# Used to derive the Convex admin key for `tale deploy`; deploy fails if unset. -# Auto-generated by `tale init`, or generate with: openssl rand -hex 32 +# The deployment marker the app-side key derivations hang off: the WebDAV +# app-password HMAC key (sha256(":webdav-hmac:v1"), unless +# WEBDAV_APP_PASSWORD_HMAC_KEY is set explicitly), and with it the in-sandbox +# host-call and stage tokens. Auto-generated by `tale init`; keep it STABLE +# across deploys — rotating it stops every issued WebDAV app password from +# verifying. Generate manually with: openssl rand -hex 32 INSTANCE_SECRET=0516d5cddc8b9bbc01238b8696f13c711983f45f6dc4dbf9dc66ba42fc16f504 # ============================================================================ @@ -123,8 +128,9 @@ TALE_AUDIT_SIGNING_KEY=4f8c2a9e7b1d6035e4a8c2f9d7b3061a5e8c4f2a9d7b30615e4c8a2f9 # METRICS_BEARER_TOKEN=your-secret-token-here # # Endpoints (requires valid bearer token): -# https:///metrics/platform -# https:///metrics/convex (Convex backend: 261 built-in metrics) +# https:///metrics/platform (the web tier) +# https:///metrics/backend (the application backend: api + worker) +# https:///metrics/sla-rules # # Prometheus scrape config example: # scrape_configs: @@ -284,9 +290,9 @@ SOPS_AGE_KEY= # must start with the reserved prefix TALE_PROVIDER_KEY_ — any other name is # rejected (fail-closed), so `secretsEnv` can never point at a deployment secret # (SOPS_AGE_KEY, BETTER_AUTH_SECRET, DB_PASSWORD, ...) and have it sent as a -# bearer token to a provider's base URL. Names must be 40 characters or fewer -# (the platform→Convex env-sync limit). Define the variable here / in your -# secret manager so the platform and the Convex backend can read it. +# bearer token to a provider's base URL. The suffix is 1-64 characters of +# A-Z, 0-9 and _. Define the variable here / in your secret manager so the +# backend (api + worker) can read it. # # TALE_PROVIDER_KEY_OPENROUTER=sk-or-... @@ -301,7 +307,7 @@ SOPS_AGE_KEY= # rejected (fail-closed), so `secretEnv` can never point at an unrelated # deployment secret and leak it to the broker. The suffix is per-credential # (one var per broker). Define the variable here / in your secret manager so -# the platform and the Convex backend can read it. +# the backend (api + worker) can read it. # # TALE_TOKEN_SOURCE_COOLAI=brk_live_... @@ -324,7 +330,7 @@ SOPS_AGE_KEY= # # PO-token provider (bgutil HTTP server) base URL — supplies the GVS tokens # that dissolve the bot wall for the mweb/web/tv_simply clients. ZERO CONFIG in -# the self-hosted stack: the bgutil plugin is baked into the convex image and +# the self-hosted stack: the bgutil plugin is baked into the platform image and # the `bgutil-provider` compose sidecar serves tokens, so this defaults to # http://bgutil-provider:4416 automatically. Only set this to point at a # provider on a different host/port. @@ -361,7 +367,7 @@ SOPS_AGE_KEY= # image; leave unset unless you know it's available. # VIDEO_INGEST_IMPERSONATE=safari # -# Toolchain location (self-provisioned yt-dlp/ffmpeg). The convex image bakes +# Toolchain location (self-provisioned yt-dlp/ffmpeg). The platform image bakes # both into the pinned PATH, so leave these unset there; set them on a host or # dev box that provides its own binaries. # VIDEO_INGEST_BIN_DIR: directory PREPENDED to the child PATH so a yt-dlp (and @@ -409,30 +415,25 @@ SOPS_AGE_KEY= ELEVENLABS_API_KEY= # ============================================================================ -# 0.5 backend (parallel build) — opt-in compose profile `backend` +# Application backend — always part of the stack # ============================================================================ -# The platform image can also start as the Postgres-backed 0.5 backend: -# docker compose --profile backend up -d -# brings up one `backend-api` and one `backend-worker` container (scale either -# with --scale). No extra secrets: the containers reuse DB_PASSWORD and store -# state in the `tale_app` database on the `db` service. TALE_ROLE and -# DATABASE_URL are set by compose — nothing to configure here. -# Optional override for the application database name: -# APP_DB_NAME=tale_app -# -# --- Cutting a deployment over to the Postgres backend -------------------- -# BACKEND_UPSTREAM is the single switch. Set it and: -# * the proxy routes auth, the app API, the hint stream, both machine -# doors, SSO/SCIM/trusted-headers, the control channel, cloud-import -# OAuth and WebDAV to the backend instead of Convex; -# * `tale deploy` rolls `backend-api` / `backend-worker` with the platform -# image and drains in-flight chat turns through the control door first; -# * `tale migrate` re-provisions every org through that door (schema -# migrations apply themselves at backend boot). -# Unset it and every lane stays on Convex — the cutover is reversible. +# The platform image runs the Postgres-backed backend as two services that +# every deployment brings up: `backend-api` (auth, the app API, the hint +# stream, both machine doors, SSO/SCIM/trusted headers, the control channel, +# cloud-import OAuth, WebDAV) and `backend-worker` (schedules, watchdogs, +# agent turns). Scale either with --scale. No extra secrets: they reuse +# DB_PASSWORD and store state in the `tale_app` database (APP_DB_NAME, above); +# TALE_ROLE and DATABASE_URL are set by compose. Schema migrations apply +# themselves at backend boot, and `tale migrate` re-provisions the built-in +# defaults into every org through the control door. +# +# BACKEND_UPSTREAM is where the proxy reaches the api. The default +# `backend-api:3005` is the in-compose service; set it only for a split +# deployment that runs the backend elsewhere. # BACKEND_UPSTREAM=backend-api:3005 # # The deploy-time machine door (`/api/control/*`) exists only when this is -# set; `tale deploy`'s drain and `tale migrate` authenticate with it. Any -# high-entropy value works — keep it out of version control. +# set (unset, the door answers 404); `tale deploy`'s drain of in-flight chat +# turns and `tale migrate` authenticate with it. Any high-entropy value works +# — keep it out of version control. # TALE_CONTROL_TOKEN= diff --git a/services/platform/backend/core/http_connectors/token_exchange.test.ts b/services/platform/backend/core/http_connectors/token_exchange.test.ts index aa73112ecb..38f7f40453 100644 --- a/services/platform/backend/core/http_connectors/token_exchange.test.ts +++ b/services/platform/backend/core/http_connectors/token_exchange.test.ts @@ -64,7 +64,7 @@ describe('exchangeAuthorizationCode', () => { }); }); - it('splits comma-separated scopes and surfaces the Slack team id', async () => { + it('splits comma-separated scopes and surfaces the Slack team id and name', async () => { const fetchImpl = vi.fn().mockResolvedValue( jsonResponse({ ok: true, @@ -84,6 +84,8 @@ describe('exchangeAuthorizationCode', () => { expiresAt: undefined, scopes: ['chat:write', 'channels:read'], teamId: 'T0EXCHANGE', + // Labels a second workspace's credential (`Slack (Workspace)`). + teamName: 'Workspace', }, }); }); diff --git a/services/platform/backend/core/http_connectors/token_exchange.ts b/services/platform/backend/core/http_connectors/token_exchange.ts index 70be0c0f19..9892cda044 100644 --- a/services/platform/backend/core/http_connectors/token_exchange.ts +++ b/services/platform/backend/core/http_connectors/token_exchange.ts @@ -40,6 +40,12 @@ export interface Oauth2Tokens { * generically instead of switching on the connector. */ readonly teamId?: string; + /** + * The workspace's display name (`team.name`), when the vendor sends one + * alongside the id — it labels a second workspace's credential so an + * organization connecting several can tell them apart. + */ + readonly teamName?: string; } export type TokenExchangeResult = @@ -199,6 +205,7 @@ export async function exchangeAuthorizationCode( : undefined, scopes: parseScopes(payload.scope), teamId: team ? getString(team, 'id') : undefined, + teamName: team ? getString(team, 'name') : undefined, }, }; } diff --git a/services/platform/backend/db/migrations/0076_messages_unique_slot.sql b/services/platform/backend/db/migrations/0076_messages_unique_slot.sql new file mode 100644 index 0000000000..a478b4e794 --- /dev/null +++ b/services/platform/backend/db/migrations/0076_messages_unique_slot.sql @@ -0,0 +1,48 @@ +-- One row per (thread, order, step) slot in app.messages. +-- +-- The message store orders a thread by ("order", step_order), and every +-- appender claims `max("order") + 1` — a read followed by a write. Under +-- READ COMMITTED two concurrent appends to one thread read the same max and +-- both land on it; the rows then TIE: readers sort them arbitrarily, a branch +-- fork "up to this order" copies both, and nothing ever corrects it. The slot +-- is now UNIQUE, so the second appender is refused at the index and re-claims +-- the next slot (`domains/threads/store.ts`, `domains/chat/store.ts` — +-- `INSERT … ON CONFLICT DO NOTHING` plus a bounded retry). +-- +-- Existing ties are repaired FIRST, deterministically and without deleting a +-- row: within every (thread_id, "order") group that holds a tie, the rows are +-- renumbered 0..n-1 by (step_order, created_at_ms, id). That keeps the order +-- readers already observe (step first, then arrival) and touches only the +-- step numbers of rows in affected groups — every other row keeps its slot. +-- +-- Rolling-deploy safe: the previous image is still serving while this applies. +-- Its appenders carry no ON CONFLICT clause, so a race it loses during the +-- roll answers an error for that one send instead of writing a tie — strictly +-- better than the corruption, and gone once the new image serves. + +WITH tied_groups AS ( + SELECT DISTINCT thread_id, "order" + FROM app.messages + GROUP BY thread_id, "order", step_order + HAVING count(*) > 1 +), +renumbered AS ( + SELECT m.id, + row_number() OVER ( + PARTITION BY m.thread_id, m."order" + ORDER BY m.step_order, m.created_at_ms, m.id + ) - 1 AS step_order + FROM app.messages m + JOIN tied_groups g ON g.thread_id = m.thread_id AND g."order" = m."order" +) +UPDATE app.messages m +SET step_order = r.step_order +FROM renumbered r +WHERE m.id = r.id AND m.step_order <> r.step_order; + +-- The unique slot replaces the plain ordering index: same columns, so every +-- (thread_id, "order", step_order) read keeps its plan. +CREATE UNIQUE INDEX IF NOT EXISTS messages_thread_slot + ON app.messages (thread_id, "order", step_order); + +DROP INDEX IF EXISTS app.messages_thread_order; diff --git a/services/platform/backend/domains/chat/append-message.test.ts b/services/platform/backend/domains/chat/append-message.test.ts new file mode 100644 index 0000000000..7ed730ca02 --- /dev/null +++ b/services/platform/backend/domains/chat/append-message.test.ts @@ -0,0 +1,89 @@ +// @vitest-environment node + +/** + * The chat appender claims its (order, step) slot the same way the generic + * store does: one statement that reads max+1 and is refused by the unique + * slot index when a concurrent turn got there first — after which it claims + * the next slot, so two racing sends never tie a thread's ordering. + */ + +import type { Sql } from 'postgres'; +import { describe, expect, it, vi } from 'vitest'; + +vi.mock('../../lib/org-config.ts', () => ({ resolveOrgSlug: vi.fn() })); +vi.mock('../../core/lib/providers/org_providers.ts', () => ({ + resolveProvidersForOrg: vi.fn(), +})); +vi.mock('../../core/lib/providers/catalog_fetch.ts', () => ({ + getProviderCatalog: vi.fn(), +})); +vi.mock('../../jobs/enqueue.ts', () => ({ addJobInTx: vi.fn() })); + +import { MESSAGE_SLOT_ATTEMPTS } from '../threads/store.ts'; +import { appendMessageRow } from './store.ts'; + +/** A `sql` whose INSERTs answer from `outcomes` in order (an empty array is a + * lost race); every other statement finds nothing. */ +function fakeSql(outcomes: { id: string; order: number }[][]): { + sql: Sql; + statements: string[]; +} { + const statements: string[] = []; + let inserts = 0; + const tag = (strings: TemplateStringsArray): Promise => { + const text = strings.join('?'); + statements.push(text); + if (text.includes('INSERT INTO app.messages')) { + const outcome = outcomes[inserts] ?? []; + inserts += 1; + return Promise.resolve(outcome); + } + return Promise.resolve([]); + }; + Object.assign(tag, { json: (value: unknown) => value }); + // oxlint-disable-next-line typescript/no-unsafe-type-assertion -- only the tag call and `json` are exercised + return { sql: tag as unknown as Sql, statements }; +} + +const MESSAGE = { + organizationId: 'org-1', + threadId: 't-1', + role: 'assistant', + parts: [], + status: 'pending', +}; + +const insertsOf = (statements: string[]): number => + statements.filter((text) => text.includes('INSERT INTO app.messages')).length; + +describe('appendMessageRow — claiming a unique slot', () => { + it('lands on the computed slot when nobody raced it', async () => { + const { sql, statements } = fakeSql([[{ id: 'm-1', order: 7 }]]); + await expect(appendMessageRow(sql, MESSAGE)).resolves.toEqual({ + id: 'm-1', + sequence: 7, + }); + expect(insertsOf(statements)).toBe(1); + expect(statements[0]).toContain( + 'ON CONFLICT (thread_id, "order", step_order) DO NOTHING', + ); + }); + + it('re-claims the next slot after losing the race for one', async () => { + const { sql, statements } = fakeSql([[], [], [{ id: 'm-3', order: 9 }]]); + await expect(appendMessageRow(sql, MESSAGE)).resolves.toEqual({ + id: 'm-3', + sequence: 9, + }); + expect(insertsOf(statements)).toBe(3); + }); + + it('fails loudly, and writes nothing else, once the attempts are spent', async () => { + const { sql, statements } = fakeSql([]); + await expect(appendMessageRow(sql, MESSAGE)).rejects.toThrow( + /no free slot/, + ); + expect(insertsOf(statements)).toBe(MESSAGE_SLOT_ATTEMPTS); + expect(statements.some((text) => text.includes('UPDATE'))).toBe(false); + }); +}); diff --git a/services/platform/backend/domains/chat/store.ts b/services/platform/backend/domains/chat/store.ts index ccc7f31c17..01812ee5d1 100644 --- a/services/platform/backend/domains/chat/store.ts +++ b/services/platform/backend/domains/chat/store.ts @@ -13,6 +13,7 @@ import { toJson } from '../../db/sql.ts'; import { addJobInTx } from '../../jobs/enqueue.ts'; import { resolveOrgSlug } from '../../lib/org-config.ts'; import { incrementUsageLedger } from '../governance/service.ts'; +import { MESSAGE_SLOT_ATTEMPTS } from '../threads/store.ts'; /** * The Postgres-backed ports the turn pipeline writes through — the 0.5 twin @@ -51,28 +52,41 @@ export async function appendMessageRow( status?: string; }, ): Promise<{ id: string; sequence: number }> { - const rows = await sql<{ id: string; order: number }[]>` - INSERT INTO app.messages ( - thread_id, org_id, "order", step_order, role, parts, text, model, - provider_slug, usage, blocked_reason, truncation, error, status, - created_at_ms - ) - SELECT ${message.threadId}, ${message.organizationId}, - coalesce(max("order"), -1) + 1, 0, ${message.role}, - ${message.parts === undefined ? null : sql.json(toJson(message.parts))}, - ${message.text ?? null}, ${message.model ?? null}, - ${message.providerSlug ?? null}, - ${message.usage === undefined ? null : sql.json(toJson(message.usage))}, - ${message.blockedReason ?? null}, - ${message.truncation === undefined ? null : sql.json(toJson(message.truncation))}, - ${message.error ?? null}, ${message.status ?? 'complete'}, - ${Date.now()} - FROM app.messages WHERE thread_id = ${message.threadId} - RETURNING id, "order" - `; - const row = rows[0]; + // The slot is UNIQUE: two turns appending to one thread at once both read + // the same max, and the one the index refuses re-claims the next slot on + // a fresh statement instead of tying the winner's ordering. + let row: { id: string; order: number } | undefined; + for ( + let attempt = 0; + row === undefined && attempt < MESSAGE_SLOT_ATTEMPTS; + attempt += 1 + ) { + const rows = await sql<{ id: string; order: number }[]>` + INSERT INTO app.messages ( + thread_id, org_id, "order", step_order, role, parts, text, model, + provider_slug, usage, blocked_reason, truncation, error, status, + created_at_ms + ) + SELECT ${message.threadId}, ${message.organizationId}, + coalesce(max("order"), -1) + 1, 0, ${message.role}, + ${message.parts === undefined ? null : sql.json(toJson(message.parts))}, + ${message.text ?? null}, ${message.model ?? null}, + ${message.providerSlug ?? null}, + ${message.usage === undefined ? null : sql.json(toJson(message.usage))}, + ${message.blockedReason ?? null}, + ${message.truncation === undefined ? null : sql.json(toJson(message.truncation))}, + ${message.error ?? null}, ${message.status ?? 'complete'}, + ${Date.now()} + FROM app.messages WHERE thread_id = ${message.threadId} + ON CONFLICT (thread_id, "order", step_order) DO NOTHING + RETURNING id, "order" + `; + row = rows[0]; + } if (!row) { - throw new Error('message insert failed'); + throw new Error( + `message insert failed: no free slot after ${MESSAGE_SLOT_ATTEMPTS} attempts`, + ); } // A turn just wrote to the thread; keep its list ordering fresh. An // assistant row also stamps the unread watermark; activity on a hidden diff --git a/services/platform/backend/domains/connector_credentials/service.ts b/services/platform/backend/domains/connector_credentials/service.ts index d9cf42919f..8eb92e88ac 100644 --- a/services/platform/backend/domains/connector_credentials/service.ts +++ b/services/platform/backend/domains/connector_credentials/service.ts @@ -102,7 +102,10 @@ const CREDENTIAL_COLUMNS = ` created_at_ms::float8 AS "createdAt", updated_at_ms::float8 AS "updatedAt" `; -const NAME_MAX = 100; +/** The longest label a credential may carry — the OAuth callback derives + * workspace-named labels and must stay within it. */ +export const CREDENTIAL_NAME_MAX = 100; +const NAME_MAX = CREDENTIAL_NAME_MAX; const SECRET_VALUE_MAX = 8192; function normalizeName(raw: string): string { diff --git a/services/platform/backend/domains/connectors/oauth.test.ts b/services/platform/backend/domains/connectors/oauth.test.ts new file mode 100644 index 0000000000..c3c3486374 --- /dev/null +++ b/services/platform/backend/domains/connectors/oauth.test.ts @@ -0,0 +1,227 @@ +// @vitest-environment node + +/** + * Where a completed consent lands. The settings card advertises Reconnect for + * an oauth2 credential, and an organization may connect several Slack + * workspaces — neither may fail on the label the first credential already + * holds. A workspace already routed here renews ITS credential; a new + * workspace (or a first connection) is stored under a label no sibling has — + * and its workspace claim commits in the same transaction, so a lost claim + * keeps nothing. + */ + +import type { Sql } from 'postgres'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const { createCredentialInTransaction, listCredentials, updateCredential } = + vi.hoisted(() => ({ + createCredentialInTransaction: vi.fn(), + listCredentials: vi.fn(), + updateCredential: vi.fn(), + })); + +vi.mock('../connector_credentials/service.ts', () => ({ + CREDENTIAL_NAME_MAX: 100, + createCredentialInTransaction, + listCredentials, + updateCredential, +})); + +import { storeOauth2Grant, uniqueCredentialName } from './oauth.ts'; + +interface Route { + organizationId: string; + credentialId: string; +} + +/** + * A tagged-template `sql` that answers the team-route lookup and the route + * claim, and runs `begin` callbacks against itself — the store-and-claim + * transaction is what a NEW credential lands in. A claim on a workspace whose + * route names another organization answers no row, exactly as the + * `ON CONFLICT … WHERE org_id = $me` insert does. + */ +function sqlWithRoutes(routes: Record): Sql { + const tag = ( + strings: TemplateStringsArray, + ...values: unknown[] + ): Promise => { + const text = strings.join('?'); + if (text.includes('INSERT INTO app.connector_team_routes')) { + const teamId = String(values[0]); + const organizationId = String(values[1]); + const route = routes[teamId]; + return Promise.resolve( + route !== undefined && route.organizationId !== organizationId + ? [] + : [{ teamId }], + ); + } + if (text.includes('FROM app.connector_team_routes')) { + const teamId = String(values[0]); + const route = routes[teamId]; + return Promise.resolve(route === undefined ? [] : [route]); + } + return Promise.resolve([]); + }; + const begin = (callback: (tx: unknown) => Promise) => callback(tag); + // oxlint-disable-next-line typescript/no-unsafe-type-assertion -- only the tag call and `begin` are exercised + return Object.assign(tag, { begin }) as unknown as Sql; +} + +const credential = ( + id: string, + name: string, + extra: Partial<{ + authMethod: string; + isDefault: boolean; + createdAt: number; + }> = {}, +) => ({ + id, + connectorSlug: 'slack', + authMethod: 'oauth2', + name, + isDefault: false, + status: 'active', + createdAt: 1, + updatedAt: 1, + ...extra, +}); + +const TOKENS = { + accessToken: 'xoxb-fresh', + refreshToken: 'xoxe-fresh', + expiresAt: 4_102_444_800_000, + scopes: ['chat:write'], +}; + +const grant = ( + sql: Sql, + tokens: typeof TOKENS & { teamId?: string; teamName?: string }, +) => + storeOauth2Grant(sql, { + organizationId: 'org-1', + connectorSlug: 'slack', + userId: 'user-1', + displayName: 'Slack', + tokens, + }); + +beforeEach(() => { + vi.clearAllMocks(); + createCredentialInTransaction.mockResolvedValue({ credentialId: 'cred-new' }); + updateCredential.mockResolvedValue(undefined); +}); + +describe('uniqueCredentialName', () => { + it('keeps the base when no sibling holds it', () => { + expect(uniqueCredentialName(['Gmail'], 'Slack')).toBe('Slack'); + }); + + it('counts past every taken label, case-insensitively', () => { + expect(uniqueCredentialName(['slack', 'Slack (2)'], 'Slack')).toBe( + 'Slack (3)', + ); + }); +}); + +describe('storeOauth2Grant', () => { + it('stores the first connection under the connector display name', async () => { + listCredentials.mockResolvedValue([]); + const outcome = await grant(sqlWithRoutes({}), { + ...TOKENS, + teamId: 'T-1', + teamName: 'Acme', + }); + expect(outcome).toEqual({ credentialId: 'cred-new', renewed: false }); + expect(updateCredential).not.toHaveBeenCalled(); + expect(createCredentialInTransaction).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ + name: 'Slack', + authMethod: 'oauth2', + secret: TOKENS, + }), + ); + }); + + it('renews the credential a connected workspace already routes to', async () => { + listCredentials.mockResolvedValue([credential('cred-1', 'Slack')]); + const outcome = await grant( + sqlWithRoutes({ + 'T-1': { organizationId: 'org-1', credentialId: 'cred-1' }, + }), + { ...TOKENS, teamId: 'T-1' }, + ); + expect(outcome).toEqual({ credentialId: 'cred-1', renewed: true }); + expect(createCredentialInTransaction).not.toHaveBeenCalled(); + expect(updateCredential).toHaveBeenCalledWith(expect.anything(), { + organizationId: 'org-1', + credentialId: 'cred-1', + secret: TOKENS, + status: 'active', + statusDetail: null, + }); + }); + + it('names a second workspace after itself instead of colliding', async () => { + listCredentials.mockResolvedValue([credential('cred-1', 'Slack')]); + const outcome = await grant( + sqlWithRoutes({ + 'T-1': { organizationId: 'org-1', credentialId: 'cred-1' }, + }), + { ...TOKENS, teamId: 'T-2', teamName: 'Second Workspace' }, + ); + expect(outcome.renewed).toBe(false); + expect(createCredentialInTransaction).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ name: 'Slack (Second Workspace)' }), + ); + }); + + it('falls back to a counter when the workspace name is unknown or taken', async () => { + listCredentials.mockResolvedValue([ + credential('cred-1', 'Slack'), + credential('cred-2', 'Slack (2)'), + ]); + await grant(sqlWithRoutes({}), { ...TOKENS, teamId: 'T-3' }); + expect(createCredentialInTransaction).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ name: 'Slack (3)' }), + ); + }); + + it("never renews another organization's credential for the same workspace — the claim refuses and nothing is kept", async () => { + listCredentials.mockResolvedValue([]); + await expect( + grant( + sqlWithRoutes({ + 'T-1': { organizationId: 'org-other', credentialId: 'cred-x' }, + }), + { ...TOKENS, teamId: 'T-1' }, + ), + ).rejects.toThrow('workspace already connected to another organization'); + expect(updateCredential).not.toHaveBeenCalled(); + // The credential row was written INSIDE the transaction the lost claim + // rolls back — the database keeps nothing. + expect(createCredentialInTransaction).toHaveBeenCalledTimes(1); + }); + + it('reconnects a connector without a workspace notion through its default grant', async () => { + listCredentials.mockResolvedValue([ + credential('cred-old', 'Gmail (2)', { createdAt: 1 }), + credential('cred-default', 'Gmail', { isDefault: true, createdAt: 2 }), + credential('cred-key', 'Gmail key', { authMethod: 'api-key' }), + ]); + const outcome = await storeOauth2Grant(sqlWithRoutes({}), { + organizationId: 'org-1', + connectorSlug: 'gmail', + userId: 'user-1', + displayName: 'Gmail', + tokens: TOKENS, + }); + expect(outcome).toEqual({ credentialId: 'cred-default', renewed: true }); + expect(createCredentialInTransaction).not.toHaveBeenCalled(); + }); +}); diff --git a/services/platform/backend/domains/connectors/oauth.ts b/services/platform/backend/domains/connectors/oauth.ts index 2f3389e9e8..3bbab10318 100644 --- a/services/platform/backend/domains/connectors/oauth.ts +++ b/services/platform/backend/domains/connectors/oauth.ts @@ -14,8 +14,16 @@ import { mintStateToken, OAUTH_STATE_TTL_MS, } from '../../core/http_connectors/oauth_state.ts'; -import { exchangeAuthorizationCode } from '../../core/http_connectors/token_exchange.ts'; -import { createCredentialInTransaction } from '../connector_credentials/service.ts'; +import { + exchangeAuthorizationCode, + type Oauth2Tokens, +} from '../../core/http_connectors/token_exchange.ts'; +import { + CREDENTIAL_NAME_MAX, + createCredentialInTransaction, + listCredentials, + updateCredential, +} from '../connector_credentials/service.ts'; import { applyMicrosoftTenant, resolveConnectorOauthApp, @@ -201,6 +209,140 @@ export async function claimTeamRoute( return { ok: true }; } +/** + * The label a NEW credential gets among the names its (organization, + * connector) siblings already hold: `base` itself when free, else + * `base (2)`, `base (3)`, … — compared case-insensitively, the way the + * table's unique index compares. Pure, so the rule is testable on its own. + */ +export function uniqueCredentialName( + taken: readonly string[], + base: string, +): string { + const held = new Set(taken.map((name) => name.trim().toLowerCase())); + let candidate = base; + let counter = 1; + while (held.has(candidate.toLowerCase())) { + counter += 1; + candidate = `${base} (${counter})`; + } + return candidate; +} + +/** Room a workspace-named label leaves for the ` (N)` a collision appends. */ +const NAME_COUNTER_ROOM = 6; + +export interface Oauth2GrantArgs { + organizationId: string; + connectorSlug: string; + userId: string; + /** The connector's catalog display name — the first credential's label. */ + displayName: string; + tokens: Oauth2Tokens; +} + +/** + * Where a completed consent lands. + * + * A workspace this organization already connected — the team route names its + * credential — is a RECONNECT: that credential takes the fresh grant and is + * active again, so the settings card's Reconnect action (and a second consent + * for the same workspace) renews what is there instead of failing on the + * label it already holds. A connector with no workspace notion reconnects + * the same way — the pair's one oauth2 credential (its default, when several + * exist) is the grant being renewed. Everything else is a NEW connection: a + * first one, or a second Slack workspace — stored under a label no sibling + * holds (the connector's display name for the first, the workspace name or a + * counter after that). + */ +export async function storeOauth2Grant( + sql: Sql, + args: Oauth2GrantArgs, +): Promise<{ credentialId: string; renewed: boolean }> { + const { tokens } = args; + const secret = { + accessToken: tokens.accessToken, + ...(tokens.refreshToken !== undefined + ? { refreshToken: tokens.refreshToken } + : {}), + ...(tokens.expiresAt !== undefined ? { expiresAt: tokens.expiresAt } : {}), + scopes: tokens.scopes, + }; + const siblings = await listCredentials( + sql, + args.organizationId, + args.connectorSlug, + ); + const grants = siblings.filter((row) => row.authMethod === 'oauth2'); + + let renewId: string | null = null; + if (tokens.teamId !== undefined) { + const route = await resolveTeamRoute(sql, tokens.teamId); + if ( + route !== null && + route.organizationId === args.organizationId && + grants.some((row) => row.id === route.credentialId) + ) { + renewId = route.credentialId; + } + } else if (grants.length > 0) { + const oldest = [...grants].sort((a, b) => a.createdAt - b.createdAt)[0]; + renewId = grants.find((row) => row.isDefault)?.id ?? oldest?.id ?? null; + } + + if (renewId !== null) { + await updateCredential(sql, { + organizationId: args.organizationId, + credentialId: renewId, + secret, + status: 'active', + statusDetail: null, + }); + console.info( + `[connectors:oauth2] "${args.connectorSlug}" grant renewed for organization ${args.organizationId}`, + ); + return { credentialId: renewId, renewed: true }; + } + + const workspace = tokens.teamName?.trim() ?? ''; + const base = + grants.length === 0 || workspace.length === 0 + ? args.displayName + : `${args.displayName} (${workspace})`.slice( + 0, + CREDENTIAL_NAME_MAX - NAME_COUNTER_ROOM, + ); + // Store the credential and claim the workspace in ONE transaction. Two + // organizations can pass the pre-check for the same workspace at once; + // the route's key decides the winner, and the loser must keep nothing — + // a committed credential for a workspace routed elsewhere would be a + // live foreign token stored (and default) for this organization. + const name = uniqueCredentialName( + siblings.map((row) => row.name), + base, + ); + let created!: { credentialId: string }; + await sql.begin(async (tx) => { + created = await createCredentialInTransaction(tx, { + organizationId: args.organizationId, + connectorSlug: args.connectorSlug, + authMethod: 'oauth2', + name, + createdBy: args.userId, + secret, + }); + if (tokens.teamId !== undefined) { + const claim = await claimTeamRoute(tx, { + teamId: tokens.teamId, + organizationId: args.organizationId, + credentialId: created.credentialId, + }); + if (!claim.ok) throw new WorkspaceClaimedError(); + } + }); + return { credentialId: created.credentialId, renewed: false }; +} + export type StartOutcome = | { kind: 'redirect'; url: string } | { kind: 'error'; error: ConnectorFlowError }; @@ -407,38 +549,17 @@ export async function completeOauth2( } } - // Store the credential and claim the workspace in ONE transaction. Two - // organizations can pass the pre-check for the same workspace at once; - // the route's key decides the winner, and the loser must keep nothing — - // a committed credential for a workspace routed elsewhere would be a - // live foreign token stored (and default) for this organization. try { - await sql.begin(async (tx) => { - const stored = await createCredentialInTransaction(tx, { - organizationId, - connectorSlug, - authMethod: 'oauth2', - name: endpoints.displayName, - createdBy: userId, - secret: { - accessToken: tokens.accessToken, - ...(tokens.refreshToken !== undefined - ? { refreshToken: tokens.refreshToken } - : {}), - ...(tokens.expiresAt !== undefined - ? { expiresAt: tokens.expiresAt } - : {}), - scopes: tokens.scopes, - }, - }); - if (tokens.teamId !== undefined) { - const claim = await claimTeamRoute(tx, { - teamId: tokens.teamId, - organizationId, - credentialId: stored.credentialId, - }); - if (!claim.ok) throw new WorkspaceClaimedError(); - } + // A renewal for a workspace (or connector) already connected here, or a + // new credential under a label no sibling holds — never a collision on + // the connector's display name. A NEW credential and its workspace claim + // commit together inside storeOauth2Grant, or not at all. + await storeOauth2Grant(sql, { + organizationId, + connectorSlug, + userId, + displayName: endpoints.displayName, + tokens, }); } catch (error) { if (error instanceof WorkspaceClaimedError) { diff --git a/services/platform/backend/domains/connectors/slack-events.ts b/services/platform/backend/domains/connectors/slack-events.ts index dd118926fc..cd7d9db368 100644 --- a/services/platform/backend/domains/connectors/slack-events.ts +++ b/services/platform/backend/domains/connectors/slack-events.ts @@ -11,6 +11,7 @@ import { import { getClientIp } from '../../core/lib/utils/client_ip.ts'; import { addJobInTx } from '../../jobs/enqueue.ts'; import { readGovernancePolicy } from '../../lib/org-config.ts'; +import { rateLimitedPlainResponse } from '../../lib/rate-limit-response.ts'; import { checkIpRateLimit, RateLimitExceededError, @@ -67,13 +68,7 @@ async function throttleAndRefuse(sql: Sql, req: Request): Promise { await checkIpRateLimit(sql, 'connector:slack-events', ip); } catch (error) { if (error instanceof RateLimitExceededError) { - return new Response('Rate limit exceeded', { - status: 429, - headers: { - 'Content-Type': 'text/plain; charset=utf-8', - 'Retry-After': String(Math.ceil(error.retryAfter / 1000)), - }, - }); + return rateLimitedPlainResponse(error); } console.error( '[connectors:slack] rate-limit check failed; refusing the request anyway', diff --git a/services/platform/backend/domains/documents/routes.ts b/services/platform/backend/domains/documents/routes.ts index 36a8d13e32..d3184e25fd 100644 --- a/services/platform/backend/domains/documents/routes.ts +++ b/services/platform/backend/domains/documents/routes.ts @@ -6,6 +6,10 @@ import { z } from 'zod'; import type { Auth } from '../../auth/auth.ts'; import { requireOrgMember, type OrgEnv } from '../../auth/org.ts'; import { requireSession } from '../../auth/session.ts'; +import { + rateLimitExceededCause, + rateLimitedResponse, +} from '../../lib/rate-limit-response.ts'; import { FileError, getFileUrl } from '../files/service.ts'; import { FolderError } from '../folders/service.ts'; import { syncRagDocumentScope } from '../knowledge/service.ts'; @@ -102,6 +106,12 @@ function handleError( c: Context, error: unknown, ): Response { + // A spent budget answers the one 429 every door speaks, whether the + // limiter threw it here or a service wrapped it as a coded refusal. + const limited = rateLimitExceededCause(error); + if (limited !== null) { + return rateLimitedResponse(c, limited); + } if (error instanceof DocumentError) { return c.json( { diff --git a/services/platform/backend/domains/documents/service.ts b/services/platform/backend/domains/documents/service.ts index 94bd314ab7..d42fac5f46 100644 --- a/services/platform/backend/domains/documents/service.ts +++ b/services/platform/backend/domains/documents/service.ts @@ -66,8 +66,10 @@ export class DocumentError extends Error { message: string, status: 400 | 403 | 404 | 429 = 400, data?: Record, + /** The refusal this one wraps (a rate limit), for the door to answer. */ + options?: ErrorOptions, ) { - super(message); + super(message, options); this.name = 'DocumentError'; this.code = code; this.status = status; @@ -1510,9 +1512,15 @@ export async function validateDocumentUploadForOrg( await checkOrganizationRateLimit(sql, 'file:upload', auth.organizationId); } catch (error) { if (error instanceof RateLimitExceededError) { - throw new DocumentError('RATE_LIMITED', error.message, 429, { - retryAfterMs: error.retryAfter, - }); + // Coded for the REST helpers and bridges that read codes; the app door + // answers the one 429 from the cause (`rateLimitExceededCause`). + throw new DocumentError( + 'RATE_LIMITED', + error.message, + 429, + { retryAfterMs: error.retryAfter }, + { cause: error }, + ); } throw error; } diff --git a/services/platform/backend/domains/feedback/access.test.ts b/services/platform/backend/domains/feedback/access.test.ts new file mode 100644 index 0000000000..ffcc3aeb4b --- /dev/null +++ b/services/platform/backend/domains/feedback/access.test.ts @@ -0,0 +1,99 @@ +// @vitest-environment node + +/** + * A vote is recorded only against a message the caller can read. The ids in + * the body are client-supplied, so the service re-derives the right to vote + * from the database: the message must exist in the caller's organization and + * thread, and the thread must be theirs or shared with a project they can + * read. Every other case is the same opaque refusal, before any write. + */ + +import type { TransactionSql } from 'postgres'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const { loadOwnedThread, loadProjectSharedThread } = vi.hoisted(() => ({ + loadOwnedThread: vi.fn(), + loadProjectSharedThread: vi.fn(), +})); + +vi.mock('../chat/threads.ts', () => ({ + loadOwnedThread, + loadProjectSharedThread, +})); + +import { FeedbackError, submitMessageFeedback } from './service.ts'; + +const SCOPE = { organizationId: 'org-1', userId: 'user-1' }; +const VOTE = { threadId: 't-1', messageId: 'm-1', rating: 'positive' } as const; + +/** A tagged-template `tx` that answers the message lookup from `messageRows` + * and records every statement, so the test can see whether a write ran. */ +function fakeTx(messageRows: { id: string }[]): { + tx: TransactionSql; + statements: string[]; +} { + const statements: string[] = []; + const tag = (strings: TemplateStringsArray): Promise => { + const text = strings.join('?'); + statements.push(text); + return Promise.resolve( + text.includes('FROM app.messages') ? messageRows : [], + ); + }; + Object.assign(tag, { json: (value: unknown) => value }); + // oxlint-disable-next-line typescript/no-unsafe-type-assertion -- only the tag call and `json` are exercised + return { tx: tag as unknown as TransactionSql, statements }; +} + +const wrote = (statements: string[]): boolean => + statements.some((text) => text.includes('INSERT INTO app.message_feedback')); + +beforeEach(() => { + vi.clearAllMocks(); + loadOwnedThread.mockResolvedValue(null); + loadProjectSharedThread.mockResolvedValue(null); +}); + +describe('submitMessageFeedback — the message must be within reach', () => { + it('refuses a message that is not in the caller organization and thread, before any write', async () => { + const { tx, statements } = fakeTx([]); + await expect(submitMessageFeedback(tx, SCOPE, VOTE)).rejects.toBeInstanceOf( + FeedbackError, + ); + expect(wrote(statements)).toBe(false); + // Nothing else is consulted once the message itself is out of reach. + expect(loadOwnedThread).not.toHaveBeenCalled(); + expect(loadProjectSharedThread).not.toHaveBeenCalled(); + }); + + it('records the vote on a message in a thread the caller owns', async () => { + loadOwnedThread.mockResolvedValue({ id: 't-1' }); + const { tx, statements } = fakeTx([{ id: 'm-1' }]); + await submitMessageFeedback(tx, SCOPE, VOTE); + expect(wrote(statements)).toBe(true); + expect(loadOwnedThread).toHaveBeenCalledWith(tx, 'org-1', 'user-1', 't-1'); + }); + + it('records the vote on a message in a thread shared with a project the caller can read', async () => { + loadProjectSharedThread.mockResolvedValue({ id: 't-1' }); + const { tx, statements } = fakeTx([{ id: 'm-1' }]); + await submitMessageFeedback(tx, SCOPE, VOTE); + expect(wrote(statements)).toBe(true); + expect(loadProjectSharedThread).toHaveBeenCalledWith( + tx, + 'org-1', + 'user-1', + 't-1', + ); + }); + + it("refuses another member's private thread with the same opaque answer", async () => { + const { tx, statements } = fakeTx([{ id: 'm-1' }]); + const refusal = await submitMessageFeedback(tx, SCOPE, VOTE).catch( + (error: unknown) => error, + ); + expect(refusal).toBeInstanceOf(FeedbackError); + expect(refusal).toMatchObject({ code: 'MESSAGE_NOT_FOUND', status: 404 }); + expect(wrote(statements)).toBe(false); + }); +}); diff --git a/services/platform/backend/domains/feedback/routes.access.test.ts b/services/platform/backend/domains/feedback/routes.access.test.ts new file mode 100644 index 0000000000..acfcc55b09 --- /dev/null +++ b/services/platform/backend/domains/feedback/routes.access.test.ts @@ -0,0 +1,70 @@ +// @vitest-environment node + +/** + * The vote door answers a message outside the caller's reach with one opaque + * 404 — the same status for "no such message", "another organization's" and + * "not your thread", so a member cannot probe a foreign message id by voting + * on it. + */ + +import type { Context } from 'hono'; +import { describe, expect, it, vi } from 'vitest'; + +import type { OrgEnv } from '../../auth/org.ts'; + +vi.mock('@tale/shared/db/serializable', () => ({ + transactSerializable: (_sql: unknown, fn: (tx: unknown) => unknown) => { + // A `tx` whose every statement finds nothing: the named message does not + // exist in the caller's organization. + const tag = (): Promise => Promise.resolve([]); + Object.assign(tag, { json: (value: unknown) => value }); + return fn(tag); + }, +})); + +vi.mock('../chat/threads.ts', () => ({ + loadOwnedThread: vi.fn(async () => null), + loadProjectSharedThread: vi.fn(async () => null), +})); + +vi.mock('../../auth/session.ts', () => ({ + requireSession: + () => async (c: Context, next: () => Promise) => { + c.set('sessionBundle', { + user: { id: 'u1', email: 'u@example.test' }, + } as never); + await next(); + }, +})); + +vi.mock('../../auth/org.ts', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + requireOrgMember: + () => async (c: Context, next: () => Promise) => { + c.set('orgId', 'o1'); + c.set('orgMember', { role: 'member' } as never); + await next(); + }, + }; +}); + +import { createFeedbackRoutes } from './routes.ts'; + +describe('POST /feedback — a message outside the caller reach', () => { + it('answers an opaque 404 and records nothing', async () => { + const app = createFeedbackRoutes({ sql: {} as never, auth: {} as never }); + const res = await app.request('/?orgId=o1', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ + threadId: 'someone-elses-thread', + messageId: 'someone-elses-message', + rating: 'positive', + }), + }); + expect(res.status).toBe(404); + expect(await res.json()).toEqual({ error: 'MESSAGE_NOT_FOUND' }); + }); +}); diff --git a/services/platform/backend/domains/feedback/routes.ts b/services/platform/backend/domains/feedback/routes.ts index ea3cff758c..036dc40c0c 100644 --- a/services/platform/backend/domains/feedback/routes.ts +++ b/services/platform/backend/domains/feedback/routes.ts @@ -9,6 +9,7 @@ import { isAdminRole } from '../../auth/membership.ts'; import { requireOrgMember, type OrgEnv } from '../../auth/org.ts'; import { requireSession } from '../../auth/session.ts'; import { + FeedbackError, listMyThreadFeedback, getMyMessageFeedback, listMessageFeedback, @@ -54,9 +55,18 @@ export function createFeedbackRoutes(deps: { return c.json({ error: 'invalid body' }, 400); } const scope = scopeOf(c); - await transactSerializable(deps.sql, (tx) => - submitMessageFeedback(tx, scope, body.data), - ); + try { + await transactSerializable(deps.sql, (tx) => + submitMessageFeedback(tx, scope, body.data), + ); + } catch (error) { + // A message outside the caller's reach — another organization's, a + // thread that is not theirs, or none at all — is one opaque refusal. + if (error instanceof FeedbackError) { + return c.json({ error: error.code }, error.status); + } + throw error; + } return c.json({ ok: true }); }); diff --git a/services/platform/backend/domains/feedback/service.ts b/services/platform/backend/domains/feedback/service.ts index a80c2e763f..0402ac9300 100644 --- a/services/platform/backend/domains/feedback/service.ts +++ b/services/platform/backend/domains/feedback/service.ts @@ -1,5 +1,7 @@ import type { Sql, TransactionSql } from 'postgres'; +import { loadOwnedThread, loadProjectSharedThread } from '../chat/threads.ts'; + /** * Message feedback — thumbs up/down on assistant messages, one row per * (message, user), upsert-on-revote. Every active member may read and write @@ -9,11 +11,61 @@ import type { Sql, TransactionSql } from 'postgres'; * partial unique index keys on. */ +export class FeedbackError extends Error { + readonly code: string; + readonly status: 404; + constructor(code: string, message: string) { + super(message); + this.name = 'FeedbackError'; + this.code = code; + this.status = 404; + } +} + export interface FeedbackScope { organizationId: string; userId: string; } +/** + * A vote lands only on a message the caller can READ: it exists in the + * caller's organization, inside the thread the client named, and that thread + * is the caller's own or one its owner shared with a project the caller can + * read — the same two grants the chat surface reads through. Anything else + * is one opaque "not found": the ids are client-supplied, and the answer must + * not confirm that a foreign message exists. + */ +async function assertVotableMessage( + tx: TransactionSql, + scope: FeedbackScope, + threadId: string, + messageId: string, +): Promise { + const refuse = (): FeedbackError => + new FeedbackError('MESSAGE_NOT_FOUND', 'Message not found.'); + const rows = await tx<{ id: string }[]>` + SELECT id FROM app.messages + WHERE id = ${messageId} AND thread_id = ${threadId} + AND org_id = ${scope.organizationId} + LIMIT 1 + `; + if (rows.length === 0) throw refuse(); + const owned = await loadOwnedThread( + tx, + scope.organizationId, + scope.userId, + threadId, + ); + if (owned !== null) return; + const shared = await loadProjectSharedThread( + tx, + scope.organizationId, + scope.userId, + threadId, + ); + if (shared === null) throw refuse(); +} + export interface SubmitFeedbackArgs { threadId: string; messageId: string; @@ -29,6 +81,7 @@ export async function submitMessageFeedback( scope: FeedbackScope, args: SubmitFeedbackArgs, ): Promise { + await assertVotableMessage(tx, scope, args.threadId, args.messageId); const now = Date.now(); // `metadata` is written NULL unconditionally: it is the vote's upsert key // (the partial unique index is `WHERE metadata IS NULL`), so a value here diff --git a/services/platform/backend/domains/files/routes.ts b/services/platform/backend/domains/files/routes.ts index 300aa76592..1f2a67f730 100644 --- a/services/platform/backend/domains/files/routes.ts +++ b/services/platform/backend/domains/files/routes.ts @@ -7,6 +7,7 @@ import { isAudioOrVideo } from '../../../lib/shared/file-types.ts'; import type { Auth } from '../../auth/auth.ts'; import { requireOrgMember, type OrgEnv } from '../../auth/org.ts'; import { requireSession } from '../../auth/session.ts'; +import { rateLimitedResponse } from '../../lib/rate-limit-response.ts'; import { checkUserRateLimit, RateLimitExceededError, @@ -58,10 +59,7 @@ function handleError( return c.json({ error: error.code }, error.status); } if (error instanceof RateLimitExceededError) { - return c.json( - { error: 'RATE_LIMITED', data: { retryAfterMs: error.retryAfter } }, - 429, - ); + return rateLimitedResponse(c, error); } throw error; } diff --git a/services/platform/backend/domains/folders/routes.ts b/services/platform/backend/domains/folders/routes.ts index 52d8823eaf..8b8fb08436 100644 --- a/services/platform/backend/domains/folders/routes.ts +++ b/services/platform/backend/domains/folders/routes.ts @@ -6,6 +6,7 @@ import { z } from 'zod'; import type { Auth } from '../../auth/auth.ts'; import { requireOrgMember, type OrgEnv } from '../../auth/org.ts'; import { requireSession } from '../../auth/session.ts'; +import { rateLimitedResponse } from '../../lib/rate-limit-response.ts'; import { checkOrganizationRateLimit, RateLimitExceededError, @@ -53,10 +54,7 @@ function handleError( return c.json({ error: error.code, message: error.message }, error.status); } if (error instanceof RateLimitExceededError) { - return c.json( - { error: 'RATE_LIMITED', data: { retryAfterMs: error.retryAfter } }, - 429, - ); + return rateLimitedResponse(c, error); } throw error; } diff --git a/services/platform/backend/domains/knowledge_entries/routes.ts b/services/platform/backend/domains/knowledge_entries/routes.ts index f9b1f3786c..689f821eed 100644 --- a/services/platform/backend/domains/knowledge_entries/routes.ts +++ b/services/platform/backend/domains/knowledge_entries/routes.ts @@ -5,6 +5,7 @@ import { z } from 'zod'; import type { Auth } from '../../auth/auth.ts'; import { requireOrgMember, type OrgEnv } from '../../auth/org.ts'; import { requireSession } from '../../auth/session.ts'; +import { rateLimitedResponse } from '../../lib/rate-limit-response.ts'; import { checkOrganizationRateLimit, RateLimitExceededError, @@ -28,9 +29,7 @@ function handleError( return c.json({ error: error.code, message: error.message }, error.status); } if (error instanceof RateLimitExceededError) { - return c.json({ error: 'RATE_LIMITED' }, 429, { - 'retry-after': String(Math.ceil(error.retryAfter / 1000)), - }); + return rateLimitedResponse(c, error); } throw error; } diff --git a/services/platform/backend/domains/projects/routes.ts b/services/platform/backend/domains/projects/routes.ts index 92d1e9dc9b..4552003c6d 100644 --- a/services/platform/backend/domains/projects/routes.ts +++ b/services/platform/backend/domains/projects/routes.ts @@ -6,6 +6,7 @@ import { z } from 'zod'; import type { Auth } from '../../auth/auth.ts'; import { requireOrgMember, type OrgEnv } from '../../auth/org.ts'; import { requireSession } from '../../auth/session.ts'; +import { rateLimitedResponse } from '../../lib/rate-limit-response.ts'; import { checkUserRateLimit, RateLimitExceededError, @@ -120,10 +121,7 @@ function handleError( ); } if (error instanceof RateLimitExceededError) { - return c.json( - { error: 'RATE_LIMITED', data: { retryAfterMs: error.retryAfter } }, - 429, - ); + return rateLimitedResponse(c, error); } throw error; } diff --git a/services/platform/backend/domains/tasks/routes.ts b/services/platform/backend/domains/tasks/routes.ts index a9302a8c4d..04474ce2d5 100644 --- a/services/platform/backend/domains/tasks/routes.ts +++ b/services/platform/backend/domains/tasks/routes.ts @@ -8,6 +8,7 @@ import { requireOrgMember, type OrgEnv } from '../../auth/org.ts'; import { requireSession } from '../../auth/session.ts'; import { resolveTaskServing } from '../../core/tasks/task_serving.ts'; import { createCtxShim } from '../../lib/ctx-shim.ts'; +import { rateLimitedResponse } from '../../lib/rate-limit-response.ts'; import { checkUserRateLimit, RateLimitExceededError, @@ -168,10 +169,7 @@ function handleError( ); } if (error instanceof RateLimitExceededError) { - return c.json( - { error: 'RATE_LIMITED', data: { retryAfterMs: error.retryAfter } }, - 429, - ); + return rateLimitedResponse(c, error); } // A comment whose @mentions could not be resolved is NOT posted — the // author sees a retryable failure instead of a comment that silently diff --git a/services/platform/backend/domains/threads/store.test.ts b/services/platform/backend/domains/threads/store.test.ts new file mode 100644 index 0000000000..2bc8dd0c8e --- /dev/null +++ b/services/platform/backend/domains/threads/store.test.ts @@ -0,0 +1,85 @@ +// @vitest-environment node + +/** + * A message slot is unique: an append that loses the race for `max+1` gets + * no row back from `ON CONFLICT DO NOTHING` and must claim the next slot on + * a fresh statement — never land on the winner's slot, never surface the + * lost race as an error while attempts remain. + */ + +import type { TransactionSql } from 'postgres'; +import { describe, expect, it } from 'vitest'; + +import { MESSAGE_SLOT_ATTEMPTS, saveMessage } from './store.ts'; + +/** A `tx` whose INSERTs answer from `outcomes` in order (an empty array is a + * lost race), recording every statement so the retry count is observable. */ +function fakeTx(outcomes: { id: string; order: number }[][]): { + tx: TransactionSql; + statements: string[]; +} { + const statements: string[] = []; + let inserts = 0; + const tag = (strings: TemplateStringsArray): Promise => { + const text = strings.join('?'); + statements.push(text); + if (text.includes('INSERT INTO app.messages')) { + const outcome = outcomes[inserts] ?? []; + inserts += 1; + return Promise.resolve(outcome); + } + return Promise.resolve([]); + }; + Object.assign(tag, { json: (value: unknown) => value }); + // oxlint-disable-next-line typescript/no-unsafe-type-assertion -- only the tag call and `json` are exercised + return { tx: tag as unknown as TransactionSql, statements }; +} + +const ARGS = { + threadId: 't-1', + organizationId: 'org-1', + role: 'user' as const, + text: 'hello', +}; + +const insertsOf = (statements: string[]): number => + statements.filter((text) => text.includes('INSERT INTO app.messages')).length; + +describe('saveMessage — claiming a unique slot', () => { + it('claims the slot in one statement that reads max+1 and refuses a tie', async () => { + const { tx, statements } = fakeTx([[{ id: 'm-1', order: 4 }]]); + await expect(saveMessage(tx, ARGS)).resolves.toEqual({ + messageId: 'm-1', + order: 4, + }); + const insert = statements.find((text) => + text.includes('INSERT INTO app.messages'), + ); + expect(insert).toContain('coalesce(max("order"), -1) + 1'); + expect(insert).toContain( + 'ON CONFLICT (thread_id, "order", step_order) DO NOTHING', + ); + // The thread's activity stamp follows a landed row. + expect(statements.some((text) => text.includes('UPDATE app.threads'))).toBe( + true, + ); + }); + + it('re-claims the next slot after a concurrent append took the computed one', async () => { + const { tx, statements } = fakeTx([[], [{ id: 'm-2', order: 5 }]]); + await expect(saveMessage(tx, ARGS)).resolves.toEqual({ + messageId: 'm-2', + order: 5, + }); + expect(insertsOf(statements)).toBe(2); + }); + + it('gives up only after the bounded number of lost races', async () => { + const { tx, statements } = fakeTx([]); + await expect(saveMessage(tx, ARGS)).rejects.toThrow(/no free slot/); + expect(insertsOf(statements)).toBe(MESSAGE_SLOT_ATTEMPTS); + expect(statements.some((text) => text.includes('UPDATE app.threads'))).toBe( + false, + ); + }); +}); diff --git a/services/platform/backend/domains/threads/store.ts b/services/platform/backend/domains/threads/store.ts index 7338dc2644..4ab616f37f 100644 --- a/services/platform/backend/domains/threads/store.ts +++ b/services/platform/backend/domains/threads/store.ts @@ -87,42 +87,55 @@ export interface SaveMessageArgs { status?: 'pending' | 'complete' | 'failed' | 'cancelled'; } +/** + * How many times an appender re-claims the next (order, step) slot after + * another appender took the one it computed. The slot is UNIQUE + * (`messages_thread_slot`), so a lost race is refused at the index rather + * than landing two rows on one slot; under READ COMMITTED each attempt is a + * fresh statement that sees the winner's row, so one retry is the norm and a + * handful is generous. Under SERIALIZABLE the same conflict surfaces as a + * serialization failure and `transactSerializable` reruns the whole + * transaction instead, so this loop never spins there. + */ +export const MESSAGE_SLOT_ATTEMPTS = 8; + /** * Append a message as the next turn: claims `max(order)+1` with - * `step_order = 0`. Callers appending STEPS of an existing turn use - * `saveMessageStep`. Runs inside the caller's serializable transaction, so - * two concurrent appends to one thread serialize (one retries). + * `step_order = 0` in ONE statement (read and write together), and re-claims + * the following slot when a concurrent append took it first. Runs inside the + * caller's transaction; under a serializable one, two concurrent appends to + * one thread serialize (one retries the transaction). */ export async function saveMessage( tx: TransactionSql, args: SaveMessageArgs, ): Promise<{ messageId: string; order: number }> { - const orderRows = await tx<{ next: number }[]>` - SELECT coalesce(max("order"), -1) + 1 AS next FROM app.messages - WHERE thread_id = ${args.threadId} - `; - const order = orderRows[0]?.next ?? 0; - const rows = await tx<{ id: string }[]>` - INSERT INTO app.messages ( - thread_id, org_id, "order", step_order, role, parts, text, author_id, - status, created_at_ms - ) VALUES ( - ${args.threadId}, ${args.organizationId}, ${order}, 0, ${args.role}, - ${args.parts === undefined ? null : tx.json(toJson(args.parts))}, - ${args.text ?? null}, ${args.authorId ?? null}, - ${args.status ?? 'complete'}, ${Date.now()} - ) - RETURNING id - `; - const id = rows[0]?.id; - if (!id) { - throw new Error('message insert failed'); + for (let attempt = 0; attempt < MESSAGE_SLOT_ATTEMPTS; attempt += 1) { + const rows = await tx<{ id: string; order: number }[]>` + INSERT INTO app.messages ( + thread_id, org_id, "order", step_order, role, parts, text, author_id, + status, created_at_ms + ) + SELECT ${args.threadId}, ${args.organizationId}, + coalesce(max("order"), -1) + 1, 0, ${args.role}, + ${args.parts === undefined ? null : tx.json(toJson(args.parts))}, + ${args.text ?? null}, ${args.authorId ?? null}, + ${args.status ?? 'complete'}, ${Date.now()} + FROM app.messages WHERE thread_id = ${args.threadId} + ON CONFLICT (thread_id, "order", step_order) DO NOTHING + RETURNING id, "order" + `; + const row = rows[0]; + if (row === undefined) continue; // the slot went to a concurrent append + await tx` + UPDATE app.threads SET updated_at_ms = ${Date.now()} + WHERE id = ${args.threadId} + `; + return { messageId: row.id, order: row.order }; } - await tx` - UPDATE app.threads SET updated_at_ms = ${Date.now()} - WHERE id = ${args.threadId} - `; - return { messageId: id, order }; + throw new Error( + `message insert failed: no free slot after ${MESSAGE_SLOT_ATTEMPTS} attempts`, + ); } /** The most messages one read may ask for, on either lane below. */ diff --git a/services/platform/backend/domains/tts/routes.ts b/services/platform/backend/domains/tts/routes.ts index 43ea5b9ce2..d41639e819 100644 --- a/services/platform/backend/domains/tts/routes.ts +++ b/services/platform/backend/domains/tts/routes.ts @@ -10,6 +10,10 @@ import { errorCodeFromCaught } from '../../core/tts/error_codes.ts'; import { createCtxShim } from '../../lib/ctx-shim.ts'; import { resolveObjectStore, s3PresignGetUrl } from '../../lib/object-store.ts'; import { resolveOrgSlug } from '../../lib/org-config.ts'; +import { + rateLimitExceededCause, + rateLimitedResponse, +} from '../../lib/rate-limit-response.ts'; import { chatShimHandlers } from '../chat/shim.ts'; import { loadOwnedThread } from '../chat/threads.ts'; import { @@ -35,6 +39,12 @@ function handleError( c: Context, error: unknown, ): Response { + // A spent budget answers the one 429 every door speaks — the service + // wraps the limiter's refusal as a coded TtsError and carries it as cause. + const limited = rateLimitExceededCause(error); + if (limited !== null) { + return rateLimitedResponse(c, limited); + } if (error instanceof TtsError) { return c.json( { diff --git a/services/platform/backend/domains/tts/service.ts b/services/platform/backend/domains/tts/service.ts index 5da06f9ee3..ee3042e398 100644 --- a/services/platform/backend/domains/tts/service.ts +++ b/services/platform/backend/domains/tts/service.ts @@ -84,8 +84,10 @@ export class TtsError extends Error { message: string, status: 400 | 403 | 404 | 429 = 400, retryAfterMs?: number, + /** The refusal this one wraps (a rate limit), for the door to answer. */ + options?: ErrorOptions, ) { - super(message); + super(message, options); this.name = 'TtsError'; this.code = code; this.status = status; @@ -496,11 +498,14 @@ async function reserveChunk( await checkUserRateLimit(tx, 'tts:synthesize:user', args.userId, 1); } catch (error) { if (error instanceof RateLimitExceededError) { + // Coded for the shim and job consumers; the app door answers the one + // 429 from the cause (`rateLimitExceededCause`). throw new TtsError( 'RATE_LIMITED', 'TTS rate limit exceeded for this user.', 429, error.retryAfter, + { cause: error }, ); } throw error; @@ -519,6 +524,7 @@ async function reserveChunk( 'TTS rate limit exceeded for this organization.', 429, error.retryAfter, + { cause: error }, ); } throw error; diff --git a/services/platform/backend/domains/webdav/routes.ts b/services/platform/backend/domains/webdav/routes.ts index 3a38438bfb..4ef8467084 100644 --- a/services/platform/backend/domains/webdav/routes.ts +++ b/services/platform/backend/domains/webdav/routes.ts @@ -14,6 +14,7 @@ import { hmacHash, requireHmacSecret, } from '../../core/webdav/helpers.ts'; +import { rateLimitedResponse } from '../../lib/rate-limit-response.ts'; import { checkOrganizationRateLimit, RateLimitExceededError, @@ -134,7 +135,7 @@ export function createWebdavAdminRoutes(deps: { ); } catch (error) { if (error instanceof RateLimitExceededError) { - return c.json({ error: 'RATE_LIMITED' }, 429); + return rateLimitedResponse(c, error); } throw error; } diff --git a/services/platform/backend/integration-check.ts b/services/platform/backend/integration-check.ts index 84e01b32c2..504f405fc0 100644 --- a/services/platform/backend/integration-check.ts +++ b/services/platform/backend/integration-check.ts @@ -45,6 +45,7 @@ import { runBootMigrations } from './db/migrate.ts'; import { createSql } from './db/sql.ts'; import { rowToHashInput } from './domains/audit_logs/hash-input.ts'; import type { AuditLogRow } from './domains/audit_logs/types.ts'; +import { appendMessageRow } from './domains/chat/store.ts'; import { setMailTransportForTesting } from './domains/connectors/service.ts'; import { writeNotificationForOrgs } from './domains/notifications/service.ts'; import { ensureDefaultObjectStore } from './domains/object_storage/bootstrap.ts'; @@ -53,6 +54,7 @@ import { addJobInTx, setEnqueueBoss } from './jobs/enqueue.ts'; import { startWorker } from './jobs/runner.ts'; import { registerSchedules } from './jobs/schedules.ts'; import { createTaskList } from './jobs/task-list.ts'; +import { RATE_LIMITS, type RateLimitName } from './lib/rate-limit.ts'; import { emitHintInTx, latestOutboxId } from './realtime/outbox.ts'; const noopPayloadSchema = z.object({ @@ -4903,6 +4905,252 @@ async function checkDocumentWriteGuards( ); } +/** + * Message ordering under concurrency: a thread's (order, step_order) slot is + * unique, so concurrent appends each land on their own slot — never a tie — + * and the migration that introduced the rule repairs pre-existing ties + * deterministically (proved by replaying its statement on a scratch table + * seeded with ties, since the live table can no longer hold any). + */ +async function checkMessageSlots( + sql: Sql, + ctx: { orgId: string; userId: string }, +): Promise { + const { orgId, userId } = ctx; + const now = Date.now(); + const threadId = `itest-slot-thread-${now}`; + await sql` + INSERT INTO app.threads (id, org_id, user_id, kind, created_at_ms, + updated_at_ms) + VALUES (${threadId}, ${orgId}, ${userId}, 'chat', ${now}, ${now}) + `; + await sql` + INSERT INTO app.thread_metadata ( + thread_id, org_id, user_id, chat_type, status, created_at_ms + ) VALUES (${threadId}, ${orgId}, ${userId}, 'assistant', 'active', ${now}) + `; + const APPENDS = 12; + const outcomes = await Promise.allSettled( + Array.from({ length: APPENDS }, (_, i) => + appendMessageRow(sql, { + organizationId: orgId, + threadId, + role: 'assistant', + parts: [{ type: 'text', text: `reply ${i}` }], + text: `reply ${i}`, + }), + ), + ); + const failures = outcomes.filter((o) => o.status === 'rejected').length; + const slots = await sql<{ order: number; stepOrder: number }[]>` + SELECT "order", step_order AS "stepOrder" FROM app.messages + WHERE thread_id = ${threadId} + ORDER BY "order", step_order + `; + const distinctOrders = new Set(slots.map((row) => row.order)); + const contiguous = [...distinctOrders] + .sort((a, b) => a - b) + .every((order, i) => order === i); + record( + 'messages: concurrent appends each take their own slot', + failures === 0 && + slots.length === APPENDS && + distinctOrders.size === APPENDS && + contiguous, + `appends=${APPENDS}, failed=${failures} (want 0), rows=${slots.length}, distinctOrders=${distinctOrders.size} (want ${APPENDS}, no ties), contiguous=${contiguous}`, + ); + + // The dedup rule, replayed from the shipped migration onto a scratch + // table: two groups tie in one thread, one of them also holding a + // non-tied step; an untied group and another thread stay untouched. + const migration = await readFile( + new URL('./db/migrations/0074_messages_unique_slot.sql', import.meta.url), + 'utf8', + ).catch(() => ''); + // Comment lines go first: the header prose may hold a ';' of its own. + const dedupStatement = migration + .split('\n') + .filter((line) => !line.trimStart().startsWith('--')) + .join('\n') + .split(';') + .find((statement) => statement.includes('UPDATE app.messages')); + await sql`DROP TABLE IF EXISTS itest_msg_slots`; + await sql` + CREATE TABLE itest_msg_slots ( + id text PRIMARY KEY, thread_id text NOT NULL, "order" int NOT NULL, + step_order int NOT NULL, created_at_ms bigint NOT NULL + ) + `; + await sql` + INSERT INTO itest_msg_slots (id, thread_id, "order", step_order, created_at_ms) + VALUES + ('a-late', 'tA', 3, 0, 300), ('a-early', 'tA', 3, 0, 100), + ('a-step', 'tA', 3, 1, 200), ('a-mid', 'tA', 3, 0, 200), + ('a-solo', 'tA', 4, 0, 400), + ('a-tie2x', 'tA', 5, 2, 500), ('a-tie2y', 'tA', 5, 2, 600), + ('b-one', 'tB', 3, 0, 100), ('b-two', 'tB', 3, 1, 200) + `; + let replayError = ''; + if (dedupStatement !== undefined) { + await sql + .unsafe(dedupStatement.replaceAll('app.messages', 'itest_msg_slots')) + .catch((error: unknown) => { + replayError = error instanceof Error ? error.message : String(error); + }); + } + const repaired = await sql< + { id: string; threadId: string; order: number; stepOrder: number }[] + >` + SELECT id, thread_id AS "threadId", "order", step_order AS "stepOrder" + FROM itest_msg_slots ORDER BY thread_id, "order", step_order, id + `; + const stepOf = (id: string): number | undefined => + repaired.find((row) => row.id === id)?.stepOrder; + const noTies = + new Set( + repaired.map((row) => `${row.threadId}/${row.order}/${row.stepOrder}`), + ).size === repaired.length; + const orderKept = + // step first, then arrival: the step-1 row stays after every step-0 row + stepOf('a-early') === 0 && + stepOf('a-mid') === 1 && + stepOf('a-late') === 2 && + stepOf('a-step') === 3 && + // a tie above step 0 compacts to the front of its group + stepOf('a-tie2x') === 0 && + stepOf('a-tie2y') === 1 && + // untied groups and other threads are untouched + stepOf('a-solo') === 0 && + stepOf('b-one') === 0 && + stepOf('b-two') === 1; + await sql`DROP TABLE IF EXISTS itest_msg_slots`; + record( + 'messages: the 0076 migration renumbers tied slots deterministically', + dedupStatement !== undefined && replayError === '' && noTies && orderKept, + `migrationFound=${dedupStatement !== undefined}, replay=${replayError === '' ? 'ok' : replayError}, noTies=${noTies}, orderKept=${orderKept} (${repaired.map((row) => `${row.id}=${row.order}.${row.stepOrder}`).join(' ')})`, + ); +} + +/** + * Every rate-limited app door refuses the same way: 429, a `Retry-After` in + * whole seconds, and `{ error: 'RATE_LIMITED', data: { retryAfterMs } }` — + * one helper (`lib/rate-limit-response.ts`) behind all of them, so no door + * answers a spent budget as an outage or without the wait. Budgets are spent + * directly in the limiter's table (a token bucket driven far negative, a + * fixed window filled for the current period) and restored afterwards so the + * rest of the suite keeps its budgets. + */ +async function checkRateLimitShapes( + sql: Sql, + base: string, + ctx: { cookie: string; orgId: string; userId: string }, +): Promise { + const { cookie, orgId, userId } = ctx; + const doors: { + name: string; + rule: RateLimitName; + key: string; + route: string; + body: string; + contentType: string; + }[] = [ + { + name: 'files', + rule: 'file:upload', + key: `user:${userId}`, + route: `/api/app/files/upload?purpose=file&orgId=${orgId}`, + body: 'bytes', + contentType: 'application/octet-stream', + }, + { + name: 'folders', + rule: 'folder:mutate', + key: `org:${orgId}`, + route: `/api/app/folders?orgId=${orgId}`, + body: JSON.stringify({ name: 'Rate limited folder' }), + contentType: 'application/json', + }, + { + name: 'projects', + rule: 'project:create', + key: `user:${userId}`, + route: `/api/app/projects?orgId=${orgId}`, + body: JSON.stringify({ name: 'Rate limited project' }), + contentType: 'application/json', + }, + { + name: 'tasks', + rule: 'task:create', + key: `user:${userId}`, + route: `/api/app/tasks?orgId=${orgId}`, + body: JSON.stringify({ projectId: 'no-such-project', title: 'x' }), + contentType: 'application/json', + }, + { + name: 'knowledge entries', + rule: 'knowledge:mutate', + key: `org:${orgId}`, + route: `/api/app/knowledge-entries?orgId=${orgId}`, + body: JSON.stringify({ topic: 'limits', content: 'spent' }), + contentType: 'application/json', + }, + { + name: 'webdav app passwords', + rule: 'webdav:app-password-create', + key: `org:${orgId}`, + route: `/api/app/webdav/app-passwords?orgId=${orgId}`, + body: JSON.stringify({ label: 'spent' }), + contentType: 'application/json', + }, + ]; + const spend = async (rule: RateLimitName, key: string): Promise => { + const spec = RATE_LIMITS[rule]; + const now = Date.now(); + if (spec.kind === 'token bucket') { + await sql` + INSERT INTO app.rate_limits (name, key, value, ts) + VALUES (${rule}, ${key}, -1e9, ${now}) + ON CONFLICT (name, key) DO UPDATE SET value = -1e9, ts = ${now} + `; + return; + } + const windowStart = Math.floor(now / spec.period) * spec.period; + await sql` + INSERT INTO app.rate_limits (name, key, value, ts) + VALUES (${rule}, ${key}, 1e9, ${windowStart}) + ON CONFLICT (name, key) DO UPDATE SET value = 1e9, ts = ${windowStart} + `; + }; + const refusal = z.object({ + error: z.literal('RATE_LIMITED'), + data: z.object({ retryAfterMs: z.number().positive() }), + }); + for (const door of doors) { + await spend(door.rule, door.key); + try { + const res = await fetch(`${base}${door.route}`, { + method: 'POST', + headers: { 'content-type': door.contentType, cookie, origin: base }, + body: door.body, + }); + const retryAfter = Number(res.headers.get('retry-after') ?? 'NaN'); + const body = refusal.safeParse(await res.json().catch(() => null)); + record( + `rate limit: ${door.name} refuses in the one 429 shape`, + res.status === 429 && + Number.isInteger(retryAfter) && + retryAfter >= 1 && + body.success, + `status=${res.status} (want 429), retry-after=${res.headers.get('retry-after') ?? 'MISSING'} (want whole seconds ≥ 1), body=${body.success ? 'RATE_LIMITED+retryAfterMs' : 'WRONG SHAPE'}`, + ); + } finally { + await sql` + DELETE FROM app.rate_limits WHERE name = ${door.rule} AND key = ${door.key} + `; + } + } +} + /** * Small-domain smoke: contacts CRUD + find-or-create shape, message * feedback upsert/toggle/stats, support case lifecycle. @@ -4910,9 +5158,9 @@ async function checkDocumentWriteGuards( async function checkSmallDomains( sql: Sql, base: string, - ctx: { cookie: string; orgId: string }, + ctx: { cookie: string; orgId: string; userId: string }, ): Promise { - const { cookie, orgId } = ctx; + const { cookie, orgId, userId } = ctx; const get = async (route: string): Promise => (await fetch(`${base}${route}`, { headers: { cookie } })).json(); const send = ( @@ -5089,15 +5337,112 @@ async function checkSmallDomains( `listed created=${hopperListed?.createdAt} updated=${hopperListed?.updatedAt}, hit updatedAt=${hopperSearched?.updatedAt} (want = listed updatedAt, > createdAt)`, ); + // Message feedback lands on a REAL message the caller can read: a chat + // thread this user owns, with one assistant reply in it. + const fbNow = Date.now(); + const fbThreadId = `itest-fb-thread-${fbNow}`; + const fbMessageId = `itest-fb-msg-${fbNow}`; + await sql` + INSERT INTO app.threads (id, org_id, user_id, kind, created_at_ms, + updated_at_ms) + VALUES (${fbThreadId}, ${orgId}, ${userId}, 'chat', ${fbNow}, ${fbNow}) + `; + await sql` + INSERT INTO app.thread_metadata ( + thread_id, org_id, user_id, chat_type, status, created_at_ms + ) VALUES (${fbThreadId}, ${orgId}, ${userId}, 'assistant', 'active', + ${fbNow}) + `; + await sql` + INSERT INTO app.messages ( + id, thread_id, org_id, "order", step_order, role, text, status, + created_at_ms + ) VALUES (${fbMessageId}, ${fbThreadId}, ${orgId}, 0, 0, 'assistant', + 'the answer', 'complete', ${fbNow}) + `; + + // A member of ANOTHER organization, voting through their own org on this + // message: well-formed request, foreign ids. The door must not confirm the + // message exists, and must record nothing. + const rivalSignUp = await fetch(`${base}/api/auth/sign-up/email`, { + method: 'POST', + headers: { 'content-type': 'application/json', origin: base }, + body: JSON.stringify({ + email: `itest-fb-rival-${fbNow}@example.com`, + password: 'itest-password-1', + name: 'Feedback Rival', + }), + }); + const rivalCookie = cookieHeaderFrom(rivalSignUp); + const rivalParsed = z + .object({ user: z.object({ id: z.string() }) }) + .safeParse(await rivalSignUp.json()); + const rivalId = rivalParsed.success ? rivalParsed.data.user.id : ''; + const rivalOrgCreate = await fetch(`${base}/api/auth/organization/create`, { + method: 'POST', + headers: { + 'content-type': 'application/json', + cookie: rivalCookie, + origin: base, + }, + body: JSON.stringify({ + name: `Feedback Rival ${fbNow}`, + slug: `itest-fb-rival-${fbNow}`, + }), + }); + const rivalOrgId = + z + .object({ id: z.string().optional() }) + .safeParse(await rivalOrgCreate.json()).data?.id ?? ''; + const crossOrgVote = await fetch( + `${base}/api/app/feedback?orgId=${rivalOrgId}`, + { + method: 'POST', + headers: { + 'content-type': 'application/json', + cookie: rivalCookie, + origin: base, + }, + body: JSON.stringify({ + threadId: fbThreadId, + messageId: fbMessageId, + rating: 'positive', + }), + }, + ); + const crossOrgBody = z + .object({ error: z.string() }) + .safeParse(await crossOrgVote.json()); + // The owner naming a message that does not exist gets the SAME answer. + const missingVote = await send('POST', `/api/app/feedback?orgId=${orgId}`, { + threadId: fbThreadId, + messageId: 'itest-fb-no-such-message', + rating: 'positive', + }); + const foreignRows = await sql<{ count: string }[]>` + SELECT count(*)::text AS count FROM app.message_feedback + WHERE message_id = ${fbMessageId} AND user_id = ${rivalId} + `; + record( + "message feedback: a vote on another organization's message is refused opaquely", + rivalOrgId !== '' && + crossOrgVote.status === 404 && + crossOrgBody.success && + crossOrgBody.data.error === 'MESSAGE_NOT_FOUND' && + missingVote.status === 404 && + foreignRows[0]?.count === '0', + `crossOrg → ${crossOrgVote.status}/${crossOrgBody.success ? crossOrgBody.data.error : 'ERR'} (want 404/MESSAGE_NOT_FOUND), missing → ${missingVote.status} (want 404), foreignRows=${foreignRows[0]?.count} (want 0)`, + ); + // Message feedback: vote → toggle → stats reflect one negative. - await send('POST', `/api/app/feedback?orgId=${orgId}`, { - threadId: 'itest-thread', - messageId: 'itest-msg-1', + const ownVote = await send('POST', `/api/app/feedback?orgId=${orgId}`, { + threadId: fbThreadId, + messageId: fbMessageId, rating: 'positive', }); await send('POST', `/api/app/feedback?orgId=${orgId}`, { - threadId: 'itest-thread', - messageId: 'itest-msg-1', + threadId: fbThreadId, + messageId: fbMessageId, rating: 'negative', comment: 'wrong answer', }); @@ -5109,11 +5454,12 @@ async function checkSmallDomains( .safeParse(await get(`/api/app/feedback?orgId=${orgId}`)); record( 'message feedback upsert + stats', - stats.success && + ownVote.status === 200 && + stats.success && stats.data.items.length === 1 && stats.data.stats.negative === 1 && stats.data.stats.positive === 0, - `items=${stats.success ? stats.data.items.length : 'ERR'} (want 1 after toggle), stats=${stats.success ? JSON.stringify(stats.data.stats) : 'ERR'}`, + `ownVote → ${ownVote.status} (want 200), items=${stats.success ? stats.data.items.length : 'ERR'} (want 1 after toggle), stats=${stats.success ? JSON.stringify(stats.data.stats) : 'ERR'}`, ); // The vote is keyed by the SERVER: whatever a client puts in `metadata` is @@ -16347,7 +16693,9 @@ async function checkConnectorOauth( // vendor shape — including `team.id`, which drives the workspace claim. const seen: { body: string; auth: string | null }[] = []; let denyExchange = false; - let vendorTeamId = 'T-ITEST-1'; + // The workspace the fake vendor reports — re-pointed below to prove a + // reconnect renews the same credential and a second workspace gets its own. + let vendorTeam: { id: string; name?: string } = { id: 'T-ITEST-1' }; const vendor = createServer((req, res) => { let body = ''; req.on('data', (chunk: unknown) => { @@ -16365,7 +16713,7 @@ async function checkConnectorOauth( refresh_token: 'xoxe-itest-refresh', expires_in: 3600, scope: 'channels:read,chat:write', - team: { id: vendorTeamId }, + team: vendorTeam, }), ); }); @@ -16593,6 +16941,91 @@ async function checkConnectorOauth( `claim=${claim.ok ? 'ALLOWED' : 'refused'}, routeOwner=${routeAfter[0]?.orgId === orgId}, resolve=${resolved?.organizationId === orgId}`, ); + // ---- reconnect: the same workspace renews ITS credential ------------ + // The settings card's Reconnect is a second consent for a workspace + // already connected here. It must land on the credential the route names + // — fresh tokens, active again — not collide with the label it holds. + const firstCredentialId = credentialRows[0]?.id ?? ''; + await sql` + UPDATE app.connector_credentials + SET status = 'needs-reauth', status_detail = 'token revoked' + WHERE id = ${firstCredentialId} + `; + const reconnectState = 'itest-oauth-state-reconnect'; + await oauth.createPendingAuthorization(sql, { + stateHash: await hashStateToken(reconnectState), + organizationId: orgId, + userId, + connectorSlug: 'slack', + codeVerifier: 'itest-verifier-value-222222222222222222222', + redirectUri: `${base}/api/connectors/oauth2/callback`, + }); + const reconnected = await oauth.completeOauth2( + sql, + { state: reconnectState, code: 'itest-auth-code-r', vendorError: null }, + { fetchImpl: vendorFetch }, + ); + const afterReconnect = await sql< + { + id: string; + name: string; + status: string; + statusDetail: string | null; + }[] + >` + SELECT id, name, status, status_detail AS "statusDetail" + FROM app.connector_credentials + WHERE org_id = ${orgId} AND connector_slug = 'slack' + ORDER BY created_at_ms + `; + const routeAfterReconnect = await oauth.resolveTeamRoute(sql, 'T-ITEST-1'); + record( + 'connector oauth: reconnecting a connected workspace renews its credential', + reconnected.kind === 'connected' && + afterReconnect.length === 1 && + afterReconnect[0]?.id === firstCredentialId && + afterReconnect[0]?.name === 'Slack' && + afterReconnect[0]?.status === 'active' && + afterReconnect[0]?.statusDetail === null && + routeAfterReconnect?.credentialId === firstCredentialId, + `outcome=${reconnected.kind}${reconnected.kind === 'error' ? `/${reconnected.error}` : ''} (want connected), credentials=${afterReconnect.length} (want 1, same row), status=${afterReconnect[0]?.status ?? '-'}/${afterReconnect[0]?.statusDetail ?? 'null'} (want active/null), route=${routeAfterReconnect?.credentialId === firstCredentialId}`, + ); + + // ---- a second workspace gets its own, distinctly named credential ---- + vendorTeam = { id: 'T-ITEST-2', name: 'Second Workspace' }; + const secondState = 'itest-oauth-state-second-workspace'; + await oauth.createPendingAuthorization(sql, { + stateHash: await hashStateToken(secondState), + organizationId: orgId, + userId, + connectorSlug: 'slack', + codeVerifier: 'itest-verifier-value-333333333333333333333', + redirectUri: `${base}/api/connectors/oauth2/callback`, + }); + const secondConnected = await oauth.completeOauth2( + sql, + { state: secondState, code: 'itest-auth-code-s', vendorError: null }, + { fetchImpl: vendorFetch }, + ); + const afterSecond = await sql<{ id: string; name: string }[]>` + SELECT id, name FROM app.connector_credentials + WHERE org_id = ${orgId} AND connector_slug = 'slack' + ORDER BY created_at_ms + `; + const secondRoute = await oauth.resolveTeamRoute(sql, 'T-ITEST-2'); + record( + 'connector oauth: a second workspace connects under its own label', + secondConnected.kind === 'connected' && + afterSecond.length === 2 && + afterSecond[0]?.name === 'Slack' && + afterSecond[1]?.name === 'Slack (Second Workspace)' && + secondRoute?.organizationId === orgId && + secondRoute.credentialId === afterSecond[1]?.id && + secondRoute.credentialId !== firstCredentialId, + `outcome=${secondConnected.kind}${secondConnected.kind === 'error' ? `/${secondConnected.error}` : ''} (want connected), credentials=${afterSecond.map((row) => row.name).join(' | ')} (want Slack | Slack (Second Workspace)), route2=${secondRoute?.credentialId === afterSecond[1]?.id}`, + ); + vendorTeam = { id: 'T-ITEST-1' }; + // ---- the claim race: the pre-check passes, another org claims first -- // Two organizations can pass the pre-check for one workspace; the // route's key decides the winner and the LOSER must keep nothing. The @@ -16603,7 +17036,7 @@ async function checkConnectorOauth( // holds this org's Slack credential (one credential per connector name), // so only a fresh organization walks the store-then-claim path — and the // credential it must not keep would have been its FIRST, i.e. its default. - vendorTeamId = 'T-ITEST-RACE'; + vendorTeam = { id: 'T-ITEST-RACE' }; const raceOrgId = 'itest-race-org'; const raceState = 'itest-oauth-state-race'; await oauth.createPendingAuthorization(sql, { @@ -16660,7 +17093,7 @@ async function checkConnectorOauth( releaseForeignClaim(); await foreignClaim; const raced = await racing; - vendorTeamId = 'T-ITEST-1'; + vendorTeam = { id: 'T-ITEST-1' }; const raceCredentials = await sql< { count: string; isDefault: boolean | null }[] >` @@ -16711,8 +17144,8 @@ async function checkConnectorOauth( 'connector oauth: a vendor-rejected exchange stores no credential', refused.kind === 'error' && (refused.kind === 'error' ? refused.error : '') === 'vendor_declined' && - credentialsAfter[0]?.count === '1', - `outcome=${refused.kind}/${refused.kind === 'error' ? refused.error : '-'}, credentials=${credentialsAfter[0]?.count} (want 1, unchanged)`, + credentialsAfter[0]?.count === '2', + `outcome=${refused.kind}/${refused.kind === 'error' ? refused.error : '-'}, credentials=${credentialsAfter[0]?.count} (want 2, unchanged)`, ); } finally { vendor.close(); @@ -38671,6 +39104,8 @@ async function main(): Promise { await checkWorkflowDocumentListing(sql, baseUrl, authCtx); await checkBlobRefAuthority(sql, baseUrl, authCtx); await checkSmallDomains(sql, baseUrl, authCtx); + await checkMessageSlots(sql, authCtx); + await checkRateLimitShapes(sql, baseUrl, authCtx); await checkAgents(baseUrl, authCtx); await checkSkills(baseUrl, authCtx); await checkProviderCredentials(sql, baseUrl, authCtx); diff --git a/services/platform/backend/lib/rate-limit-response.test.ts b/services/platform/backend/lib/rate-limit-response.test.ts index 8c647e93ee..3ca72f644f 100644 --- a/services/platform/backend/lib/rate-limit-response.test.ts +++ b/services/platform/backend/lib/rate-limit-response.test.ts @@ -6,7 +6,10 @@ import { describe, expect, it, vi } from 'vitest'; import { chargeOrgRateLimit, + rateLimitExceededCause, + rateLimitedPlainResponse, rateLimitedResponse, + retryAfterSeconds, } from './rate-limit-response.ts'; import { RateLimitExceededError } from './rate-limit.ts'; @@ -78,3 +81,44 @@ describe('chargeOrgRateLimit', () => { expect((await app.request('/fresh')).status).toBe(200); }); }); + +describe('retryAfterSeconds', () => { + it('rounds up to whole seconds and never advertises zero', () => { + expect(retryAfterSeconds(1500)).toBe('2'); + expect(retryAfterSeconds(60_000)).toBe('60'); + expect(retryAfterSeconds(0)).toBe('1'); + }); +}); + +describe('rateLimitedPlainResponse', () => { + it('answers a bare 429 with Retry-After and keeps the door headers', async () => { + const res = rateLimitedPlainResponse( + new RateLimitExceededError('spent', 2001), + { 'Cache-Control': 'no-store', Vary: 'Cookie' }, + ); + expect(res.status).toBe(429); + expect(res.headers.get('retry-after')).toBe('3'); + expect(res.headers.get('cache-control')).toBe('no-store'); + expect(res.headers.get('vary')).toBe('Cookie'); + expect(res.headers.get('content-type')).toContain('text/plain'); + expect(await res.text()).toBe('Rate limit exceeded'); + }); +}); + +describe('rateLimitExceededCause', () => { + it('finds the refusal itself, or the one a domain wrapper carries as cause', () => { + const limited = new RateLimitExceededError('spent', 10); + expect(rateLimitExceededCause(limited)).toBe(limited); + expect( + rateLimitExceededCause(new Error('wrapped', { cause: limited })), + ).toBe(limited); + }); + + it('answers null for anything else', () => { + expect(rateLimitExceededCause(new Error('plain'))).toBeNull(); + expect( + rateLimitExceededCause(new Error('other cause', { cause: 'x' })), + ).toBeNull(); + expect(rateLimitExceededCause('not an error')).toBeNull(); + }); +}); diff --git a/services/platform/backend/lib/rate-limit-response.ts b/services/platform/backend/lib/rate-limit-response.ts index 6ab79dab44..41d2ca0f19 100644 --- a/services/platform/backend/lib/rate-limit-response.ts +++ b/services/platform/backend/lib/rate-limit-response.ts @@ -46,3 +46,48 @@ export async function chargeOrgRateLimit( throw error; } } + +/** + * `Retry-After` in whole seconds, rounded up and never zero — the one figure + * every door advertises, whichever body its protocol speaks. + */ +export function retryAfterSeconds(retryAfterMs: number): string { + return String(Math.max(1, Math.ceil(retryAfterMs / 1000))); +} + +/** + * The same refusal for a door that answers with a bare `Response` and no + * Hono context — the Slack events webhook, the SSE/screencast auth + * pre-checks: 429, plain text, `Retry-After`, plus whatever headers the door + * always sends (`Cache-Control: no-store`, `Vary`). + */ +export function rateLimitedPlainResponse( + error: RateLimitExceededError, + headers: Record = {}, +): Response { + return new Response('Rate limit exceeded', { + status: 429, + headers: { + 'content-type': 'text/plain; charset=utf-8', + ...headers, + 'retry-after': retryAfterSeconds(error.retryAfter), + }, + }); +} + +/** + * The rate-limit refusal behind an error, if any: the error itself, or the + * `cause` a domain wrapper carries (a `DocumentError` / `TtsError` coded + * `RATE_LIMITED`, so the wrapper's own consumers — the REST helpers, the + * bridges — keep reading a code while the app door answers the one 429). + * Null for anything else. + */ +export function rateLimitExceededCause( + error: unknown, +): RateLimitExceededError | null { + if (error instanceof RateLimitExceededError) return error; + if (error instanceof Error && error.cause instanceof RateLimitExceededError) { + return error.cause; + } + return null; +} diff --git a/services/platform/backend/realtime/oracle-routes.ts b/services/platform/backend/realtime/oracle-routes.ts index 8979511b7c..7f453a3e07 100644 --- a/services/platform/backend/realtime/oracle-routes.ts +++ b/services/platform/backend/realtime/oracle-routes.ts @@ -23,6 +23,7 @@ import type { Sql } from 'postgres'; import { loadTrustedProxies, type Auth } from '../auth/auth.ts'; import { getUserOrganizations } from '../auth/membership.ts'; import { getClientIp } from '../core/lib/utils/client_ip.ts'; +import { rateLimitedPlainResponse } from '../lib/rate-limit-response.ts'; import { checkIpRateLimit, RateLimitExceededError } from '../lib/rate-limit.ts'; /** @@ -76,12 +77,7 @@ async function authenticateForwardedCookie( await checkIpRateLimit(deps.sql, limit, ip); } catch (error) { if (error instanceof RateLimitExceededError) { - return new Response('Rate limit exceeded', { - status: 429, - headers: noStore({ - 'Retry-After': String(Math.ceil(error.retryAfter / 1000)), - }), - }); + return rateLimitedPlainResponse(error, noStore()); } throw error; } diff --git a/services/platform/backend/rest/shared.ts b/services/platform/backend/rest/shared.ts index 8b989436a1..a03fe4f62d 100644 --- a/services/platform/backend/rest/shared.ts +++ b/services/platform/backend/rest/shared.ts @@ -5,6 +5,7 @@ import { defineAbilityFor } from '../../lib/permissions/ability.ts'; import { EDITOR_ROLES } from '../core/projects/access.ts'; import { resolveUserOrganization } from '../domains/organizations/service.ts'; import { getProjectAuthContext } from '../domains/projects/service.ts'; +import { rateLimitedResponse } from '../lib/rate-limit-response.ts'; import { RateLimitExceededError, checkUserRateLimit, @@ -139,9 +140,7 @@ export async function chargeLane( return null; } catch (error) { if (error instanceof RateLimitExceededError) { - return c.json({ error: 'Rate limit exceeded' }, 429, { - 'retry-after': String(Math.ceil(error.retryAfter / 1000)), - }); + return rateLimitedResponse(c, error); } throw error; } diff --git a/services/platform/backend/rest/v1.ts b/services/platform/backend/rest/v1.ts index f40570a4a3..a13391e63c 100644 --- a/services/platform/backend/rest/v1.ts +++ b/services/platform/backend/rest/v1.ts @@ -9,6 +9,7 @@ import { import { findOrganizationMember } from '../auth/membership.ts'; import { getClientIp, nodePeerAddress } from '../core/lib/utils/client_ip.ts'; import { resolveUserOrganization } from '../domains/organizations/service.ts'; +import { rateLimitedResponse } from '../lib/rate-limit-response.ts'; import { RateLimitExceededError, checkIpRateLimit, @@ -56,10 +57,13 @@ import { createRestWebsiteRoutes } from './v1-websites.ts'; */ /** The 429 every lane answers: the flat envelope plus `Retry-After`. */ +/** The plugin's own per-key window, answered in the one 429 shape every + * door speaks — the window is a refusal the limiter never threw. */ function rateLimited(c: Context, retryAfterMs: number): Response { - return c.json({ error: 'Rate limit exceeded' }, 429, { - 'retry-after': String(Math.ceil(retryAfterMs / 1000)), - }); + return rateLimitedResponse( + c, + new RateLimitExceededError('API key rate limit exceeded', retryAfterMs), + ); } /** @@ -120,7 +124,7 @@ export function createRestV1Routes(deps: { await checkIpRateLimit(deps.sql, 'rest:auth-fail-ip', ip); } catch (error) { if (error instanceof RateLimitExceededError) { - return rateLimited(c, error.retryAfter); + return rateLimitedResponse(c, error); } throw error; } @@ -133,7 +137,7 @@ export function createRestV1Routes(deps: { await checkUserRateLimit(deps.sql, 'rest:api', session.user.id); } catch (error) { if (error instanceof RateLimitExceededError) { - return rateLimited(c, error.retryAfter); + return rateLimitedResponse(c, error); } throw error; } diff --git a/tools/cli/src/lib/compose/generators/generate-dev-compose.test.ts b/tools/cli/src/lib/compose/generators/generate-dev-compose.test.ts index dcd35b112b..9bc11ff432 100644 --- a/tools/cli/src/lib/compose/generators/generate-dev-compose.test.ts +++ b/tools/cli/src/lib/compose/generators/generate-dev-compose.test.ts @@ -3,6 +3,8 @@ import { mkdtempSync, rmSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; +import { parse } from 'yaml'; + const warnMock = mock(); mock.module('../../../utils/logger', () => ({ @@ -47,9 +49,73 @@ describe('generateDevCompose — empty-workspace warning (R31-P2-b)', () => { const orgWarnings = warnMock.mock.calls.filter((c) => String(c[0]).includes('No org config found'), ); - // Both the convex and the platform service resolve host mounts from the - // same discovery; an empty workspace must not repeat the warning per - // service. + // Both the backend tier and the platform service resolve host mounts + // from the same discovery; an empty workspace must not repeat the warning + // per service. expect(orgWarnings).toHaveLength(1); }); }); + +interface ParsedCompose { + services: Record< + string, + { + depends_on?: string[] | Record; + volumes?: string[]; + } + >; + volumes: Record; +} + +function renderDevCompose(): ParsedCompose { + const projectDir = mkdtempSync(join(tmpdir(), 'tale-dev-compose-')); + try { + return parse( + generateDevCompose( + { version: 'latest', registry: 'ghcr.io/tale-project/tale' }, + 'localhost', + 443, + { projectDir }, + ), + ) as ParsedCompose; + } finally { + rmSync(projectDir, { recursive: true, force: true }); + } +} + +describe('generateDevCompose — the stack is self-consistent', () => { + // A `depends_on` naming a service the file does not define is a compose + // file docker refuses outright (`service "x" depends on undefined service + // "y"`), so the whole `tale dev` stack fails to start. The backend tier + // depends on the blob store it seeds at boot; the store must be IN the + // dev stack, not only in the stateful one. + test('every depends_on target is a service the dev stack defines', () => { + const { services } = renderDevCompose(); + for (const [name, service] of Object.entries(services)) { + const targets = Array.isArray(service.depends_on) + ? service.depends_on + : Object.keys(service.depends_on ?? {}); + for (const target of targets) { + expect( + Object.keys(services), + `${name} depends on ${target}, which the dev compose does not define`, + ).toContain(target); + } + } + }); + + test('runs the object store the backend tier depends on, on its dev volume', () => { + const { services, volumes } = renderDevCompose(); + expect(services['object-store']).toBeDefined(); + expect(services['backend-api']?.depends_on).toHaveProperty('object-store'); + expect(services['backend-worker']?.depends_on).toHaveProperty( + 'object-store', + ); + // The named volume the service mounts is declared (and pre-created by + // `tale dev` from DEV_VOLUME_NAMES). + expect(services['object-store']?.volumes).toContain( + 'object-store-data:/data', + ); + expect(volumes).toHaveProperty('object-store-data'); + }); +}); diff --git a/tools/cli/src/lib/compose/generators/generate-dev-compose.ts b/tools/cli/src/lib/compose/generators/generate-dev-compose.ts index 148f09485d..1a0da21ac0 100644 --- a/tools/cli/src/lib/compose/generators/generate-dev-compose.ts +++ b/tools/cli/src/lib/compose/generators/generate-dev-compose.ts @@ -15,6 +15,7 @@ import { createBackendWorkerService, } from '../services/create-backend-services'; import { createDbService } from '../services/create-db-service'; +import { createObjectStorageService } from '../services/create-object-storage-service'; import { createPlatformService } from '../services/create-platform-service'; import { createProxyService } from '../services/create-proxy-service'; import { createSandboxEgressService } from '../services/create-sandbox-egress-service'; @@ -163,6 +164,12 @@ export function generateDevCompose( const compose: ComposeConfig = { services: { db: createDbService(config), + // The blob store the backend tier depends on (it seeds the deployment- + // default blob connection at boot and refuses every upload without + // one). Same service as the stateful stack — dev writes real blobs to + // the `object-store-data` dev volume, and the proxy forwards the + // presigned `//*` URLs to it exactly as in production. + 'object-store': createObjectStorageService(config), proxy, 'backend-api': backendApi, 'backend-worker': backendWorker, From 312186d08a3aefa48a942908262627a32d07b667 Mon Sep 17 00:00:00 2001 From: larryro <371767072@qq.com> Date: Fri, 4 Sep 2026 10:36:59 +0800 Subject: [PATCH 2/2] test(platform): vote on real messages and name migration 0076 in the harness After the rebase onto #3189 and the renumbering to 0076, two probes drifted: the metadata-forgery probe voted on ids no message row carries, which the new reach gate refuses before any write, and the slot-migration probe still read the 0074 file. Both now use the lane's own message fixtures and the renumbered file. --- .../platform/backend/integration-check.ts | 34 +++++++++++++------ 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/services/platform/backend/integration-check.ts b/services/platform/backend/integration-check.ts index 504f405fc0..2aed4ebff9 100644 --- a/services/platform/backend/integration-check.ts +++ b/services/platform/backend/integration-check.ts @@ -4964,7 +4964,7 @@ async function checkMessageSlots( // table: two groups tie in one thread, one of them also holding a // non-tied step; an untied group and another thread stay untouched. const migration = await readFile( - new URL('./db/migrations/0074_messages_unique_slot.sql', import.meta.url), + new URL('./db/migrations/0076_messages_unique_slot.sql', import.meta.url), 'utf8', ).catch(() => ''); // Comment lines go first: the header prose may hold a ';' of its own. @@ -5465,22 +5465,36 @@ async function checkSmallDomains( // The vote is keyed by the SERVER: whatever a client puts in `metadata` is // dropped, so a repeated vote upserts (the partial-unique arbiter is // `metadata IS NULL`) and a forged arena verdict never lands as one — the - // arena settle lane is the only writer of metadata rows. + // arena settle lane is the only writer of metadata rows. Both votes land on + // REAL assistant rows of the caller's own thread: a vote names a message + // the caller can read, or it is refused before any write. + const fbStackedId = `${fbMessageId}-stacked`; + const fbForgedId = `${fbMessageId}-forged`; + await sql` + INSERT INTO app.messages ( + id, thread_id, org_id, "order", step_order, role, text, status, + created_at_ms + ) VALUES + (${fbStackedId}, ${fbThreadId}, ${orgId}, 1, 0, 'assistant', + 'a second answer', 'complete', ${fbNow}), + (${fbForgedId}, ${fbThreadId}, ${orgId}, 2, 0, 'assistant', + 'a third answer', 'complete', ${fbNow}) + `; await send('POST', `/api/app/feedback?orgId=${orgId}`, { - threadId: 'itest-thread', - messageId: 'itest-msg-2', + threadId: fbThreadId, + messageId: fbStackedId, rating: 'positive', metadata: {}, }); await send('POST', `/api/app/feedback?orgId=${orgId}`, { - threadId: 'itest-thread', - messageId: 'itest-msg-2', + threadId: fbThreadId, + messageId: fbStackedId, rating: 'negative', metadata: { stacked: true }, }); await send('POST', `/api/app/feedback?orgId=${orgId}`, { - threadId: 'itest-thread', - messageId: 'arena:forged-a:forged-b', + threadId: fbThreadId, + messageId: fbForgedId, rating: 'positive', metadata: { arenaVerdict: 'a_better', @@ -5491,11 +5505,11 @@ async function checkSmallDomains( const stackedVotes = await sql<{ count: string; rating: string | null }[]>` SELECT count(*)::text AS count, min(rating) AS rating FROM app.message_feedback - WHERE org_id = ${orgId} AND message_id = 'itest-msg-2' + WHERE org_id = ${orgId} AND message_id = ${fbStackedId} `; const forgedArenaRows = await sql<{ count: string }[]>` SELECT count(*)::text AS count FROM app.message_feedback - WHERE org_id = ${orgId} AND message_id = 'arena:forged-a:forged-b' + WHERE org_id = ${orgId} AND message_id = ${fbForgedId} AND metadata IS NOT NULL `; record(