From 32f9dc894fdc3a180bac7c190779b14944d11c0f Mon Sep 17 00:00:00 2001 From: olegberman Date: Fri, 4 Sep 2026 21:27:31 -0400 Subject: [PATCH 1/4] Fix mail integrity, send recovery, concurrency, and maintenance --- .github/workflows/dependency-audit.yml | 22 + api/hqbase-mail-api-v1.openapi.json | 26 + api/hqbase-mail-api-v2.openapi.json | 26 + app/app.tsx | 10 +- app/features/compose/compose-state.ts | 3 +- app/features/drafts/api.ts | 11 +- app/features/drafts/drafts-page.tsx | 16 +- app/features/drafts/sync-client.ts | 54 ++ app/features/drafts/use-drafts.ts | 36 +- app/features/events/use-mail-events.ts | 24 + app/features/messages/conversation-window.ts | 23 + app/features/messages/types.ts | 1 + app/features/messages/use-mail-sync.ts | 60 +- app/workspace-pages.tsx | 28 +- .../0004_mail_reliability_guards.sql | 37 ++ migrations/0029_mail_reliability.sql | 29 + package.json | 10 +- playwright.config.ts | 1 + pnpm-lock.yaml | 447 +++++++-------- pnpm-workspace.yaml | 7 +- scripts/hqbase/backup.mjs | 24 +- scripts/hqbase/cli.mjs | 2 +- scripts/hqbase/empty-r2.mjs | 2 +- scripts/hqbase/recovery-verification.mjs | 135 +++++ scripts/hqbase/reset-d1.sql | 10 + scripts/hqbase/restore.mjs | 95 +++- scripts/release/after-deploy-state.mjs | 21 +- scripts/release/deploy.mjs | 14 +- scripts/release/package.mjs | 2 +- test/e2e/staging/lifecycle.spec.ts | 8 +- .../worker/agent-mailboxes.test.ts | 4 +- test/integration/worker/auth.test.ts | 13 +- test/integration/worker/contacts.test.ts | 1 + test/integration/worker/conversations.test.ts | 2 + test/integration/worker/labels.test.ts | 1 + test/integration/worker/mail-api.test.ts | 2 + .../worker/mail-reliability.test.ts | 520 ++++++++++++++++++ .../worker/oauth-connections.test.ts | 4 +- .../worker/signature-routes.test.ts | 1 + test/integration/worker/users.test.ts | 50 +- .../app/drafts/draft-list-paging.test.tsx | 54 ++ test/unit/app/drafts/sync-client.test.ts | 53 ++ test/unit/app/events/use-mail-events.test.tsx | 21 + test/unit/app/messages/use-mail-sync.test.tsx | 28 +- test/unit/scripts/after-deploy-state.test.mjs | 4 +- test/unit/scripts/d1-migrations.test.mjs | 10 +- test/unit/scripts/recovery-objects.test.mjs | 56 ++ test/unit/scripts/recovery.test.mjs | 48 +- test/unit/scripts/sql-migrations.test.mjs | 15 +- test/unit/worker/auth/mailbox-access.test.ts | 6 +- .../messages/conversation-scale.test.ts | 78 +++ .../features/messages/message-thread.test.ts | 4 +- .../features/send/forward-service.test.ts | 20 +- test/unit/worker/features/send/routes.test.ts | 15 +- .../worker/features/send/send-service.test.ts | 194 +++---- .../features/updates/migration-state.test.ts | 9 +- .../worker/features/updates/service.test.ts | 6 +- .../features/updates/zone-account.test.ts | 24 + test/unit/worker/jobs/consumer.test.ts | 185 ++----- test/unit/worker/lib/request-body.test.ts | 34 ++ vitest.config.ts | 12 +- worker/auth/mailbox-access.ts | 12 +- worker/auth/session.ts | 6 + worker/db/drizzle.ts | 8 + worker/db/schema-core.ts | 3 + worker/db/schema-messages.ts | 2 + worker/db/schema-send.ts | 34 ++ worker/db/schema.ts | 1 + worker/email/parse-email.ts | 4 + worker/email/store-email.ts | 78 ++- worker/features/drafts/queries.ts | 93 ++-- worker/features/drafts/routes.ts | 3 +- worker/features/labels/queries.ts | 15 +- worker/features/mcp/mail-tools.ts | 6 +- worker/features/mcp/send-tools.ts | 38 +- .../features/messages/conversation-queries.ts | 8 +- worker/features/messages/object-references.ts | 24 + worker/features/messages/queries.ts | 95 +--- worker/features/messages/routes.ts | 12 +- worker/features/messages/storage.ts | 19 + worker/features/messages/text-storage.ts | 46 ++ worker/features/messages/types.ts | 5 + worker/features/operations/routes.ts | 7 +- worker/features/send/attachment-storage.ts | 31 +- worker/features/send/delivery.ts | 46 ++ worker/features/send/forward.ts | 27 +- worker/features/send/operations.ts | 235 ++++++++ worker/features/send/persistence.ts | 106 ++++ worker/features/send/routes.ts | 75 ++- worker/features/send/service.ts | 290 ++++------ worker/features/send/validation.ts | 3 + worker/features/updates/migration-names.ts | 50 ++ worker/features/updates/migration-state.ts | 68 +-- worker/features/updates/service.ts | 18 +- worker/features/updates/zone-account.ts | 43 ++ worker/features/users/queries.ts | 27 +- worker/features/users/routes.ts | 13 +- worker/jobs/consumer.ts | 265 ++++----- worker/jobs/maintenance.ts | 96 ++++ worker/jobs/object-scan.ts | 44 ++ worker/lib/errors.ts | 14 + worker/lib/json.ts | 5 +- worker/lib/request-body.ts | 59 ++ worker/routes/index.ts | 7 + 104 files changed, 3406 insertions(+), 1293 deletions(-) create mode 100644 .github/workflows/dependency-audit.yml create mode 100644 app/features/drafts/sync-client.ts create mode 100644 app/features/messages/conversation-window.ts create mode 100644 migrations-after-deploy/0004_mail_reliability_guards.sql create mode 100644 migrations/0029_mail_reliability.sql create mode 100644 scripts/hqbase/recovery-verification.mjs create mode 100644 test/integration/worker/mail-reliability.test.ts create mode 100644 test/unit/app/drafts/draft-list-paging.test.tsx create mode 100644 test/unit/app/drafts/sync-client.test.ts create mode 100644 test/unit/scripts/recovery-objects.test.mjs create mode 100644 test/unit/worker/features/messages/conversation-scale.test.ts create mode 100644 test/unit/worker/features/updates/zone-account.test.ts create mode 100644 test/unit/worker/lib/request-body.test.ts create mode 100644 worker/db/schema-send.ts create mode 100644 worker/features/messages/object-references.ts create mode 100644 worker/features/messages/storage.ts create mode 100644 worker/features/messages/text-storage.ts create mode 100644 worker/features/send/delivery.ts create mode 100644 worker/features/send/operations.ts create mode 100644 worker/features/send/persistence.ts create mode 100644 worker/features/updates/migration-names.ts create mode 100644 worker/features/updates/zone-account.ts create mode 100644 worker/jobs/maintenance.ts create mode 100644 worker/jobs/object-scan.ts create mode 100644 worker/lib/request-body.ts diff --git a/.github/workflows/dependency-audit.yml b/.github/workflows/dependency-audit.yml new file mode 100644 index 00000000..5408751f --- /dev/null +++ b/.github/workflows/dependency-audit.yml @@ -0,0 +1,22 @@ +name: Dependency audit +on: + pull_request: + push: + branches: [main] + schedule: + - cron: '17 10 * * 1' + workflow_dispatch: +permissions: + contents: read +jobs: + audit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: pnpm/action-setup@v6 + - uses: actions/setup-node@v6 + with: + node-version: 24 + cache: pnpm + - run: pnpm install --frozen-lockfile + - run: pnpm audit --prod --audit-level=moderate diff --git a/api/hqbase-mail-api-v1.openapi.json b/api/hqbase-mail-api-v1.openapi.json index 59f3bb30..4db4fb62 100644 --- a/api/hqbase-mail-api-v1.openapi.json +++ b/api/hqbase-mail-api-v1.openapi.json @@ -3761,6 +3761,14 @@ "items": { "$ref": "#/components/schemas/Attachment" } + }, + "replyTo": { + "type": "array", + "items": { + "type": "string", + "format": "email" + }, + "description": "Preferred recipients for replies to inbound mail when no explicit recipients are supplied." } } } @@ -4166,6 +4174,12 @@ }, "signature": { "$ref": "#/components/schemas/SignatureSelection" + }, + "idempotencyKey": { + "type": "string", + "minLength": 1, + "maxLength": 100, + "description": "Reuse this key only for a retry of the same send request. A saved draft already has a stable send identity." } } }, @@ -4222,6 +4236,12 @@ }, "signature": { "$ref": "#/components/schemas/SignatureSelection" + }, + "idempotencyKey": { + "type": "string", + "minLength": 1, + "maxLength": 100, + "description": "Reuse this key only for a retry of the same send request. A saved draft already has a stable send identity." } } }, @@ -4295,6 +4315,12 @@ }, "signature": { "$ref": "#/components/schemas/SignatureSelection" + }, + "idempotencyKey": { + "type": "string", + "minLength": 1, + "maxLength": 100, + "description": "Reuse this key only for a retry of the same send request. A saved draft already has a stable send identity." } } }, diff --git a/api/hqbase-mail-api-v2.openapi.json b/api/hqbase-mail-api-v2.openapi.json index e5af2831..d5b49058 100644 --- a/api/hqbase-mail-api-v2.openapi.json +++ b/api/hqbase-mail-api-v2.openapi.json @@ -3876,6 +3876,14 @@ "items": { "$ref": "#/components/schemas/Attachment" } + }, + "replyTo": { + "type": "array", + "items": { + "type": "string", + "format": "email" + }, + "description": "Preferred recipients for replies to inbound mail when no explicit recipients are supplied." } } } @@ -4281,6 +4289,12 @@ }, "signature": { "$ref": "#/components/schemas/SignatureSelection" + }, + "idempotencyKey": { + "type": "string", + "minLength": 1, + "maxLength": 100, + "description": "Reuse this key only for a retry of the same send request. A saved draft already has a stable send identity." } } }, @@ -4337,6 +4351,12 @@ }, "signature": { "$ref": "#/components/schemas/SignatureSelection" + }, + "idempotencyKey": { + "type": "string", + "minLength": 1, + "maxLength": 100, + "description": "Reuse this key only for a retry of the same send request. A saved draft already has a stable send identity." } } }, @@ -4410,6 +4430,12 @@ }, "signature": { "$ref": "#/components/schemas/SignatureSelection" + }, + "idempotencyKey": { + "type": "string", + "minLength": 1, + "maxLength": 100, + "description": "Reuse this key only for a retry of the same send request. A saved draft already has a stable send identity." } } }, diff --git a/app/app.tsx b/app/app.tsx index 105f6702..06bdd4fb 100644 --- a/app/app.tsx +++ b/app/app.tsx @@ -143,14 +143,20 @@ export function App(): React.ReactElement { const results = await Promise.allSettled([ refreshWorkspace(), hardRefresh ? mailSync.hardRefresh() : mailSync.refresh(), - draftState.refresh() + hardRefresh ? draftState.hardRefresh() : draftState.refresh() ]); if (results.every((result) => result.status === "rejected")) { const failure = results.find((result) => result.status === "rejected"); throw failure?.reason; } }, - [draftState.refresh, mailSync.hardRefresh, mailSync.refresh, refreshWorkspace] + [ + draftState.hardRefresh, + draftState.refresh, + mailSync.hardRefresh, + mailSync.refresh, + refreshWorkspace + ] ); const hardRefreshRealtimeState = React.useCallback( () => refreshRealtimeState(true), diff --git a/app/features/compose/compose-state.ts b/app/features/compose/compose-state.ts index 2738c036..aa7d26d0 100644 --- a/app/features/compose/compose-state.ts +++ b/app/features/compose/compose-state.ts @@ -115,7 +115,8 @@ export function hasInvalidRecipients(...values: string[]): boolean { } export function replyRecipients(message: MessageDetail): string[] { - if (message.direction === "inbound") return [message.fromAddress]; + if (message.direction === "inbound") + return message.replyTo?.length ? message.replyTo : [message.fromAddress]; const sender = message.fromAddress.toLowerCase(); return message.to.filter((address) => address.toLowerCase() !== sender); diff --git a/app/features/drafts/api.ts b/app/features/drafts/api.ts index b99c2825..38e7e1a8 100644 --- a/app/features/drafts/api.ts +++ b/app/features/drafts/api.ts @@ -1,7 +1,16 @@ -import { apiDelete, apiGetPage, apiPatch, apiPost } from "@/lib/api-client"; +import { apiDelete, apiGet, apiGetPage, apiPatch, apiPost } from "@/lib/api-client"; import type { Draft, DraftAttachment, DraftInput, DraftLabelMutationResult } from "./types"; +export type DraftChangePage = { + changes: Array<{ type: "upsert"; draft: Draft } | { type: "delete"; draftId: string }>; + nextCursor: string; + hasMore: boolean; +}; + +export const listDraftChanges = (cursor?: string): Promise => + apiGet(`/api/v2/drafts/changes${cursor ? `?cursor=${encodeURIComponent(cursor)}` : ""}`); + export async function listDrafts(): Promise { const drafts: Draft[] = []; let nextPageUrl: string | null = "/api/v2/drafts?limit=100"; diff --git a/app/features/drafts/drafts-page.tsx b/app/features/drafts/drafts-page.tsx index f2b2c545..a49768d7 100644 --- a/app/features/drafts/drafts-page.tsx +++ b/app/features/drafts/drafts-page.tsx @@ -1,4 +1,4 @@ -import type * as React from "react"; +import * as React from "react"; import { PiNotePencil, PiPaperclip } from "react-icons/pi"; import { Avatar, AvatarFallback } from "@/components/ui/avatar"; import { Button } from "@/components/ui/button"; @@ -45,6 +45,9 @@ export function DraftsPage({ onSelect, onToggleLabel }: DraftsPageProps): React.ReactElement { + const filterKey = JSON.stringify([mailboxId, search, labelIds]); + const [page, setPage] = React.useState({ filterKey, limit: 50 }); + const limit = page.filterKey === filterKey ? page.limit : 50; const normalizedSearch = search.trim().toLowerCase(); const visibleDrafts = drafts.filter((draft) => { if (mailboxId !== "all" && draft.mailboxId !== mailboxId) return false; @@ -99,7 +102,7 @@ export function DraftsPage({ ); } - const groups = groupDrafts(visibleDrafts); + const groups = groupDrafts(visibleDrafts.slice(0, limit)); return (
@@ -149,6 +152,15 @@ export function DraftsPage({
))} + {visibleDrafts.length > limit ? ( + + ) : null} )} diff --git a/app/features/drafts/sync-client.ts b/app/features/drafts/sync-client.ts new file mode 100644 index 00000000..0b764fa1 --- /dev/null +++ b/app/features/drafts/sync-client.ts @@ -0,0 +1,54 @@ +import { listDraftChanges, listDrafts } from "./api"; +import type { Draft } from "./types"; + +/** A cache belongs to one signed-in user. Access changes require a fresh snapshot. */ +export function createDraftSync() { + let drafts: Draft[] = []; + let cursor: string | null = null; + let generation = 0; + let inFlight: Promise | null = null; + const refresh = (reset = false): Promise => { + if (reset) { + cursor = null; + generation += 1; + } + if (inFlight && !reset) return inFlight; + const currentGeneration = generation; + const pending = (async () => { + let nextCursor = cursor; + let snapshot = drafts; + if (nextCursor === null) { + // Capture the journal boundary before listing so changes during bootstrap are replayed. + nextCursor = (await listDraftChanges()).nextCursor; + snapshot = await listDrafts(); + } + const byId = new Map(snapshot.map((draft) => [draft.id, draft])); + while (true) { + const page = await listDraftChanges(nextCursor); + for (const change of page.changes) { + if (change.type === "delete") byId.delete(change.draftId); + else byId.set(change.draft.id, change.draft); + } + if (page.hasMore && page.nextCursor === nextCursor) + throw new Error("Draft cursor did not advance."); + nextCursor = page.nextCursor; + if (!page.hasMore) break; + } + if (currentGeneration === generation) { + drafts = [...byId.values()].sort( + (left, right) => + right.updatedAt.localeCompare(left.updatedAt) || right.id.localeCompare(left.id) + ); + cursor = nextCursor; + } + return drafts; + })(); + inFlight = pending; + const clear = () => { + if (inFlight === pending) inFlight = null; + }; + void pending.then(clear, clear); + return pending; + }; + return { refresh }; +} diff --git a/app/features/drafts/use-drafts.ts b/app/features/drafts/use-drafts.ts index 623dc829..59f0e8d2 100644 --- a/app/features/drafts/use-drafts.ts +++ b/app/features/drafts/use-drafts.ts @@ -1,6 +1,6 @@ import * as React from "react"; import type { MailLabel } from "@/features/labels/types"; -import { listDrafts } from "./api"; +import { createDraftSync } from "./sync-client"; import type { Draft } from "./types"; export function useDrafts(userId: string | null): { @@ -8,25 +8,33 @@ export function useDrafts(userId: string | null): { isLoading: boolean; applyLabels: (draftId: string, labels: MailLabel[]) => void; refresh: () => Promise; + hardRefresh: () => Promise; } { + const cache = React.useMemo(() => ({ userId, sync: createDraftSync() }), [userId]); + const sync = cache.sync; const [drafts, setDrafts] = React.useState([]); const [isLoading, setIsLoading] = React.useState(false); const currentUserId = React.useRef(userId); currentUserId.current = userId; - const refresh = React.useCallback(async (): Promise => { - if (!userId) { - setDrafts([]); - setIsLoading(false); - return; - } + const refresh = React.useCallback( + async (reset = false): Promise => { + if (!userId) { + setDrafts([]); + setIsLoading(false); + return; + } - const nextDrafts = await listDrafts(); - if (currentUserId.current === userId) { - setDrafts(nextDrafts); - setIsLoading(false); - } - }, [userId]); + const nextDrafts = await sync.refresh(reset); + if (currentUserId.current === userId) { + setDrafts(nextDrafts); + setIsLoading(false); + } + }, + [sync, userId] + ); + + const hardRefresh = React.useCallback(() => refresh(true), [refresh]); React.useEffect(() => { if (!userId) { @@ -57,5 +65,5 @@ export function useDrafts(userId: string | null): { ); }, []); - return { applyLabels, drafts, isLoading, refresh }; + return { applyLabels, drafts, isLoading, refresh, hardRefresh }; } diff --git a/app/features/events/use-mail-events.ts b/app/features/events/use-mail-events.ts index 4152c7c8..e55ee4e0 100644 --- a/app/features/events/use-mail-events.ts +++ b/app/features/events/use-mail-events.ts @@ -9,6 +9,7 @@ const fallbackPollBaseDelayMs = 30_000; const fallbackPollMaxDelayMs = 60_000; const heartbeatIntervalMs = 30_000; const heartbeatTimeoutMs = 10_000; +const reconciliationIntervalMs = 120_000; type MailEventTopic = "drafts" | "labels" | "mailboxes" | "messages"; @@ -50,6 +51,7 @@ export function useMailEvents( let heartbeatTimeoutTimer: number | null = null; let reconnectTimer: number | null = null; let socket: WebSocket | null = null; + let reconciliationTimer: number | null = null; const pendingTopics = new Set(); let flushScheduled = false; @@ -93,6 +95,25 @@ export function useMailEvents( window.clearTimeout(fallbackTimer); fallbackTimer = null; }; + const clearReconciliationTimer = (): void => { + if (reconciliationTimer !== null) window.clearTimeout(reconciliationTimer); + reconciliationTimer = null; + }; + const scheduleReconciliation = (current: WebSocket): void => { + clearReconciliationTimer(); + reconciliationTimer = window.setTimeout(() => { + reconciliationTimer = null; + if (!canConnect() || socket !== current || !socketIsOpen()) return; + void Promise.resolve() + .then(currentHandlers.current.onFallbackPoll) + .catch(() => { + if (active && socket === current) current.close(4000, "Synchronization failed."); + }) + .finally(() => { + if (active && socket === current && socketIsOpen()) scheduleReconciliation(current); + }); + }, reconciliationIntervalMs); + }; const clearHeartbeatTimers = (): void => { if (heartbeatTimer !== null) window.clearTimeout(heartbeatTimer); if (heartbeatTimeoutTimer !== null) window.clearTimeout(heartbeatTimeoutTimer); @@ -101,6 +122,7 @@ export function useMailEvents( }; const closeSocket = (): void => { clearConnectionTimer(); + clearReconciliationTimer(); clearHeartbeatTimers(); const current = socket; socket = null; @@ -198,6 +220,7 @@ export function useMailEvents( fallbackAttempt = 0; setStatus("connected"); scheduleHeartbeat(next); + scheduleReconciliation(next); invoke(currentHandlers.current.onReconnect); }); next.addEventListener("message", (event) => { @@ -220,6 +243,7 @@ export function useMailEvents( if (socket !== next) return; socket = null; clearConnectionTimer(); + clearReconciliationTimer(); clearHeartbeatTimers(); if (!canConnect()) { if (navigator.onLine === false) setStatus("unavailable"); diff --git a/app/features/messages/conversation-window.ts b/app/features/messages/conversation-window.ts new file mode 100644 index 00000000..74af0a04 --- /dev/null +++ b/app/features/messages/conversation-window.ts @@ -0,0 +1,23 @@ +import { listConversations } from "./api"; +import type { ConversationPage } from "./types"; + +/** Re-read the loaded window so removed and filtered-out rows cannot survive a refresh. */ +export async function listConversationWindow( + params: Parameters[0], + pages: number +): Promise { + const first = await listConversations(params); + const conversations = new Map(first.conversations.map((item) => [item.threadId, item])); + let cursor = first.nextCursor; + for (let index = 1; cursor && index < pages; index += 1) { + const page = await listConversations({ ...params, cursor }); + for (const item of page.conversations) conversations.set(item.threadId, item); + if (page.nextCursor === cursor) throw new Error("Conversation cursor did not advance."); + cursor = page.nextCursor; + } + return { + conversations: [...conversations.values()], + nextCursor: cursor, + totalCount: first.totalCount + }; +} diff --git a/app/features/messages/types.ts b/app/features/messages/types.ts index 2ecb698a..a84bc166 100644 --- a/app/features/messages/types.ts +++ b/app/features/messages/types.ts @@ -50,6 +50,7 @@ export type MessageFolderAction = Extract< >; export type MessageDetail = MessageSummary & { + replyTo?: string[]; cc: string[]; bcc: string[]; deliveredToAddress: string | null; diff --git a/app/features/messages/use-mail-sync.ts b/app/features/messages/use-mail-sync.ts index af66c44f..481e1ec0 100644 --- a/app/features/messages/use-mail-sync.ts +++ b/app/features/messages/use-mail-sync.ts @@ -7,6 +7,7 @@ import { playNotificationSound } from "@/lib/notification-sounds"; import type { FolderId } from "@/lib/routes"; import { listConversations } from "./api"; +import { listConversationWindow } from "./conversation-window"; import type { ConversationAction, ConversationSummary } from "./types"; const noLabelIds: readonly string[] = []; @@ -58,7 +59,8 @@ export function useMailSync({ key: string; promise: Promise; } | null>(null); - const loadedAdditionalPages = React.useRef(false); + const loadedPageCount = React.useRef(1); + const nextCursorRef = React.useRef(null); const refreshGeneration = React.useRef(0); currentUserId.current = userId; currentSyncKey.current = syncKey; @@ -67,11 +69,12 @@ export function useMailSync({ refreshGeneration.current += 1; inFlight.current = null; loadMoreInFlight.current = null; - loadedAdditionalPages.current = false; + loadedPageCount.current = 1; setIsLoadingMore(false); setLoadMoreError(null); if (preserveVisible) return; setConversations([]); + nextCursorRef.current = null; setNextCursor(null); setTotalCount(null); }, []); @@ -79,6 +82,9 @@ export function useMailSync({ const refresh = React.useCallback((): Promise => { if (inFlight.current?.key === syncKey) return inFlight.current.promise; + if (loadMoreInFlight.current?.key === syncKey) { + return loadMoreInFlight.current.promise.then(() => refresh()); + } const generation = refreshGeneration.current; const promise = (async () => { if (!userId) { @@ -96,12 +102,15 @@ export function useMailSync({ activeFolder === "agents" || activeFolder === "drafts" ? Promise.resolve(null) - : listConversations({ - folder: activeFolder, - labelIds: labelIds.length === 0 ? undefined : labelIds, - mailboxId: mailboxId === "all" ? undefined : mailboxId, - search: search || undefined - }) + : listConversationWindow( + { + folder: activeFolder, + labelIds: labelIds.length === 0 ? undefined : labelIds, + mailboxId: mailboxId === "all" ? undefined : mailboxId, + search: search || undefined + }, + loadedPageCount.current + ) ]); if ( currentSyncKey.current !== syncKey || @@ -114,12 +123,9 @@ export function useMailSync({ if (conversationResult.status === "fulfilled" && conversationResult.value !== null) { const page = conversationResult.value; if (page.totalCount !== null) setTotalCount(page.totalCount); - if (loadedAdditionalPages.current) { - setConversations((current) => reconcileNewestPage(page.conversations, current)); - } else { - setConversations(page.conversations); - setNextCursor(page.nextCursor); - } + setConversations(page.conversations); + nextCursorRef.current = page.nextCursor; + setNextCursor(page.nextCursor); } if (notificationResult.status === "fulfilled") { const nextInboundId = notificationResult.value.latestInboundMessageId; @@ -203,9 +209,11 @@ export function useMailSync({ }, [refresh, userId]); const loadMore = React.useCallback((): Promise => { + if (inFlight.current?.key === syncKey) return inFlight.current.promise.then(() => loadMore()); + const cursor = nextCursorRef.current; if ( !userId || - !nextCursor || + !cursor || activeFolder === "settings" || activeFolder === "contacts" || activeFolder === "agents" || @@ -213,14 +221,10 @@ export function useMailSync({ ) { return Promise.resolve(); } - if ( - loadMoreInFlight.current?.key === syncKey && - loadMoreInFlight.current.cursor === nextCursor - ) { + if (loadMoreInFlight.current?.key === syncKey && loadMoreInFlight.current.cursor === cursor) { return loadMoreInFlight.current.promise; } - const cursor = nextCursor; const generation = refreshGeneration.current; setIsLoadingMore(true); setLoadMoreError(null); @@ -240,8 +244,9 @@ export function useMailSync({ ) { return; } - loadedAdditionalPages.current = true; + loadedPageCount.current += 1; setConversations((current) => appendConversationPage(current, page.conversations)); + nextCursorRef.current = page.nextCursor; setNextCursor(page.nextCursor); } catch (error: unknown) { if (currentSyncKey.current === syncKey && refreshGeneration.current === generation) { @@ -261,7 +266,7 @@ export function useMailSync({ }; void promise.then(clearInFlight, clearInFlight); return promise; - }, [activeFolder, labelIds, mailboxId, nextCursor, search, syncKey, userId]); + }, [activeFolder, labelIds, mailboxId, search, syncKey, userId]); const applyConversationAction = React.useCallback( (threadId: string, action: ConversationAction, affected: number): void => { @@ -329,17 +334,6 @@ export function useMailSync({ }; } -function reconcileNewestPage( - newest: ConversationSummary[], - current: ConversationSummary[] -): ConversationSummary[] { - const newestThreadIds = new Set(newest.map((conversation) => conversation.threadId)); - return [ - ...newest, - ...current.filter((conversation) => !newestThreadIds.has(conversation.threadId)) - ]; -} - function appendConversationPage( current: ConversationSummary[], next: ConversationSummary[] diff --git a/app/workspace-pages.tsx b/app/workspace-pages.tsx index 74b9ef6b..41606e4c 100644 --- a/app/workspace-pages.tsx +++ b/app/workspace-pages.tsx @@ -1,8 +1,8 @@ import * as React from "react"; +import { Spinner } from "@/components/ui/spinner"; -import { AgentsPage } from "@/features/agents/agents-page"; import type { CurrentUser } from "@/features/auth/types"; -import { ContactsPage } from "@/features/contacts/contacts-page"; + import { setDraftLabel } from "@/features/drafts/api"; import { DraftsPage } from "@/features/drafts/drafts-page"; import type { useDrafts } from "@/features/drafts/use-drafts"; @@ -10,13 +10,25 @@ import { InboxPage } from "@/features/inbox/inbox-page"; import type { MailLabel } from "@/features/labels/types"; import type { Mailbox } from "@/features/mailboxes/types"; import type { useMailSync } from "@/features/messages/use-mail-sync"; -import { SettingsPage } from "@/features/settings/settings-page"; + import type { SetupStatus } from "@/features/setup/types"; import type { useUpdateMonitor } from "@/features/updates/use-update-monitor"; import type { WorkspaceUser } from "@/features/users/types"; import type { AppRoute, FolderId, MailFolderId, SettingsTabId } from "@/lib/routes"; import type { useAppRoute } from "@/lib/use-app-route"; +const AgentsPage = React.lazy(() => + import("@/features/agents/agents-page").then((module) => ({ default: module.AgentsPage })) +); + +const ContactsPage = React.lazy(() => + import("@/features/contacts/contacts-page").then((module) => ({ default: module.ContactsPage })) +); + +const SettingsPage = React.lazy(() => + import("@/features/settings/settings-page").then((module) => ({ default: module.SettingsPage })) +); + const DraftComposeDialog = React.lazy(() => import("@/features/drafts/draft-compose-dialog").then((module) => ({ default: module.DraftComposeDialog @@ -94,7 +106,13 @@ export function WorkspacePages({ ); return ( - <> + + + + } + > {activeFolder === "agents" ? ( ) : null} - + ); } diff --git a/migrations-after-deploy/0004_mail_reliability_guards.sql b/migrations-after-deploy/0004_mail_reliability_guards.sql new file mode 100644 index 00000000..8f9bee4e --- /dev/null +++ b/migrations-after-deploy/0004_mail_reliability_guards.sql @@ -0,0 +1,37 @@ +CREATE TRIGGER send_operations_require_draft BEFORE INSERT ON send_operations +WHEN NEW.draft_id IS NOT NULL AND NOT EXISTS ( + SELECT 1 FROM drafts WHERE id = NEW.draft_id AND principal_id IS NEW.principal_id +) +BEGIN SELECT RAISE(ABORT, 'draft was removed before send'); END; + +CREATE TRIGGER drafts_before_update_pending_send BEFORE UPDATE ON drafts +WHEN EXISTS (SELECT 1 FROM send_operations WHERE draft_id = OLD.id AND status <> 'stored') +BEGIN SELECT RAISE(ABORT, 'draft send is pending'); END; + +CREATE TRIGGER drafts_before_delete_pending_send BEFORE DELETE ON drafts +WHEN EXISTS (SELECT 1 FROM send_operations WHERE draft_id = OLD.id AND status <> 'stored') +BEGIN SELECT RAISE(ABORT, 'draft send is pending'); END; + +CREATE TRIGGER draft_attachments_before_update_pending_send BEFORE UPDATE ON draft_attachments +WHEN EXISTS (SELECT 1 FROM send_operations WHERE draft_id = OLD.draft_id AND status <> 'stored') +BEGIN SELECT RAISE(ABORT, 'draft send is pending'); END; + +CREATE TRIGGER draft_attachments_before_delete_pending_send BEFORE DELETE ON draft_attachments +WHEN EXISTS (SELECT 1 FROM send_operations WHERE draft_id = OLD.draft_id AND status <> 'stored') +BEGIN SELECT RAISE(ABORT, 'draft send is pending'); END; + +CREATE TRIGGER draft_attachments_before_insert_pending_send BEFORE INSERT ON draft_attachments +WHEN EXISTS (SELECT 1 FROM send_operations WHERE draft_id = NEW.draft_id AND status <> 'stored') +BEGIN SELECT RAISE(ABORT, 'draft send is pending'); END; + +CREATE TRIGGER draft_labels_before_update_pending_send BEFORE UPDATE ON draft_labels +WHEN EXISTS (SELECT 1 FROM send_operations WHERE draft_id = OLD.draft_id AND status <> 'stored') +BEGIN SELECT RAISE(ABORT, 'draft send is pending'); END; + +CREATE TRIGGER draft_labels_before_delete_pending_send BEFORE DELETE ON draft_labels +WHEN EXISTS (SELECT 1 FROM send_operations WHERE draft_id = OLD.draft_id AND status <> 'stored') +BEGIN SELECT RAISE(ABORT, 'draft send is pending'); END; + +CREATE TRIGGER draft_labels_before_insert_pending_send BEFORE INSERT ON draft_labels +WHEN EXISTS (SELECT 1 FROM send_operations WHERE draft_id = NEW.draft_id AND status <> 'stored') +BEGIN SELECT RAISE(ABORT, 'draft send is pending'); END; diff --git a/migrations/0029_mail_reliability.sql b/migrations/0029_mail_reliability.sql new file mode 100644 index 00000000..d8ae16d9 --- /dev/null +++ b/migrations/0029_mail_reliability.sql @@ -0,0 +1,29 @@ +ALTER TABLE messages ADD COLUMN text_r2_key TEXT; +ALTER TABLE messages ADD COLUMN reply_to_json TEXT NOT NULL DEFAULT '[]'; +CREATE INDEX messages_text_r2_key_idx ON messages(text_r2_key) WHERE text_r2_key IS NOT NULL; +CREATE INDEX messages_html_r2_key_idx ON messages(html_r2_key) WHERE html_r2_key IS NOT NULL; +CREATE INDEX messages_raw_r2_key_idx ON messages(raw_r2_key) WHERE raw_r2_key IS NOT NULL; +CREATE INDEX message_attachments_r2_key_idx ON message_attachments(r2_key); + +CREATE TABLE send_operations ( + id TEXT PRIMARY KEY NOT NULL, + principal_id TEXT, + draft_id TEXT, + mailbox_id TEXT, + request_hash TEXT NOT NULL, + status TEXT NOT NULL CHECK (status IN ('sending', 'accepted', 'stored', 'unknown')), + message_id TEXT NOT NULL, + provider_message_id TEXT, + payload_r2_key TEXT NOT NULL UNIQUE, + receipt_r2_key TEXT NOT NULL UNIQUE, + object_keys_json TEXT NOT NULL DEFAULT '[]', + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL +); +CREATE UNIQUE INDEX send_operations_draft_idx ON send_operations(principal_id, draft_id) + WHERE draft_id IS NOT NULL; +CREATE INDEX send_operations_status_idx ON send_operations(status, updated_at); + +ALTER TABLE operation_runs ADD COLUMN attempts INTEGER NOT NULL DEFAULT 0; +ALTER TABLE operation_runs ADD COLUMN lease_token TEXT; +ALTER TABLE operation_runs ADD COLUMN lease_expires_at TEXT; diff --git a/package.json b/package.json index 3c9d611d..81cde372 100644 --- a/package.json +++ b/package.json @@ -84,10 +84,10 @@ "@radix-ui/react-slot": "^1.2.3", "@radix-ui/react-tabs": "^1.1.12", "@radix-ui/react-tooltip": "^1.2.7", - "@tiptap/extension-image": "3.27.3", - "@tiptap/extension-placeholder": "^3.27.3", - "@tiptap/react": "^3.27.3", - "@tiptap/starter-kit": "^3.27.3", + "@tiptap/extension-image": "3.30.4", + "@tiptap/extension-placeholder": "3.30.4", + "@tiptap/react": "3.30.4", + "@tiptap/starter-kit": "3.30.4", "better-auth": "1.7.0-rc.6", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", @@ -99,7 +99,7 @@ "react": "^19.1.0", "react-dom": "^19.1.0", "react-icons": "^5.7.0", - "sanitize-html": "^2.17.6", + "sanitize-html": "^2.17.7", "sonner": "^2.0.6", "tailwind-merge": "^3.3.1", "web-push": "^3.6.7", diff --git a/playwright.config.ts b/playwright.config.ts index 4cf65334..1a798c4a 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -24,6 +24,7 @@ export default defineConfig({ extraHTTPHeaders: accessClientId && accessClientSecret ? { + origin: baseURL ? new URL(baseURL).origin : "https://staging.invalid", "CF-Access-Client-Id": accessClientId, "CF-Access-Client-Secret": accessClientSecret } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 77ec2fed..99399535 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -7,7 +7,12 @@ settings: overrides: '@better-auth/core': 1.7.0-rc.6 '@hono/node-server': 2.0.12 - fast-uri: 3.1.4 + '@tiptap/extension-bubble-menu': 3.30.4 + '@tiptap/extension-floating-menu': 3.30.4 + fast-uri: 3.1.6 + ip-address: 10.3.1 + nanoid@3: 3.3.18 + qs: 6.16.0 postcss: 8.5.23 importers: @@ -51,17 +56,17 @@ importers: specifier: ^1.2.7 version: 1.2.10(@types/react-dom@19.2.3(@types/react@19.2.17))(@types/react@19.2.17)(react-dom@19.2.7(react@19.2.7))(react@19.2.7) '@tiptap/extension-image': - specifier: 3.27.3 - version: 3.27.3(@tiptap/core@3.27.3(@tiptap/pm@3.27.3)) + specifier: 3.30.4 + version: 3.30.4(@tiptap/core@3.30.4(@tiptap/pm@3.30.4)) '@tiptap/extension-placeholder': - specifier: ^3.27.3 - version: 3.27.3(@tiptap/extensions@3.27.3(@tiptap/core@3.27.3(@tiptap/pm@3.27.3))(@tiptap/pm@3.27.3)) + specifier: 3.30.4 + version: 3.30.4(@tiptap/extensions@3.30.4(@tiptap/core@3.30.4(@tiptap/pm@3.30.4))(@tiptap/pm@3.30.4)) '@tiptap/react': - specifier: ^3.27.3 - version: 3.27.3(@floating-ui/dom@1.7.6)(@tiptap/core@3.27.3(@tiptap/pm@3.27.3))(@tiptap/pm@3.27.3)(@types/react-dom@19.2.3(@types/react@19.2.17))(@types/react@19.2.17)(react-dom@19.2.7(react@19.2.7))(react@19.2.7) + specifier: 3.30.4 + version: 3.30.4(@floating-ui/dom@1.7.6)(@tiptap/core@3.30.4(@tiptap/pm@3.30.4))(@tiptap/pm@3.30.4)(@types/react-dom@19.2.3(@types/react@19.2.17))(@types/react@19.2.17)(react-dom@19.2.7(react@19.2.7))(react@19.2.7) '@tiptap/starter-kit': - specifier: ^3.27.3 - version: 3.27.3 + specifier: 3.30.4 + version: 3.30.4 better-auth: specifier: 1.7.0-rc.6 version: 1.7.0-rc.6(drizzle-orm@0.45.2(kysely@0.29.2))(react-dom@19.2.7(react@19.2.7))(react@19.2.7)(vitest@4.1.10) @@ -96,8 +101,8 @@ importers: specifier: ^5.7.0 version: 5.7.0(react@19.2.7) sanitize-html: - specifier: ^2.17.6 - version: 2.17.6 + specifier: ^2.17.7 + version: 2.17.7 sonner: specifier: ^2.0.6 version: 2.0.7(react-dom@19.2.7(react@19.2.7))(react@19.2.7) @@ -1470,164 +1475,165 @@ packages: '@standard-schema/spec@1.1.0': resolution: {integrity: sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==} - '@tiptap/core@3.27.3': - resolution: {integrity: sha512-TJj5929M96C1KlH796wS8MywfHDh49RhmakOyzyMMc9pFmRj9UXi1gj0TCXgsZtjEOG7B+m/DRvNOvnuvR9kmg==} + '@tiptap/core@3.30.4': + resolution: {integrity: sha512-V9yKuUfV8qC9WBrnVxkFWmYLBomc3d1CwXoFSjED5DRu5q2oyxv07F+jOJSRogJAY+feHjuxvjhixwxeHm2DXQ==} peerDependencies: - '@tiptap/pm': 3.27.3 + '@tiptap/pm': 3.30.4 - '@tiptap/extension-blockquote@3.27.3': - resolution: {integrity: sha512-VGoVMcqcGkkoduzEqQ70ZK5OOHBDn5iYeHJSgkvNoSHzYHl0CaKoWxoDKJjSWC5DAM4mBU+tXTlsiUp3evRMfA==} + '@tiptap/extension-blockquote@3.30.4': + resolution: {integrity: sha512-n25/pFfDpZRJS4f6Ga/PkNrQFBEr3pRvPkDlY99kl8Kt8jFa1EUxbx+uUVIilqwiUjUnHtwCfqjSDFooQjezjg==} peerDependencies: - '@tiptap/core': 3.27.3 + '@tiptap/core': 3.30.4 + '@tiptap/pm': 3.30.4 - '@tiptap/extension-bold@3.27.3': - resolution: {integrity: sha512-pchbycFppBqBmvk+OxrQLIPZLNd3rNbcm7zOa+OTjluphpAcsJ6AmHBmiRhJUYWitV6/SA+0nujcaWxBp5hNcQ==} + '@tiptap/extension-bold@3.30.4': + resolution: {integrity: sha512-eCOkf+/jdQte7lTZ4pQF10akrsFM4TzYc0ysSAkPa7QA+ex1aI8QT5UUx+mK8RwFo57lqE8PCX/Mx1gTqlCDnQ==} peerDependencies: - '@tiptap/core': 3.27.3 + '@tiptap/core': 3.30.4 - '@tiptap/extension-bubble-menu@3.27.3': - resolution: {integrity: sha512-PKSM9g8BXzO0Lxm2gOin7FAFvxO6T0QvlXTXVke/GJn/sR1tEz7w5ybsvweytUPmSliUc601pfIUOHKEhIVC2Q==} + '@tiptap/extension-bubble-menu@3.30.4': + resolution: {integrity: sha512-diB5stCiuffFIIKhDAVvSYTYW/zzEVh7QqTuk2JAqdkAmAGP2YvUXXZBVhA9ITTIYD1+cp5p6rx9pdM25fsVRQ==} peerDependencies: - '@tiptap/core': 3.27.3 - '@tiptap/pm': 3.27.3 + '@tiptap/core': 3.30.4 + '@tiptap/pm': 3.30.4 - '@tiptap/extension-bullet-list@3.27.3': - resolution: {integrity: sha512-owQW6mcgnYgQU1z5BqmslcjZFBUm1SGM/Ax++4R6XelIt/7ol6uSTjzjAzum+ASW7UlBbuJHuSgRjl8cA+RGkg==} + '@tiptap/extension-bullet-list@3.30.4': + resolution: {integrity: sha512-kJFOF3U4b1ad4T7Vm+CIIp+nxcr6wwWVeqpVe6Jhq9gudsWCKP0+KVAmOiJCePoDu9fpc/ZuGMXDefXYnyQDxg==} peerDependencies: - '@tiptap/extension-list': 3.27.3 + '@tiptap/extension-list': 3.30.4 - '@tiptap/extension-code-block@3.27.3': - resolution: {integrity: sha512-3CVnzkpGoqqI5KaXI9l9PMwLkx34SYY63tpeo0I0QjDk4LU+JTAQDTPJFSwB4zTsHZ7ZFMU0jjasFOZ1kZRVgw==} + '@tiptap/extension-code-block@3.30.4': + resolution: {integrity: sha512-eKLKxgLCvi+M5tiLkW+fIMzDtR7HvSKnigSE9RYHZmzSrv1ejVjgNj5FH8nGTim98hXiQGfWotss3zqG6bpdDw==} peerDependencies: - '@tiptap/core': 3.27.3 - '@tiptap/pm': 3.27.3 + '@tiptap/core': 3.30.4 + '@tiptap/pm': 3.30.4 - '@tiptap/extension-code@3.27.3': - resolution: {integrity: sha512-T7JF10Y3k7fyVujXXLVubdUh7fctgBUoiuxbvjotpltg1U/mPbOcT0UW/uif9j3H8TFvAwNApVw3x8bwwFUakA==} + '@tiptap/extension-code@3.30.4': + resolution: {integrity: sha512-z9v9rBA/0MecUvf8QkcKKOGgceO9X3DT/JvFNka8Mdd68V5fhBs0jGsxYrj3qDWtdXyyUQIWs0t8HzL6LTdLiA==} peerDependencies: - '@tiptap/core': 3.27.3 + '@tiptap/core': 3.30.4 - '@tiptap/extension-document@3.27.3': - resolution: {integrity: sha512-PR25MRv56Nm4ETVLwA8nAh/0RrVdvD4D2nxnAC8kUkxz1WOYDaqTfPqqEeSWDoUUTUp1I1xzDkD4x0gEnAlmZA==} + '@tiptap/extension-document@3.30.4': + resolution: {integrity: sha512-N+FbI+X1FVH8HxsM8C4fNkkzKXiyhXc9oh9oSqAIKOAAYGAjnHLMxW9TzYXbvm60SlAn6DIMypNf6nULGEc1oQ==} peerDependencies: - '@tiptap/core': 3.27.3 + '@tiptap/core': 3.30.4 - '@tiptap/extension-dropcursor@3.27.3': - resolution: {integrity: sha512-XsL0Ziual+ynXyx5JgLb0KIOShlMN3MW7+GQvf9PGRb1vOB7IOTRFDxMPrxDWcFIZ6WWMWSrbkCLOKHStt1BWg==} + '@tiptap/extension-dropcursor@3.30.4': + resolution: {integrity: sha512-P1V0y/FKdyNVBImeW3WN+uGI77Uboc0dW44izAjrII0jcW2KUoBQeNrbbm9UZR1xuRvQJ1Z0OrFHmYrxP5ClNQ==} peerDependencies: - '@tiptap/extensions': 3.27.3 + '@tiptap/extensions': 3.30.4 - '@tiptap/extension-floating-menu@3.27.3': - resolution: {integrity: sha512-qL+g1Z6MqZvYG4mHTPwub9II2fWZY+2I5VFfV8fzqy1HchBOyY6Df0mRKDNb0bmjBdFabIXnswz3iqLx3mDTbw==} + '@tiptap/extension-floating-menu@3.30.4': + resolution: {integrity: sha512-75kDLGkVqLBL7OPGNhJZQkmI3KGGVN9S5i+ZHKZEhk4HNOQUU+YMs7qu4I/XSJMXSh7mGrQlMi/OD/Fn3JmkKg==} peerDependencies: '@floating-ui/dom': ^1.0.0 - '@tiptap/core': 3.27.3 - '@tiptap/pm': 3.27.3 + '@tiptap/core': 3.30.4 + '@tiptap/pm': 3.30.4 - '@tiptap/extension-gapcursor@3.27.3': - resolution: {integrity: sha512-MVBqzwYrDOttkn2GOrvvKTDCymfSS2VGSKpahcKKhge5mweSDiqko2X1vBDnMF8aVW3WxIFfDMmO4VMmh8HDZA==} + '@tiptap/extension-gapcursor@3.30.4': + resolution: {integrity: sha512-wsuXsB8Rp9BgfWlWYsRVzoHg9LwhoPAQGaw/gku1buDSTMcbLYQvok5MwJ/uJRECaJRLGgbUe5WBXoRvSY+o3g==} peerDependencies: - '@tiptap/extensions': 3.27.3 + '@tiptap/extensions': 3.30.4 - '@tiptap/extension-hard-break@3.27.3': - resolution: {integrity: sha512-cySObJITR2CRrWII87mI2LJ12wEETTnxt2p1vvC6ZtUmKnB50fOu+ux8ZEt6KYNocIOtrEZthNFzTOx7R7UtrQ==} + '@tiptap/extension-hard-break@3.30.4': + resolution: {integrity: sha512-eZ66SyfgmMK861S5SYtQROT/+ZfXtDHxllz7ao+X+dcl+DMdffmOzBeSfwrHQENmgl7umjbdAjqC7PT95jaU+w==} peerDependencies: - '@tiptap/core': 3.27.3 + '@tiptap/core': 3.30.4 - '@tiptap/extension-heading@3.27.3': - resolution: {integrity: sha512-QHXnsNic6iId8pnsFZ8z4PkX5L+HCHa/D7rAi3nNWtPlSIAOxo4nKrALcB5/tHmY+XL8kEXKH3nsNLNEDLCYPg==} + '@tiptap/extension-heading@3.30.4': + resolution: {integrity: sha512-sVJxoRnbfK/QC7IoUw/Ezzx9b87+cqFz5d5WZvN8O8yobDWuh7IS8ZgqZiPypTW5RuhnZI+ahvmSyO3WLm8q5Q==} peerDependencies: - '@tiptap/core': 3.27.3 + '@tiptap/core': 3.30.4 - '@tiptap/extension-horizontal-rule@3.27.3': - resolution: {integrity: sha512-sNCuo0+Q001B5XTVWY+ULpWXQqtBNRmAqXxAexZl44bx3rDqHG8OzjwT4EM0LIj2+BYVbdqwaJ1vfQbZVEEbig==} + '@tiptap/extension-horizontal-rule@3.30.4': + resolution: {integrity: sha512-RmvjVVkUf5pF70XvwI7Sb4vIUSDbLTLEcFl1l5X9vPcDgDT5G0SiUL6l8F49+qYc6wrMvnLcenApnAiiQ0Vbpw==} peerDependencies: - '@tiptap/core': 3.27.3 - '@tiptap/pm': 3.27.3 + '@tiptap/core': 3.30.4 + '@tiptap/pm': 3.30.4 - '@tiptap/extension-image@3.27.3': - resolution: {integrity: sha512-3OVLRH54Xkh9yrI5oGIempuqZkhLG1plyJBRnapx0N8j9IjEImOLF78kPYoAZzaBuU78M3Zk/E7HQZgmnaNRBg==} + '@tiptap/extension-image@3.30.4': + resolution: {integrity: sha512-BwO+SJ+1ZWFLDFbebT80q0PlsbrnmWRExzQqurubM93M22SlADnQxNzS7AKfNmv/e3qKAAXXsoBOkx66BuFjaQ==} peerDependencies: - '@tiptap/core': 3.27.3 + '@tiptap/core': 3.30.4 - '@tiptap/extension-italic@3.27.3': - resolution: {integrity: sha512-7VpVi2vn8kGFAc/HLrt96J/E6+LXOGKn8xLhJS863t150yLIG3EPQLA1h+G5O6eo+/w+9YokvJAAJoeDihj1Jg==} + '@tiptap/extension-italic@3.30.4': + resolution: {integrity: sha512-6cEjcyjPRcLEMB75BwiUc6S7yzkqc7VPXCeRgY26UqMg4AvcLrseY70+pO4sLv3n2DS5w8DBbMXZJ4rb1BhpUw==} peerDependencies: - '@tiptap/core': 3.27.3 + '@tiptap/core': 3.30.4 - '@tiptap/extension-link@3.27.3': - resolution: {integrity: sha512-iXmu1tJ/3vP60c9d+zfO3+X64vod2EyBrM1qeufecAtWA6t0bT9tYeka0Zq6qDRGsZK5zViO9F2lsY7eHXiYuA==} + '@tiptap/extension-link@3.30.4': + resolution: {integrity: sha512-HPHaey3+nQZl+lsr/RvjoXSgSC67SAkNEF4qNELzzDwMUfI0UrZ3H/HhBe6XBeRnGli9gfuSpeRkh63ke5JpOw==} peerDependencies: - '@tiptap/core': 3.27.3 - '@tiptap/pm': 3.27.3 + '@tiptap/core': 3.30.4 + '@tiptap/pm': 3.30.4 - '@tiptap/extension-list-item@3.27.3': - resolution: {integrity: sha512-UddhVaQ7+V9IfQVVE8AJLNzSmyaaNpZPQsoPQxFNlJyOFxD6RwSTT8L4rb/TwbUctxSFWpyd1J7exzf33Ony/g==} + '@tiptap/extension-list-item@3.30.4': + resolution: {integrity: sha512-8E1ffdC7v3dwSrsqyxY3YP5uEQHTrwBDDmYF6o2YoHfzQ9nhXX5GmBz8fTxukjXUzSpuO76N/gwHIM8hBfdxMQ==} peerDependencies: - '@tiptap/extension-list': 3.27.3 + '@tiptap/extension-list': 3.30.4 - '@tiptap/extension-list-keymap@3.27.3': - resolution: {integrity: sha512-GZPVg4QCFm41yTMPEPeTlH2odNeQwRuWHY/pt3pI5LsfipQ4kDBv6h9we928vJQUnU2Gg9Y3cL5iRbmXIWWTUA==} + '@tiptap/extension-list-keymap@3.30.4': + resolution: {integrity: sha512-6RIzF3aThqIt4sia81K+6wA7H7bgPKjd3D66qLOGxdqdyfnAegk0OD5pUYhMZKhDCWGYKylyRVd2yIn/VNXqAg==} peerDependencies: - '@tiptap/extension-list': 3.27.3 + '@tiptap/extension-list': 3.30.4 - '@tiptap/extension-list@3.27.3': - resolution: {integrity: sha512-52rcaSzYjtVGUrNkdB8k1Bw/rId2lBirrWMvmnBLsieoeFd8zo8ow5zkwV057BYSgWXfnMAV7GOpDyq2IoSD9A==} + '@tiptap/extension-list@3.30.4': + resolution: {integrity: sha512-Usqez9DBRoG78tdLwPDcd1j1mBBU7mr5D2yrRP2JL/eXXkhAfdWqEFRYXwct5T4/o9JkGcW3aQf+gUvp12v0pw==} peerDependencies: - '@tiptap/core': 3.27.3 - '@tiptap/pm': 3.27.3 + '@tiptap/core': 3.30.4 + '@tiptap/pm': 3.30.4 - '@tiptap/extension-ordered-list@3.27.3': - resolution: {integrity: sha512-oHYoE65WfRSM5tCcXi8MtQmzPxz83E2qw0pa2yHBKwr9xCCQUu8vJHJE+EJGduWdaNImv43+6/6kVMeYYbXRKg==} + '@tiptap/extension-ordered-list@3.30.4': + resolution: {integrity: sha512-hInlH8I2UFGGULm9XLPtLWWFBECrYf/eNkloQb+udbF7ltLBL5JRCPcB3aM29qSSUQVxCCN8ICHdbwgpnTUAsw==} peerDependencies: - '@tiptap/extension-list': 3.27.3 + '@tiptap/extension-list': 3.30.4 - '@tiptap/extension-paragraph@3.27.3': - resolution: {integrity: sha512-G5XPCN7lm0nULYPelJC2eUbKbt1q97j67e/HSWxwMYEXhceec5eNAtVfpDCRyXNO9osyBi2kXqLv/IQtxbch9A==} + '@tiptap/extension-paragraph@3.30.4': + resolution: {integrity: sha512-gM0WXvOP1tNcvpyRXTCtGhwVxye0VaFMCzDPjLrVAUtZMpo3cOrqDSrFl5tCurA59Qc2p5TUOR4g3/fKw8Yx9Q==} peerDependencies: - '@tiptap/core': 3.27.3 + '@tiptap/core': 3.30.4 - '@tiptap/extension-placeholder@3.27.3': - resolution: {integrity: sha512-Y43dGaN3rA0oU4CWmXkZ2idjZjrxPX9oy0zZL92EgXOguhAUToBs0dIxRG5ZanYX9A8kKKWPlfeRFlBMtRhKvg==} + '@tiptap/extension-placeholder@3.30.4': + resolution: {integrity: sha512-Bbwzse6oTNwkQVmBQF45YJ/zpFQut52tT+s+u6cWi/SwyYTMeaU4XNOV6KidkM8urERFCkF/PplAevusvWuowA==} peerDependencies: - '@tiptap/extensions': 3.27.3 + '@tiptap/extensions': 3.30.4 - '@tiptap/extension-strike@3.27.3': - resolution: {integrity: sha512-dp6Rs3I4zLVJPvAw2Q9yrNuYKcX5LTONH3/oyULvGsrqq5DKz7pqscfwhwnDSzrgm/3aCz6B9r1BVHlYI/DXlA==} + '@tiptap/extension-strike@3.30.4': + resolution: {integrity: sha512-iM3QhkEvNwDsxyNKTZM4wwqYhIKOjgXHTamPWnCICrPQ3aYFVdidfpDMePlHl6XL8aEAqPa1Xlj0aumFvSphOQ==} peerDependencies: - '@tiptap/core': 3.27.3 + '@tiptap/core': 3.30.4 - '@tiptap/extension-text@3.27.3': - resolution: {integrity: sha512-wSatZ/bop0pleeC18nF90zvWWEoRe/ewZncvm8LjcMrXwq41kTEs3j7nZflMYAh7X8fZvu00UIMeEyCNJl+5Yg==} + '@tiptap/extension-text@3.30.4': + resolution: {integrity: sha512-bzgVlPhkVan+m6jycXkyq08fKCNyQff/cl/5U+/wQf8s3GlX+Le2/pTxbbC3ClfO1D8VLzLwcd2HJTYIoBOdAA==} peerDependencies: - '@tiptap/core': 3.27.3 + '@tiptap/core': 3.30.4 - '@tiptap/extension-underline@3.27.3': - resolution: {integrity: sha512-YH7ka/Rp7ZwlzTuQCcs0OZ4CMSx/P/MzbseQKxKebEIZreWaKMZAaED97Fnu0dA66VlreD4/1C7xHJy84ovamg==} + '@tiptap/extension-underline@3.30.4': + resolution: {integrity: sha512-h6nM3ykKswJLWvJVD1eiUFbanNmNj2SeV+2vuW666/xOy2WPyMnE2NZnsbvCKeK0wlUj7AE3l1m602mFwA4sDg==} peerDependencies: - '@tiptap/core': 3.27.3 + '@tiptap/core': 3.30.4 - '@tiptap/extensions@3.27.3': - resolution: {integrity: sha512-IA2QKUVJgzUPEhhkUfr6P5u5GFVfhiApiEaaHXBz07saL2c4HNoVs2RC18QlZDCtLNKrPR1mUScr72u7uYs+YQ==} + '@tiptap/extensions@3.30.4': + resolution: {integrity: sha512-WeBl/ggeNCOoOySX7647lwtSUHWbCh8I1Qqp8NQBsGyHXEE6/7jHbKcxpw4fplOgScgVMfPz8WZ+3yrfXYCUFw==} peerDependencies: - '@tiptap/core': 3.27.3 - '@tiptap/pm': 3.27.3 + '@tiptap/core': 3.30.4 + '@tiptap/pm': 3.30.4 - '@tiptap/pm@3.27.3': - resolution: {integrity: sha512-ppiG57RxM3HSHHgcHT0hP6Ib4P56Sd5itxdV4w0hIGHKPGyLLKiAkYp+htIHa9T7IvjMdaqxIlsfyV3ZKsS1sw==} + '@tiptap/pm@3.30.4': + resolution: {integrity: sha512-oPbE+BOzzDKkxsvF9wepTWELvq385oifDkKIigqKCPKpGqplEIHIjNztCj/nOqHCfJBiU9LcoyyWH0qkHAQ9TQ==} - '@tiptap/react@3.27.3': - resolution: {integrity: sha512-lanXxMScw/LevbRFuFw5MoQJ1grmHUDszwph2VIk4l2U5e/EDIw3rr4CeDME6Z4isVReD6zccMFVShkHqsv+jg==} + '@tiptap/react@3.30.4': + resolution: {integrity: sha512-NxGcKg4xBF6ngk6xORyzRCvG9zb7Z/cf59GpDBsiYpPpD/6m+J4kfNF0KVIOhRSAhNxm2u2ZPuGIWBtS/CDUBQ==} peerDependencies: - '@tiptap/core': 3.27.3 - '@tiptap/pm': 3.27.3 + '@tiptap/core': 3.30.4 + '@tiptap/pm': 3.30.4 '@types/react': ^17.0.0 || ^18.0.0 || ^19.0.0 '@types/react-dom': ^17.0.0 || ^18.0.0 || ^19.0.0 react: ^17.0.0 || ^18.0.0 || ^19.0.0 react-dom: ^17.0.0 || ^18.0.0 || ^19.0.0 - '@tiptap/starter-kit@3.27.3': - resolution: {integrity: sha512-xX3baFqiC30skntdhxUUvyJo755ON9c1pE83+2Wiq2g+Qnffg9knUuLCZStHoZZ318yB0aBsKAMvHWLtYDo8AA==} + '@tiptap/starter-kit@3.30.4': + resolution: {integrity: sha512-rZiv2QOqfQU4/MNsAmvbymIwx0tKrGrtdJg76aEaej+aGU1QNDhZbUNutV8AnZxFjSJMm8rt810qzAU98OmCAg==} '@types/babel__core@7.20.5': resolution: {integrity: sha512-qoQprZvz5wQFJwMDqeseRXWv3rqMvhgpbXFfVyWhbx9X47POIA6i/+dXefEmZKoAgOaTdaIgNSMqMIU61yRyzA==} @@ -2225,8 +2231,8 @@ packages: resolution: {integrity: sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg==} engines: {node: '>=8.6.0'} - fast-uri@3.1.4: - resolution: {integrity: sha512-8JnbkQ4juDyvYs4mgFGQqg4yCYtFDtUtmp2QIQq11ZZe5CFQ5wcqm1rqDgAh/QdMySuBnPzMUiJUNZG5N/AiQw==} + fast-uri@3.1.6: + resolution: {integrity: sha512-7Ical1vFEMr0onbVzEDIreM22I4khW+fzyQPwvAFWBp1iwdshSZRsL4jjRvPG9JP1uiqMHRto+YU6R2/CzDz5Q==} fastq@1.20.1: resolution: {integrity: sha512-GGToxJ/w1x32s/D2EKND7kTil4n8OVk/9mycTc4VDza13lOvpUZTGX3mFSCtV9ksdGBVzvsyAVLM6mHFThxXxw==} @@ -2349,8 +2355,8 @@ packages: inherits@2.0.4: resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==} - ip-address@10.2.0: - resolution: {integrity: sha512-/+S6j4E9AHvW9SWMSEY9Xfy66O5PWvVEJ08O0y5JGyEKQpojb0K0GKpz/v5HJ/G0vi3D2sjGK78119oXZeE0qA==} + ip-address@10.3.1: + resolution: {integrity: sha512-1e9d3kb97NHJTIJDZW9rKqW2h6+dFa50Dy0fpPSMQp2ADje5gvKsXmdiK6dwY5t76TaTt5+P5N1Y/LoToIxP6g==} engines: {node: '>= 12'} ipaddr.js@1.9.1: @@ -2513,8 +2519,8 @@ packages: mz@2.7.0: resolution: {integrity: sha512-z81GNO7nnYMEhrGh9LeymoE4+Yr0Wn5McHIZMK5cfQCl+NDX08sCZgUc9/6MHni9IWuFLm1Z3HTCXu2z9fN62Q==} - nanoid@3.3.16: - resolution: {integrity: sha512-bzlKTyNJ7+LdGIIwy8ijFpIqEQIvafahV7eYykJ8Cvh42EdJeODoJ6gUJXpQJvej1BddH8OqTXZNE/KfbWAu8Q==} + nanoid@3.3.18: + resolution: {integrity: sha512-DTg4MJbGMWkfi6VZFdNt2/caMbQy4Ou+Op/hJQvGEWcnVfoA1QA+xzRKAzw9jD6+GVOOeYr/mIcuDSdug6F6+w==} engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} hasBin: true @@ -2709,8 +2715,8 @@ packages: resolution: {integrity: sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==} engines: {node: '>= 0.10'} - qs@6.15.3: - resolution: {integrity: sha512-O9gl3zCl5h5blw1KGUzQKhA5oUXSl8rwUIM5o0S3nCXMliSvy5Dzx7/DJcI+SwgICv+IneSZwhBh1oSyEHA71A==} + qs@6.16.0: + resolution: {integrity: sha512-h6fhOIaRrID2CbEY2fqs+7t+UXZo+MLAnU5gRIq85uFtdiUPCdsApMlHhXogKVM4HM2DVbIjGNTTYH2OcmP1vA==} engines: {node: '>=0.6'} queue-microtask@1.2.3: @@ -2816,8 +2822,8 @@ packages: safer-buffer@2.1.2: resolution: {integrity: sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==} - sanitize-html@2.17.6: - resolution: {integrity: sha512-M4bo9tfv1yfhQZZKkc6dL07ALrGJtfvNOuhX3hU9AVPR/uPQ+nKOJBqTYc7LfMQblTW04mtSWDJWEyLvygJsLA==} + sanitize-html@2.17.7: + resolution: {integrity: sha512-PGtEkc9cbnedU3s9TmzDbpsZ8w086g/0Q8k8/oIO1NLNU3i5k9yn835CrjJSajp1KMmkisbO1qPXxNKO3welAg==} engines: {node: '>=22.12.0'} scheduler@0.27.0: @@ -4236,127 +4242,128 @@ snapshots: '@standard-schema/spec@1.1.0': {} - '@tiptap/core@3.27.3(@tiptap/pm@3.27.3)': + '@tiptap/core@3.30.4(@tiptap/pm@3.30.4)': dependencies: - '@tiptap/pm': 3.27.3 + '@tiptap/pm': 3.30.4 - '@tiptap/extension-blockquote@3.27.3(@tiptap/core@3.27.3(@tiptap/pm@3.27.3))': + '@tiptap/extension-blockquote@3.30.4(@tiptap/core@3.30.4(@tiptap/pm@3.30.4))(@tiptap/pm@3.30.4)': dependencies: - '@tiptap/core': 3.27.3(@tiptap/pm@3.27.3) + '@tiptap/core': 3.30.4(@tiptap/pm@3.30.4) + '@tiptap/pm': 3.30.4 - '@tiptap/extension-bold@3.27.3(@tiptap/core@3.27.3(@tiptap/pm@3.27.3))': + '@tiptap/extension-bold@3.30.4(@tiptap/core@3.30.4(@tiptap/pm@3.30.4))': dependencies: - '@tiptap/core': 3.27.3(@tiptap/pm@3.27.3) + '@tiptap/core': 3.30.4(@tiptap/pm@3.30.4) - '@tiptap/extension-bubble-menu@3.27.3(@tiptap/core@3.27.3(@tiptap/pm@3.27.3))(@tiptap/pm@3.27.3)': + '@tiptap/extension-bubble-menu@3.30.4(@tiptap/core@3.30.4(@tiptap/pm@3.30.4))(@tiptap/pm@3.30.4)': dependencies: '@floating-ui/dom': 1.7.6 - '@tiptap/core': 3.27.3(@tiptap/pm@3.27.3) - '@tiptap/pm': 3.27.3 + '@tiptap/core': 3.30.4(@tiptap/pm@3.30.4) + '@tiptap/pm': 3.30.4 optional: true - '@tiptap/extension-bullet-list@3.27.3(@tiptap/extension-list@3.27.3(@tiptap/core@3.27.3(@tiptap/pm@3.27.3))(@tiptap/pm@3.27.3))': + '@tiptap/extension-bullet-list@3.30.4(@tiptap/extension-list@3.30.4(@tiptap/core@3.30.4(@tiptap/pm@3.30.4))(@tiptap/pm@3.30.4))': dependencies: - '@tiptap/extension-list': 3.27.3(@tiptap/core@3.27.3(@tiptap/pm@3.27.3))(@tiptap/pm@3.27.3) + '@tiptap/extension-list': 3.30.4(@tiptap/core@3.30.4(@tiptap/pm@3.30.4))(@tiptap/pm@3.30.4) - '@tiptap/extension-code-block@3.27.3(@tiptap/core@3.27.3(@tiptap/pm@3.27.3))(@tiptap/pm@3.27.3)': + '@tiptap/extension-code-block@3.30.4(@tiptap/core@3.30.4(@tiptap/pm@3.30.4))(@tiptap/pm@3.30.4)': dependencies: - '@tiptap/core': 3.27.3(@tiptap/pm@3.27.3) - '@tiptap/pm': 3.27.3 + '@tiptap/core': 3.30.4(@tiptap/pm@3.30.4) + '@tiptap/pm': 3.30.4 - '@tiptap/extension-code@3.27.3(@tiptap/core@3.27.3(@tiptap/pm@3.27.3))': + '@tiptap/extension-code@3.30.4(@tiptap/core@3.30.4(@tiptap/pm@3.30.4))': dependencies: - '@tiptap/core': 3.27.3(@tiptap/pm@3.27.3) + '@tiptap/core': 3.30.4(@tiptap/pm@3.30.4) - '@tiptap/extension-document@3.27.3(@tiptap/core@3.27.3(@tiptap/pm@3.27.3))': + '@tiptap/extension-document@3.30.4(@tiptap/core@3.30.4(@tiptap/pm@3.30.4))': dependencies: - '@tiptap/core': 3.27.3(@tiptap/pm@3.27.3) + '@tiptap/core': 3.30.4(@tiptap/pm@3.30.4) - '@tiptap/extension-dropcursor@3.27.3(@tiptap/extensions@3.27.3(@tiptap/core@3.27.3(@tiptap/pm@3.27.3))(@tiptap/pm@3.27.3))': + '@tiptap/extension-dropcursor@3.30.4(@tiptap/extensions@3.30.4(@tiptap/core@3.30.4(@tiptap/pm@3.30.4))(@tiptap/pm@3.30.4))': dependencies: - '@tiptap/extensions': 3.27.3(@tiptap/core@3.27.3(@tiptap/pm@3.27.3))(@tiptap/pm@3.27.3) + '@tiptap/extensions': 3.30.4(@tiptap/core@3.30.4(@tiptap/pm@3.30.4))(@tiptap/pm@3.30.4) - '@tiptap/extension-floating-menu@3.27.3(@floating-ui/dom@1.7.6)(@tiptap/core@3.27.3(@tiptap/pm@3.27.3))(@tiptap/pm@3.27.3)': + '@tiptap/extension-floating-menu@3.30.4(@floating-ui/dom@1.7.6)(@tiptap/core@3.30.4(@tiptap/pm@3.30.4))(@tiptap/pm@3.30.4)': dependencies: '@floating-ui/dom': 1.7.6 - '@tiptap/core': 3.27.3(@tiptap/pm@3.27.3) - '@tiptap/pm': 3.27.3 + '@tiptap/core': 3.30.4(@tiptap/pm@3.30.4) + '@tiptap/pm': 3.30.4 optional: true - '@tiptap/extension-gapcursor@3.27.3(@tiptap/extensions@3.27.3(@tiptap/core@3.27.3(@tiptap/pm@3.27.3))(@tiptap/pm@3.27.3))': + '@tiptap/extension-gapcursor@3.30.4(@tiptap/extensions@3.30.4(@tiptap/core@3.30.4(@tiptap/pm@3.30.4))(@tiptap/pm@3.30.4))': dependencies: - '@tiptap/extensions': 3.27.3(@tiptap/core@3.27.3(@tiptap/pm@3.27.3))(@tiptap/pm@3.27.3) + '@tiptap/extensions': 3.30.4(@tiptap/core@3.30.4(@tiptap/pm@3.30.4))(@tiptap/pm@3.30.4) - '@tiptap/extension-hard-break@3.27.3(@tiptap/core@3.27.3(@tiptap/pm@3.27.3))': + '@tiptap/extension-hard-break@3.30.4(@tiptap/core@3.30.4(@tiptap/pm@3.30.4))': dependencies: - '@tiptap/core': 3.27.3(@tiptap/pm@3.27.3) + '@tiptap/core': 3.30.4(@tiptap/pm@3.30.4) - '@tiptap/extension-heading@3.27.3(@tiptap/core@3.27.3(@tiptap/pm@3.27.3))': + '@tiptap/extension-heading@3.30.4(@tiptap/core@3.30.4(@tiptap/pm@3.30.4))': dependencies: - '@tiptap/core': 3.27.3(@tiptap/pm@3.27.3) + '@tiptap/core': 3.30.4(@tiptap/pm@3.30.4) - '@tiptap/extension-horizontal-rule@3.27.3(@tiptap/core@3.27.3(@tiptap/pm@3.27.3))(@tiptap/pm@3.27.3)': + '@tiptap/extension-horizontal-rule@3.30.4(@tiptap/core@3.30.4(@tiptap/pm@3.30.4))(@tiptap/pm@3.30.4)': dependencies: - '@tiptap/core': 3.27.3(@tiptap/pm@3.27.3) - '@tiptap/pm': 3.27.3 + '@tiptap/core': 3.30.4(@tiptap/pm@3.30.4) + '@tiptap/pm': 3.30.4 - '@tiptap/extension-image@3.27.3(@tiptap/core@3.27.3(@tiptap/pm@3.27.3))': + '@tiptap/extension-image@3.30.4(@tiptap/core@3.30.4(@tiptap/pm@3.30.4))': dependencies: - '@tiptap/core': 3.27.3(@tiptap/pm@3.27.3) + '@tiptap/core': 3.30.4(@tiptap/pm@3.30.4) - '@tiptap/extension-italic@3.27.3(@tiptap/core@3.27.3(@tiptap/pm@3.27.3))': + '@tiptap/extension-italic@3.30.4(@tiptap/core@3.30.4(@tiptap/pm@3.30.4))': dependencies: - '@tiptap/core': 3.27.3(@tiptap/pm@3.27.3) + '@tiptap/core': 3.30.4(@tiptap/pm@3.30.4) - '@tiptap/extension-link@3.27.3(@tiptap/core@3.27.3(@tiptap/pm@3.27.3))(@tiptap/pm@3.27.3)': + '@tiptap/extension-link@3.30.4(@tiptap/core@3.30.4(@tiptap/pm@3.30.4))(@tiptap/pm@3.30.4)': dependencies: - '@tiptap/core': 3.27.3(@tiptap/pm@3.27.3) - '@tiptap/pm': 3.27.3 + '@tiptap/core': 3.30.4(@tiptap/pm@3.30.4) + '@tiptap/pm': 3.30.4 linkifyjs: 4.3.3 - '@tiptap/extension-list-item@3.27.3(@tiptap/extension-list@3.27.3(@tiptap/core@3.27.3(@tiptap/pm@3.27.3))(@tiptap/pm@3.27.3))': + '@tiptap/extension-list-item@3.30.4(@tiptap/extension-list@3.30.4(@tiptap/core@3.30.4(@tiptap/pm@3.30.4))(@tiptap/pm@3.30.4))': dependencies: - '@tiptap/extension-list': 3.27.3(@tiptap/core@3.27.3(@tiptap/pm@3.27.3))(@tiptap/pm@3.27.3) + '@tiptap/extension-list': 3.30.4(@tiptap/core@3.30.4(@tiptap/pm@3.30.4))(@tiptap/pm@3.30.4) - '@tiptap/extension-list-keymap@3.27.3(@tiptap/extension-list@3.27.3(@tiptap/core@3.27.3(@tiptap/pm@3.27.3))(@tiptap/pm@3.27.3))': + '@tiptap/extension-list-keymap@3.30.4(@tiptap/extension-list@3.30.4(@tiptap/core@3.30.4(@tiptap/pm@3.30.4))(@tiptap/pm@3.30.4))': dependencies: - '@tiptap/extension-list': 3.27.3(@tiptap/core@3.27.3(@tiptap/pm@3.27.3))(@tiptap/pm@3.27.3) + '@tiptap/extension-list': 3.30.4(@tiptap/core@3.30.4(@tiptap/pm@3.30.4))(@tiptap/pm@3.30.4) - '@tiptap/extension-list@3.27.3(@tiptap/core@3.27.3(@tiptap/pm@3.27.3))(@tiptap/pm@3.27.3)': + '@tiptap/extension-list@3.30.4(@tiptap/core@3.30.4(@tiptap/pm@3.30.4))(@tiptap/pm@3.30.4)': dependencies: - '@tiptap/core': 3.27.3(@tiptap/pm@3.27.3) - '@tiptap/pm': 3.27.3 + '@tiptap/core': 3.30.4(@tiptap/pm@3.30.4) + '@tiptap/pm': 3.30.4 - '@tiptap/extension-ordered-list@3.27.3(@tiptap/extension-list@3.27.3(@tiptap/core@3.27.3(@tiptap/pm@3.27.3))(@tiptap/pm@3.27.3))': + '@tiptap/extension-ordered-list@3.30.4(@tiptap/extension-list@3.30.4(@tiptap/core@3.30.4(@tiptap/pm@3.30.4))(@tiptap/pm@3.30.4))': dependencies: - '@tiptap/extension-list': 3.27.3(@tiptap/core@3.27.3(@tiptap/pm@3.27.3))(@tiptap/pm@3.27.3) + '@tiptap/extension-list': 3.30.4(@tiptap/core@3.30.4(@tiptap/pm@3.30.4))(@tiptap/pm@3.30.4) - '@tiptap/extension-paragraph@3.27.3(@tiptap/core@3.27.3(@tiptap/pm@3.27.3))': + '@tiptap/extension-paragraph@3.30.4(@tiptap/core@3.30.4(@tiptap/pm@3.30.4))': dependencies: - '@tiptap/core': 3.27.3(@tiptap/pm@3.27.3) + '@tiptap/core': 3.30.4(@tiptap/pm@3.30.4) - '@tiptap/extension-placeholder@3.27.3(@tiptap/extensions@3.27.3(@tiptap/core@3.27.3(@tiptap/pm@3.27.3))(@tiptap/pm@3.27.3))': + '@tiptap/extension-placeholder@3.30.4(@tiptap/extensions@3.30.4(@tiptap/core@3.30.4(@tiptap/pm@3.30.4))(@tiptap/pm@3.30.4))': dependencies: - '@tiptap/extensions': 3.27.3(@tiptap/core@3.27.3(@tiptap/pm@3.27.3))(@tiptap/pm@3.27.3) + '@tiptap/extensions': 3.30.4(@tiptap/core@3.30.4(@tiptap/pm@3.30.4))(@tiptap/pm@3.30.4) - '@tiptap/extension-strike@3.27.3(@tiptap/core@3.27.3(@tiptap/pm@3.27.3))': + '@tiptap/extension-strike@3.30.4(@tiptap/core@3.30.4(@tiptap/pm@3.30.4))': dependencies: - '@tiptap/core': 3.27.3(@tiptap/pm@3.27.3) + '@tiptap/core': 3.30.4(@tiptap/pm@3.30.4) - '@tiptap/extension-text@3.27.3(@tiptap/core@3.27.3(@tiptap/pm@3.27.3))': + '@tiptap/extension-text@3.30.4(@tiptap/core@3.30.4(@tiptap/pm@3.30.4))': dependencies: - '@tiptap/core': 3.27.3(@tiptap/pm@3.27.3) + '@tiptap/core': 3.30.4(@tiptap/pm@3.30.4) - '@tiptap/extension-underline@3.27.3(@tiptap/core@3.27.3(@tiptap/pm@3.27.3))': + '@tiptap/extension-underline@3.30.4(@tiptap/core@3.30.4(@tiptap/pm@3.30.4))': dependencies: - '@tiptap/core': 3.27.3(@tiptap/pm@3.27.3) + '@tiptap/core': 3.30.4(@tiptap/pm@3.30.4) - '@tiptap/extensions@3.27.3(@tiptap/core@3.27.3(@tiptap/pm@3.27.3))(@tiptap/pm@3.27.3)': + '@tiptap/extensions@3.30.4(@tiptap/core@3.30.4(@tiptap/pm@3.30.4))(@tiptap/pm@3.30.4)': dependencies: - '@tiptap/core': 3.27.3(@tiptap/pm@3.27.3) - '@tiptap/pm': 3.27.3 + '@tiptap/core': 3.30.4(@tiptap/pm@3.30.4) + '@tiptap/pm': 3.30.4 - '@tiptap/pm@3.27.3': + '@tiptap/pm@3.30.4': dependencies: prosemirror-changeset: 2.4.1 prosemirror-commands: 1.7.2 @@ -4372,10 +4379,10 @@ snapshots: prosemirror-transform: 1.12.0 prosemirror-view: 1.42.2 - '@tiptap/react@3.27.3(@floating-ui/dom@1.7.6)(@tiptap/core@3.27.3(@tiptap/pm@3.27.3))(@tiptap/pm@3.27.3)(@types/react-dom@19.2.3(@types/react@19.2.17))(@types/react@19.2.17)(react-dom@19.2.7(react@19.2.7))(react@19.2.7)': + '@tiptap/react@3.30.4(@floating-ui/dom@1.7.6)(@tiptap/core@3.30.4(@tiptap/pm@3.30.4))(@tiptap/pm@3.30.4)(@types/react-dom@19.2.3(@types/react@19.2.17))(@types/react@19.2.17)(react-dom@19.2.7(react@19.2.7))(react@19.2.7)': dependencies: - '@tiptap/core': 3.27.3(@tiptap/pm@3.27.3) - '@tiptap/pm': 3.27.3 + '@tiptap/core': 3.30.4(@tiptap/pm@3.30.4) + '@tiptap/pm': 3.30.4 '@types/react': 19.2.17 '@types/react-dom': 19.2.3(@types/react@19.2.17) '@types/use-sync-external-store': 0.0.6 @@ -4384,37 +4391,37 @@ snapshots: react-dom: 19.2.7(react@19.2.7) use-sync-external-store: 1.6.0(react@19.2.7) optionalDependencies: - '@tiptap/extension-bubble-menu': 3.27.3(@tiptap/core@3.27.3(@tiptap/pm@3.27.3))(@tiptap/pm@3.27.3) - '@tiptap/extension-floating-menu': 3.27.3(@floating-ui/dom@1.7.6)(@tiptap/core@3.27.3(@tiptap/pm@3.27.3))(@tiptap/pm@3.27.3) + '@tiptap/extension-bubble-menu': 3.30.4(@tiptap/core@3.30.4(@tiptap/pm@3.30.4))(@tiptap/pm@3.30.4) + '@tiptap/extension-floating-menu': 3.30.4(@floating-ui/dom@1.7.6)(@tiptap/core@3.30.4(@tiptap/pm@3.30.4))(@tiptap/pm@3.30.4) transitivePeerDependencies: - '@floating-ui/dom' - '@tiptap/starter-kit@3.27.3': - dependencies: - '@tiptap/core': 3.27.3(@tiptap/pm@3.27.3) - '@tiptap/extension-blockquote': 3.27.3(@tiptap/core@3.27.3(@tiptap/pm@3.27.3)) - '@tiptap/extension-bold': 3.27.3(@tiptap/core@3.27.3(@tiptap/pm@3.27.3)) - '@tiptap/extension-bullet-list': 3.27.3(@tiptap/extension-list@3.27.3(@tiptap/core@3.27.3(@tiptap/pm@3.27.3))(@tiptap/pm@3.27.3)) - '@tiptap/extension-code': 3.27.3(@tiptap/core@3.27.3(@tiptap/pm@3.27.3)) - '@tiptap/extension-code-block': 3.27.3(@tiptap/core@3.27.3(@tiptap/pm@3.27.3))(@tiptap/pm@3.27.3) - '@tiptap/extension-document': 3.27.3(@tiptap/core@3.27.3(@tiptap/pm@3.27.3)) - '@tiptap/extension-dropcursor': 3.27.3(@tiptap/extensions@3.27.3(@tiptap/core@3.27.3(@tiptap/pm@3.27.3))(@tiptap/pm@3.27.3)) - '@tiptap/extension-gapcursor': 3.27.3(@tiptap/extensions@3.27.3(@tiptap/core@3.27.3(@tiptap/pm@3.27.3))(@tiptap/pm@3.27.3)) - '@tiptap/extension-hard-break': 3.27.3(@tiptap/core@3.27.3(@tiptap/pm@3.27.3)) - '@tiptap/extension-heading': 3.27.3(@tiptap/core@3.27.3(@tiptap/pm@3.27.3)) - '@tiptap/extension-horizontal-rule': 3.27.3(@tiptap/core@3.27.3(@tiptap/pm@3.27.3))(@tiptap/pm@3.27.3) - '@tiptap/extension-italic': 3.27.3(@tiptap/core@3.27.3(@tiptap/pm@3.27.3)) - '@tiptap/extension-link': 3.27.3(@tiptap/core@3.27.3(@tiptap/pm@3.27.3))(@tiptap/pm@3.27.3) - '@tiptap/extension-list': 3.27.3(@tiptap/core@3.27.3(@tiptap/pm@3.27.3))(@tiptap/pm@3.27.3) - '@tiptap/extension-list-item': 3.27.3(@tiptap/extension-list@3.27.3(@tiptap/core@3.27.3(@tiptap/pm@3.27.3))(@tiptap/pm@3.27.3)) - '@tiptap/extension-list-keymap': 3.27.3(@tiptap/extension-list@3.27.3(@tiptap/core@3.27.3(@tiptap/pm@3.27.3))(@tiptap/pm@3.27.3)) - '@tiptap/extension-ordered-list': 3.27.3(@tiptap/extension-list@3.27.3(@tiptap/core@3.27.3(@tiptap/pm@3.27.3))(@tiptap/pm@3.27.3)) - '@tiptap/extension-paragraph': 3.27.3(@tiptap/core@3.27.3(@tiptap/pm@3.27.3)) - '@tiptap/extension-strike': 3.27.3(@tiptap/core@3.27.3(@tiptap/pm@3.27.3)) - '@tiptap/extension-text': 3.27.3(@tiptap/core@3.27.3(@tiptap/pm@3.27.3)) - '@tiptap/extension-underline': 3.27.3(@tiptap/core@3.27.3(@tiptap/pm@3.27.3)) - '@tiptap/extensions': 3.27.3(@tiptap/core@3.27.3(@tiptap/pm@3.27.3))(@tiptap/pm@3.27.3) - '@tiptap/pm': 3.27.3 + '@tiptap/starter-kit@3.30.4': + dependencies: + '@tiptap/core': 3.30.4(@tiptap/pm@3.30.4) + '@tiptap/extension-blockquote': 3.30.4(@tiptap/core@3.30.4(@tiptap/pm@3.30.4))(@tiptap/pm@3.30.4) + '@tiptap/extension-bold': 3.30.4(@tiptap/core@3.30.4(@tiptap/pm@3.30.4)) + '@tiptap/extension-bullet-list': 3.30.4(@tiptap/extension-list@3.30.4(@tiptap/core@3.30.4(@tiptap/pm@3.30.4))(@tiptap/pm@3.30.4)) + '@tiptap/extension-code': 3.30.4(@tiptap/core@3.30.4(@tiptap/pm@3.30.4)) + '@tiptap/extension-code-block': 3.30.4(@tiptap/core@3.30.4(@tiptap/pm@3.30.4))(@tiptap/pm@3.30.4) + '@tiptap/extension-document': 3.30.4(@tiptap/core@3.30.4(@tiptap/pm@3.30.4)) + '@tiptap/extension-dropcursor': 3.30.4(@tiptap/extensions@3.30.4(@tiptap/core@3.30.4(@tiptap/pm@3.30.4))(@tiptap/pm@3.30.4)) + '@tiptap/extension-gapcursor': 3.30.4(@tiptap/extensions@3.30.4(@tiptap/core@3.30.4(@tiptap/pm@3.30.4))(@tiptap/pm@3.30.4)) + '@tiptap/extension-hard-break': 3.30.4(@tiptap/core@3.30.4(@tiptap/pm@3.30.4)) + '@tiptap/extension-heading': 3.30.4(@tiptap/core@3.30.4(@tiptap/pm@3.30.4)) + '@tiptap/extension-horizontal-rule': 3.30.4(@tiptap/core@3.30.4(@tiptap/pm@3.30.4))(@tiptap/pm@3.30.4) + '@tiptap/extension-italic': 3.30.4(@tiptap/core@3.30.4(@tiptap/pm@3.30.4)) + '@tiptap/extension-link': 3.30.4(@tiptap/core@3.30.4(@tiptap/pm@3.30.4))(@tiptap/pm@3.30.4) + '@tiptap/extension-list': 3.30.4(@tiptap/core@3.30.4(@tiptap/pm@3.30.4))(@tiptap/pm@3.30.4) + '@tiptap/extension-list-item': 3.30.4(@tiptap/extension-list@3.30.4(@tiptap/core@3.30.4(@tiptap/pm@3.30.4))(@tiptap/pm@3.30.4)) + '@tiptap/extension-list-keymap': 3.30.4(@tiptap/extension-list@3.30.4(@tiptap/core@3.30.4(@tiptap/pm@3.30.4))(@tiptap/pm@3.30.4)) + '@tiptap/extension-ordered-list': 3.30.4(@tiptap/extension-list@3.30.4(@tiptap/core@3.30.4(@tiptap/pm@3.30.4))(@tiptap/pm@3.30.4)) + '@tiptap/extension-paragraph': 3.30.4(@tiptap/core@3.30.4(@tiptap/pm@3.30.4)) + '@tiptap/extension-strike': 3.30.4(@tiptap/core@3.30.4(@tiptap/pm@3.30.4)) + '@tiptap/extension-text': 3.30.4(@tiptap/core@3.30.4(@tiptap/pm@3.30.4)) + '@tiptap/extension-underline': 3.30.4(@tiptap/core@3.30.4(@tiptap/pm@3.30.4)) + '@tiptap/extensions': 3.30.4(@tiptap/core@3.30.4(@tiptap/pm@3.30.4))(@tiptap/pm@3.30.4) + '@tiptap/pm': 3.30.4 '@types/babel__core@7.20.5': dependencies: @@ -4559,7 +4566,7 @@ snapshots: ajv@8.20.0: dependencies: fast-deep-equal: 3.1.3 - fast-uri: 3.1.4 + fast-uri: 3.1.6 json-schema-traverse: 1.0.0 require-from-string: 2.0.2 @@ -4653,7 +4660,7 @@ snapshots: http-errors: 2.0.1 iconv-lite: 0.7.3 on-finished: 2.4.1 - qs: 6.15.3 + qs: 6.16.0 raw-body: 3.0.2 type-is: 2.1.0 transitivePeerDependencies: @@ -4895,7 +4902,7 @@ snapshots: dependencies: debug: 4.4.3 express: 5.2.1 - ip-address: 10.2.0 + ip-address: 10.3.1 transitivePeerDependencies: - supports-color @@ -4921,7 +4928,7 @@ snapshots: once: 1.4.0 parseurl: 1.3.3 proxy-addr: 2.0.7 - qs: 6.15.3 + qs: 6.16.0 range-parser: 1.3.0 router: 2.2.0 send: 1.2.1 @@ -4944,7 +4951,7 @@ snapshots: merge2: 1.4.1 micromatch: 4.0.8 - fast-uri@3.1.4: {} + fast-uri@3.1.6: {} fastq@1.20.1: dependencies: @@ -5077,7 +5084,7 @@ snapshots: inherits@2.0.4: {} - ip-address@10.2.0: {} + ip-address@10.3.1: {} ipaddr.js@1.9.1: {} @@ -5218,7 +5225,7 @@ snapshots: object-assign: 4.1.1 thenify-all: 1.6.0 - nanoid@3.3.16: {} + nanoid@3.3.18: {} nanostores@1.4.0: {} @@ -5316,7 +5323,7 @@ snapshots: postcss@8.5.23: dependencies: - nanoid: 3.3.16 + nanoid: 3.3.18 picocolors: 1.1.1 source-map-js: 1.2.1 @@ -5399,7 +5406,7 @@ snapshots: forwarded: 0.2.0 ipaddr.js: 1.9.1 - qs@6.15.3: + qs@6.16.0: dependencies: es-define-property: 1.0.1 side-channel: 1.1.1 @@ -5527,7 +5534,7 @@ snapshots: safer-buffer@2.1.2: {} - sanitize-html@2.17.6: + sanitize-html@2.17.7: dependencies: deepmerge: 4.3.1 escape-string-regexp: 4.0.0 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index cd8f3358..baa43bbc 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -9,7 +9,12 @@ allowBuilds: overrides: '@better-auth/core': 1.7.0-rc.6 '@hono/node-server': 2.0.12 - fast-uri: 3.1.4 + '@tiptap/extension-bubble-menu': 3.30.4 + '@tiptap/extension-floating-menu': 3.30.4 + fast-uri: 3.1.6 + ip-address: 10.3.1 + nanoid@3: 3.3.18 + qs: 6.16.0 postcss: 8.5.23 minimumReleaseAgeExclude: diff --git a/scripts/hqbase/backup.mjs b/scripts/hqbase/backup.mjs index 96d316bd..d48a125d 100644 --- a/scripts/hqbase/backup.mjs +++ b/scripts/hqbase/backup.mjs @@ -4,6 +4,7 @@ import path from "node:path"; import { optionalString, requireString } from "./args.mjs"; import { run } from "./command.mjs"; import { configPath, deploymentDir, loadManifest } from "./manifest.mjs"; +import { inspectRelease } from "./recovery-verification.mjs"; function findString(value, keys) { if (!value || typeof value !== "object") return null; @@ -22,6 +23,16 @@ export function parseTimeTravelBookmark(output) { } export function parseWorkerVersion(output) { + const parsed = JSON.parse(output); + const deployment = parsed.result ?? parsed; + if ( + Array.isArray(deployment.versions) && + (deployment.versions.length !== 1 || + (deployment.versions[0]?.percentage !== undefined && + deployment.versions[0].percentage !== 100)) + ) { + throw new Error("Recovery requires one Worker version serving all traffic."); + } const version = findString(JSON.parse(output), new Set(["version_id", "versionId"])); if (!version) throw new Error("Worker deployment status did not return a version ID."); return version; @@ -30,6 +41,7 @@ export function parseWorkerVersion(output) { export function createBackup(name, options = {}) { const manifest = loadManifest(name); const config = configPath(name); + const release = inspectRelease(manifest); const bookmark = parseTimeTravelBookmark( run( "pnpm", @@ -44,7 +56,7 @@ export function createBackup(name, options = {}) { "--config", config ], - { quiet: true } + { quiet: true, stdoutOnly: true } ) ); const workerVersion = parseWorkerVersion( @@ -61,7 +73,7 @@ export function createBackup(name, options = {}) { "--config", config ], - { quiet: true } + { quiet: true, stdoutOnly: true } ) ); const r2 = JSON.parse( @@ -78,17 +90,19 @@ export function createBackup(name, options = {}) { "--config", config ], - { quiet: true } + { quiet: true, stdoutOnly: true } ) ); const createdAt = new Date().toISOString(); const backup = { - format: "hqbase-backup-v1", + format: "hqbase-backup-v2", deployment: name, createdAt, + accountId: manifest.accountId, + release, d1: { name: manifest.d1.name, id: manifest.d1.id, bookmark }, worker: { name: manifest.worker.name, version: workerVersion }, - r2: { bucket: manifest.r2.bucket, inventory: r2 } + r2: { bucket: manifest.r2.bucket, metadata: r2 } }; const output = options.output ? path.resolve(process.cwd(), options.output) diff --git a/scripts/hqbase/cli.mjs b/scripts/hqbase/cli.mjs index c2818319..6ed3f9ea 100644 --- a/scripts/hqbase/cli.mjs +++ b/scripts/hqbase/cli.mjs @@ -38,7 +38,7 @@ try { backup(flags); break; case "restore": - restore(flags); + await restore(flags); break; case "reset": reset(flags); diff --git a/scripts/hqbase/empty-r2.mjs b/scripts/hqbase/empty-r2.mjs index e93fdc45..9553b93e 100644 --- a/scripts/hqbase/empty-r2.mjs +++ b/scripts/hqbase/empty-r2.mjs @@ -74,7 +74,7 @@ export function authenticationHeaders(authentication) { throw new Error("Wrangler did not return usable Cloudflare authentication."); } -function readWranglerAuthentication(options) { +export function readWranglerAuthentication(options = {}) { const runAuthentication = options.runAuthentication ?? defaultAuthenticationCommand; let authentication; try { diff --git a/scripts/hqbase/recovery-verification.mjs b/scripts/hqbase/recovery-verification.mjs new file mode 100644 index 00000000..cd45e5a1 --- /dev/null +++ b/scripts/hqbase/recovery-verification.mjs @@ -0,0 +1,135 @@ +import { parseD1Rows } from "../release/after-deploy-state.mjs"; +import { run } from "./command.mjs"; +import { authenticationHeaders, readWranglerAuthentication } from "./empty-r2.mjs"; +import { configPath } from "./manifest.mjs"; + +export const releaseInspectionSql = `SELECT s.value AS schema_product, r.product, + r.installed_version, r.installed_schema_version, r.channel + FROM hqbase_schema_state s CROSS JOIN release_state r + WHERE s.key = 'product' AND r.singleton = 1`; + +export function inspectRelease(manifest) { + const rows = parseD1Rows( + run( + "pnpm", + [ + "exec", + "wrangler", + "d1", + "execute", + manifest.d1.name, + "--remote", + "--json", + "--command", + releaseInspectionSql, + "--config", + configPath(manifest.name) + ], + { quiet: true, stdoutOnly: true } + ) + ); + return validateRelease(rows); +} + +export function validateRelease(rows, expected) { + const row = rows?.[0]; + if ( + rows?.length !== 1 || + row?.schema_product !== "hqbase" || + row.product !== "hqbase" || + row.channel !== "stable" || + !/^\d+\.\d+\.\d+$/.test(row.installed_version) || + !Number.isInteger(row.installed_schema_version) || + row.installed_schema_version < 1 || + (expected && Object.keys(row).some((key) => row[key] !== expected[key])) + ) { + throw new Error("The database release state does not match the recovery checkpoint."); + } + return row; +} + +// Compare two ordered streams. Keep at most one page from each service in memory. +export async function verifyObjectReferences(manifest, options = {}) { + const headers = authenticationHeaders(options.authentication ?? readWranglerAuthentication()); + const request = options.fetchRequest ?? fetch; + const base = `https://api.cloudflare.com/client/v4/accounts/${manifest.accountId}`; + async function json(url, init) { + try { + const response = await request(url, { + ...init, + headers: { ...headers, "Content-Type": "application/json" } + }); + const payload = await response.json(); + if (!response.ok || payload.success !== true) throw new Error(); + return payload; + } catch { + throw new Error("Could not verify mail objects in the recorded Cloudflare resources."); + } + } + async function query(sql, params = []) { + const payload = await json(`${base}/d1/database/${manifest.d1.id}/query`, { + method: "POST", + body: JSON.stringify({ sql, params }) + }); + return parseD1Rows(JSON.stringify(payload)); + } + const columns = await query("PRAGMA table_info(messages)"); + const hasTextObjects = columns.some((row) => row.name === "text_r2_key"); + const tables = await query( + "SELECT name FROM sqlite_schema WHERE type = 'table' AND name = 'send_operations'" + ); + const referenceSql = `SELECT DISTINCT value AS key FROM messages, json_each(json_array(raw_r2_key, html_r2_key${hasTextObjects ? ", text_r2_key" : ""})) WHERE value IS NOT NULL + UNION SELECT r2_key AS key FROM message_attachments + UNION SELECT r2_key AS key FROM draft_attachments + ${tables.length ? "UNION SELECT value AS key FROM send_operations, json_each(object_keys_json) WHERE status <> 'stored'" : ""}`; + async function* references() { + let previous = ""; + while (true) { + const rows = await query( + `SELECT key FROM (${referenceSql}) WHERE key > ? ORDER BY key LIMIT 1000`, + [previous] + ); + for (const row of rows) { + if ( + typeof row.key !== "string" || + Buffer.compare(Buffer.from(row.key), Buffer.from(previous)) <= 0 + ) + throw new Error("Invalid object reference order."); + previous = row.key; + yield row.key; + } + if (rows.length < 1000) return; + } + } + async function* objects() { + let cursor = ""; + do { + const page = await json( + `${base}/r2/buckets/${manifest.r2.bucket}/objects?per_page=1000${cursor ? `&cursor=${encodeURIComponent(cursor)}` : ""}`, + { method: "GET" } + ); + if (!Array.isArray(page.result)) throw new Error("Invalid mail object inventory."); + for (const object of page.result) { + if (typeof object.key !== "string") throw new Error("Invalid mail object inventory."); + yield object.key; + } + const next = page.result_info?.is_truncated ? page.result_info.cursor : ""; + if (page.result_info?.is_truncated && (!next || next === cursor)) + throw new Error("Mail object inventory did not advance."); + cursor = next; + } while (cursor); + } + const inventory = objects(); + let current = await inventory.next(); + let count = 0; + for await (const key of references()) { + while (!current.done && Buffer.compare(Buffer.from(current.value), Buffer.from(key)) < 0) + current = await inventory.next(); + if (current.done || current.value !== key) + throw new Error( + "Recovery is incomplete: a referenced mail object is missing. Restore the independent R2 copy before retrying verification." + ); + count += 1; + } + return count; +} diff --git a/scripts/hqbase/reset-d1.sql b/scripts/hqbase/reset-d1.sql index fa4d656d..09b0d05c 100644 --- a/scripts/hqbase/reset-d1.sql +++ b/scripts/hqbase/reset-d1.sql @@ -1,3 +1,12 @@ +DROP TRIGGER IF EXISTS send_operations_require_draft; +DROP TRIGGER IF EXISTS drafts_before_update_pending_send; +DROP TRIGGER IF EXISTS drafts_before_delete_pending_send; +DROP TRIGGER IF EXISTS draft_attachments_before_update_pending_send; +DROP TRIGGER IF EXISTS draft_attachments_before_delete_pending_send; +DROP TRIGGER IF EXISTS draft_attachments_before_insert_pending_send; +DROP TRIGGER IF EXISTS draft_labels_before_update_pending_send; +DROP TRIGGER IF EXISTS draft_labels_before_delete_pending_send; +DROP TRIGGER IF EXISTS draft_labels_before_insert_pending_send; PRAGMA foreign_keys = OFF; DROP TRIGGER IF EXISTS mailbox_addresses_transition_insert_guard; @@ -51,6 +60,7 @@ DROP TABLE IF EXISTS update_history; DROP TABLE IF EXISTS release_state; DROP TABLE IF EXISTS installation_identity; DROP TABLE IF EXISTS draft_attachments; +DROP TABLE IF EXISTS send_operations; DROP TABLE IF EXISTS drafts; DROP TABLE IF EXISTS email_signatures; DROP TABLE IF EXISTS draft_attachments_agent_transition; diff --git a/scripts/hqbase/restore.mjs b/scripts/hqbase/restore.mjs index 314e97d4..23aac8c5 100644 --- a/scripts/hqbase/restore.mjs +++ b/scripts/hqbase/restore.mjs @@ -2,30 +2,78 @@ import fs from "node:fs"; import path from "node:path"; import { optionalBoolean, requireString } from "./args.mjs"; -import { createBackup } from "./backup.mjs"; +import { createBackup, parseTimeTravelBookmark, parseWorkerVersion } from "./backup.mjs"; import { run } from "./command.mjs"; import { configPath, loadManifest } from "./manifest.mjs"; +import { + inspectRelease, + validateRelease, + verifyObjectReferences +} from "./recovery-verification.mjs"; -export function validateBackupManifest(value, deployment) { +export function validateBackupManifest(value, deployment, manifest, now = Date.now()) { if ( - value?.format !== "hqbase-backup-v1" || + value?.format !== "hqbase-backup-v2" || value.deployment !== deployment || typeof value.d1?.bookmark !== "string" || typeof value.worker?.version !== "string" ) { throw new Error("Backup manifest is invalid or belongs to a different deployment."); } + if ( + !Number.isFinite(Date.parse(value.createdAt)) || + Date.parse(value.createdAt) > now + 60_000 || + now - Date.parse(value.createdAt) > 30 * 86_400_000 + ) { + throw new Error( + "The recovery checkpoint date is invalid or outside the maximum Time Travel window." + ); + } + if ( + manifest && + (value.accountId !== manifest.accountId || + value.d1.id !== manifest.d1.id || + value.d1.name !== manifest.d1.name || + value.worker.name !== manifest.worker.name || + value.r2?.bucket !== manifest.r2.bucket) + ) { + throw new Error("The checkpoint resource identities do not match this deployment."); + } + validateRelease([value.release]); return value; } -export function restore(flags) { +export async function restore(flags) { const name = requireString(flags, "name"); const backupPath = path.resolve(process.cwd(), requireString(flags, "backup")); if (!optionalBoolean(flags, "yes")) { throw new Error("Restore is destructive and requires --yes."); } const manifest = loadManifest(name); - const target = validateBackupManifest(JSON.parse(fs.readFileSync(backupPath, "utf8")), name); + const target = validateBackupManifest( + JSON.parse(fs.readFileSync(backupPath, "utf8")), + name, + manifest + ); + parseTimeTravelBookmark( + run( + "pnpm", + [ + "exec", + "wrangler", + "d1", + "time-travel", + "info", + manifest.d1.name, + "--timestamp", + target.createdAt, + "--json", + "--config", + configPath(name) + ], + { quiet: true, stdoutOnly: true } + ) + ); const safety = createBackup(name); console.log(`Pre-restore safety bookmark: ${safety.backup.d1.bookmark}`); run("pnpm", [ @@ -52,17 +100,28 @@ export function restore(flags) { "--config", configPath(name) ]); - run("pnpm", [ - "exec", - "wrangler", - "d1", - "execute", - manifest.d1.name, - "--remote", - "--command", - "SELECT value FROM hqbase_schema_state WHERE key = 'product'; SELECT product, installed_version, installed_schema_version FROM release_state WHERE singleton = 1;", - "--config", - configPath(name) - ]); - console.log(`Database and Worker version ${target.worker.version} restored and verified.`); + validateRelease([inspectRelease(manifest)], target.release); + const activeVersion = parseWorkerVersion( + run( + "pnpm", + [ + "exec", + "wrangler", + "deployments", + "status", + "--name", + manifest.worker.name, + "--json", + "--config", + configPath(name) + ], + { quiet: true, stdoutOnly: true } + ) + ); + if (activeVersion !== target.worker.version) + throw new Error("The active Worker version does not match the recovery checkpoint."); + const verifiedObjects = await verifyObjectReferences(manifest); + console.log( + `Database and Worker version ${target.worker.version} restored and verified (${verifiedObjects} mail objects).` + ); } diff --git a/scripts/release/after-deploy-state.mjs b/scripts/release/after-deploy-state.mjs index bf12eb79..1f776cb5 100644 --- a/scripts/release/after-deploy-state.mjs +++ b/scripts/release/after-deploy-state.mjs @@ -3,12 +3,27 @@ import { resolve } from "node:path"; import { attemptRun, emitCommandOutput } from "./command.mjs"; +export const pendingSendGuards = [ + "send_operations_require_draft", + "drafts_before_update_pending_send", + "drafts_before_delete_pending_send", + "draft_attachments_before_update_pending_send", + "draft_attachments_before_delete_pending_send", + "draft_attachments_before_insert_pending_send", + "draft_labels_before_update_pending_send", + "draft_labels_before_delete_pending_send", + "draft_labels_before_insert_pending_send" +]; + const afterDeployMigrations = [ "0001_remove_mailbox_alias_storage.sql", "0002_finalize_agent_principals.sql", - "0003_finalize_draft_labels.sql" + "0003_finalize_draft_labels.sql", + "0004_mail_reliability_guards.sql" ]; +export const finalAfterDeployPhase = `S${afterDeployMigrations.length}`; + const addressTransitionTables = ["mailbox_addresses", "mailbox_address_migration"]; const addressTransitionColumns = [ "messages.delivered_to_address_id", @@ -99,6 +114,7 @@ const inspectedTables = [ "d1_migrations_after_deploy" ]; const inspectedTriggers = [ + ...pendingSendGuards, ...requiredTriggers, ...addressTransitionTriggers, ...principalTransitionTriggers @@ -316,7 +332,8 @@ function expectedSchemaItems(appliedCount, hasAfterDeployLedger) { ...principalTransitionForeignKeys.map((name) => `foreign-key:${name}`) ); } - if (appliedCount === 3) items.push(`foreign-key:${draftLabelsForeignKey}`); + if (appliedCount >= 3) items.push(`foreign-key:${draftLabelsForeignKey}`); + if (appliedCount >= 4) items.push(...pendingSendGuards.map((name) => `trigger:${name}`)); return new Set(items); } diff --git a/scripts/release/deploy.mjs b/scripts/release/deploy.mjs index f0bdd828..529be7ae 100644 --- a/scripts/release/deploy.mjs +++ b/scripts/release/deploy.mjs @@ -8,7 +8,7 @@ import { applyMigrationPhase } from "../d1-migrations.mjs"; import { recordWorkerDeployedForConfig } from "../hqbase/manifest.mjs"; import { windowsSystem32Executable } from "../windows-system32.mjs"; import { assertRequiredActiveBindings, inspectActiveRelease } from "./active-version.mjs"; -import { inspectRemoteAfterDeployState } from "./after-deploy-state.mjs"; +import { finalAfterDeployPhase, inspectRemoteAfterDeployState } from "./after-deploy-state.mjs"; import { capture, run } from "./command.mjs"; import { compareVersions, @@ -255,20 +255,20 @@ export function completeActiveReleaseRetry(source, manifest, recordWorkerDeploye const afterDeployState = options.afterDeployState ?? (options.inspectAfterDeployState ?? inspectRemoteAfterDeployState)(source, manifest.version); - if (!["S0", "S1", "S2", "S3"].includes(afterDeployState?.phase)) { + if (!["S0", "S1", "S2", "S3", finalAfterDeployPhase].includes(afterDeployState?.phase)) { throw new Error("Refusing to repair HQBase because the D1 post-deploy state is invalid."); } let update = afterDeployState.pendingUpdate; - if (afterDeployState.phase === "S3" && !update) { - return { phase: "S3", repaired: false, workerRecorded: false }; + if (afterDeployState.phase === finalAfterDeployPhase && !update) { + return { phase: finalAfterDeployPhase, repaired: false, workerRecorded: false }; } let checkpoint; if (update) { checkpoint = { bookmark: update.checkpoint_bookmark, - cleanupComplete: afterDeployState.phase === "S3", + cleanupComplete: afterDeployState.phase === finalAfterDeployPhase, configFile: options.configFile, name: options.workerName, workerVersion: update.worker_version @@ -304,7 +304,7 @@ export function completeActiveReleaseRetry(source, manifest, recordWorkerDeploye ); } recordWorkerDeployed(); - if (afterDeployState.phase !== "S3") { + if (afterDeployState.phase !== finalAfterDeployPhase) { applyMigrations(source, "normal", { target: "remote" }); applyMigrations(source, "after-deploy", { target: "remote" }); checkpoint.cleanupComplete = true; @@ -315,7 +315,7 @@ export function completeActiveReleaseRetry(source, manifest, recordWorkerDeploye ); return { phase: afterDeployState.phase, - repaired: afterDeployState.phase !== "S3", + repaired: afterDeployState.phase !== finalAfterDeployPhase, workerRecorded: true }; } diff --git a/scripts/release/package.mjs b/scripts/release/package.mjs index c5873205..fb1c7b90 100644 --- a/scripts/release/package.mjs +++ b/scripts/release/package.mjs @@ -10,7 +10,7 @@ import { assertStableReleaseVersion } from "./version.mjs"; const root = resolve(import.meta.dirname, "../.."); const product = "hqbase"; -const schemaVersion = 3; +const schemaVersion = 4; const packageJson = JSON.parse(readFileSync(resolve(root, "package.json"), "utf8")); const version = process.env.HQBASE_RELEASE_VERSION ?? packageJson.version; const minVersion = process.env.HQBASE_MIN_VERSION || packageJson.hqbaseRelease?.minimumVersion; diff --git a/test/e2e/staging/lifecycle.spec.ts b/test/e2e/staging/lifecycle.spec.ts index cf6a196e..9db750e0 100644 --- a/test/e2e/staging/lifecycle.spec.ts +++ b/test/e2e/staging/lifecycle.spec.ts @@ -1001,8 +1001,12 @@ function accessHeaders(): Record { const clientId = process.env.HQBASE_STAGING_ACCESS_CLIENT_ID; const clientSecret = process.env.HQBASE_STAGING_ACCESS_CLIENT_SECRET; return clientId && clientSecret - ? { "CF-Access-Client-Id": clientId, "CF-Access-Client-Secret": clientSecret } - : {}; + ? { + origin: stagingUrl, + "CF-Access-Client-Id": clientId, + "CF-Access-Client-Secret": clientSecret + } + : { origin: stagingUrl }; } function stagingSetupGrantCookie(secret: string): string { diff --git a/test/integration/worker/agent-mailboxes.test.ts b/test/integration/worker/agent-mailboxes.test.ts index 66720aa2..1767901b 100644 --- a/test/integration/worker/agent-mailboxes.test.ts +++ b/test/integration/worker/agent-mailboxes.test.ts @@ -541,7 +541,7 @@ describe("agent mailboxes", () => { try { const failedEnable = await SELF.fetch(`${origin}/management/v1/agents/${created.agent.id}`, { body: JSON.stringify({ isActive: true }), - headers: { "content-type": "application/json", cookie: ownerCookie }, + headers: { "content-type": "application/json", cookie: ownerCookie, origin }, method: "PATCH" }); expect(failedEnable.status).toBe(500); @@ -591,7 +591,7 @@ async function requestJson( token?: string; } = {} ): Promise { - const headers = new Headers(); + const headers = new Headers({ origin }); if (options.body !== undefined) headers.set("content-type", "application/json"); if (options.cookie) headers.set("cookie", options.cookie); if (options.token) headers.set("authorization", `Bearer ${options.token}`); diff --git a/test/integration/worker/auth.test.ts b/test/integration/worker/auth.test.ts index 8c38ce17..3d5bf6c8 100644 --- a/test/integration/worker/auth.test.ts +++ b/test/integration/worker/auth.test.ts @@ -260,10 +260,7 @@ describe("Better Auth schema", () => { const updated = await SELF.fetch(`${origin}/api/me`, { body: JSON.stringify({ defaultFromMailboxId: "mailbox_preferences" }), - headers: { - "content-type": "application/json", - cookie - }, + headers: { origin, "content-type": "application/json", cookie }, method: "PATCH" }); expect(updated.status, await updated.clone().text()).toBe(200); @@ -272,7 +269,7 @@ describe("Better Auth schema", () => { defaultFromMailboxId: "mailbox_preferences" }); - const current = await SELF.fetch(`${origin}/api/me`, { headers: { cookie } }); + const current = await SELF.fetch(`${origin}/api/me`, { headers: { origin, cookie } }); expect(current.status, await current.clone().text()).toBe(200); await expect(current.json()).resolves.toMatchObject({ email, @@ -316,7 +313,7 @@ describe("Better Auth schema", () => { const staleSessionCookie = extractSessionCookie(signIn); const recent = await SELF.fetch(`${origin}/api/sessions/recent-authentication`, { - headers: { cookie: staleSessionCookie } + headers: { origin, cookie: staleSessionCookie } }); expect(await recent.json()).toEqual({ recent: true }); @@ -329,7 +326,7 @@ describe("Better Auth schema", () => { .run(); const stale = await SELF.fetch(`${origin}/api/sessions/recent-authentication`, { - headers: { cookie: staleSessionCookie } + headers: { origin, cookie: staleSessionCookie } }); expect(await stale.json()).toEqual({ recent: false }); @@ -365,7 +362,7 @@ describe("Better Auth schema", () => { const recentSessionCookie = extractSessionCookie(reauthenticated); const refreshed = await SELF.fetch(`${origin}/api/sessions/recent-authentication`, { - headers: { cookie: recentSessionCookie } + headers: { origin, cookie: recentSessionCookie } }); expect(await refreshed.json()).toEqual({ recent: true }); diff --git a/test/integration/worker/contacts.test.ts b/test/integration/worker/contacts.test.ts index 906eaee7..8f0af14e 100644 --- a/test/integration/worker/contacts.test.ts +++ b/test/integration/worker/contacts.test.ts @@ -323,6 +323,7 @@ async function createUser( function sessionFetch(path: string, cookie: string, init: RequestInit = {}): Promise { const headers = new Headers(init.headers); headers.set("cookie", cookie); + headers.set("origin", origin); return SELF.fetch(`${origin}${path}`, { ...init, headers }); } diff --git a/test/integration/worker/conversations.test.ts b/test/integration/worker/conversations.test.ts index a88c589b..ad2c87b0 100644 --- a/test/integration/worker/conversations.test.ts +++ b/test/integration/worker/conversations.test.ts @@ -6,6 +6,7 @@ import workspaceMigration from "../../../migrations/0002_workspace.sql?raw"; import oauthResourcesMigration from "../../../migrations/0003_oauth_resources.sql?raw"; import conversationMigration from "../../../migrations/0004_conversations.sql?raw"; import threadRebuildMigration from "../../../migrations/0005_rebuild_threads.sql?raw"; +import senderNameMigration from "../../../migrations/0023_message_sender_names.sql?raw"; import { listConversationPage, listConversations, @@ -62,6 +63,7 @@ describe("conversation persistence", () => { occurredAt: "2026-07-28T14:00:00.000Z" }); await applyMigration(threadRebuildMigration); + await applyMigration(senderNameMigration); }); it("repairs subject-only history from message headers", async () => { diff --git a/test/integration/worker/labels.test.ts b/test/integration/worker/labels.test.ts index 77913566..52fe05c2 100644 --- a/test/integration/worker/labels.test.ts +++ b/test/integration/worker/labels.test.ts @@ -396,6 +396,7 @@ async function createUser( function sessionFetch(path: string, cookie: string, init: RequestInit = {}): Promise { const headers = new Headers(init.headers); headers.set("cookie", cookie); + headers.set("origin", origin); return SELF.fetch(`${origin}${path}`, { ...init, headers }); } diff --git a/test/integration/worker/mail-api.test.ts b/test/integration/worker/mail-api.test.ts index 4e8eb9bd..79ffbfc8 100644 --- a/test/integration/worker/mail-api.test.ts +++ b/test/integration/worker/mail-api.test.ts @@ -3,6 +3,7 @@ import { beforeAll, describe, expect, it, vi } from "vitest"; import { createAuth } from "../../../worker/auth/auth"; import { draftAttachmentObjects } from "../../../worker/features/drafts/attachment-lookups"; import { mailEventInternalHeaders } from "../../../worker/features/events/durable-object"; +import { removeExpiredOrphanedObjects } from "../../../worker/jobs/consumer"; import { applyCurrentMigrations } from "./current-migrations"; import { tokenRow } from "./mail-api-token-fixture"; @@ -941,6 +942,7 @@ describe("HQBase Mail API", () => { r2_key: expect.stringMatching(/^sent\//u) }); expect(await env.MAIL_OBJECTS.get(sentInline?.r2_key ?? "missing")).not.toBeNull(); + await removeExpiredOrphanedObjects(env, Date.now() + 25 * 60 * 60 * 1000); expect(await env.MAIL_OBJECTS.get(objectKeys.get(attachment.id) ?? "missing")).toBeNull(); expect(await env.MAIL_OBJECTS.get(objectKeys.get(unusedAttachment.id) ?? "missing")).toBeNull(); expect(await apiFetch(`/api/v2/drafts/${draft.id}`, fullToken)).toMatchObject({ status: 404 }); diff --git a/test/integration/worker/mail-reliability.test.ts b/test/integration/worker/mail-reliability.test.ts new file mode 100644 index 00000000..236e5ba2 --- /dev/null +++ b/test/integration/worker/mail-reliability.test.ts @@ -0,0 +1,520 @@ +import { createExecutionContext, env, SELF, waitOnExecutionContext } from "cloudflare:test"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { createAuth } from "../../../worker/auth/auth"; +import type { ParsedEmail } from "../../../worker/email/parse-email"; +import { parseRawEmail } from "../../../worker/email/parse-email"; +import { storeInboundEmail } from "../../../worker/email/store-email"; +import { getDraft, saveDraft } from "../../../worker/features/drafts/queries"; +import { labelsForThreadIds } from "../../../worker/features/labels/queries"; +import { getMessageDetail } from "../../../worker/features/messages/queries"; +import { sendNewMessage } from "../../../worker/features/send/service"; +import { consumeJobs, removeExpiredOrphanedObjects } from "../../../worker/jobs/consumer"; +import { applyRetention } from "../../../worker/jobs/maintenance"; +import type { Job } from "../../../worker/jobs/types"; +import type { WorkerEnv } from "../../../worker/lib/env"; +import { apiRoutes } from "../../../worker/routes"; +import { applyCurrentMigrations } from "./current-migrations"; + +const origin = "https://hqbase.test"; +let userId: string; +let cookie: string; +const stamp = "2026-09-04T00:00:00.000Z"; +const draftInput = { + mailboxId: "mbx_audit", + replyToMessageId: null, + forwardOfMessageId: null, + from: "audit@example.com", + to: ["recipient@example.net"], + cc: [], + bcc: [], + subject: "Audit fixture", + text: "Synthetic audit text", + html: "" +}; +function incoming(filename = "fixture.txt"): { + envelopeRecipient: string; + mailboxId: string; + raw: ArrayBuffer; + parsed: ParsedEmail; +} { + return { + envelopeRecipient: "audit@example.com", + mailboxId: "mbx_audit", + raw: new TextEncoder().encode("Synthetic audit raw data").buffer, + parsed: { + fromAddress: "sender@example.net", + fromName: null, + to: ["audit@example.com"], + cc: [], + bcc: [], + subject: "Audit inbound", + date: stamp, + messageId: "", + inReplyTo: null, + references: [], + textBody: "Synthetic audit text", + htmlBody: "

Synthetic audit text

", + snippet: "Audit", + attachments: [ + { + filename, + contentType: "text/plain", + contentId: null, + disposition: "attachment", + content: new TextEncoder().encode("fixture") + } + ] + } + }; +} +function queueMessage(job: Job) { + return { id: "queue-audit", body: job, ack: vi.fn(), retry: vi.fn() }; +} +async function consume(message: ReturnType, bindings: WorkerEnv) { + await consumeJobs({ messages: [message] } as unknown as MessageBatch, bindings); +} + +describe("Mail integrity and concurrency regressions", () => { + beforeEach(async () => { + await applyCurrentMigrations(); + await env.DB.batch( + [ + "send_operations", + "operation_runs", + "messages", + "threads", + "drafts", + "mailboxes", + "mail_domains", + "user" + ].map((table) => env.DB.prepare(`DELETE FROM "${table}"`)) + ); + const response = await createAuth(env, new Request(origin)).handler( + new Request(`${origin}/api/auth/sign-up/email`, { + method: "POST", + headers: { "content-type": "application/json", origin }, + body: JSON.stringify({ + email: "audit@login.example", + name: "Audit fixture", + password: "audit-test-password-123", + rememberMe: false + }) + }) + ); + expect(response.status).toBe(200); + userId = (await response.json<{ user: { id: string } }>()).user.id; + cookie = + (response.headers.get("set-cookie") ?? "").match( + /(?:^|,\s*)((?:__Secure-)?better-auth\.session_token=[^;,]+)/ + )?.[1] ?? ""; + expect(cookie).not.toBe(""); + await env.DB.batch([ + env.DB.prepare(`UPDATE "user" SET role = 'owner' WHERE id = ?`).bind(userId), + env.DB.prepare( + `INSERT INTO mail_domains (id, name, receiving_status, sending_status, dns_status, is_enabled, created_at, updated_at) VALUES ('dom_audit', 'example.com', 'ready', 'ready', 'ready', 1, ?, ?)` + ).bind(stamp, stamp), + env.DB.prepare( + `INSERT INTO mailboxes (id, address, mail_domain_id, display_name, is_active, created_at, updated_at) VALUES ('mbx_audit', 'audit@example.com', 'dom_audit', 'Audit', 1, ?, ?)` + ).bind(stamp, stamp) + ]); + }); + + it("rejects one of two concurrent saves at the same draft version", async () => { + const initial = await saveDraft(env.DB, userId, draftInput); + const results = await Promise.allSettled([ + saveDraft(env.DB, userId, { + ...draftInput, + id: initial.id, + version: initial.version, + text: "First edit" + }), + saveDraft(env.DB, userId, { + ...draftInput, + id: initial.id, + version: initial.version, + text: "Second edit" + }) + ]); + expect(results.map((result) => result.status).sort()).toEqual(["fulfilled", "rejected"]); + expect((await getDraft(env.DB, userId, initial.id))?.version).toBe(2); + }); + + it("retries the whole inbound commit after attachment storage fails", async () => { + const input = incoming(); + const bucket = { + put: vi.fn(async (key: string, body: unknown, options: unknown) => { + if (key.includes("/attachments/")) throw new Error("Injected attachment storage failure"); + return env.MAIL_OBJECTS.put(key, body as never, options as never); + }) + } as unknown as R2Bucket; + await expect(storeInboundEmail(env.DB, bucket, input)).rejects.toThrow("Injected"); + const retry = await storeInboundEmail(env.DB, env.MAIL_OBJECTS, input); + expect(retry.inserted).toBe(true); + expect( + await env.DB.prepare("SELECT COUNT(*) AS count FROM message_attachments").first("count") + ).toBe(1); + expect(await env.DB.prepare("SELECT COUNT(*) AS count FROM messages").first("count")).toBe(1); + }); + + it("recovers an accepted send without sending again after D1 storage fails", async () => { + const draft = await saveDraft(env.DB, userId, draftInput); + const send = vi.fn().mockResolvedValue({ messageId: "provider-accepted" }); + const failingDb = new Proxy(env.DB, { + get(target, property) { + if (property === "prepare") + return (query: string) => { + if (/insert into "threads"/i.test(query)) + throw new Error("Injected sent-record failure"); + return target.prepare(query); + }; + const value = Reflect.get(target, property); + return typeof value === "function" ? value.bind(target) : value; + } + }); + const bindings = { ...env, DB: failingDb, MAIL_SENDER: { send } } as unknown as WorkerEnv; + const input = { ...draftInput, draftId: draft.id, attachmentIds: [] }; + await expect(sendNewMessage(bindings, input, userId)).rejects.toThrow("accepted"); + expect(await getDraft(env.DB, userId, draft.id)).not.toBeNull(); + await sendNewMessage({ ...bindings, DB: env.DB }, input, userId); + expect(send).toHaveBeenCalledOnce(); + expect( + await env.DB.prepare( + "SELECT COUNT(*) AS count FROM messages WHERE direction = 'outbound'" + ).first("count") + ).toBe(1); + }); + + it("runs failed jobs again instead of acknowledging a failed record", async () => { + const list = vi + .fn() + .mockRejectedValueOnce(new Error("Injected list failure")) + .mockResolvedValue({ objects: [], truncated: false }); + const bindings = { ...env, MAIL_OBJECTS: { list } } as unknown as WorkerEnv; + const job: Job = { id: "integrity:retry", kind: "integrity-scan", requestedAt: stamp }; + const first = queueMessage(job); + await consume(first, bindings); + expect(first.retry).toHaveBeenCalledOnce(); + const retry = queueMessage(job); + await consume(retry, bindings); + expect(retry.ack).toHaveBeenCalledOnce(); + expect(list).toHaveBeenCalledTimes(2); + expect( + await env.DB.prepare("SELECT status FROM operation_runs WHERE id = ?") + .bind(job.id) + .first("status") + ).toBe("succeeded"); + }); + + it("resumes a saved scan cursor when queue continuation publication fails", async () => { + const job: Job = { id: "scan-continuation", kind: "integrity-scan", requestedAt: stamp }; + const list = vi + .fn() + .mockResolvedValueOnce({ objects: [], truncated: true, cursor: "second-page" }) + .mockResolvedValueOnce({ objects: [], truncated: false }); + const send = vi.fn().mockRejectedValueOnce(new Error("Queue unavailable")); + const bindings = { + ...env, + MAIL_OBJECTS: { list }, + HQBASE_JOBS: { send } + } as unknown as WorkerEnv; + const first = queueMessage(job); + await consume(first, bindings); + expect(first.retry).toHaveBeenCalledOnce(); + const retry = queueMessage(job); + await consume(retry, bindings); + expect(list).toHaveBeenLastCalledWith({ cursor: "second-page", limit: 1000 }); + expect(retry.ack).toHaveBeenCalledOnce(); + const duplicate = queueMessage(job); + await consume(duplicate, bindings); + expect(duplicate.ack).toHaveBeenCalledOnce(); + expect(list).toHaveBeenCalledTimes(2); + }); + + it("does not let a duplicate delivery take an active job lease", async () => { + const job: Job = { id: "scan-lease", kind: "integrity-scan", requestedAt: stamp }; + let start = () => {}; + let finish = () => {}; + const started = new Promise((resolve) => { + start = resolve; + }); + const pending = new Promise((resolve) => { + finish = resolve; + }); + const list = vi.fn(async () => { + start(); + await pending; + return { objects: [], truncated: false }; + }); + const bindings = { ...env, MAIL_OBJECTS: { list } } as unknown as WorkerEnv; + const first = queueMessage(job); + const active = consume(first, bindings); + await started; + const duplicate = queueMessage(job); + await consume(duplicate, bindings); + expect(duplicate.retry).toHaveBeenCalledWith({ delaySeconds: 60 }); + finish(); + await active; + expect(first.ack).toHaveBeenCalledOnce(); + expect(list).toHaveBeenCalledOnce(); + }); + + it("stores an accepted send when only the independent receipt write fails", async () => { + const draft = await saveDraft(env.DB, userId, draftInput); + const bucket = new Proxy(env.MAIL_OBJECTS, { + get(target, property) { + if (property === "put") + return (key: string, value: unknown, options: unknown) => { + if (key.endsWith("/receipt.json")) + return Promise.reject(new Error("Receipt unavailable")); + return target.put(key, value as never, options as never); + }; + const value = Reflect.get(target, property); + return typeof value === "function" ? value.bind(target) : value; + } + }); + const send = vi.fn().mockResolvedValue({ messageId: "provider-receipt-fallback" }); + const bindings = { + ...env, + MAIL_OBJECTS: bucket, + MAIL_SENDER: { send } + } as unknown as WorkerEnv; + const input = { ...draftInput, draftId: draft.id, attachmentIds: [] }; + const result = await sendNewMessage(bindings, input, userId); + expect((await sendNewMessage(bindings, input, userId)).id).toBe(result.id); + expect(send).toHaveBeenCalledOnce(); + }); + + it("waits for draft protection before sending during an update", async () => { + const trigger = await env.DB.prepare( + "SELECT sql FROM sqlite_schema WHERE name = 'drafts_before_update_pending_send'" + ).first("sql"); + if (!trigger) throw new Error("Test guard is missing"); + await env.DB.prepare("DROP TRIGGER drafts_before_update_pending_send").run(); + const send = vi.fn(); + try { + await expect( + sendNewMessage( + { ...env, MAIL_SENDER: { send } } as unknown as WorkerEnv, + { + from: draftInput.from, + to: draftInput.to, + cc: [], + bcc: [], + subject: "Update fixture", + text: "Synthetic body", + attachmentIds: [], + idempotencyKey: "during-update" + }, + userId + ) + ).rejects.toMatchObject({ code: "SEND_STORAGE_NOT_READY" }); + expect(send).not.toHaveBeenCalled(); + } finally { + await env.DB.prepare(trigger).run(); + } + }); + + it("does not reproduce a Worker error for a Unicode filename", async () => { + const stored = await storeInboundEmail(env.DB, env.MAIL_OBJECTS, incoming("报告.txt")); + const attachment = await env.DB.prepare( + "SELECT id FROM message_attachments WHERE message_id = ?" + ) + .bind(stored.message.id) + .first<{ id: string }>(); + const response = await SELF.fetch(`${origin}/api/v2/attachments/${attachment?.id}`, { + headers: { cookie } + }); + expect(response.status).toBe(200); + expect(await response.text()).toBe("fixture"); + }); + + it("tests whether alternate signup paths reach the disabled endpoint", async () => { + const results: Array<{ path: string; status: number }> = []; + for (const [index, path] of [ + "/api/auth/sign-up/email/", + "/api/auth//sign-up/email", + "/api/auth/%73ign-up/email", + "/api/auth/sign-up%2Femail" + ].entries()) { + const response = await SELF.fetch(`${origin}${path}`, { + method: "POST", + headers: { "content-type": "application/json", origin }, + body: JSON.stringify({ + email: `audit-variant-${index}@login.example`, + name: "Audit fixture", + password: "audit-test-password-123" + }) + }); + results.push({ path, status: response.status }); + } + expect(results.every((result) => result.status !== 200)).toBe(true); + }); + + it("keeps one active owner when owner demotions run together", async () => { + await env.DB.prepare(`INSERT INTO "user" (id, name, email, emailVerified, role, createdAt, updatedAt) + VALUES ('audit_second_owner', 'Second', 'second@audit.example', 1, 'owner', ?, ?)`) + .bind(stamp, stamp) + .run(); + const contexts = [createExecutionContext(), createExecutionContext()]; + const responses = await Promise.all( + [userId, "audit_second_owner"].map((id, index) => + apiRoutes.fetch( + new Request(`${origin}/api/users/${id}`, { + method: "PATCH", + headers: { cookie, origin, "content-type": "application/json" }, + body: JSON.stringify({ role: "member" }) + }), + env, + contexts[index] + ) + ) + ); + await Promise.all(contexts.map((context) => waitOnExecutionContext(context))); + expect(responses.filter((response) => response.status === 200)).toHaveLength(1); + expect( + await env.DB.prepare( + `SELECT COUNT(*) AS count FROM "user" WHERE role = 'owner' AND COALESCE(banned, 0) = 0` + ).first("count") + ).toBe(1); + }); + + it("rejects session writes from sibling origins and non-JSON media types", async () => { + for (const [requestOrigin, contentType, status] of [ + ["https://untrusted.hqbase.test", "application/json", 403], + ["https://untrusted.hqbase.test", "text/plain", 403], + [origin, "text/plain", 415], + ["null", "application/json", 403] + ] as const) { + const response = await SELF.fetch(`${origin}/api/v2/drafts`, { + method: "POST", + headers: { cookie, origin: requestOrigin, "content-type": contentType }, + body: JSON.stringify(draftInput) + }); + expect(response.status).toBe(status); + } + expect(await env.DB.prepare("SELECT COUNT(*) AS count FROM drafts").first("count")).toBe(0); + }); + + it("applies the 30-day Trash default without a policy row", async () => { + const stored = await storeInboundEmail(env.DB, env.MAIL_OBJECTS, incoming()); + await env.DB.prepare( + "UPDATE messages SET folder = 'trash', trashed_at = '2000-01-01T00:00:00.000Z', created_at = '2000-01-01T00:00:00.000Z' WHERE id = ?" + ) + .bind(stored.message.id) + .run(); + expect( + await env.DB.prepare("SELECT COUNT(*) AS count FROM retention_policies").first("count") + ).toBe(0); + await applyRetention(env, stamp); + expect( + await env.DB.prepare("SELECT id FROM messages WHERE id = ?") + .bind(stored.message.id) + .first("id") + ).toBeNull(); + }); + + it("uses two bound sets for 100 threads and one mailbox", async () => { + let largestBind = 0; + const database = new Proxy(env.DB, { + get(target, property) { + if (property === "prepare") + return (query: string) => { + const statement = target.prepare(query); + return new Proxy(statement, { + get(stmt, key) { + if (key === "bind") + return (...values: unknown[]) => { + largestBind = Math.max(largestBind, values.length); + return stmt.bind(...values); + }; + const value = Reflect.get(stmt, key); + return typeof value === "function" ? value.bind(stmt) : value; + } + }); + }; + const value = Reflect.get(target, property); + return typeof value === "function" ? value.bind(target) : value; + } + }); + await labelsForThreadIds( + database, + Array.from({ length: 100 }, (_, i) => `thread_${i}`), + { mailboxIds: ["mbx_audit"], includeUnassigned: true } + ).catch(() => undefined); + expect(largestBind).toBe(2); + }); + + it("uses one D1 query for a bounded R2 page", async () => { + let queries = 0; + const database = new Proxy(env.DB, { + get(target, property) { + if (property === "prepare") + return (query: string) => { + queries += 1; + return target.prepare(query); + }; + const value = Reflect.get(target, property); + return typeof value === "function" ? value.bind(target) : value; + } + }); + const list = vi + .fn() + .mockResolvedValueOnce({ + objects: Array.from({ length: 1000 }, (_, i) => ({ + key: `old-${i}`, + uploaded: new Date("2000-01-01") + })), + truncated: true, + cursor: "next" + }) + .mockResolvedValueOnce({ + objects: [{ key: "old-1000", uploaded: new Date("2000-01-01") }], + truncated: false + }); + const bindings = { + ...env, + DB: database, + MAIL_OBJECTS: { list, delete: vi.fn().mockResolvedValue(undefined) } + } as unknown as WorkerEnv; + await removeExpiredOrphanedObjects(bindings); + expect(queries).toBe(1); + expect(list).toHaveBeenCalledOnce(); + }); + it("preserves a large plain-text body and Reply-To while bounding D1 search text", async () => { + const text = "abcd😀".repeat(500_000); + const parsed = await parseRawEmail( + new TextEncoder().encode( + "From: sender@example.net\r\nReply-To: customer@example.net\r\nTo: audit@example.com\r\nSubject: Large text\r\nContent-Type: text/plain; charset=utf-8\r\n\r\n" + + text + ).buffer + ); + expect(parsed.replyTo).toEqual(["customer@example.net"]); + const result = await storeInboundEmail(env.DB, env.MAIL_OBJECTS, { ...incoming(), parsed }); + const row = await env.DB.prepare("SELECT text_body, text_r2_key FROM messages WHERE id = ?") + .bind(result.message.id) + .first<{ text_body: string; text_r2_key: string }>(); + expect(new TextEncoder().encode(row?.text_body).byteLength).toBeLessThanOrEqual(256 * 1024); + expect(row?.text_r2_key).toBeTruthy(); + const stored = await getMessageDetail(env.DB, result.message.id, env.MAIL_OBJECTS); + expect(stored?.textBody).toBe(parsed.textBody); + expect(stored?.replyTo).toEqual(["customer@example.net"]); + }); + + it("blocks a second provider call after an uncertain outcome and locks the draft", async () => { + const draft = await saveDraft(env.DB, userId, draftInput); + const send = vi.fn().mockRejectedValue(new Error("Provider timeout")); + const bindings = { ...env, MAIL_SENDER: { send } } as unknown as WorkerEnv; + const input = { ...draftInput, draftId: draft.id, attachmentIds: [] }; + await expect(sendNewMessage(bindings, input, userId)).rejects.toThrow("uncertain"); + await expect(sendNewMessage(bindings, input, userId)).rejects.toThrow("uncertain"); + expect(send).toHaveBeenCalledOnce(); + await expect( + saveDraft(env.DB, userId, { + ...draftInput, + id: draft.id, + version: draft.version, + text: "Changed" + }) + ).rejects.toThrow(); + }); +}); diff --git a/test/integration/worker/oauth-connections.test.ts b/test/integration/worker/oauth-connections.test.ts index b4b9c0b1..2da19856 100644 --- a/test/integration/worker/oauth-connections.test.ts +++ b/test/integration/worker/oauth-connections.test.ts @@ -69,7 +69,7 @@ describe("OAuth connections", () => { it("lists and fully revokes only the signed-in person's connection", async () => { const list = await SELF.fetch(`${origin}/api/oauth-connections`, { - headers: { cookie: first.cookie } + headers: { origin, cookie: first.cookie } }); expect(list.status, await list.clone().text()).toBe(200); await expect(list.json()).resolves.toEqual({ @@ -84,7 +84,7 @@ describe("OAuth connections", () => { }); const revoked = await SELF.fetch(`${origin}/api/oauth-connections/client_connections`, { - headers: { cookie: first.cookie }, + headers: { origin, cookie: first.cookie }, method: "DELETE" }); expect(revoked.status, await revoked.clone().text()).toBe(204); diff --git a/test/integration/worker/signature-routes.test.ts b/test/integration/worker/signature-routes.test.ts index 4a2f9a4c..c54d2f5f 100644 --- a/test/integration/worker/signature-routes.test.ts +++ b/test/integration/worker/signature-routes.test.ts @@ -366,6 +366,7 @@ async function createUser( function sessionFetch(path: string, cookie: string, init: RequestInit = {}): Promise { const headers = new Headers(init.headers); headers.set("cookie", cookie); + headers.set("origin", origin); return SELF.fetch(`${origin}${path}`, { ...init, headers }); } diff --git a/test/integration/worker/users.test.ts b/test/integration/worker/users.test.ts index 1502b01a..51afe2a3 100644 --- a/test/integration/worker/users.test.ts +++ b/test/integration/worker/users.test.ts @@ -67,11 +67,13 @@ describe("workspace user onboarding", () => { expect(account?.password).not.toBe(result.temporaryPassword); const memberCookie = await signIn("direct-user@gmail.com", result.temporaryPassword); - const meBefore = await SELF.fetch(`${origin}/api/me`, { headers: { cookie: memberCookie } }); + const meBefore = await SELF.fetch(`${origin}/api/me`, { + headers: { origin, cookie: memberCookie } + }); await expect(meBefore.json()).resolves.toMatchObject({ passwordSetupRequired: true }); const blocked = await SELF.fetch(`${origin}/api/mailboxes`, { - headers: { cookie: memberCookie } + headers: { origin, cookie: memberCookie } }); expect(blocked.status).toBe(403); await expect(blocked.json()).resolves.toMatchObject({ @@ -90,10 +92,12 @@ describe("workspace user onboarding", () => { expect(changed.status, await changed.clone().text()).toBe(200); const refreshedCookie = extractSessionCookie(changed); - const meAfter = await SELF.fetch(`${origin}/api/me`, { headers: { cookie: refreshedCookie } }); + const meAfter = await SELF.fetch(`${origin}/api/me`, { + headers: { origin, cookie: refreshedCookie } + }); await expect(meAfter.json()).resolves.toMatchObject({ passwordSetupRequired: false }); const allowed = await SELF.fetch(`${origin}/api/mailboxes`, { - headers: { cookie: refreshedCookie } + headers: { origin, cookie: refreshedCookie } }); expect(allowed.status, await allowed.clone().text()).toBe(200); }); @@ -159,7 +163,7 @@ describe("workspace user onboarding", () => { const regenerated = await SELF.fetch( `${origin}/api/users/${initial.user.id}/temporary-password`, - { headers: { cookie: ownerCookie }, method: "POST" } + { headers: { origin, cookie: ownerCookie }, method: "POST" } ); expect(regenerated.status, await regenerated.clone().text()).toBe(200); const next = (await regenerated.json()) as { temporaryPassword: string }; @@ -207,7 +211,7 @@ describe("workspace user onboarding", () => { expect(invitationLifetimeMs).toBeLessThanOrEqual(7 * 24 * 60 * 60 * 1000); const resent = await SELF.fetch(`${origin}/api/users/${result.user.id}/resend-invitation`, { - headers: { cookie: ownerCookie }, + headers: { origin, cookie: ownerCookie }, method: "POST" }); expect(resent.status, await resent.clone().text()).toBe(200); @@ -338,7 +342,9 @@ describe("workspace user onboarding", () => { .bind(result.user.id) .first<{ status: string }>(); expect(onboarding?.status).toBe("complete"); - const revoked = await SELF.fetch(`${origin}/api/me`, { headers: { cookie: temporaryCookie } }); + const revoked = await SELF.fetch(`${origin}/api/me`, { + headers: { origin, cookie: temporaryCookie } + }); expect(revoked.status).toBe(401); await expect( signIn("pending-recovery-user@gmail.com", result.temporaryPassword) @@ -446,7 +452,9 @@ describe("workspace user onboarding", () => { }); expect(reset.status, await reset.clone().text()).toBe(200); - const revoked = await SELF.fetch(`${origin}/api/me`, { headers: { cookie: activeCookie } }); + const revoked = await SELF.fetch(`${origin}/api/me`, { + headers: { origin, cookie: activeCookie } + }); expect(revoked.status).toBe(401); await expect( signIn("recovery-user@gmail.com", "first-recovery-password-123") @@ -539,7 +547,7 @@ describe("workspace user onboarding", () => { ]); const removed = await SELF.fetch(`${origin}/api/users/${result.user.id}`, { - headers: { cookie: ownerCookie }, + headers: { origin, cookie: ownerCookie }, method: "DELETE" }); expect(removed.status, await removed.clone().text()).toBe(204); @@ -575,19 +583,19 @@ describe("workspace user onboarding", () => { }); await expect(signIn("removed-user@gmail.com", result.temporaryPassword)).rejects.toThrow(); const revokedSession = await SELF.fetch(`${origin}/api/me`, { - headers: { cookie: userCookie } + headers: { origin, cookie: userCookie } }); expect(revokedSession.status).toBe(401); const listed = await SELF.fetch(`${origin}/api/users`, { - headers: { cookie: ownerCookie } + headers: { origin, cookie: ownerCookie } }); const users = (await listed.json()) as Array<{ banned: boolean; id: string }>; expect(users).toContainEqual(expect.objectContaining({ banned: true, id: result.user.id })); const roleChange = await SELF.fetch(`${origin}/api/users/${result.user.id}`, { body: JSON.stringify({ role: "admin" }), - headers: { "content-type": "application/json", cookie: ownerCookie }, + headers: { origin, "content-type": "application/json", cookie: ownerCookie }, method: "PATCH" }); expect(roleChange.status).toBe(409); @@ -596,7 +604,7 @@ describe("workspace user onboarding", () => { }); const restored = await SELF.fetch(`${origin}/api/users/${result.user.id}/restore`, { - headers: { cookie: ownerCookie }, + headers: { origin, cookie: ownerCookie }, method: "POST" }); expect(restored.status, await restored.clone().text()).toBe(200); @@ -633,7 +641,7 @@ describe("workspace user onboarding", () => { .bind("owner@login.example") .first<{ id: string }>(); const response = await SELF.fetch(`${origin}/api/users/${owner?.id}`, { - headers: { cookie: ownerCookie }, + headers: { origin, cookie: ownerCookie }, method: "DELETE" }); expect(response.status).toBe(409); @@ -673,7 +681,7 @@ describe("workspace user onboarding", () => { }); const ownerResult = (await ownerCreated.json()) as { user: { id: string } }; const deniedRemoval = await SELF.fetch(`${origin}/api/users/${ownerResult.user.id}`, { - headers: { cookie: adminCookie }, + headers: { origin, cookie: adminCookie }, method: "DELETE" }); expect(deniedRemoval.status).toBe(403); @@ -682,12 +690,12 @@ describe("workspace user onboarding", () => { }); const removed = await SELF.fetch(`${origin}/api/users/${ownerResult.user.id}`, { - headers: { cookie: ownerCookie }, + headers: { origin, cookie: ownerCookie }, method: "DELETE" }); expect(removed.status, await removed.clone().text()).toBe(204); const deniedRestore = await SELF.fetch(`${origin}/api/users/${ownerResult.user.id}/restore`, { - headers: { cookie: adminCookie }, + headers: { origin, cookie: adminCookie }, method: "POST" }); expect(deniedRestore.status).toBe(403); @@ -696,7 +704,7 @@ describe("workspace user onboarding", () => { }); const restored = await SELF.fetch(`${origin}/api/users/${ownerResult.user.id}/restore`, { - headers: { cookie: ownerCookie }, + headers: { origin, cookie: ownerCookie }, method: "POST" }); expect(restored.status, await restored.clone().text()).toBe(200); @@ -731,20 +739,20 @@ describe("workspace user onboarding", () => { if (!owner || !ownerSession) throw new Error("Owner session was not created."); const listed = await SELF.fetch(`${origin}/api/sessions?userId=${owner.id}`, { - headers: { cookie: adminCookie } + headers: { origin, cookie: adminCookie } }); expect(listed.status).toBe(403); await expect(listed.json()).resolves.toMatchObject({ error: { code: "OWNER_REQUIRED" } }); const revoked = await SELF.fetch(`${origin}/api/sessions/${ownerSession.id}`, { - headers: { cookie: adminCookie }, + headers: { origin, cookie: adminCookie }, method: "DELETE" }); expect(revoked.status).toBe(403); await expect(revoked.json()).resolves.toMatchObject({ error: { code: "OWNER_REQUIRED" } }); const ownerStillSignedIn = await SELF.fetch(`${origin}/api/me`, { - headers: { cookie: ownerCookie } + headers: { origin, cookie: ownerCookie } }); expect(ownerStillSignedIn.status).toBe(200); }); diff --git a/test/unit/app/drafts/draft-list-paging.test.tsx b/test/unit/app/drafts/draft-list-paging.test.tsx new file mode 100644 index 00000000..2808645a --- /dev/null +++ b/test/unit/app/drafts/draft-list-paging.test.tsx @@ -0,0 +1,54 @@ +// @vitest-environment happy-dom +import { expect, it, vi } from "vitest"; +import { DraftsPage } from "@/features/drafts/drafts-page"; +import type { Draft } from "@/features/drafts/types"; +import { flushHookEffects, renderComponent } from "../render-hook"; + +it("pages draft rows and searches the complete synchronized set", async () => { + const drafts = Array.from( + { length: 125 }, + (_, index) => + ({ + id: `draft-${index}`, + mailboxId: "mailbox", + from: "owner@example.test", + to: [], + cc: [], + bcc: [], + subject: `Draft ${index}`, + text: "Synthetic body", + labels: [], + attachments: [], + updatedAt: "2026-09-04T12:00:00Z" + }) as unknown as Draft + ); + const onSelect = vi.fn(); + const props = { + drafts, + isLoading: false, + labelIds: [], + labels: [], + mailboxId: "all", + search: "", + selectedId: null, + onBack: vi.fn(), + onLabelChange: vi.fn(), + onSelect, + onToggleLabel: vi.fn() + }; + const view = await renderComponent(); + try { + expect(view.container.querySelectorAll("a")).toHaveLength(50); + const more = [...view.container.querySelectorAll("button")].find( + (button) => button.textContent === "Load more drafts" + ); + await flushHookEffects(() => more?.click()); + expect(view.container.querySelectorAll("a")).toHaveLength(100); + await view.rerender(); + expect(view.container.querySelectorAll("a")).toHaveLength(1); + await flushHookEffects(() => view.container.querySelector("a")?.click()); + expect(onSelect).toHaveBeenCalledWith("draft-124"); + } finally { + await view.unmount(); + } +}); diff --git a/test/unit/app/drafts/sync-client.test.ts b/test/unit/app/drafts/sync-client.test.ts new file mode 100644 index 00000000..ce8c8f77 --- /dev/null +++ b/test/unit/app/drafts/sync-client.test.ts @@ -0,0 +1,53 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { listDraftChanges, listDrafts } from "@/features/drafts/api"; +import { createDraftSync } from "@/features/drafts/sync-client"; +import type { Draft } from "@/features/drafts/types"; + +vi.mock("@/features/drafts/api", () => ({ listDraftChanges: vi.fn(), listDrafts: vi.fn() })); +const draft = (id: string, text = "") => ({ id, text, updatedAt: "2026-09-04T12:00:00Z" }) as Draft; +const page = (nextCursor: string, changes: unknown[] = [], hasMore = false) => + ({ nextCursor, changes, hasMore }) as Awaited>; +beforeEach(() => vi.resetAllMocks()); +describe("draft journal cache", () => { + it("replays changes during bootstrap, then reads only the journal", async () => { + vi.mocked(listDrafts).mockResolvedValue([draft("a"), draft("b")]); + vi.mocked(listDraftChanges) + .mockResolvedValueOnce(page("10")) + .mockResolvedValueOnce( + page("12", [ + { type: "delete", draftId: "a" }, + { type: "upsert", draft: draft("b", "new text") } + ]) + ) + .mockResolvedValueOnce(page("13", [{ type: "upsert", draft: draft("c") }])); + const cache = createDraftSync(); + expect(await cache.refresh()).toEqual([draft("b", "new text")]); + expect(await cache.refresh()).toEqual([draft("c"), draft("b", "new text")]); + expect(listDrafts).toHaveBeenCalledOnce(); + expect(listDraftChanges).toHaveBeenLastCalledWith("12"); + }); + it("retries from the last committed cursor after a later page fails", async () => { + vi.mocked(listDrafts).mockResolvedValue([draft("a")]); + vi.mocked(listDraftChanges) + .mockResolvedValueOnce(page("10")) + .mockResolvedValueOnce(page("10")) + .mockResolvedValueOnce(page("11", [{ type: "delete", draftId: "a" }], true)) + .mockRejectedValueOnce(new Error("offline")) + .mockResolvedValueOnce(page("12")); + const cache = createDraftSync(); + await cache.refresh(); + await expect(cache.refresh()).rejects.toThrow("offline"); + expect(await cache.refresh()).toEqual([draft("a")]); + expect(listDraftChanges).toHaveBeenLastCalledWith("10"); + }); + it("replaces cached access on a hard refresh", async () => { + vi.mocked(listDrafts) + .mockResolvedValueOnce([draft("a")]) + .mockResolvedValueOnce([]); + vi.mocked(listDraftChanges).mockResolvedValue(page("10")); + const cache = createDraftSync(); + await cache.refresh(); + expect(await cache.refresh(true)).toEqual([]); + expect(listDrafts).toHaveBeenCalledTimes(2); + }); +}); diff --git a/test/unit/app/events/use-mail-events.test.tsx b/test/unit/app/events/use-mail-events.test.tsx index 75c9b2f7..58aa77be 100644 --- a/test/unit/app/events/use-mail-events.test.tsx +++ b/test/unit/app/events/use-mail-events.test.tsx @@ -65,6 +65,27 @@ describe("useMailEvents", () => { vi.unstubAllGlobals(); }); + it("reconciles a healthy connection after a missed wake-up", async () => { + vi.useFakeTimers(); + const callbacks = handlers(); + const hook = await renderHook(({ userId }) => useMailEvents(userId, callbacks), { + userId: "user-1" + }); + const socket = FakeWebSocket.instances[0]; + await flushHookEffects(() => socket?.open()); + for (let beat = 0; beat < 4; beat += 1) { + await flushHookEffects(async () => { + await vi.advanceTimersByTimeAsync(30_000); + socket?.message("pong"); + }); + } + expect(hook.result).toBe("connected"); + expect(callbacks.onFallbackPoll).toHaveBeenCalledOnce(); + await hook.unmount(); + await vi.advanceTimersByTimeAsync(120_000); + expect(callbacks.onFallbackPoll).toHaveBeenCalledOnce(); + }); + it("opens one same-origin socket and coalesces wake events by topic", async () => { const callbacks = handlers(); const hook = await renderHook(({ userId }) => useMailEvents(userId, callbacks), { diff --git a/test/unit/app/messages/use-mail-sync.test.tsx b/test/unit/app/messages/use-mail-sync.test.tsx index a074d2dd..512228cb 100644 --- a/test/unit/app/messages/use-mail-sync.test.tsx +++ b/test/unit/app/messages/use-mail-sync.test.tsx @@ -70,6 +70,31 @@ describe("useMailSync", () => { mocks.refreshNotifications.mockReset(); }); + it("removes a remotely archived conversation after additional pages were loaded", async () => { + const first = conversation("first", "2026-09-04T12:00:00Z"); + const older = conversation("older", "2026-09-03T12:00:00Z"); + mocks.refreshNotifications.mockResolvedValue(status("first")); + mocks.listConversations + .mockResolvedValueOnce({ conversations: [first], nextCursor: "older", totalCount: 2 }) + .mockResolvedValueOnce({ conversations: [older], nextCursor: null, totalCount: null }) + .mockResolvedValueOnce({ conversations: [older], nextCursor: null, totalCount: 1 }); + const hook = await renderHook(useMailSync, { + activeFolder: "inbox", + mailboxId: "all", + search: "", + userId: "audit" + }); + try { + await flushHookEffects(); + await flushHookEffects(() => hook.result.loadMore()); + await flushHookEffects(() => hook.result.refresh()); + expect(hook.result.totalCount).toBe(1); + expect(hook.result.conversations.map((row) => row.id)).toEqual(["older"]); + } finally { + await hook.unmount(); + } + }); + it("uses one refresh path for initial load, focus, unread state, and incoming sound", async () => { Object.defineProperty(document, "visibilityState", { configurable: true, @@ -158,6 +183,7 @@ describe("useMailSync", () => { nextCursor: "cursor-2", totalCount: 3 }) + .mockResolvedValueOnce({ conversations: [second], nextCursor: null, totalCount: null }) .mockReturnValueOnce(replacementRequest); mocks.refreshNotifications .mockResolvedValueOnce(status("message-1")) @@ -197,7 +223,7 @@ describe("useMailSync", () => { hardRefresh = hook.result.hardRefresh(); }); expect(mocks.listConversations).toHaveBeenNthCalledWith( - 4, + 5, expect.not.objectContaining({ cursor: expect.anything() }) ); expect(hook.result.conversations.map((item) => item.id)).toEqual([ diff --git a/test/unit/scripts/after-deploy-state.test.mjs b/test/unit/scripts/after-deploy-state.test.mjs index 672be1ef..bd5fdf51 100644 --- a/test/unit/scripts/after-deploy-state.test.mjs +++ b/test/unit/scripts/after-deploy-state.test.mjs @@ -23,7 +23,7 @@ afterEach(() => { }); describe("after-deploy state inspection", () => { - it("recognizes only the exact S0, S1, S2, and S3 ledger and schema pairs", async () => { + it("recognizes only the exact S0, S1, S2, S3, and S4 ledger and schema pairs", async () => { const database = createDatabase(); const normal = await readD1Migrations(migrationsDirectory); const afterDeploy = await readD1Migrations(afterDeployMigrationsDirectory); @@ -57,7 +57,7 @@ describe("after-deploy state inspection", () => { worker_version: "worker-before-repair" } ]) - ).toMatchObject({ phase: "S3", repairRequired: true }); + ).toMatchObject({ phase: "S4", repairRequired: true }); }); it("fails closed when a ledger is incomplete, out of order, or unlike the live schema", async () => { diff --git a/test/unit/scripts/d1-migrations.test.mjs b/test/unit/scripts/d1-migrations.test.mjs index d553d530..d315bbab 100644 --- a/test/unit/scripts/d1-migrations.test.mjs +++ b/test/unit/scripts/d1-migrations.test.mjs @@ -129,7 +129,7 @@ describe("two-phase D1 migrations", () => { }); it("publishes schema epoch 3 only after the cleanup phase", () => { - expect(releasePackageSource).toContain("const schemaVersion = 3;"); + expect(releasePackageSource).toContain("const schemaVersion = 4;"); expect(updateServiceSource).toContain("installedSchemaVersion: 3"); expect(cleanupMigrationSource).toContain("installed_schema_version = 3"); }); @@ -226,11 +226,11 @@ describe("two-phase D1 migrations", () => { expect(sql[0]).toContain("WHERE id = 'repair-original'"); }); - it("leaves a complete S3 database unchanged and finalizes only its pending repair row", () => { + it("leaves a complete S4 database unchanged and finalizes only its pending repair row", () => { const recordWorker = vi.fn(); const migrate = vi.fn(); const execute = vi.fn(); - const complete = { phase: "S3", pendingUpdate: null }; + const complete = { phase: "S4", pendingUpdate: null }; expect( completeActiveReleaseRetry( @@ -243,14 +243,14 @@ describe("two-phase D1 migrations", () => { executeSql: execute } ) - ).toEqual({ phase: "S3", repaired: false, workerRecorded: false }); + ).toEqual({ phase: "S4", repaired: false, workerRecorded: false }); expect(recordWorker).not.toHaveBeenCalled(); expect(migrate).not.toHaveBeenCalled(); expect(execute).not.toHaveBeenCalled(); completeActiveReleaseRetry("/release", { schemaVersion: 16, version: "1.2.3" }, recordWorker, { afterDeployState: { - phase: "S3", + phase: "S4", pendingUpdate: { checkpoint_bookmark: "bookmark-original", from_version: "1.2.3", diff --git a/test/unit/scripts/recovery-objects.test.mjs b/test/unit/scripts/recovery-objects.test.mjs new file mode 100644 index 00000000..7a1af7ec --- /dev/null +++ b/test/unit/scripts/recovery-objects.test.mjs @@ -0,0 +1,56 @@ +import { describe, expect, it, vi } from "vitest"; +import { + validateRelease, + verifyObjectReferences +} from "../../../scripts/hqbase/recovery-verification.mjs"; + +const manifest = { accountId: "account", d1: { id: "db" }, r2: { bucket: "mail" } }; +const authentication = { type: "api_token", token: "synthetic-token" }; +function responses(missing = false) { + return vi.fn(async (url, init) => { + if (url.includes("/objects")) + return Response.json({ + success: true, + result: [{ key: url.includes("cursor=") ? (missing ? "d" : "c") : "a" }], + result_info: url.includes("cursor=") + ? { is_truncated: false } + : { is_truncated: true, cursor: "page2" } + }); + const { sql } = JSON.parse(init.body); + const results = sql.includes("table_info") + ? [{ name: "text_r2_key" }] + : sql.includes("sqlite_schema") + ? [{ name: "send_operations" }] + : [{ key: "a" }, { key: "c" }]; + return Response.json({ success: true, result: [{ success: true, results }] }); + }); +} +describe("recovery object integrity", () => { + it("checks references across all inventory pages without reading mail content", async () => { + const fetchRequest = responses(); + await expect(verifyObjectReferences(manifest, { authentication, fetchRequest })).resolves.toBe( + 2 + ); + expect(fetchRequest.mock.calls.filter(([url]) => url.includes("/objects"))).toHaveLength(2); + expect(fetchRequest.mock.calls.every(([url]) => !url.includes("/objects/"))).toBe(true); + }); + it("fails when a referenced object is missing without disclosing its key", async () => { + await expect( + verifyObjectReferences(manifest, { authentication, fetchRequest: responses(true) }) + ).rejects.toThrow("referenced mail object is missing"); + }); + it("rejects an unrelated or mismatched release marker", () => { + const row = { + schema_product: "hqbase", + product: "hqbase", + installed_version: "1.3.4", + installed_schema_version: 3, + channel: "stable" + }; + expect(validateRelease([row], row)).toEqual(row); + expect(() => validateRelease([row], { ...row, installed_version: "1.3.3" })).toThrow( + "release state" + ); + expect(() => validateRelease([])).toThrow("release state"); + }); +}); diff --git a/test/unit/scripts/recovery.test.mjs b/test/unit/scripts/recovery.test.mjs index d3d1541a..ee9fcbcd 100644 --- a/test/unit/scripts/recovery.test.mjs +++ b/test/unit/scripts/recovery.test.mjs @@ -17,8 +17,16 @@ describe("operator recovery manifests", () => { it("rejects malformed and cross-deployment restores", () => { const valid = { - format: "hqbase-backup-v1", + format: "hqbase-backup-v2", deployment: "staging", + createdAt: new Date().toISOString(), + release: { + schema_product: "hqbase", + product: "hqbase", + installed_version: "1.3.4", + installed_schema_version: 3, + channel: "stable" + }, d1: { bookmark: "bk-123" }, worker: { version: "ver-456" } }; @@ -27,3 +35,41 @@ describe("operator recovery manifests", () => { expect(() => validateBackupManifest({}, "staging")).toThrow("invalid"); }); }); + +describe("recovery verification", () => { + it("rejects split Worker deployments", () => { + expect(() => + parseWorkerVersion( + JSON.stringify({ + versions: [ + { version_id: "a", percentage: 50 }, + { version_id: "b", percentage: 50 } + ] + }) + ) + ).toThrow("one Worker version"); + }); + it("rejects a checkpoint for a replaced database or an expired date", () => { + const manifest = { + accountId: "account", + d1: { id: "new-db", name: "mail" }, + worker: { name: "mail" }, + r2: { bucket: "mail" } + }; + const checkpoint = { + format: "hqbase-backup-v2", + deployment: "staging", + createdAt: new Date().toISOString(), + accountId: "account", + d1: { id: "old-db", name: "mail", bookmark: "bookmark" }, + worker: { name: "mail", version: "worker" }, + r2: { bucket: "mail" } + }; + expect(() => validateBackupManifest(checkpoint, "staging", manifest)).toThrow( + "resource identities" + ); + expect(() => + validateBackupManifest({ ...checkpoint, createdAt: "2020-01-01" }, "staging", manifest) + ).toThrow("Time Travel window"); + }); +}); diff --git a/test/unit/scripts/sql-migrations.test.mjs b/test/unit/scripts/sql-migrations.test.mjs index db5cf741..4255ce27 100644 --- a/test/unit/scripts/sql-migrations.test.mjs +++ b/test/unit/scripts/sql-migrations.test.mjs @@ -41,12 +41,14 @@ const expectedMigrationNames = [ "0025_activate_catch_all_policy.sql", "0026_domain_disconnect.sql", "0027_message_attachment_disposition.sql", - "0028_draft_labels.sql" + "0028_draft_labels.sql", + "0029_mail_reliability.sql" ]; const expectedAfterDeployMigrationNames = [ "0001_remove_mailbox_alias_storage.sql", "0002_finalize_agent_principals.sql", - "0003_finalize_draft_labels.sql" + "0003_finalize_draft_labels.sql", + "0004_mail_reliability_guards.sql" ]; const oneAddressMigrationSource = readFileSync( resolve(migrationsDirectory, "0016_one_address_per_mailbox.sql"), @@ -421,7 +423,7 @@ describe("SQL migration contract", () => { const tables = database .prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'") .all(); - expect(tables).toHaveLength(48); + expect(tables).toHaveLength(49); expect(tables.map((table) => table.name)).not.toContain("mailbox_addresses"); const mailboxColumns = database.prepare("PRAGMA table_info(mailboxes)").all(); @@ -569,6 +571,9 @@ describe("SQL migration contract", () => { expect(applyMigration(database, migrationNamed(migrations, "0028_draft_labels.sql"))).toBe( true ); + expect(applyMigration(database, migrationNamed(migrations, "0029_mail_reliability.sql"))).toBe( + true + ); database .prepare("UPDATE draft_attachments SET content_id = ? WHERE id = 'att_upgrade'") .run("att_upgrade@hqbase.invalid"); @@ -819,11 +824,11 @@ describe("SQL migration contract", () => { ) ).toBe(false); expect(database.prepare("SELECT count(*) AS count FROM d1_migrations").get()).toEqual({ - count: 28 + count: 29 }); expect( database.prepare("SELECT count(*) AS count FROM d1_migrations_after_deploy").get() - ).toEqual({ count: 3 }); + ).toEqual({ count: 4 }); }); it("closes deleted agent mailboxes before and after principal finalization", async () => { diff --git a/test/unit/worker/auth/mailbox-access.test.ts b/test/unit/worker/auth/mailbox-access.test.ts index 339b248b..34ec0172 100644 --- a/test/unit/worker/auth/mailbox-access.test.ts +++ b/test/unit/worker/auth/mailbox-access.test.ts @@ -55,7 +55,7 @@ describe("mailbox access levels", () => { "mailbox_id", "is_unassigned" ) - ).toEqual({ params: ["mbx_1"], sql: "(mailbox_id IN (?))" }); + ).toEqual({ params: ['["mbx_1"]'], sql: "(mailbox_id IN (SELECT value FROM json_each(?)))" }); expect( messageScopeSql( { includeUnassigned: true, mailboxIds: ["mbx_1"] }, @@ -63,8 +63,8 @@ describe("mailbox access levels", () => { "is_unassigned" ) ).toEqual({ - params: ["mbx_1"], - sql: "(mailbox_id IN (?) OR is_unassigned = 1)" + params: ['["mbx_1"]'], + sql: "(mailbox_id IN (SELECT value FROM json_each(?)) OR is_unassigned = 1)" }); expect( messageScopeSql({ includeUnassigned: true, mailboxIds: [] }, "mailbox_id", "is_unassigned") diff --git a/test/unit/worker/features/messages/conversation-scale.test.ts b/test/unit/worker/features/messages/conversation-scale.test.ts new file mode 100644 index 00000000..c28ac24e --- /dev/null +++ b/test/unit/worker/features/messages/conversation-scale.test.ts @@ -0,0 +1,78 @@ +import { DatabaseSync } from "node:sqlite"; +import { listConversationPage } from "@worker/features/messages/conversation-queries"; +import { expect, it } from "vitest"; + +it("keeps large inbox pages scoped and searchable without copying bodies into normal listings", async () => { + const database = new DatabaseSync(":memory:"); + try { + database.exec(`CREATE TABLE messages ( + id TEXT PRIMARY KEY, thread_id TEXT, mailbox_id TEXT, is_unassigned INTEGER, + direction TEXT, folder TEXT, from_address TEXT, from_name TEXT, to_json TEXT, + cc_json TEXT, bcc_json TEXT, subject TEXT, snippet TEXT, text_body TEXT, + received_at TEXT, sent_at TEXT, created_at TEXT, read_at TEXT, starred_at TEXT, + has_attachments INTEGER); + WITH RECURSIVE n(i) AS (VALUES(1) UNION ALL SELECT i+1 FROM n WHERE i<10000) + INSERT INTO messages SELECT printf('message-%05d',i), printf('thread-%05d',(i-1)/4), + CASE WHEN i%2=0 THEN 'allowed' ELSE 'private' END, 0, 'inbound', 'inbox', + 'sender@example.test', 'Sender', '["owner@example.test"]', '[]', '[]', + 'Synthetic message', 'Fixture', CASE WHEN i%100=0 THEN 'needle' ELSE printf('%.*c',4096,'x') END, + printf('2026-09-%02dT12:00:00Z',1+i%28), NULL, '2026-09-01', NULL, NULL, 0 FROM n;`); + let listingSql = ""; + let listingParams: unknown[] = []; + const client = { + prepare(query: string) { + return { + bind(...params: unknown[]) { + return { + all: async () => { + listingSql = query; + listingParams = params; + return { results: database.prepare(query).all(...(params as never[])) }; + } + }; + } + }; + } + } as unknown as D1Database; + const filters = { + scope: { mailboxIds: ["allowed"], includeUnassigned: false }, + folder: "inbox" as const, + limit: 50 + }; + const first = await listConversationPage(client, filters); + const compact = listingSql; + const params = listingParams; + const wide = compact.replace( + /SELECT messages\.id,[\s\S]*?COALESCE\(messages.received_at/, + "SELECT messages.*, COALESCE(messages.received_at" + ); + const timed = (query: string) => { + const start = performance.now(); + const rows = database.prepare(query).all(...(params as never[])); + return { rows, ms: performance.now() - start }; + }; + const previous = timed(wide); + const current = timed(compact); + expect(first.totalCount).toBe(2500); + expect(first.conversations).toHaveLength(50); + expect(first.conversations.every((row) => row.mailboxId === "allowed")).toBe(true); + expect(current.rows.map((row) => row.id)).toEqual(previous.rows.map((row) => row.id)); + expect(current.rows.every((row) => row.text_body === null)).toBe(true); + const second = await listConversationPage(client, { + ...filters, + cursor: first.nextCursor ?? undefined + }); + expect( + second.conversations.every( + (row) => !first.conversations.some((firstRow) => firstRow.threadId === row.threadId) + ) + ).toBe(true); + const search = await listConversationPage(client, { ...filters, search: "needle" }); + expect(search.totalCount).toBe(100); + console.info( + `Synthetic 10,000-message query: wide=${previous.ms.toFixed(1)}ms compact=${current.ms.toFixed(1)}ms` + ); + } finally { + database.close(); + } +}); diff --git a/test/unit/worker/features/messages/message-thread.test.ts b/test/unit/worker/features/messages/message-thread.test.ts index fb1e174d..e91e2d78 100644 --- a/test/unit/worker/features/messages/message-thread.test.ts +++ b/test/unit/worker/features/messages/message-thread.test.ts @@ -57,7 +57,7 @@ describe("message threads", () => { expect(prepare.mock.calls[0]?.[0]).toContain("ORDER BY COALESCE"); expect(prepare.mock.calls[0]?.[0]).toContain("SELECT messages.* FROM messages"); expect(prepare.mock.calls[0]?.[0]).not.toMatch(/\bLIMIT\b/u); - expect(threadBind).toHaveBeenCalledWith("thr_1", "mbx_allowed", "mbx_second"); + expect(threadBind).toHaveBeenCalledWith("thr_1", '["mbx_allowed","mbx_second"]'); expect(prepare.mock.calls[0]?.[0]).not.toContain("IS NULL"); }); @@ -74,7 +74,7 @@ describe("message threads", () => { }); expect(prepare.mock.calls[0]?.[0]).toContain("is_unassigned = 1"); - expect(threadBind).toHaveBeenCalledWith("thr_1", "mbx_allowed"); + expect(threadBind).toHaveBeenCalledWith("thr_1", '["mbx_allowed"]'); }); it("does not query when no mailbox is accessible", async () => { diff --git a/test/unit/worker/features/send/forward-service.test.ts b/test/unit/worker/features/send/forward-service.test.ts index 8a476d6f..0ffc8400 100644 --- a/test/unit/worker/features/send/forward-service.test.ts +++ b/test/unit/worker/features/send/forward-service.test.ts @@ -1,3 +1,8 @@ +vi.mock("@worker/features/send/operations", async (importOriginal) => ({ + ...(await importOriginal()), + resumeSend: vi.fn().mockResolvedValue(null) +})); + import { beforeEach, describe, expect, it, vi } from "vitest"; vi.mock("@worker/features/drafts/queries", () => ({ @@ -134,7 +139,8 @@ describe("forward service", () => { expect.objectContaining({ text: expect.stringContaining("---------- Forwarded message ---------") }), - [] + [], + expect.objectContaining({ id: expect.any(String), hash: expect.any(String) }) ); expect(saveDraft).not.toHaveBeenCalled(); }); @@ -180,7 +186,8 @@ describe("forward service", () => { "user-1", undefined, expect.any(Object), - [] + [], + expect.objectContaining({ id: expect.any(String), hash: expect.any(String) }) ); }); @@ -305,7 +312,8 @@ describe("forward service", () => { expect.objectContaining({ text: expect.stringContaining("---------- Forwarded message ---------") }), - [] + [], + expect.objectContaining({ id: expect.any(String), hash: expect.any(String) }) ); expect(vi.mocked(sendNewMessage).mock.calls[0]?.[1]).not.toHaveProperty("draftId"); expect(deleteDraft).toHaveBeenCalledWith(env.DB, env.MAIL_OBJECTS, "user-1", "draft-forward"); @@ -374,7 +382,8 @@ describe("forward service", () => { expect.objectContaining({ text: expect.stringContaining("---------- Forwarded message ---------") }), - [] + [], + expect.objectContaining({ id: expect.any(String), hash: expect.any(String) }) ); expect(saveDraft).not.toHaveBeenCalled(); expect(removeDraftAttachment).not.toHaveBeenCalled(); @@ -438,7 +447,8 @@ describe("forward service", () => { disposition: "inline", filename: "logo.png" }) - ] + ], + expect.objectContaining({ id: expect.any(String), hash: expect.any(String) }) ); }); diff --git a/test/unit/worker/features/send/routes.test.ts b/test/unit/worker/features/send/routes.test.ts index cd8da0e0..14cb4821 100644 --- a/test/unit/worker/features/send/routes.test.ts +++ b/test/unit/worker/features/send/routes.test.ts @@ -1,3 +1,8 @@ +vi.mock("@worker/features/send/operations", async (importOriginal) => ({ + ...(await importOriginal()), + resumeSend: vi.fn().mockResolvedValue(null) +})); + import type { WorkerEnv } from "@worker/lib/env"; import { errorBody, toAppError } from "@worker/lib/errors"; import { beforeEach, describe, expect, it, vi } from "vitest"; @@ -68,6 +73,7 @@ const noSignature = { mode: "none", id: null, name: "", html: "", text: "" } as describe("send routes", () => { beforeEach(() => { vi.clearAllMocks(); + mocks.recordAudit.mockResolvedValue(undefined); mocks.requireMailApiPrincipal.mockResolvedValue({ principal: { email: "person@example.com", @@ -116,7 +122,8 @@ describe("send routes", () => { { BETTER_AUTH_SECRET: "test-secret", DB: db - } as WorkerEnv + } as WorkerEnv, + { waitUntil: vi.fn(), passThroughOnException: vi.fn(), props: {} } ); expect(response.status).toBe(201); @@ -162,7 +169,8 @@ describe("send routes", () => { { BETTER_AUTH_SECRET: "test-secret", DB: db - } as WorkerEnv + } as WorkerEnv, + { waitUntil: vi.fn(), passThroughOnException: vi.fn(), props: {} } ); expect(response.status).toBe(201); @@ -335,6 +343,7 @@ async function request(db: D1Database, path: string, body: object): Promise ({ + ...(await importOriginal()), + resumeSend: vi.fn(), + reserveSend: vi.fn(), + acceptSend: vi.fn(), + markSendUnknown: vi.fn() +})); + +import { + acceptSend, + markSendUnknown, + reserveSend, + resumeSend +} from "@worker/features/send/operations"; import { beforeEach, describe, expect, it, vi } from "vitest"; vi.mock("@worker/db/client", () => ({ @@ -32,7 +46,6 @@ import { findMailboxForSending } from "@worker/features/mailboxes/queries"; import { getMessageDetail, getMessageHtmlKey, - insertAttachment, insertMessage, listThreadMessages } from "@worker/features/messages/queries"; @@ -98,6 +111,10 @@ describe("send service", () => { beforeEach(() => { vi.resetAllMocks(); + vi.mocked(resumeSend).mockResolvedValue(null); + vi.mocked(reserveSend).mockResolvedValue({ id: "operation-1" } as never); + vi.mocked(acceptSend).mockResolvedValue(sentSummary); + vi.mocked(markSendUnknown).mockResolvedValue(); vi.mocked(findMailboxForSending).mockResolvedValue(mailbox); vi.mocked(createThread).mockResolvedValue("thread-1"); vi.mocked(touchThread).mockResolvedValue(); @@ -127,15 +144,12 @@ describe("send service", () => { text: "Hello", to: ["owner@example.com"] }); - expect(insertMessage).toHaveBeenCalledWith( - env.DB, - expect.objectContaining({ - fromName: mailbox.displayName, - messageId: "" - }) - ); + expect(vi.mocked(acceptSend).mock.calls[0]?.[2].message).toMatchObject({ + fromName: mailbox.displayName + }); + expect(vi.mocked(acceptSend).mock.calls[0]?.[3]).toBe(""); expect(findMailboxForSending).toHaveBeenCalledOnce(); - expect(createThread).toHaveBeenCalledWith(env.DB, "Hello", "2026-07-10T00:00:00.000Z"); + expect(vi.mocked(acceptSend).mock.calls[0]?.[2].createThread).toBe(true); }); it("sends and stores one resolved signature after the authored content", async () => { @@ -168,10 +182,9 @@ describe("send service", () => { html: "

Hello



Jane
Support

" }) ); - expect(insertMessage).toHaveBeenCalledWith( - env.DB, - expect.objectContaining({ textBody: "Hello\n\nJane\nSupport" }) - ); + expect(vi.mocked(acceptSend).mock.calls[0]?.[2].message).toMatchObject({ + textBody: "Hello\n\nJane\nSupport" + }); }); it("converts signature data images to private CID attachments", async () => { @@ -213,16 +226,16 @@ describe("send service", () => { expect(put).toHaveBeenCalledWith("sent/2026-07-10/html-1-1.png", expect.any(ArrayBuffer), { httpMetadata: { contentType: "image/png" } }); - expect(insertMessage).toHaveBeenCalledWith( - env.DB, - expect.objectContaining({ hasAttachments: false }) - ); - expect(insertAttachment).toHaveBeenCalledWith( - env.DB, - expect.objectContaining({ - contentId: "html-1-1@hqbase.invalid", - r2Key: "sent/2026-07-10/html-1-1.png" - }) + expect(vi.mocked(acceptSend).mock.calls[0]?.[2].message).toMatchObject({ + hasAttachments: false + }); + expect(vi.mocked(acceptSend).mock.calls[0]?.[2].attachments).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + contentId: "html-1-1@hqbase.invalid", + r2Key: "sent/2026-07-10/html-1-1.png" + }) + ]) ); }); @@ -272,16 +285,17 @@ describe("send service", () => { expect(put).toHaveBeenCalledWith("sent/2026-07-10/html-1-1", expect.any(ArrayBuffer), { httpMetadata: { contentType: "image/png" } }); - expect(insertAttachment).toHaveBeenCalledWith( - env.DB, - expect.objectContaining({ - contentId: "forwarded-logo@example.com", - r2Key: "sent/2026-07-10/html-1-1" - }) + expect(vi.mocked(acceptSend).mock.calls[0]?.[2].attachments).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + contentId: "forwarded-logo@example.com", + r2Key: "sent/2026-07-10/html-1-1" + }) + ]) ); }); - it("uses only referenced private draft images and removes unused draft objects", async () => { + it("uses only referenced private draft images and leaves unused draft objects for the collector", async () => { send.mockResolvedValue({ messageId: "" }); vi.mocked(draftAttachmentObjects).mockResolvedValue([ { @@ -326,14 +340,10 @@ describe("send service", () => { filename: "chart.png" }) ]); - expect(insertAttachment).toHaveBeenCalledWith( - env.DB, - expect.objectContaining({ r2Key: "sent/2026-07-10/html-1-1" }) + expect(vi.mocked(acceptSend).mock.calls[0]?.[2].attachments).toEqual( + expect.arrayContaining([expect.objectContaining({ r2Key: "sent/2026-07-10/html-1-1" })]) ); - expect(deleteObject).toHaveBeenCalledWith([ - "drafts/user-1/draft-1/attachment-inline", - "drafts/user-1/draft-1/unused" - ]); + expect(deleteObject).not.toHaveBeenCalled(); }); it("uses the attachment's owning draft when an API client omits draftId", async () => { @@ -368,9 +378,8 @@ describe("send service", () => { const payload = send.mock.calls[0]?.[0] as Parameters[0]; expect(payload.html).toContain('src="cid:attachment-inline@hqbase.invalid"'); - expect(insertAttachment).toHaveBeenCalledWith( - env.DB, - expect.objectContaining({ r2Key: "sent/2026-07-10/html-1-1" }) + expect(vi.mocked(acceptSend).mock.calls[0]?.[2].attachments).toEqual( + expect.arrayContaining([expect.objectContaining({ r2Key: "sent/2026-07-10/html-1-1" })]) ); expect(batch).not.toHaveBeenCalled(); }); @@ -464,33 +473,7 @@ describe("send service", () => { expect(send).not.toHaveBeenCalled(); }); - it("chunks cleanup for drafts with more than 1,000 staged objects", async () => { - send.mockResolvedValue({ messageId: "" }); - vi.mocked(getRows).mockResolvedValue( - Array.from({ length: 1_001 }, (_value, index) => ({ r2_key: `drafts/object-${index}` })) - ); - - await sendNewMessage( - env, - { - attachmentIds: [], - bcc: [], - cc: [], - draftId: "draft-1", - from: mailbox.address, - subject: "Cleanup", - text: "Hello", - to: ["owner@example.com"] - }, - "user-1" - ); - - expect(deleteObject).toHaveBeenCalledTimes(2); - expect(deleteObject.mock.calls[0]?.[0]).toHaveLength(1_000); - expect(deleteObject.mock.calls[1]?.[0]).toEqual(["drafts/object-1000"]); - }); - - it("removes staged signature objects when delivery fails", async () => { + it("keeps staged signature objects when delivery is uncertain", async () => { send.mockRejectedValue(new Error("delivery failed")); await expect( @@ -514,14 +497,14 @@ describe("send service", () => { html: 'Support logo' } ) - ).rejects.toThrow("delivery failed"); + ).rejects.toMatchObject({ code: "SEND_OUTCOME_UNKNOWN" }); - expect(deleteObject).toHaveBeenCalledWith(["sent/2026-07-10/html-1-1.png"]); + expect(deleteObject).not.toHaveBeenCalled(); }); - it("removes unrecorded signature objects when post-send persistence fails", async () => { + it("keeps signature objects when accepted send storage fails", async () => { send.mockResolvedValue({ messageId: "" }); - vi.mocked(createThread).mockRejectedValue(new Error("D1 unavailable")); + vi.mocked(acceptSend).mockRejectedValue(new Error("D1 unavailable")); await expect( sendNewMessage( @@ -544,15 +527,15 @@ describe("send service", () => { html: 'Support logo' } ) - ).rejects.toThrow("D1 unavailable"); + ).rejects.toMatchObject({ code: "SEND_ACCEPTED_STORAGE_PENDING" }); expect(send).toHaveBeenCalledOnce(); - expect(deleteObject).toHaveBeenCalledWith(["sent/2026-07-10/html-1-1.png"]); + expect(deleteObject).not.toHaveBeenCalled(); }); - it("removes an unrecorded HTML body when persistence fails after upload", async () => { + it("keeps the HTML body when accepted send storage fails", async () => { send.mockResolvedValue({ messageId: "" }); - vi.mocked(touchThread).mockRejectedValue(new Error("D1 unavailable")); + vi.mocked(acceptSend).mockRejectedValue(new Error("D1 unavailable")); await expect( sendNewMessage(env, { @@ -565,12 +548,12 @@ describe("send service", () => { text: "Hello", to: ["owner@example.com"] }) - ).rejects.toThrow("D1 unavailable"); + ).rejects.toMatchObject({ code: "SEND_ACCEPTED_STORAGE_PENDING" }); - expect(put).toHaveBeenCalledWith("sent/2026-07-10/html-1.html", "

Hello

", { + expect(put).toHaveBeenCalledWith("sent/2026-07-10/html-1/body.html", "

Hello

", { httpMetadata: { contentType: "text/html; charset=utf-8" } }); - expect(deleteObject).toHaveBeenCalledWith(["sent/2026-07-10/html-1.html"]); + expect(deleteObject).not.toHaveBeenCalled(); }); it("keeps only allowlisted threading headers on replies", async () => { @@ -620,21 +603,18 @@ describe("send service", () => { text: quotedText, to: ["alternate@example.com"] }); - expect(insertMessage).toHaveBeenCalledWith( - env.DB, - expect.objectContaining({ - bcc: ["audit@example.com"], - cc: ["manager@example.com"], - fromName: mailbox.displayName, - htmlR2Key: "sent/2026-07-10/html-1.html", - messageId: "", - textBody: quotedText, - to: ["alternate@example.com"] - }) - ); - expect(createThread).not.toHaveBeenCalled(); - expect(touchThread).toHaveBeenCalledWith(env.DB, "thread-1", "2026-07-10T00:00:00.000Z"); - expect(put).toHaveBeenCalledWith("sent/2026-07-10/html-1.html", quotedHtml, { + expect(vi.mocked(acceptSend).mock.calls[0]?.[2].message).toMatchObject({ + bcc: ["audit@example.com"], + cc: ["manager@example.com"], + fromName: mailbox.displayName, + htmlR2Key: "sent/2026-07-10/html-1/body.html", + textBody: quotedText, + to: ["alternate@example.com"] + }); + expect(vi.mocked(acceptSend).mock.calls[0]?.[3]).toBe(""); + expect(vi.mocked(acceptSend).mock.calls[0]?.[2].createThread).toBe(false); + + expect(put).toHaveBeenCalledWith("sent/2026-07-10/html-1/body.html", quotedHtml, { httpMetadata: { contentType: "text/html; charset=utf-8" } }); }); @@ -724,10 +704,15 @@ describe("send service", () => { { includeUnassigned: false, mailboxIds: [mailbox.id] } ); - expect(listThreadMessages).toHaveBeenCalledWith(env.DB, "thread-1", { - includeUnassigned: false, - mailboxIds: [mailbox.id] - }); + expect(listThreadMessages).toHaveBeenCalledWith( + env.DB, + "thread-1", + { + includeUnassigned: false, + mailboxIds: [mailbox.id] + }, + env.MAIL_OBJECTS + ); const payload = send.mock.calls[0]?.[0] as Parameters[0]; expect(payload.headers).toEqual({ "In-Reply-To": "", @@ -809,13 +794,14 @@ describe("send service", () => { type: "image/png" }) ]); - expect(insertAttachment).toHaveBeenCalledWith( - env.DB, - expect.objectContaining({ - contentId: "logo@example.com", - messageId: "message-1", - r2Key: "sent/2026-07-10/html-1-1" - }) + expect(vi.mocked(acceptSend).mock.calls[0]?.[2].attachments).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + contentId: "logo@example.com", + messageId: "html-1", + r2Key: "sent/2026-07-10/html-1-1" + }) + ]) ); expect(put).toHaveBeenCalledWith("sent/2026-07-10/html-1-1", expect.any(ArrayBuffer), { httpMetadata: { contentType: "image/png" } diff --git a/test/unit/worker/features/updates/migration-state.test.ts b/test/unit/worker/features/updates/migration-state.test.ts index 144b6112..f560e2d7 100644 --- a/test/unit/worker/features/updates/migration-state.test.ts +++ b/test/unit/worker/features/updates/migration-state.test.ts @@ -3,6 +3,7 @@ import { classifyManagedMigrationState, type ManagedMigrationSnapshot, normalMigrationNames, + pendingSendGuards, transitionGuards } from "@worker/features/updates/migration-state"; import { describe, expect, it } from "vitest"; @@ -12,7 +13,8 @@ describe("managed update migration state", () => { [0, "stale", true], [1, "partial", true], [2, "partial", true], - [3, "clean", false] + [3, "partial", true], + [4, "clean", false] ] as const)("accepts the exact S%s migration state", (completed, state, repairRequired) => { expect(classifyManagedMigrationState(snapshot(completed), "1.3.3", 3)).toEqual({ completedAfterDeployMigrations: completed, @@ -106,10 +108,11 @@ describe("managed update migration state", () => { }); }); -function snapshot(completed: 0 | 1 | 2 | 3): ManagedMigrationSnapshot { +function snapshot(completed: 0 | 1 | 2 | 3 | 4): ManagedMigrationSnapshot { const aliasesAreLegacy = completed === 0; const principalsAreLegacy = completed < 2; return { + pendingSendGuards: completed === 4 ? [...pendingSendGuards] : [], afterDeployLedger: completed === 0 ? null : [...afterDeployMigrationNames.slice(0, completed)], columns: { draft_changes: ["sequence", "principal_id", ...(principalsAreLegacy ? ["user_id"] : [])], @@ -126,7 +129,7 @@ function snapshot(completed: 0 | 1 | 2 | 3): ManagedMigrationSnapshot { ] }, draftLabelForeignKeys: - completed === 3 + completed >= 3 ? [ { from: "draft_id", diff --git a/test/unit/worker/features/updates/service.test.ts b/test/unit/worker/features/updates/service.test.ts index 3410fc9f..1390c7b6 100644 --- a/test/unit/worker/features/updates/service.test.ts +++ b/test/unit/worker/features/updates/service.test.ts @@ -2,6 +2,7 @@ import { generateKeyPairSync, sign } from "node:crypto"; import { afterDeployMigrationNames, normalMigrationNames, + pendingSendGuards, transitionGuards } from "@worker/features/updates/migration-state"; import { @@ -698,7 +699,10 @@ function migrationQueryResult( results: [ { name: "d1_migrations", type: "table" }, ...(stage === 3 - ? [{ name: "d1_migrations_after_deploy", type: "table" }] + ? [ + { name: "d1_migrations_after_deploy", type: "table" }, + ...pendingSendGuards.map((name) => ({ name, type: "trigger" })) + ] : [ { name: "mailbox_address_migration", type: "table" }, { name: "mailbox_addresses", type: "table" }, diff --git a/test/unit/worker/features/updates/zone-account.test.ts b/test/unit/worker/features/updates/zone-account.test.ts new file mode 100644 index 00000000..9093cb95 --- /dev/null +++ b/test/unit/worker/features/updates/zone-account.test.ts @@ -0,0 +1,24 @@ +import { findZoneAccount } from "@worker/features/updates/zone-account"; +import { expect, it, vi } from "vitest"; + +it("finds the most specific accessible zone after the first 50 zones", async () => { + const first = Array.from({ length: 49 }, (_, index) => ({ + name: `zone${index}.example`, + account: { id: "other" } + })); + first.push({ name: "example.com", account: { id: "parent" } }); + const request = vi + .fn() + .mockResolvedValueOnce( + Response.json({ success: true, result: first, result_info: { total_pages: 2 } }) + ) + .mockResolvedValueOnce( + Response.json({ + success: true, + result: [{ name: "mail.example.com", account: { id: "workspace" } }], + result_info: { total_pages: 2 } + }) + ); + await expect(findZoneAccount("app.mail.example.com", {}, request)).resolves.toBe("workspace"); + expect(request.mock.calls[1]?.[0]).toContain("page=2"); +}); diff --git a/test/unit/worker/jobs/consumer.test.ts b/test/unit/worker/jobs/consumer.test.ts index b5452027..9323f650 100644 --- a/test/unit/worker/jobs/consumer.test.ts +++ b/test/unit/worker/jobs/consumer.test.ts @@ -1,153 +1,56 @@ -import { beforeEach, describe, expect, it, vi } from "vitest"; - -vi.mock("@worker/db/drizzle", () => ({ - createDatabase: vi.fn(), - getRow: vi.fn(), - getRows: vi.fn() -})); -vi.mock("@worker/observability/log", () => ({ operationalLog: vi.fn() })); - -import { createDatabase, getRow, getRows } from "@worker/db/drizzle"; -import { consumeJobs, removeExpiredOrphanedObjects } from "@worker/jobs/consumer"; -import type { Job } from "@worker/jobs/types"; +import { getRows } from "@worker/db/drizzle"; +import { scanObjectPage } from "@worker/jobs/object-scan"; import type { WorkerEnv } from "@worker/lib/env"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +vi.mock("@worker/db/drizzle", () => ({ getRow: vi.fn(), getRows: vi.fn() })); const now = Date.parse("2026-08-25T12:00:00.000Z"); -const oneDay = 24 * 60 * 60 * 1_000; - -function r2Object(key: string, uploaded: number) { - return { key, uploaded: new Date(uploaded) }; -} - -describe("maintenance orphan cleanup", () => { +const day = 86_400_000; +const object = (key: string, age = day + 1) => ({ key, uploaded: new Date(now - age) }); +describe("bounded maintenance object pages", () => { const list = vi.fn(); - const deleteObjects = vi.fn(); - const env = { - DB: {} as D1Database, - MAIL_OBJECTS: { delete: deleteObjects, list } - } as unknown as WorkerEnv; - + const remove = vi.fn(); + const env = { DB: {}, MAIL_OBJECTS: { list, delete: remove } } as unknown as WorkerEnv; beforeEach(() => { vi.resetAllMocks(); + vi.mocked(getRows).mockResolvedValue([]); }); - - it("removes only expired, unreferenced objects across list pages", async () => { - list - .mockResolvedValueOnce({ - cursor: "page-2", - delimitedPrefixes: [], - objects: [ - r2Object("old-orphan", now - oneDay - 1), - r2Object("old-reference", now - oneDay - 1), - r2Object("age-boundary", now - oneDay) - ], - truncated: true - }) - .mockResolvedValueOnce({ - delimitedPrefixes: [], - objects: [ - r2Object("recent-orphan", now - oneDay + 1), - r2Object("second-old-orphan", now - oneDay - 1) - ], - truncated: false - }); - vi.mocked(getRow) - .mockResolvedValueOnce(null) - .mockResolvedValueOnce({ found: 1 }) - .mockResolvedValueOnce(null); - - await expect(removeExpiredOrphanedObjects(env, now)).resolves.toBe(2); - - expect(list).toHaveBeenNthCalledWith(1, { limit: 1_000 }); - expect(list).toHaveBeenNthCalledWith(2, { cursor: "page-2", limit: 1_000 }); - expect(getRow).toHaveBeenCalledTimes(3); - expect(deleteObjects).toHaveBeenCalledWith(["old-orphan", "second-old-orphan"]); - }); - - it("limits each maintenance run to 10,000 listed objects", async () => { - let page = 0; - list.mockImplementation(async (options: { cursor?: string; limit: number }) => { - page += 1; - const count = page === 1 ? options.limit - 1 : options.limit; - return { - cursor: `page-${page + 1}`, - delimitedPrefixes: [], - objects: Array.from({ length: count }, (_, index) => - r2Object(`recent-${page}-${index}`, now) - ), - truncated: true - }; + it("keeps referenced objects and the full 24-hour upload grace period", async () => { + list.mockResolvedValue({ + objects: [object("old"), object("referenced"), object("boundary", day), object("new", 1)], + truncated: true, + cursor: "next" }); - - await expect(removeExpiredOrphanedObjects(env, now)).resolves.toBe(0); - - expect(list).toHaveBeenCalledTimes(11); - expect(list).toHaveBeenLastCalledWith({ cursor: "page-11", limit: 1 }); - expect(getRow).not.toHaveBeenCalled(); - expect(deleteObjects).not.toHaveBeenCalled(); + vi.mocked(getRows).mockResolvedValue([{ key: "referenced" }]); + await expect(scanObjectPage(env, undefined, true, now)).resolves.toEqual({ + cursor: "next", + counters: { r2ObjectsScanned: 4, removedR2Orphans: 1 } + }); + expect(remove).toHaveBeenCalledExactlyOnceWith(["old"]); + expect(list).toHaveBeenCalledExactlyOnceWith({ limit: 1000 }); + expect(getRows).toHaveBeenCalledOnce(); }); - - it("deletes orphan keys in R2 batches of at most 1,000", async () => { - const firstPage = Array.from({ length: 1_000 }, (_, index) => - r2Object(`orphan-${index}`, now - oneDay - 1) - ); - list - .mockResolvedValueOnce({ - cursor: "page-2", - delimitedPrefixes: [], - objects: firstPage, - truncated: true - }) - .mockResolvedValueOnce({ - delimitedPrefixes: [], - objects: [r2Object("orphan-1000", now - oneDay - 1)], - truncated: false - }); - vi.mocked(getRow).mockResolvedValue(null); - - await expect(removeExpiredOrphanedObjects(env, now)).resolves.toBe(1_001); - - expect(deleteObjects).toHaveBeenCalledTimes(2); - expect(deleteObjects.mock.calls[0]?.[0]).toHaveLength(1_000); - expect(deleteObjects.mock.calls[1]?.[0]).toEqual(["orphan-1000"]); + it("can continue beyond 10,000 objects with one query and one delete per page", async () => { + const keys = Array.from({ length: 1000 }, (_, index) => object(`object-${index}`)); + list.mockImplementation(({ cursor }: { cursor?: string }) => { + const page = Number(cursor ?? 0); + return { objects: keys, truncated: page < 10, cursor: String(page + 1) }; + }); + let cursor: string | undefined; + let total = 0; + do { + const page = await scanObjectPage(env, cursor, true, now); + cursor = page.cursor; + total += page.counters.removedR2Orphans ?? 0; + } while (cursor); + expect(total).toBe(11000); + expect(getRows).toHaveBeenCalledTimes(11); + expect(remove.mock.calls.every(([page]) => page.length === 1000)).toBe(true); }); - - it("records the removed orphan count on maintenance runs", async () => { - const insertRun = vi.fn().mockResolvedValue({ meta: { changes: 1 } }); - const deleteRun = vi.fn().mockResolvedValue({ meta: { changes: 3 } }); - const updateRun = vi.fn().mockResolvedValue({ meta: { changes: 1 } }); - const setUpdate = vi.fn((values: unknown) => ({ - where: () => ({ run: updateRun, values }) - })); - vi.mocked(createDatabase).mockReturnValue({ - delete: () => ({ where: () => ({ run: deleteRun }) }), - insert: () => ({ - values: () => ({ onConflictDoNothing: () => ({ run: insertRun }) }) - }), - update: () => ({ set: setUpdate }) - } as never); - vi.mocked(getRows).mockResolvedValue([]); - list.mockResolvedValue({ delimitedPrefixes: [], objects: [], truncated: false }); - const ack = vi.fn(); - const retry = vi.fn(); - const job: Job = { - id: "maintenance:2026-08-25", - kind: "maintenance", - requestedAt: "2026-08-25T12:00:00.000Z" - }; - - await consumeJobs( - { messages: [{ ack, body: job, id: "queue-1", retry }] } as unknown as MessageBatch, - env - ); - - expect(setUpdate).toHaveBeenCalledWith( - expect.objectContaining({ - counters: { rateLimits: 3, removedR2Orphans: 0, retainedMessages: 0 }, - status: "succeeded" - }) - ); - expect(ack).toHaveBeenCalledOnce(); - expect(retry).not.toHaveBeenCalled(); + it("does not delete anything when reference inspection fails", async () => { + list.mockResolvedValue({ objects: [object("protected")], truncated: false }); + vi.mocked(getRows).mockRejectedValue(new Error("D1 unavailable")); + await expect(scanObjectPage(env, undefined, true, now)).rejects.toThrow("D1 unavailable"); + expect(remove).not.toHaveBeenCalled(); }); }); diff --git a/test/unit/worker/lib/request-body.test.ts b/test/unit/worker/lib/request-body.test.ts new file mode 100644 index 00000000..8b91754d --- /dev/null +++ b/test/unit/worker/lib/request-body.test.ts @@ -0,0 +1,34 @@ +import { readJson } from "@worker/lib/json"; +import { readBoundedBody } from "@worker/lib/request-body"; +import { describe, expect, it, vi } from "vitest"; + +describe("bounded request parsing", () => { + it("stops an oversized stream without a Content-Length header", async () => { + const cancel = vi.fn(); + const body = new ReadableStream({ + start(controller) { + controller.enqueue(new Uint8Array(4)); + controller.enqueue(new Uint8Array(4)); + }, + cancel + }); + const request = new Request("https://hqbase.test", { + method: "POST", + body, + duplex: "half" + } as RequestInit); + await expect(readBoundedBody(request, 6)).rejects.toMatchObject({ status: 413 }); + expect(cancel).toHaveBeenCalledOnce(); + }); + it("rejects a non-JSON media type before parsing", async () => { + await expect( + readJson( + new Request("https://hqbase.test", { + method: "POST", + body: "{}", + headers: { "content-type": "text/plain" } + }) + ) + ).rejects.toMatchObject({ status: 415 }); + }); +}); diff --git a/vitest.config.ts b/vitest.config.ts index c672f82c..479764c1 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -21,17 +21,7 @@ export default defineConfig({ reporter: ["text", "json-summary"], reportsDirectory: "coverage", include: ["app/**/*.{ts,tsx}", "worker/**/*.ts"], - exclude: [ - "app/components/ui/**", - "app/features/ui-lab/**", - "app/main.tsx", - "worker/index.ts", - "worker/features/domains/queries.ts", - "worker/features/domains/routes.ts", - "worker/features/drafts/queries.ts", - "worker/features/drafts/routes.ts", - "worker/features/setup/cloudflare.ts" - ], + exclude: ["app/components/ui/**", "app/features/ui-lab/**", "app/main.tsx"], thresholds: { branches: 30, functions: 30, diff --git a/worker/auth/mailbox-access.ts b/worker/auth/mailbox-access.ts index daf4b23d..3f01b01a 100644 --- a/worker/auth/mailbox-access.ts +++ b/worker/auth/mailbox-access.ts @@ -32,13 +32,16 @@ export function messageScopeSql( ): { params: string[]; sql: string } | null { const clauses: string[] = []; if (scope.mailboxIds.length > 0) { - clauses.push(`${mailboxColumn} IN (${scope.mailboxIds.map(() => "?").join(", ")})`); + clauses.push(`${mailboxColumn} IN (SELECT value FROM json_each(?))`); } if (scope.includeUnassigned) { clauses.push(`${unassignedColumn} = 1`); } if (clauses.length === 0) return null; - return { params: [...scope.mailboxIds], sql: `(${clauses.join(" OR ")})` }; + return { + params: scope.mailboxIds.length ? [JSON.stringify(scope.mailboxIds)] : [], + sql: `(${clauses.join(" OR ")})` + }; } export function messageScopeCondition( @@ -49,10 +52,7 @@ export function messageScopeCondition( const clauses = []; if (scope.mailboxIds.length > 0) { clauses.push( - sql`${sql.raw(mailboxColumn)} IN (${sql.join( - scope.mailboxIds.map((mailboxId) => sql`${mailboxId}`), - sql`, ` - )})` + sql`${sql.raw(mailboxColumn)} IN (SELECT value FROM json_each(${JSON.stringify(scope.mailboxIds)}))` ); } if (scope.includeUnassigned) { diff --git a/worker/auth/session.ts b/worker/auth/session.ts index 1fbbeca4..61733932 100644 --- a/worker/auth/session.ts +++ b/worker/auth/session.ts @@ -67,6 +67,12 @@ export async function requireAuthContext( if (!authContext) { throw new AppError("UNAUTHENTICATED", "Sign in is required.", 401); } + if ( + !["GET", "HEAD", "OPTIONS"].includes(request.method) && + request.headers.get("origin") !== new URL(request.url).origin + ) { + throw new AppError("ORIGIN_FORBIDDEN", "Request origin is not allowed.", 403); + } if ( !options.allowPasswordSetupRequired && (await isPasswordSetupRequired(env.DB, authContext.user.id)) diff --git a/worker/db/drizzle.ts b/worker/db/drizzle.ts index aecb6b0c..c69b8dd5 100644 --- a/worker/db/drizzle.ts +++ b/worker/db/drizzle.ts @@ -14,3 +14,11 @@ export async function getRow(client: D1Database, query: SQL): Promise(client: D1Database, query: SQL): Promise { return createDatabase(client).all(query); } + +export function preparedStatement( + client: D1Database, + query: { toSQL(): { sql: string; params: unknown[] } } +): D1PreparedStatement { + const { sql, params } = query.toSQL(); + return client.prepare(sql).bind(...params); +} diff --git a/worker/db/schema-core.ts b/worker/db/schema-core.ts index 42dee8da..afb5db92 100644 --- a/worker/db/schema-core.ts +++ b/worker/db/schema-core.ts @@ -96,6 +96,9 @@ export const operationRuns = sqliteTable( kind: text("kind").notNull(), status: text("status", { enum: ["running", "succeeded", "failed"] }).notNull(), cursor: text("cursor"), + attempts: integer("attempts").default(0).notNull(), + leaseToken: text("lease_token"), + leaseExpiresAt: text("lease_expires_at"), counters: text("counters_json", { mode: "json" }) .$type>() .default(sql`'{}'`) diff --git a/worker/db/schema-messages.ts b/worker/db/schema-messages.ts index 3e43fa1e..e75f5a47 100644 --- a/worker/db/schema-messages.ts +++ b/worker/db/schema-messages.ts @@ -37,6 +37,8 @@ export const messages = sqliteTable( subject: text("subject").notNull(), snippet: text("snippet").notNull(), textBody: text("text_body").notNull(), + textR2Key: text("text_r2_key"), + replyTo: text("reply_to_json", { mode: "json" }).$type().default(sql`'[]'`).notNull(), htmlR2Key: text("html_r2_key"), rawR2Key: text("raw_r2_key"), messageId: text("message_id"), diff --git a/worker/db/schema-send.ts b/worker/db/schema-send.ts new file mode 100644 index 00000000..bd716d60 --- /dev/null +++ b/worker/db/schema-send.ts @@ -0,0 +1,34 @@ +import { sql } from "drizzle-orm"; +import { check, index, sqliteTable, text, uniqueIndex } from "drizzle-orm/sqlite-core"; + +export const sendOperations = sqliteTable( + "send_operations", + { + id: text("id").primaryKey().notNull(), + principalId: text("principal_id"), + draftId: text("draft_id"), + mailboxId: text("mailbox_id"), + requestHash: text("request_hash").notNull(), + status: text("status", { enum: ["sending", "accepted", "stored", "unknown"] }).notNull(), + messageId: text("message_id").notNull(), + providerMessageId: text("provider_message_id"), + payloadR2Key: text("payload_r2_key").notNull().unique(), + receiptR2Key: text("receipt_r2_key").notNull().unique(), + objectKeys: text("object_keys_json", { mode: "json" }) + .$type() + .default(sql`'[]'`) + .notNull(), + createdAt: text("created_at").notNull(), + updatedAt: text("updated_at").notNull() + }, + (table) => [ + check( + "send_operations_status_check", + sql`${table.status} IN ('sending', 'accepted', 'stored', 'unknown')` + ), + uniqueIndex("send_operations_draft_idx") + .on(table.principalId, table.draftId) + .where(sql`${table.draftId} IS NOT NULL`), + index("send_operations_status_idx").on(table.status, table.updatedAt) + ] +); diff --git a/worker/db/schema.ts b/worker/db/schema.ts index 3a045058..0bd31d48 100644 --- a/worker/db/schema.ts +++ b/worker/db/schema.ts @@ -5,4 +5,5 @@ export * from "./schema-core"; export * from "./schema-mail"; export * from "./schema-messages"; export * from "./schema-notifications"; +export * from "./schema-send"; export * from "./schema-signatures"; diff --git a/worker/email/parse-email.ts b/worker/email/parse-email.ts index 8925bd50..4b0996b2 100644 --- a/worker/email/parse-email.ts +++ b/worker/email/parse-email.ts @@ -12,6 +12,7 @@ type ParsedAttachment = { }; export type ParsedEmail = { + replyTo?: string[]; fromAddress: string; fromName: string | null; to: string[]; @@ -42,6 +43,9 @@ export async function parseRawEmail(raw: ArrayBuffer): Promise { fromName: normalizeSenderName(from.name), to: flattenAddresses(email.to), cc: flattenAddresses(email.cc), + replyTo: flattenAddresses(email.replyTo).filter( + (address) => /^[^\s@]+@[^\s@]+\.[^\s@]+$/u.test(address) && address.length <= 254 + ), bcc: flattenAddresses(email.bcc), subject: email.subject?.trim() || "(no subject)", date: parseDate(email.date), diff --git a/worker/email/store-email.ts b/worker/email/store-email.ts index c6cf7eff..4c512250 100644 --- a/worker/email/store-email.ts +++ b/worker/email/store-email.ts @@ -1,8 +1,11 @@ import { sql } from "drizzle-orm"; import { newId, nowIso } from "../db/client"; -import { getRow } from "../db/drizzle"; -import { getMessageDetail, insertAttachment, insertMessage } from "../features/messages/queries"; +import { createDatabase, getRow } from "../db/drizzle"; +import { messageAttachments, messages } from "../db/schema"; +import { getMessageDetail } from "../features/messages/queries"; +import { attachmentValues, messageValues } from "../features/messages/storage"; +import { searchTextProjection } from "../features/messages/text-storage"; import { resolveInboundThread } from "../features/messages/threading"; import type { MessageDetail, MessageSummary } from "../features/messages/types"; @@ -33,7 +36,7 @@ export async function storeInboundEmail( parsed: input.parsed }); const dedupeKey = plan.dedupeKey; - const duplicate = dedupeKey ? await findDuplicate(db, dedupeKey) : null; + const duplicate = dedupeKey ? await findDuplicate(db, dedupeKey, bucket) : null; if (duplicate) { return { inserted: false, message: duplicate }; } @@ -59,7 +62,14 @@ export async function storeInboundEmail( references: input.parsed.references, subject: input.parsed.subject }); - const message = await insertMessage(db, { + const projection = searchTextProjection(input.parsed.textBody); + const textR2Key = projection.truncated ? `${objectBase}/body.txt` : null; + if (textR2Key) { + await bucket.put(textR2Key, input.parsed.textBody, { + httpMetadata: { contentType: "text/plain; charset=utf-8" } + }); + } + const message = messageValues({ threadId, isUnassigned: plan.isUnassigned, mailboxId: plan.mailboxId, @@ -72,7 +82,9 @@ export async function storeInboundEmail( bcc: input.parsed.bcc, subject: input.parsed.subject, snippet: input.parsed.snippet, - textBody: input.parsed.textBody, + textBody: projection.text, + textR2Key, + replyTo: input.parsed.replyTo ?? [], htmlR2Key, rawR2Key, messageId: input.parsed.messageId, @@ -85,28 +97,44 @@ export async function storeInboundEmail( hasAttachments: hasDownloadableAttachments(input.parsed.attachments), deliveredToAddress: recipient }); - + const attachments = []; for (const attachment of input.parsed.attachments) { - const r2Key = `${objectBase}/attachments/${newId("att")}-${attachment.filename}`; + const r2Key = `${objectBase}/attachments/${newId("att")}`; await bucket.put(r2Key, attachmentBody(attachment.content), { httpMetadata: { contentType: attachment.contentType } }); - await insertAttachment(db, { - messageId: message.id, - filename: attachment.filename, - contentType: attachment.contentType, - sizeBytes: attachmentSize(attachment.content), - contentId: attachment.contentId, - disposition: messageAttachmentDisposition(attachment), - r2Key - }); + attachments.push( + attachmentValues({ + messageId: message.id, + filename: attachment.filename, + contentType: attachment.contentType, + sizeBytes: attachmentSize(attachment.content), + contentId: attachment.contentId, + disposition: messageAttachmentDisposition(attachment), + r2Key + }) + ); + } + const database = createDatabase(db); + const attachmentInserts = []; + for (let start = 0; start < attachments.length; start += 8) { + attachmentInserts.push( + database.insert(messageAttachments).values(attachments.slice(start, start + 8)) + ); + } + try { + await database.batch([database.insert(messages).values(message), ...attachmentInserts]); + } catch (error) { + // A concurrent delivery can win the unique key. Its batch contains every attachment. + // Keep staged objects on uncertain failures; maintenance removes only proven orphans. + const winner = dedupeKey ? await findDuplicate(db, dedupeKey, bucket) : null; + if (winner) return { inserted: false, message: winner }; + throw error; } - return { - inserted: true, - isUnassigned: plan.isUnassigned, - message: (await getMessageDetail(db, message.id)) ?? message - }; + const stored = await getMessageDetail(db, message.id, bucket); + if (!stored) throw new Error("Committed inbound message could not be read."); + return { inserted: true, isUnassigned: plan.isUnassigned, message: stored }; } export function hasDownloadableAttachments(attachments: ParsedEmail["attachments"]): boolean { @@ -122,7 +150,11 @@ function messageAttachmentDisposition( return attachment.disposition === "inline" || attachment.contentId ? "inline" : "attachment"; } -async function findDuplicate(db: D1Database, dedupeKey: string): Promise { +async function findDuplicate( + db: D1Database, + dedupeKey: string, + bucket: R2Bucket +): Promise { const row = await getRow<{ id: string }>( db, sql`SELECT id FROM messages WHERE dedupe_key = ${dedupeKey}` @@ -132,5 +164,5 @@ async function findDuplicate(db: D1Database, dedupeKey: string): Promise sql`${id}`), - sql`, ` - )}) + WHERE draft_id IN (SELECT value FROM json_each(${JSON.stringify(draftIds)})) ORDER BY draft_id, created_at` ); } @@ -160,53 +157,47 @@ export async function saveDraft( input.signature && "name" in input.signature ? input.signature : (current?.signature ?? emptySignatureSnapshot("none")); - await createDatabase(db) - .insert(drafts) - .values({ - id, - principalId, - mailboxId: input.mailboxId, - replyToMessageId: input.replyToMessageId, - forwardOfMessageId: input.forwardOfMessageId, - fromAddress: input.from, - to: input.to, - cc: input.cc, - bcc: input.bcc, - subject: input.subject, - textBody: input.text, - htmlBody: input.html, - signatureMode: signature.mode, - signatureId: signature.id, - signatureNameSnapshot: signature.name, - signatureHtmlSnapshot: signature.html, - signatureTextSnapshot: signature.text, - version: nextVersion, - createdAt: current?.updatedAt ?? now, - updatedAt: now - }) - .onConflictDoUpdate({ - target: drafts.id, - set: { - mailboxId: input.mailboxId, - replyToMessageId: input.replyToMessageId, - forwardOfMessageId: input.forwardOfMessageId, - fromAddress: input.from, - to: input.to, - cc: input.cc, - bcc: input.bcc, - subject: input.subject, - textBody: input.text, - htmlBody: input.html, - signatureMode: signature.mode, - signatureId: signature.id, - signatureNameSnapshot: signature.name, - signatureHtmlSnapshot: signature.html, - signatureTextSnapshot: signature.text, - version: nextVersion, - updatedAt: now - } - }) - .run(); + const database = createDatabase(db); + const values = { + mailboxId: input.mailboxId, + replyToMessageId: input.replyToMessageId, + forwardOfMessageId: input.forwardOfMessageId, + fromAddress: input.from, + to: input.to, + cc: input.cc, + bcc: input.bcc, + subject: input.subject, + textBody: input.text, + htmlBody: input.html, + signatureMode: signature.mode, + signatureId: signature.id, + signatureNameSnapshot: signature.name, + signatureHtmlSnapshot: signature.html, + signatureTextSnapshot: signature.text, + version: nextVersion, + updatedAt: now + }; + if (current) { + const result = await database + .update(drafts) + .set(values) + .where( + and( + eq(drafts.id, id), + eq(drafts.principalId, principalId), + eq(drafts.version, current.version) + ) + ) + .run(); + if ((result.meta.changes ?? 0) === 0) { + throw new AppError("DRAFT_CONFLICT", "This draft changed in another session.", 409); + } + } else { + await database + .insert(drafts) + .values({ ...values, id, principalId, createdAt: now }) + .run(); + } const saved = await getDraft(db, principalId, id); if (!saved) throw new AppError("DRAFT_SAVE_FAILED", "Draft could not be saved.", 500); return saved; diff --git a/worker/features/drafts/routes.ts b/worker/features/drafts/routes.ts index bedeeba1..1f160c3a 100644 --- a/worker/features/drafts/routes.ts +++ b/worker/features/drafts/routes.ts @@ -3,6 +3,7 @@ import { requireMailApiPrincipal } from "../../auth/mail-api"; import type { HonoApp } from "../../lib/env"; import { AppError } from "../../lib/errors"; import { readJson } from "../../lib/json"; +import { readUpload } from "../../lib/request-body"; import { parseWith } from "../../lib/validation"; import { ignoreMailEventFailure, publishUserMailEvent } from "../events/service"; import { requireLabel, setDraftLabel } from "../labels/queries"; @@ -143,7 +144,7 @@ draftRoutes.delete("/:id", async (c) => { draftRoutes.post("/:id/attachments", async (c) => { const auth = await requireMailApiPrincipal(c.env, c.req.raw, "mail:send"); await getAccessibleDraft(c.env, draftPrincipal(auth), c.req.param("id")); - const body = await c.req.raw.formData(); + const body = await readUpload(c.req.raw); const file = body.get("file"); if (!(file instanceof File)) throw new AppError("FILE_REQUIRED", "Choose a file.", 400); const attachment = await storeDraftAttachment( diff --git a/worker/features/labels/queries.ts b/worker/features/labels/queries.ts index 3edd9414..4e3d5416 100644 --- a/worker/features/labels/queries.ts +++ b/worker/features/labels/queries.ts @@ -116,10 +116,7 @@ export async function labelsForMessageIds( label.created_at, label.updated_at FROM message_labels assignment JOIN labels label ON label.id = assignment.label_id - WHERE assignment.message_id IN (${sql.join( - messageIds.map((id) => sql`${id}`), - sql`, ` - )}) + WHERE assignment.message_id IN (SELECT value FROM json_each(${JSON.stringify(messageIds)})) ORDER BY label.name COLLATE NOCASE ASC, label.id ASC` ); return groupedLabels(rows, "message_id"); @@ -136,10 +133,7 @@ export async function labelsForDraftIds( label.created_at, label.updated_at FROM draft_labels assignment JOIN labels label ON label.id = assignment.label_id - WHERE assignment.draft_id IN (${sql.join( - draftIds.map((id) => sql`${id}`), - sql`, ` - )}) + WHERE assignment.draft_id IN (SELECT value FROM json_each(${JSON.stringify(draftIds)})) ORDER BY label.name COLLATE NOCASE ASC, label.id ASC` ); return groupedLabels(rows, "draft_id"); @@ -161,10 +155,7 @@ export async function labelsForThreadIds( JOIN message_labels assignment ON assignment.message_id = message.id JOIN labels label ON label.id = assignment.label_id WHERE ${access} - AND message.thread_id IN (${sql.join( - threadIds.map((id) => sql`${id}`), - sql`, ` - )}) + AND message.thread_id IN (SELECT value FROM json_each(${JSON.stringify(threadIds)})) ORDER BY label.name COLLATE NOCASE ASC, label.id ASC` ); return groupedLabels(rows, "thread_id"); diff --git a/worker/features/mcp/mail-tools.ts b/worker/features/mcp/mail-tools.ts index d3cd05f9..3aebc4c3 100644 --- a/worker/features/mcp/mail-tools.ts +++ b/worker/features/mcp/mail-tools.ts @@ -165,7 +165,9 @@ function registerReadTools(server: McpServer, env: WorkerEnv, principal: McpPrin ); return withMessageLabels( env.DB, - (await listThreadMessages(env.DB, message.threadId, scope)).map(publicMessage) + (await listThreadMessages(env.DB, message.threadId, scope, env.MAIL_OBJECTS)).map( + publicMessage + ) ); }) ); @@ -273,7 +275,7 @@ function registerWriteTools( async function readMessage(env: WorkerEnv, principal: McpPrincipal, messageId: string) { await requireMessageAccess(env.DB, principal.userId, principal.role, messageId, "read"); - const message = await getMessageDetail(env.DB, messageId); + const message = await getMessageDetail(env.DB, messageId, env.MAIL_OBJECTS); if (!message) throw new AppError("MESSAGE_NOT_FOUND", "Message not found.", 404); const [result] = await withMessageLabels(env.DB, [publicMessage(message)]); if (!result) throw new AppError("MESSAGE_NOT_FOUND", "Message not found.", 404); diff --git a/worker/features/mcp/send-tools.ts b/worker/features/mcp/send-tools.ts index ca383c57..6d20b326 100644 --- a/worker/features/mcp/send-tools.ts +++ b/worker/features/mcp/send-tools.ts @@ -5,6 +5,7 @@ import { accessibleMessageScope, requireMailboxAccess } from "../../auth/mailbox import type { WorkerEnv } from "../../lib/env"; import { AppError } from "../../lib/errors"; import { parseWith } from "../../lib/validation"; +import { operationalLog } from "../../observability/log"; import { enforceRateLimit } from "../../security/rate-limit"; import { recordAudit } from "../audit/service"; import { getAccessibleDraft, requireDraftAttachmentIdsAccess } from "../drafts/access"; @@ -12,6 +13,7 @@ import { type MailEventScheduler, scheduleSentMailEvents } from "../events/servi import { findMailboxForSending } from "../mailboxes/queries"; import { requireMessageAccess } from "../messages/access"; import { forwardMessage, sendForwardDraft } from "../send/forward"; +import { identifySend, resumeSend } from "../send/operations"; import { replyToMessage, sendNewMessage } from "../send/service"; import { forwardMessageSchema, replyMessageSchema, sendMessageSchema } from "../send/validation"; import { resolveSendSignature } from "../signatures/service"; @@ -45,6 +47,7 @@ export function registerSendTools( text: z.string().trim().min(1).max(100_000), html: z.string().trim().max(200_000).optional(), attachmentIds, + idempotencyKey: z.string().min(1).max(100).optional(), draftId: z.string().min(1).max(100).optional(), signature: signatureSelectionSchema.default({ mode: "automatic" }) }, @@ -55,6 +58,11 @@ export function registerSendTools( await enforceSendRateLimit(env, principal.userId); const parsed = parseWith(sendMessageSchema, input); const mailboxId = await requireSendingAccess(env, principal, parsed.from); + const previous = await resumeSend( + env, + await identifySend(principal.userId, parsed, "send") + ); + if (previous) return previous; const draft = parsed.draftId ? await getAccessibleDraft(env, principal, parsed.draftId) : null; @@ -80,7 +88,11 @@ export function registerSendTools( mailboxId, userId: principal.userId }); - await recordSend(env, principal, "mcp.message.send", mailboxId); + schedule( + recordSend(env, principal, "mcp.message.send", mailboxId).catch(() => + operationalLog("error", "send_audit_failed", {}) + ) + ); return message; }) ); @@ -99,6 +111,7 @@ export function registerSendTools( text: z.string().trim().min(1).max(100_000), html: z.string().trim().max(200_000).optional(), attachmentIds, + idempotencyKey: z.string().min(1).max(100).optional(), draftId: z.string().min(1).max(100).optional(), signature: signatureSelectionSchema.default({ mode: "automatic" }) }, @@ -110,6 +123,11 @@ export function registerSendTools( const parsed = parseWith(replyMessageSchema, input); await requireSourceAccess(env, principal, parsed.messageId); const mailboxId = await requireSendingAccess(env, principal, parsed.from); + const previous = await resumeSend( + env, + await identifySend(principal.userId, parsed, "reply") + ); + if (previous) return previous; const draft = parsed.draftId ? await getAccessibleDraft(env, principal, parsed.draftId) : null; @@ -138,7 +156,11 @@ export function registerSendTools( mailboxId, userId: principal.userId }); - await recordSend(env, principal, "mcp.message.reply", mailboxId); + schedule( + recordSend(env, principal, "mcp.message.reply", mailboxId).catch(() => + operationalLog("error", "send_audit_failed", {}) + ) + ); return message; }) ); @@ -158,6 +180,7 @@ export function registerSendTools( text: z.string().trim().max(100_000).default(""), html: z.string().trim().max(200_000).optional(), attachmentIds, + idempotencyKey: z.string().min(1).max(100).optional(), includeOriginalAttachments: z.boolean().default(true), signature: signatureSelectionSchema.default({ mode: "automatic" }) }, @@ -169,6 +192,11 @@ export function registerSendTools( const parsed = parseWith(forwardMessageSchema, input); await requireSourceAccess(env, principal, parsed.messageId); const mailboxId = await requireSendingAccess(env, principal, parsed.from); + const previous = await resumeSend( + env, + await identifySend(principal.userId, parsed, "forward") + ); + if (previous) return previous; await requireDraftAttachmentIdsAccess(env, principal, parsed.attachmentIds); const signature = await resolveSendSignature(env.DB, signaturePrincipal(principal), { from: parsed.from, @@ -176,7 +204,11 @@ export function registerSendTools( }); const message = await forwardMessage(env, parsed, principal.userId, signature); scheduleSentMailEvents(env, schedule, { mailboxId, userId: principal.userId }); - await recordSend(env, principal, "mcp.message.forward", mailboxId); + schedule( + recordSend(env, principal, "mcp.message.forward", mailboxId).catch(() => + operationalLog("error", "send_audit_failed", {}) + ) + ); return message; }) ); diff --git a/worker/features/messages/conversation-queries.ts b/worker/features/messages/conversation-queries.ts index 8fc6f091..b8171df9 100644 --- a/worker/features/messages/conversation-queries.ts +++ b/worker/features/messages/conversation-queries.ts @@ -123,7 +123,13 @@ export async function listConversationPage( const rows = await getRows( db, sql`WITH accessible AS ( - SELECT messages.*, + SELECT messages.id, messages.thread_id, messages.mailbox_id, messages.direction, + messages.folder, messages.from_address, messages.from_name, messages.to_json, + messages.subject, messages.snippet, messages.received_at, messages.sent_at, + messages.created_at, messages.read_at, messages.starred_at, messages.has_attachments, + ${filters.search ? sql`messages.text_body` : sql`NULL`} AS text_body, + ${filters.correspondentEmail ? sql`messages.cc_json` : sql`NULL`} AS cc_json, + ${filters.correspondentEmail ? sql`messages.bcc_json` : sql`NULL`} AS bcc_json, COALESCE(messages.received_at, messages.sent_at, messages.created_at) AS activity_at FROM messages WHERE ${scope} AND ${visibility} diff --git a/worker/features/messages/object-references.ts b/worker/features/messages/object-references.ts new file mode 100644 index 00000000..f444fd54 --- /dev/null +++ b/worker/features/messages/object-references.ts @@ -0,0 +1,24 @@ +import { sql } from "drizzle-orm"; +import { getRows } from "../../db/drizzle"; + +/** One bound set and one D1 query for a whole R2 page, including unfinished sends. */ +export async function referencedObjectKeys( + db: D1Database, + keys: readonly string[] +): Promise> { + if (keys.length === 0) return new Set(); + const rows = await getRows<{ key: string }>( + db, + sql` + WITH candidates AS (SELECT value AS key FROM json_each(${JSON.stringify(keys)})) + SELECT key FROM candidates WHERE + EXISTS (SELECT 1 FROM messages WHERE raw_r2_key = candidates.key) OR + EXISTS (SELECT 1 FROM messages WHERE html_r2_key = candidates.key) OR + EXISTS (SELECT 1 FROM messages WHERE text_r2_key = candidates.key) OR + EXISTS (SELECT 1 FROM message_attachments WHERE r2_key = candidates.key) OR + EXISTS (SELECT 1 FROM draft_attachments WHERE r2_key = candidates.key) OR + EXISTS (SELECT 1 FROM send_operations operation, json_each(operation.object_keys_json) object + WHERE operation.status <> 'stored' AND object.value = candidates.key)` + ); + return new Set(rows.map((row) => row.key)); +} diff --git a/worker/features/messages/queries.ts b/worker/features/messages/queries.ts index facec1fd..118c2752 100644 --- a/worker/features/messages/queries.ts +++ b/worker/features/messages/queries.ts @@ -2,7 +2,7 @@ import { and, eq, type SQL, sql } from "drizzle-orm"; import type { MessageScope } from "../../auth/mailbox-access"; import { messageScopeCondition } from "../../auth/mailbox-access"; -import { newId, nowIso } from "../../db/client"; +import { nowIso } from "../../db/client"; import { createDatabase, getRow, getRows } from "../../db/drizzle"; import { messageAttachments, messages as messagesTable } from "../../db/schema"; import { AppError } from "../../lib/errors"; @@ -10,6 +10,8 @@ import type { MessageAction } from "./actions"; import { buildMessageActionPatch } from "./actions"; import { decodeKeysetCursor, encodeKeysetCursor, type KeysetCursor } from "./keyset-cursor"; import { literalContains } from "./search"; +import { attachmentValues, messageValues } from "./storage"; +import { loadMessageText } from "./text-storage"; import type { AttachmentRow, InsertAttachmentInput, @@ -55,41 +57,9 @@ export async function insertMessage( db: D1Database, input: InsertMessageInput ): Promise { - const id = newId("msg"); - const timestamp = nowIso(); - - await createDatabase(db) - .insert(messagesTable) - .values({ - id, - threadId: input.threadId, - mailboxId: input.mailboxId, - isUnassigned: input.isUnassigned, - direction: input.direction, - folder: input.folder, - fromAddress: input.fromAddress, - fromName: input.fromName, - to: input.to, - cc: input.cc, - bcc: input.bcc, - subject: input.subject, - snippet: input.snippet, - textBody: input.textBody, - htmlR2Key: input.htmlR2Key, - rawR2Key: input.rawR2Key, - messageId: input.messageId, - dedupeKey: input.dedupeKey, - inReplyTo: input.inReplyTo, - references: input.references, - receivedAt: input.receivedAt, - sentAt: input.sentAt, - readAt: input.readAt, - hasAttachments: input.hasAttachments, - createdAt: timestamp, - updatedAt: timestamp, - deliveredToAddress: input.deliveredToAddress ?? null - }) - .run(); + const values = messageValues(input); + const id = values.id; + await createDatabase(db).insert(messagesTable).values(values).run(); const row = await getMessageRow(db, id); if (!row) { @@ -102,34 +72,9 @@ export async function insertAttachment( db: D1Database, input: InsertAttachmentInput ): Promise { - const id = newId("att"); - const timestamp = nowIso(); - await createDatabase(db) - .insert(messageAttachments) - .values({ - id, - messageId: input.messageId, - filename: input.filename, - contentType: input.contentType, - sizeBytes: input.sizeBytes, - contentId: input.contentId, - disposition: input.disposition, - r2Key: input.r2Key, - createdAt: timestamp - }) - .run(); - - return { - id, - messageId: input.messageId, - filename: input.filename, - contentType: input.contentType, - sizeBytes: input.sizeBytes, - contentId: input.contentId, - disposition: input.disposition, - r2Key: input.r2Key, - createdAt: timestamp - }; + const values = attachmentValues(input); + await createDatabase(db).insert(messageAttachments).values(values).run(); + return values; } export async function listMessages( @@ -211,19 +156,24 @@ function messageActivityOf(row: MessageRow): string { return row.received_at ?? row.sent_at ?? row.created_at; } -export async function getMessageDetail(db: D1Database, id: string): Promise { +export async function getMessageDetail( + db: D1Database, + id: string, + bucket?: R2Bucket +): Promise { const row = await getMessageRow(db, id); if (!row) { return null; } - return mapMessageDetail(db, row); + return mapMessageDetail(db, row, bucket); } export async function listThreadMessages( db: D1Database, threadId: string, - scope: MessageScope + scope: MessageScope, + bucket?: R2Bucket ): Promise { const scopeCondition = messageScopeCondition(scope, "mailbox_id", "is_unassigned"); if (!scopeCondition) return []; @@ -233,16 +183,21 @@ export async function listThreadMessages( WHERE thread_id = ${threadId} AND ${scopeCondition} ORDER BY COALESCE(received_at, sent_at, created_at) ASC` ); - return Promise.all(rows.map((row) => mapMessageDetail(db, row))); + return Promise.all(rows.map((row) => mapMessageDetail(db, row, bucket))); } -async function mapMessageDetail(db: D1Database, row: MessageRow): Promise { +async function mapMessageDetail( + db: D1Database, + row: MessageRow, + bucket?: R2Bucket +): Promise { return { ...mapMessageSummary(row), cc: parseJsonList(row.cc_json), bcc: parseJsonList(row.bcc_json), deliveredToAddress: row.delivered_to_address, - textBody: row.text_body, + replyTo: parseJsonList(row.reply_to_json ?? "[]"), + textBody: await loadMessageText(bucket, row.text_r2_key, row.text_body), htmlAvailable: row.html_r2_key !== null, messageId: row.message_id, inReplyTo: row.in_reply_to, diff --git a/worker/features/messages/routes.ts b/worker/features/messages/routes.ts index f9ac5929..fb55cdc0 100644 --- a/worker/features/messages/routes.ts +++ b/worker/features/messages/routes.ts @@ -82,7 +82,7 @@ messageRoutes.get("/", async (c) => { messageRoutes.get("/:id/thread", async (c) => { const auth = await requireMailApiPrincipal(c.env, c.req.raw, "mail:read"); - const message = await getMessageDetail(c.env.DB, c.req.param("id")); + const message = await getMessageDetail(c.env.DB, c.req.param("id"), c.env.MAIL_OBJECTS); if (!message) { throw new AppError("MESSAGE_NOT_FOUND", "Message not found.", 404); } @@ -93,7 +93,9 @@ messageRoutes.get("/:id/thread", async (c) => { auth.principal.role, "read" ); - const messages = (await listThreadMessages(c.env.DB, message.threadId, scope)).map(publicMessage); + const messages = ( + await listThreadMessages(c.env.DB, message.threadId, scope, c.env.MAIL_OBJECTS) + ).map(publicMessage); return c.json(includeLabels(c.req.raw) ? await withMessageLabels(c.env.DB, messages) : messages); }); @@ -106,7 +108,7 @@ messageRoutes.get("/:id", async (c) => { c.req.param("id"), "read" ); - const message = await getMessageDetail(c.env.DB, c.req.param("id")); + const message = await getMessageDetail(c.env.DB, c.req.param("id"), c.env.MAIL_OBJECTS); if (!message) { throw new AppError("MESSAGE_NOT_FOUND", "Message not found.", 404); } @@ -125,7 +127,7 @@ messageRoutes.get("/:id/html", async (c) => { c.req.param("id"), "read" ); - const message = await getMessageDetail(c.env.DB, c.req.param("id")); + const message = await getMessageDetail(c.env.DB, c.req.param("id"), c.env.MAIL_OBJECTS); if (!message) { throw new AppError("MESSAGE_NOT_FOUND", "Message not found.", 404); } @@ -167,7 +169,7 @@ messageRoutes.post("/:id/remote-media/trust", async (c) => { c.req.param("id"), "read" ); - const message = await getMessageDetail(c.env.DB, c.req.param("id")); + const message = await getMessageDetail(c.env.DB, c.req.param("id"), c.env.MAIL_OBJECTS); if (!message) { throw new AppError("MESSAGE_NOT_FOUND", "Message not found.", 404); } diff --git a/worker/features/messages/storage.ts b/worker/features/messages/storage.ts new file mode 100644 index 00000000..3215f71a --- /dev/null +++ b/worker/features/messages/storage.ts @@ -0,0 +1,19 @@ +import { newId, nowIso } from "../../db/client"; +import type { InsertAttachmentInput, InsertMessageInput } from "./types"; + +export function messageValues(input: InsertMessageInput, id = newId("msg")) { + const timestamp = nowIso(); + return { + ...input, + id, + replyTo: input.replyTo ?? [], + textR2Key: input.textR2Key ?? null, + deliveredToAddress: input.deliveredToAddress ?? null, + createdAt: timestamp, + updatedAt: timestamp + }; +} + +export function attachmentValues(input: InsertAttachmentInput, id = newId("att")) { + return { ...input, id, createdAt: nowIso() }; +} diff --git a/worker/features/messages/text-storage.ts b/worker/features/messages/text-storage.ts new file mode 100644 index 00000000..eceb9556 --- /dev/null +++ b/worker/features/messages/text-storage.ts @@ -0,0 +1,46 @@ +import { AppError } from "../../lib/errors"; + +export const maxSearchTextBytes = 256 * 1024; + +export function searchTextProjection(value: string): { text: string; truncated: boolean } { + const encoded = new TextEncoder().encode(value); + if (encoded.byteLength <= maxSearchTextBytes) return { text: value, truncated: false }; + // Drop an incomplete final code point instead of inserting a replacement character. + let end = maxSearchTextBytes; + while (end > 0 && ((encoded[end] ?? 0) & 0xc0) === 0x80) end -= 1; + return { text: new TextDecoder().decode(encoded.subarray(0, end)), truncated: true }; +} + +export async function loadMessageText( + bucket: R2Bucket | undefined, + key: string | null | undefined, + projection: string +): Promise { + if (!key) return projection; + if (!bucket) + throw new AppError("MESSAGE_BODY_UNAVAILABLE", "Message storage is unavailable.", 503); + const object = await bucket.get(key); + if (!object) + throw new AppError("MESSAGE_BODY_UNAVAILABLE", "The full message body is unavailable.", 503); + return object.text(); +} + +export async function storeMessageBody( + bucket: R2Bucket, + text: string, + html: string | undefined, + base: string +) { + const projection = searchTextProjection(text); + const textR2Key = projection.truncated ? `${base}/body.txt` : null; + const htmlR2Key = html ? `${base}/body.html` : null; + if (textR2Key) + await bucket.put(textR2Key, text, { + httpMetadata: { contentType: "text/plain; charset=utf-8" } + }); + if (htmlR2Key) + await bucket.put(htmlR2Key, html ?? "", { + httpMetadata: { contentType: "text/html; charset=utf-8" } + }); + return { textBody: projection.text, textR2Key, htmlR2Key }; +} diff --git a/worker/features/messages/types.ts b/worker/features/messages/types.ts index e343f9fa..1258f3b0 100644 --- a/worker/features/messages/types.ts +++ b/worker/features/messages/types.ts @@ -45,6 +45,7 @@ export type MessageSummary = { }; export type MessageDetail = MessageSummary & { + replyTo?: string[]; cc: string[]; bcc: string[]; deliveredToAddress: string | null; @@ -83,6 +84,8 @@ export type MessageRow = { subject: string; snippet: string; text_body: string; + text_r2_key?: string | null; + reply_to_json?: string; html_r2_key: string | null; raw_r2_key: string | null; message_id: string | null; @@ -123,6 +126,8 @@ export type AttachmentRow = { }; export type InsertMessageInput = { + textR2Key?: string | null; + replyTo?: string[]; threadId: string; mailboxId: string | null; isUnassigned: boolean; diff --git a/worker/features/operations/routes.ts b/worker/features/operations/routes.ts index 0bba72ee..3dcc799e 100644 --- a/worker/features/operations/routes.ts +++ b/worker/features/operations/routes.ts @@ -7,6 +7,7 @@ import { requireAuthContext, requireRole } from "../../auth/session"; import { nowIso } from "../../db/client"; import { createDatabase, getRow, getRows } from "../../db/drizzle"; import { retentionPolicies } from "../../db/schema"; +import { defaultTrashDays } from "../../jobs/maintenance"; import type { HonoApp } from "../../lib/env"; import { readJson } from "../../lib/json"; import { parseWith } from "../../lib/validation"; @@ -106,7 +107,11 @@ operationRoutes.get("/retention/:mailboxId", async (c) => { FROM retention_policies WHERE mailbox_id = ${c.req.param("mailboxId")}` ); return c.json( - policy ?? { mailbox_id: c.req.param("mailboxId"), message_days: null, trash_days: 30 } + policy ?? { + mailbox_id: c.req.param("mailboxId"), + message_days: null, + trash_days: defaultTrashDays + } ); }); diff --git a/worker/features/send/attachment-storage.ts b/worker/features/send/attachment-storage.ts index 4b837541..ab302327 100644 --- a/worker/features/send/attachment-storage.ts +++ b/worker/features/send/attachment-storage.ts @@ -1,7 +1,5 @@ -import { sql } from "drizzle-orm"; - -import { getRows } from "../../db/drizzle"; import type { WorkerEnv } from "../../lib/env"; +import { referencedObjectKeys } from "../messages/object-references"; import type { StoredOutgoingAttachment } from "./content-attachments"; @@ -21,19 +19,6 @@ export async function stageOutgoingAttachments( } } -export async function sendWithStagedCleanup( - env: Pick, - email: Parameters[0], - stagedAttachments: StoredOutgoingAttachment[] -): Promise>> { - try { - return await env.MAIL_SENDER.send(email); - } catch (error) { - await cleanupStagedObjects(env.MAIL_OBJECTS, stagedAttachments); - throw error; - } -} - async function cleanupStagedObjects( bucket: R2Bucket, attachments: StoredOutgoingAttachment[] @@ -61,19 +46,7 @@ export async function cleanupUnstoredObjectKeys( const uniqueKeys = [...new Set(keys)]; if (uniqueKeys.length === 0) return; try { - const values = sql.join( - uniqueKeys.map((key) => sql`${key}`), - sql`, ` - ); - const referenced = await getRows<{ r2_key: string }>( - env.DB, - sql`SELECT r2_key FROM message_attachments WHERE r2_key IN (${values}) - UNION - SELECT html_r2_key AS r2_key FROM messages WHERE html_r2_key IN (${values}) - UNION - SELECT raw_r2_key AS r2_key FROM messages WHERE raw_r2_key IN (${values})` - ); - const referencedKeys = new Set(referenced.map((object) => object.r2_key)); + const referencedKeys = await referencedObjectKeys(env.DB, uniqueKeys); await deleteObjectKeys( env.MAIL_OBJECTS, uniqueKeys.filter((key) => !referencedKeys.has(key)) diff --git a/worker/features/send/delivery.ts b/worker/features/send/delivery.ts new file mode 100644 index 00000000..f0af4156 --- /dev/null +++ b/worker/features/send/delivery.ts @@ -0,0 +1,46 @@ +import type { WorkerEnv } from "../../lib/env"; +import { AppError } from "../../lib/errors"; +import { cleanupUnstoredObjects, stageOutgoingAttachments } from "./attachment-storage"; +import type { StoredOutgoingAttachment } from "./content-attachments"; +import { + acceptSend, + markSendUnknown, + reserveSend, + resumeSend, + type SendIdentity, + type SendPayload, + uncertain +} from "./operations"; + +export async function deliverPreparedMail( + env: WorkerEnv, + identity: SendIdentity, + payload: SendPayload, + email: Parameters[0], + attachments: StoredOutgoingAttachment[] +) { + await stageOutgoingAttachments(env.MAIL_OBJECTS, attachments); + const operation = await reserveSend(env, identity, payload); + if (!operation) { + await cleanupUnstoredObjects(env, attachments); + const previous = await resumeSend(env, identity); + if (previous) return previous; + throw new AppError("DRAFT_SEND_PENDING", "This draft already has a send operation.", 409); + } + let result: Awaited>; + try { + result = await env.MAIL_SENDER.send(email); + } catch { + await markSendUnknown(env, identity); + throw uncertain(identity.id); + } + try { + return await acceptSend(env, operation, payload, result.messageId); + } catch { + throw new AppError( + "SEND_ACCEPTED_STORAGE_PENDING", + `Mail was accepted. Retry this same operation to finish storage; do not send a new copy. Reference: ${identity.id}`, + 503 + ); + } +} diff --git a/worker/features/send/forward.ts b/worker/features/send/forward.ts index 37a9a754..7ab237c6 100644 --- a/worker/features/send/forward.ts +++ b/worker/features/send/forward.ts @@ -22,6 +22,7 @@ import { type StoredOutgoingAttachment, totalAttachmentBytes } from "./content-attachments"; +import { identifySend, resumeSend } from "./operations"; import { sendNewMessage } from "./service"; import type { ForwardMessageInput, SendMessageInput } from "./validation"; @@ -33,7 +34,10 @@ export async function forwardMessage( principalId: string, signature?: SignatureSnapshot ) { - const original = await getMessageDetail(env.DB, input.messageId); + const identity = await identifySend(principalId, input, "forward"); + const previous = await resumeSend(env, identity); + if (previous) return previous; + const original = await getMessageDetail(env.DB, input.messageId, env.MAIL_OBJECTS); if (!original) throw new AppError("MESSAGE_NOT_FOUND", "Message not found.", 404); const mailbox = await findMailboxForSending(env.DB, input.from); @@ -69,7 +73,8 @@ export async function forwardMessage( principalId, signature, forwarded.context, - forwarded.inlineAttachments + forwarded.inlineAttachments, + identity ); } @@ -121,7 +126,8 @@ export async function forwardMessage( principalId, signature, forwarded.context, - forwarded.inlineAttachments + forwarded.inlineAttachments, + identity ); } finally { try { @@ -140,7 +146,10 @@ export async function sendForwardDraft( principalId: string, signature?: SignatureSnapshot ) { - const original = await getMessageDetail(env.DB, originalMessageId); + const identity = await identifySend(principalId, input, "send"); + const previous = await resumeSend(env, identity); + if (previous) return previous; + const original = await getMessageDetail(env.DB, originalMessageId, env.MAIL_OBJECTS); if (!original) throw new AppError("MESSAGE_NOT_FOUND", "Message not found.", 404); const authored = stripLegacyForwardContext(input); const originalAttachments = forwardableAttachments(original.attachments); @@ -158,7 +167,8 @@ export async function sendForwardDraft( principalId, signature, forwarded.context, - forwarded.inlineAttachments + forwarded.inlineAttachments, + identity ); } @@ -189,10 +199,13 @@ export async function sendForwardDraft( principalId, signature, forwarded.context, - forwarded.inlineAttachments + forwarded.inlineAttachments, + identity ); } catch (error) { - await removeCopiedAttachments(env, principalId, draftId, copiedAttachmentIds); + if (!(error instanceof AppError && error.code.startsWith("SEND_"))) { + await removeCopiedAttachments(env, principalId, draftId, copiedAttachmentIds); + } throw error; } } diff --git a/worker/features/send/operations.ts b/worker/features/send/operations.ts new file mode 100644 index 00000000..1405b85d --- /dev/null +++ b/worker/features/send/operations.ts @@ -0,0 +1,235 @@ +import { eq, sql } from "drizzle-orm"; +import { newId, nowIso } from "../../db/client"; +import { createDatabase, getRow } from "../../db/drizzle"; +import { sendOperations } from "../../db/schema"; +import type { WorkerEnv } from "../../lib/env"; +import { AppError } from "../../lib/errors"; +import { attachmentValues, messageValues } from "../messages/storage"; +import type { InsertMessageInput, MessageSummary } from "../messages/types"; +import { pendingSendGuards } from "../updates/migration-names"; +import { deleteObjectKeys } from "./attachment-storage"; +import type { StoredOutgoingAttachment } from "./content-attachments"; +import { finishSend, storedResult } from "./persistence"; + +export type SendIdentity = { + id: string; + hash: string; + principalId: string | null; + draftId: string | null; +}; +export type SendPayload = { + message: ReturnType; + attachments: ReturnType[]; + createThread: boolean; +}; +export type SendOperation = { + id: string; + principal_id: string | null; + draft_id: string | null; + request_hash: string; + status: "sending" | "accepted" | "stored" | "unknown"; + message_id: string; + provider_message_id: string | null; + payload_r2_key: string; + receipt_r2_key: string; +}; + +export async function identifySend( + principalId: string | undefined, + input: { draftId?: string | undefined; idempotencyKey?: string | undefined }, + kind: string +): Promise { + const key = input.idempotencyKey ?? (input.draftId ? `draft:${input.draftId}` : null); + return { + id: key ? `snd_${await digest(JSON.stringify([principalId ?? null, key]))}` : newId("snd"), + hash: await digest(canonicalJson({ kind, input })), + principalId: principalId ?? null, + draftId: input.draftId ?? null + }; +} + +export async function resumeSend( + env: WorkerEnv, + identity: SendIdentity +): Promise { + const row = await getRow( + env.DB, + sql`SELECT * FROM send_operations WHERE id = ${identity.id}` + ); + if (!row) return null; + if (row.principal_id !== identity.principalId || row.request_hash !== identity.hash) { + throw new AppError("SEND_KEY_CONFLICT", "This send key belongs to a different request.", 409); + } + if (row.status === "stored") return storedResult(env.DB, row.message_id); + let providerId = row.provider_message_id; + if (!providerId) { + const receipt = await env.MAIL_OBJECTS.get(row.receipt_r2_key); + if (receipt) { + const value = await receipt.json<{ messageId: string }>(); + providerId = value.messageId; + } + } + if (!providerId) throw uncertain(row.id); + const object = await env.MAIL_OBJECTS.get(row.payload_r2_key); + if (!object) { + const latest = await getRow( + env.DB, + sql`SELECT * FROM send_operations WHERE id = ${identity.id}` + ); + if (latest?.status === "stored") return storedResult(env.DB, latest.message_id); + } + if (!object) + throw new AppError( + "SEND_RECOVERY_UNAVAILABLE", + "Accepted mail needs storage recovery. Do not send it again.", + 503 + ); + return finishSend(env, row, await object.json(), providerId); +} + +export async function reserveSend( + env: WorkerEnv, + identity: SendIdentity, + payload: SendPayload +): Promise { + const guards = await getRow<{ count: number }>( + env.DB, + sql` + SELECT COUNT(*) AS count FROM sqlite_schema WHERE type = 'trigger' + AND name IN (SELECT value FROM json_each(${JSON.stringify(pendingSendGuards)}))` + ); + if (guards?.count !== pendingSendGuards.length) { + throw new AppError( + "SEND_STORAGE_NOT_READY", + "Sending is unavailable until the database update finishes.", + 503 + ); + } + const timestamp = nowIso(); + // Each attempt stages to a distinct key. A losing attempt cannot overwrite the winner's payload. + const base = `send-operations/${identity.id}/${newId("obj")}`; + const payloadR2Key = `${base}/payload.json`; + const receiptR2Key = `${base}/receipt.json`; + await env.MAIL_OBJECTS.put(payloadR2Key, JSON.stringify(payload), { + httpMetadata: { contentType: "application/json" } + }); + const database = createDatabase(env.DB); + const result = await database + .insert(sendOperations) + .values({ + id: identity.id, + principalId: identity.principalId, + draftId: identity.draftId, + mailboxId: payload.message.mailboxId, + requestHash: identity.hash, + status: "sending", + messageId: payload.message.id, + payloadR2Key, + receiptR2Key, + objectKeys: [ + payloadR2Key, + receiptR2Key, + payload.message.htmlR2Key, + payload.message.textR2Key, + ...payload.attachments.map((attachment) => attachment.r2Key) + ].filter((key): key is string => Boolean(key)), + createdAt: timestamp, + updatedAt: timestamp + }) + .onConflictDoNothing() + .run(); + if ((result.meta.changes ?? 0) !== 0) + return { + id: identity.id, + principal_id: identity.principalId, + draft_id: identity.draftId, + request_hash: identity.hash, + status: "sending", + message_id: payload.message.id, + provider_message_id: null, + payload_r2_key: payloadR2Key, + receipt_r2_key: receiptR2Key + }; + await deleteObjectKeys(env.MAIL_OBJECTS, [payloadR2Key]); + return null; +} + +export async function acceptSend( + env: WorkerEnv, + row: SendOperation, + payload: SendPayload, + providerId: string +): Promise { + // R2 receipt lets a later request finish storage even if D1 fails after provider acceptance. + await env.MAIL_OBJECTS.put(row.receipt_r2_key, JSON.stringify({ messageId: providerId }), { + httpMetadata: { contentType: "application/json" } + }).catch(() => undefined); + // D1 can still preserve acceptance when the independent R2 receipt is unavailable. + await createDatabase(env.DB) + .update(sendOperations) + .set({ + status: "accepted", + providerMessageId: providerId, + updatedAt: nowIso() + }) + .where(eq(sendOperations.id, row.id)) + .run(); + return finishSend(env, row, payload, providerId); +} + +export async function markSendUnknown(env: WorkerEnv, identity: SendIdentity): Promise { + await createDatabase(env.DB) + .update(sendOperations) + .set({ status: "unknown", updatedAt: nowIso() }) + .where(eq(sendOperations.id, identity.id)) + .run() + .catch(() => undefined); +} + +export function uncertain(id: string): AppError { + return new AppError( + "SEND_OUTCOME_UNKNOWN", + `Delivery is pending or uncertain. Do not send another copy. Reference: ${id}`, + 409 + ); +} + +export function makeSendPayload( + input: InsertMessageInput, + attachments: StoredOutgoingAttachment[], + createThread: boolean +): SendPayload { + const message = messageValues(input); + return { + message, + createThread, + attachments: attachments.map((attachment) => + attachmentValues({ + messageId: message.id, + filename: attachment.filename, + contentType: attachment.contentType, + contentId: attachment.contentId, + disposition: attachment.disposition, + sizeBytes: attachment.sizeBytes, + r2Key: attachment.r2Key + }) + ) + }; +} + +async function digest(value: string): Promise { + const hash = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(value)); + return Array.from(new Uint8Array(hash), (byte) => byte.toString(16).padStart(2, "0")).join(""); +} + +function canonicalJson(value: unknown): string { + if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`; + if (value && typeof value === "object") { + return `{${Object.entries(value) + .filter(([, item]) => item !== undefined) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([key, item]) => `${JSON.stringify(key)}:${canonicalJson(item)}`) + .join(",")}}`; + } + return JSON.stringify(value) ?? "null"; +} diff --git a/worker/features/send/persistence.ts b/worker/features/send/persistence.ts new file mode 100644 index 00000000..b379aa00 --- /dev/null +++ b/worker/features/send/persistence.ts @@ -0,0 +1,106 @@ +import { eq, sql } from "drizzle-orm"; +import { nowIso } from "../../db/client"; +import { createDatabase, getRow, preparedStatement } from "../../db/drizzle"; +import { messageAttachments, messages, sendOperations, threads } from "../../db/schema"; +import type { WorkerEnv } from "../../lib/env"; +import { AppError } from "../../lib/errors"; +import { normalizeSubject } from "../messages/headers"; +import { mapMessageSummary } from "../messages/queries"; +import type { MessageRow, MessageSummary } from "../messages/types"; +import { deleteObjectKeys } from "./attachment-storage"; +import type { SendOperation, SendPayload } from "./operations"; + +export async function finishSend( + env: WorkerEnv, + operation: SendOperation, + payload: SendPayload, + providerId: string +): Promise { + const database = createDatabase(env.DB); + const prepare = (query: Parameters[1]) => + preparedStatement(env.DB, query); + const statements = [ + prepare( + database + .update(sendOperations) + .set({ + status: "stored", + providerMessageId: providerId, + objectKeys: [], + updatedAt: nowIso() + }) + .where(eq(sendOperations.id, operation.id)) + ) + ]; + if (payload.createThread) + statements.push( + prepare( + database + .insert(threads) + .values({ + id: payload.message.threadId, + subjectNormalized: normalizeSubject(payload.message.subject), + lastMessageAt: payload.message.sentAt ?? payload.message.createdAt, + createdAt: payload.message.createdAt, + updatedAt: payload.message.updatedAt + }) + .onConflictDoNothing() + ) + ); + statements.push( + prepare( + database + .insert(messages) + .values({ ...payload.message, messageId: providerId }) + .onConflictDoNothing({ target: messages.id }) + ) + ); + for (let start = 0; start < payload.attachments.length; start += 8) { + statements.push( + prepare( + database + .insert(messageAttachments) + .values(payload.attachments.slice(start, start + 8)) + .onConflictDoNothing() + ) + ); + } + statements.push( + env.DB.prepare( + `UPDATE threads SET last_message_at = MAX(last_message_at, ?), updated_at = ? WHERE id = ?` + ).bind(payload.message.sentAt, nowIso(), payload.message.threadId) + ); + if (operation.draft_id && operation.principal_id) { + statements.push( + env.DB.prepare(`INSERT OR IGNORE INTO message_labels + (message_id, label_id, assigned_by_principal_id, created_at) + SELECT ?, assignment.label_id, assignment.assigned_by_principal_id, assignment.created_at + FROM draft_labels assignment JOIN drafts draft ON draft.id = assignment.draft_id + WHERE draft.id = ? AND draft.principal_id = ?`).bind( + payload.message.id, + operation.draft_id, + operation.principal_id + ) + ); + statements.push( + env.DB.prepare("DELETE FROM drafts WHERE id = ? AND principal_id = ?").bind( + operation.draft_id, + operation.principal_id + ) + ); + } + await env.DB.batch(statements); + await deleteObjectKeys(env.MAIL_OBJECTS, [operation.payload_r2_key, operation.receipt_r2_key]); + return storedResult(env.DB, payload.message.id); +} + +export async function storedResult(db: D1Database, id: string): Promise { + const message = await getRow(db, sql`SELECT * FROM messages WHERE id = ${id}`); + if (!message) + throw new AppError( + "SEND_RESULT_UNAVAILABLE", + "This send completed, but its stored message is no longer available.", + 410 + ); + return mapMessageSummary(message); +} diff --git a/worker/features/send/routes.ts b/worker/features/send/routes.ts index 19ff19a2..52cb8823 100644 --- a/worker/features/send/routes.ts +++ b/worker/features/send/routes.ts @@ -5,6 +5,7 @@ import type { HonoApp } from "../../lib/env"; import { AppError } from "../../lib/errors"; import { readJson } from "../../lib/json"; import { parseWith } from "../../lib/validation"; +import { operationalLog } from "../../observability/log"; import { enforceRateLimit } from "../../security/rate-limit"; import { recordAudit } from "../audit/service"; import { getAccessibleDraft, requireDraftAttachmentIdsAccess } from "../drafts/access"; @@ -12,8 +13,8 @@ import { scheduleSentMailEvents } from "../events/service"; import { findMailboxForSending } from "../mailboxes/queries"; import { requireMessageAccess } from "../messages/access"; import { resolveSendSignature } from "../signatures/service"; - import { forwardMessage, sendForwardDraft } from "./forward"; +import { identifySend, resumeSend } from "./operations"; import { replyToMessage, sendNewMessage } from "./service"; import { forwardMessageSchema, replyMessageSchema, sendMessageSchema } from "./validation"; @@ -32,6 +33,8 @@ sendRoutes.post("/send", async (c) => { const mailbox = await findMailboxForSending(c.env.DB, input.from); if (!mailbox) throw new AppError("MAILBOX_NOT_FOUND", "Sending mailbox not found.", 404); await requireMailboxAccess(c.env.DB, principal.id, principal.role, mailbox.id, "agent"); + const previous = await resumeSend(c.env, await identifySend(principal.id, input, "send")); + if (previous) return c.json(previous, 201); const draftPrincipal = { id: principal.id, role: principal.role }; const draft = input.draftId ? await getAccessibleDraft(c.env, draftPrincipal, input.draftId) @@ -58,15 +61,19 @@ sendRoutes.post("/send", async (c) => { mailboxId: mailbox.id, userId: principal.id }); - await recordAudit(c.env.DB, { - correlationId: c.get("correlationId"), - actorType: principal.type, - actorId: principal.id, - action: "message.send", - resourceType: "mailbox", - resourceId: mailbox.id, - outcome: "success" - }); + c.executionCtx.waitUntil( + recordAudit(c.env.DB, { + correlationId: c.get("correlationId"), + actorType: principal.type, + actorId: principal.id, + action: "message.send", + resourceType: "mailbox", + resourceId: mailbox.id, + outcome: "success" + }).catch(() => + operationalLog("error", "send_audit_failed", { requestId: c.get("correlationId") }) + ) + ); return c.json(sent, 201); }); @@ -90,6 +97,8 @@ sendRoutes.post("/reply", async (c) => { const mailbox = await findMailboxForSending(c.env.DB, input.from); if (!mailbox) throw new AppError("MAILBOX_NOT_FOUND", "Sending mailbox not found.", 404); await requireMailboxAccess(c.env.DB, principal.id, principal.role, mailbox.id, "agent"); + const previous = await resumeSend(c.env, await identifySend(principal.id, input, "reply")); + if (previous) return c.json(previous, 201); const draftPrincipal = { id: principal.id, role: principal.role }; const draft = input.draftId ? await getAccessibleDraft(c.env, draftPrincipal, input.draftId) @@ -107,15 +116,19 @@ sendRoutes.post("/reply", async (c) => { mailboxId: mailbox.id, userId: principal.id }); - await recordAudit(c.env.DB, { - correlationId: c.get("correlationId"), - actorType: principal.type, - actorId: principal.id, - action: "message.reply", - resourceType: "mailbox", - resourceId: mailbox.id, - outcome: "success" - }); + c.executionCtx.waitUntil( + recordAudit(c.env.DB, { + correlationId: c.get("correlationId"), + actorType: principal.type, + actorId: principal.id, + action: "message.reply", + resourceType: "mailbox", + resourceId: mailbox.id, + outcome: "success" + }).catch(() => + operationalLog("error", "send_audit_failed", { requestId: c.get("correlationId") }) + ) + ); return c.json(sent, 201); }); @@ -133,6 +146,8 @@ sendRoutes.post("/forward", async (c) => { const mailbox = await findMailboxForSending(c.env.DB, input.from); if (!mailbox) throw new AppError("MAILBOX_NOT_FOUND", "Sending mailbox not found.", 404); await requireMailboxAccess(c.env.DB, principal.id, principal.role, mailbox.id, "agent"); + const previous = await resumeSend(c.env, await identifySend(principal.id, input, "forward")); + if (previous) return c.json(previous, 201); const draftPrincipal = { id: principal.id, role: principal.role }; await requireDraftAttachmentIdsAccess(c.env, draftPrincipal, input.attachmentIds); const signature = await resolveSendSignature(c.env.DB, principal, { @@ -144,14 +159,18 @@ sendRoutes.post("/forward", async (c) => { mailboxId: mailbox.id, userId: principal.id }); - await recordAudit(c.env.DB, { - correlationId: c.get("correlationId"), - actorType: principal.type, - actorId: principal.id, - action: "message.forward", - resourceType: "mailbox", - resourceId: mailbox.id, - outcome: "success" - }); + c.executionCtx.waitUntil( + recordAudit(c.env.DB, { + correlationId: c.get("correlationId"), + actorType: principal.type, + actorId: principal.id, + action: "message.forward", + resourceType: "mailbox", + resourceId: mailbox.id, + outcome: "success" + }).catch(() => + operationalLog("error", "send_audit_failed", { requestId: c.get("correlationId") }) + ) + ); return c.json(sent, 201); }); diff --git a/worker/features/send/service.ts b/worker/features/send/service.ts index 5ab66406..3c1ee3d0 100644 --- a/worker/features/send/service.ts +++ b/worker/features/send/service.ts @@ -1,29 +1,14 @@ -import { sql } from "drizzle-orm"; - import type { MessageScope } from "../../auth/mailbox-access"; import { newId, nowIso } from "../../db/client"; -import { getRows } from "../../db/drizzle"; import type { WorkerEnv } from "../../lib/env"; import { AppError } from "../../lib/errors"; import { findMailboxForSending } from "../mailboxes/queries"; import type { Mailbox } from "../mailboxes/types"; import { ensureReplySubject } from "../messages/headers"; -import { - getMessageDetail, - insertAttachment, - insertMessage, - listThreadMessages -} from "../messages/queries"; -import { createThread, touchThread } from "../messages/threading"; +import { getMessageDetail, listThreadMessages } from "../messages/queries"; +import { storeMessageBody } from "../messages/text-storage"; import type { MessageSummary } from "../messages/types"; import type { SignatureSnapshot } from "../signatures/types"; -import { - cleanupUnstoredObjectKeys, - cleanupUnstoredObjects, - deleteObjectKeys, - sendWithStagedCleanup, - stageOutgoingAttachments -} from "./attachment-storage"; import { assembleMessageBody, type MessageBodyPart } from "./body"; import { asEmailAttachment, @@ -39,6 +24,8 @@ import { type StoredOutgoingAttachment, totalAttachmentBytes } from "./content-attachments"; +import { deliverPreparedMail } from "./delivery"; +import { identifySend, makeSendPayload, resumeSend, type SendIdentity } from "./operations"; import { buildReplyChainContext } from "./reply-body"; import type { ReplyMessageInput, SendMessageInput } from "./validation"; @@ -48,9 +35,13 @@ export async function sendNewMessage( principalId?: string, signature?: SignatureSnapshot, context?: MessageBodyPart, - contextAttachments: StoredOutgoingAttachment[] = [] + contextAttachments: StoredOutgoingAttachment[] = [], + operationIdentity?: SendIdentity ): Promise { const mailbox = await ensureActiveMailbox(env.DB, input.from); + const identity = operationIdentity ?? (await identifySend(principalId, input, "send")); + const previous = await resumeSend(env, identity); + if (previous) return previous; const timestamp = nowIso(); const draftAttachments = await loadAttachments(env, input.attachmentIds, principalId); @@ -81,14 +72,44 @@ export async function sendNewMessage( subject: input.subject, text: body.text }; - const stagedAttachments = [ - ...preparedDraftAttachments, - ...preparedSignature.attachments, - ...preparedContextAttachments - ]; - await stageOutgoingAttachments(env.MAIL_OBJECTS, stagedAttachments); - const sendResult = await sendWithStagedCleanup( + const storedBody = await storeMessageBody( + env.MAIL_OBJECTS, + body.text, + body.html, + `sent/${timestamp.slice(0, 10)}/${newId("obj")}` + ); + const payload = makeSendPayload( + { + threadId: newId("thr"), + isUnassigned: false, + mailboxId: mailbox.id, + direction: "outbound", + folder: "sent", + fromAddress: input.from, + fromName: mailbox.displayName, + to: input.to, + cc: input.cc, + bcc: input.bcc, + subject: input.subject, + snippet: body.text.replace(/\s+/g, " ").trim().slice(0, 180), + ...storedBody, + rawR2Key: null, + messageId: null, + dedupeKey: null, + inReplyTo: null, + references: [], + receivedAt: null, + sentAt: timestamp, + readAt: timestamp, + hasAttachments: attachments.some((attachment) => attachment.disposition === "attachment") + }, + attachments, + true + ); + return deliverPreparedMail( env, + identity, + payload, { ...email, ...(input.cc.length ? { cc: input.cc } : {}), @@ -96,30 +117,8 @@ export async function sendNewMessage( ...(body.html ? { html: body.html } : {}), ...(attachments.length ? { attachments: attachments.map(asEmailAttachment) } : {}) }, - stagedAttachments + attachments ); - try { - const threadId = await createThread(env.DB, input.subject, timestamp); - return await storeSentMessage(env, { - ...input, - fromName: mailbox.displayName, - text: body.text, - ...(body.html ? { html: body.html } : { html: undefined }), - inReplyTo: null, - messageId: sendResult.messageId, - mailboxId: mailbox.id, - references: [], - sentAt: timestamp, - subject: input.subject, - threadId, - storedAttachments: attachments, - draftId: input.draftId ?? null, - principalId: principalId ?? null - }); - } catch (error) { - await cleanupUnstoredObjects(env, stagedAttachments); - throw error; - } } export async function replyToMessage( @@ -130,13 +129,16 @@ export async function replyToMessage( messageScope?: MessageScope ): Promise { const mailbox = await ensureActiveMailbox(env.DB, input.from); + const identity = await identifySend(principalId, input, "reply"); + const previous = await resumeSend(env, identity); + if (previous) return previous; - const original = await getMessageDetail(env.DB, input.messageId); + const original = await getMessageDetail(env.DB, input.messageId, env.MAIL_OBJECTS); if (!original) { throw new AppError("MESSAGE_NOT_FOUND", "Message not found.", 404); } const threadMessages = messageScope - ? (await listThreadMessages(env.DB, original.threadId, messageScope)).filter( + ? (await listThreadMessages(env.DB, original.threadId, messageScope, env.MAIL_OBJECTS)).filter( (message) => (message.folder === "trash") === (original.folder === "trash") ) : [original]; @@ -147,7 +149,14 @@ export async function replyToMessage( const references = [...original.references, original.messageId].filter( (value): value is string => value !== null ); - const to = input.to?.length ? input.to : [original.fromAddress]; + const to = input.to?.length + ? input.to + : original.replyTo?.length + ? original.replyTo + : [original.fromAddress]; + if (to.length + input.cc.length + input.bcc.length > 50) { + throw new AppError("RECIPIENTS_TOO_MANY", "Choose at most 50 total recipients.", 400); + } const draftAttachments = await loadAttachments(env, input.attachmentIds, principalId); const authoredDraftId = resolveAuthoredDraftId(input.draftId, draftAttachments); const authored = prepareAuthoredContent({ @@ -186,14 +195,46 @@ export async function replyToMessage( const preparedQuotedAttachments = prepareStoredAttachments(quoted.inlineAttachments, timestamp); const outgoingAttachments = [...baseAttachments, ...preparedQuotedAttachments]; requireAttachmentLimits(outgoingAttachments); - const stagedAttachments = [ - ...preparedDraftAttachments, - ...preparedSignature.attachments, - ...preparedQuotedAttachments - ]; - await stageOutgoingAttachments(env.MAIL_OBJECTS, stagedAttachments); - const sendResult = await sendWithStagedCleanup( + const storedBody = await storeMessageBody( + env.MAIL_OBJECTS, + body.text, + body.html, + `sent/${timestamp.slice(0, 10)}/${newId("obj")}` + ); + const payload = makeSendPayload( + { + threadId: original.threadId, + isUnassigned: false, + mailboxId: mailbox.id, + direction: "outbound", + folder: "sent", + fromAddress: input.from, + fromName: mailbox.displayName, + to, + cc: input.cc, + bcc: input.bcc, + subject: ensureReplySubject(original.subject), + snippet: body.text.replace(/\s+/g, " ").trim().slice(0, 180), + ...storedBody, + rawR2Key: null, + messageId: null, + dedupeKey: null, + inReplyTo: original.messageId, + references, + receivedAt: null, + sentAt: timestamp, + readAt: timestamp, + hasAttachments: outgoingAttachments.some( + (attachment) => attachment.disposition === "attachment" + ) + }, + outgoingAttachments, + false + ); + return deliverPreparedMail( env, + identity, + payload, { from: { name: mailbox.displayName, email: mailbox.address }, to, @@ -202,7 +243,7 @@ export async function replyToMessage( subject: ensureReplySubject(original.subject), text: body.text, headers: { - "In-Reply-To": original.messageId ?? original.id, + ...(original.messageId ? { "In-Reply-To": original.messageId } : {}), References: references.join(" ") }, ...(body.html ? { html: body.html } : {}), @@ -210,32 +251,8 @@ export async function replyToMessage( ? { attachments: outgoingAttachments.map(asEmailAttachment) } : {}) }, - stagedAttachments + outgoingAttachments ); - try { - return await storeSentMessage(env, { - from: input.from, - fromName: mailbox.displayName, - to, - cc: input.cc, - bcc: input.bcc, - subject: ensureReplySubject(original.subject), - text: body.text, - ...(body.html ? { html: body.html } : {}), - inReplyTo: original.messageId ?? original.id, - messageId: sendResult.messageId, - mailboxId: mailbox.id, - references, - sentAt: timestamp, - threadId: original.threadId, - storedAttachments: outgoingAttachments, - draftId: input.draftId ?? null, - principalId: principalId ?? null - }); - } catch (error) { - await cleanupUnstoredObjects(env, stagedAttachments); - throw error; - } } async function ensureActiveMailbox(db: D1Database, address: string): Promise { @@ -248,108 +265,3 @@ async function ensureActiveMailbox(db: D1Database, address: string): Promise { - const htmlR2Key = input.html ? `sent/${input.sentAt.slice(0, 10)}/${newId("html")}.html` : null; - try { - if (input.html && htmlR2Key) { - await env.MAIL_OBJECTS.put(htmlR2Key, input.html, { - httpMetadata: { contentType: "text/html; charset=utf-8" } - }); - } - - await touchThread(env.DB, input.threadId, input.sentAt); - const message = await insertMessage(env.DB, { - threadId: input.threadId, - isUnassigned: false, - mailboxId: input.mailboxId, - direction: "outbound", - folder: "sent", - fromAddress: input.from, - fromName: input.fromName, - to: input.to, - cc: input.cc, - bcc: input.bcc, - subject: input.subject, - snippet: input.text.replace(/\s+/g, " ").trim().slice(0, 180), - textBody: input.text, - htmlR2Key, - rawR2Key: null, - messageId: input.messageId, - dedupeKey: null, - inReplyTo: input.inReplyTo, - references: input.references, - receivedAt: null, - sentAt: input.sentAt, - readAt: input.sentAt, - hasAttachments: input.storedAttachments.some( - (attachment) => attachment.disposition === "attachment" - ) - }); - for (const attachment of input.storedAttachments) { - await insertAttachment(env.DB, { - messageId: message.id, - filename: attachment.filename, - contentType: attachment.contentType, - sizeBytes: attachment.sizeBytes, - contentId: attachment.contentId, - disposition: attachment.disposition, - r2Key: attachment.r2Key - }); - } - if (input.draftId && input.principalId) { - const draftObjects = await getRows<{ r2_key: string }>( - env.DB, - sql`SELECT a.r2_key - FROM draft_attachments a - JOIN drafts d ON d.id = a.draft_id - WHERE d.id = ${input.draftId} AND d.principal_id = ${input.principalId}` - ); - await env.DB.batch([ - env.DB.prepare( - `INSERT OR IGNORE INTO message_labels - (message_id, label_id, assigned_by_principal_id, created_at) - SELECT ?, assignment.label_id, assignment.assigned_by_principal_id, - assignment.created_at - FROM draft_labels assignment - JOIN drafts draft ON draft.id = assignment.draft_id - WHERE draft.id = ? AND draft.principal_id = ?` - ).bind(message.id, input.draftId, input.principalId), - env.DB.prepare("DELETE FROM drafts WHERE id = ? AND principal_id = ?").bind( - input.draftId, - input.principalId - ) - ]); - const retainedKeys = new Set(input.storedAttachments.map((attachment) => attachment.r2Key)); - const unusedKeys = draftObjects - .map((object) => object.r2_key) - .filter((key) => !retainedKeys.has(key)); - await deleteObjectKeys(env.MAIL_OBJECTS, unusedKeys); - } - return message; - } catch (error) { - if (htmlR2Key) await cleanupUnstoredObjectKeys(env, [htmlR2Key]); - throw error; - } -} diff --git a/worker/features/send/validation.ts b/worker/features/send/validation.ts index 4706e8e7..9db7b4ab 100644 --- a/worker/features/send/validation.ts +++ b/worker/features/send/validation.ts @@ -9,6 +9,7 @@ const maxTotalRecipients = 50; export const sendMessageSchema = z .object({ + idempotencyKey: z.string().min(1).max(100).optional(), from: emailAddressSchema, to: recipientListSchema, cc: optionalRecipientListSchema, @@ -34,6 +35,7 @@ export const sendMessageSchema = z export const replyMessageSchema = z .object({ messageId: z.string().min(1), + idempotencyKey: z.string().min(1).max(100).optional(), from: emailAddressSchema, to: z.array(emailAddressSchema).max(50).optional(), cc: optionalRecipientListSchema, @@ -58,6 +60,7 @@ export const replyMessageSchema = z export const forwardMessageSchema = z .object({ messageId: z.string().min(1).max(100), + idempotencyKey: z.string().min(1).max(100).optional(), from: emailAddressSchema, to: recipientListSchema, cc: optionalRecipientListSchema, diff --git a/worker/features/updates/migration-names.ts b/worker/features/updates/migration-names.ts new file mode 100644 index 00000000..30196370 --- /dev/null +++ b/worker/features/updates/migration-names.ts @@ -0,0 +1,50 @@ +export const pendingSendGuards = [ + "send_operations_require_draft", + "drafts_before_update_pending_send", + "drafts_before_delete_pending_send", + "draft_attachments_before_update_pending_send", + "draft_attachments_before_delete_pending_send", + "draft_attachments_before_insert_pending_send", + "draft_labels_before_update_pending_send", + "draft_labels_before_delete_pending_send", + "draft_labels_before_insert_pending_send" +] as const; + +export const normalMigrationNames = [ + "0001_initial.sql", + "0002_workspace.sql", + "0003_oauth_resources.sql", + "0004_conversations.sql", + "0005_rebuild_threads.sql", + "0006_push_notifications.sql", + "0007_user_mail_preferences.sql", + "0008_user_onboarding.sql", + "0009_login_email_domain_isolation.sql", + "0010_oauth_device_authorization.sql", + "0011_latest_password_reset_token.sql", + "0012_message_activity_index.sql", + "0013_message_changes.sql", + "0014_unassigned_messages.sql", + "0015_draft_changes.sql", + "0016_one_address_per_mailbox.sql", + "0017_agent_principals.sql", + "0018_mailbox_lifecycle.sql", + "0019_contacts.sql", + "0020_labels.sql", + "0021_email_signatures.sql", + "0022_login_email_domain_exact_match.sql", + "0023_message_sender_names.sql", + "0024_draft_inline_images.sql", + "0025_activate_catch_all_policy.sql", + "0026_domain_disconnect.sql", + "0027_message_attachment_disposition.sql", + "0028_draft_labels.sql", + "0029_mail_reliability.sql" +] as const; + +export const afterDeployMigrationNames = [ + "0001_remove_mailbox_alias_storage.sql", + "0002_finalize_agent_principals.sql", + "0003_finalize_draft_labels.sql", + "0004_mail_reliability_guards.sql" +] as const; diff --git a/worker/features/updates/migration-state.ts b/worker/features/updates/migration-state.ts index a486d738..9bd1bf47 100644 --- a/worker/features/updates/migration-state.ts +++ b/worker/features/updates/migration-state.ts @@ -1,39 +1,14 @@ -export const normalMigrationNames = [ - "0001_initial.sql", - "0002_workspace.sql", - "0003_oauth_resources.sql", - "0004_conversations.sql", - "0005_rebuild_threads.sql", - "0006_push_notifications.sql", - "0007_user_mail_preferences.sql", - "0008_user_onboarding.sql", - "0009_login_email_domain_isolation.sql", - "0010_oauth_device_authorization.sql", - "0011_latest_password_reset_token.sql", - "0012_message_activity_index.sql", - "0013_message_changes.sql", - "0014_unassigned_messages.sql", - "0015_draft_changes.sql", - "0016_one_address_per_mailbox.sql", - "0017_agent_principals.sql", - "0018_mailbox_lifecycle.sql", - "0019_contacts.sql", - "0020_labels.sql", - "0021_email_signatures.sql", - "0022_login_email_domain_exact_match.sql", - "0023_message_sender_names.sql", - "0024_draft_inline_images.sql", - "0025_activate_catch_all_policy.sql", - "0026_domain_disconnect.sql", - "0027_message_attachment_disposition.sql", - "0028_draft_labels.sql" -] as const; +import { + afterDeployMigrationNames, + normalMigrationNames, + pendingSendGuards +} from "./migration-names"; -export const afterDeployMigrationNames = [ - "0001_remove_mailbox_alias_storage.sql", - "0002_finalize_agent_principals.sql", - "0003_finalize_draft_labels.sql" -] as const; +export { + afterDeployMigrationNames, + normalMigrationNames, + pendingSendGuards +} from "./migration-names"; const aliasTables = ["mailbox_address_migration", "mailbox_addresses"] as const; const aliasMessageColumns = ["delivered_to_address_id", "sent_from_address_id"] as const; @@ -57,7 +32,7 @@ export const transitionGuards = [ ] as const; export type ManagedMigrationState = { - completedAfterDeployMigrations: 0 | 1 | 2 | 3; + completedAfterDeployMigrations: 0 | 1 | 2 | 3 | 4; repairRequired: boolean; state: "stale" | "partial" | "clean"; }; @@ -88,6 +63,7 @@ export type ManagedMigrationSnapshot = { } | null; tables: string[]; transitionGuards: string[]; + pendingSendGuards: string[]; }; export async function inspectManagedMigrationState( @@ -104,6 +80,7 @@ export async function inspectManagedMigrationState( 'd1_migrations_after_deploy', 'mailbox_address_migration', 'mailbox_addresses', + ${pendingSendGuards.map((name) => `'${name}'`).join(", ")}, ${transitionGuards.map((name) => `'${name}'`).join(", ")} )` ) @@ -166,8 +143,16 @@ export async function inspectManagedMigrationState( tables: foundTables.filter((name) => aliasTables.includes(name as (typeof aliasTables)[number]) ), + pendingSendGuards: objects.results + .filter( + ({ type, name }) => + type === "trigger" && pendingSendGuards.some((guard) => guard === name) + ) + .map(({ name }) => name), transitionGuards: objects.results - .filter(({ type }) => type === "trigger") + .filter( + ({ type, name }) => type === "trigger" && transitionGuards.some((guard) => guard === name) + ) .map(({ name }) => name) }, expectedVersion, @@ -245,19 +230,22 @@ export function classifyManagedMigrationState( const valid = baseSchemaPresent && + (completedCount === 4 + ? includesExactly(snapshot.pendingSendGuards, pendingSendGuards) + : snapshot.pendingSendGuards.length === 0) && ((completedCount === 0 && aliasLegacy && principalLegacy && draftForeignKeys.length === 0) || (completedCount === 1 && aliasFinal && principalLegacy && draftForeignKeys.length === 0) || (completedCount === 2 && aliasFinal && principalFinal && draftForeignKeys.length === 0) || - (completedCount === 3 && aliasFinal && principalFinal && hasFinalDraftLabelForeignKey)); + (completedCount >= 3 && aliasFinal && principalFinal && hasFinalDraftLabelForeignKey)); if (!valid) throw inconsistentState(); return { - completedAfterDeployMigrations: completedCount as 0 | 1 | 2 | 3, + completedAfterDeployMigrations: completedCount as 0 | 1 | 2 | 3 | 4, repairRequired: completedCount < afterDeployMigrationNames.length || pendingUpdate !== null, state: completedCount === 0 ? "stale" - : completedCount === 3 && pendingUpdate === null + : completedCount === afterDeployMigrationNames.length && pendingUpdate === null ? "clean" : "partial" }; diff --git a/worker/features/updates/service.ts b/worker/features/updates/service.ts index 1f86aff1..227afffc 100644 --- a/worker/features/updates/service.ts +++ b/worker/features/updates/service.ts @@ -26,6 +26,7 @@ import { import { cloudflare, isAmbiguousCloudflareOperation } from "./cloudflare"; import { inspectManagedMigrationState, type ManagedMigrationState } from "./migration-state"; import type { ReleaseManifest, UpdateStatus } from "./types"; +import { findZoneAccount } from "./zone-account"; export { isManagedDeployCommand, managedDeployCommand, managedUpdaterLoader }; @@ -139,22 +140,7 @@ export async function triggerUpdate( if (!domain) throw new AppError("UPDATE_DOMAIN_REQUIRED", "Configure the workspace portal first.", 409); const headers = { authorization: `Bearer ${apiToken}`, "content-type": "application/json" }; - const zones = await cloudflare<{ result: Array<{ name: string; account: { id: string } }> }>( - "https://api.cloudflare.com/client/v4/zones?per_page=50", - { headers }, - fetcher, - "read_zones" - ); - const zone = zones.result - .filter((candidate) => domain === candidate.name || domain.endsWith(`.${candidate.name}`)) - .sort((left, right) => right.name.length - left.name.length)[0]; - if (!zone) - throw new AppError( - "UPDATE_ACCOUNT_NOT_FOUND", - "The token cannot access the workspace zone.", - 403 - ); - const accountId = zone.account.id; + const accountId = await findZoneAccount(domain, headers, fetcher); const scripts = await cloudflare<{ result: Array<{ id: string; tag?: string }> }>( `https://api.cloudflare.com/client/v4/accounts/${accountId}/workers/scripts`, { headers }, diff --git a/worker/features/updates/zone-account.ts b/worker/features/updates/zone-account.ts new file mode 100644 index 00000000..4973e7e5 --- /dev/null +++ b/worker/features/updates/zone-account.ts @@ -0,0 +1,43 @@ +import { AppError } from "../../lib/errors"; +import { cloudflare } from "./cloudflare"; + +type ZonePage = { + result: Array<{ name: string; account: { id: string } }>; + result_info?: { total_pages?: number }; +}; + +export async function findZoneAccount( + domain: string, + headers: HeadersInit, + fetcher: typeof fetch +): Promise { + let match: ZonePage["result"][number] | undefined; + for (let page = 1; page <= 1_000; page += 1) { + const zones = await cloudflare( + `https://api.cloudflare.com/client/v4/zones?per_page=50${page === 1 ? "" : `&page=${page}`}`, + { headers }, + fetcher, + "read_zones" + ); + for (const zone of zones.result) { + if ( + (domain === zone.name || domain.endsWith(`.${zone.name}`)) && + (!match || zone.name.length > match.name.length) + ) + match = zone; + } + if (zones.result.length < 50 || page >= (zones.result_info?.total_pages ?? Infinity)) { + if (match) return match.account.id; + throw new AppError( + "UPDATE_ACCOUNT_NOT_FOUND", + "No accessible zone matches the workspace portal.", + 403 + ); + } + } + throw new AppError( + "UPDATE_ZONE_LIMIT", + "Zone discovery exceeded its limit. Use a token scoped to the workspace zone.", + 503 + ); +} diff --git a/worker/features/users/queries.ts b/worker/features/users/queries.ts index dd666102..3e469840 100644 --- a/worker/features/users/queries.ts +++ b/worker/features/users/queries.ts @@ -1,7 +1,8 @@ import { and, eq, like, sql } from "drizzle-orm"; import { createDatabase, getRow, getRows } from "../../db/drizzle"; -import { userOnboarding, users, verifications } from "../../db/schema"; +import { userOnboarding, verifications } from "../../db/schema"; +import { AppError } from "../../lib/errors"; import type { WorkspaceRole } from "../../lib/validation"; import type { UserOnboardingMethod, UserRow, WorkspaceUser } from "./types"; @@ -84,13 +85,25 @@ export async function clearInvitationSentAt(db: D1Database, userId: string): Pro export async function setWorkspaceUserRole( db: D1Database, userId: string, - role: WorkspaceRole + role: WorkspaceRole, + actorId: string ): Promise { - await createDatabase(db) - .update(users) - .set({ role, updatedAt: new Date().toISOString() }) - .where(eq(users.id, userId)) - .run(); + const result = await createDatabase(db).run(sql` + UPDATE "user" SET role = ${role}, updatedAt = ${new Date().toISOString()} + WHERE id = ${userId} AND COALESCE(banned, 0) = 0 + AND (role <> 'owner' OR ${role} = 'owner' OR + (SELECT COUNT(*) FROM "user" WHERE role = 'owner' AND COALESCE(banned, 0) = 0) > 1) + AND EXISTS (SELECT 1 FROM "user" actor WHERE actor.id = ${actorId} + AND COALESCE(actor.banned, 0) = 0 + AND (actor.role = 'owner' OR + (actor.role = 'admin' AND "user".role <> 'owner' AND ${role} <> 'owner')))`); + if ((result.meta.changes ?? 0) === 0) { + throw new AppError( + "USER_ROLE_CONFLICT", + "The role could not change. Keep an active owner and refresh the user list.", + 409 + ); + } } function mapUser(row: UserRow): WorkspaceUser { diff --git a/worker/features/users/routes.ts b/worker/features/users/routes.ts index e611fbf1..8c535b54 100644 --- a/worker/features/users/routes.ts +++ b/worker/features/users/routes.ts @@ -107,18 +107,7 @@ userRoutes.patch("/:id", async (c) => { if ((input.role === "owner" || target.role === "owner") && authContext.user.role !== "owner") { throw new AppError("OWNER_REQUIRED", "Only an owner can change owner membership.", 403); } - if (target.role === "owner" && input.role !== "owner") { - const owners = await getRow<{ count: number }>( - c.env.DB, - sql`SELECT COUNT(*) AS count - FROM "user" - WHERE role = 'owner' AND COALESCE(banned, 0) = 0` - ); - if ((owners?.count ?? 0) <= 1) { - throw new AppError("LAST_OWNER", "The last active owner cannot be demoted.", 409); - } - } - await setWorkspaceUserRole(c.env.DB, c.req.param("id"), input.role); + await setWorkspaceUserRole(c.env.DB, c.req.param("id"), input.role, authContext.user.id); await recordAudit(c.env.DB, { correlationId: c.get("correlationId"), actorType: "user", diff --git a/worker/jobs/consumer.ts b/worker/jobs/consumer.ts index b08fdcc1..96d4ca19 100644 --- a/worker/jobs/consumer.ts +++ b/worker/jobs/consumer.ts @@ -1,171 +1,132 @@ -import { eq, lt, sql } from "drizzle-orm"; - +import { and, eq, sql } from "drizzle-orm"; import { nowIso } from "../db/client"; -import { createDatabase, getRow, getRows } from "../db/drizzle"; -import { messages, operationRuns, rateLimits } from "../db/schema"; -import { - ignoreMailEventFailure, - type MessageEventTarget, - publishMessageMailEvent -} from "../features/events/service"; +import { createDatabase, getRow } from "../db/drizzle"; +import { operationRuns } from "../db/schema"; import type { WorkerEnv } from "../lib/env"; import { operationalLog } from "../observability/log"; +import { applyRetention, deleteExpiredRows } from "./maintenance"; +import { countObjectReferences, scanObjectPage } from "./object-scan"; import { isJob, type Job } from "./types"; -const batchSize = 100; -const orphanGraceMs = 24 * 60 * 60 * 1_000; -// ponytail: Persist a cursor if a workspace needs complete sweeps beyond this daily bound. -const r2ObjectScanLimit = 10_000; -const r2PageSize = 1_000; - -async function deleteExpiredRows(env: WorkerEnv): Promise> { - const nowSeconds = Math.floor(Date.now() / 1000); - const result = await createDatabase(env.DB) - .delete(rateLimits) - .where(lt(rateLimits.expiresAt, nowSeconds)) - .run(); - return { - rateLimits: result.meta.changes ?? 0 - }; -} - -async function applyRetention(env: WorkerEnv): Promise { - const expired = await getRows<{ - id: string; - is_unassigned: number; - mailbox_id: string | null; - raw_r2_key: string | null; - }>( - env.DB, - sql`SELECT m.id, m.mailbox_id, m.is_unassigned, m.raw_r2_key FROM messages m - JOIN retention_policies p ON p.mailbox_id = m.mailbox_id - WHERE (m.folder = 'trash' - AND COALESCE(m.trashed_at, m.updated_at) < datetime('now', '-' || p.trash_days || ' days')) - OR (p.message_days IS NOT NULL - AND m.created_at < datetime('now', '-' || p.message_days || ' days')) - ORDER BY m.created_at, m.id LIMIT ${batchSize}` - ); - const database = createDatabase(env.DB); - for (const message of expired) { - const attachments = await getRows<{ r2_key: string }>( - env.DB, - sql`SELECT r2_key FROM message_attachments WHERE message_id = ${message.id}` - ); - await database.delete(messages).where(eq(messages.id, message.id)).run(); - const candidates = [message.raw_r2_key, ...attachments.map((row) => row.r2_key)].filter( - (key): key is string => Boolean(key) - ); - const keys: string[] = []; - for (const key of candidates) { - const reference = await hasObjectReference(env.DB, key); - if (!reference) keys.push(key); - } - if (keys.length) await env.MAIL_OBJECTS.delete(keys); - } - const targets: MessageEventTarget[] = expired.map((message) => ({ - isUnassigned: message.is_unassigned === 1, - mailboxId: message.mailbox_id - })); - await ignoreMailEventFailure(publishMessageMailEvent(env, targets)); - return expired.length; -} - -async function integrityCounters(env: WorkerEnv): Promise> { - const database = await getRow<{ raw_refs: number; attachment_refs: number }>( - env.DB, - sql`SELECT - (SELECT COUNT(*) FROM messages WHERE raw_r2_key IS NOT NULL) AS raw_refs, - (SELECT COUNT(*) FROM message_attachments) AS attachment_refs` - ); - let listed = 0; - let orphaned = 0; - let cursor: string | undefined; - do { - const limit = Math.min(r2PageSize, r2ObjectScanLimit - listed); - const page = await env.MAIL_OBJECTS.list(cursor ? { cursor, limit } : { limit }); - listed += page.objects.length; - for (const object of page.objects) { - const referenced = await hasObjectReference(env.DB, object.key); - if (!referenced) orphaned += 1; - } - cursor = page.truncated ? page.cursor : undefined; - } while (cursor && listed < r2ObjectScanLimit); - return { - rawReferences: database?.raw_refs ?? 0, - attachmentReferences: database?.attachment_refs ?? 0, - r2ObjectsScanned: listed, - orphanedR2Objects: orphaned - }; -} - -export async function removeExpiredOrphanedObjects( - env: WorkerEnv, - now = Date.now() -): Promise { - const cutoff = now - orphanGraceMs; - const orphaned: string[] = []; - let listed = 0; - let cursor: string | undefined; - do { - const limit = Math.min(r2PageSize, r2ObjectScanLimit - listed); - const page = await env.MAIL_OBJECTS.list(cursor ? { cursor, limit } : { limit }); - listed += page.objects.length; - for (const object of page.objects) { - if (object.uploaded.getTime() >= cutoff) continue; - const referenced = await hasObjectReference(env.DB, object.key); - if (!referenced) orphaned.push(object.key); - } - cursor = page.truncated ? page.cursor : undefined; - } while (cursor && listed < r2ObjectScanLimit); - - for (let start = 0; start < orphaned.length; start += r2PageSize) { - await env.MAIL_OBJECTS.delete(orphaned.slice(start, start + r2PageSize)); - } - return orphaned.length; -} +type Progress = { phase: "expiry" | "retention" | "objects"; r2Cursor?: string }; +type Operation = { + status: string; + cursor: string | null; + counters_json: string; + lease_token: string | null; +}; +const leaseMs = 5 * 60 * 1_000; -async function processJob(env: WorkerEnv, job: Job): Promise { - const startedAt = nowIso(); +async function processJob(env: WorkerEnv, job: Job): Promise { const database = createDatabase(env.DB); - const inserted = await database + const token = crypto.randomUUID(); + const now = nowIso(); + await database .insert(operationRuns) - .values({ id: job.id, kind: job.kind, status: "running", counters: {}, startedAt }) + .values({ + id: job.id, + kind: job.kind, + status: "running", + counters: {}, + startedAt: now + }) .onConflictDoNothing() .run(); - if ((inserted.meta.changes ?? 0) === 0) return; + const claim = await database + .update(operationRuns) + .set({ + status: "running", + attempts: sql`${operationRuns.attempts} + 1`, + leaseToken: token, + leaseExpiresAt: new Date(Date.now() + leaseMs).toISOString(), + errorCode: null, + finishedAt: null + }) + .where( + and( + eq(operationRuns.id, job.id), + sql`${operationRuns.status} <> 'succeeded'`, + sql`(${operationRuns.leaseToken} IS NULL OR ${operationRuns.leaseExpiresAt} <= ${now})` + ) + ) + .run(); + const row = await getRow( + env.DB, + sql`SELECT status, cursor, counters_json, lease_token FROM operation_runs WHERE id = ${job.id}` + ); + if ((claim.meta.changes ?? 0) === 0) return row?.status === "succeeded"; + if (!row || row.lease_token !== token) return false; + const owned = and(eq(operationRuns.id, job.id), eq(operationRuns.leaseToken, token)); try { - const counters = - job.kind === "maintenance" - ? { - ...(await deleteExpiredRows(env)), - retainedMessages: await applyRetention(env), - removedR2Orphans: await removeExpiredOrphanedObjects(env) - } - : await integrityCounters(env); - await database + const progress: Progress = row.cursor + ? JSON.parse(row.cursor) + : { phase: job.kind === "maintenance" ? "expiry" : "objects" }; + const counters: Record = JSON.parse(row.counters_json); + const portion = await runPortion(env, job, progress); + for (const [key, value] of Object.entries(portion.counters)) + counters[key] = (counters[key] ?? 0) + value; + const complete = portion.next === null; + const saved = await database .update(operationRuns) - .set({ status: "succeeded", counters, finishedAt: nowIso() }) - .where(eq(operationRuns.id, job.id)) + .set({ + status: complete ? "succeeded" : "running", + counters, + cursor: portion.next ? JSON.stringify(portion.next) : null, + leaseToken: null, + leaseExpiresAt: null, + finishedAt: complete ? nowIso() : null + }) + .where(owned) .run(); - operationalLog("info", "job_succeeded", { jobId: job.id, kind: job.kind }); + if ((saved.meta.changes ?? 0) === 0) return false; + if (!complete) { + if (!env.HQBASE_JOBS) throw new Error("HQBASE_JOBS binding is required."); + await env.HQBASE_JOBS.send(job); + } else operationalLog("info", "job_succeeded", { jobId: job.id, kind: job.kind }); + return true; } catch (error) { await database .update(operationRuns) - .set({ status: "failed", errorCode: "JOB_FAILED", finishedAt: nowIso() }) - .where(eq(operationRuns.id, job.id)) - .run(); + .set({ + status: "failed", + errorCode: "JOB_FAILED", + leaseToken: null, + leaseExpiresAt: null, + finishedAt: nowIso() + }) + .where(owned) + .run() + .catch(() => undefined); operationalLog("error", "job_failed", { jobId: job.id, kind: job.kind }); throw error; } } -function hasObjectReference(db: D1Database, key: string): Promise { - return getRow( - db, - sql`SELECT 1 FROM messages WHERE raw_r2_key = ${key} OR html_r2_key = ${key} - UNION ALL SELECT 1 FROM message_attachments WHERE r2_key = ${key} - UNION ALL SELECT 1 FROM draft_attachments WHERE r2_key = ${key} LIMIT 1` +async function runPortion( + env: WorkerEnv, + job: Job, + progress: Progress +): Promise<{ next: Progress | null; counters: Record }> { + if (progress.phase === "expiry") { + const result = await deleteExpiredRows(env, job.requestedAt); + return { next: { phase: result.more ? "expiry" : "retention" }, counters: result.counters }; + } + if (progress.phase === "retention") { + const result = await applyRetention(env, job.requestedAt); + return { next: { phase: result.more ? "retention" : "objects" }, counters: result.counters }; + } + const references = + !progress.r2Cursor && job.kind === "integrity-scan" ? await countObjectReferences(env) : {}; + const page = await scanObjectPage( + env, + progress.r2Cursor, + job.kind === "maintenance", + Date.parse(job.requestedAt) ); + return { + next: page.cursor ? { phase: "objects", r2Cursor: page.cursor } : null, + counters: { ...references, ...page.counters } + }; } export async function consumeJobs(batch: MessageBatch, env: WorkerEnv): Promise { @@ -176,10 +137,18 @@ export async function consumeJobs(batch: MessageBatch, env: WorkerEnv): Pro continue; } try { - await processJob(env, message.body); - message.ack(); + if (await processJob(env, message.body)) message.ack(); + else message.retry({ delaySeconds: 60 }); } catch { message.retry(); } } } + +/** One portion; queue consumers use the saved cursor to continue the complete scan. */ +export async function removeExpiredOrphanedObjects( + env: WorkerEnv, + now = Date.now() +): Promise { + return (await scanObjectPage(env, undefined, true, now)).counters.removedR2Orphans ?? 0; +} diff --git a/worker/jobs/maintenance.ts b/worker/jobs/maintenance.ts new file mode 100644 index 00000000..f4bf4c9e --- /dev/null +++ b/worker/jobs/maintenance.ts @@ -0,0 +1,96 @@ +import { sql } from "drizzle-orm"; +import { getRows } from "../db/drizzle"; +import { ignoreMailEventFailure, publishMessageMailEvent } from "../features/events/service"; +import { referencedObjectKeys } from "../features/messages/object-references"; +import type { WorkerEnv } from "../lib/env"; + +export const defaultTrashDays = 30; +const batchSize = 100; + +export async function deleteExpiredRows(env: WorkerEnv, now: string) { + const oldOperations = new Date(Date.parse(now) - 90 * 24 * 60 * 60 * 1000).toISOString(); + const statements = [ + env.DB.prepare( + "DELETE FROM rate_limits WHERE rowid IN (SELECT rowid FROM rate_limits WHERE expires_at < ? LIMIT ?)" + ).bind(Math.floor(Date.parse(now) / 1000), batchSize), + env.DB.prepare( + 'DELETE FROM "session" WHERE id IN (SELECT id FROM "session" WHERE "expiresAt" < ? LIMIT ?)' + ).bind(now, batchSize), + env.DB.prepare( + 'DELETE FROM verification WHERE id IN (SELECT id FROM verification WHERE "expiresAt" < ? LIMIT ?)' + ).bind(now, batchSize), + env.DB.prepare( + "DELETE FROM operation_runs WHERE id IN (SELECT id FROM operation_runs WHERE status = 'succeeded' AND finished_at < ? LIMIT ?)" + ).bind(oldOperations, batchSize) + ]; + const results = await env.DB.batch(statements); + const names = ["rateLimits", "expiredSessions", "expiredVerifications", "oldOperations"]; + const counters = Object.fromEntries( + results.map((result, index) => [names[index], result.meta.changes ?? 0]) + ); + return { more: results.some((result) => (result.meta.changes ?? 0) === batchSize), counters }; +} + +export async function applyRetention(env: WorkerEnv, now: string) { + const expired = await getRows<{ + id: string; + is_unassigned: number; + mailbox_id: string | null; + raw_r2_key: string | null; + html_r2_key: string | null; + text_r2_key: string | null; + }>( + env.DB, + sql`SELECT m.id, m.mailbox_id, m.is_unassigned, m.raw_r2_key, m.html_r2_key, m.text_r2_key + FROM messages m LEFT JOIN retention_policies p ON p.mailbox_id = m.mailbox_id + WHERE (m.folder = 'trash' AND julianday(COALESCE(m.trashed_at, m.updated_at)) < julianday(${now}) - COALESCE(p.trash_days, ${defaultTrashDays})) + OR (p.message_days IS NOT NULL AND julianday(m.created_at) < julianday(${now}) - p.message_days) + ORDER BY m.created_at, m.id LIMIT ${batchSize}` + ); + if (!expired.length) return { more: false, counters: { retainedMessages: 0 } }; + const ids = JSON.stringify(expired.map((message) => message.id)); + const attachments = await getRows<{ r2_key: string }>( + env.DB, + sql` + SELECT r2_key FROM message_attachments WHERE message_id IN (SELECT value FROM json_each(${ids}))` + ); + // Check inside the write so a concurrent restore is not deleted by an old read. + const removed = + await env.DB.prepare(`DELETE FROM messages WHERE id IN (SELECT value FROM json_each(?)) + AND ((folder = 'trash' AND julianday(COALESCE(trashed_at, updated_at)) < julianday(?) - COALESCE( + (SELECT trash_days FROM retention_policies WHERE mailbox_id = messages.mailbox_id), ?)) + OR julianday(created_at) < julianday(?) - (SELECT message_days FROM retention_policies WHERE mailbox_id = messages.mailbox_id))`) + .bind(ids, now, defaultTrashDays, now) + .run(); + const keys = [ + ...new Set( + [ + ...expired.flatMap((message) => [ + message.raw_r2_key, + message.html_r2_key, + message.text_r2_key + ]), + ...attachments.map((attachment) => attachment.r2_key) + ].filter((key): key is string => Boolean(key)) + ) + ]; + for (let start = 0; start < keys.length; start += 1_000) { + const portion = keys.slice(start, start + 1_000); + const referenced = await referencedObjectKeys(env.DB, portion); + const unused = portion.filter((key) => !referenced.has(key)); + if (unused.length) await env.MAIL_OBJECTS.delete(unused); + } + await ignoreMailEventFailure( + publishMessageMailEvent( + env, + expired.map((message) => ({ + isUnassigned: message.is_unassigned === 1, + mailboxId: message.mailbox_id + })) + ) + ); + return { + more: expired.length === batchSize, + counters: { retainedMessages: removed.meta.changes ?? 0 } + }; +} diff --git a/worker/jobs/object-scan.ts b/worker/jobs/object-scan.ts new file mode 100644 index 00000000..d408832a --- /dev/null +++ b/worker/jobs/object-scan.ts @@ -0,0 +1,44 @@ +import { sql } from "drizzle-orm"; +import { getRow } from "../db/drizzle"; +import { referencedObjectKeys } from "../features/messages/object-references"; +import type { WorkerEnv } from "../lib/env"; + +const pageSize = 1_000; +const orphanGraceMs = 24 * 60 * 60 * 1_000; + +export async function scanObjectPage( + env: WorkerEnv, + cursor: string | undefined, + remove: boolean, + now = Date.now() +) { + const page = await env.MAIL_OBJECTS.list({ limit: pageSize, ...(cursor ? { cursor } : {}) }); + const candidates = page.objects.filter( + (object) => !remove || object.uploaded.getTime() < now - orphanGraceMs + ); + const referenced = await referencedObjectKeys( + env.DB, + candidates.map((object) => object.key) + ); + const orphaned = candidates + .filter((object) => !referenced.has(object.key)) + .map((object) => object.key); + if (remove && orphaned.length) await env.MAIL_OBJECTS.delete(orphaned); + return { + cursor: page.truncated ? page.cursor : undefined, + counters: { + r2ObjectsScanned: page.objects.length, + [remove ? "removedR2Orphans" : "orphanedR2Objects"]: orphaned.length + } + }; +} + +export async function countObjectReferences(env: WorkerEnv): Promise> { + const row = await getRow<{ raw: number; attachments: number }>( + env.DB, + sql`SELECT + (SELECT COUNT(*) FROM messages WHERE raw_r2_key IS NOT NULL) AS raw, + (SELECT COUNT(*) FROM message_attachments) AS attachments` + ); + return { rawReferences: row?.raw ?? 0, attachmentReferences: row?.attachments ?? 0 }; +} diff --git a/worker/lib/errors.ts b/worker/lib/errors.ts index b02ae6c9..f75bdcf3 100644 --- a/worker/lib/errors.ts +++ b/worker/lib/errors.ts @@ -26,5 +26,19 @@ export function toAppError(error: unknown): AppError { return error; } + let cause = error; + for (let depth = 0; depth < 5 && cause instanceof Error; depth += 1) { + if (cause.message.includes("draft send is pending")) { + return new AppError( + "DRAFT_SEND_PENDING", + "This draft has a pending or uncertain delivery. Do not send another copy.", + 409 + ); + } + if (cause.message.includes("draft was removed before send")) { + return new AppError("DRAFT_CONFLICT", "The draft was removed before sending.", 409); + } + cause = cause.cause; + } return new AppError("INTERNAL_ERROR", "An internal error occurred.", 500); } diff --git a/worker/lib/json.ts b/worker/lib/json.ts index da56e74d..2e485416 100644 --- a/worker/lib/json.ts +++ b/worker/lib/json.ts @@ -1,8 +1,11 @@ import { AppError } from "./errors"; +import { maxJsonBytes, readBoundedBody, requireMediaType } from "./request-body"; export async function readJson(request: Request): Promise { + requireMediaType(request, "application/json"); + const body = await readBoundedBody(request, maxJsonBytes); try { - return await request.json(); + return JSON.parse(new TextDecoder().decode(body)); } catch { throw new AppError("INVALID_JSON", "Request body must be valid JSON.", 400); } diff --git a/worker/lib/request-body.ts b/worker/lib/request-body.ts new file mode 100644 index 00000000..2cb3b04f --- /dev/null +++ b/worker/lib/request-body.ts @@ -0,0 +1,59 @@ +import { AppError } from "./errors"; + +export const maxJsonBytes = 2 * 1024 * 1024; +export const maxUploadBytes = 26 * 1024 * 1024; + +export async function readBoundedBody( + request: Request, + limit: number +): Promise> { + const declared = Number(request.headers.get("content-length")); + if (declared > limit) throw tooLarge(); + if (!request.body) return new Uint8Array(); + const reader = request.body.getReader(); + const chunks: Uint8Array[] = []; + let length = 0; + try { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + length += value.byteLength; + if (length > limit) { + await reader.cancel().catch(() => undefined); + throw tooLarge(); + } + chunks.push(value); + } + } finally { + reader.releaseLock(); + } + const body = new Uint8Array(length); + let offset = 0; + for (const chunk of chunks) { + body.set(chunk, offset); + offset += chunk.byteLength; + } + return body; +} + +export function requireMediaType(request: Request, expected: string): void { + if (request.headers.get("content-type")?.split(";", 1)[0]?.trim().toLowerCase() !== expected) { + throw new AppError("UNSUPPORTED_MEDIA_TYPE", `Content-Type must be ${expected}.`, 415); + } +} + +export async function readUpload(request: Request): Promise { + requireMediaType(request, "multipart/form-data"); + const body = await readBoundedBody(request, maxUploadBytes); + try { + return await new Response(body, { + headers: { "content-type": request.headers.get("content-type") ?? "" } + }).formData(); + } catch { + throw new AppError("INVALID_UPLOAD", "Upload must be valid multipart data.", 400); + } +} + +function tooLarge(): AppError { + return new AppError("REQUEST_TOO_LARGE", "Request body is too large.", 413); +} diff --git a/worker/routes/index.ts b/worker/routes/index.ts index 9eeb443d..5094660b 100644 --- a/worker/routes/index.ts +++ b/worker/routes/index.ts @@ -31,6 +31,7 @@ import { userRoutes } from "../features/users/routes"; import type { HonoApp } from "../lib/env"; import { errorBody, toAppError } from "../lib/errors"; import { jsonResponse } from "../lib/json"; +import { operationalLog } from "../observability/log"; import { enforceRateLimit } from "../security/rate-limit"; import { healthRoutes } from "./health"; @@ -51,6 +52,12 @@ apiRoutes.use("*", async (c, next) => { apiRoutes.onError((error, c) => { const appError = toAppError(error); + if (appError.status >= 500) { + operationalLog("error", "api_request_failed", { + code: appError.code, + requestId: c.get("correlationId") ?? "unavailable" + }); + } const response = jsonResponse(errorBody(appError.code, appError.message), { status: appError.status }); From 7aa5c5f8111ff6c07439aeabc6eed7ab109c27d6 Mon Sep 17 00:00:00 2001 From: olegberman Date: Fri, 4 Sep 2026 21:38:07 -0400 Subject: [PATCH 2/4] Align staging migration checks and completed schema version --- .github/workflows/staging-e2e.yml | 4 ++-- .../0004_mail_reliability_guards.sql | 6 ++++++ test/integration/worker/local-reset.test.ts | 2 +- test/unit/scripts/sql-migrations.test.mjs | 4 ++-- test/unit/scripts/staging-workflow.test.mjs | 16 +++++++++++----- 5 files changed, 22 insertions(+), 10 deletions(-) diff --git a/.github/workflows/staging-e2e.yml b/.github/workflows/staging-e2e.yml index 048848ac..4bef6349 100644 --- a/.github/workflows/staging-e2e.yml +++ b/.github/workflows/staging-e2e.yml @@ -101,7 +101,7 @@ jobs: pnpm exec wrangler d1 migrations apply DB --remote --config "$config" before_deploy=$(pnpm exec wrangler d1 execute DB --remote --config "$config" --json \ --command "SELECT (SELECT value_json FROM app_settings WHERE key = 'sql-upgrade-probe') AS value_json, (SELECT is_unassigned FROM messages WHERE id = 'msg_sql_upgrade') AS is_unassigned, (SELECT from_name FROM messages WHERE id = 'msg_sql_upgrade') AS from_name, (SELECT is_unassigned FROM message_changes WHERE message_id = 'msg_sql_upgrade' LIMIT 1) AS change_is_unassigned, (SELECT catch_all_policy FROM mail_domains WHERE id = 'dom_sql_upgrade') AS catch_all_policy, (SELECT disconnected_at FROM mail_domains WHERE id = 'dom_sql_upgrade') AS disconnected_at, (SELECT COUNT(*) FROM mailboxes WHERE mail_domain_id = 'dom_sql_upgrade') AS migrated_mailbox_count, (SELECT id FROM mailboxes WHERE address = 'alias@sql-upgrade.example.test') AS alias_mailbox_id, (SELECT mailbox_id FROM messages WHERE id = 'msg_sql_alias_upgrade') AS reassigned_mailbox_id, (SELECT delivered_to_address FROM messages WHERE id = 'msg_sql_alias_upgrade') AS delivered_to_address, (SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = 'mailbox_addresses') AS alias_table_count, (SELECT COUNT(*) FROM pragma_table_info('messages') WHERE name IN ('delivered_to_address_id', 'sent_from_address_id')) AS address_id_column_count, (SELECT COUNT(*) FROM pragma_table_info('draft_attachments') WHERE name = 'content_id') AS draft_content_id_column_count, (SELECT COUNT(*) FROM d1_migrations) AS migration_count, (SELECT installed_schema_version FROM release_state WHERE singleton = 1) AS schema_version") - jq -e '.[0].results[0] == {"value_json":"{\"state\":\"preserved\"}","is_unassigned":1,"from_name":null,"change_is_unassigned":1,"catch_all_policy":"unassigned","disconnected_at":null,"migrated_mailbox_count":2,"alias_mailbox_id":"mbx_migrated_addr_sql_upgrade_alias","reassigned_mailbox_id":"mbx_migrated_addr_sql_upgrade_alias","delivered_to_address":"alias@sql-upgrade.example.test","alias_table_count":1,"address_id_column_count":2,"draft_content_id_column_count":1,"migration_count":28,"schema_version":2}' <<<"$before_deploy" + jq -e '.[0].results[0] == {"value_json":"{\"state\":\"preserved\"}","is_unassigned":1,"from_name":null,"change_is_unassigned":1,"catch_all_policy":"unassigned","disconnected_at":null,"migrated_mailbox_count":2,"alias_mailbox_id":"mbx_migrated_addr_sql_upgrade_alias","reassigned_mailbox_id":"mbx_migrated_addr_sql_upgrade_alias","delivered_to_address":"alias@sql-upgrade.example.test","alias_table_count":1,"address_id_column_count":2,"draft_content_id_column_count":1,"migration_count":29,"schema_version":2}' <<<"$before_deploy" - name: Deploy reviewed source candidate run: | config=".hqbase/deployments/$DEPLOYMENT_NAME/wrangler.jsonc" @@ -147,7 +147,7 @@ jobs: pnpm exec wrangler d1 migrations apply DB --remote --config "$after_deploy_config" result=$(pnpm exec wrangler d1 execute DB --remote --config "$config" --json \ --command "SELECT (SELECT value_json FROM app_settings WHERE key = 'sql-upgrade-probe') AS value_json, (SELECT is_unassigned FROM messages WHERE id = 'msg_sql_upgrade') AS is_unassigned, (SELECT from_name FROM messages WHERE id = 'msg_sql_upgrade') AS from_name, (SELECT is_unassigned FROM message_changes WHERE message_id = 'msg_sql_upgrade' LIMIT 1) AS change_is_unassigned, (SELECT catch_all_policy FROM mail_domains WHERE id = 'dom_sql_upgrade') AS catch_all_policy, (SELECT disconnected_at FROM mail_domains WHERE id = 'dom_sql_upgrade') AS disconnected_at, (SELECT COUNT(*) FROM mailboxes WHERE mail_domain_id = 'dom_sql_upgrade') AS migrated_mailbox_count, (SELECT id FROM mailboxes WHERE address = 'alias@sql-upgrade.example.test') AS alias_mailbox_id, (SELECT mailbox_id FROM messages WHERE id = 'msg_sql_alias_upgrade') AS reassigned_mailbox_id, (SELECT delivered_to_address FROM messages WHERE id = 'msg_sql_alias_upgrade') AS delivered_to_address, (SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name IN ('mailbox_addresses', 'mailbox_address_migration')) AS alias_table_count, (SELECT COUNT(*) FROM pragma_table_info('messages') WHERE name IN ('delivered_to_address_id', 'sent_from_address_id')) AS address_id_column_count, (SELECT COUNT(*) FROM pragma_table_info('draft_attachments') WHERE name = 'content_id') AS draft_content_id_column_count, (SELECT COUNT(*) FROM d1_migrations) AS migration_count, (SELECT COUNT(*) FROM d1_migrations_after_deploy) AS post_migration_count, (SELECT installed_schema_version FROM release_state WHERE singleton = 1) AS schema_version") - jq -e '.[0].results[0] == {"value_json":"{\"state\":\"preserved\"}","is_unassigned":1,"from_name":null,"change_is_unassigned":1,"catch_all_policy":"unassigned","disconnected_at":null,"migrated_mailbox_count":2,"alias_mailbox_id":"mbx_migrated_addr_sql_upgrade_alias","reassigned_mailbox_id":"mbx_migrated_addr_sql_upgrade_alias","delivered_to_address":"alias@sql-upgrade.example.test","alias_table_count":0,"address_id_column_count":0,"draft_content_id_column_count":1,"migration_count":28,"post_migration_count":3,"schema_version":3}' <<<"$result" + jq -e '.[0].results[0] == {"value_json":"{\"state\":\"preserved\"}","is_unassigned":1,"from_name":null,"change_is_unassigned":1,"catch_all_policy":"unassigned","disconnected_at":null,"migrated_mailbox_count":2,"alias_mailbox_id":"mbx_migrated_addr_sql_upgrade_alias","reassigned_mailbox_id":"mbx_migrated_addr_sql_upgrade_alias","delivered_to_address":"alias@sql-upgrade.example.test","alias_table_count":0,"address_id_column_count":0,"draft_content_id_column_count":1,"migration_count":29,"post_migration_count":4,"schema_version":4}' <<<"$result" pnpm exec wrangler d1 execute DB --remote --config "$config" \ --command "DELETE FROM messages WHERE id IN ('msg_sql_upgrade', 'msg_sql_alias_upgrade'); DELETE FROM message_changes WHERE message_id IN ('msg_sql_upgrade', 'msg_sql_alias_upgrade'); DELETE FROM threads WHERE id = 'thr_sql_upgrade'; DELETE FROM mailboxes WHERE id IN ('mbx_sql_upgrade', 'mbx_migrated_addr_sql_upgrade_alias'); DELETE FROM mail_domains WHERE id = 'dom_sql_upgrade'; DELETE FROM app_settings WHERE key = 'sql-upgrade-probe'" diff --git a/migrations-after-deploy/0004_mail_reliability_guards.sql b/migrations-after-deploy/0004_mail_reliability_guards.sql index 8f9bee4e..db13f958 100644 --- a/migrations-after-deploy/0004_mail_reliability_guards.sql +++ b/migrations-after-deploy/0004_mail_reliability_guards.sql @@ -35,3 +35,9 @@ BEGIN SELECT RAISE(ABORT, 'draft send is pending'); END; CREATE TRIGGER draft_labels_before_insert_pending_send BEFORE INSERT ON draft_labels WHEN EXISTS (SELECT 1 FROM send_operations WHERE draft_id = NEW.draft_id AND status <> 'stored') BEGIN SELECT RAISE(ABORT, 'draft send is pending'); END; + +UPDATE release_state +SET installed_schema_version = 4, updated_at = datetime('now') +WHERE singleton = 1; + +PRAGMA foreign_key_check; diff --git a/test/integration/worker/local-reset.test.ts b/test/integration/worker/local-reset.test.ts index d27eb4fd..7ee20db3 100644 --- a/test/integration/worker/local-reset.test.ts +++ b/test/integration/worker/local-reset.test.ts @@ -84,7 +84,7 @@ describe("local database reset", () => { env.DB.prepare( "SELECT installed_schema_version FROM release_state WHERE singleton = 1" ).first() - ).resolves.toEqual({ installed_schema_version: 3 }); + ).resolves.toEqual({ installed_schema_version: 4 }); await expect( env.DB.prepare( "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'mailbox_addresses'" diff --git a/test/unit/scripts/sql-migrations.test.mjs b/test/unit/scripts/sql-migrations.test.mjs index 4255ce27..0084b5f1 100644 --- a/test/unit/scripts/sql-migrations.test.mjs +++ b/test/unit/scripts/sql-migrations.test.mjs @@ -456,7 +456,7 @@ describe("SQL migration contract", () => { database .prepare("SELECT installed_schema_version FROM release_state WHERE singleton = 1") .get() - ).toEqual({ installed_schema_version: 3 }); + ).toEqual({ installed_schema_version: 4 }); }); it("preserves populated data through the latest upgrade and skips it on retry", async () => { @@ -765,7 +765,7 @@ describe("SQL migration contract", () => { database .prepare("SELECT installed_schema_version FROM release_state WHERE singleton = 1") .get() - ).toEqual({ installed_schema_version: 3 }); + ).toEqual({ installed_schema_version: 4 }); expect( database.prepare("SELECT 1 FROM sqlite_master WHERE name = 'mailbox_addresses'").get() ).toBeUndefined(); diff --git a/test/unit/scripts/staging-workflow.test.mjs b/test/unit/scripts/staging-workflow.test.mjs index 6cb08a0c..dc9d28b9 100644 --- a/test/unit/scripts/staging-workflow.test.mjs +++ b/test/unit/scripts/staging-workflow.test.mjs @@ -1,4 +1,4 @@ -import { readFileSync } from "node:fs"; +import { readdirSync, readFileSync } from "node:fs"; import { describe, expect, it } from "vitest"; const workflow = readFileSync( @@ -9,6 +9,10 @@ const releaseWorkflow = readFileSync( new URL("../../../.github/workflows/release.yml", import.meta.url), "utf8" ); +const migrationCount = (directory) => + readdirSync(new URL(`../../../${directory}/`, import.meta.url)).filter((name) => + name.endsWith(".sql") + ).length; const readme = readFileSync(new URL("../../../README.md", import.meta.url), "utf8"); describe("staging workflow lifecycle record", () => { @@ -23,8 +27,8 @@ describe("staging workflow lifecycle record", () => { const afterDeployDirectory = workflow.indexOf('"../../../migrations-after-deploy"'); const afterAliasAssertion = workflow.indexOf('"alias_table_count":0'); const afterAddressIdAssertion = workflow.indexOf('"address_id_column_count":0'); - const afterSchemaAssertion = workflow.indexOf('"schema_version":3'); - const finalAssertion = workflow.indexOf('"post_migration_count":3'); + const afterSchemaAssertion = workflow.indexOf('"schema_version":4'); + const finalAssertion = workflow.indexOf('"post_migration_count":4'); const cleanup = workflow.indexOf( "DELETE FROM messages WHERE id IN ('msg_sql_upgrade', 'msg_sql_alias_upgrade')" ); @@ -57,9 +61,11 @@ describe("staging workflow lifecycle record", () => { expect(workflow).toContain('"delivered_to_address":"alias@sql-upgrade.example.test"'); expect(workflow).toContain('"draft_content_id_column_count":1'); expect(workflow).toContain('"disconnected_at":null'); - expect(workflow).toContain('"migration_count":28'); + expect(workflow).toContain(`"migration_count":${migrationCount("migrations")}`); expect(workflow).toContain('"from_name":null'); - expect(workflow).toContain('"post_migration_count":3'); + expect(workflow).toContain( + `"post_migration_count":${migrationCount("migrations-after-deploy")}` + ); expect(workflow).toContain('migrations_table = "d1_migrations_after_deploy"'); expect(workflow).toContain("del(.d1_databases[0].migrations_pattern)"); expect(normalUpgrade.match(/migrations apply DB --remote --config "\$config"/g)).toHaveLength( From 24f2bb45672c74a42325213e7b0a20f884ae486d Mon Sep 17 00:00:00 2001 From: olegberman Date: Fri, 4 Sep 2026 21:47:27 -0400 Subject: [PATCH 3/4] Make authenticated staging probes initialize a fresh workspace --- test/e2e/staging/event-socket.spec.ts | 2 + test/e2e/staging/lifecycle.spec.ts | 43 +------------------- test/e2e/staging/setup.ts | 57 +++++++++++++++++++++++++++ 3 files changed, 61 insertions(+), 41 deletions(-) create mode 100644 test/e2e/staging/setup.ts diff --git a/test/e2e/staging/event-socket.spec.ts b/test/e2e/staging/event-socket.spec.ts index 9fd966a7..63448ee4 100644 --- a/test/e2e/staging/event-socket.spec.ts +++ b/test/e2e/staging/event-socket.spec.ts @@ -2,6 +2,7 @@ import { createHash, randomBytes } from "node:crypto"; import { request as httpsRequest } from "node:https"; import { expect, test } from "@playwright/test"; +import { ensureStagingSetup } from "./setup"; const accessClientId = required("HQBASE_STAGING_ACCESS_CLIENT_ID"); const accessClientSecret = required("HQBASE_STAGING_ACCESS_CLIENT_SECRET"); @@ -24,6 +25,7 @@ type ProbeOutcome = | { kind: "network-error"; code: string }; test("authenticated event WebSocket opens", async ({ page }) => { + await ensureStagingSetup(page.context().request); const login = await page.context().request.post("/api/auth/sign-in/email", { data: { email, password, rememberMe: false }, headers: { origin: stagingUrl } diff --git a/test/e2e/staging/lifecycle.spec.ts b/test/e2e/staging/lifecycle.spec.ts index 9db750e0..30a27d74 100644 --- a/test/e2e/staging/lifecycle.spec.ts +++ b/test/e2e/staging/lifecycle.spec.ts @@ -1,4 +1,3 @@ -import { createCipheriv, createHash, randomBytes } from "node:crypto"; import { type APIRequestContext, expect, @@ -6,6 +5,7 @@ import { test } from "@playwright/test"; import { stagingMailApiPath } from "./mail-api-path"; +import { ensureStagingSetup } from "./setup"; const email = required("HQBASE_STAGING_OWNER_EMAIL"); const password = required("HQBASE_STAGING_OWNER_PASSWORD"); @@ -48,34 +48,7 @@ test("HQBase web lifecycle remains healthy", async ({ page, request }) => { ) .toBe(200); - const status = await request.get("/api/setup/status"); - expect(status.ok()).toBeTruthy(); - const setup = (await status.json()) as { isComplete: boolean }; - if (!setup.isComplete) { - const grantCookie = stagingSetupGrantCookie(required("HQBASE_STAGING_AUTH_SECRET")); - const bootstrap = await request.post("/api/setup/bootstrap", { - data: { - checklistAcknowledged: true, - defaultFromMailboxAddress: sender, - emailDomains: [ - { - catchAllMailboxAddress: sender, - catchAllPolicy: "mailbox", - name: domain - } - ], - mailboxes: [{ address: sender, displayName: "HQBase E2E" }], - ownerEmail: email, - ownerName: "HQBase E2E Owner", - ownerPassword: password, - portalHostname: new URL(stagingUrl).hostname, - primaryDomain: domain - }, - headers: { cookie: grantCookie } - }); - expect(bootstrap.status()).toBe(201); - await expect(bootstrap.json()).resolves.toMatchObject({ setup: { isComplete: true } }); - } + await ensureStagingSetup(request); await expect .poll(async () => { @@ -1009,18 +982,6 @@ function accessHeaders(): Record { : { origin: stagingUrl }; } -function stagingSetupGrantCookie(secret: string): string { - const iv = randomBytes(12); - const key = createHash("sha256").update(`hqbase-runtime-cloudflare-oauth:${secret}`).digest(); - const cipher = createCipheriv("aes-256-gcm", key, iv); - const encrypted = Buffer.concat([ - cipher.update("hqbase-staging-oauth-grant", "utf8"), - cipher.final(), - cipher.getAuthTag() - ]); - return `hqb_cf_oauth_grant=${encodeURIComponent(`${iv.toString("base64url")}.${encrypted.toString("base64url")}`)}`; -} - function required(name: string): string { const value = process.env[name]; if (!value) throw new Error(`${name} is required for HQBase staging E2E.`); diff --git a/test/e2e/staging/setup.ts b/test/e2e/staging/setup.ts new file mode 100644 index 00000000..aa6f7baf --- /dev/null +++ b/test/e2e/staging/setup.ts @@ -0,0 +1,57 @@ +import { createCipheriv, createHash, randomBytes } from "node:crypto"; +import { type APIRequestContext, expect } from "@playwright/test"; + +// Each authenticated probe also works on a fresh disposable installation. +export async function ensureStagingSetup(request: APIRequestContext): Promise { + const stagingUrl = required("HQBASE_STAGING_URL"); + const status = await request.get("/api/setup/status"); + expect(status.ok()).toBeTruthy(); + const setup = (await status.json()) as { isComplete: boolean }; + if (!setup.isComplete) { + const email = required("HQBASE_STAGING_OWNER_EMAIL"); + const password = required("HQBASE_STAGING_OWNER_PASSWORD"); + const sender = required("HQBASE_STAGING_SENDER"); + const domain = required("HQBASE_STAGING_EMAIL_DOMAIN"); + const grantCookie = stagingSetupGrantCookie(required("HQBASE_STAGING_AUTH_SECRET")); + const bootstrap = await request.post("/api/setup/bootstrap", { + data: { + checklistAcknowledged: true, + defaultFromMailboxAddress: sender, + emailDomains: [ + { + catchAllMailboxAddress: sender, + catchAllPolicy: "mailbox", + name: domain + } + ], + mailboxes: [{ address: sender, displayName: "HQBase E2E" }], + ownerEmail: email, + ownerName: "HQBase E2E Owner", + ownerPassword: password, + portalHostname: new URL(stagingUrl).hostname, + primaryDomain: domain + }, + headers: { cookie: grantCookie, origin: new URL(stagingUrl).origin } + }); + expect(bootstrap.status()).toBe(201); + await expect(bootstrap.json()).resolves.toMatchObject({ setup: { isComplete: true } }); + } +} + +function stagingSetupGrantCookie(secret: string): string { + const iv = randomBytes(12); + const key = createHash("sha256").update(`hqbase-runtime-cloudflare-oauth:${secret}`).digest(); + const cipher = createCipheriv("aes-256-gcm", key, iv); + const encrypted = Buffer.concat([ + cipher.update("hqbase-staging-oauth-grant", "utf8"), + cipher.final(), + cipher.getAuthTag() + ]); + return `hqb_cf_oauth_grant=${encodeURIComponent(`${iv.toString("base64url")}.${encrypted.toString("base64url")}`)}`; +} + +function required(name: string): string { + const value = process.env[name]; + if (!value) throw new Error(`${name} is required for HQBase staging setup.`); + return value; +} From 7766795c35b15063b9fdd42281d4179a4fc0dcd1 Mon Sep 17 00:00:00 2001 From: olegberman Date: Fri, 4 Sep 2026 21:56:03 -0400 Subject: [PATCH 4/4] test: separate MCP send signature cases --- test/integration/worker/mcp.test.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/test/integration/worker/mcp.test.ts b/test/integration/worker/mcp.test.ts index 1e4bedce..4cf5336e 100644 --- a/test/integration/worker/mcp.test.ts +++ b/test/integration/worker/mcp.test.ts @@ -652,7 +652,7 @@ describe("HQBase MCP server", () => { ).resolves.toEqual({ count: 1 }); }); - it("defaults MCP sends, replies, and forwards to the automatic signature", async () => { + it("defaults MCP sends to the automatic signature", async () => { const sent = (await callTool( "send_email", { @@ -665,7 +665,9 @@ describe("HQBase MCP server", () => { "/mcp/full" )) as { id: string }; expect(await storedText(sent.id)).toBe("Send authored\n\nMCP signature"); + }); + it("defaults MCP replies to the automatic signature", async () => { const replied = (await callTool( "reply_to_message", { @@ -681,7 +683,9 @@ describe("HQBase MCP server", () => { expect(replyText.indexOf("MCP signature")).toBeLessThan( replyText.indexOf("Earlier signature body") ); + }); + it("defaults MCP forwards to the automatic signature", async () => { const forwarded = (await callTool( "forward_message", { @@ -701,7 +705,9 @@ describe("HQBase MCP server", () => { expect(forwardText.indexOf("MCP signature")).toBeLessThan( forwardText.indexOf("Forwarded message") ); + }); + it("keeps the automatic signature and attachments when sending a saved MCP forward", async () => { const savedForwardDraft = (await callTool( "create_draft", { @@ -744,7 +750,9 @@ describe("HQBase MCP server", () => { expect( await env.DB.prepare("SELECT id FROM drafts WHERE id = ?").bind(savedForwardDraft.id).first() ).toBeNull(); + }); + it("omits the MCP signature when requested", async () => { const unsigned = (await callTool( "send_email", {