Fix mail integrity, send recovery, concurrency, and maintenance

.github/workflows/dependency-audit.yml

@@ -0,0 +1,22 @@
+name: Dependency audit
+on:
+  pull_request:
+  push:
+    branches: [main]
+  schedule:
+    - cron: '17 10 * * 1'
+  workflow_dispatch:
+permissions:
+  contents: read
+jobs:
+  audit:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+      - uses: pnpm/action-setup@v6
+      - uses: actions/setup-node@v6
+        with:
+          node-version: 24
+          cache: pnpm
+      - run: pnpm install --frozen-lockfile
+      - run: pnpm audit --prod --audit-level=moderate

.github/workflows/staging-e2e.yml

@@ -101,7 +101,7 @@ jobs:
           pnpm exec wrangler d1 migrations apply DB --remote --config "$config"
           before_deploy=$(pnpm exec wrangler d1 execute DB --remote --config "$config" --json \
             --command "SELECT (SELECT value_json FROM app_settings WHERE key = 'sql-upgrade-probe') AS value_json, (SELECT is_unassigned FROM messages WHERE id = 'msg_sql_upgrade') AS is_unassigned, (SELECT from_name FROM messages WHERE id = 'msg_sql_upgrade') AS from_name, (SELECT is_unassigned FROM message_changes WHERE message_id = 'msg_sql_upgrade' LIMIT 1) AS change_is_unassigned, (SELECT catch_all_policy FROM mail_domains WHERE id = 'dom_sql_upgrade') AS catch_all_policy, (SELECT disconnected_at FROM mail_domains WHERE id = 'dom_sql_upgrade') AS disconnected_at, (SELECT COUNT(*) FROM mailboxes WHERE mail_domain_id = 'dom_sql_upgrade') AS migrated_mailbox_count, (SELECT id FROM mailboxes WHERE address = 'alias@sql-upgrade.example.test') AS alias_mailbox_id, (SELECT mailbox_id FROM messages WHERE id = 'msg_sql_alias_upgrade') AS reassigned_mailbox_id, (SELECT delivered_to_address FROM messages WHERE id = 'msg_sql_alias_upgrade') AS delivered_to_address, (SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = 'mailbox_addresses') AS alias_table_count, (SELECT COUNT(*) FROM pragma_table_info('messages') WHERE name IN ('delivered_to_address_id', 'sent_from_address_id')) AS address_id_column_count, (SELECT COUNT(*) FROM pragma_table_info('draft_attachments') WHERE name = 'content_id') AS draft_content_id_column_count, (SELECT COUNT(*) FROM d1_migrations) AS migration_count, (SELECT installed_schema_version FROM release_state WHERE singleton = 1) AS schema_version")
-          jq -e '.[0].results[0] == {"value_json":"{\"state\":\"preserved\"}","is_unassigned":1,"from_name":null,"change_is_unassigned":1,"catch_all_policy":"unassigned","disconnected_at":null,"migrated_mailbox_count":2,"alias_mailbox_id":"mbx_migrated_addr_sql_upgrade_alias","reassigned_mailbox_id":"mbx_migrated_addr_sql_upgrade_alias","delivered_to_address":"alias@sql-upgrade.example.test","alias_table_count":1,"address_id_column_count":2,"draft_content_id_column_count":1,"migration_count":28,"schema_version":2}' <<<"$before_deploy"
+          jq -e '.[0].results[0] == {"value_json":"{\"state\":\"preserved\"}","is_unassigned":1,"from_name":null,"change_is_unassigned":1,"catch_all_policy":"unassigned","disconnected_at":null,"migrated_mailbox_count":2,"alias_mailbox_id":"mbx_migrated_addr_sql_upgrade_alias","reassigned_mailbox_id":"mbx_migrated_addr_sql_upgrade_alias","delivered_to_address":"alias@sql-upgrade.example.test","alias_table_count":1,"address_id_column_count":2,"draft_content_id_column_count":1,"migration_count":29,"schema_version":2}' <<<"$before_deploy"
       - name: Deploy reviewed source candidate
         run: |
           config=".hqbase/deployments/$DEPLOYMENT_NAME/wrangler.jsonc"
@@ -147,7 +147,7 @@ jobs:
           pnpm exec wrangler d1 migrations apply DB --remote --config "$after_deploy_config"
           result=$(pnpm exec wrangler d1 execute DB --remote --config "$config" --json \
             --command "SELECT (SELECT value_json FROM app_settings WHERE key = 'sql-upgrade-probe') AS value_json, (SELECT is_unassigned FROM messages WHERE id = 'msg_sql_upgrade') AS is_unassigned, (SELECT from_name FROM messages WHERE id = 'msg_sql_upgrade') AS from_name, (SELECT is_unassigned FROM message_changes WHERE message_id = 'msg_sql_upgrade' LIMIT 1) AS change_is_unassigned, (SELECT catch_all_policy FROM mail_domains WHERE id = 'dom_sql_upgrade') AS catch_all_policy, (SELECT disconnected_at FROM mail_domains WHERE id = 'dom_sql_upgrade') AS disconnected_at, (SELECT COUNT(*) FROM mailboxes WHERE mail_domain_id = 'dom_sql_upgrade') AS migrated_mailbox_count, (SELECT id FROM mailboxes WHERE address = 'alias@sql-upgrade.example.test') AS alias_mailbox_id, (SELECT mailbox_id FROM messages WHERE id = 'msg_sql_alias_upgrade') AS reassigned_mailbox_id, (SELECT delivered_to_address FROM messages WHERE id = 'msg_sql_alias_upgrade') AS delivered_to_address, (SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name IN ('mailbox_addresses', 'mailbox_address_migration')) AS alias_table_count, (SELECT COUNT(*) FROM pragma_table_info('messages') WHERE name IN ('delivered_to_address_id', 'sent_from_address_id')) AS address_id_column_count, (SELECT COUNT(*) FROM pragma_table_info('draft_attachments') WHERE name = 'content_id') AS draft_content_id_column_count, (SELECT COUNT(*) FROM d1_migrations) AS migration_count, (SELECT COUNT(*) FROM d1_migrations_after_deploy) AS post_migration_count, (SELECT installed_schema_version FROM release_state WHERE singleton = 1) AS schema_version")
-          jq -e '.[0].results[0] == {"value_json":"{\"state\":\"preserved\"}","is_unassigned":1,"from_name":null,"change_is_unassigned":1,"catch_all_policy":"unassigned","disconnected_at":null,"migrated_mailbox_count":2,"alias_mailbox_id":"mbx_migrated_addr_sql_upgrade_alias","reassigned_mailbox_id":"mbx_migrated_addr_sql_upgrade_alias","delivered_to_address":"alias@sql-upgrade.example.test","alias_table_count":0,"address_id_column_count":0,"draft_content_id_column_count":1,"migration_count":28,"post_migration_count":3,"schema_version":3}' <<<"$result"
+          jq -e '.[0].results[0] == {"value_json":"{\"state\":\"preserved\"}","is_unassigned":1,"from_name":null,"change_is_unassigned":1,"catch_all_policy":"unassigned","disconnected_at":null,"migrated_mailbox_count":2,"alias_mailbox_id":"mbx_migrated_addr_sql_upgrade_alias","reassigned_mailbox_id":"mbx_migrated_addr_sql_upgrade_alias","delivered_to_address":"alias@sql-upgrade.example.test","alias_table_count":0,"address_id_column_count":0,"draft_content_id_column_count":1,"migration_count":29,"post_migration_count":4,"schema_version":4}' <<<"$result"
 
           pnpm exec wrangler d1 execute DB --remote --config "$config" \
             --command "DELETE FROM messages WHERE id IN ('msg_sql_upgrade', 'msg_sql_alias_upgrade'); DELETE FROM message_changes WHERE message_id IN ('msg_sql_upgrade', 'msg_sql_alias_upgrade'); DELETE FROM threads WHERE id = 'thr_sql_upgrade'; DELETE FROM mailboxes WHERE id IN ('mbx_sql_upgrade', 'mbx_migrated_addr_sql_upgrade_alias'); DELETE FROM mail_domains WHERE id = 'dom_sql_upgrade'; DELETE FROM app_settings WHERE key = 'sql-upgrade-probe'"

api/hqbase-mail-api-v1.openapi.json

@@ -3761,6 +3761,14 @@
                 "items": {
                   "$ref": "#/components/schemas/Attachment"
                 }
+              },
+              "replyTo": {
+                "type": "array",
+                "items": {
+                  "type": "string",
+                  "format": "email"
+                },
+                "description": "Preferred recipients for replies to inbound mail when no explicit recipients are supplied."
               }
             }
           }
@@ -4166,6 +4174,12 @@
           },
           "signature": {
             "$ref": "#/components/schemas/SignatureSelection"
+          },
+          "idempotencyKey": {
+            "type": "string",
+            "minLength": 1,
+            "maxLength": 100,
+            "description": "Reuse this key only for a retry of the same send request. A saved draft already has a stable send identity."
           }
         }
       },
@@ -4222,6 +4236,12 @@
           },
           "signature": {
             "$ref": "#/components/schemas/SignatureSelection"
+          },
+          "idempotencyKey": {
+            "type": "string",
+            "minLength": 1,
+            "maxLength": 100,
+            "description": "Reuse this key only for a retry of the same send request. A saved draft already has a stable send identity."
           }
         }
       },
@@ -4295,6 +4315,12 @@
           },
           "signature": {
             "$ref": "#/components/schemas/SignatureSelection"
+          },
+          "idempotencyKey": {
+            "type": "string",
+            "minLength": 1,
+            "maxLength": 100,
+            "description": "Reuse this key only for a retry of the same send request. A saved draft already has a stable send identity."
           }
         }
       },

api/hqbase-mail-api-v2.openapi.json

@@ -3876,6 +3876,14 @@
                 "items": {
                   "$ref": "#/components/schemas/Attachment"
                 }
+              },
+              "replyTo": {
+                "type": "array",
+                "items": {
+                  "type": "string",
+                  "format": "email"
+                },
+                "description": "Preferred recipients for replies to inbound mail when no explicit recipients are supplied."
               }
             }
           }
@@ -4281,6 +4289,12 @@
           },
           "signature": {
             "$ref": "#/components/schemas/SignatureSelection"
+          },
+          "idempotencyKey": {
+            "type": "string",
+            "minLength": 1,
+            "maxLength": 100,
+            "description": "Reuse this key only for a retry of the same send request. A saved draft already has a stable send identity."
           }
         }
       },
@@ -4337,6 +4351,12 @@
           },
           "signature": {
             "$ref": "#/components/schemas/SignatureSelection"
+          },
+          "idempotencyKey": {
+            "type": "string",
+            "minLength": 1,
+            "maxLength": 100,
+            "description": "Reuse this key only for a retry of the same send request. A saved draft already has a stable send identity."
           }
         }
       },
@@ -4410,6 +4430,12 @@
           },
           "signature": {
             "$ref": "#/components/schemas/SignatureSelection"
+          },
+          "idempotencyKey": {
+            "type": "string",
+            "minLength": 1,
+            "maxLength": 100,
+            "description": "Reuse this key only for a retry of the same send request. A saved draft already has a stable send identity."
           }
         }
       },

app/app.tsx

@@ -143,14 +143,20 @@ export function App(): React.ReactElement {
       const results = await Promise.allSettled([
         refreshWorkspace(),
         hardRefresh ? mailSync.hardRefresh() : mailSync.refresh(),
-        draftState.refresh()
+        hardRefresh ? draftState.hardRefresh() : draftState.refresh()
       ]);
       if (results.every((result) => result.status === "rejected")) {
         const failure = results.find((result) => result.status === "rejected");
         throw failure?.reason;
       }
     },
-    [draftState.refresh, mailSync.hardRefresh, mailSync.refresh, refreshWorkspace]
+    [
+      draftState.hardRefresh,
+      draftState.refresh,
+      mailSync.hardRefresh,
+      mailSync.refresh,
+      refreshWorkspace
+    ]
   );
   const hardRefreshRealtimeState = React.useCallback(
     () => refreshRealtimeState(true),

app/features/compose/compose-state.ts

@@ -115,7 +115,8 @@ export function hasInvalidRecipients(...values: string[]): boolean {
 }
 
 export function replyRecipients(message: MessageDetail): string[] {
-  if (message.direction === "inbound") return [message.fromAddress];
+  if (message.direction === "inbound")
+    return message.replyTo?.length ? message.replyTo : [message.fromAddress];
 
   const sender = message.fromAddress.toLowerCase();
   return message.to.filter((address) => address.toLowerCase() !== sender);

app/features/drafts/api.ts

@@ -1,7 +1,16 @@
-import { apiDelete, apiGetPage, apiPatch, apiPost } from "@/lib/api-client";
+import { apiDelete, apiGet, apiGetPage, apiPatch, apiPost } from "@/lib/api-client";
 
 import type { Draft, DraftAttachment, DraftInput, DraftLabelMutationResult } from "./types";
 
+export type DraftChangePage = {
+  changes: Array<{ type: "upsert"; draft: Draft } | { type: "delete"; draftId: string }>;
+  nextCursor: string;
+  hasMore: boolean;
+};
+
+export const listDraftChanges = (cursor?: string): Promise<DraftChangePage> =>
+  apiGet(`/api/v2/drafts/changes${cursor ? `?cursor=${encodeURIComponent(cursor)}` : ""}`);
+
 export async function listDrafts(): Promise<Draft[]> {
   const drafts: Draft[] = [];
   let nextPageUrl: string | null = "/api/v2/drafts?limit=100";

app/features/drafts/drafts-page.tsx

@@ -1,4 +1,4 @@
-import type * as React from "react";
+import * as React from "react";
 import { PiNotePencil, PiPaperclip } from "react-icons/pi";
 import { Avatar, AvatarFallback } from "@/components/ui/avatar";
 import { Button } from "@/components/ui/button";
@@ -45,6 +45,9 @@ export function DraftsPage({
   onSelect,
   onToggleLabel
 }: DraftsPageProps): React.ReactElement {
+  const filterKey = JSON.stringify([mailboxId, search, labelIds]);
+  const [page, setPage] = React.useState({ filterKey, limit: 50 });
+  const limit = page.filterKey === filterKey ? page.limit : 50;
   const normalizedSearch = search.trim().toLowerCase();
   const visibleDrafts = drafts.filter((draft) => {
     if (mailboxId !== "all" && draft.mailboxId !== mailboxId) return false;
@@ -99,7 +102,7 @@ export function DraftsPage({
     );
   }
 
-  const groups = groupDrafts(visibleDrafts);
+  const groups = groupDrafts(visibleDrafts.slice(0, limit));
 
   return (
     <div className="flex h-full flex-col bg-list" data-mobile-view="message-list">
@@ -149,6 +152,15 @@ export function DraftsPage({
                   </div>
                 </section>
               ))}
+              {visibleDrafts.length > limit ? (
+                <Button
+                  className="mt-4"
+                  variant="outline"
+                  onClick={() => setPage({ filterKey, limit: limit + 50 })}
+                >
+                  Load more drafts
+                </Button>
+              ) : null}
             </div>
           )}
         </div>

app/features/drafts/sync-client.ts

@@ -0,0 +1,54 @@
+import { listDraftChanges, listDrafts } from "./api";
+import type { Draft } from "./types";
+
+/** A cache belongs to one signed-in user. Access changes require a fresh snapshot. */
+export function createDraftSync() {
+  let drafts: Draft[] = [];
+  let cursor: string | null = null;
+  let generation = 0;
+  let inFlight: Promise<Draft[]> | null = null;
+  const refresh = (reset = false): Promise<Draft[]> => {
+    if (reset) {
+      cursor = null;
+      generation += 1;
+    }
+    if (inFlight && !reset) return inFlight;
+    const currentGeneration = generation;
+    const pending = (async () => {
+      let nextCursor = cursor;
+      let snapshot = drafts;
+      if (nextCursor === null) {
+        // Capture the journal boundary before listing so changes during bootstrap are replayed.
+        nextCursor = (await listDraftChanges()).nextCursor;
+        snapshot = await listDrafts();
+      }
+      const byId = new Map(snapshot.map((draft) => [draft.id, draft]));
+      while (true) {
+        const page = await listDraftChanges(nextCursor);
+        for (const change of page.changes) {
+          if (change.type === "delete") byId.delete(change.draftId);
+          else byId.set(change.draft.id, change.draft);
+        }
+        if (page.hasMore && page.nextCursor === nextCursor)
+          throw new Error("Draft cursor did not advance.");
+        nextCursor = page.nextCursor;
+        if (!page.hasMore) break;
+      }
+      if (currentGeneration === generation) {
+        drafts = [...byId.values()].sort(
+          (left, right) =>
+            right.updatedAt.localeCompare(left.updatedAt) || right.id.localeCompare(left.id)
+        );
+        cursor = nextCursor;
+      }
+      return drafts;
+    })();
+    inFlight = pending;
+    const clear = () => {
+      if (inFlight === pending) inFlight = null;
+    };
+    void pending.then(clear, clear);
+    return pending;
+  };
+  return { refresh };
+}

app/features/drafts/use-drafts.ts

@@ -1,32 +1,40 @@
 import * as React from "react";
 import type { MailLabel } from "@/features/labels/types";
-import { listDrafts } from "./api";
+import { createDraftSync } from "./sync-client";
 import type { Draft } from "./types";
 
 export function useDrafts(userId: string | null): {
   drafts: Draft[];
   isLoading: boolean;
   applyLabels: (draftId: string, labels: MailLabel[]) => void;
   refresh: () => Promise<void>;
+  hardRefresh: () => Promise<void>;
 } {
+  const cache = React.useMemo(() => ({ userId, sync: createDraftSync() }), [userId]);
+  const sync = cache.sync;
   const [drafts, setDrafts] = React.useState<Draft[]>([]);
   const [isLoading, setIsLoading] = React.useState(false);
   const currentUserId = React.useRef(userId);
   currentUserId.current = userId;
 
-  const refresh = React.useCallback(async (): Promise<void> => {
-    if (!userId) {
-      setDrafts([]);
-      setIsLoading(false);
-      return;
-    }
+  const refresh = React.useCallback(
+    async (reset = false): Promise<void> => {
+      if (!userId) {
+        setDrafts([]);
+        setIsLoading(false);
+        return;
+      }
 
-    const nextDrafts = await listDrafts();
-    if (currentUserId.current === userId) {
-      setDrafts(nextDrafts);
-      setIsLoading(false);
-    }
-  }, [userId]);
+      const nextDrafts = await sync.refresh(reset);
+      if (currentUserId.current === userId) {
+        setDrafts(nextDrafts);
+        setIsLoading(false);
+      }
+    },
+    [sync, userId]
+  );
+
+  const hardRefresh = React.useCallback(() => refresh(true), [refresh]);
 
   React.useEffect(() => {
     if (!userId) {
@@ -57,5 +65,5 @@ export function useDrafts(userId: string | null): {
     );
   }, []);
 
-  return { applyLabels, drafts, isLoading, refresh };
+  return { applyLabels, drafts, isLoading, refresh, hardRefresh };
 }