diff --git a/.changeset/sys-email-headers-attachments.md b/.changeset/sys-email-headers-attachments.md new file mode 100644 index 0000000000..56e02a752d --- /dev/null +++ b/.changeset/sys-email-headers-attachments.md @@ -0,0 +1,48 @@ +--- +"@objectstack/platform-objects": minor +"@objectstack/plugin-email": minor +--- + +feat(plugin-email,platform-objects): `sys_email` carries headers and small attachments, so those messages become durably deliverable (#5177) + +Durable email delivery works from the **row**, not from the in-memory message: +`send()` publishes an `{ rowId }` job (#5160), the boot sweep re-reads rows +(#5161), and both end at `rowToNormalized`. So anything a `sys_email` row could +not carry, a row-based delivery would have dropped — and custom headers and +attachments were exactly that. The honest workaround was to refuse: a message +with either was pushed back onto inline delivery so that it would at least go +out whole, which closed the durable path to precisely the mail most worth +making durable (a signed receipt, a `List-Unsubscribe` header, an invoice PDF). + +`sys_email` now has two columns, and those messages are queueable. + +**`headers_json`** — the custom headers, as a JSON object. Written in both +delivery modes (it is audit evidence as much as delivery input) and rebuilt on +read. Headers are no longer a reason to fall back to inline delivery. + +**`attachments_json`** — attachments as a JSON array of +`{ filename, contentType?, size, hash, cid?, contentForm, inline?, storageKey? }`, +content base64 in `inline`. Written when the **combined raw size of one +message's attachments is within `SYS_EMAIL_ATTACHMENT_LIMIT_BYTES` (256 KiB, +exported from `@objectstack/plugin-email`)** — worst case ~350 KB of base64, so +a row stays bounded. Both arms of the declared `content: string | Buffer` +contract round-trip as the arm they were sent as: restoring a text attachment +as a Buffer would silently drop `charset=utf-8` from its MIME part and let the +recipient's client mis-decode a UTF-8 file, so `contentForm` records which one +it was. `cid` travels too — an inline `` is unusable without +it. + +**Over the limit, nothing changes.** The message is delivered inline exactly as +before, whole, and the row stores no attachment content; the reason is stated +at `info` (a bound, not a degradation — the worst outcome is today's +behaviour). Out-of-row storage for large attachments is #5172; `storageKey` is +declared now so that lands as a new *producer* rather than a data migration. + +Rows written before these columns exist read exactly as they did. A column that +is present but does not describe what it claims — malformed JSON, a size or +hash that disagrees with the content, a missing `contentForm` — is **rejected**, +and the row lands at `failed` carrying the reason, rather than being delivered +with a part quietly missing. + +The `sys_email` schema change is additive (two optional textarea columns); no +migration is required and default inline delivery is unchanged. diff --git a/packages/platform-objects/src/apps/translations/en.objects.generated.ts b/packages/platform-objects/src/apps/translations/en.objects.generated.ts index d3935d09c5..49f57694e9 100644 --- a/packages/platform-objects/src/apps/translations/en.objects.generated.ts +++ b/packages/platform-objects/src/apps/translations/en.objects.generated.ts @@ -2045,6 +2045,14 @@ export const enObjects: NonNullable = { body_html: { label: "Body (HTML)" }, + headers_json: { + label: "Headers (JSON)", + help: "Custom headers supplied to IEmailService.send, as a JSON object of name → value. Written in both delivery modes (it is audit evidence as much as delivery input). Absent on rows written before this column existed, which read back as \"no custom headers\"." + }, + attachments_json: { + label: "Attachments (JSON)", + help: "Attachments as a JSON array of { filename, contentType?, size, hash, cid?, contentForm, inline?, storageKey? }, with content base64 in `inline`. Written only when the combined raw size is within the plugin-email budget (SYS_EMAIL_ATTACHMENT_LIMIT_BYTES, 256 KiB — ~350 KB of base64 at worst); a larger message is delivered inline and stores nothing here, so the row stays bounded. `storageKey` (out-of-row content) has no producer yet — objectstack#5172." + }, status: { label: "Status", help: "Lifecycle state — queued by IEmailService.send before transport call", diff --git a/packages/platform-objects/src/apps/translations/es-ES.objects.generated.ts b/packages/platform-objects/src/apps/translations/es-ES.objects.generated.ts index a4ed943ac4..b5b43b58c2 100644 --- a/packages/platform-objects/src/apps/translations/es-ES.objects.generated.ts +++ b/packages/platform-objects/src/apps/translations/es-ES.objects.generated.ts @@ -2045,6 +2045,14 @@ export const esESObjects: NonNullable = { body_html: { label: "Contenido (HTML)" }, + headers_json: { + label: "Cabeceras (JSON)", + help: "Cabeceras personalizadas facilitadas a IEmailService.send, como un objeto JSON de nombre → valor. Se escriben en ambos modos de entrega (son tanto evidencia de auditoría como entrada de la entrega). Ausentes en las filas escritas antes de que existiera esta columna, que se leen como «sin cabeceras personalizadas»." + }, + attachments_json: { + label: "Adjuntos (JSON)", + help: "Adjuntos como un array JSON de { filename, contentType?, size, hash, cid?, contentForm, inline?, storageKey? }, con el contenido en base64 en `inline`. Solo se escribe cuando el tamaño bruto combinado está dentro del presupuesto de plugin-email (SYS_EMAIL_ATTACHMENT_LIMIT_BYTES, 256 KiB — ~350 KB de base64 en el peor caso); un mensaje mayor se entrega en línea y aquí no se almacena nada, de modo que la fila permanece acotada. `storageKey` (contenido fuera de la fila) todavía no tiene productor — objectstack#5172." + }, status: { label: "Estado", help: "Estado del ciclo de vida; se pone en cola mediante IEmailService.send antes de la llamada al transporte.", diff --git a/packages/platform-objects/src/apps/translations/ja-JP.objects.generated.ts b/packages/platform-objects/src/apps/translations/ja-JP.objects.generated.ts index 2d0d293327..f828fa921c 100644 --- a/packages/platform-objects/src/apps/translations/ja-JP.objects.generated.ts +++ b/packages/platform-objects/src/apps/translations/ja-JP.objects.generated.ts @@ -2045,6 +2045,14 @@ export const jaJPObjects: NonNullable = { body_html: { label: "本文（HTML）" }, + headers_json: { + label: "ヘッダー（JSON）", + help: "IEmailService.send に渡されたカスタムヘッダーを、name → value の JSON オブジェクトとして保持します。両方の配信モードで書き込まれます（配信の入力であると同時に監査証跡でもあるため）。この列が存在する前に書き込まれた行には値がなく、「カスタムヘッダーなし」として読み戻されます。" + }, + attachments_json: { + label: "添付ファイル（JSON）", + help: "添付ファイルの JSON 配列で、要素は { filename, contentType?, size, hash, cid?, contentForm, inline?, storageKey? } の形をとり、内容は base64 で `inline` に格納されます。添付の合計生サイズが plugin-email の予算（SYS_EMAIL_ATTACHMENT_LIMIT_BYTES、256 KiB —— 最悪でも base64 で約 350 KB）に収まる場合にのみ書き込まれます。上限を超えるメッセージはインライン配信され、この列には何も保存されないため、行のサイズは有界に保たれます。`storageKey`（行外の内容）にはまだプロデューサーがありません —— objectstack#5172。" + }, status: { label: "ステータス", help: "ライフサイクル状態 — トランスポート呼び出し前に IEmailService.send がキューに入れます", diff --git a/packages/platform-objects/src/apps/translations/zh-CN.objects.generated.ts b/packages/platform-objects/src/apps/translations/zh-CN.objects.generated.ts index 6246e12c78..ec3799e80a 100644 --- a/packages/platform-objects/src/apps/translations/zh-CN.objects.generated.ts +++ b/packages/platform-objects/src/apps/translations/zh-CN.objects.generated.ts @@ -2045,6 +2045,14 @@ export const zhCNObjects: NonNullable = { body_html: { label: "正文（HTML）" }, + headers_json: { + label: "邮件头（JSON）", + help: "传给 IEmailService.send 的自定义邮件头，以 name → value 的 JSON 对象存储。两种投递模式下都会写入（它既是投递输入，也是审计证据）。在该列出现之前写入的行没有此值，读回时按「无自定义邮件头」处理。" + }, + attachments_json: { + label: "附件（JSON）", + help: "附件的 JSON 数组，元素形状为 { filename, contentType?, size, hash, cid?, contentForm, inline?, storageKey? }，内容以 base64 存放在 `inline` 中。仅当附件合计原始大小在 plugin-email 的预算内（SYS_EMAIL_ATTACHMENT_LIMIT_BYTES，256 KiB —— 最坏约 350 KB 的 base64）时才写入；超出上限的邮件改走内联投递，此列不落任何内容，因此行体积有界。`storageKey`（行外内容）目前还没有生产者 —— 见 objectstack#5172。" + }, status: { label: "状态", help: "生命周期状态——在调用传输层之前由 IEmailService.send 排队", diff --git a/packages/platform-objects/src/audit/sys-email.object.ts b/packages/platform-objects/src/audit/sys-email.object.ts index 6b8e67d337..e0c4ee0aeb 100644 --- a/packages/platform-objects/src/audit/sys-email.object.ts +++ b/packages/platform-objects/src/audit/sys-email.object.ts @@ -108,6 +108,35 @@ export const SysEmail = ObjectSchema.create({ group: 'Content', }), + // ── Message parts a row must carry to be deliverable (#5177) ─ + // Delivery of a queued / stranded / app-inserted message happens FROM + // THIS ROW, not from the in-memory message: `send()` publishes an + // `{ rowId }` job (#5160) and the boot sweep re-reads rows (#5161). Any + // part of the message the row cannot carry is therefore a part a durable + // delivery silently drops — which is why messages with headers or + // attachments used to be pushed back onto inline delivery instead. + headers_json: Field.textarea({ + label: 'Headers (JSON)', + required: false, + description: + 'Custom headers supplied to IEmailService.send, as a JSON object of name → value. ' + + 'Written in both delivery modes (it is audit evidence as much as delivery input). ' + + 'Absent on rows written before this column existed, which read back as "no custom headers".', + group: 'Content', + }), + + attachments_json: Field.textarea({ + label: 'Attachments (JSON)', + required: false, + description: + 'Attachments as a JSON array of { filename, contentType?, size, hash, cid?, contentForm, ' + + 'inline?, storageKey? }, with content base64 in `inline`. Written only when the combined raw ' + + 'size is within the plugin-email budget (SYS_EMAIL_ATTACHMENT_LIMIT_BYTES, 256 KiB — ~350 KB of ' + + 'base64 at worst); a larger message is delivered inline and stores nothing here, so the row ' + + 'stays bounded. `storageKey` (out-of-row content) has no producer yet — objectstack#5172.', + group: 'Content', + }), + // ── Delivery state ─────────────────────────────────────────── status: Field.select( ['queued', 'sent', 'failed'], diff --git a/packages/plugins/plugin-email/src/email-plugin.outbox-sweep.test.ts b/packages/plugins/plugin-email/src/email-plugin.outbox-sweep.test.ts index df9ea011d5..6d110f08ef 100644 --- a/packages/plugins/plugin-email/src/email-plugin.outbox-sweep.test.ts +++ b/packages/plugins/plugin-email/src/email-plugin.outbox-sweep.test.ts @@ -17,6 +17,7 @@ import { assertEngineDeleteDispatch } from '@objectstack/objectql'; import { DbQueueAdapter } from '@objectstack/service-queue'; import { EmailServicePlugin } from './email-plugin.js'; import { EmailService, EMAIL_SEND_QUEUE } from './email-service.js'; +import { encodeAttachmentsForRow, encodeHeadersForRow } from './sys-email-payload.js'; // ── harness ──────────────────────────────────────────────────────────────── @@ -172,8 +173,32 @@ async function boot(opts: BootOpts = {}) { created_at: createdAt, }); + /** + * The same crash, for a message that carried custom headers and a small + * attachment (#5177) — the columns are written exactly as `send()` writes + * them, via the same encoders. + */ + const strandWithParts = (id: string, createdAt: string) => { + const encoded = encodeAttachmentsForRow([ + { filename: '对账单.txt', content: '金额:¥1.00', cid: 'stmt@inline' }, + ]); + if (encoded.kind !== 'inline') throw new Error(`fixture is not storable: ${encoded.kind}`); + engine.seed('sys_email', { + id, + from_address: 'no-reply@example.test', + to_addresses: 'user@example.test', + subject: `Stranded ${id}`, + body_text: 'hello', + headers_json: encodeHeadersForRow({ 'X-Campaign': 'spring' }), + attachments_json: encoded.json, + status: 'queued', + attempt_count: 0, + created_at: createdAt, + }); + }; + return { - plugin, ctx, engine, adapter, clock, transport, strand, + plugin, ctx, engine, adapter, clock, transport, strand, strandWithParts, service: () => services.email as EmailService, sysEmail: () => engine.rows('sys_email'), jobs: () => engine.rows('sys_job_queue'), @@ -221,6 +246,22 @@ describe('boot sweep — inline delivery', () => { expect(errorLines(h.ctx).join('\n')).toMatch(/never reached a recipient/); }); + it('re-delivers a stranded row WITH its headers and attachment, inline too (#5177)', async () => { + // Inline mode reaches the transport straight from the sweep, so this is + // the shortest path from a persisted row to the wire — and the one that + // proves the columns, not the queue, are what carries the parts. + const h = await boot(); + h.strandWithParts('row-rich-inline', ago(min(30))); + + const swept = await h.ready(); + + expect(swept).toMatchObject({ scanned: 1, sent: 1 }); + expect(vi.mocked(h.transport.send).mock.calls[0][0]).toMatchObject({ + headers: { 'X-Campaign': 'spring' }, + attachments: [{ filename: '对账单.txt', content: '金额:¥1.00', cid: 'stmt@inline' }], + }); + }); + it('does not touch a row that was inserted seconds ago', async () => { // Another instance is delivering it right now; a boot must not race it. const h = await boot(); @@ -259,6 +300,24 @@ describe('boot sweep — durable queue delivery', () => { expect(h.jobs()[0]).toMatchObject({ status: 'completed' }); }); + it('re-delivers a stranded row WITH its headers and attachment (#5177)', async () => { + // A crash must not silently downgrade the message. Before #5177 the row + // had nowhere to keep either part, so a swept row was necessarily sent + // stripped — the loss looked exactly like a successful delivery. + const h = await boot({ queue: true, plugin: { queueDelivery: true } }); + h.strandWithParts('row-rich', ago(min(30))); + + await h.ready(); + await h.adapter.pollOnce(); + + expect(h.sysEmail()[0]).toMatchObject({ id: 'row-rich', status: 'sent' }); + expect(h.transport.send).toHaveBeenCalledTimes(1); + expect(vi.mocked(h.transport.send).mock.calls[0][0]).toMatchObject({ + headers: { 'X-Campaign': 'spring' }, + attachments: [{ filename: '对账单.txt', content: '金额:¥1.00', cid: 'stmt@inline' }], + }); + }); + it('collapses onto an existing pending job instead of racing a second worker', async () => { // The other half of a crash: the row AND its job survived (the process died // between publishing and the worker running). Re-publishing a second job diff --git a/packages/plugins/plugin-email/src/email-plugin.queue-delivery.test.ts b/packages/plugins/plugin-email/src/email-plugin.queue-delivery.test.ts index 76d5f48399..cd9254c452 100644 --- a/packages/plugins/plugin-email/src/email-plugin.queue-delivery.test.ts +++ b/packages/plugins/plugin-email/src/email-plugin.queue-delivery.test.ts @@ -225,6 +225,43 @@ describe('queue delivery — round trip', () => { expect(h.jobs()[0]).toMatchObject({ status: 'completed', attempts: 1 }); }); + it('carries custom headers and a small attachment all the way to the worker (#5177)', async () => { + // The capability #5177 adds, end to end: these messages used to be pushed + // back onto inline delivery because a row could not rebuild them, so the + // durable path was closed to exactly the mail most worth making durable. + const h = await boot({ plugin: { queueDelivery: true } }); + await h.ready(); + + const res = await h.service().send({ + to: 'a@b.com', + subject: '对账单', + text: 'hello', + headers: { 'X-Campaign': 'spring', 'List-Unsubscribe': '' }, + attachments: [ + { filename: '对账单.txt', content: '金额:¥1.00' }, + { filename: 'logo.png', content: Buffer.from([0x89, 0x50, 0x4e, 0x47]), contentType: 'image/png', cid: 'logo@inline' }, + ], + }); + + expect(res.status).toBe('queued'); + expect(h.transport.send).not.toHaveBeenCalled(); + expect(h.sysEmail()[0].headers_json).toBeTruthy(); + expect(h.sysEmail()[0].attachments_json).toBeTruthy(); + + await h.adapter.pollOnce(); + + expect(h.sysEmail()).toHaveLength(1); + expect(h.sysEmail()[0]).toMatchObject({ id: res.id, status: 'sent' }); + const delivered = vi.mocked(h.transport.send).mock.calls[0][0]; + expect(delivered.headers).toEqual({ 'X-Campaign': 'spring', 'List-Unsubscribe': '' }); + // Both content forms come back as the arm they were sent as. + expect(delivered.attachments[0]).toEqual({ filename: '对账单.txt', content: '金额:¥1.00' }); + expect(delivered.attachments[1]).toMatchObject({ + filename: 'logo.png', contentType: 'image/png', cid: 'logo@inline', + }); + expect(Buffer.compare(delivered.attachments[1].content, Buffer.from([0x89, 0x50, 0x4e, 0x47]))).toBe(0); + }); + it('SMTP 535: the queue retries with backoff, exhausts, and DLQs — one row throughout', async () => { // The regression the issue names: the old subscriber called `send()`, so // each redelivery INSERTED a new sys_email row. Five attempts, five rows, diff --git a/packages/plugins/plugin-email/src/email-service.queue-delivery.test.ts b/packages/plugins/plugin-email/src/email-service.queue-delivery.test.ts index 727b4c8989..d59ea67328 100644 --- a/packages/plugins/plugin-email/src/email-service.queue-delivery.test.ts +++ b/packages/plugins/plugin-email/src/email-service.queue-delivery.test.ts @@ -15,6 +15,7 @@ // not. import { describe, it, expect, vi } from 'vitest'; +import { createHash } from 'node:crypto'; import type { IQueueService } from '@objectstack/spec/contracts'; import { EmailService, @@ -22,6 +23,7 @@ import { type EmailPersistence, type EmailQueueDelivery, } from './email-service.js'; +import { SYS_EMAIL_ATTACHMENT_LIMIT_BYTES } from './sys-email-payload.js'; interface Published { queue: string; @@ -201,13 +203,13 @@ describe('EmailService — queue delivery on', () => { expect(queue.published).toHaveLength(0); }); - it('delivers a message with attachments inline rather than queueing it stripped', async () => { - // sys_email has no attachment / header columns, so a row cannot rebuild - // them. Queueing such a message would deliver it WITHOUT the attachment — - // silent data loss wearing durability's clothes. - const transport = { send: vi.fn(async () => ({ messageId: '' })) }; + it('queues a message with a SMALL attachment, storing its content on the row (#5177)', async () => { + // Was: pushed back to inline delivery, because no column could carry the + // attachment and queueing it would have delivered it stripped. Now the + // row carries it, so the durable path is available to it too. + const transport = { send: vi.fn(async () => ({ messageId: '' })) }; const queue = makeQueue(); - const { p } = makePersistence(); + const { p, rows } = makePersistence(); const logger = makeLogger(); const svc = new EmailService({ transport, defaultFrom: 'no@reply.com', persistence: p, logger, queueDelivery: wiring(queue), @@ -215,28 +217,136 @@ describe('EmailService — queue delivery on', () => { const res = await svc.send({ ...MSG, attachments: [{ filename: 'a.txt', content: 'hi' }] }); + expect(res.status).toBe('queued'); + expect(queue.published).toHaveLength(1); + expect(transport.send).not.toHaveBeenCalled(); + const stored = JSON.parse(String(rows.get(res.id)!.attachments_json)); + expect(stored).toEqual([{ + filename: 'a.txt', + size: 2, + hash: `sha256:${createHash('sha256').update('hi').digest('hex')}`, + contentForm: 'string', + inline: Buffer.from('hi').toString('base64'), + }]); + expect(logger.error).not.toHaveBeenCalled(); + }); + + it('queues a message with custom headers, storing them on the row (#5177)', async () => { + const transport = { send: vi.fn(async () => ({ messageId: '' })) }; + const queue = makeQueue(); + const { p, rows } = makePersistence(); + const svc = new EmailService({ + transport, defaultFrom: 'no@reply.com', persistence: p, queueDelivery: wiring(queue), + }); + + const res = await svc.send({ ...MSG, headers: { 'X-Campaign': 'spring', 'List-Unsubscribe': '' } }); + + expect(res.status).toBe('queued'); + expect(queue.published).toHaveLength(1); + expect(transport.send).not.toHaveBeenCalled(); + expect(JSON.parse(String(rows.get(res.id)!.headers_json))) + .toEqual({ 'X-Campaign': 'spring', 'List-Unsubscribe': '' }); + }); + + it('still refuses the queue for attachments OVER the limit, and stores nothing (#5177)', async () => { + const transport = { send: vi.fn(async () => ({ messageId: '' })) }; + const queue = makeQueue(); + const { p, rows } = makePersistence(); + const logger = makeLogger(); + const svc = new EmailService({ + transport, defaultFrom: 'no@reply.com', persistence: p, logger, queueDelivery: wiring(queue), + }); + const huge = Buffer.alloc(SYS_EMAIL_ATTACHMENT_LIMIT_BYTES + 1, 0x41); + + const res = await svc.send({ ...MSG, attachments: [{ filename: 'big.bin', content: huge }] }); + + // Pre-#5177 behaviour, unchanged: delivered inline and delivered WHOLE. expect(res.status).toBe('sent'); expect(queue.published).toHaveLength(0); expect(transport.send).toHaveBeenCalledWith(expect.objectContaining({ - attachments: [{ filename: 'a.txt', content: 'hi' }], + attachments: [{ filename: 'big.bin', content: huge }], })); - expect(logger.info).toHaveBeenCalledWith(expect.stringContaining('attachments')); - // A capability gap, not a failure: nothing is logged at error. + // The row must stay bounded: over-limit content never lands in the column. + expect(rows.get(res.id)!.attachments_json).toBeUndefined(); + const info = logger.info.mock.calls.map((c) => String(c[0])).join('\n'); + expect(info).toMatch(/over the \d+-byte limit/); + expect(info).toContain(String(SYS_EMAIL_ATTACHMENT_LIMIT_BYTES + 1)); + // A capability bound, not a failure: nothing is logged at error. expect(logger.error).not.toHaveBeenCalled(); }); - it('does the same for custom headers', async () => { - const transport = { send: vi.fn(async () => ({ messageId: '' })) }; + it('counts the limit across ALL attachments of one message, not per attachment', async () => { + const transport = { send: vi.fn(async () => ({ messageId: '' })) }; const queue = makeQueue(); - const { p } = makePersistence(); + const { p, rows } = makePersistence(); const svc = new EmailService({ transport, defaultFrom: 'no@reply.com', persistence: p, queueDelivery: wiring(queue), }); + const half = Buffer.alloc(SYS_EMAIL_ATTACHMENT_LIMIT_BYTES / 2 + 1, 0x42); + + const res = await svc.send({ + ...MSG, + attachments: [{ filename: 'a.bin', content: half }, { filename: 'b.bin', content: half }], + }); + + expect(res.status).toBe('sent'); // each fits; together they do not + expect(queue.published).toHaveLength(0); + expect(rows.get(res.id)!.attachments_json).toBeUndefined(); + }); + + it('keeps a message inline when `content` is outside the string | Buffer contract', async () => { + // A Uint8Array/stream is off-contract for EmailAttachment but nodemailer + // accepts it, so inline delivery must keep working exactly as before — + // what it must NOT do is queue a row that cannot rebuild it. + const transport = { send: vi.fn(async () => ({ messageId: '' })) }; + const queue = makeQueue(); + const { p, rows } = makePersistence(); + const logger = makeLogger(); + const svc = new EmailService({ + transport, defaultFrom: 'no@reply.com', persistence: p, logger, queueDelivery: wiring(queue), + }); - const res = await svc.send({ ...MSG, headers: { 'X-Campaign': 'spring' } }); + const res = await svc.send({ + ...MSG, + attachments: [{ filename: 'odd.bin', content: new Uint8Array([1, 2, 3]) as unknown as Buffer }], + }); expect(res.status).toBe('sent'); expect(queue.published).toHaveLength(0); + expect(rows.get(res.id)!.attachments_json).toBeUndefined(); + expect(logger.info.mock.calls.map((c) => String(c[0])).join('\n')) + .toMatch(/neither a string nor a Buffer/); + expect(logger.error).not.toHaveBeenCalled(); + }); + + it('persists headers and attachments in INLINE mode too, so a stranded row is whole', async () => { + // The boot sweep (#5161) re-delivers from the row in inline mode as well. + // A row that survived a crash without its parts would be re-sent stripped. + const transport = { send: vi.fn(async () => ({ messageId: '' })) }; + const { p, rows } = makePersistence(); + const svc = new EmailService({ transport, defaultFrom: 'no@reply.com', persistence: p }); + + const res = await svc.send({ + ...MSG, + headers: { 'X-Campaign': 'spring' }, + attachments: [{ filename: 'a.txt', content: 'hi' }], + }); + + expect(res.status).toBe('sent'); + const row = rows.get(res.id)!; + expect(JSON.parse(String(row.headers_json))).toEqual({ 'X-Campaign': 'spring' }); + expect(JSON.parse(String(row.attachments_json))[0]).toMatchObject({ filename: 'a.txt', size: 2 }); + }); + + it('writes neither column for a message that has neither', async () => { + const transport = { send: vi.fn(async () => ({ messageId: '' })) }; + const { p, rows } = makePersistence(); + const svc = new EmailService({ transport, defaultFrom: 'no@reply.com', persistence: p }); + + const res = await svc.send(MSG); + + expect(Object.keys(rows.get(res.id)!)).not.toContain('headers_json'); + expect(Object.keys(rows.get(res.id)!)).not.toContain('attachments_json'); }); }); diff --git a/packages/plugins/plugin-email/src/email-service.ts b/packages/plugins/plugin-email/src/email-service.ts index 24624a91a3..55ffcdc3fd 100644 --- a/packages/plugins/plugin-email/src/email-service.ts +++ b/packages/plugins/plugin-email/src/email-service.ts @@ -14,6 +14,14 @@ import type { QueueBackoffPolicy, } from '@objectstack/spec/contracts'; import { renderTemplate, requireVars, htmlToText } from './template-engine.js'; +import { + SYS_EMAIL_ATTACHMENT_LIMIT_BYTES, + encodeAttachmentsForRow, + encodeHeadersForRow, + decodeAttachmentsFromRow, + decodeHeadersFromRow, + type EncodedAttachments, +} from './sys-email-payload.js'; /** * Queue topic durable email delivery is published to and consumed from @@ -204,6 +212,16 @@ export function rowToNormalized(row: Record): NormalizedEmailMessag const bcc = splitAddresses(row.bcc_addresses); if (bcc.length > 0) msg.bcc = bcc; if (row.reply_to) msg.replyTo = String(row.reply_to); + // Headers and attachments (#5177). Both decoders return `undefined` for a + // row that simply has no such column — every row written before #5177 — and + // THROW for a column that is present but does not describe what it claims + // to. Throwing here is the point: the caller (`deliverPersistedRow`) turns + // it into a `failed` row carrying the reason, which is strictly better than + // handing the transport a message with an attachment quietly missing. + const headers = decodeHeadersFromRow(row.headers_json); + if (headers) msg.headers = headers; + const attachments = decodeAttachmentsFromRow(row.attachments_json); + if (attachments) msg.attachments = attachments; return msg; } @@ -411,10 +429,26 @@ export class EmailService implements IEmailService { throw err; } + // Encode the attachments ONCE (#5177). The same verdict answers both + // questions that follow — "may this message be queued?" and "what goes in + // `attachments_json`?" — so the row can never disagree with the routing + // decision (e.g. an over-limit message queued against a column that was + // never written). + // + // Skipped entirely without persistence: there is no row to write the + // column to and no queue job that could reference one (queue delivery + // reports that as its own degradation, below), so base64-ing a large + // attachment for nobody is pure cost on the path that opted out of + // persistence. + const encodedAttachments: EncodedAttachments = this.options.persistence + ? encodeAttachmentsForRow(normalized.attachments) + : { kind: 'none' }; + // `undefined` ⇒ every statement below is the pre-#5160 inline path. - const queue = allowQueue ? this.resolveQueueForSend(input) : undefined; + const queue = allowQueue ? this.resolveQueueForSend(encodedAttachments) : undefined; const id = newId(); + const headersJson = encodeHeadersForRow(normalized.headers); const baseRow: Record = { id, from_address: normalized.from, @@ -425,6 +459,15 @@ export class EmailService implements IEmailService { subject: normalized.subject, ...(normalized.text !== undefined ? { body_text: normalized.text } : {}), ...(normalized.html !== undefined ? { body_html: normalized.html } : {}), + // Written in BOTH delivery modes, not only the queued one. Two reasons: + // the row is the audit record of what was actually sent, and the boot + // sweep (#5161) re-delivers stranded rows in inline mode too — a row + // that survived a crash without its headers/attachments would be + // re-sent stripped, which is the exact loss these columns exist to + // prevent. Over-limit attachments store NOTHING (see the verdict + // above), so the row stays bounded. + ...(headersJson !== undefined ? { headers_json: headersJson } : {}), + ...(encodedAttachments.kind === 'inline' ? { attachments_json: encodedAttachments.json } : {}), ...(input.relatedObject ? { related_object: input.relatedObject } : {}), ...(input.relatedId ? { related_id: input.relatedId } : {}), ...(input.sentBy ? { sent_by: input.sentBy } : {}), @@ -473,25 +516,47 @@ export class EmailService implements IEmailService { * Resolve the queue to publish THIS message to, or `undefined` to deliver * it inline. * - * Returning `undefined` is never silent when queue delivery was asked for: - * the first time it happens the service reports at `error` what is no - * longer durable and how to restore it, then stays quiet. Mail keeps - * flowing either way — what degrades is persistence of the retry, not - * delivery, so a missing queue must not become a missing email. + * Returning `undefined` is never silent when queue delivery was asked for, + * but the level distinguishes two different things: + * + * - the wiring is **broken** (no queue service, no persistence) — the + * durability the operator switched on is not in force while everything + * still looks normal, so that is reported at `error`, once, with the fix; + * - this one message is **outside what a row can carry** (attachments over + * the budget) — nothing regressed, the outcome is exactly the pre-#5160 + * inline path, so it is stated at `info`. + * + * Mail keeps flowing either way — what degrades is persistence of the + * retry, not delivery, so a missing queue must not become a missing email. */ - private resolveQueueForSend(input: SendEmailInput): IQueueService | undefined { + private resolveQueueForSend(encodedAttachments: EncodedAttachments): IQueueService | undefined { const wiring = this.options.queueDelivery; if (!wiring) return undefined; - // `sys_email` carries no attachment or header columns, so a row cannot - // reconstruct them (`rowToNormalized`). Queueing such a message would - // deliver it stripped — silent data loss dressed as durability. Deliver - // it inline, where the in-memory message is still intact. Tracked for a - // real fix (attachment storage) rather than papered over. - if (input.attachments?.length || (input.headers && Object.keys(input.headers).length > 0)) { + // Custom headers are NO LONGER a reason to refuse the queue (#5177): + // `headers_json` carries them and `rowToNormalized` rebuilds them. + // + // Attachments are queueable too, up to the row budget. Past it, the row + // deliberately carries nothing, so a queued job could only deliver the + // message stripped — silent data loss dressed as durability. Fall back to + // inline delivery, where the in-memory message is still whole. This is + // not a failure and does not degrade anything that was previously + // working: the outcome is exactly today's behaviour, which is why it is + // stated at `info` and not `error`. + if (encodedAttachments.kind === 'over-limit') { + this.options.logger?.info( + `EmailService: queue delivery skipped for one message — its attachments total ` + + `${encodedAttachments.totalBytes} bytes, over the ${SYS_EMAIL_ATTACHMENT_LIMIT_BYTES}-byte limit a ` + + 'sys_email row carries, so the message was delivered inline (in-process retries only) rather than ' + + 'queued without them. Out-of-row storage for large attachments is objectstack#5172.', + ); + return undefined; + } + if (encodedAttachments.kind === 'unsupported') { this.options.logger?.info( - 'EmailService: queue delivery skipped for one message — sys_email cannot carry attachments or custom ' - + 'headers, so the message was delivered inline (in-process retries only) rather than stripped.', + 'EmailService: queue delivery skipped for one message — ' + + `${encodedAttachments.detail}, so it cannot be reconstructed from a sys_email row. The message was ` + + 'delivered inline (in-process retries only) rather than queued without the attachment.', ); return undefined; } diff --git a/packages/plugins/plugin-email/src/index.ts b/packages/plugins/plugin-email/src/index.ts index 3c0ebf6f38..028771074b 100644 --- a/packages/plugins/plugin-email/src/index.ts +++ b/packages/plugins/plugin-email/src/index.ts @@ -26,6 +26,15 @@ export type { EmailSendQueuePayload, DeliverAttemptOptions, } from './email-service.js'; +export { + SYS_EMAIL_ATTACHMENT_LIMIT_BYTES, + encodeAttachmentsForRow, + decodeAttachmentsFromRow, + encodeHeadersForRow, + decodeHeadersFromRow, + type PersistedEmailAttachment, + type EncodedAttachments, +} from './sys-email-payload.js'; export { renderTemplate, requireVars, htmlToText } from './template-engine.js'; export { ResendTransport, diff --git a/packages/plugins/plugin-email/src/sys-email-payload.test.ts b/packages/plugins/plugin-email/src/sys-email-payload.test.ts new file mode 100644 index 0000000000..d9e11b472b --- /dev/null +++ b/packages/plugins/plugin-email/src/sys-email-payload.test.ts @@ -0,0 +1,240 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. +// +// `sys_email.headers_json` / `attachments_json` — the codec and the row +// round trip (#5177). +// +// What these pin, in one sentence each: +// - a message's headers and attachments survive the trip through a row, in +// the SAME JS shapes the EmailAttachment contract declares; +// - a row written before these columns existed still reads; +// - a column that is present but does not describe what it claims is +// REJECTED, never silently reduced to a message missing a part. + +import { describe, it, expect } from 'vitest'; +import { createHash } from 'node:crypto'; +import { + SYS_EMAIL_ATTACHMENT_LIMIT_BYTES, + encodeAttachmentsForRow, + decodeAttachmentsFromRow, + encodeHeadersForRow, + decodeHeadersFromRow, + type PersistedEmailAttachment, +} from './sys-email-payload.js'; +import { rowToNormalized } from './email-service.js'; + +const sha = (b: Buffer | string) => `sha256:${createHash('sha256').update(b).digest('hex')}`; + +/** The minimum a row needs before `rowToNormalized` will look at anything else. */ +const baseRow = { + id: 'row-1', + from_address: 'no-reply@example.test', + to_addresses: 'user@example.test', + subject: 'Hi', + body_text: 'hello', +}; + +/** Encode → column value, as `send()` would write it. */ +function column(attachments: Parameters[0]): string { + const encoded = encodeAttachmentsForRow(attachments); + if (encoded.kind !== 'inline') throw new Error(`expected inline, got ${encoded.kind}`); + return encoded.json; +} + +describe('attachments_json — encode', () => { + it('records size, a tagged hash, the cid, and base64 content', () => { + const json = column([ + { filename: 'invoice.pdf', content: Buffer.from([1, 2, 3]), contentType: 'application/pdf' }, + { filename: 'logo.png', content: Buffer.from('png'), cid: 'logo@inline' }, + ]); + + expect(JSON.parse(json)).toEqual([ + { + filename: 'invoice.pdf', + contentType: 'application/pdf', + size: 3, + hash: sha(Buffer.from([1, 2, 3])), + contentForm: 'buffer', + inline: Buffer.from([1, 2, 3]).toString('base64'), + }, + { + filename: 'logo.png', + size: 3, + hash: sha(Buffer.from('png')), + cid: 'logo@inline', + contentForm: 'buffer', + inline: Buffer.from('png').toString('base64'), + }, + ] satisfies PersistedEmailAttachment[]); + }); + + it('does NOT invent a contentType the sender never supplied', () => { + // Writing `application/octet-stream` here would make a queued `.txt` + // arrive as a binary blob while the same message delivered inline arrives + // as text/plain — the transport infers from the filename when we say + // nothing, and the two paths must agree. + const [item] = JSON.parse(column([{ filename: 'notes.txt', content: 'hi' }])); + expect(item).not.toHaveProperty('contentType'); + }); + + it('measures RAW bytes (not base64, not UTF-16 code units) for the limit', () => { + // '中' is 3 UTF-8 bytes; 1 JS char. A char-count limit would let ~3x + // through. + const encoded = encodeAttachmentsForRow([{ filename: 'cn.txt', content: '中'.repeat(1000) }]); + expect(encoded).toMatchObject({ kind: 'inline', totalBytes: 3000 }); + }); + + it('reports over-limit with the true total, and yields no column value', () => { + const encoded = encodeAttachmentsForRow([ + { filename: 'a.bin', content: Buffer.alloc(SYS_EMAIL_ATTACHMENT_LIMIT_BYTES) }, + { filename: 'b.bin', content: Buffer.alloc(10) }, + ]); + expect(encoded).toEqual({ kind: 'over-limit', totalBytes: SYS_EMAIL_ATTACHMENT_LIMIT_BYTES + 10 }); + }); + + it('accepts a message exactly AT the limit (the bound is inclusive)', () => { + const encoded = encodeAttachmentsForRow([ + { filename: 'a.bin', content: Buffer.alloc(SYS_EMAIL_ATTACHMENT_LIMIT_BYTES) }, + ]); + expect(encoded.kind).toBe('inline'); + }); + + it('reports content outside the string | Buffer contract instead of guessing', () => { + const encoded = encodeAttachmentsForRow([ + { filename: 'odd.bin', content: new Uint8Array([1]) as unknown as Buffer }, + ]); + expect(encoded).toMatchObject({ kind: 'unsupported' }); + expect((encoded as { detail: string }).detail).toContain('odd.bin'); + }); + + it('treats no attachments as no column', () => { + expect(encodeAttachmentsForRow(undefined)).toEqual({ kind: 'none' }); + expect(encodeAttachmentsForRow([])).toEqual({ kind: 'none' }); + }); +}); + +describe('attachments_json — round trip through a row', () => { + it('restores a Buffer attachment as a Buffer, byte for byte', () => { + const content = Buffer.from([0x00, 0xff, 0x10, 0x80]); // not valid UTF-8 + const msg = rowToNormalized({ ...baseRow, attachments_json: column([{ filename: 'b.bin', content }]) }); + + const [att] = msg.attachments!; + expect(Buffer.isBuffer(att.content)).toBe(true); + expect(Buffer.compare(att.content as Buffer, content)).toBe(0); + }); + + it('restores a string attachment as a string — the charset declaration depends on it', () => { + const content = '你好,世界'; + const msg = rowToNormalized({ ...baseRow, attachments_json: column([{ filename: '中文.txt', content }]) }); + + const [att] = msg.attachments!; + expect(typeof att.content).toBe('string'); + expect(att.content).toBe(content); + expect(att.filename).toBe('中文.txt'); + }); + + it('carries contentType and cid across, so an inline image still resolves', () => { + const msg = rowToNormalized({ + ...baseRow, + attachments_json: column([ + { filename: 'logo.png', content: Buffer.from('png'), contentType: 'image/png', cid: 'logo@inline' }, + ]), + }); + expect(msg.attachments).toEqual([ + { filename: 'logo.png', content: Buffer.from('png'), contentType: 'image/png', cid: 'logo@inline' }, + ]); + }); +}); + +describe('headers_json', () => { + it('round-trips custom headers', () => { + const headers = { 'X-Campaign': 'spring', 'List-Unsubscribe': '' }; + const msg = rowToNormalized({ ...baseRow, headers_json: encodeHeadersForRow(headers) }); + expect(msg.headers).toEqual(headers); + }); + + it('writes no column for an empty or missing header bag', () => { + expect(encodeHeadersForRow(undefined)).toBeUndefined(); + expect(encodeHeadersForRow({})).toBeUndefined(); + }); +}); + +describe('rows written before these columns existed', () => { + it('reads with neither column, exactly as before', () => { + const msg = rowToNormalized({ ...baseRow }); + expect(msg.headers).toBeUndefined(); + expect(msg.attachments).toBeUndefined(); + expect(msg).toMatchObject({ to: ['user@example.test'], subject: 'Hi', text: 'hello' }); + }); + + it('reads with the columns explicitly null / empty', () => { + for (const empty of [null, undefined, '', ' ', 'null']) { + const msg = rowToNormalized({ ...baseRow, headers_json: empty, attachments_json: empty }); + expect(msg.headers).toBeUndefined(); + expect(msg.attachments).toBeUndefined(); + } + }); + + it('reads an empty array / empty object as "nothing", not as a broken row', () => { + const msg = rowToNormalized({ ...baseRow, headers_json: '{}', attachments_json: '[]' }); + expect(msg.headers).toBeUndefined(); + expect(msg.attachments).toBeUndefined(); + }); +}); + +describe('a column that lies is rejected, never partially delivered', () => { + const decodeAtt = (v: unknown) => () => decodeAttachmentsFromRow(v); + const decodeHdr = (v: unknown) => () => decodeHeadersFromRow(v); + + it('rejects malformed JSON', () => { + expect(decodeAtt('{ nope')).toThrow(/attachments_json is not valid JSON/); + expect(decodeHdr('{ nope')).toThrow(/headers_json is not valid JSON/); + }); + + it('rejects the wrong top-level shape', () => { + expect(decodeAtt('{"a":1}')).toThrow(/must decode to an array/); + expect(decodeHdr('["a"]')).toThrow(/must decode to an object/); + }); + + it('rejects a non-string header value rather than coercing it', () => { + expect(decodeHdr('{"X-Retry":3}')).toThrow(/must be a string/); + }); + + it('rejects a missing contentForm instead of guessing which union arm to rebuild', () => { + const [item] = JSON.parse(column([{ filename: 'a.txt', content: 'hi' }])); + delete item.contentForm; + expect(decodeAtt(JSON.stringify([item]))).toThrow(/contentForm must be 'string' or 'buffer'/); + }); + + it('rejects an attachment with no content at all', () => { + expect(decodeAtt(JSON.stringify([ + { filename: 'a.txt', size: 2, hash: sha('hi'), contentForm: 'string' }, + ]))).toThrow(/carries no content/); + }); + + it('rejects a storageKey-only attachment and names the issue that will implement it', () => { + expect(decodeAtt(JSON.stringify([ + { filename: 'a.txt', size: 2, hash: sha('hi'), contentForm: 'buffer', storageKey: 'blob/abc' }, + ]))).toThrow(/objectstack#5172/); + }); + + it('rejects truncated content (size disagrees)', () => { + const [item] = JSON.parse(column([{ filename: 'a.txt', content: 'hello' }])); + item.inline = Buffer.from('he').toString('base64'); + expect(decodeAtt(JSON.stringify([item]))).toThrow(/decodes to 2 byte\(s\) but the row records size 5/); + }); + + it('rejects rewritten content (hash disagrees)', () => { + const [item] = JSON.parse(column([{ filename: 'a.txt', content: 'hi' }])); + item.inline = Buffer.from('HI').toString('base64'); + expect(decodeAtt(JSON.stringify([item]))).toThrow(/is not the content that was sent/); + }); + + it('names the offending index so a multi-attachment row is debuggable', () => { + const items = JSON.parse(column([ + { filename: 'ok.txt', content: 'ok' }, + { filename: 'bad.txt', content: 'bad' }, + ])); + delete items[1].hash; + expect(decodeAtt(JSON.stringify(items))).toThrow(/\[1\]\.hash/); + }); +}); diff --git a/packages/plugins/plugin-email/src/sys-email-payload.ts b/packages/plugins/plugin-email/src/sys-email-payload.ts new file mode 100644 index 0000000000..6bef1d5dd2 --- /dev/null +++ b/packages/plugins/plugin-email/src/sys-email-payload.ts @@ -0,0 +1,327 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * `sys_email.headers_json` / `sys_email.attachments_json` — the codec (#5177). + * + * ## What these two columns are for + * + * Everything durable about email delivery in this package hangs off ONE fact: + * a queued (or stranded, or app-inserted) message is delivered from its + * **`sys_email` row**, never from an in-memory object — `send()` publishes an + * `{ rowId }` job (#5160), the boot sweep re-reads rows (#5161), and both end + * at {@link rowToNormalized}. So a part of the message that no column can + * carry is a part of the message that a row-based delivery **silently drops**. + * + * Before this module, custom headers and attachments were exactly that, and + * the honest workaround was to refuse: a message carrying either was pushed + * back onto inline delivery so it would at least go out whole. That trade + * bought correctness by giving up durability for precisely the messages most + * likely to matter (a signed receipt, a `List-Unsubscribe` header, an invoice + * PDF). These columns buy it back. + * + * ## Why attachments are bounded, and why the bound is a constant + * + * `sys_email` is an append-only audit log, not a blob store. Base64 inflates + * content by 4/3, so an unbounded attachment column would let one message + * write an arbitrarily large row into a table nobody prunes. Phase 1 therefore + * carries attachments **only up to {@link SYS_EMAIL_ATTACHMENT_LIMIT_BYTES} + * combined raw bytes** (~350 KB of base64 in the worst case) and leaves + * anything larger on the pre-existing inline path — where it behaves exactly + * as it does today. Over-limit is not an error; the worst outcome is the + * status quo. + * + * The bound is a **constant, not a setting**, deliberately: a knob here would + * be a second place for the row-size budget to drift, and there is no evidence + * yet of a deployment that needs a different number. Raising it is a code + * change with a reason attached, which is what a change to a storage budget + * should cost. + * + * Out-of-row storage for large attachments (`storageKey`) is phase 2, tracked + * by objectstack#5172. The element shape already declares that key so phase 2 + * adds a *producer* rather than migrating data — see + * {@link PersistedEmailAttachment.storageKey}. + * + * ## Why decoding is strict + * + * Every failure mode here is "the recipient gets a message that is not the one + * that was sent" — a stripped attachment, a header that vanished, bytes that + * decoded to something else. None of it is visible from the outside: the row + * says `sent`, the transport said 250. So a row whose payload columns do not + * say exactly what they claim is **rejected loudly** (the row lands at + * `failed` with the reason) instead of being delivered partially. That is + * AGENTS.md Prime Directive #12 applied to data at rest: no `??`, no silent + * coercion, no "best effort" reconstruction of a message. + * + * The strictness runs as far as re-hashing the content on read — `size` and + * `hash` are not decoration, they are what turns a truncated or corrupted + * column into an error instead of a wrong email. + */ + +import { createHash } from 'node:crypto'; +import type { EmailAttachment } from '@objectstack/spec/contracts'; + +/** + * Combined **raw** (pre-base64) byte budget for ALL attachments on ONE + * message, above which `sys_email` does not carry them. + * + * 256 KiB. A message at the limit stores ~350 KB of base64 in + * `attachments_json`, which is the real bound on a `sys_email` row. + * + * Not configurable on purpose — see the module header. + */ +export const SYS_EMAIL_ATTACHMENT_LIMIT_BYTES = 256 * 1024; + +/** + * One element of `sys_email.attachments_json`. + * + * Shaped for phase 2 from the start: the content of an attachment is either + * carried in the row ({@link inline}) or referenced out of it + * ({@link storageKey}), and adding the second producer must not require + * migrating rows written by the first. + */ +export interface PersistedEmailAttachment { + /** `EmailAttachment.filename`, verbatim (UTF-8 — non-ASCII names included). */ + filename: string; + /** + * `EmailAttachment.contentType`, present **only when the sender supplied + * one**. + * + * Deliberately optional rather than defaulted to `application/octet-stream`: + * transports infer the type from the filename when it is absent (nodemailer + * maps `report.pdf` → `application/pdf`), so writing a default here would + * make a queued message arrive with a *different* MIME type than the same + * message delivered inline. Recording only what the caller actually said is + * what keeps the two paths byte-identical. + */ + contentType?: string; + /** Raw content size in bytes, before base64. Verified on read. */ + size: number; + /** + * Digest of the raw content, `sha256:`. + * + * Algorithm-tagged rather than a bare hex string so phase 2 can verify + * storage-backed content without inferring which algorithm produced a + * 64-character value. + */ + hash: string; + /** `EmailAttachment.cid` — an inline image in an HTML body is `cid:`-referenced and unusable without it. */ + cid?: string; + /** + * Which arm of the `content: string | Buffer` contract this attachment was + * sent as, so the message is rebuilt as the same JS type it was sent with. + * + * Not cosmetic: nodemailer emits `Content-Type: text/plain; charset=utf-8` + * for string content and omits the charset for a Buffer, so restoring a + * text attachment as a Buffer would drop the charset declaration and let a + * receiving client mis-decode a UTF-8 file it can no longer identify. + * Required (not defaulted) because guessing it is exactly the silent + * coercion this module refuses to do. + */ + contentForm: 'string' | 'buffer'; + /** Base64 of the raw content, when the row carries it (phase 1's only producer). */ + inline?: string; + /** + * Reference to content held outside the row. + * + * **Phase 1 has no producer for this key** — nothing in this repo writes it + * today, and {@link decodeAttachmentsFromRow} rejects a row that only has + * it. It is declared now so that objectstack#5172 (large attachments via + * storage, deferred by the maintainer) ships a producer + reader against an + * already-persisted shape instead of migrating rows. Declared-not-yet-live + * on purpose; not a liveness finding. + */ + storageKey?: string; +} + +/** Outcome of {@link encodeAttachmentsForRow}. */ +export type EncodedAttachments = + /** No attachments — the column is not written. */ + | { kind: 'none' } + /** Within budget: `json` goes into `attachments_json`. */ + | { kind: 'inline'; json: string; totalBytes: number } + /** Over {@link SYS_EMAIL_ATTACHMENT_LIMIT_BYTES} — nothing is written to the row. */ + | { kind: 'over-limit'; totalBytes: number } + /** Content this codec cannot represent; nothing is written to the row. */ + | { kind: 'unsupported'; detail: string }; + +/** `sha256:` of the raw bytes. */ +function digestOf(bytes: Buffer): string { + return `sha256:${createHash('sha256').update(bytes).digest('hex')}`; +} + +/** + * Encode a message's attachments for `sys_email.attachments_json`. + * + * Never throws: an attachment this codec cannot carry (too large, or a + * `content` value outside the declared `string | Buffer` contract) is reported + * as a verdict, because the caller's response is to fall back to inline + * delivery — which is today's behaviour and always whole — not to fail the + * send. Failing a send that works today would be a regression dressed as + * strictness. + */ +export function encodeAttachmentsForRow( + attachments: EmailAttachment[] | undefined, +): EncodedAttachments { + if (!attachments || attachments.length === 0) return { kind: 'none' }; + + const parts: Array<{ att: EmailAttachment; bytes: Buffer; contentForm: 'string' | 'buffer' }> = []; + for (const att of attachments) { + const content = att?.content; + if (typeof content === 'string') { + parts.push({ att, bytes: Buffer.from(content, 'utf8'), contentForm: 'string' }); + } else if (Buffer.isBuffer(content)) { + parts.push({ att, bytes: content, contentForm: 'buffer' }); + } else { + return { + kind: 'unsupported', + detail: `attachment '${String(att?.filename ?? '(unnamed)')}' carries a 'content' value that is neither ` + + 'a string nor a Buffer, which is the whole of the EmailAttachment contract', + }; + } + } + + const totalBytes = parts.reduce((n, p) => n + p.bytes.byteLength, 0); + if (totalBytes > SYS_EMAIL_ATTACHMENT_LIMIT_BYTES) return { kind: 'over-limit', totalBytes }; + + const items: PersistedEmailAttachment[] = parts.map(({ att, bytes, contentForm }) => ({ + filename: String(att.filename ?? ''), + ...(att.contentType ? { contentType: String(att.contentType) } : {}), + size: bytes.byteLength, + hash: digestOf(bytes), + ...(att.cid ? { cid: String(att.cid) } : {}), + contentForm, + inline: bytes.toString('base64'), + })); + return { kind: 'inline', json: JSON.stringify(items), totalBytes }; +} + +/** Shared prefix so every rejection names the column it came from. */ +function reject(column: string, detail: string): never { + throw new Error(`VALIDATION_FAILED: sys_email.${column} ${detail}`); +} + +/** `null` / `undefined` / `''` — i.e. a row written before this column existed. */ +function isAbsent(value: unknown): boolean { + return value == null || (typeof value === 'string' && value.trim() === ''); +} + +function parseJson(column: string, value: unknown): unknown { + if (typeof value !== 'string') return value; + try { + return JSON.parse(value); + } catch (err: any) { + return reject(column, `is not valid JSON (${String(err?.message ?? err)})`); + } +} + +/** + * Rebuild `attachments` from `sys_email.attachments_json`. + * + * Returns `undefined` for a row that has no such column (every row written + * before #5177) — reading an old row must stay safe. Everything else is + * verified: a column that is present but does not describe the message it + * claims to describe throws, so the row lands at `failed` with the reason + * rather than being delivered without the attachment. + */ +export function decodeAttachmentsFromRow(value: unknown): EmailAttachment[] | undefined { + if (isAbsent(value)) return undefined; + const parsed = parseJson('attachments_json', value); + if (parsed == null) return undefined; + if (!Array.isArray(parsed)) reject('attachments_json', 'must decode to an array of attachments'); + if (parsed.length === 0) return undefined; + + return parsed.map((raw: any, i: number): EmailAttachment => { + const at = `[${i}]`; + if (!raw || typeof raw !== 'object') reject('attachments_json', `${at} is not an object`); + const filename = raw.filename; + if (typeof filename !== 'string' || filename === '') { + reject('attachments_json', `${at}.filename is required and must be a non-empty string`); + } + if (typeof raw.size !== 'number' || !Number.isFinite(raw.size) || raw.size < 0) { + reject('attachments_json', `${at}.size is required and must be a non-negative number`); + } + if (typeof raw.hash !== 'string' || raw.hash === '') { + reject('attachments_json', `${at}.hash is required and must be a non-empty string`); + } + if (raw.contentForm !== 'string' && raw.contentForm !== 'buffer') { + reject( + 'attachments_json', + `${at}.contentForm must be 'string' or 'buffer' — it says which arm of the EmailAttachment ` + + "`content: string | Buffer` contract to rebuild, and guessing it would change the attachment's " + + 'declared charset', + ); + } + if (typeof raw.inline !== 'string' || raw.inline === '') { + if (typeof raw.storageKey === 'string' && raw.storageKey !== '') { + reject( + 'attachments_json', + `${at} references content by storageKey, which no producer writes and nothing reads yet — ` + + 'out-of-row attachment storage is objectstack#5172. Refusing rather than delivering the message ' + + 'without this attachment', + ); + } + reject('attachments_json', `${at} carries no content: neither 'inline' nor a readable reference`); + } + + const bytes = Buffer.from(raw.inline, 'base64'); + if (bytes.byteLength !== raw.size) { + reject( + 'attachments_json', + `${at}.inline decodes to ${bytes.byteLength} byte(s) but the row records size ${raw.size} — the ` + + 'column was truncated or rewritten', + ); + } + const actual = digestOf(bytes); + if (actual !== raw.hash) { + reject( + 'attachments_json', + `${at}.inline hashes to ${actual} but the row records ${String(raw.hash)} — the stored content is not ` + + 'the content that was sent', + ); + } + + return { + filename, + content: raw.contentForm === 'string' ? bytes.toString('utf8') : bytes, + ...(typeof raw.contentType === 'string' && raw.contentType ? { contentType: raw.contentType } : {}), + ...(typeof raw.cid === 'string' && raw.cid ? { cid: raw.cid } : {}), + }; + }); +} + +/** + * Encode custom headers for `sys_email.headers_json`, or `undefined` when + * there are none (the column stays unwritten rather than storing `{}`). + */ +export function encodeHeadersForRow(headers: Record | undefined): string | undefined { + if (!headers) return undefined; + const keys = Object.keys(headers); + if (keys.length === 0) return undefined; + return JSON.stringify(headers); +} + +/** + * Rebuild `headers` from `sys_email.headers_json`. + * + * `undefined` for a row without the column (pre-#5177 rows read safely). + * A present-but-malformed column throws: a message whose `List-Unsubscribe` + * or `X-Campaign` header quietly disappeared is not the message that was sent. + */ +export function decodeHeadersFromRow(value: unknown): Record | undefined { + if (isAbsent(value)) return undefined; + const parsed = parseJson('headers_json', value); + if (parsed == null) return undefined; + if (typeof parsed !== 'object' || Array.isArray(parsed)) { + reject('headers_json', 'must decode to an object of header name → value'); + } + const entries = Object.entries(parsed as Record); + if (entries.length === 0) return undefined; + const out: Record = {}; + for (const [name, v] of entries) { + if (typeof v !== 'string') { + reject('headers_json', `['${name}'] must be a string (headers are Record< string, string >), got ${typeof v}`); + } + out[name] = v; + } + return out; +} diff --git a/packages/plugins/plugin-email/src/sys-email-payload.wire.test.ts b/packages/plugins/plugin-email/src/sys-email-payload.wire.test.ts new file mode 100644 index 0000000000..cbff9a656f --- /dev/null +++ b/packages/plugins/plugin-email/src/sys-email-payload.wire.test.ts @@ -0,0 +1,141 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. +// +// The claim #5177 actually has to make: a message rebuilt from a `sys_email` +// row reaches the wire as the SAME message that would have gone out inline. +// +// Nothing short of a real transport can prove that. A unit test comparing +// `EmailAttachment` objects proves the codec round-trips JS values; it says +// nothing about whether nodemailer then encodes those values into the same +// MIME parts — which is exactly where a plausible-looking shortcut goes wrong +// (restoring a text attachment as a Buffer round-trips the bytes perfectly and +// silently drops `charset=utf-8` from the part header, so the recipient's +// client mis-decodes a UTF-8 file it can no longer identify). +// +// So: REAL nodemailer, in-process fake SMTP (the `smtp.wire.test.ts` facility, +// shared via `fake-smtp.testkit.ts`), and a byte comparison of the DATA +// payload the server received in each mode. + +import { describe, it, expect, afterEach } from 'vitest'; +import { SmtpTransport } from './transports/smtp.js'; +import { startFakeSmtp, wireAttachments, type FakeSmtp } from './transports/fake-smtp.testkit.js'; +import { EmailService, normalizeMessage, rowToNormalized, type EmailPersistence } from './email-service.js'; +import type { NormalizedEmailMessage, SendEmailInput } from '@objectstack/spec/contracts'; + +let server: FakeSmtp | undefined; +afterEach(async () => { + await server?.close(); + server = undefined; +}); + +/** A 1x1 transparent PNG — real binary bytes, including a NUL and 0xFF. */ +const PNG = Buffer.from( + 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==', + 'base64', +); + +/** The message under test: Chinese filename, a cid inline image, both content forms. */ +const MESSAGE: SendEmailInput = { + to: { name: '收件人', address: 'rcpt@example.test' }, + from: { name: 'ObjectStack 通知', address: 'no-reply@example.test' }, + subject: '【ObjectStack】您的对账单', + html: '

你好 详见附件。

', + text: '你好,详见附件。', + headers: { 'X-Campaign': 'spring-2026', 'List-Unsubscribe': '' }, + attachments: [ + // string content — a text part whose charset declaration is load-bearing + { filename: '对账单.txt', content: '账户:张三\n金额:¥1,234.56\n' }, + // Buffer content with an explicit type + cid — the inline image + { filename: 'logo.png', content: PNG, contentType: 'image/png', cid: 'logo@inline' }, + ], +}; + +/** Deliver `msg` through a real SMTP conversation and return the raw DATA payload. */ +async function onTheWire(msg: NormalizedEmailMessage): Promise { + const fake = await startFakeSmtp(); + try { + const transport = new SmtpTransport({ host: '127.0.0.1', port: fake.port, secure: false, timeout: 5_000 }); + await transport.send(msg); + await transport.close(); + expect(fake.messages).toHaveLength(1); + return fake.messages[0]; + } finally { + await fake.close(); + } +} + +/** Capture the row `send()` would persist, without delivering anything. */ +async function persistedRow(input: SendEmailInput): Promise> { + let row: Record | undefined; + const persistence: EmailPersistence = { + async insert(r) { row = { ...r }; return { id: r.id }; }, + async update() { /* not reached: the transport below never succeeds */ }, + }; + const svc = new EmailService({ + transport: { async send() { throw new Error('not delivered in this fixture'); } }, + persistence, + }); + await svc.send(input); + if (!row) throw new Error('no row was persisted'); + return row; +} + +/** + * Strip the parts of a MIME message that legitimately differ between two + * sends: the generated Message-ID, the Date, and the multipart boundaries. + */ +function stable(raw: string): string { + const boundaries = [...raw.matchAll(/boundary="?([^"\r\n;]+)"?/g)].map((m) => m[1]); + let out = raw + .replace(/^Message-ID:.*$/gim, 'Message-ID: ') + .replace(/^Date:.*$/gim, 'Date: '); + boundaries.forEach((b, i) => { out = out.split(b).join(``); }); + return out; +} + +describe('a message rebuilt from a sys_email row reaches the wire unchanged (#5177)', () => { + it('produces a byte-identical MIME message, inline vs row round trip', async () => { + const row = await persistedRow(MESSAGE); + // The row really is carrying the parts — not silently dropping them. + expect(row.headers_json).toBeTruthy(); + expect(row.attachments_json).toBeTruthy(); + + // Normalized the same way `send()` normalizes it, so the ONLY difference + // under test is the trip through the row. + const direct = await onTheWire(normalizeMessage(MESSAGE)); + const viaRow = await onTheWire(rowToNormalized(row)); + + expect(stable(viaRow)).toBe(stable(direct)); + }, 30_000); + + it('delivers both attachments byte for byte, with the Chinese filename intact', async () => { + const row = await persistedRow(MESSAGE); + const raw = await onTheWire(rowToNormalized(row)); + + const parts = wireAttachments(raw); + expect([...parts.keys()].sort()).toEqual(['logo.png', '对账单.txt']); + expect(Buffer.compare(parts.get('logo.png')!, PNG)).toBe(0); + expect(parts.get('对账单.txt')!.toString('utf8')).toBe('账户:张三\n金额:¥1,234.56\n'); + }, 30_000); + + it('keeps the text attachment declared as UTF-8 — the reason contentForm exists', async () => { + const row = await persistedRow(MESSAGE); + const raw = await onTheWire(rowToNormalized(row)); + + const textPart = raw + .split(/--[-_A-Za-z0-9]{10,}/) + .find((p) => /filename\*0\*=utf-8''%E5%AF%B9/i.test(p.replace(/\r\n[ \t]+/g, ''))); + expect(textPart).toBeTruthy(); + expect(textPart!.replace(/\r\n[ \t]+/g, '')).toMatch(/Content-Type:\s*text\/plain;\s*charset=utf-8/i); + }, 30_000); + + it('carries the custom headers and the cid the HTML body references', async () => { + const row = await persistedRow(MESSAGE); + const raw = await onTheWire(rowToNormalized(row)); + const unfolded = raw.replace(/\r\n[ \t]+/g, ' '); + + expect(unfolded).toMatch(/^X-Campaign: spring-2026$/m); + expect(unfolded).toMatch(/^List-Unsubscribe: $/m); + // Without the cid the inline resolves to nothing. + expect(unfolded).toMatch(/Content-ID: /); + }, 30_000); +}); diff --git a/packages/plugins/plugin-email/src/transports/fake-smtp.testkit.ts b/packages/plugins/plugin-email/src/transports/fake-smtp.testkit.ts new file mode 100644 index 0000000000..4d5c66f4ca --- /dev/null +++ b/packages/plugins/plugin-email/src/transports/fake-smtp.testkit.ts @@ -0,0 +1,184 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. +// +// In-process fake ESMTP server + the decoders needed to read what a REAL +// nodemailer put on the wire. +// +// Extracted from `smtp.wire.test.ts` (#5087) when a second suite needed it +// (#5177: proving a message rebuilt from a `sys_email` row reaches the wire +// with the same bytes as the same message sent inline). Shared rather than +// copied on purpose: two hand-written fake servers drift, and the whole value +// of this facility is that nothing stands between the transport and the wire — +// a claim only one implementation can keep making. +// +// Not a test file (`.testkit.ts` ⇒ vitest does not collect it) and not part of +// the published bundle (tsup builds from `src/index.ts`, which never imports +// it). + +import net from 'node:net'; + +export interface FakeSmtp { + port: number; + /** Commands the client sent, uppercased verb + raw line. */ + commands: string[]; + /** Raw DATA payloads (headers + body), one per delivered message. */ + messages: string[]; + close(): Promise; +} + +/** + * Minimal ESMTP server: greeting, EHLO, AUTH PLAIN/LOGIN, MAIL/RCPT/DATA/QUIT. + * Deliberately does NOT advertise STARTTLS — callers connect with + * `secure: false` so nodemailer stays in the clear against localhost. + */ +export async function startFakeSmtp(opts: { authOk?: boolean } = {}): Promise { + const authOk = opts.authOk !== false; + const commands: string[] = []; + const messages: string[] = []; + + const server = net.createServer((socket) => { + let buffer = ''; + let inData = false; + let dataBuf = ''; + socket.setEncoding('utf8'); + socket.write('220 fake.smtp.test ESMTP ready\r\n'); + + socket.on('data', (chunk: string) => { + if (inData) { + dataBuf += chunk; + const end = dataBuf.indexOf('\r\n.\r\n'); + if (end === -1) return; + messages.push(dataBuf.slice(0, end)); + dataBuf = ''; + inData = false; + socket.write('250 2.0.0 Ok: queued as FAKE123\r\n'); + return; + } + buffer += chunk; + let idx: number; + while ((idx = buffer.indexOf('\r\n')) !== -1) { + const line = buffer.slice(0, idx); + buffer = buffer.slice(idx + 2); + commands.push(line); + const verb = line.split(/[ :]/)[0].toUpperCase(); + if (verb === 'EHLO' || verb === 'HELO') { + socket.write('250-fake.smtp.test\r\n250-AUTH PLAIN LOGIN\r\n250 SMTPUTF8\r\n'); + } else if (verb === 'AUTH') { + if (/LOGIN/i.test(line)) { + // LOGIN is a 3-step challenge; accept/deny at the end. + socket.write('334 VXNlcm5hbWU6\r\n'); + } else { + socket.write(authOk ? '235 2.7.0 Accepted\r\n' : '535 5.7.8 Error: authentication failed\r\n'); + } + } else if (/^[A-Za-z0-9+/=]+$/.test(line) && commands.some((c) => /^AUTH LOGIN/i.test(c))) { + // base64 continuation of AUTH LOGIN (username, then password) + const step = commands.filter((c) => /^[A-Za-z0-9+/=]+$/.test(c)).length; + if (step === 1) socket.write('334 UGFzc3dvcmQ6\r\n'); + else socket.write(authOk ? '235 2.7.0 Accepted\r\n' : '535 5.7.8 Error: authentication failed\r\n'); + } else if (verb === 'MAIL' || verb === 'RCPT') { + socket.write('250 2.1.0 Ok\r\n'); + } else if (verb === 'DATA') { + inData = true; + socket.write('354 End data with .\r\n'); + } else if (verb === 'QUIT') { + socket.write('221 2.0.0 Bye\r\n'); + socket.end(); + } else { + socket.write('250 2.0.0 Ok\r\n'); + } + } + }); + socket.on('error', () => { /* client hung up */ }); + }); + + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + const port = (server.address() as net.AddressInfo).port; + return { + port, + commands, + messages, + close: () => new Promise((resolve) => server.close(() => resolve())), + }; +} + +/** Decode an RFC 2047 encoded-word run (`=?UTF-8?B?..?=` / `?Q?..?=`). */ +export function decodeEncodedWords(input: string): string { + return input.replace(/=\?utf-8\?(b|q)\?([^?]+)\?=/gi, (_m, enc: string, payload: string) => { + if (enc.toLowerCase() === 'b') return Buffer.from(payload, 'base64').toString('utf8'); + const bytes = payload + .replace(/_/g, ' ') + .replace(/=([0-9A-Fa-f]{2})/g, (_x, hex: string) => String.fromCharCode(parseInt(hex, 16))); + return Buffer.from(bytes, 'binary').toString('utf8'); + }); +} + +/** Decode every transfer-encoded body part so the text can be asserted. */ +export function decodeBodies(raw: string): string { + const out: string[] = [raw]; + // quoted-printable + out.push(Buffer.from( + raw.replace(/=\r?\n/g, '').replace(/=([0-9A-Fa-f]{2})/g, (_m, hex: string) => String.fromCharCode(parseInt(hex, 16))), + 'binary', + ).toString('utf8')); + // base64 blocks (4+ full base64 lines in a row) + for (const block of raw.match(/(?:^[A-Za-z0-9+/=]{20,}\r?\n?){1,}/gm) ?? []) { + out.push(Buffer.from(block.replace(/\s+/g, ''), 'base64').toString('utf8')); + } + return out.join('\n'); +} + +/** One MIME part of a multipart message, as it appeared on the wire. */ +export interface WireMimePart { + /** Unfolded header block of the part. */ + headers: string; + /** Raw (still transfer-encoded) body of the part. */ + body: string; +} + +/** + * Split a raw DATA payload into its MIME parts, flattening nested multiparts. + * + * The nesting is not incidental: a message with a `cid:` inline image AND a + * regular attachment comes out as `multipart/mixed` wrapping a + * `multipart/related`, so a splitter that only knows the outermost boundary + * simply cannot see the inline image — which is precisely the part a `cid` + * test is about. Every declared boundary is therefore used at once. + * + * Header continuation lines are unfolded so a single + * `Content-Disposition: attachment; filename*0*=…` can be matched in one go. + */ +export function splitMimeParts(raw: string): WireMimePart[] { + const boundaries = [...new Set([...raw.matchAll(/boundary="?([^"\r\n;]+)"?/g)].map((m) => m[1]))]; + if (boundaries.length === 0) return []; + const alternation = boundaries.map((b) => b.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')).join('|'); + const parts: WireMimePart[] = []; + for (const chunk of raw.split(new RegExp(`--(?:${alternation})(?:--)?`))) { + const body = chunk.replace(/^\r\n/, ''); + const sep = body.indexOf('\r\n\r\n'); + if (sep === -1) continue; + parts.push({ + headers: body.slice(0, sep).replace(/\r\n[ \t]+/g, ''), + body: body.slice(sep + 4).trim(), + }); + } + return parts; +} + +/** + * The base64-decoded content of every `Content-Disposition: attachment` / + * `inline` part, keyed by the decoded filename. This is what "the attachment + * arrived byte for byte" is asserted against. + */ +export function wireAttachments(raw: string): Map { + const out = new Map(); + for (const part of splitMimeParts(raw)) { + if (!/Content-Disposition:\s*(attachment|inline)/i.test(part.headers)) continue; + // RFC 2231 (`filename*0*=utf-8''%E4%B8%AD`) or an RFC 2047 encoded word. + const ext = /filename\*\d*\*?=(?:utf-8'')?([^;\r\n]+)/i.exec(part.headers)?.[1]; + const plain = /filename="?([^";\r\n]+)"?/i.exec(part.headers)?.[1]; + const filename = ext + ? decodeURIComponent(ext) + : decodeEncodedWords(plain ?? ''); + out.set(filename, Buffer.from(part.body.replace(/\s+/g, ''), 'base64')); + } + return out; +} diff --git a/packages/plugins/plugin-email/src/transports/smtp.wire.test.ts b/packages/plugins/plugin-email/src/transports/smtp.wire.test.ts index 2b0551ee64..faf7f4dbfd 100644 --- a/packages/plugins/plugin-email/src/transports/smtp.wire.test.ts +++ b/packages/plugins/plugin-email/src/transports/smtp.wire.test.ts @@ -11,118 +11,11 @@ // * an AUTH rejection (535) reaches the caller instead of being swallowed. import { describe, it, expect, afterEach } from 'vitest'; -import net from 'node:net'; import { SmtpTransport } from './smtp.js'; - -interface FakeSmtp { - port: number; - /** Commands the client sent, uppercased verb + raw line. */ - commands: string[]; - /** Raw DATA payloads (headers + body), one per delivered message. */ - messages: string[]; - close(): Promise; -} - -/** - * Minimal ESMTP server: greeting, EHLO, AUTH PLAIN/LOGIN, MAIL/RCPT/DATA/QUIT. - * Deliberately does NOT advertise STARTTLS — the tests connect with - * `secure: false` so nodemailer stays in the clear against localhost. - */ -async function startFakeSmtp(opts: { authOk?: boolean } = {}): Promise { - const authOk = opts.authOk !== false; - const commands: string[] = []; - const messages: string[] = []; - - const server = net.createServer((socket) => { - let buffer = ''; - let inData = false; - let dataBuf = ''; - socket.setEncoding('utf8'); - socket.write('220 fake.smtp.test ESMTP ready\r\n'); - - socket.on('data', (chunk: string) => { - if (inData) { - dataBuf += chunk; - const end = dataBuf.indexOf('\r\n.\r\n'); - if (end === -1) return; - messages.push(dataBuf.slice(0, end)); - dataBuf = ''; - inData = false; - socket.write('250 2.0.0 Ok: queued as FAKE123\r\n'); - return; - } - buffer += chunk; - let idx: number; - while ((idx = buffer.indexOf('\r\n')) !== -1) { - const line = buffer.slice(0, idx); - buffer = buffer.slice(idx + 2); - commands.push(line); - const verb = line.split(/[ :]/)[0].toUpperCase(); - if (verb === 'EHLO' || verb === 'HELO') { - socket.write('250-fake.smtp.test\r\n250-AUTH PLAIN LOGIN\r\n250 SMTPUTF8\r\n'); - } else if (verb === 'AUTH') { - if (/LOGIN/i.test(line)) { - // LOGIN is a 3-step challenge; accept/deny at the end. - socket.write('334 VXNlcm5hbWU6\r\n'); - } else { - socket.write(authOk ? '235 2.7.0 Accepted\r\n' : '535 5.7.8 Error: authentication failed\r\n'); - } - } else if (/^[A-Za-z0-9+/=]+$/.test(line) && commands.some((c) => /^AUTH LOGIN/i.test(c))) { - // base64 continuation of AUTH LOGIN (username, then password) - const step = commands.filter((c) => /^[A-Za-z0-9+/=]+$/.test(c)).length; - if (step === 1) socket.write('334 UGFzc3dvcmQ6\r\n'); - else socket.write(authOk ? '235 2.7.0 Accepted\r\n' : '535 5.7.8 Error: authentication failed\r\n'); - } else if (verb === 'MAIL' || verb === 'RCPT') { - socket.write('250 2.1.0 Ok\r\n'); - } else if (verb === 'DATA') { - inData = true; - socket.write('354 End data with .\r\n'); - } else if (verb === 'QUIT') { - socket.write('221 2.0.0 Bye\r\n'); - socket.end(); - } else { - socket.write('250 2.0.0 Ok\r\n'); - } - } - }); - socket.on('error', () => { /* client hung up */ }); - }); - - await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); - const port = (server.address() as net.AddressInfo).port; - return { - port, - commands, - messages, - close: () => new Promise((resolve) => server.close(() => resolve())), - }; -} - -/** Decode an RFC 2047 encoded-word run (`=?UTF-8?B?..?=` / `?Q?..?=`). */ -function decodeEncodedWords(input: string): string { - return input.replace(/=\?utf-8\?(b|q)\?([^?]+)\?=/gi, (_m, enc: string, payload: string) => { - if (enc.toLowerCase() === 'b') return Buffer.from(payload, 'base64').toString('utf8'); - const bytes = payload - .replace(/_/g, ' ') - .replace(/=([0-9A-Fa-f]{2})/g, (_x, hex: string) => String.fromCharCode(parseInt(hex, 16))); - return Buffer.from(bytes, 'binary').toString('utf8'); - }); -} - -/** Decode every transfer-encoded body part so the text can be asserted. */ -function decodeBodies(raw: string): string { - const out: string[] = [raw]; - // quoted-printable - out.push(Buffer.from( - raw.replace(/=\r?\n/g, '').replace(/=([0-9A-Fa-f]{2})/g, (_m, hex: string) => String.fromCharCode(parseInt(hex, 16))), - 'binary', - ).toString('utf8')); - // base64 blocks (4+ full base64 lines in a row) - for (const block of raw.match(/(?:^[A-Za-z0-9+/=]{20,}\r?\n?){1,}/gm) ?? []) { - out.push(Buffer.from(block.replace(/\s+/g, ''), 'base64').toString('utf8')); - } - return out.join('\n'); -} +// The fake server + wire decoders live in a testkit since #5177, so the +// sys_email round-trip suite proves itself against the SAME wire, not a second +// hand-written copy of it. +import { startFakeSmtp, decodeEncodedWords, decodeBodies, type FakeSmtp } from './fake-smtp.testkit.js'; let server: FakeSmtp | undefined; afterEach(async () => {