From 379febb79176288f614b8c33130feafb06e9b5d2 Mon Sep 17 00:00:00 2001 From: necessarylion Date: Mon, 2 Nov 2026 06:52:17 +0700 Subject: [PATCH 1/6] fix: no mail data in the errors of an encrypted mail job With Mail.queue(message, encrypt: true), the job data is encrypted. But a send error can name the recipient, eg. an SMTP reply like "550 : mailbox unavailable". The worker writes the error to its log and to the failed jobs in plain text, so the address leaked. Now the encrypted mail job catches the send error and throws a RedactedMailError. Every email address and every text of the mail is replaced by ***. The error type, the SMTP or HTTP code and the stack trace stay, to help find the problem. Mail jobs without encrypt keep the full error, as before. --- CHANGELOG.md | 1 + README.md | 2 + lib/src/send_queued_mail.dart | 88 ++++++++++++++++++++++++++++++++++- test/queue_mail_test.dart | 78 ++++++++++++++++++++++++++++++- 4 files changed, 167 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8c466c5..a45e089 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ ## Unreleased +- Fix: with `Mail.queue(message, encrypt: true)`, a send error could name the recipient (eg. an SMTP reply `550 : mailbox unavailable`). The worker wrote that error to its log and to the failed jobs in plain text. Now the encrypted mail job throws a `RedactedMailError`: every email address and every text of the mail is `***`. The error type, the SMTP or HTTP code and the rest of the text stay. Mail jobs without `encrypt` keep the full error, as before. - New: a built-in job for `Mail.queue()` and `Mail.later()` of dox_mailer. `Queue.configure()` turns it on. Apps do not add it to `lib/generated/jobs.g.dart`: every `JobRegistry` finds the built-in jobs (`JobRegistry.builtIn`) after its own jobs. With `encrypt: true`, the mail is saved like a `ShouldBeEncrypted` job. - Now depends on dox_mailer. - Deploy the queue worker together with the web app. An old worker does not know the new mail job, so it fails the queued emails. diff --git a/README.md b/README.md index 8d6dcaa..79e1d0f 100644 --- a/README.md +++ b/README.md @@ -277,6 +277,8 @@ dox_queue has a built-in job for it. It is not in `lib/generated/jobs.g.dart`, a Deploy the worker together with the web app. An old worker does not know this job, so it fails the queued emails. +With `encrypt: true`, a send error is logged and saved with `***` in place of every email address and every text of the mail. The error type and the SMTP code stay. + ## Run the worker ```bash diff --git a/lib/src/send_queued_mail.dart b/lib/src/send_queued_mail.dart index 31294cd..32e88ee 100644 --- a/lib/src/send_queued_mail.dart +++ b/lib/src/send_queued_mail.dart @@ -1,4 +1,5 @@ -import 'package:dox_mailer/dox_mailer.dart' show Mail, MailQueue, Message; +import 'package:dox_mailer/dox_mailer.dart' + show Mail, MailException, MailQueue, Message; import 'job.dart'; import 'job_registry.dart'; @@ -62,6 +63,19 @@ class SendEncryptedQueuedMail extends SendQueuedMail with ShouldBeEncrypted { _decode, ); + /// Sends the mail. When the send fails, it throws a [RedactedMailError] + /// in place of the error. An SMTP reply or a [MailException] can name + /// an address, and the worker writes the error to its log and to the + /// failed jobs in plain text. + @override + Future handle() async { + try { + await super.handle(); + } catch (error, stack) { + Error.throwWithStackTrace(RedactedMailError.of(error, message), stack); + } + } + static DoxJob _decode(Map data) { return SendEncryptedQueuedMail( mailer: data['mailer'] as String, @@ -72,6 +86,78 @@ class SendEncryptedQueuedMail extends SendQueuedMail with ShouldBeEncrypted { } } +/// The error of a mail of `Mail.queue(message, encrypt: true)` that +/// could not be sent. Every email address, and every text of the mail +/// (names, subject, body, headers, file names), is replaced by `***`. +/// The type of the first error and the SMTP or HTTP code stay, to help +/// find the problem. +class RedactedMailError implements Exception { + RedactedMailError(this.type, this.message, {this.statusCode}); + + /// [error] with the data of [mail] (from `Message.toJson()`) taken out + factory RedactedMailError.of(Object error, Map mail) { + return RedactedMailError( + '${error.runtimeType}', + redact('$error', mail), + statusCode: error is MailException ? error.statusCode : null, + ); + } + + /// the class of the first error, eg. `MailException` + final String type; + + /// the text of the first error, with `***` for the mail data + final String message; + + /// the SMTP reply code or the HTTP status, when there is one + final int? statusCode; + + /// any email address, also one that is not in the mail + static final RegExp _email = RegExp( + r'''[^\s<>()\[\]"',;:]+@[^\s<>()\[\]"',;:]+''', + ); + + /// [text] with `***` for every email address and every text of [mail]. + /// Texts shorter than 3 letters stay, so the error can still be read. + static String redact(String text, Map mail) { + List secrets = []; + _collect(mail, secrets); + secrets.sort(_longestFirst); + String result = text; + for (String secret in secrets) { + result = result.replaceAll( + RegExp(RegExp.escape(secret), caseSensitive: false), + '***', + ); + } + return result.replaceAll(_email, '***'); + } + + /// every text in [value], but not the file bytes and content types + static void _collect(Object? value, List into, [String? key]) { + if (key == 'bytes' || key == 'contentType') return; + if (value is String) { + if (value.trim().length >= 3) into.add(value); + } else if (value is Map) { + for (MapEntry entry in value.entries) { + _collect(entry.value, into, '${entry.key}'); + } + } else if (value is List) { + for (Object? item in value) { + _collect(item, into); + } + } + } + + static int _longestFirst(String a, String b) => b.length - a.length; + + @override + String toString() { + String code = statusCode == null ? '' : ' (code $statusCode)'; + return 'RedactedMailError$code: $message'; + } +} + /// Puts the mails of `Mail.queue()` on this queue. `Queue.configure()` /// sets it with `Mail.useQueue()`. class QueueMailQueue implements MailQueue { diff --git a/test/queue_mail_test.dart b/test/queue_mail_test.dart index b2f2f8d..f3dc311 100644 --- a/test/queue_mail_test.dart +++ b/test/queue_mail_test.dart @@ -1,3 +1,6 @@ +import 'dart:async'; + +import 'package:dox_core/dox_core.dart' show AppConfig, Dox, LoggerConfig; import 'package:dox_mailer/dox_mailer.dart'; import 'package:dox_queue/dox_queue.dart'; import 'package:test/test.dart'; @@ -20,6 +23,38 @@ class BrokenTransport extends Transport { late BrokenTransport brokenMailer; +/// a mail server that says no, and names the recipient and the subject, +/// like many SMTP servers do +class RejectTransport extends Transport { + @override + Future send(Message message) async { + throw MailException( + '550 5.1.1 : mailbox unavailable ' + '(subject "Your invoice", for Ann, forwarded to ann.lee@other.org)', + statusCode: 550, + ); + } +} + +/// every line the worker logs while [body] runs +Future _logOf(Future Function() body) async { + StringBuffer log = StringBuffer(); + void keep(Zone self, ZoneDelegate parent, Zone zone, String line) { + log.writeln(line); + } + + await runZoned(body, zoneSpecification: ZoneSpecification(print: keep)); + return log.toString(); +} + +/// the parts of the test mail that must not be in a log +const List _secrets = [ + 'ann@example.com', + 'ann.lee@other.org', + 'Your invoice', + 'Ann', +]; + Message _message() => Message( to:
[const Address('ann@example.com', 'Ann')], bcc:
[const Address('boss@example.com')], @@ -45,7 +80,17 @@ Future _work([String queue = 'default', int tries = 1]) async { } void main() { - setUpAll(setUpTestApp); + setUpAll(() { + setUpTestApp(); + // log on, so the tests can read what the worker logs + Dox().initialize( + AppConfig( + appKey: 'k' * 32, + totalIsolate: 1, + logger: LoggerConfig(enabled: true), + ), + ); + }); setUp(() { Mail.restore(); @@ -60,6 +105,7 @@ void main() { 'main': mainMailer, 'other': otherMailer, 'broken': brokenMailer, + 'reject': RejectTransport(), }, ), ); @@ -123,6 +169,30 @@ void main() { expect(await Queue.driver.pop('default'), isNull); }); + test('encrypt: true: a send error shows no mail data', () async { + await Mail.mailer('reject').queue(_message(), encrypt: true); + String log = await _logOf(_retryThenFail); + List failed = await Queue.driver.failed(); + String error = failed.single.error; + for (String secret in _secrets) { + expect(log, isNot(contains(secret))); + expect(error, isNot(contains(secret))); + } + // still enough to find the problem + printOnFailure(log); + expect(log, contains('failed try 1 of 2')); + expect(log, contains('550 5.1.1 <***>: mailbox unavailable')); + expect(error, contains('RedactedMailError (code 550)')); + expect(error, contains('MailException: 550 5.1.1 <***>')); + }); + + test('a plain mail job keeps the full error, as before', () async { + await Mail.mailer('reject').queue(_message()); + await _logOf(_retryThenFail); + List failed = await Queue.driver.failed(); + expect(failed.single.error, contains('')); + }); + test('encrypt: true saves the mail encrypted', () async { await Mail.queue(_message(), encrypt: true); JobPayload payload = await _work(); @@ -135,3 +205,9 @@ void main() { }); } } + +/// two tries: the first is released, the second fails +Future _retryThenFail() async { + await _work('default', 2); + await _work('default', 2); +} From 3be26aef439fb5826586146ff71c7b93cd5a1211 Mon Sep 17 00:00:00 2001 From: necessarylion Date: Mon, 2 Nov 2026 06:59:51 +0700 Subject: [PATCH 2/6] fix: redact a long mail body without a RegExp redact() made one RegExp per text of the mail. For a text over about 30 KB, Dart throws "Regular expression too large", and the text of that error holds the start of the body. It came out of handle(), so the worker logged it and saved it. The real send error was lost. Now redact() uses a plain search, and skips a text that is longer than the error. When the redaction fails, the error has only the type and the code. A text must be a whole word, and tags are skipped, so a tag like smtp does not hide SMTP: in the error. The email regex has length limits, so it stays fast on a long error. The README and the changelog now say what is really hidden. --- CHANGELOG.md | 2 +- README.md | 7 +- lib/src/send_queued_mail.dart | 94 +++++++++++++++++++++------ test/queue_mail_test.dart | 25 ++++++++ test/redacted_mail_error_test.dart | 100 +++++++++++++++++++++++++++++ 5 files changed, 205 insertions(+), 23 deletions(-) create mode 100644 test/redacted_mail_error_test.dart diff --git a/CHANGELOG.md b/CHANGELOG.md index a45e089..d8d84d9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ ## Unreleased -- Fix: with `Mail.queue(message, encrypt: true)`, a send error could name the recipient (eg. an SMTP reply `550 : mailbox unavailable`). The worker wrote that error to its log and to the failed jobs in plain text. Now the encrypted mail job throws a `RedactedMailError`: every email address and every text of the mail is `***`. The error type, the SMTP or HTTP code and the rest of the text stay. Mail jobs without `encrypt` keep the full error, as before. +- Fix: with `Mail.queue(message, encrypt: true)`, a send error could name the recipient (eg. an SMTP reply `550 : mailbox unavailable`). The worker wrote that error to its log and to the failed jobs in plain text. Now the encrypted mail job throws a `RedactedMailError`. It hides every email address, and each whole text of the mail (names, subject, body, header and metadata values, file names) when it is a whole word in the error. A part of a text, or an encoded form (JSON, MIME, URL), may still show. The error type and the SMTP or HTTP code stay. When the hiding fails, only the type and the code are kept. Mail jobs without `encrypt` keep the full error, as before. - New: a built-in job for `Mail.queue()` and `Mail.later()` of dox_mailer. `Queue.configure()` turns it on. Apps do not add it to `lib/generated/jobs.g.dart`: every `JobRegistry` finds the built-in jobs (`JobRegistry.builtIn`) after its own jobs. With `encrypt: true`, the mail is saved like a `ShouldBeEncrypted` job. - Now depends on dox_mailer. - Deploy the queue worker together with the web app. An old worker does not know the new mail job, so it fails the queued emails. diff --git a/README.md b/README.md index 79e1d0f..ba15e79 100644 --- a/README.md +++ b/README.md @@ -277,7 +277,12 @@ dox_queue has a built-in job for it. It is not in `lib/generated/jobs.g.dart`, a Deploy the worker together with the web app. An old worker does not know this job, so it fails the queued emails. -With `encrypt: true`, a send error is logged and saved with `***` in place of every email address and every text of the mail. The error type and the SMTP code stay. +With `encrypt: true`, a send error is logged and saved with `***` in place of: + +- every email address, +- each whole text of the mail: names, subject, body, header and metadata values, file names. + +The error type and the SMTP code stay. A part of a text, or an encoded form (JSON, MIME, URL), may still show. For strict needs, log only the error type and code. ## Run the worker diff --git a/lib/src/send_queued_mail.dart b/lib/src/send_queued_mail.dart index 32e88ee..229ad10 100644 --- a/lib/src/send_queued_mail.dart +++ b/lib/src/send_queued_mail.dart @@ -87,20 +87,24 @@ class SendEncryptedQueuedMail extends SendQueuedMail with ShouldBeEncrypted { } /// The error of a mail of `Mail.queue(message, encrypt: true)` that -/// could not be sent. Every email address, and every text of the mail -/// (names, subject, body, headers, file names), is replaced by `***`. -/// The type of the first error and the SMTP or HTTP code stay, to help -/// find the problem. +/// could not be sent, with the mail data taken out (see [redact]). The +/// type of the first error and the SMTP or HTTP code stay, to help find +/// the problem. class RedactedMailError implements Exception { RedactedMailError(this.type, this.message, {this.statusCode}); - /// [error] with the data of [mail] (from `Message.toJson()`) taken out + /// [error] with the data of [mail] (from `Message.toJson()`) taken out. + /// When that fails, the text is only the type, never the first text. factory RedactedMailError.of(Object error, Map mail) { - return RedactedMailError( - '${error.runtimeType}', - redact('$error', mail), - statusCode: error is MailException ? error.statusCode : null, - ); + String type = '${error.runtimeType}'; + int? code = error is MailException ? error.statusCode : null; + String text; + try { + text = redact('$error', mail); + } catch (_) { + text = '$type: (could not redact)'; + } + return RedactedMailError(type, text, statusCode: code); } /// the class of the first error, eg. `MailException` @@ -112,32 +116,80 @@ class RedactedMailError implements Exception { /// the SMTP reply code or the HTTP status, when there is one final int? statusCode; - /// any email address, also one that is not in the mail + /// any email address, also one that is not in the mail. The length + /// limits (as in the email RFC) keep it fast on a long text. static final RegExp _email = RegExp( - r'''[^\s<>()\[\]"',;:]+@[^\s<>()\[\]"',;:]+''', + r'''[^\s<>()\[\]"',;:@]{1,64}@[^\s<>()\[\]"',;:@]{1,255}''', ); - /// [text] with `***` for every email address and every text of [mail]. - /// Texts shorter than 3 letters stay, so the error can still be read. + static final RegExp _wordChar = RegExp(r'[\p{L}\p{N}_]', unicode: true); + + /// [text] with `***` for: + /// - every email address, + /// - every whole text of [mail]: names, subject, body, header and + /// metadata values, file names. Case does not count. + /// + /// A text must be a whole word in [text], so the name `Ann` does not + /// break `Annual`. Texts shorter than 3 letters and tags stay (tags + /// are labels like `smtp`, so they would hide useful words). A part of + /// a text, or an encoded form (JSON, MIME, URL), is not found. static String redact(String text, Map mail) { List secrets = []; _collect(mail, secrets); secrets.sort(_longestFirst); String result = text; for (String secret in secrets) { - result = result.replaceAll( - RegExp(RegExp.escape(secret), caseSensitive: false), - '***', - ); + // a text longer than the error can not be in it + if (secret.length > result.length) continue; + result = _hide(result, secret); } return result.replaceAll(_email, '***'); } - /// every text in [value], but not the file bytes and content types + /// [text] with `***` for each whole-word [secret], found with a plain + /// search (a RegExp of a long body is too large for Dart) + static String _hide(String text, String secret) { + String lower = text.toLowerCase(); + String needle = secret.toLowerCase(); + // ponytail: rare letters change length in lower case. Then the + // places do not match, so work on the lower case text. + String source = lower.length == text.length ? text : lower; + bool wordStart = _isWord(needle, 0); + bool wordEnd = _isWord(needle, needle.length - 1); + StringBuffer out = StringBuffer(); + int start = 0; + int search = 0; + while (true) { + int at = lower.indexOf(needle, search); + if (at < 0) break; + int end = at + needle.length; + if ((wordStart && _isWord(lower, at - 1)) || + (wordEnd && _isWord(lower, end))) { + search = at + 1; + continue; + } + out + ..write(source.substring(start, at)) + ..write('***'); + start = end; + search = end; + } + out.write(source.substring(start)); + return out.toString(); + } + + /// true when [text] has a letter, digit or `_` at [index] + static bool _isWord(String text, int index) { + if (index < 0 || index >= text.length) return false; + return _wordChar.hasMatch(text[index]); + } + + /// every text in [value], but not the file bytes, content types and tags static void _collect(Object? value, List into, [String? key]) { - if (key == 'bytes' || key == 'contentType') return; + if (key == 'bytes' || key == 'contentType' || key == 'tags') return; if (value is String) { - if (value.trim().length >= 3) into.add(value); + String secret = value.trim(); + if (secret.length >= 3) into.add(secret); } else if (value is Map) { for (MapEntry entry in value.entries) { _collect(entry.value, into, '${entry.key}'); diff --git a/test/queue_mail_test.dart b/test/queue_mail_test.dart index f3dc311..cd89d63 100644 --- a/test/queue_mail_test.dart +++ b/test/queue_mail_test.dart @@ -36,6 +36,17 @@ class RejectTransport extends Transport { } } +/// a mail server that quotes the whole body in its answer +class QuoteTransport extends Transport { + @override + Future send(Message message) async { + throw MailException( + '552 message refused: ${message.html}', + statusCode: 552, + ); + } +} + /// every line the worker logs while [body] runs Future _logOf(Future Function() body) async { StringBuffer log = StringBuffer(); @@ -106,6 +117,7 @@ void main() { 'other': otherMailer, 'broken': brokenMailer, 'reject': RejectTransport(), + 'quote': QuoteTransport(), }, ), ); @@ -186,6 +198,19 @@ void main() { expect(error, contains('MailException: 550 5.1.1 <***>')); }); + test('encrypt: true: a 100 KB body is not in the error', () async { + String html = '

Your reset code is 918273

${'x' * 100 * 1024}'; + Message message = _message().copyWith(html: html); + await Mail.mailer('quote').queue(message, encrypt: true); + String log = await _logOf(_retryThenFail); + String error = (await Queue.driver.failed()).single.error; + for (String text in [log, error]) { + expect(text, isNot(contains('918273'))); + expect(text, isNot(contains('xxxx'))); + expect(text, contains('(code 552): MailException: 552')); + } + }); + test('a plain mail job keeps the full error, as before', () async { await Mail.mailer('reject').queue(_message()); await _logOf(_retryThenFail); diff --git a/test/redacted_mail_error_test.dart b/test/redacted_mail_error_test.dart new file mode 100644 index 0000000..a50da7e --- /dev/null +++ b/test/redacted_mail_error_test.dart @@ -0,0 +1,100 @@ +import 'dart:io'; + +import 'package:dox_mailer/dox_mailer.dart'; +import 'package:dox_queue/src/send_queued_mail.dart'; +import 'package:test/test.dart'; + +/// an error whose text can not be made, so the redaction fails +class BadTextError extends MailException { + BadTextError() : super('ann@example.com', statusCode: 421); + + @override + String toString() => throw StateError('no text'); +} + +Map _mail({String body = 'Hi Ann', List? tags}) { + return Message( + from: const Address('app@example.com', 'App'), + to:
[const Address('ann@example.com', 'Ann')], + subject: 'Your reset code', + text: body, + tags: tags ?? [], + ).toJson(); +} + +/// a body of about [size] letters with a secret code in it +String _bigBody(int size) { + return 'Hi Ann, your reset code is 918273. ${'x' * size}'; +} + +void main() { + for (int size in [100 * 1024, 1024 * 1024]) { + test('a ${size ~/ 1024} KB body is hidden, type and code stay', () { + String body = _bigBody(size); + RedactedMailError error = RedactedMailError.of( + MailException('552 message refused: $body', statusCode: 552), + _mail(body: body), + ); + String text = '$error'; + expect(text, isNot(contains('918273'))); + expect(text, isNot(contains('xxxx'))); + expect( + text, + 'RedactedMailError (code 552): ' + 'MailException: 552 message refused: ***', + ); + expect(error.type, 'MailException'); + }); + } + + test('a big body that is not in the error does not break it', () { + RedactedMailError error = RedactedMailError.of( + MailException('550 : no mailbox', statusCode: 550), + _mail(body: _bigBody(1024 * 1024)), + ); + expect( + '$error', + 'RedactedMailError (code 550): ' + 'MailException: 550 <***>: no mailbox', + ); + }); + + test('when the redaction fails, only the type and code are left', () { + RedactedMailError error = RedactedMailError.of(BadTextError(), _mail()); + expect(error.type, 'BadTextError'); + expect(error.statusCode, 421); + expect( + '$error', + 'RedactedMailError (code 421): ' + 'BadTextError: (could not redact)', + ); + }); + + test('a tag or a name does not hide other words', () { + RedactedMailError error = RedactedMailError.of( + MailException('SMTP: Annual limit, Ann can not get more'), + _mail(tags: ['smtp']), + ); + expect( + '$error', + 'RedactedMailError: ' + 'MailException: SMTP: Annual limit, *** can not get more', + ); + }); + + test('a SocketException keeps its host and port, no code', () { + RedactedMailError error = RedactedMailError.of( + SocketException( + 'Connection refused', + address: InternetAddress('10.0.0.5'), + port: 587, + ), + _mail(), + ); + expect(error.type, 'SocketException'); + expect(error.statusCode, isNull); + expect('$error', contains('Connection refused')); + expect('$error', contains('10.0.0.5')); + expect('$error', contains('587')); + }); +} From ed3131509cfdec367076cdbca75fa0214cae6389 Mon Sep 17 00:00:00 2001 From: necessarylion Date: Mon, 2 Nov 2026 07:09:41 +0700 Subject: [PATCH 3/6] fix: hide long addresses whole and redact in one pass An address that is not in the mail, with more than 64 letters before the @, kept its first letters. Now each @ grows left and right over the address letters by hand. So the whole address is hidden, of any length, in one pass. The texts of the mail were searched one by one, so many recipients and a long error were slow. Now the error is put in lower case once, and read once. The texts are grouped by their first 3 letters and their length, so each place needs only a few set lookups. The address pass runs only when the error has an @. 5000 recipients and a 1 MB error now take about 60 to 200 ms. The docs now say that other Unicode forms may still show, and that header names and metadata keys are not hidden. --- CHANGELOG.md | 2 +- README.md | 2 +- lib/src/send_queued_mail.dart | 171 ++++++++++++++++++++++------- test/redacted_mail_error_test.dart | 59 ++++++++++ 4 files changed, 190 insertions(+), 44 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d8d84d9..9f518c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ ## Unreleased -- Fix: with `Mail.queue(message, encrypt: true)`, a send error could name the recipient (eg. an SMTP reply `550 : mailbox unavailable`). The worker wrote that error to its log and to the failed jobs in plain text. Now the encrypted mail job throws a `RedactedMailError`. It hides every email address, and each whole text of the mail (names, subject, body, header and metadata values, file names) when it is a whole word in the error. A part of a text, or an encoded form (JSON, MIME, URL), may still show. The error type and the SMTP or HTTP code stay. When the hiding fails, only the type and the code are kept. Mail jobs without `encrypt` keep the full error, as before. +- Fix: with `Mail.queue(message, encrypt: true)`, a send error could name the recipient (eg. an SMTP reply `550 : mailbox unavailable`). The worker wrote that error to its log and to the failed jobs in plain text. Now the encrypted mail job throws a `RedactedMailError`. It hides every email address, and each whole text of the mail (names, subject, body, header and metadata values, file names) when it is a whole word in the error. A part of a text, an encoded form (JSON, MIME, URL), or another Unicode form (eg. `Zoë` as NFC or NFD) may still show. Header names and metadata keys are not hidden, only their values. The error type and the SMTP or HTTP code stay. When the hiding fails, only the type and the code are kept. Mail jobs without `encrypt` keep the full error, as before. - New: a built-in job for `Mail.queue()` and `Mail.later()` of dox_mailer. `Queue.configure()` turns it on. Apps do not add it to `lib/generated/jobs.g.dart`: every `JobRegistry` finds the built-in jobs (`JobRegistry.builtIn`) after its own jobs. With `encrypt: true`, the mail is saved like a `ShouldBeEncrypted` job. - Now depends on dox_mailer. - Deploy the queue worker together with the web app. An old worker does not know the new mail job, so it fails the queued emails. diff --git a/README.md b/README.md index ba15e79..98db80d 100644 --- a/README.md +++ b/README.md @@ -282,7 +282,7 @@ With `encrypt: true`, a send error is logged and saved with `***` in place of: - every email address, - each whole text of the mail: names, subject, body, header and metadata values, file names. -The error type and the SMTP code stay. A part of a text, or an encoded form (JSON, MIME, URL), may still show. For strict needs, log only the error type and code. +The error type and the SMTP code stay. A part of a text, an encoded form (JSON, MIME, URL), or another Unicode form (eg. `Zoë` as NFC or NFD) may still show. Header names and metadata keys are not hidden, only their values. For strict needs, log only the error type and code. ## Run the worker diff --git a/lib/src/send_queued_mail.dart b/lib/src/send_queued_mail.dart index 229ad10..66ba43f 100644 --- a/lib/src/send_queued_mail.dart +++ b/lib/src/send_queued_mail.dart @@ -116,13 +116,10 @@ class RedactedMailError implements Exception { /// the SMTP reply code or the HTTP status, when there is one final int? statusCode; - /// any email address, also one that is not in the mail. The length - /// limits (as in the email RFC) keep it fast on a long text. - static final RegExp _email = RegExp( - r'''[^\s<>()\[\]"',;:@]{1,64}@[^\s<>()\[\]"',;:@]{1,255}''', - ); - - static final RegExp _wordChar = RegExp(r'[\p{L}\p{N}_]', unicode: true); + /// the characters that end an email address: space, `<>()[]"',;:@` + static final Set _notAddress = { + ...' \t\r\n<>()[]"\',;:@'.codeUnits, + }; /// [text] with `***` for: /// - every email address, @@ -132,64 +129,117 @@ class RedactedMailError implements Exception { /// A text must be a whole word in [text], so the name `Ann` does not /// break `Annual`. Texts shorter than 3 letters and tags stay (tags /// are labels like `smtp`, so they would hide useful words). A part of - /// a text, or an encoded form (JSON, MIME, URL), is not found. + /// a text, an encoded form (JSON, MIME, URL) or another Unicode form + /// (NFC or NFD) is not found. Header names and metadata keys stay. + /// + /// It reads [text] once for the texts and once for the addresses, so + /// a long error with many recipients stays fast. No RegExp is made + /// from mail data (a RegExp of a long body is too large for Dart). static String redact(String text, Map mail) { - List secrets = []; - _collect(mail, secrets); - secrets.sort(_longestFirst); - String result = text; - for (String secret in secrets) { - // a text longer than the error can not be in it - if (secret.length > result.length) continue; - result = _hide(result, secret); - } - return result.replaceAll(_email, '***'); - } - - /// [text] with `***` for each whole-word [secret], found with a plain - /// search (a RegExp of a long body is too large for Dart) - static String _hide(String text, String secret) { String lower = text.toLowerCase(); - String needle = secret.toLowerCase(); // ponytail: rare letters change length in lower case. Then the // places do not match, so work on the lower case text. String source = lower.length == text.length ? text : lower; - bool wordStart = _isWord(needle, 0); - bool wordEnd = _isWord(needle, needle.length - 1); + Map byStart = _secrets(mail, lower.length); + String result = byStart.isEmpty + ? source + : _hideTexts(source, lower, byStart); + return result.contains('@') ? _hideAddresses(result) : result; + } + + /// the lower case texts of [mail] that fit in [max] letters, by their + /// first 3 letters + static Map _secrets(Map mail, int max) { + List texts = []; + _collect(mail, texts); + Map byStart = {}; + for (String value in texts) { + String secret = value.trim().toLowerCase(); + if (secret.length < 3 || secret.length > max) continue; + byStart.putIfAbsent(_key(secret, 0), _Texts.new).add(secret); + } + return byStart; + } + + /// the first 3 code units of [text] at [at], as one number + static int _key(String text, int at) { + return (text.codeUnitAt(at) << 32) | + (text.codeUnitAt(at + 1) << 16) | + text.codeUnitAt(at + 2); + } + + /// one pass over [lower]: at each place, the longest whole-word text + /// of [byStart] that starts there is replaced by `***` + static String _hideTexts( + String source, + String lower, + Map byStart, + ) { StringBuffer out = StringBuffer(); int start = 0; - int search = 0; - while (true) { - int at = lower.indexOf(needle, search); - if (at < 0) break; - int end = at + needle.length; - if ((wordStart && _isWord(lower, at - 1)) || - (wordEnd && _isWord(lower, end))) { - search = at + 1; + int at = 0; + while (at + 3 <= lower.length) { + _Texts? texts = byStart[_key(lower, at)]; + int? length = texts?.longestAt(lower, at); + if (length == null) { + at++; continue; } out ..write(source.substring(start, at)) ..write('***'); - start = end; - search = end; + at += length; + start = at; } out.write(source.substring(start)); return out.toString(); } - /// true when [text] has a letter, digit or `_` at [index] + /// [text] with `***` for every email address, of any length. Each `@` + /// grows left and right over address letters, so it is one pass. + static String _hideAddresses(String text) { + StringBuffer out = StringBuffer(); + int start = 0; + int at = text.indexOf('@'); + while (at >= 0) { + int left = at; + while (left > start && !_notAddress.contains(text.codeUnitAt(left - 1))) { + left--; + } + int right = at + 1; + while (right < text.length && + !_notAddress.contains(text.codeUnitAt(right))) { + right++; + } + if (left < at && right > at + 1) { + out + ..write(text.substring(start, left)) + ..write('***'); + start = right; + } + at = text.indexOf('@', at + 1); + } + out.write(text.substring(start)); + return out.toString(); + } + + /// true when [text] has a letter, digit or `_` at [index]. Every + /// letter that is not ASCII counts as a letter. static bool _isWord(String text, int index) { if (index < 0 || index >= text.length) return false; - return _wordChar.hasMatch(text[index]); + int c = text.codeUnitAt(index); + return (c >= 0x30 && c <= 0x39) || + (c >= 0x41 && c <= 0x5a) || + (c >= 0x61 && c <= 0x7a) || + c == 0x5f || + c >= 0x80; } /// every text in [value], but not the file bytes, content types and tags static void _collect(Object? value, List into, [String? key]) { if (key == 'bytes' || key == 'contentType' || key == 'tags') return; if (value is String) { - String secret = value.trim(); - if (secret.length >= 3) into.add(secret); + into.add(value); } else if (value is Map) { for (MapEntry entry in value.entries) { _collect(entry.value, into, '${entry.key}'); @@ -201,8 +251,6 @@ class RedactedMailError implements Exception { } } - static int _longestFirst(String a, String b) => b.length - a.length; - @override String toString() { String code = statusCode == null ? '' : ' (code $statusCode)'; @@ -210,6 +258,45 @@ class RedactedMailError implements Exception { } } +/// The texts of a mail that start with the same 3 letters, by length. +class _Texts { + final Map> _byLength = >{}; + List _lengths = []; + + void add(String text) { + Set? same = _byLength[text.length]; + if (same == null) { + same = {}; + _byLength[text.length] = same; + _lengths = [..._lengths, text.length]..sort(_down); + } + same.add(text); + } + + static int _down(int a, int b) => b - a; + + /// the length of the longest text that is a whole word in [lower] at + /// [at], or null + int? longestAt(String lower, int at) { + for (int length in _lengths) { + int end = at + length; + if (end > lower.length) continue; + String part = lower.substring(at, end); + if (!_byLength[length]!.contains(part)) continue; + if (RedactedMailError._isWord(part, 0) && + RedactedMailError._isWord(lower, at - 1)) { + continue; + } + if (RedactedMailError._isWord(part, length - 1) && + RedactedMailError._isWord(lower, end)) { + continue; + } + return length; + } + return null; + } +} + /// Puts the mails of `Mail.queue()` on this queue. `Queue.configure()` /// sets it with `Mail.useQueue()`. class QueueMailQueue implements MailQueue { diff --git a/test/redacted_mail_error_test.dart b/test/redacted_mail_error_test.dart index a50da7e..1f7183c 100644 --- a/test/redacted_mail_error_test.dart +++ b/test/redacted_mail_error_test.dart @@ -97,4 +97,63 @@ void main() { expect('$error', contains('10.0.0.5')); expect('$error', contains('587')); }); + + test('a long address that is not in the mail is hidden whole', () { + String local = 'L' * 200; + RedactedMailError error = RedactedMailError.of( + MailException('550 <$local@other.org>: no mailbox', statusCode: 550), + _mail(), + ); + expect( + '$error', + 'RedactedMailError (code 550): ' + 'MailException: 550 <***>: no mailbox', + ); + }); + + test('a 5 MB error with no spaces is fast', () { + String half = 'x' * (5 * 512 * 1024); + Stopwatch watch = Stopwatch()..start(); + RedactedMailError plain = RedactedMailError.of( + MailException('$half$half'), + _mail(), + ); + RedactedMailError address = RedactedMailError.of( + MailException('$half@$half'), + _mail(), + ); + watch.stop(); + print('5 MB errors with no spaces: ${watch.elapsedMilliseconds} ms'); + expect(plain.message.length, greaterThan(5 * 1024 * 1024)); + expect(address.message, 'MailException: ***'); + // generous, so CI is not flaky. Locally it is far less. + expect(watch.elapsed, lessThan(const Duration(seconds: 5))); + }); + + test('5000 recipients and a 1 MB error are fast', () { + List
to =
[ + for (int i = 0; i < 5000; i++) Address('user$i@example.com', 'User $i'), + ]; + Map mail = Message( + from: const Address('app@example.com'), + to: to, + subject: 'Hi', + text: 'Hello', + ).toJson(); + StringBuffer text = StringBuffer('550 refused: '); + while (text.length < 1024 * 1024) { + text.write('User 42 said no to user42@example.com. '); + } + Stopwatch watch = Stopwatch()..start(); + RedactedMailError error = RedactedMailError.of( + MailException('$text', statusCode: 550), + mail, + ); + watch.stop(); + expect(error.message, isNot(contains('User 42'))); + expect(error.message, isNot(contains('example.com'))); + expect(error.message, contains('550 refused: *** said no to ***.')); + print('5000 recipients, 1 MB error: ${watch.elapsedMilliseconds} ms'); + expect(watch.elapsed, lessThan(const Duration(seconds: 5))); + }); } From 78c1c5b628db6ff28bb9a587cda475247835fad0 Mon Sep 17 00:00:00 2001 From: necessarylion Date: Mon, 2 Nov 2026 07:14:10 +0700 Subject: [PATCH 4/6] fix: cut huge mail errors and hide header names and metadata keys - Cut an error to its first 16 KB before the redaction, and say it was cut. A huge error could block the worker for seconds. A mail text that goes past the cut is hidden up to the cut. - Check each place with a rolling hash before copying the text. Many mail texts with the same start were slow on a long error. - Hide the names of headers and the keys of metadata. A metadata key named tags, bytes or contentType no longer makes its value stay. - Export RedactedMailError from package:dox_queue/dox_queue.dart. - README: say which mail data can still show. --- CHANGELOG.md | 2 +- README.md | 13 ++- lib/dox_queue.dart | 1 + lib/src/send_queued_mail.dart | 159 +++++++++++++++++++++++++---- test/redacted_mail_error_test.dart | 87 +++++++++++++++- 5 files changed, 234 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f518c6..538dca2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ ## Unreleased -- Fix: with `Mail.queue(message, encrypt: true)`, a send error could name the recipient (eg. an SMTP reply `550 : mailbox unavailable`). The worker wrote that error to its log and to the failed jobs in plain text. Now the encrypted mail job throws a `RedactedMailError`. It hides every email address, and each whole text of the mail (names, subject, body, header and metadata values, file names) when it is a whole word in the error. A part of a text, an encoded form (JSON, MIME, URL), or another Unicode form (eg. `Zoë` as NFC or NFD) may still show. Header names and metadata keys are not hidden, only their values. The error type and the SMTP or HTTP code stay. When the hiding fails, only the type and the code are kept. Mail jobs without `encrypt` keep the full error, as before. +- Fix: with `Mail.queue(message, encrypt: true)`, a send error could name the recipient (eg. an SMTP reply `550 : mailbox unavailable`). The worker wrote that error to its log and to the failed jobs in plain text. Now the encrypted mail job throws a `RedactedMailError`. It hides every email address, and each whole text of the mail (names, subject, body, header names and values, metadata keys and values, file names) when it is a whole word in the error. Texts shorter than 3 letters and tags stay. A part of a text, an encoded form (JSON, MIME, URL), or another Unicode form (eg. `Zoë` as NFC or NFD) may still show. An error longer than 16 KB is cut to its first 16 KB, so the worker stays fast. `RedactedMailError` is exported. The error type and the SMTP or HTTP code stay. When the hiding fails, only the type and the code are kept. Mail jobs without `encrypt` keep the full error, as before. - New: a built-in job for `Mail.queue()` and `Mail.later()` of dox_mailer. `Queue.configure()` turns it on. Apps do not add it to `lib/generated/jobs.g.dart`: every `JobRegistry` finds the built-in jobs (`JobRegistry.builtIn`) after its own jobs. With `encrypt: true`, the mail is saved like a `ShouldBeEncrypted` job. - Now depends on dox_mailer. - Deploy the queue worker together with the web app. An old worker does not know the new mail job, so it fails the queued emails. diff --git a/README.md b/README.md index 98db80d..49b7661 100644 --- a/README.md +++ b/README.md @@ -280,9 +280,18 @@ Deploy the worker together with the web app. An old worker does not know this jo With `encrypt: true`, a send error is logged and saved with `***` in place of: - every email address, -- each whole text of the mail: names, subject, body, header and metadata values, file names. +- each whole text of the mail: names, subject, body, header names and values, metadata keys and values, file names. -The error type and the SMTP code stay. A part of a text, an encoded form (JSON, MIME, URL), or another Unicode form (eg. `Zoë` as NFC or NFD) may still show. Header names and metadata keys are not hidden, only their values. For strict needs, log only the error type and code. +The error type and the SMTP code stay. Catch it as `RedactedMailError`: it has `type`, `message` and `statusCode`. + +Some mail data can still show: + +- A text shorter than 3 letters stays. +- Tags are never hidden. They are labels like `smtp`, so hiding them would hide useful words. +- A text is hidden only as a whole word. Text joined to other letters stays, eg. `code2` when the subject is `code`. +- A part of a text, an encoded form (JSON, MIME, URL), or another Unicode form (eg. `Zoë` as NFC or NFD) is not found. + +An error longer than 16 KB is cut to its first 16 KB, and the text says it was cut. This keeps the worker fast. For strict needs, log only the error type and code. ## Run the worker diff --git a/lib/dox_queue.dart b/lib/dox_queue.dart index 13e416b..9201ca6 100644 --- a/lib/dox_queue.dart +++ b/lib/dox_queue.dart @@ -14,4 +14,5 @@ export 'src/queue_driver.dart'; export 'src/queue_provider.dart'; export 'src/queue_tables.dart'; export 'src/queue_worker.dart'; +export 'src/send_queued_mail.dart' show RedactedMailError; export 'src/worker.dart'; diff --git a/lib/src/send_queued_mail.dart b/lib/src/send_queued_mail.dart index 66ba43f..0f46e84 100644 --- a/lib/src/send_queued_mail.dart +++ b/lib/src/send_queued_mail.dart @@ -116,6 +116,10 @@ class RedactedMailError implements Exception { /// the SMTP reply code or the HTTP status, when there is one final int? statusCode; + /// the most letters of an error that are redacted. The rest is cut, + /// so a huge error can not block the worker for seconds. + static const int maxLength = 16 * 1024; + /// the characters that end an email address: space, `<>()[]"',;:@` static final Set _notAddress = { ...' \t\r\n<>()[]"\',;:@'.codeUnits, @@ -123,44 +127,104 @@ class RedactedMailError implements Exception { /// [text] with `***` for: /// - every email address, - /// - every whole text of [mail]: names, subject, body, header and - /// metadata values, file names. Case does not count. + /// - every whole text of [mail]: names, subject, body, header names and + /// values, metadata keys and values, file names. Case does not count. + /// + /// Only the first [maxLength] letters are kept, and the text then says + /// it was cut. A text of [mail] that starts near the cut and goes past + /// it is hidden up to the cut. /// /// A text must be a whole word in [text], so the name `Ann` does not /// break `Annual`. Texts shorter than 3 letters and tags stay (tags /// are labels like `smtp`, so they would hide useful words). A part of /// a text, an encoded form (JSON, MIME, URL) or another Unicode form - /// (NFC or NFD) is not found. Header names and metadata keys stay. + /// (NFC or NFD) is not found. /// /// It reads [text] once for the texts and once for the addresses, so /// a long error with many recipients stays fast. No RegExp is made /// from mail data (a RegExp of a long body is too large for Dart). static String redact(String text, Map mail) { + int fullLength = text.length; + bool cut = fullLength > maxLength; + if (cut) text = text.substring(0, maxLength); String lower = text.toLowerCase(); // ponytail: rare letters change length in lower case. Then the // places do not match, so work on the lower case text. String source = lower.length == text.length ? text : lower; - Map byStart = _secrets(mail, lower.length); + List secrets = _secrets(mail); + String tail = ''; + if (cut) { + tail = ' ... (cut, $fullLength letters in all)'; + int? at = _cutStart(lower, secrets); + if (at != null) { + source = source.substring(0, at); + lower = lower.substring(0, at); + tail = '***$tail'; + } + } + Map byStart = _byStart(secrets, lower.length); String result = byStart.isEmpty ? source : _hideTexts(source, lower, byStart); - return result.contains('@') ? _hideAddresses(result) : result; + if (result.contains('@')) result = _hideAddresses(result); + return '$result$tail'; } - /// the lower case texts of [mail] that fit in [max] letters, by their - /// first 3 letters - static Map _secrets(Map mail, int max) { + /// the trimmed, lower case texts of [mail], 3 letters or more + static List _secrets(Map mail) { List texts = []; _collect(mail, texts); + return [ + for (String value in texts) + if (value.trim().length >= 3) value.trim().toLowerCase(), + ]; + } + + /// the [secrets] that fit in [max] letters, by their first 3 letters + static Map _byStart(List secrets, int max) { Map byStart = {}; - for (String value in texts) { - String secret = value.trim().toLowerCase(); - if (secret.length < 3 || secret.length > max) continue; + for (String secret in secrets) { + if (secret.length > max) continue; byStart.putIfAbsent(_key(secret, 0), _Texts.new).add(secret); } return byStart; } + /// the first place where a whole-word text of [secrets] starts at the + /// end of the cut [lower] text and goes past the cut, or null + static int? _cutStart(String lower, List secrets) { + int? first; + for (String needle in secrets) { + int from = lower.length - needle.length + 1; + if (from < 0) from = 0; + // find the start with a short head of the text, then check the rest + String head = needle.length > 16 ? needle.substring(0, 16) : needle; + int last = lower.length - head.length; + int at = lower.indexOf(head, from); + while (at >= 0 && at <= last && !_startsAt(lower, needle, at)) { + at = lower.indexOf(head, at + 1); + } + if (at < 0 || at > last) { + at = -1; + for (int i = last + 1 > from ? last + 1 : from; i < lower.length; i++) { + if (_startsAt(lower, needle, i)) { + at = i; + break; + } + } + } + if (at >= 0 && (first == null || at < first)) first = at; + } + return first; + } + + /// true when the end of [lower] from [at] is the start of [needle], at + /// the start of a word + static bool _startsAt(String lower, String needle, int at) { + if (_isWord(needle, 0) && _isWord(lower, at - 1)) return false; + return needle.startsWith(lower.substring(at)); + } + /// the first 3 code units of [text] at [at], as one number static int _key(String text, int at) { return (text.codeUnitAt(at) << 32) | @@ -175,12 +239,13 @@ class RedactedMailError implements Exception { String lower, Map byStart, ) { + _Hash hash = _Hash(lower); StringBuffer out = StringBuffer(); int start = 0; int at = 0; while (at + 3 <= lower.length) { _Texts? texts = byStart[_key(lower, at)]; - int? length = texts?.longestAt(lower, at); + int? length = texts?.longestAt(lower, hash, at); if (length == null) { at++; continue; @@ -235,18 +300,43 @@ class RedactedMailError implements Exception { c >= 0x80; } - /// every text in [value], but not the file bytes, content types and tags - static void _collect(Object? value, List into, [String? key]) { - if (key == 'bytes' || key == 'contentType' || key == 'tags') return; + /// every text of [mail] (from `Message.toJson()`), with the names of + /// the headers and the keys of the metadata, but not the tags and the + /// file bytes and content types + static void _collect(Map mail, List into) { + for (MapEntry entry in mail.entries) { + Object? value = entry.value; + switch (entry.key) { + case 'tags': + break; + case 'attachments': + for (Object? file in value is List ? value : const []) { + if (file is! Map) continue; + for (MapEntry field in file.entries) { + if (field.key == 'bytes' || field.key == 'contentType') continue; + _add(field.value, into); + } + } + case 'headers' || 'metadata': + if (value is Map) _add(value.keys.toList(), into); + _add(value, into); + default: + _add(value, into); + } + } + } + + /// every text in [value], with the values of a map, not its keys + static void _add(Object? value, List into) { if (value is String) { into.add(value); } else if (value is Map) { - for (MapEntry entry in value.entries) { - _collect(entry.value, into, '${entry.key}'); + for (Object? item in value.values) { + _add(item, into); } } else if (value is List) { for (Object? item in value) { - _collect(item, into); + _add(item, into); } } } @@ -261,6 +351,7 @@ class RedactedMailError implements Exception { /// The texts of a mail that start with the same 3 letters, by length. class _Texts { final Map> _byLength = >{}; + final Map> _hashes = >{}; List _lengths = []; void add(String text) { @@ -268,19 +359,23 @@ class _Texts { if (same == null) { same = {}; _byLength[text.length] = same; + _hashes[text.length] = {}; _lengths = [..._lengths, text.length]..sort(_down); } same.add(text); + _hashes[text.length]!.add(_Hash(text).of(0, text.length)); } static int _down(int a, int b) => b - a; /// the length of the longest text that is a whole word in [lower] at - /// [at], or null - int? longestAt(String lower, int at) { + /// [at], or null. The [hash] of [lower] skips most places without + /// copying the text, so many texts of the same start stay fast. + int? longestAt(String lower, _Hash hash, int at) { for (int length in _lengths) { int end = at + length; if (end > lower.length) continue; + if (!_hashes[length]!.contains(hash.of(at, end))) continue; String part = lower.substring(at, end); if (!_byLength[length]!.contains(part)) continue; if (RedactedMailError._isWord(part, 0) && @@ -297,6 +392,30 @@ class _Texts { } } +/// A rolling hash of a text: the hash of any part in constant time. +class _Hash { + _Hash(String text) + : _prefix = List.filled(text.length + 1, 0), + _power = List.filled(text.length + 1, 1) { + for (int i = 0; i < text.length; i++) { + _prefix[i + 1] = (_prefix[i] * _base + text.codeUnitAt(i)) % _mod; + _power[i + 1] = _power[i] * _base % _mod; + } + } + + static const int _mod = 2147483647; + static const int _base = 131; + + final List _prefix; + final List _power; + + /// the hash of the part from [start] to [end] + int of(int start, int end) { + int value = (_prefix[end] - _prefix[start] * _power[end - start]) % _mod; + return value < 0 ? value + _mod : value; + } +} + /// Puts the mails of `Mail.queue()` on this queue. `Queue.configure()` /// sets it with `Mail.useQueue()`. class QueueMailQueue implements MailQueue { diff --git a/test/redacted_mail_error_test.dart b/test/redacted_mail_error_test.dart index 1f7183c..9db4546 100644 --- a/test/redacted_mail_error_test.dart +++ b/test/redacted_mail_error_test.dart @@ -1,7 +1,8 @@ import 'dart:io'; +import 'dart:typed_data'; import 'package:dox_mailer/dox_mailer.dart'; -import 'package:dox_queue/src/send_queued_mail.dart'; +import 'package:dox_queue/dox_queue.dart'; import 'package:test/test.dart'; /// an error whose text can not be made, so the redaction fails @@ -12,13 +13,22 @@ class BadTextError extends MailException { String toString() => throw StateError('no text'); } -Map _mail({String body = 'Hi Ann', List? tags}) { +Map _mail({ + String body = 'Hi Ann', + List? tags, + Map? headers, + Map? metadata, + List? attachments, +}) { return Message( from: const Address('app@example.com', 'App'), to:
[const Address('ann@example.com', 'Ann')], subject: 'Your reset code', text: body, tags: tags ?? [], + headers: headers ?? {}, + metadata: metadata ?? {}, + attachments: attachments ?? [], ).toJson(); } @@ -41,7 +51,9 @@ void main() { expect( text, 'RedactedMailError (code 552): ' - 'MailException: 552 message refused: ***', + 'MailException: 552 message refused: *** ' + '... (cut, ${'MailException: 552 message refused: $body'.length} ' + 'letters in all)', ); expect(error.type, 'MailException'); }); @@ -59,6 +71,67 @@ void main() { ); }); + test('a 1 MB error is cut and redacted fast', () { + List files = [ + for (int i = 0; i < 2000; i++) + Attachment(Uint8List(0), 'file-$i.pdf', contentType: 'text/plain'), + ]; + String body = '${'a' * (200 * 1024)}b'; + Stopwatch watch = Stopwatch()..start(); + RedactedMailError error = RedactedMailError.of( + MailException('${'a' * (1024 * 1024)} ann@example.com'), + _mail(body: body, attachments: files), + ); + watch.stop(); + expect(watch.elapsedMilliseconds, lessThan(500)); + expect(error.message.length, lessThan(RedactedMailError.maxLength + 100)); + expect(error.message, endsWith('letters in all)')); + }); + + test('a mail text that goes past the cut is hidden up to the cut', () { + String body = 'Hi Ann, your reset code is 918273. ${'y' * 20000}'; + RedactedMailError error = RedactedMailError.of( + MailException('refused: ${'z ' * 8000}$body'), + _mail(body: body), + ); + expect(error.message, isNot(contains('918273'))); + expect(error.message, isNot(contains('reset'))); + expect(error.message, contains('z z *** ... (cut,')); + }); + + test('header names, metadata keys and their values are hidden', () { + RedactedMailError error = RedactedMailError.of( + MailException( + 'bad X-Customer-Ref: 7781-secret, order-9912 = vip, ' + 'tags = gold-tier, bytes = blue-sky, contentType = top-plan', + ), + _mail( + headers: {'X-Customer-Ref': '7781-secret'}, + metadata: { + 'order-9912': 'vip', + 'tags': 'gold-tier', + 'bytes': 'blue-sky', + 'contentType': 'top-plan', + }, + ), + ); + expect( + error.message, + 'MailException: bad ***: ***, *** = ***, ' + '*** = ***, *** = ***, *** = ***', + ); + }); + + test('a file name is hidden, its content type is not', () { + RedactedMailError error = RedactedMailError.of( + MailException('bad file ann-invoice.pdf (application/pdf)'), + _mail( + attachments: [Attachment(Uint8List(0), 'ann-invoice.pdf')], + ), + ); + expect(error.message, 'MailException: bad file *** (application/pdf)'); + }); + test('when the redaction fails, only the type and code are left', () { RedactedMailError error = RedactedMailError.of(BadTextError(), _mail()); expect(error.type, 'BadTextError'); @@ -124,8 +197,12 @@ void main() { ); watch.stop(); print('5 MB errors with no spaces: ${watch.elapsedMilliseconds} ms'); - expect(plain.message.length, greaterThan(5 * 1024 * 1024)); - expect(address.message, 'MailException: ***'); + expect( + plain.message, + endsWith('(cut, ${5 * 1024 * 1024 + 15} letters in all)'), + ); + expect(plain.message.length, lessThan(RedactedMailError.maxLength + 100)); + expect(address.message, isNot(contains('@'))); // generous, so CI is not flaky. Locally it is far less. expect(watch.elapsed, lessThan(const Duration(seconds: 5))); }); From 026bd92e5592ac403349c16e18a5c09908967816 Mon Sep 17 00:00:00 2001 From: necessarylion Date: Mon, 2 Nov 2026 07:21:27 +0700 Subject: [PATCH 5/6] fix: encrypt the send error of an encrypted mail job Hiding mail data in the error text by search left gaps each round: curly quotes, overlapping texts, and slow cases. Now the error that the worker logs and saves has only the error type and the SMTP or HTTP code. The full error text is encrypted with the app key, like the job data. At most 64 KB of it is encrypted, so a huge error stays small. RedactedMailError.decrypt() reads it back. When the error can not be encrypted, only the type and the code stay. The search code is removed. --- CHANGELOG.md | 2 +- README.md | 17 +- lib/src/send_queued_mail.dart | 363 +++++------------------------ test/queue_mail_test.dart | 12 +- test/redacted_mail_error_test.dart | 259 ++++++-------------- 5 files changed, 139 insertions(+), 514 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 538dca2..0a8268d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ ## Unreleased -- Fix: with `Mail.queue(message, encrypt: true)`, a send error could name the recipient (eg. an SMTP reply `550 : mailbox unavailable`). The worker wrote that error to its log and to the failed jobs in plain text. Now the encrypted mail job throws a `RedactedMailError`. It hides every email address, and each whole text of the mail (names, subject, body, header names and values, metadata keys and values, file names) when it is a whole word in the error. Texts shorter than 3 letters and tags stay. A part of a text, an encoded form (JSON, MIME, URL), or another Unicode form (eg. `Zoë` as NFC or NFD) may still show. An error longer than 16 KB is cut to its first 16 KB, so the worker stays fast. `RedactedMailError` is exported. The error type and the SMTP or HTTP code stay. When the hiding fails, only the type and the code are kept. Mail jobs without `encrypt` keep the full error, as before. +- Fix: with `Mail.queue(message, encrypt: true)`, a send error could name the recipient (eg. an SMTP reply `550 : mailbox unavailable`). The worker wrote that error to its log and to the failed jobs in plain text. Now the encrypted mail job throws a `RedactedMailError`. Its text has only the error type and the SMTP or HTTP code. The full error text (at most 64 KB) is encrypted with the app key. Read it with `RedactedMailError.decrypt(job.error)`. When the error can not be encrypted, only the type and the code are kept. Mail jobs without `encrypt` keep the full error, as before. - New: a built-in job for `Mail.queue()` and `Mail.later()` of dox_mailer. `Queue.configure()` turns it on. Apps do not add it to `lib/generated/jobs.g.dart`: every `JobRegistry` finds the built-in jobs (`JobRegistry.builtIn`) after its own jobs. With `encrypt: true`, the mail is saved like a `ShouldBeEncrypted` job. - Now depends on dox_mailer. - Deploy the queue worker together with the web app. An old worker does not know the new mail job, so it fails the queued emails. diff --git a/README.md b/README.md index 49b7661..2ef645d 100644 --- a/README.md +++ b/README.md @@ -277,21 +277,8 @@ dox_queue has a built-in job for it. It is not in `lib/generated/jobs.g.dart`, a Deploy the worker together with the web app. An old worker does not know this job, so it fails the queued emails. -With `encrypt: true`, a send error is logged and saved with `***` in place of: - -- every email address, -- each whole text of the mail: names, subject, body, header names and values, metadata keys and values, file names. - -The error type and the SMTP code stay. Catch it as `RedactedMailError`: it has `type`, `message` and `statusCode`. - -Some mail data can still show: - -- A text shorter than 3 letters stays. -- Tags are never hidden. They are labels like `smtp`, so hiding them would hide useful words. -- A text is hidden only as a whole word. Text joined to other letters stays, eg. `code2` when the subject is `code`. -- A part of a text, an encoded form (JSON, MIME, URL), or another Unicode form (eg. `Zoë` as NFC or NFD) is not found. - -An error longer than 16 KB is cut to its first 16 KB, and the text says it was cut. This keeps the worker fast. For strict needs, log only the error type and code. +With `encrypt: true`, a send error is logged and saved as its type and code only, eg. `RedactedMailError: MailException (code 550). Details are encrypted: ...`. +The full error text (at most 64 KB) is encrypted with the app key. To read it, pass the saved error to `RedactedMailError.decrypt(job.error)`. ## Run the worker diff --git a/lib/src/send_queued_mail.dart b/lib/src/send_queued_mail.dart index 0f46e84..cb23bb3 100644 --- a/lib/src/send_queued_mail.dart +++ b/lib/src/send_queued_mail.dart @@ -72,7 +72,7 @@ class SendEncryptedQueuedMail extends SendQueuedMail with ShouldBeEncrypted { try { await super.handle(); } catch (error, stack) { - Error.throwWithStackTrace(RedactedMailError.of(error, message), stack); + Error.throwWithStackTrace(RedactedMailError.of(error), stack); } } @@ -87,332 +87,79 @@ class SendEncryptedQueuedMail extends SendQueuedMail with ShouldBeEncrypted { } /// The error of a mail of `Mail.queue(message, encrypt: true)` that -/// could not be sent, with the mail data taken out (see [redact]). The -/// type of the first error and the SMTP or HTTP code stay, to help find -/// the problem. +/// could not be sent. The worker logs and saves it in place of the +/// first error, so no address or text of the mail is written in plain +/// text. +/// +/// Its text has only the type of the first error and the SMTP or HTTP +/// code. The full text of the first error is encrypted with the app key, +/// like the job data. At most the first 64 KB of it is encrypted, so a +/// huge error does not fill the log. Read it with [decrypt]. +/// ``` +/// RedactedMailError: MailException (code 550). Details are encrypted: +/// ``` class RedactedMailError implements Exception { - RedactedMailError(this.type, this.message, {this.statusCode}); + RedactedMailError(this.type, {this.statusCode, this.details}); - /// [error] with the data of [mail] (from `Message.toJson()`) taken out. - /// When that fails, the text is only the type, never the first text. - factory RedactedMailError.of(Object error, Map mail) { + /// [error] with its text encrypted. When that fails, only the type + /// is kept, never the text. + factory RedactedMailError.of(Object error) { String type = '${error.runtimeType}'; - int? code = error is MailException ? error.statusCode : null; - String text; + int? code; + String? details; try { - text = redact('$error', mail); + code = error is MailException ? error.statusCode : null; + String text = '$error'; + if (text.length > maxDetails) text = text.substring(0, maxDetails); + details = Queue.encryptData(text, _purpose); } catch (_) { - text = '$type: (could not redact)'; + details = null; } - return RedactedMailError(type, text, statusCode: code); + return RedactedMailError(type, statusCode: code, details: details); } + /// the most letters of the first error that are encrypted + static const int maxDetails = 64 * 1024; + + /// bound to the encrypted details, so job data can not be read as them + static const String _purpose = 'dox_queue:RedactedMailError'; + + static const String _marker = 'Details are encrypted: '; + /// the class of the first error, eg. `MailException` final String type; - /// the text of the first error, with `***` for the mail data - final String message; - /// the SMTP reply code or the HTTP status, when there is one final int? statusCode; - /// the most letters of an error that are redacted. The rest is cut, - /// so a huge error can not block the worker for seconds. - static const int maxLength = 16 * 1024; - - /// the characters that end an email address: space, `<>()[]"',;:@` - static final Set _notAddress = { - ...' \t\r\n<>()[]"\',;:@'.codeUnits, - }; - - /// [text] with `***` for: - /// - every email address, - /// - every whole text of [mail]: names, subject, body, header names and - /// values, metadata keys and values, file names. Case does not count. - /// - /// Only the first [maxLength] letters are kept, and the text then says - /// it was cut. A text of [mail] that starts near the cut and goes past - /// it is hidden up to the cut. - /// - /// A text must be a whole word in [text], so the name `Ann` does not - /// break `Annual`. Texts shorter than 3 letters and tags stay (tags - /// are labels like `smtp`, so they would hide useful words). A part of - /// a text, an encoded form (JSON, MIME, URL) or another Unicode form - /// (NFC or NFD) is not found. - /// - /// It reads [text] once for the texts and once for the addresses, so - /// a long error with many recipients stays fast. No RegExp is made - /// from mail data (a RegExp of a long body is too large for Dart). - static String redact(String text, Map mail) { - int fullLength = text.length; - bool cut = fullLength > maxLength; - if (cut) text = text.substring(0, maxLength); - String lower = text.toLowerCase(); - // ponytail: rare letters change length in lower case. Then the - // places do not match, so work on the lower case text. - String source = lower.length == text.length ? text : lower; - List secrets = _secrets(mail); - String tail = ''; - if (cut) { - tail = ' ... (cut, $fullLength letters in all)'; - int? at = _cutStart(lower, secrets); - if (at != null) { - source = source.substring(0, at); - lower = lower.substring(0, at); - tail = '***$tail'; - } - } - Map byStart = _byStart(secrets, lower.length); - String result = byStart.isEmpty - ? source - : _hideTexts(source, lower, byStart); - if (result.contains('@')) result = _hideAddresses(result); - return '$result$tail'; - } - - /// the trimmed, lower case texts of [mail], 3 letters or more - static List _secrets(Map mail) { - List texts = []; - _collect(mail, texts); - return [ - for (String value in texts) - if (value.trim().length >= 3) value.trim().toLowerCase(), - ]; - } - - /// the [secrets] that fit in [max] letters, by their first 3 letters - static Map _byStart(List secrets, int max) { - Map byStart = {}; - for (String secret in secrets) { - if (secret.length > max) continue; - byStart.putIfAbsent(_key(secret, 0), _Texts.new).add(secret); - } - return byStart; - } - - /// the first place where a whole-word text of [secrets] starts at the - /// end of the cut [lower] text and goes past the cut, or null - static int? _cutStart(String lower, List secrets) { - int? first; - for (String needle in secrets) { - int from = lower.length - needle.length + 1; - if (from < 0) from = 0; - // find the start with a short head of the text, then check the rest - String head = needle.length > 16 ? needle.substring(0, 16) : needle; - int last = lower.length - head.length; - int at = lower.indexOf(head, from); - while (at >= 0 && at <= last && !_startsAt(lower, needle, at)) { - at = lower.indexOf(head, at + 1); - } - if (at < 0 || at > last) { - at = -1; - for (int i = last + 1 > from ? last + 1 : from; i < lower.length; i++) { - if (_startsAt(lower, needle, i)) { - at = i; - break; - } - } - } - if (at >= 0 && (first == null || at < first)) first = at; - } - return first; - } - - /// true when the end of [lower] from [at] is the start of [needle], at - /// the start of a word - static bool _startsAt(String lower, String needle, int at) { - if (_isWord(needle, 0) && _isWord(lower, at - 1)) return false; - return needle.startsWith(lower.substring(at)); - } - - /// the first 3 code units of [text] at [at], as one number - static int _key(String text, int at) { - return (text.codeUnitAt(at) << 32) | - (text.codeUnitAt(at + 1) << 16) | - text.codeUnitAt(at + 2); - } - - /// one pass over [lower]: at each place, the longest whole-word text - /// of [byStart] that starts there is replaced by `***` - static String _hideTexts( - String source, - String lower, - Map byStart, - ) { - _Hash hash = _Hash(lower); - StringBuffer out = StringBuffer(); - int start = 0; - int at = 0; - while (at + 3 <= lower.length) { - _Texts? texts = byStart[_key(lower, at)]; - int? length = texts?.longestAt(lower, hash, at); - if (length == null) { - at++; - continue; - } - out - ..write(source.substring(start, at)) - ..write('***'); - at += length; - start = at; - } - out.write(source.substring(start)); - return out.toString(); - } - - /// [text] with `***` for every email address, of any length. Each `@` - /// grows left and right over address letters, so it is one pass. - static String _hideAddresses(String text) { - StringBuffer out = StringBuffer(); - int start = 0; - int at = text.indexOf('@'); - while (at >= 0) { - int left = at; - while (left > start && !_notAddress.contains(text.codeUnitAt(left - 1))) { - left--; - } - int right = at + 1; - while (right < text.length && - !_notAddress.contains(text.codeUnitAt(right))) { - right++; - } - if (left < at && right > at + 1) { - out - ..write(text.substring(start, left)) - ..write('***'); - start = right; - } - at = text.indexOf('@', at + 1); - } - out.write(text.substring(start)); - return out.toString(); - } - - /// true when [text] has a letter, digit or `_` at [index]. Every - /// letter that is not ASCII counts as a letter. - static bool _isWord(String text, int index) { - if (index < 0 || index >= text.length) return false; - int c = text.codeUnitAt(index); - return (c >= 0x30 && c <= 0x39) || - (c >= 0x41 && c <= 0x5a) || - (c >= 0x61 && c <= 0x7a) || - c == 0x5f || - c >= 0x80; - } - - /// every text of [mail] (from `Message.toJson()`), with the names of - /// the headers and the keys of the metadata, but not the tags and the - /// file bytes and content types - static void _collect(Map mail, List into) { - for (MapEntry entry in mail.entries) { - Object? value = entry.value; - switch (entry.key) { - case 'tags': - break; - case 'attachments': - for (Object? file in value is List ? value : const []) { - if (file is! Map) continue; - for (MapEntry field in file.entries) { - if (field.key == 'bytes' || field.key == 'contentType') continue; - _add(field.value, into); - } - } - case 'headers' || 'metadata': - if (value is Map) _add(value.keys.toList(), into); - _add(value, into); - default: - _add(value, into); - } - } - } - - /// every text in [value], with the values of a map, not its keys - static void _add(Object? value, List into) { - if (value is String) { - into.add(value); - } else if (value is Map) { - for (Object? item in value.values) { - _add(item, into); - } - } else if (value is List) { - for (Object? item in value) { - _add(item, into); - } - } + /// the encrypted text of the first error, or null when it could not + /// be encrypted + final String? details; + + /// The text of the first error, from a saved or logged [error] text + /// (eg. the `error` of a failed job). Needs the same app key, or one + /// of `AppConfig.previousKeys`. Null when it has no details. Throws a + /// `DecryptException` when the key is wrong. + /// ``` + /// for (FailedJob job in await Queue.driver.failed()) { + /// print(RedactedMailError.decrypt(job.error)); + /// } + /// ``` + static String? decrypt(String error) { + int at = error.indexOf(_marker); + if (at < 0) return null; + String rest = error.substring(at + _marker.length); + String details = rest.split(RegExp(r'\s')).first; + return Queue.readData(details, _purpose); } @override String toString() { String code = statusCode == null ? '' : ' (code $statusCode)'; - return 'RedactedMailError$code: $message'; - } -} - -/// The texts of a mail that start with the same 3 letters, by length. -class _Texts { - final Map> _byLength = >{}; - final Map> _hashes = >{}; - List _lengths = []; - - void add(String text) { - Set? same = _byLength[text.length]; - if (same == null) { - same = {}; - _byLength[text.length] = same; - _hashes[text.length] = {}; - _lengths = [..._lengths, text.length]..sort(_down); - } - same.add(text); - _hashes[text.length]!.add(_Hash(text).of(0, text.length)); - } - - static int _down(int a, int b) => b - a; - - /// the length of the longest text that is a whole word in [lower] at - /// [at], or null. The [hash] of [lower] skips most places without - /// copying the text, so many texts of the same start stay fast. - int? longestAt(String lower, _Hash hash, int at) { - for (int length in _lengths) { - int end = at + length; - if (end > lower.length) continue; - if (!_hashes[length]!.contains(hash.of(at, end))) continue; - String part = lower.substring(at, end); - if (!_byLength[length]!.contains(part)) continue; - if (RedactedMailError._isWord(part, 0) && - RedactedMailError._isWord(lower, at - 1)) { - continue; - } - if (RedactedMailError._isWord(part, length - 1) && - RedactedMailError._isWord(lower, end)) { - continue; - } - return length; - } - return null; - } -} - -/// A rolling hash of a text: the hash of any part in constant time. -class _Hash { - _Hash(String text) - : _prefix = List.filled(text.length + 1, 0), - _power = List.filled(text.length + 1, 1) { - for (int i = 0; i < text.length; i++) { - _prefix[i + 1] = (_prefix[i] * _base + text.codeUnitAt(i)) % _mod; - _power[i + 1] = _power[i] * _base % _mod; - } - } - - static const int _mod = 2147483647; - static const int _base = 131; - - final List _prefix; - final List _power; - - /// the hash of the part from [start] to [end] - int of(int start, int end) { - int value = (_prefix[end] - _prefix[start] * _power[end - start]) % _mod; - return value < 0 ? value + _mod : value; + String text = details == null + ? 'details could not be encrypted' + : '$_marker$details'; + return 'RedactedMailError: $type$code. $text'; } } diff --git a/test/queue_mail_test.dart b/test/queue_mail_test.dart index cd89d63..9879a73 100644 --- a/test/queue_mail_test.dart +++ b/test/queue_mail_test.dart @@ -193,9 +193,13 @@ void main() { // still enough to find the problem printOnFailure(log); expect(log, contains('failed try 1 of 2')); - expect(log, contains('550 5.1.1 <***>: mailbox unavailable')); - expect(error, contains('RedactedMailError (code 550)')); - expect(error, contains('MailException: 550 5.1.1 <***>')); + expect(log, contains('RedactedMailError: MailException (code 550).')); + expect(error, contains('RedactedMailError: MailException (code 550).')); + // the app key gives back the full error + expect( + RedactedMailError.decrypt(error), + startsWith('MailException: 550 5.1.1 '), + ); }); test('encrypt: true: a 100 KB body is not in the error', () async { @@ -207,7 +211,7 @@ void main() { for (String text in [log, error]) { expect(text, isNot(contains('918273'))); expect(text, isNot(contains('xxxx'))); - expect(text, contains('(code 552): MailException: 552')); + expect(text, contains('MailException (code 552). Details are')); } }); diff --git a/test/redacted_mail_error_test.dart b/test/redacted_mail_error_test.dart index 9db4546..b9fec71 100644 --- a/test/redacted_mail_error_test.dart +++ b/test/redacted_mail_error_test.dart @@ -1,11 +1,11 @@ import 'dart:io'; -import 'dart:typed_data'; +import 'package:dox_core/dox_core.dart' show AppConfig, DecryptException, Dox; import 'package:dox_mailer/dox_mailer.dart'; import 'package:dox_queue/dox_queue.dart'; import 'package:test/test.dart'; -/// an error whose text can not be made, so the redaction fails +/// an error whose text can not be made, so it can not be encrypted class BadTextError extends MailException { BadTextError() : super('ann@example.com', statusCode: 421); @@ -13,224 +13,111 @@ class BadTextError extends MailException { String toString() => throw StateError('no text'); } -Map _mail({ - String body = 'Hi Ann', - List? tags, - Map? headers, - Map? metadata, - List? attachments, -}) { - return Message( - from: const Address('app@example.com', 'App'), - to:
[const Address('ann@example.com', 'Ann')], - subject: 'Your reset code', - text: body, - tags: tags ?? [], - headers: headers ?? {}, - metadata: metadata ?? {}, - attachments: attachments ?? [], - ).toJson(); -} - -/// a body of about [size] letters with a secret code in it -String _bigBody(int size) { - return 'Hi Ann, your reset code is 918273. ${'x' * size}'; -} +/// the texts that must never be in the plain text of the error +const List _secrets = [ + 'ann@example.com', + 'ANN@EXAMPLE.COM', + 'Ann Lee', + 'Zoë', + '918273', + 'xxxx', + 'LLLL', + 'other.org', +]; + +/// Many kinds of error text that leaked with text redaction: names in +/// curly quotes, texts that overlap, a 1 MB body, a long local part. +List _errors = [ + '550 5.1.1 : mailbox unavailable', + 'rejected for “Ann Lee” and ‘Zoë’', + 'Ann Leeann@example.com overlaps Ann Lee', + '552 refused: Hi Ann, your code is 918273. ${'x' * 1024 * 1024}', + '550 <${'L' * 200}@other.org>: no mailbox', +]; + +/// the plain text part: the error without its encrypted details +String _plain(String error) => error.split('Details are encrypted: ').first; void main() { - for (int size in [100 * 1024, 1024 * 1024]) { - test('a ${size ~/ 1024} KB body is hidden, type and code stay', () { - String body = _bigBody(size); - RedactedMailError error = RedactedMailError.of( - MailException('552 message refused: $body', statusCode: 552), - _mail(body: body), - ); - String text = '$error'; - expect(text, isNot(contains('918273'))); - expect(text, isNot(contains('xxxx'))); + setUpAll(() { + Dox().initialize(AppConfig(appKey: 'k' * 32, totalIsolate: 1)); + }); + + for (String text in _errors) { + test('no mail data in: ${text.substring(0, 30)}', () { + String error = + '${RedactedMailError.of(MailException(text, statusCode: 550))}'; + for (String secret in _secrets) { + expect(_plain(error), isNot(contains(secret))); + expect(error, isNot(contains(secret))); + } expect( - text, - 'RedactedMailError (code 552): ' - 'MailException: 552 message refused: *** ' - '... (cut, ${'MailException: 552 message refused: $body'.length} ' - 'letters in all)', + error, + startsWith('RedactedMailError: MailException (code 550). '), ); - expect(error.type, 'MailException'); }); } - test('a big body that is not in the error does not break it', () { - RedactedMailError error = RedactedMailError.of( - MailException('550 : no mailbox', statusCode: 550), - _mail(body: _bigBody(1024 * 1024)), - ); - expect( - '$error', - 'RedactedMailError (code 550): ' - 'MailException: 550 <***>: no mailbox', - ); - }); - - test('a 1 MB error is cut and redacted fast', () { - List files = [ - for (int i = 0; i < 2000; i++) - Attachment(Uint8List(0), 'file-$i.pdf', contentType: 'text/plain'), - ]; - String body = '${'a' * (200 * 1024)}b'; - Stopwatch watch = Stopwatch()..start(); - RedactedMailError error = RedactedMailError.of( - MailException('${'a' * (1024 * 1024)} ann@example.com'), - _mail(body: body, attachments: files), - ); - watch.stop(); - expect(watch.elapsedMilliseconds, lessThan(500)); - expect(error.message.length, lessThan(RedactedMailError.maxLength + 100)); - expect(error.message, endsWith('letters in all)')); + test('decrypt gives back the full error text', () { + String text = '550 5.1.1 : mailbox unavailable'; + String error = + '${RedactedMailError.of(MailException(text, statusCode: 550))}'; + // like a failed job: the error, then the stack trace + String saved = '$error\n#0 main (file:///app.dart:1:1)'; + expect(RedactedMailError.decrypt(saved), 'MailException: $text'); }); - test('a mail text that goes past the cut is hidden up to the cut', () { - String body = 'Hi Ann, your reset code is 918273. ${'y' * 20000}'; - RedactedMailError error = RedactedMailError.of( - MailException('refused: ${'z ' * 8000}$body'), - _mail(body: body), + test('only the first 64 KB of a huge error is encrypted', () { + String text = 'x' * (5 * 1024 * 1024); + RedactedMailError error = RedactedMailError.of(MailException(text)); + expect('$error'.length, lessThan(100 * 1024)); + expect( + RedactedMailError.decrypt('$error'), + hasLength(RedactedMailError.maxDetails), ); - expect(error.message, isNot(contains('918273'))); - expect(error.message, isNot(contains('reset'))); - expect(error.message, contains('z z *** ... (cut,')); }); - test('header names, metadata keys and their values are hidden', () { - RedactedMailError error = RedactedMailError.of( - MailException( - 'bad X-Customer-Ref: 7781-secret, order-9912 = vip, ' - 'tags = gold-tier, bytes = blue-sky, contentType = top-plan', - ), - _mail( - headers: {'X-Customer-Ref': '7781-secret'}, - metadata: { - 'order-9912': 'vip', - 'tags': 'gold-tier', - 'bytes': 'blue-sky', - 'contentType': 'top-plan', - }, - ), - ); + test('a wrong key can not read the details', () { + String error = '${RedactedMailError.of(MailException('ann@example.com'))}'; + Dox().initialize(AppConfig(appKey: 'z' * 32, totalIsolate: 1)); + addTearDown(_resetKey); expect( - error.message, - 'MailException: bad ***: ***, *** = ***, ' - '*** = ***, *** = ***, *** = ***', + () => RedactedMailError.decrypt(error), + throwsA(isA()), ); }); - test('a file name is hidden, its content type is not', () { - RedactedMailError error = RedactedMailError.of( - MailException('bad file ann-invoice.pdf (application/pdf)'), - _mail( - attachments: [Attachment(Uint8List(0), 'ann-invoice.pdf')], - ), - ); - expect(error.message, 'MailException: bad file *** (application/pdf)'); + test('an error with no details gives null', () { + expect(RedactedMailError.decrypt('StateError: boom'), isNull); }); - test('when the redaction fails, only the type and code are left', () { - RedactedMailError error = RedactedMailError.of(BadTextError(), _mail()); + test('when the text can not be made, only type and code stay', () { + RedactedMailError error = RedactedMailError.of(BadTextError()); expect(error.type, 'BadTextError'); expect(error.statusCode, 421); + expect(error.details, isNull); expect( '$error', - 'RedactedMailError (code 421): ' - 'BadTextError: (could not redact)', + 'RedactedMailError: BadTextError (code 421). ' + 'details could not be encrypted', ); }); - test('a tag or a name does not hide other words', () { - RedactedMailError error = RedactedMailError.of( - MailException('SMTP: Annual limit, Ann can not get more'), - _mail(tags: ['smtp']), - ); - expect( - '$error', - 'RedactedMailError: ' - 'MailException: SMTP: Annual limit, *** can not get more', - ); - }); - - test('a SocketException keeps its host and port, no code', () { + test('a SocketException keeps its type, with no code', () { RedactedMailError error = RedactedMailError.of( SocketException( 'Connection refused', address: InternetAddress('10.0.0.5'), port: 587, ), - _mail(), ); expect(error.type, 'SocketException'); expect(error.statusCode, isNull); - expect('$error', contains('Connection refused')); - expect('$error', contains('10.0.0.5')); - expect('$error', contains('587')); - }); - - test('a long address that is not in the mail is hidden whole', () { - String local = 'L' * 200; - RedactedMailError error = RedactedMailError.of( - MailException('550 <$local@other.org>: no mailbox', statusCode: 550), - _mail(), - ); - expect( - '$error', - 'RedactedMailError (code 550): ' - 'MailException: 550 <***>: no mailbox', - ); - }); - - test('a 5 MB error with no spaces is fast', () { - String half = 'x' * (5 * 512 * 1024); - Stopwatch watch = Stopwatch()..start(); - RedactedMailError plain = RedactedMailError.of( - MailException('$half$half'), - _mail(), - ); - RedactedMailError address = RedactedMailError.of( - MailException('$half@$half'), - _mail(), - ); - watch.stop(); - print('5 MB errors with no spaces: ${watch.elapsedMilliseconds} ms'); - expect( - plain.message, - endsWith('(cut, ${5 * 1024 * 1024 + 15} letters in all)'), - ); - expect(plain.message.length, lessThan(RedactedMailError.maxLength + 100)); - expect(address.message, isNot(contains('@'))); - // generous, so CI is not flaky. Locally it is far less. - expect(watch.elapsed, lessThan(const Duration(seconds: 5))); + expect('$error', startsWith('RedactedMailError: SocketException. ')); + expect(RedactedMailError.decrypt('$error'), contains('10.0.0.5')); }); +} - test('5000 recipients and a 1 MB error are fast', () { - List
to =
[ - for (int i = 0; i < 5000; i++) Address('user$i@example.com', 'User $i'), - ]; - Map mail = Message( - from: const Address('app@example.com'), - to: to, - subject: 'Hi', - text: 'Hello', - ).toJson(); - StringBuffer text = StringBuffer('550 refused: '); - while (text.length < 1024 * 1024) { - text.write('User 42 said no to user42@example.com. '); - } - Stopwatch watch = Stopwatch()..start(); - RedactedMailError error = RedactedMailError.of( - MailException('$text', statusCode: 550), - mail, - ); - watch.stop(); - expect(error.message, isNot(contains('User 42'))); - expect(error.message, isNot(contains('example.com'))); - expect(error.message, contains('550 refused: *** said no to ***.')); - print('5000 recipients, 1 MB error: ${watch.elapsedMilliseconds} ms'); - expect(watch.elapsed, lessThan(const Duration(seconds: 5))); - }); +void _resetKey() { + Dox().initialize(AppConfig(appKey: 'k' * 32, totalIsolate: 1)); } From 413f040155e7d431e5761bd72aefedeeb9bd9887 Mon Sep 17 00:00:00 2001 From: necessarylion Date: Mon, 2 Nov 2026 07:28:59 +0700 Subject: [PATCH 6/6] fix: review nits for RedactedMailError Read the type and code of the error inside the try, so an error that throws there can not leak its text. decrypt() now only takes real encrypted details. It no longer gives back plain text that starts with {. The cut to 64K characters no longer splits an emoji in two. The answer of the server (MailException.body) is now in the encrypted details too. The docs now say 64K characters (UTF-16 code units), not 64 KB. --- CHANGELOG.md | 2 +- README.md | 2 +- lib/src/send_queued_mail.dart | 45 +++++++++++++++++----- test/redacted_mail_error_test.dart | 62 +++++++++++++++++++++++++++++- 4 files changed, 98 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a8268d..6594af4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ ## Unreleased -- Fix: with `Mail.queue(message, encrypt: true)`, a send error could name the recipient (eg. an SMTP reply `550 : mailbox unavailable`). The worker wrote that error to its log and to the failed jobs in plain text. Now the encrypted mail job throws a `RedactedMailError`. Its text has only the error type and the SMTP or HTTP code. The full error text (at most 64 KB) is encrypted with the app key. Read it with `RedactedMailError.decrypt(job.error)`. When the error can not be encrypted, only the type and the code are kept. Mail jobs without `encrypt` keep the full error, as before. +- Fix: with `Mail.queue(message, encrypt: true)`, a send error could name the recipient (eg. an SMTP reply `550 : mailbox unavailable`). The worker wrote that error to its log and to the failed jobs in plain text. Now the encrypted mail job throws a `RedactedMailError`. Its text has only the error type and the SMTP or HTTP code. The full error text, with the answer of the server, is encrypted with the app key (at most the first 64K characters, counted in UTF-16 code units). Read it with `RedactedMailError.decrypt(job.error)`. When the error can not be encrypted, only the type and the code are kept. Mail jobs without `encrypt` keep the full error, as before. - New: a built-in job for `Mail.queue()` and `Mail.later()` of dox_mailer. `Queue.configure()` turns it on. Apps do not add it to `lib/generated/jobs.g.dart`: every `JobRegistry` finds the built-in jobs (`JobRegistry.builtIn`) after its own jobs. With `encrypt: true`, the mail is saved like a `ShouldBeEncrypted` job. - Now depends on dox_mailer. - Deploy the queue worker together with the web app. An old worker does not know the new mail job, so it fails the queued emails. diff --git a/README.md b/README.md index 2ef645d..71feab2 100644 --- a/README.md +++ b/README.md @@ -278,7 +278,7 @@ dox_queue has a built-in job for it. It is not in `lib/generated/jobs.g.dart`, a Deploy the worker together with the web app. An old worker does not know this job, so it fails the queued emails. With `encrypt: true`, a send error is logged and saved as its type and code only, eg. `RedactedMailError: MailException (code 550). Details are encrypted: ...`. -The full error text (at most 64 KB) is encrypted with the app key. To read it, pass the saved error to `RedactedMailError.decrypt(job.error)`. +The full error text, with the answer of the server, is encrypted with the app key (at most the first 64K characters, counted in UTF-16 code units). To read it, take a `FailedJob` from `Queue.driver.failed()` and call `RedactedMailError.decrypt(job.error)`. ## Run the worker diff --git a/lib/src/send_queued_mail.dart b/lib/src/send_queued_mail.dart index cb23bb3..2a855ee 100644 --- a/lib/src/send_queued_mail.dart +++ b/lib/src/send_queued_mail.dart @@ -1,3 +1,4 @@ +import 'package:dox_core/dox_core.dart' show AESEncryptor, Dox; import 'package:dox_mailer/dox_mailer.dart' show Mail, MailException, MailQueue, Message; @@ -92,9 +93,11 @@ class SendEncryptedQueuedMail extends SendQueuedMail with ShouldBeEncrypted { /// text. /// /// Its text has only the type of the first error and the SMTP or HTTP -/// code. The full text of the first error is encrypted with the app key, -/// like the job data. At most the first 64 KB of it is encrypted, so a -/// huge error does not fill the log. Read it with [decrypt]. +/// code. The full text of the first error (and the answer of the server, +/// `MailException.body`) is encrypted with the app key, like the job +/// data. At most the first 64K characters (UTF-16 code units) are +/// encrypted, so a huge error does not fill the log. Read it with +/// [decrypt]. /// ``` /// RedactedMailError: MailException (code 550). Details are encrypted: /// ``` @@ -102,25 +105,40 @@ class RedactedMailError implements Exception { RedactedMailError(this.type, {this.statusCode, this.details}); /// [error] with its text encrypted. When that fails, only the type - /// is kept, never the text. + /// (and the code) is kept, never the text. Everything is read from + /// [error] inside the `try`, so an error that throws can not leak. factory RedactedMailError.of(Object error) { - String type = '${error.runtimeType}'; + String type = 'unknown'; int? code; String? details; try { + type = '${error.runtimeType}'; code = error is MailException ? error.statusCode : null; String text = '$error'; - if (text.length > maxDetails) text = text.substring(0, maxDetails); - details = Queue.encryptData(text, _purpose); + if (error is MailException && (error.body ?? '').isNotEmpty) { + text = '$text\nbody: ${error.body}'; + } + details = Queue.encryptData(_cut(text), _purpose); } catch (_) { details = null; } return RedactedMailError(type, statusCode: code, details: details); } - /// the most letters of the first error that are encrypted + /// the most characters (UTF-16 code units) of the error that are + /// encrypted static const int maxDetails = 64 * 1024; + /// the first [maxDetails] characters of [text]. It does not cut an + /// emoji (a surrogate pair) in two. + static String _cut(String text) { + if (text.length <= maxDetails) return text; + int end = maxDetails; + int last = text.codeUnitAt(end - 1); + if (last >= 0xD800 && last <= 0xDBFF) end--; + return text.substring(0, end); + } + /// bound to the encrypted details, so job data can not be read as them static const String _purpose = 'dox_queue:RedactedMailError'; @@ -139,7 +157,8 @@ class RedactedMailError implements Exception { /// The text of the first error, from a saved or logged [error] text /// (eg. the `error` of a failed job). Needs the same app key, or one /// of `AppConfig.previousKeys`. Null when it has no details. Throws a - /// `DecryptException` when the key is wrong. + /// `DecryptException` when the key is wrong, or the details are not + /// real encrypted details. /// ``` /// for (FailedJob job in await Queue.driver.failed()) { /// print(RedactedMailError.decrypt(job.error)); @@ -150,7 +169,13 @@ class RedactedMailError implements Exception { if (at < 0) return null; String rest = error.substring(at + _marker.length); String details = rest.split(RegExp(r'\s')).first; - return Queue.readData(details, _purpose); + return AESEncryptor.decode( + details, + Dox().config.appKey, + associatedData: _purpose, + purpose: Queue.encryptPurpose, + previousSecrets: Dox().config.previousKeys, + ); } @override diff --git a/test/redacted_mail_error_test.dart b/test/redacted_mail_error_test.dart index b9fec71..425b528 100644 --- a/test/redacted_mail_error_test.dart +++ b/test/redacted_mail_error_test.dart @@ -13,6 +13,25 @@ class BadTextError extends MailException { String toString() => throw StateError('no text'); } +/// an error with a given text +class TextError implements Exception { + TextError(this.text); + + final String text; + + @override + String toString() => text; +} + +/// an error whose type can not be read +class BadTypeError implements Exception { + @override + Type get runtimeType => throw StateError('ann@example.com'); + + @override + String toString() => 'ann@example.com'; +} + /// the texts that must never be in the plain text of the error const List _secrets = [ 'ann@example.com', @@ -67,7 +86,7 @@ void main() { expect(RedactedMailError.decrypt(saved), 'MailException: $text'); }); - test('only the first 64 KB of a huge error is encrypted', () { + test('only the first 64K characters of a huge error are encrypted', () { String text = 'x' * (5 * 1024 * 1024); RedactedMailError error = RedactedMailError.of(MailException(text)); expect('$error'.length, lessThan(100 * 1024)); @@ -116,6 +135,47 @@ void main() { expect('$error', startsWith('RedactedMailError: SocketException. ')); expect(RedactedMailError.decrypt('$error'), contains('10.0.0.5')); }); + + test('the cut does not split an emoji', () { + int max = RedactedMailError.maxDetails; + String text = '${'x' * (max - 1)}😀 and more'; + String? back = RedactedMailError.decrypt( + '${RedactedMailError.of(TextError(text))}', + ); + expect(back, 'x' * (max - 1)); + }); + + test('the answer of the server is in the encrypted details', () { + RedactedMailError error = RedactedMailError.of( + MailException( + 'the API said no', + statusCode: 422, + body: '{"to":"ann@example.com"}', + ), + ); + expect('$error', isNot(contains('ann@example.com'))); + expect( + RedactedMailError.decrypt('$error'), + 'MailException: the API said no\nbody: {"to":"ann@example.com"}', + ); + }); + + test('an error whose type throws gives only "unknown"', () { + RedactedMailError error = RedactedMailError.of(BadTypeError()); + expect( + '$error', + 'RedactedMailError: unknown. details could not be encrypted', + ); + }); + + test('decrypt only takes real encrypted details', () { + expect( + () => RedactedMailError.decrypt( + 'RedactedMailError: X. Details are encrypted: {"to":"ann@example.com"}', + ), + throwsA(isA()), + ); + }); } void _resetKey() {