From 6c927dee34d3f913b20ab7592e87b93c480bcf8f Mon Sep 17 00:00:00 2001 From: Petra-1012 Date: Wed, 7 Oct 2026 15:12:41 -0400 Subject: [PATCH 1/7] feat(mcpl): stamp each inbound item's delivered-body and stored-copy digests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The ingestion stamp the delivery-receipt lane and the source-header work share (room-220 #46752–#47210; isolated as its own foundation at Tessa-974's request, #47586, so receipts need not wait on outgoing routing). mcpl/inbound-source.ts gains the one implementation: canonicalJson (keys sorted at every level, undefined dropped) and sourceBodyDigest(blocks) = SHA-256 hex of canonicalJson([blocks]) — the one-element framing an undecorated, unsharded stored copy has always hashed to, so stamped and older copies of the same body share a version (Hugo-1004 #46835). Both lanes stamp, outside the frozen admission envelope (they describe the delivery — a materialization, a correction — not the admission): - metadata.sourceBodyDigest: the body as delivered, after MCPL conversion and before any host decoration (on the push lane, the closed-channel invitation) or storage sharding; - metadata.storedBodyDigest: the same function over exactly the blocks handed to addMessage, decorations included (Oscar-1005 #47210). A copy that no longer hashes to it was changed after delivery — CM's editMessage keeps metadata — so a consumer can tell an intact copy from an edited one. A silent heartbeat stores nothing and gets neither field. Neither proves later presence or completeness. Tests (inbound-source.test.ts): a channels/incoming item's two digests agree and equal the legacy unsharded hash, sit outside the envelope, and an edit through the context manager leaves the stored digest unmatched; a DM push into a closed channel stores the invitation, so its stored digest covers it while the delivered digest is the adapter body alone. Co-Authored-By: Claude Opus 5.5 --- changelog.d/ingestion-body-digests.added.md | 13 +++++++ src/framework.ts | 14 +++++++- src/mcpl/inbound-source.ts | 39 +++++++++++++++++++++ test/inbound-source.test.ts | 38 ++++++++++++++++++++ 4 files changed, 103 insertions(+), 1 deletion(-) create mode 100644 changelog.d/ingestion-body-digests.added.md diff --git a/changelog.d/ingestion-body-digests.added.md b/changelog.d/ingestion-body-digests.added.md new file mode 100644 index 000000000..287bc8f7e --- /dev/null +++ b/changelog.d/ingestion-body-digests.added.md @@ -0,0 +1,13 @@ +- **Delivered-body digests on every inbound item.** Both MCPL lanes stamp two + fields at ingestion, outside the frozen source envelope: + - `metadata.sourceBodyDigest` is the version identity of the body as + delivered, taken after MCPL conversion and before any host decoration or + storage sharding. It is SHA-256 of `canonicalJson([blocks])`, the framing + an undecorated, unsharded stored copy already hashes to. + - `metadata.storedBodyDigest` uses the same function over exactly the + blocks handed to storage, decorations included. + + A copy that no longer hashes to its `storedBodyDigest` was changed after it + arrived (`editMessage` keeps metadata). `canonicalJson` and + `sourceBodyDigest` are exported from `mcpl/inbound-source.ts` as the one + implementation consumers share. diff --git a/src/framework.ts b/src/framework.ts index 9263fb408..395c224d8 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -84,7 +84,7 @@ import { InferenceRouter } from './mcpl/inference-router.js'; import { ChannelRegistry, type ChannelToolOrigin, type PublishDestination, type PublishOutcome } from './mcpl/channel-registry.js'; import { ProseDraftStore, draftState, uncertainAttempt, type Draft, type DraftReason, type DraftSource, type DraftState, type InheritedRisk } from './prose-drafts.js'; import { ConversationRouter, CONVERSATION_ROUTING_DEPRECATION_NOTICE } from './mcpl/conversation-router.js'; -import { INBOUND_SOURCE_KEY, type InboundAcceptanceObserver, type InboundChannelSource, type InboundLane, type InboundSource, type InboundUnscopedSource } from './mcpl/inbound-source.js'; +import { INBOUND_SOURCE_KEY, sourceBodyDigest, type InboundAcceptanceObserver, type InboundChannelSource, type InboundLane, type InboundSource, type InboundUnscopedSource } from './mcpl/inbound-source.js'; import { safeSlice } from './safe-slice.js'; import type { WorkspaceModule } from './modules/workspace/index.js'; import { @@ -7214,6 +7214,12 @@ export class AgentFramework { this.noteInboundAccepted(source); } metadata[INBOUND_SOURCE_KEY] = source; + // The delivered body's version identity, and the stored copy's own + // digest (mcpl/inbound-source.ts sourceBodyDigest; room-220 + // #46752–#47210). This lane stores the body as delivered, so the two + // agree until a decoration or a later edit sets them apart. + metadata.sourceBodyDigest = sourceBodyDigest(event.content); + metadata.storedBodyDigest = sourceBodyDigest(event.content); // Per-channel conversation routing: messages go to the channel's fork // agent (spawned from the template on first qualifying message), never @@ -8381,6 +8387,9 @@ export class AgentFramework { metadata[INBOUND_SOURCE_KEY] = source; const content = [...event.content]; + // The delivered body's version identity, before any host decoration + // (the closed-channel invitation below). + if (content.length > 0) metadata.sourceBodyDigest = sourceBodyDigest(content); if (triggerChannel) { const origin = (event.origin ?? {}) as Record; const invitation = this.buildClosedChannelInvitation({ @@ -8411,6 +8420,9 @@ export class AgentFramework { event.origin?.source === 'heartbeat' && event.origin?.silent === true && content.length === 0; + // The stored copy's own digest, over exactly what is stored (decorations + // included, before storage shards it). A silent heartbeat stores nothing. + if (!silentHeartbeat) metadata.storedBodyDigest = sourceBodyDigest(content); const placement: CoalescingPlacement = { agent: '' }; if (!silentHeartbeat) { diff --git a/src/mcpl/inbound-source.ts b/src/mcpl/inbound-source.ts index 29c2e65e0..0812c45f4 100644 --- a/src/mcpl/inbound-source.ts +++ b/src/mcpl/inbound-source.ts @@ -21,6 +21,8 @@ * key: `eventId` is present only when the producer supplied one. */ +import { createHash } from 'node:crypto'; + /** The MCPL admission lane that accepted an item. Its contract decides what * an `eventId` is worth: `push/event` deduplicates by eventId, and RFC-006 * coalesced admission (`coalesced: true`) guarantees stable retries and @@ -152,3 +154,40 @@ export function conversationKey(source: InboundSource): string | undefined { if (source.kind === 'surface') return `surface\u0000${source.surface}`; return undefined; } + +/** + * JSON with object keys sorted at every level and `undefined` values dropped, + * so a value hashes alike however its keys were ordered (content read back + * from the store has its keys in a different order than it was written). + */ +export function canonicalJson(value: unknown): string { + if (Array.isArray(value)) return `[${value.map(canonicalJson).join(',')}]`; + if (value && typeof value === 'object') { + const entries = Object.entries(value as Record) + .filter(([, v]) => v !== undefined) + .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)); + return `{${entries.map(([k, v]) => `${JSON.stringify(k)}:${canonicalJson(v)}`).join(',')}}`; + } + return JSON.stringify(value) ?? 'null'; +} + +/** + * The version identity of a delivered body (agreed with the receipts lane, + * room-220 #46752–#47210): SHA-256 hex of `canonicalJson([blocks])` over the + * ContentBlock[] the framework stores for the body — after MCPL conversion, + * before any host decoration is added and before storage shards it. The + * one-element array keeps the framing an undecorated, unsharded stored copy + * has always hashed to, so a stamped copy and an older copy of the same body + * share a version. + * + * Ingestion stamps it as `metadata.sourceBodyDigest`, and the same function + * over exactly the blocks handed to storage (decorations included) as + * `metadata.storedBodyDigest`, on both MCPL lanes. Both live outside the + * frozen admission envelope: they describe the body actually delivered (a + * materialization, a correction), not the admission. Neither proves later + * presence or completeness; a copy that no longer hashes to its + * storedBodyDigest was changed after delivery (editMessage keeps metadata). + */ +export function sourceBodyDigest(blocks: readonly unknown[]): string { + return createHash('sha256').update(canonicalJson([blocks])).digest('hex'); +} diff --git a/test/inbound-source.test.ts b/test/inbound-source.test.ts index 5516268dc..de9efada8 100644 --- a/test/inbound-source.test.ts +++ b/test/inbound-source.test.ts @@ -27,6 +27,8 @@ import { type ToolResult, } from '../src/index.js'; import { MockMembrane } from './helpers/mock-membrane.js'; +import { sourceBodyDigest, canonicalJson } from '../src/mcpl/inbound-source.js'; +import { createHash } from 'node:crypto'; import { fixture, eventually, TS } from './helpers/coalescing-fixture.js'; const FIXTURE = join(import.meta.dirname, 'fixtures/speech-route-mcpl-server.mjs'); @@ -160,6 +162,42 @@ describe('inbound source envelope', () => { assert.equal(observed.length, 1); }); + it('stamps the delivered body\'s digest and the stored copy\'s digest at ingestion', async () => { + command({ op: 'incoming', channelId: ROOM, messageId: 'm-d1', mode: 'ambient', text: 'digest me' }); + await waitFor(() => !!storedWith((m) => m.messageId === 'm-d1'), 'message stored'); + const message = storedWith((m) => m.messageId === 'm-d1')!; + const meta = message.metadata as Record; + // The unsharded framing receipts already hash: canonicalJson([blocks]). + const expected = createHash('sha256').update(canonicalJson([[{ type: 'text', text: 'digest me' }]])).digest('hex'); + assert.equal(meta.sourceBodyDigest, expected); + assert.equal(sourceBodyDigest([{ text: 'digest me', type: 'text' }]), expected, 'key order does not matter'); + // This lane stores the body as delivered: the stored copy hashes the same. + assert.equal(meta.storedBodyDigest, sourceBodyDigest(message.content)); + assert.equal(meta.storedBodyDigest, meta.sourceBodyDigest); + // Outside the frozen admission envelope. + assert.equal((readInboundSource(message.metadata) as unknown as Record).sourceBodyDigest, undefined); + + // An edit through the supported path keeps the metadata, so only the + // stored digest can reveal that the copy no longer holds the delivery. + const cm = framework.getAgent('scout')!.getContextManager(); + cm.editMessage(message.id, [{ type: 'text', text: 'edited later' }]); + const edited = cm.getAllMessages().find((m) => m.id === message.id)!; + assert.equal((edited.metadata as Record).storedBodyDigest, meta.storedBodyDigest); + assert.notEqual(sourceBodyDigest(edited.content), meta.storedBodyDigest); + }); + + it('on the push lane, the delivered digest excludes host decoration and the stored digest covers it', async () => { + command({ op: 'dm', eventId: 'ev-dig', authorId: '134', authorName: 'antra', rawChannelId: RAW_DM, text: 'psst' }); + await waitFor(() => !!storedWith((m) => m.eventId === 'ev-dig'), 'dm stored'); + const message = storedWith((m) => m.eventId === 'ev-dig')!; + const meta = message.metadata as Record; + assert.equal(meta.sourceBodyDigest, sourceBodyDigest([{ type: 'text', text: 'psst' }]), 'the adapter body alone'); + assert.equal(meta.storedBodyDigest, sourceBodyDigest(message.content), 'exactly what was stored'); + // The DM's channel is closed, so the host appends its invitation. + assert.equal(message.content.length, 2); + assert.notEqual(meta.storedBodyDigest, meta.sourceBodyDigest, 'the closed-channel invitation decorates the stored copy'); + }); + it('keeps a stored envelope when the channel is renamed later', async () => { command({ op: 'incoming', channelId: ROOM, messageId: 'm-2', mode: 'ambient', text: 'before rename' }); await waitFor(() => !!storedWith((m) => m.messageId === 'm-2'), 'first message stored'); From 711b400a9255d1edc90d23734807a9a7f2330bdf Mon Sep 17 00:00:00 2001 From: Petra-1012 Date: Wed, 7 Oct 2026 16:28:02 -0400 Subject: [PATCH 2/7] fix(mcpl): take the stored-copy digest at each storage site, over the blocks stored there MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tessa-974's recheck (room-220 #48282): on channels/incoming, storedBodyDigest was taken at ingestion, but the main storage path can still append the closed-channel invitation afterwards — when a message accepted while its channel was open is processed after the channel closed. The untouched stored copy (header + body + invitation) then failed its own witness, which the receipts lane would read as an edit. The stored digest is now taken where each path stores, over exactly the blocks it hands to storage, through one helper (withStoredDigest): - channels/incoming, main path: after the invitation is appended; - channels/incoming, an RFC-006 correction delivered to a conversation fork; - channels/incoming, conversation-router delivery; - push/event: at its addMessage (a silent heartbeat stores nothing). sourceBodyDigest stays at ingestion, from the undecorated delivered body. Deferred storage keeps the content and metadata it was given, so a deferred copy's witness still matches. Test (inbound-source.test.ts), reproducing Tessa's discriminator: an incoming message is accepted and held while its channel is open, the channel is closed, then the message is processed — it is stored with the invitation and its storedBodyDigest equals the hash of its stored blocks, while its sourceBodyDigest is the delivered body alone. Fails before this change. Co-Authored-By: Claude Opus 5.5 --- src/framework.ts | 37 +++++++++++++++++++++++++------------ test/inbound-source.test.ts | 25 +++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 12 deletions(-) diff --git a/src/framework.ts b/src/framework.ts index 395c224d8..20acf6a1d 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -7214,12 +7214,11 @@ export class AgentFramework { this.noteInboundAccepted(source); } metadata[INBOUND_SOURCE_KEY] = source; - // The delivered body's version identity, and the stored copy's own - // digest (mcpl/inbound-source.ts sourceBodyDigest; room-220 - // #46752–#47210). This lane stores the body as delivered, so the two - // agree until a decoration or a later edit sets them apart. + // The delivered body's version identity (mcpl/inbound-source.ts + // sourceBodyDigest; room-220 #46752–#47210), from the body as delivered, + // before any host decoration. The stored copy's own digest is taken at + // each storage site below, over exactly the blocks stored there. metadata.sourceBodyDigest = sourceBodyDigest(event.content); - metadata.storedBodyDigest = sourceBodyDigest(event.content); // Per-channel conversation routing: messages go to the channel's fork // agent (spawned from the template on first qualifying message), never @@ -7232,7 +7231,7 @@ export class AgentFramework { if (!target) return undefined; if (this.conversationRouter && this.conversationAgentHomes.has(target.name)) { metadata.triggered = event.triggerInference ?? false; - const id = target.getContextManager().addMessage('user', event.content, metadata); + const id = target.getContextManager().addMessage('user', event.content, AgentFramework.withStoredDigest(metadata, event.content)); // A correction may target an older engagement. Refresh only the // binding whose agent actually received the message. if (this.conversationRouter.getBinding(event.channelId)?.agentName === target.name) { @@ -7301,7 +7300,7 @@ export class AgentFramework { } const placement: CoalescingPlacement = { agent: '' }; - const id = this.addMessage('user', incomingContent, metadata, { placement, bypassDeferralFor: event.assemblingFor }); + const id = this.addMessage('user', incomingContent, AgentFramework.withStoredDigest(metadata, incomingContent), { placement, bypassDeferralFor: event.assemblingFor }); this.emitTrace({ type: 'message:added', messageId: id, source: 'mcpl:channel-incoming' }); if (event.triggerInference && !divert) { @@ -7416,7 +7415,7 @@ export class AgentFramework { const trigger = decision.trigger && event.triggerInference !== false; messageMetadata.triggered = trigger; - const id = agent.getContextManager().addMessage('user', event.content, messageMetadata); + const id = agent.getContextManager().addMessage('user', event.content, AgentFramework.withStoredDigest(messageMetadata, event.content)); router.touch(event.channelId); this.emitTrace({ type: 'message:added', messageId: id, source: 'mcpl:channel-incoming' }); @@ -7793,6 +7792,23 @@ export class AgentFramework { }; } + /** + * An inbound item's metadata with its stored copy's own digest + * (`storedBodyDigest`): the delivered-body digest's function over exactly + * the blocks handed to storage here — every decoration included, before + * storage shards them. Taken at each storage site, never earlier, because + * a path can still decorate the body after ingestion stamped it (the + * closed-channel invitation; room-220 #48282). An edit through + * editMessage keeps metadata but not this hash. + */ + private static withStoredDigest( + metadata: Record, + content: readonly ContentBlock[], + ): Record { + metadata.storedBodyDigest = sourceBodyDigest(content); + return metadata; + } + /** Source envelope of a channels/incoming event (lane facts as given). */ private channelEventSource( event: { @@ -8420,13 +8436,10 @@ export class AgentFramework { event.origin?.source === 'heartbeat' && event.origin?.silent === true && content.length === 0; - // The stored copy's own digest, over exactly what is stored (decorations - // included, before storage shards it). A silent heartbeat stores nothing. - if (!silentHeartbeat) metadata.storedBodyDigest = sourceBodyDigest(content); const placement: CoalescingPlacement = { agent: '' }; if (!silentHeartbeat) { - const id = this.addMessage('user', content, metadata, { placement, bypassDeferralFor: event.assemblingFor }); + const id = this.addMessage('user', content, AgentFramework.withStoredDigest(metadata, content), { placement, bypassDeferralFor: event.assemblingFor }); this.emitTrace({ type: 'message:added', messageId: id, source: 'mcpl:push-event' }); } else { console.error(`[heartbeat] ${event.serverId}: accepted silent scheduled wake ${event.eventId}`); diff --git a/test/inbound-source.test.ts b/test/inbound-source.test.ts index de9efada8..31e37ab95 100644 --- a/test/inbound-source.test.ts +++ b/test/inbound-source.test.ts @@ -186,6 +186,31 @@ describe('inbound source envelope', () => { assert.notEqual(sourceBodyDigest(edited.content), meta.storedBodyDigest); }); + it('a message accepted while its channel was open but stored after it closed: the stored digest covers the invitation', async () => { + // Tessa-974's discriminator (room-220 #48282): the invitation is + // appended after ingestion stamped the delivered digest, at storage + // time, when the channel has closed in between. + const registry = (framework as unknown as { + channelRegistry: { handleChannelToolCall(name: string, input: unknown, caller: unknown): Promise; isChannelOpen(id: string): boolean }; + }).channelRegistry; + const fw = framework as unknown as { pushEvent(e: { type: string }): void }; + const held: Array<{ type: string }> = []; + const push = fw.pushEvent.bind(framework); + fw.pushEvent = (e) => { if (e.type === 'mcpl:channel-incoming') held.push(e); else push(e); }; + command({ op: 'incoming', channelId: ROOM, messageId: 'm-closed', mode: 'addressed', text: 'hello while open' }); + await waitFor(() => held.length > 0, 'incoming accepted and held'); + await registry.handleChannelToolCall('channel_close', { channelId: ROOM }, { kind: 'agent', agentName: 'scout' }); + assert.equal(registry.isChannelOpen(ROOM), false); + fw.pushEvent = push; + for (const e of held) push(e); + await waitFor(() => !!storedWith((m) => m.messageId === 'm-closed'), 'message stored'); + const message = storedWith((m) => m.messageId === 'm-closed')!; + const meta = message.metadata as Record; + assert.equal(meta.channelInvitation, true, 'stored with the closed-channel invitation'); + assert.equal(meta.storedBodyDigest, sourceBodyDigest(message.content), 'the untouched copy matches its own witness'); + assert.equal(meta.sourceBodyDigest, sourceBodyDigest([{ type: 'text', text: 'hello while open' }]), 'the delivered body alone'); + }); + it('on the push lane, the delivered digest excludes host decoration and the stored digest covers it', async () => { command({ op: 'dm', eventId: 'ev-dig', authorId: '134', authorName: 'antra', rawChannelId: RAW_DM, text: 'psst' }); await waitFor(() => !!storedWith((m) => m.eventId === 'ev-dig'), 'dm stored'); From d36a5ad5d37b274055c1b1e527fde1df40334c1c Mon Sep 17 00:00:00 2001 From: Petra-1759 Date: Wed, 7 Oct 2026 18:54:27 -0400 Subject: [PATCH 3/7] fix(mcpl): every stored push gets the host's delivered-body digest; export sourceBodyDigest from the package root Greptile's review of #252 at 711b400 (review 5448991826): P1 (discussion r4212520929). handleMcplPushEvent stamped metadata.sourceBodyDigest only when the body had blocks, a guard meant for the silent heartbeat, which stores nothing. But PushHandler maps an ordinary empty payload to [], and handleMcplPushEvent stores it unless the heartbeat's exact marker holds. So an ordinary empty push, or an addressed empty push into a closed channel (stored as the host's invitation alone), was stored with a storedBodyDigest and no host-computed sourceBodyDigest; and since push metadata starts from the adapter's origin, an origin.sourceBodyDigest survived on the stored copy as if the host had computed it. The digest is now taken for every push, an empty body included, before any decoration, written over whatever the origin carried. A silent heartbeat still stores nothing; its metadata is never kept. channels/incoming already stamped unconditionally, and every storage site on both lanes already wrote storedBodyDigest over the adapter's value. P2 (discussion r4212520946). The helpers were reachable only from src/mcpl/inbound-source.ts, and the package's exports map exposes only the root, so a package consumer could not import them and would have to copy the hashing code. sourceBodyDigest is now re-exported from src/mcpl/index.ts and the package root. canonicalJson stays out of the package API: it is the digest's serializer, and keeping serialization and framing inside sourceBodyDigest saves consumers from duplicating either, which is the drift the finding is about (a consumer hashing canonicalJson(blocks) itself gets a different digest from sourceBodyDigest(blocks)). It keeps its module export for in-repo tests that pin the framing. Agreed with Petra-1012 (room-203 #49948) and June-1758 (room-262 #49931); wording per June-1758's review (#50126). The changelog fragment said both helpers were exported as the shared implementation and that both fields were stamped at ingestion. It now names the one public function and states the algorithm, says the stored digest is taken where each path stores (711b400), that an empty body has a digest, and that adapter-supplied values are replaced. Tests (inbound-source.test.ts, over the real MCPL child): - an ordinary empty unscoped push whose origin carries forged digests is stored with sourceBodyDigest = storedBodyDigest = SHA-256 of "[[]]"; - an empty DM into a closed channel is stored as the invitation alone, its sourceBodyDigest the empty body's and its storedBodyDigest the stored blocks'. Both fail before this change: sourceBodyDigest came back as the adapter's 'adapter-value'. The test file now imports sourceBodyDigest from the package root, so the public export is typechecked. The speech-route fixture's dm and push ops accept "content" (blocks, [] included) in place of "text", as its incoming op does on the receipts branch. Co-Authored-By: Claude Opus 5.5 --- changelog.d/ingestion-body-digests.added.md | 23 ++++++++----- src/framework.ts | 6 ++-- src/index.ts | 6 ++-- src/mcpl/inbound-source.ts | 7 ++++ src/mcpl/index.ts | 5 +-- test/fixtures/speech-route-mcpl-server.mjs | 10 +++--- test/inbound-source.test.ts | 37 ++++++++++++++++++++- 7 files changed, 74 insertions(+), 20 deletions(-) diff --git a/changelog.d/ingestion-body-digests.added.md b/changelog.d/ingestion-body-digests.added.md index 287bc8f7e..ea2ac3d73 100644 --- a/changelog.d/ingestion-body-digests.added.md +++ b/changelog.d/ingestion-body-digests.added.md @@ -1,13 +1,18 @@ -- **Delivered-body digests on every inbound item.** Both MCPL lanes stamp two - fields at ingestion, outside the frozen source envelope: +- **Delivered-body digests on every stored inbound item.** Both MCPL lanes + stamp two fields, outside the frozen source envelope: - `metadata.sourceBodyDigest` is the version identity of the body as - delivered, taken after MCPL conversion and before any host decoration or - storage sharding. It is SHA-256 of `canonicalJson([blocks])`, the framing - an undecorated, unsharded stored copy already hashes to. + delivered, taken at ingestion, after MCPL conversion and before any host + decoration or storage sharding. It is the SHA-256 (hex) of `[blocks]` + as canonical JSON (keys sorted at every level, `undefined` dropped), the + framing an undecorated, unsharded stored copy already hashes to. An + empty body has one too. - `metadata.storedBodyDigest` uses the same function over exactly the - blocks handed to storage, decorations included. + blocks handed to storage, decorations included, taken where each path + stores. + Both are always the host's own: values an adapter supplies are replaced. A copy that no longer hashes to its `storedBodyDigest` was changed after it - arrived (`editMessage` keeps metadata). `canonicalJson` and - `sourceBodyDigest` are exported from `mcpl/inbound-source.ts` as the one - implementation consumers share. + arrived (`editMessage` keeps metadata). `sourceBodyDigest(blocks)` is + exported from the package root as the one implementation consumers share; + it carries the framing, so check a copy with it rather than hashing your + own serialization. diff --git a/src/framework.ts b/src/framework.ts index 20acf6a1d..b2d8ec8f4 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -8404,8 +8404,10 @@ export class AgentFramework { const content = [...event.content]; // The delivered body's version identity, before any host decoration - // (the closed-channel invitation below). - if (content.length > 0) metadata.sourceBodyDigest = sourceBodyDigest(content); + // (the closed-channel invitation below). Every push gets one, an empty + // body included, written over any value the adapter's origin carried. + // A silent heartbeat stores nothing, so its metadata is never kept. + metadata.sourceBodyDigest = sourceBodyDigest(content); if (triggerChannel) { const origin = (event.origin ?? {}) as Record; const invitation = this.buildClosedChannelInvitation({ diff --git a/src/index.ts b/src/index.ts index 9c1cc4ac4..db533b6d5 100644 --- a/src/index.ts +++ b/src/index.ts @@ -83,8 +83,10 @@ export type { GateConfig, GateOptions, GatePolicy, GatePolicyMatch, GateBehavior // MCPL channel registry (exposed for modules that need channel-level operations) export { ChannelRegistry } from './mcpl/index.js'; -// Where each accepted inbound item came from (`metadata.inboundSource`) -export { INBOUND_SOURCE_KEY, readInboundSource, conversationKey } from './mcpl/index.js'; +// Where each accepted inbound item came from (`metadata.inboundSource`), and +// the digest its delivered and stored bodies are stamped with +// (`metadata.sourceBodyDigest`, `metadata.storedBodyDigest`) +export { INBOUND_SOURCE_KEY, readInboundSource, conversationKey, sourceBodyDigest } from './mcpl/index.js'; export type { InboundSource, InboundChannelSource, diff --git a/src/mcpl/inbound-source.ts b/src/mcpl/inbound-source.ts index 0812c45f4..4f51a1860 100644 --- a/src/mcpl/inbound-source.ts +++ b/src/mcpl/inbound-source.ts @@ -159,6 +159,10 @@ export function conversationKey(source: InboundSource): string | undefined { * JSON with object keys sorted at every level and `undefined` values dropped, * so a value hashes alike however its keys were ordered (content read back * from the store has its keys in a different order than it was written). + * + * sourceBodyDigest's serializer, kept out of the package API: serialization + * and framing both stay inside sourceBodyDigest, so consumers don't + * duplicate either (`canonicalJson(blocks)` alone hashes differently). */ export function canonicalJson(value: unknown): string { if (Array.isArray(value)) return `[${value.map(canonicalJson).join(',')}]`; @@ -187,6 +191,9 @@ export function canonicalJson(value: unknown): string { * materialization, a correction), not the admission. Neither proves later * presence or completeness; a copy that no longer hashes to its * storedBodyDigest was changed after delivery (editMessage keeps metadata). + * + * Exported from the package root, so a consumer checks a copy against either + * field with this same function rather than a copy of it. */ export function sourceBodyDigest(blocks: readonly unknown[]): string { return createHash('sha256').update(canonicalJson([blocks])).digest('hex'); diff --git a/src/mcpl/index.ts b/src/mcpl/index.ts index 1b933017a..e7a54fb9b 100644 --- a/src/mcpl/index.ts +++ b/src/mcpl/index.ts @@ -157,8 +157,9 @@ export { InferenceRouter } from './inference-router.js'; // Channel registry (channel lifecycle, incoming messages, synthesized tools) export { ChannelRegistry } from './channel-registry.js'; -// Inbound source envelope: where each accepted item came from, stamped once -export { INBOUND_SOURCE_KEY, readInboundSource, conversationKey } from './inbound-source.js'; +// Inbound source envelope: where each accepted item came from, stamped once; +// and the digest of a delivered or stored body +export { INBOUND_SOURCE_KEY, readInboundSource, conversationKey, sourceBodyDigest } from './inbound-source.js'; export type { InboundSource, InboundChannelSource, diff --git a/test/fixtures/speech-route-mcpl-server.mjs b/test/fixtures/speech-route-mcpl-server.mjs index 8cebbf70c..6d6a757d5 100644 --- a/test/fixtures/speech-route-mcpl-server.mjs +++ b/test/fixtures/speech-route-mcpl-server.mjs @@ -5,8 +5,10 @@ // BOTH MCPL lanes. The TEST appends one JSON object per line to COMMAND_PATH: // {"op":"incoming","channelId":…,"messageId":…,"mode":"ambient"|"addressed", // "text":…, "threadId"?, "metadata"?, "eventId"?, "coalesce"?} -// {"op":"dm","eventId":…,"authorId":…,"authorName"?,"rawChannelId":…,"text":…} -// {"op":"push","eventId":…,"origin"?,"tags"?,"text":…,"featureSet"?} +// {"op":"dm","eventId":…,"authorId":…,"authorName"?,"rawChannelId":…,"text":…, +// "content"? (content blocks, instead of text; [] included), "origin"?} +// {"op":"push","eventId":…,"origin"?,"tags"?,"text":…,"featureSet"?, +// "content"? (content blocks, instead of text; [] included)} // {"op":"rename","channelId":…,"label":…} (channels/changed) // {"op":"publish-mode","mode":"delivered"|"not-delivered"|"no-receipt"|"error"|"hang"} // {"op":"history","channelId":…,"history":[ChannelIncomingMessage…]} @@ -90,7 +92,7 @@ function pollCommands() { ...(c.origin ?? {}), }, tags: ['chat:dm', 'chat:addressed', 'chat:private'], - payload: { content: textBlock(c.text ?? '') }, + payload: { content: c.content ?? textBlock(c.text ?? '') }, }, }); } else if (c.op === 'push') { @@ -105,7 +107,7 @@ function pollCommands() { timestamp: c.timestamp ?? new Date().toISOString(), ...(c.origin ? { origin: c.origin } : {}), ...(c.tags ? { tags: c.tags } : {}), - payload: { content: textBlock(c.text ?? '') }, + payload: { content: c.content ?? textBlock(c.text ?? '') }, }, }); } else if (c.op === 'rename') { diff --git a/test/inbound-source.test.ts b/test/inbound-source.test.ts index 31e37ab95..5d3f3dafb 100644 --- a/test/inbound-source.test.ts +++ b/test/inbound-source.test.ts @@ -16,6 +16,8 @@ import { INBOUND_SOURCE_KEY, readInboundSource, conversationKey, + // The package's public digest: consumers import it from the root. + sourceBodyDigest, type InboundSource, type Module, type ModuleContext, @@ -27,7 +29,7 @@ import { type ToolResult, } from '../src/index.js'; import { MockMembrane } from './helpers/mock-membrane.js'; -import { sourceBodyDigest, canonicalJson } from '../src/mcpl/inbound-source.js'; +import { canonicalJson } from '../src/mcpl/inbound-source.js'; import { createHash } from 'node:crypto'; import { fixture, eventually, TS } from './helpers/coalescing-fixture.js'; @@ -223,6 +225,39 @@ describe('inbound source envelope', () => { assert.notEqual(meta.storedBodyDigest, meta.sourceBodyDigest, 'the closed-channel invitation decorates the stored copy'); }); + it('an ordinary empty push is stored with the host\'s digests of its empty body, never the adapter\'s', async () => { + // Only a silent heartbeat stores nothing (test/silent-heartbeat.test.ts); + // any other push is stored, an empty one included, so it is stamped like + // any other — over whatever its origin claimed. + command({ + op: 'push', eventId: 'empty-1', content: [], + origin: { source: 'timer', sourceBodyDigest: 'adapter-value', storedBodyDigest: 'adapter-value' }, + }); + await waitFor(() => !!storedWith((m) => m.eventId === 'empty-1'), 'empty push stored'); + const message = storedWith((m) => m.eventId === 'empty-1')!; + const meta = message.metadata as Record; + assert.deepEqual(message.content, []); + // The empty body in the one-element framing: SHA-256 of `[[]]`. + const empty = createHash('sha256').update('[[]]').digest('hex'); + assert.equal(meta.sourceBodyDigest, empty, 'the empty delivered body'); + assert.equal(meta.storedBodyDigest, empty, 'stored undecorated'); + assert.equal(meta.storedBodyDigest, sourceBodyDigest(message.content), 'the untouched copy matches its own witness'); + }); + + it('an empty DM pushed into a closed channel: the delivered digest is the empty body, the stored digest covers the invitation', async () => { + command({ + op: 'dm', eventId: 'ev-empty', authorId: '134', authorName: 'antra', rawChannelId: RAW_DM, content: [], + origin: { sourceBodyDigest: 'adapter-value', storedBodyDigest: 'adapter-value' }, + }); + await waitFor(() => !!storedWith((m) => m.eventId === 'ev-empty'), 'empty dm stored'); + const message = storedWith((m) => m.eventId === 'ev-empty')!; + const meta = message.metadata as Record; + assert.equal(meta.channelInvitation, true, 'stored with the closed-channel invitation'); + assert.equal(message.content.length, 1, 'the invitation alone'); + assert.equal(meta.sourceBodyDigest, sourceBodyDigest([]), 'the empty delivered body, not the adapter\'s value'); + assert.equal(meta.storedBodyDigest, sourceBodyDigest(message.content), 'exactly what was stored, not the adapter\'s value'); + }); + it('keeps a stored envelope when the channel is renamed later', async () => { command({ op: 'incoming', channelId: ROOM, messageId: 'm-2', mode: 'ambient', text: 'before rename' }); await waitFor(() => !!storedWith((m) => m.messageId === 'm-2'), 'first message stored'); From a2accc1d8de1686de26ef629eb518938edff5e0d Mon Sep 17 00:00:00 2001 From: Ogden-1862 Date: Thu, 8 Oct 2026 14:28:43 -0400 Subject: [PATCH 4/7] fix(mcpl): digest a body as the store keeps it, so an untouched stored copy matches its storedBodyDigest slimepriestess's review of #252 (pullrequestreview-5460863368) found an untouched image copy that permanently failed its storedBodyDigest: storage takes an image's media type from its bytes' signature and re-encodes its base64, so the digest of the blocks handed to storage didn't match the copy read back. Writing bodies through the store directly (context-manager 0.13.0 over chronicle 0.4.0, read back live and after reopening) showed the class is wider than images: - every inline media block (an image whose source isn't a URL; any document, audio or video) is kept as a blob and comes back as { type, source: { type: 'base64', data, mediaType } } alone: its base64 re-encoded from the decoded bytes (padding, whitespace, URL-safe alphabet, unused bits), an image's media type taken from its signature, and its other fields (sourceUrl, tokenEstimate, filename, duration) dropped; - the store writes strings as UTF-8: a lone surrogate comes back as U+FFFD, and an entry whose key has one is dropped. sourceBodyDigest now hashes each block as the store keeps it. storedBlock maps inline media to its stored form; its base64 is re-encoded only when it isn't already canonical, so a stored copy's data is never decoded in full, and the store's signature test is copied (context-manager doesn't export it). canonicalJson writes strings well-formed and drops entries whose keys aren't. The normalization is inside the shared digest, not only in how storedBodyDigest is taken, so every copy of a body hashes alike: as delivered, as handed to storage, as injected into a live turn (#253's injectedEvidence hashes those very blocks), and as read back. Had only the stored digest been taken over the stored form, the injected copy would have failed it instead of the read-back one. An undecorated body's two digests stay equal whatever its media, and a legacy copy's recovered digest matches the delivered one. Docs: canonicalJson says it expects JSON-shaped values (review item 3): any other object is written by its own enumerable entries, which the store's own write matches (a Date as {}, a typed array by its indices, as probed) but a deferred write recovered from its JSON file doesn't. sourceBodyDigest says to check copies read back with blobs resolved (item 2). tsconfig gains ES2024.String for isWellFormed/toWellFormed, which Node has from 20, the engines floor. Tests: the reviewer's reproduction through the real framework's push lane (a PNG labeled image/jpeg, base64 broken by a newline), and a store contract test that writes nineteen bodies through a real ContextManager and checks that each hashes as it was handed over, live and after reopening, and which ones the store hands back changed. Its rows of image bytes the store has no signature for (SVG, BMP, AVIF) are labeled image/png and expected back unchanged, so a context-manager that learns to sniff one fails the test rather than letting the copied sniff drift. All three new tests fail at 88a24e6 and pass here. Reviewed independently by Lottie-1863 (room-310 #57630), who found that the store blobs an image of any non-URL source, not only a base64 one, and asked for the unsniffed rows. --- changelog.d/ingestion-body-digests.added.md | 20 ++-- src/framework.ts | 4 +- src/mcpl/inbound-source.ts | 96 ++++++++++++++++-- test/inbound-source.test.ts | 102 ++++++++++++++++++++ tsconfig.json | 2 +- 5 files changed, 205 insertions(+), 19 deletions(-) diff --git a/changelog.d/ingestion-body-digests.added.md b/changelog.d/ingestion-body-digests.added.md index f97bc691a..6e181ed8c 100644 --- a/changelog.d/ingestion-body-digests.added.md +++ b/changelog.d/ingestion-body-digests.added.md @@ -3,15 +3,21 @@ - `metadata.sourceBodyDigest` is the version identity of the body as delivered, taken at ingestion, after MCPL conversion and before any host decoration or storage sharding. It is the SHA-256 (hex) of `[blocks]` - as canonical JSON (keys sorted at every level, `undefined` dropped), the - framing an undecorated, unsharded stored copy already hashes to. + as canonical JSON (keys sorted at every level, `undefined` dropped), with + the blocks as the store keeps them: inline media as the store hands it + back (base64 re-encoded from its bytes, an image's media type taken from + its bytes' signature, only `type` and `source` kept) and strings + well-formed (a lone surrogate as U+FFFD). So the body hashes alike as + delivered, as handed to storage and as read back, and an undecorated, + unsharded stored copy hashes to it. - `metadata.storedBodyDigest` uses the same function over exactly the blocks handed to storage, decorations included, taken where each path stores. Both are always the host's own: values an adapter supplies are replaced. - A copy that no longer hashes to its `storedBodyDigest` was changed after it - arrived (`editMessage` keeps metadata). `sourceBodyDigest(blocks)` is - exported from the package root as the one implementation consumers share; - it carries the framing, so check a copy with it rather than hashing your - own serialization. + A stored copy, read back with its blobs resolved, that no longer hashes to + its `storedBodyDigest` was changed after it arrived (`editMessage` keeps + metadata). `sourceBodyDigest(blocks)` is exported from the package root as + the one implementation consumers share; it carries the framing and the + store's normalization, so check a copy with it rather than hashing your own + serialization. diff --git a/src/framework.ts b/src/framework.ts index 81957b3bf..5e44cc63c 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -7837,7 +7837,9 @@ export class AgentFramework { * An inbound item's metadata with its stored copy's own digest * (`storedBodyDigest`): the delivered-body digest's function over exactly * the blocks handed to storage here — every decoration included, before - * storage shards them. Taken at each storage site, never earlier, because + * storage shards them. That function hashes blocks as the store keeps them + * (inline media re-encoded and relabeled from its bytes), so the copy read + * back hashes to this too. Taken at each storage site, never earlier, because * a path can still decorate the body after ingestion stamped it (the * closed-channel invitation; room-220 #48282). An edit through * editMessage keeps metadata but not this hash. diff --git a/src/mcpl/inbound-source.ts b/src/mcpl/inbound-source.ts index 4f51a1860..b7c39c107 100644 --- a/src/mcpl/inbound-source.ts +++ b/src/mcpl/inbound-source.ts @@ -156,9 +156,18 @@ export function conversationKey(source: InboundSource): string | undefined { } /** - * JSON with object keys sorted at every level and `undefined` values dropped, - * so a value hashes alike however its keys were ordered (content read back - * from the store has its keys in a different order than it was written). + * JSON with object keys sorted at every level, written as the store keeps a + * value: `undefined` values are dropped, a string is well-formed (the store + * writes UTF-8, so each lone surrogate comes back as U+FFFD), and an entry + * whose key isn't well-formed is dropped (the store doesn't keep it). So a + * value hashes alike however its keys were ordered (content read back from + * the store has them in a different order than it was written), and alike + * before and after the store. + * + * It expects JSON-shaped values, as the MCPL lanes deliver them (parsed from + * JSON). Any other object is written by its own enumerable entries, unlike + * JSON.stringify: a Date becomes `{}`, as the store's own write keeps it, but + * a deferred write recovered from its JSON file brings it back as a string. * * sourceBodyDigest's serializer, kept out of the package API: serialization * and framing both stay inside sourceBodyDigest, so consumers don't @@ -168,33 +177,100 @@ export function canonicalJson(value: unknown): string { if (Array.isArray(value)) return `[${value.map(canonicalJson).join(',')}]`; if (value && typeof value === 'object') { const entries = Object.entries(value as Record) - .filter(([, v]) => v !== undefined) + .filter(([k, v]) => v !== undefined && k.isWellFormed()) .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)); return `{${entries.map(([k, v]) => `${JSON.stringify(k)}:${canonicalJson(v)}`).join(',')}}`; } + if (typeof value === 'string') return JSON.stringify(value.toWellFormed()); return JSON.stringify(value) ?? 'null'; } +/** + * A block as the store hands it back. Context-manager keeps inline media as + * blobs (blob-manager.ts): an image whose source isn't a URL, and every + * document, audio or video block, comes back as `{ type, source: { type: + * 'base64', data, mediaType } }` and nothing else, its data re-encoded from + * the decoded bytes, and an image's media type taken from its bytes' + * signature where they have one. Any other block comes back as it was given. + */ +function storedBlock(block: unknown): unknown { + const media = block as { type?: unknown; source?: { type?: unknown; data?: unknown; mediaType?: unknown } } | null; + const blob = media?.type === 'image' + ? media.source?.type !== 'url' + : media?.type === 'document' || media?.type === 'audio' || media?.type === 'video'; + const source = media?.source; + if (!blob || typeof source?.data !== 'string') return block; + const data = canonicalBase64(source.data); + const mediaType = media!.type === 'image' + ? sniffRasterImageMediaType(Buffer.from(data.slice(0, 32), 'base64')) ?? source.mediaType + : source.mediaType; + return { type: media!.type, source: { type: 'base64', data, mediaType } }; +} + +/** + * Base64 as the store writes it back from the decoded bytes: padded, with no + * whitespace, the standard alphabet and zero unused bits. Data already in + * that form, as every stored copy's is, is returned without decoding it all. + */ +function canonicalBase64(data: string): string { + if (data.length % 4 === 0 && /^[A-Za-z0-9+/]*={0,2}$/.test(data)) { + const last = data.slice(-4); + if (Buffer.from(last, 'base64').toString('base64') === last) return data; + } + return Buffer.from(data, 'base64').toString('base64'); +} + +/** + * The store's own test for raster image types with an unambiguous byte + * signature, copied from context-manager's blob-manager.ts (it isn't + * exported): the store relabels an image whose bytes match one of these. + */ +function sniffRasterImageMediaType(bytes: Uint8Array): string | null { + if (bytes.length >= 8 && + bytes[0] === 0x89 && bytes[1] === 0x50 && bytes[2] === 0x4e && bytes[3] === 0x47 && + bytes[4] === 0x0d && bytes[5] === 0x0a && bytes[6] === 0x1a && bytes[7] === 0x0a) { + return 'image/png'; + } + if (bytes.length >= 3 && bytes[0] === 0xff && bytes[1] === 0xd8 && bytes[2] === 0xff) { + return 'image/jpeg'; + } + if (bytes.length >= 6) { + const signature = Buffer.from(bytes.subarray(0, 6)).toString('ascii'); + if (signature === 'GIF87a' || signature === 'GIF89a') return 'image/gif'; + } + if (bytes.length >= 12 && + Buffer.from(bytes.subarray(0, 4)).toString('ascii') === 'RIFF' && + Buffer.from(bytes.subarray(8, 12)).toString('ascii') === 'WEBP') { + return 'image/webp'; + } + return null; +} + /** * The version identity of a delivered body (agreed with the receipts lane, * room-220 #46752–#47210): SHA-256 hex of `canonicalJson([blocks])` over the * ContentBlock[] the framework stores for the body — after MCPL conversion, - * before any host decoration is added and before storage shards it. The - * one-element array keeps the framing an undecorated, unsharded stored copy - * has always hashed to, so a stamped copy and an older copy of the same body - * share a version. + * before any host decoration is added and before storage shards it — with + * each block as the store keeps it (storedBlock, canonicalJson). So a body + * hashes alike as delivered, as handed to storage, as injected into a live + * turn, and as read back from the store. The one-element array keeps the + * framing an undecorated, unsharded stored copy has always hashed to, so a + * stamped copy and an older copy of the same body share a version. * * Ingestion stamps it as `metadata.sourceBodyDigest`, and the same function * over exactly the blocks handed to storage (decorations included) as * `metadata.storedBodyDigest`, on both MCPL lanes. Both live outside the * frozen admission envelope: they describe the body actually delivered (a * materialization, a correction), not the admission. Neither proves later - * presence or completeness; a copy that no longer hashes to its + * presence or completeness; a stored copy that no longer hashes to its * storedBodyDigest was changed after delivery (editMessage keeps metadata). + * Check a copy read back with its blobs resolved (getAllMessages, getMessage, + * or getMessageWindow without `resolveBlobs: false`): a blob reference never + * matches. * * Exported from the package root, so a consumer checks a copy against either * field with this same function rather than a copy of it. */ export function sourceBodyDigest(blocks: readonly unknown[]): string { - return createHash('sha256').update(canonicalJson([blocks])).digest('hex'); + return createHash('sha256').update(canonicalJson([blocks.map(storedBlock)])).digest('hex'); } diff --git a/test/inbound-source.test.ts b/test/inbound-source.test.ts index 8a8b797a6..65232a7ca 100644 --- a/test/inbound-source.test.ts +++ b/test/inbound-source.test.ts @@ -18,6 +18,7 @@ import { conversationKey, // The package's public digest: consumers import it from the root. sourceBodyDigest, + PassthroughStrategy, type InboundSource, type Module, type ModuleContext, @@ -28,6 +29,7 @@ import { type ToolCall, type ToolResult, } from '../src/index.js'; +import { ContextManager } from '@animalabs/context-manager'; import { MockMembrane } from './helpers/mock-membrane.js'; import { canonicalJson } from '../src/mcpl/inbound-source.js'; import { createHash } from 'node:crypto'; @@ -36,6 +38,9 @@ import { fixture, eventually, TS } from './helpers/coalescing-fixture.js'; const FIXTURE = join(import.meta.dirname, 'fixtures/speech-route-mcpl-server.mjs'); const ROOM = 'discord:g1:room'; const RAW_DM = '1548000000000000001'; +/** Bytes that open with PNG's signature, which is all the store reads of an + * image. 14 bytes, so their base64 is padded and has a '/' in it. */ +const PNG = Buffer.from('89504e470d0a1a0afbefbeffffff', 'hex'); async function waitFor(cond: () => boolean, what: string, timeoutMs = 15_000): Promise { const deadline = Date.now() + timeoutMs; @@ -256,6 +261,29 @@ describe('inbound source envelope', () => { assert.notEqual(meta.storedBodyDigest, meta.sourceBodyDigest, 'the invitation decorates the stored copy'); }); + it('an image the store relabels or re-encodes still matches both its digests once read back', async () => { + // slimepriestess's review of #252: the store takes an image's media type + // from its bytes and re-encodes its base64, so a digest of the blocks as + // handed to storage failed against the untouched copy read back. + const png = PNG.toString('base64'); + const pushes: Record = { + 'img-labeled': { type: 'image', data: png, mimeType: 'image/png' }, + 'img-mislabeled': { type: 'image', data: png, mimeType: 'image/jpeg' }, + 'img-wrapped': { type: 'image', data: `${png.slice(0, 11)}\n${png.slice(11)}`, mimeType: 'image/png' }, + }; + for (const [eventId, block] of Object.entries(pushes)) command({ op: 'push', eventId, content: [block] }); + await waitFor(() => Object.keys(pushes).every((id) => !!storedWith((m) => m.eventId === id)), 'images stored'); + for (const eventId of Object.keys(pushes)) { + const message = storedWith((m) => m.eventId === eventId)!; + const meta = message.metadata as Record; + const image = message.content.find((b) => b.type === 'image'); + assert.deepEqual(image, { type: 'image', source: { type: 'base64', data: png, mediaType: 'image/png' } }, + `${eventId}: stored as the bytes' own type, in canonical base64`); + assert.equal(sourceBodyDigest(message.content), meta.storedBodyDigest, `${eventId}: the untouched copy matches its stored digest`); + assert.equal(meta.sourceBodyDigest, sourceBodyDigest([image]), `${eventId}: the delivered body hashes as the image stored for it`); + } + }); + it('keeps a stored envelope when the channel is renamed later', async () => { command({ op: 'incoming', channelId: ROOM, messageId: 'm-2', mode: 'ambient', text: 'before rename' }); await waitFor(() => !!storedWith((m) => m.messageId === 'm-2'), 'first message stored'); @@ -443,3 +471,77 @@ describe('inbound source envelope under RFC-006 coalescing', () => { assert.deepEqual(observed, [stored]); }); }); + +describe('sourceBodyDigest hashes a body as the store keeps it', () => { + // Every agent stores through context-manager over chronicle. Whatever the + // store changes on the way in (an image's label, base64's spelling, fields + // it doesn't keep, a lone surrogate), the body read back must hash as it + // did when it was handed over, live and after the store is reopened. + const png = PNG.toString('base64'); + const image = (data: string, mediaType = 'image/png', extra: Record = {}) => + ({ type: 'image', source: { type: 'base64', data, mediaType }, ...extra }); + const media = (type: string, data: string, mediaType: string, extra: Record = {}) => + ({ type, source: { type: 'base64', data, mediaType }, ...extra }); + /** Each body, and whether the store hands it back changed. */ + const bodies: Record = { + 'a labeled image': { blocks: [image(png)], changed: false }, + 'PNG bytes labeled image/jpeg': { blocks: [image(png, 'image/jpeg')], changed: true }, + 'JPEG bytes labeled image/png': { blocks: [image(Buffer.from('ffd8ffe000104a464946', 'hex').toString('base64'))], changed: true }, + 'base64 with a newline in it': { blocks: [image(`${png.slice(0, 11)}\n${png.slice(11)}`)], changed: true }, + 'base64 with whitespace around it': { blocks: [image(` ${png}\n`)], changed: true }, + 'base64 without its padding': { blocks: [image(png.replace(/=+$/, ''))], changed: true }, + 'base64 with non-zero unused bits': { blocks: [image(png.replace(/8=$/, '9='))], changed: true }, + 'base64 in the URL-safe alphabet': { blocks: [image(png.replace(/\//g, '_'))], changed: true }, + // Bytes the store has no signature for keep their label. Labeled + // image/png, so a context-manager that learns to sniff one fails here. + 'SVG bytes labeled image/png': { blocks: [image(Buffer.from('').toString('base64'))], changed: false }, + 'BMP bytes labeled image/png': { blocks: [image(Buffer.from('424d3a0000000000000036000000', 'hex').toString('base64'))], changed: false }, + 'AVIF bytes labeled image/png': { blocks: [image(Buffer.from('0000001c6674797061766966000000006d696631', 'hex').toString('base64'))], changed: false }, + 'an image whose source is neither base64 nor a URL': { blocks: [{ type: 'image', source: { type: 'file', data: png, mediaType: 'image/png' } }], changed: true }, + 'an image with fields the store drops': { blocks: [image(png, 'image/png', { sourceUrl: 'https://cdn.example/a.png', tokenEstimate: 85 })], changed: true }, + 'an image by URL, its fields kept': { blocks: [{ type: 'image', source: { type: 'url', url: 'https://cdn.example/a.png' }, sourceUrl: 'https://cdn.example/a.png' }], changed: false }, + 'audio, wrapped, with a duration': { blocks: [media('audio', 'SUQz\nAwAAAAAA', 'audio/mpeg', { duration: 3 })], changed: true }, + 'a document with a filename': { blocks: [media('document', 'JVBERi0xLjQ=', 'application/pdf', { filename: 'a.pdf' })], changed: true }, + 'text with lone surrogates': { blocks: [{ type: 'text', text: 'cut \ud83d here, \ude00 there, \ud83d\ud83d twice' }], changed: true }, + 'a key that is not well-formed': { blocks: [{ type: 'text', text: 'x', rawItem: { 'k\ud800': 1, kept: 2 } }], changed: true }, + 'a relabeled image with a text decoration': { blocks: [image(png, 'image/jpeg'), { type: 'text', text: '[invitation]' }], changed: true }, + }; + + it('every body hashes alike as handed to the store and as read back, live and after reopening', async () => { + const dir = mkdtempSync(join(tmpdir(), 'stored-digest-')); + const path = join(dir, 'store.chronicle'); + try { + const strategy = () => new PassthroughStrategy(); + const ids = new Map(); + let cm = await ContextManager.open({ path, strategy: strategy() }); + for (const [name, { blocks }] of Object.entries(bodies)) { + ids.set(name, cm.addMessage('user', blocks as never, { storedBodyDigest: sourceBodyDigest(blocks) })); + } + const check = (when: string) => { + for (const [name, { blocks, changed }] of Object.entries(bodies)) { + const back = cm.getMessage(ids.get(name)!)!; + if (changed) assert.notDeepEqual(back.content, blocks, `${when}, ${name}: the store hands it back changed`); + else assert.deepEqual(back.content, blocks, `${when}, ${name}: the store hands it back as it was`); + assert.equal(sourceBodyDigest(back.content), (back.metadata as Record).storedBodyDigest, `${when}, ${name}: it still hashes as handed over`); + } + }; + check('live'); + cm.close(); + cm = await ContextManager.open({ path, strategy: strategy() }); + check('reopened'); + cm.close(); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it('what the store keeps still changes the digest', () => { + const digest = (blocks: unknown[]) => sourceBodyDigest(blocks); + assert.notEqual(digest([image(png)]), digest([image(Buffer.from([...PNG, 0]).toString('base64'))]), 'other bytes'); + const svg = Buffer.from('').toString('base64'); + assert.notEqual(digest([image(svg, 'image/svg+xml')]), digest([image(svg, 'image/png')]), 'a label the store keeps'); + assert.notEqual(digest([media('audio', 'SUQzAwAAAAAA', 'audio/mpeg')]), digest([media('audio', 'SUQzAwAAAAAA', 'audio/wav')]), 'an audio label'); + assert.notEqual(digest([{ type: 'text', text: 'a' }]), digest([{ type: 'text', text: 'b' }]), 'other text'); + assert.equal(digest([image(png, 'image/jpeg')]), digest([image(png)]), 'a label the store replaces is not part of the body'); + }); +}); diff --git a/tsconfig.json b/tsconfig.json index c10092cff..beb230fc8 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -3,7 +3,7 @@ "target": "ES2022", "module": "NodeNext", "moduleResolution": "NodeNext", - "lib": ["ES2022"], + "lib": ["ES2022", "ES2024.String"], "outDir": "./dist", "rootDir": ".", "strict": true, From d981e494806b69ef1b2a4ed17b281d14c24edab2 Mon Sep 17 00:00:00 2001 From: Petra-1012 Date: Fri, 9 Oct 2026 13:07:10 -0400 Subject: [PATCH 5/7] test(mcpl): pin every branch of the copied raster sniffer with a mislabeled fixture slimepriestess approved #252 at a2accc1 with one non-blocking ask: the sniffer that sourceBodyDigest uses (sniffRasterImageMediaType, copied from context-manager's unexported blob-manager code) was pinned by real-store round-trip fixtures only for png and jpeg (and svg as unsniffed), so a context-manager change to its gif or webp branch would silently bring back the stored-digest mismatch for those formats. Three rows join the real-store table, each labeled as something else so the store relabels it on the way in: GIF87a bytes labeled image/png, GIF89a bytes labeled image/jpeg, and WebP (RIFF....WEBP) bytes labeled image/png. Each must come back changed and still hash as handed over, live and after the store is reopened, against the installed @animalabs/context-manager 0.13.0. Red checks, each a one-line edit to the copied sniffer, rebuilt and run: - the gif branch removed fails at "GIF87a bytes labeled image/png"; - only GIF87a kept fails at "GIF89a bytes labeled image/jpeg"; - the webp branch removed fails at "WebP bytes labeled image/png". Restored, inbound-source passes 22/22. The existing SVG, BMP and AVIF rows already pin the other direction (a store that starts sniffing a new type). Test-only; no changelog fragment. Co-Authored-By: Claude Opus 5.5 --- test/inbound-source.test.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/inbound-source.test.ts b/test/inbound-source.test.ts index 65232a7ca..b50b14a70 100644 --- a/test/inbound-source.test.ts +++ b/test/inbound-source.test.ts @@ -487,6 +487,11 @@ describe('sourceBodyDigest hashes a body as the store keeps it', () => { 'a labeled image': { blocks: [image(png)], changed: false }, 'PNG bytes labeled image/jpeg': { blocks: [image(png, 'image/jpeg')], changed: true }, 'JPEG bytes labeled image/png': { blocks: [image(Buffer.from('ffd8ffe000104a464946', 'hex').toString('base64'))], changed: true }, + // One fixture per branch of the copied sniffer, each labeled as something + // else, so a context-manager that stops sniffing one fails here. + 'GIF87a bytes labeled image/png': { blocks: [image(Buffer.from('GIF87a\x01\x00\x01\x00\x00\x00\x00', 'latin1').toString('base64'))], changed: true }, + 'GIF89a bytes labeled image/jpeg': { blocks: [image(Buffer.from('GIF89a\x01\x00\x01\x00\x00\x00\x00', 'latin1').toString('base64'), 'image/jpeg')], changed: true }, + 'WebP bytes labeled image/png': { blocks: [image(Buffer.from('RIFF\x1a\x00\x00\x00WEBPVP8L\x0d\x00\x00\x00', 'latin1').toString('base64'))], changed: true }, 'base64 with a newline in it': { blocks: [image(`${png.slice(0, 11)}\n${png.slice(11)}`)], changed: true }, 'base64 with whitespace around it': { blocks: [image(` ${png}\n`)], changed: true }, 'base64 without its padding': { blocks: [image(png.replace(/=+$/, ''))], changed: true }, From 97d860dea8efa30f5c73965d6ca3db2e4e811447 Mon Sep 17 00:00:00 2001 From: Petra-1012 Date: Sun, 11 Oct 2026 15:13:16 -0400 Subject: [PATCH 6/7] test(digests): every storage site and store normalization is pinned, and the texts say where a stored copy can be checked sw-af252 (the house's haiku review of #252), its fix round. Text: - I2, I5, I9, I11, I17, I18 (changelog, sourceBodyDigest's, storedBlock's, canonicalJson's and withStoredDigest's docs): - the digests are on every stored MCPL item; console and API input and the host's notices carry none; - canonicalJson mirrors the store's append: an undefined object entry is dropped, an undefined array element is null, a lone surrogate is U+FFFD, and an ill-formed key is dropped; editMessage writes JSON instead; - the copy check is of an unsharded copy: a body the store shards carries the whole body's digests on every shard (bodyGroupId and shardIndex mark them), and no shard hashes to them; - a blob_ref is digested as given, which its copy read back (resolved) doesn't match; the MCPL lanes never store one; - sourceBodyDigest no longer claims a live-turn injection this change neither performs nor tests. Tests (inbound-source): - I1: the conversation router's site and a fork correction's site (a coalesced replacement following its prior into an unbound fork), each with forged digests in the adapter's metadata, store the host's. - I10: a channels/incoming item's forged digests are replaced. - I2: with a strategy whose store shards a two-block push, both shards carry the whole body's digests and neither hashes to them; a one-block body is not sharded and matches its own. - I7: the materialized RFC-006 copy's digests are the host's, over the rendered body, not the deferred placeholder. - I13: a channel message deferred mid-turn and stored at the boundary still hashes to its stored digest. - I3, I4, I12 (the real-store table): a video with a duration; an image without a media type, and with mediaType undefined (both take their bytes' type); a text block with an undefined field; a URL image with a stray data string, kept as given. A document without a media type, and an image without one whose bytes have no signature, are refused at write. A blob_ref is resolved on read and doesn't match its digest. A URL image and a text block are digested as given (plain canonical hash). Red checks: dropping withStoredDigest at the router site, at the fork correction site or on the channels/incoming path fails its test; so do blobbing a URL image and keeping undefined object entries. Runs (on-vps, Node 24.21.0): typecheck clean; inbound-source 28/28. --- changelog.d/ingestion-body-digests.added.md | 34 ++-- src/framework.ts | 6 +- src/mcpl/inbound-source.ts | 41 +++-- test/inbound-source.test.ts | 165 ++++++++++++++++++++ 4 files changed, 214 insertions(+), 32 deletions(-) diff --git a/changelog.d/ingestion-body-digests.added.md b/changelog.d/ingestion-body-digests.added.md index 6e181ed8c..558bf4832 100644 --- a/changelog.d/ingestion-body-digests.added.md +++ b/changelog.d/ingestion-body-digests.added.md @@ -1,15 +1,18 @@ -- **Delivered-body digests on every stored inbound item.** Both MCPL lanes - stamp two fields, outside the frozen source envelope: +- **Delivered-body digests on every stored MCPL inbound item.** Both MCPL + lanes stamp two fields, outside the frozen source envelope (console and API + input, and the host's own notices, carry neither): - `metadata.sourceBodyDigest` is the version identity of the body as delivered, taken at ingestion, after MCPL conversion and before any host decoration or storage sharding. It is the SHA-256 (hex) of `[blocks]` - as canonical JSON (keys sorted at every level, `undefined` dropped), with - the blocks as the store keeps them: inline media as the store hands it - back (base64 re-encoded from its bytes, an image's media type taken from - its bytes' signature, only `type` and `source` kept) and strings - well-formed (a lone surrogate as U+FFFD). So the body hashes alike as - delivered, as handed to storage and as read back, and an undecorated, - unsharded stored copy hashes to it. + as canonical JSON (keys sorted at every level, `undefined` object entries + dropped and an `undefined` array element written as `null`, as JSON + writes them), with the blocks as the store keeps them: inline media as + the store hands it back (base64 re-encoded from its bytes, an image's + media type taken from its bytes' signature, only `type` and `source` + kept), strings well-formed (a lone surrogate as U+FFFD) and an entry + whose key isn't well-formed dropped, as the store's append drops it. So + the body hashes alike as delivered, as handed to storage and as read + back, and an undecorated, unsharded stored copy hashes to it. - `metadata.storedBodyDigest` uses the same function over exactly the blocks handed to storage, decorations included, taken where each path stores. @@ -17,7 +20,12 @@ Both are always the host's own: values an adapter supplies are replaced. A stored copy, read back with its blobs resolved, that no longer hashes to its `storedBodyDigest` was changed after it arrived (`editMessage` keeps - metadata). `sourceBodyDigest(blocks)` is exported from the package root as - the one implementation consumers share; it carries the framing and the - store's normalization, so check a copy with it rather than hashing your own - serialization. + metadata). That check is of an unsharded copy: a body the store shards + (context-manager's adaptive ingress chunking) carries the whole body's two + digests on every shard (`bodyGroupId` and `shardIndex` mark them), and no + shard hashes to them. A `blob_ref` block is digested as given, while the + store resolves it on read, so a copy holding one doesn't match; the MCPL + lanes never store one. `sourceBodyDigest(blocks)` is exported from the + package root as the one implementation consumers share; it carries the + framing and the store's normalization, so check a copy with it rather than + hashing your own serialization. diff --git a/src/framework.ts b/src/framework.ts index 5e44cc63c..63a72154f 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -7838,8 +7838,10 @@ export class AgentFramework { * (`storedBodyDigest`): the delivered-body digest's function over exactly * the blocks handed to storage here — every decoration included, before * storage shards them. That function hashes blocks as the store keeps them - * (inline media re-encoded and relabeled from its bytes), so the copy read - * back hashes to this too. Taken at each storage site, never earlier, because + * (inline media re-encoded and relabeled from its bytes), so an unsharded + * copy read back hashes to this too. A body the store shards (adaptive + * ingress chunking) carries this on every shard, with its metadata, and no + * shard hashes to it. Taken at each storage site, never earlier, because * a path can still decorate the body after ingestion stamped it (the * closed-channel invitation; room-220 #48282). An edit through * editMessage keeps metadata but not this hash. diff --git a/src/mcpl/inbound-source.ts b/src/mcpl/inbound-source.ts index b7c39c107..3d53dc307 100644 --- a/src/mcpl/inbound-source.ts +++ b/src/mcpl/inbound-source.ts @@ -156,13 +156,16 @@ export function conversationKey(source: InboundSource): string | undefined { } /** - * JSON with object keys sorted at every level, written as the store keeps a - * value: `undefined` values are dropped, a string is well-formed (the store - * writes UTF-8, so each lone surrogate comes back as U+FFFD), and an entry - * whose key isn't well-formed is dropped (the store doesn't keep it). So a - * value hashes alike however its keys were ordered (content read back from - * the store has them in a different order than it was written), and alike - * before and after the store. + * JSON with object keys sorted at every level, written as the store's append + * keeps a value (its native UTF-8 write, which the real-store digest table + * checks): an `undefined` object entry is dropped and an `undefined` array + * element is written as `null`, as JSON writes them; a string is + * well-formed (each lone surrogate comes back as U+FFFD); and an entry whose + * key isn't well-formed is dropped (the store doesn't keep it). So a value + * hashes alike however its keys were ordered (content read back from the + * store has them in a different order than it was written), and alike + * before and after the store. editMessage writes JSON instead, so an edited + * copy can keep what an append wouldn't; it has changed anyway. * * It expects JSON-shaped values, as the MCPL lanes deliver them (parsed from * JSON). Any other object is written by its own enumerable entries, unlike @@ -191,7 +194,10 @@ export function canonicalJson(value: unknown): string { * document, audio or video block, comes back as `{ type, source: { type: * 'base64', data, mediaType } }` and nothing else, its data re-encoded from * the decoded bytes, and an image's media type taken from its bytes' - * signature where they have one. Any other block comes back as it was given. + * signature where they have one. Any other block comes back as it was given, + * except a `blob_ref`, which the store resolves to its inline media on read: + * it is digested as given here, so a copy holding one doesn't match its + * digest (the MCPL lanes never store one). */ function storedBlock(block: unknown): unknown { const media = block as { type?: unknown; source?: { type?: unknown; data?: unknown; mediaType?: unknown } } | null; @@ -252,21 +258,22 @@ function sniffRasterImageMediaType(bytes: Uint8Array): string | null { * ContentBlock[] the framework stores for the body — after MCPL conversion, * before any host decoration is added and before storage shards it — with * each block as the store keeps it (storedBlock, canonicalJson). So a body - * hashes alike as delivered, as handed to storage, as injected into a live - * turn, and as read back from the store. The one-element array keeps the - * framing an undecorated, unsharded stored copy has always hashed to, so a - * stamped copy and an older copy of the same body share a version. + * hashes alike as delivered, as handed to storage, and as read back from + * the store. The one-element array keeps the framing an undecorated, + * unsharded stored copy has always hashed to, so a stamped copy and an + * older copy of the same body share a version. * * Ingestion stamps it as `metadata.sourceBodyDigest`, and the same function * over exactly the blocks handed to storage (decorations included) as * `metadata.storedBodyDigest`, on both MCPL lanes. Both live outside the * frozen admission envelope: they describe the body actually delivered (a * materialization, a correction), not the admission. Neither proves later - * presence or completeness; a stored copy that no longer hashes to its - * storedBodyDigest was changed after delivery (editMessage keeps metadata). - * Check a copy read back with its blobs resolved (getAllMessages, getMessage, - * or getMessageWindow without `resolveBlobs: false`): a blob reference never - * matches. + * presence or completeness; an unsharded stored copy that no longer hashes + * to its storedBodyDigest was changed after delivery (editMessage keeps + * metadata). A body the store shards carries the whole body's two digests + * on every shard, and no shard hashes to them. Check a copy read back with + * its blobs resolved (getAllMessages, getMessage, or getMessageWindow + * without `resolveBlobs: false`): a blob reference never matches. * * Exported from the package root, so a consumer checks a copy against either * field with this same function rather than a copy of it. diff --git a/test/inbound-source.test.ts b/test/inbound-source.test.ts index b50b14a70..d632dc1d8 100644 --- a/test/inbound-source.test.ts +++ b/test/inbound-source.test.ts @@ -30,6 +30,7 @@ import { type ToolResult, } from '../src/index.js'; import { ContextManager } from '@animalabs/context-manager'; +import type { ContentBlock } from '@animalabs/membrane'; import { MockMembrane } from './helpers/mock-membrane.js'; import { canonicalJson } from '../src/mcpl/inbound-source.js'; import { createHash } from 'node:crypto'; @@ -261,6 +262,18 @@ describe('inbound source envelope', () => { assert.notEqual(meta.storedBodyDigest, meta.sourceBodyDigest, 'the invitation decorates the stored copy'); }); + it('a channels/incoming item carries the host\'s digests, never the adapter\'s', async () => { + command({ + op: 'incoming', channelId: ROOM, messageId: 'm-forged', mode: 'ambient', text: 'forged digests', + metadata: { sourceBodyDigest: 'adapter-value', storedBodyDigest: 'adapter-value' }, + }); + await waitFor(() => !!storedWith((m) => m.messageId === 'm-forged'), 'message stored'); + const message = storedWith((m) => m.messageId === 'm-forged')!; + const meta = message.metadata as Record; + assert.equal(meta.sourceBodyDigest, sourceBodyDigest([{ type: 'text', text: 'forged digests' }]), 'the delivered body, not the adapter\'s value'); + assert.equal(meta.storedBodyDigest, sourceBodyDigest(message.content), 'exactly what was stored, not the adapter\'s value'); + }); + it('an image the store relabels or re-encodes still matches both its digests once read back', async () => { // slimepriestess's review of #252: the store takes an image's media type // from its bytes and re-encodes its base64, so a digest of the blocks as @@ -411,6 +424,61 @@ describe('readInboundSource', () => { }); }); +describe('stored digests when the store shards a body', () => { + /** A strategy whose store splits a body of more than one block into one + * shard per block, as adaptive ingress chunking does. */ + class ShardingStrategy extends PassthroughStrategy { + chunkIngressMessage(_participant: string, content: ContentBlock[]) { + if (content.length < 2) return null; + return { bodyGroupId: `group-${content.length}-${Date.now()}`, shards: content.map((block, shardIndex) => ({ content: [block], shardIndex })) }; + } + } + + it("every shard carries the whole body's digests, and no shard hashes to them; an unsharded copy matches its own", async () => { + const tempDir = mkdtempSync(join(tmpdir(), 'inbound-shards-')); + const commandPath = join(tempDir, 'commands.jsonl'); + writeFileSync(commandPath, ''); + const framework = await AgentFramework.create({ + storePath: join(tempDir, 'test.chronicle'), + membrane: new MockMembrane().asMembrane(), + agents: [{ name: 'scout', model: 'test-model', systemPrompt: 'You are scout.', strategy: new ShardingStrategy() }], + mcplServers: [{ + id: 'discord', + command: process.execPath, + args: [FIXTURE], + env: { STATUS_PATH: join(tempDir, 'status.jsonl'), COMMAND_PATH: commandPath }, + }], + modules: [], + }); + try { + await framework.start(); + const command = (c: Record): void => appendFileSync(commandPath, JSON.stringify(c) + '\n'); + const blocks = [{ type: 'text', text: 'first part' }, { type: 'text', text: 'second part' }]; + command({ op: 'push', eventId: 'sharded', content: blocks }); + command({ op: 'push', eventId: 'whole', text: 'one block' }); + const stored = () => framework.getAgent('scout')!.getContextManager().getAllMessages(); + await waitFor(() => stored().some((m) => (m.metadata as { eventId?: string } | undefined)?.eventId === 'whole'), 'both pushes stored'); + const shards = stored().filter((m) => (m.metadata as { eventId?: string } | undefined)?.eventId === 'sharded'); + assert.equal(shards.length, 2, 'stored as two shards'); + assert.ok(shards[0]!.bodyGroupId && shards.every((m) => m.bodyGroupId === shards[0]!.bodyGroupId)); + assert.deepEqual(shards.map((m) => m.shardIndex), [0, 1]); + const whole = sourceBodyDigest(blocks); + for (const shard of shards) { + const meta = shard.metadata as Record; + assert.equal(meta.sourceBodyDigest, whole, 'the whole delivered body'); + assert.equal(meta.storedBodyDigest, whole, 'the whole body handed to storage'); + assert.notEqual(sourceBodyDigest(shard.content), whole, 'a shard alone is not the body'); + } + const one = stored().find((m) => (m.metadata as { eventId?: string } | undefined)?.eventId === 'whole')!; + assert.equal(one.bodyGroupId, undefined, 'a one-block body is not sharded'); + assert.equal(sourceBodyDigest(one.content), (one.metadata as Record).storedBodyDigest, 'an unsharded copy matches'); + } finally { + await framework.stop(); + rmSync(tempDir, { recursive: true, force: true }); + } + }); +}); + describe('inbound source envelope under RFC-006 coalescing', () => { const install = (framework: AgentFramework): InboundSource[] => { const observed: InboundSource[] = []; @@ -452,6 +520,14 @@ describe('inbound source envelope under RFC-006 coalescing', () => { assert.equal(observed.length, 1, 'materialization is not a second acceptance'); const stored = sourceOf(f, 'document_diff'); assert.deepEqual(stored, { ...accepted, materialized: true }); + // The materialized copy is its own storage site: both digests are the + // host's, over the rendered body, not the deferred placeholder. + const copy = f.framework.getAgent('agent')!.getContextManager().getAllMessages() + .find((m) => JSON.stringify(m.content).includes('document_diff'))!; + const meta = copy.metadata as Record; + assert.equal(meta.storedBodyDigest, sourceBodyDigest(copy.content), 'exactly what was stored'); + assert.equal(meta.sourceBodyDigest, sourceBodyDigest(copy.content), 'the rendered body, undecorated'); + assert.notEqual(meta.sourceBodyDigest, sourceBodyDigest([{ type: 'text', text: 'fallback_unavailable' }]), 'not the placeholder'); }); it('a channel-scoped coalesced message belongs to its channel, frozen at acceptance', async (t) => { @@ -470,6 +546,46 @@ describe('inbound source envelope under RFC-006 coalescing', () => { assert.equal(stored.sourceTimestamp, TS); assert.deepEqual(observed, [stored]); }); + + it('a channel message deferred mid-turn and stored at the boundary still hashes to its stored digest', async (t) => { + const f = await fixture(); t.after(f.close); + await f.register('chat'); + const internals = f.framework as unknown as { activeTurnTokens: Map; flushDeferredWrites(label: string): Promise }; + internals.activeTurnTokens.set('agent', 100); + await f.send('channels/incoming', { messages: [f.channel('e1', 'deferred_words', { initial: true })] }); + assert.ok(!f.context().includes('deferred_words'), 'held until the boundary'); + internals.activeTurnTokens.delete('agent'); + await internals.flushDeferredWrites('test'); + const copy = f.framework.getAgent('agent')!.getContextManager().getAllMessages() + .find((m) => JSON.stringify(m.content).includes('deferred_words'))!; + const meta = copy.metadata as Record; + assert.equal(meta.storedBodyDigest, sourceBodyDigest(copy.content), 'the copy stored at the boundary matches'); + assert.equal(meta.sourceBodyDigest, sourceBodyDigest([{ type: 'text', text: 'deferred_words' }]), 'the delivered body, unchanged'); + }); + + it("the conversation router's and a fork correction's storage sites carry the host's digests, never the adapter's", async (t) => { + const f = await fixture({ framework: { conversations: { templateAgent: 'agent', bind: { channel: 'always' }, trigger: { channel: 'always' } } } as never }); t.after(f.close); + await f.register(); + const forged = { sourceBodyDigest: 'adapter-value', storedBodyDigest: 'adapter-value' }; + const router = (f.framework as unknown as { conversationRouter: { getBinding(id: string): { agentName: string } | undefined; unbind(id: string): void } }).conversationRouter; + const check = (text: string, fork: string) => { + const copy = f.framework.getAgent(fork)!.getContextManager().getAllMessages().find((m) => JSON.stringify(m.content).includes(text)); + assert.ok(copy, `${text} stored in ${fork}`); + const meta = copy.metadata as Record; + assert.equal(meta.sourceBodyDigest, sourceBodyDigest([{ type: 'text', text }]), `${text}: the delivered body, not the adapter's value`); + assert.equal(meta.storedBodyDigest, sourceBodyDigest(copy.content), `${text}: exactly what was stored, not the adapter's value`); + }; + // Routed into a fresh fork by the conversation router. + await f.send('channels/incoming', { messages: [{ ...f.channel('c1', 'routed_original', { initial: true }), metadata: forged }] }); + const fork = router.getBinding('chat')!.agentName; + check('routed_original', fork); + // A correction follows the prior delivery into that fork, though the + // channel is no longer bound to it. + router.unbind('chat'); + await f.send('channels/incoming', { messages: [{ ...f.channel('c2', 'fork_correction'), metadata: forged }] }); + assert.equal(router.getBinding('chat'), undefined, 'no fresh fork was spawned for the correction'); + check('fork_correction', fork); + }); }); describe('sourceBodyDigest hashes a body as the store keeps it', () => { @@ -510,6 +626,15 @@ describe('sourceBodyDigest hashes a body as the store keeps it', () => { 'text with lone surrogates': { blocks: [{ type: 'text', text: 'cut \ud83d here, \ude00 there, \ud83d\ud83d twice' }], changed: true }, 'a key that is not well-formed': { blocks: [{ type: 'text', text: 'x', rawItem: { 'k\ud800': 1, kept: 2 } }], changed: true }, 'a relabeled image with a text decoration': { blocks: [image(png, 'image/jpeg'), { type: 'text', text: '[invitation]' }], changed: true }, + 'a video with a duration': { blocks: [media('video', 'AAAAGGZ0eXBtcDQy', 'video/mp4', { duration: 2 })], changed: true }, + // No media type: an image takes its bytes' own (an image without a + // signature, or other media, is refused at write: see below). + 'an image without a media type': { blocks: [{ type: 'image', source: { type: 'base64', data: png } }], changed: true }, + 'an image whose media type is undefined': { blocks: [{ type: 'image', source: { type: 'base64', data: png, mediaType: undefined } }], changed: true }, + 'a text block with an undefined field': { blocks: [{ type: 'text', text: 'x', note: undefined }], changed: true }, + // Only a non-URL image is blobbed: a URL image keeps even a stray data + // string as given, and storedBlock must not re-encode it. + 'an image by URL with a stray data string': { blocks: [{ type: 'image', source: { type: 'url', url: 'https://cdn.example/a.png', data: 'not base64 at all' } }], changed: false }, }; it('every body hashes alike as handed to the store and as read back, live and after reopening', async () => { @@ -540,6 +665,46 @@ describe('sourceBodyDigest hashes a body as the store keeps it', () => { } }); + it('a block the store refuses is refused at write, and a blob_ref is digested as given, which its copy read back does not match', async () => { + const dir = mkdtempSync(join(tmpdir(), 'stored-digest-refused-')); + try { + const cm = await ContextManager.open({ path: join(dir, 'store.chronicle'), strategy: new PassthroughStrategy() }); + try { + const svg = Buffer.from('').toString('base64'); + for (const [what, block] of [ + ['a document without a media type', { type: 'document', source: { type: 'base64', data: 'JVBERi0xLjQ=' } }], + ['an image without a media type whose bytes have no signature', { type: 'image', source: { type: 'base64', data: svg } }], + ] as const) { + assert.throws(() => cm.addMessage('user', [block] as never), Error, `${what}: refused at write`); + } + // A blob_ref names media the store already holds; it is resolved to + // inline media on read, so the copy read back hashes differently. + const held = cm.addMessage('user', [image(png)] as never); + const ref = cm.getMessageWindow(0, cm.getMessageCount(), { resolveBlobs: false }).messages + .find((m) => m.id === held)!.content[0] as { type: string }; + assert.equal(ref.type, 'blob_ref'); + const given = [ref]; + const id = cm.addMessage('user', given as never, { storedBodyDigest: sourceBodyDigest(given) }); + const back = cm.getMessage(id)!; + assert.deepEqual(back.content, [image(png)], 'resolved on read'); + assert.notEqual(sourceBodyDigest(back.content), (back.metadata as Record).storedBodyDigest, 'the documented mismatch'); + } finally { + cm.close(); + } + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it('a block the store keeps as given is digested as given', () => { + const plain = (blocks: unknown[]) => createHash('sha256').update(canonicalJson([blocks])).digest('hex'); + // Only a non-URL image is blobbed, so a URL image's stray data string is + // neither decoded nor re-encoded. + const urlImage = { type: 'image', source: { type: 'url', url: 'https://cdn.example/a.png', data: 'not base64 at all' } }; + assert.equal(sourceBodyDigest([urlImage]), plain([urlImage])); + assert.equal(sourceBodyDigest([{ type: 'text', text: 'x' }]), plain([{ type: 'text', text: 'x' }])); + }); + it('what the store keeps still changes the digest', () => { const digest = (blocks: unknown[]) => sourceBodyDigest(blocks); assert.notEqual(digest([image(png)]), digest([image(Buffer.from([...PNG, 0]).toString('base64'))]), 'other bytes'); From 7ce228ed9f2b789adfb0e23de98587e7bbb3a80b Mon Sep 17 00:00:00 2001 From: Petra-1012 Date: Sun, 11 Oct 2026 15:32:12 -0400 Subject: [PATCH 7/7] test(digests): the store refuses an undefined array element, and canonicalJson writes one as null Oscar-1808's take-or-leave in his review of 97d860d (room-203 #72199): no test pinned the new text's claim that canonicalJson writes an undefined array element as null, so a canonicalJson that skipped the element passed. He suggested a real-store row, expecting the store's append to write the element as null. The row showed the store doesn't: context-manager 0.13.0's append throws ("undefined cannot be represented as a serde_json::Value"), so no stored copy holds one. So instead: - the refused-at-write test gains a body with ['a', undefined]; - the digested-as-given test asserts the element digests as null, as JSON writes it, which fails for a canonicalJson that skips it (red-checked); - canonicalJson's doc and the changelog say an undefined array element is written as null as JSON writes it, and that the store refuses a body holding one. The doc no longer implies the store writes null. Runs (on-vps, Node 24.21.0): typecheck clean; inbound-source 29/29. --- changelog.d/ingestion-body-digests.added.md | 5 +++-- src/mcpl/inbound-source.ts | 9 +++++---- test/inbound-source.test.ts | 8 ++++++++ 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/changelog.d/ingestion-body-digests.added.md b/changelog.d/ingestion-body-digests.added.md index 558bf4832..0242f0bf8 100644 --- a/changelog.d/ingestion-body-digests.added.md +++ b/changelog.d/ingestion-body-digests.added.md @@ -5,8 +5,9 @@ delivered, taken at ingestion, after MCPL conversion and before any host decoration or storage sharding. It is the SHA-256 (hex) of `[blocks]` as canonical JSON (keys sorted at every level, `undefined` object entries - dropped and an `undefined` array element written as `null`, as JSON - writes them), with the blocks as the store keeps them: inline media as + dropped, and an `undefined` array element written as `null` as JSON + writes it, though the store refuses a body holding one), with the blocks + as the store keeps them: inline media as the store hands it back (base64 re-encoded from its bytes, an image's media type taken from its bytes' signature, only `type` and `source` kept), strings well-formed (a lone surrogate as U+FFFD) and an entry diff --git a/src/mcpl/inbound-source.ts b/src/mcpl/inbound-source.ts index d11a8f8de..e2bc75645 100644 --- a/src/mcpl/inbound-source.ts +++ b/src/mcpl/inbound-source.ts @@ -160,10 +160,11 @@ export function conversationKey(source: InboundSource): string | undefined { /** * JSON with object keys sorted at every level, written as the store's append * keeps a value (its native UTF-8 write, which the real-store digest table - * checks): an `undefined` object entry is dropped and an `undefined` array - * element is written as `null`, as JSON writes them; a string is - * well-formed (each lone surrogate comes back as U+FFFD); and an entry whose - * key isn't well-formed is dropped (the store doesn't keep it). So a value + * checks): an `undefined` object entry is dropped; a string is well-formed + * (each lone surrogate comes back as U+FFFD); and an entry whose key isn't + * well-formed is dropped (the store doesn't keep it). An `undefined` array + * element is written as `null`, as JSON writes it: the store's append + * refuses a body holding one, so no stored copy does. So a value * hashes alike however its keys were ordered (content read back from the * store has them in a different order than it was written), and alike * before and after the store. editMessage writes JSON instead, so an edited diff --git a/test/inbound-source.test.ts b/test/inbound-source.test.ts index 12de580de..b68e0e495 100644 --- a/test/inbound-source.test.ts +++ b/test/inbound-source.test.ts @@ -712,6 +712,8 @@ describe('sourceBodyDigest hashes a body as the store keeps it', () => { for (const [what, block] of [ ['a document without a media type', { type: 'document', source: { type: 'base64', data: 'JVBERi0xLjQ=' } }], ['an image without a media type whose bytes have no signature', { type: 'image', source: { type: 'base64', data: svg } }], + // The store's append can't represent an undefined array element. + ['an array with an undefined element', { type: 'text', text: 'x', parts: ['a', undefined] }], ] as const) { assert.throws(() => cm.addMessage('user', [block] as never), Error, `${what}: refused at write`); } @@ -741,6 +743,12 @@ describe('sourceBodyDigest hashes a body as the store keeps it', () => { const urlImage = { type: 'image', source: { type: 'url', url: 'https://cdn.example/a.png', data: 'not base64 at all' } }; assert.equal(sourceBodyDigest([urlImage]), plain([urlImage])); assert.equal(sourceBodyDigest([{ type: 'text', text: 'x' }]), plain([{ type: 'text', text: 'x' }])); + // An undefined array element is written as null, as JSON writes it (the + // store refuses such a body, above, so no stored copy holds one). + assert.equal( + sourceBodyDigest([{ type: 'text', text: 'x', parts: ['a', undefined] }]), + sourceBodyDigest([{ type: 'text', text: 'x', parts: ['a', null] }]), + ); }); it('what the store keeps still changes the digest', () => {