diff --git a/CHANGELOG.md b/CHANGELOG.md index 03e957a..9b6e9c9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), ### Fixed +- A `Date` inside a `map` field now uses the tagged + `{ "__t": "D", "__v": }` wire form, the same as every other + position. It was copied raw before, so it round-tripped as an ISO string while + its checksum tagged it as a `Date`, marking the delta commit as corrupted on + reload. Commits written by earlier builds with such values stay corrupted and + must be re-created. - JSON decoders use a disposable lifetime. Parent disposal releases owned children and preserves children acquired by another owner. Disposed decoders reject reads until acquired again. `finalize()` remains a disposal alias. diff --git a/base/core-types/encoding/base-encoder.ts b/base/core-types/encoding/base-encoder.ts index 0325c55..a068bd6 100644 --- a/base/core-types/encoding/base-encoder.ts +++ b/base/core-types/encoding/base-encoder.ts @@ -8,11 +8,7 @@ import type { ReadonlyJSONValue, } from '../../interfaces.ts'; import { serializeDate } from '../../date.ts'; -import { - type Dictionary, - dictToPrimitive, - isDictionary, -} from '../../collections/dict.ts'; +import { type Dictionary, isDictionary } from '../../collections/dict.ts'; import { type CoreKey, type CoreOptions, @@ -322,9 +318,17 @@ export abstract class JSONBaseEncoder< convertDictionary( value: Dictionary, - _options?: OT, - ) { - return dictToPrimitive(value); + options?: OT, + ): JSONObject { + // WHY: every dictionary value must pass through the shared value encoder. + // Copying raw values (dictToPrimitive) skipped Date/Set/Encodable tagging, + // so a Date in a map field landed as an ISO string while its checksum + // tagged it as a Date, marking the commit corrupted on reload. + const result: JSONObject = {}; + for (const [k, v] of value) { + result[this.convertKey(k)] = this.convertValue(v, options); + } + return result; } convertGenerator(g: Generator, options?: OT) { diff --git a/base/core-types/encoding/json.ts b/base/core-types/encoding/json.ts index 127f4aa..c7a109f 100644 --- a/base/core-types/encoding/json.ts +++ b/base/core-types/encoding/json.ts @@ -225,12 +225,26 @@ export class JSONDecoder implements Decoder { return decoded; } // Non-Encodable: wrap the raw value so the child decodes lazily. - const child = new JSONDecoder(value as ReadonlyJSONObject | undefined); - child._acquire(); + const child = this.createChildDecoder( + value as ReadonlyJSONObject | undefined, + ); this._addChild(child); return child; } + /** + * Creates an acquired child decoder for nested data. Subclasses override the + * factory so a cyclical parent yields cyclical children; a plain + * `new JSONDecoder` would drop the `__r`/`__d` context. + */ + protected createChildDecoder( + data: ReadonlyJSONObject | undefined, + ): OwnedDecoder { + const child = new JSONDecoder(data); + child._acquire(); + return child; + } + decodeValue(value: ReadonlyJSONValue): DecodedValue { this._assertAlive(); if (value instanceof Array) { @@ -259,8 +273,9 @@ export class JSONDecoder implements Decoder { // child is tracked on _children so the parent's dispose releases it even // when the consumer never disposes the child directly (e.g. when it is // reached inside a decoded array via get()). - const nested = new JSONDecoder(value.__v as ReadonlyJSONObject); - nested._acquire(); + const nested = this.createChildDecoder( + value.__v as ReadonlyJSONObject, + ); this._addChild(nested); return nested; } @@ -380,6 +395,13 @@ export class JSONCyclicalDecoder extends JSONDecoder { return super.decodeObject(jsonObj); } + protected override createChildDecoder( + data: ReadonlyJSONObject | undefined, + ): OwnedDecoder { + // Pooled factory already acquires and resets `_tempRefs`. + return JSONCyclicalDecoder.get(data); + } + override getDecoder( key: string, offset?: number, diff --git a/cfds/richtext/legacy-ref.ts b/cfds/richtext/legacy-ref.ts index aa4aba5..6517444 100644 --- a/cfds/richtext/legacy-ref.ts +++ b/cfds/richtext/legacy-ref.ts @@ -39,7 +39,8 @@ function assertNodeIsNotLegacyRef(node: unknown): void { if (!isLegacyRefNode(node)) return; throw new Error( 'Legacy rich text ref is not supported (RefMarker/RefNode): ' + - `${JSON.stringify(node)}. Delete the store and re-sync it.`, + `${JSON.stringify(node)}. Back up the store, then migrate the document ` + + 'with an older release or delete and re-sync it.', ); } diff --git a/tests/core-value-serialization.test.ts b/tests/core-value-serialization.test.ts index 850f39e..47365be 100644 --- a/tests/core-value-serialization.test.ts +++ b/tests/core-value-serialization.test.ts @@ -1,5 +1,7 @@ import { deserialize } from '../cfds/base/object.ts'; -import { kChecksumSerOptions } from '../cfds/base/item.ts'; +import { Item, kChecksumSerOptions } from '../cfds/base/item.ts'; +import { DataRegistry } from '../cfds/base/data-registry.ts'; +import { decoderOutstandingCount } from '../base/core-types/encoding/json.ts'; import { getTypeOperations, kValueTypeCoreTypes, @@ -99,6 +101,108 @@ function assertLegacyIngress(doc: RichText): void { } export default function setup(): void { + TEST( + 'CoreValueSerialization', + 'nested Encodable decodes cyclical refs through get()', + () => { + // WHY: a nested Encodable must be created by the parent's decoder class. + // A plain JSONDecoder child loses the __r/__d context and returns the + // raw ref wrappers instead of the referenced values. + const shared = { id: 'shared', n: 1 }; + const inner = new JSONCyclicalEncoder(); + inner.set('x', { first: shared, second: shared }); + const innerEncoded = inner.getOutput(); + const base = decoderOutstandingCount(); + { + using outer = JSONCyclicalDecoder.get({ + nested: { __t: 'E', __v: innerEncoded }, + } as unknown as Record); + const nested = outer.get('nested') as JSONDecoder; + const decoded = nested.get('x') as { + first: unknown; + second: unknown; + }; + assertEquals(decoded.first, { id: 'shared', n: 1 }); + assertTrue(decoded.first === decoded.second); + } + // The parent dispose cascades to the pooled child. + assertEquals(decoderOutstandingCount(), base); + }, + ); + + TEST( + 'CoreValueSerialization', + 'dictionary values use the shared value encoder', + () => { + // WHY: a raw dictToPrimitive copy skipped Date/Set/Encodable tagging, so + // a Date in a map landed as an ISO string while the checksum tagged it + // as a Date. Wire form must match the top-level Date form. + const dict = mkDict([ + ['when', new Date(1_700_000_000_000)], + ['nested', { d: new Date(1000) }], + ]); + const encoded = new JSONEncoder().convertValue(dict as never); + assertEquals( + JSON.stringify(encoded), + '{"when":{"__t":"D","__v":1700000000},"nested":{"d":{"__t":"D","__v":1}}}', + ); + }, + ); + + TEST( + 'CoreValueSerialization', + 'dictionary Date round-trips without changing the checksum', + () => { + const date = new Date(1_700_000_000_000); + const dict = mkDict([['when', date]]); + const encoded = new JSONEncoder().convertValue(dict as never); + using decoder = JSONDecoder.get(encoded as never); + const decodedDate = decoder.get('when') as Date; + assertEquals(decodedDate instanceof Date, true); + assertEquals(decodedDate.getTime(), date.getTime()); + const reloaded = mkDict([['when', decodedDate]]); + assertEquals( + new Murmur3Checksum({ typeSafe: true }).checksumForValue( + reloaded as never, + ), + new Murmur3Checksum({ typeSafe: true }).checksumForValue(dict as never), + ); + }, + ); + + TEST( + 'CoreValueSerialization', + 'typed map Date survives encode/decode with a matching Item checksum', + () => { + // Mirrors the delta-commit corruption path: the restored item is rebuilt + // from decoded data and its recomputed checksum must equal the original, + // otherwise repo/repo.ts marks the commit corrupted. + const schema: Schema = { + ns: 'core-value-serialization-map-date', + version: 1, + fields: { m: { type: 'map' } }, + }; + const registry = new DataRegistry(); + registry.registerSchema(schema); + const date = new Date(1_700_000_000_000); + const item = new Item( + { schema, data: { m: mkDict([['when', date]]) } as never }, + registry, + ); + const js = JSON.parse(JSON.stringify(item.toJS())); + using dec = JSONCyclicalDecoder.get(js); + const restored = new Item({ decoder: dec }, registry); + const restoredMap = restored.get('m') as Map; + assertEquals(restoredMap.get('when') instanceof Date, true); + assertEquals(restoredMap.get('when')!.getTime(), date.getTime()); + const rebuilt = new Item( + { schema, data: restored.dataUnsafe() as never }, + registry, + ); + assertEquals(rebuilt.checksum, item.checksum); + }, + ); + TEST( 'CoreValueSerialization', 'kTypeOperations is frozen and exhaustive', diff --git a/tests/storage-format-epoch.test.ts b/tests/storage-format-epoch.test.ts index 077a8b3..a91ed3f 100644 --- a/tests/storage-format-epoch.test.ts +++ b/tests/storage-format-epoch.test.ts @@ -14,6 +14,14 @@ import { assertEquals, assertExists, assertTrue } from './asserts.ts'; import * as path from '../base/path.ts'; import type { GoatDB } from '../db/db.ts'; import { JSONDecoder } from '../base/core-types/encoding/json.ts'; +import { JSONCyclicalEncoder } from '../base/core-types/encoding/json.ts'; +import { + MD5Checksum, + Murmur3Checksum, +} from '../base/core-types/encoding/checksum.ts'; +import type { CoreValue } from '../base/core-types/base.ts'; +import { Item } from '../cfds/base/item.ts'; +import { DataRegistry } from '../cfds/base/data-registry.ts'; import { deserialize } from '../cfds/base/object.ts'; import type { Schema } from '../cfds/base/schema.ts'; import { @@ -75,6 +83,73 @@ function rejectionMessage(r: PromiseRejectedResult): string { return r.reason instanceof Error ? r.reason.message : String(r.reason); } +/** + * Expected codec fingerprint per storage epoch. + * + * WHY: `kStorageFormatEpoch` used to depend on a "BUMP THIS" comment only, so + * a codec/checksum change could ship without a bump and old stores would + * mis-decode instead of failing fast. A codec change now fails this test until + * the author bumps the epoch and records the new fingerprint. + */ +const kEpochFingerprints: Readonly> = { + 1: 'aca89293fe85f54d4e3e3f65fb80d41a', +}; + +const kFingerprintSchema: Schema = { + ns: 'epoch-fingerprint', + version: 1, + fields: { + d: { type: 'date' }, + s: { type: 'set' }, + m: { type: 'map' }, + }, +}; +const kFingerprintRegistry = new DataRegistry(); +kFingerprintRegistry.registerSchema(kFingerprintSchema); + +const kMapDateSchema: Schema = { + ns: 'epoch-map-date', + version: 1, + fields: { m: { type: 'map' } }, +}; +const kMapDateRegistry = new DataRegistry(); +kMapDateRegistry.registerSchema(kMapDateSchema); + +/** + * Deterministic fingerprint over the wire formats the epoch protects. + * Covers core values (Date, Set, Map with a Date, shared refs) and the + * authoritative Item checksum. + */ +function codecFingerprint(): string { + const date = new Date(1_700_000_000_000); + const shared = { id: 'shared', when: date }; + const fixtures: CoreValue[] = [ + date, + new Set([date, 'x', 2]), + new Map([['when', date], ['n', 1]]), + { first: shared, second: shared } as CoreValue, + ]; + const parts = fixtures.map((value) => { + const encoder = new JSONCyclicalEncoder(); + encoder.set('v', value); + return JSON.stringify(encoder.getOutput()) + '|' + + new Murmur3Checksum({ typeSafe: true }).checksumForValue(value); + }); + const item = new Item( + { + schema: kFingerprintSchema, + data: { + d: date, + s: new Set([date]), + m: new Map([['when', date]]), + } as never, + }, + kFingerprintRegistry, + ); + parts.push(JSON.stringify(item.toJS()) + '|' + item.checksum); + return new MD5Checksum().checksumForValue(parts.join('\n')); +} + /** * Matches the OPFS concurrent/rapid-reopen platform errors (see * file-impl-opfs.ts). These are storage-layer limitations, not epoch bugs. @@ -173,6 +248,46 @@ export default function setup() { }, ); + TEST( + 'StorageFormatEpoch', + 'a Date inside a map survives a store reopen and keeps its checksum', + async (ctx) => { + // Regression for the raw dictToPrimitive copy: the Date was stored as an + // ISO string while the checksum tagged it as a Date, so the delta commit + // was marked corrupted after reload. + const date = new Date(1_700_000_000_000); + const first = await ctx.createDB('map-date', { + registry: kMapDateRegistry, + }); + try { + await first.readyPromise(); + await first.create('/data/maps/m1', kMapDateSchema, { + m: new Map([['when', date]]), + }).commit(); + await first.flushAll(); + } finally { + await first.close(); + } + + const second = await ctx.createDB('map-date', { + registry: kMapDateRegistry, + }); + try { + await second.readyPromise(); + await second.open('/data/maps'); + const map = second.item('/data/maps/m1').get('m') as Map< + string, + Date + >; + assertTrue(map.get('when') instanceof Date); + assertEquals(map.get('when')!.getTime(), date.getTime()); + } finally { + await second.flushAll(); + await second.close(); + } + }, + ); + TEST( 'StorageFormatEpoch', 'a non-empty store without a marker is rejected', @@ -234,6 +349,26 @@ export default function setup() { }, ); + TEST( + 'StorageFormatEpoch', + 'codec fingerprint matches the pinned epoch', + () => { + const expected = kEpochFingerprints[kStorageFormatEpoch]; + assertExists( + expected, + `No codec fingerprint for storage epoch ${kStorageFormatEpoch}. ` + + 'A codec change requires bumping kStorageFormatEpoch and adding its ' + + 'fingerprint to kEpochFingerprints.', + ); + assertEquals( + codecFingerprint(), + expected, + `Codec bytes changed. If intentional, bump kStorageFormatEpoch ` + + `(currently ${kStorageFormatEpoch}) and add the new fingerprint.`, + ); + }, + ); + TEST( 'StorageFormatEpoch', 'a store with a mismatched epoch is rejected',