Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),

### Fixed

- A `Date` inside a `map` field now uses the tagged
`{ "__t": "D", "__v": <epoch-seconds> }` wire form, the same as every other
position. It was copied raw before, so it round-tripped as an ISO string while
its checksum tagged it as a `Date`, marking the delta commit as corrupted on
reload. Commits written by earlier builds with such values stay corrupted and
must be re-created.
- JSON decoders use a disposable lifetime. Parent disposal releases owned
children and preserves children acquired by another owner. Disposed decoders
reject reads until acquired again. `finalize()` remains a disposal alias.
Expand Down
20 changes: 12 additions & 8 deletions base/core-types/encoding/base-encoder.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,7 @@ import type {
ReadonlyJSONValue,
} from '../../interfaces.ts';
import { serializeDate } from '../../date.ts';
import {
type Dictionary,
dictToPrimitive,
isDictionary,
} from '../../collections/dict.ts';
import { type Dictionary, isDictionary } from '../../collections/dict.ts';
import {
type CoreKey,
type CoreOptions,
Expand Down Expand Up @@ -322,9 +318,17 @@ export abstract class JSONBaseEncoder<

convertDictionary(
value: Dictionary<CoreKey, CoreValue>,
_options?: OT,
) {
return dictToPrimitive(value);
options?: OT,
): JSONObject {
// WHY: every dictionary value must pass through the shared value encoder.
// Copying raw values (dictToPrimitive) skipped Date/Set/Encodable tagging,
// so a Date in a map field landed as an ISO string while its checksum
// tagged it as a Date, marking the commit corrupted on reload.
const result: JSONObject = {};
for (const [k, v] of value) {
result[this.convertKey(k)] = this.convertValue(v, options);
}
return result;
}

convertGenerator(g: Generator<CoreValue>, options?: OT) {
Expand Down
30 changes: 26 additions & 4 deletions base/core-types/encoding/json.ts
Original file line number Diff line number Diff line change
Expand Up @@ -225,12 +225,26 @@ export class JSONDecoder implements Decoder {
return decoded;
}
// Non-Encodable: wrap the raw value so the child decodes lazily.
const child = new JSONDecoder(value as ReadonlyJSONObject | undefined);
child._acquire();
const child = this.createChildDecoder(
value as ReadonlyJSONObject | undefined,
);
this._addChild(child);
return child;
}

/**
* Creates an acquired child decoder for nested data. Subclasses override the
* factory so a cyclical parent yields cyclical children; a plain
* `new JSONDecoder` would drop the `__r`/`__d` context.
*/
protected createChildDecoder(
data: ReadonlyJSONObject | undefined,
): OwnedDecoder<string, ReadonlyJSONValue> {
const child = new JSONDecoder(data);
child._acquire();
return child;
}

decodeValue(value: ReadonlyJSONValue): DecodedValue {
this._assertAlive();
if (value instanceof Array) {
Expand Down Expand Up @@ -259,8 +273,9 @@ export class JSONDecoder implements Decoder {
// child is tracked on _children so the parent's dispose releases it even
// when the consumer never disposes the child directly (e.g. when it is
// reached inside a decoded array via get()).
const nested = new JSONDecoder(value.__v as ReadonlyJSONObject);
nested._acquire();
const nested = this.createChildDecoder(
value.__v as ReadonlyJSONObject,
);
this._addChild(nested);
return nested;
}
Expand Down Expand Up @@ -380,6 +395,13 @@ export class JSONCyclicalDecoder extends JSONDecoder {
return super.decodeObject(jsonObj);
}

protected override createChildDecoder(
data: ReadonlyJSONObject | undefined,
): OwnedDecoder<string, ReadonlyJSONValue> {
// Pooled factory already acquires and resets `_tempRefs`.
return JSONCyclicalDecoder.get(data);
}

override getDecoder(
key: string,
offset?: number,
Expand Down
3 changes: 2 additions & 1 deletion cfds/richtext/legacy-ref.ts
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,8 @@ function assertNodeIsNotLegacyRef(node: unknown): void {
if (!isLegacyRefNode(node)) return;
throw new Error(
'Legacy rich text ref is not supported (RefMarker/RefNode): ' +
`${JSON.stringify(node)}. Delete the store and re-sync it.`,
`${JSON.stringify(node)}. Back up the store, then migrate the document ` +
'with an older release or delete and re-sync it.',
);
}

Expand Down
106 changes: 105 additions & 1 deletion tests/core-value-serialization.test.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
import { deserialize } from '../cfds/base/object.ts';
import { kChecksumSerOptions } from '../cfds/base/item.ts';
import { Item, kChecksumSerOptions } from '../cfds/base/item.ts';
import { DataRegistry } from '../cfds/base/data-registry.ts';
import { decoderOutstandingCount } from '../base/core-types/encoding/json.ts';
import {
getTypeOperations,
kValueTypeCoreTypes,
Expand Down Expand Up @@ -99,6 +101,108 @@ function assertLegacyIngress(doc: RichText): void {
}

export default function setup(): void {
TEST(
'CoreValueSerialization',
'nested Encodable decodes cyclical refs through get()',
() => {
// WHY: a nested Encodable must be created by the parent's decoder class.
// A plain JSONDecoder child loses the __r/__d context and returns the
// raw ref wrappers instead of the referenced values.
const shared = { id: 'shared', n: 1 };
const inner = new JSONCyclicalEncoder();
inner.set('x', { first: shared, second: shared });
const innerEncoded = inner.getOutput();
const base = decoderOutstandingCount();
{
using outer = JSONCyclicalDecoder.get({
nested: { __t: 'E', __v: innerEncoded },
} as unknown as Record<string, never>);
const nested = outer.get('nested') as JSONDecoder;
const decoded = nested.get('x') as {
first: unknown;
second: unknown;
};
assertEquals(decoded.first, { id: 'shared', n: 1 });
assertTrue(decoded.first === decoded.second);
}
// The parent dispose cascades to the pooled child.
assertEquals(decoderOutstandingCount(), base);
},
);

TEST(
'CoreValueSerialization',
'dictionary values use the shared value encoder',
() => {
// WHY: a raw dictToPrimitive copy skipped Date/Set/Encodable tagging, so
// a Date in a map landed as an ISO string while the checksum tagged it
// as a Date. Wire form must match the top-level Date form.
const dict = mkDict([
['when', new Date(1_700_000_000_000)],
['nested', { d: new Date(1000) }],
]);
const encoded = new JSONEncoder().convertValue(dict as never);
assertEquals(
JSON.stringify(encoded),
'{"when":{"__t":"D","__v":1700000000},"nested":{"d":{"__t":"D","__v":1}}}',
);
},
);

TEST(
'CoreValueSerialization',
'dictionary Date round-trips without changing the checksum',
() => {
const date = new Date(1_700_000_000_000);
const dict = mkDict([['when', date]]);
const encoded = new JSONEncoder().convertValue(dict as never);
using decoder = JSONDecoder.get(encoded as never);
const decodedDate = decoder.get('when') as Date;
assertEquals(decodedDate instanceof Date, true);
assertEquals(decodedDate.getTime(), date.getTime());
const reloaded = mkDict([['when', decodedDate]]);
assertEquals(
new Murmur3Checksum({ typeSafe: true }).checksumForValue(
reloaded as never,
),
new Murmur3Checksum({ typeSafe: true }).checksumForValue(dict as never),
);
},
);

TEST(
'CoreValueSerialization',
'typed map Date survives encode/decode with a matching Item checksum',
() => {
// Mirrors the delta-commit corruption path: the restored item is rebuilt
// from decoded data and its recomputed checksum must equal the original,
// otherwise repo/repo.ts marks the commit corrupted.
const schema: Schema = {
ns: 'core-value-serialization-map-date',
version: 1,
fields: { m: { type: 'map' } },
};
const registry = new DataRegistry();
registry.registerSchema(schema);
const date = new Date(1_700_000_000_000);
const item = new Item(
{ schema, data: { m: mkDict([['when', date]]) } as never },
registry,
);
const js = JSON.parse(JSON.stringify(item.toJS()));
using dec = JSONCyclicalDecoder.get(js);
const restored = new Item({ decoder: dec }, registry);
const restoredMap = restored.get('m') as Map<string, Date>;
assertEquals(restoredMap.get('when') instanceof Date, true);
assertEquals(restoredMap.get('when')!.getTime(), date.getTime());
const rebuilt = new Item(
{ schema, data: restored.dataUnsafe() as never },
registry,
);
assertEquals(rebuilt.checksum, item.checksum);
},
);

TEST(
'CoreValueSerialization',
'kTypeOperations is frozen and exhaustive',
Expand Down
135 changes: 135 additions & 0 deletions tests/storage-format-epoch.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,14 @@ import { assertEquals, assertExists, assertTrue } from './asserts.ts';
import * as path from '../base/path.ts';
import type { GoatDB } from '../db/db.ts';
import { JSONDecoder } from '../base/core-types/encoding/json.ts';
import { JSONCyclicalEncoder } from '../base/core-types/encoding/json.ts';
import {
MD5Checksum,
Murmur3Checksum,
} from '../base/core-types/encoding/checksum.ts';
import type { CoreValue } from '../base/core-types/base.ts';
import { Item } from '../cfds/base/item.ts';
import { DataRegistry } from '../cfds/base/data-registry.ts';
import { deserialize } from '../cfds/base/object.ts';
import type { Schema } from '../cfds/base/schema.ts';
import {
Expand Down Expand Up @@ -75,6 +83,73 @@ function rejectionMessage(r: PromiseRejectedResult): string {
return r.reason instanceof Error ? r.reason.message : String(r.reason);
}

/**
* Expected codec fingerprint per storage epoch.
*
* WHY: `kStorageFormatEpoch` used to depend on a "BUMP THIS" comment only, so
* a codec/checksum change could ship without a bump and old stores would
* mis-decode instead of failing fast. A codec change now fails this test until
* the author bumps the epoch and records the new fingerprint.
*/
const kEpochFingerprints: Readonly<Record<number, string>> = {
1: 'aca89293fe85f54d4e3e3f65fb80d41a',
};

const kFingerprintSchema: Schema = {
ns: 'epoch-fingerprint',
version: 1,
fields: {
d: { type: 'date' },
s: { type: 'set' },
m: { type: 'map' },
},
};
const kFingerprintRegistry = new DataRegistry();
kFingerprintRegistry.registerSchema(kFingerprintSchema);

const kMapDateSchema: Schema = {
ns: 'epoch-map-date',
version: 1,
fields: { m: { type: 'map' } },
};
const kMapDateRegistry = new DataRegistry();
kMapDateRegistry.registerSchema(kMapDateSchema);

/**
* Deterministic fingerprint over the wire formats the epoch protects.
* Covers core values (Date, Set, Map with a Date, shared refs) and the
* authoritative Item checksum.
*/
function codecFingerprint(): string {
const date = new Date(1_700_000_000_000);
const shared = { id: 'shared', when: date };
const fixtures: CoreValue[] = [
date,
new Set<CoreValue>([date, 'x', 2]),
new Map<string, CoreValue>([['when', date], ['n', 1]]),
{ first: shared, second: shared } as CoreValue,
];
const parts = fixtures.map((value) => {
const encoder = new JSONCyclicalEncoder();
encoder.set('v', value);
return JSON.stringify(encoder.getOutput()) + '|' +
new Murmur3Checksum({ typeSafe: true }).checksumForValue(value);
});
const item = new Item(
{
schema: kFingerprintSchema,
data: {
d: date,
s: new Set([date]),
m: new Map([['when', date]]),
} as never,
},
kFingerprintRegistry,
);
parts.push(JSON.stringify(item.toJS()) + '|' + item.checksum);
return new MD5Checksum().checksumForValue(parts.join('\n'));
}

/**
* Matches the OPFS concurrent/rapid-reopen platform errors (see
* file-impl-opfs.ts). These are storage-layer limitations, not epoch bugs.
Expand Down Expand Up @@ -173,6 +248,46 @@ export default function setup() {
},
);

TEST(
'StorageFormatEpoch',
'a Date inside a map survives a store reopen and keeps its checksum',
async (ctx) => {
// Regression for the raw dictToPrimitive copy: the Date was stored as an
// ISO string while the checksum tagged it as a Date, so the delta commit
// was marked corrupted after reload.
const date = new Date(1_700_000_000_000);
const first = await ctx.createDB('map-date', {
registry: kMapDateRegistry,
});
try {
await first.readyPromise();
await first.create('/data/maps/m1', kMapDateSchema, {
m: new Map([['when', date]]),
}).commit();
await first.flushAll();
} finally {
await first.close();
}

const second = await ctx.createDB('map-date', {
registry: kMapDateRegistry,
});
try {
await second.readyPromise();
await second.open('/data/maps');
const map = second.item('/data/maps/m1').get('m') as Map<
string,
Date
>;
assertTrue(map.get('when') instanceof Date);
assertEquals(map.get('when')!.getTime(), date.getTime());
} finally {
await second.flushAll();
await second.close();
}
},
);

TEST(
'StorageFormatEpoch',
'a non-empty store without a marker is rejected',
Expand Down Expand Up @@ -234,6 +349,26 @@ export default function setup() {
},
);

TEST(
'StorageFormatEpoch',
'codec fingerprint matches the pinned epoch',
() => {
const expected = kEpochFingerprints[kStorageFormatEpoch];
assertExists(
expected,
`No codec fingerprint for storage epoch ${kStorageFormatEpoch}. ` +
'A codec change requires bumping kStorageFormatEpoch and adding its ' +
'fingerprint to kEpochFingerprints.',
);
assertEquals(
codecFingerprint(),
expected,
`Codec bytes changed. If intentional, bump kStorageFormatEpoch ` +
`(currently ${kStorageFormatEpoch}) and add the new fingerprint.`,
);
},
);

TEST(
'StorageFormatEpoch',
'a store with a mismatched epoch is rejected',
Expand Down
Loading