Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),

### Fixed

- JSON decoders use a disposable lifetime. Parent disposal releases owned
children and preserves children acquired by another owner. Disposed decoders
reject reads until acquired again. `finalize()` remains a disposal alias.
- Bare `__rId` references without an enclosing reference table now throw.
Objects with additional fields remain user data.
- Store initialization ignores unrelated files. Missing markers reject only
known GoatDB artifacts, name the artifact, and require a backup before removal.
- Storage epoch file operations run through the worker so browser OPFS opens
Expand Down
7 changes: 1 addition & 6 deletions base/common.ts
Original file line number Diff line number Diff line change
Expand Up @@ -85,8 +85,7 @@ export function* keysOf<T extends Record<string, unknown>>(
obj: T,
): Generator<string> {
for (const k in obj) {
// deno-lint-ignore no-prototype-builtins
if (obj.hasOwnProperty(k)) {
if (Object.hasOwn(obj, k)) {
yield k;
}
}
Expand Down Expand Up @@ -130,10 +129,6 @@ export function* unionIter<T>(...args: Iterable<T>[]): Iterable<T> {
}
}

export function newInstance<T = any>(instance: any, ...args: any[]): T {
return new (instance.constructor as any)(...args) as T;
}

/**
* Maps values from an iterable to new values using a mapping function.
*
Expand Down
1 change: 1 addition & 0 deletions base/core-types/encoding/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ export type {
DecodableKey,
DecodedValue,
Decoder,
OwnedDecoder,
ReadonlyDecodedArray,
ReadonlyDecodedObject,
} from './types.ts';
Expand Down
233 changes: 170 additions & 63 deletions base/core-types/encoding/json.ts
Original file line number Diff line number Diff line change
@@ -1,29 +1,43 @@
import {
import type {
JSONObject,
JSONValue,
ReadonlyJSONObject,
ReadonlyJSONValue,
} from '../../interfaces.ts';
import { CoreOptions, CoreType, CoreValue, Encoder } from '../base.ts';
import {
type CoreOptions,
CoreType,
type CoreValue,
type Encoder,
} from '../base.ts';
import { getCoreType } from '../utils.ts';
import {
EncodedEncodable,
isEncodedDate,
isEncodedEncodable,
isEncodedSet,
JSONBaseEncoder,
} from './base-encoder.ts';
import {
BaseCyclicalEncoder,
EncodedRef,
type EncodedRef,
isEncodedRef,
isEncodedRefObject,
} from './cyclical.ts';
import { DecodedValue, Decoder } from './types.ts';
import { newInstance } from '../../common.ts';
import type { DecodedValue, Decoder, OwnedDecoder } from './types.ts';
import { deserializeDate } from '../../date.ts';
import { Encodable } from '../index.ts';
import { ReadonlyDecodedArray } from './index.ts';
import { assert } from '../../error.ts';
import type { Encodable } from '../index.ts';

// L4 accounting: number of decoders currently acquired but not yet disposed.
// Balanced by `_acquire()` on every acquire and `[Symbol.dispose]` on first
// release. The accessor below is for tests only; it is deliberately not
// re-exported from mod.ts.
let gOutstandingDecoders = 0;

/** @internal Test-visible L4 outstanding-decoder count. */
export function decoderOutstandingCount(): number {
return gOutstandingDecoders;
}

export class JSONEncoder extends JSONBaseEncoder<ReadonlyJSONValue> {
private _encodedObj: JSONObject;
Expand All @@ -38,7 +52,7 @@ export class JSONEncoder extends JSONBaseEncoder<ReadonlyJSONValue> {
}

newEncoder() {
return newInstance<JSONEncoder>(this);
return new JSONEncoder();
}

static toJS(value: CoreValue): ReadonlyJSONValue {
Expand Down Expand Up @@ -106,39 +120,119 @@ export class JSONCyclicalEncoder extends BaseCyclicalEncoder<

export class JSONDecoder implements Decoder {
protected _data: ReadonlyJSONObject;
// L3: set on dispose, cleared on re-acquire. Guards reads and re-pooling.
protected _finalized = false;
// L4: true while this instance is counted in gOutstandingDecoders.
private _tracked = false;
// Acquire generation. Bumped on every `_acquire`; a parent records a child's
// generation and only cascades to it while that generation is still current.
// WHY: a consumer can dispose a pooled child and another decode re-acquires
// it; the parent must not then dispose the new owner's decoder.
private _generation = 0;
// L4: children created via decodeValue/getDecoder, each paired with the
// generation seen when it was added. Disposed with the parent when still
// current. Lazily allocated to avoid overhead for leaf decoders.
protected _children:
| { decoder: OwnedDecoder; generation: number }[]
| undefined;

constructor(encodedValue: ReadonlyJSONObject | undefined) {
this._data = encodedValue || {};
}

/** Acquiring factory for roots (e.g. settings). Tracks L4 like children. */
static get(encodedValue: ReadonlyJSONObject | undefined): JSONDecoder {
const decoder = new JSONDecoder(encodedValue);
decoder._acquire();
return decoder;
}

/** L4 acquire accounting. Re-acquiring clears the disposed state (L3). */
protected _acquire(): void {
this._finalized = false;
this._tracked = true;
this._generation++;
gOutstandingDecoders++;
}

/** L3/L4 release. Idempotent: a second call is a no-op and never re-pools. */
[Symbol.dispose](): void {
if (this._finalized) return;
this._finalized = true;
// Release tracked children first so their L4 count drops before ours. Skip
// any child re-acquired since it was added (generation changed): it belongs
// to a newer owner now and must stay alive.
if (this._children) {
for (const { decoder, generation } of this._children) {
if ((decoder as JSONDecoder)._generation === generation) {
decoder[Symbol.dispose]();
}
}
this._children = undefined;
}
if (this._tracked) {
this._tracked = false;
gOutstandingDecoders--;
}
}

protected _assertAlive(): void {
assert(!this._finalized, 'JSON decoder used after dispose');
}

protected _addChild(child: OwnedDecoder): void {
if (!this._children) this._children = [];
this._children.push({
decoder: child,
generation: (child as JSONDecoder)._generation,
});
}

get<T extends DecodedValue>(key: string, defaultValue?: T): T | undefined {
this._assertAlive();
if (!this.has(key)) {
return defaultValue;
}
const val = this._data[key];
if (!valueNeedsJSONDecode(val)) {
// Primitives are returned as-is; every non-null object needs decoding.
if (val === null || typeof val !== 'object') {
return val as T;
}
return this.decodeValue(this._data[key]) as T;
return this.decodeValue(val) as T;
}

has(key: string): boolean {
// deno-lint-ignore no-prototype-builtins
return this._data.hasOwnProperty(key);
this._assertAlive();
return Object.hasOwn(this._data, key);
}

getDecoder(key: string, offset?: number): Decoder<string, ReadonlyJSONValue> {
let value = this.get(key);
getDecoder(
key: string,
offset?: number,
): OwnedDecoder<string, ReadonlyJSONValue> {
this._assertAlive();
let value = this._data[key];
if (offset !== undefined && value instanceof Array) {
value = value[offset];
}
if (value instanceof JSONDecoder) {
return value;
}
return newInstance(this, value);
// decodeValue handles {__t:'E'} by returning an already-acquired decoder
// (tracked via _addChild inside decodeValue) — no extra acquire or tracking.
const decoded = this.decodeValue(value);
if (decoded instanceof JSONDecoder) {
return decoded;
}
// Non-Encodable: wrap the raw value so the child decodes lazily.
const child = new JSONDecoder(value as ReadonlyJSONObject | undefined);
child._acquire();
this._addChild(child);
return child;
}

decodeValue(value: ReadonlyJSONValue): DecodedValue {
this._assertAlive();
if (value instanceof Array) {
const res: DecodedValue[] = [];
for (const v of value) {
Expand All @@ -161,7 +255,14 @@ export class JSONDecoder implements Decoder {
return deserializeDate(value.__v);
}
if (isEncodedEncodable(value)) {
return newInstance<JSONDecoder>(this, value.__v);
// Nested Encodable: acquire so the L4 total balances on dispose. The
// child is tracked on _children so the parent's dispose releases it even
// when the consumer never disposes the child directly (e.g. when it is
// reached inside a decoded array via get()).
const nested = new JSONDecoder(value.__v as ReadonlyJSONObject);
nested._acquire();
this._addChild(nested);
return nested;
}
if (getCoreType(value) === CoreType.Object) {
return this.decodeObject(value as ReadonlyJSONObject);
Expand Down Expand Up @@ -190,7 +291,20 @@ export class JSONDecoder implements Decoder {
}
}

// Pool of finalized cyclical decoders, reused LIFO to avoid allocation on the
// .goat/JSON read path. Membership is managed exclusively by
// `JSONCyclicalDecoder.get` (pop) and `[Symbol.dispose]`/`finalize` (push).
// WHY: the pool is bounded so a peak of transient nested decoders cannot pin
// memory on long-lived processes; overflow decoders are dropped for GC and
// counted (test-only) via decoderPoolDroppedCount.
export const kMaxActiveDecoders = 32;
const gActiveDecoders: JSONCyclicalDecoder[] = [];
let gPoolDrops = 0;

/** @internal Test-visible count of pool-overflow drops. */
export function decoderPoolDroppedCount(): number {
return gPoolDrops;
}

export class JSONCyclicalDecoder extends JSONDecoder {
private _tempRefs?: ReadonlyJSONObject[][];
Expand All @@ -200,7 +314,7 @@ export class JSONCyclicalDecoder extends JSONDecoder {
super(encodedValue);
}

static get(
static override get(
encodedValue: ReadonlyJSONObject | undefined,
): JSONCyclicalDecoder {
let r = gActiveDecoders.pop();
Expand All @@ -210,21 +324,33 @@ export class JSONCyclicalDecoder extends JSONDecoder {
} else {
r = new JSONCyclicalDecoder(encodedValue);
}
r._acquire();
return r;
}

finalize(): void {
gActiveDecoders.push(this);
override [Symbol.dispose](): void {
if (this._finalized) return;
super[Symbol.dispose]();
if (gActiveDecoders.length < kMaxActiveDecoders) {
gActiveDecoders.push(this);
} else {
gPoolDrops++;
}
}

// needDecode(value: ReadonlyJSONValue) {
// if (isEncodedRefObject(value)) {
// return true;
// }
// return valueNeedsJSONDecode(value);
// }
/**
* Named alias for `[Symbol.dispose]`; kept for callers not using `using`.
*
* Dispose at most once per acquire. This instance is pooled and re-acquired
* by later decodes, so calling dispose again through a stale reference
* releases the *new* owner's decoder. `using` guarantees single disposal.
*/
finalize(): void {
this[Symbol.dispose]();
}

override decodeValue(value: ReadonlyJSONValue): DecodedValue {
this._assertAlive();
if (isEncodedRefObject(value)) {
if (!this._tempRefs) this._tempRefs = [];
this._tempRefs.push(value.__r);
Expand All @@ -238,7 +364,16 @@ export class JSONCyclicalDecoder extends JSONDecoder {
protected override decodeObject(jsonObj: ReadonlyJSONObject): {
[key: string]: DecodedValue;
} {
if (isEncodedRef(jsonObj) && this._tempRefs) {
if (isEncodedRef(jsonObj)) {
// WHY: a lone `{__rId}` outside `__r/__d` is corrupt input, never
// data. Fail loudly like bare-number dates and legacy refs. Wider
// shapes (e.g. `{__rId:5,other:1}`) are legal user data: fall through
// as plain objects so the stray assert never fires on them.
if (Object.keys(jsonObj).length !== 1) return super.decodeObject(jsonObj);
assert(
this._tempRefs !== undefined,
'Stray __rId ref without __r/__d context',
);
const refs = this._tempRefs[this._tempRefs.length - 1];
return refs[jsonObj.__rId as number];
}
Expand All @@ -248,51 +383,23 @@ export class JSONCyclicalDecoder extends JSONDecoder {
override getDecoder(
key: string,
offset?: number,
): Decoder<string, ReadonlyJSONValue> {
): OwnedDecoder<string, ReadonlyJSONValue> {
this._assertAlive();
let value = this._data[key];
if (isEncodedEncodable(value)) {
value = value.__v;
}
if (offset !== undefined && value instanceof Array) {
value = value[offset];
}
if (value instanceof JSONDecoder) {
return value;
}
return JSONCyclicalDecoder.get(value as ReadonlyJSONObject);
}
}

function valueNeedsJSONDecode(value: ReadonlyJSONValue) {
if (typeof value === 'object') {
if (value instanceof Array) {
// for (const item of value) {
// if (this.needDecode(item)) {
// return true;
// }
// }
return true;
}
if (isEncodedSet(value)) {
return true;
}
if (isEncodedDate(value)) {
return true;
}
if (isEncodedEncodable(value)) {
return true;
}
if (getCoreType(value) === CoreType.Object) {
// for (const val of Object.values(value as ReadonlyJSONObject)) {
// if (this.needDecode(val as ReadonlyJSONValue)) {
// return true;
// }
// }
return true;
value = value.__v;
}
if (isEncodedRefObject(value)) {
return true;
if (value instanceof JSONDecoder) {
return value;
}
const child = JSONCyclicalDecoder.get(value as ReadonlyJSONObject);
this._addChild(child);
return child;
}
return false;
}
Loading
Loading