Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),

### Breaking Changes

- Unified `Date` wire format and commit checksum (0.7.0). Typed fields now
encode as `{ "__t": "D", "__v": <epoch-seconds> }`, not a bare number.
Dates carry their tag in every checksum position. The typed-field checksum
changes from `416277608` to `1391363808`. Older stores fail fast on open.
Back up the store before removal or re-sync. Mixed-version fleets are not
supported; typed-field decode rejects bare-number dates from older peers.
- Non-`typeSafe` strings can collide with tagged Dates (for example,
`"41700000000"`). Use `typeSafe` checksums for untrusted input.
- Legacy rich-text refs (`RefMarker` leaves or `RefNode` elements) now throw
on read, normalize, and serialize instead of being silently stripped. Back up
the store first. Use a release that supports legacy refs to migrate affected
Expand All @@ -27,6 +35,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),

### Fixed

- Store initialization ignores unrelated files. Missing markers reject only
known GoatDB artifacts, name the artifact, and require a backup before removal.
- Storage epoch file operations run through the worker so browser OPFS opens
succeed. Strict file operations preserve filesystem error names and codes.
- Recursive `mkdir` now rejects on failure in every runtime instead of returning
a runtime-dependent boolean.
- `coreValueEquals` is symmetric when `objectFilterFields` inspects its object.
The second comparison loop previously passed the wrong object to the filter.

Expand Down
15 changes: 14 additions & 1 deletion base/core-types/encoding/checksum.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,14 @@ import type {
ReadonlyCoreArray,
ReadonlyCoreObject,
} from '../base.ts';
import { CoreType } from '../base.ts';
import { getCoreTypeOrUndef } from '../utils.ts';
import * as StreamMD5 from '../../../external/md5.ts';

export interface ChecksumEncoderOpts extends CoreOptions {
// Require `typeSafe: true` where checksums must distinguish value types.
// Require `typeSafe: true` for untrusted input. A plain string such as
// "41700000000" collides with Date (tag "4" + epoch seconds). Type-safe
// checksums prefix each type, which separates the colliding string.
typeSafe?: boolean;
flatRep?: boolean;
local?: boolean;
Expand Down Expand Up @@ -101,6 +104,16 @@ export abstract class ChecksumEncoder<
return v;
}
const converted = super.convertValue(value as CoreValue, options);
// A Date tags as Date in every checksum position (issue #58 §5). Nested
// Dates tag only when typeSafe is true, yet the typed-field path drops
// typeSafe before BaseEncoder.set, so an untagged top-level Date collides
// with a Number. Unconditional Date tagging removes that Date-vs-Number
// asymmetry. Limit: non-typeSafe strings stay untagged, so a string such
// as "41700000000" can still collide with a tagged Date. Use typeSafe
// checksums where untrusted input matters; no other gate changes.
if (type === CoreType.Date) {
return String(type) + converted;
}
const typeSafe = typeof options?.typeSafe === 'boolean'
? options.typeSafe
: this.typeSafe;
Expand Down
9 changes: 3 additions & 6 deletions base/json-log/file-impl-deno.ts
Original file line number Diff line number Diff line change
Expand Up @@ -77,13 +77,10 @@ export const FileImplDeno: FileImpl<Deno.FsFile> = {
return Deno.makeTempDir();
},

// Throw on failure like the Node and OPFS impls. Swallowing the error made
// the contract runtime-dependent and hid the cause from callers.
async mkdir(path: string) {
try {
await Deno.mkdir(path, { recursive: true });
return true;
} catch (_: unknown) {
return false;
}
await Deno.mkdir(path, { recursive: true });
},

async exists(path: string): Promise<boolean> {
Expand Down
6 changes: 3 additions & 3 deletions base/json-log/file-impl-interface.ts
Original file line number Diff line number Diff line change
Expand Up @@ -115,10 +115,10 @@ export interface FileImpl<T> {
* and "/a/b/c" directories.
*
* @param path The path where to create the directory
* @returns A promise that resolves to true if the directory was created,
* false otherwise
* @returns Resolves on success and rejects on failure. All impls (Deno,
* Node, OPFS) behave the same.
*/
mkdir(path: string): Promise<boolean>;
mkdir(path: string): Promise<void>;

/**
* Checks if a file or directory exists at the specified path.
Expand Down
1 change: 0 additions & 1 deletion base/json-log/file-impl-node.ts
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,6 @@ export const FileImplNode: FileImpl<NodeFsFile> = {

async mkdir(path: string) {
await fs.mkdir(path, { recursive: true });
return true;
},

async exists(path: string): Promise<boolean> {
Expand Down
8 changes: 5 additions & 3 deletions base/json-log/file-impl-opfs.ts
Original file line number Diff line number Diff line change
Expand Up @@ -204,7 +204,6 @@ export const FileImplOPFS: FileImpl<OPFSFile> = {

async mkdir(path: string) {
await getDir(path);
return true;
},

async exists(filePath: string): Promise<boolean> {
Expand Down Expand Up @@ -254,8 +253,11 @@ export const FileImplOPFS: FileImpl<OPFSFile> = {
async readDir(dirPath: string): Promise<DirEntry[]> {
const dir = await getDirReadOnly(dirPath);
if (!dir) {
throw new Error(
`ENOENT: no such file or directory, scandir '${dirPath}'`,
// Attach ENOENT so the worker error transport can classify a missing
// directory as "not found" (see isNotFound in db/storage-epoch.ts).
throw Object.assign(
new Error(`ENOENT: no such file or directory, scandir '${dirPath}'`),
{ code: 'ENOENT' },
);
}
const entries: DirEntry[] = [];
Expand Down
9 changes: 6 additions & 3 deletions base/json-log/file-impl.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@ import { getRuntime } from '../runtime/index.ts';
import * as path from '../path.ts';
import type { DirEntry, FileImpl } from './file-impl-interface.ts';

// Worker-only. Main-thread callers use json-log.ts wrappers (OPFS handles
// exist only in the worker). Rule source: tests/worker-boundary.test.ts.

export type { DirEntry } from './file-impl-interface.ts';

let gFileImpl: FileImpl<unknown> | undefined;
Expand Down Expand Up @@ -172,11 +175,11 @@ export async function copyFile(
* This operation is always recursive.
*
* @param path The path where to create the directory
* @returns Promise<boolean> True if the directory was created, false otherwise
* @returns Rejects on failure.
*/
export async function mkdir(path: string): Promise<boolean> {
export async function mkdir(path: string): Promise<void> {
const impl = await FileImplGet();
return impl.mkdir(path);
await impl.mkdir(path);
}

/**
Expand Down
56 changes: 55 additions & 1 deletion base/json-log/json-log-worker-req.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,33 @@
import type { DirEntry } from './file-impl-interface.ts';

/** Strict file operations run in the worker, where OPFS sync handles exist. */
export type WorkerReadFileReq = {
type: 'readFile';
id: number;
path: string;
};
export type WorkerWriteFileReq = {
type: 'writeFile';
id: number;
path: string;
bytes: Uint8Array;
};
export type WorkerMkdirReq = { type: 'mkdir'; id: number; path: string };
export type WorkerReadDirReq = { type: 'readDir'; id: number; path: string };

export type WorkerReadFileResp = {
type: 'readFile';
id: number;
bytes: Uint8Array;
};
export type WorkerWriteFileResp = { type: 'writeFile'; id: number };
export type WorkerMkdirResp = { type: 'mkdir'; id: number };
export type WorkerReadDirResp = {
type: 'readDir';
id: number;
entries: DirEntry[];
};

/**
* A request to open a JSON log file.
*
Expand Down Expand Up @@ -130,7 +160,11 @@ export type WorkerFileReq =
| WorkerFileReqAppend
| WorkerReadTextFileReq
| WorkerWriteTextFileReq
| WorkerRemoveReq;
| WorkerRemoveReq
| WorkerReadFileReq
| WorkerWriteFileReq
| WorkerMkdirReq
| WorkerReadDirReq;

/**
* A response to a request to open a JSON log file.
Expand Down Expand Up @@ -272,8 +306,20 @@ export type WorkerErrorResp = {
type: 'error';
id: number;
error: WorkerErrorType;
// Explicit fields preserve filesystem codes and DOMException names on clone.
details?: { message: string; name: string; code?: string | number };
};

/** Unknown-worker-command details. Shared so the worker and tests agree. */
export function unknownCommandDetails(type: string): {
name: string;
message: string;
} {
// WHY: include the command so a stale worker bundle (one that predates a
// request type) is diagnosable instead of an opaque "UnknownCommand".
return { name: 'Error', message: `Unknown worker command: ${type}` };
}

export type WorkerFileResp =
| WorkerFileRespOpen
| WorkerFileRespClose
Expand All @@ -284,6 +330,10 @@ export type WorkerFileResp =
| WorkerReadTextFileResp
| WorkerWriteTextFileResp
| WorkerRemoveResp
| WorkerReadFileResp
| WorkerWriteFileResp
| WorkerMkdirResp
| WorkerReadDirResp
| WorkerErrorResp;

export type WorkerFileRespForReq<T extends WorkerFileReq = WorkerFileReq> =
Expand All @@ -296,5 +346,9 @@ export type WorkerFileRespForReq<T extends WorkerFileReq = WorkerFileReq> =
: T['type'] extends 'readTextFile' ? WorkerReadTextFileResp
: T['type'] extends 'writeTextFile' ? WorkerWriteTextFileResp
: T['type'] extends 'remove' ? WorkerRemoveResp
: T['type'] extends 'readFile' ? WorkerReadFileResp
: T['type'] extends 'writeFile' ? WorkerWriteFileResp
: T['type'] extends 'mkdir' ? WorkerMkdirResp
: T['type'] extends 'readDir' ? WorkerReadDirResp
: T['type'] extends 'error' ? WorkerErrorResp
: WorkerFileResp;
60 changes: 58 additions & 2 deletions base/json-log/json-log-worker.ts
Original file line number Diff line number Diff line change
Expand Up @@ -56,12 +56,25 @@ import type {
WorkerFileRespFlush,
WorkerFileRespOpen,
WorkerFileRespScan,
WorkerMkdirResp,
WorkerReadDirResp,
WorkerReadFileResp,
WorkerReadTextFileResp,
WorkerRemoveResp,
WorkerWriteFileResp,
WorkerWriteTextFileResp,
} from './json-log-worker-req.ts';
import { unknownCommandDetails } from './json-log-worker-req.ts';
import type { FileImpl } from './file-impl-interface.ts';
import { FileImplGet, readTextFile, writeTextFile } from './file-impl.ts';
import {
FileImplGet,
mkdir,
readDir,
readFile,
readTextFile,
writeFile,
writeTextFile,
} from './file-impl.ts';
import { isNode } from '../common.ts';

const FILE_READ_BUF_SIZE_BYTES = 1024 * 1024;
Expand Down Expand Up @@ -774,6 +787,34 @@ async function handleRequest(
break;
}

case 'readFile': {
const bytes = await readFile(req.path);
const resp: WorkerReadFileResp = { type: 'readFile', id: req.id, bytes };
postWithLogs(resp, [bytes.buffer as ArrayBuffer]);
break;
}

case 'writeFile': {
await writeFile(req.path, req.bytes);
const resp: WorkerWriteFileResp = { type: 'writeFile', id: req.id };
postWithLogs(resp);
break;
}

case 'mkdir': {
await mkdir(req.path);
const resp: WorkerMkdirResp = { type: 'mkdir', id: req.id };
postWithLogs(resp);
break;
}

case 'readDir': {
const entries = await readDir(req.path);
const resp: WorkerReadDirResp = { type: 'readDir', id: req.id, entries };
postWithLogs(resp);
break;
}

case 'readTextFile': {
const resp: WorkerReadTextFileResp = {
type: 'readTextFile',
Expand Down Expand Up @@ -809,13 +850,27 @@ async function handleRequest(
type: 'error',
id: (req as WorkerFileReq).id,
error: 'UnknownCommand',
details: unknownCommandDetails((req as WorkerFileReq).type),
};
postWithLogs(resp);
break;
}
}
}

function workerErrorDetails(error: unknown): WorkerErrorResp['details'] {
const err = error as
| { message?: unknown; name?: unknown; code?: unknown }
| null;
return {
message: typeof err?.message === 'string' ? err.message : String(error),
name: typeof err?.name === 'string' ? err.name : 'Error',
code: typeof err?.code === 'string' || typeof err?.code === 'number'
? err.code
: undefined,
};
}

export function jsonLogWorkerMain(): void {
// Redirect all log() calls in the worker to the collector so entries are
// relayed to the main thread via responses rather than going to console.
Expand All @@ -827,14 +882,15 @@ export function jsonLogWorkerMain(): void {
processingChain = processingChain.then(
() => handleRequest(event),
() => handleRequest(event),
).catch((_err) => {
).catch((err: unknown) => {
const req = isNode()
? (event as WorkerFileReq)
: (event as MessageEvent<WorkerFileReq>).data;
const resp: WorkerErrorResp = {
type: 'error',
id: req.id,
error: 'InternalError',
details: workerErrorDetails(err),
};
postWithLogs(resp as unknown as Record<string, unknown>);
});
Expand Down
Loading
Loading