Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
its checksum tagged it as a `Date`, marking the delta commit as corrupted on
reload. Commits written by earlier builds with such values stay corrupted and
must be re-created.
- Reading a directory now fails with `EISDIR` on Deno, Node.js, and the
browser instead of returning empty bytes on Windows.
- JSON decoders use a disposable lifetime. Parent disposal releases owned
children and preserves children acquired by another owner. Disposed decoders
reject reads until acquired again. `finalize()` remains a disposal alias.
Expand Down
13 changes: 10 additions & 3 deletions base/json-log/file-impl-deno.ts
Original file line number Diff line number Diff line change
@@ -1,16 +1,23 @@
import * as path from '../path.ts';
import type { DirEntry, FileImpl } from './file-impl-interface.ts';
import { eisdirError } from './file-impl-errors.ts';

export const FileImplDeno: FileImpl<Deno.FsFile> = {
async open(filePath, write) {
// try {
await Deno.mkdir(path.dirname(filePath), { recursive: true });
// } catch (_: unknown) {}
return Deno.open(filePath, {
const file = await Deno.open(filePath, {
read: true,
write,
create: write,
});
// A directory must never open as a file. Windows reports a directory's
// length as 0, so without this guard readFile() returns empty bytes
// instead of surfacing EISDIR.
if ((await file.stat()).isDirectory) {
file.close();
throw eisdirError(filePath);
}
return file;
},

seek(handle, offset, from) {
Expand Down
17 changes: 17 additions & 0 deletions base/json-log/file-impl-errors.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
/**
* Filesystem error identity shared by every FileImpl.
*
* WHY: reading a directory must fail on every runtime. Windows reports a
* directory's length as 0 (Win32 file size describes a data stream), so the
* size-driven read loop never fires and the OS never raises EISDIR. The impls
* therefore reject directories at open() and throw this normalized error, so
* callers and the worker error transport (which preserves name + code) see one
* stable identity. Regression contracts: worker-boundary identity in
* tests/json-log-worker-io.test.ts; per-impl behavior in tests/file-impl.test.ts.
*/
export function eisdirError(path: string): Error {
return Object.assign(
new Error(`EISDIR: illegal operation on a directory, open '${path}'`),
{ code: 'EISDIR' },
);
}
4 changes: 4 additions & 0 deletions base/json-log/file-impl-interface.ts
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,10 @@ export interface FileImpl<T> {
* @param path The path to the file
* @param write Whether to open the file for writing
* @returns A promise that resolves to a platform-specific file handle
* @throws A read open of a directory rejects with code 'EISDIR'. A directory
* is never a valid read target; Windows reports its length as 0, so
* the size-driven read loop would return empty bytes. A write open
* surfaces the platform error unchanged (Windows may reject earlier).
*/
open(path: string, write: boolean): Promise<T>;

Expand Down
36 changes: 20 additions & 16 deletions base/json-log/file-impl-node.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,32 +2,36 @@ import * as path from '../path.ts';
import * as fs from 'node:fs/promises';
import * as os from 'node:os';
import type { DirEntry, FileImpl } from './file-impl-interface.ts';
import { eisdirError } from './file-impl-errors.ts';

interface NodeFsFile {
file: fs.FileHandle;
pos: number;
}

/** Opens writable, falling back to create when the file does not exist yet. */
async function openWritable(filePath: string): Promise<fs.FileHandle> {
try {
return await fs.open(filePath, 'r+');
} catch (_: unknown) {
return await fs.open(filePath, 'w+');
}
}

export const FileImplNode: FileImpl<NodeFsFile> = {
async open(filePath, write) {
await fs.mkdir(path.dirname(filePath), { recursive: true });
if (write) {
try {
return {
file: await fs.open(filePath, 'r+'),
pos: 0,
};
} catch (_: unknown) {
return {
file: await fs.open(filePath, 'w+'),
pos: 0,
};
}
const file = write
? await openWritable(filePath)
: await fs.open(filePath, 'r');
// A directory must never open as a file. Windows reports a directory's
// length as 0, so without this guard readFile() returns empty bytes
// instead of surfacing EISDIR.
if ((await file.stat()).isDirectory()) {
await file.close();
throw eisdirError(filePath);
}
return {
file: await fs.open(filePath, 'r'),
pos: 0,
};
return { file, pos: 0 };
},

async seek(handle, offset, from) {
Expand Down
17 changes: 14 additions & 3 deletions base/json-log/file-impl-opfs.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import * as path from '../path.ts';
import type { DirEntry, FileImpl } from './file-impl-interface.ts';
import { retry, TryAgain } from '../time.ts';
import { log } from '../../logging/log.ts';
import { eisdirError } from './file-impl-errors.ts';

interface FileSystemSyncAccessHandle {
close(): void;
Expand Down Expand Up @@ -66,9 +67,19 @@ export async function clearOPFS(): Promise<void> {
export const FileImplOPFS: FileImpl<OPFSFile> = {
async open(filePath, write) {
const dir = await getDir(path.dirname(filePath));
const handle = await dir.getFileHandle(path.basename(filePath), {
create: write,
});
let handle: FileSystemFileHandle;
try {
handle = await dir.getFileHandle(path.basename(filePath), {
create: write,
});
} catch (e: unknown) {
// OPFS refuses to hand out a file handle for a directory. Normalize its
// DOMException to the shared EISDIR identity so all runtimes agree.
if (e instanceof DOMException && e.name === 'TypeMismatchError') {
throw eisdirError(filePath);
}
throw e;
}

// For benchmarks and tests that rapidly close and reopen files,
// we need to handle the race condition where the browser hasn't
Expand Down
169 changes: 126 additions & 43 deletions base/json-log/json-log.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@
* code to work seamlessly in both environments.
*/
import { kStaticAssetsSystem } from '../../system-assets/system-assets.ts';
import { assert } from '../error.ts';
import type {
WorkerErrorResp,
WorkerFileReq,
Expand Down Expand Up @@ -41,23 +40,16 @@ import type { LogEntry } from '../../logging/log.ts';

let gWorker: Worker | NodeWorker | undefined;

type NodeWorker =
& NodeWorkerOnError
& NodeWorkerOnMessage
& NodeWorkerPostMessage;

type NodeWorkerOnMessage = {
on: (
event: 'message',
handler: (event: WorkerFileResp) => void,
) => void;
};
type NodeWorkerOnError = {
on: (event: 'error', handler: (error: unknown) => void) => void;
};

type NodeWorkerPostMessage = {
postMessage: (message: unknown) => void;
/**
* Minimal Node worker surface used here. Node's Worker is an EventEmitter with
* these events, while a browser Worker exposes properties instead.
*/
type NodeWorker = {
on(event: 'message', handler: (event: WorkerFileResp) => void): void;
on(event: 'error', handler: (error: unknown) => void): void;
on(event: 'exit', handler: (code: number) => void): void;
postMessage(message: unknown): void;
terminate(): void;
};

/**
Expand All @@ -74,27 +66,98 @@ type NodeWorkerPostMessage = {
export async function startJSONLogWorkerIfNeeded(): Promise<
Worker | NodeWorker
> {
if (gWorker === undefined) {
const workerJs = new TextDecoder().decode(
kStaticAssetsSystem['/system-assets/json-log-worker.js'].data,
if (gWorker !== undefined) return gWorker;
const workerJs = new TextDecoder().decode(
kStaticAssetsSystem['/system-assets/json-log-worker.js'].data,
);
const worker = getRuntime().createWorker(workerJs) as Worker | NodeWorker;
// WHY publish after wiring: an unwatched worker would hang every request it
// owns if it failed.
watchWorker(worker);
gWorker = worker;
return worker;
}

/**
* Fails every in-flight request and drops the worker, so the next call spawns
* a fresh one.
*
* WHY: a dead worker never answers. Without this, a request would wait forever
* and a DB open would never settle. Rejecting with the cause keeps the failure
* visible; clearing the handle lets a later request recover instead of talking
* to a corpse.
*/
function failWorker(error: Error): void {
const worker = gWorker;
gWorker = undefined;
const pending = [...gPendingResolveFuncs.values()];
gPendingResolveFuncs.clear();
for (const entry of pending) entry.reject(error);
// Handles and cursors belong to the dead worker; the next open recreates them.
gPendingFileOperations.clear();
gPendingScanPromise.clear();
gCursorToFile.clear();
if (worker === undefined) return;
try {
worker.terminate();
} catch {
// WHY best effort: termination is cleanup, not the error path.
}
}

/** Logs a worker failure and returns the error pending requests receive. */
function workerFailure(kind: string, cause: unknown): Error {
const error = cause instanceof Error
? cause
: new Error(describeCause(cause));
log({
severity: 'ERROR',
error: 'JsonLogWorkerError',
message: `JSON log worker ${kind}: ${error.message}`,
trace: error.stack,
});
return error;
}

/** Extracts text from Errors, ErrorEvents, and plain values. */
function describeCause(cause: unknown): string {
const message = (cause as { message?: unknown } | undefined)?.message;
return typeof message === 'string' && message !== ''
? message
: String(cause);
}

/** Attaches the fatal-failure wiring for the worker kind in use. */
function watchWorker(worker: Worker | NodeWorker): void {
// WHY ignore stale workers: terminating a worker emits its error/exit events
// asynchronously, after a replacement may already be serving requests.
// Failing then would kill the healthy worker and reject its requests.
const fail = (kind: string, cause: unknown) => {
if (gWorker !== worker) return;
failWorker(workerFailure(kind, cause));
};
if (getRuntime().id === 'node') {
const node = worker as NodeWorker;
node.on('message', (resp) => handleResponse(resp));
node.on('error', (cause) => fail('error', cause));
// WHY exit too: a worker that dies silently emits only 'exit', and its
// pending requests would otherwise never settle.
node.on(
'exit',
(code) => fail('exit', new Error(`exited with code ${code}`)),
);
gWorker = getRuntime().createWorker(workerJs) as Worker | NodeWorker;

if (getRuntime().id === 'node') {
(gWorker as NodeWorker).on('message', handleResponse);
(gWorker as NodeWorker).on('error', (error: unknown) => {
console.error(error);
});
} else {
(gWorker as Worker).onmessage = (
event: MessageEvent<WorkerFileResp>,
) => handleResponse(event.data);
(gWorker as Worker).onmessageerror = (event) => {
console.error(event);
};
}
return;
}
return gWorker;
const web = worker as Worker;
web.onmessage = (event: MessageEvent<WorkerFileResp>) =>
handleResponse(event.data);
web.onerror = (cause) => fail('error', cause);
web.onmessageerror = (cause) => fail('messageerror', cause);
}

/** Simulates a fatal worker failure. Test-only hook. @internal */
export function failJSONLogWorkerForTests(error: Error): void {
failWorker(error);
}

/**
Expand Down Expand Up @@ -150,11 +213,21 @@ async function sendRequest<T extends WorkerFileReq>(
resolve,
reject,
});
const worker = await startJSONLogWorkerIfNeeded();
worker.postMessage({
...req,
id,
});
try {
const worker = await startJSONLogWorkerIfNeeded();
worker.postMessage({
...req,
id,
});
} catch (cause) {
// WHY: a worker that cannot start, or a message that cannot be cloned,
// must reject this request; the caller would otherwise wait forever.
const entry = gPendingResolveFuncs.get(id);
if (entry !== undefined) {
gPendingResolveFuncs.delete(id);
entry.reject(workerFailure('send', cause));
}
}
return promise as Promise<WorkerFileRespForReq<T>>;
}

Expand All @@ -179,7 +252,17 @@ function handleResponse(resp: WorkerFileResp): void {
}
}
const entry = gPendingResolveFuncs.get(resp.id);
assert(entry !== undefined, 'Received unknown response from worker');
if (entry === undefined) {
// WHY warn and drop: a response can arrive after its request was failed by
// a worker restart. Asserting here would turn a recoverable race into a
// crash of the whole process.
log({
severity: 'WARNING',
error: 'UnknownWorkerResponse',
message: `Ignoring response ${resp.id} for an unknown request`,
});
return;
}
gPendingResolveFuncs.delete(resp.id);
if (resp.type === 'error') {
entry.reject(workerResponseError(resp));
Expand Down
5 changes: 4 additions & 1 deletion logging/errors.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,13 +12,15 @@ import type { EmailType } from './metrics.ts';
* impossible
* - FailedAssertion: Used when an assertion check fails
* - UncaughtServerError: Used for unhandled exceptions on the server
* - JsonLogWorkerError: Used when the JSON log worker fails or dies
* - MissingConfiguration: Used when required configuration options are missing
*/
export type DeveloperError =
| 'NotImplemented'
| 'NotReached'
| 'FailedAssertion'
| 'UncaughtServerError';
| 'UncaughtServerError'
| 'JsonLogWorkerError';

/**
* Base interface for all error log entries. Extends BaseLogEntry with an
Expand Down Expand Up @@ -69,6 +71,7 @@ export type OperationalError =
| 'EmailSendFailed'
| 'MissingConfiguration'
| 'StorageError'
| 'UnknownWorkerResponse'
| 'ValidationError';

/**
Expand Down
2 changes: 1 addition & 1 deletion system-assets/assets.json

Large diffs are not rendered by default.

Loading
Loading