Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
64 changes: 64 additions & 0 deletions scripts/bench-message-content-read.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
import assert from "node:assert/strict";
import { createHash } from "node:crypto";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { performance } from "node:perf_hooks";
import { renderSessionJsonlFromTables } from "../src/export.ts";
import { SqliteBackend } from "../src/storage/backend.ts";
import { UnifiedStore, type IngestSessionInput } from "../src/storage/store.ts";

const count = Number(process.argv[2] ?? 10000);
assert.ok(Number.isSafeInteger(count) && count > 0);
const dir = mkdtempSync(join(tmpdir(), "message-content-read-"));
const store = UnifiedStore.open(join(dir, "index.db"), "node:sqlite");
try {
const input: IngestSessionInput = {
session: {
sessionId: "benchmark", createdAt: 1, updatedAt: 2, nextBlockId: 1, nextRunId: 1,
metaJson: "{}", statsJson: "{}", metadataJson: "{}", kstateJson: "{}", ccrJson: null,
tailJson: JSON.stringify({ tailSeqs: Array.from({ length: count }, (_, seq) => seq) }),
},
messages: Array.from({ length: count }, (_, seq) => ({
seq, msgId: `message-${seq}`, ref: `m${seq + 1}`, refNum: seq + 1, role: "user", kind: "text", ts: 1000 + seq,
content: JSON.stringify({ id: `message-${seq}`, role: "user", contentType: "text", text: `${seq}: ${"stored original 日本語 😀 ".repeat(20)}` }),
})),
blocks: [], deadRefs: [], refs: [], ccrEntries: [],
};
store.ingestSession(input);
const selected = input.messages.filter((row) => row.seq % 2 === 1).map((row) => row.seq).reverse();
const covered = Array.from({ length: Math.ceil(count / 100) }, (_, i) => [i * 100, Math.min(i * 100 + 49, count - 1)] as [number, number]);
const uncovered = input.messages.filter((row) => row.seq % 100 >= 50).length;
const backend = new SqliteBackend(store);
const workloads: { name: string; rows: number; run(): unknown | Promise<unknown> }[] = [
{ name: "getMessages", rows: count, run: () => store.getMessages("benchmark") },
{ name: "getMessagesAt", rows: selected.length, run: () => store.getMessagesAt("benchmark", selected) },
{ name: "uncoveredRows", rows: uncovered, run: () => store.uncoveredRows("benchmark", covered) },
{ name: "jsonlExport", rows: count, run: () => renderSessionJsonlFromTables(backend, "benchmark", false) },
];
const measurements = [];
for (const workload of workloads) {
await workload.run();
const cpuStart = process.cpuUsage();
const started = performance.now();
const result = await workload.run();
const wallMs = performance.now() - started;
const cpu = process.cpuUsage(cpuStart);
const serialized = typeof result === "string" ? result : JSON.stringify(result);
if (typeof result === "string") {
const records = result.trimEnd().split("\n").map((line) => JSON.parse(line) as { type: string; message?: unknown });
assert.equal(records.filter((row) => row.type === "message").length, workload.rows);
assert.deepEqual(records.at(-1)!.message, JSON.parse(input.messages.at(-1)!.content));
} else {
assert.ok(Array.isArray(result));
assert.equal(result.length, workload.rows);
for (const row of result as { seq: number; content: string }[]) assert.equal(row.content, input.messages[row.seq]!.content);
}
measurements.push({ name: workload.name, rows: workload.rows, wallMs, cpuMs: (cpu.user + cpu.system) / 1000,
outputSha256: createHash("sha256").update(serialized).digest("hex") });
}
process.stdout.write(`${JSON.stringify({ count, warmups: 1, measurements })}\n`);
} finally {
store.close();
rmSync(dir, { recursive: true, force: true });
}
44 changes: 28 additions & 16 deletions src/storage/mysql.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1563,7 +1563,8 @@ export class MysqlBackend implements StorageBackend {
const chunk = [...new Set(pointers.slice(i, i + CHUNK))];
if (chunk.length === 0) continue;
const [rows] = (await this.pool.query(
`SELECT seq, msg_id, ref, ref_num, role, kind, content_hash, ts FROM messages
`SELECT seq, msg_id, ref, ref_num, role, kind, content_hash, ts, c.bytes AS content_bytes FROM messages
LEFT JOIN content c ON c.hash = messages.content_hash
WHERE session_id = ? AND retired_at IS NULL AND seq IN (${chunk.map(() => "?").join(", ")})`,
[sessionId, ...chunk],
)) as unknown as Row[][];
Expand All @@ -1578,7 +1579,7 @@ export class MysqlBackend implements StorageBackend {
kind: str(r.kind),
contentHash,
ts: nullableNum(r.ts),
content: await this.hydrateMediaAsync((await this.getContent(this.pool, contentHash)) ?? "", this.pool),
content: await this.hydrateMediaAsync(str(r.content_bytes), this.pool),
});
}
}
Expand Down Expand Up @@ -1606,20 +1607,31 @@ export class MysqlBackend implements StorageBackend {
[sessionId, ...(opts?.seqs ?? [])],
)) as unknown as Row[][];
const out: (MessageRow & { contentHash: string; retiredAt: number | null })[] = [];
for (const r of rows) {
const contentHash = str(r.content_hash);
out.push({
seq: num(r.seq),
msgId: str(r.msg_id),
ref: nullableStr(r.ref),
refNum: nullableNum(r.ref_num),
role: str(r.role),
kind: str(r.kind),
contentHash,
retiredAt: nullableNum(r.retired_at),
ts: nullableNum(r.ts),
content: await this.hydrateMediaAsync((await this.getContent(conn, contentHash)) ?? "", conn),
});
// Keep body batches bounded even when the selected metadata spans a
// whole historian log. Every query stays on the caller's connection.
const CHUNK = 500;
for (let i = 0; i < rows.length; i += CHUNK) {
const chunk = rows.slice(i, i + CHUNK);
const hashes = [...new Set(chunk.map((row) => str(row.content_hash)))];
const [contents] = (await conn.query(
`SELECT hash, bytes FROM content WHERE hash IN (${hashes.map(() => "?").join(",")})`, hashes,
)) as unknown as Row[][];
const byHash = new Map(contents.map((row) => [str(row.hash), str(row.bytes)]));
for (const r of chunk) {
const contentHash = str(r.content_hash);
out.push({
seq: num(r.seq),
msgId: str(r.msg_id),
ref: nullableStr(r.ref),
refNum: nullableNum(r.ref_num),
role: str(r.role),
kind: str(r.kind),
contentHash,
retiredAt: nullableNum(r.retired_at),
ts: nullableNum(r.ts),
content: await this.hydrateMediaAsync(byHash.get(contentHash) ?? "", conn),
});
}
}
return out;
}
Expand Down
33 changes: 26 additions & 7 deletions src/storage/store.ts
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ export function rewriteArchiveLog(file: string, text: string): void {
}
}
/** #4739 test instrumentation: message rows materialized by getMessages /
* getMessagesAt (each row = one CAS read + media hydration). Production
* getMessagesAt (including content decoding + media hydration). Production
* code never reads this; tests assert ordinary loads stop decoding the
* whole historical log. */
export const messageRowReads = { count: 0 };
Expand Down Expand Up @@ -3340,7 +3340,7 @@ export class UnifiedStore {
+ ` AND (${clauses.join(" OR ")}) ORDER BY seq`;
for (const row of this.db.prepare(sql).all(...params) as Record<string, unknown>[]) rows.push(row);
}
return rows.map((r) => ({
return Array.from(this.messageContents(rows), ([r, content]) => ({
seq: r.seq as number,
msgId: r.msg_id as string,
ref: (r.ref as string | null) ?? null,
Expand All @@ -3349,10 +3349,28 @@ export class UnifiedStore {
kind: r.kind as string,
contentHash: r.content_hash as string,
ts: (r.ts as number | null) ?? null,
content: this.hydrateMedia(this.getContent(r.content_hash as string) ?? ""),
content: this.hydrateMedia(content),
}));
}

/** Fetch bodies for captured message rows in bounded batches. Keeping
* the metadata query separate avoids holding a whole history's BLOBs
* alongside all of its decoded strings. Hashes preserve row identity. */
private *messageContents(rows: readonly Record<string, unknown>[]): Generator<readonly [Record<string, unknown>, string]> {
const CHUNK = 500;
for (let i = 0; i < rows.length; i += CHUNK) {
const chunk = rows.slice(i, i + CHUNK);
const hashes = [...new Set(chunk.map((row) => row.content_hash as string))];
const contents = this.db.prepare(`SELECT hash, bytes FROM content WHERE hash IN (${hashes.map(() => "?").join(",")})`)
.all(...hashes) as { hash: string; bytes: Uint8Array }[];
const byHash = new Map(contents.map((row) => [row.hash, row.bytes]));
for (const row of chunk) {
const bytes = byHash.get(row.content_hash as string);
yield [row, bytes === undefined ? "" : Buffer.from(bytes).toString("utf8")];
}
}
}

getMessages(sessionId: string, opts?: { includeRetired?: boolean; seqs?: readonly number[]; fromSeqExclusive?: number }): (MessageRow & { contentHash: string; retiredAt: number | null })[] {
// #4390: retired rows (rewound/undo history) stay out of the unified
// replay by default — they are history, not view. Callers that need
Expand All @@ -3371,7 +3389,7 @@ export class UnifiedStore {
`SELECT seq, msg_id, ref, ref_num, role, kind, content_hash, retired_at, ts FROM messages WHERE session_id = ?${retiredFilter}${seqFilter}${afterClause} ORDER BY seq`,
)
.all(...params) as Record<string, unknown>[];
return rows.map((r) => {
return Array.from(this.messageContents(rows), ([r, content]) => {
messageRowReads.count++;
return {
seq: r.seq as number,
Expand All @@ -3383,7 +3401,7 @@ export class UnifiedStore {
contentHash: r.content_hash as string,
retiredAt: (r.retired_at as number | null) ?? null,
ts: (r.ts as number | null) ?? null,
content: this.hydrateMedia(this.getContent(r.content_hash as string) ?? ""),
content: this.hydrateMedia(content),
};
});
}
Expand All @@ -3403,7 +3421,8 @@ export class UnifiedStore {
const chunk = unique.slice(i, i + CHUNK);
const rows = this.db
.prepare(
`SELECT seq, msg_id, ref, ref_num, role, kind, content_hash, ts FROM messages
`SELECT seq, msg_id, ref, ref_num, role, kind, content_hash, ts, c.bytes AS content_bytes FROM messages
LEFT JOIN content c ON c.hash = messages.content_hash
WHERE session_id = ? AND retired_at IS NULL AND seq IN (${chunk.map(() => "?").join(", ")})`,
)
.all(sessionId, ...chunk) as Record<string, unknown>[];
Expand All @@ -3418,7 +3437,7 @@ export class UnifiedStore {
kind: r.kind as string,
contentHash: r.content_hash as string,
ts: (r.ts as number | null) ?? null,
content: this.hydrateMedia(this.getContent(r.content_hash as string) ?? ""),
content: this.hydrateMedia(r.content_bytes === null ? "" : Buffer.from(r.content_bytes as Uint8Array).toString("utf8")),
});
}
}
Expand Down
2 changes: 1 addition & 1 deletion tests/issue4739-storage-read-snapshot.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,7 @@ test("mysql snapshot pins SQL, CAS and media hydration to its read-only connecti
if (sql.includes("FROM content")) {
const bytes = values?.[0] === "msg" ? JSON.stringify({ imageBase64: { $media: mediaHash } })
: values?.[0] === mediaHash ? "image-bytes" : '{"text":"summary","count":1}';
return [[{ bytes }]];
return [[{ hash: values?.[0], bytes }]];
}
return [[]];
},
Expand Down
Loading
Loading