Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
62 changes: 48 additions & 14 deletions src/web/blog-refresh.ts
Original file line number Diff line number Diff line change
Expand Up @@ -307,7 +307,8 @@ async function getJsonList(url: string): Promise<ContentsEntry[]> {
if (!Array.isArray(raw)) throw new Error("contents listing is not a JSON array");
return raw as ContentsEntry[];
} finally {
result.stopIdleTimer(); // body fully consumed (fetch-util #552 contract)
await releaseUnreadBody(result.response);
result.stopIdleTimer();
}
}

Expand All @@ -322,11 +323,12 @@ async function fetchPlainText(url: string, account?: (bytes: number) => void): P
if (declared !== undefined && declared > MAX_TEXT_BYTES) {
throw new Error(`text document exceeds ${MAX_TEXT_BYTES} bytes (content-length ${declared}): ${url}`);
}
const buf = Buffer.from(await result.response.arrayBuffer());
if (buf.byteLength > MAX_TEXT_BYTES) throw new Error(`text document exceeds ${MAX_TEXT_BYTES} bytes: ${url}`);
const buf = await readCappedBytes(result.response.body, MAX_TEXT_BYTES);
if (buf === undefined) throw new Error(`text document exceeds ${MAX_TEXT_BYTES} bytes: ${url}`);
account?.(buf.byteLength);
return buf.toString("utf8");
} finally {
await releaseUnreadBody(result.response);
result.stopIdleTimer();
}
}
Expand All @@ -343,15 +345,12 @@ async function fetchApiText(rel: string, account?: (bytes: number) => void): Pro
if (typeof data.content !== "string") {
throw new Error(`contents API omitted the body (>1 MiB cap): ${rel}`);
}
// Size gate BEFORE decoding: base64 inflates payloads by ~4/3.
if (Math.ceil(data.content.length * 3 / 4) > MAX_TEXT_BYTES) {
throw new Error(`text document exceeds ${MAX_TEXT_BYTES} bytes: ${rel}`);
}
const buf = Buffer.from(data.content, "base64");
if (buf.byteLength > MAX_TEXT_BYTES) throw new Error(`text document exceeds ${MAX_TEXT_BYTES} bytes: ${rel}`);
const buf = decodeCappedBase64(data.content, MAX_TEXT_BYTES);
if (buf === undefined) throw new Error(`text document exceeds ${MAX_TEXT_BYTES} bytes: ${rel}`);
account?.(buf.byteLength);
return buf.toString("utf8");
} finally {
await releaseUnreadBody(result.response);
result.stopIdleTimer();
}
}
Expand Down Expand Up @@ -394,11 +393,12 @@ async function fetchPlainImage(url: string, account?: (bytes: number) => void):
if (!result.response.ok) throw new Error(`image GET ${url} → HTTP ${result.response.status}`);
const declared = contentLengthOf(result.response.headers);
if (declared !== undefined && declared > MAX_IMAGE_BYTES) return undefined; // never download the body
const buf = Buffer.from(await result.response.arrayBuffer());
if (buf.byteLength === 0 || buf.byteLength > MAX_IMAGE_BYTES) return undefined;
const buf = await readCappedBytes(result.response.body, MAX_IMAGE_BYTES);
if (buf === undefined || buf.byteLength === 0) return undefined;
account?.(buf.byteLength);
return { buf, type };
} finally {
await releaseUnreadBody(result.response);
result.stopIdleTimer();
}
}
Expand All @@ -419,16 +419,50 @@ async function fetchApiImage(rel: string, account?: (bytes: number) => void): Pr
if (!result.response.ok) throw new Error(`contents image GET ${url} → HTTP ${result.response.status}`);
const data = (await result.response.json()) as { content?: unknown };
if (typeof data.content !== "string") return undefined; // >1 MiB omission → next source
if (Math.ceil(data.content.length * 3 / 4) > MAX_IMAGE_BYTES) return undefined;
const buf = Buffer.from(data.content, "base64");
if (buf.byteLength === 0 || buf.byteLength > MAX_IMAGE_BYTES) return undefined;
const buf = decodeCappedBase64(data.content, MAX_IMAGE_BYTES);
if (buf === undefined || buf.byteLength === 0) return undefined;
account?.(buf.byteLength);
return { buf, type };
} finally {
await releaseUnreadBody(result.response);
result.stopIdleTimer();
}
}

async function releaseUnreadBody(response: Response): Promise<void> {
if (!response.bodyUsed) await response.body?.cancel().catch(() => {});
}

function decodeCappedBase64(content: string, maxBytes: number): Buffer | undefined {
const compact = content.replace(/\s/g, "");
if (Buffer.byteLength(compact, "base64") > maxBytes) return undefined;
const buf = Buffer.from(compact, "base64");
return buf.byteLength > maxBytes ? undefined : buf;
}

async function readCappedBytes(body: Response["body"], maxBytes: number): Promise<Buffer | undefined> {
if (body === null) return Buffer.alloc(0);
const reader = body.getReader();
const chunks: Uint8Array[] = [];
let total = 0;
try {
for (;;) {
const { done, value } = await reader.read();
if (done) break;
if (value.byteLength === 0) continue;
if (value.byteLength > maxBytes - total) {
await reader.cancel().catch(() => {});
return undefined;
}
total += value.byteLength;
chunks.push(value);
}
} finally {
reader.releaseLock();
}
return Buffer.concat(chunks, total);
}

/** Image fetch through the fallback chain. Misses (true 404s / >1 MiB API
* omissions) keep walking — a file absent on one host is absent nowhere we
* look, and walking costs bounded calls; the chain exhausts to a MISS
Expand Down
106 changes: 106 additions & 0 deletions tests/web-blog-refresh.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -226,6 +226,30 @@ test("raw unreachable → the contents-API chain serves the whole round", async
"first-party hosts only: no third-party host was ever contacted");
});

for (const lineWrapped of [false, true]) {
for (const delta of [-1, 0, 1]) {
test(`API text byte cap counts decoded bytes (${lineWrapped ? "wrapped" : "plain"} base64, delta ${delta})`, async () => {
const prefix = "# Fresh\n\n";
const text = prefix + "x".repeat(512 * 1024 + delta - prefix.length);
const encoded = Buffer.from(text).toString("base64");
const content = lineWrapped ? encoded.replace(/.{1,60}/g, "$&\n") : encoded;
router = (url) => {
if (url === CONTENTS_URL) return new Response(JSON.stringify([{ name: "2026-10-09_fresh.md", type: "file" }]));
if (url.startsWith("https://raw.githubusercontent.com/")) throw new Error("raw down");
if (url === apiFileUrl(`${POSTS_REL}/2026-10-09_fresh.md`)) {
return new Response(JSON.stringify({ content, encoding: "base64" }));
}
return new Response("nf", { status: 404 });
};

await runBlogRefreshRound({ force: true });

assert.equal(getBlogFeed().error, delta > 0 ? "offline" : undefined);
assert.equal(getBlogFeed().items.some((item) => item.id === "fresh"), delta <= 0);
});
}
}

test("raw AND contents-API dead (rate-limited) → the round fails soft with the offline marker", async () => {
router = (url) => {
if (url === CONTENTS_URL) return new Response(CHAIN_LISTING, { status: 200 }); // listing still reachable
Expand Down Expand Up @@ -536,6 +560,88 @@ test("#4906 P3 lying-small content-length: the post-read byte check stays the au
assert.equal(getBlogFeed().error, "offline", "header said 10 bytes; the real 512KB+1 body still blew the limit");
});

function chunkedBody(chunkSize: number, chunkCount: number, headers?: Record<string, string>): {
response: Response;
reads(): number;
cancelled(): boolean;
} {
let reads = 0;
let cancelled = false;
const body = new ReadableStream<Uint8Array>({
pull(controller) {
if (reads === chunkCount) {
controller.close();
return;
}
reads++;
controller.enqueue(Buffer.alloc(chunkSize, "x"));
},
cancel() { cancelled = true; },
}, { highWaterMark: 0 });
return { response: new Response(body, { headers }), reads: () => reads, cancelled: () => cancelled };
}

for (const headers of [undefined, { "content-length": "10" }]) {
test(`text download stops at its byte cap with ${headers === undefined ? "no" : "understated"} content-length`, async () => {
installGithubRouter();
await runBlogRefreshRound({ force: true });
const body = chunkedBody(64 * 1024, 32, headers);
router = (url) => {
if (url === CONTENTS_URL) return new Response(JSON.stringify([{ name: "2026-10-09_fresh.md", type: "file" }]));
if (url === rawUrl("website/blog/posts/2026-10-09_fresh.md")) return body.response;
return new Response("not here", { status: 404 });
};

await runBlogRefreshRound({ force: true });

assert.equal(body.cancelled(), true, "oversized body is canceled before EOF");
assert.ok(body.reads() <= 10, `expected the cap plus one prefetched chunk, read ${body.reads()}`);
assert.equal(getBlogFeed().error, "offline");
assert.ok(getBlogFeed().items.some((item) => item.id === "demo"), "the last good overlay survives");
});
}

test("an oversized streamed image is canceled and remains a clean miss", async () => {
installGithubRouter();
const base = router;
const body = chunkedBody(256 * 1024, 32);
router = (url) => url === rawUrl("website/blog/posts/figures/good.png") ? body.response : base(url);

await runBlogRefreshRound({ force: true });

assert.equal(body.cancelled(), true);
assert.ok(body.reads() <= 10, `expected the cap plus one prefetched chunk, read ${body.reads()}`);
assert.equal(getBlogFeed().error, undefined);
assert.equal(getRemoteBlogAsset("website/blog/posts/figures/good.png"), undefined);
assert.doesNotMatch(getBlogFeed().items.find((item) => item.id === "fresh")?.html ?? "", /good\.png/);
});

test("a streamed image exactly at the byte cap is retained", async () => {
installGithubRouter();
const base = router;
const body = chunkedBody(256 * 1024, 8);
router = (url) => url === rawUrl("website/blog/posts/figures/good.png") ? body.response : base(url);

await runBlogRefreshRound({ force: true });

assert.equal(body.cancelled(), false);
assert.equal(getBlogFeed().error, undefined);
assert.equal(getRemoteBlogAsset("website/blog/posts/figures/good.png")?.buf.byteLength, 2 * 1024 * 1024);
});

test("content-length rejection cancels the unread image body", async () => {
installGithubRouter();
const base = router;
const body = chunkedBody(256 * 1024, 32, { "content-length": String(3 * 1024 * 1024) });
router = (url) => url === rawUrl("website/blog/posts/figures/good.png") ? body.response : base(url);

await runBlogRefreshRound({ force: true });

assert.equal(body.cancelled(), true);
assert.ok(body.reads() <= 1, "only the fetch wrapper's prefetch may run");
assert.equal(getBlogFeed().error, undefined);
});

test("#4906 P3 settleRoundImmune: a rejecting round never leaks an unhandledRejection", async () => {
let unhandled: unknown = undefined;
const onUn = (e: unknown) => { unhandled = e; };
Expand Down
Loading