diff --git a/src/web/blog-refresh.ts b/src/web/blog-refresh.ts index 06fadcedf4..4f5ae9951e 100644 --- a/src/web/blog-refresh.ts +++ b/src/web/blog-refresh.ts @@ -307,7 +307,8 @@ async function getJsonList(url: string): Promise { if (!Array.isArray(raw)) throw new Error("contents listing is not a JSON array"); return raw as ContentsEntry[]; } finally { - result.stopIdleTimer(); // body fully consumed (fetch-util #552 contract) + await releaseUnreadBody(result.response); + result.stopIdleTimer(); } } @@ -322,11 +323,12 @@ async function fetchPlainText(url: string, account?: (bytes: number) => void): P if (declared !== undefined && declared > MAX_TEXT_BYTES) { throw new Error(`text document exceeds ${MAX_TEXT_BYTES} bytes (content-length ${declared}): ${url}`); } - const buf = Buffer.from(await result.response.arrayBuffer()); - if (buf.byteLength > MAX_TEXT_BYTES) throw new Error(`text document exceeds ${MAX_TEXT_BYTES} bytes: ${url}`); + const buf = await readCappedBytes(result.response.body, MAX_TEXT_BYTES); + if (buf === undefined) throw new Error(`text document exceeds ${MAX_TEXT_BYTES} bytes: ${url}`); account?.(buf.byteLength); return buf.toString("utf8"); } finally { + await releaseUnreadBody(result.response); result.stopIdleTimer(); } } @@ -343,15 +345,12 @@ async function fetchApiText(rel: string, account?: (bytes: number) => void): Pro if (typeof data.content !== "string") { throw new Error(`contents API omitted the body (>1 MiB cap): ${rel}`); } - // Size gate BEFORE decoding: base64 inflates payloads by ~4/3. - if (Math.ceil(data.content.length * 3 / 4) > MAX_TEXT_BYTES) { - throw new Error(`text document exceeds ${MAX_TEXT_BYTES} bytes: ${rel}`); - } - const buf = Buffer.from(data.content, "base64"); - if (buf.byteLength > MAX_TEXT_BYTES) throw new Error(`text document exceeds ${MAX_TEXT_BYTES} bytes: ${rel}`); + const buf = decodeCappedBase64(data.content, MAX_TEXT_BYTES); + if (buf === undefined) throw new Error(`text document exceeds ${MAX_TEXT_BYTES} bytes: ${rel}`); account?.(buf.byteLength); return buf.toString("utf8"); } finally { + await releaseUnreadBody(result.response); result.stopIdleTimer(); } } @@ -394,11 +393,12 @@ async function fetchPlainImage(url: string, account?: (bytes: number) => void): if (!result.response.ok) throw new Error(`image GET ${url} → HTTP ${result.response.status}`); const declared = contentLengthOf(result.response.headers); if (declared !== undefined && declared > MAX_IMAGE_BYTES) return undefined; // never download the body - const buf = Buffer.from(await result.response.arrayBuffer()); - if (buf.byteLength === 0 || buf.byteLength > MAX_IMAGE_BYTES) return undefined; + const buf = await readCappedBytes(result.response.body, MAX_IMAGE_BYTES); + if (buf === undefined || buf.byteLength === 0) return undefined; account?.(buf.byteLength); return { buf, type }; } finally { + await releaseUnreadBody(result.response); result.stopIdleTimer(); } } @@ -419,16 +419,50 @@ async function fetchApiImage(rel: string, account?: (bytes: number) => void): Pr if (!result.response.ok) throw new Error(`contents image GET ${url} → HTTP ${result.response.status}`); const data = (await result.response.json()) as { content?: unknown }; if (typeof data.content !== "string") return undefined; // >1 MiB omission → next source - if (Math.ceil(data.content.length * 3 / 4) > MAX_IMAGE_BYTES) return undefined; - const buf = Buffer.from(data.content, "base64"); - if (buf.byteLength === 0 || buf.byteLength > MAX_IMAGE_BYTES) return undefined; + const buf = decodeCappedBase64(data.content, MAX_IMAGE_BYTES); + if (buf === undefined || buf.byteLength === 0) return undefined; account?.(buf.byteLength); return { buf, type }; } finally { + await releaseUnreadBody(result.response); result.stopIdleTimer(); } } +async function releaseUnreadBody(response: Response): Promise { + if (!response.bodyUsed) await response.body?.cancel().catch(() => {}); +} + +function decodeCappedBase64(content: string, maxBytes: number): Buffer | undefined { + const compact = content.replace(/\s/g, ""); + if (Buffer.byteLength(compact, "base64") > maxBytes) return undefined; + const buf = Buffer.from(compact, "base64"); + return buf.byteLength > maxBytes ? undefined : buf; +} + +async function readCappedBytes(body: Response["body"], maxBytes: number): Promise { + if (body === null) return Buffer.alloc(0); + const reader = body.getReader(); + const chunks: Uint8Array[] = []; + let total = 0; + try { + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + if (value.byteLength === 0) continue; + if (value.byteLength > maxBytes - total) { + await reader.cancel().catch(() => {}); + return undefined; + } + total += value.byteLength; + chunks.push(value); + } + } finally { + reader.releaseLock(); + } + return Buffer.concat(chunks, total); +} + /** Image fetch through the fallback chain. Misses (true 404s / >1 MiB API * omissions) keep walking — a file absent on one host is absent nowhere we * look, and walking costs bounded calls; the chain exhausts to a MISS diff --git a/tests/web-blog-refresh.test.ts b/tests/web-blog-refresh.test.ts index 58782a9ef1..6a4a1d3522 100644 --- a/tests/web-blog-refresh.test.ts +++ b/tests/web-blog-refresh.test.ts @@ -226,6 +226,30 @@ test("raw unreachable → the contents-API chain serves the whole round", async "first-party hosts only: no third-party host was ever contacted"); }); +for (const lineWrapped of [false, true]) { + for (const delta of [-1, 0, 1]) { + test(`API text byte cap counts decoded bytes (${lineWrapped ? "wrapped" : "plain"} base64, delta ${delta})`, async () => { + const prefix = "# Fresh\n\n"; + const text = prefix + "x".repeat(512 * 1024 + delta - prefix.length); + const encoded = Buffer.from(text).toString("base64"); + const content = lineWrapped ? encoded.replace(/.{1,60}/g, "$&\n") : encoded; + router = (url) => { + if (url === CONTENTS_URL) return new Response(JSON.stringify([{ name: "2026-10-09_fresh.md", type: "file" }])); + if (url.startsWith("https://raw.githubusercontent.com/")) throw new Error("raw down"); + if (url === apiFileUrl(`${POSTS_REL}/2026-10-09_fresh.md`)) { + return new Response(JSON.stringify({ content, encoding: "base64" })); + } + return new Response("nf", { status: 404 }); + }; + + await runBlogRefreshRound({ force: true }); + + assert.equal(getBlogFeed().error, delta > 0 ? "offline" : undefined); + assert.equal(getBlogFeed().items.some((item) => item.id === "fresh"), delta <= 0); + }); + } +} + test("raw AND contents-API dead (rate-limited) → the round fails soft with the offline marker", async () => { router = (url) => { if (url === CONTENTS_URL) return new Response(CHAIN_LISTING, { status: 200 }); // listing still reachable @@ -536,6 +560,88 @@ test("#4906 P3 lying-small content-length: the post-read byte check stays the au assert.equal(getBlogFeed().error, "offline", "header said 10 bytes; the real 512KB+1 body still blew the limit"); }); +function chunkedBody(chunkSize: number, chunkCount: number, headers?: Record): { + response: Response; + reads(): number; + cancelled(): boolean; +} { + let reads = 0; + let cancelled = false; + const body = new ReadableStream({ + pull(controller) { + if (reads === chunkCount) { + controller.close(); + return; + } + reads++; + controller.enqueue(Buffer.alloc(chunkSize, "x")); + }, + cancel() { cancelled = true; }, + }, { highWaterMark: 0 }); + return { response: new Response(body, { headers }), reads: () => reads, cancelled: () => cancelled }; +} + +for (const headers of [undefined, { "content-length": "10" }]) { + test(`text download stops at its byte cap with ${headers === undefined ? "no" : "understated"} content-length`, async () => { + installGithubRouter(); + await runBlogRefreshRound({ force: true }); + const body = chunkedBody(64 * 1024, 32, headers); + router = (url) => { + if (url === CONTENTS_URL) return new Response(JSON.stringify([{ name: "2026-10-09_fresh.md", type: "file" }])); + if (url === rawUrl("website/blog/posts/2026-10-09_fresh.md")) return body.response; + return new Response("not here", { status: 404 }); + }; + + await runBlogRefreshRound({ force: true }); + + assert.equal(body.cancelled(), true, "oversized body is canceled before EOF"); + assert.ok(body.reads() <= 10, `expected the cap plus one prefetched chunk, read ${body.reads()}`); + assert.equal(getBlogFeed().error, "offline"); + assert.ok(getBlogFeed().items.some((item) => item.id === "demo"), "the last good overlay survives"); + }); +} + +test("an oversized streamed image is canceled and remains a clean miss", async () => { + installGithubRouter(); + const base = router; + const body = chunkedBody(256 * 1024, 32); + router = (url) => url === rawUrl("website/blog/posts/figures/good.png") ? body.response : base(url); + + await runBlogRefreshRound({ force: true }); + + assert.equal(body.cancelled(), true); + assert.ok(body.reads() <= 10, `expected the cap plus one prefetched chunk, read ${body.reads()}`); + assert.equal(getBlogFeed().error, undefined); + assert.equal(getRemoteBlogAsset("website/blog/posts/figures/good.png"), undefined); + assert.doesNotMatch(getBlogFeed().items.find((item) => item.id === "fresh")?.html ?? "", /good\.png/); +}); + +test("a streamed image exactly at the byte cap is retained", async () => { + installGithubRouter(); + const base = router; + const body = chunkedBody(256 * 1024, 8); + router = (url) => url === rawUrl("website/blog/posts/figures/good.png") ? body.response : base(url); + + await runBlogRefreshRound({ force: true }); + + assert.equal(body.cancelled(), false); + assert.equal(getBlogFeed().error, undefined); + assert.equal(getRemoteBlogAsset("website/blog/posts/figures/good.png")?.buf.byteLength, 2 * 1024 * 1024); +}); + +test("content-length rejection cancels the unread image body", async () => { + installGithubRouter(); + const base = router; + const body = chunkedBody(256 * 1024, 32, { "content-length": String(3 * 1024 * 1024) }); + router = (url) => url === rawUrl("website/blog/posts/figures/good.png") ? body.response : base(url); + + await runBlogRefreshRound({ force: true }); + + assert.equal(body.cancelled(), true); + assert.ok(body.reads() <= 1, "only the fetch wrapper's prefetch may run"); + assert.equal(getBlogFeed().error, undefined); +}); + test("#4906 P3 settleRoundImmune: a rejecting round never leaks an unhandledRejection", async () => { let unhandled: unknown = undefined; const onUn = (e: unknown) => { unhandled = e; };