Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 29 additions & 7 deletions src/web/blog-refresh.ts
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,9 @@ const MAX_IMAGE_CANDIDATES = 50; // deduped relative targets per
type TextFileMap = Map<string, string>; // repo-relative path → raw text
type BinaryAssetMap = Map<string, { buf: Buffer; type: string }>; // repo-relative path → bytes

/** A confirmed missing source is distinct from a failed download. */
class TextFileNotFound extends Error {}

let lastRoundStartedAt = 0;
let inflight: Promise<void> | undefined;
let warmAttempted = false;
Expand Down Expand Up @@ -203,13 +206,23 @@ async function runRound(): Promise<void> {
// kept in `files` under MANIFEST_REL — buildRemoteBlogPosts re-reads
// it by that key when assembling the article section.
let entries: ArticleEntry[] = [];
let manifestText: string | undefined;
try {
const manifestText = await getTextChain(MANIFEST_REL, chain, account);
manifestText = await getTextChain(MANIFEST_REL, chain, account);
} catch (error) {
// Only a clean absence can remove the previous article set. A
// failed read must keep the last complete overlay, like a post.
if (!(error instanceof TextFileNotFound)) throw error;
}
if (manifestText !== undefined) {
files.set(MANIFEST_REL, manifestText);
const parsed = JSON.parse(manifestText) as { articles?: unknown };
if (Array.isArray(parsed.articles)) entries = parsed.articles.filter(isArticleEntry);
} catch {
entries = [];
try {
const parsed = JSON.parse(manifestText) as { articles?: unknown };
if (Array.isArray(parsed.articles)) entries = parsed.articles.filter(isArticleEntry);
} catch {
// A completely downloaded but malformed manifest retains the
// existing posts-only behavior; it is not a transport failure.
}
}
const wanted: string[] = [];
for (const e of entries) {
Expand Down Expand Up @@ -317,6 +330,7 @@ async function getJsonList(url: string): Promise<ContentsEntry[]> {
async function fetchPlainText(url: string, account?: (bytes: number) => void): Promise<string> {
const result = await fetchWithTimeout(url, ghHeaders(), FETCH_TIMEOUT_MS);
try {
if (result.response.status === 404) throw new TextFileNotFound(`plain GET ${url} → HTTP 404`);
if (!result.response.ok) throw new Error(`plain GET ${url} → HTTP ${result.response.status}`);
const declared = contentLengthOf(result.response.headers);
if (declared !== undefined && declared > MAX_TEXT_BYTES) {
Expand All @@ -338,6 +352,7 @@ async function fetchApiText(rel: string, account?: (bytes: number) => void): Pro
const url = `${GITHUB_CONTENTS_URL}/${encRelPath(rel)}?ref=${BRANCH}`;
const result = await fetchWithTimeout(url, ghHeaders("application/json"), FETCH_TIMEOUT_MS);
try {
if (result.response.status === 404) throw new TextFileNotFound(`contents GET ${url} → HTTP 404`);
if (!result.response.ok) throw new Error(`contents GET ${url} → HTTP ${result.response.status}`);
const data = (await result.response.json()) as { content?: unknown; encoding?: unknown };
if (typeof data.content !== "string") {
Expand All @@ -359,11 +374,14 @@ async function fetchApiText(rel: string, account?: (bytes: number) => void): Pro
/** Text fetch through the fallback chain for ONE repo-relative path. Any
* per-source failure (network, HTTP, oversize, omitted body) walks to the
* next member; the round only fails when EVERY member fails, and then with
* the last error — half-fetched trees never render. On success the walk
* the last hard error (or a confirmed absence if all attempts are 404s) —
* half-fetched trees never render. On success the walk
* point advances so later files skip already-proven-dead sources. */
async function getTextChain(rel: string, chain: ChainState, account?: (bytes: number) => void): Promise<string> {
const sources = sourceChainFor(rel);
let lastError: unknown = new Error(`all sources failed for ${rel}`);
let lastHardError: unknown;
let hadHardError = false;
for (let i = Math.min(chain.start, sources.length - 1); i < sources.length; i++) {
try {
const text = i === 1 ? await fetchApiText(rel, account)
Expand All @@ -373,9 +391,13 @@ async function getTextChain(rel: string, chain: ChainState, account?: (bytes: nu
return text;
} catch (e) {
lastError = e; // walk to the next fallback source
if (!(e instanceof TextFileNotFound)) {
lastHardError = e;
hadHardError = true;
}
}
}
throw lastError;
throw hadHardError ? lastHardError : lastError;
}

/** Download ONE image from the PLAIN source (raw host); 404/oversize are
Expand Down
38 changes: 38 additions & 0 deletions tests/web-blog-refresh.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -450,6 +450,44 @@ test("an oversize document blows the round budget: fail-soft keeps the last good
assert.ok(feed.items.some((i) => i.id === "demo"), "previous good overlay survives");
});

for (const failure of ["http", "network", "raw-missing", "api-missing", "body"] as const) {
test(`a ${failure} failure fetching the optional manifest preserves the last complete overlay`, async () => {
const manifest = JSON.stringify({ articles: [{ file: "website/articles/kept.md", slug: "kept", title: "Kept article" }] });
const good = (url: string): Response => {
if (url === CONTENTS_URL) return Response.json([{ name: "2026-10-09_fresh.md", type: "file" }]);
if (url === rawUrl("website/blog/posts/2026-10-09_fresh.md")) return new Response("# Original post\n\noriginal post body\n");
if (url === rawUrl("website/articles.json")) return new Response(manifest);
if (url === rawUrl("website/articles/kept.md")) return new Response("# Kept article\n\nlast complete article\n");
return new Response("not here", { status: 404 });
};
router = good;
await runBlogRefreshRound({ force: true });
assert.ok(getBlogFeed().items.some((item) => item.id === "kept"));

router = (url) => {
if (url === rawUrl("website/blog/posts/2026-10-09_fresh.md")) return new Response("# Partial new post\n\nincomplete round\n");
if (url === rawUrl("website/articles.json") || url === apiFileUrl("website/articles.json")) {
if (failure === "network") throw new Error("fixture manifest unavailable");
if (failure === "body") return new Response(new ReadableStream({ start(controller) { controller.error(new Error("fixture truncated body")); } }));
const missing = (failure === "raw-missing" && url === rawUrl("website/articles.json"))
|| (failure === "api-missing" && url === apiFileUrl("website/articles.json"));
return new Response("unavailable", { status: missing ? 404 : 503 });
}
return good(url);
};
await runBlogRefreshRound({ force: true });
const failed = getBlogFeed();
assert.ok(failed.items.some((item) => item.id === "kept" && item.html.includes("last complete article")));
assert.ok(failed.items.some((item) => item.id === "fresh" && item.html.includes("original post body")));
assert.ok(!failed.items.some((item) => item.html.includes("incomplete round")));
assert.equal(failed.error, "offline");

router = good;
await runBlogRefreshRound({ force: true });
assert.equal(getBlogFeed().error, undefined, "a complete later refresh clears the failure marker");
});
}

test("malformed articles.json degrades to a posts-only round without failing", async () => {
router = (url) => {
if (url === CONTENTS_URL) return new Response(JSON.stringify([{ name: "2026-10-09_fresh.md", type: "file" }]), { status: 200 });
Expand Down
Loading