Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 22 additions & 1 deletion .github/workflows/pages.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,19 @@ on:
branches: [master]
paths:
- "website/**"
# Article sources live outside website/ and are rendered at deploy time — any change to a
# listed source must retrigger the pipeline (see website/articles.json for what is published).
- "*.md"
- "paper/**"
- "tools/build-blog.mjs"
- "tools/build-articles.mjs"
- "tools/lib/**"
- ".github/workflows/pages.yml"
workflow_dispatch:
# Scheduled refresh: published articles may drift from the repository between pushes; re-render
# + redeploy every half hour so the deployed copy lags master by at most one tick.
schedule:
- cron: "*/30 * * * *"

permissions:
contents: read
Expand All @@ -25,8 +36,18 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version: 22
# Re-render generated pages from their markdown sources on every deploy (deterministic;
# committed copies under website/ exist for local preview only — deployment always ships
# fresh output). No other build step: the site itself stays dependency-free static HTML.
- name: Build generated pages
run: |
node tools/build-blog.mjs
node tools/build-articles.mjs
- uses: actions/configure-pages@v5
# No build step — publish the folder contents as-is, so the site root is website/
# Publish the folder contents as-is; the site root is website/
- uses: actions/upload-pages-artifact@v3
with:
path: website
Expand Down
149 changes: 149 additions & 0 deletions tools/build-articles.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,149 @@
#!/usr/bin/env node
// Static article sync for the billion-context website. Zero dependencies (Node >= 18 builtins only).
// Renders long-form markdown that lives ELSEWHERE in the repository (paper/, root *.md, ...) into
// committed static pages under website/docs/articles/, sharing the rendering kit with tools/build-blog.mjs.
//
// Usage: node tools/build-articles.mjs (regenerate all)
// Input: website/articles.json manifest, repo-relative sources
// { "articles": [ { file, zh_file?, slug?, title?, title_zh?, date? } ] }
// Output: website/docs/articles/<slug>.html (+ <slug>.zh.html for a zh twin), index.html
// website/docs/assets/<slug>/... image/link targets referenced by the source
//
// Rules: only listed files are published (explicit over implicit — internal notes stay private
// unless someone adds them here). Relative media references resolve against the source directory
// and are copied next to the rendered page; external/absolute anchors pass through untouched.
// Deterministic: same inputs, same bytes (safe to rerun in CI on every deploy).

import { readFileSync, writeFileSync, mkdirSync, rmSync, copyFileSync, existsSync, statSync } from "node:fs";
import { dirname, join, resolve, relative, sep } from "node:path";
import { fileURLToPath } from "node:url";
import { esc, mdToHtml, parseFrontmatter, page } from "./lib/md-web.mjs";

const ROOT = join(dirname(fileURLToPath(import.meta.url)), "..");
const MANIFEST = join(ROOT, "website", "articles.json");
const OUT = join(ROOT, "website", "docs", "articles");
const ASSETS = join(ROOT, "website", "docs", "assets");
const CRUMB = "文库 Articles";
const SRC_BASE = "https://github.com/ranxianglei/billion-context/blob/master/";
const ROOT_PREFIX = "../../";
// Article pages live in docs/articles/ while copied media lands in docs/assets/ —
// one level up from the page, not two (the site-root prefix above is the depth
// to styles.css/topbar links, not to the sibling assets dir).
const MEDIA_PREFIX = "../";

function fail(msg) {
console.error("[build-articles] " + msg);
process.exit(1);
}

const encPath = (p) => p.split("/").map(encodeURIComponent).join("/");
const srcUrl = (p) => SRC_BASE + encPath(p);

function isZhhName(name) {
return /\.zh(-CN)?\.md$/.test(name);
}

function readSource(relPath, ctx) {
const abs = join(ROOT, relPath);
if (!abs.startsWith(ROOT + sep)) fail(`${ctx}: path escapes the repository: ${relPath}`);
if (!existsSync(abs) || !statSync(abs).isFile()) fail(`${ctx}: missing source file: ${relPath}`);
return readFileSync(abs, "utf8");
}

// Rewrite relative link/image targets so they point at the copied asset tree; copy the files.
function prepareMedia(md, relPath, slug) {
const srcDir = dirname(join(ROOT, relPath));
const destDir = join(ASSETS, slug);
return md.replace(/(!?\[[^\]]*\])\(([^()\s]+)(?:#[^()]*?)?\)/g, (whole, pre, target) => {
if (/^(?:[a-z][a-z0-9+.-]*:|\/\/|\/|#)/i.test(target)) return whole;
const abs = resolve(srcDir, target);
if (!abs.startsWith(resolve(srcDir) + sep) || !existsSync(abs) || !statSync(abs).isFile()) return whole;
const rel = relative(resolve(srcDir), abs).split(sep);
mkdirSync(dirname(join(destDir, ...rel)), { recursive: true });
copyFileSync(abs, join(destDir, ...rel));
return `${pre}(${MEDIA_PREFIX}assets/${encPath(slug + "/" + rel.join("/"))})`;
});
}

function renderEntry(entry) {
const ctx = entry.slug || entry.file;
const file = entry.file;
const lang = isZhhName(file) ? "zh" : "en";
const { fm, body } = parseFrontmatter(readSource(file, ctx));
const slug = entry.slug || file.replace(/\.md$/, "").split("/").pop();
rmSync(join(ASSETS, slug), { recursive: true, force: true });

const h1Raw = body.match(/^#\s+(.+?)\s*$/m)?.[1];
const title = entry.title || fm.title || h1Raw?.replace(/[*_`]/g, "") || slug;
const dated = entry.date || fm.date || "";
const dateShown = /^\d{4}-\d{2}-\d{2}$/.test(dated) ? dated : "";

const variants = [{ key: "", lang, rel: file }];
if (entry.zh_file) variants.push({ key: ".zh", lang: "zh", rel: entry.zh_file });
const titles = {};

for (const v of variants) {
const parsed = parseFrontmatter(readSource(v.rel, ctx));
const h1 = parsed.body.match(/^#\s+(.+?)\s*$/m)?.[1]?.replace(/[*_`]/g, "");
const variantTitle = v.key === ".zh"
? entry.title_zh || parsed.fm.title_zh || parsed.fm.title || h1 || title
: entry.title || parsed.fm.title || h1 || title;
titles[v.key] = variantTitle;
const bodyHtml = mdToHtml(prepareMedia(parsed.body.replace(/^#[ \t]+[^\n]*\r?\n/, ""), v.rel, slug));
const metaParts = [];
if (dateShown) metaParts.push(`<time datetime="${dateShown}">${dateShown}</time>`);
metaParts.push(`<a href="${srcUrl(v.rel)}">source on GitHub</a>`);
const siblingKey = v.key === ".zh" ? "" : ".zh";
const sibling = variants.find((x) => x.key === siblingKey);
if (sibling) {
const label = sibling.lang === "zh" ? "中文版" : "English";
metaParts.push(`<a href="${slug}${sibling.key}.html">${label}</a>`);
}
const inner = `<article class="blog-wrap post-article">
<h1>${esc(variantTitle)}</h1>
<p class="post-meta">${metaParts.join(" · ")}</p>
${bodyHtml}
<p class="post-foot"><a href="./index.html">&larr; 文库 Articles</a></p>
</article>`;
writeFileSync(join(OUT, `${slug}${v.key}.html`), page({ lang: v.lang, title: variantTitle, crumb: CRUMB, root: ROOT_PREFIX, current: "articles", inner }));
}
return { entry, slug, title, titleZh: titles[".zh"] || "", lang, dateShown };
}

function main() {
if (!existsSync(MANIFEST)) {
console.log("build-articles: no website/articles.json — nothing to do");
return;
}
const manifest = JSON.parse(readFileSync(MANIFEST, "utf8"));
const entries = manifest.articles ?? [];
if (!Array.isArray(entries)) fail("manifest.articles must be an array");
mkdirSync(OUT, { recursive: true });
const rendered = entries.map(renderEntry);

const cards = rendered
.map(({ slug, title, titleZh, dateShown, entry }) => {
const lines = dateShown ? [`<time datetime="${dateShown}">${dateShown}</time>`] : [];
const titles = [`<a href="${slug}.html">${esc(title)}</a>`];
if (entry.zh_file) titles.push(`<a href="${slug}.zh.html" class="post-title-zh">${esc(titleZh)}</a>`);
const exs = `<span><a href="${srcUrl(entry.file)}">source on GitHub</a></span>`;
return `<li class="post-card">
${[...lines, `<h3>${titles.join(" &nbsp;/&nbsp; ")}</h3>`, `<p class="post-ex">${exs}</p>`].join("\n")}
</li>`;
})
.join("\n");

const indexInner = `<div class="blog-wrap blog-index-page">
<h1>文库 <span class="blog-alt">Articles</span></h1>
<p class="blog-sub">长文档同步自本仓库(paper、技术笔记等)。由 CI 定时重新渲染发布,与仓库保持半小时内一致。</p>
<ul class="blog-index">
${cards || "<li class=\"post-ex\">暂无文章 / No articles yet.</li>"}
</ul>
<p class="post-foot"><a href="../index.html">&larr; Docs / 文档</a> · <a href="../../blog/">博客 Blog &rarr;</a></p>
</div>`;
writeFileSync(join(OUT, "index.html"), page({ lang: "en", title: "billion-context", crumb: CRUMB, root: ROOT_PREFIX, current: "articles", inner: indexInner }));
const pages = rendered.reduce((n, r) => n + (r.entry.zh_file ? 2 : 1), 0);
console.log(`build-articles: wrote ${pages} article page(s) + index into website/docs/articles (${entries.length} manifest entr(y/ies))`);
}

main();
141 changes: 6 additions & 135 deletions tools/build-blog.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
// Static blog builder for the billion-context website. Zero dependencies (Node >= 18 builtins only),
// same house style as tools/gen-config-docs.mjs. Keeps the site build-free: generated HTML is
// committed next to the markdown sources and published verbatim by GitHub Pages.
// Rendering kit lives in tools/lib/md-web.mjs (shared with tools/build-articles.mjs).
//
// Usage: node tools/build-blog.mjs (regenerate all)
// Input: website/blog/posts/<YYYY-MM-DD_slug>.md English post
Expand All @@ -10,147 +11,16 @@
//
// Frontmatter (key: value lines between --- fences):
// title, title_zh, date (YYYY-MM-DD), excerpt, excerpt_zh
//
// Markdown subset rendered: headings (#..#### -> h2..h4), paragraphs, fenced ```code blocks,
// inline code, **bold**, *italic*, [links](url), ![img](url), - / 1. lists (flat), > blockquote,
// | tables | (with dash separator row), --- hr. Nested lists unsupported — restructure instead.

import { readFileSync, writeFileSync, readdirSync, mkdirSync } from "node:fs";
import { dirname, join } from "node:path";
import { fileURLToPath } from "node:url";
import { esc, mdToHtml, parseFrontmatter, page } from "./lib/md-web.mjs";

const ROOT = join(dirname(fileURLToPath(import.meta.url)), "..");
const BLOG = join(ROOT, "website", "blog");
const POSTS_DIR = join(BLOG, "posts");

const esc = (s) => s.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;");

function inline(s) {
let out = esc(s);
const codes = [];
out = out.replace(/`([^`]+)`/g, (_m, c) => {
codes.push(`<code>${c}</code>`);
return `\u0000${codes.length - 1}\u0000`;
});
out = out.replace(/!\[([^\]]*)\]\(([^)\s]+)\)/g, '<img src="$2" alt="$1">');
out = out.replace(/\[([^\]]+)\]\(([^)\s]+)\)/g, '<a href="$2">$1</a>');
out = out.replace(/\*\*([^*]+)\*\*/g, "<strong>$1</strong>");
out = out.replace(/\*([^*]+)\*/g, "<em>$1</em>");
return out.replace(/\u0000(\d+)\u0000/g, (_m, i) => codes[Number(i)]);
}

function mdToHtml(src) {
const lines = src.split("\n");
const html = [];
let para = [];
const flushP = () => {
if (para.length) { html.push(`<p>${inline(para.join(" "))}</p>`); para = []; }
};
let i = 0;
while (i < lines.length) {
const l = lines[i];
let m;
if (/^```/.test(l)) {
flushP();
const lang = l.slice(3).trim();
const buf = [];
i++;
while (i < lines.length && !/^```/.test(lines[i])) { buf.push(lines[i]); i++; }
i++; // closing fence
html.push(`<pre><code${lang ? ` class="lang-${esc(lang)}"` : ""}>${esc(buf.join("\n"))}</code></pre>`);
continue;
}
if (/^\s*$/.test(l)) { flushP(); i++; continue; }
if ((m = l.match(/^(#{1,4})\s+(.*)$/))) {
flushP();
const lv = Math.min(m[1].length + 1, 4);
html.push(`<h${lv}>${inline(m[2])}</h${lv}>`);
i++;
continue;
}
if (/^-{3,}\s*$/.test(l)) { flushP(); html.push("<hr>"); i++; continue; }
if (/^>\s?/.test(l)) {
flushP();
const buf = [];
while (i < lines.length && /^>\s?/.test(lines[i])) { buf.push(inline(lines[i].replace(/^>\s?/, ""))); i++; }
html.push(`<blockquote><p>${buf.join(" ")}</p></blockquote>`);
continue;
}
if (/^\s*\|.*\|\s*$/.test(l) && i + 1 < lines.length && /^\s*\|[\s:|-]+\|\s*$/.test(lines[i + 1])) {
flushP();
const cells = (row) => row.trim().replace(/^\|/, "").replace(/\|$/, "").split("|").map((c) => inline(c.trim()));
const head = cells(l);
i += 2;
const rows = [];
while (i < lines.length && /^\s*\|.*\|\s*$/.test(lines[i])) { rows.push(cells(lines[i])); i++; }
html.push(
`<table class="param-table"><thead><tr>${head.map((c) => `<th>${c}</th>`).join("")}</tr></thead>` +
`<tbody>${rows.map((r) => `<tr>${r.map((c) => `<td>${c}</td>`).join("")}</tr>`).join("")}</tbody></table>`
);
continue;
}
if (/^\s*[-*]\s+/.test(l)) {
flushP();
const items = [];
while (i < lines.length && /^\s*[-*]\s+/.test(lines[i])) { items.push(`<li>${inline(lines[i].trim().replace(/^[-*]\s+/, ""))}</li>`); i++; }
html.push(`<ul>${items.join("")}</ul>`);
continue;
}
if (/^\s*\d+\.\s+/.test(l)) {
flushP();
const items = [];
while (i < lines.length && /^\s*\d+\.\s+/.test(lines[i])) { items.push(`<li>${inline(lines[i].trim().replace(/^\d+\.\s+/, ""))}</li>`); i++; }
html.push(`<ol>${items.join("")}</ol>`);
continue;
}
para.push(l.trim());
i++;
}
flushP();
return html.join("\n");
}

function parseFrontmatter(text) {
const fm = {};
let body = text;
if (text.startsWith("---\n")) {
const end = text.indexOf("\n---\n", 4);
if (end !== -1) {
for (const line of text.slice(4, end).split("\n")) {
const j = line.indexOf(":");
if (j > 0) fm[line.slice(0, j).trim()] = line.slice(j + 1).trim().replace(/^["']|["']$/g, "");
}
body = text.slice(end + 5);
}
}
return { fm, body };
}

function shell({ lang, title, inner }) {
const htmlLang = lang === "zh" ? "zh-CN" : "en";
return `<!doctype html>
<html lang="${htmlLang}">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>${esc(title)} · billion-context 博客 Blog</title>
<link rel="stylesheet" href="../styles.css">
<!-- GENERATED by tools/build-blog.mjs — edit website/blog/posts/*.md and rerun -->
</head>
<body>
<div class="blog-topbar">
<a class="blog-brand" href="../">billion-context</a>
<nav class="blog-nav" aria-label="Blog">
<a href="../index.html">Home</a>
<a href="../docs/">Docs / 文档</a>
<a href="./index.html">Blog / 博客</a>
</nav>
</div>
${inner}
</body>
</html>
`;
}
const CRUMB = "博客 Blog";

function collectPosts() {
let files;
Expand Down Expand Up @@ -195,7 +65,7 @@ function main() {
${bodyHtml}
<p class="post-foot"><a href="./index.html">&larr; Blog / 博客</a></p>
</article>`;
writeFileSync(join(BLOG, p.htmlFile), shell({ lang: p.lang, title: p.title, inner }));
writeFileSync(join(BLOG, p.htmlFile), page({ lang: p.lang, title: p.title, crumb: CRUMB, root: "../", current: "blog", inner }));
written++;
}

Expand Down Expand Up @@ -233,8 +103,9 @@ ${bodyHtml}
<ul class="blog-index">
${cards || "<li class=\"post-ex\">暂无文章 / No posts yet.</li>"}
</ul>
<p class="post-foot"><a href="../docs/articles/">文库 Long-form articles &rarr;</a></p>
</div>`;
writeFileSync(join(BLOG, "index.html"), shell({ lang: "en", title: "博客 Blog", inner: indexInner }));
writeFileSync(join(BLOG, "index.html"), page({ lang: "en", title: "billion-context", crumb: CRUMB, root: "../", current: "blog", inner: indexInner }));
written++;
console.log(`build-blog: wrote ${written} file(s) into website/blog (${posts.length} post file(s))`);
}
Expand Down
Loading
Loading