From eb6e0d8dfb55b171928e0a636e2d5f93d978d576 Mon Sep 17 00:00:00 2001 From: FortuneCatOfSummer <132820009+FortuneCatOfSummer@users.noreply.github.com> Date: Sat, 24 Oct 2026 00:18:18 +0800 Subject: [PATCH] fix(online): allow a configurable raw repository base The online commands hard-code raw.githubusercontent.com for the public SUSTech Online Markdown. On networks where that host is unreachable, every online command fails with NETWORK_ERROR even though the rendered site is available. Resolve the raw base from SUSTECH_ONLINE_RAW_BASE (HTTPS, no query or fragment), defaulting to the current public repository URL. The allowlist origin and path are derived from the same base, so exact URL matching and redirect rejection remain enforced. Add tests for the default, a mirror override, invalid values, and import-time resolution, and document the variable in docs/ONLINE.md. --- docs/ONLINE.md | 18 ++++++++ src/online/manual.ts | 12 +++--- src/online/shared.ts | 39 ++++++++++++++++-- src/services/sustech-online.ts | 9 ++-- src/test/online-raw-base.test.ts | 70 ++++++++++++++++++++++++++++++++ 5 files changed, 131 insertions(+), 17 deletions(-) create mode 100644 src/test/online-raw-base.test.ts diff --git a/docs/ONLINE.md b/docs/ONLINE.md index e14ac1b..15d5984 100644 --- a/docs/ONLINE.md +++ b/docs/ONLINE.md @@ -100,6 +100,24 @@ the community site. Poster links are limited to those hosts plus the exact image-mirror host currently used by the upstream talks archive; unrelated, social, document-sharing, and deceptive lookalike domains are omitted. +### Mirrors and restricted networks + +Some networks cannot reach `raw.githubusercontent.com`. Set +`SUSTECH_ONLINE_RAW_BASE` to a plain HTTPS base that serves the same repository; +the default is +`https://raw.githubusercontent.com/SUSTech-CRA/sustech-online-ng/master`. + +```bash +export SUSTECH_ONLINE_RAW_BASE="https://ghfast.top/https://raw.githubusercontent.com/SUSTech-CRA/sustech-online-ng/master" +sustech online talks list --limit 5 +``` + +The raw allowlist is derived from this base. Redirects are still rejected and +the final URL must still match the constructed target exactly, so mirrors that +answer with a 3xx redirect cannot be used. Values that are not HTTPS URLs +without a query string or fragment are rejected with +`ONLINE_RAW_BASE_INVALID`. + Rendered-page metadata is optional; raw Markdown is required. Tests use frozen synthetic fixtures and do not copy the upstream contact or talks content into the package. diff --git a/src/online/manual.ts b/src/online/manual.ts index 7b21374..75409d2 100644 --- a/src/online/manual.ts +++ b/src/online/manual.ts @@ -7,11 +7,10 @@ import { ONLINE_DEFAULT_STALE_AFTER_DAYS, ONLINE_DEFAULT_TIMEOUT_MS, ONLINE_MAX_DOCUMENT_BYTES, + ONLINE_RAW_BASE_PATH, ONLINE_RAW_ORIGIN, - ONLINE_REPO_BRANCH, - ONLINE_REPO_NAME, - ONLINE_REPO_OWNER, ONLINE_SITE_ORIGIN, + onlineRawBase, scoreSearchMatch, type OnlineFetchOptions, } from "./shared.js"; @@ -242,7 +241,7 @@ export function createOnlineManualAdapter(fetchImpl: typeof fetch = globalThis.f export function onlineManualRawUrl(repoPath: string): string { assertAllowedManualRepoPath(repoPath); const segments = repoPath.split("/").map((segment) => encodeURIComponent(segment)); - return `${ONLINE_RAW_ORIGIN}/${ONLINE_REPO_OWNER}/${ONLINE_REPO_NAME}/${ONLINE_REPO_BRANCH}/${segments.join("/")}`; + return `${onlineRawBase()}/${segments.join("/")}`; } export function onlineManualSiteUrl(sitePath: string): string { @@ -774,13 +773,12 @@ function validateFetchedManualUrl(value: string, expected: string, kind: "raw" | ); } if (kind === "raw") { - const expectedPrefix = `/${ONLINE_REPO_OWNER}/${ONLINE_REPO_NAME}/${ONLINE_REPO_BRANCH}/docs/`; - if (url.origin !== ONLINE_RAW_ORIGIN || !url.pathname.startsWith(expectedPrefix)) { + if (url.origin !== ONLINE_RAW_ORIGIN || !url.pathname.startsWith(`${ONLINE_RAW_BASE_PATH}docs/`)) { throw new CliError("The fetched SUSTech Online manual source escaped the raw allowlist.", "ONLINE_SOURCE_NOT_ALLOWED", 2, { sourceUrl: value, }); } - assertAllowedManualRepoPath(decodeOnlinePath(url.pathname.slice(expectedPrefix.length - "docs/".length))); + assertAllowedManualRepoPath(decodeOnlinePath(url.pathname.slice(ONLINE_RAW_BASE_PATH.length))); return; } if (url.origin !== ONLINE_SITE_ORIGIN) { diff --git a/src/online/shared.ts b/src/online/shared.ts index 52695d9..5811ab6 100644 --- a/src/online/shared.ts +++ b/src/online/shared.ts @@ -13,7 +13,39 @@ export const ONLINE_SITE_ORIGIN = "https://sustech.online"; export const ONLINE_REPO_OWNER = "SUSTech-CRA"; export const ONLINE_REPO_NAME = "sustech-online-ng"; export const ONLINE_REPO_BRANCH = "master"; -export const ONLINE_RAW_ORIGIN = "https://raw.githubusercontent.com"; +const DEFAULT_ONLINE_RAW_BASE = `https://raw.githubusercontent.com/${ONLINE_REPO_OWNER}/${ONLINE_REPO_NAME}/${ONLINE_REPO_BRANCH}`; + +function normaliseOnlineRawBase(value: string): string | undefined { + const candidate = value.trim().replace(/\/+$/, ""); + if (!candidate) return undefined; + try { + const url = new URL(candidate); + if (url.protocol !== "https:" || url.search || url.hash) return undefined; + } catch { + return undefined; + } + return candidate; +} + +export function onlineRawBase(env: NodeJS.ProcessEnv = process.env): string { + const configured = env.SUSTECH_ONLINE_RAW_BASE?.trim(); + if (!configured) return DEFAULT_ONLINE_RAW_BASE; + const normalised = normaliseOnlineRawBase(configured); + if (!normalised) { + throw new CliError( + "SUSTECH_ONLINE_RAW_BASE must be a plain HTTPS URL without a query string or fragment.", + "ONLINE_RAW_BASE_INVALID", + 2, + { variable: "SUSTECH_ONLINE_RAW_BASE" }, + ); + } + return normalised; +} + +export const ONLINE_RAW_BASE = normaliseOnlineRawBase(process.env.SUSTECH_ONLINE_RAW_BASE ?? "") ?? DEFAULT_ONLINE_RAW_BASE; +const ONLINE_RAW_BASE_URL = new URL(ONLINE_RAW_BASE); +export const ONLINE_RAW_ORIGIN = ONLINE_RAW_BASE_URL.origin; +export const ONLINE_RAW_BASE_PATH = `${ONLINE_RAW_BASE_URL.pathname.replace(/\/+$/, "")}/`; export const ONLINE_TALKS_INDEX_REPO_PATH = "docs/study/talks/README.md"; export const ONLINE_TALKS_DIRECTORY_REPO_PATH = "docs/study/talks"; export const ONLINE_CONTACT_REPO_PATH = "docs/contact/README.md"; @@ -95,7 +127,7 @@ export function buildOnlineProvenance( export function onlineRawUrl(repoPath: string): string { assertAllowedOnlineRepoPath(repoPath); const segments = repoPath.split("/").map((segment) => encodeURIComponent(segment)); - return `${ONLINE_RAW_ORIGIN}/${ONLINE_REPO_OWNER}/${ONLINE_REPO_NAME}/${ONLINE_REPO_BRANCH}/${segments.join("/")}`; + return `${onlineRawBase()}/${segments.join("/")}`; } export function onlineSiteUrl(sitePath: string): string { @@ -387,8 +419,7 @@ function validateFetchedUrl(value: string, expected: string, kind: "raw" | "site }); } if (kind === "raw") { - const expectedPrefix = `/${ONLINE_REPO_OWNER}/${ONLINE_REPO_NAME}/${ONLINE_REPO_BRANCH}/docs/`; - if (url.origin !== ONLINE_RAW_ORIGIN || !url.pathname.startsWith(expectedPrefix)) { + if (url.origin !== ONLINE_RAW_ORIGIN || !url.pathname.startsWith(`${ONLINE_RAW_BASE_PATH}docs/`)) { throw new CliError("The fetched SUSTech Online source escaped the raw allowlist.", "ONLINE_SOURCE_NOT_ALLOWED", 2, { sourceUrl: value, }); diff --git a/src/services/sustech-online.ts b/src/services/sustech-online.ts index 94fe2ea..33ea23b 100644 --- a/src/services/sustech-online.ts +++ b/src/services/sustech-online.ts @@ -1,10 +1,7 @@ import { ONLINE_CONTACT_REPO_PATH, ONLINE_CONTACT_SITE_PATH, - ONLINE_RAW_ORIGIN, - ONLINE_REPO_BRANCH, - ONLINE_REPO_NAME, - ONLINE_REPO_OWNER, + ONLINE_RAW_BASE, ONLINE_SITE_ORIGIN, ONLINE_TALKS_INDEX_REPO_PATH, ONLINE_TALKS_INDEX_SITE_PATH, @@ -24,8 +21,8 @@ export const SUSTECH_ONLINE_STATUS: ServiceStatus = { "Handbook ingestion is constrained to a fixed allowlist; high-stakes, financial, personal, dining/chat, and professor-list content is excluded.", ], endpoints: [ - `${ONLINE_RAW_ORIGIN}/${ONLINE_REPO_OWNER}/${ONLINE_REPO_NAME}/${ONLINE_REPO_BRANCH}/${ONLINE_TALKS_INDEX_REPO_PATH}`, - `${ONLINE_RAW_ORIGIN}/${ONLINE_REPO_OWNER}/${ONLINE_REPO_NAME}/${ONLINE_REPO_BRANCH}/${ONLINE_CONTACT_REPO_PATH}`, + `${ONLINE_RAW_BASE}/${ONLINE_TALKS_INDEX_REPO_PATH}`, + `${ONLINE_RAW_BASE}/${ONLINE_CONTACT_REPO_PATH}`, `${ONLINE_SITE_ORIGIN}${ONLINE_TALKS_INDEX_SITE_PATH}`, `${ONLINE_SITE_ORIGIN}${ONLINE_CONTACT_SITE_PATH}`, ...ONLINE_MANUAL_ENDPOINTS, diff --git a/src/test/online-raw-base.test.ts b/src/test/online-raw-base.test.ts new file mode 100644 index 0000000..20bc970 --- /dev/null +++ b/src/test/online-raw-base.test.ts @@ -0,0 +1,70 @@ +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import { resolve } from "node:path"; +import test from "node:test"; +import { pathToFileURL } from "node:url"; +import { CliError } from "../core/errors.js"; +import { + ONLINE_RAW_BASE, + ONLINE_RAW_BASE_PATH, + ONLINE_RAW_ORIGIN, + onlineRawBase, + onlineRawUrl, +} from "../online/index.js"; + +const DEFAULT_BASE = "https://raw.githubusercontent.com/SUSTech-CRA/sustech-online-ng/master"; +const MIRROR_BASE = "https://ghfast.top/https://raw.githubusercontent.com/SUSTech-CRA/sustech-online-ng/master"; +const CONTACT_PATH = "docs/contact/README.md"; + +test("defaults to the public GitHub raw base", () => { + assert.equal(onlineRawBase({}), DEFAULT_BASE); + assert.equal(onlineRawUrl(CONTACT_PATH), `${DEFAULT_BASE}/${CONTACT_PATH}`); +}); + +test("accepts an HTTPS mirror override and trims surrounding slashes and spaces", () => { + assert.equal(onlineRawBase({ SUSTECH_ONLINE_RAW_BASE: MIRROR_BASE }), MIRROR_BASE); + assert.equal(onlineRawBase({ SUSTECH_ONLINE_RAW_BASE: `${MIRROR_BASE}/` }), MIRROR_BASE); + assert.equal(onlineRawBase({ SUSTECH_ONLINE_RAW_BASE: ` ${MIRROR_BASE} ` }), MIRROR_BASE); +}); + +test("rejects unsafe or malformed overrides", () => { + const values = [ + "http://mirror.example/repo", + "https://mirror.example/repo?token=1", + "https://mirror.example/repo#fragment", + "not a url", + ]; + for (const value of values) { + assert.throws( + () => onlineRawBase({ SUSTECH_ONLINE_RAW_BASE: value }), + (error: unknown) => error instanceof CliError && error.code === "ONLINE_RAW_BASE_INVALID", + ); + } +}); + +test( + "import-time allowlist derives from the default base", + { skip: process.env.SUSTECH_ONLINE_RAW_BASE ? "SUSTECH_ONLINE_RAW_BASE is set in this environment" : false }, + () => { + assert.equal(ONLINE_RAW_BASE, DEFAULT_BASE); + assert.equal(ONLINE_RAW_ORIGIN, "https://raw.githubusercontent.com"); + assert.equal(ONLINE_RAW_BASE_PATH, "/SUSTech-CRA/sustech-online-ng/master/"); + }, +); + +test("mirror override is applied when the module loads", () => { + const sharedModule = pathToFileURL(resolve("dist", "online", "shared.js")).href; + const script = [ + `const m = await import(${JSON.stringify(sharedModule)});`, + `console.log(m.onlineRawUrl(${JSON.stringify(CONTACT_PATH)}));`, + "console.log(m.ONLINE_RAW_BASE_PATH);", + ].join(""); + const [builtUrl, basePath] = execFileSync(process.execPath, ["--input-type=module", "-e", script], { + encoding: "utf8", + env: { ...process.env, SUSTECH_ONLINE_RAW_BASE: MIRROR_BASE }, + }) + .trim() + .split(/\r?\n/); + assert.equal(builtUrl, `${MIRROR_BASE}/${CONTACT_PATH}`); + assert.equal(basePath, "/https://raw.githubusercontent.com/SUSTech-CRA/sustech-online-ng/master/"); +});