Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
"bili-proxy": "./dist/index.js"
},
"scripts": {
"build": "npm --prefix kernel run build && npm --prefix pi-subagents run build && tsup && node scripts/write-build-info.mjs && node scripts/check-dist-import-annotations.mjs && node scripts/check-dist-external-zod.mjs",
"build": "npm --prefix kernel run build && npm --prefix pi-subagents run build && tsup && node scripts/write-build-info.mjs && node scripts/check-dist-import-annotations.mjs && node scripts/check-dist-external.mjs",
"prepare": "node scripts/check-dist-ready.mjs",
"registry:snapshot": "node scripts/update-registry-snapshot.mjs",
"codex-models:snapshot": "node scripts/update-codex-models-snapshot.mjs",
Expand Down
86 changes: 0 additions & 86 deletions scripts/check-dist-external-zod.mjs

This file was deleted.

169 changes: 169 additions & 0 deletions scripts/check-dist-external.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,169 @@
// Build gate: fail if any external bare import survives into dist outside
// the sanctioned allowlist below.
//
// dist must stay self-contained for thin installs — the package copied
// WITHOUT its node_modules tree (dsh profile bare copies, plugin copies,
// bili's own e2e rigs). A STATIC bare import resolves at module load with NO
// degrade path: the process dies before it binds a port and model traffic
// silently goes direct, uncompressed. History of the class this pins:
// #3425/#3511 P0 — jimp inlining leaked 16 static `from"zod"` imports into
// dist (this script started life as the zod-only guard);
// #3516 — jimp itself moved to external + lazy dynamic load (allowlisted);
// #4008 — mysql2 (declared dep, auto-externalized by tsup) arrived with a
// TOP-LEVEL static import in src/storage/mysql.ts; openBackend's
// internal-module dynamic import got hoisted by the single-file bundle
// and the driver import rode along to the top of dist/index.js and
// dist/agent/pi-native.js — every thin install crashed at startup with
// ERR_MODULE_NOT_FOUND even when sqlite was selected. This
// generalization closes the class: ANY new external specifier fails the
// build unless explicitly listed below WITH a reason, and listed
// specifiers may only appear in their sanctioned forms (dynamic-only
// entries keep their lazy-load contract — a static form of them is still
// a violation).
//
// Node builtins (node:* and module.builtinModules) and relative paths are
// fine. Opaque variable specifiers (`const s="zod"; import(s)`) are invisible
// to a text scan by design — that is the sanctioned lazy path (opencode V1
// interop loads the HOST's zod instance through a variable; see
// src/agent/opencode-native.ts). A literal `import("zod")` is NOT sanctioned
// (esbuild would bundle or rewrite it) and fails via the allowlist.

import { builtinModules } from "node:module";
import { readdirSync, readFileSync, realpathSync, statSync } from "node:fs";
import path from "node:path";
import process from "node:process";
import { fileURLToPath } from "node:url";

const NODE_BUILTINS = new Set(builtinModules);

/** Sanctioned external specifiers. kind "static" covers every form that
* resolves at module load (import decl / export-from / require); "dynamic"
* covers import(). `files` restricts an entry to specific dist-relative
* artifact paths when given. Every entry needs a reason — the allowlist is
* evidence, not convenience. */
const ALLOWED_EXTERNALS = [
{
spec: "jimp",
kinds: ["dynamic"],
reason: "#3516 optional image-rescale dependency — loaded at first use (src/image-rescale.ts) and degrades to pass-through when absent; a static form would crash thin installs at module load",
},
{
spec: "react",
kinds: ["static"],
files: ["agent/dsh-native-client.js"],
reason: "#1590 browser CJS bundle for the dsh web profile — react resolves through dsh's own module system at runtime (classic-script wrapper); nothing else may reference it",
},
{
spec: "mysql2/promise",
kinds: ["dynamic"],
reason: "#4008 storage driver — loaded at openBackend(\"mysql\") time only (src/storage/backend.ts); a static form crashes thin installs at module load even when sqlite is selected",
},
];

// Import forms, each capturing the bare specifier. Static forms first; the
// dynamic form is separate because its KIND is what the allowlist polices.
const STATIC_REF_PATTERNS = [
// side-effect: import "x"
/\bimport\s*["']([^"'\n]+)["']/g,
// named: import { a, b as c } from "x" (minified or spaced)
/\bimport\s*\{[^}]*\}\s*from\s*["']([^"'\n]+)["']/g,
// namespace (optionally with named tail): import * as ns from "x"
/\bimport\s*\*\s*as\s+[\w$]+\s*,?\s*(?:\{[^}]*\})?\s*from\s*["']([^"'\n]+)["']/g,
// default + namespace: import def, * as ns from "x"
/\bimport\s+[\w$]+\s*,\s*\*\s*as\s+[\w$]+\s+from\s*["']([^"'\n]+)["']/g,
// default + named: import def, { a } from "x"
/\bimport\s+[\w$]+\s*,\s*\{[^}]*\}\s*from\s*["']([^"'\n]+)["']/g,
// default: import def from "x"
/\bimport\s+[\w$]+\s+from\s*["']([^"'\n]+)["']/g,
// re-export: export * [as ns] from "x" / export { a } from "x"
// (\s* after export and before from: minified output writes
// export{x}from"x" — braces need no separators, unlike identifier-from
// which always has one; exports.foo never matches — the group demands * or {)
/\bexport\s*(?:\*\s*(?:as\s+[\w$]+\s*)?|\{[^}]*\})\s*from\s*["']([^"'\n]+)["']/g,
// CJS require (resolves at call, i.e. factory execution for our bundles)
/\brequire\s*\(\s*["']([^"'\n]+)["']\s*\)/g,
];
const DYNAMIC_REF_PATTERN = /\bimport\s*\(\s*["']([^"'\n]+)["']\s*\)/g;

function lineOf(text, index) {
return text.slice(0, index).split("\n").length;
}

function collectRefs(text, re, kind) {
const refs = [];
re.lastIndex = 0;
let m;
while ((m = re.exec(text)) !== null) {
const spec = m[1];
if (!spec || spec.startsWith(".") || spec.startsWith("/")) continue;
const base = spec.replace(/^node:/, "").split("/")[0];
if (NODE_BUILTINS.has(base)) continue;
refs.push({ spec, kind, line: lineOf(text, m.index) });
}
return refs;
}

/** All bare external refs in a source text (builtins/relative excluded),
* each tagged with the import kind that produced it. */
export function findExternalRefs(text) {
const refs = [];
for (const re of STATIC_REF_PATTERNS) refs.push(...collectRefs(text, re, "static"));
refs.push(...collectRefs(text, DYNAMIC_REF_PATTERN, "dynamic"));
return refs;
}

function classify(refs, relFile) {
const violations = [];
for (const ref of refs) {
const entry = ALLOWED_EXTERNALS.find((e) => e.spec === ref.spec);
if (!entry) {
violations.push(`${ref.kind} ${ref.spec} @ line ${ref.line} — not on the allowlist (bundle it via tsup noExternal, or add it here WITH a reason)`);
continue;
}
if (!entry.kinds.includes(ref.kind)) {
violations.push(`${ref.kind} ${ref.spec} @ line ${ref.line} — allowed kinds: ${entry.kinds.join(", ")} (${entry.reason})`);
continue;
}
if (entry.files && !entry.files.includes(relFile)) {
violations.push(`${ref.kind} ${ref.spec} @ line ${ref.line} — restricted to ${entry.files.join(", ")}`);
}
}
return violations;
}

/** Walk a dist tree and report every disallowed external ref per file. */
export function checkDistExternal(distDir) {
const violations = [];
const walk = (dir) => {
for (const name of readdirSync(dir)) {
const full = path.join(dir, name);
const st = statSync(full);
if (st.isDirectory()) { walk(full); continue; }
if (!name.endsWith(".js")) continue;
const relFile = path.relative(distDir, full).split(path.sep).join("/");
const refs = findExternalRefs(readFileSync(full, "utf8"));
const bad = classify(refs, relFile);
if (bad.length > 0) violations.push({ file: full, refs: bad });
}
};
walk(distDir);
return violations;
}

const invoked = process.argv[1];
// Realpath both sides: Node resolves entry-point symlinks, so argv[1] can differ from import.meta.url
const isMain = !!invoked && realpathSync(fileURLToPath(import.meta.url)) === realpathSync(invoked);
if (isMain) {
const distDir = process.argv[2] ?? "dist";
const violations = checkDistExternal(distDir);
if (violations.length > 0) {
for (const v of violations) {
console.error(`dist-external: ${v.file}:`);
for (const r of v.refs.slice(0, 5)) console.error(` ${r}`);
if (v.refs.length > 5) console.error(` … (+${v.refs.length - 5} more)`);
}
console.error(`dist-external: FAIL — ${violations.reduce((n, v) => n + v.refs.length, 0)} disallowed external import ref(s) across ${violations.length} file(s). dist must stay self-contained for thin installs (no node_modules): bundle new deps via tsup noExternal, keep lazy loads DYNAMIC, or extend the allowlist in scripts/check-dist-external.mjs with a reason (#3511/#3516/#4008 class).`);
process.exit(1);
}
console.log("dist-external: OK");
}
4 changes: 2 additions & 2 deletions scripts/check-dist-thin-install.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@
// 2. every Node-consumed dist/agent/*.js entry IMPORTS in its own process
// without throwing — an undeclared runtime dependency fails here at
// import time, exactly where it would fail in a real consumer's process
// (static analysis like check-dist-external-zod can only see one known
// offender; this catches any future one).
// (the static gate scripts/check-dist-external.mjs pins the allowlist;
// this smoke proves the installed artifact actually RUNS end to end).
// Run after `npm run build` in CI (static-gates). Deliberately NOT part of
// the local build chain or the prepare gate: it needs registry access for
// the package's declared deps and must never slow a consumer install.
Expand Down
30 changes: 23 additions & 7 deletions src/storage/backend.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import type { SqliteEngineName } from "./driver.ts";
import type { MessageContentStore } from "acp-kernel";
import type { Mysql2Module } from "./mysql.js";
import { UnifiedStore } from "./store.ts";

/** #3931 / #3388 step 1 — the storage backend contract.
Expand All @@ -14,10 +15,12 @@ import { UnifiedStore } from "./store.ts";
*
* `sqlite` is the default. `mysql` is IMPLEMENTED (#3388 step 2, #3931):
* MysqlBackend implements the full contract against MySQL 8.0 (mysql2
* driver, a bundled dependency — #3931 owner decision). The module loads
* lazily inside openBackend("mysql", dsn), so boots that do not select
* mysql never load mysql2; the `indexPath` argument doubles as the DSN for
* mysql, mirroring how SqliteBackend.open consumes the index path.
* driver — #3931 owner decision). Driver AND module load lazily inside
* openBackend("mysql", dsn) through dynamic imports that survive bundling
* as real runtime imports (#4008: a static import of the driver would hoist
* to the top of dist and crash thin installs), so boots that do not select
* mysql never resolve mysql2; the `indexPath` argument doubles as the DSN
* for mysql, mirroring how SqliteBackend.open consumes the index path.
*/

export type {
Expand Down Expand Up @@ -299,8 +302,9 @@ export function listBackendNames(): string[] {
* is the DSN (mysql://user:pass@host:port/db), the pool connects lazily,
* and `init()` (DDL + schema stamp) must run once before first use — the
* contract suite does this in `before`. Unknown names throw with the valid
* list. Async because the mysql module is dynamically imported (#3388
* step-2 decision: bundled dependency, lazy load). */
* list. Async because the mysql driver + module are dynamically imported at
* open time (#3388 step-2 decision; #4008: the driver stays external to the
* bundle and must never be statically imported). */
export async function openBackend(name: string, indexPath: string, engine?: SqliteEngineName): Promise<StorageBackend> {
if (name === "sqlite") return SqliteBackend.open(indexPath, engine);
if (name === "mysql") {
Expand All @@ -309,8 +313,20 @@ export async function openBackend(name: string, indexPath: string, engine?: Sqli
`[storage] backend "mysql" needs a DSN — set BILI_MYSQL_DSN or storage.mysql.dsn (mysql://user:pass@host:3306/db); runs on "sqlite" until then (#3388/#3931)`,
);
}
// #4008: literal dynamic import of an EXTERNAL specifier — it survives
// the single-file bundle as a real runtime import and resolves only
// now, never at module load. The pool itself still opens lazily; this
// only moves driver resolution from boot time to open time.
let driver: Mysql2Module;
try {
driver = await import("mysql2/promise");
} catch (err) {
throw new Error(
`[storage] mysql backend selected but the mysql2 driver cannot be loaded (${err instanceof Error ? err.message : String(err)}) — installs without a node_modules tree cannot run the mysql backend; set storage.backend="sqlite" or install with dependencies (#4008)`,
);
}
const { MysqlBackend } = await import("./mysql.js");
return new MysqlBackend(indexPath);
return new MysqlBackend(indexPath, driver);
}
throw new Error(`[storage] unknown backend "${name}" — valid names: ${REGISTERED_BACKENDS.join(", ")}`);
}
Expand Down
26 changes: 18 additions & 8 deletions src/storage/mysql.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,10 @@
* MySQL backend for the storage contract (#3388 step 2, #3931).
*
* Implements the same StorageBackend operation surface as the sqlite
* store, against MySQL 8.0 (mysql2 driver, bundled dependency per the
* 2026-10-20 owner decision — see the #3931 comment; this module is only
* ever reached through openBackend("mysql", dsn), so boots that do not
* select mysql never load it).
* store, against MySQL 8.0 (mysql2 driver per the 2026-10-20 owner decision
* — see the #3931 comment). The driver stays EXTERNAL to the bundle (#4008):
* openBackend("mysql", dsn) loads it lazily and injects it into the
* constructor, so boots that do not select mysql never resolve it.
*
* Representation discipline mirrors src/storage/schema.ts: verbatim facts
* (payload_json / content bytes / *_json bags) are the truth; scalar
Expand All @@ -27,7 +27,7 @@
* hardening plan).
*/

import { createPool, type Pool, type PoolConnection, type PoolOptions } from "mysql2/promise";
import type { Pool, PoolConnection, PoolOptions } from "mysql2/promise";
import { mkdirSync, writeFileSync, appendFileSync, readFileSync, existsSync } from "node:fs";
import { join } from "node:path";
import { log as loggerLog } from "../logger.js";
Expand All @@ -46,6 +46,15 @@ import type {
IngestSessionInput,
} from "./store.js";

/** The mysql2/promise driver module, injected by openBackend() at open time.
* THIS FILE MUST NEVER HOLD A TOP-LEVEL VALUE IMPORT OF THE DRIVER (#4008):
* under tsup's single-file bundle (splitting:false) any value import here
* hoists to the top of dist/index.js and resolves at MODULE LOAD — crashing
* every install without a node_modules tree with ERR_MODULE_NOT_FOUND even
* when sqlite is selected. Type imports erase at compile time and are safe.
* Same discipline as the jimp lazy load (src/image-rescale.ts, #3516). */
export type Mysql2Module = typeof import("mysql2/promise");

const MYSQL_SCHEMA_VERSION = 7;

/** Fresh-install DDL: the schema.ts v7 shape in MySQL dialect. LONGTEXT for
Expand Down Expand Up @@ -222,9 +231,10 @@ function nullableStr(v: unknown): string | null {
export class MysqlBackend implements StorageBackend {
private readonly pool: Pool;

constructor(dsnOrOpts: string | PoolOptions) {
this.pool =
typeof dsnOrOpts === "string" ? createPool(dsnOrOpts) : createPool(dsnOrOpts);
constructor(dsnOrOpts: string | PoolOptions, driver: Mysql2Module) {
this.pool = typeof dsnOrOpts === "string"
? driver.createPool(dsnOrOpts)
: driver.createPool(dsnOrOpts);
}

/** Applies the DDL + schema-version stamp. Idempotent. */
Expand Down
Loading
Loading