Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion skills/reverse-engineering-saas/scripts/build-report.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,8 @@ const UNSEEN = /^\s*(not tested|not measured|not reached|not studied|not seen|no
const GALLERY_LIMIT = 6;

const escapeHtml = (text) => String(text).replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;').replace(/"/g, '&quot;');
const read = (kb, file) => (existsSync(join(kb, file)) ? readFileSync(join(kb, file), 'utf8') : '');
// Files saved on Windows end lines with CRLF; the parsers expect LF.
const read = (kb, file) => (existsSync(join(kb, file)) ? readFileSync(join(kb, file), 'utf8').replace(/\r\n?/g, '\n') : '');
// A comment on its own line goes with its line, so tables around it stay intact.
const stripComments = (text) => text.replace(/^[ \t]*<!--[\s\S]*?-->[ \t]*\r?\n/gm, '').replace(/<!--[\s\S]*?-->/g, '');
const listDir = (kb, dir, filter = () => true) =>
Expand Down
27 changes: 15 additions & 12 deletions skills/reverse-engineering-saas/scripts/check-kb.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
// evidence tags, the change ledger and STATUS.md.
// Exit code 1 when there are errors, 2 on bad usage.
import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs';
import { dirname, join, relative, resolve } from 'node:path';
import { dirname, join, relative, resolve, sep } from 'node:path';

const USAGE = 'Usage: node check-kb.mjs <kb-dir> [--closing] [--next] [--help]';
const ID_PATTERN = /\b(FL|SRC|EV|BR|F|S|C|R|E|B|Q|L)-(\d{3})\b/g;
Expand Down Expand Up @@ -35,6 +35,9 @@ const ROW_ID_FILES = {
const HEADING_ID_FILES = { 'findings.md': ['B'], 'questions.md': ['Q'], 'clone/barriers.md': ['BR'] };
const ARTIFACT_DIRS = ['screens', 'components', 'flows', 'topics'];

// Files saved on Windows end lines with CRLF and relative() returns backslashes; the checks expect LF and "/".
const readText = (path) => readFileSync(path, 'utf8').replace(/\r\n?/g, '\n');

const errors = [];
let closingRun = false;
const warnings = [];
Expand All @@ -46,7 +49,7 @@ function walk(dir, root = dir) {
if (SKIP_DIRS.has(entry)) continue;
const full = join(dir, entry);
if (statSync(full).isDirectory()) files.push(...walk(full, root));
else files.push(relative(root, full));
else files.push(relative(root, full).split(sep).join('/'));
}
return files;
}
Expand Down Expand Up @@ -110,7 +113,7 @@ function collectDefinitions(kb, files) {
}
const markdownFiles = files.filter((file) => file.endsWith('.md'));
for (const file of markdownFiles) {
const lines = stripIgnored(readFileSync(join(kb, file), 'utf8')).split('\n');
const lines = stripIgnored(readText(join(kb, file))).split('\n');
const rowPrefixes = ROW_ID_FILES[file] ?? [];
const headingPrefixes = file.startsWith('rules/') ? ['R'] : HEADING_ID_FILES[file] ?? [];
lines.forEach((line, index) => {
Expand All @@ -137,7 +140,7 @@ function checkDuplicates() {

function checkReferencesAndLinks(kb, files) {
for (const file of files.filter((name) => SCANNED_EXTENSIONS.has(extensionOf(name)))) {
const text = stripIgnored(readFileSync(join(kb, file), 'utf8'));
const text = stripIgnored(readText(join(kb, file)));
const lines = text.split('\n');
lines.forEach((line, index) => {
const where = `${file}:${index + 1}`;
Expand Down Expand Up @@ -174,7 +177,7 @@ function checkEvidenceSections(kb, files) {
...files.filter((name) => name === 'clone/barriers.md').map((file) => [file, 'BR']),
];
for (const [file, prefix] of targets) {
const text = stripIgnored(readFileSync(join(kb, file), 'utf8'));
const text = stripIgnored(readText(join(kb, file)));
const start = new RegExp(`^(?=#{1,6}\\s+${prefix}-\\d{3}\\b)`, 'm');
const sections = text.split(start).filter((section) => new RegExp(`^#{1,6}\\s+${prefix}-\\d{3}`).test(section));
for (const section of sections) {
Expand All @@ -189,7 +192,7 @@ function checkEvidenceSections(kb, files) {
function checkArtifacts(files, kb) {
for (const file of files) {
if (!file.endsWith('.md') || !ARTIFACT_DIRS.includes(dirname(file))) continue;
if (!/^##\s+Not tested\b/m.test(readFileSync(join(kb, file), 'utf8'))) warnings.push(`${file}: no "## Not tested" section`);
if (!/^##\s+Not tested\b/m.test(readText(join(kb, file)))) warnings.push(`${file}: no "## Not tested" section`);
}
}

Expand All @@ -199,7 +202,7 @@ function checkLedger(kb, closing) {
errors.push('sandbox/ledger.csv is missing');
return { rows: 0, open: 0 };
}
const [header = [], ...rows] = parseCsv(readFileSync(path, 'utf8'));
const [header = [], ...rows] = parseCsv(readText(path));
if (header.join(',') !== LEDGER_COLUMNS.join(',')) {
errors.push(`sandbox/ledger.csv: header must be ${LEDGER_COLUMNS.join(',')}`);
return { rows: rows.length, open: 0 };
Expand Down Expand Up @@ -231,7 +234,7 @@ function checkStatus(kb, files, ledger) {
errors.push('STATUS.md is missing');
return;
}
const text = readFileSync(path, 'utf8');
const text = readText(path);
const updated = text.match(/^Updated:\s*(\d{4}-\d{2}-\d{2})/m);
if (!updated) warnings.push('STATUS.md: no "Updated: YYYY-MM-DD" line');
for (const section of ['Summary', 'Next actions', 'Needs human', 'Blocked']) {
Expand All @@ -255,7 +258,7 @@ function checkEvidenceIndex(kb, files) {
if (evidence.length) warnings.push(`${EVIDENCE_INDEX} is missing`);
return;
}
const [, ...rows] = parseCsv(readFileSync(path, 'utf8'));
const [, ...rows] = parseCsv(readText(path));
const indexed = new Set(rows.map((cells) => (cells[0] ?? '').trim()));
for (const file of evidence) {
const id = file.match(/EV-\d{3}/)[0];
Expand All @@ -270,7 +273,7 @@ function checkEvidenceIndex(kb, files) {
function checkEntities(kb, files) {
for (const file of files.filter((name) => name.startsWith('data-model/entities/') && name.endsWith('.md'))) {
let section = '';
stripIgnored(readFileSync(join(kb, file), 'utf8'))
stripIgnored(readText(join(kb, file)))
.split('\n')
.forEach((line, index) => {
const heading = line.match(/^##\s+(.+)/);
Expand All @@ -286,7 +289,7 @@ function checkEntities(kb, files) {
function checkCoverageRows(kb) {
const path = join(kb, 'inventory', 'coverage.md');
if (!existsSync(path)) return;
const covered = new Set(stripIgnored(readFileSync(path, 'utf8')).match(/^\|\s*F-\d{3}/gm)?.map((cell) => cell.replace(/^\|\s*/, '')) ?? []);
const covered = new Set(stripIgnored(readText(path)).match(/^\|\s*F-\d{3}/gm)?.map((cell) => cell.replace(/^\|\s*/, '')) ?? []);
for (const [id, places] of definitions) {
if (id.startsWith('F-') && places.some((place) => place.file === 'inventory/features.md') && !covered.has(id)) {
warnings.push(`inventory/coverage.md: ${id} has no row`);
Expand All @@ -298,7 +301,7 @@ function coverageSummary(kb) {
const path = join(kb, 'inventory', 'coverage.md');
const counts = Object.fromEntries([...COVERAGE_VALUES].map((value) => [value, 0]));
if (!existsSync(path)) return counts;
stripIgnored(readFileSync(path, 'utf8'))
stripIgnored(readText(path))
.split('\n')
.forEach((line, index) => {
if (!/^\|\s*F-\d{3}\b/.test(line)) return;
Expand Down
8 changes: 4 additions & 4 deletions skills/reverse-engineering-saas/scripts/pack-kb.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
// check-kb and shows how the study left the product. No dependencies.
import { spawnSync } from 'node:child_process';
import { existsSync, readdirSync, readFileSync, statSync, writeFileSync } from 'node:fs';
import { basename, dirname, join, relative, resolve } from 'node:path';
import { basename, dirname, join, relative, resolve, sep } from 'node:path';
import { fileURLToPath } from 'node:url';
import { deflateRawSync } from 'node:zlib';
import { buildReport } from './build-report.mjs';
Expand Down Expand Up @@ -97,7 +97,7 @@ function walk(dir, root, skipDirs) {
for (const entry of readdirSync(dir).sort()) {
if (ALWAYS_SKIP.has(entry)) continue;
const full = join(dir, entry);
const rel = relative(root, full);
const rel = relative(root, full).split(sep).join('/');
if (statSync(full).isDirectory()) {
if (!skipDirs.includes(rel)) files.push(...walk(full, root, skipDirs));
} else {
Expand Down Expand Up @@ -142,7 +142,7 @@ const csvCell = (cell) => (/[",\r\n]/.test(cell) ? `"${cell.replaceAll('"', '""'

// The ledger as the zip holds it: object IDs and locations blanked, URLs replaced.
function sanitizedLedger(path) {
const [header = [], ...rows] = parseCsv(readFileSync(path, 'utf8'));
const [header = [], ...rows] = parseCsv(readFileSync(path, 'utf8').replace(/\r\n?/g, '\n'));
const blank = new Set(LEDGER_PRIVATE_COLUMNS.map((column) => header.indexOf(column)).filter((index) => index >= 0));
const clean = rows.map((cells) => cells.map((cell, index) => (blank.has(index) ? '' : cell.replace(/\b(?:https?|wss?):\/\/[^\s",]+/gi, '[url]'))));
return Buffer.from(`${[header, ...clean].map((cells) => cells.map(csvCell).join(',')).join('\n')}\n`);
Expand Down Expand Up @@ -204,7 +204,7 @@ function main() {

const skipped = options.full ? [] : PRIVATE_DIRS.filter((dir) => existsSync(join(kb, dir)));
const folder = basename(kb);
const entries = walk(kb, kb, skipped).map((file) => ({ name: `${folder}/${file.split('\\').join('/')}`, data: readFileSync(join(kb, file)) }));
const entries = walk(kb, kb, skipped).map((file) => ({ name: `${folder}/${file}`, data: readFileSync(join(kb, file)) }));
const ledger = skipped.includes('sandbox') && existsSync(join(kb, LEDGER));
if (ledger) entries.push({ name: `${folder}/${LEDGER}`, data: sanitizedLedger(join(kb, LEDGER)) });
const zip = buildZip(entries);
Expand Down
37 changes: 33 additions & 4 deletions tests/kb-scripts.test.mjs
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
// Tests for init-kb.mjs and check-kb.mjs. No dependencies.
import assert from 'node:assert/strict';
import { spawnSync } from 'node:child_process';
import { appendFileSync, existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
import { appendFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { dirname, join } from 'node:path';
import { afterEach, beforeEach, test } from 'node:test';
Expand Down Expand Up @@ -156,7 +156,7 @@ test('--git creates a repository for the KB even inside another repository', ()
test('--git without a git identity still creates the repository', () => {
const target = join(dir, 're-no-identity');
const empty = join(dir, 'empty-home');
spawnSync('mkdir', ['-p', empty]);
mkdirSync(empty, { recursive: true });
// useConfigOnly stops git from guessing an identity from the OS user, as on CI runners.
const env = { ...process.env, HOME: empty, XDG_CONFIG_HOME: empty, GIT_CONFIG_NOSYSTEM: '1', GIT_CONFIG_COUNT: '1', GIT_CONFIG_KEY_0: 'user.useConfigOnly', GIT_CONFIG_VALUE_0: 'true' };
for (const key of ['GIT_AUTHOR_NAME', 'GIT_AUTHOR_EMAIL', 'GIT_COMMITTER_NAME', 'GIT_COMMITTER_EMAIL', 'EMAIL']) delete env[key];
Expand Down Expand Up @@ -216,9 +216,13 @@ test('init-kb defaults to ./re-<product-slug>', () => {
});

// Lists and verifies a zip with Python's zipfile, an independent implementation.
// "python3" is a Microsoft Store stub on Windows: use the first interpreter that actually runs.
const python = ['python3', 'python', 'py'].find((command) => spawnSync(command, ['-c', 'import zipfile'], { encoding: 'utf8' }).status === 0);

const readZip = (file) => {
assert.ok(python, 'Python 3 is needed to verify zips with an independent implementation');
const code = 'import json,sys,zipfile\nz=zipfile.ZipFile(sys.argv[1])\nprint(json.dumps({"bad": z.testzip(), "names": z.namelist(), "build": z.read([n for n in z.namelist() if n.endswith("/BUILD.md")][0]).decode()}))';
const result = spawnSync('python3', ['-c', code, file], { encoding: 'utf8' });
const result = spawnSync(python, ['-c', code, file], { encoding: 'utf8' });
assert.equal(result.status, 0, result.stderr);
return JSON.parse(result.stdout);
};
Expand Down Expand Up @@ -255,7 +259,7 @@ test('an unzipped handoff passes check-kb --closing; its ledger has no IDs, loca
assert.equal(packed.status, 0, packed.stdout + packed.stderr);
assert.match(packed.stdout, /Kept sandbox\/ledger\.csv without object IDs/);
const target = join(dir, 'unzipped');
const unzip = spawnSync('python3', ['-c', 'import sys,zipfile\nzipfile.ZipFile(sys.argv[1]).extractall(sys.argv[2])', out, target], { encoding: 'utf8' });
const unzip = spawnSync(python, ['-c', 'import sys,zipfile\nzipfile.ZipFile(sys.argv[1]).extractall(sys.argv[2])', out, target], { encoding: 'utf8' });
assert.equal(unzip.status, 0, unzip.stderr);

const ledger = readFileSync(join(target, 're-acme', 'sandbox', 'ledger.csv'), 'utf8');
Expand Down Expand Up @@ -367,3 +371,28 @@ test('--next prints the next free ID per prefix', () => {
assert.match(result.stdout, /EV-013/);
assert.match(result.stdout, /R-001/);
});

test('check-kb and build-report accept files with CRLF line endings', () => {
const crlf = (text) => text.replace(/\n/g, '\r\n');
writeFileSync(join(kb, 'evidence/EV-001-deal-won.png'), 'png');
appendFileSync(join(kb, 'evidence/index.csv'), crlf('EV-001,2026-10-15,s01,/deal/:id,won deal,yes\n'));
write('rules/deals.md', crlf('# Rules: deals\n\n### R-001 Won deals leave the board\n- Given a deal\n- When won\n- Then hidden\n- Evidence: [obs EV-001]\n'));
const result = check();
assert.equal(result.status, 0, result.stdout);
assert.doesNotMatch(result.stdout, /R-001 has no evidence tag/);
const report = run('build-report.mjs', kb);
assert.equal(report.status, 0, report.stderr);
assert.match(readFileSync(join(kb, 'REPORT.html'), 'utf8'), /Won deals leave the board/);
});

test('pack-kb keeps a CRLF ledger readable and writes zip entries with forward slashes', () => {
write('sandbox/ledger.csv', 'id,time,class,action,object,location,object_id,before,undo,status,note\r\nL-001,2026-10-16T10:00:00Z,A2,create,task,Acme > Sandbox,4471,,Delete it at https://acme.example.com/deal/4471,removed,ok\r\n');
appendFileSync(join(kb, 'STATUS.md'), '- L-001 removed and verified.\n');
const out = join(dir, 'crlf.zip');
const packed = run('pack-kb.mjs', kb, '--out', out);
assert.equal(packed.status, 0, packed.stdout + packed.stderr);
const zip = readZip(out);
assert.ok(zip.names.every((name) => !name.includes('\\')), 'zip entry names use forward slashes');
assert.ok(zip.names.includes('re-acme/clone/prd.md'));
assert.ok(zip.names.includes('re-acme/sandbox/ledger.csv'));
});