diff --git a/skills/reverse-engineering-saas/scripts/build-report.mjs b/skills/reverse-engineering-saas/scripts/build-report.mjs index 132d2ed..11d9e7a 100644 --- a/skills/reverse-engineering-saas/scripts/build-report.mjs +++ b/skills/reverse-engineering-saas/scripts/build-report.mjs @@ -18,7 +18,8 @@ const UNSEEN = /^\s*(not tested|not measured|not reached|not studied|not seen|no const GALLERY_LIMIT = 6; const escapeHtml = (text) => String(text).replace(/&/g, '&').replace(//g, '>').replace(/"/g, '"'); -const read = (kb, file) => (existsSync(join(kb, file)) ? readFileSync(join(kb, file), 'utf8') : ''); +// Files saved on Windows end lines with CRLF; the parsers expect LF. +const read = (kb, file) => (existsSync(join(kb, file)) ? readFileSync(join(kb, file), 'utf8').replace(/\r\n?/g, '\n') : ''); // A comment on its own line goes with its line, so tables around it stay intact. const stripComments = (text) => text.replace(/^[ \t]*[ \t]*\r?\n/gm, '').replace(//g, ''); const listDir = (kb, dir, filter = () => true) => diff --git a/skills/reverse-engineering-saas/scripts/check-kb.mjs b/skills/reverse-engineering-saas/scripts/check-kb.mjs index 4d00f7e..e2c5b81 100644 --- a/skills/reverse-engineering-saas/scripts/check-kb.mjs +++ b/skills/reverse-engineering-saas/scripts/check-kb.mjs @@ -3,7 +3,7 @@ // evidence tags, the change ledger and STATUS.md. // Exit code 1 when there are errors, 2 on bad usage. import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs'; -import { dirname, join, relative, resolve } from 'node:path'; +import { dirname, join, relative, resolve, sep } from 'node:path'; const USAGE = 'Usage: node check-kb.mjs [--closing] [--next] [--help]'; const ID_PATTERN = /\b(FL|SRC|EV|BR|F|S|C|R|E|B|Q|L)-(\d{3})\b/g; @@ -35,6 +35,9 @@ const ROW_ID_FILES = { const HEADING_ID_FILES = { 'findings.md': ['B'], 'questions.md': ['Q'], 'clone/barriers.md': ['BR'] }; const ARTIFACT_DIRS = ['screens', 'components', 'flows', 'topics']; +// Files saved on Windows end lines with CRLF and relative() returns backslashes; the checks expect LF and "/". +const readText = (path) => readFileSync(path, 'utf8').replace(/\r\n?/g, '\n'); + const errors = []; let closingRun = false; const warnings = []; @@ -46,7 +49,7 @@ function walk(dir, root = dir) { if (SKIP_DIRS.has(entry)) continue; const full = join(dir, entry); if (statSync(full).isDirectory()) files.push(...walk(full, root)); - else files.push(relative(root, full)); + else files.push(relative(root, full).split(sep).join('/')); } return files; } @@ -110,7 +113,7 @@ function collectDefinitions(kb, files) { } const markdownFiles = files.filter((file) => file.endsWith('.md')); for (const file of markdownFiles) { - const lines = stripIgnored(readFileSync(join(kb, file), 'utf8')).split('\n'); + const lines = stripIgnored(readText(join(kb, file))).split('\n'); const rowPrefixes = ROW_ID_FILES[file] ?? []; const headingPrefixes = file.startsWith('rules/') ? ['R'] : HEADING_ID_FILES[file] ?? []; lines.forEach((line, index) => { @@ -137,7 +140,7 @@ function checkDuplicates() { function checkReferencesAndLinks(kb, files) { for (const file of files.filter((name) => SCANNED_EXTENSIONS.has(extensionOf(name)))) { - const text = stripIgnored(readFileSync(join(kb, file), 'utf8')); + const text = stripIgnored(readText(join(kb, file))); const lines = text.split('\n'); lines.forEach((line, index) => { const where = `${file}:${index + 1}`; @@ -174,7 +177,7 @@ function checkEvidenceSections(kb, files) { ...files.filter((name) => name === 'clone/barriers.md').map((file) => [file, 'BR']), ]; for (const [file, prefix] of targets) { - const text = stripIgnored(readFileSync(join(kb, file), 'utf8')); + const text = stripIgnored(readText(join(kb, file))); const start = new RegExp(`^(?=#{1,6}\\s+${prefix}-\\d{3}\\b)`, 'm'); const sections = text.split(start).filter((section) => new RegExp(`^#{1,6}\\s+${prefix}-\\d{3}`).test(section)); for (const section of sections) { @@ -189,7 +192,7 @@ function checkEvidenceSections(kb, files) { function checkArtifacts(files, kb) { for (const file of files) { if (!file.endsWith('.md') || !ARTIFACT_DIRS.includes(dirname(file))) continue; - if (!/^##\s+Not tested\b/m.test(readFileSync(join(kb, file), 'utf8'))) warnings.push(`${file}: no "## Not tested" section`); + if (!/^##\s+Not tested\b/m.test(readText(join(kb, file)))) warnings.push(`${file}: no "## Not tested" section`); } } @@ -199,7 +202,7 @@ function checkLedger(kb, closing) { errors.push('sandbox/ledger.csv is missing'); return { rows: 0, open: 0 }; } - const [header = [], ...rows] = parseCsv(readFileSync(path, 'utf8')); + const [header = [], ...rows] = parseCsv(readText(path)); if (header.join(',') !== LEDGER_COLUMNS.join(',')) { errors.push(`sandbox/ledger.csv: header must be ${LEDGER_COLUMNS.join(',')}`); return { rows: rows.length, open: 0 }; @@ -231,7 +234,7 @@ function checkStatus(kb, files, ledger) { errors.push('STATUS.md is missing'); return; } - const text = readFileSync(path, 'utf8'); + const text = readText(path); const updated = text.match(/^Updated:\s*(\d{4}-\d{2}-\d{2})/m); if (!updated) warnings.push('STATUS.md: no "Updated: YYYY-MM-DD" line'); for (const section of ['Summary', 'Next actions', 'Needs human', 'Blocked']) { @@ -255,7 +258,7 @@ function checkEvidenceIndex(kb, files) { if (evidence.length) warnings.push(`${EVIDENCE_INDEX} is missing`); return; } - const [, ...rows] = parseCsv(readFileSync(path, 'utf8')); + const [, ...rows] = parseCsv(readText(path)); const indexed = new Set(rows.map((cells) => (cells[0] ?? '').trim())); for (const file of evidence) { const id = file.match(/EV-\d{3}/)[0]; @@ -270,7 +273,7 @@ function checkEvidenceIndex(kb, files) { function checkEntities(kb, files) { for (const file of files.filter((name) => name.startsWith('data-model/entities/') && name.endsWith('.md'))) { let section = ''; - stripIgnored(readFileSync(join(kb, file), 'utf8')) + stripIgnored(readText(join(kb, file))) .split('\n') .forEach((line, index) => { const heading = line.match(/^##\s+(.+)/); @@ -286,7 +289,7 @@ function checkEntities(kb, files) { function checkCoverageRows(kb) { const path = join(kb, 'inventory', 'coverage.md'); if (!existsSync(path)) return; - const covered = new Set(stripIgnored(readFileSync(path, 'utf8')).match(/^\|\s*F-\d{3}/gm)?.map((cell) => cell.replace(/^\|\s*/, '')) ?? []); + const covered = new Set(stripIgnored(readText(path)).match(/^\|\s*F-\d{3}/gm)?.map((cell) => cell.replace(/^\|\s*/, '')) ?? []); for (const [id, places] of definitions) { if (id.startsWith('F-') && places.some((place) => place.file === 'inventory/features.md') && !covered.has(id)) { warnings.push(`inventory/coverage.md: ${id} has no row`); @@ -298,7 +301,7 @@ function coverageSummary(kb) { const path = join(kb, 'inventory', 'coverage.md'); const counts = Object.fromEntries([...COVERAGE_VALUES].map((value) => [value, 0])); if (!existsSync(path)) return counts; - stripIgnored(readFileSync(path, 'utf8')) + stripIgnored(readText(path)) .split('\n') .forEach((line, index) => { if (!/^\|\s*F-\d{3}\b/.test(line)) return; diff --git a/skills/reverse-engineering-saas/scripts/pack-kb.mjs b/skills/reverse-engineering-saas/scripts/pack-kb.mjs index 2800447..3a8a693 100644 --- a/skills/reverse-engineering-saas/scripts/pack-kb.mjs +++ b/skills/reverse-engineering-saas/scripts/pack-kb.mjs @@ -6,7 +6,7 @@ // check-kb and shows how the study left the product. No dependencies. import { spawnSync } from 'node:child_process'; import { existsSync, readdirSync, readFileSync, statSync, writeFileSync } from 'node:fs'; -import { basename, dirname, join, relative, resolve } from 'node:path'; +import { basename, dirname, join, relative, resolve, sep } from 'node:path'; import { fileURLToPath } from 'node:url'; import { deflateRawSync } from 'node:zlib'; import { buildReport } from './build-report.mjs'; @@ -97,7 +97,7 @@ function walk(dir, root, skipDirs) { for (const entry of readdirSync(dir).sort()) { if (ALWAYS_SKIP.has(entry)) continue; const full = join(dir, entry); - const rel = relative(root, full); + const rel = relative(root, full).split(sep).join('/'); if (statSync(full).isDirectory()) { if (!skipDirs.includes(rel)) files.push(...walk(full, root, skipDirs)); } else { @@ -142,7 +142,7 @@ const csvCell = (cell) => (/[",\r\n]/.test(cell) ? `"${cell.replaceAll('"', '""' // The ledger as the zip holds it: object IDs and locations blanked, URLs replaced. function sanitizedLedger(path) { - const [header = [], ...rows] = parseCsv(readFileSync(path, 'utf8')); + const [header = [], ...rows] = parseCsv(readFileSync(path, 'utf8').replace(/\r\n?/g, '\n')); const blank = new Set(LEDGER_PRIVATE_COLUMNS.map((column) => header.indexOf(column)).filter((index) => index >= 0)); const clean = rows.map((cells) => cells.map((cell, index) => (blank.has(index) ? '' : cell.replace(/\b(?:https?|wss?):\/\/[^\s",]+/gi, '[url]')))); return Buffer.from(`${[header, ...clean].map((cells) => cells.map(csvCell).join(',')).join('\n')}\n`); @@ -204,7 +204,7 @@ function main() { const skipped = options.full ? [] : PRIVATE_DIRS.filter((dir) => existsSync(join(kb, dir))); const folder = basename(kb); - const entries = walk(kb, kb, skipped).map((file) => ({ name: `${folder}/${file.split('\\').join('/')}`, data: readFileSync(join(kb, file)) })); + const entries = walk(kb, kb, skipped).map((file) => ({ name: `${folder}/${file}`, data: readFileSync(join(kb, file)) })); const ledger = skipped.includes('sandbox') && existsSync(join(kb, LEDGER)); if (ledger) entries.push({ name: `${folder}/${LEDGER}`, data: sanitizedLedger(join(kb, LEDGER)) }); const zip = buildZip(entries); diff --git a/tests/kb-scripts.test.mjs b/tests/kb-scripts.test.mjs index 3b912eb..572b1bb 100644 --- a/tests/kb-scripts.test.mjs +++ b/tests/kb-scripts.test.mjs @@ -1,7 +1,7 @@ // Tests for init-kb.mjs and check-kb.mjs. No dependencies. import assert from 'node:assert/strict'; import { spawnSync } from 'node:child_process'; -import { appendFileSync, existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { appendFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { dirname, join } from 'node:path'; import { afterEach, beforeEach, test } from 'node:test'; @@ -156,7 +156,7 @@ test('--git creates a repository for the KB even inside another repository', () test('--git without a git identity still creates the repository', () => { const target = join(dir, 're-no-identity'); const empty = join(dir, 'empty-home'); - spawnSync('mkdir', ['-p', empty]); + mkdirSync(empty, { recursive: true }); // useConfigOnly stops git from guessing an identity from the OS user, as on CI runners. const env = { ...process.env, HOME: empty, XDG_CONFIG_HOME: empty, GIT_CONFIG_NOSYSTEM: '1', GIT_CONFIG_COUNT: '1', GIT_CONFIG_KEY_0: 'user.useConfigOnly', GIT_CONFIG_VALUE_0: 'true' }; for (const key of ['GIT_AUTHOR_NAME', 'GIT_AUTHOR_EMAIL', 'GIT_COMMITTER_NAME', 'GIT_COMMITTER_EMAIL', 'EMAIL']) delete env[key]; @@ -216,9 +216,13 @@ test('init-kb defaults to ./re-', () => { }); // Lists and verifies a zip with Python's zipfile, an independent implementation. +// "python3" is a Microsoft Store stub on Windows: use the first interpreter that actually runs. +const python = ['python3', 'python', 'py'].find((command) => spawnSync(command, ['-c', 'import zipfile'], { encoding: 'utf8' }).status === 0); + const readZip = (file) => { + assert.ok(python, 'Python 3 is needed to verify zips with an independent implementation'); const code = 'import json,sys,zipfile\nz=zipfile.ZipFile(sys.argv[1])\nprint(json.dumps({"bad": z.testzip(), "names": z.namelist(), "build": z.read([n for n in z.namelist() if n.endswith("/BUILD.md")][0]).decode()}))'; - const result = spawnSync('python3', ['-c', code, file], { encoding: 'utf8' }); + const result = spawnSync(python, ['-c', code, file], { encoding: 'utf8' }); assert.equal(result.status, 0, result.stderr); return JSON.parse(result.stdout); }; @@ -255,7 +259,7 @@ test('an unzipped handoff passes check-kb --closing; its ledger has no IDs, loca assert.equal(packed.status, 0, packed.stdout + packed.stderr); assert.match(packed.stdout, /Kept sandbox\/ledger\.csv without object IDs/); const target = join(dir, 'unzipped'); - const unzip = spawnSync('python3', ['-c', 'import sys,zipfile\nzipfile.ZipFile(sys.argv[1]).extractall(sys.argv[2])', out, target], { encoding: 'utf8' }); + const unzip = spawnSync(python, ['-c', 'import sys,zipfile\nzipfile.ZipFile(sys.argv[1]).extractall(sys.argv[2])', out, target], { encoding: 'utf8' }); assert.equal(unzip.status, 0, unzip.stderr); const ledger = readFileSync(join(target, 're-acme', 'sandbox', 'ledger.csv'), 'utf8'); @@ -367,3 +371,28 @@ test('--next prints the next free ID per prefix', () => { assert.match(result.stdout, /EV-013/); assert.match(result.stdout, /R-001/); }); + +test('check-kb and build-report accept files with CRLF line endings', () => { + const crlf = (text) => text.replace(/\n/g, '\r\n'); + writeFileSync(join(kb, 'evidence/EV-001-deal-won.png'), 'png'); + appendFileSync(join(kb, 'evidence/index.csv'), crlf('EV-001,2026-10-15,s01,/deal/:id,won deal,yes\n')); + write('rules/deals.md', crlf('# Rules: deals\n\n### R-001 Won deals leave the board\n- Given a deal\n- When won\n- Then hidden\n- Evidence: [obs EV-001]\n')); + const result = check(); + assert.equal(result.status, 0, result.stdout); + assert.doesNotMatch(result.stdout, /R-001 has no evidence tag/); + const report = run('build-report.mjs', kb); + assert.equal(report.status, 0, report.stderr); + assert.match(readFileSync(join(kb, 'REPORT.html'), 'utf8'), /Won deals leave the board/); +}); + +test('pack-kb keeps a CRLF ledger readable and writes zip entries with forward slashes', () => { + write('sandbox/ledger.csv', 'id,time,class,action,object,location,object_id,before,undo,status,note\r\nL-001,2026-10-16T10:00:00Z,A2,create,task,Acme > Sandbox,4471,,Delete it at https://acme.example.com/deal/4471,removed,ok\r\n'); + appendFileSync(join(kb, 'STATUS.md'), '- L-001 removed and verified.\n'); + const out = join(dir, 'crlf.zip'); + const packed = run('pack-kb.mjs', kb, '--out', out); + assert.equal(packed.status, 0, packed.stdout + packed.stderr); + const zip = readZip(out); + assert.ok(zip.names.every((name) => !name.includes('\\')), 'zip entry names use forward slashes'); + assert.ok(zip.names.includes('re-acme/clone/prd.md')); + assert.ok(zip.names.includes('re-acme/sandbox/ledger.csv')); +});