|
52 | 52 | * step in CI — alongside `check:api-surface` / the example-app typecheck, its |
53 | 53 | * fellow "real consumer" gates — not before it like `check:skill-refs`. |
54 | 54 | * |
| 55 | + * ── The third anti-idle assertion: no bare `any` in a marked block (#5943) ─── |
| 56 | + * A marker is the author's claim "this block compiles", and the two guards above |
| 57 | + * (orphan marker, zero blocks) exist because a gate that checks nothing must not |
| 58 | + * report success. A bare `any` inside a marked block is the same failure wearing |
| 59 | + * a green badge: every property access on an `any` is unchecked, so `tsc` proves |
| 60 | + * exactly nothing about the lines a reader copies. |
| 61 | + * |
| 62 | + * #5720 is the measured specimen. Two marked hook examples were written |
| 63 | + * `export async function beforeUpdate(ctx: any)` and read `ctx.services`, which |
| 64 | + * a hook context does not have (the engine builds nine keys by name, the sandbox |
| 65 | + * ten, neither of them `services`) — so the copied hook short-circuits on the |
| 66 | + * optional chain and throws `PERMISSION_DENIED` on every write. This gate was |
| 67 | + * green throughout. Re-annotate the identical function bodies with the honest |
| 68 | + * `HookContext` and both report the same line: |
| 69 | + * |
| 70 | + * error TS2339: Property 'services' does not exist on type |
| 71 | + * '{ object: string; event: …; input: Record<string, unknown>; … }' |
| 72 | + * |
| 73 | + * The same `any` also hid #5605's `ctx.session?.positions`. One `any`, two |
| 74 | + * defects, zero diagnostics. |
| 75 | + * |
| 76 | + * SCOPE — the annotation must BE `any`, in a position where it erases checking |
| 77 | + * wholesale: a parameter, a variable/property/return annotation, a type alias, |
| 78 | + * or an `as any` / `satisfies any` / angle-bracket assertion. `any` NESTED inside a |
| 79 | + * larger type (`Record<string, any>`, `any[]`, `Promise<any>`) is deliberately |
| 80 | + * NOT flagged — the same line `check-exported-any.ts` draws for the same reason: |
| 81 | + * a nested `any` is a much broader question, and holding the gate at zero false |
| 82 | + * positives is what keeps red meaning broken. |
| 83 | + * |
| 84 | + * Casts and locals are in scope, and not for symmetry: a parameter-only rule is |
| 85 | + * defeated by exactly the edit an author reaches for when it goes red — move the |
| 86 | + * `any` one line down (`const c: any = ctx`) or into the access |
| 87 | + * (`(ctx as any).services`) — which would leave the gate green over an unchanged |
| 88 | + * defect. Measured at the time of writing, the whole 208-block corpus contained |
| 89 | + * three bare `any` annotations, all in one block, so the wider scope cost nothing |
| 90 | + * to adopt and there is no ratchet file: the baseline is zero and stays zero. |
| 91 | + * |
55 | 92 | * Usage: |
56 | 93 | * tsx scripts/check-skill-examples.ts # extract + type-check (CI) |
| 94 | + * tsx scripts/check-skill-examples.ts --self-test # pin the `any` detector, both directions |
57 | 95 | * tsx scripts/check-skill-examples.ts --keep # also leave the build dir for inspection |
58 | 96 | */ |
59 | 97 |
|
60 | 98 | import { spawnSync } from 'child_process'; |
61 | 99 | import fs from 'fs'; |
| 100 | +import os from 'os'; |
62 | 101 | import path from 'path'; |
| 102 | +import ts from 'typescript'; |
63 | 103 |
|
64 | 104 | // ── Paths ──────────────────────────────────────────────────────────────────── |
65 | 105 |
|
@@ -111,6 +151,7 @@ const SOURCE_ROOTS: Array<{ |
111 | 151 | const ALL_MARKERS = ['<!-- os:check -->', '{/* os:check */}']; |
112 | 152 |
|
113 | 153 | const KEEP = process.argv.includes('--keep'); |
| 154 | +const SELF_TEST = process.argv.includes('--self-test'); |
114 | 155 |
|
115 | 156 | const rel = (p: string) => path.relative(REPO_ROOT, p); |
116 | 157 |
|
@@ -205,6 +246,74 @@ function extractFromFile( |
205 | 246 | return { examples, orphans }; |
206 | 247 | } |
207 | 248 |
|
| 249 | +// ── Bare-`any` guard (#5943) ───────────────────────────────────────────────── |
| 250 | + |
| 251 | +interface AnyFinding { |
| 252 | + /** 1-based line WITHIN the block body (body[0] is line 1). */ |
| 253 | + line: number; |
| 254 | + /** 1-based column. */ |
| 255 | + col: number; |
| 256 | + /** Human-readable position, e.g. "parameter `ctx`" — the prescription's subject. */ |
| 257 | + where: string; |
| 258 | +} |
| 259 | + |
| 260 | +/** |
| 261 | + * Every position in which a bare `any` erases checking wholesale, keyed by the |
| 262 | + * PARENT node kind. The check is `parent.type === node` (or the assertion's own |
| 263 | + * type slot), so an `any` nested in a larger type — `Record<string, any>`, |
| 264 | + * `any[]`, `Promise<any>` — has a TypeReference/ArrayType parent and is not a |
| 265 | + * finding. That boundary is the gate's zero-false-positive line; widening it is |
| 266 | + * a different question with a different (much larger) baseline. |
| 267 | + */ |
| 268 | +function describeAnyPosition(node: ts.Node): string | null { |
| 269 | + const parent = node.parent; |
| 270 | + if (!parent) return null; |
| 271 | + |
| 272 | + const named = (name: ts.BindingName | ts.PropertyName | undefined): string => |
| 273 | + name && ts.isIdentifier(name) ? ` \`${name.text}\`` : ''; |
| 274 | + |
| 275 | + if (ts.isParameter(parent) && parent.type === node) return `parameter${named(parent.name)}`; |
| 276 | + if (ts.isVariableDeclaration(parent) && parent.type === node) return `variable${named(parent.name)}`; |
| 277 | + if ((ts.isPropertyDeclaration(parent) || ts.isPropertySignature(parent)) && parent.type === node) |
| 278 | + return `property${named(parent.name)}`; |
| 279 | + if (ts.isTypeAliasDeclaration(parent) && parent.type === node) return `type alias \`${parent.name.text}\``; |
| 280 | + if (ts.isAsExpression(parent) && parent.type === node) return '`as any` assertion'; |
| 281 | + if (ts.isSatisfiesExpression(parent) && parent.type === node) return '`satisfies any` assertion'; |
| 282 | + if (ts.isTypeAssertionExpression(parent) && parent.type === node) return '`< any >` type assertion'; |
| 283 | + // Return annotations: functions, methods, arrows, getters, signatures. |
| 284 | + if (ts.isFunctionLike(parent) && parent.type === node) return 'return type'; |
| 285 | + return null; |
| 286 | +} |
| 287 | + |
| 288 | +/** |
| 289 | + * Parse ONE marked block and report every bare `any` annotation in it. |
| 290 | + * |
| 291 | + * Parsing (not regex) because the corpus is prose: `'any'` appears in string |
| 292 | + * literal unions, in JSDoc, and in ordinary English inside comments — a |
| 293 | + * line-wise regex reported three such lines on this repo's own corpus and none |
| 294 | + * of them was a type annotation. `createSourceFile` never throws on malformed |
| 295 | + * input; a block too broken to parse yields no findings here and is caught by |
| 296 | + * the `tsc` pass that follows, which is the right division of labour. |
| 297 | + */ |
| 298 | +function findBareAny(code: string, fileName: string): AnyFinding[] { |
| 299 | + const sf = ts.createSourceFile(fileName, code, ts.ScriptTarget.ES2020, /* setParentNodes */ true, ts.ScriptKind.TS); |
| 300 | + const findings: AnyFinding[] = []; |
| 301 | + |
| 302 | + const visit = (node: ts.Node): void => { |
| 303 | + if (node.kind === ts.SyntaxKind.AnyKeyword) { |
| 304 | + const where = describeAnyPosition(node); |
| 305 | + if (where) { |
| 306 | + const { line, character } = sf.getLineAndCharacterOfPosition(node.getStart(sf)); |
| 307 | + findings.push({ line: line + 1, col: character + 1, where }); |
| 308 | + } |
| 309 | + } |
| 310 | + ts.forEachChild(node, visit); |
| 311 | + }; |
| 312 | + ts.forEachChild(sf, visit); |
| 313 | + |
| 314 | + return findings; |
| 315 | +} |
| 316 | + |
208 | 317 | // ── Module resolution derived from the spec's own `exports` ────────────────── |
209 | 318 |
|
210 | 319 | interface ExportEntry { |
@@ -331,7 +440,177 @@ function fail(message: string): never { |
331 | 440 | process.exit(1); |
332 | 441 | } |
333 | 442 |
|
| 443 | +// ── Self-test ──────────────────────────────────────────────────────────────── |
| 444 | + |
| 445 | +/** |
| 446 | + * Pin the bare-`any` detector on BOTH edges, over the real `extractFromFile`. |
| 447 | + * |
| 448 | + * A false negative makes the assertion dormant — green forever, which is |
| 449 | + * indistinguishable from a clean corpus and is the exact state #5720 shipped in. |
| 450 | + * A false positive is just as costly the other way: the three shapes below |
| 451 | + * (`Record<string, any>`, `any[]`, a `'any'` string-literal union member, the |
| 452 | + * word "any" in prose) all occur in the real corpus, and flagging any of them |
| 453 | + * would force a corpus-wide rewrite for no defect. |
| 454 | + * |
| 455 | + * Line numbers are asserted literally, not recomputed, because the block-line → |
| 456 | + * page-line arithmetic (`bodyStartLine + line - 1`) is the part that silently |
| 457 | + * drifts: a diagnostic pointing at the wrong line is worse than none. |
| 458 | + */ |
| 459 | +function selfTest(): never { |
| 460 | + const failures: string[] = []; |
| 461 | + const check = (cond: boolean, msg: string): void => { |
| 462 | + if (!cond) failures.push(msg); |
| 463 | + }; |
| 464 | + |
| 465 | + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-examples-selftest-')); |
| 466 | + try { |
| 467 | + const docsRoot = { dir, ext: '.mdx', label: 'docs', marker: '{/* os:check */}' }; |
| 468 | + const skillsRoot = { dir, ext: '.md', label: 'skills', marker: '<!-- os:check -->' }; |
| 469 | + |
| 470 | + // ── RED, docs: the #5720 shape, verbatim. `any` on line 9 of the page. ─── |
| 471 | + const redDocs = path.join(dir, 'red.mdx'); |
| 472 | + fs.writeFileSync( |
| 473 | + redDocs, |
| 474 | + [ |
| 475 | + '# Fixture page', // 1 |
| 476 | + '', // 2 |
| 477 | + 'Prose above the marked block.', // 3 |
| 478 | + '', // 4 |
| 479 | + '{/* os:check */}', // 5 |
| 480 | + '```ts', // 6 |
| 481 | + `import type { HookContext } from '@objectstack/spec';`, // 7 |
| 482 | + '', // 8 |
| 483 | + 'export async function beforeUpdate(ctx: any): Promise<void> {', // 9 ← the defect |
| 484 | + ' void ctx;', // 10 |
| 485 | + '}', // 11 |
| 486 | + '```', // 12 |
| 487 | + '', |
| 488 | + ].join('\n'), |
| 489 | + 'utf8', |
| 490 | + ); |
| 491 | + const red = extractFromFile(redDocs, docsRoot); |
| 492 | + check(red.examples.length === 1, `red fixture: extracted ${red.examples.length} block(s), expected 1`); |
| 493 | + check(red.orphans.length === 0, `red fixture: reported ${red.orphans.length} orphan marker(s), expected 0`); |
| 494 | + if (red.examples.length === 1) { |
| 495 | + const hits = findBareAny(red.examples[0].code, 'red.ts'); |
| 496 | + check(hits.length === 1, `red fixture: found ${hits.length} bare \`any\`, expected 1 — the guard is DORMANT`); |
| 497 | + if (hits.length === 1) { |
| 498 | + const pageLine = red.examples[0].bodyStartLine + hits[0].line - 1; |
| 499 | + check(pageLine === 9, `red fixture: reported page line ${pageLine}, expected 9 — line mapping is wrong`); |
| 500 | + check( |
| 501 | + hits[0].where === 'parameter `ctx`', |
| 502 | + `red fixture: described the position as "${hits[0].where}", expected "parameter \`ctx\`"`, |
| 503 | + ); |
| 504 | + const expectedCol = |
| 505 | + 'export async function beforeUpdate(ctx: any): Promise<void> {'.indexOf(': any') + 3; |
| 506 | + check(hits[0].col === expectedCol, `red fixture: reported column ${hits[0].col}, expected ${expectedCol}`); |
| 507 | + } |
| 508 | + } |
| 509 | + |
| 510 | + // ── RED, skills: the four NON-parameter positions, all in one block. ───── |
| 511 | + const redSkill = path.join(dir, 'red.md'); |
| 512 | + fs.writeFileSync( |
| 513 | + redSkill, |
| 514 | + [ |
| 515 | + '# Skill fixture', // 1 |
| 516 | + '', // 2 |
| 517 | + '<!-- os:check -->', // 3 |
| 518 | + '```ts', // 4 |
| 519 | + 'const api = ({} as unknown) as any;', // 5 |
| 520 | + 'const loose: any = 1;', // 6 |
| 521 | + 'type Loose = any;', // 7 |
| 522 | + 'function widen(): any {', // 8 |
| 523 | + ' return 1;', // 9 |
| 524 | + '}', // 10 |
| 525 | + '```', // 11 |
| 526 | + '', |
| 527 | + ].join('\n'), |
| 528 | + 'utf8', |
| 529 | + ); |
| 530 | + const skill = extractFromFile(redSkill, skillsRoot); |
| 531 | + check(skill.examples.length === 1, `skills fixture: extracted ${skill.examples.length} block(s), expected 1`); |
| 532 | + if (skill.examples.length === 1) { |
| 533 | + const ex = skill.examples[0]; |
| 534 | + const got = findBareAny(ex.code, 'red-skill.ts') |
| 535 | + .map((h) => `${ex.bodyStartLine + h.line - 1}:${h.where}`) |
| 536 | + .sort(); |
| 537 | + const want = [ |
| 538 | + '5:`as any` assertion', |
| 539 | + '6:variable `loose`', |
| 540 | + '7:type alias `Loose`', |
| 541 | + '8:return type', |
| 542 | + ].sort(); |
| 543 | + check( |
| 544 | + JSON.stringify(got) === JSON.stringify(want), |
| 545 | + `skills fixture: got ${JSON.stringify(got)}, expected ${JSON.stringify(want)} — a bare-\`any\` position is unguarded, ` + |
| 546 | + `and moving the \`any\` there is exactly the edit a red parameter invites`, |
| 547 | + ); |
| 548 | + } |
| 549 | + |
| 550 | + // ── GREEN: the same function honestly typed, plus every shape that must |
| 551 | + // NOT be flagged, plus an UNMARKED block that must not be read at all. ─ |
| 552 | + const green = path.join(dir, 'green.mdx'); |
| 553 | + fs.writeFileSync( |
| 554 | + green, |
| 555 | + [ |
| 556 | + '# Green fixture', // 1 |
| 557 | + '', // 2 |
| 558 | + '{/* os:check */}', // 3 |
| 559 | + '```ts', // 4 |
| 560 | + `import type { HookContext } from '@objectstack/spec';`, // 5 |
| 561 | + '', // 6 |
| 562 | + `type Kind = 'string' | 'number' | 'any';`, // 7 string literal, not a type |
| 563 | + '', // 8 |
| 564 | + '/** Prose mentioning any old thing, and Record<string, any> in a comment. */', // 9 |
| 565 | + 'export async function beforeUpdate(ctx: HookContext): Promise<void> {', // 10 |
| 566 | + ' const bag: Record<string, any> = {};', // 11 nested — out of scope by design |
| 567 | + ' const rows: any[] = [];', // 12 nested |
| 568 | + ` const kind: Kind = 'any';`, // 13 string literal |
| 569 | + ' void ctx; void bag; void rows; void kind;', // 14 |
| 570 | + '}', // 15 |
| 571 | + '```', // 16 |
| 572 | + '', // 17 |
| 573 | + 'An UNMARKED block — the gate judges only what the author marked:', // 18 |
| 574 | + '', // 19 |
| 575 | + '```ts', // 20 |
| 576 | + 'export function unchecked(ctx: any) { void ctx; }', // 21 |
| 577 | + '```', // 22 |
| 578 | + '', |
| 579 | + ].join('\n'), |
| 580 | + 'utf8', |
| 581 | + ); |
| 582 | + const ok = extractFromFile(green, docsRoot); |
| 583 | + check( |
| 584 | + ok.examples.length === 1, |
| 585 | + `green fixture: extracted ${ok.examples.length} block(s), expected 1 — the UNMARKED block must not be read`, |
| 586 | + ); |
| 587 | + if (ok.examples.length >= 1) { |
| 588 | + const hits = findBareAny(ok.examples[0].code, 'green.ts'); |
| 589 | + check( |
| 590 | + hits.length === 0, |
| 591 | + `green fixture: ${hits.length} false positive(s) — ${hits.map((h) => `line ${h.line} (${h.where})`).join(', ')}. ` + |
| 592 | + `Nested \`any\`, \`'any'\` string literals and the word "any" in prose all occur in the real corpus.`, |
| 593 | + ); |
| 594 | + } |
| 595 | + } finally { |
| 596 | + fs.rmSync(dir, { recursive: true, force: true }); |
| 597 | + } |
| 598 | + |
| 599 | + if (failures.length > 0) { |
| 600 | + for (const f of failures) console.error(`✗ self-test: ${f}`); |
| 601 | + console.error(`\ncheck-skill-examples --self-test: ${failures.length} failure(s).\n`); |
| 602 | + process.exit(1); |
| 603 | + } |
| 604 | + console.log( |
| 605 | + '✅ self-test: flags a bare `any` parameter / variable / property / return / alias / cast in a\n' + |
| 606 | + ' marked block at the right page line, and flags nothing in an honestly typed one.', |
| 607 | + ); |
| 608 | + process.exit(0); |
| 609 | +} |
| 610 | + |
334 | 611 | function main() { |
| 612 | + if (SELF_TEST) selfTest(); |
| 613 | + |
335 | 614 | console.log('🧪 Type-checking prose TypeScript examples (skills + docs)...\n'); |
336 | 615 |
|
337 | 616 | const files = sourceFiles(); |
@@ -369,6 +648,34 @@ function main() { |
369 | 648 | ); |
370 | 649 | } |
371 | 650 |
|
| 651 | + // Third anti-idle assertion (#5943): a marked block that annotates anything |
| 652 | + // `any` compiles by definition and proves nothing about it. Runs BEFORE the |
| 653 | + // build dir is written, so the author reads one crisp verdict instead of a |
| 654 | + // clean `tsc` run that silently covered nothing. |
| 655 | + const anyHits: string[] = []; |
| 656 | + for (const ex of examples) { |
| 657 | + for (const f of findBareAny(ex.code, ex.fileName)) { |
| 658 | + anyHits.push(` ${rel(ex.source)}:${ex.bodyStartLine + f.line - 1}:${f.col} ${f.where}`); |
| 659 | + } |
| 660 | + } |
| 661 | + if (anyHits.length > 0) { |
| 662 | + fail( |
| 663 | + `os:check block(s) annotate ${anyHits.length === 1 ? 'a value' : 'values'} \`any\` — the marker claims\n` + |
| 664 | + `"this compiles" while every property access on that value goes unchecked:\n\n` + |
| 665 | + anyHits.join('\n') + |
| 666 | + `\n\n Fix it one of two ways:\n` + |
| 667 | + ` 1. Annotate the real type (import it from @objectstack/spec) — that is the\n` + |
| 668 | + ` whole point of marking the block, and it is what catches the drift:\n` + |
| 669 | + ` \`(ctx: any)\` hid a hook example reading a \`ctx.services\` that no hook\n` + |
| 670 | + ` context has (#5720), and a \`ctx.session?.positions\` before it (#5605).\n` + |
| 671 | + ` 2. Remove the os:check marker if the block is an illustrative fragment\n` + |
| 672 | + ` that cannot be typed against the spec (generated third-party code, a\n` + |
| 673 | + ` partial subtree). An unmarked block is honest; a marked \`any\` is not.\n\n` + |
| 674 | + ` Nested \`any\` (\`Record<string, any>\`, \`any[]\`, \`Promise<any>\`) is NOT flagged\n` + |
| 675 | + ` — only an annotation, cast or alias that IS \`any\`.`, |
| 676 | + ); |
| 677 | + } |
| 678 | + |
372 | 679 | const { paths, missing } = specPaths(); |
373 | 680 | if (missing.some((m) => m.endsWith('index.d.ts')) && !fs.existsSync(paths['@objectstack/spec']?.[0] ?? '')) { |
374 | 681 | fail( |
|
0 commit comments