Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions src/core/factsheet.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,10 @@
const PATTERNS: readonly RegExp[] = [
/\b[A-Z][A-Z0-9_]{2,}=[^\s)"'<>]+/g, // semantic LABEL=value pair (preserve association)
/\bhttps?:\/\/[^\s)"'<>]+/g, // URLs
/\b[A-Za-z0-9.!#$%&'*+/=?^_`{|}~-]+@[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?(?:\.[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?)+\b/g, // email address
/\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b/g, // UUID
/\b[A-Z]{2}\d{2}[A-Z0-9]{8,30}\b/g, // IBAN-like account string
/(?:[$€£¥]|(?:USD|EUR|GBP|CAD|AUD|CHF|JPY))\d(?:[\d,_]*\d)?(?:\.\d{2})?\b/g, // currency amount
/(?:[\w@~+-]+)?(?:\/[\w.@+-]+)+\.[A-Za-z]\w{0,8}\b/g, // path with a file extension (multi-dot ok: .test.ts)
/\/[\w.@+-]+(?:\/[\w.@+-]+)+\/?/g, // dir path (>=2 segments)
/\b(?=[0-9a-f]*\d)[0-9a-f]{7,40}\b/g, // git sha / long hex (must contain a digit)
Expand Down Expand Up @@ -56,6 +59,9 @@ const MAX_CHUNK = 512; // whitespace-free chunks longer than this are blobs (bas
* identifiers outrank long URLs when the budget is tight. Pure + total → deterministic →
* cache-stable. Tiers: 0 = protect always, 1 = paths/versions/misc, 2 = URLs (cap + last). */
const SHAPE_UUID = /^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$/;
const SHAPE_EMAIL = /^[A-Za-z0-9.!#$%&'*+/=?^_`{|}~-]+@[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?(?:\.[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?)+$/;
const SHAPE_IBAN = /^[A-Z]{2}\d{2}[A-Z0-9]{8,30}$/;
const SHAPE_CURRENCY = /^(?:[$€£¥]|(?:USD|EUR|GBP|CAD|AUD|CHF|JPY))\d(?:[\d,_]*\d)?(?:\.\d{2})?$/;
const SHAPE_HEX = /^(?=[0-9a-f]*\d)[0-9a-f]{7,40}$/; // git sha / opaque hex
const SHAPE_CONST = /^[A-Z][A-Z0-9]{2,}(?:_[A-Z0-9]+)+$/; // CONST_IDS / env vars
const SHAPE_TICKET = /^(?=[A-Z0-9-]*\d)[A-Z][A-Z0-9]+(?:-[A-Z0-9]+)+$/; // PROJ-1482 / CVE-2024-30078
Expand All @@ -71,6 +77,9 @@ function priorityTier(tok: string): 0 | 1 | 2 {
SHAPE_ASSIGNMENT.test(tok) ||
SHAPE_HEX.test(tok) ||
SHAPE_UUID.test(tok) ||
SHAPE_EMAIL.test(tok) ||
SHAPE_IBAN.test(tok) ||
SHAPE_CURRENCY.test(tok) ||
SHAPE_CONST.test(tok) ||
SHAPE_TICKET.test(tok) ||
SHAPE_FLAG.test(tok) ||
Expand Down
17 changes: 17 additions & 0 deletions tests/factsheet.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,23 @@ describe('factsheet extraction', () => {
expect(toks).toContain('97.82');
});

it('captures issue 55 exact-token classes for transactional text', () => {
const replyTo = 'ops.reply+invoice-55@example.co.uk';
const iban = 'UA383220010000026008';
const invoiceAmount = '$14,360';
const text = [
`reply-to ${replyTo}`,
`settle IBAN ${iban}`,
`invoice total ${invoiceAmount}, due today`,
].join('\n');

const toks = extractFactSheetTokens(text);

expect(toks).toContain(replyTo);
expect(toks).toContain(iban);
expect(toks).toContain(invoiceAmount);
});

it('drops substrings of longer kept tokens', () => {
const toks = extractFactSheetTokens('see https://github.com/o/r/pull/9 in repo');
// The bare /github.com path must collapse into the full URL.
Expand Down