Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 25 additions & 1 deletion high-signal-tokens.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,30 @@ export function extractHighSignalTokens(text) {
* every prior version); when supplied it enables the ledger exemption below.
* @returns {Array<{id,oldLen,newLen,ratio,dropped,droppedCount}>} one entry per suspicious pair
*/
// A token is still carried if the new text names it OR names something it is the
// tail of. Writing `memory/index.mjs` out and `E:/Project/ws/memory/index.mjs` in
// is not a loss — it is the same file, said more precisely, and set difference
// alone calls it a drop.
//
// This matters more than it looks. Consolidating several memories almost always
// expands relative references into absolute ones, so the merge that a maintainer
// is most likely to perform is exactly the one that fires the most bogus
// warnings. A guard that cries wolf on good edits gets ignored on the bad ones,
// and this one exists to be read.
//
// Suffix must break on a separator: `send.mjs` is not carried by `feishu-send.mjs`
// (a different file), while `memory/index.mjs` is carried by `E:/x/memory/index.mjs`.
export function isStillCarried(token, newTokens) {
if (newTokens.has(token)) return true
for (const t of newTokens) {
if (t.length > token.length && t.endsWith(token)) {
const boundary = t[t.length - token.length - 1]
if (boundary === '/' || boundary === '\\') return true
}
}
return false
}

export function checkSupersedeShrink(newContent, olds) {
const warnings = []
const newTokens = new Set(extractHighSignalTokens(newContent))
Expand Down Expand Up @@ -83,7 +107,7 @@ export function checkSupersedeShrink(newContent, olds) {
const peakLen = Math.max(old.peakLen || 0, oldLen)
if (oneToOne && newLen >= peakLen) continue

const dropped = extractHighSignalTokens(old.content).filter(t => !newTokens.has(t))
const dropped = extractHighSignalTokens(old.content).filter(t => !isStillCarried(t, newTokens))
const ratio = oldLen ? +(newLen / oldLen).toFixed(2) : 1
const shrank = oneToOne && oldLen >= SHRINK_MIN_OLD_LEN && ratio < SHRINK_RATIO_FLOOR
if (!dropped.length && !shrank) continue
Expand Down
8 changes: 6 additions & 2 deletions memory-health.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ import { existsSync } from 'node:fs'
import { dirname, resolve } from 'node:path'
import { fileURLToPath, pathToFileURL } from 'node:url'
import { createRequire } from 'node:module'
import { extractHighSignalTokens } from './high-signal-tokens.mjs'
import { extractHighSignalTokens, isStillCarried } from './high-signal-tokens.mjs'

const require = createRequire(import.meta.url)
const __dirname = dirname(fileURLToPath(import.meta.url))
Expand Down Expand Up @@ -535,7 +535,11 @@ export function detectShrinkVictims(db, opts = {}) {
const nowTokens = new Set(extractHighSignalTokens(r.content))
const lost = new Set()
for (const p of priors) {
for (const t of extractHighSignalTokens(p.content)) if (!nowTokens.has(t)) lost.add(t)
// Same "is it still carried" rule as the write-time guard, imported rather
// than reimplemented — these two drifted apart once before and the audit
// queue is only trustworthy if it agrees with the gate that let the write
// through in the first place.
for (const t of extractHighSignalTokens(p.content)) if (!isStillCarried(t, nowTokens)) lost.add(t)
}
if (!lost.size) continue
lostAny++
Expand Down
42 changes: 42 additions & 0 deletions supersede-shrink.integration.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -225,5 +225,47 @@ try {
}

dbRead.close()
// ── a token spelled more precisely is not a token dropped ──
//
// All three cases below are real merges from one night of memory curation. Each
// one warned, each warning was wrong, and the reason was always the same: the
// consolidated text expanded a relative reference into an absolute one, so plain
// set difference saw the short form vanish.
//
// That is the merge a maintainer performs most often, which made this the
// warning they would learn to ignore first — on a guard whose whole value is
// being read.
{
const { checkSupersedeShrink, isStillCarried } = await import('./high-signal-tokens.mjs')

const carried = [
['memory/index.mjs', 'E:/Project/ws/memory/index.mjs'],
['session-summarize.mjs', 'E:/Project/ws/memory/scripts/session-summarize.mjs'],
['scripts/run.sh', 'C:/tools/scripts/run.sh'],
]
for (const [short, long] of carried) {
check(`"${short}" counts as carried by "${long}"`, isStillCarried(short, new Set([long])))
}

// The boundary is a path separator. Same suffix, different file — must still warn.
check('a different file that merely ends the same way is NOT carried',
!isStillCarried('send.mjs', new Set(['E:/Project/ws/feishu-send.mjs'])))
check('a bare substring is not carried either',
!isStillCarried('index.mjs', new Set(['reindex.mjs'])))

const oldContent = 'runner lives at memory/index.mjs and the log rotates via scripts/run.sh, token in API_TOKEN, see https://ops.example.com/dash'
const newContent = 'runner lives at E:/Project/ws/memory/index.mjs and the log rotates via C:/tools/scripts/run.sh, token in API_TOKEN, see https://ops.example.com/dash — now also covers the nightly path'
const clean = checkSupersedeShrink(newContent, [{ id: '1', content: oldContent }])
check('expanding relative paths to absolute raises no warning',
clean.length === 0, JSON.stringify(clean))

// And the real loss still lands: same expansion, but the URL is gone.
const lossy = 'runner lives at E:/Project/ws/memory/index.mjs and the log rotates via C:/tools/scripts/run.sh, token in API_TOKEN'
const warned = checkSupersedeShrink(lossy, [{ id: '1', content: oldContent }])
check('a genuinely dropped identifier still warns',
warned.length === 1 && warned[0].dropped.some(d => d.includes('ops.example.com')),
JSON.stringify(warned))
}

console.log(`\n${fail === 0 ? 'PASS' : 'FAIL'}: ${pass} passed / ${fail} failed`)
process.exit(fail === 0 ? 0 : 1)
Loading