From b17d59a127eaa2e8153f840c74b1742214bb0fbe Mon Sep 17 00:00:00 2001 From: Ocean572 <45051223+Ocean572@users.noreply.github.com> Date: Wed, 4 Nov 2026 14:03:23 -0600 Subject: [PATCH 1/6] Keep the scoped queries alive past a thousand folders MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit store.scope spelled every chosen folder as its own (path >= ? AND path < ?) range, all ORed into one expression: at 1,179 folders (#166's reporter, working around #165) every pending query blew past SQLite's expression-depth cap of 1,000 and the indexer died at each startup. The ranges now live in a temp table and the queries test them with a correlated EXISTS — depth stays constant whatever the folder count, and the table grows with the list for free. setScope(_:) is throwing now; a 1,200-folder test reproduces the old failure shape. Closes #166 --- Sources/DigUpKit/IndexStore.swift | 43 +++++++++++++++-------- Sources/digup/IndexService.swift | 6 ++-- Tests/DigUpKitTests/IndexStoreTests.swift | 25 ++++++++++--- 3 files changed, 52 insertions(+), 22 deletions(-) diff --git a/Sources/DigUpKit/IndexStore.swift b/Sources/DigUpKit/IndexStore.swift index 7a9b8b82..d1adbf01 100644 --- a/Sources/DigUpKit/IndexStore.swift +++ b/Sources/DigUpKit/IndexStore.swift @@ -33,17 +33,30 @@ public final class IndexStore { /// When set, `pending`, `pendingCount` and `counts` see only files under these folders: the app's helper sets the /// folders that are connected right now, so files on an unplugged drive neither fail nor count as waiting. - public var scope: [String]? { - didSet { - guard let scope else { (scopeSQL, scopeValues) = ("", []); return } - let ranges = scope.flatMap(Self.pathRanges(under:)) - scopeSQL = ranges.isEmpty ? " AND 0" - : " AND (" + ranges.map { _ in "(path >= ? AND path < ?)" }.joined(separator: " OR ") + ")" - scopeValues = ranges.flatMap { [SQLValue.text($0.from), .text($0.to)] } + /// The ranges live in a temp table and the scoped queries test them with a correlated `EXISTS`: one + /// `(path >= ? AND path < ?)` per folder, ORed into a single expression, hits SQLite's expression-depth cap of + /// 1,000 around 500 folders (#166: 1,179 folders, the indexer died at every startup), while the table grows + /// with the folder list for free. + public private(set) var scope: [String]? + private var scopeSQL = "" + + public func setScope(_ folders: [String]?) throws { + scope = folders + guard let folders else { scopeSQL = ""; return } + let ranges = folders.flatMap(Self.pathRanges(under:)) + scopeSQL = ranges.isEmpty ? " AND 0" + : " AND EXISTS (SELECT 1 FROM scope s WHERE files.path >= s.lo AND files.path < s.hi)" + try db.run("CREATE TEMP TABLE IF NOT EXISTS scope(lo TEXT NOT NULL, hi TEXT NOT NULL)") + try db.run("DELETE FROM scope") + var start = 0 + while start < ranges.count { // multi-row INSERTs, kept under SQLite's cap on a compound select + let chunk = ranges[start.. Int { var count = 0 - try db.query("SELECT COUNT(*) FROM files WHERE state = 'pending'\(scopeSQL)", scopeValues) { + try db.query("SELECT COUNT(*) FROM files WHERE state = 'pending'\(scopeSQL)") { count = Int($0.int(0)) } return count @@ -352,7 +365,7 @@ public final class IndexStore { try db.query(""" SELECT \(Self.fileColumns) FROM files WHERE state = 'partial'\(scopeSQL) ORDER BY \(FileKind.indexingOrderSQL), mtime DESC - """, scopeValues) { out.append(Self.record($0)) } + """) { out.append(Self.record($0)) } return out } @@ -363,7 +376,7 @@ public final class IndexStore { SELECT COUNT(*), SUM(MAX(0, CAST(json_extract(info, '$.read_end') AS INTEGER) - CAST(json_extract(info, '$.read_to') AS INTEGER))) FROM files WHERE state = 'partial'\(scopeSQL) - """, scopeValues) { count = (Int($0.int(0)), Int($0.int(1))) } + """) { count = (Int($0.int(0)), Int($0.int(1))) } return count } @@ -494,12 +507,12 @@ public final class IndexStore { public func counts() throws -> Counts { var counts = Counts() - try db.query("SELECT kind, state, COUNT(*) FROM files WHERE 1\(scopeSQL) GROUP BY kind, state", scopeValues) { + try db.query("SELECT kind, state, COUNT(*) FROM files WHERE 1\(scopeSQL) GROUP BY kind, state") { counts.byKindAndState[$0.text(0) ?? "?", default: [:]][$0.text(1) ?? "?"] = Int($0.int(2)) } (counts.unfinished, counts.left) = try unfinishedCount() let inScope = scope == nil ? "" : " AND file IN (SELECT id FROM files WHERE 1\(scopeSQL))" - try db.query("SELECT COUNT(*), COUNT(vec) FROM segments WHERE kind != 'name'\(inScope)", scopeValues) { + try db.query("SELECT COUNT(*), COUNT(vec) FROM segments WHERE kind != 'name'\(inScope)") { counts.segments = Int($0.int(0)) counts.vectors = Int($0.int(1)) } diff --git a/Sources/digup/IndexService.swift b/Sources/digup/IndexService.swift index db2a807f..56b1f9d0 100644 --- a/Sources/digup/IndexService.swift +++ b/Sources/digup/IndexService.swift @@ -199,7 +199,7 @@ final class IndexService: @unchecked Sendable { } /// A part's chosen folders that are there and readable right now; also what `pending` and the counts look at. - private func availableRoots(_ part: Part) -> [URL] { + private func availableRoots(_ part: Part) throws -> [URL] { let missing = part.roots.filter { !Arguments.isFolder($0) } let unreadable = part.roots.filter { !missing.contains($0) && !Arguments.isReadable($0) } let report = (missing.map(\.path), unreadable.map(\.path)) @@ -208,7 +208,7 @@ final class IndexService: @unchecked Sendable { } unavailable[part.code] = report let available = part.roots.filter { !missing.contains($0) && !unreadable.contains($0) } - part.store.scope = available.map(\.path) + try part.store.setScope(available.map(\.path)) return available } @@ -219,7 +219,7 @@ final class IndexService: @unchecked Sendable { lock.withLock { yieldAt = Date() + max(took, 2) } } do { - let roots = availableRoots(part) + let roots = try availableRoots(part) emit(["event": "scanning", "checked": 0, "code": part.code]) let summary = try part.indexer.sync(roots) { progress in var event: [String: Any] = ["event": "scanning", "checked": progress.checked, "code": part.code] diff --git a/Tests/DigUpKitTests/IndexStoreTests.swift b/Tests/DigUpKitTests/IndexStoreTests.swift index ee3bbc51..5ad70d00 100644 --- a/Tests/DigUpKitTests/IndexStoreTests.swift +++ b/Tests/DigUpKitTests/IndexStoreTests.swift @@ -55,13 +55,30 @@ import WinSDK let store = try IndexStore(directory: folder) try store.insert(file("/a/x.png", inode: 1)) try store.insert(file("/drive/y.png", inode: 2)) // on a drive that's unplugged - store.scope = ["/a"] + try store.setScope(["/a"]) #expect(try store.pendingCount() == 1) #expect(try store.pending(limit: nil).map(\.path) == ["/a/x.png"]) #expect(try store.counts().byKindAndState["image"]?["pending"] == 1) - store.scope = [] + try store.setScope([]) #expect(try store.pendingCount() == 0) - store.scope = nil + try store.setScope(nil) + #expect(try store.pendingCount() == 2) + } + + @Test func aThousandFoldersFitTheScopedQueries() throws { + // #166: 1,179 chosen folders broke every scoped query at startup — one `(path >= ? AND path < ?)` + // range per folder, all ORed into one expression, past SQLite's expression-depth cap of 1,000. + let store = try IndexStore(directory: folder) + try store.insert(file("/a/x.png", inode: 1)) + try store.insert(file("/b/y.png", inode: 2)) + let folders = (0..<1_200).map { "/root-\($0)" } + try store.setScope(["/a"] + folders) + #expect(try store.pendingCount() == 1) + #expect(try store.pending(limit: nil).map(\.path) == ["/a/x.png"]) + #expect(try store.counts().byKindAndState["image"]?["pending"] == 1) + try store.setScope(folders) // no chosen folder has its files in this index + #expect(try store.pendingCount() == 0) + try store.setScope(nil) #expect(try store.pendingCount() == 2) } @@ -170,7 +187,7 @@ import WinSDK let again = try indexer.sync(chosen) #expect(again.added == 0 && again.unchanged == 2) #expect(try store.files(under: chosen.map(\.path)).count == 2) - store.scope = chosen.map(\.path) + try store.setScope(chosen.map(\.path)) #expect(try store.pendingCount() == 2) #expect(try store.prune(keepingUnder: chosen.map(\.path)) == 0) } From 174c409c71a231dfb5134163aac5ae067e906116 Mon Sep 17 00:00:00 2001 From: Ocean572 <45051223+Ocean572@users.noreply.github.com> Date: Wed, 4 Nov 2026 19:36:48 -0600 Subject: [PATCH 2/6] Keep setScope commit-or-nothing and give the scope table its lookup index MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The #191 review found the scope write is not atomic: scope moved in memory before any table write, and CREATE, DELETE and the INSERT chunks each committed on their own, so a write that failed mid-flight (disk full) left the queries answering as if the chosen folders were empty — pendingCount 0, counts 0, no error — until the next setScope succeeded. The write now runs inside one transaction and scope/scopeSQL move only after it commits: a failed setScope leaves the previous scope standing, in memory and in the table, and every scoped query answers as before. A RAISE(ABORT) trigger test reproduces the disk-full shape; ROLLBACK undoing temp-table writes (measured on macOS's SQLite 3.51 and a 3.49 bundled build; the vendored amalgamation is 3.54) is what keeps the table's contents intact. PRIMARY KEY(lo, hi) WITHOUT ROWID is the probe's lookup: EXPLAIN shows SEARCH s USING PRIMARY KEY where the review measured SCAN s. Measured on an Apple M1 (macOS 26.5.1, Swift 6.4.2, system SQLite 3.51) over a 1.2M-file index with 1,200 folders: pendingCount 4.22 s to 2.05 s, pending(limit: 50) 4.05 s to 1.80 s, counts() 153 s to 70.7 s, setScope 4.0 ms to 2.7 ms. DigUpKit 120/120 passing, LlamaRuntime 7/7 with the model. --- Sources/DigUpKit/IndexStore.swift | 41 +++++++++++++++-------- Tests/DigUpKitTests/IndexStoreTests.swift | 34 +++++++++++++++++++ 2 files changed, 61 insertions(+), 14 deletions(-) diff --git a/Sources/DigUpKit/IndexStore.swift b/Sources/DigUpKit/IndexStore.swift index d1adbf01..94fa82ed 100644 --- a/Sources/DigUpKit/IndexStore.swift +++ b/Sources/DigUpKit/IndexStore.swift @@ -36,26 +36,39 @@ public final class IndexStore { /// The ranges live in a temp table and the scoped queries test them with a correlated `EXISTS`: one /// `(path >= ? AND path < ?)` per folder, ORed into a single expression, hits SQLite's expression-depth cap of /// 1,000 around 500 folders (#166: 1,179 folders, the indexer died at every startup), while the table grows - /// with the folder list for free. + /// with the folder list for free. Its PRIMARY KEY is the probe's lookup — scoped query rows `SEARCH s USING + /// PRIMARY KEY`; without it every row scans every range, which was slower than the old chain past ~400 folders. public private(set) var scope: [String]? private var scopeSQL = "" + /// Writes the ranges in one transaction and moves `scope`/`scopeSQL` only after it commits: a write that + /// fails mid-flight (disk full) leaves the previous scope standing, in memory and in the table, and the + /// scoped queries go on answering as before. Spell that order any other way and a failed write leaves + /// the in-memory scope claiming folders the emptied table says nothing about — the indexer quietly + /// sees zero pending files until the next `setScope` succeeds. public func setScope(_ folders: [String]?) throws { - scope = folders - guard let folders else { scopeSQL = ""; return } + guard let folders else { (scope, scopeSQL) = (nil, ""); return } let ranges = folders.flatMap(Self.pathRanges(under:)) - scopeSQL = ranges.isEmpty ? " AND 0" - : " AND EXISTS (SELECT 1 FROM scope s WHERE files.path >= s.lo AND files.path < s.hi)" - try db.run("CREATE TEMP TABLE IF NOT EXISTS scope(lo TEXT NOT NULL, hi TEXT NOT NULL)") - try db.run("DELETE FROM scope") - var start = 0 - while start < ranges.count { // multi-row INSERTs, kept under SQLite's cap on a compound select - let chunk = ranges[start..= s.lo AND files.path < s.hi)") } /// The ranges of the UNIQUE path index that hold the files under `folder`. Paths are stored as a walk reports them, diff --git a/Tests/DigUpKitTests/IndexStoreTests.swift b/Tests/DigUpKitTests/IndexStoreTests.swift index 5ad70d00..89b98909 100644 --- a/Tests/DigUpKitTests/IndexStoreTests.swift +++ b/Tests/DigUpKitTests/IndexStoreTests.swift @@ -82,6 +82,40 @@ import WinSDK #expect(try store.pendingCount() == 2) } + @Test func aFailedScopeWriteKeepsTheWorkingScope() throws { + // The #191 review's disk-full shape: the scope write touches the table in several statements, + // so a failure needs to land *inside* the transaction, after `DELETE` has emptied it. A + // RAISE(ABORT) trigger on the table injects that failure; the transaction must leave the + // previous scope standing in memory and its ranges in the table, and every scoped query keep + // answering as it did before the failed write. + let store = try IndexStore(directory: folder) + try store.insert(file("/a/x.png", inode: 1)) + try store.insert(file("/drive/y.png", inode: 2)) // on a drive that's unplugged + try store.setScope(["/a"]) + var rowsBefore = -1 + try store.db.query("SELECT COUNT(*) FROM scope") { rowsBefore = Int($0.int(0)) } + #expect(rowsBefore > 0) + try store.db.run(""" + CREATE TEMP TRIGGER disk_full BEFORE INSERT ON scope + BEGIN SELECT RAISE(ABORT, 'disk full'); END + """) + var threw = false + do { try store.setScope(["/drive", "/b"]) } catch { threw = true } + #expect(threw, "the injected disk full must reach the caller") + #expect(store.scope == ["/a"]) // the in-memory scope of the last good write, untouched + var rowsAfter = -1 + try store.db.query("SELECT COUNT(*) FROM scope") { rowsAfter = Int($0.int(0)) } + #expect(rowsAfter == rowsBefore) // its ranges, in the table, untouched + #expect(try store.pendingCount() == 1) + #expect(try store.pending(limit: nil).map(\.path) == ["/a/x.png"]) + #expect(try store.counts().byKindAndState["image"]?["pending"] == 1) + // The next write goes through: table and scope move together, in one transaction. + try store.db.run("DROP TRIGGER disk_full") + try store.setScope(["/a", "/drive"]) + #expect(store.scope == ["/a", "/drive"]) + #expect(try store.pendingCount() == 2) + } + @Test func aFileTheHelperStoppedOnIsNotReadAgain() throws { let store = try IndexStore(directory: folder) try store.insert(file("/a/stuck.pdf", inode: 1)) From 8da015b82bec874bda4289c12f53b4cefa6954d5 Mon Sep 17 00:00:00 2001 From: Ocean572 <45051223+Ocean572@users.noreply.github.com> Date: Wed, 4 Nov 2026 21:22:46 -0600 Subject: [PATCH 3/6] Ask the scope's one covering range, not its thousand probes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The scoped queries asked every range with lo <= path whether the path sat under its hi — up to 1,200 PRIMARY KEY seeks per row. Merging the list into disjoint spans in setScope lets one seek answer the same question: the range with the greatest lo at or below a path is the only one that can cover it, so the queries compare that range's hi. Merging is bytewise, SQLite's BINARY order for TEXT; nested ranges fold, and a folder listed twice folds with them, so it never trips the PRIMARY KEY. Measured on an Apple M1 (macOS 26.5.1, Swift 6.4.2, system SQLite 3.51) over a 1.2M-file index with 1,200 folders: pendingCount 878 ms to 27.7 ms, pending(limit: 50) 831 ms to 30.2 ms, unfinishedCount 78 ms to 3.9 ms, counts() 36.0 s to 0.70 s. The EXISTS form it replaces cost 2.05 s and 70.7 s on the same index with a zero-match scope; main's OR-chain crashed there outright. A research harness says the merged predecessor answers exactly as the EXISTS form everywhere: the 1.2M benchmark agrees on all four queries, and a 48- comparison edge matrix (nested, adjacent-with-gaps, sibling /ab vs /a, dual /tmp spellings, precomposed and decomposed names, unplugged drives, empty, nil, tie fetches) is exact — the new mergedScopeKeepsNestedAdjacentAndDupli- cateFoldersExact and countsSeeOnlyTheScopedFilesSegments tests lock the shapes in the suite. DigUpKit 122/122 passing, LlamaRuntime 7/7 with the model. --- Sources/DigUpKit/IndexStore.swift | 52 +++++++++++++++++------ Tests/DigUpKitTests/IndexStoreTests.swift | 50 ++++++++++++++++++++++ 2 files changed, 90 insertions(+), 12 deletions(-) diff --git a/Sources/DigUpKit/IndexStore.swift b/Sources/DigUpKit/IndexStore.swift index 94fa82ed..766b044e 100644 --- a/Sources/DigUpKit/IndexStore.swift +++ b/Sources/DigUpKit/IndexStore.swift @@ -33,22 +33,26 @@ public final class IndexStore { /// When set, `pending`, `pendingCount` and `counts` see only files under these folders: the app's helper sets the /// folders that are connected right now, so files on an unplugged drive neither fail nor count as waiting. - /// The ranges live in a temp table and the scoped queries test them with a correlated `EXISTS`: one - /// `(path >= ? AND path < ?)` per folder, ORed into a single expression, hits SQLite's expression-depth cap of - /// 1,000 around 500 folders (#166: 1,179 folders, the indexer died at every startup), while the table grows - /// with the folder list for free. Its PRIMARY KEY is the probe's lookup — scoped query rows `SEARCH s USING - /// PRIMARY KEY`; without it every row scans every range, which was slower than the old chain past ~400 folders. + /// The ranges live in a temp table, merged to disjoint, and the scoped queries test a path against the one + /// range that can contain it: `(SELECT s.hi FROM scope s WHERE s.lo <= files.path ORDER BY s.lo DESC + /// LIMIT 1) > files.path`. When the ranges don't overlap, the greatest `lo` at or below a path is the + /// only candidate — one PRIMARY KEY seek per row, where a correlated `EXISTS` asked every range with + /// `lo <= path` to say yes (26× the time at 1,200 folders, measured in the #191 review's follow-up). + /// One `(path >= ? AND path < ?)` per folder, ORed into a single expression, is what died past #166's + /// 1,179 folders: SQLite caps expression trees at a depth of 1,000. The table grows with the list for free. public private(set) var scope: [String]? private var scopeSQL = "" - /// Writes the ranges in one transaction and moves `scope`/`scopeSQL` only after it commits: a write that - /// fails mid-flight (disk full) leaves the previous scope standing, in memory and in the table, and the - /// scoped queries go on answering as before. Spell that order any other way and a failed write leaves - /// the in-memory scope claiming folders the emptied table says nothing about — the indexer quietly - /// sees zero pending files until the next `setScope` succeeds. + /// Writes the merged ranges in one transaction and moves `scope`/`scopeSQL` only after it commits: a + /// write that fails mid-flight (disk full) leaves the previous scope standing, in memory and in the + /// table, and the scoped queries go on answering as before. Spell that order any other way and a + /// failed write leaves the in-memory scope claiming folders the emptied table says nothing about — + /// the indexer quietly sees zero pending files until the next `setScope` succeeds. Merging makes the + /// range list unique on its own, so a folder listed twice folds into its twin instead of tripping + /// the PRIMARY KEY. public func setScope(_ folders: [String]?) throws { guard let folders else { (scope, scopeSQL) = (nil, ""); return } - let ranges = folders.flatMap(Self.pathRanges(under:)) + let ranges = Self.mergedRanges(folders.flatMap(Self.pathRanges(under:))) try db.transaction { try db.run(""" CREATE TEMP TABLE IF NOT EXISTS scope( @@ -68,7 +72,7 @@ public final class IndexStore { } } (scope, scopeSQL) = (folders, ranges.isEmpty ? " AND 0" - : " AND EXISTS (SELECT 1 FROM scope s WHERE files.path >= s.lo AND files.path < s.hi)") + : " AND (SELECT s.hi FROM scope s WHERE s.lo <= files.path ORDER BY s.lo DESC LIMIT 1) > files.path") } /// The ranges of the UNIQUE path index that hold the files under `folder`. Paths are stored as a walk reports them, @@ -86,6 +90,30 @@ public final class IndexStore { } } + /// The range list sorted and folded into disjoint spans: nested ranges (a chosen folder inside another), + /// repeats, and touching ones all merge. The scope test above trusts the one range with the greatest + /// `lo` at or below a path, and that answer is exact only when the ranges don't overlap — merging is + /// what makes them not. Comparisons are bytewise, the order SQLite keeps TEXT in (BINARY): it is what + /// keeps `/ab` out of `/a`'s range and a decomposed name out of its precomposed folder. + static func mergedRanges(_ ranges: [(from: String, to: String)]) -> [(from: String, to: String)] { + func byteLess(_ a: String, _ b: String) -> Bool { + let x = Array(a.utf8), y = Array(b.utf8) + for (i, byte) in x.enumerated() { + if i >= y.count { return false } // b ran out: a is longer, a is after b + if byte != y[i] { return byte < y[i] } + } + return x.count < y.count // a is a strict prefix: a is before b + } + var out: [(from: String, to: String)] = [] + for r in ranges.filter { byteLess($0.from, $0.to) }.sorted { byteLess($0.from, $1.from) } { + if var last = out.last, !byteLess(last.to, r.from) { // r.from <= last.to: touching or overlapping + if byteLess(last.to, r.to) { last.to = r.to } + out[out.count - 1] = last + } else { out.append(r) } + } + return out + } + /// The main index's file in the index folder. public static let mainFile = "index.sqlite" /// The code index (`IndexOptions.code`) is a file of its own beside it: code search never moves the rest (its diff --git a/Tests/DigUpKitTests/IndexStoreTests.swift b/Tests/DigUpKitTests/IndexStoreTests.swift index 89b98909..309b3529 100644 --- a/Tests/DigUpKitTests/IndexStoreTests.swift +++ b/Tests/DigUpKitTests/IndexStoreTests.swift @@ -116,6 +116,56 @@ import WinSDK #expect(try store.pendingCount() == 2) } + @Test func mergedScopeKeepsNestedAdjacentAndDuplicateFoldersExact() throws { + // The scope test asks the one range with the greatest `lo` at or below a path, and that answer + // is exact only when the ranges don't overlap — so `setScope` merges the list first. These are + // the merge's shapes: a chosen folder inside a chosen folder (nested), the same folder twice + // (it must fold, not trip the PRIMARY KEY), `/a` next to `/a0` (adjacent — a `/a1` file sits in + // the gap between their ranges), and `/ab` (a sibling in byte order whose files are its own). + let store = try IndexStore(directory: folder) + try store.insert(file("/a/x.png", inode: 1)) + try store.insert(file("/a/b/y.png", inode: 2)) + try store.insert(file("/ab/z.png", inode: 3)) + try store.insert(file("/a0/g.png", inode: 4)) + try store.insert(file("/a1/h.png", inode: 5)) + try store.insert(file("/drive-off/d.png", inode: 6)) + try store.setScope(["/a", "/a/b"]) // nested: the subfolder's files were in /a's range all along + #expect(Set(try store.pending(limit: nil).map(\.path)) == ["/a/x.png", "/a/b/y.png"]) + try store.setScope(["/a", "/a"]) // the same folder twice: folds, the PK never sees a repeat + #expect(try store.pendingCount() == 2) // a file under a chosen folder's subfolder is under it too + try store.setScope(["/a", "/a0"]) // adjacent, gap between: the /a1 file stays out of both + #expect(Set(try store.pending(limit: nil).map(\.path)) == ["/a/x.png", "/a/b/y.png", "/a0/g.png"]) + try store.setScope(["/ab"]) // sibling trap: byte order must not pull /a's files in + #expect(try store.pending(limit: nil).map(\.path) == ["/ab/z.png"]) + try store.setScope(["/a", "/a/b", "/drive-off"]) + #expect(try store.pendingCount() == 3) + #expect(try store.counts().byKindAndState["image"]?["pending"] == 3) + try store.setScope([]) + #expect(try store.pendingCount() == 0) + try store.setScope(nil) + #expect(try store.pendingCount() == 6) + } + + @Test func countsSeeOnlyTheScopedFilesSegments() throws { + // counts() reaches segments through the same scope string the file probes use: the predecessor + // form must filter them exactly as the EXISTS form did (measured in the #191 review's follow-up). + let store = try IndexStore(directory: folder) + try store.insert(file("/a/x.png", inode: 1)) + try store.insert(file("/drive/y.png", inode: 2)) // the unplugged drive + for (path, id) in try store.pending(limit: nil).map({ ($0.path, $0.id) }) { + for _ in 0..<(path == "/a/x.png" ? 2 : 3) { + try store.addSegment(file: id, kind: .chunk, modality: .text, loc: 1, + vector: [Float](repeating: 0.1, count: 768)) + } + } + try store.setScope(["/a"]) + let scoped = try store.counts() + #expect(scoped.segments == 2 && scoped.vectors == 2) // the drive's 3 stay out + try store.setScope(nil) + let all = try store.counts() + #expect(all.segments == 5 && all.vectors == 5) + } + @Test func aFileTheHelperStoppedOnIsNotReadAgain() throws { let store = try IndexStore(directory: folder) try store.insert(file("/a/stuck.pdf", inode: 1)) From c8f052703cfad04ab7f71739d56732081d31317f Mon Sep 17 00:00:00 2001 From: Ocean572 <45051223+Ocean572@users.noreply.github.com> Date: Thu, 5 Nov 2026 18:08:48 -0600 Subject: [PATCH 4/6] Ship the counters as schema v6: the migration, the eight triggers, and the covering index MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The bucket column, counter tables, triggers, fill, and content check all run inside the one transaction migrate() opened, with the version stamp last: a crash at any point leaves v5 exactly intact, and a re-run after such a crash is safe because every step is guarded. The state-first covering index and the prepared-statement path in Database exist for the counter reads that follow in the next commit — the cache's only user is the migration's per-row bucket backfill, where re-parsing per row costs more than the update itself. --- Sources/DigUpKit/Database.swift | 30 +++- Sources/DigUpKit/IndexStore.swift | 279 +++++++++++++++++++++++++++++- 2 files changed, 303 insertions(+), 6 deletions(-) diff --git a/Sources/DigUpKit/Database.swift b/Sources/DigUpKit/Database.swift index 9672c4bb..a3865fdb 100644 --- a/Sources/DigUpKit/Database.swift +++ b/Sources/DigUpKit/Database.swift @@ -61,6 +61,24 @@ public final class Database { public func run(_ sql: String, _ values: [SQLValue] = []) throws { try query(sql, values) } + private var prepared: [String: Statement] = [:] + + /// `run` with the statement kept for the next call of the same SQL: the counters migration updates + /// 1.2M rows one at a time, and preparing per row costs more than the update itself (measured). + /// SQLite re-prepares a cached statement by itself when the schema changes under it (the migration + /// adds triggers after the backfill uses this), so the cache is safe across DDL. + public func run(_ sql: String, _ values: [SQLValue] = [], keeping: Bool) throws { + guard keeping else { try query(sql, values); return } + let statement: Statement + if let cached = prepared[sql] { + statement = cached + } else { + statement = try Statement(self, sql) + prepared[sql] = statement + } + try statement.reuse(values) + } + /// Inside another transaction, a savepoint: its failure undoes only its own writes, and the outer one goes on. private var transactionDepth = 0 @@ -100,6 +118,7 @@ public enum SQLValue { public final class Statement { private let statement: OpaquePointer private let db: Database + private let sql: String init(_ db: Database, _ sql: String) throws { var statement: OpaquePointer? @@ -108,11 +127,13 @@ public final class Statement { } self.statement = statement self.db = db + self.sql = sql } deinit { sqlite3_finalize(statement) } func bind(_ values: [SQLValue]) throws { + sqlite3_clear_bindings(statement) // a reused statement: drop last call's values first for (offset, value) in values.enumerated() { let index = Int32(offset + 1) let status: Int32 @@ -132,10 +153,17 @@ public final class Statement { switch sqlite3_step(statement) { case SQLITE_ROW: return true case SQLITE_DONE: return false - default: throw Database.Failure(message: db.errorMessage) + default: throw Database.Failure(message: "\(db.errorMessage) (step: \(sql.prefix(160)))") } } + /// Bind a fresh set of values into a kept statement and run it to the end (`Database.run(keeping:)`). + func reuse(_ values: [SQLValue]) throws { + guard sqlite3_reset(statement) == SQLITE_OK else { throw Database.Failure(message: db.errorMessage) } + try bind(values) + while try step() {} + } + public func int(_ column: Int32) -> Int64 { sqlite3_column_int64(statement, column) } public func double(_ column: Int32) -> Double { sqlite3_column_double(statement, column) } public func isNull(_ column: Int32) -> Bool { sqlite3_column_type(statement, column) == SQLITE_NULL } diff --git a/Sources/DigUpKit/IndexStore.swift b/Sources/DigUpKit/IndexStore.swift index 766b044e..15af29a7 100644 --- a/Sources/DigUpKit/IndexStore.swift +++ b/Sources/DigUpKit/IndexStore.swift @@ -22,7 +22,7 @@ public struct FileRecord: Sendable { public final class IndexStore { public let directory: URL public let db: Database - static let schemaVersion: Int64 = 5 + static let schemaVersion: Int64 = 6 /// Keyword search's tokenizer. Marks (M*) count as letters, so Bengali vowel signs and Arabic harakat stay inside /// their words: by default SQLite splits on them ("কলকাতায়" became "কলক", "ত", "য"). @@ -114,6 +114,20 @@ public final class IndexStore { return out } + /// The folder a file's name sits in: the path up to the last `/`, the slash dropped. A root-level file + /// ("/x.png") has bucket `""`; a file directly under a Windows drive root ("C:/x") has bucket `"C:"`. + /// Scanned bytewise, and that is the safe way to cut: a 0x2F byte is always a real separator (every + /// UTF-8 continuation byte is ≥ 0x80, so 0x2F never hides inside a multi-byte character), the last + /// slash byte is the last slash character, and cutting at a slash byte always yields valid UTF-8. + /// Buckets are compared bytewise (BINARY) exactly like paths — same order, same rules — which is what + /// makes `bucketRanges` the exact bucket analogue of `pathRanges`. + static func bucket(of path: String) -> String { + var bytes = [UInt8](path.utf8) + guard let last = bytes.lastIndex(of: 0x2F) else { return "" } + bytes.removeSubrange(last...) // everything before the final slash + return String(decoding: bytes, as: UTF8.self) + } + /// The main index's file in the index folder. public static let mainFile = "index.sqlite" /// The code index (`IndexOptions.code`) is a file of its own beside it: code search never moves the rest (its @@ -157,6 +171,7 @@ public final class IndexStore { if (1...2).contains(version) { try migrateScreenshotText() } if (1...3).contains(version) { try migrateVectorNames() } if (1...4).contains(version) { try migrateLongFiles() } + try migrateCounters() // 5 → 6, fresh databases included: an empty one fills trivially } } @@ -190,7 +205,7 @@ public final class IndexStore { ); CREATE INDEX segments_file ON segments(file); CREATE VIRTUAL TABLE fts USING fts5(\(Self.ftsColumns)); - PRAGMA user_version = \(Self.schemaVersion); + PRAGMA user_version = 5; -- base layout: the counters step (6) follows in the same transaction """) } @@ -259,6 +274,258 @@ public final class IndexStore { PRAGMA user_version = 5; """) } + + /// Schema 5 → 6 (issue #193): the counters. The `files.bucket` column, the counter tables, the + /// maintenance triggers, the aggregate fill, and the content check all run inside the one transaction + /// `migrate()` opened, with the version stamp last: a crash at any point leaves v5 exactly intact + /// (audit Finding 12, measured: SIGKILL mid-backfill rolls the whole thing back, ~2 s WAL replay), + /// and a re-run after such a crash is safe because every step is guarded (Finding 12's amendment — + /// a blind `ALTER TABLE` re-run raises `duplicate column name: bucket`, measured). + /// + /// The order is deliberate: column and backfill come first, while no triggers exist, so the backfill's + /// per-row UPDATEs cost one write each and fire no counter maintenance; the tables, triggers, and + /// index come next (creating a trigger needs its tables to exist); the fill then computes every group + /// once from full aggregation, and the check must pass before the stamp — v6 always means "checked". + /// The fill's aggregate and the triggers' row terms share one source (`partialTerm`), so a fill can + /// never disagree with the suite it seeds. + private func migrateCounters() throws { + var hasBucket = false // the pragma_table_info guard Finding 12 measured the need for + try db.query("SELECT 1 FROM pragma_table_info('files') WHERE name = 'bucket'") { _ in hasBucket = true } + if !hasBucket { + try db.execute("ALTER TABLE files ADD COLUMN bucket TEXT NOT NULL DEFAULT ''") + } + // Backfill by rowid in batches of 5,000. Root-level files' true bucket *is* '', so the audit's + // resumable-scan sketch (`WHERE bucket = ''`) would rescan them forever; the rowid walk is the + // resumable form here — a re-run after a crash walks the table again, and the `AND bucket != ?` + // makes the re-run write nothing where the answer is already right. The statement is prepared + // once (`keeping:`): re-parsing per row costs more than the update itself at 1.2M rows. + // Each batch is read whole before it is written: updating a table while its own SELECT cursor is + // open is a live-read hazard. + var last: Int64 = -1 + var count = 5000 + while count == 5000 { + var batch: [(id: Int64, bucket: String)] = [] + try db.query("SELECT id, path FROM files WHERE id > ? ORDER BY id LIMIT 5000", [.int(last)]) { row in + batch.append((row.int(0), Self.bucket(of: row.text(1) ?? ""))) + last = row.int(0) + } + for (id, bucket) in batch { + try db.run("UPDATE files SET bucket = ? WHERE id = ? AND bucket != ?", + [.text(bucket), .int(id), .text(bucket)], keeping: true) + } + count = batch.count + } + try db.execute(Self.countersTablesSQL) + try db.execute(Self.counterTriggersSQL) + try fillCounters() + guard try countersAreConsistent() else { + // The fill disagrees with files: throw, roll the transaction back, stay v5 — the stamp is the + // commit flag, and the next open retries from the guards. + throw Database.Failure(message: "counter fill disagrees with the files table") + } + try db.execute("PRAGMA user_version = 6") + } + + /// A row's pages/chunks left, as a SQL expression over `state`/`info` expressions (Finding 8's + /// amendment): COALESCE **outside** MAX — the exact contribution of one row to `unfinishedCount`'s + /// SUM. A row with one key (`{"read_end":"9"}`) contributes 0, exactly as production measures it; + /// the issue's filed nesting (COALESCE inside the subtraction) would count 9 for it. The CASE keeps + /// `json_extract` away from non-partial rows: malformed `info` on a partial row is refused at the + /// WRITE (the trigger dies, the statement rolls back) where today's form crashes at read time — + /// a net improvement, measured in the audit. One source for all six trigger sites and the fill. + private static func partialTerm(_ state: String, _ info: String) -> String { + "CASE WHEN \(state) = 'partial' THEN COALESCE(MAX(0, CAST(json_extract(\(info), '$.read_end') AS INTEGER)" + + " - CAST(json_extract(\(info), '$.read_to') AS INTEGER)), 0) ELSE 0 END" + } + + /// The aggregate sibling of `partialTerm` for full-scan SELECTs (fill and check). The extra + /// `json_valid`: a v5 row can already hold malformed `info` (the audit: today's `unfinishedCount` + /// crashes reading one), and a migration must not be blocked by a legacy row it cannot read — + /// the fill/check treat it as contributing 0 (and the counters' read path is json-free, so + /// `unfinishedCount` answers where the shipped form dies). The triggers keep the ungarded term: + /// writing a new malformed partial row is refused (Finding 8's write-time rejection, intended). + private static func partialSum(_ prefix: String = "") -> String { + "SUM(CASE WHEN \(prefix)state = 'partial' AND json_valid(\(prefix)info) THEN COALESCE(" + + "MAX(0, CAST(json_extract(\(prefix)info, '$.read_end') AS INTEGER)" + + " - CAST(json_extract(\(prefix)info, '$.read_to') AS INTEGER)), 0) ELSE 0 END)" + } + + /// The counter tables. Audit Finding 4: the filed suite is **eight** triggers + /// (`files_ai`/`files_ad`, `files_au_state`/`files_au_path`, `files_bd`, `seg_ai`/`seg_ad`/`seg_au_vec`) + /// — the issue prose's "nine" is a typo; an implementation following the SQL ships exactly eight. + /// + /// Invariants the suite rests on (audit Findings 1/5/6, each pinned by a CounterTests test): + /// * Nothing updates `segments.kind`/`segments.file` in place — no such statement exists in + /// `Sources/` today. A future reparenting query would desync `segcounters`; `rebuildCounters()` is + /// the documented self-heal for any path that breaks an invariant. + /// * `foreign_keys = ON` on every connection — `Database.init` is the only open site in the codebase + /// and sets it (the Tauri shell opens no SQLite at all). The cascade scheme needs it: for a parent + /// delete, `files_bd` is the ONLY effective `segcounters` decrement, and the cascade-fired `seg_ad` + /// firings see a NULL bucket lookup (the parent row is already gone) and stand down; in app order + /// (`deleteSegments` then `DELETE FROM files`) `seg_ad` decrements per row and `files_bd` then + /// decrements by zero. Both orders land on exactly one decrement per segment (measured firing-by- + /// firing in the audit; re-measured in `cascadeFiringCountsOnceInBothOrders`). + /// * The WHEN guards split `files_au_state` (path unchanged) and `files_au_path` (path or bucket + /// changed — the amended guard, Finding 2) so that every file UPDATE in the codebase lands on + /// exactly one counter maintenance (Finding 3's five measured shapes). `bucket` is derived from + /// `path` and is written only by `insert`, `move`, and the migration backfill — a statement that + /// sets `bucket` alone is exactly the shape the amended guard now catches. + private static let countersTablesSQL = """ + CREATE TABLE IF NOT EXISTS counters( + bucket TEXT NOT NULL, + kind TEXT NOT NULL, + state TEXT NOT NULL, + n INTEGER NOT NULL DEFAULT 0, + leftsum INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY(bucket, kind, state) + ) WITHOUT ROWID; + CREATE TABLE IF NOT EXISTS segcounters( + bucket TEXT NOT NULL, + hasvec INTEGER NOT NULL, + n INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY(bucket, hasvec) + ) WITHOUT ROWID; + """ + + /// The eight triggers, both amendments applied (`files_au_path`'s bucket guard, Finding 2; + /// COALESCE-outside-MAX at all six row-term sites, Finding 8), plus the state-first covering index + /// that makes the scoped reads Θ(folders) (Finding 9: the state-filtered reads as filed cost + /// rows × ranges, 11.1 s measured at 2.985M rows × 100 ranges; with `counters_state` and a + /// scope-driven join the same query measured 0.4 ms, exact against live aggregation). + /// `IF NOT EXISTS` on everything: the migration re-run after a partial state (Finding 12). + static let counterTriggersSQL = { + func term(_ s: String, _ i: String) -> String { IndexStore.partialTerm(s, i) } + let moveCounters = """ + UPDATE counters SET n = counters.n - 1, + leftsum = counters.leftsum - \(term("old.state", "old.info")) + WHERE bucket = old.bucket AND kind = old.kind AND state = old.state; + INSERT INTO counters(bucket, kind, state, n, leftsum) + VALUES (new.bucket, new.kind, new.state, 1, \(term("new.state", "new.info"))) + ON CONFLICT(bucket, kind, state) DO UPDATE SET n = counters.n + 1, leftsum = counters.leftsum + excluded.leftsum; + """ + var sql = """ + CREATE TRIGGER IF NOT EXISTS files_ai AFTER INSERT ON files BEGIN + INSERT INTO counters(bucket, kind, state, n, leftsum) + VALUES (new.bucket, new.kind, new.state, 1, \(term("new.state", "new.info"))) + ON CONFLICT(bucket, kind, state) DO UPDATE SET n = counters.n + 1, leftsum = counters.leftsum + excluded.leftsum; + END; + CREATE TRIGGER IF NOT EXISTS files_ad AFTER DELETE ON files BEGIN + UPDATE counters SET n = counters.n - 1, + leftsum = counters.leftsum - \(term("old.state", "old.info")) + WHERE bucket = old.bucket AND kind = old.kind AND state = old.state; + END; + CREATE TRIGGER IF NOT EXISTS files_au_state AFTER UPDATE OF kind, state, info ON files WHEN old.path IS new.path BEGIN + \(moveCounters) + END; + CREATE TRIGGER IF NOT EXISTS files_au_path AFTER UPDATE OF path, bucket ON files + WHEN old.path IS NOT new.path OR old.bucket IS NOT new.bucket BEGIN + """ + sql += """ + \(moveCounters) + UPDATE segcounters SET n = segcounters.n - (SELECT COUNT(*) FROM segments WHERE file = new.id AND kind != 'name' AND vec IS NULL) + WHERE bucket = old.bucket AND hasvec = 0; + UPDATE segcounters SET n = segcounters.n - (SELECT COUNT(*) FROM segments WHERE file = new.id AND kind != 'name' AND vec IS NOT NULL) + WHERE bucket = old.bucket AND hasvec = 1; + INSERT INTO segcounters(bucket, hasvec, n) + SELECT new.bucket, 0, COUNT(*) FROM segments WHERE file = new.id AND kind != 'name' AND vec IS NULL HAVING COUNT(*) > 0 + ON CONFLICT(bucket, hasvec) DO UPDATE SET n = segcounters.n + excluded.n; + INSERT INTO segcounters(bucket, hasvec, n) + SELECT new.bucket, 1, COUNT(*) FROM segments WHERE file = new.id AND kind != 'name' AND vec IS NOT NULL HAVING COUNT(*) > 0 + ON CONFLICT(bucket, hasvec) DO UPDATE SET n = segcounters.n + excluded.n; + END; + CREATE TRIGGER IF NOT EXISTS files_bd BEFORE DELETE ON files BEGIN + UPDATE segcounters SET n = segcounters.n - (SELECT COUNT(*) FROM segments WHERE file = old.id AND kind != 'name' AND vec IS NULL) + WHERE bucket = old.bucket AND hasvec = 0; + UPDATE segcounters SET n = segcounters.n - (SELECT COUNT(*) FROM segments WHERE file = old.id AND kind != 'name' AND vec IS NOT NULL) + WHERE bucket = old.bucket AND hasvec = 1; + END; + CREATE TRIGGER IF NOT EXISTS seg_ai AFTER INSERT ON segments + WHEN new.kind != 'name' AND EXISTS (SELECT 1 FROM files WHERE id = new.file) BEGIN + INSERT INTO segcounters(bucket, hasvec, n) + VALUES ((SELECT bucket FROM files WHERE id = new.file), new.vec IS NOT NULL, 1) + ON CONFLICT(bucket, hasvec) DO UPDATE SET n = segcounters.n + 1; + END; + CREATE TRIGGER IF NOT EXISTS seg_ad AFTER DELETE ON segments WHEN old.kind != 'name' BEGIN + UPDATE segcounters SET n = segcounters.n - 1 + WHERE bucket = (SELECT bucket FROM files WHERE id = old.file) AND hasvec = (old.vec IS NOT NULL); + END; + CREATE TRIGGER IF NOT EXISTS seg_au_vec AFTER UPDATE OF vec ON segments WHEN old.kind != 'name' AND new.kind != 'name' BEGIN + UPDATE segcounters SET n = segcounters.n - 1 + WHERE bucket = (SELECT bucket FROM files WHERE id = new.file) AND hasvec = (old.vec IS NOT NULL); + INSERT INTO segcounters(bucket, hasvec, n) + VALUES ((SELECT bucket FROM files WHERE id = new.file), (new.vec IS NOT NULL), 1) + ON CONFLICT(bucket, hasvec) DO UPDATE SET n = segcounters.n + 1; + END; + CREATE INDEX IF NOT EXISTS counters_state ON counters(state, bucket, kind, n, leftsum); + """ + return sql + }() + + /// Fill counters and segcounters from full aggregation (the migration's fill and `rebuildCounters`' + /// repair; the aggregate uses `partialTerm`, so it agrees with the triggers by construction). + private func fillCounters() throws { + try db.execute(""" + DELETE FROM counters; + DELETE FROM segcounters; + INSERT INTO counters(bucket, kind, state, n, leftsum) + SELECT bucket, kind, state, COUNT(*), \(Self.partialSum()) FROM files GROUP BY bucket, kind, state; + INSERT INTO segcounters(bucket, hasvec, n) + SELECT f.bucket, s.vec IS NOT NULL, COUNT(*) FROM segments s JOIN files f ON f.id = s.file + WHERE s.kind != 'name' GROUP BY f.bucket, s.vec IS NOT NULL; + """) + } + + /// Do counters and segcounters equal full aggregation, row for row, over the whole library? A + /// NULL-safe `IS NOT` full join against the live GROUP BY: a group missing its counter row, a + /// non-zero row with no group, and any n/leftsum disagreement are mismatches. Zero-valued counter + /// rows with no group stand tolerated (Finding 9's churn residue — they add 0, which is the truth, + /// and the triggers' upserts reuse them): the join filter keeps them out of the match, and the + /// `NOT (key IS NULL AND n = 0 …)` clause forgives the counters-only row that remains, without + /// forgiving a zeroed row that stands over files — that group then appears on its own side of the + /// join (its counter row was filtered away) and is flagged there. The migration's gate before the + /// version stamp, the check `rebuildCounters` ends with, and the tests' oracle companion. + public func countersAreConsistent() throws -> Bool { + var mismatches: Int64 = 0 + try db.query(""" + SELECT (SELECT COUNT(*) FROM ( + SELECT bucket, kind, state, COUNT(*) AS n, \(Self.partialSum()) AS leftsum + FROM files GROUP BY bucket, kind, state) f + FULL OUTER JOIN counters c + ON c.bucket = f.bucket AND c.kind = f.kind AND c.state = f.state + AND (c.n != 0 OR c.leftsum != 0) + WHERE (f.n IS NOT c.n OR f.leftsum IS NOT c.leftsum) + AND NOT (f.bucket IS NULL AND c.n = 0 AND c.leftsum = 0)) + + (SELECT COUNT(*) FROM ( + SELECT f.bucket, s.vec IS NOT NULL AS hasvec, COUNT(*) AS n + FROM segments s JOIN files f ON f.id = s.file + WHERE s.kind != 'name' GROUP BY f.bucket, s.vec IS NOT NULL) g + FULL OUTER JOIN segcounters q ON q.bucket = g.bucket AND q.hasvec = g.hasvec AND q.n != 0 + WHERE g.n IS NOT q.n AND NOT (g.bucket IS NULL AND q.n = 0)) + """) { mismatches = $0.int(0) } + return mismatches == 0 + } + + /// Recompute every counter row from full aggregation, in one transaction, and verify it: the + /// documented self-heal (audit Findings 1/3) — the full aggregation is the truth, and any future + /// path that breaks a trigger invariant can be repaired here without reindexing (the audit measured + /// the full aggregation over 1.2M rows at 36 s; failure rolls back and the counters stand as they were). + public func rebuildCounters() throws { + try db.transaction { + try fillCounters() + guard try countersAreConsistent() else { + throw Database.Failure(message: "counter rebuild disagrees with the files table") + } + } + } + + /// Housekeeping (Finding 9): emptied groups keep their counter rows at zero — reads stay exact with + /// them (they add 0, and `countersAreConsistent` allows them), and they were ~75% of rows after + /// heavy churn in the 5M-file matrix. This clears them; the triggers' upserts re-create rows on + /// demand, so nothing is needed to keep them. No app path calls this automatically — it is + /// maintenance, exercised by `zeroRowsPruneAndRegrow`. + public func pruneZeroCounterRows() throws { + try db.execute("DELETE FROM counters WHERE n = 0 AND leftsum = 0; DELETE FROM segcounters WHERE n = 0;") + } // MARK: Meta public func meta(_ key: String) throws -> String? { @@ -307,9 +574,10 @@ public final class IndexStore { @discardableResult public func insert(_ file: Candidate) throws -> Int64 { try db.transaction { // the row and its name entry together - try db.run("INSERT INTO files(path, kind, size, mtime, inode, device) VALUES(?, ?, ?, ?, ?, ?)", + try db.run("INSERT INTO files(path, kind, size, mtime, inode, device, bucket) VALUES(?, ?, ?, ?, ?, ?, ?)", [.text(file.path), .text(file.kind.rawValue), .int(file.size), .double(file.mtime), - .int(Int64(bitPattern: file.inode)), .int(Int64(bitPattern: file.device))]) + .int(Int64(bitPattern: file.inode)), .int(Int64(bitPattern: file.device)), + .text(Self.bucket(of: file.path))]) let id = db.lastInsertID try addNameRow(file: id, path: file.path) return id @@ -341,7 +609,8 @@ public final class IndexStore { /// (Screenshot tools move screenshots into project folders all the time.) public func move(_ id: Int64, to path: String) throws { try db.transaction { // the path and its name entry together - try db.run("UPDATE files SET path = ? WHERE id = ?", [.text(path), .int(id)]) + try db.run("UPDATE files SET path = ?, bucket = ? WHERE id = ?", // counters: path and bucket + [.text(path), .text(Self.bucket(of: path)), .int(id)]) // move in one statement try db.run(""" UPDATE fts SET name = ?, body = ? WHERE rowid IN (SELECT id FROM segments WHERE file = ? AND kind = 'name') """, [.text(SearchText.normalized(Self.nameText(path))), .text(SearchText.normalized(Self.folderText(path))), From 5718d156c93fe6dc5cdf595066a4a4c27462f639 Mon Sep 17 00:00:00 2001 From: Ocean572 <45051223+Ocean572@users.noreply.github.com> Date: Thu, 5 Nov 2026 18:08:57 -0600 Subject: [PATCH 5/6] Read counts, pending, and unfinished from the counter rows, with aggregation kept as the oracle MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A folder scope drives from the scopeb temp table so the cost follows folders, not rows: each range seeks the covering counters_state index, Θ(folders) measured against live aggregation at every scale (0.4 ms vs 11.1 s for the rows-driven join at 2.985M rows × 100 ranges). fullCounts, fullPendingCount, and fullUnfinishedCount keep the aggregation form as the tests' oracle; bucketRanges spells the counters' bucket-key ranges exactly like pathRanges spells the path ranges, same caveats included. --- Sources/DigUpKit/IndexStore.swift | 140 ++++++++++++++++++++++++++++-- 1 file changed, 131 insertions(+), 9 deletions(-) diff --git a/Sources/DigUpKit/IndexStore.swift b/Sources/DigUpKit/IndexStore.swift index 15af29a7..5bcabb7e 100644 --- a/Sources/DigUpKit/IndexStore.swift +++ b/Sources/DigUpKit/IndexStore.swift @@ -53,6 +53,7 @@ public final class IndexStore { public func setScope(_ folders: [String]?) throws { guard let folders else { (scope, scopeSQL) = (nil, ""); return } let ranges = Self.mergedRanges(folders.flatMap(Self.pathRanges(under:))) + let bucketRanges = Self.mergedRanges(folders.flatMap(Self.bucketRanges(under:))) try db.transaction { try db.run(""" CREATE TEMP TABLE IF NOT EXISTS scope( @@ -70,6 +71,22 @@ public final class IndexStore { chunk.flatMap { [.text($0.from), .text($0.to)] }) start += 200 } + try db.run(""" + CREATE TEMP TABLE IF NOT EXISTS scopeb( + lo TEXT NOT NULL, + hi TEXT NOT NULL, + PRIMARY KEY(lo, hi) + ) WITHOUT ROWID + """) + try db.run("DELETE FROM scopeb") + var bStart = 0 + while bStart < bucketRanges.count { + let chunk = bucketRanges[bStart.. files.path") @@ -128,6 +145,35 @@ public final class IndexStore { return String(decoding: bytes, as: UTF8.self) } + /// The ranges of the counters table's bucket key that hold the files under `folder` — `pathRanges`' + /// dual spellings (the folder as given, plus its real path) mapped into the two shapes buckets come in: + /// * `(own, own + "\u{1}")` where `own` is the folder name without its trailing slash: the files sitting + /// in the folder itself, whose bucket is exactly `own`. A range spanning one code unit past `own` + /// contains that one bucket value and nothing else: every longer bucket extending `own` starts with + /// a byte ≥ 0x01, and 0x2F (`/`) is ≥ 0x01, so a sibling folder named "notes." or "notes!" — which is + /// NOT inside "notes" — has its files' buckets sort past this range's end. Root-level files + /// (bucket `""`) and drive-root files (bucket `"C:"`) land in this range for the root spellings. + /// * `(folder + "/", folder + "/" + "\u{10FFFF}")`: the files in the folder's subfolders — their bucket + /// starts with `folder + "/"`, and a bucket under `folder`'s path bound is a bucket under its name + /// bound, because a bucket is a prefix of its files' paths. + /// The `"\u{10FFFF}"` bound is inherited from `pathRanges` verbatim (it is that range's own `to`): on + /// macOS a name containing it is unrepresentable — Foundation **and** raw `mkdir`/`open` refuse it + /// (APFS, measured in the audit) — so the bound is unreachable here; Linux (ext4: any UTF-8 sans + /// NUL/`/`) and Windows (NTFS: full UTF-16, surrogate pairs included) can represent a name starting + /// with it, and such a file/folder escapes the range in *both* forms identically — behavior inherited + /// from the shipped path machinery, not introduced here (audit Finding 10). The one asymmetry buckets + /// add: a direct child named starting with U+10FFFF has bucket `own`, so the counters count it while + /// the path query's bound excludes it (audit Finding 10's caveat, disclosed in the report). + static func bucketRanges(under folder: String) -> [(from: String, to: String)] { + pathRanges(under: folder).flatMap { range in + let own = String(range.from.dropLast()) // the prefix's trailing slash IS the cut: the folder + // name is its own bucket. `bucket(of:)` is NOT applicable to a folder path — it cuts the last + // segment, so it maps `/t/a` to its parent `/t`, which would make the exact range one folder + // too broad and drop the files sitting in the folder itself. + return [(own, own + "\u{1}"), (range.from, range.to)] + } + } + /// The main index's file in the index folder. public static let mainFile = "index.sqlite" /// The code index (`IndexOptions.code`) is a file of its own beside it: code search never moves the rest (its @@ -661,10 +707,26 @@ public final class IndexStore { return out } + /// Files waiting to be embedded (#193: read from the counters, not aggregated). Nil scope sums the + /// `counters` table's pending section; a folder scope joins from `scopeb` so the cost is one index + /// seek per connected folder, not a sweep of the counter table (audit Finding 9, measured: the + /// state-filtered plain join costs rows × ranges — 11.1 s at 2.985M rows × 100 ranges; the CROSS + /// JOIN form seeks `counters_state (state=? AND bucket>? AND bucket Int { var count = 0 - try db.query("SELECT COUNT(*) FROM files WHERE state = 'pending'\(scopeSQL)") { - count = Int($0.int(0)) + if scope == nil { + try db.query("SELECT COALESCE(SUM(n), 0) FROM counters WHERE state = 'pending'") { + count = Int($0.int(0)) + } + } else { + try db.query(""" + SELECT COALESCE(SUM(c.n), 0) FROM scopeb b CROSS JOIN counters c + ON c.state = 'pending' AND c.bucket >= b.lo AND c.bucket < b.hi + """) { count = Int($0.int(0)) } } return count } @@ -679,14 +741,24 @@ public final class IndexStore { return out } - /// How many long files are unfinished, and how many of their pages and chunks are left to read. + /// How many long files are unfinished, and how many of their pages and chunks are left to read + /// (#193: `n` and `leftsum` from the counters; `leftsum` is maintained with the exact per-row shape + /// this query's SUM produced — Finding 8's amended term, one source with the triggers and the fill). + /// Same join contract as `pendingCount`; a partial row with one key (`{"read_end":"9"}`) counts 0 + /// toward `left`, as production measures it. public func unfinishedCount() throws -> (files: Int, left: Int) { var count = (files: 0, left: 0) - try db.query(""" - SELECT COUNT(*), SUM(MAX(0, CAST(json_extract(info, '$.read_end') AS INTEGER) - - CAST(json_extract(info, '$.read_to') AS INTEGER))) - FROM files WHERE state = 'partial'\(scopeSQL) - """) { count = (Int($0.int(0)), Int($0.int(1))) } + if scope == nil { + try db.query("SELECT COALESCE(SUM(n), 0), COALESCE(SUM(leftsum), 0) FROM counters WHERE state = 'partial'") { + count = (Int($0.int(0)), Int($0.int(1))) + } + } else { + try db.query(""" + SELECT COALESCE(SUM(c.n), 0), COALESCE(SUM(c.leftsum), 0) + FROM scopeb b CROSS JOIN counters c + ON c.state = 'partial' AND c.bucket >= b.lo AND c.bucket < b.hi + """) { count = (Int($0.int(0)), Int($0.int(1))) } + } return count } @@ -815,12 +887,62 @@ public final class IndexStore { public var left = 0 } + /// What indexing learned, in totals (#193: counters reads; `fullCounts` is the aggregation form kept + /// as the tests' oracle). The `byKindAndState` dictionary lists only groups with files — zero-valued + /// counter rows (Finding 9's churn residue) are skipped, exactly as the old GROUP BY returned no row + /// for an absent group. Nil scope scans the two covering counter tables; a folder scope drives from + /// `scopeb` (PK bucket seek, Θ(folders)); an empty scope joins to zero, matching the old ` AND 0`. public func counts() throws -> Counts { + var counts = Counts() + let rowSQL = scope == nil + ? "SELECT kind, state, SUM(n) FROM counters GROUP BY kind, state" + : "SELECT c.kind, c.state, SUM(c.n) FROM scopeb b CROSS JOIN counters c ON c.bucket >= b.lo AND c.bucket < b.hi GROUP BY c.kind, c.state" + try db.query(rowSQL) { row in + let n = Int(row.int(2)) + guard n != 0 else { return } + counts.byKindAndState[row.text(0) ?? "?", default: [:]][row.text(1) ?? "?"] = n + } + (counts.unfinished, counts.left) = try unfinishedCount() + let segSQL = scope == nil + ? "SELECT hasvec, SUM(n) FROM segcounters GROUP BY hasvec" + : "SELECT c.hasvec, SUM(c.n) FROM scopeb b CROSS JOIN segcounters c ON c.bucket >= b.lo AND c.bucket < b.hi GROUP BY c.hasvec" + try db.query(segSQL) { row in + counts.segments += Int(row.int(1)) + if row.int(0) == 1 { counts.vectors += Int(row.int(1)) } + } + return counts + } + + // MARK: Full aggregation — the oracle (#193 keeps it) + + /// `pendingCount`'s aggregation form, verbatim as shipped before the counters. Every counters test + /// compares the maintained answer against this; `rebuildCounters` is built from this class of query. + public func fullPendingCount() throws -> Int { + var count = 0 + try db.query("SELECT COUNT(*) FROM files WHERE state = 'pending'\(scopeSQL)") { + count = Int($0.int(0)) + } + return count + } + + /// `unfinishedCount`'s aggregation form, verbatim as shipped before the counters. + public func fullUnfinishedCount() throws -> (files: Int, left: Int) { + var count = (files: 0, left: 0) + try db.query(""" + SELECT COUNT(*), SUM(MAX(0, CAST(json_extract(info, '$.read_end') AS INTEGER) + - CAST(json_extract(info, '$.read_to') AS INTEGER))) + FROM files WHERE state = 'partial'\(scopeSQL) + """) { count = (Int($0.int(0)), Int($0.int(1))) } + return count + } + + /// `counts`' aggregation form, verbatim as shipped before the counters. + public func fullCounts() throws -> Counts { var counts = Counts() try db.query("SELECT kind, state, COUNT(*) FROM files WHERE 1\(scopeSQL) GROUP BY kind, state") { counts.byKindAndState[$0.text(0) ?? "?", default: [:]][$0.text(1) ?? "?"] = Int($0.int(2)) } - (counts.unfinished, counts.left) = try unfinishedCount() + (counts.unfinished, counts.left) = try fullUnfinishedCount() let inScope = scope == nil ? "" : " AND file IN (SELECT id FROM files WHERE 1\(scopeSQL))" try db.query("SELECT COUNT(*), COUNT(vec) FROM segments WHERE kind != 'name'\(inScope)") { counts.segments = Int($0.int(0)) From 2e2d50cbd09e46c8922385b68f017ee60442ffd2 Mon Sep 17 00:00:00 2001 From: Ocean572 <45051223+Ocean572@users.noreply.github.com> Date: Thu, 5 Nov 2026 18:09:06 -0600 Subject: [PATCH 6/6] Cover the counter lifecycle, migration recovery, and trigger invariants in 25 tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every write path is followed by a row-for-row comparison against the aggregation oracle; migration tests include a SIGKILL of the real CLI child mid-migration (rolls back to a clean v5, the next open completes to v6), the legacy malformed-partial shape, and two connections on one WAL index. The two invariant pins record the suite's assumptions as behavior: no production statement reparents a segment row in place, and the WHEN guards fire once per file update — the forbidden state-plus-bucket shape is shown to drift the counters and rebuildCounters is shown to heal it. --- Tests/DigUpKitTests/CounterTests.swift | 966 +++++++++++++++++++++++++ 1 file changed, 966 insertions(+) create mode 100644 Tests/DigUpKitTests/CounterTests.swift diff --git a/Tests/DigUpKitTests/CounterTests.swift b/Tests/DigUpKitTests/CounterTests.swift new file mode 100644 index 00000000..3507f56b --- /dev/null +++ b/Tests/DigUpKitTests/CounterTests.swift @@ -0,0 +1,966 @@ +import Foundation +import Testing +@testable import DigUpKit + +/// Issue #193: the counter tables, their triggers, the scoped joins, and the schema-v6 migration. +/// Every test compares the maintained counters against full aggregation over `files`/`segments` — +/// `fullCounts`/`fullPendingCount`/`fullUnfinishedCount` are the pre-counters production forms kept +/// verbatim as the oracle, and `countersAreConsistent` is the row-for-row check the migration runs +/// before it stamps v6. +@Suite struct CounterTests { + let folder: URL = { + let url = FileManager.default.temporaryDirectory.appendingPathComponent("DigUpCounters-\(UUID().uuidString)") + try? FileManager.default.createDirectory(at: url, withIntermediateDirectories: true) + return url + }() + + func file(_ path: String, inode: UInt64, kind: FileKind = .image) -> Candidate { + Candidate(path: path, kind: kind, size: 10, mtime: 1, inode: inode, device: 1) + } + + /// The oracle pair: maintained totals equal aggregated totals, every field. + func expectEqual(_ a: IndexStore.Counts, _ b: IndexStore.Counts, _ note: String = "") throws { + #expect(a.byKindAndState == b.byKindAndState, "byKindAndState \(note)") + #expect(a.segments == b.segments, "segments \(note)") + #expect(a.vectors == b.vectors, "vectors \(note)") + #expect(a.unfinished == b.unfinished, "unfinished \(note)") + #expect(a.left == b.left, "left \(note)") + } + + /// All four comparisons at once, under the scope the store currently carries. + func expectAllMatch(_ store: IndexStore, _ note: String = "") throws { + #expect(try store.countersAreConsistent(), "row-for-row consistency \(note)") + try expectEqual(store.counts(), store.fullCounts(), note) + #expect(try store.pendingCount() == store.fullPendingCount(), "pendingCount \(note)") + #expect(try store.unfinishedCount() == store.fullUnfinishedCount(), "unfinishedCount \(note)") + } + + func version(_ store: IndexStore) throws -> Int64 { + var v: Int64 = -1 + try store.db.query("PRAGMA user_version") { v = $0.int(0) } + return v + } + + /// Back to the v5 shape: the counters layout and the bucket column go, the version stamp drops. + /// `keepColumn` simulates a migration interrupted after the ALTER (Finding 12's partial states). + /// (DROP TRIGGER/INDEX/TABLE take `IF EXISTS` — SQLite has no `IF NOT EXISTS` on DROP: that's the + /// syntax the interrupted-migration re-runs must tolerate, not the test's setup.) + func dropToV5(_ store: IndexStore, keepColumn: Bool = false, keepCounters: Bool = false) throws { + var sql = """ + DROP TRIGGER IF EXISTS files_ai; DROP TRIGGER IF EXISTS files_ad; + DROP TRIGGER IF EXISTS files_au_state; DROP TRIGGER IF EXISTS files_au_path; + DROP TRIGGER IF EXISTS files_bd; DROP TRIGGER IF EXISTS seg_ai; + DROP TRIGGER IF EXISTS seg_ad; DROP TRIGGER IF EXISTS seg_au_vec; + DROP INDEX IF EXISTS counters_state; + """ + if !keepCounters { sql += " DROP TABLE IF EXISTS counters; DROP TABLE IF EXISTS segcounters;" } + if !keepColumn { sql += " ALTER TABLE files DROP COLUMN bucket;" } + sql += " PRAGMA user_version = 5;" + try store.db.execute(sql) + } + + // MARK: Write paths + + @Test func countersFollowEveryWritePath() throws { + let store = try IndexStore(directory: folder) + try expectAllMatch(store, "empty") // fresh database: counters exist, all zero + // insert (+ the name segment seg_ai guards out) + let a = try store.insert(file("/t/a/x.png", inode: 1)) + let b = try store.insert(file("/t/a/sub/y.pdf", inode: 2)) + let c = try store.insert(file("/t/b/z.doc", inode: 3)) + try expectAllMatch(store, "inserts") + // segments with and without vectors + let vec = [Float](repeating: 0.25, count: 768) + try store.addSegment(file: a, kind: .image, modality: .image, vector: vec) + try store.addSegment(file: a, kind: .ocr, modality: .text, text: "road report") + try store.addSegment(file: b, kind: .page, modality: .text, loc: 1, text: "harbour pages") + try store.addSegment(file: c, kind: .chunk, modality: .text, loc: 1, text: "long text", vector: vec) + try expectAllMatch(store, "segments") + // finish: every state shape + try store.finish(a, state: "done", info: ["pixels": "800x600"]) + try store.finish(b, state: "partial", detail: "read to page 2", + info: ["read_to": "2", "read_end": "9", "pages": "9"]) + try store.finish(c, state: "skipped", detail: "too small", info: [:]) + try expectAllMatch(store, "finishes") + // the vector-NULL migration shape (migrateLongFiles/migrateScreenshotText): seg_au_vec + var nulled: [Int64] = [] + try store.db.query("SELECT id FROM segments WHERE vec IS NOT NULL") { nulled.append($0.int(0)) } + for id in nulled { + try store.db.run("UPDATE segments SET vec = NULL, modality = 'none' WHERE id = ?", [.int(id)]) + } + try expectAllMatch(store, "vectors nulled") + // rename: path + bucket in one statement, segcounters relocate with it + try store.move(a, to: "/t/c/x.png") + try expectAllMatch(store, "rename") + // requeue: deleteSegments (app order) + the state/kind UPDATE + try store.requeue(b, as: file("/t/a/sub/y.pdf", inode: 22, kind: .doc)) + try expectAllMatch(store, "requeue as candidate") + try store.requeue(a) + try expectAllMatch(store, "requeue plain") + // markStopped: a long file being finished becomes done + #expect(try store.markStopped("/t/b/z.doc", why: "ran away")) + try expectAllMatch(store, "markStopped") + // delete: cascade, files_bd + files_ad (Finding 5's cascade order) + try store.delete(c) + try expectAllMatch(store, "delete") + // prune: per-id deletes of everything outside the kept roots + #expect(try store.prune(keepingUnder: ["/t/a"]) == 1) + try expectAllMatch(store, "prune") + // resetAll: bulk seg_ad + one files_au_state per file + try store.resetAll() + try expectAllMatch(store, "resetAll") + // the failed inner transaction: its counters undo, the outer's stand (Database.savepoint nesting) + struct Oops: Error {} + try store.db.transaction { + try store.insert(file("/t/a/keep.png", inode: 40)) + #expect(throws: Oops.self) { + try store.db.transaction { + try store.insert(file("/t/a/gone.png", inode: 41)) + throw Oops() + } + } + } + try expectAllMatch(store, "failed inner transaction") + } + + @Test func stateTransitionsInBothDirections() throws { + let store = try IndexStore(directory: folder) + let id = try store.insert(file("/t/doc.txt", inode: 1, kind: .doc)) + let pair: [String: String] = ["read_to": "10", "read_end": "40"] + let done: [String: String] = ["chars": "60000"] + // pending → done → pending → partial → done → partial → skipped → partial → failed → pending + for step: (String, [String: String]) in [ + ("done", done), ("pending", [:]), ("partial", pair), ("done", done), ("partial", pair), + ("skipped", [:]), ("partial", pair), ("failed", [:]), ("pending", [:]), + ] { + try store.finish(id, state: step.0, info: step.1) + try expectAllMatch(store, "to \(step.0)") + } + // the same-value UPDATE (OF list names the columns, values equal): −1/+1 on one row + try store.finish(id, state: "pending", info: [:]) + try expectAllMatch(store, "same-value requeue") + } + + @Test func oneKeyPartialRowsCountAsProduction() throws { + // Finding 8's amendment: COALESCE outside MAX. Production's per-row term is + // MAX(0, read_end − read_to): a NULL key makes it NULL, and SUM skips the row → 0. + let store = try IndexStore(directory: folder) + let oneKey = try store.insert(file("/t/p/one.pdf", inode: 1, kind: .pdf)) + let bothKeys = try store.insert(file("/t/p/both.pdf", inode: 2, kind: .pdf)) + let reversed = try store.insert(file("/t/p/back.pdf", inode: 3, kind: .pdf)) + try store.finish(oneKey, state: "partial", info: ["read_end": "9"]) + try store.finish(bothKeys, state: "partial", info: ["read_to": "12", "read_end": "40"]) + try store.finish(reversed, state: "partial", info: ["read_to": "40", "read_end": "12"]) + try expectAllMatch(store, "one-key partials") + let (files, left) = try store.unfinishedCount() + #expect(files == 3 && left == 28, "3 partial rows, 28 left: production's answer") + // a malformed info on a NON-partial row is accepted (the CASE never extracts it) + try store.finish(oneKey, state: "skipped", detail: "not json", info: [:]) + try store.db.run("UPDATE files SET info = 'not json' WHERE id = ?", [.int(oneKey)]) + try expectAllMatch(store, "malformed on a non-partial row") + // transitioning it to partial via finish() is ACCEPTED: finish writes valid JSON, and the + // trigger's CASE short-circuits — json_extract sees only the NEW info (old is 'skipped', its + // term is 0 without extracting). Refusal is for malformed NEW info: the raw INSERT below. + try store.finish(oneKey, state: "partial", info: [:]) + try expectAllMatch(store, "malformed old info transitioned out") + #expect(try store.unfinishedCount().left == 28, "the transitioned row contributes 0 (empty info)") + #expect(throws: (any Error).self) { + try store.db.run("INSERT INTO files(path, kind, size, mtime, inode, device, bucket, state, info) " + + "VALUES('/t/p/bad.pdf', 'pdf', 10, 1, 99, 1, '/t/p', 'partial', 'not json')") + } + try expectAllMatch(store, "malformed partial refused") + } + + @Test func vectorsAddRemoveAndFlip() throws { + let store = try IndexStore(directory: folder) + let id = try store.insert(file("/t/v/img.png", inode: 1)) + try store.addSegment(file: id, kind: .image, modality: .image, vector: [Float](repeating: 0.1, count: 768)) + try expectAllMatch(store, "vector added") + try store.db.run("UPDATE segments SET vec = NULL, modality = 'none' WHERE file = ?", [.int(id)]) + try expectAllMatch(store, "vector removed") + try store.db.run("UPDATE segments SET vec = ? WHERE file = ? AND kind != 'name'", + [.blob(Vectors.half([Float](repeating: 0.2, count: 768))), .int(id)]) + try expectAllMatch(store, "vector back") + try store.db.run("DELETE FROM segments WHERE file = ? AND kind != 'name'", [.int(id)]) + try expectAllMatch(store, "segments gone (app-order seg_ad)") + } + + @Test func renameMovesCountersAcrossBuckets() throws { + let store = try IndexStore(directory: folder) + let id = try store.insert(file("/t/a/x.png", inode: 1)) + try store.addSegment(file: id, kind: .image, modality: .image, vector: [Float](repeating: 0.1, count: 768)) + try store.finish(id, state: "done") + // out of /t/a, into /t/b: counts, segments and vectors follow + try store.setScope(["/t/a"]) + try expectAllMatch(store, "scoped at the old folder") + try store.move(id, to: "/t/b/x.png") + try expectAllMatch(store, "moved, scope still the old folder") + #expect(try store.pendingCount() == 0 && store.counts().segments == 0) + try store.setScope(["/t/b"]) + try expectAllMatch(store, "scoped at the new folder") + #expect(try store.counts().segments == 1 && store.counts().vectors == 1) + // a no-op move: the guard skips, counters stand + try store.move(id, to: "/t/b/x.png") + try expectAllMatch(store, "no-op move") + } + + @Test func bucketOnlyUpdateRepairKeepsCountersFollowing() throws { + // Finding 2's amendment, the G1 shape: `UPDATE files SET bucket = …` alone — exactly the + // migration-backfill/repair shape — must maintain the counters, not drift them. + let store = try IndexStore(directory: folder) + let id = try store.insert(file("/t/a/x.png", inode: 1)) + try store.addSegment(file: id, kind: .image, modality: .image, vector: [Float](repeating: 0.1, count: 768)) + try store.db.run("UPDATE files SET bucket = '/t/drift' WHERE id = ?", [.int(id)]) + try #require(try store.countersAreConsistent(), "counters followed the bucket surgery") + var bucket: String? + try store.db.query("SELECT bucket FROM counters WHERE bucket = '/t/drift'") { bucket = $0.text(0) } + #expect(bucket == "/t/drift", "the counter row moved with the file") + // the scoped counters (bucket space) and the scoped path query agree only on canonical + // buckets: surgery shows the two spaces are joined, not glued — the documented contract + try store.setScope(["/t/drift"]) + #expect(try store.pendingCount() == 1, "counters count the file at its new bucket") + #expect(try store.fullPendingCount() == 0, "the path test still sees the file at its path") + // repair back to canonical: the counters move back and the scoped forms agree again + try store.db.run("UPDATE files SET bucket = ? WHERE id = ?", [.text(IndexStore.bucket(of: "/t/a/x.png")), .int(id)]) + try store.setScope(["/t/a"]) + try expectAllMatch(store, "repaired bucket") + } + + @Test func cascadeFiringCountsOnceInBothOrders() throws { + // Findings 3/5's firing evidence (the audit's cascade.py pattern, re-run in production SQL): + // cascade order — files_bd decrements, cascade-fired seg_ad sees a NULL bucket and stands + // down; app order (production `delete`'s shape) — seg_ad decrements per row with the parent + // alive, files_bd fires on an empty nest. Two stores: the cascade half deliberately uses a + // RAW parent delete to fire the cascade — and a raw cascade leaves the nest's fts5 rows + // behind (nothing deletes them; production never raw-cascades: `delete` removes the fts rows + // first). Left in one connection, the leaked rows collide with the next insert's reused + // segment rowids, so the app-order half runs on a fresh store. + func logTriggers(_ store: IndexStore) throws { + try store.db.execute(""" + CREATE TEMP TABLE fired(name TEXT NOT NULL, detail TEXT); + CREATE TEMP TRIGGER log_files_bd BEFORE DELETE ON files BEGIN + INSERT INTO fired VALUES('files_bd', CAST(old.id AS TEXT)); + END; + CREATE TEMP TRIGGER log_seg_ad AFTER DELETE ON segments BEGIN + INSERT INTO fired VALUES('seg_ad', (SELECT bucket FROM files WHERE id = old.file) IS NULL); + END; + """) + } + func fires(_ store: IndexStore, _ name: String) throws -> [String] { + var out: [String] = [] + try store.db.query("SELECT detail FROM fired WHERE name = ? ORDER BY rowid", [.text(name)]) { + out.append($0.text(0) ?? "?") + } + return out + } + let vec = [Float](repeating: 0.1, count: 768) + func nest(_ store: IndexStore, _ candidate: Candidate, _ inode: UInt64) throws -> Int64 { + let id = try store.insert(candidate) + for i in 0..<5 { + try store.addSegment(file: id, kind: i < 3 ? .image : .ocr, modality: .image, vector: i < 3 ? vec : nil) + } + return id + } + // cascade order: raw DELETE FROM files with a live nest + let cascade = try IndexStore(directory: folder) + try logTriggers(cascade) + let cascaded = try nest(cascade, file("/c/one.png", inode: 1), 1) + try cascade.finish(cascaded, state: "done") + try cascade.db.run("DELETE FROM fired") + try cascade.db.run("DELETE FROM files WHERE id = ?", [.int(cascaded)]) + #expect(try fires(cascade, "files_bd").count == 1, "files_bd fires once") + #expect(try fires(cascade, "seg_ad") == Array(repeating: "1", count: 6), + "every cascade-fired seg_ad row (the 5-segment nest + the name row) sees the parent gone (NULL bucket)") + try #require(try cascade.countersAreConsistent(), "cascade: decremented exactly once") + try expectAllMatch(cascade, "cascade store after") + // app order: the production delete() shape — explicit segment deletes (fts first), then the file + let app = try IndexStore(directory: folder.appendingPathComponent("app-order")) + try logTriggers(app) + let moved = try nest(app, file("/c/two.png", inode: 2), 2) + try app.db.run("DELETE FROM fired") + try app.db.run("DELETE FROM fts WHERE rowid IN (SELECT id FROM segments WHERE file = ? AND kind != 'name')", [.int(moved)]) + try app.db.run("DELETE FROM segments WHERE file = ? AND kind != 'name'", [.int(moved)]) + #expect(try fires(app, "seg_ad") == Array(repeating: "0", count: 5), + "the app-order seg_ad rows all see a live parent (real bucket)") + try app.delete(moved) // fts rows, then the parent: the name row goes by cascade + #expect(try fires(app, "files_bd").count == 1, "files_bd fires once, on a nest of only the name row") + try #require(try app.countersAreConsistent(), "app order: decremented exactly once") + try expectAllMatch(app, "app store after") + } + + @Test func triggerSuiteIsTheEightAmendedTriggers() throws { + let store = try IndexStore(directory: folder) + var names: [String] = [] + try store.db.query("SELECT name FROM sqlite_master WHERE type = 'trigger' ORDER BY name") { + names.append($0.text(0) ?? "?") + } + #expect(names == ["files_ad", "files_ai", "files_au_path", "files_au_state", + "files_bd", "seg_ad", "seg_ai", "seg_au_vec"], + "eight triggers (Finding 4: the prose's 'nine' is a typo)") + let q = "%" // Swift 6.2 reads % in literals as interpolation — SQL's wildcard goes through a value + var amended = 0 + try store.db.query("SELECT COUNT(*) FROM sqlite_master WHERE type = 'trigger' " + + "AND sql LIKE '\(q)OR old.bucket IS NOT new.bucket\(q)'") { amended = Int($0.int(0)) } + #expect(amended == 1, "Finding 2's guard amendment is in files_au_path") + var coalesced = 0 + try store.db.query("SELECT COUNT(*) FROM sqlite_master WHERE type = 'trigger' " + + "AND sql LIKE '\(q)COALESCE(MAX(0\(q)'") { coalesced = Int($0.int(0)) } + #expect(coalesced == 4, "Finding 8's nesting: COALESCE outside MAX at every row-term trigger (4 carry terms)") + var covering = 0 + try store.db.query("SELECT COUNT(*) FROM sqlite_master WHERE type = 'index' AND name = 'counters_state'") { + covering = Int($0.int(0)) + } + #expect(covering == 1, "Finding 9's state-first covering index") + var fk: Int64 = 0 + try store.db.query("PRAGMA foreign_keys") { fk = $0.int(0) } // Finding 6 + #expect(fk == 1, "writable connections carry foreign_keys=ON") + // the segments.kind/.file immutability invariant (Finding 1) is pinned by the write-path test + } + + // MARK: Scope shapes + + @Test func scopedReadsMatchAggregationOnEveryShape() throws { + let store = try IndexStore(directory: folder) + var inode: UInt64 = 1 + let vec = [Float](repeating: 0.1, count: 768) + let seeds: [(path: String, kind: FileKind, state: String?, info: [String: String])] = [ + ("/x.png", .image, "done", [:]), + ("/t/a/x.png", .image, nil, [:]), + ("/t/a/y.png", .image, "done", [:]), + ("/t/a/sub/z.pdf", .pdf, "partial", ["read_to": "2", "read_end": "9"]), + ("/t/a/sub/deep/w.doc", .doc, "partial", ["read_end": "9"]), + ("/t/ab/q.png", .image, nil, [:]), + ("/t/a./d.png", .image, "done", [:]), + ("/t/a !/e.png", .image, nil, [:]), + ("/t/a\tt/f.png", .image, nil, [:]), + ("/t/notes\u{00E9}/g.png", .image, nil, [:]), + ("/t/notes\u{0301}/h.png", .image, nil, [:]), + ("/t/mix/one.pdf", .pdf, "partial", ["read_to": "40", "read_end": "12"]), + ("/t/mix/done.doc", .doc, "skipped", [:]), + ] + var ids: [Int64] = [] + for s in seeds { + ids.append(try store.insert(file(s.path, inode: inode, kind: s.kind))) + inode += 1 + if let state = s.state { try store.finish(ids.last!, state: state, info: s.info) } + } + try store.addSegment(file: ids[1], kind: .image, modality: .image, vector: vec) // /t/a/x.png + try store.addSegment(file: ids[3], kind: .page, modality: .text, loc: 1, text: "harbour pages") + try store.addSegment(file: ids[11], kind: .chunk, modality: .text, loc: 1, text: "long text", vector: vec) + let shapes: [[String]?] = [ + nil, [], ["/"], ["/t"], ["/t/a"], ["/t/a/sub"], ["/t/a", "/t/a/sub"], + ["/t/a", "/t/ab"], ["/t/a", "/t/a"], ["/t/a/sub", "/t/a/sub/deep"], + ["/t", "/t/a"], ["/t/a."], ["/t/a "], ["/t/a\tt"], ["/nope"], ["/notes"], + ["/t/notes\u{0301}"], ["/t/notes\u{00E9}"], + ["/t/a"] + (0..<1_200).map { "/root-\($0)" }, + (0..<2_400).map { "/root-\($0)" } + ["/t/a"], + ] + for shape in shapes { + try store.setScope(shape) + try expectAllMatch(store, "scope of \(shape?.count ?? -1)") + } + // What /t/a's own counters see: its direct children plus everything in its subfolders, + // nothing of the "ab" / "a." / "a !" / "a\tt" siblings, nothing at the filesystem root. + try store.setScope(["/t/a"]) + let c = try store.counts() + #expect((c.byKindAndState["image"]?["done"] ?? 0) == 1, "y.png: /t/a direct child") + #expect((c.byKindAndState["image"]?["pending"] ?? 0) == 1, "x.png: /t/a direct child") + #expect((c.byKindAndState["pdf"]?["partial"] ?? 0) == 1, "z.pdf: the scoped folder's one pdf, in a subfolder") + #expect((c.byKindAndState["doc"]?["partial"] ?? 0) == 1, "deep w.doc: a doc two folders down rides in /t/a's counters") + #expect(c.unfinished == 2 && c.left == 7, "the one-key deep file contributes 0, as production") + #expect(c.segments == 2 && c.vectors == 1, "the /t/mix chunk belongs to its own folder") + try store.setScope([]) + let z = try store.counts() + #expect(z.byKindAndState.isEmpty && z.segments == 0 && z.vectors == 0 + && z.unfinished == 0 && z.left == 0, "empty scope: all zero (the old ' AND 0' answer)") + try expectAllMatch(store, "empty scope") + } + + @Test func bucketRuleSlashAndRootForms() throws { + let cases: [(String, String)] = [ + ("/a/b/c.png", "/a/b"), ("/a/b", "/a"), ("/x.png", ""), ("/", ""), ("", ""), + ("C:/x", "C:"), ("C:/x/y", "C:/x"), ("/a//x", "/a/"), ("/a/", "/a"), + ("/notes./x", "/notes."), ("/a !/x", "/a !"), ("/a\tt/x", "/a\tt"), + ("/a\u{E002F}tag/x", "/a\u{E002F}tag"), // tag chars encode no 0x2F byte: the scan cuts at the real slash + ("/\u{1F1E6}\u{1F1E6}/x", "/\u{1F1E6}\u{1F1E6}"), // a regional-indicator pair + ("/\u{10FFFE}/x", "/\u{10FFFE}"), + ] + for (path, want) in cases { + #expect(IndexStore.bucket(of: path) == want, "bucket(of: \(path.debugDescription))") + } + // The reference rule: everything up to the last slash CHARACTER — the bytewise scan and + // the character scan agree, because 0x2F bytes are always real separators in UTF-8. + for (path, _) in cases where path.lastIndex(of: "/") != nil { + let last = try #require(path.lastIndex(of: "/")) + #expect(IndexStore.bucket(of: path) == String(path[.. Bool { + let x = Array(a.utf8), y = Array(b.utf8) + for (i, byte) in x.enumerated() { + if i >= y.count { return false } + if byte != y[i] { return byte < y[i] } + } + return x.count < y.count + } + + func sameRanges(_ a: [(String, String)], _ b: [(String, String)]) -> Bool { + a.count == b.count && zip(a, b).allSatisfy { $0.0 == $1.0 && $0.1 == $1.1 } + } + + @Test func setScopeWritesTheMergedBucketRanges() throws { + let store = try IndexStore(directory: folder) + let folders = ["/t/a", "/t/a/sub"] + try store.setScope(folders) + let expected = IndexStore.mergedRanges(folders.flatMap(IndexStore.bucketRanges(under:))) + .map { ($0.from, $0.to) }.sorted { byteLess($0.0, $1.0) } + var rows: [(String, String)] = [] + try store.db.query("SELECT lo, hi FROM scopeb") { rows.append(($0.text(0) ?? "", $0.text(1) ?? "")) } + #expect(sameRanges(rows.sorted { byteLess($0.0, $1.0) }, expected)) + #expect(expected.count == 2, + "nested scopes fold: /t/a/sub's exact and deeper ranges merge into /t/a's deeper span") + // past the chunk size (200): the 1,200-folder case from #166's review + let many = (0..<1_200).map { "/root-\($0)" } + ["/t/a"] + try store.setScope(many) + let expectedMany = IndexStore.mergedRanges(many.flatMap(IndexStore.bucketRanges(under:))) + var manyRows = 0 + try store.db.query("SELECT COUNT(*) FROM scopeb") { manyRows = Int($0.int(0)) } + #expect(manyRows == expectedMany.count, "every merged range is in the table (chunked inserts)") + try store.setScope(nil) // nil: the counters reads go covering, the table is left unused + } + + // MARK: Migration + + @Test func migrationFromV5FillsCountersAndStampsV6() throws { + var store: IndexStore? = try IndexStore(directory: folder) + #expect(try version(store!) == 6, "a fresh index is created and migrated in one open") + var inode: UInt64 = 1 + for i in 0..<120 { + let id = try store!.insert(file("/m/f\(i % 12)/p\(i).png", inode: inode)) + inode += 1 + if i % 5 == 0 { try store!.finish(id, state: "done") } + if i % 7 == 0 { try store!.finish(id, state: "partial", info: ["read_to": "5", "read_end": "12"]) } + if i % 3 == 0 { try store!.addSegment(file: id, kind: .image, modality: .image, vector: [Float](repeating: 0.1, count: 768)) } + if i % 4 == 0 { try store!.addSegment(file: id, kind: .ocr, modality: .text, text: "words \(i)") } + } + try expectAllMatch(store!, "before the downgrade") + try dropToV5(store!) + #expect(try version(store!) == 5) + store = nil // close: a fresh open runs the migration + store = try IndexStore(directory: folder) + #expect(try version(store!) == 6, "the stamp lands last, after the check passed") + try #require(try store!.countersAreConsistent(), "the fill agrees with files row for row") + try expectAllMatch(store!, "migrated (nil scope)") + try store!.setScope(["/m/f1"]) + try expectAllMatch(store!, "migrated (scoped)") + var drift = 0 + try store!.db.query("SELECT path, bucket FROM files") { + if $0.text(1) ?? "" != IndexStore.bucket(of: $0.text(0) ?? "") { drift += 1 } + } + #expect(drift == 0, "the backfill reproduces bucket(of:) on every row") + store = nil + } + + @Test func partialMigrationStatesReRunSafely() throws { + var store: IndexStore? = try IndexStore(directory: folder) + for i in 0..<40 { _ = try store!.insert(file("/p/f\(i % 5)/p\(i).png", inode: UInt64(i))) } + try expectAllMatch(store!, "seeded") + // shape 1: interrupted right after the ALTER — the column stands, no counters, version 5 + try dropToV5(store!, keepColumn: true) + store = nil + store = try IndexStore(directory: folder) + #expect(try version(store!) == 6) + try #require(try store!.countersAreConsistent()) + try expectAllMatch(store!, "resumed after the ALTER (no duplicate column name)") + // shape 2: interrupted after a fill — the counter tables stand, triggers gone, version 5 + try dropToV5(store!, keepColumn: true, keepCounters: true) + try store!.db.run("INSERT INTO counters(bucket, kind, state, n, leftsum) VALUES('/junk', 'image', 'pending', 9, 0)") + store = nil + store = try IndexStore(directory: folder) + #expect(try version(store!) == 6) + try #require(try store!.countersAreConsistent(), "the leftover fill is refilled, junk and all") + try expectAllMatch(store!, "resumed over a leftover fill") + // opening a finished index is a no-op + store = nil + store = try IndexStore(directory: folder) + #expect(try version(store!) == 6) + try #require(try store!.countersAreConsistent()) + store = nil + } + + @Test func migrationToleratesALegacyMalformedPartialRow() throws { + var store: IndexStore? = try IndexStore(directory: folder) + _ = try store!.insert(file("/mm/p.pdf", inode: 1, kind: .pdf)) + try dropToV5(store!) + // the v5 world writes it freely (no triggers): today's read-time hazard (Finding 8b) + try store!.db.run("UPDATE files SET state = 'partial', info = 'not json' WHERE path = '/mm/p.pdf'") + store = nil + store = try IndexStore(directory: folder) // the migration must not die on a legacy row + #expect(try version(store!) == 6, "json_valid-guarded fill: a v5 row it cannot read counts 0") + #expect(try store!.unfinishedCount() == (1, 0), "counted in files, 0 left") + try #require(try store!.countersAreConsistent()) + #expect(throws: (any Error).self) { _ = try store!.fullUnfinishedCount() } // the shipped form dies on it + // the new world refuses such rows at the write + #expect(throws: (any Error).self) { + try store!.db.run("INSERT INTO files(path, kind, size, mtime, inode, device, bucket, state, info) " + + "VALUES('/mm/bad.pdf', 'pdf', 10, 1, 99, 1, '/mm', 'partial', 'not json')") + } + try #require(try store!.countersAreConsistent()) + store = nil + } + + @Test func interruptedMigrationRollsBackToV5OnSIGKILL() throws { + // Finding 12's measured pattern, against the Swift migration itself: kill the child mid- + // transaction; the next open replays the WAL and finds v5 exactly intact; opening again + // completes. The child is the built CLI (`digup status --index DIR`) so the kill lands in + // real migration code, not a test stand-in. `swift test` runs from the package root, so the + // debug binary sits at .build/debug/digup (DIGUP_CLI overrides, e.g. a release build). + let candidates: [String] = [ProcessInfo.processInfo.environment["DIGUP_CLI"]].compactMap { $0 } + [ + FileManager.default.currentDirectoryPath + "/.build/debug/digup", + FileManager.default.currentDirectoryPath + "/.build/arm64-apple-macosx/debug/digup", + Bundle.main.bundleURL.deletingLastPathComponent().appendingPathComponent("digup").path, + ] + guard let cliPath = candidates.first(where: { FileManager.default.isExecutableFile(atPath: $0) }) else { + Issue.record("digup CLI not found (tried: \(candidates.joined(separator: ", ")))") + return + } + let cli = URL(fileURLWithPath: cliPath) + var killedMid = false + let delays = [0.70, 0.30, 0.12] + for attempt in 0.. String { + var lines: [String] = [] + try store.db.query("EXPLAIN QUERY PLAN \(sql)") { lines.append($0.text(3) ?? "") } + return lines.joined(separator: " | ") + } + let nilPlan = try plan("SELECT COALESCE(SUM(n), 0) FROM counters WHERE state = 'pending'") + #expect(nilPlan.contains("SEARCH counters") && nilPlan.contains("counters_state"), + "nil-scope pendingCount seeks the state section: \(nilPlan)") + try store.setScope(["/t/f0", "/t/f1", "/t/f2"]) + let scoped = try plan("SELECT COALESCE(SUM(c.n), 0) FROM scopeb b CROSS JOIN counters c " + + "ON c.state = 'pending' AND c.bucket >= b.lo AND c.bucket < b.hi") + #expect(scoped.contains("SCAN b") && scoped.contains("SEARCH c USING COVERING INDEX counters_state"), + "scope-driven seek (Finding 9): \(scoped)") + let partial = try plan("SELECT COALESCE(SUM(c.n), 0), COALESCE(SUM(c.leftsum), 0) FROM scopeb b " + + "CROSS JOIN counters c ON c.state = 'partial' AND c.bucket >= b.lo AND c.bucket < b.hi") + #expect(partial.contains("SCAN b") && partial.contains("SEARCH c USING COVERING INDEX counters_state"), + "partial seek: \(partial)") + let kinds = try plan("SELECT c.kind, c.state, SUM(c.n) FROM scopeb b CROSS JOIN counters c " + + "ON c.bucket >= b.lo AND c.bucket < b.hi GROUP BY c.kind, c.state") + #expect(kinds.contains("SCAN b") && kinds.contains("SEARCH c USING PRIMARY KEY"), + "kind read seeks the counters PK (a WITHOUT ROWID table's PK is its only index): \(kinds)") + let segs = try plan("SELECT c.hasvec, SUM(c.n) FROM scopeb b CROSS JOIN segcounters c " + + "ON c.bucket >= b.lo AND c.bucket < b.hi GROUP BY c.hasvec") + #expect(segs.contains("SCAN b") && segs.contains("SEARCH c USING PRIMARY KEY"), "segment read: \(segs)") + } + + @Test func reopenKeepsCountersExact() throws { + var store: IndexStore? = try IndexStore(directory: folder) + for i in 0..<40 { _ = try store!.insert(file("/o/f\(i % 4)/p\(i).png", inode: UInt64(i))) } + try store!.setScope(["/o/f1"]) + #expect(try store!.pendingCount() == 10) + store = nil + store = try IndexStore(directory: folder) // a fresh connection: the counters stand as committed + try #require(try store!.countersAreConsistent()) + try expectAllMatch(store!, "reopened (nil scope — the temp scope tables start empty)") + try store!.setScope(["/o/f1"]) + try expectAllMatch(store!, "reopened (scoped)") + store = nil + } + + @Test func twoConnectionsOnOneIndexStayConsistent() throws { + var a: IndexStore? = try IndexStore(directory: folder) + for i in 0..<50 { _ = try a!.insert(file("/n/f\(i % 5)/p\(i).png", inode: UInt64(i))) } + try a!.setScope(["/n"]) + let b = try IndexStore(directory: folder) // the same file: WAL, second connection + var fk: Int64 = 0 + try b.db.query("PRAGMA foreign_keys") { fk = $0.int(0) } + #expect(fk == 1, "every writable connection carries foreign_keys=ON (Finding 6)") + try #require(try b.countersAreConsistent()) + let id = try b.insert(file("/n/x.png", inode: 100)) + try b.finish(id, state: "done", info: ["read_to": "3", "read_end": "10"]) + try expectAllMatch(a!, "A sees B's commits") + try expectAllMatch(b, "B sees itself") + // mid-write reads are not blocked (Finding 12's reader result): B reads A's open transaction away + try a!.db.transaction { + _ = try a!.insert(file("/n/holding.png", inode: 101)) + try expectAllMatch(b, "reading through A's open transaction") + } + // writer contention: the second BEGIN IMMEDIATE hits the 5 s busy timeout, then goes through + #expect(throws: (any Error).self) { + try a!.db.transaction { + _ = try a!.insert(file("/n/lock.png", inode: 102)) + _ = try b.insert(file("/n/rival.png", inode: 103)) // fails: A holds the write lock + } + } + _ = try b.insert(file("/n/late.png", inode: 104)) // the lock is free + try expectAllMatch(a!, "after contention") + try expectAllMatch(b, "after contention") + a = nil + } + + @Test func randomLifecycleAgainstTheOracle() throws { + // A fixed-seed random walk through every write path, comparing counters to full + // aggregation after every single operation, under rotating scopes (nil, empty, nested, + // adjacent, sibling, root, duplicates, the 200+ chunk case). + struct Rng { + var state: UInt64 + init(seed: UInt64) { state = seed } + mutating func next(_ n: Int) -> Int { + state = state &+ 0x9E3779B97F4A7C15 + var z = state + z = (z ^ (z >> 30)) &* 0xBF58476D1CE4E5B9 + z = (z ^ (z >> 27)) &* 0x94D049BB133111EB + z = z ^ (z >> 31) + return Int(z % UInt64(n)) + } + } + let store = try IndexStore(directory: folder) + var rng = Rng(seed: 0x5EED_193) + let dirs = ["/r/a", "/r/a/sub", "/r/a/sub/deep", "/r/ab", "/r/a.", "/r/a !", "/r/a\tt", + "/r/keep x", "/r/nó", "/r/nó"] + let finishes: [(String, [String: String])] = [ + ("done", [:]), ("done", ["pages": "12"]), + ("partial", ["read_to": "10", "read_end": "40"]), ("partial", ["read_to": "40", "read_end": "12"]), + ("partial", ["read_end": "9"]), ("partial", ["read_to": "3"]), ("partial", [:]), + ("skipped", [:]), ("failed", ["why": "x"]), ("pending", [:]), + ] + let kinds = Array(FileKind.allCases) + let segKinds: [SegmentKind] = [.page, .chunk, .ocr, .pageImage, .image, .audio] + var seq: UInt64 = 1_000 + for step in 0..<500 { + let roll = rng.next(100) + var ids: [Int64] = [] + try store.db.query("SELECT id FROM files") { ids.append($0.int(0)) } + switch roll { + case 0..<22: // insert + if rng.next(12) == 0 { + _ = try store.insert(file("/x\(seq).png", inode: seq)) // root-level file + } else { + let kind = kinds[rng.next(kinds.count)] + _ = try store.insert(file("\(dirs[rng.next(dirs.count)])/\(seq).bin", inode: seq, kind: kind)) + } + seq += 1 + case 22..<48: // finish + if !ids.isEmpty { + let f = finishes[rng.next(finishes.count)] + try store.finish(ids[rng.next(ids.count)], state: f.0, detail: "step \(step)", info: f.1) + } + case 48..<55: // move + if !ids.isEmpty { + try store.move(ids[rng.next(ids.count)], + to: "\(dirs[rng.next(dirs.count)])/\(seq).png") + seq += 1 + } + case 55..<60: // requeue as a new candidate + if !ids.isEmpty { + let id = ids[rng.next(ids.count)] + var path = "" + try store.db.query("SELECT path FROM files WHERE id = ?", [.int(id)]) { path = $0.text(0) ?? "" } + try store.requeue(id, as: file(path, inode: seq, kind: kinds[rng.next(kinds.count)])) + seq += 1 + } + case 60..<63: // requeue plain + if !ids.isEmpty { try store.requeue(ids[rng.next(ids.count)]) } + case 63..<72: // addSegment + if !ids.isEmpty { + try store.addSegment(file: ids[rng.next(ids.count)], kind: segKinds[rng.next(segKinds.count)], + modality: Modality.allCases[rng.next(Modality.allCases.count)], + loc: 1, text: rng.next(2) == 0 ? "a \(rng.next(50)) word walk" : nil, + vector: rng.next(2) == 0 ? [Float](repeating: 0.25, count: 768) : nil) + } + case 72..<77: // app-order segment delete (production deleteSegments shape: fts first) + if !ids.isEmpty { + let victim = ids[rng.next(ids.count)] + try store.db.run("DELETE FROM fts WHERE rowid IN (SELECT id FROM segments WHERE file = ? AND kind != 'name')", [.int(victim)]) + try store.db.run("DELETE FROM segments WHERE file = ? AND kind != 'name'", [.int(victim)]) + } + case 77..<81: // file delete (cascade order) + if !ids.isEmpty { try store.delete(ids[rng.next(ids.count)]) } + case 81..<89: // setScope + let roll2 = rng.next(100) + if roll2 < 45 { try store.setScope(nil) } + else if roll2 < 55 { try store.setScope([]) } + else { + var picked = [dirs[rng.next(dirs.count)]] + for _ in 0.. [Int64: (file: Int64, kind: String)] { + var out: [Int64: (Int64, String)] = [:] + try store.db.query("SELECT id, file, kind FROM segments ORDER BY id") { + out[$0.int(0)] = ($0.int(1), $0.text(2) ?? "?") + } + return out + } + // The invariant is about UPDATEs, not inserts: a segment row that SURVIVES a call (present + // in two consecutive snapshots under the same id) must carry the same (file, kind). Rowids + // legitimately recycle after deletions — a reused id in a later snapshot is a NEW row, not + // a reparented one — so each step compares against the immediately preceding snapshot. + var prev = try snapshot() + func check(_ tag: String) throws { + let now = try snapshot() + for (id, row) in now { + if let was = prev[id] { + #expect(was.file == row.file && was.kind == row.kind, + "segment \(id) keeps its (file, kind) after \(tag)") + } + } + prev = now + try #require(try store.countersAreConsistent(), "consistency after \(tag)") + try expectAllMatch(store, "invariant \(tag)") + } + try check("seed") + try store.finish(ids[1], state: "partial", info: ["read_to": "2", "read_end": "9"]) + try check("finish partial") + try store.move(ids[2], to: "/v/moved/f2.png") + try check("move") + try store.addSegment(file: ids[3], kind: .chunk, modality: .text, text: "more words") + try check("addSegment text") + try store.addSegment(file: ids[3], kind: .image, modality: .image, vector: vec) + try check("addSegment vector") + try store.requeue(ids[4], as: file("/v/f4.png", inode: 104)) + try check("requeue as candidate") + try store.requeue(ids[3]) + try check("requeue plain") + try store.markStopped("/v/f5.png", why: "ejected") + try check("markStopped") + try store.resetAll() + try check("resetAll") + try store.delete(ids[0]) + try check("delete") + } + + /// Finding 3's boundary, pinned as behavior: the WHEN guards give every file UPDATE in the + /// codebase exactly one counter maintenance because no production statement sets `state` and + /// `bucket` together without `path` (bucket is derived from path; `move` sets path+bucket, + /// `finish`/`requeue`/`markStopped`/`resetAll` never touch bucket). This test writes exactly + /// the forbidden shape — state + bucket, path untouched — shows both triggers fire and the + /// counters drift, and shows `rebuildCounters` heals it: the documented self-heal is the + /// answer to any future statement that breaks a suite invariant. + @Test func guardsFireOnceAndTheForbiddenShapeIsHealedByRebuild() throws { + let store = try IndexStore(directory: folder) + let id = try store.insert(file("/g/a/x.png", inode: 1)) + try store.finish(id, state: "done") + try #require(try store.countersAreConsistent(), "clean start") + try store.db.run("UPDATE files SET state = 'pending', bucket = '/g/b' WHERE id = ?", [.int(id)]) + let consistentAfterForbidden = try store.countersAreConsistent() + #expect(!consistentAfterForbidden, + "the forbidden shape double-fires and drifts the counters — the invariant's boundary") + try store.rebuildCounters() + try #require(try store.countersAreConsistent(), "rebuildCounters heals the drift") + try expectAllMatch(store, "after heal") + } +}