diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..e460881 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,25 @@ +name: CI + +on: + push: + branches: [master] + pull_request: + +permissions: + contents: read + +jobs: + test: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - name: Install Zig 0.16.0 + uses: mlugg/setup-zig@d1434d08867e3ee9daa34448df10607b98908d29 # v2 + with: + version: 0.16.0 + - name: Run tests + run: zig build test diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..db3cb54 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +/.zig-cache/ +/zig-out/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..cc169fb --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,20 @@ +# Contributing + +Chi is pre-1.0 and its public contracts are intentionally narrow. Before a +large change, open an issue describing the corpus shape, memory constraint, +failure mode, and checkpoint behavior it is meant to address. + +Pull requests should: + +- remain compatible with Zig 0.16.0; +- keep steady-state memory bounds explicit; +- document the lifetime of every borrowed slice; +- preserve deterministic behavior across input chunk boundaries; +- add tests for checkpoints and failure rollback when state changes; and +- run `zig build test` before submission. + +Do not include customer data, model corpora, credentials, generated secrets, +or code whose license is incompatible with Apache-2.0. + +By submitting a contribution, you agree that it may be distributed under this +repository's Apache-2.0 license. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..ce45249 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2026 Cameron Emery + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/NOTICE b/NOTICE new file mode 100644 index 0000000..9158d0b --- /dev/null +++ b/NOTICE @@ -0,0 +1,9 @@ +Chi +Copyright 2026 Cameron Emery + +This product was developed with material AI assistance under human direction +and review. Human review, release decisions, and responsibility remain with +the maintainer. + +This product includes software developed by Cameron Emery and is licensed +under the Apache License, Version 2.0. diff --git a/README.md b/README.md index e69de29..598e77d 100644 --- a/README.md +++ b/README.md @@ -0,0 +1,83 @@ +# Chi + +Streaming, checkpointable building blocks for large text-corpus curation in +Zig. + +Chi is a pre-1.0 library for data infrastructure that cannot keep an entire +corpus—or even an entire document—in memory. Its four modules cover bounded +ingestion, text cleanup, document filtering, and exact/approximate +deduplication. Each module exposes explicit ownership boundaries and a +versioned checkpoint format for stopping or moving work between processes. + +The current release targets **Zig 0.16.0** and has no third-party runtime +dependencies. + +## Modules + +| Module | Current scope | +|---|---| +| `corpus_reader` | Deterministic file and repository traversal plus pull-based stream adapters | +| `text_cleaner` | Streaming UTF-8/UTF-16/Latin-1/Windows-1252 decoding and conservative whitespace/control cleanup | +| `document_filter` | Bounded-memory length, UTF-8, visibility, replacement, cleaning-change, and repeated-run gates | +| `deduplicator` | Chunk-stable exact fingerprints, SimHash candidates, sampled duplicate spans, and a transactional index boundary | + +```zig +const chi = @import("chi"); + +const CorpusReader = chi.data.corpus_reader.CorpusReader; +const TextCleaner = chi.data.text_cleaner.TextCleaner; +const DocumentFilter = chi.data.document_filter.DocumentFilter; +const Deduplicator = chi.data.deduplicator.Deduplicator; +``` + +Each module has a focused README with API behavior, memory ownership, and +checkpoint guarantees: + +- [`data/corpus-reader`](data/corpus-reader/README.md) +- [`data/text-cleaner`](data/text-cleaner/README.md) +- [`data/document-filter`](data/document-filter/README.md) +- [`data/deduplicator`](data/deduplicator/README.md) + +## Verify + +Install Zig 0.16.0, clone the repository, and run: + +```sh +zig build test +``` + +The suite exercises arbitrary chunk boundaries, checkpoint transfer, +deterministic repository traversal, duplicate classification, and rollback +behavior. Windows and Linux run the same test command in CI. + +## What Chi does not claim + +Chi is a set of library components, not a turnkey distributed data platform. +This release does not include: + +- NFC or NFKC normalization; +- built-in language or safety classifiers; +- a database driver; +- a distributed deduplication backend; +- Unicode word segmentation or token-shingle MinHash; or +- published performance comparisons. + +The provided approximate match signal uses byte-level ASCII word features. +Downstream systems must evaluate whether that is appropriate for their data. + +## Status + +Chi is early, intentionally narrow software. APIs and checkpoint formats may +change before 1.0. The current release is useful as an inspectable foundation +for streaming corpus tooling and as research infrastructure for YY's data +systems; it is not offered with an SLA. + +See [SECURITY.md](SECURITY.md) for private vulnerability reporting and +[CONTRIBUTING.md](CONTRIBUTING.md) before proposing changes. + +## License and authorship + +Copyright 2026 Cameron Emery. Licensed under Apache-2.0. + +Chi was developed under Cameron Emery's direction with material AI assistance. +Human review, release decisions, and responsibility remain with the maintainer. diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..92226da --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,16 @@ +# Security + +Please do not publish suspected vulnerabilities, credential exposures, or +unsafe parsing cases in a public issue before the maintainer has had a chance +to assess them. + +Report them to **security@yycore.ai** with: + +- the affected commit and module; +- a minimal reproducer that contains no sensitive corpus data; +- the observed impact; and +- any suggested mitigation. + +Chi is pre-1.0 research infrastructure and does not have a guaranteed response +time. Receipt will be acknowledged when possible, and confirmed issues will be +coordinated privately before disclosure. diff --git a/build.zig b/build.zig new file mode 100644 index 0000000..f303b1e --- /dev/null +++ b/build.zig @@ -0,0 +1,18 @@ +const std = @import("std"); + +pub fn build(b: *std.Build) void { + const target = b.standardTargetOptions(.{}); + const optimize = b.standardOptimizeOption(.{}); + + const chi = b.addModule("chi", .{ + .root_source_file = b.path("root.zig"), + .target = target, + .optimize = optimize, + }); + + const unit_tests = b.addTest(.{ .root_module = chi }); + const run_unit_tests = b.addRunArtifact(unit_tests); + + const test_step = b.step("test", "Run unit tests"); + test_step.dependOn(&run_unit_tests.step); +} diff --git a/build.zig.zon b/build.zig.zon new file mode 100644 index 0000000..c496472 --- /dev/null +++ b/build.zig.zon @@ -0,0 +1,17 @@ +.{ + .name = .chi, + .version = "0.1.0", + .minimum_zig_version = "0.16.0", + .paths = .{ + "build.zig", + "build.zig.zon", + "root.zig", + "data", + "LICENSE", + "NOTICE", + "README.md", + "SECURITY.md", + "CONTRIBUTING.md", + }, + .fingerprint = 0x2c9a9438dc98f8f3, +} diff --git a/data/corpus-reader/README.md b/data/corpus-reader/README.md new file mode 100644 index 0000000..f0447e5 --- /dev/null +++ b/data/corpus-reader/README.md @@ -0,0 +1,26 @@ +# Corpus reader + +`CorpusReader` turns raw sources into bounded, borrowed chunks for the future +preprocessing pipeline. It currently supports: + +- regular files through positional reads; +- local repository trees with deterministic, bytewise-sorted traversal; +- streams through a zero-copy pull adapter; +- databases through the same adapter boundary, ready for concrete drivers. + +The reader owns three arenas with source, document, and chunk lifetimes. A chunk +is invalidated by the next reader operation. This keeps steady-state ingestion +memory near `chunk_size` while retaining allocation capacity after warm-up. +Repository traversal holds only the sorted directory entries on the active +depth-first stack, not an index of every file in the checkout. + +Every chunk carries `state_after`. Encode that value before the next operation +to transfer or persist a restore point. Checkpoints assume the underlying corpus +is immutable; changing a file or repository while it is being read is an input +consistency error, not something the reader attempts to hide. + +Run the component tests from the repository root: + +```sh +zig build test +``` diff --git a/data/corpus-reader/corpus_reader.zig b/data/corpus-reader/corpus_reader.zig new file mode 100644 index 0000000..1f7ab00 --- /dev/null +++ b/data/corpus-reader/corpus_reader.zig @@ -0,0 +1,1219 @@ +const std = @import("std"); + +const Allocator = std.mem.Allocator; +const ArenaAllocator = std.heap.ArenaAllocator; +const Dir = std.Io.Dir; +const File = std.Io.File; +const Io = std.Io; + +pub const SourceKind = enum(u8) { + file = 1, + repository = 2, + database = 3, + stream = 4, +}; + +/// Controls the hot path. A chunk is the largest allocation made per `next` call. +pub const Options = struct { + chunk_size: usize = 256 * 1024, + + /// Retaining the chunk arena avoids allocator traffic after warm-up. Set a + /// limit when readers may occasionally receive unusually large chunks. + retained_chunk_bytes: ?usize = null, +}; + +pub const RepositoryOptions = struct { + reader: Options = .{}, + + /// Git, Mercurial, and Subversion internals are metadata rather than source + /// corpus in the common case. Dotfiles outside these directories are read. + include_vcs_metadata: bool = false, +}; + +/// State for the first unread byte after a chunk. It is deliberately independent +/// of pointers owned by `CorpusReader`, so it can be encoded and sent to a +/// different machine in a memory/compute-disaggregated pipeline. +/// +/// Slices returned by `decode` borrow the encoded input. Slices obtained from a +/// `Chunk` remain valid only until the next reader operation. +pub const StreamingState = struct { + source_kind: SourceKind, + sequence: u64, + document_index: u64, + byte_offset: u64, + document_done: bool, + source_done: bool, + source_id: []const u8, + document_id: []const u8, + resume_token: []const u8 = "", + + const magic = "CRS1"; + const header_len = 44; + + pub const EncodeError = Allocator.Error || error{StateTooLarge}; + pub const DecodeError = error{ + InvalidState, + UnsupportedVersion, + }; + + pub fn encode(self: StreamingState, allocator: Allocator) EncodeError![]u8 { + if (self.source_id.len > std.math.maxInt(u32) or + self.document_id.len > std.math.maxInt(u32) or + self.resume_token.len > std.math.maxInt(u32)) + { + return error.StateTooLarge; + } + + var total = std.math.add(usize, header_len, self.source_id.len) catch + return error.StateTooLarge; + total = std.math.add(usize, total, self.document_id.len) catch + return error.StateTooLarge; + total = std.math.add(usize, total, self.resume_token.len) catch + return error.StateTooLarge; + + const encoded = try allocator.alloc(u8, total); + @memcpy(encoded[0..magic.len], magic); + + var flags: u8 = 0; + if (self.document_done) flags |= 1 << 0; + if (self.source_done) flags |= 1 << 1; + encoded[4] = flags; + encoded[5] = @intFromEnum(self.source_kind); + encoded[6] = 0; + encoded[7] = 0; + + putU64(encoded[8..16], self.sequence); + putU64(encoded[16..24], self.document_index); + putU64(encoded[24..32], self.byte_offset); + putU32(encoded[32..36], @intCast(self.source_id.len)); + putU32(encoded[36..40], @intCast(self.document_id.len)); + putU32(encoded[40..44], @intCast(self.resume_token.len)); + + var cursor: usize = header_len; + @memcpy(encoded[cursor..][0..self.source_id.len], self.source_id); + cursor += self.source_id.len; + @memcpy(encoded[cursor..][0..self.document_id.len], self.document_id); + cursor += self.document_id.len; + @memcpy(encoded[cursor..][0..self.resume_token.len], self.resume_token); + return encoded; + } + + pub fn decode(encoded: []const u8) DecodeError!StreamingState { + if (encoded.len < header_len) return error.InvalidState; + if (!std.mem.eql(u8, encoded[0..4], magic)) { + return error.UnsupportedVersion; + } + if (encoded[6] != 0 or encoded[7] != 0) return error.InvalidState; + if (encoded[4] & ~@as(u8, 0b11) != 0) return error.InvalidState; + + const source_kind: SourceKind = switch (encoded[5]) { + @intFromEnum(SourceKind.file) => .file, + @intFromEnum(SourceKind.repository) => .repository, + @intFromEnum(SourceKind.database) => .database, + @intFromEnum(SourceKind.stream) => .stream, + else => return error.InvalidState, + }; + const source_len: usize = getU32(encoded[32..36]); + const document_len: usize = getU32(encoded[36..40]); + const token_len: usize = getU32(encoded[40..44]); + + var expected_len = std.math.add(usize, header_len, source_len) catch + return error.InvalidState; + expected_len = std.math.add(usize, expected_len, document_len) catch + return error.InvalidState; + expected_len = std.math.add(usize, expected_len, token_len) catch + return error.InvalidState; + if (expected_len != encoded.len) return error.InvalidState; + + var cursor: usize = header_len; + const source_id = encoded[cursor..][0..source_len]; + cursor += source_len; + const document_id = encoded[cursor..][0..document_len]; + cursor += document_len; + const resume_token = encoded[cursor..][0..token_len]; + + return .{ + .source_kind = source_kind, + .sequence = getU64(encoded[8..16]), + .document_index = getU64(encoded[16..24]), + .byte_offset = getU64(encoded[24..32]), + .document_done = encoded[4] & (1 << 0) != 0, + .source_done = encoded[4] & (1 << 1) != 0, + .source_id = source_id, + .document_id = document_id, + .resume_token = resume_token, + }; + } +}; + +/// A borrowed view into the reader's resettable arenas. Consume or copy it +/// before calling `next`, `restore`, or `destroy` again. +pub const Chunk = struct { + bytes: []const u8, + source_kind: SourceKind, + source_id: []const u8, + document_id: []const u8, + document_index: u64, + byte_offset: u64, + sequence: u64, + is_document_start: bool, + is_document_end: bool, + state_after: StreamingState, +}; + +/// Pull-based boundary for network streams and database drivers. The adapter +/// writes directly into the reader's chunk buffer, avoiding an intermediate +/// copy. `document_id` and `resume_token` only need to live until `pull` returns. +/// +/// The reader owns the adapter lifecycle and calls `deinit` when present. +pub const ExternalSource = struct { + context: *anyopaque, + source_id: []const u8, + vtable: *const VTable, + + pub const PullResult = struct { + bytes_written: usize, + document_id: []const u8, + is_document_start: bool, + is_document_end: bool, + is_source_end: bool, + resume_token: []const u8 = "", + }; + + pub const VTable = struct { + pull: *const fn (context: *anyopaque, destination: []u8) anyerror!PullResult, + restore: ?*const fn (context: *anyopaque, token: []const u8) anyerror!void = null, + deinit: ?*const fn (context: *anyopaque) void = null, + }; +}; + +/// Deterministic depth-first traversal. Only the entries for directories on +/// the active stack are resident, so memory scales with traversal breadth and +/// depth rather than with the total repository size. +const RepositoryWalker = struct { + allocator: Allocator, + frames: std.ArrayList(Frame) = .empty, + include_vcs_metadata: bool, + + const InitError = Allocator.Error || Dir.Iterator.Error || Dir.StatFileError; + const NextError = InitError || Dir.OpenError; + + const Entry = struct { + name: []const u8, + kind: File.Kind, + }; + + const Frame = struct { + dir: Dir, + arena: ArenaAllocator, + canonical_prefix: []const u8, + entries: []Entry, + next_index: usize = 0, + owns_dir: bool, + + fn init( + allocator: Allocator, + io: Io, + dir: Dir, + parent_prefix: []const u8, + directory_name: ?[]const u8, + owns_dir: bool, + ) InitError!Frame { + errdefer if (owns_dir) dir.close(io); + var arena: ArenaAllocator = .init(allocator); + errdefer arena.deinit(); + const arena_allocator = arena.allocator(); + + const canonical_prefix = if (directory_name) |name| + try joinCanonical(arena_allocator, parent_prefix, name) + else + try arena_allocator.dupe(u8, parent_prefix); + var entries: std.ArrayList(Entry) = .empty; + + var iterator = dir.iterate(); + while (try iterator.next(io)) |entry| { + const kind = if (entry.kind == .unknown) + (try dir.statFile(io, entry.name, .{ .follow_symlinks = false })).kind + else + entry.kind; + if (kind != .file and kind != .directory) continue; + + try entries.append(arena_allocator, .{ + .name = try arena_allocator.dupe(u8, entry.name), + .kind = kind, + }); + } + + std.mem.sortUnstable(Entry, entries.items, {}, lessThanEntry); + return .{ + .dir = dir, + .arena = arena, + .canonical_prefix = canonical_prefix, + .entries = entries.items, + .owns_dir = owns_dir, + }; + } + + fn deinit(self: *Frame, io: Io) void { + self.arena.deinit(); + if (self.owns_dir) self.dir.close(io); + self.* = undefined; + } + }; + + const FileEntry = struct { + dir: Dir, + basename: []const u8, + canonical_prefix: []const u8, + }; + + fn init( + allocator: Allocator, + io: Io, + root: Dir, + include_vcs_metadata: bool, + ) InitError!RepositoryWalker { + var self: RepositoryWalker = .{ + .allocator = allocator, + .include_vcs_metadata = include_vcs_metadata, + }; + errdefer self.deinit(io); + + var root_frame = try Frame.init(allocator, io, root, "", null, false); + errdefer root_frame.deinit(io); + try self.frames.append(allocator, root_frame); + return self; + } + + fn deinit(self: *RepositoryWalker, io: Io) void { + while (self.frames.pop()) |frame_value| { + var frame = frame_value; + frame.deinit(io); + } + self.frames.deinit(self.allocator); + self.* = undefined; + } + + fn nextFile(self: *RepositoryWalker, io: Io) NextError!?FileEntry { + while (self.frames.items.len != 0) { + const frame = &self.frames.items[self.frames.items.len - 1]; + if (frame.next_index == frame.entries.len) { + var completed = self.frames.pop().?; + completed.deinit(io); + continue; + } + + const entry = &frame.entries[frame.next_index]; + frame.next_index += 1; + switch (entry.kind) { + .file => return .{ + .dir = frame.dir, + .basename = entry.name, + .canonical_prefix = frame.canonical_prefix, + }, + .directory => { + if (!self.include_vcs_metadata and isVcsDirectory(entry.name)) { + continue; + } + + const child = try frame.dir.openDir(io, entry.name, .{ .iterate = true }); + var child_frame = try Frame.init( + self.allocator, + io, + child, + frame.canonical_prefix, + entry.name, + true, + ); + self.frames.append(self.allocator, child_frame) catch |err| { + child_frame.deinit(io); + return err; + }; + }, + else => unreachable, + } + } + return null; + } + + fn lessThanEntry(_: void, left: Entry, right: Entry) bool { + return std.mem.order(u8, left.name, right.name) == .lt; + } +}; + +/// Bounded-memory raw corpus reader. +/// +/// Three arenas make the ownership periods explicit: +/// - source arena: source configuration and traversal state; +/// - document arena: the current repository/stream document identity; +/// - chunk arena: bytes and checkpoint token, reset on every operation. +/// +/// Creation returns a pointer because persistent adapters store allocators that +/// refer to these embedded arenas. Do not move the value; call `destroy` once. +pub const CorpusReader = struct { + backing_allocator: Allocator, + io: Io, + options: Options, + source_arena: ArenaAllocator, + document_arena: ArenaAllocator, + chunk_arena: ArenaAllocator, + sequence: u64, + source: Source, + + const Source = union(SourceKind) { + file: FileState, + repository: RepositoryState, + database: ExternalState, + stream: ExternalState, + }; + + const FileState = struct { + source_id: []const u8, + file: File, + size: u64, + offset: u64 = 0, + emitted_empty: bool = false, + done: bool = false, + }; + + const CurrentFile = struct { + file: File, + path: []const u8, + size: u64, + offset: u64 = 0, + emitted_empty: bool = false, + done: bool = false, + }; + + const RepositoryState = struct { + source_id: []const u8, + root: Dir, + walker: RepositoryWalker, + current: ?CurrentFile = null, + current_document_index: u64 = 0, + next_document_index: u64 = 0, + done: bool = false, + }; + + const ExternalState = struct { + adapter: ExternalSource, + source_id: []const u8, + current_document_id: []const u8 = "", + current_document_index: u64 = 0, + next_document_index: u64 = 0, + offset: u64 = 0, + in_document: bool = false, + done: bool = false, + }; + + pub const CreateError = Allocator.Error || error{InvalidChunkSize} || + File.OpenError || File.StatError || Dir.OpenError || Dir.Iterator.Error; + + pub fn openFile( + backing_allocator: Allocator, + io: Io, + base_dir: Dir, + path: []const u8, + options: Options, + ) CreateError!*CorpusReader { + if (options.chunk_size == 0) return error.InvalidChunkSize; + + const self = try allocate(backing_allocator, io, options); + errdefer freeUninitialized(self); + + const source_id = try self.source_arena.allocator().dupe(u8, path); + const file = try base_dir.openFile(io, path, .{}); + errdefer file.close(io); + + const stat = try file.stat(io); + if (stat.kind != .file) return error.Streaming; + self.source = .{ .file = .{ + .source_id = source_id, + .file = file, + .size = stat.size, + } }; + return self; + } + + pub fn openRepository( + backing_allocator: Allocator, + io: Io, + base_dir: Dir, + path: []const u8, + options: RepositoryOptions, + ) CreateError!*CorpusReader { + if (options.reader.chunk_size == 0) return error.InvalidChunkSize; + + const self = try allocate(backing_allocator, io, options.reader); + errdefer freeUninitialized(self); + + const source_id = try self.source_arena.allocator().dupe(u8, path); + const root = try base_dir.openDir(io, path, .{ .iterate = true }); + errdefer root.close(io); + const walker = try RepositoryWalker.init( + backing_allocator, + io, + root, + options.include_vcs_metadata, + ); + + self.source = .{ .repository = .{ + .source_id = source_id, + .root = root, + .walker = walker, + } }; + return self; + } + + /// Creates a database-backed reader through a driver adapter. + /// + /// TODO(data/database): add concrete SQL and object-store adapters once the + /// database component defines connection, query, and row ownership APIs. + pub fn openDatabase( + backing_allocator: Allocator, + io: Io, + adapter: ExternalSource, + options: Options, + ) CreateError!*CorpusReader { + return openExternal(backing_allocator, io, adapter, options, .database); + } + + pub fn openStream( + backing_allocator: Allocator, + io: Io, + adapter: ExternalSource, + options: Options, + ) CreateError!*CorpusReader { + return openExternal(backing_allocator, io, adapter, options, .stream); + } + + pub fn destroy(self: *CorpusReader) void { + switch (self.source) { + .file => |state| state.file.close(self.io), + .repository => |*state| { + if (state.current) |current| current.file.close(self.io); + state.walker.deinit(self.io); + state.root.close(self.io); + }, + .database, .stream => |state| { + if (state.adapter.vtable.deinit) |deinitFn| { + deinitFn(state.adapter.context); + } + }, + } + + self.chunk_arena.deinit(); + self.document_arena.deinit(); + self.source_arena.deinit(); + const backing_allocator = self.backing_allocator; + backing_allocator.destroy(self); + } + + /// Returns null only after the complete source has been consumed. + pub fn next(self: *CorpusReader) anyerror!?Chunk { + self.resetChunkArena(); + return switch (self.source) { + .file => |*state| self.nextFile(state), + .repository => |*state| self.nextRepository(state), + .database => |*state| self.nextExternal(.database, state), + .stream => |*state| self.nextExternal(.stream, state), + }; + } + + /// Moves the reader to a previously emitted `state_after` checkpoint. + /// The source must still describe the same immutable data. + pub fn restore(self: *CorpusReader, state: StreamingState) anyerror!void { + // A state taken directly from a Chunk may borrow both the document and + // chunk arenas that restoration resets. Stabilize it outside the arenas; + // restore is a control-plane operation, not part of the ingestion hot path. + const source_id = try self.backing_allocator.dupe(u8, state.source_id); + defer self.backing_allocator.free(source_id); + const document_id = try self.backing_allocator.dupe(u8, state.document_id); + defer self.backing_allocator.free(document_id); + const resume_token = try self.backing_allocator.dupe(u8, state.resume_token); + defer self.backing_allocator.free(resume_token); + var stable_state = state; + stable_state.source_id = source_id; + stable_state.document_id = document_id; + stable_state.resume_token = resume_token; + + if (stable_state.source_kind != std.meta.activeTag(self.source)) { + return error.CheckpointSourceMismatch; + } + + switch (self.source) { + .file => |*file_state| try self.restoreFile(file_state, stable_state), + .repository => |*repository_state| { + try self.restoreRepository(repository_state, stable_state); + }, + .database => |*external_state| { + try self.restoreExternal(external_state, stable_state); + }, + .stream => |*external_state| { + try self.restoreExternal(external_state, stable_state); + }, + } + self.sequence = stable_state.sequence; + self.resetChunkArena(); + } + + fn allocate( + backing_allocator: Allocator, + io: Io, + options: Options, + ) Allocator.Error!*CorpusReader { + const self = try backing_allocator.create(CorpusReader); + self.* = .{ + .backing_allocator = backing_allocator, + .io = io, + .options = options, + .source_arena = .init(backing_allocator), + .document_arena = .init(backing_allocator), + .chunk_arena = .init(backing_allocator), + .sequence = 0, + .source = undefined, + }; + return self; + } + + fn freeUninitialized(self: *CorpusReader) void { + self.chunk_arena.deinit(); + self.document_arena.deinit(); + self.source_arena.deinit(); + const backing_allocator = self.backing_allocator; + backing_allocator.destroy(self); + } + + fn openExternal( + backing_allocator: Allocator, + io: Io, + adapter: ExternalSource, + options: Options, + kind: SourceKind, + ) CreateError!*CorpusReader { + if (options.chunk_size == 0) return error.InvalidChunkSize; + + const self = try allocate(backing_allocator, io, options); + errdefer freeUninitialized(self); + const source_id = try self.source_arena.allocator().dupe(u8, adapter.source_id); + const state: ExternalState = .{ + .adapter = adapter, + .source_id = source_id, + }; + switch (kind) { + .database => self.source = .{ .database = state }, + .stream => self.source = .{ .stream = state }, + else => unreachable, + } + return self; + } + + fn nextFile(self: *CorpusReader, state: *FileState) anyerror!?Chunk { + if (state.done) return null; + + const start = state.offset; + const is_empty = state.size == 0; + if (is_empty and state.emitted_empty) { + state.done = true; + return null; + } + if (!is_empty and start >= state.size) { + state.done = true; + return null; + } + + const wanted: usize = if (is_empty) + 0 + else + @intCast(@min(@as(u64, self.options.chunk_size), state.size - start)); + const buffer = try self.chunk_arena.allocator().alloc(u8, wanted); + const bytes_read = try state.file.readPositionalAll(self.io, buffer, start); + if (bytes_read != wanted) return error.SourceTruncated; + + state.offset += bytes_read; + state.emitted_empty = is_empty; + const is_end = state.offset == state.size; + if (is_end) state.done = true; + + return self.makeFileChunk( + .file, + state.source_id, + state.source_id, + 0, + start, + buffer[0..bytes_read], + start == 0, + is_end, + is_end, + ); + } + + fn nextRepository( + self: *CorpusReader, + state: *RepositoryState, + ) anyerror!?Chunk { + if (state.done) return null; + + while (true) { + if (state.current) |*current| { + if (current.done) { + current.file.close(self.io); + state.current = null; + _ = self.document_arena.reset(.retain_capacity); + continue; + } + return try self.readRepositoryFile(state, current); + } + + if (!try self.advanceRepository(state)) { + state.done = true; + return null; + } + } + } + + fn advanceRepository( + self: *CorpusReader, + state: *RepositoryState, + ) anyerror!bool { + while (try state.walker.nextFile(self.io)) |entry| { + _ = self.document_arena.reset(.retain_capacity); + const path = try joinCanonical( + self.document_arena.allocator(), + entry.canonical_prefix, + entry.basename, + ); + const file = try entry.dir.openFile(self.io, entry.basename, .{}); + errdefer file.close(self.io); + const stat = try file.stat(self.io); + if (stat.kind != .file) { + file.close(self.io); + continue; + } + + state.current = .{ + .file = file, + .path = path, + .size = stat.size, + }; + state.current_document_index = state.next_document_index; + state.next_document_index += 1; + return true; + } + return false; + } + + fn readRepositoryFile( + self: *CorpusReader, + state: *RepositoryState, + current: *CurrentFile, + ) anyerror!Chunk { + const start = current.offset; + const is_empty = current.size == 0; + if (is_empty and current.emitted_empty) unreachable; + if (!is_empty and start >= current.size) unreachable; + + const wanted: usize = if (is_empty) + 0 + else + @intCast(@min(@as(u64, self.options.chunk_size), current.size - start)); + const buffer = try self.chunk_arena.allocator().alloc(u8, wanted); + const bytes_read = try current.file.readPositionalAll(self.io, buffer, start); + if (bytes_read != wanted) return error.SourceTruncated; + + current.offset += bytes_read; + current.emitted_empty = is_empty; + const is_end = current.offset == current.size; + if (is_end) current.done = true; + + return self.makeFileChunk( + .repository, + state.source_id, + current.path, + state.current_document_index, + start, + buffer[0..bytes_read], + start == 0, + is_end, + false, + ); + } + + fn makeFileChunk( + self: *CorpusReader, + kind: SourceKind, + source_id: []const u8, + document_id: []const u8, + document_index: u64, + byte_offset: u64, + bytes: []const u8, + is_start: bool, + is_end: bool, + source_done: bool, + ) Chunk { + const sequence = self.sequence; + self.sequence += 1; + return .{ + .bytes = bytes, + .source_kind = kind, + .source_id = source_id, + .document_id = document_id, + .document_index = document_index, + .byte_offset = byte_offset, + .sequence = sequence, + .is_document_start = is_start, + .is_document_end = is_end, + .state_after = .{ + .source_kind = kind, + .sequence = self.sequence, + .document_index = document_index, + .byte_offset = byte_offset + bytes.len, + .document_done = is_end, + .source_done = source_done, + .source_id = source_id, + .document_id = document_id, + }, + }; + } + + fn nextExternal( + self: *CorpusReader, + kind: SourceKind, + state: *ExternalState, + ) anyerror!?Chunk { + if (state.done) return null; + + const buffer = try self.chunk_arena.allocator().alloc(u8, self.options.chunk_size); + const result = try state.adapter.vtable.pull(state.adapter.context, buffer); + if (result.bytes_written > buffer.len) return error.AdapterOverflow; + if (result.bytes_written == 0 and + !result.is_document_end and + !result.is_source_end) + { + return error.AdapterMadeNoProgress; + } + + if (result.is_document_start) { + if (state.in_document) return error.AdapterProtocolViolation; + _ = self.document_arena.reset(.retain_capacity); + state.current_document_id = try self.document_arena.allocator().dupe( + u8, + result.document_id, + ); + state.current_document_index = state.next_document_index; + state.next_document_index += 1; + state.offset = 0; + state.in_document = true; + } else if (!state.in_document) { + if (result.is_source_end and result.bytes_written == 0) { + state.done = true; + return null; + } + return error.AdapterProtocolViolation; + } else if (!std.mem.eql(u8, state.current_document_id, result.document_id)) { + return error.AdapterProtocolViolation; + } + + if (result.is_source_end and !result.is_document_end) { + return error.AdapterProtocolViolation; + } + + const token = try self.chunk_arena.allocator().dupe(u8, result.resume_token); + const start = state.offset; + state.offset += result.bytes_written; + const document_index = state.current_document_index; + const document_id = state.current_document_id; + if (result.is_document_end) state.in_document = false; + if (result.is_source_end) state.done = true; + + const sequence = self.sequence; + self.sequence += 1; + return .{ + .bytes = buffer[0..result.bytes_written], + .source_kind = kind, + .source_id = state.source_id, + .document_id = document_id, + .document_index = document_index, + .byte_offset = start, + .sequence = sequence, + .is_document_start = result.is_document_start, + .is_document_end = result.is_document_end, + .state_after = .{ + .source_kind = kind, + .sequence = self.sequence, + .document_index = document_index, + .byte_offset = state.offset, + .document_done = result.is_document_end, + .source_done = result.is_source_end, + .source_id = state.source_id, + .document_id = document_id, + .resume_token = token, + }, + }; + } + + fn restoreFile( + self: *CorpusReader, + file_state: *FileState, + state: StreamingState, + ) !void { + _ = self; + if (!std.mem.eql(u8, file_state.source_id, state.source_id) or + !std.mem.eql(u8, file_state.source_id, state.document_id) or + state.document_index != 0 or + state.byte_offset > file_state.size) + { + return error.CheckpointSourceMismatch; + } + file_state.offset = state.byte_offset; + file_state.emitted_empty = file_state.size == 0 and state.document_done; + file_state.done = state.source_done; + } + + fn restoreRepository( + self: *CorpusReader, + repository_state: *RepositoryState, + state: StreamingState, + ) anyerror!void { + if (!std.mem.eql(u8, repository_state.source_id, state.source_id)) { + return error.CheckpointSourceMismatch; + } + + try self.resetRepository(repository_state); + while (try self.advanceRepository(repository_state)) { + const current = &repository_state.current.?; + if (std.mem.eql(u8, current.path, state.document_id)) { + if (current.size < state.byte_offset) return error.CheckpointSourceMismatch; + current.offset = state.byte_offset; + current.emitted_empty = current.size == 0 and state.document_done; + current.done = state.document_done; + repository_state.current_document_index = state.document_index; + repository_state.next_document_index = state.document_index + 1; + repository_state.done = state.source_done; + return; + } + current.file.close(self.io); + repository_state.current = null; + } + return error.CheckpointSourceMismatch; + } + + fn resetRepository(self: *CorpusReader, state: *RepositoryState) anyerror!void { + if (state.current) |current| current.file.close(self.io); + state.current = null; + const new_walker = try RepositoryWalker.init( + self.backing_allocator, + self.io, + state.root, + state.walker.include_vcs_metadata, + ); + state.walker.deinit(self.io); + state.walker = new_walker; + state.current_document_index = 0; + state.next_document_index = 0; + state.done = false; + _ = self.document_arena.reset(.retain_capacity); + } + + fn restoreExternal( + self: *CorpusReader, + external_state: *ExternalState, + state: StreamingState, + ) anyerror!void { + if (!std.mem.eql(u8, external_state.source_id, state.source_id)) { + return error.CheckpointSourceMismatch; + } + const restoreFn = external_state.adapter.vtable.restore orelse + return error.RestoreUnsupported; + try restoreFn(external_state.adapter.context, state.resume_token); + + _ = self.document_arena.reset(.retain_capacity); + external_state.current_document_index = state.document_index; + external_state.next_document_index = state.document_index + 1; + external_state.offset = state.byte_offset; + external_state.in_document = !state.document_done; + external_state.done = state.source_done; + if (external_state.in_document) { + external_state.current_document_id = try self.document_arena.allocator().dupe( + u8, + state.document_id, + ); + } else { + external_state.current_document_id = ""; + } + } + + fn resetChunkArena(self: *CorpusReader) void { + const mode: ArenaAllocator.ResetMode = if (self.options.retained_chunk_bytes) |limit| + .{ .retain_with_limit = limit } + else + .retain_capacity; + _ = self.chunk_arena.reset(mode); + } +}; + +fn isVcsDirectory(name: []const u8) bool { + return std.mem.eql(u8, name, ".git") or + std.mem.eql(u8, name, ".hg") or + std.mem.eql(u8, name, ".svn"); +} + +fn joinCanonical( + allocator: Allocator, + prefix: []const u8, + name: []const u8, +) Allocator.Error![]u8 { + if (prefix.len == 0) return allocator.dupe(u8, name); + + const result = try allocator.alloc(u8, prefix.len + 1 + name.len); + @memcpy(result[0..prefix.len], prefix); + result[prefix.len] = '/'; + @memcpy(result[prefix.len + 1 ..], name); + return result; +} + +fn putU32(destination: *[4]u8, value: u32) void { + for (destination, 0..) |*byte, index| { + byte.* = @truncate(value >> @intCast(index * 8)); + } +} + +fn putU64(destination: *[8]u8, value: u64) void { + for (destination, 0..) |*byte, index| { + byte.* = @truncate(value >> @intCast(index * 8)); + } +} + +fn getU32(source: *const [4]u8) u32 { + var value: u32 = 0; + for (source, 0..) |byte, index| { + value |= @as(u32, byte) << @intCast(index * 8); + } + return value; +} + +fn getU64(source: *const [8]u8) u64 { + var value: u64 = 0; + for (source, 0..) |byte, index| { + value |= @as(u64, byte) << @intCast(index * 8); + } + return value; +} + +test "streaming state has a transport-stable encoding" { + const original: StreamingState = .{ + .source_kind = .database, + .sequence = 42, + .document_index = 7, + .byte_offset = 1_000_000, + .document_done = true, + .source_done = false, + .source_id = "books-db", + .document_id = "row/99", + .resume_token = &.{ 0, 1, 2, 255 }, + }; + + const encoded = try original.encode(std.testing.allocator); + defer std.testing.allocator.free(encoded); + const decoded = try StreamingState.decode(encoded); + + try std.testing.expectEqual(original.source_kind, decoded.source_kind); + try std.testing.expectEqual(original.sequence, decoded.sequence); + try std.testing.expectEqual(original.document_index, decoded.document_index); + try std.testing.expectEqual(original.byte_offset, decoded.byte_offset); + try std.testing.expectEqual(original.document_done, decoded.document_done); + try std.testing.expectEqual(original.source_done, decoded.source_done); + try std.testing.expectEqualStrings(original.source_id, decoded.source_id); + try std.testing.expectEqualStrings(original.document_id, decoded.document_id); + try std.testing.expectEqualSlices(u8, original.resume_token, decoded.resume_token); +} + +const TestStream = struct { + bytes: []const u8, + position: usize = 0, + token: [8]u8 = @splat(0), + + const vtable: ExternalSource.VTable = .{ + .pull = pull, + .restore = restore, + }; + + fn source(self: *TestStream) ExternalSource { + return .{ + .context = self, + .source_id = "test-stream", + .vtable = &vtable, + }; + } + + fn pull(context: *anyopaque, destination: []u8) anyerror!ExternalSource.PullResult { + const self: *TestStream = @ptrCast(@alignCast(context)); + if (self.position == self.bytes.len) { + return .{ + .bytes_written = 0, + .document_id = "", + .is_document_start = false, + .is_document_end = false, + .is_source_end = true, + }; + } + + const start = self.position; + const count = @min(destination.len, self.bytes.len - start); + @memcpy(destination[0..count], self.bytes[start..][0..count]); + self.position += count; + putU64(&self.token, self.position); + const is_end = self.position == self.bytes.len; + return .{ + .bytes_written = count, + .document_id = "stream-document", + .is_document_start = start == 0, + .is_document_end = is_end, + .is_source_end = is_end, + .resume_token = &self.token, + }; + } + + fn restore(context: *anyopaque, token: []const u8) anyerror!void { + const self: *TestStream = @ptrCast(@alignCast(context)); + if (token.len != self.token.len) return error.InvalidToken; + const position = getU64(token[0..8]); + if (position > self.bytes.len) return error.InvalidToken; + self.position = @intCast(position); + } +}; + +test "stream adapters write directly into chunks and restore from tokens" { + var stream: TestStream = .{ .bytes = "streamed corpus" }; + const reader = try CorpusReader.openStream( + std.testing.allocator, + std.testing.io, + stream.source(), + .{ .chunk_size = 8 }, + ); + defer reader.destroy(); + + const first = (try reader.next()).?; + try std.testing.expectEqualStrings("streamed", first.bytes); + try std.testing.expect(first.is_document_start); + const encoded = try first.state_after.encode(std.testing.allocator); + defer std.testing.allocator.free(encoded); + + // Restoring directly from a borrowed chunk state must stabilize its token + // before resetting the chunk arena. + try reader.restore(first.state_after); + const directly_replayed = (try reader.next()).?; + try std.testing.expectEqualStrings(" corpus", directly_replayed.bytes); + try std.testing.expect(directly_replayed.is_document_end); + + try reader.restore(try StreamingState.decode(encoded)); + const replayed = (try reader.next()).?; + try std.testing.expectEqualStrings(" corpus", replayed.bytes); + try std.testing.expectEqual(@as(u64, 1), replayed.sequence); +} + +test "file source emits bounded chunks and restores exactly" { + const io = std.testing.io; + var temporary = std.testing.tmpDir(.{}); + defer temporary.cleanup(); + try temporary.dir.writeFile(io, .{ + .sub_path = "corpus.txt", + .data = "abcdefghij", + }); + + const reader = try CorpusReader.openFile( + std.testing.allocator, + io, + temporary.dir, + "corpus.txt", + .{ .chunk_size = 4 }, + ); + defer reader.destroy(); + + const first = (try reader.next()).?; + try std.testing.expectEqualStrings("abcd", first.bytes); + try std.testing.expect(first.is_document_start); + try std.testing.expect(!first.is_document_end); + + const encoded = try first.state_after.encode(std.testing.allocator); + defer std.testing.allocator.free(encoded); + + const second = (try reader.next()).?; + try std.testing.expectEqualStrings("efgh", second.bytes); + + try reader.restore(try StreamingState.decode(encoded)); + const replayed = (try reader.next()).?; + try std.testing.expectEqualStrings("efgh", replayed.bytes); + try std.testing.expectEqual(@as(u64, 1), replayed.sequence); + + const last = (try reader.next()).?; + try std.testing.expectEqualStrings("ij", last.bytes); + try std.testing.expect(last.is_document_end); + try std.testing.expect(last.state_after.source_done); + try std.testing.expectEqual(@as(?Chunk, null), try reader.next()); +} + +test "repository source skips VCS metadata and preserves document boundaries" { + const io = std.testing.io; + var temporary = std.testing.tmpDir(.{ .iterate = true }); + defer temporary.cleanup(); + try temporary.dir.writeFile(io, .{ .sub_path = "one.txt", .data = "one" }); + try temporary.dir.writeFile(io, .{ .sub_path = "z-last.txt", .data = "z" }); + try temporary.dir.createDir(io, ".git", .default_dir); + var git_dir = try temporary.dir.openDir(io, ".git", .{}); + defer git_dir.close(io); + try git_dir.writeFile(io, .{ .sub_path = "secret", .data = "skip" }); + + const reader = try CorpusReader.openRepository( + std.testing.allocator, + io, + temporary.dir, + ".", + .{ .reader = .{ .chunk_size = 2 } }, + ); + defer reader.destroy(); + + const first = (try reader.next()).?; + try std.testing.expectEqualStrings("one.txt", first.document_id); + try std.testing.expectEqualStrings("on", first.bytes); + try std.testing.expect(first.is_document_start); + const checkpoint = try first.state_after.encode(std.testing.allocator); + defer std.testing.allocator.free(checkpoint); + + const second = (try reader.next()).?; + try std.testing.expectEqualStrings("e", second.bytes); + try std.testing.expect(second.is_document_end); + + const third = (try reader.next()).?; + try std.testing.expectEqualStrings("z-last.txt", third.document_id); + try std.testing.expectEqualStrings("z", third.bytes); + try std.testing.expectEqual(@as(?Chunk, null), try reader.next()); + + try reader.restore(try StreamingState.decode(checkpoint)); + const replayed = (try reader.next()).?; + try std.testing.expectEqualStrings("e", replayed.bytes); + const deterministic_next = (try reader.next()).?; + try std.testing.expectEqualStrings("z-last.txt", deterministic_next.document_id); +} + +test "empty files are represented as one boundary chunk" { + const io = std.testing.io; + var temporary = std.testing.tmpDir(.{}); + defer temporary.cleanup(); + try temporary.dir.writeFile(io, .{ .sub_path = "empty", .data = "" }); + + const reader = try CorpusReader.openFile( + std.testing.allocator, + io, + temporary.dir, + "empty", + .{}, + ); + defer reader.destroy(); + + const chunk = (try reader.next()).?; + try std.testing.expectEqual(@as(usize, 0), chunk.bytes.len); + try std.testing.expect(chunk.is_document_start); + try std.testing.expect(chunk.is_document_end); + try std.testing.expectEqual(@as(?Chunk, null), try reader.next()); +} diff --git a/data/corpus-reader/root.zig b/data/corpus-reader/root.zig new file mode 100644 index 0000000..9ec4e15 --- /dev/null +++ b/data/corpus-reader/root.zig @@ -0,0 +1,13 @@ +//! Raw, bounded-memory corpus ingestion. + +pub const CorpusReader = @import("corpus_reader.zig").CorpusReader; +pub const Chunk = @import("corpus_reader.zig").Chunk; +pub const ExternalSource = @import("corpus_reader.zig").ExternalSource; +pub const Options = @import("corpus_reader.zig").Options; +pub const RepositoryOptions = @import("corpus_reader.zig").RepositoryOptions; +pub const SourceKind = @import("corpus_reader.zig").SourceKind; +pub const StreamingState = @import("corpus_reader.zig").StreamingState; + +test { + _ = @import("corpus_reader.zig"); +} diff --git a/data/deduplicator/README.md b/data/deduplicator/README.md new file mode 100644 index 0000000..6dbd53f --- /dev/null +++ b/data/deduplicator/README.md @@ -0,0 +1,36 @@ +# Deduplicator + +The deduplicator consumes document fragments and classifies a document only at +its end boundary. It supports three complementary signals: + +- a 256-bit, chunk-boundary-independent BLAKE3 Merkle fingerprint for exact + document matches; +- a streaming 64-bit SimHash over normalized word features for approximate + document matches; +- sampled rolling windows, verified with 128-bit BLAKE3 fingerprints, for + exact duplicate text spans. + +`Deduplicator` retains only fixed-size hashing state and one span-sized ring +buffer per active document. Per-call results use a resettable arena. The index +is an explicit backend interface: `InMemoryIndex` is provided for a single +worker, while a remote implementation can move fingerprints and transactional +state to a memory service. Span probes are batched once per input chunk to +avoid turning sampled windows into individual network round trips. + +An active document can be serialized with `checkpoint`, transported, decoded +as `StreamingState`, and resumed with `restore`. The checkpoint includes the +Merkle peaks, partial block, SimHash accumulator, span ring, and the backend's +uncommitted transaction token. + +The backend transaction starts with the document and commits only if the final +classification is unique. Exact or approximate duplicates never publish their +staged spans. A production distributed backend must make `finishDocument` +atomic so two workers cannot both claim the same content as canonical. + +Approximate lookup uses four 16-bit SimHash bands, so the configured Hamming +distance is capped at three. This guarantees that every qualifying signature +shares at least one band with its candidate. + +Returned slices remain valid until the next call to `observe`, `abortDocument`, +or `destroy`. Canonical identifiers owned by the in-memory index remain valid +for the lifetime of that index. diff --git a/data/deduplicator/deduplicator.zig b/data/deduplicator/deduplicator.zig new file mode 100644 index 0000000..950a0a0 --- /dev/null +++ b/data/deduplicator/deduplicator.zig @@ -0,0 +1,1465 @@ +const std = @import("std"); + +const Allocator = std.mem.Allocator; +const ArenaAllocator = std.heap.ArenaAllocator; +const Blake3 = std.crypto.hash.Blake3; +const CorpusChunk = @import("../corpus-reader/root.zig").Chunk; +const CleanedChunk = @import("../text-cleaner/root.zig").CleanedChunk; + +pub const ExactDigest = [32]u8; +pub const SpanDigest = [16]u8; + +pub const Boundary = struct { + document_start: bool = false, + document_end: bool = false, + + pub const whole: Boundary = .{ + .document_start = true, + .document_end = true, + }; +}; + +pub const DocumentChunk = struct { + document_id: []const u8, + bytes: []const u8, + boundary: Boundary, +}; + +pub const DocumentFingerprint = struct { + document_id: []const u8, + exact: ExactDigest, + simhash: u64, + feature_count: u64, + byte_count: u64, +}; + +pub const SpanFingerprint = struct { + digest: SpanDigest, + start_byte: u64, + end_byte: u64, +}; + +pub const SpanOccurrence = struct { + document_id: []const u8, + start_byte: u64, + end_byte: u64, +}; + +pub const DuplicateSpan = struct { + current_start_byte: u64, + current_end_byte: u64, + canonical_document_id: []const u8, + canonical_start_byte: u64, + canonical_end_byte: u64, +}; + +pub const CanonicalMatch = struct { + document_id: []const u8, +}; + +pub const ApproximateMatch = struct { + document_id: []const u8, + hamming_distance: u8, +}; + +pub const Classification = union(enum) { + unique, + exact_duplicate: CanonicalMatch, + approximate_duplicate: ApproximateMatch, +}; + +pub const Status = union(enum) { + pending, + complete: Classification, +}; + +/// All borrowed slices in an observation are valid until the next mutating +/// deduplicator call. +pub const Observation = struct { + document_id: []const u8, + status: Status, + duplicate_spans: []const DuplicateSpan, + fingerprint: ?DocumentFingerprint = null, +}; + +pub const ApproximatePolicy = struct { + enabled: bool = true, + /// Four 16-bit bands guarantee candidate discovery only through distance 3. + max_hamming_distance: u8 = 3, + min_features: u64 = 8, +}; + +pub const SpanPolicy = struct { + // TODO(data/tokenizer): add token-shingle MinHash for approximate span + // matches after the tokenizer exposes versioned streaming word boundaries. + enabled: bool = true, + window_bytes: usize = 128, + /// A window is retained when the low `sampling_bits` rolling-hash bits are 0. + sampling_bits: u8 = 6, + min_start_distance: u64 = 64, +}; + +pub const Options = struct { + approximate: ApproximatePolicy = .{}, + spans: SpanPolicy = .{}, + retained_output_bytes: ?usize = null, +}; + +/// Validated view of a transportable deduplicator checkpoint. The encoded +/// bytes are borrowed; `Deduplicator.checkpoint` returns caller-owned storage. +pub const StreamingState = struct { + encoded: []const u8, + + pub const DecodeError = error{ + InvalidState, + UnsupportedVersion, + }; + + pub fn decode(encoded: []const u8) DecodeError!StreamingState { + try validateEncodedState(encoded); + return .{ .encoded = encoded }; + } +}; + +/// Transactional index boundary for memory/compute disaggregation. `finish` +/// must atomically classify and publish a unique document. The backend may +/// borrow all input only for the duration of a call; returned identifiers must +/// remain valid until the next call on the same backend context. +pub const Index = struct { + context: *anyopaque, + vtable: *const VTable, + version: u64 = 1, + + pub const VTable = struct { + startDocument: *const fn (*anyopaque, []const u8) anyerror!void, + probeSpans: *const fn ( + *anyopaque, + []const SpanFingerprint, + []?SpanOccurrence, + ) anyerror!void, + finishDocument: *const fn ( + *anyopaque, + DocumentFingerprint, + ApproximatePolicy, + ) anyerror!Classification, + abortDocument: *const fn (*anyopaque) void, + checkpointDocument: ?*const fn (*anyopaque, Allocator) anyerror![]const u8 = null, + restoreDocument: ?*const fn (*anyopaque, []const u8, []const u8) anyerror!void = null, + deinit: ?*const fn (*anyopaque) void = null, + }; +}; + +const band_count = 4; +const StoredDocument = struct { + document_id: []const u8, + exact: ExactDigest, + simhash: u64, + feature_count: u64, + next_by_band: [band_count]?usize, + visited_epoch: u64 = 0, +}; + +const StoredSpan = struct { + document_index: usize, + start_byte: u64, + end_byte: u64, +}; + +/// Single-threaded reference backend. Give each worker its own instance, or +/// protect it externally. A distributed backend should preserve the same +/// transaction semantics with a compare-and-publish operation. +pub const InMemoryIndex = struct { + allocator: Allocator, + id_arena: ArenaAllocator, + exact: std.AutoHashMapUnmanaged(ExactDigest, usize) = .empty, + bands: std.AutoHashMapUnmanaged(u32, usize) = .empty, + spans: std.AutoHashMapUnmanaged(SpanDigest, StoredSpan) = .empty, + documents: std.ArrayList(StoredDocument) = .empty, + staged_spans: std.ArrayList(SpanFingerprint) = .empty, + active: bool = false, + query_epoch: u64 = 0, + + pub fn create(allocator: Allocator) Allocator.Error!*InMemoryIndex { + const self = try allocator.create(InMemoryIndex); + self.* = .{ + .allocator = allocator, + .id_arena = .init(allocator), + }; + return self; + } + + pub fn destroy(self: *InMemoryIndex) void { + self.staged_spans.deinit(self.allocator); + self.documents.deinit(self.allocator); + self.spans.deinit(self.allocator); + self.bands.deinit(self.allocator); + self.exact.deinit(self.allocator); + self.id_arena.deinit(); + const allocator = self.allocator; + allocator.destroy(self); + } + + /// Transfers backend ownership to a successfully-created Deduplicator. + pub fn ownedIndex(self: *InMemoryIndex) Index { + return .{ .context = self, .vtable = &owned_vtable }; + } + + /// The caller remains responsible for destroying the backend. + pub fn borrowedIndex(self: *InMemoryIndex) Index { + return .{ .context = self, .vtable = &borrowed_vtable }; + } + + pub fn documentCount(self: *const InMemoryIndex) usize { + return self.documents.items.len; + } + + fn indexStart(context: *anyopaque, _: []const u8) anyerror!void { + const self: *InMemoryIndex = @ptrCast(@alignCast(context)); + if (self.active) return error.IndexTransactionActive; + self.staged_spans.clearRetainingCapacity(); + self.active = true; + } + + fn indexProbeSpans( + context: *anyopaque, + fingerprints: []const SpanFingerprint, + matches: []?SpanOccurrence, + ) anyerror!void { + const self: *InMemoryIndex = @ptrCast(@alignCast(context)); + if (!self.active) return error.IndexTransactionNotActive; + if (fingerprints.len != matches.len) return error.InvalidProbeBatch; + try self.staged_spans.ensureUnusedCapacity(self.allocator, fingerprints.len); + + for (fingerprints, matches) |fingerprint, *match| { + if (self.spans.get(fingerprint.digest)) |occurrence| { + const document = self.documents.items[occurrence.document_index]; + match.* = .{ + .document_id = document.document_id, + .start_byte = occurrence.start_byte, + .end_byte = occurrence.end_byte, + }; + } else { + match.* = null; + } + self.staged_spans.appendAssumeCapacity(fingerprint); + } + } + + fn indexFinish( + context: *anyopaque, + fingerprint: DocumentFingerprint, + policy: ApproximatePolicy, + ) anyerror!Classification { + const self: *InMemoryIndex = @ptrCast(@alignCast(context)); + if (!self.active) return error.IndexTransactionNotActive; + + if (self.exact.get(fingerprint.exact)) |document_index| { + self.closeTransaction(); + return .{ .exact_duplicate = .{ + .document_id = self.documents.items[document_index].document_id, + } }; + } + + if (policy.enabled and fingerprint.feature_count >= policy.min_features) { + if (self.closestApproximate(fingerprint, policy)) |match| { + self.closeTransaction(); + return .{ .approximate_duplicate = match }; + } + } + + try self.commit(fingerprint, policy.enabled); + self.closeTransaction(); + return .unique; + } + + fn closestApproximate( + self: *InMemoryIndex, + fingerprint: DocumentFingerprint, + policy: ApproximatePolicy, + ) ?ApproximateMatch { + self.advanceQueryEpoch(); + var closest_index: ?usize = null; + var closest_distance: u8 = std.math.maxInt(u8); + + for (0..band_count) |band_index| { + var cursor = self.bands.get(bandKey(fingerprint.simhash, band_index)); + while (cursor) |document_index| { + const candidate = &self.documents.items[document_index]; + cursor = candidate.next_by_band[band_index]; + if (candidate.visited_epoch == self.query_epoch) continue; + candidate.visited_epoch = self.query_epoch; + if (candidate.feature_count >= policy.min_features) { + const distance: u8 = @intCast(@popCount( + fingerprint.simhash ^ candidate.simhash, + )); + if (distance <= policy.max_hamming_distance and + (distance < closest_distance or + (distance == closest_distance and + (closest_index == null or + document_index < closest_index.?)))) + { + closest_index = document_index; + closest_distance = distance; + } + } + } + } + + const index = closest_index orelse return null; + return .{ + .document_id = self.documents.items[index].document_id, + .hamming_distance = closest_distance, + }; + } + + fn advanceQueryEpoch(self: *InMemoryIndex) void { + if (self.query_epoch == std.math.maxInt(u64)) { + for (self.documents.items) |*document| document.visited_epoch = 0; + self.query_epoch = 1; + } else { + self.query_epoch += 1; + } + } + + /// Reserves every fallible allocation before publishing any index entry. + fn commit( + self: *InMemoryIndex, + fingerprint: DocumentFingerprint, + index_approximate: bool, + ) anyerror!void { + try self.documents.ensureUnusedCapacity(self.allocator, 1); + try self.exact.ensureUnusedCapacity(self.allocator, 1); + if (index_approximate) { + try self.bands.ensureUnusedCapacity(self.allocator, band_count); + } + const staged_count = std.math.cast(u32, self.staged_spans.items.len) orelse + return error.IndexCapacityExceeded; + try self.spans.ensureUnusedCapacity(self.allocator, staged_count); + const owned_id = try self.id_arena.allocator().dupe(u8, fingerprint.document_id); + + const document_index = self.documents.items.len; + var next_by_band: [band_count]?usize = @splat(null); + if (index_approximate) { + for (0..band_count) |band_index| { + next_by_band[band_index] = self.bands.get( + bandKey(fingerprint.simhash, band_index), + ); + } + } + + self.documents.appendAssumeCapacity(.{ + .document_id = owned_id, + .exact = fingerprint.exact, + .simhash = fingerprint.simhash, + .feature_count = fingerprint.feature_count, + .next_by_band = next_by_band, + }); + self.exact.putAssumeCapacityNoClobber(fingerprint.exact, document_index); + if (index_approximate) { + for (0..band_count) |band_index| { + self.bands.putAssumeCapacity( + bandKey(fingerprint.simhash, band_index), + document_index, + ); + } + } + for (self.staged_spans.items) |span| { + if (!self.spans.contains(span.digest)) { + self.spans.putAssumeCapacityNoClobber(span.digest, .{ + .document_index = document_index, + .start_byte = span.start_byte, + .end_byte = span.end_byte, + }); + } + } + } + + fn indexAbort(context: *anyopaque) void { + const self: *InMemoryIndex = @ptrCast(@alignCast(context)); + self.closeTransaction(); + } + + fn indexCheckpoint(context: *anyopaque, allocator: Allocator) anyerror![]const u8 { + const self: *InMemoryIndex = @ptrCast(@alignCast(context)); + if (!self.active) return error.IndexTransactionNotActive; + if (self.staged_spans.items.len > std.math.maxInt(u32)) { + return error.IndexCapacityExceeded; + } + const item_bytes = std.math.mul(usize, self.staged_spans.items.len, 32) catch + return error.IndexCapacityExceeded; + const encoded = try allocator.alloc(u8, 8 + item_bytes); + @memcpy(encoded[0..4], "IM01"); + putU32(encoded[4..8], @intCast(self.staged_spans.items.len)); + var cursor: usize = 8; + for (self.staged_spans.items) |span| { + @memcpy(encoded[cursor..][0..16], &span.digest); + putU64(encoded[cursor + 16 ..][0..8], span.start_byte); + putU64(encoded[cursor + 24 ..][0..8], span.end_byte); + cursor += 32; + } + return encoded; + } + + fn indexRestore( + context: *anyopaque, + _: []const u8, + token: []const u8, + ) anyerror!void { + const self: *InMemoryIndex = @ptrCast(@alignCast(context)); + if (self.active) return error.IndexTransactionActive; + if (token.len < 8 or !std.mem.eql(u8, token[0..4], "IM01")) { + return error.InvalidIndexCheckpoint; + } + const count: usize = getU32(token[4..8]); + const item_bytes = std.math.mul(usize, count, 32) catch + return error.InvalidIndexCheckpoint; + if (token.len != 8 + item_bytes) return error.InvalidIndexCheckpoint; + try self.staged_spans.ensureUnusedCapacity(self.allocator, count); + self.staged_spans.clearRetainingCapacity(); + var cursor: usize = 8; + for (0..count) |_| { + var digest: SpanDigest = undefined; + @memcpy(&digest, token[cursor..][0..16]); + const start_byte = getU64(token[cursor + 16 ..][0..8]); + const end_byte = getU64(token[cursor + 24 ..][0..8]); + if (end_byte < start_byte) return error.InvalidIndexCheckpoint; + self.staged_spans.appendAssumeCapacity(.{ + .digest = digest, + .start_byte = start_byte, + .end_byte = end_byte, + }); + cursor += 32; + } + self.active = true; + } + + fn closeTransaction(self: *InMemoryIndex) void { + self.staged_spans.clearRetainingCapacity(); + self.active = false; + } + + fn indexDeinit(context: *anyopaque) void { + const self: *InMemoryIndex = @ptrCast(@alignCast(context)); + self.destroy(); + } + + const borrowed_vtable: Index.VTable = .{ + .startDocument = indexStart, + .probeSpans = indexProbeSpans, + .finishDocument = indexFinish, + .abortDocument = indexAbort, + .checkpointDocument = indexCheckpoint, + .restoreDocument = indexRestore, + }; + + const owned_vtable: Index.VTable = .{ + .startDocument = indexStart, + .probeSpans = indexProbeSpans, + .finishDocument = indexFinish, + .abortDocument = indexAbort, + .checkpointDocument = indexCheckpoint, + .restoreDocument = indexRestore, + .deinit = indexDeinit, + }; +}; + +const exact_block_bytes = 4096; +const ExactHasher = struct { + partial: [exact_block_bytes]u8 = @splat(0), + partial_len: usize = 0, + peaks: [64]ExactDigest = @splat(@splat(0)), + occupied: u64 = 0, + block_count: u64 = 0, + total_bytes: u64 = 0, + + fn update(self: *ExactHasher, input: []const u8) error{DocumentTooLarge}!void { + self.total_bytes = std.math.add(u64, self.total_bytes, input.len) catch + return error.DocumentTooLarge; + + var remaining = input; + while (remaining.len != 0) { + const take = @min(exact_block_bytes - self.partial_len, remaining.len); + @memcpy(self.partial[self.partial_len..][0..take], remaining[0..take]); + self.partial_len += take; + remaining = remaining[take..]; + if (self.partial_len == exact_block_bytes) { + self.addLeaf(hashLeaf(self.partial[0..])); + self.partial_len = 0; + self.block_count += 1; + } + } + } + + fn digest(self: ExactHasher) ExactDigest { + var copy = self; + if (copy.partial_len != 0 or copy.block_count == 0) { + copy.addLeaf(hashLeaf(copy.partial[0..copy.partial_len])); + copy.block_count += 1; + } + + var hash = Blake3.init(.{}); + hash.update("chi.dedup.root.v1\x00"); + var number: [8]u8 = undefined; + putU64(&number, copy.total_bytes); + hash.update(&number); + putU64(&number, copy.block_count); + hash.update(&number); + for (0..64) |reverse_index| { + const level = 63 - reverse_index; + if (copy.occupied & (@as(u64, 1) << @intCast(level)) != 0) { + const encoded_level: [1]u8 = .{@intCast(level)}; + hash.update(&encoded_level); + hash.update(©.peaks[level]); + } + } + var result: ExactDigest = undefined; + hash.final(&result); + return result; + } + + fn addLeaf(self: *ExactHasher, leaf: ExactDigest) void { + var node = leaf; + var level: usize = 0; + while (self.occupied & (@as(u64, 1) << @intCast(level)) != 0) { + node = hashParent(self.peaks[level], node); + self.occupied &= ~(@as(u64, 1) << @intCast(level)); + level += 1; + } + self.peaks[level] = node; + self.occupied |= @as(u64, 1) << @intCast(level); + } +}; + +const SimHasher = struct { + weights: [64]i64 = @splat(0), + token_hash: u64 = fnv_offset, + token_len: u64 = 0, + feature_count: u64 = 0, + + fn update(self: *SimHasher, bytes: []const u8) error{DocumentTooLarge}!void { + for (bytes) |byte| { + if (isTokenByte(byte)) { + const normalized = if (byte >= 'A' and byte <= 'Z') byte + 32 else byte; + self.token_hash ^= normalized; + self.token_hash *%= fnv_prime; + self.token_len = std.math.add(u64, self.token_len, 1) catch + return error.DocumentTooLarge; + } else { + try self.finishToken(); + } + } + } + + fn finishToken(self: *SimHasher) error{DocumentTooLarge}!void { + if (self.token_len == 0) return; + const mixed = mix64(self.token_hash ^ (self.token_len *% 0x9e3779b97f4a7c15)); + for (&self.weights, 0..) |*weight, bit| { + if (mixed & (@as(u64, 1) << @intCast(bit)) != 0) { + weight.* +|= 1; + } else { + weight.* -|= 1; + } + } + self.feature_count = std.math.add(u64, self.feature_count, 1) catch + return error.DocumentTooLarge; + self.token_hash = fnv_offset; + self.token_len = 0; + } + + fn digest(self: SimHasher) u64 { + var result: u64 = 0; + for (self.weights, 0..) |weight, bit| { + if (weight > 0) result |= @as(u64, 1) << @intCast(bit); + } + return result; + } +}; + +const SpanHasher = struct { + ring: []u8 = &.{}, + next: usize = 0, + filled: usize = 0, + rolling: u64 = 0, + outgoing_factor: u64 = 1, + last_sample_start: ?u64 = null, + + fn init(ring: []u8) SpanHasher { + var factor: u64 = 1; + for (0..ring.len) |_| factor *%= rolling_base; + return .{ .ring = ring, .outgoing_factor = factor }; + } +}; + +const DocumentState = struct { + in_document: bool = false, + document_id: []const u8 = "", + exact: ExactHasher = .{}, + approximate: SimHasher = .{}, + spans: SpanHasher = .{}, +}; + +pub const Deduplicator = struct { + backing_allocator: Allocator, + index: Index, + options: Options, + document_arena: ArenaAllocator, + output_arena: ArenaAllocator, + state: DocumentState = .{}, + + pub const CreateError = Allocator.Error || error{ + InvalidApproximateDistance, + InvalidSpanPolicy, + }; + + pub fn create( + backing_allocator: Allocator, + index: Index, + options: Options, + ) CreateError!*Deduplicator { + try validateOptions(options); + const self = try backing_allocator.create(Deduplicator); + self.* = .{ + .backing_allocator = backing_allocator, + .index = index, + .options = options, + .document_arena = .init(backing_allocator), + .output_arena = .init(backing_allocator), + }; + return self; + } + + pub fn destroy(self: *Deduplicator) void { + if (self.state.in_document) self.index.vtable.abortDocument(self.index.context); + self.output_arena.deinit(); + self.document_arena.deinit(); + if (self.index.vtable.deinit) |deinitFn| deinitFn(self.index.context); + const allocator = self.backing_allocator; + allocator.destroy(self); + } + + pub fn observe(self: *Deduplicator, chunk: DocumentChunk) anyerror!Observation { + try self.validateBoundary(chunk); + self.resetOutputArena(); + + if (chunk.boundary.document_start) try self.startDocument(chunk.document_id); + errdefer self.abortDocument(); + + try self.state.exact.update(chunk.bytes); + if (self.options.approximate.enabled) try self.state.approximate.update(chunk.bytes); + + var duplicate_spans: std.ArrayList(DuplicateSpan) = .empty; + if (self.options.spans.enabled) { + try self.updateSpans(chunk.bytes, &duplicate_spans); + } + + if (!chunk.boundary.document_end) { + return .{ + .document_id = self.state.document_id, + .status = .pending, + .duplicate_spans = duplicate_spans.items, + }; + } + + if (self.options.approximate.enabled) try self.state.approximate.finishToken(); + const fingerprint = DocumentFingerprint{ + .document_id = self.state.document_id, + .exact = self.state.exact.digest(), + .simhash = self.state.approximate.digest(), + .feature_count = self.state.approximate.feature_count, + .byte_count = self.state.exact.total_bytes, + }; + const backend_classification = try self.index.vtable.finishDocument( + self.index.context, + fingerprint, + self.options.approximate, + ); + self.state.in_document = false; + return .{ + .document_id = self.state.document_id, + .status = .{ .complete = backend_classification }, + .duplicate_spans = duplicate_spans.items, + .fingerprint = fingerprint, + }; + } + + pub fn observeCorpusChunk( + self: *Deduplicator, + chunk: CorpusChunk, + ) anyerror!Observation { + return self.observe(.{ + .document_id = chunk.document_id, + .bytes = chunk.bytes, + .boundary = .{ + .document_start = chunk.is_document_start, + .document_end = chunk.is_document_end, + }, + }); + } + + pub fn observeCleanedChunk( + self: *Deduplicator, + document_id: []const u8, + chunk: CleanedChunk, + ) anyerror!Observation { + return self.observe(.{ + .document_id = document_id, + .bytes = chunk.bytes, + .boundary = .{ + .document_start = chunk.document_start, + .document_end = chunk.document_end, + }, + }); + } + + /// Captures local streaming accumulators plus the index's unpublished + /// transaction. The caller owns the returned bytes. + pub fn checkpoint(self: *Deduplicator, allocator: Allocator) anyerror![]u8 { + var scratch: ArenaAllocator = .init(self.backing_allocator); + defer scratch.deinit(); + + var backend_token: []const u8 = ""; + if (self.state.in_document) { + const checkpointFn = self.index.vtable.checkpointDocument orelse + return error.IndexCheckpointUnsupported; + backend_token = try checkpointFn(self.index.context, scratch.allocator()); + } + + const document_id = if (self.state.in_document) self.state.document_id else ""; + const partial_len = if (self.state.in_document) self.state.exact.partial_len else 0; + const ring = if (self.state.in_document) self.state.spans.ring else &.{}; + if (document_id.len > std.math.maxInt(u32) or + backend_token.len > std.math.maxInt(u32) or + partial_len > std.math.maxInt(u16)) + { + return error.StateTooLarge; + } + + var total: usize = checkpoint_header_len; + total = std.math.add(usize, total, document_id.len) catch return error.StateTooLarge; + total = std.math.add(usize, total, partial_len) catch return error.StateTooLarge; + total = std.math.add(usize, total, ring.len) catch return error.StateTooLarge; + total = std.math.add(usize, total, backend_token.len) catch return error.StateTooLarge; + + const encoded = try allocator.alloc(u8, total); + @memset(encoded, 0); + @memcpy(encoded[0..4], checkpoint_magic); + putU64(encoded[8..16], self.configurationFingerprint()); + putU32(encoded[16..20], @intCast(document_id.len)); + putU32(encoded[20..24], @intCast(backend_token.len)); + + if (self.state.in_document) { + encoded[4] = 1 << 0; + if (self.state.spans.last_sample_start != null) encoded[4] |= 1 << 1; + putU16(encoded[24..26], @intCast(partial_len)); + putU64(encoded[28..36], self.state.exact.occupied); + putU64(encoded[36..44], self.state.exact.block_count); + putU64(encoded[44..52], self.state.exact.total_bytes); + putU64(encoded[52..60], self.state.approximate.token_hash); + putU64(encoded[60..68], self.state.approximate.token_len); + putU64(encoded[68..76], self.state.approximate.feature_count); + putU64(encoded[76..84], self.state.spans.rolling); + putU64( + encoded[84..92], + self.state.spans.last_sample_start orelse 0, + ); + putU64(encoded[92..100], self.state.spans.next); + putU64(encoded[100..108], self.state.spans.filled); + putU64(encoded[108..116], ring.len); + for (self.state.approximate.weights, 0..) |weight, index| { + putU64( + encoded[checkpoint_weights_offset + index * 8 ..][0..8], + @as(u64, @bitCast(weight)), + ); + } + for (self.state.exact.peaks, 0..) |peak, index| { + @memcpy( + encoded[checkpoint_peaks_offset + index * 32 ..][0..32], + &peak, + ); + } + } + + var cursor: usize = checkpoint_header_len; + @memcpy(encoded[cursor..][0..document_id.len], document_id); + cursor += document_id.len; + if (self.state.in_document) { + @memcpy( + encoded[cursor..][0..partial_len], + self.state.exact.partial[0..partial_len], + ); + } + cursor += partial_len; + @memcpy(encoded[cursor..][0..ring.len], ring); + cursor += ring.len; + @memcpy(encoded[cursor..][0..backend_token.len], backend_token); + std.debug.assert(cursor + backend_token.len == encoded.len); + const checksum = checkpointChecksum(encoded); + @memcpy(encoded[checkpoint_checksum_offset..][0..checksum.len], &checksum); + validateEncodedState(encoded) catch unreachable; + return encoded; + } + + /// Restores onto a worker with identical options and index protocol + /// version. The backend receives its staged transaction before processing + /// can continue. + pub fn restore( + self: *Deduplicator, + streaming_state: StreamingState, + ) anyerror!void { + try validateEncodedState(streaming_state.encoded); + const stable = try self.backing_allocator.dupe(u8, streaming_state.encoded); + defer self.backing_allocator.free(stable); + if (getU64(stable[8..16]) != self.configurationFingerprint()) { + return error.CheckpointConfigurationMismatch; + } + + const in_document = stable[4] & (1 << 0) != 0; + const restoreFn = if (in_document) + self.index.vtable.restoreDocument orelse + return error.IndexRestoreUnsupported + else + null; + + if (self.state.in_document) self.index.vtable.abortDocument(self.index.context); + self.state = .{}; + _ = self.document_arena.reset(.retain_capacity); + self.resetOutputArena(); + if (!in_document) return; + + const document_len: usize = getU32(stable[16..20]); + const backend_len: usize = getU32(stable[20..24]); + const partial_len: usize = getU16(stable[24..26]); + const ring_len: usize = @intCast(getU64(stable[108..116])); + var cursor: usize = checkpoint_header_len; + const encoded_document_id = stable[cursor..][0..document_len]; + cursor += document_len; + const encoded_partial = stable[cursor..][0..partial_len]; + cursor += partial_len; + const encoded_ring = stable[cursor..][0..ring_len]; + cursor += ring_len; + const backend_token = stable[cursor..][0..backend_len]; + + const document_id = try self.document_arena.allocator().dupe( + u8, + encoded_document_id, + ); + const ring = try self.document_arena.allocator().dupe(u8, encoded_ring); + var restored: DocumentState = .{ + .in_document = true, + .document_id = document_id, + .spans = .init(ring), + }; + restored.exact.partial_len = partial_len; + @memcpy(restored.exact.partial[0..partial_len], encoded_partial); + restored.exact.occupied = getU64(stable[28..36]); + restored.exact.block_count = getU64(stable[36..44]); + restored.exact.total_bytes = getU64(stable[44..52]); + restored.approximate.token_hash = getU64(stable[52..60]); + restored.approximate.token_len = getU64(stable[60..68]); + restored.approximate.feature_count = getU64(stable[68..76]); + restored.spans.rolling = getU64(stable[76..84]); + restored.spans.last_sample_start = if (stable[4] & (1 << 1) != 0) + getU64(stable[84..92]) + else + null; + restored.spans.next = @intCast(getU64(stable[92..100])); + restored.spans.filled = @intCast(getU64(stable[100..108])); + for (&restored.approximate.weights, 0..) |*weight, index| { + weight.* = @bitCast(getU64( + stable[checkpoint_weights_offset + index * 8 ..][0..8], + )); + } + for (&restored.exact.peaks, 0..) |*peak, index| { + @memcpy( + peak, + stable[checkpoint_peaks_offset + index * 32 ..][0..32], + ); + } + + restoreFn.?(self.index.context, document_id, backend_token) catch |err| { + _ = self.document_arena.reset(.retain_capacity); + return err; + }; + self.state = restored; + } + + pub fn abortDocument(self: *Deduplicator) void { + if (self.state.in_document) self.index.vtable.abortDocument(self.index.context); + self.state = .{}; + _ = self.document_arena.reset(.retain_capacity); + self.resetOutputArena(); + } + + fn validateBoundary(self: *const Deduplicator, chunk: DocumentChunk) !void { + if (chunk.boundary.document_start) { + if (self.state.in_document) return error.DocumentAlreadyStarted; + } else { + if (!self.state.in_document) return error.DocumentNotStarted; + if (!std.mem.eql(u8, self.state.document_id, chunk.document_id)) { + return error.DocumentIdChanged; + } + } + } + + fn startDocument(self: *Deduplicator, document_id: []const u8) anyerror!void { + _ = self.document_arena.reset(.retain_capacity); + self.state = .{}; + const owned_id = try self.document_arena.allocator().dupe(u8, document_id); + var spans: SpanHasher = .{}; + if (self.options.spans.enabled) { + spans = .init(try self.document_arena.allocator().alloc( + u8, + self.options.spans.window_bytes, + )); + @memset(spans.ring, 0); + } + self.state = .{ + .in_document = true, + .document_id = owned_id, + .spans = spans, + }; + self.index.vtable.startDocument(self.index.context, owned_id) catch |err| { + self.state = .{}; + _ = self.document_arena.reset(.retain_capacity); + return err; + }; + } + + fn updateSpans( + self: *Deduplicator, + bytes: []const u8, + duplicate_spans: *std.ArrayList(DuplicateSpan), + ) anyerror!void { + const policy = self.options.spans; + var span_state = &self.state.spans; + var fingerprints: std.ArrayList(SpanFingerprint) = .empty; + const fragment_start = self.state.exact.total_bytes - @as(u64, @intCast(bytes.len)); + for (bytes, 0..) |byte, byte_index| { + const encoded: u64 = @as(u64, byte) + 1; + if (span_state.filled < span_state.ring.len) { + span_state.ring[span_state.next] = byte; + span_state.next += 1; + if (span_state.next == span_state.ring.len) span_state.next = 0; + span_state.filled += 1; + span_state.rolling = span_state.rolling *% rolling_base +% encoded; + if (span_state.filled != span_state.ring.len) continue; + } else { + const outgoing: u64 = @as(u64, span_state.ring[span_state.next]) + 1; + span_state.ring[span_state.next] = byte; + span_state.next += 1; + if (span_state.next == span_state.ring.len) span_state.next = 0; + span_state.rolling = span_state.rolling *% rolling_base +% encoded -% + (outgoing *% span_state.outgoing_factor); + } + + // The exact hasher consumed the whole fragment before span probing. + const end_byte = fragment_start + @as(u64, @intCast(byte_index)) + 1; + const start_byte = end_byte - @as(u64, @intCast(span_state.ring.len)); + if (!shouldSample(span_state.*, policy, start_byte)) continue; + + try fingerprints.append(self.output_arena.allocator(), .{ + .digest = digestWindow(span_state.*), + .start_byte = start_byte, + .end_byte = end_byte, + }); + span_state.last_sample_start = start_byte; + } + + if (fingerprints.items.len == 0) return; + const matches = try self.output_arena.allocator().alloc( + ?SpanOccurrence, + fingerprints.items.len, + ); + try self.index.vtable.probeSpans( + self.index.context, + fingerprints.items, + matches, + ); + for (fingerprints.items, matches) |fingerprint, possible_match| { + if (possible_match) |match| { + try duplicate_spans.append(self.output_arena.allocator(), .{ + .current_start_byte = fingerprint.start_byte, + .current_end_byte = fingerprint.end_byte, + .canonical_document_id = try self.output_arena.allocator().dupe( + u8, + match.document_id, + ), + .canonical_start_byte = match.start_byte, + .canonical_end_byte = match.end_byte, + }); + } + } + } + + fn resetOutputArena(self: *Deduplicator) void { + const mode: ArenaAllocator.ResetMode = if (self.options.retained_output_bytes) |limit| + .{ .retain_with_limit = limit } + else + .retain_capacity; + _ = self.output_arena.reset(mode); + } + + fn configurationFingerprint(self: *const Deduplicator) u64 { + return optionsFingerprint(self.options, self.index.version); + } +}; + +const fnv_offset: u64 = 0xcbf29ce484222325; +const fnv_prime: u64 = 0x100000001b3; +const rolling_base: u64 = 0x9e3779b185ebca87; +const checkpoint_magic = "DD01"; +const checkpoint_checksum_offset = 116; +const checkpoint_weights_offset = checkpoint_checksum_offset + 16; +const checkpoint_peaks_offset = checkpoint_weights_offset + 64 * 8; +const checkpoint_header_len = checkpoint_peaks_offset + 64 * 32; + +fn validateEncodedState(encoded: []const u8) StreamingState.DecodeError!void { + if (encoded.len < checkpoint_header_len) return error.InvalidState; + if (!std.mem.eql(u8, encoded[0..4], checkpoint_magic)) { + return error.UnsupportedVersion; + } + const expected_checksum = checkpointChecksum(encoded); + if (!std.mem.eql( + u8, + encoded[checkpoint_checksum_offset..][0..expected_checksum.len], + &expected_checksum, + )) return error.InvalidState; + if (encoded[4] & ~@as(u8, 0b0000_0011) != 0) return error.InvalidState; + for (encoded[5..8]) |byte| { + if (byte != 0) return error.InvalidState; + } + if (getU16(encoded[26..28]) != 0) return error.InvalidState; + + const in_document = encoded[4] & (1 << 0) != 0; + const has_last_sample = encoded[4] & (1 << 1) != 0; + const document_len: usize = getU32(encoded[16..20]); + const backend_len: usize = getU32(encoded[20..24]); + const partial_len: usize = getU16(encoded[24..26]); + const ring_len_u64 = getU64(encoded[108..116]); + const ring_len = std.math.cast(usize, ring_len_u64) orelse + return error.InvalidState; + + var expected_len: usize = checkpoint_header_len; + expected_len = std.math.add(usize, expected_len, document_len) catch + return error.InvalidState; + expected_len = std.math.add(usize, expected_len, partial_len) catch + return error.InvalidState; + expected_len = std.math.add(usize, expected_len, ring_len) catch + return error.InvalidState; + expected_len = std.math.add(usize, expected_len, backend_len) catch + return error.InvalidState; + if (encoded.len != expected_len) return error.InvalidState; + + if (!in_document) { + if (encoded[4] != 0 or document_len != 0 or backend_len != 0 or + partial_len != 0 or ring_len != 0) + { + return error.InvalidState; + } + for (encoded[16..checkpoint_checksum_offset]) |byte| { + if (byte != 0) return error.InvalidState; + } + for (encoded[checkpoint_weights_offset..checkpoint_header_len]) |byte| { + if (byte != 0) return error.InvalidState; + } + return; + } + + if (partial_len >= exact_block_bytes) return error.InvalidState; + const occupied = getU64(encoded[28..36]); + const block_count = getU64(encoded[36..44]); + const total_bytes = getU64(encoded[44..52]); + if (occupied != block_count or + block_count != total_bytes / exact_block_bytes or + partial_len != total_bytes % exact_block_bytes) + { + return error.InvalidState; + } + + const next_u64 = getU64(encoded[92..100]); + const filled_u64 = getU64(encoded[100..108]); + if (ring_len == 0) { + if (next_u64 != 0 or filled_u64 != 0 or + getU64(encoded[76..84]) != 0 or has_last_sample) + { + return error.InvalidState; + } + } else { + if (next_u64 >= ring_len_u64 or filled_u64 > ring_len_u64) { + return error.InvalidState; + } + const expected_filled = @min(total_bytes, ring_len_u64); + const expected_next = total_bytes % ring_len_u64; + if (filled_u64 != expected_filled or next_u64 != expected_next) { + return error.InvalidState; + } + if (has_last_sample and + (total_bytes < ring_len_u64 or + getU64(encoded[84..92]) > total_bytes - ring_len_u64)) + { + return error.InvalidState; + } + } + if (!has_last_sample and getU64(encoded[84..92]) != 0) { + return error.InvalidState; + } +} + +fn checkpointChecksum(encoded: []const u8) [16]u8 { + var hash = Blake3.init(.{}); + hash.update("chi.dedup.checkpoint.v1\x00"); + hash.update(encoded[0..checkpoint_checksum_offset]); + hash.update(encoded[checkpoint_weights_offset..]); + var result: [16]u8 = undefined; + hash.final(&result); + return result; +} + +fn optionsFingerprint(options: Options, index_version: u64) u64 { + var hash: u64 = fnv_offset; + fingerprintValue(&hash, index_version); + fingerprintValue(&hash, @intFromBool(options.approximate.enabled)); + fingerprintValue(&hash, options.approximate.max_hamming_distance); + fingerprintValue(&hash, options.approximate.min_features); + fingerprintValue(&hash, @intFromBool(options.spans.enabled)); + fingerprintValue(&hash, options.spans.window_bytes); + fingerprintValue(&hash, options.spans.sampling_bits); + fingerprintValue(&hash, options.spans.min_start_distance); + return hash; +} + +fn fingerprintValue(hash: *u64, value: anytype) void { + const canonical: u64 = @intCast(value); + for (0..8) |index| { + hash.* ^= @as(u8, @truncate(canonical >> @intCast(index * 8))); + hash.* *%= fnv_prime; + } +} + +fn validateOptions(options: Options) !void { + if (options.approximate.max_hamming_distance > 3) { + return error.InvalidApproximateDistance; + } + if (options.spans.enabled and + (options.spans.window_bytes == 0 or options.spans.sampling_bits > 63)) + { + return error.InvalidSpanPolicy; + } +} + +fn isTokenByte(byte: u8) bool { + // TODO(data/unicode-tables): replace byte-level non-ASCII grouping with + // Unicode word segmentation and case folding once versioned tables exist. + return byte >= 0x80 or (byte >= 'a' and byte <= 'z') or + (byte >= 'A' and byte <= 'Z') or (byte >= '0' and byte <= '9') or + byte == '_'; +} + +fn mix64(input: u64) u64 { + var value = input; + value ^= value >> 30; + value *%= 0xbf58476d1ce4e5b9; + value ^= value >> 27; + value *%= 0x94d049bb133111eb; + return value ^ (value >> 31); +} + +fn bandKey(simhash: u64, band_index: usize) u32 { + const value: u16 = @truncate(simhash >> @intCast(band_index * 16)); + return (@as(u32, @intCast(band_index)) << 16) | value; +} + +fn hashLeaf(bytes: []const u8) ExactDigest { + var hash = Blake3.init(.{}); + hash.update("chi.dedup.leaf.v1\x00"); + var length: [8]u8 = undefined; + putU64(&length, bytes.len); + hash.update(&length); + hash.update(bytes); + var result: ExactDigest = undefined; + hash.final(&result); + return result; +} + +fn hashParent(left: ExactDigest, right: ExactDigest) ExactDigest { + var hash = Blake3.init(.{}); + hash.update("chi.dedup.parent.v1\x00"); + hash.update(&left); + hash.update(&right); + var result: ExactDigest = undefined; + hash.final(&result); + return result; +} + +fn shouldSample(state: SpanHasher, policy: SpanPolicy, start_byte: u64) bool { + if (policy.sampling_bits != 0) { + const mask = (@as(u64, 1) << @intCast(policy.sampling_bits)) - 1; + if (state.rolling & mask != 0) return false; + } + if (state.last_sample_start) |last| { + if (start_byte - last < policy.min_start_distance) return false; + } + return true; +} + +fn digestWindow(state: SpanHasher) SpanDigest { + var hash = Blake3.init(.{}); + hash.update("chi.dedup.span.v1\x00"); + var length: [8]u8 = undefined; + putU64(&length, state.ring.len); + hash.update(&length); + hash.update(state.ring[state.next..]); + hash.update(state.ring[0..state.next]); + var result: SpanDigest = undefined; + hash.final(&result); + return result; +} + +fn putU64(destination: *[8]u8, value: anytype) void { + const canonical: u64 = @intCast(value); + for (destination, 0..) |*byte, index| { + byte.* = @truncate(canonical >> @intCast(index * 8)); + } +} + +fn putU32(destination: *[4]u8, value: u32) void { + for (destination, 0..) |*byte, index| { + byte.* = @truncate(value >> @intCast(index * 8)); + } +} + +fn putU16(destination: *[2]u8, value: u16) void { + for (destination, 0..) |*byte, index| { + byte.* = @truncate(value >> @intCast(index * 8)); + } +} + +fn getU64(source: *const [8]u8) u64 { + var value: u64 = 0; + for (source, 0..) |byte, index| { + value |= @as(u64, byte) << @intCast(index * 8); + } + return value; +} + +fn getU32(source: *const [4]u8) u32 { + var value: u32 = 0; + for (source, 0..) |byte, index| { + value |= @as(u32, byte) << @intCast(index * 8); + } + return value; +} + +fn getU16(source: *const [2]u8) u16 { + var value: u16 = 0; + for (source, 0..) |byte, index| { + value |= @as(u16, byte) << @intCast(index * 8); + } + return value; +} + +test "exact fingerprints ignore fragment boundaries" { + const backend = try InMemoryIndex.create(std.testing.allocator); + const deduplicator = try Deduplicator.create( + std.testing.allocator, + backend.ownedIndex(), + .{ .spans = .{ .enabled = false } }, + ); + defer deduplicator.destroy(); + + var input: [exact_block_bytes * 2 + 137]u8 = undefined; + for (&input, 0..) |*byte, index| byte.* = @truncate(index *% 131 +% 17); + + const first = try deduplicator.observe(.{ + .document_id = "first", + .bytes = &input, + .boundary = .whole, + }); + try std.testing.expect(first.status.complete == .unique); + + _ = try deduplicator.observe(.{ + .document_id = "second", + .bytes = input[0..123], + .boundary = .{ .document_start = true }, + }); + _ = try deduplicator.observe(.{ + .document_id = "second", + .bytes = input[123..5001], + .boundary = .{}, + }); + const second = try deduplicator.observe(.{ + .document_id = "second", + .bytes = input[5001..], + .boundary = .{ .document_end = true }, + }); + switch (second.status.complete) { + .exact_duplicate => |match| try std.testing.expectEqualStrings("first", match.document_id), + else => return error.TestExpectedEqual, + } + try std.testing.expectEqual(@as(usize, 1), backend.documentCount()); +} + +test "approximate matches require enough features" { + const backend = try InMemoryIndex.create(std.testing.allocator); + const deduplicator = try Deduplicator.create( + std.testing.allocator, + backend.ownedIndex(), + .{ + .approximate = .{ .max_hamming_distance = 3, .min_features = 4 }, + .spans = .{ .enabled = false }, + }, + ); + defer deduplicator.destroy(); + + const original = try deduplicator.observe(.{ + .document_id = "original", + .bytes = "alpha beta gamma delta epsilon zeta eta theta", + .boundary = .whole, + }); + try std.testing.expect(original.status.complete == .unique); + const variant = try deduplicator.observe(.{ + .document_id = "variant", + .bytes = "ALPHA beta gamma delta epsilon zeta eta theta!", + .boundary = .whole, + }); + switch (variant.status.complete) { + .approximate_duplicate => |match| { + try std.testing.expectEqualStrings("original", match.document_id); + try std.testing.expectEqual(@as(u8, 0), match.hamming_distance); + }, + else => return error.TestExpectedEqual, + } +} + +test "duplicate spans are found across shifted text" { + const backend = try InMemoryIndex.create(std.testing.allocator); + const deduplicator = try Deduplicator.create( + std.testing.allocator, + backend.ownedIndex(), + .{ + .approximate = .{ .enabled = false }, + .spans = .{ + .window_bytes = 5, + .sampling_bits = 0, + .min_start_distance = 1, + }, + }, + ); + defer deduplicator.destroy(); + + _ = try deduplicator.observe(.{ + .document_id = "canonical", + .bytes = "abcdefghij", + .boundary = .whole, + }); + const shifted = try deduplicator.observe(.{ + .document_id = "shifted", + .bytes = "XXcdefgYY", + .boundary = .whole, + }); + try std.testing.expect(shifted.duplicate_spans.len >= 1); + const match = shifted.duplicate_spans[0]; + try std.testing.expectEqualStrings("canonical", match.canonical_document_id); + try std.testing.expectEqual(@as(u64, 2), match.current_start_byte); + try std.testing.expectEqual(@as(u64, 2), match.canonical_start_byte); +} + +test "failed and duplicate documents do not publish staged spans" { + const backend = try InMemoryIndex.create(std.testing.allocator); + const deduplicator = try Deduplicator.create( + std.testing.allocator, + backend.ownedIndex(), + .{ + .approximate = .{ .enabled = false }, + .spans = .{ .window_bytes = 4, .sampling_bits = 0, .min_start_distance = 1 }, + }, + ); + defer deduplicator.destroy(); + + _ = try deduplicator.observe(.{ + .document_id = "one", + .bytes = "abcdefgh", + .boundary = .whole, + }); + _ = try deduplicator.observe(.{ + .document_id = "duplicate", + .bytes = "abcdefgh", + .boundary = .whole, + }); + try std.testing.expectEqual(@as(usize, 1), backend.documentCount()); + + _ = try deduplicator.observe(.{ + .document_id = "aborted", + .bytes = "unique-prefix", + .boundary = .{ .document_start = true }, + }); + deduplicator.abortDocument(); + try std.testing.expectEqual(@as(usize, 1), backend.documentCount()); +} + +test "checkpoint transfers hashing span and backend transaction state" { + const backend = try InMemoryIndex.create(std.testing.allocator); + defer backend.destroy(); + const options = Options{ + .approximate = .{ .min_features = 4 }, + .spans = .{ .window_bytes = 5, .sampling_bits = 0, .min_start_distance = 1 }, + }; + + const source = try Deduplicator.create( + std.testing.allocator, + backend.borrowedIndex(), + options, + ); + defer source.destroy(); + _ = try source.observe(.{ + .document_id = "migrating", + .bytes = "alpha beta gam", + .boundary = .{ .document_start = true }, + }); + + const encoded = try source.checkpoint(std.testing.allocator); + defer std.testing.allocator.free(encoded); + const streaming_state = try StreamingState.decode(encoded); + const corrupted = try std.testing.allocator.dupe(u8, encoded); + defer std.testing.allocator.free(corrupted); + corrupted[corrupted.len - 1] ^= 1; + try std.testing.expectError(error.InvalidState, StreamingState.decode(corrupted)); + source.abortDocument(); + + const destination = try Deduplicator.create( + std.testing.allocator, + backend.borrowedIndex(), + options, + ); + defer destination.destroy(); + try destination.restore(streaming_state); + const completed = try destination.observe(.{ + .document_id = "migrating", + .bytes = "ma delta epsilon", + .boundary = .{ .document_end = true }, + }); + try std.testing.expect(completed.status.complete == .unique); + + const duplicate = try destination.observe(.{ + .document_id = "whole", + .bytes = "alpha beta gamma delta epsilon", + .boundary = .whole, + }); + switch (duplicate.status.complete) { + .exact_duplicate => |match| { + try std.testing.expectEqualStrings("migrating", match.document_id); + }, + else => return error.TestExpectedEqual, + } + + const incompatible = try Deduplicator.create( + std.testing.allocator, + backend.borrowedIndex(), + .{ .spans = .{ .enabled = false } }, + ); + defer incompatible.destroy(); + try std.testing.expectError( + error.CheckpointConfigurationMismatch, + incompatible.restore(streaming_state), + ); +} diff --git a/data/deduplicator/root.zig b/data/deduplicator/root.zig new file mode 100644 index 0000000..07c576e --- /dev/null +++ b/data/deduplicator/root.zig @@ -0,0 +1,28 @@ +//! Streaming exact, approximate, and text-span deduplication. + +const dedup = @import("deduplicator.zig"); + +pub const ApproximateMatch = dedup.ApproximateMatch; +pub const ApproximatePolicy = dedup.ApproximatePolicy; +pub const Boundary = dedup.Boundary; +pub const CanonicalMatch = dedup.CanonicalMatch; +pub const Classification = dedup.Classification; +pub const Deduplicator = dedup.Deduplicator; +pub const DocumentChunk = dedup.DocumentChunk; +pub const DocumentFingerprint = dedup.DocumentFingerprint; +pub const DuplicateSpan = dedup.DuplicateSpan; +pub const ExactDigest = dedup.ExactDigest; +pub const InMemoryIndex = dedup.InMemoryIndex; +pub const Index = dedup.Index; +pub const Observation = dedup.Observation; +pub const Options = dedup.Options; +pub const SpanDigest = dedup.SpanDigest; +pub const SpanFingerprint = dedup.SpanFingerprint; +pub const SpanOccurrence = dedup.SpanOccurrence; +pub const SpanPolicy = dedup.SpanPolicy; +pub const Status = dedup.Status; +pub const StreamingState = dedup.StreamingState; + +test { + _ = dedup; +} diff --git a/data/document-filter/README.md b/data/document-filter/README.md new file mode 100644 index 0000000..dd76e6e --- /dev/null +++ b/data/document-filter/README.md @@ -0,0 +1,27 @@ +# Document filter + +`DocumentFilter` is a bounded-memory document gate. It observes UTF-8 chunks, +returns early rejection when possible, and returns acceptance only after the +document-end boundary. The pipeline can therefore keep undecided bytes in a +spool or quarantine tier instead of buffering every document in this component. + +Built-in rules cover byte, codepoint, and line lengths; UTF-8 validity; visible, +replacement, and cleaning-change ratios; and repeated-codepoint runs. Threshold +ratios use integer basis points for deterministic behavior. + +Language identification, safety classification, and project-specific policies +use zero-copy `Rule` adapters. Rules are categorized for attribution, have +explicit lifecycle hooks, and can reject before the full document is consumed. +Stateful rules provide checkpoint/restore hooks; stateless rules need no token. + +Versioned checkpoints contain built-in metrics, partial UTF-8 sequences, sticky +rejections, and active rule tokens. Configuration fingerprints prevent a worker +from restoring state under different thresholds or rule versions. + +Decision document IDs and rejection details are borrowed until the next document +starts, the current document is aborted, or the filter is destroyed. + +TODOs for concrete language and safety model adapters remain until their model +inference/tokenization components define stable input and ownership contracts. + +Run all tests from the repository root with `zig build test`. diff --git a/data/document-filter/document_filter.zig b/data/document-filter/document_filter.zig new file mode 100644 index 0000000..5b265ff --- /dev/null +++ b/data/document-filter/document_filter.zig @@ -0,0 +1,1370 @@ +const std = @import("std"); + +const Allocator = std.mem.Allocator; +const ArenaAllocator = std.heap.ArenaAllocator; +const CorpusChunk = @import("../corpus-reader/root.zig").Chunk; +const CleanedChunk = @import("../text-cleaner/root.zig").CleanedChunk; + +pub const Boundary = struct { + document_start: bool = false, + document_end: bool = false, +}; + +/// Cleaning events are kept separate from cleaned bytes because destructive +/// cleaning removes the evidence needed by quality rules. +pub const CleaningSignals = struct { + malformed_sequences: u64 = 0, + controls_removed: u64 = 0, + format_characters_removed: u64 = 0, + whitespace_normalized: u64 = 0, + + pub fn fromCleanedChunk(chunk: CleanedChunk) CleaningSignals { + return .{ + .malformed_sequences = chunk.stats.malformed_sequences, + .controls_removed = chunk.stats.controls_removed, + .format_characters_removed = chunk.stats.format_characters_removed, + .whitespace_normalized = chunk.stats.whitespace_normalized, + }; + } +}; + +pub const DocumentChunk = struct { + document_id: []const u8, + bytes: []const u8, + boundary: Boundary, + cleaning: CleaningSignals = .{}, +}; + +pub const LengthPolicy = struct { + min_bytes: u64 = 1, + max_bytes: ?u64 = null, + min_codepoints: u64 = 1, + max_codepoints: ?u64 = null, + min_lines: ?u64 = null, + max_lines: ?u64 = null, +}; + +/// Ratios use basis points: 10_000 is 100%, 100 is 1%. Null disables a rule. +pub const QualityPolicy = struct { + require_valid_utf8: bool = true, + min_visible_basis_points: ?u16 = null, + max_replacement_basis_points: ?u16 = null, + /// Cleaning events (malformed sequences and removed control/format scalars) + /// divided by emitted UTF-8 bytes. + max_cleaning_change_basis_points: ?u16 = null, + max_repeated_codepoints: ?u64 = null, +}; + +pub const Options = struct { + length: LengthPolicy = .{}, + quality: QualityPolicy = .{}, +}; + +pub const Metrics = struct { + bytes: u64 = 0, + codepoints: u64 = 0, + newline_codepoints: u64 = 0, + visible_codepoints: u64 = 0, + replacement_codepoints: u64 = 0, + invalid_utf8_sequences: u64 = 0, + cleaning_changes: u64 = 0, + whitespace_normalized: u64 = 0, + longest_repeated_run: u64 = 0, + + pub fn lines(self: Metrics) u64 { + return if (self.codepoints == 0) 0 else self.newline_codepoints +| 1; + } +}; + +pub const Summary = struct { + document_id: []const u8, + metrics: Metrics, +}; + +pub const RejectionReason = enum(u8) { + bytes_below_minimum, + bytes_above_maximum, + codepoints_below_minimum, + codepoints_above_maximum, + lines_below_minimum, + lines_above_maximum, + invalid_utf8, + low_visible_ratio, + excessive_replacement_ratio, + excessive_cleaning_ratio, + repeated_codepoint_run, + rule, +}; + +pub const Rejection = struct { + document_id: []const u8, + reason: RejectionReason, + rule_id: ?[]const u8 = null, + rule_kind: ?RuleKind = null, + detail: []const u8, + metrics: Metrics, +}; + +pub const Decision = union(enum) { + pending: Summary, + accepted: Summary, + rejected: Rejection, +}; + +/// A validated view over an encoded filter checkpoint. The encoded bytes are +/// borrowed; `checkpoint` returns an owned byte slice for transport/storage. +pub const StreamingState = struct { + encoded: []const u8, + + pub const DecodeError = error{ + InvalidState, + UnsupportedVersion, + }; + + pub fn decode(encoded: []const u8) DecodeError!StreamingState { + try validateEncodedState(encoded); + return .{ .encoded = encoded }; + } +}; + +pub const RuleKind = enum(u8) { + language, + safety, + quality, + custom, +}; + +pub const RuleDecision = union(enum) { + keep, + reject: []const u8, +}; + +pub const RuleChunk = struct { + document_id: []const u8, + bytes: []const u8, + boundary: Boundary, + cleaning: CleaningSignals, + metrics: Metrics, +}; + +/// Adapter boundary for language identification, safety classifiers, and +/// project-specific filters. The filter owns the adapter lifecycle after a +/// successful `create` call. +// TODO(data/classification): add concrete language-ID and safety adapters once +// tokenizer/inference components define stable streaming input contracts. +pub const Rule = struct { + id: []const u8, + kind: RuleKind, + context: *anyopaque, + vtable: *const VTable, + version: u64 = 1, + stateful: bool = false, + + pub const VTable = struct { + evaluate: *const fn (context: *anyopaque, chunk: RuleChunk) anyerror!RuleDecision, + abort: ?*const fn (context: *anyopaque) void = null, + checkpoint: ?*const fn (context: *anyopaque, allocator: Allocator) anyerror![]const u8 = null, + restore: ?*const fn (context: *anyopaque, token: []const u8) anyerror!void = null, + deinit: ?*const fn (context: *anyopaque) void = null, + }; +}; + +const FilterState = struct { + in_document: bool = false, + document_id: []const u8 = "", + metrics: Metrics = .{}, + rejection: ?StoredRejection = null, + active_rule_count: usize = 0, + chunks: u64 = 0, + + utf8_pending: [4]u8 = @splat(0), + utf8_pending_len: u3 = 0, + utf8_expected_len: u3 = 0, + + last_visible_codepoint: ?u21 = null, + current_repeated_run: u64 = 0, +}; + +const StoredRejection = struct { + reason: RejectionReason, + rule_id: ?[]const u8, + rule_kind: ?RuleKind, + detail: []const u8, +}; + +pub const DocumentFilter = struct { + backing_allocator: Allocator, + options: Options, + source_arena: ArenaAllocator, + document_arena: ArenaAllocator, + rules: []Rule, + state: FilterState = .{}, + + pub const CreateError = Allocator.Error || error{ + DuplicateRuleId, + EmptyRuleId, + InvalidLengthPolicy, + InvalidQualityRatio, + }; + + pub fn create( + backing_allocator: Allocator, + options: Options, + input_rules: []const Rule, + ) CreateError!*DocumentFilter { + try validateOptions(options); + + const self = try backing_allocator.create(DocumentFilter); + self.* = .{ + .backing_allocator = backing_allocator, + .options = options, + .source_arena = .init(backing_allocator), + .document_arena = .init(backing_allocator), + .rules = &.{}, + }; + errdefer { + self.document_arena.deinit(); + self.source_arena.deinit(); + backing_allocator.destroy(self); + } + + const rules = try self.source_arena.allocator().alloc(Rule, input_rules.len); + for (input_rules, 0..) |input_rule, index| { + if (input_rule.id.len == 0) return error.EmptyRuleId; + for (rules[0..index]) |previous| { + if (std.mem.eql(u8, previous.id, input_rule.id)) { + return error.DuplicateRuleId; + } + } + rules[index] = input_rule; + rules[index].id = try self.source_arena.allocator().dupe(u8, input_rule.id); + } + self.rules = rules; + return self; + } + + pub fn destroy(self: *DocumentFilter) void { + if (self.state.active_rule_count != 0) self.abortRules(); + for (self.rules) |rule| { + if (rule.vtable.deinit) |deinitFn| deinitFn(rule.context); + } + self.document_arena.deinit(); + self.source_arena.deinit(); + const backing_allocator = self.backing_allocator; + backing_allocator.destroy(self); + } + + /// Observes one UTF-8 document fragment. Acceptance is only returned at an + /// end boundary; rejection may happen earlier and remains sticky while the + /// rest of that document is drained. + pub fn observe(self: *DocumentFilter, chunk: DocumentChunk) anyerror!Decision { + if (chunk.boundary.document_start) { + if (self.state.in_document) return error.DocumentAlreadyStarted; + try self.startDocument(chunk.document_id); + } else { + if (!self.state.in_document) return error.DocumentNotStarted; + if (!std.mem.eql(u8, self.state.document_id, chunk.document_id)) { + return error.DocumentIdChanged; + } + } + + if (self.state.rejection == null) { + try self.measureChunk(chunk); + if (self.state.rejection == null) try self.evaluateRules(chunk); + } + self.state.chunks = try addU64(self.state.chunks, 1); + + if (chunk.boundary.document_end) { + if (self.state.rejection == null) { + try self.finishUtf8(); + if (self.state.rejection == null) try self.applyFinalPolicies(); + } + self.state.in_document = false; + self.state.active_rule_count = 0; + return self.finalDecision(); + } + return self.currentDecision(.pending); + } + + pub fn observeCorpusChunk(self: *DocumentFilter, chunk: CorpusChunk) anyerror!Decision { + return self.observe(.{ + .document_id = chunk.document_id, + .bytes = chunk.bytes, + .boundary = .{ + .document_start = chunk.is_document_start, + .document_end = chunk.is_document_end, + }, + }); + } + + pub fn observeCleanedChunk( + self: *DocumentFilter, + document_id: []const u8, + chunk: CleanedChunk, + ) anyerror!Decision { + return self.observe(.{ + .document_id = document_id, + .bytes = chunk.bytes, + .boundary = .{ + .document_start = chunk.document_start, + .document_end = chunk.document_end, + }, + .cleaning = .fromCleanedChunk(chunk), + }); + } + + /// Encodes built-in metrics, partial UTF-8 state, sticky rejection, and all + /// active stateful rule tokens. The caller owns the returned bytes. + pub fn checkpoint( + self: *DocumentFilter, + allocator: Allocator, + ) anyerror![]u8 { + var scratch: ArenaAllocator = .init(self.backing_allocator); + defer scratch.deinit(); + + const active_count = if (self.state.in_document) + self.state.active_rule_count + else + 0; + const tokens = try scratch.allocator().alloc([]const u8, active_count); + var token_bytes: usize = 0; + for (self.rules[0..active_count], 0..) |rule, index| { + if (rule.stateful) { + const checkpointFn = rule.vtable.checkpoint orelse + return error.RuleCheckpointUnsupported; + tokens[index] = try checkpointFn(rule.context, scratch.allocator()); + } else { + tokens[index] = ""; + } + token_bytes = std.math.add(usize, token_bytes, 4) catch + return error.StateTooLarge; + token_bytes = std.math.add(usize, token_bytes, tokens[index].len) catch + return error.StateTooLarge; + if (tokens[index].len > std.math.maxInt(u32)) return error.StateTooLarge; + } + + const document_id = if (self.state.in_document) self.state.document_id else ""; + const stored_rejection = if (self.state.in_document) self.state.rejection else null; + const rejection_rule_id = if (stored_rejection) |stored| + stored.rule_id orelse "" + else + ""; + const rejection_detail = if (stored_rejection) |stored| stored.detail else ""; + if (document_id.len > std.math.maxInt(u32) or + rejection_rule_id.len > std.math.maxInt(u32) or + rejection_detail.len > std.math.maxInt(u32) or + active_count > std.math.maxInt(u32)) + { + return error.StateTooLarge; + } + + var total: usize = checkpoint_header_len; + total = std.math.add(usize, total, document_id.len) catch return error.StateTooLarge; + total = std.math.add(usize, total, rejection_rule_id.len) catch return error.StateTooLarge; + total = std.math.add(usize, total, rejection_detail.len) catch return error.StateTooLarge; + total = std.math.add(usize, total, token_bytes) catch return error.StateTooLarge; + + const encoded = try allocator.alloc(u8, total); + @memset(encoded, 0); + @memcpy(encoded[0..4], checkpoint_magic); + putU64(encoded[108..116], self.configurationFingerprint()); + putU32(encoded[116..120], @intCast(document_id.len)); + putU32(encoded[120..124], @intCast(rejection_rule_id.len)); + putU32(encoded[124..128], @intCast(rejection_detail.len)); + putU32(encoded[128..132], @intCast(active_count)); + + if (self.state.in_document) { + var flags: u8 = 1 << 0; + if (stored_rejection != null) flags |= 1 << 1; + if (self.state.last_visible_codepoint != null) flags |= 1 << 2; + encoded[4] = flags; + encoded[5] = if (stored_rejection) |stored| + @intFromEnum(stored.reason) + 1 + else + 0; + encoded[6] = self.state.utf8_pending_len; + encoded[7] = self.state.utf8_expected_len; + putU32(encoded[8..12], @intCast(active_count)); + putU64(encoded[12..20], self.state.chunks); + putU64(encoded[20..28], self.state.current_repeated_run); + encodeMetrics(encoded[28..100], self.state.metrics); + putU32( + encoded[100..104], + if (self.state.last_visible_codepoint) |codepoint| codepoint else 0, + ); + @memcpy( + encoded[104..][0..self.state.utf8_pending_len], + self.state.utf8_pending[0..self.state.utf8_pending_len], + ); + } + + var cursor: usize = checkpoint_header_len; + @memcpy(encoded[cursor..][0..document_id.len], document_id); + cursor += document_id.len; + @memcpy(encoded[cursor..][0..rejection_rule_id.len], rejection_rule_id); + cursor += rejection_rule_id.len; + @memcpy(encoded[cursor..][0..rejection_detail.len], rejection_detail); + cursor += rejection_detail.len; + for (tokens) |token| { + putU32(encoded[cursor..][0..4], @intCast(token.len)); + cursor += 4; + @memcpy(encoded[cursor..][0..token.len], token); + cursor += token.len; + } + std.debug.assert(cursor == encoded.len); + validateEncodedState(encoded) catch unreachable; + return encoded; + } + + /// Restores a checkpoint on a worker with identical policy and rule IDs, + /// versions, ordering, and statefulness declarations. + pub fn restore( + self: *DocumentFilter, + streaming_state: StreamingState, + ) anyerror!void { + try validateEncodedState(streaming_state.encoded); + const stable = try self.backing_allocator.dupe(u8, streaming_state.encoded); + defer self.backing_allocator.free(stable); + if (getU64(stable[108..116]) != self.configurationFingerprint()) { + return error.CheckpointConfigurationMismatch; + } + + if (self.state.active_rule_count != 0) self.abortRules(); + _ = self.document_arena.reset(.retain_capacity); + self.state = .{}; + + if (stable[4] & (1 << 0) == 0) return; + const active_count: usize = getU32(stable[8..12]); + if (active_count > self.rules.len) return error.InvalidState; + + const document_len: usize = getU32(stable[116..120]); + const rule_id_len: usize = getU32(stable[120..124]); + const detail_len: usize = getU32(stable[124..128]); + var cursor: usize = checkpoint_header_len; + const document_id = stable[cursor..][0..document_len]; + cursor += document_len; + const rejection_rule_id = stable[cursor..][0..rule_id_len]; + cursor += rule_id_len; + const rejection_detail = stable[cursor..][0..detail_len]; + cursor += detail_len; + + var restored_rules: usize = 0; + errdefer { + for (self.rules[0..restored_rules]) |rule| { + if (rule.vtable.abort) |abortFn| abortFn(rule.context); + } + self.state = .{}; + } + for (self.rules[0..active_count]) |rule| { + const token_len: usize = getU32(stable[cursor..][0..4]); + cursor += 4; + const token = stable[cursor..][0..token_len]; + cursor += token_len; + if (rule.stateful) { + const restoreFn = rule.vtable.restore orelse + return error.RuleRestoreUnsupported; + try restoreFn(rule.context, token); + } else if (token.len != 0) { + return error.InvalidState; + } + restored_rules += 1; + } + + self.state = .{ + .in_document = true, + .document_id = try self.document_arena.allocator().dupe(u8, document_id), + .metrics = decodeMetrics(stable[28..100]), + .rejection = if (stable[4] & (1 << 1) != 0) .{ + .reason = (decodeRejectionReason(stable[5]) catch unreachable).?, + .rule_id = if (rule_id_len == 0) + null + else + try self.document_arena.allocator().dupe(u8, rejection_rule_id), + .rule_kind = if (rule_id_len == 0) + null + else + self.ruleKindForId(rejection_rule_id) orelse return error.InvalidState, + .detail = try self.document_arena.allocator().dupe(u8, rejection_detail), + } else null, + .active_rule_count = active_count, + .chunks = getU64(stable[12..20]), + .utf8_pending_len = @intCast(stable[6]), + .utf8_expected_len = @intCast(stable[7]), + .last_visible_codepoint = if (stable[4] & (1 << 2) != 0) + @intCast(getU32(stable[100..104])) + else + null, + .current_repeated_run = getU64(stable[20..28]), + }; + @memcpy(&self.state.utf8_pending, stable[104..108]); + } + + pub fn abortDocument(self: *DocumentFilter) void { + if (self.state.active_rule_count != 0) self.abortRules(); + self.state = .{}; + _ = self.document_arena.reset(.retain_capacity); + } + + fn startDocument(self: *DocumentFilter, document_id: []const u8) Allocator.Error!void { + _ = self.document_arena.reset(.retain_capacity); + self.state = .{ + .in_document = true, + .document_id = try self.document_arena.allocator().dupe(u8, document_id), + }; + } + + fn measureChunk(self: *DocumentFilter, chunk: DocumentChunk) anyerror!void { + self.state.metrics.bytes = try addU64(self.state.metrics.bytes, chunk.bytes.len); + self.state.metrics.cleaning_changes = try addU64( + self.state.metrics.cleaning_changes, + try cleaningChanges(chunk.cleaning), + ); + self.state.metrics.whitespace_normalized = try addU64( + self.state.metrics.whitespace_normalized, + chunk.cleaning.whitespace_normalized, + ); + + if (self.options.length.max_bytes) |maximum| { + if (self.state.metrics.bytes > maximum) { + try self.rejectBuiltIn(.bytes_above_maximum, "document exceeds max_bytes"); + return; + } + } + + try self.measureUtf8(chunk.bytes); + if (self.state.rejection != null) return; + + if (self.options.length.max_codepoints) |maximum| { + if (self.state.metrics.codepoints > maximum) { + try self.rejectBuiltIn( + .codepoints_above_maximum, + "document exceeds max_codepoints", + ); + return; + } + } + if (self.options.length.max_lines) |maximum| { + if (self.state.metrics.lines() > maximum) { + try self.rejectBuiltIn(.lines_above_maximum, "document exceeds max_lines"); + return; + } + } + if (self.options.quality.max_repeated_codepoints) |maximum| { + if (self.state.metrics.longest_repeated_run > maximum) { + try self.rejectBuiltIn( + .repeated_codepoint_run, + "visible codepoint run exceeds quality limit", + ); + } + } + } + + fn measureUtf8(self: *DocumentFilter, bytes: []const u8) anyerror!void { + var index: usize = 0; + while (index < bytes.len and self.state.rejection == null) { + const byte = bytes[index]; + if (self.state.utf8_pending_len == 0) { + if (byte >= 0x21 and byte <= 0x7E) { + const run_start = index; + index += 1; + while (index < bytes.len and + bytes[index] >= 0x21 and bytes[index] <= 0x7E) + { + index += 1; + } + try self.measureAsciiRun(bytes[run_start..index]); + continue; + } + if (byte < 0x80) { + try self.measureCodepoint(byte); + index += 1; + continue; + } + const expected = std.unicode.utf8ByteSequenceLength(byte) catch { + try self.recordInvalidUtf8(); + index += 1; + continue; + }; + self.state.utf8_pending[0] = byte; + self.state.utf8_pending_len = 1; + self.state.utf8_expected_len = expected; + index += 1; + continue; + } + + if (byte & 0b1100_0000 != 0b1000_0000) { + try self.recordInvalidUtf8(); + self.state.utf8_pending_len = 0; + self.state.utf8_expected_len = 0; + continue; + } + + self.state.utf8_pending[self.state.utf8_pending_len] = byte; + self.state.utf8_pending_len += 1; + index += 1; + if (self.state.utf8_pending_len == self.state.utf8_expected_len) { + const sequence = self.state.utf8_pending[0..self.state.utf8_pending_len]; + const codepoint = std.unicode.utf8Decode(sequence) catch { + try self.recordInvalidUtf8(); + self.state.utf8_pending_len = 0; + self.state.utf8_expected_len = 0; + continue; + }; + self.state.utf8_pending_len = 0; + self.state.utf8_expected_len = 0; + try self.measureCodepoint(codepoint); + } + } + } + + fn measureAsciiRun(self: *DocumentFilter, bytes: []const u8) anyerror!void { + self.state.metrics.codepoints = try addU64(self.state.metrics.codepoints, bytes.len); + self.state.metrics.visible_codepoints = try addU64( + self.state.metrics.visible_codepoints, + bytes.len, + ); + + var index: usize = 0; + while (index < bytes.len) { + const codepoint: u21 = bytes[index]; + var end = index + 1; + while (end < bytes.len and bytes[end] == bytes[index]) end += 1; + const run_len: u64 = @intCast(end - index); + if (self.state.last_visible_codepoint == codepoint) { + self.state.current_repeated_run = try addU64( + self.state.current_repeated_run, + run_len, + ); + } else { + self.state.last_visible_codepoint = codepoint; + self.state.current_repeated_run = run_len; + } + self.state.metrics.longest_repeated_run = @max( + self.state.metrics.longest_repeated_run, + self.state.current_repeated_run, + ); + index = end; + } + } + + fn measureCodepoint(self: *DocumentFilter, codepoint: u21) anyerror!void { + self.state.metrics.codepoints = try addU64(self.state.metrics.codepoints, 1); + if (codepoint == '\n') { + self.state.metrics.newline_codepoints = try addU64( + self.state.metrics.newline_codepoints, + 1, + ); + } + if (codepoint == 0xFFFD) { + self.state.metrics.replacement_codepoints = try addU64( + self.state.metrics.replacement_codepoints, + 1, + ); + } + + if (isVisible(codepoint)) { + self.state.metrics.visible_codepoints = try addU64( + self.state.metrics.visible_codepoints, + 1, + ); + if (self.state.last_visible_codepoint == codepoint) { + self.state.current_repeated_run = try addU64( + self.state.current_repeated_run, + 1, + ); + } else { + self.state.last_visible_codepoint = codepoint; + self.state.current_repeated_run = 1; + } + self.state.metrics.longest_repeated_run = @max( + self.state.metrics.longest_repeated_run, + self.state.current_repeated_run, + ); + } else { + self.state.last_visible_codepoint = null; + self.state.current_repeated_run = 0; + } + } + + fn recordInvalidUtf8(self: *DocumentFilter) anyerror!void { + self.state.metrics.invalid_utf8_sequences = try addU64( + self.state.metrics.invalid_utf8_sequences, + 1, + ); + if (self.options.quality.require_valid_utf8) { + try self.rejectBuiltIn(.invalid_utf8, "document is not valid UTF-8"); + } + } + + fn finishUtf8(self: *DocumentFilter) anyerror!void { + if (self.state.utf8_pending_len != 0) { + self.state.utf8_pending_len = 0; + self.state.utf8_expected_len = 0; + try self.recordInvalidUtf8(); + } + } + + fn evaluateRules(self: *DocumentFilter, chunk: DocumentChunk) anyerror!void { + if (self.rules.len == 0) return; + const rule_chunk: RuleChunk = .{ + .document_id = self.state.document_id, + .bytes = chunk.bytes, + .boundary = chunk.boundary, + .cleaning = chunk.cleaning, + .metrics = self.state.metrics, + }; + for (self.rules, 0..) |rule, index| { + self.state.active_rule_count = @max(self.state.active_rule_count, index + 1); + switch (try rule.vtable.evaluate(rule.context, rule_chunk)) { + .keep => {}, + .reject => |detail| { + try self.rejectRule(rule, detail); + return; + }, + } + } + if (chunk.boundary.document_end) self.state.active_rule_count = 0; + } + + fn applyFinalPolicies(self: *DocumentFilter) anyerror!void { + const metrics = self.state.metrics; + const length = self.options.length; + if (metrics.bytes < length.min_bytes) { + try self.rejectBuiltIn(.bytes_below_minimum, "document is below min_bytes"); + return; + } + if (metrics.codepoints < length.min_codepoints) { + try self.rejectBuiltIn( + .codepoints_below_minimum, + "document is below min_codepoints", + ); + return; + } + if (length.min_lines) |minimum| { + if (metrics.lines() < minimum) { + try self.rejectBuiltIn(.lines_below_minimum, "document is below min_lines"); + return; + } + } + + const quality = self.options.quality; + if (quality.min_visible_basis_points) |minimum| { + if (!ratioAtLeast(metrics.visible_codepoints, metrics.codepoints, minimum)) { + try self.rejectBuiltIn(.low_visible_ratio, "visible ratio is below quality limit"); + return; + } + } + if (quality.max_replacement_basis_points) |maximum| { + if (!ratioAtMost(metrics.replacement_codepoints, metrics.codepoints, maximum)) { + try self.rejectBuiltIn( + .excessive_replacement_ratio, + "replacement ratio exceeds quality limit", + ); + return; + } + } + if (quality.max_cleaning_change_basis_points) |maximum| { + if (!ratioAtMost(metrics.cleaning_changes, metrics.bytes, maximum)) { + try self.rejectBuiltIn( + .excessive_cleaning_ratio, + "cleaning-change ratio exceeds quality limit", + ); + } + } + } + + fn rejectBuiltIn( + self: *DocumentFilter, + reason: RejectionReason, + detail: []const u8, + ) Allocator.Error!void { + self.state.rejection = .{ + .reason = reason, + .rule_id = null, + .rule_kind = null, + .detail = detail, + }; + if (self.state.active_rule_count != 0) self.abortRules(); + } + + fn rejectRule( + self: *DocumentFilter, + rule: Rule, + detail: []const u8, + ) Allocator.Error!void { + self.state.rejection = .{ + .reason = .rule, + .rule_id = rule.id, + .rule_kind = rule.kind, + .detail = try self.document_arena.allocator().dupe(u8, detail), + }; + self.abortRules(); + } + + fn abortRules(self: *DocumentFilter) void { + for (self.rules[0..self.state.active_rule_count]) |rule| { + if (rule.vtable.abort) |abortFn| abortFn(rule.context); + } + self.state.active_rule_count = 0; + } + + fn currentDecision(self: *const DocumentFilter, comptime tag: enum { pending }) Decision { + _ = tag; + if (self.state.rejection) |stored| return .{ .rejected = self.makeRejection(stored) }; + return .{ .pending = self.summary() }; + } + + fn finalDecision(self: *const DocumentFilter) Decision { + if (self.state.rejection) |stored| return .{ .rejected = self.makeRejection(stored) }; + return .{ .accepted = self.summary() }; + } + + fn summary(self: *const DocumentFilter) Summary { + return .{ + .document_id = self.state.document_id, + .metrics = self.state.metrics, + }; + } + + fn makeRejection(self: *const DocumentFilter, stored: StoredRejection) Rejection { + return .{ + .document_id = self.state.document_id, + .reason = stored.reason, + .rule_id = stored.rule_id, + .rule_kind = stored.rule_kind, + .detail = stored.detail, + .metrics = self.state.metrics, + }; + } + + fn configurationFingerprint(self: *const DocumentFilter) u64 { + var hash: u64 = 0xcbf29ce484222325; + fingerprintOptional(&hash, self.options.length.max_bytes); + fingerprintValue(&hash, self.options.length.min_bytes); + fingerprintOptional(&hash, self.options.length.max_codepoints); + fingerprintValue(&hash, self.options.length.min_codepoints); + fingerprintOptional(&hash, self.options.length.min_lines); + fingerprintOptional(&hash, self.options.length.max_lines); + fingerprintValue(&hash, @intFromBool(self.options.quality.require_valid_utf8)); + fingerprintOptional(&hash, self.options.quality.min_visible_basis_points); + fingerprintOptional(&hash, self.options.quality.max_replacement_basis_points); + fingerprintOptional(&hash, self.options.quality.max_cleaning_change_basis_points); + fingerprintOptional(&hash, self.options.quality.max_repeated_codepoints); + fingerprintValue(&hash, self.rules.len); + for (self.rules) |rule| { + fingerprintBytes(&hash, rule.id); + fingerprintValue(&hash, @intFromEnum(rule.kind)); + fingerprintValue(&hash, rule.version); + fingerprintValue(&hash, @intFromBool(rule.stateful)); + } + return hash; + } + + fn ruleKindForId(self: *const DocumentFilter, id: []const u8) ?RuleKind { + for (self.rules) |rule| { + if (std.mem.eql(u8, rule.id, id)) return rule.kind; + } + return null; + } +}; + +const checkpoint_magic = "DFL1"; +const checkpoint_header_len = 132; + +fn validateEncodedState(encoded: []const u8) StreamingState.DecodeError!void { + if (encoded.len < checkpoint_header_len) return error.InvalidState; + if (!std.mem.eql(u8, encoded[0..4], checkpoint_magic)) { + return error.UnsupportedVersion; + } + if (encoded[4] & ~@as(u8, 0b0000_0111) != 0) return error.InvalidState; + + const in_document = encoded[4] & (1 << 0) != 0; + const rejected = encoded[4] & (1 << 1) != 0; + const has_last_codepoint = encoded[4] & (1 << 2) != 0; + const reason = try decodeRejectionReason(encoded[5]); + const pending_len = encoded[6]; + const expected_len = encoded[7]; + const active_count = getU32(encoded[8..12]); + const current_run = getU64(encoded[20..28]); + const last_codepoint = getU32(encoded[100..104]); + const document_len: usize = getU32(encoded[116..120]); + const rule_id_len: usize = getU32(encoded[120..124]); + const detail_len: usize = getU32(encoded[124..128]); + const token_count = getU32(encoded[128..132]); + + if (active_count != token_count) return error.InvalidState; + if (pending_len > 3 or expected_len > 4) return error.InvalidState; + if ((pending_len == 0) != (expected_len == 0)) return error.InvalidState; + if (pending_len != 0 and (expected_len < 2 or pending_len >= expected_len)) { + return error.InvalidState; + } + for (encoded[104 + pending_len .. 108]) |byte| { + if (byte != 0) return error.InvalidState; + } + if (pending_len != 0) { + const expected = std.unicode.utf8ByteSequenceLength(encoded[104]) catch + return error.InvalidState; + if (expected != expected_len) return error.InvalidState; + for (encoded[105 .. 104 + pending_len]) |byte| { + if (byte & 0b1100_0000 != 0b1000_0000) return error.InvalidState; + } + } + if (has_last_codepoint) { + if (last_codepoint > 0x10FFFF or + !std.unicode.utf8ValidCodepoint(@intCast(last_codepoint)) or + current_run == 0) + { + return error.InvalidState; + } + } else if (last_codepoint != 0 or current_run != 0) { + return error.InvalidState; + } + if (rejected != (reason != null)) return error.InvalidState; + if (!rejected and (rule_id_len != 0 or detail_len != 0)) return error.InvalidState; + if (reason == .rule and rule_id_len == 0) return error.InvalidState; + if (reason != null and reason != .rule and rule_id_len != 0) return error.InvalidState; + if (rejected and active_count != 0) return error.InvalidState; + + const metrics = decodeMetrics(encoded[28..100]); + if (metrics.newline_codepoints > metrics.codepoints or + metrics.codepoints > metrics.bytes or + metrics.visible_codepoints > metrics.codepoints or + metrics.replacement_codepoints > metrics.codepoints or + metrics.longest_repeated_run > metrics.visible_codepoints or + current_run > metrics.longest_repeated_run) + { + return error.InvalidState; + } + + var cursor = std.math.add(usize, checkpoint_header_len, document_len) catch + return error.InvalidState; + cursor = std.math.add(usize, cursor, rule_id_len) catch return error.InvalidState; + cursor = std.math.add(usize, cursor, detail_len) catch return error.InvalidState; + if (cursor > encoded.len) return error.InvalidState; + for (0..token_count) |_| { + const length_end = std.math.add(usize, cursor, 4) catch return error.InvalidState; + if (length_end > encoded.len) return error.InvalidState; + const token_len: usize = getU32(encoded[cursor..][0..4]); + cursor = std.math.add(usize, length_end, token_len) catch return error.InvalidState; + if (cursor > encoded.len) return error.InvalidState; + } + if (cursor != encoded.len) return error.InvalidState; + + if (!in_document) { + if (encoded[4] != 0 or encoded[5] != 0 or pending_len != 0 or + expected_len != 0 or active_count != 0 or document_len != 0 or + rule_id_len != 0 or detail_len != 0) + { + return error.InvalidState; + } + for (encoded[8..108]) |byte| { + if (byte != 0) return error.InvalidState; + } + } else if (getU64(encoded[12..20]) == 0) { + return error.InvalidState; + } +} + +fn encodeMetrics(destination: *[72]u8, metrics: Metrics) void { + putU64(destination[0..8], metrics.bytes); + putU64(destination[8..16], metrics.codepoints); + putU64(destination[16..24], metrics.newline_codepoints); + putU64(destination[24..32], metrics.visible_codepoints); + putU64(destination[32..40], metrics.replacement_codepoints); + putU64(destination[40..48], metrics.invalid_utf8_sequences); + putU64(destination[48..56], metrics.cleaning_changes); + putU64(destination[56..64], metrics.whitespace_normalized); + putU64(destination[64..72], metrics.longest_repeated_run); +} + +fn decodeMetrics(source: *const [72]u8) Metrics { + return .{ + .bytes = getU64(source[0..8]), + .codepoints = getU64(source[8..16]), + .newline_codepoints = getU64(source[16..24]), + .visible_codepoints = getU64(source[24..32]), + .replacement_codepoints = getU64(source[32..40]), + .invalid_utf8_sequences = getU64(source[40..48]), + .cleaning_changes = getU64(source[48..56]), + .whitespace_normalized = getU64(source[56..64]), + .longest_repeated_run = getU64(source[64..72]), + }; +} + +fn decodeRejectionReason(encoded: u8) StreamingState.DecodeError!?RejectionReason { + if (encoded == 0) return null; + return switch (encoded - 1) { + @intFromEnum(RejectionReason.bytes_below_minimum) => .bytes_below_minimum, + @intFromEnum(RejectionReason.bytes_above_maximum) => .bytes_above_maximum, + @intFromEnum(RejectionReason.codepoints_below_minimum) => .codepoints_below_minimum, + @intFromEnum(RejectionReason.codepoints_above_maximum) => .codepoints_above_maximum, + @intFromEnum(RejectionReason.lines_below_minimum) => .lines_below_minimum, + @intFromEnum(RejectionReason.lines_above_maximum) => .lines_above_maximum, + @intFromEnum(RejectionReason.invalid_utf8) => .invalid_utf8, + @intFromEnum(RejectionReason.low_visible_ratio) => .low_visible_ratio, + @intFromEnum(RejectionReason.excessive_replacement_ratio) => .excessive_replacement_ratio, + @intFromEnum(RejectionReason.excessive_cleaning_ratio) => .excessive_cleaning_ratio, + @intFromEnum(RejectionReason.repeated_codepoint_run) => .repeated_codepoint_run, + @intFromEnum(RejectionReason.rule) => .rule, + else => error.InvalidState, + }; +} + +fn fingerprintOptional(hash: *u64, value: anytype) void { + if (value) |present| { + fingerprintValue(hash, 1); + fingerprintValue(hash, present); + } else { + fingerprintValue(hash, 0); + } +} + +fn fingerprintValue(hash: *u64, value: anytype) void { + const canonical: u64 = @intCast(value); + for (0..8) |index| { + hash.* ^= @as(u8, @truncate(canonical >> @intCast(index * 8))); + hash.* *%= 0x100000001b3; + } +} + +fn fingerprintBytes(hash: *u64, bytes: []const u8) void { + fingerprintValue(hash, bytes.len); + for (bytes) |byte| { + hash.* ^= byte; + hash.* *%= 0x100000001b3; + } +} + +fn putU32(destination: *[4]u8, value: u32) void { + for (destination, 0..) |*byte, index| { + byte.* = @truncate(value >> @intCast(index * 8)); + } +} + +fn putU64(destination: *[8]u8, value: u64) void { + for (destination, 0..) |*byte, index| { + byte.* = @truncate(value >> @intCast(index * 8)); + } +} + +fn getU32(source: *const [4]u8) u32 { + var value: u32 = 0; + for (source, 0..) |byte, index| { + value |= @as(u32, byte) << @intCast(index * 8); + } + return value; +} + +fn getU64(source: *const [8]u8) u64 { + var value: u64 = 0; + for (source, 0..) |byte, index| { + value |= @as(u64, byte) << @intCast(index * 8); + } + return value; +} + +fn validateOptions(options: Options) DocumentFilter.CreateError!void { + const length = options.length; + if (length.max_bytes) |maximum| { + if (maximum < length.min_bytes) return error.InvalidLengthPolicy; + } + if (length.max_codepoints) |maximum| { + if (maximum < length.min_codepoints) return error.InvalidLengthPolicy; + } + if (length.min_lines) |minimum| { + if (length.max_lines) |maximum| { + if (maximum < minimum) return error.InvalidLengthPolicy; + } + } + + const quality = options.quality; + inline for (.{ + quality.min_visible_basis_points, + quality.max_replacement_basis_points, + quality.max_cleaning_change_basis_points, + }) |threshold| { + if (threshold) |value| { + if (value > 10_000) return error.InvalidQualityRatio; + } + } +} + +fn cleaningChanges(signals: CleaningSignals) error{CounterOverflow}!u64 { + var total = try addU64(signals.malformed_sequences, signals.controls_removed); + total = try addU64(total, signals.format_characters_removed); + return total; +} + +fn addU64(left: u64, right: anytype) error{CounterOverflow}!u64 { + const right_u64 = std.math.cast(u64, right) orelse return error.CounterOverflow; + return std.math.add(u64, left, right_u64) catch error.CounterOverflow; +} + +fn ratioAtLeast(numerator: u64, denominator: u64, basis_points: u16) bool { + if (denominator == 0) return numerator == 0 and basis_points == 0; + return @as(u128, numerator) * 10_000 >= + @as(u128, denominator) * basis_points; +} + +fn ratioAtMost(numerator: u64, denominator: u64, basis_points: u16) bool { + if (denominator == 0) return numerator == 0; + return @as(u128, numerator) * 10_000 <= + @as(u128, denominator) * basis_points; +} + +fn isVisible(codepoint: u21) bool { + return !isWhitespace(codepoint) and !isControl(codepoint) and !isFormat(codepoint); +} + +fn isWhitespace(codepoint: u21) bool { + return switch (codepoint) { + 0x0009...0x000D, 0x0020, 0x0085, 0x00A0, 0x1680, 0x2028, 0x2029, 0x202F, 0x205F, 0x3000 => true, + 0x2000...0x200A => true, + else => false, + }; +} + +fn isControl(codepoint: u21) bool { + return codepoint <= 0x001F or (codepoint >= 0x007F and codepoint <= 0x009F); +} + +fn isFormat(codepoint: u21) bool { + return switch (codepoint) { + 0x00AD, 0x061C, 0x06DD, 0x070F, 0x08E2, 0x180E, 0xFEFF, 0x110BD, 0x110CD, 0xE0001 => true, + 0x0600...0x0605, 0x0890...0x0891, 0x200B...0x200F, 0x202A...0x202E => true, + 0x2060...0x206F, 0xFFF9...0xFFFB, 0x13430...0x1343F, 0x1BCA0...0x1BCA3 => true, + 0x1D173...0x1D17A, 0xE0020...0xE007F => true, + else => false, + }; +} + +test "length policies reject early and accept only at document end" { + const filter = try DocumentFilter.create(std.testing.allocator, .{ + .length = .{ .min_bytes = 3, .max_bytes = 5, .min_codepoints = 3 }, + }, &.{}); + defer filter.destroy(); + + const pending = try filter.observe(.{ + .document_id = "good", + .bytes = "abc", + .boundary = .{ .document_start = true }, + }); + try std.testing.expect(pending == .pending); + const accepted = try filter.observe(.{ + .document_id = "good", + .bytes = "de", + .boundary = .{ .document_end = true }, + }); + try std.testing.expect(accepted == .accepted); + try std.testing.expectEqual(@as(u64, 5), accepted.accepted.metrics.bytes); + + const rejected = try filter.observe(.{ + .document_id = "large", + .bytes = "123456", + .boundary = .{ .document_start = true }, + }); + try std.testing.expect(rejected == .rejected); + try std.testing.expectEqual(RejectionReason.bytes_above_maximum, rejected.rejected.reason); + const drained = try filter.observe(.{ + .document_id = "large", + .bytes = "ignored", + .boundary = .{ .document_end = true }, + }); + try std.testing.expect(drained == .rejected); +} + +test "quality policies use UTF-8 and cleaner evidence" { + const filter = try DocumentFilter.create(std.testing.allocator, .{ + .quality = .{ + .max_replacement_basis_points = 1_000, + .max_cleaning_change_basis_points = 2_000, + .max_repeated_codepoints = 3, + }, + }, &.{}); + defer filter.destroy(); + + const repeated = try filter.observe(.{ + .document_id = "repeat", + .bytes = "aaaa", + .boundary = .{ .document_start = true, .document_end = true }, + }); + try std.testing.expectEqual( + RejectionReason.repeated_codepoint_run, + repeated.rejected.reason, + ); + + const dirty = try filter.observe(.{ + .document_id = "dirty", + .bytes = "clean", + .boundary = .{ .document_start = true, .document_end = true }, + .cleaning = .{ .malformed_sequences = 2 }, + }); + try std.testing.expectEqual( + RejectionReason.excessive_cleaning_ratio, + dirty.rejected.reason, + ); + + const invalid = try filter.observe(.{ + .document_id = "invalid", + .bytes = "bad\xFFutf8", + .boundary = .{ .document_start = true, .document_end = true }, + }); + try std.testing.expectEqual(RejectionReason.invalid_utf8, invalid.rejected.reason); +} + +const TestLanguageRule = struct { + saw_start: bool = false, + aborted: bool = false, + + const vtable: Rule.VTable = .{ + .evaluate = evaluate, + .abort = abort, + }; + + fn rule(self: *TestLanguageRule) Rule { + return .{ + .id = "language/allow-english", + .kind = .language, + .context = self, + .vtable = &vtable, + }; + } + + fn evaluate(context: *anyopaque, chunk: RuleChunk) anyerror!RuleDecision { + const self: *TestLanguageRule = @ptrCast(@alignCast(context)); + if (chunk.boundary.document_start) self.saw_start = true; + if (std.mem.indexOf(u8, chunk.bytes, "bonjour") != null) { + return .{ .reject = "language is not allowed" }; + } + return .keep; + } + + fn abort(context: *anyopaque) void { + const self: *TestLanguageRule = @ptrCast(@alignCast(context)); + self.aborted = true; + } +}; + +test "language and safety adapters can reject without buffering documents" { + var language: TestLanguageRule = .{}; + const filter = try DocumentFilter.create( + std.testing.allocator, + .{}, + &.{language.rule()}, + ); + defer filter.destroy(); + + const decision = try filter.observe(.{ + .document_id = "french", + .bytes = "bonjour le monde", + .boundary = .{ .document_start = true, .document_end = true }, + }); + try std.testing.expect(decision == .rejected); + try std.testing.expectEqualStrings("language/allow-english", decision.rejected.rule_id.?); + try std.testing.expect(language.saw_start); + try std.testing.expect(language.aborted); +} + +const StatefulByteRule = struct { + bytes: u64 = 0, + + const vtable: Rule.VTable = .{ + .evaluate = evaluate, + .abort = abort, + .checkpoint = checkpoint, + .restore = restore, + }; + + fn rule(self: *StatefulByteRule) Rule { + return .{ + .id = "safety/stateful-byte-limit", + .kind = .safety, + .context = self, + .vtable = &vtable, + .stateful = true, + }; + } + + fn evaluate(context: *anyopaque, chunk: RuleChunk) anyerror!RuleDecision { + const self: *StatefulByteRule = @ptrCast(@alignCast(context)); + self.bytes = try addU64(self.bytes, chunk.bytes.len); + if (chunk.boundary.document_end and self.bytes > 5) { + return .{ .reject = "stateful safety byte limit exceeded" }; + } + return .keep; + } + + fn abort(context: *anyopaque) void { + const self: *StatefulByteRule = @ptrCast(@alignCast(context)); + self.bytes = 0; + } + + fn checkpoint(context: *anyopaque, allocator: Allocator) anyerror![]const u8 { + const self: *StatefulByteRule = @ptrCast(@alignCast(context)); + const token = try allocator.alloc(u8, 8); + putU64(token[0..8], self.bytes); + return token; + } + + fn restore(context: *anyopaque, token: []const u8) anyerror!void { + if (token.len != 8) return error.InvalidToken; + const self: *StatefulByteRule = @ptrCast(@alignCast(context)); + self.bytes = getU64(token[0..8]); + } +}; + +test "checkpoints transfer metrics partial UTF-8 and stateful rule tokens" { + var source_rule: StatefulByteRule = .{}; + const source = try DocumentFilter.create( + std.testing.allocator, + .{}, + &.{source_rule.rule()}, + ); + defer source.destroy(); + _ = try source.observe(.{ + .document_id = "transfer", + .bytes = "ab\xF0\x9F", + .boundary = .{ .document_start = true }, + }); + + const encoded = try source.checkpoint(std.testing.allocator); + defer std.testing.allocator.free(encoded); + const checkpoint_state = try StreamingState.decode(encoded); + + var destination_rule: StatefulByteRule = .{}; + const destination = try DocumentFilter.create( + std.testing.allocator, + .{}, + &.{destination_rule.rule()}, + ); + defer destination.destroy(); + try destination.restore(checkpoint_state); + const decision = try destination.observe(.{ + .document_id = "transfer", + .bytes = "\x98\x80cd", + .boundary = .{ .document_end = true }, + }); + try std.testing.expect(decision == .rejected); + try std.testing.expectEqualStrings( + "safety/stateful-byte-limit", + decision.rejected.rule_id.?, + ); + try std.testing.expectEqual(@as(u64, 5), decision.rejected.metrics.codepoints); +} + +test "UTF-8 codepoints may span filter chunks" { + const filter = try DocumentFilter.create(std.testing.allocator, .{}, &.{}); + defer filter.destroy(); + _ = try filter.observe(.{ + .document_id = "unicode", + .bytes = "\xF0\x9F", + .boundary = .{ .document_start = true }, + }); + const decision = try filter.observe(.{ + .document_id = "unicode", + .bytes = "\x98\x80", + .boundary = .{ .document_end = true }, + }); + try std.testing.expect(decision == .accepted); + try std.testing.expectEqual(@as(u64, 1), decision.accepted.metrics.codepoints); +} diff --git a/data/document-filter/root.zig b/data/document-filter/root.zig new file mode 100644 index 0000000..d46e4d2 --- /dev/null +++ b/data/document-filter/root.zig @@ -0,0 +1,25 @@ +//! Streaming document filtering and rule adapters. + +const filter = @import("document_filter.zig"); + +pub const Boundary = filter.Boundary; +pub const CleaningSignals = filter.CleaningSignals; +pub const Decision = filter.Decision; +pub const DocumentChunk = filter.DocumentChunk; +pub const DocumentFilter = filter.DocumentFilter; +pub const LengthPolicy = filter.LengthPolicy; +pub const Metrics = filter.Metrics; +pub const Options = filter.Options; +pub const QualityPolicy = filter.QualityPolicy; +pub const Rejection = filter.Rejection; +pub const RejectionReason = filter.RejectionReason; +pub const Rule = filter.Rule; +pub const RuleChunk = filter.RuleChunk; +pub const RuleDecision = filter.RuleDecision; +pub const RuleKind = filter.RuleKind; +pub const Summary = filter.Summary; +pub const StreamingState = filter.StreamingState; + +test { + _ = filter; +} diff --git a/data/text-cleaner/README.md b/data/text-cleaner/README.md new file mode 100644 index 0000000..4b1feef --- /dev/null +++ b/data/text-cleaner/README.md @@ -0,0 +1,27 @@ +# Text cleaner + +`TextCleaner` is a stateful streaming transform for raw corpus bytes. It emits +UTF-8 while preserving decoder state across corpus chunk boundaries. + +Supported input encodings are UTF-8, UTF-16LE, UTF-16BE, ISO-8859-1, and +Windows-1252. Automatic mode recognizes UTF byte-order marks and otherwise uses +UTF-8. Invalid input can be replaced, removed, or rejected. + +Whitespace is normalized to ASCII space and LF, CRLF is coalesced even when it +crosses chunks, line-edge spaces are removed, and blank-line runs are bounded. +Control and Unicode formatting characters have independent policies. Output is +owned by a resettable arena and is invalidated by the next cleaning operation; +retained arena capacity can be capped for workloads with untrusted chunk sizes. + +`cleanCorpusChunk` maps corpus-reader document boundaries directly into the +stateful decoder. Empty cleaned chunks are valid because a trailing code unit or +whitespace run may need the next corpus chunk before it can be resolved. + +`checkpoint` produces a fixed-size, versioned state containing partial encoding +and formatting state. It can be decoded and restored on another worker with the +same semantic options; memory-retention settings may differ. + +Full NFC/NFKC normalization remains an explicit TODO until the repository has a +versioned Unicode data-table component. + +Run all tests from the repository root with `zig build test`. diff --git a/data/text-cleaner/root.zig b/data/text-cleaner/root.zig new file mode 100644 index 0000000..2c464d8 --- /dev/null +++ b/data/text-cleaner/root.zig @@ -0,0 +1,18 @@ +//! Stateful, bounded-memory text normalization. + +const cleaner = @import("text_cleaner.zig"); + +pub const Boundary = cleaner.Boundary; +pub const CleanedChunk = cleaner.CleanedChunk; +pub const InputEncoding = cleaner.InputEncoding; +pub const MalformedPolicy = cleaner.MalformedPolicy; +pub const Options = cleaner.Options; +pub const ScalarPolicy = cleaner.ScalarPolicy; +pub const Stats = cleaner.Stats; +pub const StreamingState = cleaner.StreamingState; +pub const TextCleaner = cleaner.TextCleaner; +pub const Totals = cleaner.Totals; + +test { + _ = cleaner; +} diff --git a/data/text-cleaner/text_cleaner.zig b/data/text-cleaner/text_cleaner.zig new file mode 100644 index 0000000..e881018 --- /dev/null +++ b/data/text-cleaner/text_cleaner.zig @@ -0,0 +1,1192 @@ +const std = @import("std"); + +const Allocator = std.mem.Allocator; +const ArenaAllocator = std.heap.ArenaAllocator; +const CorpusChunk = @import("../corpus-reader/root.zig").Chunk; + +pub const InputEncoding = enum { + /// Detects UTF-8, UTF-16LE, or UTF-16BE from a byte-order mark. Input + /// without a byte-order mark is interpreted as UTF-8. + auto, + utf8, + utf16_le, + utf16_be, + latin1, + windows_1252, +}; + +pub const MalformedPolicy = enum { + replace, + remove, + reject, +}; + +pub const ScalarPolicy = enum { + preserve, + remove, + reject, +}; + +pub const Options = struct { + input_encoding: InputEncoding = .auto, + strip_byte_order_mark: bool = true, + malformed: MalformedPolicy = .replace, + replacement: u21 = 0xFFFD, + + /// All Unicode White_Space code points are mapped to ASCII space or LF. + /// Line-leading and line-trailing horizontal whitespace is removed. + collapse_horizontal_whitespace: bool = true, + + /// Two LFs represent one empty line. Null preserves every normalized LF. + max_consecutive_newlines: ?usize = 2, + + /// Leading newline runs and trailing normalized whitespace are removed. + /// Trailing whitespace is delayed across chunks until the boundary is known. + /// Horizontal whitespace at line edges is always removed. + trim_document_edges: bool = true, + + control_characters: ScalarPolicy = .remove, + unicode_format_characters: ScalarPolicy = .remove, + + /// Null retains peak output capacity for maximum throughput. A limit keeps + /// one unusually expansive or malformed chunk from raising steady-state use. + retained_output_bytes: ?usize = null, + + // TODO(data/unicode): add NFC/NFKC normalization after a Unicode data-table + // component provides versioned decomposition and composition tables. +}; + +pub const Boundary = struct { + document_start: bool = false, + document_end: bool = false, + + pub const whole: Boundary = .{ + .document_start = true, + .document_end = true, + }; +}; + +pub const Stats = struct { + input_bytes: usize = 0, + output_bytes: usize = 0, + malformed_sequences: usize = 0, + whitespace_normalized: usize = 0, + controls_removed: usize = 0, + format_characters_removed: usize = 0, +}; + +pub const Totals = struct { + input_bytes: u64 = 0, + output_bytes: u64 = 0, + documents: u64 = 0, + malformed_sequences: u64 = 0, + whitespace_normalized: u64 = 0, + controls_removed: u64 = 0, + format_characters_removed: u64 = 0, + + fn add(self: *Totals, stats: Stats, document_ended: bool) void { + self.input_bytes += stats.input_bytes; + self.output_bytes += stats.output_bytes; + self.malformed_sequences += stats.malformed_sequences; + self.whitespace_normalized += stats.whitespace_normalized; + self.controls_removed += stats.controls_removed; + self.format_characters_removed += stats.format_characters_removed; + if (document_ended) self.documents += 1; + } +}; + +/// Borrowed output. `bytes` remains valid until the next call to `clean`, +/// `abortDocument`, or `destroy`. +pub const CleanedChunk = struct { + bytes: []const u8, + stats: Stats, + document_start: bool, + document_end: bool, +}; + +/// Fixed-size, transport-stable cleaner state. It includes partial decoder and +/// delayed-whitespace state but intentionally excludes monitoring counters. +pub const StreamingState = struct { + encoded: [wire_len]u8, + + pub const wire_len = 48; + const magic = "TCL1"; + + pub const DecodeError = error{ + InvalidState, + UnsupportedVersion, + }; + + pub fn asBytes(self: *const StreamingState) *const [wire_len]u8 { + return &self.encoded; + } + + pub fn decode(bytes: []const u8) DecodeError!StreamingState { + if (bytes.len != wire_len) return error.InvalidState; + var state: StreamingState = undefined; + @memcpy(&state.encoded, bytes); + try validateStreamingState(state); + return state; + } +}; + +const ResolvedEncoding = enum { + utf8, + utf16_le, + utf16_be, + latin1, + windows_1252, +}; + +const CleanerState = struct { + in_document: bool = false, + encoding: ?ResolvedEncoding = null, + probe: [3]u8 = @splat(0), + probe_len: u2 = 0, + + utf8_pending: [4]u8 = @splat(0), + utf8_pending_len: u3 = 0, + utf8_expected_len: u3 = 0, + + utf16_pending_byte: ?u8 = null, + utf16_high_surrogate: ?u16 = null, + + previous_was_cr: bool = false, + pending_spaces: usize = 0, + pending_newlines: usize = 0, + line_has_content: bool = false, + document_has_content: bool = false, +}; + +const ProbeDecision = struct { + encoding: ResolvedEncoding, + skip: u2, +}; + +pub const TextCleaner = struct { + backing_allocator: Allocator, + options: Options, + output_arena: ArenaAllocator, + state: CleanerState = .{}, + counters: Totals = .{}, + + pub const Error = Allocator.Error || error{ + ByteOrderMarkMismatch, + ControlCharacterRejected, + DocumentAlreadyStarted, + DocumentNotStarted, + FormatCharacterRejected, + InputTooLarge, + InvalidReplacement, + MalformedEncoding, + }; + + pub const CheckpointError = error{StateTooLarge}; + pub const RestoreError = StreamingState.DecodeError || error{ + CheckpointOptionsMismatch, + }; + + pub fn create(backing_allocator: Allocator, options: Options) Error!*TextCleaner { + if (!std.unicode.utf8ValidCodepoint(options.replacement)) { + return error.InvalidReplacement; + } + + const self = try backing_allocator.create(TextCleaner); + self.* = .{ + .backing_allocator = backing_allocator, + .options = options, + .output_arena = .init(backing_allocator), + }; + return self; + } + + pub fn destroy(self: *TextCleaner) void { + self.output_arena.deinit(); + const backing_allocator = self.backing_allocator; + backing_allocator.destroy(self); + } + + /// Cleans one streaming input fragment. Fragments may split UTF-8 code + /// points, UTF-16 code units, surrogate pairs, BOMs, and CRLF sequences. + /// On error, decoder and formatting state are rolled back to the state at + /// the beginning of this call. + pub fn clean( + self: *TextCleaner, + input: []const u8, + boundary: Boundary, + ) Error!CleanedChunk { + const state_before = self.state; + errdefer self.state = state_before; + + self.resetOutputArena(); + const output_allocator = self.output_arena.allocator(); + var output: std.ArrayList(u8) = .empty; + + const reserve = std.math.mul(usize, input.len, reserveFactor(self.options.input_encoding)) catch + return error.InputTooLarge; + try output.ensureTotalCapacityPrecise(output_allocator, reserve); + + if (boundary.document_start) { + if (self.state.in_document) return error.DocumentAlreadyStarted; + self.startDocument(); + } else if (!self.state.in_document) { + return error.DocumentNotStarted; + } + + var stats: Stats = .{ .input_bytes = input.len }; + try self.decode(input, boundary.document_end, &output, &stats); + + if (boundary.document_end) { + try self.finishDocument(&output, &stats); + self.state.in_document = false; + } + + stats.output_bytes = output.items.len; + self.counters.add(stats, boundary.document_end); + return .{ + .bytes = output.items, + .stats = stats, + .document_start = boundary.document_start, + .document_end = boundary.document_end, + }; + } + + /// Convenience boundary for the corpus reader without coupling ownership: + /// source metadata remains on the corpus chunk and only text bytes are copied. + pub fn cleanCorpusChunk( + self: *TextCleaner, + chunk: CorpusChunk, + ) Error!CleanedChunk { + return self.clean(chunk.bytes, .{ + .document_start = chunk.is_document_start, + .document_end = chunk.is_document_end, + }); + } + + /// Captures all semantic state required to continue on another worker. + pub fn checkpoint(self: *const TextCleaner) CheckpointError!StreamingState { + var encoded: [StreamingState.wire_len]u8 = @splat(0); + @memcpy(encoded[0..4], StreamingState.magic); + putU64(encoded[36..44], optionsFingerprint(self.options)); + + if (!self.state.in_document) return .{ .encoded = encoded }; + + var flags: u8 = 1 << 0; + if (self.state.utf16_pending_byte != null) flags |= 1 << 1; + if (self.state.utf16_high_surrogate != null) flags |= 1 << 2; + if (self.state.previous_was_cr) flags |= 1 << 3; + if (self.state.line_has_content) flags |= 1 << 4; + if (self.state.document_has_content) flags |= 1 << 5; + encoded[4] = flags; + encoded[5] = encodeResolvedEncoding(self.state.encoding); + encoded[6] = self.state.probe_len; + encoded[7] = self.state.utf8_pending_len; + encoded[8] = self.state.utf8_expected_len; + @memcpy( + encoded[10..][0..self.state.probe_len], + self.state.probe[0..self.state.probe_len], + ); + @memcpy( + encoded[13..][0..self.state.utf8_pending_len], + self.state.utf8_pending[0..self.state.utf8_pending_len], + ); + encoded[17] = self.state.utf16_pending_byte orelse 0; + putU16(encoded[18..20], self.state.utf16_high_surrogate orelse 0); + putU64( + encoded[20..28], + std.math.cast(u64, self.state.pending_spaces) orelse + return error.StateTooLarge, + ); + putU64( + encoded[28..36], + std.math.cast(u64, self.state.pending_newlines) orelse + return error.StateTooLarge, + ); + const result: StreamingState = .{ .encoded = encoded }; + validateStreamingState(result) catch unreachable; + return result; + } + + /// Restores a decoded checkpoint. Semantic options must match the worker + /// that created it; arena-retention settings may differ between workers. + pub fn restore( + self: *TextCleaner, + streaming_state: StreamingState, + ) RestoreError!void { + try validateStreamingState(streaming_state); + if (getU64(streaming_state.encoded[36..44]) != optionsFingerprint(self.options)) { + return error.CheckpointOptionsMismatch; + } + + self.state = try unpackStreamingState(streaming_state); + self.resetOutputArena(); + } + + /// Discards decoder state when an upstream source abandons a document. + pub fn abortDocument(self: *TextCleaner) void { + self.state = .{}; + self.resetOutputArena(); + } + + pub fn totals(self: *const TextCleaner) Totals { + return self.counters; + } + + fn startDocument(self: *TextCleaner) void { + self.state = .{ .in_document = true }; + self.state.encoding = switch (self.options.input_encoding) { + .latin1 => .latin1, + .windows_1252 => .windows_1252, + .utf8 => if (self.options.strip_byte_order_mark) null else .utf8, + .utf16_le => if (self.options.strip_byte_order_mark) null else .utf16_le, + .utf16_be => if (self.options.strip_byte_order_mark) null else .utf16_be, + .auto => null, + }; + } + + fn decode( + self: *TextCleaner, + input: []const u8, + document_end: bool, + output: *std.ArrayList(u8), + stats: *Stats, + ) Error!void { + var input_index: usize = 0; + while (self.state.encoding == null and input_index < input.len) { + self.state.probe[self.state.probe_len] = input[input_index]; + self.state.probe_len += 1; + input_index += 1; + + if (try self.detectEncoding(false)) |decision| { + try self.applyProbeDecision(decision, output, stats); + } + } + + if (self.state.encoding == null and document_end) { + const decision = (try self.detectEncoding(true)).?; + try self.applyProbeDecision(decision, output, stats); + } + + if (self.state.encoding) |encoding| { + try self.decodeResolved(encoding, input[input_index..], output, stats); + } + } + + fn detectEncoding(self: *TextCleaner, final: bool) Error!?ProbeDecision { + const bytes = self.state.probe[0..self.state.probe_len]; + return switch (self.options.input_encoding) { + .auto => detectAutomatic(bytes, final, self.options.strip_byte_order_mark), + .utf8 => detectExplicitUtf8(bytes, final, self.options.strip_byte_order_mark), + .utf16_le => try detectExplicitUtf16( + bytes, + final, + .utf16_le, + self.options.strip_byte_order_mark, + ), + .utf16_be => try detectExplicitUtf16( + bytes, + final, + .utf16_be, + self.options.strip_byte_order_mark, + ), + .latin1 => .{ .encoding = .latin1, .skip = 0 }, + .windows_1252 => .{ .encoding = .windows_1252, .skip = 0 }, + }; + } + + fn applyProbeDecision( + self: *TextCleaner, + decision: ProbeDecision, + output: *std.ArrayList(u8), + stats: *Stats, + ) Error!void { + var buffered: [3]u8 = undefined; + const buffered_len: usize = self.state.probe_len - decision.skip; + @memcpy(buffered[0..buffered_len], self.state.probe[decision.skip..self.state.probe_len]); + self.state.probe_len = 0; + self.state.encoding = decision.encoding; + try self.decodeResolved(decision.encoding, buffered[0..buffered_len], output, stats); + } + + fn decodeResolved( + self: *TextCleaner, + encoding: ResolvedEncoding, + bytes: []const u8, + output: *std.ArrayList(u8), + stats: *Stats, + ) Error!void { + switch (encoding) { + .utf8 => try self.decodeUtf8(bytes, output, stats), + .utf16_le => try self.decodeUtf16(bytes, .little, output, stats), + .utf16_be => try self.decodeUtf16(bytes, .big, output, stats), + .latin1 => { + for (bytes) |byte| try self.processCodepoint(byte, output, stats); + }, + .windows_1252 => { + for (bytes) |byte| { + if (windows1252Codepoint(byte)) |codepoint| { + try self.processCodepoint(codepoint, output, stats); + } else { + try self.handleMalformed(output, stats); + } + } + }, + } + } + + fn decodeUtf8( + self: *TextCleaner, + bytes: []const u8, + output: *std.ArrayList(u8), + stats: *Stats, + ) Error!void { + var index: usize = 0; + while (index < bytes.len) { + const byte = bytes[index]; + if (self.state.utf8_pending_len == 0) { + if (byte >= 0x21 and byte <= 0x7E) { + const run_start = index; + index += 1; + while (index < bytes.len and + bytes[index] >= 0x21 and bytes[index] <= 0x7E) + { + index += 1; + } + try self.emitAsciiRun(bytes[run_start..index], output); + continue; + } + if (byte < 0x80) { + try self.processCodepoint(byte, output, stats); + index += 1; + continue; + } + + const expected = std.unicode.utf8ByteSequenceLength(byte) catch { + try self.handleMalformed(output, stats); + index += 1; + continue; + }; + self.state.utf8_pending[0] = byte; + self.state.utf8_pending_len = 1; + self.state.utf8_expected_len = expected; + index += 1; + continue; + } + + if (byte & 0b1100_0000 != 0b1000_0000) { + try self.handleMalformed(output, stats); + self.state.utf8_pending_len = 0; + self.state.utf8_expected_len = 0; + continue; + } + + self.state.utf8_pending[self.state.utf8_pending_len] = byte; + self.state.utf8_pending_len += 1; + index += 1; + if (self.state.utf8_pending_len == self.state.utf8_expected_len) { + const sequence = self.state.utf8_pending[0..self.state.utf8_pending_len]; + const codepoint = std.unicode.utf8Decode(sequence) catch { + try self.handleMalformed(output, stats); + self.state.utf8_pending_len = 0; + self.state.utf8_expected_len = 0; + continue; + }; + self.state.utf8_pending_len = 0; + self.state.utf8_expected_len = 0; + try self.processCodepoint(codepoint, output, stats); + } + } + } + + fn decodeUtf16( + self: *TextCleaner, + bytes: []const u8, + endian: std.builtin.Endian, + output: *std.ArrayList(u8), + stats: *Stats, + ) Error!void { + for (bytes) |byte| { + if (self.state.utf16_pending_byte) |first| { + self.state.utf16_pending_byte = null; + const code_unit = switch (endian) { + .little => @as(u16, first) | (@as(u16, byte) << 8), + .big => (@as(u16, first) << 8) | byte, + }; + try self.processUtf16CodeUnit(code_unit, output, stats); + } else { + self.state.utf16_pending_byte = byte; + } + } + } + + fn processUtf16CodeUnit( + self: *TextCleaner, + code_unit: u16, + output: *std.ArrayList(u8), + stats: *Stats, + ) Error!void { + if (self.state.utf16_high_surrogate) |high| { + if (std.unicode.utf16IsLowSurrogate(code_unit)) { + const pair = [_]u16{ high, code_unit }; + self.state.utf16_high_surrogate = null; + const codepoint = std.unicode.utf16DecodeSurrogatePair(&pair) catch unreachable; + try self.processCodepoint(codepoint, output, stats); + return; + } + self.state.utf16_high_surrogate = null; + try self.handleMalformed(output, stats); + // The current code unit begins a new scalar and must be reprocessed. + } + + if (std.unicode.utf16IsHighSurrogate(code_unit)) { + self.state.utf16_high_surrogate = code_unit; + } else if (std.unicode.utf16IsLowSurrogate(code_unit)) { + try self.handleMalformed(output, stats); + } else { + try self.processCodepoint(code_unit, output, stats); + } + } + + fn processCodepoint( + self: *TextCleaner, + codepoint: u21, + output: *std.ArrayList(u8), + stats: *Stats, + ) Error!void { + if (self.state.previous_was_cr) { + self.state.previous_was_cr = false; + if (codepoint == '\n') { + stats.whitespace_normalized += 1; + return; + } + } + + if (codepoint == '\r') { + self.state.previous_was_cr = true; + stats.whitespace_normalized += 1; + try self.queueNewline(); + return; + } + if (isLineBreak(codepoint)) { + stats.whitespace_normalized += 1; + try self.queueNewline(); + return; + } + if (isHorizontalWhitespace(codepoint)) { + stats.whitespace_normalized += 1; + try self.queueSpace(); + return; + } + if (isControl(codepoint)) { + switch (self.options.control_characters) { + .preserve => try self.emitVisible(codepoint, output), + .remove => stats.controls_removed += 1, + .reject => return error.ControlCharacterRejected, + } + return; + } + if (isFormatCharacter(codepoint)) { + switch (self.options.unicode_format_characters) { + .preserve => try self.emitVisible(codepoint, output), + .remove => stats.format_characters_removed += 1, + .reject => return error.FormatCharacterRejected, + } + return; + } + + try self.emitVisible(codepoint, output); + } + + fn queueSpace(self: *TextCleaner) Error!void { + // Horizontal whitespace at line edges is always removed. + if (!self.state.line_has_content or self.state.pending_newlines != 0) return; + if (self.options.collapse_horizontal_whitespace) { + self.state.pending_spaces = 1; + } else { + self.state.pending_spaces = std.math.add( + usize, + self.state.pending_spaces, + 1, + ) catch return error.InputTooLarge; + } + } + + fn queueNewline(self: *TextCleaner) Error!void { + self.state.pending_spaces = 0; + self.state.line_has_content = false; + if (self.options.trim_document_edges and !self.state.document_has_content) return; + + if (self.options.max_consecutive_newlines) |maximum| { + self.state.pending_newlines = @min( + maximum, + self.state.pending_newlines +| 1, + ); + } else { + self.state.pending_newlines = std.math.add( + usize, + self.state.pending_newlines, + 1, + ) catch return error.InputTooLarge; + } + } + + fn emitVisible( + self: *TextCleaner, + codepoint: u21, + output: *std.ArrayList(u8), + ) Allocator.Error!void { + const allocator = self.output_arena.allocator(); + try self.flushPending(output); + + var encoded: [4]u8 = undefined; + const encoded_len = std.unicode.utf8Encode(codepoint, &encoded) catch unreachable; + try output.appendSlice(allocator, encoded[0..encoded_len]); + self.state.line_has_content = true; + self.state.document_has_content = true; + } + + fn emitAsciiRun( + self: *TextCleaner, + bytes: []const u8, + output: *std.ArrayList(u8), + ) Allocator.Error!void { + const allocator = self.output_arena.allocator(); + try self.flushPending(output); + try output.appendSlice(allocator, bytes); + self.state.line_has_content = true; + self.state.document_has_content = true; + } + + fn flushPending( + self: *TextCleaner, + output: *std.ArrayList(u8), + ) Allocator.Error!void { + const allocator = self.output_arena.allocator(); + if (self.state.pending_newlines != 0) { + const newlines = try output.addManyAsSlice( + allocator, + self.state.pending_newlines, + ); + @memset(newlines, '\n'); + self.state.pending_newlines = 0; + } + if (self.state.pending_spaces != 0) { + const spaces = try output.addManyAsSlice( + allocator, + self.state.pending_spaces, + ); + @memset(spaces, ' '); + self.state.pending_spaces = 0; + } + } + + fn handleMalformed( + self: *TextCleaner, + output: *std.ArrayList(u8), + stats: *Stats, + ) Error!void { + stats.malformed_sequences += 1; + switch (self.options.malformed) { + .replace => try self.processCodepoint(self.options.replacement, output, stats), + .remove => {}, + .reject => return error.MalformedEncoding, + } + } + + fn finishDocument( + self: *TextCleaner, + output: *std.ArrayList(u8), + stats: *Stats, + ) Error!void { + switch (self.state.encoding.?) { + .utf8 => { + if (self.state.utf8_pending_len != 0) { + self.state.utf8_pending_len = 0; + self.state.utf8_expected_len = 0; + try self.handleMalformed(output, stats); + } + }, + .utf16_le, .utf16_be => { + if (self.state.utf16_pending_byte != null) { + self.state.utf16_pending_byte = null; + try self.handleMalformed(output, stats); + } + if (self.state.utf16_high_surrogate != null) { + self.state.utf16_high_surrogate = null; + try self.handleMalformed(output, stats); + } + }, + .latin1, .windows_1252 => {}, + } + + self.state.previous_was_cr = false; + if (self.options.trim_document_edges) { + self.state.pending_spaces = 0; + self.state.pending_newlines = 0; + } else if (self.state.pending_newlines != 0) { + const newlines = try output.addManyAsSlice( + self.output_arena.allocator(), + self.state.pending_newlines, + ); + @memset(newlines, '\n'); + self.state.pending_newlines = 0; + self.state.pending_spaces = 0; + } else if (self.state.pending_spaces != 0) { + const spaces = try output.addManyAsSlice( + self.output_arena.allocator(), + self.state.pending_spaces, + ); + @memset(spaces, ' '); + self.state.pending_spaces = 0; + } + } + + fn resetOutputArena(self: *TextCleaner) void { + const mode: ArenaAllocator.ResetMode = if (self.options.retained_output_bytes) |limit| + .{ .retain_with_limit = limit } + else + .retain_capacity; + _ = self.output_arena.reset(mode); + } +}; + +fn reserveFactor(encoding: InputEncoding) usize { + return switch (encoding) { + .auto, .utf8 => 1, + .utf16_le, .utf16_be, .latin1 => 2, + .windows_1252 => 3, + }; +} + +fn validateStreamingState(state: StreamingState) StreamingState.DecodeError!void { + const bytes = state.encoded; + if (!std.mem.eql(u8, bytes[0..4], StreamingState.magic)) { + return error.UnsupportedVersion; + } + if (bytes[4] & ~@as(u8, 0b0011_1111) != 0) return error.InvalidState; + if (bytes[9] != 0) return error.InvalidState; + for (bytes[44..48]) |byte| { + if (byte != 0) return error.InvalidState; + } + + const in_document = bytes[4] & (1 << 0) != 0; + const has_pending_byte = bytes[4] & (1 << 1) != 0; + const has_high_surrogate = bytes[4] & (1 << 2) != 0; + const line_has_content = bytes[4] & (1 << 4) != 0; + const document_has_content = bytes[4] & (1 << 5) != 0; + const encoding = try decodeResolvedEncoding(bytes[5]); + const probe_len = bytes[6]; + const utf8_pending_len = bytes[7]; + const utf8_expected_len = bytes[8]; + const pending_spaces = getU64(bytes[20..28]); + const pending_newlines = getU64(bytes[28..36]); + + if (probe_len > 3 or utf8_pending_len > 4 or utf8_expected_len > 4) { + return error.InvalidState; + } + if ((utf8_pending_len == 0) != (utf8_expected_len == 0)) { + return error.InvalidState; + } + if (utf8_pending_len != 0 and + (utf8_expected_len < 2 or utf8_pending_len >= utf8_expected_len)) + { + return error.InvalidState; + } + if (encoding != null and probe_len != 0) return error.InvalidState; + if (encoding == null and probe_len > 2) return error.InvalidState; + if (encoding != .utf8 and (utf8_pending_len != 0 or utf8_expected_len != 0)) { + return error.InvalidState; + } + for (bytes[10 + probe_len .. 13]) |byte| { + if (byte != 0) return error.InvalidState; + } + for (bytes[13 + utf8_pending_len .. 17]) |byte| { + if (byte != 0) return error.InvalidState; + } + if (utf8_pending_len != 0) { + const expected = std.unicode.utf8ByteSequenceLength(bytes[13]) catch + return error.InvalidState; + if (expected != utf8_expected_len) return error.InvalidState; + for (bytes[14 .. 13 + utf8_pending_len]) |byte| { + if (byte & 0b1100_0000 != 0b1000_0000) return error.InvalidState; + } + } + if (encoding != .utf16_le and encoding != .utf16_be and + (has_pending_byte or has_high_surrogate)) + { + return error.InvalidState; + } + if (!has_pending_byte and bytes[17] != 0) return error.InvalidState; + const high_surrogate = getU16(bytes[18..20]); + if (has_high_surrogate != std.unicode.utf16IsHighSurrogate(high_surrogate)) { + return error.InvalidState; + } + if (pending_spaces != 0 and pending_newlines != 0) return error.InvalidState; + if (pending_spaces != 0 and !line_has_content) return error.InvalidState; + if (line_has_content and !document_has_content) return error.InvalidState; + + if (!in_document) { + if (bytes[4] != 0 or bytes[5] != 0 or probe_len != 0 or + utf8_pending_len != 0 or utf8_expected_len != 0 or + pending_spaces != 0 or pending_newlines != 0) + { + return error.InvalidState; + } + for (bytes[10..20]) |byte| { + if (byte != 0) return error.InvalidState; + } + } else if (encoding == null and probe_len == 0) { + // An empty document may be between start and its first byte. + } +} + +fn unpackStreamingState(state: StreamingState) TextCleaner.RestoreError!CleanerState { + const bytes = state.encoded; + if (bytes[4] & (1 << 0) == 0) return .{}; + + const pending_spaces = std.math.cast(usize, getU64(bytes[20..28])) orelse + return error.InvalidState; + const pending_newlines = std.math.cast(usize, getU64(bytes[28..36])) orelse + return error.InvalidState; + var result: CleanerState = .{ + .in_document = true, + .encoding = try decodeResolvedEncoding(bytes[5]), + .probe_len = @intCast(bytes[6]), + .utf8_pending_len = @intCast(bytes[7]), + .utf8_expected_len = @intCast(bytes[8]), + .utf16_pending_byte = if (bytes[4] & (1 << 1) != 0) bytes[17] else null, + .utf16_high_surrogate = if (bytes[4] & (1 << 2) != 0) + getU16(bytes[18..20]) + else + null, + .previous_was_cr = bytes[4] & (1 << 3) != 0, + .line_has_content = bytes[4] & (1 << 4) != 0, + .document_has_content = bytes[4] & (1 << 5) != 0, + .pending_spaces = pending_spaces, + .pending_newlines = pending_newlines, + }; + @memcpy(&result.probe, bytes[10..13]); + @memcpy(&result.utf8_pending, bytes[13..17]); + return result; +} + +fn encodeResolvedEncoding(encoding: ?ResolvedEncoding) u8 { + return if (encoding) |value| switch (value) { + .utf8 => 1, + .utf16_le => 2, + .utf16_be => 3, + .latin1 => 4, + .windows_1252 => 5, + } else 0; +} + +fn decodeResolvedEncoding(encoded: u8) StreamingState.DecodeError!?ResolvedEncoding { + return switch (encoded) { + 0 => null, + 1 => .utf8, + 2 => .utf16_le, + 3 => .utf16_be, + 4 => .latin1, + 5 => .windows_1252, + else => error.InvalidState, + }; +} + +fn optionsFingerprint(options: Options) u64 { + var hash: u64 = 0xcbf29ce484222325; + fingerprintValue(&hash, @intFromEnum(options.input_encoding)); + fingerprintValue(&hash, @intFromBool(options.strip_byte_order_mark)); + fingerprintValue(&hash, @intFromEnum(options.malformed)); + fingerprintValue(&hash, options.replacement); + fingerprintValue(&hash, @intFromBool(options.collapse_horizontal_whitespace)); + fingerprintValue(&hash, if (options.max_consecutive_newlines) |value| + @as(u64, value) + else + std.math.maxInt(u64)); + fingerprintValue(&hash, @intFromBool(options.trim_document_edges)); + fingerprintValue(&hash, @intFromEnum(options.control_characters)); + fingerprintValue(&hash, @intFromEnum(options.unicode_format_characters)); + return hash; +} + +fn fingerprintValue(hash: *u64, value: anytype) void { + const canonical: u64 = @intCast(value); + for (0..8) |index| { + hash.* ^= @as(u8, @truncate(canonical >> @intCast(index * 8))); + hash.* *%= 0x100000001b3; + } +} + +fn putU16(destination: *[2]u8, value: u16) void { + for (destination, 0..) |*byte, index| { + byte.* = @truncate(value >> @intCast(index * 8)); + } +} + +fn putU64(destination: *[8]u8, value: u64) void { + for (destination, 0..) |*byte, index| { + byte.* = @truncate(value >> @intCast(index * 8)); + } +} + +fn getU16(source: *const [2]u8) u16 { + var value: u16 = 0; + for (source, 0..) |byte, index| { + value |= @as(u16, byte) << @intCast(index * 8); + } + return value; +} + +fn getU64(source: *const [8]u8) u64 { + var value: u64 = 0; + for (source, 0..) |byte, index| { + value |= @as(u64, byte) << @intCast(index * 8); + } + return value; +} + +fn detectAutomatic( + bytes: []const u8, + final: bool, + strip_bom: bool, +) ?ProbeDecision { + if (bytes.len >= 3 and std.mem.eql(u8, bytes[0..3], &.{ 0xEF, 0xBB, 0xBF })) { + return .{ .encoding = .utf8, .skip = if (strip_bom) 3 else 0 }; + } + if (bytes.len >= 2 and bytes[0] == 0xFF and bytes[1] == 0xFE) { + return .{ .encoding = .utf16_le, .skip = if (strip_bom) 2 else 0 }; + } + if (bytes.len >= 2 and bytes[0] == 0xFE and bytes[1] == 0xFF) { + return .{ .encoding = .utf16_be, .skip = if (strip_bom) 2 else 0 }; + } + if (bytes.len == 0) return if (final) .{ .encoding = .utf8, .skip = 0 } else null; + + const could_be_bom = bytes[0] == 0xEF or bytes[0] == 0xFF or bytes[0] == 0xFE; + if (!could_be_bom or final) return .{ .encoding = .utf8, .skip = 0 }; + if (bytes[0] == 0xEF) { + if (bytes.len >= 2 and bytes[1] != 0xBB) return .{ .encoding = .utf8, .skip = 0 }; + if (bytes.len == 3) return .{ .encoding = .utf8, .skip = 0 }; + } else if (bytes.len == 2) { + return .{ .encoding = .utf8, .skip = 0 }; + } + return null; +} + +fn detectExplicitUtf8( + bytes: []const u8, + final: bool, + strip_bom: bool, +) ?ProbeDecision { + if (!strip_bom) return .{ .encoding = .utf8, .skip = 0 }; + if (bytes.len >= 3) { + const has_bom = std.mem.eql(u8, bytes[0..3], &.{ 0xEF, 0xBB, 0xBF }); + return .{ .encoding = .utf8, .skip = if (has_bom) 3 else 0 }; + } + if (bytes.len >= 1 and bytes[0] != 0xEF) return .{ .encoding = .utf8, .skip = 0 }; + if (bytes.len >= 2 and bytes[1] != 0xBB) return .{ .encoding = .utf8, .skip = 0 }; + return if (final) .{ .encoding = .utf8, .skip = 0 } else null; +} + +fn detectExplicitUtf16( + bytes: []const u8, + final: bool, + encoding: ResolvedEncoding, + strip_bom: bool, +) TextCleaner.Error!?ProbeDecision { + if (!strip_bom) return .{ .encoding = encoding, .skip = 0 }; + if (bytes.len < 2) { + return if (final) .{ .encoding = encoding, .skip = 0 } else null; + } + + const is_little_bom = bytes[0] == 0xFF and bytes[1] == 0xFE; + const is_big_bom = bytes[0] == 0xFE and bytes[1] == 0xFF; + if ((encoding == .utf16_le and is_big_bom) or + (encoding == .utf16_be and is_little_bom)) + { + return error.ByteOrderMarkMismatch; + } + const matching_bom = (encoding == .utf16_le and is_little_bom) or + (encoding == .utf16_be and is_big_bom); + return .{ .encoding = encoding, .skip = if (matching_bom) 2 else 0 }; +} + +fn isLineBreak(codepoint: u21) bool { + return codepoint == '\n' or codepoint == 0x0085 or + codepoint == 0x2028 or codepoint == 0x2029; +} + +fn isHorizontalWhitespace(codepoint: u21) bool { + return switch (codepoint) { + '\t', 0x000B, 0x000C, ' ', 0x00A0, 0x1680, 0x202F, 0x205F, 0x3000 => true, + 0x2000...0x200A => true, + else => false, + }; +} + +fn isControl(codepoint: u21) bool { + return codepoint <= 0x001F or (codepoint >= 0x007F and codepoint <= 0x009F); +} + +fn isFormatCharacter(codepoint: u21) bool { + return switch (codepoint) { + 0x00AD, 0x061C, 0x06DD, 0x070F, 0x08E2, 0x180E, 0xFEFF, 0x110BD, 0x110CD, 0xE0001 => true, + 0x0600...0x0605, 0x0890...0x0891, 0x200B...0x200F, 0x202A...0x202E => true, + 0x2060...0x206F, 0xFFF9...0xFFFB, 0x13430...0x1343F, 0x1BCA0...0x1BCA3 => true, + 0x1D173...0x1D17A, 0xE0020...0xE007F => true, + else => false, + }; +} + +fn windows1252Codepoint(byte: u8) ?u21 { + if (byte < 0x80 or byte >= 0xA0) return byte; + return switch (byte) { + 0x80 => 0x20AC, + 0x82 => 0x201A, + 0x83 => 0x0192, + 0x84 => 0x201E, + 0x85 => 0x2026, + 0x86 => 0x2020, + 0x87 => 0x2021, + 0x88 => 0x02C6, + 0x89 => 0x2030, + 0x8A => 0x0160, + 0x8B => 0x2039, + 0x8C => 0x0152, + 0x8E => 0x017D, + 0x91 => 0x2018, + 0x92 => 0x2019, + 0x93 => 0x201C, + 0x94 => 0x201D, + 0x95 => 0x2022, + 0x96 => 0x2013, + 0x97 => 0x2014, + 0x98 => 0x02DC, + 0x99 => 0x2122, + 0x9A => 0x0161, + 0x9B => 0x203A, + 0x9C => 0x0153, + 0x9E => 0x017E, + 0x9F => 0x0178, + else => null, + }; +} + +test "UTF-8 cleaning is stable across arbitrary chunk boundaries" { + const cleaner = try TextCleaner.create(std.testing.allocator, .{}); + defer cleaner.destroy(); + + const first = try cleaner.clean("\xEF\xBB\xBF Hello\t", .{ .document_start = true }); + try std.testing.expectEqualStrings("Hello", first.bytes); + + const second = try cleaner.clean(" world\r", .{}); + try std.testing.expectEqualStrings(" world", second.bytes); + + const third = try cleaner.clean("\n\n\x00next \xE2", .{}); + try std.testing.expectEqualStrings("\n\nnext", third.bytes); + try std.testing.expectEqual(@as(usize, 1), third.stats.controls_removed); + + const fourth = try cleaner.clean("\x82\xAC ", .{ .document_end = true }); + try std.testing.expectEqualStrings(" €", fourth.bytes); + try std.testing.expectEqual(@as(u64, 1), cleaner.totals().documents); +} + +test "byte-at-a-time cleaning matches whole-document cleaning" { + const input = "\xEF\xBB\xBF Alpha\tβ\r\n\nGamma\xE2( end "; + + const whole_cleaner = try TextCleaner.create(std.testing.allocator, .{}); + defer whole_cleaner.destroy(); + const whole = try whole_cleaner.clean(input, .whole); + + const streaming_cleaner = try TextCleaner.create(std.testing.allocator, .{}); + defer streaming_cleaner.destroy(); + var assembled: std.ArrayList(u8) = .empty; + defer assembled.deinit(std.testing.allocator); + for (input, 0..) |_, index| { + const cleaned = try streaming_cleaner.clean(input[index .. index + 1], .{ + .document_start = index == 0, + .document_end = index + 1 == input.len, + }); + try assembled.appendSlice(std.testing.allocator, cleaned.bytes); + } + + try std.testing.expectEqualStrings(whole.bytes, assembled.items); +} + +test "streaming checkpoints transfer partial decoder and formatting state" { + const source = try TextCleaner.create(std.testing.allocator, .{}); + defer source.destroy(); + const partial = try source.clean("prefix \xE2", .{ .document_start = true }); + try std.testing.expectEqualStrings("prefix", partial.bytes); + + const checkpoint = try source.checkpoint(); + const encoded = checkpoint.asBytes().*; + const decoded = try StreamingState.decode(&encoded); + + const destination = try TextCleaner.create(std.testing.allocator, .{}); + defer destination.destroy(); + try destination.restore(decoded); + const continued = try destination.clean("\x82\xAC\r\nnext", .{ .document_end = true }); + try std.testing.expectEqualStrings(" €\nnext", continued.bytes); + + const incompatible = try TextCleaner.create(std.testing.allocator, .{ + .collapse_horizontal_whitespace = false, + }); + defer incompatible.destroy(); + try std.testing.expectError( + error.CheckpointOptionsMismatch, + incompatible.restore(decoded), + ); +} + +test "malformed UTF-8 can be replaced, removed, or rejected" { + const replacing = try TextCleaner.create(std.testing.allocator, .{}); + defer replacing.destroy(); + const replaced = try replacing.clean("a\xFFb", .whole); + try std.testing.expectEqualStrings("a\xEF\xBF\xBDb", replaced.bytes); + try std.testing.expectEqual(@as(usize, 1), replaced.stats.malformed_sequences); + + const removing = try TextCleaner.create(std.testing.allocator, .{ .malformed = .remove }); + defer removing.destroy(); + const removed = try removing.clean("a\xFFb", .whole); + try std.testing.expectEqualStrings("ab", removed.bytes); + + const rejecting = try TextCleaner.create(std.testing.allocator, .{ .malformed = .reject }); + defer rejecting.destroy(); + try std.testing.expectError(error.MalformedEncoding, rejecting.clean("a\xFFb", .whole)); + // State rollback makes the cleaner reusable after a rejected fragment. + const valid = try rejecting.clean("valid", .whole); + try std.testing.expectEqualStrings("valid", valid.bytes); +} + +test "UTF-16 BOMs code units and surrogate pairs may span chunks" { + const cleaner = try TextCleaner.create(std.testing.allocator, .{}); + defer cleaner.destroy(); + + const first = try cleaner.clean("\xFF", .{ .document_start = true }); + try std.testing.expectEqual(@as(usize, 0), first.bytes.len); + const second = try cleaner.clean("\xFE\x41\x00\x3D\xD8", .{}); + try std.testing.expectEqualStrings("A", second.bytes); + const third = try cleaner.clean( + "\x00\xDE\x0D\x00\x0A\x00\x42\x00", + .{ .document_end = true }, + ); + try std.testing.expectEqualStrings("😀\nB", third.bytes); +} + +test "format and control policies remove unsafe invisible scalars" { + const cleaner = try TextCleaner.create(std.testing.allocator, .{}); + defer cleaner.destroy(); + const cleaned = try cleaner.clean("a\xE2\x80\x8Bb\x07c", .whole); + try std.testing.expectEqualStrings("abc", cleaned.bytes); + try std.testing.expectEqual(@as(usize, 1), cleaned.stats.format_characters_removed); + try std.testing.expectEqual(@as(usize, 1), cleaned.stats.controls_removed); +} + +test "Windows-1252 is converted to UTF-8" { + const cleaner = try TextCleaner.create(std.testing.allocator, .{ + .input_encoding = .windows_1252, + }); + defer cleaner.destroy(); + const cleaned = try cleaner.clean("\x93quoted\x94 \x80", .whole); + try std.testing.expectEqualStrings("“quoted” €", cleaned.bytes); +} diff --git a/root.zig b/root.zig new file mode 100644 index 0000000..00265d1 --- /dev/null +++ b/root.zig @@ -0,0 +1,13 @@ +pub const data = struct { + pub const corpus_reader = @import("data/corpus-reader/root.zig"); + pub const deduplicator = @import("data/deduplicator/root.zig"); + pub const document_filter = @import("data/document-filter/root.zig"); + pub const text_cleaner = @import("data/text-cleaner/root.zig"); +}; + +test { + _ = data.corpus_reader; + _ = data.deduplicator; + _ = data.document_filter; + _ = data.text_cleaner; +}