From 13eedf9e35437aa3a3db881b67ab0771572e9faa Mon Sep 17 00:00:00 2001 From: Cameron Date: Sat, 1 Aug 2026 01:08:58 -0400 Subject: [PATCH] Release Chi streaming corpus library --- .github/workflows/ci.yml | 25 + .gitignore | 2 + CONTRIBUTING.md | 20 + LICENSE | 201 +++ NOTICE | 9 + README.md | 83 ++ SECURITY.md | 16 + build.zig | 18 + build.zig.zon | 17 + data/corpus-reader/README.md | 26 + data/corpus-reader/corpus_reader.zig | 1219 ++++++++++++++++++ data/corpus-reader/root.zig | 13 + data/deduplicator/README.md | 36 + data/deduplicator/deduplicator.zig | 1465 ++++++++++++++++++++++ data/deduplicator/root.zig | 28 + data/document-filter/README.md | 27 + data/document-filter/document_filter.zig | 1370 ++++++++++++++++++++ data/document-filter/root.zig | 25 + data/text-cleaner/README.md | 27 + data/text-cleaner/root.zig | 18 + data/text-cleaner/text_cleaner.zig | 1192 ++++++++++++++++++ root.zig | 13 + 22 files changed, 5850 insertions(+) create mode 100644 .github/workflows/ci.yml create mode 100644 .gitignore create mode 100644 CONTRIBUTING.md create mode 100644 LICENSE create mode 100644 NOTICE create mode 100644 SECURITY.md create mode 100644 build.zig create mode 100644 build.zig.zon create mode 100644 data/corpus-reader/README.md create mode 100644 data/corpus-reader/corpus_reader.zig create mode 100644 data/corpus-reader/root.zig create mode 100644 data/deduplicator/README.md create mode 100644 data/deduplicator/deduplicator.zig create mode 100644 data/deduplicator/root.zig create mode 100644 data/document-filter/README.md create mode 100644 data/document-filter/document_filter.zig create mode 100644 data/document-filter/root.zig create mode 100644 data/text-cleaner/README.md create mode 100644 data/text-cleaner/root.zig create mode 100644 data/text-cleaner/text_cleaner.zig create mode 100644 root.zig diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..e460881 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,25 @@ +name: CI + +on: + push: + branches: [master] + pull_request: + +permissions: + contents: read + +jobs: + test: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - name: Install Zig 0.16.0 + uses: mlugg/setup-zig@d1434d08867e3ee9daa34448df10607b98908d29 # v2 + with: + version: 0.16.0 + - name: Run tests + run: zig build test diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..db3cb54 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +/.zig-cache/ +/zig-out/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..cc169fb --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,20 @@ +# Contributing + +Chi is pre-1.0 and its public contracts are intentionally narrow. Before a +large change, open an issue describing the corpus shape, memory constraint, +failure mode, and checkpoint behavior it is meant to address. + +Pull requests should: + +- remain compatible with Zig 0.16.0; +- keep steady-state memory bounds explicit; +- document the lifetime of every borrowed slice; +- preserve deterministic behavior across input chunk boundaries; +- add tests for checkpoints and failure rollback when state changes; and +- run `zig build test` before submission. + +Do not include customer data, model corpora, credentials, generated secrets, +or code whose license is incompatible with Apache-2.0. + +By submitting a contribution, you agree that it may be distributed under this +repository's Apache-2.0 license. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..ce45249 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2026 Cameron Emery + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/NOTICE b/NOTICE new file mode 100644 index 0000000..9158d0b --- /dev/null +++ b/NOTICE @@ -0,0 +1,9 @@ +Chi +Copyright 2026 Cameron Emery + +This product was developed with material AI assistance under human direction +and review. Human review, release decisions, and responsibility remain with +the maintainer. + +This product includes software developed by Cameron Emery and is licensed +under the Apache License, Version 2.0. diff --git a/README.md b/README.md index e69de29..598e77d 100644 --- a/README.md +++ b/README.md @@ -0,0 +1,83 @@ +# Chi + +Streaming, checkpointable building blocks for large text-corpus curation in +Zig. + +Chi is a pre-1.0 library for data infrastructure that cannot keep an entire +corpus—or even an entire document—in memory. Its four modules cover bounded +ingestion, text cleanup, document filtering, and exact/approximate +deduplication. Each module exposes explicit ownership boundaries and a +versioned checkpoint format for stopping or moving work between processes. + +The current release targets **Zig 0.16.0** and has no third-party runtime +dependencies. + +## Modules + +| Module | Current scope | +|---|---| +| `corpus_reader` | Deterministic file and repository traversal plus pull-based stream adapters | +| `text_cleaner` | Streaming UTF-8/UTF-16/Latin-1/Windows-1252 decoding and conservative whitespace/control cleanup | +| `document_filter` | Bounded-memory length, UTF-8, visibility, replacement, cleaning-change, and repeated-run gates | +| `deduplicator` | Chunk-stable exact fingerprints, SimHash candidates, sampled duplicate spans, and a transactional index boundary | + +```zig +const chi = @import("chi"); + +const CorpusReader = chi.data.corpus_reader.CorpusReader; +const TextCleaner = chi.data.text_cleaner.TextCleaner; +const DocumentFilter = chi.data.document_filter.DocumentFilter; +const Deduplicator = chi.data.deduplicator.Deduplicator; +``` + +Each module has a focused README with API behavior, memory ownership, and +checkpoint guarantees: + +- [`data/corpus-reader`](data/corpus-reader/README.md) +- [`data/text-cleaner`](data/text-cleaner/README.md) +- [`data/document-filter`](data/document-filter/README.md) +- [`data/deduplicator`](data/deduplicator/README.md) + +## Verify + +Install Zig 0.16.0, clone the repository, and run: + +```sh +zig build test +``` + +The suite exercises arbitrary chunk boundaries, checkpoint transfer, +deterministic repository traversal, duplicate classification, and rollback +behavior. Windows and Linux run the same test command in CI. + +## What Chi does not claim + +Chi is a set of library components, not a turnkey distributed data platform. +This release does not include: + +- NFC or NFKC normalization; +- built-in language or safety classifiers; +- a database driver; +- a distributed deduplication backend; +- Unicode word segmentation or token-shingle MinHash; or +- published performance comparisons. + +The provided approximate match signal uses byte-level ASCII word features. +Downstream systems must evaluate whether that is appropriate for their data. + +## Status + +Chi is early, intentionally narrow software. APIs and checkpoint formats may +change before 1.0. The current release is useful as an inspectable foundation +for streaming corpus tooling and as research infrastructure for YY's data +systems; it is not offered with an SLA. + +See [SECURITY.md](SECURITY.md) for private vulnerability reporting and +[CONTRIBUTING.md](CONTRIBUTING.md) before proposing changes. + +## License and authorship + +Copyright 2026 Cameron Emery. Licensed under Apache-2.0. + +Chi was developed under Cameron Emery's direction with material AI assistance. +Human review, release decisions, and responsibility remain with the maintainer. diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..92226da --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,16 @@ +# Security + +Please do not publish suspected vulnerabilities, credential exposures, or +unsafe parsing cases in a public issue before the maintainer has had a chance +to assess them. + +Report them to **security@yycore.ai** with: + +- the affected commit and module; +- a minimal reproducer that contains no sensitive corpus data; +- the observed impact; and +- any suggested mitigation. + +Chi is pre-1.0 research infrastructure and does not have a guaranteed response +time. Receipt will be acknowledged when possible, and confirmed issues will be +coordinated privately before disclosure. diff --git a/build.zig b/build.zig new file mode 100644 index 0000000..f303b1e --- /dev/null +++ b/build.zig @@ -0,0 +1,18 @@ +const std = @import("std"); + +pub fn build(b: *std.Build) void { + const target = b.standardTargetOptions(.{}); + const optimize = b.standardOptimizeOption(.{}); + + const chi = b.addModule("chi", .{ + .root_source_file = b.path("root.zig"), + .target = target, + .optimize = optimize, + }); + + const unit_tests = b.addTest(.{ .root_module = chi }); + const run_unit_tests = b.addRunArtifact(unit_tests); + + const test_step = b.step("test", "Run unit tests"); + test_step.dependOn(&run_unit_tests.step); +} diff --git a/build.zig.zon b/build.zig.zon new file mode 100644 index 0000000..c496472 --- /dev/null +++ b/build.zig.zon @@ -0,0 +1,17 @@ +.{ + .name = .chi, + .version = "0.1.0", + .minimum_zig_version = "0.16.0", + .paths = .{ + "build.zig", + "build.zig.zon", + "root.zig", + "data", + "LICENSE", + "NOTICE", + "README.md", + "SECURITY.md", + "CONTRIBUTING.md", + }, + .fingerprint = 0x2c9a9438dc98f8f3, +} diff --git a/data/corpus-reader/README.md b/data/corpus-reader/README.md new file mode 100644 index 0000000..f0447e5 --- /dev/null +++ b/data/corpus-reader/README.md @@ -0,0 +1,26 @@ +# Corpus reader + +`CorpusReader` turns raw sources into bounded, borrowed chunks for the future +preprocessing pipeline. It currently supports: + +- regular files through positional reads; +- local repository trees with deterministic, bytewise-sorted traversal; +- streams through a zero-copy pull adapter; +- databases through the same adapter boundary, ready for concrete drivers. + +The reader owns three arenas with source, document, and chunk lifetimes. A chunk +is invalidated by the next reader operation. This keeps steady-state ingestion +memory near `chunk_size` while retaining allocation capacity after warm-up. +Repository traversal holds only the sorted directory entries on the active +depth-first stack, not an index of every file in the checkout. + +Every chunk carries `state_after`. Encode that value before the next operation +to transfer or persist a restore point. Checkpoints assume the underlying corpus +is immutable; changing a file or repository while it is being read is an input +consistency error, not something the reader attempts to hide. + +Run the component tests from the repository root: + +```sh +zig build test +``` diff --git a/data/corpus-reader/corpus_reader.zig b/data/corpus-reader/corpus_reader.zig new file mode 100644 index 0000000..1f7ab00 --- /dev/null +++ b/data/corpus-reader/corpus_reader.zig @@ -0,0 +1,1219 @@ +const std = @import("std"); + +const Allocator = std.mem.Allocator; +const ArenaAllocator = std.heap.ArenaAllocator; +const Dir = std.Io.Dir; +const File = std.Io.File; +const Io = std.Io; + +pub const SourceKind = enum(u8) { + file = 1, + repository = 2, + database = 3, + stream = 4, +}; + +/// Controls the hot path. A chunk is the largest allocation made per `next` call. +pub const Options = struct { + chunk_size: usize = 256 * 1024, + + /// Retaining the chunk arena avoids allocator traffic after warm-up. Set a + /// limit when readers may occasionally receive unusually large chunks. + retained_chunk_bytes: ?usize = null, +}; + +pub const RepositoryOptions = struct { + reader: Options = .{}, + + /// Git, Mercurial, and Subversion internals are metadata rather than source + /// corpus in the common case. Dotfiles outside these directories are read. + include_vcs_metadata: bool = false, +}; + +/// State for the first unread byte after a chunk. It is deliberately independent +/// of pointers owned by `CorpusReader`, so it can be encoded and sent to a +/// different machine in a memory/compute-disaggregated pipeline. +/// +/// Slices returned by `decode` borrow the encoded input. Slices obtained from a +/// `Chunk` remain valid only until the next reader operation. +pub const StreamingState = struct { + source_kind: SourceKind, + sequence: u64, + document_index: u64, + byte_offset: u64, + document_done: bool, + source_done: bool, + source_id: []const u8, + document_id: []const u8, + resume_token: []const u8 = "", + + const magic = "CRS1"; + const header_len = 44; + + pub const EncodeError = Allocator.Error || error{StateTooLarge}; + pub const DecodeError = error{ + InvalidState, + UnsupportedVersion, + }; + + pub fn encode(self: StreamingState, allocator: Allocator) EncodeError![]u8 { + if (self.source_id.len > std.math.maxInt(u32) or + self.document_id.len > std.math.maxInt(u32) or + self.resume_token.len > std.math.maxInt(u32)) + { + return error.StateTooLarge; + } + + var total = std.math.add(usize, header_len, self.source_id.len) catch + return error.StateTooLarge; + total = std.math.add(usize, total, self.document_id.len) catch + return error.StateTooLarge; + total = std.math.add(usize, total, self.resume_token.len) catch + return error.StateTooLarge; + + const encoded = try allocator.alloc(u8, total); + @memcpy(encoded[0..magic.len], magic); + + var flags: u8 = 0; + if (self.document_done) flags |= 1 << 0; + if (self.source_done) flags |= 1 << 1; + encoded[4] = flags; + encoded[5] = @intFromEnum(self.source_kind); + encoded[6] = 0; + encoded[7] = 0; + + putU64(encoded[8..16], self.sequence); + putU64(encoded[16..24], self.document_index); + putU64(encoded[24..32], self.byte_offset); + putU32(encoded[32..36], @intCast(self.source_id.len)); + putU32(encoded[36..40], @intCast(self.document_id.len)); + putU32(encoded[40..44], @intCast(self.resume_token.len)); + + var cursor: usize = header_len; + @memcpy(encoded[cursor..][0..self.source_id.len], self.source_id); + cursor += self.source_id.len; + @memcpy(encoded[cursor..][0..self.document_id.len], self.document_id); + cursor += self.document_id.len; + @memcpy(encoded[cursor..][0..self.resume_token.len], self.resume_token); + return encoded; + } + + pub fn decode(encoded: []const u8) DecodeError!StreamingState { + if (encoded.len < header_len) return error.InvalidState; + if (!std.mem.eql(u8, encoded[0..4], magic)) { + return error.UnsupportedVersion; + } + if (encoded[6] != 0 or encoded[7] != 0) return error.InvalidState; + if (encoded[4] & ~@as(u8, 0b11) != 0) return error.InvalidState; + + const source_kind: SourceKind = switch (encoded[5]) { + @intFromEnum(SourceKind.file) => .file, + @intFromEnum(SourceKind.repository) => .repository, + @intFromEnum(SourceKind.database) => .database, + @intFromEnum(SourceKind.stream) => .stream, + else => return error.InvalidState, + }; + const source_len: usize = getU32(encoded[32..36]); + const document_len: usize = getU32(encoded[36..40]); + const token_len: usize = getU32(encoded[40..44]); + + var expected_len = std.math.add(usize, header_len, source_len) catch + return error.InvalidState; + expected_len = std.math.add(usize, expected_len, document_len) catch + return error.InvalidState; + expected_len = std.math.add(usize, expected_len, token_len) catch + return error.InvalidState; + if (expected_len != encoded.len) return error.InvalidState; + + var cursor: usize = header_len; + const source_id = encoded[cursor..][0..source_len]; + cursor += source_len; + const document_id = encoded[cursor..][0..document_len]; + cursor += document_len; + const resume_token = encoded[cursor..][0..token_len]; + + return .{ + .source_kind = source_kind, + .sequence = getU64(encoded[8..16]), + .document_index = getU64(encoded[16..24]), + .byte_offset = getU64(encoded[24..32]), + .document_done = encoded[4] & (1 << 0) != 0, + .source_done = encoded[4] & (1 << 1) != 0, + .source_id = source_id, + .document_id = document_id, + .resume_token = resume_token, + }; + } +}; + +/// A borrowed view into the reader's resettable arenas. Consume or copy it +/// before calling `next`, `restore`, or `destroy` again. +pub const Chunk = struct { + bytes: []const u8, + source_kind: SourceKind, + source_id: []const u8, + document_id: []const u8, + document_index: u64, + byte_offset: u64, + sequence: u64, + is_document_start: bool, + is_document_end: bool, + state_after: StreamingState, +}; + +/// Pull-based boundary for network streams and database drivers. The adapter +/// writes directly into the reader's chunk buffer, avoiding an intermediate +/// copy. `document_id` and `resume_token` only need to live until `pull` returns. +/// +/// The reader owns the adapter lifecycle and calls `deinit` when present. +pub const ExternalSource = struct { + context: *anyopaque, + source_id: []const u8, + vtable: *const VTable, + + pub const PullResult = struct { + bytes_written: usize, + document_id: []const u8, + is_document_start: bool, + is_document_end: bool, + is_source_end: bool, + resume_token: []const u8 = "", + }; + + pub const VTable = struct { + pull: *const fn (context: *anyopaque, destination: []u8) anyerror!PullResult, + restore: ?*const fn (context: *anyopaque, token: []const u8) anyerror!void = null, + deinit: ?*const fn (context: *anyopaque) void = null, + }; +}; + +/// Deterministic depth-first traversal. Only the entries for directories on +/// the active stack are resident, so memory scales with traversal breadth and +/// depth rather than with the total repository size. +const RepositoryWalker = struct { + allocator: Allocator, + frames: std.ArrayList(Frame) = .empty, + include_vcs_metadata: bool, + + const InitError = Allocator.Error || Dir.Iterator.Error || Dir.StatFileError; + const NextError = InitError || Dir.OpenError; + + const Entry = struct { + name: []const u8, + kind: File.Kind, + }; + + const Frame = struct { + dir: Dir, + arena: ArenaAllocator, + canonical_prefix: []const u8, + entries: []Entry, + next_index: usize = 0, + owns_dir: bool, + + fn init( + allocator: Allocator, + io: Io, + dir: Dir, + parent_prefix: []const u8, + directory_name: ?[]const u8, + owns_dir: bool, + ) InitError!Frame { + errdefer if (owns_dir) dir.close(io); + var arena: ArenaAllocator = .init(allocator); + errdefer arena.deinit(); + const arena_allocator = arena.allocator(); + + const canonical_prefix = if (directory_name) |name| + try joinCanonical(arena_allocator, parent_prefix, name) + else + try arena_allocator.dupe(u8, parent_prefix); + var entries: std.ArrayList(Entry) = .empty; + + var iterator = dir.iterate(); + while (try iterator.next(io)) |entry| { + const kind = if (entry.kind == .unknown) + (try dir.statFile(io, entry.name, .{ .follow_symlinks = false })).kind + else + entry.kind; + if (kind != .file and kind != .directory) continue; + + try entries.append(arena_allocator, .{ + .name = try arena_allocator.dupe(u8, entry.name), + .kind = kind, + }); + } + + std.mem.sortUnstable(Entry, entries.items, {}, lessThanEntry); + return .{ + .dir = dir, + .arena = arena, + .canonical_prefix = canonical_prefix, + .entries = entries.items, + .owns_dir = owns_dir, + }; + } + + fn deinit(self: *Frame, io: Io) void { + self.arena.deinit(); + if (self.owns_dir) self.dir.close(io); + self.* = undefined; + } + }; + + const FileEntry = struct { + dir: Dir, + basename: []const u8, + canonical_prefix: []const u8, + }; + + fn init( + allocator: Allocator, + io: Io, + root: Dir, + include_vcs_metadata: bool, + ) InitError!RepositoryWalker { + var self: RepositoryWalker = .{ + .allocator = allocator, + .include_vcs_metadata = include_vcs_metadata, + }; + errdefer self.deinit(io); + + var root_frame = try Frame.init(allocator, io, root, "", null, false); + errdefer root_frame.deinit(io); + try self.frames.append(allocator, root_frame); + return self; + } + + fn deinit(self: *RepositoryWalker, io: Io) void { + while (self.frames.pop()) |frame_value| { + var frame = frame_value; + frame.deinit(io); + } + self.frames.deinit(self.allocator); + self.* = undefined; + } + + fn nextFile(self: *RepositoryWalker, io: Io) NextError!?FileEntry { + while (self.frames.items.len != 0) { + const frame = &self.frames.items[self.frames.items.len - 1]; + if (frame.next_index == frame.entries.len) { + var completed = self.frames.pop().?; + completed.deinit(io); + continue; + } + + const entry = &frame.entries[frame.next_index]; + frame.next_index += 1; + switch (entry.kind) { + .file => return .{ + .dir = frame.dir, + .basename = entry.name, + .canonical_prefix = frame.canonical_prefix, + }, + .directory => { + if (!self.include_vcs_metadata and isVcsDirectory(entry.name)) { + continue; + } + + const child = try frame.dir.openDir(io, entry.name, .{ .iterate = true }); + var child_frame = try Frame.init( + self.allocator, + io, + child, + frame.canonical_prefix, + entry.name, + true, + ); + self.frames.append(self.allocator, child_frame) catch |err| { + child_frame.deinit(io); + return err; + }; + }, + else => unreachable, + } + } + return null; + } + + fn lessThanEntry(_: void, left: Entry, right: Entry) bool { + return std.mem.order(u8, left.name, right.name) == .lt; + } +}; + +/// Bounded-memory raw corpus reader. +/// +/// Three arenas make the ownership periods explicit: +/// - source arena: source configuration and traversal state; +/// - document arena: the current repository/stream document identity; +/// - chunk arena: bytes and checkpoint token, reset on every operation. +/// +/// Creation returns a pointer because persistent adapters store allocators that +/// refer to these embedded arenas. Do not move the value; call `destroy` once. +pub const CorpusReader = struct { + backing_allocator: Allocator, + io: Io, + options: Options, + source_arena: ArenaAllocator, + document_arena: ArenaAllocator, + chunk_arena: ArenaAllocator, + sequence: u64, + source: Source, + + const Source = union(SourceKind) { + file: FileState, + repository: RepositoryState, + database: ExternalState, + stream: ExternalState, + }; + + const FileState = struct { + source_id: []const u8, + file: File, + size: u64, + offset: u64 = 0, + emitted_empty: bool = false, + done: bool = false, + }; + + const CurrentFile = struct { + file: File, + path: []const u8, + size: u64, + offset: u64 = 0, + emitted_empty: bool = false, + done: bool = false, + }; + + const RepositoryState = struct { + source_id: []const u8, + root: Dir, + walker: RepositoryWalker, + current: ?CurrentFile = null, + current_document_index: u64 = 0, + next_document_index: u64 = 0, + done: bool = false, + }; + + const ExternalState = struct { + adapter: ExternalSource, + source_id: []const u8, + current_document_id: []const u8 = "", + current_document_index: u64 = 0, + next_document_index: u64 = 0, + offset: u64 = 0, + in_document: bool = false, + done: bool = false, + }; + + pub const CreateError = Allocator.Error || error{InvalidChunkSize} || + File.OpenError || File.StatError || Dir.OpenError || Dir.Iterator.Error; + + pub fn openFile( + backing_allocator: Allocator, + io: Io, + base_dir: Dir, + path: []const u8, + options: Options, + ) CreateError!*CorpusReader { + if (options.chunk_size == 0) return error.InvalidChunkSize; + + const self = try allocate(backing_allocator, io, options); + errdefer freeUninitialized(self); + + const source_id = try self.source_arena.allocator().dupe(u8, path); + const file = try base_dir.openFile(io, path, .{}); + errdefer file.close(io); + + const stat = try file.stat(io); + if (stat.kind != .file) return error.Streaming; + self.source = .{ .file = .{ + .source_id = source_id, + .file = file, + .size = stat.size, + } }; + return self; + } + + pub fn openRepository( + backing_allocator: Allocator, + io: Io, + base_dir: Dir, + path: []const u8, + options: RepositoryOptions, + ) CreateError!*CorpusReader { + if (options.reader.chunk_size == 0) return error.InvalidChunkSize; + + const self = try allocate(backing_allocator, io, options.reader); + errdefer freeUninitialized(self); + + const source_id = try self.source_arena.allocator().dupe(u8, path); + const root = try base_dir.openDir(io, path, .{ .iterate = true }); + errdefer root.close(io); + const walker = try RepositoryWalker.init( + backing_allocator, + io, + root, + options.include_vcs_metadata, + ); + + self.source = .{ .repository = .{ + .source_id = source_id, + .root = root, + .walker = walker, + } }; + return self; + } + + /// Creates a database-backed reader through a driver adapter. + /// + /// TODO(data/database): add concrete SQL and object-store adapters once the + /// database component defines connection, query, and row ownership APIs. + pub fn openDatabase( + backing_allocator: Allocator, + io: Io, + adapter: ExternalSource, + options: Options, + ) CreateError!*CorpusReader { + return openExternal(backing_allocator, io, adapter, options, .database); + } + + pub fn openStream( + backing_allocator: Allocator, + io: Io, + adapter: ExternalSource, + options: Options, + ) CreateError!*CorpusReader { + return openExternal(backing_allocator, io, adapter, options, .stream); + } + + pub fn destroy(self: *CorpusReader) void { + switch (self.source) { + .file => |state| state.file.close(self.io), + .repository => |*state| { + if (state.current) |current| current.file.close(self.io); + state.walker.deinit(self.io); + state.root.close(self.io); + }, + .database, .stream => |state| { + if (state.adapter.vtable.deinit) |deinitFn| { + deinitFn(state.adapter.context); + } + }, + } + + self.chunk_arena.deinit(); + self.document_arena.deinit(); + self.source_arena.deinit(); + const backing_allocator = self.backing_allocator; + backing_allocator.destroy(self); + } + + /// Returns null only after the complete source has been consumed. + pub fn next(self: *CorpusReader) anyerror!?Chunk { + self.resetChunkArena(); + return switch (self.source) { + .file => |*state| self.nextFile(state), + .repository => |*state| self.nextRepository(state), + .database => |*state| self.nextExternal(.database, state), + .stream => |*state| self.nextExternal(.stream, state), + }; + } + + /// Moves the reader to a previously emitted `state_after` checkpoint. + /// The source must still describe the same immutable data. + pub fn restore(self: *CorpusReader, state: StreamingState) anyerror!void { + // A state taken directly from a Chunk may borrow both the document and + // chunk arenas that restoration resets. Stabilize it outside the arenas; + // restore is a control-plane operation, not part of the ingestion hot path. + const source_id = try self.backing_allocator.dupe(u8, state.source_id); + defer self.backing_allocator.free(source_id); + const document_id = try self.backing_allocator.dupe(u8, state.document_id); + defer self.backing_allocator.free(document_id); + const resume_token = try self.backing_allocator.dupe(u8, state.resume_token); + defer self.backing_allocator.free(resume_token); + var stable_state = state; + stable_state.source_id = source_id; + stable_state.document_id = document_id; + stable_state.resume_token = resume_token; + + if (stable_state.source_kind != std.meta.activeTag(self.source)) { + return error.CheckpointSourceMismatch; + } + + switch (self.source) { + .file => |*file_state| try self.restoreFile(file_state, stable_state), + .repository => |*repository_state| { + try self.restoreRepository(repository_state, stable_state); + }, + .database => |*external_state| { + try self.restoreExternal(external_state, stable_state); + }, + .stream => |*external_state| { + try self.restoreExternal(external_state, stable_state); + }, + } + self.sequence = stable_state.sequence; + self.resetChunkArena(); + } + + fn allocate( + backing_allocator: Allocator, + io: Io, + options: Options, + ) Allocator.Error!*CorpusReader { + const self = try backing_allocator.create(CorpusReader); + self.* = .{ + .backing_allocator = backing_allocator, + .io = io, + .options = options, + .source_arena = .init(backing_allocator), + .document_arena = .init(backing_allocator), + .chunk_arena = .init(backing_allocator), + .sequence = 0, + .source = undefined, + }; + return self; + } + + fn freeUninitialized(self: *CorpusReader) void { + self.chunk_arena.deinit(); + self.document_arena.deinit(); + self.source_arena.deinit(); + const backing_allocator = self.backing_allocator; + backing_allocator.destroy(self); + } + + fn openExternal( + backing_allocator: Allocator, + io: Io, + adapter: ExternalSource, + options: Options, + kind: SourceKind, + ) CreateError!*CorpusReader { + if (options.chunk_size == 0) return error.InvalidChunkSize; + + const self = try allocate(backing_allocator, io, options); + errdefer freeUninitialized(self); + const source_id = try self.source_arena.allocator().dupe(u8, adapter.source_id); + const state: ExternalState = .{ + .adapter = adapter, + .source_id = source_id, + }; + switch (kind) { + .database => self.source = .{ .database = state }, + .stream => self.source = .{ .stream = state }, + else => unreachable, + } + return self; + } + + fn nextFile(self: *CorpusReader, state: *FileState) anyerror!?Chunk { + if (state.done) return null; + + const start = state.offset; + const is_empty = state.size == 0; + if (is_empty and state.emitted_empty) { + state.done = true; + return null; + } + if (!is_empty and start >= state.size) { + state.done = true; + return null; + } + + const wanted: usize = if (is_empty) + 0 + else + @intCast(@min(@as(u64, self.options.chunk_size), state.size - start)); + const buffer = try self.chunk_arena.allocator().alloc(u8, wanted); + const bytes_read = try state.file.readPositionalAll(self.io, buffer, start); + if (bytes_read != wanted) return error.SourceTruncated; + + state.offset += bytes_read; + state.emitted_empty = is_empty; + const is_end = state.offset == state.size; + if (is_end) state.done = true; + + return self.makeFileChunk( + .file, + state.source_id, + state.source_id, + 0, + start, + buffer[0..bytes_read], + start == 0, + is_end, + is_end, + ); + } + + fn nextRepository( + self: *CorpusReader, + state: *RepositoryState, + ) anyerror!?Chunk { + if (state.done) return null; + + while (true) { + if (state.current) |*current| { + if (current.done) { + current.file.close(self.io); + state.current = null; + _ = self.document_arena.reset(.retain_capacity); + continue; + } + return try self.readRepositoryFile(state, current); + } + + if (!try self.advanceRepository(state)) { + state.done = true; + return null; + } + } + } + + fn advanceRepository( + self: *CorpusReader, + state: *RepositoryState, + ) anyerror!bool { + while (try state.walker.nextFile(self.io)) |entry| { + _ = self.document_arena.reset(.retain_capacity); + const path = try joinCanonical( + self.document_arena.allocator(), + entry.canonical_prefix, + entry.basename, + ); + const file = try entry.dir.openFile(self.io, entry.basename, .{}); + errdefer file.close(self.io); + const stat = try file.stat(self.io); + if (stat.kind != .file) { + file.close(self.io); + continue; + } + + state.current = .{ + .file = file, + .path = path, + .size = stat.size, + }; + state.current_document_index = state.next_document_index; + state.next_document_index += 1; + return true; + } + return false; + } + + fn readRepositoryFile( + self: *CorpusReader, + state: *RepositoryState, + current: *CurrentFile, + ) anyerror!Chunk { + const start = current.offset; + const is_empty = current.size == 0; + if (is_empty and current.emitted_empty) unreachable; + if (!is_empty and start >= current.size) unreachable; + + const wanted: usize = if (is_empty) + 0 + else + @intCast(@min(@as(u64, self.options.chunk_size), current.size - start)); + const buffer = try self.chunk_arena.allocator().alloc(u8, wanted); + const bytes_read = try current.file.readPositionalAll(self.io, buffer, start); + if (bytes_read != wanted) return error.SourceTruncated; + + current.offset += bytes_read; + current.emitted_empty = is_empty; + const is_end = current.offset == current.size; + if (is_end) current.done = true; + + return self.makeFileChunk( + .repository, + state.source_id, + current.path, + state.current_document_index, + start, + buffer[0..bytes_read], + start == 0, + is_end, + false, + ); + } + + fn makeFileChunk( + self: *CorpusReader, + kind: SourceKind, + source_id: []const u8, + document_id: []const u8, + document_index: u64, + byte_offset: u64, + bytes: []const u8, + is_start: bool, + is_end: bool, + source_done: bool, + ) Chunk { + const sequence = self.sequence; + self.sequence += 1; + return .{ + .bytes = bytes, + .source_kind = kind, + .source_id = source_id, + .document_id = document_id, + .document_index = document_index, + .byte_offset = byte_offset, + .sequence = sequence, + .is_document_start = is_start, + .is_document_end = is_end, + .state_after = .{ + .source_kind = kind, + .sequence = self.sequence, + .document_index = document_index, + .byte_offset = byte_offset + bytes.len, + .document_done = is_end, + .source_done = source_done, + .source_id = source_id, + .document_id = document_id, + }, + }; + } + + fn nextExternal( + self: *CorpusReader, + kind: SourceKind, + state: *ExternalState, + ) anyerror!?Chunk { + if (state.done) return null; + + const buffer = try self.chunk_arena.allocator().alloc(u8, self.options.chunk_size); + const result = try state.adapter.vtable.pull(state.adapter.context, buffer); + if (result.bytes_written > buffer.len) return error.AdapterOverflow; + if (result.bytes_written == 0 and + !result.is_document_end and + !result.is_source_end) + { + return error.AdapterMadeNoProgress; + } + + if (result.is_document_start) { + if (state.in_document) return error.AdapterProtocolViolation; + _ = self.document_arena.reset(.retain_capacity); + state.current_document_id = try self.document_arena.allocator().dupe( + u8, + result.document_id, + ); + state.current_document_index = state.next_document_index; + state.next_document_index += 1; + state.offset = 0; + state.in_document = true; + } else if (!state.in_document) { + if (result.is_source_end and result.bytes_written == 0) { + state.done = true; + return null; + } + return error.AdapterProtocolViolation; + } else if (!std.mem.eql(u8, state.current_document_id, result.document_id)) { + return error.AdapterProtocolViolation; + } + + if (result.is_source_end and !result.is_document_end) { + return error.AdapterProtocolViolation; + } + + const token = try self.chunk_arena.allocator().dupe(u8, result.resume_token); + const start = state.offset; + state.offset += result.bytes_written; + const document_index = state.current_document_index; + const document_id = state.current_document_id; + if (result.is_document_end) state.in_document = false; + if (result.is_source_end) state.done = true; + + const sequence = self.sequence; + self.sequence += 1; + return .{ + .bytes = buffer[0..result.bytes_written], + .source_kind = kind, + .source_id = state.source_id, + .document_id = document_id, + .document_index = document_index, + .byte_offset = start, + .sequence = sequence, + .is_document_start = result.is_document_start, + .is_document_end = result.is_document_end, + .state_after = .{ + .source_kind = kind, + .sequence = self.sequence, + .document_index = document_index, + .byte_offset = state.offset, + .document_done = result.is_document_end, + .source_done = result.is_source_end, + .source_id = state.source_id, + .document_id = document_id, + .resume_token = token, + }, + }; + } + + fn restoreFile( + self: *CorpusReader, + file_state: *FileState, + state: StreamingState, + ) !void { + _ = self; + if (!std.mem.eql(u8, file_state.source_id, state.source_id) or + !std.mem.eql(u8, file_state.source_id, state.document_id) or + state.document_index != 0 or + state.byte_offset > file_state.size) + { + return error.CheckpointSourceMismatch; + } + file_state.offset = state.byte_offset; + file_state.emitted_empty = file_state.size == 0 and state.document_done; + file_state.done = state.source_done; + } + + fn restoreRepository( + self: *CorpusReader, + repository_state: *RepositoryState, + state: StreamingState, + ) anyerror!void { + if (!std.mem.eql(u8, repository_state.source_id, state.source_id)) { + return error.CheckpointSourceMismatch; + } + + try self.resetRepository(repository_state); + while (try self.advanceRepository(repository_state)) { + const current = &repository_state.current.?; + if (std.mem.eql(u8, current.path, state.document_id)) { + if (current.size < state.byte_offset) return error.CheckpointSourceMismatch; + current.offset = state.byte_offset; + current.emitted_empty = current.size == 0 and state.document_done; + current.done = state.document_done; + repository_state.current_document_index = state.document_index; + repository_state.next_document_index = state.document_index + 1; + repository_state.done = state.source_done; + return; + } + current.file.close(self.io); + repository_state.current = null; + } + return error.CheckpointSourceMismatch; + } + + fn resetRepository(self: *CorpusReader, state: *RepositoryState) anyerror!void { + if (state.current) |current| current.file.close(self.io); + state.current = null; + const new_walker = try RepositoryWalker.init( + self.backing_allocator, + self.io, + state.root, + state.walker.include_vcs_metadata, + ); + state.walker.deinit(self.io); + state.walker = new_walker; + state.current_document_index = 0; + state.next_document_index = 0; + state.done = false; + _ = self.document_arena.reset(.retain_capacity); + } + + fn restoreExternal( + self: *CorpusReader, + external_state: *ExternalState, + state: StreamingState, + ) anyerror!void { + if (!std.mem.eql(u8, external_state.source_id, state.source_id)) { + return error.CheckpointSourceMismatch; + } + const restoreFn = external_state.adapter.vtable.restore orelse + return error.RestoreUnsupported; + try restoreFn(external_state.adapter.context, state.resume_token); + + _ = self.document_arena.reset(.retain_capacity); + external_state.current_document_index = state.document_index; + external_state.next_document_index = state.document_index + 1; + external_state.offset = state.byte_offset; + external_state.in_document = !state.document_done; + external_state.done = state.source_done; + if (external_state.in_document) { + external_state.current_document_id = try self.document_arena.allocator().dupe( + u8, + state.document_id, + ); + } else { + external_state.current_document_id = ""; + } + } + + fn resetChunkArena(self: *CorpusReader) void { + const mode: ArenaAllocator.ResetMode = if (self.options.retained_chunk_bytes) |limit| + .{ .retain_with_limit = limit } + else + .retain_capacity; + _ = self.chunk_arena.reset(mode); + } +}; + +fn isVcsDirectory(name: []const u8) bool { + return std.mem.eql(u8, name, ".git") or + std.mem.eql(u8, name, ".hg") or + std.mem.eql(u8, name, ".svn"); +} + +fn joinCanonical( + allocator: Allocator, + prefix: []const u8, + name: []const u8, +) Allocator.Error![]u8 { + if (prefix.len == 0) return allocator.dupe(u8, name); + + const result = try allocator.alloc(u8, prefix.len + 1 + name.len); + @memcpy(result[0..prefix.len], prefix); + result[prefix.len] = '/'; + @memcpy(result[prefix.len + 1 ..], name); + return result; +} + +fn putU32(destination: *[4]u8, value: u32) void { + for (destination, 0..) |*byte, index| { + byte.* = @truncate(value >> @intCast(index * 8)); + } +} + +fn putU64(destination: *[8]u8, value: u64) void { + for (destination, 0..) |*byte, index| { + byte.* = @truncate(value >> @intCast(index * 8)); + } +} + +fn getU32(source: *const [4]u8) u32 { + var value: u32 = 0; + for (source, 0..) |byte, index| { + value |= @as(u32, byte) << @intCast(index * 8); + } + return value; +} + +fn getU64(source: *const [8]u8) u64 { + var value: u64 = 0; + for (source, 0..) |byte, index| { + value |= @as(u64, byte) << @intCast(index * 8); + } + return value; +} + +test "streaming state has a transport-stable encoding" { + const original: StreamingState = .{ + .source_kind = .database, + .sequence = 42, + .document_index = 7, + .byte_offset = 1_000_000, + .document_done = true, + .source_done = false, + .source_id = "books-db", + .document_id = "row/99", + .resume_token = &.{ 0, 1, 2, 255 }, + }; + + const encoded = try original.encode(std.testing.allocator); + defer std.testing.allocator.free(encoded); + const decoded = try StreamingState.decode(encoded); + + try std.testing.expectEqual(original.source_kind, decoded.source_kind); + try std.testing.expectEqual(original.sequence, decoded.sequence); + try std.testing.expectEqual(original.document_index, decoded.document_index); + try std.testing.expectEqual(original.byte_offset, decoded.byte_offset); + try std.testing.expectEqual(original.document_done, decoded.document_done); + try std.testing.expectEqual(original.source_done, decoded.source_done); + try std.testing.expectEqualStrings(original.source_id, decoded.source_id); + try std.testing.expectEqualStrings(original.document_id, decoded.document_id); + try std.testing.expectEqualSlices(u8, original.resume_token, decoded.resume_token); +} + +const TestStream = struct { + bytes: []const u8, + position: usize = 0, + token: [8]u8 = @splat(0), + + const vtable: ExternalSource.VTable = .{ + .pull = pull, + .restore = restore, + }; + + fn source(self: *TestStream) ExternalSource { + return .{ + .context = self, + .source_id = "test-stream", + .vtable = &vtable, + }; + } + + fn pull(context: *anyopaque, destination: []u8) anyerror!ExternalSource.PullResult { + const self: *TestStream = @ptrCast(@alignCast(context)); + if (self.position == self.bytes.len) { + return .{ + .bytes_written = 0, + .document_id = "", + .is_document_start = false, + .is_document_end = false, + .is_source_end = true, + }; + } + + const start = self.position; + const count = @min(destination.len, self.bytes.len - start); + @memcpy(destination[0..count], self.bytes[start..][0..count]); + self.position += count; + putU64(&self.token, self.position); + const is_end = self.position == self.bytes.len; + return .{ + .bytes_written = count, + .document_id = "stream-document", + .is_document_start = start == 0, + .is_document_end = is_end, + .is_source_end = is_end, + .resume_token = &self.token, + }; + } + + fn restore(context: *anyopaque, token: []const u8) anyerror!void { + const self: *TestStream = @ptrCast(@alignCast(context)); + if (token.len != self.token.len) return error.InvalidToken; + const position = getU64(token[0..8]); + if (position > self.bytes.len) return error.InvalidToken; + self.position = @intCast(position); + } +}; + +test "stream adapters write directly into chunks and restore from tokens" { + var stream: TestStream = .{ .bytes = "streamed corpus" }; + const reader = try CorpusReader.openStream( + std.testing.allocator, + std.testing.io, + stream.source(), + .{ .chunk_size = 8 }, + ); + defer reader.destroy(); + + const first = (try reader.next()).?; + try std.testing.expectEqualStrings("streamed", first.bytes); + try std.testing.expect(first.is_document_start); + const encoded = try first.state_after.encode(std.testing.allocator); + defer std.testing.allocator.free(encoded); + + // Restoring directly from a borrowed chunk state must stabilize its token + // before resetting the chunk arena. + try reader.restore(first.state_after); + const directly_replayed = (try reader.next()).?; + try std.testing.expectEqualStrings(" corpus", directly_replayed.bytes); + try std.testing.expect(directly_replayed.is_document_end); + + try reader.restore(try StreamingState.decode(encoded)); + const replayed = (try reader.next()).?; + try std.testing.expectEqualStrings(" corpus", replayed.bytes); + try std.testing.expectEqual(@as(u64, 1), replayed.sequence); +} + +test "file source emits bounded chunks and restores exactly" { + const io = std.testing.io; + var temporary = std.testing.tmpDir(.{}); + defer temporary.cleanup(); + try temporary.dir.writeFile(io, .{ + .sub_path = "corpus.txt", + .data = "abcdefghij", + }); + + const reader = try CorpusReader.openFile( + std.testing.allocator, + io, + temporary.dir, + "corpus.txt", + .{ .chunk_size = 4 }, + ); + defer reader.destroy(); + + const first = (try reader.next()).?; + try std.testing.expectEqualStrings("abcd", first.bytes); + try std.testing.expect(first.is_document_start); + try std.testing.expect(!first.is_document_end); + + const encoded = try first.state_after.encode(std.testing.allocator); + defer std.testing.allocator.free(encoded); + + const second = (try reader.next()).?; + try std.testing.expectEqualStrings("efgh", second.bytes); + + try reader.restore(try StreamingState.decode(encoded)); + const replayed = (try reader.next()).?; + try std.testing.expectEqualStrings("efgh", replayed.bytes); + try std.testing.expectEqual(@as(u64, 1), replayed.sequence); + + const last = (try reader.next()).?; + try std.testing.expectEqualStrings("ij", last.bytes); + try std.testing.expect(last.is_document_end); + try std.testing.expect(last.state_after.source_done); + try std.testing.expectEqual(@as(?Chunk, null), try reader.next()); +} + +test "repository source skips VCS metadata and preserves document boundaries" { + const io = std.testing.io; + var temporary = std.testing.tmpDir(.{ .iterate = true }); + defer temporary.cleanup(); + try temporary.dir.writeFile(io, .{ .sub_path = "one.txt", .data = "one" }); + try temporary.dir.writeFile(io, .{ .sub_path = "z-last.txt", .data = "z" }); + try temporary.dir.createDir(io, ".git", .default_dir); + var git_dir = try temporary.dir.openDir(io, ".git", .{}); + defer git_dir.close(io); + try git_dir.writeFile(io, .{ .sub_path = "secret", .data = "skip" }); + + const reader = try CorpusReader.openRepository( + std.testing.allocator, + io, + temporary.dir, + ".", + .{ .reader = .{ .chunk_size = 2 } }, + ); + defer reader.destroy(); + + const first = (try reader.next()).?; + try std.testing.expectEqualStrings("one.txt", first.document_id); + try std.testing.expectEqualStrings("on", first.bytes); + try std.testing.expect(first.is_document_start); + const checkpoint = try first.state_after.encode(std.testing.allocator); + defer std.testing.allocator.free(checkpoint); + + const second = (try reader.next()).?; + try std.testing.expectEqualStrings("e", second.bytes); + try std.testing.expect(second.is_document_end); + + const third = (try reader.next()).?; + try std.testing.expectEqualStrings("z-last.txt", third.document_id); + try std.testing.expectEqualStrings("z", third.bytes); + try std.testing.expectEqual(@as(?Chunk, null), try reader.next()); + + try reader.restore(try StreamingState.decode(checkpoint)); + const replayed = (try reader.next()).?; + try std.testing.expectEqualStrings("e", replayed.bytes); + const deterministic_next = (try reader.next()).?; + try std.testing.expectEqualStrings("z-last.txt", deterministic_next.document_id); +} + +test "empty files are represented as one boundary chunk" { + const io = std.testing.io; + var temporary = std.testing.tmpDir(.{}); + defer temporary.cleanup(); + try temporary.dir.writeFile(io, .{ .sub_path = "empty", .data = "" }); + + const reader = try CorpusReader.openFile( + std.testing.allocator, + io, + temporary.dir, + "empty", + .{}, + ); + defer reader.destroy(); + + const chunk = (try reader.next()).?; + try std.testing.expectEqual(@as(usize, 0), chunk.bytes.len); + try std.testing.expect(chunk.is_document_start); + try std.testing.expect(chunk.is_document_end); + try std.testing.expectEqual(@as(?Chunk, null), try reader.next()); +} diff --git a/data/corpus-reader/root.zig b/data/corpus-reader/root.zig new file mode 100644 index 0000000..9ec4e15 --- /dev/null +++ b/data/corpus-reader/root.zig @@ -0,0 +1,13 @@ +//! Raw, bounded-memory corpus ingestion. + +pub const CorpusReader = @import("corpus_reader.zig").CorpusReader; +pub const Chunk = @import("corpus_reader.zig").Chunk; +pub const ExternalSource = @import("corpus_reader.zig").ExternalSource; +pub const Options = @import("corpus_reader.zig").Options; +pub const RepositoryOptions = @import("corpus_reader.zig").RepositoryOptions; +pub const SourceKind = @import("corpus_reader.zig").SourceKind; +pub const StreamingState = @import("corpus_reader.zig").StreamingState; + +test { + _ = @import("corpus_reader.zig"); +} diff --git a/data/deduplicator/README.md b/data/deduplicator/README.md new file mode 100644 index 0000000..6dbd53f --- /dev/null +++ b/data/deduplicator/README.md @@ -0,0 +1,36 @@ +# Deduplicator + +The deduplicator consumes document fragments and classifies a document only at +its end boundary. It supports three complementary signals: + +- a 256-bit, chunk-boundary-independent BLAKE3 Merkle fingerprint for exact + document matches; +- a streaming 64-bit SimHash over normalized word features for approximate + document matches; +- sampled rolling windows, verified with 128-bit BLAKE3 fingerprints, for + exact duplicate text spans. + +`Deduplicator` retains only fixed-size hashing state and one span-sized ring +buffer per active document. Per-call results use a resettable arena. The index +is an explicit backend interface: `InMemoryIndex` is provided for a single +worker, while a remote implementation can move fingerprints and transactional +state to a memory service. Span probes are batched once per input chunk to +avoid turning sampled windows into individual network round trips. + +An active document can be serialized with `checkpoint`, transported, decoded +as `StreamingState`, and resumed with `restore`. The checkpoint includes the +Merkle peaks, partial block, SimHash accumulator, span ring, and the backend's +uncommitted transaction token. + +The backend transaction starts with the document and commits only if the final +classification is unique. Exact or approximate duplicates never publish their +staged spans. A production distributed backend must make `finishDocument` +atomic so two workers cannot both claim the same content as canonical. + +Approximate lookup uses four 16-bit SimHash bands, so the configured Hamming +distance is capped at three. This guarantees that every qualifying signature +shares at least one band with its candidate. + +Returned slices remain valid until the next call to `observe`, `abortDocument`, +or `destroy`. Canonical identifiers owned by the in-memory index remain valid +for the lifetime of that index. diff --git a/data/deduplicator/deduplicator.zig b/data/deduplicator/deduplicator.zig new file mode 100644 index 0000000..950a0a0 --- /dev/null +++ b/data/deduplicator/deduplicator.zig @@ -0,0 +1,1465 @@ +const std = @import("std"); + +const Allocator = std.mem.Allocator; +const ArenaAllocator = std.heap.ArenaAllocator; +const Blake3 = std.crypto.hash.Blake3; +const CorpusChunk = @import("../corpus-reader/root.zig").Chunk; +const CleanedChunk = @import("../text-cleaner/root.zig").CleanedChunk; + +pub const ExactDigest = [32]u8; +pub const SpanDigest = [16]u8; + +pub const Boundary = struct { + document_start: bool = false, + document_end: bool = false, + + pub const whole: Boundary = .{ + .document_start = true, + .document_end = true, + }; +}; + +pub const DocumentChunk = struct { + document_id: []const u8, + bytes: []const u8, + boundary: Boundary, +}; + +pub const DocumentFingerprint = struct { + document_id: []const u8, + exact: ExactDigest, + simhash: u64, + feature_count: u64, + byte_count: u64, +}; + +pub const SpanFingerprint = struct { + digest: SpanDigest, + start_byte: u64, + end_byte: u64, +}; + +pub const SpanOccurrence = struct { + document_id: []const u8, + start_byte: u64, + end_byte: u64, +}; + +pub const DuplicateSpan = struct { + current_start_byte: u64, + current_end_byte: u64, + canonical_document_id: []const u8, + canonical_start_byte: u64, + canonical_end_byte: u64, +}; + +pub const CanonicalMatch = struct { + document_id: []const u8, +}; + +pub const ApproximateMatch = struct { + document_id: []const u8, + hamming_distance: u8, +}; + +pub const Classification = union(enum) { + unique, + exact_duplicate: CanonicalMatch, + approximate_duplicate: ApproximateMatch, +}; + +pub const Status = union(enum) { + pending, + complete: Classification, +}; + +/// All borrowed slices in an observation are valid until the next mutating +/// deduplicator call. +pub const Observation = struct { + document_id: []const u8, + status: Status, + duplicate_spans: []const DuplicateSpan, + fingerprint: ?DocumentFingerprint = null, +}; + +pub const ApproximatePolicy = struct { + enabled: bool = true, + /// Four 16-bit bands guarantee candidate discovery only through distance 3. + max_hamming_distance: u8 = 3, + min_features: u64 = 8, +}; + +pub const SpanPolicy = struct { + // TODO(data/tokenizer): add token-shingle MinHash for approximate span + // matches after the tokenizer exposes versioned streaming word boundaries. + enabled: bool = true, + window_bytes: usize = 128, + /// A window is retained when the low `sampling_bits` rolling-hash bits are 0. + sampling_bits: u8 = 6, + min_start_distance: u64 = 64, +}; + +pub const Options = struct { + approximate: ApproximatePolicy = .{}, + spans: SpanPolicy = .{}, + retained_output_bytes: ?usize = null, +}; + +/// Validated view of a transportable deduplicator checkpoint. The encoded +/// bytes are borrowed; `Deduplicator.checkpoint` returns caller-owned storage. +pub const StreamingState = struct { + encoded: []const u8, + + pub const DecodeError = error{ + InvalidState, + UnsupportedVersion, + }; + + pub fn decode(encoded: []const u8) DecodeError!StreamingState { + try validateEncodedState(encoded); + return .{ .encoded = encoded }; + } +}; + +/// Transactional index boundary for memory/compute disaggregation. `finish` +/// must atomically classify and publish a unique document. The backend may +/// borrow all input only for the duration of a call; returned identifiers must +/// remain valid until the next call on the same backend context. +pub const Index = struct { + context: *anyopaque, + vtable: *const VTable, + version: u64 = 1, + + pub const VTable = struct { + startDocument: *const fn (*anyopaque, []const u8) anyerror!void, + probeSpans: *const fn ( + *anyopaque, + []const SpanFingerprint, + []?SpanOccurrence, + ) anyerror!void, + finishDocument: *const fn ( + *anyopaque, + DocumentFingerprint, + ApproximatePolicy, + ) anyerror!Classification, + abortDocument: *const fn (*anyopaque) void, + checkpointDocument: ?*const fn (*anyopaque, Allocator) anyerror![]const u8 = null, + restoreDocument: ?*const fn (*anyopaque, []const u8, []const u8) anyerror!void = null, + deinit: ?*const fn (*anyopaque) void = null, + }; +}; + +const band_count = 4; +const StoredDocument = struct { + document_id: []const u8, + exact: ExactDigest, + simhash: u64, + feature_count: u64, + next_by_band: [band_count]?usize, + visited_epoch: u64 = 0, +}; + +const StoredSpan = struct { + document_index: usize, + start_byte: u64, + end_byte: u64, +}; + +/// Single-threaded reference backend. Give each worker its own instance, or +/// protect it externally. A distributed backend should preserve the same +/// transaction semantics with a compare-and-publish operation. +pub const InMemoryIndex = struct { + allocator: Allocator, + id_arena: ArenaAllocator, + exact: std.AutoHashMapUnmanaged(ExactDigest, usize) = .empty, + bands: std.AutoHashMapUnmanaged(u32, usize) = .empty, + spans: std.AutoHashMapUnmanaged(SpanDigest, StoredSpan) = .empty, + documents: std.ArrayList(StoredDocument) = .empty, + staged_spans: std.ArrayList(SpanFingerprint) = .empty, + active: bool = false, + query_epoch: u64 = 0, + + pub fn create(allocator: Allocator) Allocator.Error!*InMemoryIndex { + const self = try allocator.create(InMemoryIndex); + self.* = .{ + .allocator = allocator, + .id_arena = .init(allocator), + }; + return self; + } + + pub fn destroy(self: *InMemoryIndex) void { + self.staged_spans.deinit(self.allocator); + self.documents.deinit(self.allocator); + self.spans.deinit(self.allocator); + self.bands.deinit(self.allocator); + self.exact.deinit(self.allocator); + self.id_arena.deinit(); + const allocator = self.allocator; + allocator.destroy(self); + } + + /// Transfers backend ownership to a successfully-created Deduplicator. + pub fn ownedIndex(self: *InMemoryIndex) Index { + return .{ .context = self, .vtable = &owned_vtable }; + } + + /// The caller remains responsible for destroying the backend. + pub fn borrowedIndex(self: *InMemoryIndex) Index { + return .{ .context = self, .vtable = &borrowed_vtable }; + } + + pub fn documentCount(self: *const InMemoryIndex) usize { + return self.documents.items.len; + } + + fn indexStart(context: *anyopaque, _: []const u8) anyerror!void { + const self: *InMemoryIndex = @ptrCast(@alignCast(context)); + if (self.active) return error.IndexTransactionActive; + self.staged_spans.clearRetainingCapacity(); + self.active = true; + } + + fn indexProbeSpans( + context: *anyopaque, + fingerprints: []const SpanFingerprint, + matches: []?SpanOccurrence, + ) anyerror!void { + const self: *InMemoryIndex = @ptrCast(@alignCast(context)); + if (!self.active) return error.IndexTransactionNotActive; + if (fingerprints.len != matches.len) return error.InvalidProbeBatch; + try self.staged_spans.ensureUnusedCapacity(self.allocator, fingerprints.len); + + for (fingerprints, matches) |fingerprint, *match| { + if (self.spans.get(fingerprint.digest)) |occurrence| { + const document = self.documents.items[occurrence.document_index]; + match.* = .{ + .document_id = document.document_id, + .start_byte = occurrence.start_byte, + .end_byte = occurrence.end_byte, + }; + } else { + match.* = null; + } + self.staged_spans.appendAssumeCapacity(fingerprint); + } + } + + fn indexFinish( + context: *anyopaque, + fingerprint: DocumentFingerprint, + policy: ApproximatePolicy, + ) anyerror!Classification { + const self: *InMemoryIndex = @ptrCast(@alignCast(context)); + if (!self.active) return error.IndexTransactionNotActive; + + if (self.exact.get(fingerprint.exact)) |document_index| { + self.closeTransaction(); + return .{ .exact_duplicate = .{ + .document_id = self.documents.items[document_index].document_id, + } }; + } + + if (policy.enabled and fingerprint.feature_count >= policy.min_features) { + if (self.closestApproximate(fingerprint, policy)) |match| { + self.closeTransaction(); + return .{ .approximate_duplicate = match }; + } + } + + try self.commit(fingerprint, policy.enabled); + self.closeTransaction(); + return .unique; + } + + fn closestApproximate( + self: *InMemoryIndex, + fingerprint: DocumentFingerprint, + policy: ApproximatePolicy, + ) ?ApproximateMatch { + self.advanceQueryEpoch(); + var closest_index: ?usize = null; + var closest_distance: u8 = std.math.maxInt(u8); + + for (0..band_count) |band_index| { + var cursor = self.bands.get(bandKey(fingerprint.simhash, band_index)); + while (cursor) |document_index| { + const candidate = &self.documents.items[document_index]; + cursor = candidate.next_by_band[band_index]; + if (candidate.visited_epoch == self.query_epoch) continue; + candidate.visited_epoch = self.query_epoch; + if (candidate.feature_count >= policy.min_features) { + const distance: u8 = @intCast(@popCount( + fingerprint.simhash ^ candidate.simhash, + )); + if (distance <= policy.max_hamming_distance and + (distance < closest_distance or + (distance == closest_distance and + (closest_index == null or + document_index < closest_index.?)))) + { + closest_index = document_index; + closest_distance = distance; + } + } + } + } + + const index = closest_index orelse return null; + return .{ + .document_id = self.documents.items[index].document_id, + .hamming_distance = closest_distance, + }; + } + + fn advanceQueryEpoch(self: *InMemoryIndex) void { + if (self.query_epoch == std.math.maxInt(u64)) { + for (self.documents.items) |*document| document.visited_epoch = 0; + self.query_epoch = 1; + } else { + self.query_epoch += 1; + } + } + + /// Reserves every fallible allocation before publishing any index entry. + fn commit( + self: *InMemoryIndex, + fingerprint: DocumentFingerprint, + index_approximate: bool, + ) anyerror!void { + try self.documents.ensureUnusedCapacity(self.allocator, 1); + try self.exact.ensureUnusedCapacity(self.allocator, 1); + if (index_approximate) { + try self.bands.ensureUnusedCapacity(self.allocator, band_count); + } + const staged_count = std.math.cast(u32, self.staged_spans.items.len) orelse + return error.IndexCapacityExceeded; + try self.spans.ensureUnusedCapacity(self.allocator, staged_count); + const owned_id = try self.id_arena.allocator().dupe(u8, fingerprint.document_id); + + const document_index = self.documents.items.len; + var next_by_band: [band_count]?usize = @splat(null); + if (index_approximate) { + for (0..band_count) |band_index| { + next_by_band[band_index] = self.bands.get( + bandKey(fingerprint.simhash, band_index), + ); + } + } + + self.documents.appendAssumeCapacity(.{ + .document_id = owned_id, + .exact = fingerprint.exact, + .simhash = fingerprint.simhash, + .feature_count = fingerprint.feature_count, + .next_by_band = next_by_band, + }); + self.exact.putAssumeCapacityNoClobber(fingerprint.exact, document_index); + if (index_approximate) { + for (0..band_count) |band_index| { + self.bands.putAssumeCapacity( + bandKey(fingerprint.simhash, band_index), + document_index, + ); + } + } + for (self.staged_spans.items) |span| { + if (!self.spans.contains(span.digest)) { + self.spans.putAssumeCapacityNoClobber(span.digest, .{ + .document_index = document_index, + .start_byte = span.start_byte, + .end_byte = span.end_byte, + }); + } + } + } + + fn indexAbort(context: *anyopaque) void { + const self: *InMemoryIndex = @ptrCast(@alignCast(context)); + self.closeTransaction(); + } + + fn indexCheckpoint(context: *anyopaque, allocator: Allocator) anyerror![]const u8 { + const self: *InMemoryIndex = @ptrCast(@alignCast(context)); + if (!self.active) return error.IndexTransactionNotActive; + if (self.staged_spans.items.len > std.math.maxInt(u32)) { + return error.IndexCapacityExceeded; + } + const item_bytes = std.math.mul(usize, self.staged_spans.items.len, 32) catch + return error.IndexCapacityExceeded; + const encoded = try allocator.alloc(u8, 8 + item_bytes); + @memcpy(encoded[0..4], "IM01"); + putU32(encoded[4..8], @intCast(self.staged_spans.items.len)); + var cursor: usize = 8; + for (self.staged_spans.items) |span| { + @memcpy(encoded[cursor..][0..16], &span.digest); + putU64(encoded[cursor + 16 ..][0..8], span.start_byte); + putU64(encoded[cursor + 24 ..][0..8], span.end_byte); + cursor += 32; + } + return encoded; + } + + fn indexRestore( + context: *anyopaque, + _: []const u8, + token: []const u8, + ) anyerror!void { + const self: *InMemoryIndex = @ptrCast(@alignCast(context)); + if (self.active) return error.IndexTransactionActive; + if (token.len < 8 or !std.mem.eql(u8, token[0..4], "IM01")) { + return error.InvalidIndexCheckpoint; + } + const count: usize = getU32(token[4..8]); + const item_bytes = std.math.mul(usize, count, 32) catch + return error.InvalidIndexCheckpoint; + if (token.len != 8 + item_bytes) return error.InvalidIndexCheckpoint; + try self.staged_spans.ensureUnusedCapacity(self.allocator, count); + self.staged_spans.clearRetainingCapacity(); + var cursor: usize = 8; + for (0..count) |_| { + var digest: SpanDigest = undefined; + @memcpy(&digest, token[cursor..][0..16]); + const start_byte = getU64(token[cursor + 16 ..][0..8]); + const end_byte = getU64(token[cursor + 24 ..][0..8]); + if (end_byte < start_byte) return error.InvalidIndexCheckpoint; + self.staged_spans.appendAssumeCapacity(.{ + .digest = digest, + .start_byte = start_byte, + .end_byte = end_byte, + }); + cursor += 32; + } + self.active = true; + } + + fn closeTransaction(self: *InMemoryIndex) void { + self.staged_spans.clearRetainingCapacity(); + self.active = false; + } + + fn indexDeinit(context: *anyopaque) void { + const self: *InMemoryIndex = @ptrCast(@alignCast(context)); + self.destroy(); + } + + const borrowed_vtable: Index.VTable = .{ + .startDocument = indexStart, + .probeSpans = indexProbeSpans, + .finishDocument = indexFinish, + .abortDocument = indexAbort, + .checkpointDocument = indexCheckpoint, + .restoreDocument = indexRestore, + }; + + const owned_vtable: Index.VTable = .{ + .startDocument = indexStart, + .probeSpans = indexProbeSpans, + .finishDocument = indexFinish, + .abortDocument = indexAbort, + .checkpointDocument = indexCheckpoint, + .restoreDocument = indexRestore, + .deinit = indexDeinit, + }; +}; + +const exact_block_bytes = 4096; +const ExactHasher = struct { + partial: [exact_block_bytes]u8 = @splat(0), + partial_len: usize = 0, + peaks: [64]ExactDigest = @splat(@splat(0)), + occupied: u64 = 0, + block_count: u64 = 0, + total_bytes: u64 = 0, + + fn update(self: *ExactHasher, input: []const u8) error{DocumentTooLarge}!void { + self.total_bytes = std.math.add(u64, self.total_bytes, input.len) catch + return error.DocumentTooLarge; + + var remaining = input; + while (remaining.len != 0) { + const take = @min(exact_block_bytes - self.partial_len, remaining.len); + @memcpy(self.partial[self.partial_len..][0..take], remaining[0..take]); + self.partial_len += take; + remaining = remaining[take..]; + if (self.partial_len == exact_block_bytes) { + self.addLeaf(hashLeaf(self.partial[0..])); + self.partial_len = 0; + self.block_count += 1; + } + } + } + + fn digest(self: ExactHasher) ExactDigest { + var copy = self; + if (copy.partial_len != 0 or copy.block_count == 0) { + copy.addLeaf(hashLeaf(copy.partial[0..copy.partial_len])); + copy.block_count += 1; + } + + var hash = Blake3.init(.{}); + hash.update("chi.dedup.root.v1\x00"); + var number: [8]u8 = undefined; + putU64(&number, copy.total_bytes); + hash.update(&number); + putU64(&number, copy.block_count); + hash.update(&number); + for (0..64) |reverse_index| { + const level = 63 - reverse_index; + if (copy.occupied & (@as(u64, 1) << @intCast(level)) != 0) { + const encoded_level: [1]u8 = .{@intCast(level)}; + hash.update(&encoded_level); + hash.update(©.peaks[level]); + } + } + var result: ExactDigest = undefined; + hash.final(&result); + return result; + } + + fn addLeaf(self: *ExactHasher, leaf: ExactDigest) void { + var node = leaf; + var level: usize = 0; + while (self.occupied & (@as(u64, 1) << @intCast(level)) != 0) { + node = hashParent(self.peaks[level], node); + self.occupied &= ~(@as(u64, 1) << @intCast(level)); + level += 1; + } + self.peaks[level] = node; + self.occupied |= @as(u64, 1) << @intCast(level); + } +}; + +const SimHasher = struct { + weights: [64]i64 = @splat(0), + token_hash: u64 = fnv_offset, + token_len: u64 = 0, + feature_count: u64 = 0, + + fn update(self: *SimHasher, bytes: []const u8) error{DocumentTooLarge}!void { + for (bytes) |byte| { + if (isTokenByte(byte)) { + const normalized = if (byte >= 'A' and byte <= 'Z') byte + 32 else byte; + self.token_hash ^= normalized; + self.token_hash *%= fnv_prime; + self.token_len = std.math.add(u64, self.token_len, 1) catch + return error.DocumentTooLarge; + } else { + try self.finishToken(); + } + } + } + + fn finishToken(self: *SimHasher) error{DocumentTooLarge}!void { + if (self.token_len == 0) return; + const mixed = mix64(self.token_hash ^ (self.token_len *% 0x9e3779b97f4a7c15)); + for (&self.weights, 0..) |*weight, bit| { + if (mixed & (@as(u64, 1) << @intCast(bit)) != 0) { + weight.* +|= 1; + } else { + weight.* -|= 1; + } + } + self.feature_count = std.math.add(u64, self.feature_count, 1) catch + return error.DocumentTooLarge; + self.token_hash = fnv_offset; + self.token_len = 0; + } + + fn digest(self: SimHasher) u64 { + var result: u64 = 0; + for (self.weights, 0..) |weight, bit| { + if (weight > 0) result |= @as(u64, 1) << @intCast(bit); + } + return result; + } +}; + +const SpanHasher = struct { + ring: []u8 = &.{}, + next: usize = 0, + filled: usize = 0, + rolling: u64 = 0, + outgoing_factor: u64 = 1, + last_sample_start: ?u64 = null, + + fn init(ring: []u8) SpanHasher { + var factor: u64 = 1; + for (0..ring.len) |_| factor *%= rolling_base; + return .{ .ring = ring, .outgoing_factor = factor }; + } +}; + +const DocumentState = struct { + in_document: bool = false, + document_id: []const u8 = "", + exact: ExactHasher = .{}, + approximate: SimHasher = .{}, + spans: SpanHasher = .{}, +}; + +pub const Deduplicator = struct { + backing_allocator: Allocator, + index: Index, + options: Options, + document_arena: ArenaAllocator, + output_arena: ArenaAllocator, + state: DocumentState = .{}, + + pub const CreateError = Allocator.Error || error{ + InvalidApproximateDistance, + InvalidSpanPolicy, + }; + + pub fn create( + backing_allocator: Allocator, + index: Index, + options: Options, + ) CreateError!*Deduplicator { + try validateOptions(options); + const self = try backing_allocator.create(Deduplicator); + self.* = .{ + .backing_allocator = backing_allocator, + .index = index, + .options = options, + .document_arena = .init(backing_allocator), + .output_arena = .init(backing_allocator), + }; + return self; + } + + pub fn destroy(self: *Deduplicator) void { + if (self.state.in_document) self.index.vtable.abortDocument(self.index.context); + self.output_arena.deinit(); + self.document_arena.deinit(); + if (self.index.vtable.deinit) |deinitFn| deinitFn(self.index.context); + const allocator = self.backing_allocator; + allocator.destroy(self); + } + + pub fn observe(self: *Deduplicator, chunk: DocumentChunk) anyerror!Observation { + try self.validateBoundary(chunk); + self.resetOutputArena(); + + if (chunk.boundary.document_start) try self.startDocument(chunk.document_id); + errdefer self.abortDocument(); + + try self.state.exact.update(chunk.bytes); + if (self.options.approximate.enabled) try self.state.approximate.update(chunk.bytes); + + var duplicate_spans: std.ArrayList(DuplicateSpan) = .empty; + if (self.options.spans.enabled) { + try self.updateSpans(chunk.bytes, &duplicate_spans); + } + + if (!chunk.boundary.document_end) { + return .{ + .document_id = self.state.document_id, + .status = .pending, + .duplicate_spans = duplicate_spans.items, + }; + } + + if (self.options.approximate.enabled) try self.state.approximate.finishToken(); + const fingerprint = DocumentFingerprint{ + .document_id = self.state.document_id, + .exact = self.state.exact.digest(), + .simhash = self.state.approximate.digest(), + .feature_count = self.state.approximate.feature_count, + .byte_count = self.state.exact.total_bytes, + }; + const backend_classification = try self.index.vtable.finishDocument( + self.index.context, + fingerprint, + self.options.approximate, + ); + self.state.in_document = false; + return .{ + .document_id = self.state.document_id, + .status = .{ .complete = backend_classification }, + .duplicate_spans = duplicate_spans.items, + .fingerprint = fingerprint, + }; + } + + pub fn observeCorpusChunk( + self: *Deduplicator, + chunk: CorpusChunk, + ) anyerror!Observation { + return self.observe(.{ + .document_id = chunk.document_id, + .bytes = chunk.bytes, + .boundary = .{ + .document_start = chunk.is_document_start, + .document_end = chunk.is_document_end, + }, + }); + } + + pub fn observeCleanedChunk( + self: *Deduplicator, + document_id: []const u8, + chunk: CleanedChunk, + ) anyerror!Observation { + return self.observe(.{ + .document_id = document_id, + .bytes = chunk.bytes, + .boundary = .{ + .document_start = chunk.document_start, + .document_end = chunk.document_end, + }, + }); + } + + /// Captures local streaming accumulators plus the index's unpublished + /// transaction. The caller owns the returned bytes. + pub fn checkpoint(self: *Deduplicator, allocator: Allocator) anyerror![]u8 { + var scratch: ArenaAllocator = .init(self.backing_allocator); + defer scratch.deinit(); + + var backend_token: []const u8 = ""; + if (self.state.in_document) { + const checkpointFn = self.index.vtable.checkpointDocument orelse + return error.IndexCheckpointUnsupported; + backend_token = try checkpointFn(self.index.context, scratch.allocator()); + } + + const document_id = if (self.state.in_document) self.state.document_id else ""; + const partial_len = if (self.state.in_document) self.state.exact.partial_len else 0; + const ring = if (self.state.in_document) self.state.spans.ring else &.{}; + if (document_id.len > std.math.maxInt(u32) or + backend_token.len > std.math.maxInt(u32) or + partial_len > std.math.maxInt(u16)) + { + return error.StateTooLarge; + } + + var total: usize = checkpoint_header_len; + total = std.math.add(usize, total, document_id.len) catch return error.StateTooLarge; + total = std.math.add(usize, total, partial_len) catch return error.StateTooLarge; + total = std.math.add(usize, total, ring.len) catch return error.StateTooLarge; + total = std.math.add(usize, total, backend_token.len) catch return error.StateTooLarge; + + const encoded = try allocator.alloc(u8, total); + @memset(encoded, 0); + @memcpy(encoded[0..4], checkpoint_magic); + putU64(encoded[8..16], self.configurationFingerprint()); + putU32(encoded[16..20], @intCast(document_id.len)); + putU32(encoded[20..24], @intCast(backend_token.len)); + + if (self.state.in_document) { + encoded[4] = 1 << 0; + if (self.state.spans.last_sample_start != null) encoded[4] |= 1 << 1; + putU16(encoded[24..26], @intCast(partial_len)); + putU64(encoded[28..36], self.state.exact.occupied); + putU64(encoded[36..44], self.state.exact.block_count); + putU64(encoded[44..52], self.state.exact.total_bytes); + putU64(encoded[52..60], self.state.approximate.token_hash); + putU64(encoded[60..68], self.state.approximate.token_len); + putU64(encoded[68..76], self.state.approximate.feature_count); + putU64(encoded[76..84], self.state.spans.rolling); + putU64( + encoded[84..92], + self.state.spans.last_sample_start orelse 0, + ); + putU64(encoded[92..100], self.state.spans.next); + putU64(encoded[100..108], self.state.spans.filled); + putU64(encoded[108..116], ring.len); + for (self.state.approximate.weights, 0..) |weight, index| { + putU64( + encoded[checkpoint_weights_offset + index * 8 ..][0..8], + @as(u64, @bitCast(weight)), + ); + } + for (self.state.exact.peaks, 0..) |peak, index| { + @memcpy( + encoded[checkpoint_peaks_offset + index * 32 ..][0..32], + &peak, + ); + } + } + + var cursor: usize = checkpoint_header_len; + @memcpy(encoded[cursor..][0..document_id.len], document_id); + cursor += document_id.len; + if (self.state.in_document) { + @memcpy( + encoded[cursor..][0..partial_len], + self.state.exact.partial[0..partial_len], + ); + } + cursor += partial_len; + @memcpy(encoded[cursor..][0..ring.len], ring); + cursor += ring.len; + @memcpy(encoded[cursor..][0..backend_token.len], backend_token); + std.debug.assert(cursor + backend_token.len == encoded.len); + const checksum = checkpointChecksum(encoded); + @memcpy(encoded[checkpoint_checksum_offset..][0..checksum.len], &checksum); + validateEncodedState(encoded) catch unreachable; + return encoded; + } + + /// Restores onto a worker with identical options and index protocol + /// version. The backend receives its staged transaction before processing + /// can continue. + pub fn restore( + self: *Deduplicator, + streaming_state: StreamingState, + ) anyerror!void { + try validateEncodedState(streaming_state.encoded); + const stable = try self.backing_allocator.dupe(u8, streaming_state.encoded); + defer self.backing_allocator.free(stable); + if (getU64(stable[8..16]) != self.configurationFingerprint()) { + return error.CheckpointConfigurationMismatch; + } + + const in_document = stable[4] & (1 << 0) != 0; + const restoreFn = if (in_document) + self.index.vtable.restoreDocument orelse + return error.IndexRestoreUnsupported + else + null; + + if (self.state.in_document) self.index.vtable.abortDocument(self.index.context); + self.state = .{}; + _ = self.document_arena.reset(.retain_capacity); + self.resetOutputArena(); + if (!in_document) return; + + const document_len: usize = getU32(stable[16..20]); + const backend_len: usize = getU32(stable[20..24]); + const partial_len: usize = getU16(stable[24..26]); + const ring_len: usize = @intCast(getU64(stable[108..116])); + var cursor: usize = checkpoint_header_len; + const encoded_document_id = stable[cursor..][0..document_len]; + cursor += document_len; + const encoded_partial = stable[cursor..][0..partial_len]; + cursor += partial_len; + const encoded_ring = stable[cursor..][0..ring_len]; + cursor += ring_len; + const backend_token = stable[cursor..][0..backend_len]; + + const document_id = try self.document_arena.allocator().dupe( + u8, + encoded_document_id, + ); + const ring = try self.document_arena.allocator().dupe(u8, encoded_ring); + var restored: DocumentState = .{ + .in_document = true, + .document_id = document_id, + .spans = .init(ring), + }; + restored.exact.partial_len = partial_len; + @memcpy(restored.exact.partial[0..partial_len], encoded_partial); + restored.exact.occupied = getU64(stable[28..36]); + restored.exact.block_count = getU64(stable[36..44]); + restored.exact.total_bytes = getU64(stable[44..52]); + restored.approximate.token_hash = getU64(stable[52..60]); + restored.approximate.token_len = getU64(stable[60..68]); + restored.approximate.feature_count = getU64(stable[68..76]); + restored.spans.rolling = getU64(stable[76..84]); + restored.spans.last_sample_start = if (stable[4] & (1 << 1) != 0) + getU64(stable[84..92]) + else + null; + restored.spans.next = @intCast(getU64(stable[92..100])); + restored.spans.filled = @intCast(getU64(stable[100..108])); + for (&restored.approximate.weights, 0..) |*weight, index| { + weight.* = @bitCast(getU64( + stable[checkpoint_weights_offset + index * 8 ..][0..8], + )); + } + for (&restored.exact.peaks, 0..) |*peak, index| { + @memcpy( + peak, + stable[checkpoint_peaks_offset + index * 32 ..][0..32], + ); + } + + restoreFn.?(self.index.context, document_id, backend_token) catch |err| { + _ = self.document_arena.reset(.retain_capacity); + return err; + }; + self.state = restored; + } + + pub fn abortDocument(self: *Deduplicator) void { + if (self.state.in_document) self.index.vtable.abortDocument(self.index.context); + self.state = .{}; + _ = self.document_arena.reset(.retain_capacity); + self.resetOutputArena(); + } + + fn validateBoundary(self: *const Deduplicator, chunk: DocumentChunk) !void { + if (chunk.boundary.document_start) { + if (self.state.in_document) return error.DocumentAlreadyStarted; + } else { + if (!self.state.in_document) return error.DocumentNotStarted; + if (!std.mem.eql(u8, self.state.document_id, chunk.document_id)) { + return error.DocumentIdChanged; + } + } + } + + fn startDocument(self: *Deduplicator, document_id: []const u8) anyerror!void { + _ = self.document_arena.reset(.retain_capacity); + self.state = .{}; + const owned_id = try self.document_arena.allocator().dupe(u8, document_id); + var spans: SpanHasher = .{}; + if (self.options.spans.enabled) { + spans = .init(try self.document_arena.allocator().alloc( + u8, + self.options.spans.window_bytes, + )); + @memset(spans.ring, 0); + } + self.state = .{ + .in_document = true, + .document_id = owned_id, + .spans = spans, + }; + self.index.vtable.startDocument(self.index.context, owned_id) catch |err| { + self.state = .{}; + _ = self.document_arena.reset(.retain_capacity); + return err; + }; + } + + fn updateSpans( + self: *Deduplicator, + bytes: []const u8, + duplicate_spans: *std.ArrayList(DuplicateSpan), + ) anyerror!void { + const policy = self.options.spans; + var span_state = &self.state.spans; + var fingerprints: std.ArrayList(SpanFingerprint) = .empty; + const fragment_start = self.state.exact.total_bytes - @as(u64, @intCast(bytes.len)); + for (bytes, 0..) |byte, byte_index| { + const encoded: u64 = @as(u64, byte) + 1; + if (span_state.filled < span_state.ring.len) { + span_state.ring[span_state.next] = byte; + span_state.next += 1; + if (span_state.next == span_state.ring.len) span_state.next = 0; + span_state.filled += 1; + span_state.rolling = span_state.rolling *% rolling_base +% encoded; + if (span_state.filled != span_state.ring.len) continue; + } else { + const outgoing: u64 = @as(u64, span_state.ring[span_state.next]) + 1; + span_state.ring[span_state.next] = byte; + span_state.next += 1; + if (span_state.next == span_state.ring.len) span_state.next = 0; + span_state.rolling = span_state.rolling *% rolling_base +% encoded -% + (outgoing *% span_state.outgoing_factor); + } + + // The exact hasher consumed the whole fragment before span probing. + const end_byte = fragment_start + @as(u64, @intCast(byte_index)) + 1; + const start_byte = end_byte - @as(u64, @intCast(span_state.ring.len)); + if (!shouldSample(span_state.*, policy, start_byte)) continue; + + try fingerprints.append(self.output_arena.allocator(), .{ + .digest = digestWindow(span_state.*), + .start_byte = start_byte, + .end_byte = end_byte, + }); + span_state.last_sample_start = start_byte; + } + + if (fingerprints.items.len == 0) return; + const matches = try self.output_arena.allocator().alloc( + ?SpanOccurrence, + fingerprints.items.len, + ); + try self.index.vtable.probeSpans( + self.index.context, + fingerprints.items, + matches, + ); + for (fingerprints.items, matches) |fingerprint, possible_match| { + if (possible_match) |match| { + try duplicate_spans.append(self.output_arena.allocator(), .{ + .current_start_byte = fingerprint.start_byte, + .current_end_byte = fingerprint.end_byte, + .canonical_document_id = try self.output_arena.allocator().dupe( + u8, + match.document_id, + ), + .canonical_start_byte = match.start_byte, + .canonical_end_byte = match.end_byte, + }); + } + } + } + + fn resetOutputArena(self: *Deduplicator) void { + const mode: ArenaAllocator.ResetMode = if (self.options.retained_output_bytes) |limit| + .{ .retain_with_limit = limit } + else + .retain_capacity; + _ = self.output_arena.reset(mode); + } + + fn configurationFingerprint(self: *const Deduplicator) u64 { + return optionsFingerprint(self.options, self.index.version); + } +}; + +const fnv_offset: u64 = 0xcbf29ce484222325; +const fnv_prime: u64 = 0x100000001b3; +const rolling_base: u64 = 0x9e3779b185ebca87; +const checkpoint_magic = "DD01"; +const checkpoint_checksum_offset = 116; +const checkpoint_weights_offset = checkpoint_checksum_offset + 16; +const checkpoint_peaks_offset = checkpoint_weights_offset + 64 * 8; +const checkpoint_header_len = checkpoint_peaks_offset + 64 * 32; + +fn validateEncodedState(encoded: []const u8) StreamingState.DecodeError!void { + if (encoded.len < checkpoint_header_len) return error.InvalidState; + if (!std.mem.eql(u8, encoded[0..4], checkpoint_magic)) { + return error.UnsupportedVersion; + } + const expected_checksum = checkpointChecksum(encoded); + if (!std.mem.eql( + u8, + encoded[checkpoint_checksum_offset..][0..expected_checksum.len], + &expected_checksum, + )) return error.InvalidState; + if (encoded[4] & ~@as(u8, 0b0000_0011) != 0) return error.InvalidState; + for (encoded[5..8]) |byte| { + if (byte != 0) return error.InvalidState; + } + if (getU16(encoded[26..28]) != 0) return error.InvalidState; + + const in_document = encoded[4] & (1 << 0) != 0; + const has_last_sample = encoded[4] & (1 << 1) != 0; + const document_len: usize = getU32(encoded[16..20]); + const backend_len: usize = getU32(encoded[20..24]); + const partial_len: usize = getU16(encoded[24..26]); + const ring_len_u64 = getU64(encoded[108..116]); + const ring_len = std.math.cast(usize, ring_len_u64) orelse + return error.InvalidState; + + var expected_len: usize = checkpoint_header_len; + expected_len = std.math.add(usize, expected_len, document_len) catch + return error.InvalidState; + expected_len = std.math.add(usize, expected_len, partial_len) catch + return error.InvalidState; + expected_len = std.math.add(usize, expected_len, ring_len) catch + return error.InvalidState; + expected_len = std.math.add(usize, expected_len, backend_len) catch + return error.InvalidState; + if (encoded.len != expected_len) return error.InvalidState; + + if (!in_document) { + if (encoded[4] != 0 or document_len != 0 or backend_len != 0 or + partial_len != 0 or ring_len != 0) + { + return error.InvalidState; + } + for (encoded[16..checkpoint_checksum_offset]) |byte| { + if (byte != 0) return error.InvalidState; + } + for (encoded[checkpoint_weights_offset..checkpoint_header_len]) |byte| { + if (byte != 0) return error.InvalidState; + } + return; + } + + if (partial_len >= exact_block_bytes) return error.InvalidState; + const occupied = getU64(encoded[28..36]); + const block_count = getU64(encoded[36..44]); + const total_bytes = getU64(encoded[44..52]); + if (occupied != block_count or + block_count != total_bytes / exact_block_bytes or + partial_len != total_bytes % exact_block_bytes) + { + return error.InvalidState; + } + + const next_u64 = getU64(encoded[92..100]); + const filled_u64 = getU64(encoded[100..108]); + if (ring_len == 0) { + if (next_u64 != 0 or filled_u64 != 0 or + getU64(encoded[76..84]) != 0 or has_last_sample) + { + return error.InvalidState; + } + } else { + if (next_u64 >= ring_len_u64 or filled_u64 > ring_len_u64) { + return error.InvalidState; + } + const expected_filled = @min(total_bytes, ring_len_u64); + const expected_next = total_bytes % ring_len_u64; + if (filled_u64 != expected_filled or next_u64 != expected_next) { + return error.InvalidState; + } + if (has_last_sample and + (total_bytes < ring_len_u64 or + getU64(encoded[84..92]) > total_bytes - ring_len_u64)) + { + return error.InvalidState; + } + } + if (!has_last_sample and getU64(encoded[84..92]) != 0) { + return error.InvalidState; + } +} + +fn checkpointChecksum(encoded: []const u8) [16]u8 { + var hash = Blake3.init(.{}); + hash.update("chi.dedup.checkpoint.v1\x00"); + hash.update(encoded[0..checkpoint_checksum_offset]); + hash.update(encoded[checkpoint_weights_offset..]); + var result: [16]u8 = undefined; + hash.final(&result); + return result; +} + +fn optionsFingerprint(options: Options, index_version: u64) u64 { + var hash: u64 = fnv_offset; + fingerprintValue(&hash, index_version); + fingerprintValue(&hash, @intFromBool(options.approximate.enabled)); + fingerprintValue(&hash, options.approximate.max_hamming_distance); + fingerprintValue(&hash, options.approximate.min_features); + fingerprintValue(&hash, @intFromBool(options.spans.enabled)); + fingerprintValue(&hash, options.spans.window_bytes); + fingerprintValue(&hash, options.spans.sampling_bits); + fingerprintValue(&hash, options.spans.min_start_distance); + return hash; +} + +fn fingerprintValue(hash: *u64, value: anytype) void { + const canonical: u64 = @intCast(value); + for (0..8) |index| { + hash.* ^= @as(u8, @truncate(canonical >> @intCast(index * 8))); + hash.* *%= fnv_prime; + } +} + +fn validateOptions(options: Options) !void { + if (options.approximate.max_hamming_distance > 3) { + return error.InvalidApproximateDistance; + } + if (options.spans.enabled and + (options.spans.window_bytes == 0 or options.spans.sampling_bits > 63)) + { + return error.InvalidSpanPolicy; + } +} + +fn isTokenByte(byte: u8) bool { + // TODO(data/unicode-tables): replace byte-level non-ASCII grouping with + // Unicode word segmentation and case folding once versioned tables exist. + return byte >= 0x80 or (byte >= 'a' and byte <= 'z') or + (byte >= 'A' and byte <= 'Z') or (byte >= '0' and byte <= '9') or + byte == '_'; +} + +fn mix64(input: u64) u64 { + var value = input; + value ^= value >> 30; + value *%= 0xbf58476d1ce4e5b9; + value ^= value >> 27; + value *%= 0x94d049bb133111eb; + return value ^ (value >> 31); +} + +fn bandKey(simhash: u64, band_index: usize) u32 { + const value: u16 = @truncate(simhash >> @intCast(band_index * 16)); + return (@as(u32, @intCast(band_index)) << 16) | value; +} + +fn hashLeaf(bytes: []const u8) ExactDigest { + var hash = Blake3.init(.{}); + hash.update("chi.dedup.leaf.v1\x00"); + var length: [8]u8 = undefined; + putU64(&length, bytes.len); + hash.update(&length); + hash.update(bytes); + var result: ExactDigest = undefined; + hash.final(&result); + return result; +} + +fn hashParent(left: ExactDigest, right: ExactDigest) ExactDigest { + var hash = Blake3.init(.{}); + hash.update("chi.dedup.parent.v1\x00"); + hash.update(&left); + hash.update(&right); + var result: ExactDigest = undefined; + hash.final(&result); + return result; +} + +fn shouldSample(state: SpanHasher, policy: SpanPolicy, start_byte: u64) bool { + if (policy.sampling_bits != 0) { + const mask = (@as(u64, 1) << @intCast(policy.sampling_bits)) - 1; + if (state.rolling & mask != 0) return false; + } + if (state.last_sample_start) |last| { + if (start_byte - last < policy.min_start_distance) return false; + } + return true; +} + +fn digestWindow(state: SpanHasher) SpanDigest { + var hash = Blake3.init(.{}); + hash.update("chi.dedup.span.v1\x00"); + var length: [8]u8 = undefined; + putU64(&length, state.ring.len); + hash.update(&length); + hash.update(state.ring[state.next..]); + hash.update(state.ring[0..state.next]); + var result: SpanDigest = undefined; + hash.final(&result); + return result; +} + +fn putU64(destination: *[8]u8, value: anytype) void { + const canonical: u64 = @intCast(value); + for (destination, 0..) |*byte, index| { + byte.* = @truncate(canonical >> @intCast(index * 8)); + } +} + +fn putU32(destination: *[4]u8, value: u32) void { + for (destination, 0..) |*byte, index| { + byte.* = @truncate(value >> @intCast(index * 8)); + } +} + +fn putU16(destination: *[2]u8, value: u16) void { + for (destination, 0..) |*byte, index| { + byte.* = @truncate(value >> @intCast(index * 8)); + } +} + +fn getU64(source: *const [8]u8) u64 { + var value: u64 = 0; + for (source, 0..) |byte, index| { + value |= @as(u64, byte) << @intCast(index * 8); + } + return value; +} + +fn getU32(source: *const [4]u8) u32 { + var value: u32 = 0; + for (source, 0..) |byte, index| { + value |= @as(u32, byte) << @intCast(index * 8); + } + return value; +} + +fn getU16(source: *const [2]u8) u16 { + var value: u16 = 0; + for (source, 0..) |byte, index| { + value |= @as(u16, byte) << @intCast(index * 8); + } + return value; +} + +test "exact fingerprints ignore fragment boundaries" { + const backend = try InMemoryIndex.create(std.testing.allocator); + const deduplicator = try Deduplicator.create( + std.testing.allocator, + backend.ownedIndex(), + .{ .spans = .{ .enabled = false } }, + ); + defer deduplicator.destroy(); + + var input: [exact_block_bytes * 2 + 137]u8 = undefined; + for (&input, 0..) |*byte, index| byte.* = @truncate(index *% 131 +% 17); + + const first = try deduplicator.observe(.{ + .document_id = "first", + .bytes = &input, + .boundary = .whole, + }); + try std.testing.expect(first.status.complete == .unique); + + _ = try deduplicator.observe(.{ + .document_id = "second", + .bytes = input[0..123], + .boundary = .{ .document_start = true }, + }); + _ = try deduplicator.observe(.{ + .document_id = "second", + .bytes = input[123..5001], + .boundary = .{}, + }); + const second = try deduplicator.observe(.{ + .document_id = "second", + .bytes = input[5001..], + .boundary = .{ .document_end = true }, + }); + switch (second.status.complete) { + .exact_duplicate => |match| try std.testing.expectEqualStrings("first", match.document_id), + else => return error.TestExpectedEqual, + } + try std.testing.expectEqual(@as(usize, 1), backend.documentCount()); +} + +test "approximate matches require enough features" { + const backend = try InMemoryIndex.create(std.testing.allocator); + const deduplicator = try Deduplicator.create( + std.testing.allocator, + backend.ownedIndex(), + .{ + .approximate = .{ .max_hamming_distance = 3, .min_features = 4 }, + .spans = .{ .enabled = false }, + }, + ); + defer deduplicator.destroy(); + + const original = try deduplicator.observe(.{ + .document_id = "original", + .bytes = "alpha beta gamma delta epsilon zeta eta theta", + .boundary = .whole, + }); + try std.testing.expect(original.status.complete == .unique); + const variant = try deduplicator.observe(.{ + .document_id = "variant", + .bytes = "ALPHA beta gamma delta epsilon zeta eta theta!", + .boundary = .whole, + }); + switch (variant.status.complete) { + .approximate_duplicate => |match| { + try std.testing.expectEqualStrings("original", match.document_id); + try std.testing.expectEqual(@as(u8, 0), match.hamming_distance); + }, + else => return error.TestExpectedEqual, + } +} + +test "duplicate spans are found across shifted text" { + const backend = try InMemoryIndex.create(std.testing.allocator); + const deduplicator = try Deduplicator.create( + std.testing.allocator, + backend.ownedIndex(), + .{ + .approximate = .{ .enabled = false }, + .spans = .{ + .window_bytes = 5, + .sampling_bits = 0, + .min_start_distance = 1, + }, + }, + ); + defer deduplicator.destroy(); + + _ = try deduplicator.observe(.{ + .document_id = "canonical", + .bytes = "abcdefghij", + .boundary = .whole, + }); + const shifted = try deduplicator.observe(.{ + .document_id = "shifted", + .bytes = "XXcdefgYY", + .boundary = .whole, + }); + try std.testing.expect(shifted.duplicate_spans.len >= 1); + const match = shifted.duplicate_spans[0]; + try std.testing.expectEqualStrings("canonical", match.canonical_document_id); + try std.testing.expectEqual(@as(u64, 2), match.current_start_byte); + try std.testing.expectEqual(@as(u64, 2), match.canonical_start_byte); +} + +test "failed and duplicate documents do not publish staged spans" { + const backend = try InMemoryIndex.create(std.testing.allocator); + const deduplicator = try Deduplicator.create( + std.testing.allocator, + backend.ownedIndex(), + .{ + .approximate = .{ .enabled = false }, + .spans = .{ .window_bytes = 4, .sampling_bits = 0, .min_start_distance = 1 }, + }, + ); + defer deduplicator.destroy(); + + _ = try deduplicator.observe(.{ + .document_id = "one", + .bytes = "abcdefgh", + .boundary = .whole, + }); + _ = try deduplicator.observe(.{ + .document_id = "duplicate", + .bytes = "abcdefgh", + .boundary = .whole, + }); + try std.testing.expectEqual(@as(usize, 1), backend.documentCount()); + + _ = try deduplicator.observe(.{ + .document_id = "aborted", + .bytes = "unique-prefix", + .boundary = .{ .document_start = true }, + }); + deduplicator.abortDocument(); + try std.testing.expectEqual(@as(usize, 1), backend.documentCount()); +} + +test "checkpoint transfers hashing span and backend transaction state" { + const backend = try InMemoryIndex.create(std.testing.allocator); + defer backend.destroy(); + const options = Options{ + .approximate = .{ .min_features = 4 }, + .spans = .{ .window_bytes = 5, .sampling_bits = 0, .min_start_distance = 1 }, + }; + + const source = try Deduplicator.create( + std.testing.allocator, + backend.borrowedIndex(), + options, + ); + defer source.destroy(); + _ = try source.observe(.{ + .document_id = "migrating", + .bytes = "alpha beta gam", + .boundary = .{ .document_start = true }, + }); + + const encoded = try source.checkpoint(std.testing.allocator); + defer std.testing.allocator.free(encoded); + const streaming_state = try StreamingState.decode(encoded); + const corrupted = try std.testing.allocator.dupe(u8, encoded); + defer std.testing.allocator.free(corrupted); + corrupted[corrupted.len - 1] ^= 1; + try std.testing.expectError(error.InvalidState, StreamingState.decode(corrupted)); + source.abortDocument(); + + const destination = try Deduplicator.create( + std.testing.allocator, + backend.borrowedIndex(), + options, + ); + defer destination.destroy(); + try destination.restore(streaming_state); + const completed = try destination.observe(.{ + .document_id = "migrating", + .bytes = "ma delta epsilon", + .boundary = .{ .document_end = true }, + }); + try std.testing.expect(completed.status.complete == .unique); + + const duplicate = try destination.observe(.{ + .document_id = "whole", + .bytes = "alpha beta gamma delta epsilon", + .boundary = .whole, + }); + switch (duplicate.status.complete) { + .exact_duplicate => |match| { + try std.testing.expectEqualStrings("migrating", match.document_id); + }, + else => return error.TestExpectedEqual, + } + + const incompatible = try Deduplicator.create( + std.testing.allocator, + backend.borrowedIndex(), + .{ .spans = .{ .enabled = false } }, + ); + defer incompatible.destroy(); + try std.testing.expectError( + error.CheckpointConfigurationMismatch, + incompatible.restore(streaming_state), + ); +} diff --git a/data/deduplicator/root.zig b/data/deduplicator/root.zig new file mode 100644 index 0000000..07c576e --- /dev/null +++ b/data/deduplicator/root.zig @@ -0,0 +1,28 @@ +//! Streaming exact, approximate, and text-span deduplication. + +const dedup = @import("deduplicator.zig"); + +pub const ApproximateMatch = dedup.ApproximateMatch; +pub const ApproximatePolicy = dedup.ApproximatePolicy; +pub const Boundary = dedup.Boundary; +pub const CanonicalMatch = dedup.CanonicalMatch; +pub const Classification = dedup.Classification; +pub const Deduplicator = dedup.Deduplicator; +pub const DocumentChunk = dedup.DocumentChunk; +pub const DocumentFingerprint = dedup.DocumentFingerprint; +pub const DuplicateSpan = dedup.DuplicateSpan; +pub const ExactDigest = dedup.ExactDigest; +pub const InMemoryIndex = dedup.InMemoryIndex; +pub const Index = dedup.Index; +pub const Observation = dedup.Observation; +pub const Options = dedup.Options; +pub const SpanDigest = dedup.SpanDigest; +pub const SpanFingerprint = dedup.SpanFingerprint; +pub const SpanOccurrence = dedup.SpanOccurrence; +pub const SpanPolicy = dedup.SpanPolicy; +pub const Status = dedup.Status; +pub const StreamingState = dedup.StreamingState; + +test { + _ = dedup; +} diff --git a/data/document-filter/README.md b/data/document-filter/README.md new file mode 100644 index 0000000..dd76e6e --- /dev/null +++ b/data/document-filter/README.md @@ -0,0 +1,27 @@ +# Document filter + +`DocumentFilter` is a bounded-memory document gate. It observes UTF-8 chunks, +returns early rejection when possible, and returns acceptance only after the +document-end boundary. The pipeline can therefore keep undecided bytes in a +spool or quarantine tier instead of buffering every document in this component. + +Built-in rules cover byte, codepoint, and line lengths; UTF-8 validity; visible, +replacement, and cleaning-change ratios; and repeated-codepoint runs. Threshold +ratios use integer basis points for deterministic behavior. + +Language identification, safety classification, and project-specific policies +use zero-copy `Rule` adapters. Rules are categorized for attribution, have +explicit lifecycle hooks, and can reject before the full document is consumed. +Stateful rules provide checkpoint/restore hooks; stateless rules need no token. + +Versioned checkpoints contain built-in metrics, partial UTF-8 sequences, sticky +rejections, and active rule tokens. Configuration fingerprints prevent a worker +from restoring state under different thresholds or rule versions. + +Decision document IDs and rejection details are borrowed until the next document +starts, the current document is aborted, or the filter is destroyed. + +TODOs for concrete language and safety model adapters remain until their model +inference/tokenization components define stable input and ownership contracts. + +Run all tests from the repository root with `zig build test`. diff --git a/data/document-filter/document_filter.zig b/data/document-filter/document_filter.zig new file mode 100644 index 0000000..5b265ff --- /dev/null +++ b/data/document-filter/document_filter.zig @@ -0,0 +1,1370 @@ +const std = @import("std"); + +const Allocator = std.mem.Allocator; +const ArenaAllocator = std.heap.ArenaAllocator; +const CorpusChunk = @import("../corpus-reader/root.zig").Chunk; +const CleanedChunk = @import("../text-cleaner/root.zig").CleanedChunk; + +pub const Boundary = struct { + document_start: bool = false, + document_end: bool = false, +}; + +/// Cleaning events are kept separate from cleaned bytes because destructive +/// cleaning removes the evidence needed by quality rules. +pub const CleaningSignals = struct { + malformed_sequences: u64 = 0, + controls_removed: u64 = 0, + format_characters_removed: u64 = 0, + whitespace_normalized: u64 = 0, + + pub fn fromCleanedChunk(chunk: CleanedChunk) CleaningSignals { + return .{ + .malformed_sequences = chunk.stats.malformed_sequences, + .controls_removed = chunk.stats.controls_removed, + .format_characters_removed = chunk.stats.format_characters_removed, + .whitespace_normalized = chunk.stats.whitespace_normalized, + }; + } +}; + +pub const DocumentChunk = struct { + document_id: []const u8, + bytes: []const u8, + boundary: Boundary, + cleaning: CleaningSignals = .{}, +}; + +pub const LengthPolicy = struct { + min_bytes: u64 = 1, + max_bytes: ?u64 = null, + min_codepoints: u64 = 1, + max_codepoints: ?u64 = null, + min_lines: ?u64 = null, + max_lines: ?u64 = null, +}; + +/// Ratios use basis points: 10_000 is 100%, 100 is 1%. Null disables a rule. +pub const QualityPolicy = struct { + require_valid_utf8: bool = true, + min_visible_basis_points: ?u16 = null, + max_replacement_basis_points: ?u16 = null, + /// Cleaning events (malformed sequences and removed control/format scalars) + /// divided by emitted UTF-8 bytes. + max_cleaning_change_basis_points: ?u16 = null, + max_repeated_codepoints: ?u64 = null, +}; + +pub const Options = struct { + length: LengthPolicy = .{}, + quality: QualityPolicy = .{}, +}; + +pub const Metrics = struct { + bytes: u64 = 0, + codepoints: u64 = 0, + newline_codepoints: u64 = 0, + visible_codepoints: u64 = 0, + replacement_codepoints: u64 = 0, + invalid_utf8_sequences: u64 = 0, + cleaning_changes: u64 = 0, + whitespace_normalized: u64 = 0, + longest_repeated_run: u64 = 0, + + pub fn lines(self: Metrics) u64 { + return if (self.codepoints == 0) 0 else self.newline_codepoints +| 1; + } +}; + +pub const Summary = struct { + document_id: []const u8, + metrics: Metrics, +}; + +pub const RejectionReason = enum(u8) { + bytes_below_minimum, + bytes_above_maximum, + codepoints_below_minimum, + codepoints_above_maximum, + lines_below_minimum, + lines_above_maximum, + invalid_utf8, + low_visible_ratio, + excessive_replacement_ratio, + excessive_cleaning_ratio, + repeated_codepoint_run, + rule, +}; + +pub const Rejection = struct { + document_id: []const u8, + reason: RejectionReason, + rule_id: ?[]const u8 = null, + rule_kind: ?RuleKind = null, + detail: []const u8, + metrics: Metrics, +}; + +pub const Decision = union(enum) { + pending: Summary, + accepted: Summary, + rejected: Rejection, +}; + +/// A validated view over an encoded filter checkpoint. The encoded bytes are +/// borrowed; `checkpoint` returns an owned byte slice for transport/storage. +pub const StreamingState = struct { + encoded: []const u8, + + pub const DecodeError = error{ + InvalidState, + UnsupportedVersion, + }; + + pub fn decode(encoded: []const u8) DecodeError!StreamingState { + try validateEncodedState(encoded); + return .{ .encoded = encoded }; + } +}; + +pub const RuleKind = enum(u8) { + language, + safety, + quality, + custom, +}; + +pub const RuleDecision = union(enum) { + keep, + reject: []const u8, +}; + +pub const RuleChunk = struct { + document_id: []const u8, + bytes: []const u8, + boundary: Boundary, + cleaning: CleaningSignals, + metrics: Metrics, +}; + +/// Adapter boundary for language identification, safety classifiers, and +/// project-specific filters. The filter owns the adapter lifecycle after a +/// successful `create` call. +// TODO(data/classification): add concrete language-ID and safety adapters once +// tokenizer/inference components define stable streaming input contracts. +pub const Rule = struct { + id: []const u8, + kind: RuleKind, + context: *anyopaque, + vtable: *const VTable, + version: u64 = 1, + stateful: bool = false, + + pub const VTable = struct { + evaluate: *const fn (context: *anyopaque, chunk: RuleChunk) anyerror!RuleDecision, + abort: ?*const fn (context: *anyopaque) void = null, + checkpoint: ?*const fn (context: *anyopaque, allocator: Allocator) anyerror![]const u8 = null, + restore: ?*const fn (context: *anyopaque, token: []const u8) anyerror!void = null, + deinit: ?*const fn (context: *anyopaque) void = null, + }; +}; + +const FilterState = struct { + in_document: bool = false, + document_id: []const u8 = "", + metrics: Metrics = .{}, + rejection: ?StoredRejection = null, + active_rule_count: usize = 0, + chunks: u64 = 0, + + utf8_pending: [4]u8 = @splat(0), + utf8_pending_len: u3 = 0, + utf8_expected_len: u3 = 0, + + last_visible_codepoint: ?u21 = null, + current_repeated_run: u64 = 0, +}; + +const StoredRejection = struct { + reason: RejectionReason, + rule_id: ?[]const u8, + rule_kind: ?RuleKind, + detail: []const u8, +}; + +pub const DocumentFilter = struct { + backing_allocator: Allocator, + options: Options, + source_arena: ArenaAllocator, + document_arena: ArenaAllocator, + rules: []Rule, + state: FilterState = .{}, + + pub const CreateError = Allocator.Error || error{ + DuplicateRuleId, + EmptyRuleId, + InvalidLengthPolicy, + InvalidQualityRatio, + }; + + pub fn create( + backing_allocator: Allocator, + options: Options, + input_rules: []const Rule, + ) CreateError!*DocumentFilter { + try validateOptions(options); + + const self = try backing_allocator.create(DocumentFilter); + self.* = .{ + .backing_allocator = backing_allocator, + .options = options, + .source_arena = .init(backing_allocator), + .document_arena = .init(backing_allocator), + .rules = &.{}, + }; + errdefer { + self.document_arena.deinit(); + self.source_arena.deinit(); + backing_allocator.destroy(self); + } + + const rules = try self.source_arena.allocator().alloc(Rule, input_rules.len); + for (input_rules, 0..) |input_rule, index| { + if (input_rule.id.len == 0) return error.EmptyRuleId; + for (rules[0..index]) |previous| { + if (std.mem.eql(u8, previous.id, input_rule.id)) { + return error.DuplicateRuleId; + } + } + rules[index] = input_rule; + rules[index].id = try self.source_arena.allocator().dupe(u8, input_rule.id); + } + self.rules = rules; + return self; + } + + pub fn destroy(self: *DocumentFilter) void { + if (self.state.active_rule_count != 0) self.abortRules(); + for (self.rules) |rule| { + if (rule.vtable.deinit) |deinitFn| deinitFn(rule.context); + } + self.document_arena.deinit(); + self.source_arena.deinit(); + const backing_allocator = self.backing_allocator; + backing_allocator.destroy(self); + } + + /// Observes one UTF-8 document fragment. Acceptance is only returned at an + /// end boundary; rejection may happen earlier and remains sticky while the + /// rest of that document is drained. + pub fn observe(self: *DocumentFilter, chunk: DocumentChunk) anyerror!Decision { + if (chunk.boundary.document_start) { + if (self.state.in_document) return error.DocumentAlreadyStarted; + try self.startDocument(chunk.document_id); + } else { + if (!self.state.in_document) return error.DocumentNotStarted; + if (!std.mem.eql(u8, self.state.document_id, chunk.document_id)) { + return error.DocumentIdChanged; + } + } + + if (self.state.rejection == null) { + try self.measureChunk(chunk); + if (self.state.rejection == null) try self.evaluateRules(chunk); + } + self.state.chunks = try addU64(self.state.chunks, 1); + + if (chunk.boundary.document_end) { + if (self.state.rejection == null) { + try self.finishUtf8(); + if (self.state.rejection == null) try self.applyFinalPolicies(); + } + self.state.in_document = false; + self.state.active_rule_count = 0; + return self.finalDecision(); + } + return self.currentDecision(.pending); + } + + pub fn observeCorpusChunk(self: *DocumentFilter, chunk: CorpusChunk) anyerror!Decision { + return self.observe(.{ + .document_id = chunk.document_id, + .bytes = chunk.bytes, + .boundary = .{ + .document_start = chunk.is_document_start, + .document_end = chunk.is_document_end, + }, + }); + } + + pub fn observeCleanedChunk( + self: *DocumentFilter, + document_id: []const u8, + chunk: CleanedChunk, + ) anyerror!Decision { + return self.observe(.{ + .document_id = document_id, + .bytes = chunk.bytes, + .boundary = .{ + .document_start = chunk.document_start, + .document_end = chunk.document_end, + }, + .cleaning = .fromCleanedChunk(chunk), + }); + } + + /// Encodes built-in metrics, partial UTF-8 state, sticky rejection, and all + /// active stateful rule tokens. The caller owns the returned bytes. + pub fn checkpoint( + self: *DocumentFilter, + allocator: Allocator, + ) anyerror![]u8 { + var scratch: ArenaAllocator = .init(self.backing_allocator); + defer scratch.deinit(); + + const active_count = if (self.state.in_document) + self.state.active_rule_count + else + 0; + const tokens = try scratch.allocator().alloc([]const u8, active_count); + var token_bytes: usize = 0; + for (self.rules[0..active_count], 0..) |rule, index| { + if (rule.stateful) { + const checkpointFn = rule.vtable.checkpoint orelse + return error.RuleCheckpointUnsupported; + tokens[index] = try checkpointFn(rule.context, scratch.allocator()); + } else { + tokens[index] = ""; + } + token_bytes = std.math.add(usize, token_bytes, 4) catch + return error.StateTooLarge; + token_bytes = std.math.add(usize, token_bytes, tokens[index].len) catch + return error.StateTooLarge; + if (tokens[index].len > std.math.maxInt(u32)) return error.StateTooLarge; + } + + const document_id = if (self.state.in_document) self.state.document_id else ""; + const stored_rejection = if (self.state.in_document) self.state.rejection else null; + const rejection_rule_id = if (stored_rejection) |stored| + stored.rule_id orelse "" + else + ""; + const rejection_detail = if (stored_rejection) |stored| stored.detail else ""; + if (document_id.len > std.math.maxInt(u32) or + rejection_rule_id.len > std.math.maxInt(u32) or + rejection_detail.len > std.math.maxInt(u32) or + active_count > std.math.maxInt(u32)) + { + return error.StateTooLarge; + } + + var total: usize = checkpoint_header_len; + total = std.math.add(usize, total, document_id.len) catch return error.StateTooLarge; + total = std.math.add(usize, total, rejection_rule_id.len) catch return error.StateTooLarge; + total = std.math.add(usize, total, rejection_detail.len) catch return error.StateTooLarge; + total = std.math.add(usize, total, token_bytes) catch return error.StateTooLarge; + + const encoded = try allocator.alloc(u8, total); + @memset(encoded, 0); + @memcpy(encoded[0..4], checkpoint_magic); + putU64(encoded[108..116], self.configurationFingerprint()); + putU32(encoded[116..120], @intCast(document_id.len)); + putU32(encoded[120..124], @intCast(rejection_rule_id.len)); + putU32(encoded[124..128], @intCast(rejection_detail.len)); + putU32(encoded[128..132], @intCast(active_count)); + + if (self.state.in_document) { + var flags: u8 = 1 << 0; + if (stored_rejection != null) flags |= 1 << 1; + if (self.state.last_visible_codepoint != null) flags |= 1 << 2; + encoded[4] = flags; + encoded[5] = if (stored_rejection) |stored| + @intFromEnum(stored.reason) + 1 + else + 0; + encoded[6] = self.state.utf8_pending_len; + encoded[7] = self.state.utf8_expected_len; + putU32(encoded[8..12], @intCast(active_count)); + putU64(encoded[12..20], self.state.chunks); + putU64(encoded[20..28], self.state.current_repeated_run); + encodeMetrics(encoded[28..100], self.state.metrics); + putU32( + encoded[100..104], + if (self.state.last_visible_codepoint) |codepoint| codepoint else 0, + ); + @memcpy( + encoded[104..][0..self.state.utf8_pending_len], + self.state.utf8_pending[0..self.state.utf8_pending_len], + ); + } + + var cursor: usize = checkpoint_header_len; + @memcpy(encoded[cursor..][0..document_id.len], document_id); + cursor += document_id.len; + @memcpy(encoded[cursor..][0..rejection_rule_id.len], rejection_rule_id); + cursor += rejection_rule_id.len; + @memcpy(encoded[cursor..][0..rejection_detail.len], rejection_detail); + cursor += rejection_detail.len; + for (tokens) |token| { + putU32(encoded[cursor..][0..4], @intCast(token.len)); + cursor += 4; + @memcpy(encoded[cursor..][0..token.len], token); + cursor += token.len; + } + std.debug.assert(cursor == encoded.len); + validateEncodedState(encoded) catch unreachable; + return encoded; + } + + /// Restores a checkpoint on a worker with identical policy and rule IDs, + /// versions, ordering, and statefulness declarations. + pub fn restore( + self: *DocumentFilter, + streaming_state: StreamingState, + ) anyerror!void { + try validateEncodedState(streaming_state.encoded); + const stable = try self.backing_allocator.dupe(u8, streaming_state.encoded); + defer self.backing_allocator.free(stable); + if (getU64(stable[108..116]) != self.configurationFingerprint()) { + return error.CheckpointConfigurationMismatch; + } + + if (self.state.active_rule_count != 0) self.abortRules(); + _ = self.document_arena.reset(.retain_capacity); + self.state = .{}; + + if (stable[4] & (1 << 0) == 0) return; + const active_count: usize = getU32(stable[8..12]); + if (active_count > self.rules.len) return error.InvalidState; + + const document_len: usize = getU32(stable[116..120]); + const rule_id_len: usize = getU32(stable[120..124]); + const detail_len: usize = getU32(stable[124..128]); + var cursor: usize = checkpoint_header_len; + const document_id = stable[cursor..][0..document_len]; + cursor += document_len; + const rejection_rule_id = stable[cursor..][0..rule_id_len]; + cursor += rule_id_len; + const rejection_detail = stable[cursor..][0..detail_len]; + cursor += detail_len; + + var restored_rules: usize = 0; + errdefer { + for (self.rules[0..restored_rules]) |rule| { + if (rule.vtable.abort) |abortFn| abortFn(rule.context); + } + self.state = .{}; + } + for (self.rules[0..active_count]) |rule| { + const token_len: usize = getU32(stable[cursor..][0..4]); + cursor += 4; + const token = stable[cursor..][0..token_len]; + cursor += token_len; + if (rule.stateful) { + const restoreFn = rule.vtable.restore orelse + return error.RuleRestoreUnsupported; + try restoreFn(rule.context, token); + } else if (token.len != 0) { + return error.InvalidState; + } + restored_rules += 1; + } + + self.state = .{ + .in_document = true, + .document_id = try self.document_arena.allocator().dupe(u8, document_id), + .metrics = decodeMetrics(stable[28..100]), + .rejection = if (stable[4] & (1 << 1) != 0) .{ + .reason = (decodeRejectionReason(stable[5]) catch unreachable).?, + .rule_id = if (rule_id_len == 0) + null + else + try self.document_arena.allocator().dupe(u8, rejection_rule_id), + .rule_kind = if (rule_id_len == 0) + null + else + self.ruleKindForId(rejection_rule_id) orelse return error.InvalidState, + .detail = try self.document_arena.allocator().dupe(u8, rejection_detail), + } else null, + .active_rule_count = active_count, + .chunks = getU64(stable[12..20]), + .utf8_pending_len = @intCast(stable[6]), + .utf8_expected_len = @intCast(stable[7]), + .last_visible_codepoint = if (stable[4] & (1 << 2) != 0) + @intCast(getU32(stable[100..104])) + else + null, + .current_repeated_run = getU64(stable[20..28]), + }; + @memcpy(&self.state.utf8_pending, stable[104..108]); + } + + pub fn abortDocument(self: *DocumentFilter) void { + if (self.state.active_rule_count != 0) self.abortRules(); + self.state = .{}; + _ = self.document_arena.reset(.retain_capacity); + } + + fn startDocument(self: *DocumentFilter, document_id: []const u8) Allocator.Error!void { + _ = self.document_arena.reset(.retain_capacity); + self.state = .{ + .in_document = true, + .document_id = try self.document_arena.allocator().dupe(u8, document_id), + }; + } + + fn measureChunk(self: *DocumentFilter, chunk: DocumentChunk) anyerror!void { + self.state.metrics.bytes = try addU64(self.state.metrics.bytes, chunk.bytes.len); + self.state.metrics.cleaning_changes = try addU64( + self.state.metrics.cleaning_changes, + try cleaningChanges(chunk.cleaning), + ); + self.state.metrics.whitespace_normalized = try addU64( + self.state.metrics.whitespace_normalized, + chunk.cleaning.whitespace_normalized, + ); + + if (self.options.length.max_bytes) |maximum| { + if (self.state.metrics.bytes > maximum) { + try self.rejectBuiltIn(.bytes_above_maximum, "document exceeds max_bytes"); + return; + } + } + + try self.measureUtf8(chunk.bytes); + if (self.state.rejection != null) return; + + if (self.options.length.max_codepoints) |maximum| { + if (self.state.metrics.codepoints > maximum) { + try self.rejectBuiltIn( + .codepoints_above_maximum, + "document exceeds max_codepoints", + ); + return; + } + } + if (self.options.length.max_lines) |maximum| { + if (self.state.metrics.lines() > maximum) { + try self.rejectBuiltIn(.lines_above_maximum, "document exceeds max_lines"); + return; + } + } + if (self.options.quality.max_repeated_codepoints) |maximum| { + if (self.state.metrics.longest_repeated_run > maximum) { + try self.rejectBuiltIn( + .repeated_codepoint_run, + "visible codepoint run exceeds quality limit", + ); + } + } + } + + fn measureUtf8(self: *DocumentFilter, bytes: []const u8) anyerror!void { + var index: usize = 0; + while (index < bytes.len and self.state.rejection == null) { + const byte = bytes[index]; + if (self.state.utf8_pending_len == 0) { + if (byte >= 0x21 and byte <= 0x7E) { + const run_start = index; + index += 1; + while (index < bytes.len and + bytes[index] >= 0x21 and bytes[index] <= 0x7E) + { + index += 1; + } + try self.measureAsciiRun(bytes[run_start..index]); + continue; + } + if (byte < 0x80) { + try self.measureCodepoint(byte); + index += 1; + continue; + } + const expected = std.unicode.utf8ByteSequenceLength(byte) catch { + try self.recordInvalidUtf8(); + index += 1; + continue; + }; + self.state.utf8_pending[0] = byte; + self.state.utf8_pending_len = 1; + self.state.utf8_expected_len = expected; + index += 1; + continue; + } + + if (byte & 0b1100_0000 != 0b1000_0000) { + try self.recordInvalidUtf8(); + self.state.utf8_pending_len = 0; + self.state.utf8_expected_len = 0; + continue; + } + + self.state.utf8_pending[self.state.utf8_pending_len] = byte; + self.state.utf8_pending_len += 1; + index += 1; + if (self.state.utf8_pending_len == self.state.utf8_expected_len) { + const sequence = self.state.utf8_pending[0..self.state.utf8_pending_len]; + const codepoint = std.unicode.utf8Decode(sequence) catch { + try self.recordInvalidUtf8(); + self.state.utf8_pending_len = 0; + self.state.utf8_expected_len = 0; + continue; + }; + self.state.utf8_pending_len = 0; + self.state.utf8_expected_len = 0; + try self.measureCodepoint(codepoint); + } + } + } + + fn measureAsciiRun(self: *DocumentFilter, bytes: []const u8) anyerror!void { + self.state.metrics.codepoints = try addU64(self.state.metrics.codepoints, bytes.len); + self.state.metrics.visible_codepoints = try addU64( + self.state.metrics.visible_codepoints, + bytes.len, + ); + + var index: usize = 0; + while (index < bytes.len) { + const codepoint: u21 = bytes[index]; + var end = index + 1; + while (end < bytes.len and bytes[end] == bytes[index]) end += 1; + const run_len: u64 = @intCast(end - index); + if (self.state.last_visible_codepoint == codepoint) { + self.state.current_repeated_run = try addU64( + self.state.current_repeated_run, + run_len, + ); + } else { + self.state.last_visible_codepoint = codepoint; + self.state.current_repeated_run = run_len; + } + self.state.metrics.longest_repeated_run = @max( + self.state.metrics.longest_repeated_run, + self.state.current_repeated_run, + ); + index = end; + } + } + + fn measureCodepoint(self: *DocumentFilter, codepoint: u21) anyerror!void { + self.state.metrics.codepoints = try addU64(self.state.metrics.codepoints, 1); + if (codepoint == '\n') { + self.state.metrics.newline_codepoints = try addU64( + self.state.metrics.newline_codepoints, + 1, + ); + } + if (codepoint == 0xFFFD) { + self.state.metrics.replacement_codepoints = try addU64( + self.state.metrics.replacement_codepoints, + 1, + ); + } + + if (isVisible(codepoint)) { + self.state.metrics.visible_codepoints = try addU64( + self.state.metrics.visible_codepoints, + 1, + ); + if (self.state.last_visible_codepoint == codepoint) { + self.state.current_repeated_run = try addU64( + self.state.current_repeated_run, + 1, + ); + } else { + self.state.last_visible_codepoint = codepoint; + self.state.current_repeated_run = 1; + } + self.state.metrics.longest_repeated_run = @max( + self.state.metrics.longest_repeated_run, + self.state.current_repeated_run, + ); + } else { + self.state.last_visible_codepoint = null; + self.state.current_repeated_run = 0; + } + } + + fn recordInvalidUtf8(self: *DocumentFilter) anyerror!void { + self.state.metrics.invalid_utf8_sequences = try addU64( + self.state.metrics.invalid_utf8_sequences, + 1, + ); + if (self.options.quality.require_valid_utf8) { + try self.rejectBuiltIn(.invalid_utf8, "document is not valid UTF-8"); + } + } + + fn finishUtf8(self: *DocumentFilter) anyerror!void { + if (self.state.utf8_pending_len != 0) { + self.state.utf8_pending_len = 0; + self.state.utf8_expected_len = 0; + try self.recordInvalidUtf8(); + } + } + + fn evaluateRules(self: *DocumentFilter, chunk: DocumentChunk) anyerror!void { + if (self.rules.len == 0) return; + const rule_chunk: RuleChunk = .{ + .document_id = self.state.document_id, + .bytes = chunk.bytes, + .boundary = chunk.boundary, + .cleaning = chunk.cleaning, + .metrics = self.state.metrics, + }; + for (self.rules, 0..) |rule, index| { + self.state.active_rule_count = @max(self.state.active_rule_count, index + 1); + switch (try rule.vtable.evaluate(rule.context, rule_chunk)) { + .keep => {}, + .reject => |detail| { + try self.rejectRule(rule, detail); + return; + }, + } + } + if (chunk.boundary.document_end) self.state.active_rule_count = 0; + } + + fn applyFinalPolicies(self: *DocumentFilter) anyerror!void { + const metrics = self.state.metrics; + const length = self.options.length; + if (metrics.bytes < length.min_bytes) { + try self.rejectBuiltIn(.bytes_below_minimum, "document is below min_bytes"); + return; + } + if (metrics.codepoints < length.min_codepoints) { + try self.rejectBuiltIn( + .codepoints_below_minimum, + "document is below min_codepoints", + ); + return; + } + if (length.min_lines) |minimum| { + if (metrics.lines() < minimum) { + try self.rejectBuiltIn(.lines_below_minimum, "document is below min_lines"); + return; + } + } + + const quality = self.options.quality; + if (quality.min_visible_basis_points) |minimum| { + if (!ratioAtLeast(metrics.visible_codepoints, metrics.codepoints, minimum)) { + try self.rejectBuiltIn(.low_visible_ratio, "visible ratio is below quality limit"); + return; + } + } + if (quality.max_replacement_basis_points) |maximum| { + if (!ratioAtMost(metrics.replacement_codepoints, metrics.codepoints, maximum)) { + try self.rejectBuiltIn( + .excessive_replacement_ratio, + "replacement ratio exceeds quality limit", + ); + return; + } + } + if (quality.max_cleaning_change_basis_points) |maximum| { + if (!ratioAtMost(metrics.cleaning_changes, metrics.bytes, maximum)) { + try self.rejectBuiltIn( + .excessive_cleaning_ratio, + "cleaning-change ratio exceeds quality limit", + ); + } + } + } + + fn rejectBuiltIn( + self: *DocumentFilter, + reason: RejectionReason, + detail: []const u8, + ) Allocator.Error!void { + self.state.rejection = .{ + .reason = reason, + .rule_id = null, + .rule_kind = null, + .detail = detail, + }; + if (self.state.active_rule_count != 0) self.abortRules(); + } + + fn rejectRule( + self: *DocumentFilter, + rule: Rule, + detail: []const u8, + ) Allocator.Error!void { + self.state.rejection = .{ + .reason = .rule, + .rule_id = rule.id, + .rule_kind = rule.kind, + .detail = try self.document_arena.allocator().dupe(u8, detail), + }; + self.abortRules(); + } + + fn abortRules(self: *DocumentFilter) void { + for (self.rules[0..self.state.active_rule_count]) |rule| { + if (rule.vtable.abort) |abortFn| abortFn(rule.context); + } + self.state.active_rule_count = 0; + } + + fn currentDecision(self: *const DocumentFilter, comptime tag: enum { pending }) Decision { + _ = tag; + if (self.state.rejection) |stored| return .{ .rejected = self.makeRejection(stored) }; + return .{ .pending = self.summary() }; + } + + fn finalDecision(self: *const DocumentFilter) Decision { + if (self.state.rejection) |stored| return .{ .rejected = self.makeRejection(stored) }; + return .{ .accepted = self.summary() }; + } + + fn summary(self: *const DocumentFilter) Summary { + return .{ + .document_id = self.state.document_id, + .metrics = self.state.metrics, + }; + } + + fn makeRejection(self: *const DocumentFilter, stored: StoredRejection) Rejection { + return .{ + .document_id = self.state.document_id, + .reason = stored.reason, + .rule_id = stored.rule_id, + .rule_kind = stored.rule_kind, + .detail = stored.detail, + .metrics = self.state.metrics, + }; + } + + fn configurationFingerprint(self: *const DocumentFilter) u64 { + var hash: u64 = 0xcbf29ce484222325; + fingerprintOptional(&hash, self.options.length.max_bytes); + fingerprintValue(&hash, self.options.length.min_bytes); + fingerprintOptional(&hash, self.options.length.max_codepoints); + fingerprintValue(&hash, self.options.length.min_codepoints); + fingerprintOptional(&hash, self.options.length.min_lines); + fingerprintOptional(&hash, self.options.length.max_lines); + fingerprintValue(&hash, @intFromBool(self.options.quality.require_valid_utf8)); + fingerprintOptional(&hash, self.options.quality.min_visible_basis_points); + fingerprintOptional(&hash, self.options.quality.max_replacement_basis_points); + fingerprintOptional(&hash, self.options.quality.max_cleaning_change_basis_points); + fingerprintOptional(&hash, self.options.quality.max_repeated_codepoints); + fingerprintValue(&hash, self.rules.len); + for (self.rules) |rule| { + fingerprintBytes(&hash, rule.id); + fingerprintValue(&hash, @intFromEnum(rule.kind)); + fingerprintValue(&hash, rule.version); + fingerprintValue(&hash, @intFromBool(rule.stateful)); + } + return hash; + } + + fn ruleKindForId(self: *const DocumentFilter, id: []const u8) ?RuleKind { + for (self.rules) |rule| { + if (std.mem.eql(u8, rule.id, id)) return rule.kind; + } + return null; + } +}; + +const checkpoint_magic = "DFL1"; +const checkpoint_header_len = 132; + +fn validateEncodedState(encoded: []const u8) StreamingState.DecodeError!void { + if (encoded.len < checkpoint_header_len) return error.InvalidState; + if (!std.mem.eql(u8, encoded[0..4], checkpoint_magic)) { + return error.UnsupportedVersion; + } + if (encoded[4] & ~@as(u8, 0b0000_0111) != 0) return error.InvalidState; + + const in_document = encoded[4] & (1 << 0) != 0; + const rejected = encoded[4] & (1 << 1) != 0; + const has_last_codepoint = encoded[4] & (1 << 2) != 0; + const reason = try decodeRejectionReason(encoded[5]); + const pending_len = encoded[6]; + const expected_len = encoded[7]; + const active_count = getU32(encoded[8..12]); + const current_run = getU64(encoded[20..28]); + const last_codepoint = getU32(encoded[100..104]); + const document_len: usize = getU32(encoded[116..120]); + const rule_id_len: usize = getU32(encoded[120..124]); + const detail_len: usize = getU32(encoded[124..128]); + const token_count = getU32(encoded[128..132]); + + if (active_count != token_count) return error.InvalidState; + if (pending_len > 3 or expected_len > 4) return error.InvalidState; + if ((pending_len == 0) != (expected_len == 0)) return error.InvalidState; + if (pending_len != 0 and (expected_len < 2 or pending_len >= expected_len)) { + return error.InvalidState; + } + for (encoded[104 + pending_len .. 108]) |byte| { + if (byte != 0) return error.InvalidState; + } + if (pending_len != 0) { + const expected = std.unicode.utf8ByteSequenceLength(encoded[104]) catch + return error.InvalidState; + if (expected != expected_len) return error.InvalidState; + for (encoded[105 .. 104 + pending_len]) |byte| { + if (byte & 0b1100_0000 != 0b1000_0000) return error.InvalidState; + } + } + if (has_last_codepoint) { + if (last_codepoint > 0x10FFFF or + !std.unicode.utf8ValidCodepoint(@intCast(last_codepoint)) or + current_run == 0) + { + return error.InvalidState; + } + } else if (last_codepoint != 0 or current_run != 0) { + return error.InvalidState; + } + if (rejected != (reason != null)) return error.InvalidState; + if (!rejected and (rule_id_len != 0 or detail_len != 0)) return error.InvalidState; + if (reason == .rule and rule_id_len == 0) return error.InvalidState; + if (reason != null and reason != .rule and rule_id_len != 0) return error.InvalidState; + if (rejected and active_count != 0) return error.InvalidState; + + const metrics = decodeMetrics(encoded[28..100]); + if (metrics.newline_codepoints > metrics.codepoints or + metrics.codepoints > metrics.bytes or + metrics.visible_codepoints > metrics.codepoints or + metrics.replacement_codepoints > metrics.codepoints or + metrics.longest_repeated_run > metrics.visible_codepoints or + current_run > metrics.longest_repeated_run) + { + return error.InvalidState; + } + + var cursor = std.math.add(usize, checkpoint_header_len, document_len) catch + return error.InvalidState; + cursor = std.math.add(usize, cursor, rule_id_len) catch return error.InvalidState; + cursor = std.math.add(usize, cursor, detail_len) catch return error.InvalidState; + if (cursor > encoded.len) return error.InvalidState; + for (0..token_count) |_| { + const length_end = std.math.add(usize, cursor, 4) catch return error.InvalidState; + if (length_end > encoded.len) return error.InvalidState; + const token_len: usize = getU32(encoded[cursor..][0..4]); + cursor = std.math.add(usize, length_end, token_len) catch return error.InvalidState; + if (cursor > encoded.len) return error.InvalidState; + } + if (cursor != encoded.len) return error.InvalidState; + + if (!in_document) { + if (encoded[4] != 0 or encoded[5] != 0 or pending_len != 0 or + expected_len != 0 or active_count != 0 or document_len != 0 or + rule_id_len != 0 or detail_len != 0) + { + return error.InvalidState; + } + for (encoded[8..108]) |byte| { + if (byte != 0) return error.InvalidState; + } + } else if (getU64(encoded[12..20]) == 0) { + return error.InvalidState; + } +} + +fn encodeMetrics(destination: *[72]u8, metrics: Metrics) void { + putU64(destination[0..8], metrics.bytes); + putU64(destination[8..16], metrics.codepoints); + putU64(destination[16..24], metrics.newline_codepoints); + putU64(destination[24..32], metrics.visible_codepoints); + putU64(destination[32..40], metrics.replacement_codepoints); + putU64(destination[40..48], metrics.invalid_utf8_sequences); + putU64(destination[48..56], metrics.cleaning_changes); + putU64(destination[56..64], metrics.whitespace_normalized); + putU64(destination[64..72], metrics.longest_repeated_run); +} + +fn decodeMetrics(source: *const [72]u8) Metrics { + return .{ + .bytes = getU64(source[0..8]), + .codepoints = getU64(source[8..16]), + .newline_codepoints = getU64(source[16..24]), + .visible_codepoints = getU64(source[24..32]), + .replacement_codepoints = getU64(source[32..40]), + .invalid_utf8_sequences = getU64(source[40..48]), + .cleaning_changes = getU64(source[48..56]), + .whitespace_normalized = getU64(source[56..64]), + .longest_repeated_run = getU64(source[64..72]), + }; +} + +fn decodeRejectionReason(encoded: u8) StreamingState.DecodeError!?RejectionReason { + if (encoded == 0) return null; + return switch (encoded - 1) { + @intFromEnum(RejectionReason.bytes_below_minimum) => .bytes_below_minimum, + @intFromEnum(RejectionReason.bytes_above_maximum) => .bytes_above_maximum, + @intFromEnum(RejectionReason.codepoints_below_minimum) => .codepoints_below_minimum, + @intFromEnum(RejectionReason.codepoints_above_maximum) => .codepoints_above_maximum, + @intFromEnum(RejectionReason.lines_below_minimum) => .lines_below_minimum, + @intFromEnum(RejectionReason.lines_above_maximum) => .lines_above_maximum, + @intFromEnum(RejectionReason.invalid_utf8) => .invalid_utf8, + @intFromEnum(RejectionReason.low_visible_ratio) => .low_visible_ratio, + @intFromEnum(RejectionReason.excessive_replacement_ratio) => .excessive_replacement_ratio, + @intFromEnum(RejectionReason.excessive_cleaning_ratio) => .excessive_cleaning_ratio, + @intFromEnum(RejectionReason.repeated_codepoint_run) => .repeated_codepoint_run, + @intFromEnum(RejectionReason.rule) => .rule, + else => error.InvalidState, + }; +} + +fn fingerprintOptional(hash: *u64, value: anytype) void { + if (value) |present| { + fingerprintValue(hash, 1); + fingerprintValue(hash, present); + } else { + fingerprintValue(hash, 0); + } +} + +fn fingerprintValue(hash: *u64, value: anytype) void { + const canonical: u64 = @intCast(value); + for (0..8) |index| { + hash.* ^= @as(u8, @truncate(canonical >> @intCast(index * 8))); + hash.* *%= 0x100000001b3; + } +} + +fn fingerprintBytes(hash: *u64, bytes: []const u8) void { + fingerprintValue(hash, bytes.len); + for (bytes) |byte| { + hash.* ^= byte; + hash.* *%= 0x100000001b3; + } +} + +fn putU32(destination: *[4]u8, value: u32) void { + for (destination, 0..) |*byte, index| { + byte.* = @truncate(value >> @intCast(index * 8)); + } +} + +fn putU64(destination: *[8]u8, value: u64) void { + for (destination, 0..) |*byte, index| { + byte.* = @truncate(value >> @intCast(index * 8)); + } +} + +fn getU32(source: *const [4]u8) u32 { + var value: u32 = 0; + for (source, 0..) |byte, index| { + value |= @as(u32, byte) << @intCast(index * 8); + } + return value; +} + +fn getU64(source: *const [8]u8) u64 { + var value: u64 = 0; + for (source, 0..) |byte, index| { + value |= @as(u64, byte) << @intCast(index * 8); + } + return value; +} + +fn validateOptions(options: Options) DocumentFilter.CreateError!void { + const length = options.length; + if (length.max_bytes) |maximum| { + if (maximum < length.min_bytes) return error.InvalidLengthPolicy; + } + if (length.max_codepoints) |maximum| { + if (maximum < length.min_codepoints) return error.InvalidLengthPolicy; + } + if (length.min_lines) |minimum| { + if (length.max_lines) |maximum| { + if (maximum < minimum) return error.InvalidLengthPolicy; + } + } + + const quality = options.quality; + inline for (.{ + quality.min_visible_basis_points, + quality.max_replacement_basis_points, + quality.max_cleaning_change_basis_points, + }) |threshold| { + if (threshold) |value| { + if (value > 10_000) return error.InvalidQualityRatio; + } + } +} + +fn cleaningChanges(signals: CleaningSignals) error{CounterOverflow}!u64 { + var total = try addU64(signals.malformed_sequences, signals.controls_removed); + total = try addU64(total, signals.format_characters_removed); + return total; +} + +fn addU64(left: u64, right: anytype) error{CounterOverflow}!u64 { + const right_u64 = std.math.cast(u64, right) orelse return error.CounterOverflow; + return std.math.add(u64, left, right_u64) catch error.CounterOverflow; +} + +fn ratioAtLeast(numerator: u64, denominator: u64, basis_points: u16) bool { + if (denominator == 0) return numerator == 0 and basis_points == 0; + return @as(u128, numerator) * 10_000 >= + @as(u128, denominator) * basis_points; +} + +fn ratioAtMost(numerator: u64, denominator: u64, basis_points: u16) bool { + if (denominator == 0) return numerator == 0; + return @as(u128, numerator) * 10_000 <= + @as(u128, denominator) * basis_points; +} + +fn isVisible(codepoint: u21) bool { + return !isWhitespace(codepoint) and !isControl(codepoint) and !isFormat(codepoint); +} + +fn isWhitespace(codepoint: u21) bool { + return switch (codepoint) { + 0x0009...0x000D, 0x0020, 0x0085, 0x00A0, 0x1680, 0x2028, 0x2029, 0x202F, 0x205F, 0x3000 => true, + 0x2000...0x200A => true, + else => false, + }; +} + +fn isControl(codepoint: u21) bool { + return codepoint <= 0x001F or (codepoint >= 0x007F and codepoint <= 0x009F); +} + +fn isFormat(codepoint: u21) bool { + return switch (codepoint) { + 0x00AD, 0x061C, 0x06DD, 0x070F, 0x08E2, 0x180E, 0xFEFF, 0x110BD, 0x110CD, 0xE0001 => true, + 0x0600...0x0605, 0x0890...0x0891, 0x200B...0x200F, 0x202A...0x202E => true, + 0x2060...0x206F, 0xFFF9...0xFFFB, 0x13430...0x1343F, 0x1BCA0...0x1BCA3 => true, + 0x1D173...0x1D17A, 0xE0020...0xE007F => true, + else => false, + }; +} + +test "length policies reject early and accept only at document end" { + const filter = try DocumentFilter.create(std.testing.allocator, .{ + .length = .{ .min_bytes = 3, .max_bytes = 5, .min_codepoints = 3 }, + }, &.{}); + defer filter.destroy(); + + const pending = try filter.observe(.{ + .document_id = "good", + .bytes = "abc", + .boundary = .{ .document_start = true }, + }); + try std.testing.expect(pending == .pending); + const accepted = try filter.observe(.{ + .document_id = "good", + .bytes = "de", + .boundary = .{ .document_end = true }, + }); + try std.testing.expect(accepted == .accepted); + try std.testing.expectEqual(@as(u64, 5), accepted.accepted.metrics.bytes); + + const rejected = try filter.observe(.{ + .document_id = "large", + .bytes = "123456", + .boundary = .{ .document_start = true }, + }); + try std.testing.expect(rejected == .rejected); + try std.testing.expectEqual(RejectionReason.bytes_above_maximum, rejected.rejected.reason); + const drained = try filter.observe(.{ + .document_id = "large", + .bytes = "ignored", + .boundary = .{ .document_end = true }, + }); + try std.testing.expect(drained == .rejected); +} + +test "quality policies use UTF-8 and cleaner evidence" { + const filter = try DocumentFilter.create(std.testing.allocator, .{ + .quality = .{ + .max_replacement_basis_points = 1_000, + .max_cleaning_change_basis_points = 2_000, + .max_repeated_codepoints = 3, + }, + }, &.{}); + defer filter.destroy(); + + const repeated = try filter.observe(.{ + .document_id = "repeat", + .bytes = "aaaa", + .boundary = .{ .document_start = true, .document_end = true }, + }); + try std.testing.expectEqual( + RejectionReason.repeated_codepoint_run, + repeated.rejected.reason, + ); + + const dirty = try filter.observe(.{ + .document_id = "dirty", + .bytes = "clean", + .boundary = .{ .document_start = true, .document_end = true }, + .cleaning = .{ .malformed_sequences = 2 }, + }); + try std.testing.expectEqual( + RejectionReason.excessive_cleaning_ratio, + dirty.rejected.reason, + ); + + const invalid = try filter.observe(.{ + .document_id = "invalid", + .bytes = "bad\xFFutf8", + .boundary = .{ .document_start = true, .document_end = true }, + }); + try std.testing.expectEqual(RejectionReason.invalid_utf8, invalid.rejected.reason); +} + +const TestLanguageRule = struct { + saw_start: bool = false, + aborted: bool = false, + + const vtable: Rule.VTable = .{ + .evaluate = evaluate, + .abort = abort, + }; + + fn rule(self: *TestLanguageRule) Rule { + return .{ + .id = "language/allow-english", + .kind = .language, + .context = self, + .vtable = &vtable, + }; + } + + fn evaluate(context: *anyopaque, chunk: RuleChunk) anyerror!RuleDecision { + const self: *TestLanguageRule = @ptrCast(@alignCast(context)); + if (chunk.boundary.document_start) self.saw_start = true; + if (std.mem.indexOf(u8, chunk.bytes, "bonjour") != null) { + return .{ .reject = "language is not allowed" }; + } + return .keep; + } + + fn abort(context: *anyopaque) void { + const self: *TestLanguageRule = @ptrCast(@alignCast(context)); + self.aborted = true; + } +}; + +test "language and safety adapters can reject without buffering documents" { + var language: TestLanguageRule = .{}; + const filter = try DocumentFilter.create( + std.testing.allocator, + .{}, + &.{language.rule()}, + ); + defer filter.destroy(); + + const decision = try filter.observe(.{ + .document_id = "french", + .bytes = "bonjour le monde", + .boundary = .{ .document_start = true, .document_end = true }, + }); + try std.testing.expect(decision == .rejected); + try std.testing.expectEqualStrings("language/allow-english", decision.rejected.rule_id.?); + try std.testing.expect(language.saw_start); + try std.testing.expect(language.aborted); +} + +const StatefulByteRule = struct { + bytes: u64 = 0, + + const vtable: Rule.VTable = .{ + .evaluate = evaluate, + .abort = abort, + .checkpoint = checkpoint, + .restore = restore, + }; + + fn rule(self: *StatefulByteRule) Rule { + return .{ + .id = "safety/stateful-byte-limit", + .kind = .safety, + .context = self, + .vtable = &vtable, + .stateful = true, + }; + } + + fn evaluate(context: *anyopaque, chunk: RuleChunk) anyerror!RuleDecision { + const self: *StatefulByteRule = @ptrCast(@alignCast(context)); + self.bytes = try addU64(self.bytes, chunk.bytes.len); + if (chunk.boundary.document_end and self.bytes > 5) { + return .{ .reject = "stateful safety byte limit exceeded" }; + } + return .keep; + } + + fn abort(context: *anyopaque) void { + const self: *StatefulByteRule = @ptrCast(@alignCast(context)); + self.bytes = 0; + } + + fn checkpoint(context: *anyopaque, allocator: Allocator) anyerror![]const u8 { + const self: *StatefulByteRule = @ptrCast(@alignCast(context)); + const token = try allocator.alloc(u8, 8); + putU64(token[0..8], self.bytes); + return token; + } + + fn restore(context: *anyopaque, token: []const u8) anyerror!void { + if (token.len != 8) return error.InvalidToken; + const self: *StatefulByteRule = @ptrCast(@alignCast(context)); + self.bytes = getU64(token[0..8]); + } +}; + +test "checkpoints transfer metrics partial UTF-8 and stateful rule tokens" { + var source_rule: StatefulByteRule = .{}; + const source = try DocumentFilter.create( + std.testing.allocator, + .{}, + &.{source_rule.rule()}, + ); + defer source.destroy(); + _ = try source.observe(.{ + .document_id = "transfer", + .bytes = "ab\xF0\x9F", + .boundary = .{ .document_start = true }, + }); + + const encoded = try source.checkpoint(std.testing.allocator); + defer std.testing.allocator.free(encoded); + const checkpoint_state = try StreamingState.decode(encoded); + + var destination_rule: StatefulByteRule = .{}; + const destination = try DocumentFilter.create( + std.testing.allocator, + .{}, + &.{destination_rule.rule()}, + ); + defer destination.destroy(); + try destination.restore(checkpoint_state); + const decision = try destination.observe(.{ + .document_id = "transfer", + .bytes = "\x98\x80cd", + .boundary = .{ .document_end = true }, + }); + try std.testing.expect(decision == .rejected); + try std.testing.expectEqualStrings( + "safety/stateful-byte-limit", + decision.rejected.rule_id.?, + ); + try std.testing.expectEqual(@as(u64, 5), decision.rejected.metrics.codepoints); +} + +test "UTF-8 codepoints may span filter chunks" { + const filter = try DocumentFilter.create(std.testing.allocator, .{}, &.{}); + defer filter.destroy(); + _ = try filter.observe(.{ + .document_id = "unicode", + .bytes = "\xF0\x9F", + .boundary = .{ .document_start = true }, + }); + const decision = try filter.observe(.{ + .document_id = "unicode", + .bytes = "\x98\x80", + .boundary = .{ .document_end = true }, + }); + try std.testing.expect(decision == .accepted); + try std.testing.expectEqual(@as(u64, 1), decision.accepted.metrics.codepoints); +} diff --git a/data/document-filter/root.zig b/data/document-filter/root.zig new file mode 100644 index 0000000..d46e4d2 --- /dev/null +++ b/data/document-filter/root.zig @@ -0,0 +1,25 @@ +//! Streaming document filtering and rule adapters. + +const filter = @import("document_filter.zig"); + +pub const Boundary = filter.Boundary; +pub const CleaningSignals = filter.CleaningSignals; +pub const Decision = filter.Decision; +pub const DocumentChunk = filter.DocumentChunk; +pub const DocumentFilter = filter.DocumentFilter; +pub const LengthPolicy = filter.LengthPolicy; +pub const Metrics = filter.Metrics; +pub const Options = filter.Options; +pub const QualityPolicy = filter.QualityPolicy; +pub const Rejection = filter.Rejection; +pub const RejectionReason = filter.RejectionReason; +pub const Rule = filter.Rule; +pub const RuleChunk = filter.RuleChunk; +pub const RuleDecision = filter.RuleDecision; +pub const RuleKind = filter.RuleKind; +pub const Summary = filter.Summary; +pub const StreamingState = filter.StreamingState; + +test { + _ = filter; +} diff --git a/data/text-cleaner/README.md b/data/text-cleaner/README.md new file mode 100644 index 0000000..4b1feef --- /dev/null +++ b/data/text-cleaner/README.md @@ -0,0 +1,27 @@ +# Text cleaner + +`TextCleaner` is a stateful streaming transform for raw corpus bytes. It emits +UTF-8 while preserving decoder state across corpus chunk boundaries. + +Supported input encodings are UTF-8, UTF-16LE, UTF-16BE, ISO-8859-1, and +Windows-1252. Automatic mode recognizes UTF byte-order marks and otherwise uses +UTF-8. Invalid input can be replaced, removed, or rejected. + +Whitespace is normalized to ASCII space and LF, CRLF is coalesced even when it +crosses chunks, line-edge spaces are removed, and blank-line runs are bounded. +Control and Unicode formatting characters have independent policies. Output is +owned by a resettable arena and is invalidated by the next cleaning operation; +retained arena capacity can be capped for workloads with untrusted chunk sizes. + +`cleanCorpusChunk` maps corpus-reader document boundaries directly into the +stateful decoder. Empty cleaned chunks are valid because a trailing code unit or +whitespace run may need the next corpus chunk before it can be resolved. + +`checkpoint` produces a fixed-size, versioned state containing partial encoding +and formatting state. It can be decoded and restored on another worker with the +same semantic options; memory-retention settings may differ. + +Full NFC/NFKC normalization remains an explicit TODO until the repository has a +versioned Unicode data-table component. + +Run all tests from the repository root with `zig build test`. diff --git a/data/text-cleaner/root.zig b/data/text-cleaner/root.zig new file mode 100644 index 0000000..2c464d8 --- /dev/null +++ b/data/text-cleaner/root.zig @@ -0,0 +1,18 @@ +//! Stateful, bounded-memory text normalization. + +const cleaner = @import("text_cleaner.zig"); + +pub const Boundary = cleaner.Boundary; +pub const CleanedChunk = cleaner.CleanedChunk; +pub const InputEncoding = cleaner.InputEncoding; +pub const MalformedPolicy = cleaner.MalformedPolicy; +pub const Options = cleaner.Options; +pub const ScalarPolicy = cleaner.ScalarPolicy; +pub const Stats = cleaner.Stats; +pub const StreamingState = cleaner.StreamingState; +pub const TextCleaner = cleaner.TextCleaner; +pub const Totals = cleaner.Totals; + +test { + _ = cleaner; +} diff --git a/data/text-cleaner/text_cleaner.zig b/data/text-cleaner/text_cleaner.zig new file mode 100644 index 0000000..e881018 --- /dev/null +++ b/data/text-cleaner/text_cleaner.zig @@ -0,0 +1,1192 @@ +const std = @import("std"); + +const Allocator = std.mem.Allocator; +const ArenaAllocator = std.heap.ArenaAllocator; +const CorpusChunk = @import("../corpus-reader/root.zig").Chunk; + +pub const InputEncoding = enum { + /// Detects UTF-8, UTF-16LE, or UTF-16BE from a byte-order mark. Input + /// without a byte-order mark is interpreted as UTF-8. + auto, + utf8, + utf16_le, + utf16_be, + latin1, + windows_1252, +}; + +pub const MalformedPolicy = enum { + replace, + remove, + reject, +}; + +pub const ScalarPolicy = enum { + preserve, + remove, + reject, +}; + +pub const Options = struct { + input_encoding: InputEncoding = .auto, + strip_byte_order_mark: bool = true, + malformed: MalformedPolicy = .replace, + replacement: u21 = 0xFFFD, + + /// All Unicode White_Space code points are mapped to ASCII space or LF. + /// Line-leading and line-trailing horizontal whitespace is removed. + collapse_horizontal_whitespace: bool = true, + + /// Two LFs represent one empty line. Null preserves every normalized LF. + max_consecutive_newlines: ?usize = 2, + + /// Leading newline runs and trailing normalized whitespace are removed. + /// Trailing whitespace is delayed across chunks until the boundary is known. + /// Horizontal whitespace at line edges is always removed. + trim_document_edges: bool = true, + + control_characters: ScalarPolicy = .remove, + unicode_format_characters: ScalarPolicy = .remove, + + /// Null retains peak output capacity for maximum throughput. A limit keeps + /// one unusually expansive or malformed chunk from raising steady-state use. + retained_output_bytes: ?usize = null, + + // TODO(data/unicode): add NFC/NFKC normalization after a Unicode data-table + // component provides versioned decomposition and composition tables. +}; + +pub const Boundary = struct { + document_start: bool = false, + document_end: bool = false, + + pub const whole: Boundary = .{ + .document_start = true, + .document_end = true, + }; +}; + +pub const Stats = struct { + input_bytes: usize = 0, + output_bytes: usize = 0, + malformed_sequences: usize = 0, + whitespace_normalized: usize = 0, + controls_removed: usize = 0, + format_characters_removed: usize = 0, +}; + +pub const Totals = struct { + input_bytes: u64 = 0, + output_bytes: u64 = 0, + documents: u64 = 0, + malformed_sequences: u64 = 0, + whitespace_normalized: u64 = 0, + controls_removed: u64 = 0, + format_characters_removed: u64 = 0, + + fn add(self: *Totals, stats: Stats, document_ended: bool) void { + self.input_bytes += stats.input_bytes; + self.output_bytes += stats.output_bytes; + self.malformed_sequences += stats.malformed_sequences; + self.whitespace_normalized += stats.whitespace_normalized; + self.controls_removed += stats.controls_removed; + self.format_characters_removed += stats.format_characters_removed; + if (document_ended) self.documents += 1; + } +}; + +/// Borrowed output. `bytes` remains valid until the next call to `clean`, +/// `abortDocument`, or `destroy`. +pub const CleanedChunk = struct { + bytes: []const u8, + stats: Stats, + document_start: bool, + document_end: bool, +}; + +/// Fixed-size, transport-stable cleaner state. It includes partial decoder and +/// delayed-whitespace state but intentionally excludes monitoring counters. +pub const StreamingState = struct { + encoded: [wire_len]u8, + + pub const wire_len = 48; + const magic = "TCL1"; + + pub const DecodeError = error{ + InvalidState, + UnsupportedVersion, + }; + + pub fn asBytes(self: *const StreamingState) *const [wire_len]u8 { + return &self.encoded; + } + + pub fn decode(bytes: []const u8) DecodeError!StreamingState { + if (bytes.len != wire_len) return error.InvalidState; + var state: StreamingState = undefined; + @memcpy(&state.encoded, bytes); + try validateStreamingState(state); + return state; + } +}; + +const ResolvedEncoding = enum { + utf8, + utf16_le, + utf16_be, + latin1, + windows_1252, +}; + +const CleanerState = struct { + in_document: bool = false, + encoding: ?ResolvedEncoding = null, + probe: [3]u8 = @splat(0), + probe_len: u2 = 0, + + utf8_pending: [4]u8 = @splat(0), + utf8_pending_len: u3 = 0, + utf8_expected_len: u3 = 0, + + utf16_pending_byte: ?u8 = null, + utf16_high_surrogate: ?u16 = null, + + previous_was_cr: bool = false, + pending_spaces: usize = 0, + pending_newlines: usize = 0, + line_has_content: bool = false, + document_has_content: bool = false, +}; + +const ProbeDecision = struct { + encoding: ResolvedEncoding, + skip: u2, +}; + +pub const TextCleaner = struct { + backing_allocator: Allocator, + options: Options, + output_arena: ArenaAllocator, + state: CleanerState = .{}, + counters: Totals = .{}, + + pub const Error = Allocator.Error || error{ + ByteOrderMarkMismatch, + ControlCharacterRejected, + DocumentAlreadyStarted, + DocumentNotStarted, + FormatCharacterRejected, + InputTooLarge, + InvalidReplacement, + MalformedEncoding, + }; + + pub const CheckpointError = error{StateTooLarge}; + pub const RestoreError = StreamingState.DecodeError || error{ + CheckpointOptionsMismatch, + }; + + pub fn create(backing_allocator: Allocator, options: Options) Error!*TextCleaner { + if (!std.unicode.utf8ValidCodepoint(options.replacement)) { + return error.InvalidReplacement; + } + + const self = try backing_allocator.create(TextCleaner); + self.* = .{ + .backing_allocator = backing_allocator, + .options = options, + .output_arena = .init(backing_allocator), + }; + return self; + } + + pub fn destroy(self: *TextCleaner) void { + self.output_arena.deinit(); + const backing_allocator = self.backing_allocator; + backing_allocator.destroy(self); + } + + /// Cleans one streaming input fragment. Fragments may split UTF-8 code + /// points, UTF-16 code units, surrogate pairs, BOMs, and CRLF sequences. + /// On error, decoder and formatting state are rolled back to the state at + /// the beginning of this call. + pub fn clean( + self: *TextCleaner, + input: []const u8, + boundary: Boundary, + ) Error!CleanedChunk { + const state_before = self.state; + errdefer self.state = state_before; + + self.resetOutputArena(); + const output_allocator = self.output_arena.allocator(); + var output: std.ArrayList(u8) = .empty; + + const reserve = std.math.mul(usize, input.len, reserveFactor(self.options.input_encoding)) catch + return error.InputTooLarge; + try output.ensureTotalCapacityPrecise(output_allocator, reserve); + + if (boundary.document_start) { + if (self.state.in_document) return error.DocumentAlreadyStarted; + self.startDocument(); + } else if (!self.state.in_document) { + return error.DocumentNotStarted; + } + + var stats: Stats = .{ .input_bytes = input.len }; + try self.decode(input, boundary.document_end, &output, &stats); + + if (boundary.document_end) { + try self.finishDocument(&output, &stats); + self.state.in_document = false; + } + + stats.output_bytes = output.items.len; + self.counters.add(stats, boundary.document_end); + return .{ + .bytes = output.items, + .stats = stats, + .document_start = boundary.document_start, + .document_end = boundary.document_end, + }; + } + + /// Convenience boundary for the corpus reader without coupling ownership: + /// source metadata remains on the corpus chunk and only text bytes are copied. + pub fn cleanCorpusChunk( + self: *TextCleaner, + chunk: CorpusChunk, + ) Error!CleanedChunk { + return self.clean(chunk.bytes, .{ + .document_start = chunk.is_document_start, + .document_end = chunk.is_document_end, + }); + } + + /// Captures all semantic state required to continue on another worker. + pub fn checkpoint(self: *const TextCleaner) CheckpointError!StreamingState { + var encoded: [StreamingState.wire_len]u8 = @splat(0); + @memcpy(encoded[0..4], StreamingState.magic); + putU64(encoded[36..44], optionsFingerprint(self.options)); + + if (!self.state.in_document) return .{ .encoded = encoded }; + + var flags: u8 = 1 << 0; + if (self.state.utf16_pending_byte != null) flags |= 1 << 1; + if (self.state.utf16_high_surrogate != null) flags |= 1 << 2; + if (self.state.previous_was_cr) flags |= 1 << 3; + if (self.state.line_has_content) flags |= 1 << 4; + if (self.state.document_has_content) flags |= 1 << 5; + encoded[4] = flags; + encoded[5] = encodeResolvedEncoding(self.state.encoding); + encoded[6] = self.state.probe_len; + encoded[7] = self.state.utf8_pending_len; + encoded[8] = self.state.utf8_expected_len; + @memcpy( + encoded[10..][0..self.state.probe_len], + self.state.probe[0..self.state.probe_len], + ); + @memcpy( + encoded[13..][0..self.state.utf8_pending_len], + self.state.utf8_pending[0..self.state.utf8_pending_len], + ); + encoded[17] = self.state.utf16_pending_byte orelse 0; + putU16(encoded[18..20], self.state.utf16_high_surrogate orelse 0); + putU64( + encoded[20..28], + std.math.cast(u64, self.state.pending_spaces) orelse + return error.StateTooLarge, + ); + putU64( + encoded[28..36], + std.math.cast(u64, self.state.pending_newlines) orelse + return error.StateTooLarge, + ); + const result: StreamingState = .{ .encoded = encoded }; + validateStreamingState(result) catch unreachable; + return result; + } + + /// Restores a decoded checkpoint. Semantic options must match the worker + /// that created it; arena-retention settings may differ between workers. + pub fn restore( + self: *TextCleaner, + streaming_state: StreamingState, + ) RestoreError!void { + try validateStreamingState(streaming_state); + if (getU64(streaming_state.encoded[36..44]) != optionsFingerprint(self.options)) { + return error.CheckpointOptionsMismatch; + } + + self.state = try unpackStreamingState(streaming_state); + self.resetOutputArena(); + } + + /// Discards decoder state when an upstream source abandons a document. + pub fn abortDocument(self: *TextCleaner) void { + self.state = .{}; + self.resetOutputArena(); + } + + pub fn totals(self: *const TextCleaner) Totals { + return self.counters; + } + + fn startDocument(self: *TextCleaner) void { + self.state = .{ .in_document = true }; + self.state.encoding = switch (self.options.input_encoding) { + .latin1 => .latin1, + .windows_1252 => .windows_1252, + .utf8 => if (self.options.strip_byte_order_mark) null else .utf8, + .utf16_le => if (self.options.strip_byte_order_mark) null else .utf16_le, + .utf16_be => if (self.options.strip_byte_order_mark) null else .utf16_be, + .auto => null, + }; + } + + fn decode( + self: *TextCleaner, + input: []const u8, + document_end: bool, + output: *std.ArrayList(u8), + stats: *Stats, + ) Error!void { + var input_index: usize = 0; + while (self.state.encoding == null and input_index < input.len) { + self.state.probe[self.state.probe_len] = input[input_index]; + self.state.probe_len += 1; + input_index += 1; + + if (try self.detectEncoding(false)) |decision| { + try self.applyProbeDecision(decision, output, stats); + } + } + + if (self.state.encoding == null and document_end) { + const decision = (try self.detectEncoding(true)).?; + try self.applyProbeDecision(decision, output, stats); + } + + if (self.state.encoding) |encoding| { + try self.decodeResolved(encoding, input[input_index..], output, stats); + } + } + + fn detectEncoding(self: *TextCleaner, final: bool) Error!?ProbeDecision { + const bytes = self.state.probe[0..self.state.probe_len]; + return switch (self.options.input_encoding) { + .auto => detectAutomatic(bytes, final, self.options.strip_byte_order_mark), + .utf8 => detectExplicitUtf8(bytes, final, self.options.strip_byte_order_mark), + .utf16_le => try detectExplicitUtf16( + bytes, + final, + .utf16_le, + self.options.strip_byte_order_mark, + ), + .utf16_be => try detectExplicitUtf16( + bytes, + final, + .utf16_be, + self.options.strip_byte_order_mark, + ), + .latin1 => .{ .encoding = .latin1, .skip = 0 }, + .windows_1252 => .{ .encoding = .windows_1252, .skip = 0 }, + }; + } + + fn applyProbeDecision( + self: *TextCleaner, + decision: ProbeDecision, + output: *std.ArrayList(u8), + stats: *Stats, + ) Error!void { + var buffered: [3]u8 = undefined; + const buffered_len: usize = self.state.probe_len - decision.skip; + @memcpy(buffered[0..buffered_len], self.state.probe[decision.skip..self.state.probe_len]); + self.state.probe_len = 0; + self.state.encoding = decision.encoding; + try self.decodeResolved(decision.encoding, buffered[0..buffered_len], output, stats); + } + + fn decodeResolved( + self: *TextCleaner, + encoding: ResolvedEncoding, + bytes: []const u8, + output: *std.ArrayList(u8), + stats: *Stats, + ) Error!void { + switch (encoding) { + .utf8 => try self.decodeUtf8(bytes, output, stats), + .utf16_le => try self.decodeUtf16(bytes, .little, output, stats), + .utf16_be => try self.decodeUtf16(bytes, .big, output, stats), + .latin1 => { + for (bytes) |byte| try self.processCodepoint(byte, output, stats); + }, + .windows_1252 => { + for (bytes) |byte| { + if (windows1252Codepoint(byte)) |codepoint| { + try self.processCodepoint(codepoint, output, stats); + } else { + try self.handleMalformed(output, stats); + } + } + }, + } + } + + fn decodeUtf8( + self: *TextCleaner, + bytes: []const u8, + output: *std.ArrayList(u8), + stats: *Stats, + ) Error!void { + var index: usize = 0; + while (index < bytes.len) { + const byte = bytes[index]; + if (self.state.utf8_pending_len == 0) { + if (byte >= 0x21 and byte <= 0x7E) { + const run_start = index; + index += 1; + while (index < bytes.len and + bytes[index] >= 0x21 and bytes[index] <= 0x7E) + { + index += 1; + } + try self.emitAsciiRun(bytes[run_start..index], output); + continue; + } + if (byte < 0x80) { + try self.processCodepoint(byte, output, stats); + index += 1; + continue; + } + + const expected = std.unicode.utf8ByteSequenceLength(byte) catch { + try self.handleMalformed(output, stats); + index += 1; + continue; + }; + self.state.utf8_pending[0] = byte; + self.state.utf8_pending_len = 1; + self.state.utf8_expected_len = expected; + index += 1; + continue; + } + + if (byte & 0b1100_0000 != 0b1000_0000) { + try self.handleMalformed(output, stats); + self.state.utf8_pending_len = 0; + self.state.utf8_expected_len = 0; + continue; + } + + self.state.utf8_pending[self.state.utf8_pending_len] = byte; + self.state.utf8_pending_len += 1; + index += 1; + if (self.state.utf8_pending_len == self.state.utf8_expected_len) { + const sequence = self.state.utf8_pending[0..self.state.utf8_pending_len]; + const codepoint = std.unicode.utf8Decode(sequence) catch { + try self.handleMalformed(output, stats); + self.state.utf8_pending_len = 0; + self.state.utf8_expected_len = 0; + continue; + }; + self.state.utf8_pending_len = 0; + self.state.utf8_expected_len = 0; + try self.processCodepoint(codepoint, output, stats); + } + } + } + + fn decodeUtf16( + self: *TextCleaner, + bytes: []const u8, + endian: std.builtin.Endian, + output: *std.ArrayList(u8), + stats: *Stats, + ) Error!void { + for (bytes) |byte| { + if (self.state.utf16_pending_byte) |first| { + self.state.utf16_pending_byte = null; + const code_unit = switch (endian) { + .little => @as(u16, first) | (@as(u16, byte) << 8), + .big => (@as(u16, first) << 8) | byte, + }; + try self.processUtf16CodeUnit(code_unit, output, stats); + } else { + self.state.utf16_pending_byte = byte; + } + } + } + + fn processUtf16CodeUnit( + self: *TextCleaner, + code_unit: u16, + output: *std.ArrayList(u8), + stats: *Stats, + ) Error!void { + if (self.state.utf16_high_surrogate) |high| { + if (std.unicode.utf16IsLowSurrogate(code_unit)) { + const pair = [_]u16{ high, code_unit }; + self.state.utf16_high_surrogate = null; + const codepoint = std.unicode.utf16DecodeSurrogatePair(&pair) catch unreachable; + try self.processCodepoint(codepoint, output, stats); + return; + } + self.state.utf16_high_surrogate = null; + try self.handleMalformed(output, stats); + // The current code unit begins a new scalar and must be reprocessed. + } + + if (std.unicode.utf16IsHighSurrogate(code_unit)) { + self.state.utf16_high_surrogate = code_unit; + } else if (std.unicode.utf16IsLowSurrogate(code_unit)) { + try self.handleMalformed(output, stats); + } else { + try self.processCodepoint(code_unit, output, stats); + } + } + + fn processCodepoint( + self: *TextCleaner, + codepoint: u21, + output: *std.ArrayList(u8), + stats: *Stats, + ) Error!void { + if (self.state.previous_was_cr) { + self.state.previous_was_cr = false; + if (codepoint == '\n') { + stats.whitespace_normalized += 1; + return; + } + } + + if (codepoint == '\r') { + self.state.previous_was_cr = true; + stats.whitespace_normalized += 1; + try self.queueNewline(); + return; + } + if (isLineBreak(codepoint)) { + stats.whitespace_normalized += 1; + try self.queueNewline(); + return; + } + if (isHorizontalWhitespace(codepoint)) { + stats.whitespace_normalized += 1; + try self.queueSpace(); + return; + } + if (isControl(codepoint)) { + switch (self.options.control_characters) { + .preserve => try self.emitVisible(codepoint, output), + .remove => stats.controls_removed += 1, + .reject => return error.ControlCharacterRejected, + } + return; + } + if (isFormatCharacter(codepoint)) { + switch (self.options.unicode_format_characters) { + .preserve => try self.emitVisible(codepoint, output), + .remove => stats.format_characters_removed += 1, + .reject => return error.FormatCharacterRejected, + } + return; + } + + try self.emitVisible(codepoint, output); + } + + fn queueSpace(self: *TextCleaner) Error!void { + // Horizontal whitespace at line edges is always removed. + if (!self.state.line_has_content or self.state.pending_newlines != 0) return; + if (self.options.collapse_horizontal_whitespace) { + self.state.pending_spaces = 1; + } else { + self.state.pending_spaces = std.math.add( + usize, + self.state.pending_spaces, + 1, + ) catch return error.InputTooLarge; + } + } + + fn queueNewline(self: *TextCleaner) Error!void { + self.state.pending_spaces = 0; + self.state.line_has_content = false; + if (self.options.trim_document_edges and !self.state.document_has_content) return; + + if (self.options.max_consecutive_newlines) |maximum| { + self.state.pending_newlines = @min( + maximum, + self.state.pending_newlines +| 1, + ); + } else { + self.state.pending_newlines = std.math.add( + usize, + self.state.pending_newlines, + 1, + ) catch return error.InputTooLarge; + } + } + + fn emitVisible( + self: *TextCleaner, + codepoint: u21, + output: *std.ArrayList(u8), + ) Allocator.Error!void { + const allocator = self.output_arena.allocator(); + try self.flushPending(output); + + var encoded: [4]u8 = undefined; + const encoded_len = std.unicode.utf8Encode(codepoint, &encoded) catch unreachable; + try output.appendSlice(allocator, encoded[0..encoded_len]); + self.state.line_has_content = true; + self.state.document_has_content = true; + } + + fn emitAsciiRun( + self: *TextCleaner, + bytes: []const u8, + output: *std.ArrayList(u8), + ) Allocator.Error!void { + const allocator = self.output_arena.allocator(); + try self.flushPending(output); + try output.appendSlice(allocator, bytes); + self.state.line_has_content = true; + self.state.document_has_content = true; + } + + fn flushPending( + self: *TextCleaner, + output: *std.ArrayList(u8), + ) Allocator.Error!void { + const allocator = self.output_arena.allocator(); + if (self.state.pending_newlines != 0) { + const newlines = try output.addManyAsSlice( + allocator, + self.state.pending_newlines, + ); + @memset(newlines, '\n'); + self.state.pending_newlines = 0; + } + if (self.state.pending_spaces != 0) { + const spaces = try output.addManyAsSlice( + allocator, + self.state.pending_spaces, + ); + @memset(spaces, ' '); + self.state.pending_spaces = 0; + } + } + + fn handleMalformed( + self: *TextCleaner, + output: *std.ArrayList(u8), + stats: *Stats, + ) Error!void { + stats.malformed_sequences += 1; + switch (self.options.malformed) { + .replace => try self.processCodepoint(self.options.replacement, output, stats), + .remove => {}, + .reject => return error.MalformedEncoding, + } + } + + fn finishDocument( + self: *TextCleaner, + output: *std.ArrayList(u8), + stats: *Stats, + ) Error!void { + switch (self.state.encoding.?) { + .utf8 => { + if (self.state.utf8_pending_len != 0) { + self.state.utf8_pending_len = 0; + self.state.utf8_expected_len = 0; + try self.handleMalformed(output, stats); + } + }, + .utf16_le, .utf16_be => { + if (self.state.utf16_pending_byte != null) { + self.state.utf16_pending_byte = null; + try self.handleMalformed(output, stats); + } + if (self.state.utf16_high_surrogate != null) { + self.state.utf16_high_surrogate = null; + try self.handleMalformed(output, stats); + } + }, + .latin1, .windows_1252 => {}, + } + + self.state.previous_was_cr = false; + if (self.options.trim_document_edges) { + self.state.pending_spaces = 0; + self.state.pending_newlines = 0; + } else if (self.state.pending_newlines != 0) { + const newlines = try output.addManyAsSlice( + self.output_arena.allocator(), + self.state.pending_newlines, + ); + @memset(newlines, '\n'); + self.state.pending_newlines = 0; + self.state.pending_spaces = 0; + } else if (self.state.pending_spaces != 0) { + const spaces = try output.addManyAsSlice( + self.output_arena.allocator(), + self.state.pending_spaces, + ); + @memset(spaces, ' '); + self.state.pending_spaces = 0; + } + } + + fn resetOutputArena(self: *TextCleaner) void { + const mode: ArenaAllocator.ResetMode = if (self.options.retained_output_bytes) |limit| + .{ .retain_with_limit = limit } + else + .retain_capacity; + _ = self.output_arena.reset(mode); + } +}; + +fn reserveFactor(encoding: InputEncoding) usize { + return switch (encoding) { + .auto, .utf8 => 1, + .utf16_le, .utf16_be, .latin1 => 2, + .windows_1252 => 3, + }; +} + +fn validateStreamingState(state: StreamingState) StreamingState.DecodeError!void { + const bytes = state.encoded; + if (!std.mem.eql(u8, bytes[0..4], StreamingState.magic)) { + return error.UnsupportedVersion; + } + if (bytes[4] & ~@as(u8, 0b0011_1111) != 0) return error.InvalidState; + if (bytes[9] != 0) return error.InvalidState; + for (bytes[44..48]) |byte| { + if (byte != 0) return error.InvalidState; + } + + const in_document = bytes[4] & (1 << 0) != 0; + const has_pending_byte = bytes[4] & (1 << 1) != 0; + const has_high_surrogate = bytes[4] & (1 << 2) != 0; + const line_has_content = bytes[4] & (1 << 4) != 0; + const document_has_content = bytes[4] & (1 << 5) != 0; + const encoding = try decodeResolvedEncoding(bytes[5]); + const probe_len = bytes[6]; + const utf8_pending_len = bytes[7]; + const utf8_expected_len = bytes[8]; + const pending_spaces = getU64(bytes[20..28]); + const pending_newlines = getU64(bytes[28..36]); + + if (probe_len > 3 or utf8_pending_len > 4 or utf8_expected_len > 4) { + return error.InvalidState; + } + if ((utf8_pending_len == 0) != (utf8_expected_len == 0)) { + return error.InvalidState; + } + if (utf8_pending_len != 0 and + (utf8_expected_len < 2 or utf8_pending_len >= utf8_expected_len)) + { + return error.InvalidState; + } + if (encoding != null and probe_len != 0) return error.InvalidState; + if (encoding == null and probe_len > 2) return error.InvalidState; + if (encoding != .utf8 and (utf8_pending_len != 0 or utf8_expected_len != 0)) { + return error.InvalidState; + } + for (bytes[10 + probe_len .. 13]) |byte| { + if (byte != 0) return error.InvalidState; + } + for (bytes[13 + utf8_pending_len .. 17]) |byte| { + if (byte != 0) return error.InvalidState; + } + if (utf8_pending_len != 0) { + const expected = std.unicode.utf8ByteSequenceLength(bytes[13]) catch + return error.InvalidState; + if (expected != utf8_expected_len) return error.InvalidState; + for (bytes[14 .. 13 + utf8_pending_len]) |byte| { + if (byte & 0b1100_0000 != 0b1000_0000) return error.InvalidState; + } + } + if (encoding != .utf16_le and encoding != .utf16_be and + (has_pending_byte or has_high_surrogate)) + { + return error.InvalidState; + } + if (!has_pending_byte and bytes[17] != 0) return error.InvalidState; + const high_surrogate = getU16(bytes[18..20]); + if (has_high_surrogate != std.unicode.utf16IsHighSurrogate(high_surrogate)) { + return error.InvalidState; + } + if (pending_spaces != 0 and pending_newlines != 0) return error.InvalidState; + if (pending_spaces != 0 and !line_has_content) return error.InvalidState; + if (line_has_content and !document_has_content) return error.InvalidState; + + if (!in_document) { + if (bytes[4] != 0 or bytes[5] != 0 or probe_len != 0 or + utf8_pending_len != 0 or utf8_expected_len != 0 or + pending_spaces != 0 or pending_newlines != 0) + { + return error.InvalidState; + } + for (bytes[10..20]) |byte| { + if (byte != 0) return error.InvalidState; + } + } else if (encoding == null and probe_len == 0) { + // An empty document may be between start and its first byte. + } +} + +fn unpackStreamingState(state: StreamingState) TextCleaner.RestoreError!CleanerState { + const bytes = state.encoded; + if (bytes[4] & (1 << 0) == 0) return .{}; + + const pending_spaces = std.math.cast(usize, getU64(bytes[20..28])) orelse + return error.InvalidState; + const pending_newlines = std.math.cast(usize, getU64(bytes[28..36])) orelse + return error.InvalidState; + var result: CleanerState = .{ + .in_document = true, + .encoding = try decodeResolvedEncoding(bytes[5]), + .probe_len = @intCast(bytes[6]), + .utf8_pending_len = @intCast(bytes[7]), + .utf8_expected_len = @intCast(bytes[8]), + .utf16_pending_byte = if (bytes[4] & (1 << 1) != 0) bytes[17] else null, + .utf16_high_surrogate = if (bytes[4] & (1 << 2) != 0) + getU16(bytes[18..20]) + else + null, + .previous_was_cr = bytes[4] & (1 << 3) != 0, + .line_has_content = bytes[4] & (1 << 4) != 0, + .document_has_content = bytes[4] & (1 << 5) != 0, + .pending_spaces = pending_spaces, + .pending_newlines = pending_newlines, + }; + @memcpy(&result.probe, bytes[10..13]); + @memcpy(&result.utf8_pending, bytes[13..17]); + return result; +} + +fn encodeResolvedEncoding(encoding: ?ResolvedEncoding) u8 { + return if (encoding) |value| switch (value) { + .utf8 => 1, + .utf16_le => 2, + .utf16_be => 3, + .latin1 => 4, + .windows_1252 => 5, + } else 0; +} + +fn decodeResolvedEncoding(encoded: u8) StreamingState.DecodeError!?ResolvedEncoding { + return switch (encoded) { + 0 => null, + 1 => .utf8, + 2 => .utf16_le, + 3 => .utf16_be, + 4 => .latin1, + 5 => .windows_1252, + else => error.InvalidState, + }; +} + +fn optionsFingerprint(options: Options) u64 { + var hash: u64 = 0xcbf29ce484222325; + fingerprintValue(&hash, @intFromEnum(options.input_encoding)); + fingerprintValue(&hash, @intFromBool(options.strip_byte_order_mark)); + fingerprintValue(&hash, @intFromEnum(options.malformed)); + fingerprintValue(&hash, options.replacement); + fingerprintValue(&hash, @intFromBool(options.collapse_horizontal_whitespace)); + fingerprintValue(&hash, if (options.max_consecutive_newlines) |value| + @as(u64, value) + else + std.math.maxInt(u64)); + fingerprintValue(&hash, @intFromBool(options.trim_document_edges)); + fingerprintValue(&hash, @intFromEnum(options.control_characters)); + fingerprintValue(&hash, @intFromEnum(options.unicode_format_characters)); + return hash; +} + +fn fingerprintValue(hash: *u64, value: anytype) void { + const canonical: u64 = @intCast(value); + for (0..8) |index| { + hash.* ^= @as(u8, @truncate(canonical >> @intCast(index * 8))); + hash.* *%= 0x100000001b3; + } +} + +fn putU16(destination: *[2]u8, value: u16) void { + for (destination, 0..) |*byte, index| { + byte.* = @truncate(value >> @intCast(index * 8)); + } +} + +fn putU64(destination: *[8]u8, value: u64) void { + for (destination, 0..) |*byte, index| { + byte.* = @truncate(value >> @intCast(index * 8)); + } +} + +fn getU16(source: *const [2]u8) u16 { + var value: u16 = 0; + for (source, 0..) |byte, index| { + value |= @as(u16, byte) << @intCast(index * 8); + } + return value; +} + +fn getU64(source: *const [8]u8) u64 { + var value: u64 = 0; + for (source, 0..) |byte, index| { + value |= @as(u64, byte) << @intCast(index * 8); + } + return value; +} + +fn detectAutomatic( + bytes: []const u8, + final: bool, + strip_bom: bool, +) ?ProbeDecision { + if (bytes.len >= 3 and std.mem.eql(u8, bytes[0..3], &.{ 0xEF, 0xBB, 0xBF })) { + return .{ .encoding = .utf8, .skip = if (strip_bom) 3 else 0 }; + } + if (bytes.len >= 2 and bytes[0] == 0xFF and bytes[1] == 0xFE) { + return .{ .encoding = .utf16_le, .skip = if (strip_bom) 2 else 0 }; + } + if (bytes.len >= 2 and bytes[0] == 0xFE and bytes[1] == 0xFF) { + return .{ .encoding = .utf16_be, .skip = if (strip_bom) 2 else 0 }; + } + if (bytes.len == 0) return if (final) .{ .encoding = .utf8, .skip = 0 } else null; + + const could_be_bom = bytes[0] == 0xEF or bytes[0] == 0xFF or bytes[0] == 0xFE; + if (!could_be_bom or final) return .{ .encoding = .utf8, .skip = 0 }; + if (bytes[0] == 0xEF) { + if (bytes.len >= 2 and bytes[1] != 0xBB) return .{ .encoding = .utf8, .skip = 0 }; + if (bytes.len == 3) return .{ .encoding = .utf8, .skip = 0 }; + } else if (bytes.len == 2) { + return .{ .encoding = .utf8, .skip = 0 }; + } + return null; +} + +fn detectExplicitUtf8( + bytes: []const u8, + final: bool, + strip_bom: bool, +) ?ProbeDecision { + if (!strip_bom) return .{ .encoding = .utf8, .skip = 0 }; + if (bytes.len >= 3) { + const has_bom = std.mem.eql(u8, bytes[0..3], &.{ 0xEF, 0xBB, 0xBF }); + return .{ .encoding = .utf8, .skip = if (has_bom) 3 else 0 }; + } + if (bytes.len >= 1 and bytes[0] != 0xEF) return .{ .encoding = .utf8, .skip = 0 }; + if (bytes.len >= 2 and bytes[1] != 0xBB) return .{ .encoding = .utf8, .skip = 0 }; + return if (final) .{ .encoding = .utf8, .skip = 0 } else null; +} + +fn detectExplicitUtf16( + bytes: []const u8, + final: bool, + encoding: ResolvedEncoding, + strip_bom: bool, +) TextCleaner.Error!?ProbeDecision { + if (!strip_bom) return .{ .encoding = encoding, .skip = 0 }; + if (bytes.len < 2) { + return if (final) .{ .encoding = encoding, .skip = 0 } else null; + } + + const is_little_bom = bytes[0] == 0xFF and bytes[1] == 0xFE; + const is_big_bom = bytes[0] == 0xFE and bytes[1] == 0xFF; + if ((encoding == .utf16_le and is_big_bom) or + (encoding == .utf16_be and is_little_bom)) + { + return error.ByteOrderMarkMismatch; + } + const matching_bom = (encoding == .utf16_le and is_little_bom) or + (encoding == .utf16_be and is_big_bom); + return .{ .encoding = encoding, .skip = if (matching_bom) 2 else 0 }; +} + +fn isLineBreak(codepoint: u21) bool { + return codepoint == '\n' or codepoint == 0x0085 or + codepoint == 0x2028 or codepoint == 0x2029; +} + +fn isHorizontalWhitespace(codepoint: u21) bool { + return switch (codepoint) { + '\t', 0x000B, 0x000C, ' ', 0x00A0, 0x1680, 0x202F, 0x205F, 0x3000 => true, + 0x2000...0x200A => true, + else => false, + }; +} + +fn isControl(codepoint: u21) bool { + return codepoint <= 0x001F or (codepoint >= 0x007F and codepoint <= 0x009F); +} + +fn isFormatCharacter(codepoint: u21) bool { + return switch (codepoint) { + 0x00AD, 0x061C, 0x06DD, 0x070F, 0x08E2, 0x180E, 0xFEFF, 0x110BD, 0x110CD, 0xE0001 => true, + 0x0600...0x0605, 0x0890...0x0891, 0x200B...0x200F, 0x202A...0x202E => true, + 0x2060...0x206F, 0xFFF9...0xFFFB, 0x13430...0x1343F, 0x1BCA0...0x1BCA3 => true, + 0x1D173...0x1D17A, 0xE0020...0xE007F => true, + else => false, + }; +} + +fn windows1252Codepoint(byte: u8) ?u21 { + if (byte < 0x80 or byte >= 0xA0) return byte; + return switch (byte) { + 0x80 => 0x20AC, + 0x82 => 0x201A, + 0x83 => 0x0192, + 0x84 => 0x201E, + 0x85 => 0x2026, + 0x86 => 0x2020, + 0x87 => 0x2021, + 0x88 => 0x02C6, + 0x89 => 0x2030, + 0x8A => 0x0160, + 0x8B => 0x2039, + 0x8C => 0x0152, + 0x8E => 0x017D, + 0x91 => 0x2018, + 0x92 => 0x2019, + 0x93 => 0x201C, + 0x94 => 0x201D, + 0x95 => 0x2022, + 0x96 => 0x2013, + 0x97 => 0x2014, + 0x98 => 0x02DC, + 0x99 => 0x2122, + 0x9A => 0x0161, + 0x9B => 0x203A, + 0x9C => 0x0153, + 0x9E => 0x017E, + 0x9F => 0x0178, + else => null, + }; +} + +test "UTF-8 cleaning is stable across arbitrary chunk boundaries" { + const cleaner = try TextCleaner.create(std.testing.allocator, .{}); + defer cleaner.destroy(); + + const first = try cleaner.clean("\xEF\xBB\xBF Hello\t", .{ .document_start = true }); + try std.testing.expectEqualStrings("Hello", first.bytes); + + const second = try cleaner.clean(" world\r", .{}); + try std.testing.expectEqualStrings(" world", second.bytes); + + const third = try cleaner.clean("\n\n\x00next \xE2", .{}); + try std.testing.expectEqualStrings("\n\nnext", third.bytes); + try std.testing.expectEqual(@as(usize, 1), third.stats.controls_removed); + + const fourth = try cleaner.clean("\x82\xAC ", .{ .document_end = true }); + try std.testing.expectEqualStrings(" €", fourth.bytes); + try std.testing.expectEqual(@as(u64, 1), cleaner.totals().documents); +} + +test "byte-at-a-time cleaning matches whole-document cleaning" { + const input = "\xEF\xBB\xBF Alpha\tβ\r\n\nGamma\xE2( end "; + + const whole_cleaner = try TextCleaner.create(std.testing.allocator, .{}); + defer whole_cleaner.destroy(); + const whole = try whole_cleaner.clean(input, .whole); + + const streaming_cleaner = try TextCleaner.create(std.testing.allocator, .{}); + defer streaming_cleaner.destroy(); + var assembled: std.ArrayList(u8) = .empty; + defer assembled.deinit(std.testing.allocator); + for (input, 0..) |_, index| { + const cleaned = try streaming_cleaner.clean(input[index .. index + 1], .{ + .document_start = index == 0, + .document_end = index + 1 == input.len, + }); + try assembled.appendSlice(std.testing.allocator, cleaned.bytes); + } + + try std.testing.expectEqualStrings(whole.bytes, assembled.items); +} + +test "streaming checkpoints transfer partial decoder and formatting state" { + const source = try TextCleaner.create(std.testing.allocator, .{}); + defer source.destroy(); + const partial = try source.clean("prefix \xE2", .{ .document_start = true }); + try std.testing.expectEqualStrings("prefix", partial.bytes); + + const checkpoint = try source.checkpoint(); + const encoded = checkpoint.asBytes().*; + const decoded = try StreamingState.decode(&encoded); + + const destination = try TextCleaner.create(std.testing.allocator, .{}); + defer destination.destroy(); + try destination.restore(decoded); + const continued = try destination.clean("\x82\xAC\r\nnext", .{ .document_end = true }); + try std.testing.expectEqualStrings(" €\nnext", continued.bytes); + + const incompatible = try TextCleaner.create(std.testing.allocator, .{ + .collapse_horizontal_whitespace = false, + }); + defer incompatible.destroy(); + try std.testing.expectError( + error.CheckpointOptionsMismatch, + incompatible.restore(decoded), + ); +} + +test "malformed UTF-8 can be replaced, removed, or rejected" { + const replacing = try TextCleaner.create(std.testing.allocator, .{}); + defer replacing.destroy(); + const replaced = try replacing.clean("a\xFFb", .whole); + try std.testing.expectEqualStrings("a\xEF\xBF\xBDb", replaced.bytes); + try std.testing.expectEqual(@as(usize, 1), replaced.stats.malformed_sequences); + + const removing = try TextCleaner.create(std.testing.allocator, .{ .malformed = .remove }); + defer removing.destroy(); + const removed = try removing.clean("a\xFFb", .whole); + try std.testing.expectEqualStrings("ab", removed.bytes); + + const rejecting = try TextCleaner.create(std.testing.allocator, .{ .malformed = .reject }); + defer rejecting.destroy(); + try std.testing.expectError(error.MalformedEncoding, rejecting.clean("a\xFFb", .whole)); + // State rollback makes the cleaner reusable after a rejected fragment. + const valid = try rejecting.clean("valid", .whole); + try std.testing.expectEqualStrings("valid", valid.bytes); +} + +test "UTF-16 BOMs code units and surrogate pairs may span chunks" { + const cleaner = try TextCleaner.create(std.testing.allocator, .{}); + defer cleaner.destroy(); + + const first = try cleaner.clean("\xFF", .{ .document_start = true }); + try std.testing.expectEqual(@as(usize, 0), first.bytes.len); + const second = try cleaner.clean("\xFE\x41\x00\x3D\xD8", .{}); + try std.testing.expectEqualStrings("A", second.bytes); + const third = try cleaner.clean( + "\x00\xDE\x0D\x00\x0A\x00\x42\x00", + .{ .document_end = true }, + ); + try std.testing.expectEqualStrings("😀\nB", third.bytes); +} + +test "format and control policies remove unsafe invisible scalars" { + const cleaner = try TextCleaner.create(std.testing.allocator, .{}); + defer cleaner.destroy(); + const cleaned = try cleaner.clean("a\xE2\x80\x8Bb\x07c", .whole); + try std.testing.expectEqualStrings("abc", cleaned.bytes); + try std.testing.expectEqual(@as(usize, 1), cleaned.stats.format_characters_removed); + try std.testing.expectEqual(@as(usize, 1), cleaned.stats.controls_removed); +} + +test "Windows-1252 is converted to UTF-8" { + const cleaner = try TextCleaner.create(std.testing.allocator, .{ + .input_encoding = .windows_1252, + }); + defer cleaner.destroy(); + const cleaned = try cleaner.clean("\x93quoted\x94 \x80", .whole); + try std.testing.expectEqualStrings("“quoted” €", cleaned.bytes); +} diff --git a/root.zig b/root.zig new file mode 100644 index 0000000..00265d1 --- /dev/null +++ b/root.zig @@ -0,0 +1,13 @@ +pub const data = struct { + pub const corpus_reader = @import("data/corpus-reader/root.zig"); + pub const deduplicator = @import("data/deduplicator/root.zig"); + pub const document_filter = @import("data/document-filter/root.zig"); + pub const text_cleaner = @import("data/text-cleaner/root.zig"); +}; + +test { + _ = data.corpus_reader; + _ = data.deduplicator; + _ = data.document_filter; + _ = data.text_cleaner; +}