From 87c2128252ed4d318b2091b8ab75b57e60753a11 Mon Sep 17 00:00:00 2001 From: kris927b Date: Tue, 24 Jun 2025 13:12:20 +0200 Subject: [PATCH 1/7] Minor fixes here and there to match the output of a datatrove pipeline. Still missing a normalisation step. --- Cargo.lock | 1274 +++++++++++++++++++---- Cargo.toml | 3 +- config/pipeline_config.yaml | 26 +- src/config/producer.rs | 2 +- src/pipeline/filters/c4_filters.rs | 41 +- src/pipeline/filters/fineweb_quality.rs | 84 +- src/pipeline/filters/gopher_quality.rs | 2 +- src/pipeline/filters/language_filter.rs | 114 +- src/pipeline/readers/parquet_reader.rs | 4 +- test.json | 35 + tests/producer_tests.rs | 2 +- 11 files changed, 1308 insertions(+), 279 deletions(-) create mode 100644 test.json diff --git a/Cargo.lock b/Cargo.lock index ee2bcaa..27c5ed1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -14,11 +14,13 @@ dependencies = [ "chrono", "clap", "futures", + "html-escape", "icu", "indicatif", "itertools 0.14.0", "lapin", "lazy_static", + "lingua", "once_cell", "parquet", "polars", @@ -43,7 +45,6 @@ dependencies = [ "tracing-appender", "tracing-subscriber", "uuid", - "whatlang", ] [[package]] @@ -935,7 +936,18 @@ checksum = "cc97b8f16f944bba54f0433f07e30be199b6dc2bd25937444bbad560bcea29bd" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", - "brotli-decompressor", + "brotli-decompressor 4.0.3", +] + +[[package]] +name = "brotli" +version = "8.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9991eea70ea4f293524138648e41ee89b0b2b12ddef3b255effa43c8056e0e0d" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor 5.0.0", ] [[package]] @@ -948,6 +960,16 @@ dependencies = [ "alloc-stdlib", ] +[[package]] +name = "brotli-decompressor" +version = "5.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "874bb8112abecc98cbd6d81ea4fa7e94fb9449648c93cc89aa40c81c24d7de03" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + [[package]] name = "bumpalo" version = "3.18.1" @@ -1170,6 +1192,20 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "compact_str" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdb1325a1cece981e8a296ab8f0f9b63ae357bd0784a9faaf548cc7b480707a" +dependencies = [ + "castaway", + "cfg-if", + "itoa", + "rustversion", + "ryu", + "static_assertions", +] + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -1420,6 +1456,20 @@ dependencies = [ "syn", ] +[[package]] +name = "dashmap" +version = "6.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] + [[package]] name = "data-encoding" version = "2.9.0" @@ -1809,6 +1859,16 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fraction" +version = "0.15.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f158e3ff0a1b334408dc9fb811cd99b446986f4d8b741bb08f9df1604085ae7" +dependencies = [ + "lazy_static", + "num", +] + [[package]] name = "fs4" version = "0.13.1" @@ -2126,6 +2186,15 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "html-escape" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d1ad449764d627e22bfd7cd5e8868264fc9236e07c752972b4080cd351cb476" +dependencies = [ + "utf8-width", +] + [[package]] name = "http" version = "0.2.12" @@ -2818,317 +2887,1110 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aa91ba6a585939a020c787235daa8aee856d9bceebd6355e283c0c310bc6de96" dependencies = [ - "displaydoc", - "icu_calendar", - "icu_provider 1.5.0", - "icu_timezone_data", - "tinystr 0.7.6", - "zerotrie 0.1.3", - "zerovec 0.10.4", + "displaydoc", + "icu_calendar", + "icu_provider 1.5.0", + "icu_timezone_data", + "tinystr 0.7.6", + "zerotrie 0.1.3", + "zerovec 0.10.4", +] + +[[package]] +name = "icu_timezone_data" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1adcf7b613a268af025bc2a2532b4b9ee294e6051c5c0832d8bff20ac0232e68" + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "idna" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +dependencies = [ + "icu_normalizer 2.0.0", + "icu_properties 2.0.1", +] + +[[package]] +name = "include_dir" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "923d117408f1e49d914f1a379a309cffe4f18c05cf4e3d12e613a15fc81bd0dd" +dependencies = [ + "include_dir_macros", +] + +[[package]] +name = "include_dir_macros" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cab85a7ed0bd5f0e76d93846e0147172bed2e2d3f859bcc33a8d9699cad1a75" +dependencies = [ + "proc-macro2", + "quote", +] + +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", + "serde", +] + +[[package]] +name = "indexmap" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cea70ddb795996207ad57735b50c5982d8844f38ba9ee5f1aedcfb708a2aa11e" +dependencies = [ + "equivalent", + "hashbrown 0.15.4", + "serde", +] + +[[package]] +name = "indicatif" +version = "0.17.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "183b3088984b400f4cfac3620d5e076c84da5364016b4f49473de574b2586235" +dependencies = [ + "console", + "number_prefix", + "portable-atomic", + "tokio", + "unicode-width", + "web-time", +] + +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "block-padding", + "generic-array", +] + +[[package]] +name = "instant" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "integer-encoding" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" + +[[package]] +name = "io-lifetimes" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" +dependencies = [ + "hermit-abi 0.3.9", + "libc", + "windows-sys 0.48.0", +] + +[[package]] +name = "ipnet" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" + +[[package]] +name = "iri-string" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbc5ebe9c3a1a7a5127f920a418f7585e9e758e911d0466ed004f393b0e380b2" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" + +[[package]] +name = "itertools" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" + +[[package]] +name = "jobserver" +version = "0.1.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38f262f097c174adebe41eb73d66ae9c06b2844fb0da69969647bbddd9b0538a" +dependencies = [ + "getrandom 0.3.3", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "lapin" +version = "2.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4273975142078ed200dedd77f09c8903dec110d0b02a0c8ad45796b39b691ea9" +dependencies = [ + "amq-protocol", + "async-global-executor-trait", + "async-reactor-trait", + "async-trait", + "executor-trait", + "flume", + "futures-core", + "futures-io", + "parking_lot", + "pinky-swear", + "reactor-trait", + "serde", + "tracing", + "waker-fn", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "lexical-core" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b765c31809609075565a70b4b71402281283aeda7ecaf4818ac14a7b2ade8958" +dependencies = [ + "lexical-parse-float", + "lexical-parse-integer", + "lexical-util", + "lexical-write-float", + "lexical-write-integer", +] + +[[package]] +name = "lexical-parse-float" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de6f9cb01fb0b08060209a057c048fcbab8717b4c1ecd2eac66ebfe39a65b0f2" +dependencies = [ + "lexical-parse-integer", + "lexical-util", + "static_assertions", +] + +[[package]] +name = "lexical-parse-integer" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72207aae22fc0a121ba7b6d479e42cbfea549af1479c3f3a4f12c70dd66df12e" +dependencies = [ + "lexical-util", + "static_assertions", +] + +[[package]] +name = "lexical-util" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a82e24bf537fd24c177ffbbdc6ebcc8d54732c35b50a3f28cc3f4e4c949a0b3" +dependencies = [ + "static_assertions", +] + +[[package]] +name = "lexical-write-float" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5afc668a27f460fb45a81a757b6bf2f43c2d7e30cb5a2dcd3abf294c78d62bd" +dependencies = [ + "lexical-util", + "lexical-write-integer", + "static_assertions", +] + +[[package]] +name = "lexical-write-integer" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "629ddff1a914a836fb245616a7888b62903aae58fa771e1d83943035efa0f978" +dependencies = [ + "lexical-util", + "static_assertions", +] + +[[package]] +name = "libc" +version = "0.2.174" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1171693293099992e19cddea4e8b849964e9846f4acee11b3948bcc337be8776" + +[[package]] +name = "libm" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" + +[[package]] +name = "libredox" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" +dependencies = [ + "bitflags 2.9.1", + "libc", + "redox_syscall 0.5.13", +] + +[[package]] +name = "libz-rs-sys" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "172a788537a2221661b480fee8dc5f96c580eb34fa88764d3205dc356c7e4221" +dependencies = [ + "zlib-rs", +] + +[[package]] +name = "lingua" +version = "1.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e501c98d473068ce13ddfa5ca86846a6b60e24e3020ba9d2f49af654e9219ec" +dependencies = [ + "ahash", + "brotli 8.0.1", + "compact_str 0.9.0", + "dashmap", + "fraction", + "include_dir", + "itertools 0.14.0", + "lingua-afrikaans-language-model", + "lingua-albanian-language-model", + "lingua-arabic-language-model", + "lingua-armenian-language-model", + "lingua-azerbaijani-language-model", + "lingua-basque-language-model", + "lingua-belarusian-language-model", + "lingua-bengali-language-model", + "lingua-bokmal-language-model", + "lingua-bosnian-language-model", + "lingua-bulgarian-language-model", + "lingua-catalan-language-model", + "lingua-chinese-language-model", + "lingua-croatian-language-model", + "lingua-czech-language-model", + "lingua-danish-language-model", + "lingua-dutch-language-model", + "lingua-english-language-model", + "lingua-esperanto-language-model", + "lingua-estonian-language-model", + "lingua-finnish-language-model", + "lingua-french-language-model", + "lingua-ganda-language-model", + "lingua-georgian-language-model", + "lingua-german-language-model", + "lingua-greek-language-model", + "lingua-gujarati-language-model", + "lingua-hebrew-language-model", + "lingua-hindi-language-model", + "lingua-hungarian-language-model", + "lingua-icelandic-language-model", + "lingua-indonesian-language-model", + "lingua-irish-language-model", + "lingua-italian-language-model", + "lingua-japanese-language-model", + "lingua-kazakh-language-model", + "lingua-korean-language-model", + "lingua-latin-language-model", + "lingua-latvian-language-model", + "lingua-lithuanian-language-model", + "lingua-macedonian-language-model", + "lingua-malay-language-model", + "lingua-maori-language-model", + "lingua-marathi-language-model", + "lingua-mongolian-language-model", + "lingua-nynorsk-language-model", + "lingua-persian-language-model", + "lingua-polish-language-model", + "lingua-portuguese-language-model", + "lingua-punjabi-language-model", + "lingua-romanian-language-model", + "lingua-russian-language-model", + "lingua-serbian-language-model", + "lingua-shona-language-model", + "lingua-slovak-language-model", + "lingua-slovene-language-model", + "lingua-somali-language-model", + "lingua-sotho-language-model", + "lingua-spanish-language-model", + "lingua-swahili-language-model", + "lingua-swedish-language-model", + "lingua-tagalog-language-model", + "lingua-tamil-language-model", + "lingua-telugu-language-model", + "lingua-thai-language-model", + "lingua-tsonga-language-model", + "lingua-tswana-language-model", + "lingua-turkish-language-model", + "lingua-ukrainian-language-model", + "lingua-urdu-language-model", + "lingua-vietnamese-language-model", + "lingua-welsh-language-model", + "lingua-xhosa-language-model", + "lingua-yoruba-language-model", + "lingua-zulu-language-model", + "maplit", + "rayon", + "regex", + "serde", + "serde-wasm-bindgen", + "serde_json", + "strum", + "strum_macros 0.27.1", + "wasm-bindgen", +] + +[[package]] +name = "lingua-afrikaans-language-model" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1287f2382e4505a9884982a208129fa0f19d331c819667ac0396e3435ffca176" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-albanian-language-model" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2843d913f39870c0226ada84959d1f796b12f10db0e4b9cdcc9588df8dcc4fe3" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-arabic-language-model" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a54f30346c57deb114fc9083047a441fc9a0fca5dd4b11db82a96ac04d4c3c53" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-armenian-language-model" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1582c3543a18d82ddd747e77ecca9ca0c6c344bbb85aa9aeafe4afaf4c674396" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-azerbaijani-language-model" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d955c944289ef7919710c181575c93401732e08a1d2c901d2e0171748ddb9c39" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-basque-language-model" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c04f4beb9512e0b1bb9d44bd51cc012ca075a88a8773dc971a22900e9b43db4" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-belarusian-language-model" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b6a22cafc7351c8f4ebb48dd4f513f0bb4d1b9bd4a5d38f4955bfa35e378f0b" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-bengali-language-model" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad51861506bf06428b95c90662803700f365331959b3007bde47aaaacfbcd5ac" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-bokmal-language-model" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d10116c60af75f59d61058f7402fe866af56c18c1c69b363fb9c4ee28f7f555c" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-bosnian-language-model" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0ba20e4fc22116b6e3873d1d15fbc5c6371de12b5f5a4fa4f583f37328521d7" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-bulgarian-language-model" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbf7243e09160b87a4b4543747e7ea4549cbf022538a493af93e8633da9ad97c" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-catalan-language-model" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71aa462a7c191e75c3949f8a7beca9031d188e8ea28f2eb76c99b2d526927d21" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-chinese-language-model" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d31600b0cbae8a5c0622038d90e7d3830193d78feb2741f8678267d9f76e90f0" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-croatian-language-model" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7438e2753dc02a1dd0c355afd1399ce26fbc13205454d81ed611e249987f134" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-czech-language-model" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e1b3749cb0a0e40f7a732d1ae9b8cb4bf35444e92a2dd7db9164971e4157c9c" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-danish-language-model" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9f83ef0885a702e6ddf776473f0eea13e3b7c20bac94ec4245cfabec555143e" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-dutch-language-model" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f390bbffcb405f9a36fbb014cfd1953f2a5edc0b4313ca2005aba51e6ed6741" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-english-language-model" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a4b2666c2dc3669b9cc2235f318184f3cc3426bf7075336bcf8f9ddba6f81c0" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-esperanto-language-model" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f169804327d4ae648c9045e139e19f2c1a8426cd2afcad6db9223d6de3cd3855" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-estonian-language-model" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b675de6a28fc39d7af5b19240ad2c34dc5646b1d1ef812cb93c6b2c589cd13d6" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-finnish-language-model" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bc6109e57ed65b665ee7f911ed451cd6587304a0bfda9359c8794c33dffc425" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-french-language-model" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7fd27da82335b66738d137ee64221f479973467b9178d49d802acbe847de355" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-ganda-language-model" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd847988a427b3f6b9bd39bcc271643edf190c1608eff2a3c4208f717c66752a" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-georgian-language-model" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4c91bb5d3f24e8760a1411d0c5868b8e614ecb7b480328c7dcea4fc1b4b7201" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-german-language-model" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa568bbca8beb3b0cfc691cf2e87d1d24cb457c8e19ed168dd22d739c24a15e" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-greek-language-model" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b473b60c61e0aee31bdd2cf35642942dde8dcfd7b1ea4c7534df98994c3b429f" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-gujarati-language-model" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfc48e1164860f35cac2ff6f3369d7c2e36feef54bd5eb6cafd32ba5fc7ad63d" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-hebrew-language-model" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4c36ff9bfba287adc3a05cd6ca2a512412155d0fbfd1f4eaddcf800f42fe3d9" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-hindi-language-model" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b0930f8845e398914d0f4b25342774b199f6486971c79fd225fd0f3f747b8d2" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-hungarian-language-model" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "286a84ab04128d25fadb162d852a327d32174ab16b7ab4725e4047820b92a781" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-icelandic-language-model" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "499bfe35307f587a18c3c8492debb84c3b8cab733035b6852cdf0bf508b6d99f" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-indonesian-language-model" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1eb4112ce3f4c5d24d739c047e6d815a4ed283ccf71a47669f6f2132e36b5c7a" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-irish-language-model" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29a90343c0e74060362793a19a7139f58533f9be61156805fab635ad5cfc5378" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-italian-language-model" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ab586cc7454f4df2a224ce245f81e199dba57d47259fc628b29d5c066697059" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-japanese-language-model" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5756b671f8a2f0ba8ae81852ab44d38485e60c590cacdff23400c47a48d04244" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-kazakh-language-model" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccbe5bbdb753152eb3c3cb6d37df58c81c12e57122898286fc1b17ad5667f419" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-korean-language-model" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b30b14af9b0f1335e6117d63b6a71afac660f4c83e81a8fd593a963222cc1b9c" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-latin-language-model" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f02abfb6beb5c53b345a3e599c9faeff3e765cf6f79b77f6650585967568c5e" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-latvian-language-model" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef790c9c49220157d09f007b9c4ae0adb7b3ffe33e3aa6c0bcdc81cd995b919a" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-lithuanian-language-model" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d546fe945b7bc4c5879fb7e59c589396277e3f007972391e55730940ffe4ef03" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-macedonian-language-model" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6368874d9d400cc3efc5d74de6c66fa5ebbf89589523e1995420ba2fd677359" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-malay-language-model" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c003d235500af772243973dfcd26cc077e850308b6da873cac365b1c91fce3af" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-maori-language-model" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f0aa75bfac8296dc681b3d9abbeb62b3bee925893d48070ac3ec87858e539f6" +dependencies = [ + "include_dir", ] [[package]] -name = "icu_timezone_data" -version = "1.5.1" +name = "lingua-marathi-language-model" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1adcf7b613a268af025bc2a2532b4b9ee294e6051c5c0832d8bff20ac0232e68" +checksum = "89511520fae381b87e25153ac5b08146d8fe91b11650433cd6fba60cff342bdb" +dependencies = [ + "include_dir", +] [[package]] -name = "ident_case" -version = "1.0.1" +name = "lingua-mongolian-language-model" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" +checksum = "4983ac733c95c8e12917f3db984c6f02b41587acac22ca9b4f37cf6cc32a8a52" +dependencies = [ + "include_dir", +] [[package]] -name = "idna" -version = "1.0.3" +name = "lingua-nynorsk-language-model" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e" +checksum = "c7ee1fa246487b683138da7684035489ab5fafb2751d0715b33c38f6a6eadb5f" dependencies = [ - "idna_adapter", - "smallvec", - "utf8_iter", + "include_dir", ] [[package]] -name = "idna_adapter" -version = "1.2.1" +name = "lingua-persian-language-model" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +checksum = "29a97bc46fe85d9f06d50668615eb903b1e7ed77dad228dac98228d417c6e84d" dependencies = [ - "icu_normalizer 2.0.0", - "icu_properties 2.0.1", + "include_dir", ] [[package]] -name = "indexmap" -version = "1.9.3" +name = "lingua-polish-language-model" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +checksum = "1534fd0f8c717bffd9ec80e9b27b8627fa069b46e9474b0e29780eb5fdddb53f" dependencies = [ - "autocfg", - "hashbrown 0.12.3", - "serde", + "include_dir", ] [[package]] -name = "indexmap" -version = "2.9.0" +name = "lingua-portuguese-language-model" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cea70ddb795996207ad57735b50c5982d8844f38ba9ee5f1aedcfb708a2aa11e" +checksum = "3659ec08115bbec9151bc23e62a4c3ee768c138e791854c8fe71cfcde0e5de84" dependencies = [ - "equivalent", - "hashbrown 0.15.4", - "serde", + "include_dir", ] [[package]] -name = "indicatif" -version = "0.17.11" +name = "lingua-punjabi-language-model" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "183b3088984b400f4cfac3620d5e076c84da5364016b4f49473de574b2586235" +checksum = "e165678767e73db517e9aeb43817517b28f5b31d13f20853847bc8f05a1ae3bd" dependencies = [ - "console", - "number_prefix", - "portable-atomic", - "tokio", - "unicode-width", - "web-time", + "include_dir", ] [[package]] -name = "inout" -version = "0.1.4" +name = "lingua-romanian-language-model" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +checksum = "836d7f9116c098e863c05dbb4739df77c516485b5e3f2e9457d376ba83834183" dependencies = [ - "block-padding", - "generic-array", + "include_dir", ] [[package]] -name = "instant" -version = "0.1.13" +name = "lingua-russian-language-model" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" +checksum = "9058ef9a1b6d09727f63c8831c38d81c09456a2b433e4d02aa296ef83c19b631" dependencies = [ - "cfg-if", + "include_dir", ] [[package]] -name = "integer-encoding" -version = "3.0.4" +name = "lingua-serbian-language-model" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" +checksum = "e4a106952e53e9162c8a96bd9eea9b1874df5ace221bac1ab7868528b4b37e78" +dependencies = [ + "include_dir", +] [[package]] -name = "io-lifetimes" -version = "1.0.11" +name = "lingua-shona-language-model" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" +checksum = "924047ff3712b3ed640e0e950faf6e5682ee57592a141da6ae5a149b44451c38" dependencies = [ - "hermit-abi 0.3.9", - "libc", - "windows-sys 0.48.0", + "include_dir", ] [[package]] -name = "ipnet" -version = "2.11.0" +name = "lingua-slovak-language-model" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" +checksum = "44afb5363903e2c6b10b8dc14f9b3c4abc1e125ecb4763ab189382a951e8f1f2" +dependencies = [ + "include_dir", +] [[package]] -name = "iri-string" -version = "0.7.8" +name = "lingua-slovene-language-model" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbc5ebe9c3a1a7a5127f920a418f7585e9e758e911d0466ed004f393b0e380b2" +checksum = "bd43bd4567a3bffa3d20b3e2c62e3a3d17988c34948df88353d59eebcdcb5a3d" dependencies = [ - "memchr", - "serde", + "include_dir", ] [[package]] -name = "is_terminal_polyfill" -version = "1.70.1" +name = "lingua-somali-language-model" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" +checksum = "7a912003628ee6b007b6aca330aefa1cf67369f7d9e2c925e38e66e058abe070" +dependencies = [ + "include_dir", +] [[package]] -name = "itertools" -version = "0.11.0" +name = "lingua-sotho-language-model" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" +checksum = "c6fd83a7172be44cd2fc7a630b96b3d9ba1a07cbe295937e19b94dcba2144502" dependencies = [ - "either", + "include_dir", ] [[package]] -name = "itertools" -version = "0.13.0" +name = "lingua-spanish-language-model" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +checksum = "05df7766652773b2292e7ad12221c0090954baed18ca2ff631a5f134ce08d91d" dependencies = [ - "either", + "include_dir", ] [[package]] -name = "itertools" -version = "0.14.0" +name = "lingua-swahili-language-model" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +checksum = "bc6a2e6ec74d0de7409e81a4ecac90fc061c87f397e9358c1fc7b06fb71375b1" dependencies = [ - "either", + "include_dir", ] [[package]] -name = "itoa" -version = "1.0.15" +name = "lingua-swedish-language-model" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" +checksum = "fc0abcbfc81c994fa2e433571febecbf7a3011c1f46164d7ca5013691656c9b3" +dependencies = [ + "include_dir", +] [[package]] -name = "jobserver" -version = "0.1.33" +name = "lingua-tagalog-language-model" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38f262f097c174adebe41eb73d66ae9c06b2844fb0da69969647bbddd9b0538a" +checksum = "0275f0d641f8193aba7dabe8ced95a7b12e14a9652bb9b3067b95861472be791" dependencies = [ - "getrandom 0.3.3", - "libc", + "include_dir", ] [[package]] -name = "js-sys" -version = "0.3.77" +name = "lingua-tamil-language-model" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" +checksum = "e8badccb67d6eec015563538ffcfa6d2a6ff151bcb787ce0d24097c26d410e07" dependencies = [ - "once_cell", - "wasm-bindgen", + "include_dir", ] [[package]] -name = "lapin" -version = "2.5.3" +name = "lingua-telugu-language-model" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4273975142078ed200dedd77f09c8903dec110d0b02a0c8ad45796b39b691ea9" +checksum = "828376db1408126c403656447eaceaede6f39c20d81f78a3c32e6cb9d78ceb20" dependencies = [ - "amq-protocol", - "async-global-executor-trait", - "async-reactor-trait", - "async-trait", - "executor-trait", - "flume", - "futures-core", - "futures-io", - "parking_lot", - "pinky-swear", - "reactor-trait", - "serde", - "tracing", - "waker-fn", + "include_dir", ] [[package]] -name = "lazy_static" -version = "1.5.0" +name = "lingua-thai-language-model" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +checksum = "22f72427dd733f84054777b8c229d489766a880b5e5207c0bc57f8983423256b" +dependencies = [ + "include_dir", +] [[package]] -name = "lexical-core" -version = "1.0.5" +name = "lingua-tsonga-language-model" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b765c31809609075565a70b4b71402281283aeda7ecaf4818ac14a7b2ade8958" +checksum = "ac4de2ed21a6e3a82a66a712fbaddfb150846eefa6cde4f8b33d732e6debe89d" dependencies = [ - "lexical-parse-float", - "lexical-parse-integer", - "lexical-util", - "lexical-write-float", - "lexical-write-integer", + "include_dir", ] [[package]] -name = "lexical-parse-float" -version = "1.0.5" +name = "lingua-tswana-language-model" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de6f9cb01fb0b08060209a057c048fcbab8717b4c1ecd2eac66ebfe39a65b0f2" +checksum = "1164e518ba79371e8d10801bdc48ba704f9260cbc45a45d0bf8710d860382369" dependencies = [ - "lexical-parse-integer", - "lexical-util", - "static_assertions", + "include_dir", ] [[package]] -name = "lexical-parse-integer" -version = "1.0.5" +name = "lingua-turkish-language-model" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72207aae22fc0a121ba7b6d479e42cbfea549af1479c3f3a4f12c70dd66df12e" +checksum = "10e15e6c8aa9c7b807933aa257468309c9ae41b84898fd087356dea4aa0ec036" dependencies = [ - "lexical-util", - "static_assertions", + "include_dir", ] [[package]] -name = "lexical-util" -version = "1.0.6" +name = "lingua-ukrainian-language-model" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a82e24bf537fd24c177ffbbdc6ebcc8d54732c35b50a3f28cc3f4e4c949a0b3" +checksum = "4f3509a5a27c043e76c31a9c8a5a16deb3dc6e1fd26de029cb83b20d16ca1fe2" dependencies = [ - "static_assertions", + "include_dir", ] [[package]] -name = "lexical-write-float" -version = "1.0.5" +name = "lingua-urdu-language-model" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5afc668a27f460fb45a81a757b6bf2f43c2d7e30cb5a2dcd3abf294c78d62bd" +checksum = "13b529ccb188dddd222c613b71e86d9401cb5e5a46dbd62a867906da093fec64" dependencies = [ - "lexical-util", - "lexical-write-integer", - "static_assertions", + "include_dir", ] [[package]] -name = "lexical-write-integer" -version = "1.0.5" +name = "lingua-vietnamese-language-model" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "629ddff1a914a836fb245616a7888b62903aae58fa771e1d83943035efa0f978" +checksum = "489ba31146a87c04a783bcca606b961320a9cace365378bd329933a8b3ae2f50" dependencies = [ - "lexical-util", - "static_assertions", + "include_dir", ] [[package]] -name = "libc" -version = "0.2.174" +name = "lingua-welsh-language-model" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1171693293099992e19cddea4e8b849964e9846f4acee11b3948bcc337be8776" +checksum = "f285d84e053c405a97a00e59a1e7aa82ac967a091d2de0f484322dd539c81080" +dependencies = [ + "include_dir", +] [[package]] -name = "libm" -version = "0.2.15" +name = "lingua-xhosa-language-model" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" +checksum = "a3080b71235d1834423ef0e4c772e24f99699d427718efab67160792d7bd5984" +dependencies = [ + "include_dir", +] [[package]] -name = "libredox" -version = "0.1.3" +name = "lingua-yoruba-language-model" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" +checksum = "78b8033c5058586fd180b1f42b2ece059c5cefb620703ecc92f379ac0a142194" dependencies = [ - "bitflags 2.9.1", - "libc", - "redox_syscall 0.5.13", + "include_dir", ] [[package]] -name = "libz-rs-sys" -version = "0.5.1" +name = "lingua-zulu-language-model" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "172a788537a2221661b480fee8dc5f96c580eb34fa88764d3205dc356c7e4221" +checksum = "ad9dfb5039e1f9268f2310277493a5dc44e9448e55e0f26938c8d1d8cd92f98c" dependencies = [ - "zlib-rs", + "include_dir", ] [[package]] @@ -3227,6 +4089,12 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "670fdfda89751bc4a84ac13eaa63e205cf0fd22b4c9a5fbfa085b63c1f1d3a30" +[[package]] +name = "maplit" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" + [[package]] name = "matchers" version = "0.1.0" @@ -3697,7 +4565,7 @@ dependencies = [ "arrow-schema", "arrow-select", "base64 0.22.1", - "brotli", + "brotli 7.0.0", "bytes", "chrono", "flate2", @@ -3945,7 +4813,7 @@ dependencies = [ "serde", "simdutf8", "streaming-iterator", - "strum_macros", + "strum_macros 0.26.4", "version_check", "zstd", ] @@ -3982,7 +4850,7 @@ dependencies = [ "serde", "skiplist", "strength_reduce", - "strum_macros", + "strum_macros 0.26.4", "version_check", ] @@ -4016,7 +4884,7 @@ dependencies = [ "regex", "serde", "serde_json", - "strum_macros", + "strum_macros 0.26.4", "uuid", "version_check", "xxhash-rust", @@ -4175,7 +5043,7 @@ dependencies = [ "rayon", "regex", "regex-syntax 0.8.5", - "strum_macros", + "strum_macros 0.26.4", "unicode-normalization", "unicode-reverse", "version_check", @@ -4189,7 +5057,7 @@ checksum = "7c849c10edd9511ccd4ec4130e283ee3a8b3bb48a7d74ac6354c1c20add81065" dependencies = [ "async-stream", "base64 0.22.1", - "brotli", + "brotli 7.0.0", "bytemuck", "ethnum", "flate2", @@ -4247,7 +5115,7 @@ dependencies = [ "rayon", "recursive", "regex", - "strum_macros", + "strum_macros 0.26.4", "version_check", ] @@ -4356,7 +5224,7 @@ dependencies = [ "polars-utils", "rayon", "regex", - "strum_macros", + "strum_macros 0.26.4", ] [[package]] @@ -4368,7 +5236,7 @@ dependencies = [ "bincode", "bytemuck", "bytes", - "compact_str", + "compact_str 0.8.1", "flate2", "foldhash", "hashbrown 0.15.4", @@ -5325,6 +6193,17 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde-wasm-bindgen" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8302e169f0eddcc139c70f139d19d6467353af16f9fce27e8c30158036a1e16b" +dependencies = [ + "js-sys", + "serde", + "wasm-bindgen", +] + [[package]] name = "serde_derive" version = "1.0.219" @@ -5688,6 +6567,12 @@ dependencies = [ "syn", ] +[[package]] +name = "strum" +version = "0.27.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f64def088c51c9510a8579e3c5d67c65349dcf755e5479ad3d010aa6454e2c32" + [[package]] name = "strum_macros" version = "0.26.4" @@ -5701,6 +6586,19 @@ dependencies = [ "syn", ] +[[package]] +name = "strum_macros" +version = "0.27.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c77a8c5abcaf0f9ce05d62342b7d298c346515365c36b673df4ebe3ced01fde8" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "rustversion", + "syn", +] + [[package]] name = "subtle" version = "2.6.1" @@ -6362,6 +7260,12 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" +[[package]] +name = "utf8-width" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86bd8d4e895da8537e5315b8254664e6b769c4ff3db18321b297a1e7004392e3" + [[package]] name = "utf8_iter" version = "1.0.4" @@ -6571,16 +7475,6 @@ dependencies = [ "rustls-pki-types", ] -[[package]] -name = "whatlang" -version = "0.16.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "471d1c1645d361eb782a1650b1786a8fb58dd625e681a04c09f5ff7c8764a7b0" -dependencies = [ - "hashbrown 0.14.5", - "once_cell", -] - [[package]] name = "wide" version = "0.7.32" diff --git a/Cargo.toml b/Cargo.toml index d6417a6..ba781c2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,9 +37,10 @@ tokio-executor-trait = "2.1" tokio-reactor-trait = "1.1" chrono = { version = "0.4.41", features = ["serde"] } serde_yaml = "0.9.34" -whatlang = "0.16.4" anyhow = "1.0.98" tokenizers = { version = "0.21", features = ["http"] } # Specify version and add http feature +html-escape = "0.2.13" +lingua = "1.7.2" # {{ Define the library (implicitly done by having src/lib.rs) }} # [lib] diff --git a/config/pipeline_config.yaml b/config/pipeline_config.yaml index 8fe9cc1..1dc64c4 100644 --- a/config/pipeline_config.yaml +++ b/config/pipeline_config.yaml @@ -2,18 +2,6 @@ pipeline: - type: LanguageDetectionFilter min_confidence: 0.65 allowed_languages: [ "dan" ] - - - type: C4QualityFilter - split_paragraph: false - remove_citations: true - filter_no_terminal_punct: true - min_num_sentences: 10 - min_words_per_line: 3 - max_word_length: 15 - filter_lorem_ipsum: true - filter_javascript: true - filter_curly_bracket: true - filter_policy: true - type: GopherRepetitionFilter dup_line_frac: 0.3 @@ -34,7 +22,7 @@ pipeline: - type: GopherQualityFilter min_doc_words: 50 - max_doc_words: 1000000 + max_doc_words: 100000 min_avg_word_length: 3.0 max_avg_word_length: 10.0 max_symbol_word_ratio: 0.1 @@ -65,6 +53,18 @@ pipeline: "vore", "vores", "vær", "være", "været", "øh" ] + - type: C4QualityFilter + split_paragraph: true + remove_citations: true + filter_no_terminal_punct: true + min_num_sentences: 5 + min_words_per_line: 3 + max_word_length: 1000 + filter_lorem_ipsum: true + filter_javascript: true + filter_curly_bracket: true + filter_policy: true + # - type: C4BadWordsFilter # keep_fraction: 0.1 # Keep 10% of documents with bad words # fail_on_missing_language: false # Continue if a language's bad word list is not found diff --git a/src/config/producer.rs b/src/config/producer.rs index ba7a905..b50d8a7 100644 --- a/src/config/producer.rs +++ b/src/config/producer.rs @@ -14,7 +14,7 @@ pub struct Args { pub text_column: String, /// Optional ID column name in the Parquet file - #[arg(long)] + #[arg(long, default_value = "id")] pub id_column: Option, /// RabbitMQ connection string (e.g., amqp://guest:guest@localhost:5672/%2f) diff --git a/src/pipeline/filters/c4_filters.rs b/src/pipeline/filters/c4_filters.rs index 1b19d29..ed60618 100644 --- a/src/pipeline/filters/c4_filters.rs +++ b/src/pipeline/filters/c4_filters.rs @@ -9,26 +9,25 @@ use lazy_static::lazy_static; use rand::rngs::StdRng; use rand::{Rng, SeedableRng}; use regex::Regex; -use reqwest::blocking::Client; // For HTTP client +use reqwest::blocking::Client; use std::collections::HashMap; use std::fs::{self}; // Added fs::File use std::path::PathBuf; // Added Path, PathBuf -use std::sync::Mutex; // Added Mutex for interior mutability +use std::sync::Mutex; +use tracing::info; // For HTTP client // Added Mutex for interior mutability // Constants based on C4 and reference implementation -const END_PUNCTUATION: [char; 4] = ['.', '!', '?', '"']; +const END_PUNCTUATION: [char; 6] = ['.', '!', '?', '"', '\'', '”']; const ELLIPSIS: &str = "..."; lazy_static! { static ref POLICY_SUBSTRINGS: Vec<&'static str> = vec![ - "cookie", - "cookies", "terms of use", "privacy policy", - "gdpr", - "ccpa", - "california consumer privacy act", - "do not sell my personal information", + "cookie policy", + "uses cookies", + "use of cookies", + "use cookies", ]; // Simple regex for Wikipedia-style citations like [1], [2, 3], [45] static ref CITATION_REGEX: Regex = Regex::new(r"\[\d+(?:,\s*\d+)*\]").expect("Invalid regex"); @@ -156,6 +155,10 @@ impl ProcessingStep for C4QualityFilter { split_into_sentences(&document.content) // Default to lines for now }; + if document.id == "danske-taler_2572" { + info!("{}", document.content); + } + let mut kept_lines: Vec = Vec::new(); let mut filter_reasons: Vec = Vec::new(); @@ -163,7 +166,9 @@ impl ProcessingStep for C4QualityFilter { if self.filter_lorem_ipsum && original_content.to_lowercase().contains("lorem ipsum") { filter_reasons.push("lorem_ipsum".to_string()); } - if self.filter_curly_bracket && original_content.contains('{') { + if self.filter_curly_bracket + && (original_content.contains('{') || original_content.contains('}')) + { filter_reasons.push("curly_bracket".to_string()); } @@ -181,6 +186,8 @@ impl ProcessingStep for C4QualityFilter { }); } + let mut line_stats: HashMap = HashMap::new(); + // Line-by-line filtering for line in lines { let current_line = line.trim().to_string(); @@ -202,6 +209,9 @@ impl ProcessingStep for C4QualityFilter { .iter() .any(|w| w.chars().count() > self.max_word_length) { + *line_stats + .entry("line-filter-too_long_word".to_string()) + .or_insert(0) += 1; continue; // Drop line } @@ -214,12 +224,18 @@ impl ProcessingStep for C4QualityFilter { let ends_with_ellipsis = processed_line.ends_with(ELLIPSIS); if !ends_with_terminal_punct || ends_with_ellipsis { + *line_stats + .entry("line-filter-no_terminal_punc".to_string()) + .or_insert(0) += 1; continue; // Drop line } } // min words per line if self.min_words_per_line > 0 && words.len() < self.min_words_per_line { + *line_stats + .entry("line-filter-too_few_words".to_string()) + .or_insert(0) += 1; continue; // Drop line } @@ -261,6 +277,11 @@ impl ProcessingStep for C4QualityFilter { document .metadata .insert("c4_filter_reasons".to_string(), reasons_string.clone()); + + for (key, value) in &line_stats { + document.metadata.insert(key.to_string(), value.to_string()); + } + Err(PipelineError::DocumentFiltered { document: Box::new(document), reason: reasons_string, diff --git a/src/pipeline/filters/fineweb_quality.rs b/src/pipeline/filters/fineweb_quality.rs index e6dc426..5559834 100644 --- a/src/pipeline/filters/fineweb_quality.rs +++ b/src/pipeline/filters/fineweb_quality.rs @@ -23,9 +23,7 @@ use std::collections::HashSet; const DEFAULT_FILTER_NAME: &str = "FineWebQualityFilter"; // TERMINAL_PUNCTUATION in Python: (".", "?", "!", "。", "？", "！") -fn default_stop_chars() -> HashSet { - vec!['.', '?', '!', '。', '？', '！'].into_iter().collect() -} +const END_PUNCTUATION: [char; 6] = ['.', '!', '?', '"', '\'', '”']; #[derive(Debug)] pub struct FineWebQualityFilter { @@ -50,7 +48,7 @@ impl FineWebQualityFilter { // language: String, stop_chars: Option>, ) -> Self { - let sc = stop_chars.unwrap_or_else(|| default_stop_chars().iter().copied().collect()); + let sc = stop_chars.unwrap_or_else(|| END_PUNCTUATION.into()); FineWebQualityFilter { line_punct_thr, line_punct_exclude_zero, @@ -71,6 +69,7 @@ impl ProcessingStep for FineWebQualityFilter { } async fn process(&self, document: TextDocument) -> Result { + let mut document = document; let text_content = &document.content; let lines: Vec<&str> = text_content .lines() @@ -78,6 +77,12 @@ impl ProcessingStep for FineWebQualityFilter { .collect(); if lines.is_empty() { + document + .metadata + .insert("fineweb_filter_status".into(), "filtered".into()); + document + .metadata + .insert("fineweb_filter_reason".into(), "empty document".into()); return Err(PipelineError::DocumentFiltered { document: Box::new(document), reason: "empty".to_string(), @@ -101,12 +106,19 @@ impl ProcessingStep for FineWebQualityFilter { if line_punct_actual_ratio < self.line_punct_thr && !(line_punct_actual_ratio == 0.0 && self.line_punct_exclude_zero) { + let reason = format!( + "line_punct_ratio: {:.4} < threshold {:.4} (exclude_zero: {})", + line_punct_actual_ratio, self.line_punct_thr, self.line_punct_exclude_zero + ); + document + .metadata + .insert("fineweb_filter_status".into(), "filtered".into()); + document + .metadata + .insert("fineweb_filter_reason".into(), reason.clone()); return Err(PipelineError::DocumentFiltered { document: Box::new(document), - reason: format!( - "line_punct_ratio: {:.4} < threshold {:.4} (exclude_zero: {})", - line_punct_actual_ratio, self.line_punct_thr, self.line_punct_exclude_zero - ), + reason: reason, }); } @@ -117,12 +129,19 @@ impl ProcessingStep for FineWebQualityFilter { .count(); let short_line_actual_ratio = short_lines_count as f64 / lines.len() as f64; if short_line_actual_ratio > self.short_line_thr { + let reason = format!( + "short_line_ratio: {:.4} > threshold {:.4}", + short_line_actual_ratio, self.short_line_thr + ); + document + .metadata + .insert("fineweb_filter_status".into(), "filtered".into()); + document + .metadata + .insert("fineweb_filter_reason".into(), reason.clone()); return Err(PipelineError::DocumentFiltered { document: Box::new(document), - reason: format!( - "short_line_ratio: {:.4} > threshold {:.4}", - short_line_actual_ratio, self.short_line_thr - ), + reason, }); } @@ -135,12 +154,19 @@ impl ProcessingStep for FineWebQualityFilter { 0.0 }; if char_dup_actual_ratio > self.char_duplicates_ratio { + let reason = format!( + "char_dup_ratio: {:.4} > threshold {:.4}", + char_dup_actual_ratio, self.char_duplicates_ratio + ); + document + .metadata + .insert("fineweb_filter_status".into(), "filtered".into()); + document + .metadata + .insert("fineweb_filter_reason".into(), reason.clone()); return Err(PipelineError::DocumentFiltered { document: Box::new(document), - reason: format!( - "char_dup_ratio: {:.4} > threshold {:.4}", - char_dup_actual_ratio, self.char_duplicates_ratio - ), + reason: reason, }); } @@ -150,20 +176,34 @@ impl ProcessingStep for FineWebQualityFilter { if words.is_empty() { if new_line_count > 0 { + let reason = "list_ratio_no_words (newlines present but no words)".to_string(); + document + .metadata + .insert("fineweb_filter_status".into(), "filtered".into()); + document + .metadata + .insert("fineweb_filter_reason".into(), reason.clone()); return Err(PipelineError::DocumentFiltered { document: Box::new(document), - reason: "list_ratio_no_words (newlines present but no words)".to_string(), + reason: reason, }); } } else { let list_actual_ratio = new_line_count as f64 / words.len() as f64; if list_actual_ratio > self.new_line_ratio { + let reason = format!( + "list_ratio: {:.4} > threshold {:.4}", + list_actual_ratio, self.new_line_ratio + ); + document + .metadata + .insert("fineweb_filter_status".into(), "filtered".into()); + document + .metadata + .insert("fineweb_filter_reason".into(), reason.clone()); return Err(PipelineError::DocumentFiltered { document: Box::new(document), - reason: format!( - "list_ratio: {:.4} > threshold {:.4}", - list_actual_ratio, self.new_line_ratio - ), + reason: reason, }); } } @@ -190,7 +230,7 @@ mod tests { FineWebQualityFilter { line_punct_thr: DEFAULT_LINE_PUNCT_THR, line_punct_exclude_zero: DEFAULT_LINE_PUNCT_EXCLUDE_ZERO, - stop_chars: default_stop_chars(), + stop_chars: END_PUNCTUATION.into(), short_line_thr: DEFAULT_SHORT_LINE_THR, short_line_length: DEFAULT_SHORT_LINE_LENGTH, char_duplicates_ratio: 0.95, // Temporarily very high to isolate other test failures diff --git a/src/pipeline/filters/gopher_quality.rs b/src/pipeline/filters/gopher_quality.rs index 7749b3c..dc8b3a8 100644 --- a/src/pipeline/filters/gopher_quality.rs +++ b/src/pipeline/filters/gopher_quality.rs @@ -135,7 +135,7 @@ impl ProcessingStep for GopherQualityFilter { .iter() .filter(|&&l| { let ls = l.trim_start(); - ls.starts_with('•') || ls.starts_with('-') || ls.starts_with('*') + ls.starts_with('•') || ls.starts_with('-') // Common bullet chars }) .count(); diff --git a/src/pipeline/filters/language_filter.rs b/src/pipeline/filters/language_filter.rs index ad81caa..f0d39ed 100644 --- a/src/pipeline/filters/language_filter.rs +++ b/src/pipeline/filters/language_filter.rs @@ -1,17 +1,24 @@ -use async_trait::async_trait; -use whatlang::{detect, Lang}; - use crate::data_model::TextDocument; use crate::error::{PipelineError, Result}; use crate::executor::ProcessingStep; +use async_trait::async_trait; +use lingua::{IsoCode639_3, Language, LanguageDetectorBuilder}; pub struct LanguageDetectionFilter { min_confidence: f64, - allowed_languages: Vec, + allowed_languages: Vec, } impl LanguageDetectionFilter { - pub fn new(min_confidence: f64, allowed_languages: Vec) -> Self { + pub fn new(min_confidence: f64, allowed_langs: Vec) -> Self { + let allowed_languages: Vec = allowed_langs + .iter() + .filter_map(|code| { + IsoCode639_3::try_from(code.as_str()) + .ok() + .and_then(|iso| Some(Language::from_iso_code_639_3(&iso))) + }) + .collect(); LanguageDetectionFilter { min_confidence, allowed_languages, @@ -29,38 +36,59 @@ impl ProcessingStep for LanguageDetectionFilter { let mut document = document; let text = &document.content; - let lang_detect = detect(text).unwrap(); - let lang: Lang = lang_detect.lang(); - let confidence = lang_detect.confidence(); - - document - .metadata - .insert("Detected language".into(), lang.name().into()); - document.metadata.insert( - "Detected language confidence".into(), - confidence.to_string(), - ); - - if !self.allowed_languages.contains(&lang.code().into()) { - let reason = format!( - "Document is not any of the following languages: {}", - self.allowed_languages.join("; ") - ); - Err(PipelineError::DocumentFiltered { - document: Box::new(document), - reason, - }) - } else if confidence < self.min_confidence { - let reason = format!( - "Language detection confidence is not satified: {} < {}", - confidence, self.min_confidence + let detector = LanguageDetectorBuilder::from_languages(&[ + Language::English, + Language::Danish, + Language::Swedish, + Language::Nynorsk, + Language::Bokmal, + ]) + .build(); + + if let Some(lang) = detector.detect_language_of(text) { + let confidence = detector.compute_language_confidence(text, lang); + + document + .metadata + .insert("Detected language".into(), lang.to_string()); + document.metadata.insert( + "Detected language confidence".into(), + confidence.to_string(), ); + + if !self.allowed_languages.contains(&lang) { + let langs: Vec = self + .allowed_languages + .clone() + .into_iter() + .map(|lang| lang.iso_code_639_3().to_string()) + .collect(); + let reason = format!( + "Document is not any of the following languages: {:?}", + langs.join("; ") + ); + Err(PipelineError::DocumentFiltered { + document: Box::new(document), + reason, + }) + } else if confidence < self.min_confidence { + let reason = format!( + "Language detection confidence is not satified: {} < {}", + confidence, self.min_confidence + ); + Err(PipelineError::DocumentFiltered { + document: Box::new(document), + reason, + }) + } else { + Ok(document) + } + } else { + let reason = "Language could not be confidently detected".to_string(); Err(PipelineError::DocumentFiltered { document: Box::new(document), reason, }) - } else { - Ok(document) } } } @@ -107,17 +135,25 @@ mod tests { #[tokio::test] async fn test_disallowed_language() { - let document = create_test_doc("doc2", "Ceci est un document de test en français."); + let document = create_test_doc("doc2", "Hej med dig. Dette er Dansk"); let filter = LanguageDetectionFilter::new(0.8, vec!["eng".to_string()]); // Only allow English let result = filter.process(document).await; - assert!(result.is_err()); + assert!( + result.is_err(), + "Expected error. Got text: {}", + result.unwrap().content + ); match result.err().unwrap() { PipelineError::DocumentFiltered { document: _, reason, } => { - assert!(reason.contains("Document is not any of the following languages: eng")); + assert!( + reason.contains("Document is not any of the following languages: \"eng\""), + "The actual reason: {}", + reason + ); } _ => panic!("Expected DocumentFiltered error"), } @@ -126,7 +162,7 @@ mod tests { #[tokio::test] async fn test_confident_but_disallowed_language() { // "Hola Mundo" is Spanish and should be detected with high confidence. - let document = create_test_doc("doc3", "Hola Mundo"); + let document = create_test_doc("doc3", "Jag talar lite svenska."); // Allow only English, with a standard confidence threshold. let filter = LanguageDetectionFilter::new(0.8, vec!["eng".to_string()]); let result = filter.process(document).await; @@ -140,7 +176,7 @@ mod tests { // The primary reason for filtering should be the language not being allowed, // even if confidence would have been sufficient. assert!( - reason.contains("Document is not any of the following languages: eng"), + reason.contains("Document is not any of the following languages: \"eng\""), "Unexpected reason: {}", reason ); @@ -156,7 +192,7 @@ mod tests { #[tokio::test] async fn test_allowed_language_low_confidence() { // "a b c" is English, but very short, likely leading to low confidence. - let document = create_test_doc("doc5", "text arrives out of thin air"); + let document = create_test_doc("doc5", "Text arrives out of thin air"); // Allow English, but require a very high confidence that "a b c" won't meet. let filter = LanguageDetectionFilter::new(0.99, vec!["eng".to_string()]); let result = filter.process(document).await; @@ -172,7 +208,7 @@ mod tests { // and that the reason is low confidence. assert!( processed_doc.metadata.get("Detected language").is_some(), - "Detected language should be in metadata even if filtered for low confidence" + "Detected language should be in metadata even if filtered for low confidence. Reason: {}",reason ); // whatlang detects "a b c" as English. assert_eq!( diff --git a/src/pipeline/readers/parquet_reader.rs b/src/pipeline/readers/parquet_reader.rs index ee47018..9c46f51 100644 --- a/src/pipeline/readers/parquet_reader.rs +++ b/src/pipeline/readers/parquet_reader.rs @@ -174,7 +174,9 @@ impl BaseReader for ParquetReader { let id = id_arr.value(i).to_string(); - let content = text_arr.value(i).to_string(); + let raw_text = text_arr.value(i); + let content = + html_escape::decode_html_entities(raw_text).to_string(); let source_val = source_arr .as_ref() diff --git a/test.json b/test.json new file mode 100644 index 0000000..106fd7b --- /dev/null +++ b/test.json @@ -0,0 +1,35 @@ +{ + "gopher_dup_line_elems_ratio": "0.0000", + "c4_filter_status": "passed", + "gopher_top_4_gram_char_ratio": "0.0167", + "gopher_all_dup_9_gram_char_ratio": "0.0000", + "gopher_quality_metric_alphabetic_word_ratio_vs_total_words": "0.9978", + "gopher_quality_metric_total_word_count": "447", + "gopher_dup_para_elems_ratio": "0.0000", + "gopher_quality_metric_ellipsis_unit_ratio_vs_total_words": "0.0000", + "gopher_repetition_filter_status": "passed", + "gopher_dup_line_chars_ratio": "0.0000", + "gopher_top_2_gram_char_ratio": "0.0167", + "gopher_all_dup_8_gram_char_ratio": "0.0000", + "gopher_all_dup_7_gram_char_ratio": "0.0000", + "Detected language confidence": "1", + "gopher_all_dup_5_gram_char_ratio": "0.0000", + "gopher_quality_metric_bullet_line_ratio": "0.0000", + "gopher_quality_metric_hash_char_ratio_vs_total_words": "0.0000", + "gopher_all_dup_6_gram_char_ratio": "0.0000", + "gopher_quality_metric_ellipsis_line_ratio": "0.0000", + "gopher_all_dup_10_gram_char_ratio": "0.0000", + "gopher_quality_metric_hash_char_count": "0", + "gopher_quality_metric_total_line_count": "1", + "gopher_quality_metric_alphabetic_word_count": "446", + "gopher_quality_metric_non_symbol_word_count": "447", + "gopher_dup_para_chars_ratio": "0.0000", + "gopher_quality_filter_status": "passed", + "Detected language": "Danish", + "gopher_quality_metric_ellipsis_line_count": "0", + "gopher_quality_metric_ellipsis_unit_count": "0", + "gopher_quality_metric_avg_word_len_char_non_symbol": "4.2416", + "gopher_quality_metric_stop_word_count": "288", + "gopher_quality_metric_bullet_line_count": "0", + "gopher_top_3_gram_char_ratio": "0.0200" +} \ No newline at end of file diff --git a/tests/producer_tests.rs b/tests/producer_tests.rs index 2c34139..58a16a1 100644 --- a/tests/producer_tests.rs +++ b/tests/producer_tests.rs @@ -41,7 +41,7 @@ mod args_tests { let args = Args::parse_from(&["producer", "-i", "input.parquet"]); assert_eq!(args.input_file, "input.parquet"); assert_eq!(args.text_column, "text"); - assert_eq!(args.id_column, None); + assert_eq!(args.id_column, Some("id".into())); assert_eq!(args.amqp_addr, "amqp://guest:guest@localhost:5672/%2f"); assert_eq!(args.task_queue, "task_queue"); assert_eq!(args.results_queue, "results_queue"); From 681d515d7bf842403365fa16caa215005e6fedd3 Mon Sep 17 00:00:00 2001 From: kris927b Date: Tue, 24 Jun 2025 14:02:32 +0200 Subject: [PATCH 2/7] Minimising the number of datapoints in the document metadata. --- src/pipeline/filters/gopher_quality.rs | 120 ++++++++++++------------- src/pipeline/filters/gopher_rep.rs | 99 +++++++------------- 2 files changed, 94 insertions(+), 125 deletions(-) diff --git a/src/pipeline/filters/gopher_quality.rs b/src/pipeline/filters/gopher_quality.rs index dc8b3a8..c9679e0 100644 --- a/src/pipeline/filters/gopher_quality.rs +++ b/src/pipeline/filters/gopher_quality.rs @@ -72,10 +72,10 @@ impl ProcessingStep for GopherQualityFilter { let words = split_into_words(text); let n_total_words = words.len(); - document.metadata.insert( - "gopher_quality_metric_total_word_count".to_string(), - n_total_words.to_string(), - ); + // document.metadata.insert( + // "gopher_quality_metric_total_word_count".to_string(), + // n_total_words.to_string(), + // ); let non_symbol_words: Vec<&str> = words .iter() @@ -83,10 +83,10 @@ impl ProcessingStep for GopherQualityFilter { .copied() .collect(); let n_non_symbol_words = non_symbol_words.len(); - document.metadata.insert( - "gopher_quality_metric_non_symbol_word_count".to_string(), - n_non_symbol_words.to_string(), - ); + // document.metadata.insert( + // "gopher_quality_metric_non_symbol_word_count".to_string(), + // n_non_symbol_words.to_string(), + // ); let avg_word_len_non_symbol = if n_non_symbol_words > 0 { let sum_len: usize = non_symbol_words.iter().map(|w| w.chars().count()).sum(); @@ -94,42 +94,42 @@ impl ProcessingStep for GopherQualityFilter { } else { 0.0 }; - document.metadata.insert( - "gopher_quality_metric_avg_word_len_char_non_symbol".to_string(), - format!("{:.4}", avg_word_len_non_symbol), - ); + // document.metadata.insert( + // "gopher_quality_metric_avg_word_len_char_non_symbol".to_string(), + // format!("{:.4}", avg_word_len_non_symbol), + // ); let n_total_words_calc = n_total_words.max(1) as f64; // Denominator for ratios let hash_char_count = text.chars().filter(|&c| c == '#').count(); let hash_char_ratio_vs_total_words = hash_char_count as f64 / n_total_words_calc; - document.metadata.insert( - "gopher_quality_metric_hash_char_count".to_string(), - hash_char_count.to_string(), - ); - document.metadata.insert( - "gopher_quality_metric_hash_char_ratio_vs_total_words".to_string(), - format!("{:.4}", hash_char_ratio_vs_total_words), - ); + // document.metadata.insert( + // "gopher_quality_metric_hash_char_count".to_string(), + // hash_char_count.to_string(), + // ); + // document.metadata.insert( + // "gopher_quality_metric_hash_char_ratio_vs_total_words".to_string(), + // format!("{:.4}", hash_char_ratio_vs_total_words), + // ); let ellipsis_unit_count = text.matches("...").count() + text.matches("…").count(); let ellipsis_unit_ratio_vs_total_words = ellipsis_unit_count as f64 / n_total_words_calc; - document.metadata.insert( - "gopher_quality_metric_ellipsis_unit_count".to_string(), - ellipsis_unit_count.to_string(), - ); - document.metadata.insert( - "gopher_quality_metric_ellipsis_unit_ratio_vs_total_words".to_string(), - format!("{:.4}", ellipsis_unit_ratio_vs_total_words), - ); + // document.metadata.insert( + // "gopher_quality_metric_ellipsis_unit_count".to_string(), + // ellipsis_unit_count.to_string(), + // ); + // document.metadata.insert( + // "gopher_quality_metric_ellipsis_unit_ratio_vs_total_words".to_string(), + // format!("{:.4}", ellipsis_unit_ratio_vs_total_words), + // ); let lines: Vec<&str> = text.lines().collect(); let n_lines_val = lines.len(); let n_lines_calc = n_lines_val.max(1) as f64; // Denominator for line ratios - document.metadata.insert( - "gopher_quality_metric_total_line_count".to_string(), - n_lines_val.to_string(), - ); + // document.metadata.insert( + // "gopher_quality_metric_total_line_count".to_string(), + // n_lines_val.to_string(), + // ); let bullet_line_count = lines .iter() @@ -140,14 +140,14 @@ impl ProcessingStep for GopherQualityFilter { }) .count(); let bullet_line_ratio = bullet_line_count as f64 / n_lines_calc; - document.metadata.insert( - "gopher_quality_metric_bullet_line_count".to_string(), - bullet_line_count.to_string(), - ); - document.metadata.insert( - "gopher_quality_metric_bullet_line_ratio".to_string(), - format!("{:.4}", bullet_line_ratio), - ); + // document.metadata.insert( + // "gopher_quality_metric_bullet_line_count".to_string(), + // bullet_line_count.to_string(), + // ); + // document.metadata.insert( + // "gopher_quality_metric_bullet_line_ratio".to_string(), + // format!("{:.4}", bullet_line_ratio), + // ); let ellipsis_line_count = lines .iter() @@ -157,14 +157,14 @@ impl ProcessingStep for GopherQualityFilter { }) .count(); let ellipsis_line_ratio = ellipsis_line_count as f64 / n_lines_calc; - document.metadata.insert( - "gopher_quality_metric_ellipsis_line_count".to_string(), - ellipsis_line_count.to_string(), - ); - document.metadata.insert( - "gopher_quality_metric_ellipsis_line_ratio".to_string(), - format!("{:.4}", ellipsis_line_ratio), - ); + // document.metadata.insert( + // "gopher_quality_metric_ellipsis_line_count".to_string(), + // ellipsis_line_count.to_string(), + // ); + // document.metadata.insert( + // "gopher_quality_metric_ellipsis_line_ratio".to_string(), + // format!("{:.4}", ellipsis_line_ratio), + // ); let alphabetic_word_count = words .iter() @@ -172,23 +172,23 @@ impl ProcessingStep for GopherQualityFilter { .count(); let alphabetic_word_ratio_vs_total_words = alphabetic_word_count as f64 / n_total_words_calc; - document.metadata.insert( - "gopher_quality_metric_alphabetic_word_count".to_string(), - alphabetic_word_count.to_string(), - ); - document.metadata.insert( - "gopher_quality_metric_alphabetic_word_ratio_vs_total_words".to_string(), - format!("{:.4}", alphabetic_word_ratio_vs_total_words), - ); + // document.metadata.insert( + // "gopher_quality_metric_alphabetic_word_count".to_string(), + // alphabetic_word_count.to_string(), + // ); + // document.metadata.insert( + // "gopher_quality_metric_alphabetic_word_ratio_vs_total_words".to_string(), + // format!("{:.4}", alphabetic_word_ratio_vs_total_words), + // ); let stop_word_count = words .iter() .filter(|w| self.stop_words.contains(w.to_lowercase().as_str())) .count(); - document.metadata.insert( - "gopher_quality_metric_stop_word_count".to_string(), - stop_word_count.to_string(), - ); + // document.metadata.insert( + // "gopher_quality_metric_stop_word_count".to_string(), + // stop_word_count.to_string(), + // ); let mut filter_reasons: Vec = Vec::new(); diff --git a/src/pipeline/filters/gopher_rep.rs b/src/pipeline/filters/gopher_rep.rs index 3ef80b1..86a7fd0 100644 --- a/src/pipeline/filters/gopher_rep.rs +++ b/src/pipeline/filters/gopher_rep.rs @@ -60,48 +60,17 @@ impl ProcessingStep for GopherRepetitionFilter { if trimmed_text.is_empty() { document.metadata.insert( "gopher_repetition_filter_status".to_string(), - "skipped_empty_content".to_string(), + "filtered".to_string(), ); - // Add 0.0 for all potential ratios for consistency if downstream expects them - // For thresholds that are Some (i.e., active) - if self.dup_para_frac.is_some() { - document.metadata.insert( - "gopher_dup_para_elems_ratio".to_string(), - "0.0000".to_string(), - ); - } - if self.dup_para_char_frac.is_some() { - document.metadata.insert( - "gopher_dup_para_chars_ratio".to_string(), - "0.0000".to_string(), - ); - } - if self.dup_line_frac.is_some() { - document.metadata.insert( - "gopher_dup_line_elems_ratio".to_string(), - "0.0000".to_string(), - ); - } - if self.dup_line_char_frac.is_some() { - document.metadata.insert( - "gopher_dup_line_chars_ratio".to_string(), - "0.0000".to_string(), - ); - } - // For configured n-grams - for &(n, _) in &self.top_n_grams { - document.metadata.insert( - format!("gopher_top_{}_gram_char_ratio", n), - "0.0000".to_string(), - ); - } - for &(n, _) in &self.dup_n_grams { - document.metadata.insert( - format!("gopher_all_dup_{}_gram_char_ratio", n), - "0.0000".to_string(), - ); - } - return Ok(document); + document.metadata.insert( + "gopher_repetition_filter_reason".to_string(), + "skipping empty content".to_string(), + ); + + return Err(PipelineError::DocumentFiltered { + document: Box::new(document), + reason: "skipping empty content".to_string(), + }); } let mut filter_reasons: Vec = Vec::new(); @@ -119,10 +88,10 @@ impl ProcessingStep for GopherRepetitionFilter { let para_len = paragraphs.len().max(1) as f64; let ratio_para_dup_elems = para_dup_elems as f64 / para_len; - document.metadata.insert( - "gopher_dup_para_elems_ratio".to_string(), - format!("{:.4}", ratio_para_dup_elems), - ); + // document.metadata.insert( + // "gopher_dup_para_elems_ratio".to_string(), + // format!("{:.4}", ratio_para_dup_elems), + // ); if let Some(threshold) = self.dup_para_frac { if ratio_para_dup_elems > threshold { filter_reasons.push(format!( @@ -133,10 +102,10 @@ impl ProcessingStep for GopherRepetitionFilter { } let ratio_para_dup_chars = para_dup_chars as f64 / text_char_len; - document.metadata.insert( - "gopher_dup_para_chars_ratio".to_string(), - format!("{:.4}", ratio_para_dup_chars), - ); + // document.metadata.insert( + // "gopher_dup_para_chars_ratio".to_string(), + // format!("{:.4}", ratio_para_dup_chars), + // ); if let Some(threshold) = self.dup_para_char_frac { if ratio_para_dup_chars > threshold { filter_reasons.push(format!( @@ -159,10 +128,10 @@ impl ProcessingStep for GopherRepetitionFilter { let line_len = lines.len().max(1) as f64; let ratio_line_dup_elems = line_dup_elems as f64 / line_len; - document.metadata.insert( - "gopher_dup_line_elems_ratio".to_string(), - format!("{:.4}", ratio_line_dup_elems), - ); + // document.metadata.insert( + // "gopher_dup_line_elems_ratio".to_string(), + // format!("{:.4}", ratio_line_dup_elems), + // ); if let Some(threshold) = self.dup_line_frac { if ratio_line_dup_elems > threshold { filter_reasons.push(format!( @@ -173,10 +142,10 @@ impl ProcessingStep for GopherRepetitionFilter { } let ratio_line_dup_chars = line_dup_chars as f64 / text_char_len; - document.metadata.insert( - "gopher_dup_line_chars_ratio".to_string(), - format!("{:.4}", ratio_line_dup_chars), - ); + // document.metadata.insert( + // "gopher_dup_line_chars_ratio".to_string(), + // format!("{:.4}", ratio_line_dup_chars), + // ); if let Some(threshold) = self.dup_line_char_frac { if ratio_line_dup_chars > threshold { filter_reasons.push(format!( @@ -196,10 +165,10 @@ impl ProcessingStep for GopherRepetitionFilter { let top_len_contribution = find_top_duplicate(&n_grams); let ratio_top_n_gram_chars = top_len_contribution as f64 / text_char_len; - document.metadata.insert( - format!("gopher_top_{}_gram_char_ratio", n), - format!("{:.4}", ratio_top_n_gram_chars), - ); + // document.metadata.insert( + // format!("gopher_top_{}_gram_char_ratio", n), + // format!("{:.4}", ratio_top_n_gram_chars), + // ); // Only apply filter if n > 0, as n=0 calculations result in 0 and thresholds are not meaningful. if n > 0 && ratio_top_n_gram_chars > frac_threshold { filter_reasons.push(format!( @@ -213,10 +182,10 @@ impl ProcessingStep for GopherRepetitionFilter { let dup_n_gram_chars = find_all_duplicate(&words, n); let ratio_all_dup_n_gram_chars = dup_n_gram_chars as f64 / text_char_len; - document.metadata.insert( - format!("gopher_all_dup_{}_gram_char_ratio", n), - format!("{:.4}", ratio_all_dup_n_gram_chars), - ); + // document.metadata.insert( + // format!("gopher_all_dup_{}_gram_char_ratio", n), + // format!("{:.4}", ratio_all_dup_n_gram_chars), + // ); // Only apply filter if n > 0. if n > 0 && ratio_all_dup_n_gram_chars > frac_threshold { filter_reasons.push(format!( From 154f43826b9906de4334d95a854411333cea87cc Mon Sep 17 00:00:00 2001 From: kris927b Date: Tue, 24 Jun 2025 14:04:41 +0200 Subject: [PATCH 3/7] Removing an test for empty documents that pass. Empty documents should no longer pass gopher rep. --- src/pipeline/filters/gopher_rep.rs | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/src/pipeline/filters/gopher_rep.rs b/src/pipeline/filters/gopher_rep.rs index 86a7fd0..e1872a5 100644 --- a/src/pipeline/filters/gopher_rep.rs +++ b/src/pipeline/filters/gopher_rep.rs @@ -418,28 +418,6 @@ mod tests { assert!(filter.process(doc).await.is_ok()); } - #[tokio::test] - async fn test_rep_filter_empty_doc_passes() { - let filter = GopherRepetitionFilter::new( - Some(0.1), - Some(0.1), - Some(0.1), - Some(0.1), - vec![(2, 0.1)], - vec![(2, 0.1)], - ); - let doc = create_rep_test_doc("empty_rep", ""); - assert!( - filter.process(doc).await.is_ok(), - "Empty doc should pass all repetition checks" - ); - let doc_whitespace = create_rep_test_doc("whitespace_rep", " \n\n "); - assert!( - filter.process(doc_whitespace).await.is_ok(), - "Whitespace doc should pass all repetition checks" - ); - } - // --- Paragraph Repetition Tests --- #[tokio::test] async fn test_duplicate_paragraphs() { From ef257e8044b15f148bd730b2af28ed2346f91ac1 Mon Sep 17 00:00:00 2001 From: kris927b Date: Mon, 30 Jun 2025 14:28:59 +0200 Subject: [PATCH 4/7] Fixing char dup error --- src/pipeline/filters/fineweb_quality.rs | 21 ++++++++++++--- test.json | 35 ------------------------- 2 files changed, 17 insertions(+), 39 deletions(-) delete mode 100644 test.json diff --git a/src/pipeline/filters/fineweb_quality.rs b/src/pipeline/filters/fineweb_quality.rs index 5559834..4aa4aba 100644 --- a/src/pipeline/filters/fineweb_quality.rs +++ b/src/pipeline/filters/fineweb_quality.rs @@ -146,10 +146,23 @@ impl ProcessingStep for FineWebQualityFilter { } // Character duplication ratio - let vec_line: Vec = lines.iter().map(|l| l.to_string()).collect(); - let (repeated_char_count, total_chars_for_dup_ratio) = find_duplicates(&vec_line); - let char_dup_actual_ratio = if total_chars_for_dup_ratio > 0 { - repeated_char_count as f64 / total_chars_for_dup_ratio as f64 + // 1. Filter non-empty lines + let non_empty_lines: Vec = lines + .iter() + .filter(|line| !line.trim().is_empty()) + .map(|line| line.to_string()) + .collect(); + + // 2. Count total characters in doc (excluding newlines) + let total_chars_no_newlines: usize = + document.content.chars().filter(|&c| c != '\n').count(); + + // 3. Find duplicates among non-empty lines + let (_dup_count, dup_char_count) = find_duplicates(&non_empty_lines); + + // 4. Compute ratio + let char_dup_actual_ratio = if total_chars_no_newlines > 0 { + dup_char_count as f64 / total_chars_no_newlines as f64 } else { 0.0 }; diff --git a/test.json b/test.json deleted file mode 100644 index 106fd7b..0000000 --- a/test.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "gopher_dup_line_elems_ratio": "0.0000", - "c4_filter_status": "passed", - "gopher_top_4_gram_char_ratio": "0.0167", - "gopher_all_dup_9_gram_char_ratio": "0.0000", - "gopher_quality_metric_alphabetic_word_ratio_vs_total_words": "0.9978", - "gopher_quality_metric_total_word_count": "447", - "gopher_dup_para_elems_ratio": "0.0000", - "gopher_quality_metric_ellipsis_unit_ratio_vs_total_words": "0.0000", - "gopher_repetition_filter_status": "passed", - "gopher_dup_line_chars_ratio": "0.0000", - "gopher_top_2_gram_char_ratio": "0.0167", - "gopher_all_dup_8_gram_char_ratio": "0.0000", - "gopher_all_dup_7_gram_char_ratio": "0.0000", - "Detected language confidence": "1", - "gopher_all_dup_5_gram_char_ratio": "0.0000", - "gopher_quality_metric_bullet_line_ratio": "0.0000", - "gopher_quality_metric_hash_char_ratio_vs_total_words": "0.0000", - "gopher_all_dup_6_gram_char_ratio": "0.0000", - "gopher_quality_metric_ellipsis_line_ratio": "0.0000", - "gopher_all_dup_10_gram_char_ratio": "0.0000", - "gopher_quality_metric_hash_char_count": "0", - "gopher_quality_metric_total_line_count": "1", - "gopher_quality_metric_alphabetic_word_count": "446", - "gopher_quality_metric_non_symbol_word_count": "447", - "gopher_dup_para_chars_ratio": "0.0000", - "gopher_quality_filter_status": "passed", - "Detected language": "Danish", - "gopher_quality_metric_ellipsis_line_count": "0", - "gopher_quality_metric_ellipsis_unit_count": "0", - "gopher_quality_metric_avg_word_len_char_non_symbol": "4.2416", - "gopher_quality_metric_stop_word_count": "288", - "gopher_quality_metric_bullet_line_count": "0", - "gopher_top_3_gram_char_ratio": "0.0200" -} \ No newline at end of file From afa3df0ed6df9ce1354d2f8c5560dbf29766f4f2 Mon Sep 17 00:00:00 2001 From: kris927b Date: Mon, 1 Sep 2025 08:24:44 +0200 Subject: [PATCH 5/7] Minor fix to gopher quality --- src/pipeline/filters/gopher_quality.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/pipeline/filters/gopher_quality.rs b/src/pipeline/filters/gopher_quality.rs index c9679e0..ef6190c 100644 --- a/src/pipeline/filters/gopher_quality.rs +++ b/src/pipeline/filters/gopher_quality.rs @@ -275,11 +275,10 @@ impl ProcessingStep for GopherQualityFilter { // Alphabetic-word ratio if let Some(max_non_alpha_ratio) = self.max_non_alpha_words_ratio { - let min_required_alpha_ratio = 1.0 - max_non_alpha_ratio; - if alphabetic_word_ratio_vs_total_words < min_required_alpha_ratio { + if alphabetic_word_ratio_vs_total_words < max_non_alpha_ratio { filter_reasons.push(format!( "gopher_below_alpha_threshold (alpha ratio {:.2}, required min {:.2})", - alphabetic_word_ratio_vs_total_words, min_required_alpha_ratio + alphabetic_word_ratio_vs_total_words, max_non_alpha_ratio )); } } From 4d2be3554d3d5e1673dba4aa35d8aa1390813057 Mon Sep 17 00:00:00 2001 From: kris927b Date: Mon, 1 Sep 2025 09:33:43 +0200 Subject: [PATCH 6/7] Fixing clippy error --- src/pipeline/filters/language_filter.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pipeline/filters/language_filter.rs b/src/pipeline/filters/language_filter.rs index f0d39ed..8d4737a 100644 --- a/src/pipeline/filters/language_filter.rs +++ b/src/pipeline/filters/language_filter.rs @@ -16,7 +16,7 @@ impl LanguageDetectionFilter { .filter_map(|code| { IsoCode639_3::try_from(code.as_str()) .ok() - .and_then(|iso| Some(Language::from_iso_code_639_3(&iso))) + .map(|iso| Language::from_iso_code_639_3(&iso)) }) .collect(); LanguageDetectionFilter { From f329678d6607672246da06b6cf04bf9ee25a6afa Mon Sep 17 00:00:00 2001 From: kris927b Date: Mon, 1 Sep 2025 09:34:00 +0200 Subject: [PATCH 7/7] Fixing clippy and test error --- src/pipeline/filters/fineweb_quality.rs | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/pipeline/filters/fineweb_quality.rs b/src/pipeline/filters/fineweb_quality.rs index 4aa4aba..c2002da 100644 --- a/src/pipeline/filters/fineweb_quality.rs +++ b/src/pipeline/filters/fineweb_quality.rs @@ -118,7 +118,7 @@ impl ProcessingStep for FineWebQualityFilter { .insert("fineweb_filter_reason".into(), reason.clone()); return Err(PipelineError::DocumentFiltered { document: Box::new(document), - reason: reason, + reason, }); } @@ -166,6 +166,7 @@ impl ProcessingStep for FineWebQualityFilter { } else { 0.0 }; + if char_dup_actual_ratio > self.char_duplicates_ratio { let reason = format!( "char_dup_ratio: {:.4} > threshold {:.4}", @@ -179,7 +180,7 @@ impl ProcessingStep for FineWebQualityFilter { .insert("fineweb_filter_reason".into(), reason.clone()); return Err(PipelineError::DocumentFiltered { document: Box::new(document), - reason: reason, + reason, }); } @@ -198,7 +199,7 @@ impl ProcessingStep for FineWebQualityFilter { .insert("fineweb_filter_reason".into(), reason.clone()); return Err(PipelineError::DocumentFiltered { document: Box::new(document), - reason: reason, + reason, }); } } else { @@ -216,7 +217,7 @@ impl ProcessingStep for FineWebQualityFilter { .insert("fineweb_filter_reason".into(), reason.clone()); return Err(PipelineError::DocumentFiltered { document: Box::new(document), - reason: reason, + reason, }); } } @@ -442,17 +443,18 @@ mod tests { filter.short_line_thr = 1.0; filter.new_line_ratio = 1.0; // Allow many newlines - filter.char_duplicates_ratio = 0.9; // Specific for this test to fail - let content = "a\na\na."; // Ends with '.', 31 chars, 29 'a' repeats. Ratio ~0.935 + filter.char_duplicates_ratio = 0.66; // Specific for this test to fail + let content = "Hello World\nHello World\nHello World"; // Ends with '.', 31 chars, 29 'a' repeats. Ratio ~0.935 let doc = create_test_doc("char_dup_all_same", content); let result = filter.process(doc).await; + println!("{:?}", result); assert!(result.is_err()); match result.err().unwrap() { PipelineError::DocumentFiltered { reason, .. } => { // Ratio 29/31 = 0.93548... assert!( - reason.starts_with("char_dup_ratio: 1.0000 > threshold 0.9000"), + reason.starts_with("char_dup_ratio: 0.6667 > threshold 0.6600"), "Actual reason: {}", reason );