diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 39a6690..a06a98c 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -100,3 +100,23 @@ jobs: env: PYWHAT_GOLDEN: target/golden.jsonl PYWHAT_GOLDEN_FILES: target/golden-files.jsonl + + cli: + name: Rust CLI prints what Python prints (parity) + runs-on: ubuntu-latest + env: + PYWHAT_CLI: target/release/what + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 + - uses: dtolnay/rust-toolchain@b725eda525362e7cf34af9a7243cac1e3f01a7ec # stable + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2 + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 + with: + python-version: '3.10' + # The versions in poetry.lock + - run: 'python -m pip install --only-binary :all: click==8.1.2 rich==12.2.0 pytest==7.1.2' + - run: cargo build --locked --release --bin what + # The stdout bytes and exit codes of ~270 command lines, against python -m pywhat + - run: python scripts/parity_cli.py + # pyWhat's CLI tests, run against the binary + - run: python -m pytest -o addopts="" tests/test_click.py diff --git a/Cargo.lock b/Cargo.lock index c088587..97ba645 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11,6 +11,21 @@ dependencies = [ "memchr", ] +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + [[package]] name = "bit-set" version = "0.5.3" @@ -41,6 +56,12 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + [[package]] name = "block-buffer" version = "0.12.1" @@ -56,18 +77,83 @@ version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cc" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "375989decec1a0fa424442bed520fa12ede43a48475ce8c7417d2c69b32b5f6b" +dependencies = [ + "find-msvc-tools", + "shlex", +] + [[package]] name = "cfg-if" version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" +[[package]] +name = "combine" +version = "4.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4356eba6976435498d05a311ee9fc486da3bc81c200d3f0d7dd9703b41913f92" +dependencies = [ + "bytes", + "memchr", +] + +[[package]] +name = "comfy-table" +version = "8.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b96a46ad395a5e00676eddaca4a45ed78b29c08be5a878d35e4b071c625637dd" +dependencies = [ + "crossterm", + "unicode-segmentation", + "unicode-width", +] + +[[package]] +name = "console" +version = "0.16.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e96a4956774c13c126a8b5af4daa79384f4d826534c95a02d76afb39e2ab64e3" +dependencies = [ + "encode_unicode", + "libc", + "unicode-width", + "windows-sys 0.61.2", +] + [[package]] name = "const-oid" version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + [[package]] name = "cpufeatures" version = "0.3.1" @@ -102,6 +188,29 @@ version = "0.8.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" +[[package]] +name = "crossterm" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8b9f2e4c67f833b660cdb0a3523065869fb35570177239812ed4c905aeff87b" +dependencies = [ + "bitflags", + "crossterm_winapi", + "document-features", + "parking_lot", + "rustix", + "winapi", +] + +[[package]] +name = "crossterm_winapi" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acdd7c62a3665c7f6830a51635d9ac9b23ed385797f70a83bb8bafe9c572ab2b" +dependencies = [ + "winapi", +] + [[package]] name = "crypto-common" version = "0.2.2" @@ -122,18 +231,43 @@ dependencies = [ "crypto-common", ] +[[package]] +name = "document-features" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4b8a88685455ed29a21542a33abd9cb6510b6b129abadabdcef0f4c55bc8f61" +dependencies = [ + "litrs", +] + [[package]] name = "either" version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be" +[[package]] +name = "encode_unicode" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" + [[package]] name = "equivalent" version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826" +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + [[package]] name = "fancy-regex" version = "0.11.0" @@ -155,12 +289,29 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "find-msvc-tools" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c66915cc5391097de7a2b0b972e8505aa33c0df401fd3a6811dbdb32881ba48" + [[package]] name = "foldhash" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + [[package]] name = "hashbrown" version = "0.17.1" @@ -173,6 +324,22 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "http" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + [[package]] name = "hybrid-array" version = "0.4.15" @@ -192,12 +359,73 @@ dependencies = [ "hashbrown", ] +[[package]] +name = "indicatif" +version = "0.18.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9433806cd6b4ec1aba79c021c7e4c58fb4c3b9977c085062e611ac929998fb0c" +dependencies = [ + "console", + "portable-atomic", + "unicode-width", + "unit-prefix", +] + [[package]] name = "itoa" version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "jni" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" +dependencies = [ + "cfg-if", + "combine", + "jni-macros", + "jni-sys", + "log", + "simd_cesu8", + "thiserror", + "walkdir", + "windows-link", +] + +[[package]] +name = "jni-macros" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "simd_cesu8", + "syn 2.0.119", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn 2.0.119", +] + [[package]] name = "lemmeknow" version = "0.8.0" @@ -218,6 +446,33 @@ version = "0.2.191" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litrs" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4744e383959f0db86ede514b809b1c53251889093803c05267acc7d4e7030d70" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d15f86d12da44cd8d4ec3c73984537eb7f4cadcb9776a575a0a8a05ca11005c" + [[package]] name = "memchr" version = "2.8.3" @@ -230,6 +485,41 @@ version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +[[package]] +name = "openssl-probe" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e036b411754132fd6b74e94b74e9a13c5cf606fa93d1885c35860d3060820b1" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + [[package]] name = "portable-atomic" version = "1.15.0" @@ -306,14 +596,20 @@ dependencies = [ name = "pywhat" version = "6.0.0-alpha.1" dependencies = [ + "anstyle-query", + "comfy-table", "fancy-regex 0.20.0", + "indicatif", "lemmeknow", "rayon", "regex", "regex-automata", "regex-syntax", + "rustls", "serde_json", "sha2", + "terminal_size", + "ureq", "walkdir", "wasm-bindgen", ] @@ -355,6 +651,15 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + [[package]] name = "regex" version = "1.13.1" @@ -384,6 +689,116 @@ version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rustix" +version = "1.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5e76742e5294153d4dde1207dbec5bdd81eef6fabdc0d2118babadabfd858aa" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls" +version = "0.23.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf" +dependencies = [ + "log", + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a073f5dc7469f984c52ad2752b63b0807745133b6de880b7b64c1ac4c48aec4" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pki-types" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" +dependencies = [ + "zeroize", +] + +[[package]] +name = "rustls-platform-verifier" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1167586491e2b18b8bfbb293e8180ec17c201c4f076d7cb3070ca964e7598f98" +dependencies = [ + "core-foundation", + "core-foundation-sys", + "jni", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustls-platform-verifier-android" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eec689c0bc40ff2458a5977b6619cb718087084a18e02a131c599b62d05e1a5f" + +[[package]] +name = "rustls-webpki" +version = "0.103.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4443b1b6d39d7c663139942cc4558262562d2c84123bb6282ef177009ca8b281" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + [[package]] name = "rustversion" version = "1.0.23" @@ -399,6 +814,50 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "schannel" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cb23c424d8ee841bb4d813677ca237e2a610067d594fec130f87d6093495ab5" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + [[package]] name = "serde" version = "1.0.229" @@ -455,6 +914,40 @@ dependencies = [ "digest", ] +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "simd_cesu8" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11031e251abf8611c80f460e19dbdeb54a66db918e49c65a7065b46ac7aec520" +dependencies = [ + "rustc_version", + "simdutf8", +] + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + +[[package]] +name = "smallvec" +version = "1.16.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0d5d4b1981b8bbbb7aed204cae5726d683d5d6ac7e74ec54122ea28f315e7c8" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + [[package]] name = "syn" version = "2.0.119" @@ -483,6 +976,36 @@ version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca" +[[package]] +name = "terminal_size" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "230a1b821ccbd75b185820a1f1ff7b14d21da1e442e22c0863ea5f08771a8874" +dependencies = [ + "rustix", + "windows-sys 0.61.2", +] + +[[package]] +name = "thiserror" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + [[package]] name = "typenum" version = "1.20.1" @@ -495,6 +1018,64 @@ version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + +[[package]] +name = "unicode-width" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca718471876992059f251e33aafc334346ada08042b7661dd1fe313a9bced557" + +[[package]] +name = "unit-prefix" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81e544489bf3d8ef66c953931f56617f423cd4b5494be343d9b9d3dda037b9a3" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "ureq" +version = "3.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7be76686bdf7198002062423ca0d8cf9b75a27c8e1de804513b97e97b555d9d" +dependencies = [ + "base64", + "log", + "percent-encoding", + "rustls", + "rustls-pki-types", + "rustls-platform-verifier", + "ureq-proto", + "utf8-zero", +] + +[[package]] +name = "ureq-proto" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622dce33c252115466d83f63e9744a18f12b4406aae51a01c15ad8fcf1185eb3" +dependencies = [ + "base64", + "http", + "httparse", + "log", +] + +[[package]] +name = "utf8-zero" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8c0a043c9540bae7c578c88f91dda8bd82e59ae27c21baca69c8b191aaf5a6e" + [[package]] name = "walkdir" version = "2.5.0" @@ -505,6 +1086,12 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + [[package]] name = "wasm-bindgen" version = "0.2.130" @@ -549,21 +1136,61 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "webpki-root-certs" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e49d0e65abc00c142e8f9428da991da9f1a250c39be97459901eb12b9b49f62a" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + [[package]] name = "winapi-util" version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys", + "windows-sys 0.61.2", ] +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + [[package]] name = "windows-link" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + [[package]] name = "windows-sys" version = "0.61.2" @@ -573,6 +1200,76 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "zeroize" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13084392c5e4bc371903e2935a5eaeed24905a7511356b883835e18a78f6879" + [[package]] name = "zmij" version = "1.0.23" diff --git a/Cargo.toml b/Cargo.toml index 6f8a424..7baa1fb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,7 +31,21 @@ categories = ["command-line-utilities", "text-processing", "wasm"] include = ["/src", "/pywhat/Data/*.json", "/pywhat/Data/regexes/*.json", "/LICENSE", "/NOTICE", "/README.md"] [features] -default = ["parallel", "files", "children", "mac-vendors"] +default = ["parallel", "files", "children", "mac-vendors", "cli", "verify"] +# The `what` and `pywhat` commands (src/cli/), with the options and the output of the +# Python package's CLI: tables, colours and progress bars. pywhat::cli::main is what they +# run, and what the Python package's `pywhat` and `what` commands call. They need the file +# signatures and every Children table to print what the Python CLI prints. +cli = [ + "files", + "mac-vendors", + "dep:comfy-table", + "dep:indicatif", + "dep:anstyle-query", + "dep:terminal_size", +] +# `what --verify`: asks Google whether the Google API keys found are valid (over HTTPS). +verify = ["cli", "dep:ureq", "dep:rustls"] # Search the regexes of a large text, and compile them, on rayon's thread pool. Not on # wasm32, which has no threads. parallel = ["dep:rayon"] @@ -60,6 +74,20 @@ rayon = { version = "1.12.0", optional = true } walkdir = { version = "2.5.0", optional = true } wasm-bindgen = { version = "=0.2.130", optional = true } +# The CLI (feature `cli`): the --format pretty table, progress bars, colours on Windows +# consoles, and the terminal sizes that rich and click use. Not on wasm32, which has no +# terminal: there the features build the library only. +[target.'cfg(not(target_arch = "wasm32"))'.dependencies] +comfy-table = { version = "8.0.1", optional = true } +indicatif = { version = "0.18.6", optional = true, default-features = false, features = ["unicode-width"] } +anstyle-query = { version = "1.1.5", optional = true } +terminal_size = { version = "0.4.4", optional = true } +# --verify (feature `verify`), without gzip, as Python's urllib. TLS with rustls and ring, +# trusting the operating system's root certificates as Python does (platform-verifier), with +# no list of root certificates of its own +ureq = { version = "3.5.1", optional = true, default-features = false, features = ["rustls-no-provider", "platform-verifier"] } +rustls = { version = "0.23.46", optional = true, default-features = false, features = ["ring", "std", "tls12"] } + [dev-dependencies] serde_json = { version = "1.0.154", features = ["preserve_order"] } # Only for benches/whole.rs, which compares the speed with lemmeknow, the crate that Ciphey @@ -68,6 +96,20 @@ lemmeknow = { version = "=0.8.0", default-features = false } # tests/charset_only.rs builds the DFA of every regex regex-automata = { version = "0.4.18", features = ["dfa-build"] } +[[bin]] +name = "what" +path = "src/bin/what.rs" +required-features = ["cli"] + +[[bin]] +name = "pywhat" +path = "src/bin/pywhat.rs" +required-features = ["cli"] + +[[test]] +name = "cli" +required-features = ["cli"] + [[example]] name = "bench_file" required-features = ["files"] diff --git a/docs/cli.md b/docs/cli.md new file mode 100644 index 0000000..df67a1e --- /dev/null +++ b/docs/cli.md @@ -0,0 +1,90 @@ +# The `what` command in Rust + +`cargo install pywhat` installs `what` and `pywhat`: the command line of the Python package in +Rust, with the same options, defaults, help, error messages and exit codes (`what --help`), and the +same output. `pywhat::cli::main(std::env::args_os())` runs it and returns the exit code. Both +binaries do just that, and the Python package's commands can call it too. It is the crate's `cli` +feature, and `--verify` is its `verify` feature (both are default features, and add nothing on +wasm32). + +- The arguments are parsed the way click parses them: `-br`, `-db` and the other options with + several letters after one `-`, clusters such as `-ok name`, `--top=5`, `--` before inputs that + start with `-`, and unknown options are inputs (`what -1606586780` searches `-1606586780`). +- Raw output, custom `--format` strings, `--json`, `--stream` (with `--json`: a JSON object per + line, without `"Fragment"`) and `Nothing found!` are byte for byte what the Python CLI prints. + `--format pretty` has the same cells, in a table that comfy-table draws. +- Colours (bold `#D7AFFF` labels) only on a terminal, without colours if `NO_COLOR` is set. + Progress bars follow the Python CLI's rules: only on a terminal, after half a second, on stderr + if stdout is a file, never into a pipe, and `--no-progress` turns them off. +- `--verify` sends each Google API key to Google once, like `GoogleAPIKeyVerifier`, with the same + messages. It reads Google's answer as Python's `json.loads()` does, and connects like + `urlopen(..., timeout=10)`: 10 s to connect and for every read, redirects (301, 302, 303 and 307) + followed, and the common network errors described the same way (`timed out`, ``). It trusts the operating system's root certificates, as Python + does (rustls-platform-verifier), and has no list of its own. The verifier is `pywhat::cli::verify::GoogleApiKeyVerifier`, a + `Processor` whose `with_http` takes the HTTP call, so that tests don't send anything. +- `--interactive` is only in the Python package: `what --interactive` says so and exits with 2. + +## Differences from the Python CLI + +- The matched text and the file names are printed as they are. The Python CLI takes them for rich + markup and emoji codes: `pywhat 'thm{[bold]x[/bold]}'` prints `thm{x}`, and `pywhat 'thm{[/x]}'` + crashes with a `MarkupError` (#291, Python bug 4). The markup in the data files (some + descriptions) and in `--format` strings is rendered as rich renders it. A `--format` string with + a closing tag that closes nothing (`[/b]%m`) prints nothing and exits with 1 in both; Rust says + why on stderr (`Error: closing tag '[/b]' at position 0 doesn't match any open tag`), where + Python shows a traceback. Emoji codes such as `:smile:` in a `--format` string are printed as + they are, where rich prints the emoji. +- In `--format pretty`, a tab in a file name is shown expanded on one line; rich counts it as 0 + columns wide when it sizes the column, and then wraps the cell. +- The width of the terminal (or `COLUMNS`) only changes the table of `--format pretty` and the + help. rich also wraps the lines that pyWhat doesn't print with `soft_wrap` (the tags of `--tags`, + `Nothing found!`, the file lines above the table) when they are wider than the terminal, removes + spaces at the end of lines that are wider than it, and prints nothing at all with `COLUMNS=0`. + These are the wrapping problems of #291's Python bug 3, which Rust doesn't have. +- On a terminal, the colours of raw and custom output are written as rich writes them, for the same + `TERM`, `COLORTERM` and `NO_COLOR`. The table has comfy-table's escape sequences; rich gives the + links in the data's markup a random `id`, and colours the numbers of the tags of `--tags` + separately. +- `-v` prints the version of the crate. +- Directories are searched in the order of their names, where Python takes the order of the file + system (#291, Python bug 5). A file that can't be read is skipped and reported on stderr, with + exit code 1, where Python stops with a traceback. +- The output is always UTF-8. Where stdout's encoding can't encode a character (output redirected + to a file on Windows, or `PYTHONIOENCODING=latin-1`), Python writes an escape such as `\u20bf`. + Arguments and the names of files in directories that aren't valid UTF-8 get U+FFFD for the + invalid bytes, where Python keeps them as surrogate escapes (`\udcff`). +- Ctrl-C ends the command at once (exit code 130), and can leave the last line of the progress + bars on the terminal; Python prints `Aborted!` and exits with 1. +- Without arguments, `what` reads stdin, as `pywhat` does; `python -m pywhat` shows the help. + +## Parity tests + +`scripts/parity_cli.py` runs 288 command lines with both CLIs and compares their stdout, their +stderr (unless Python stops with a traceback) and their exit codes: every case of +`tests/test_click.py` (read from it), every option on `fixtures/file`, the examples of the README +and of the help, and corner cases of the argument parser, of stdin, of files and of markup. Python runs with glob listing directories in name order, so that the output of directories +can be compared byte for byte. `tests/conftest.py` runs `tests/test_click.py` against the binary +when `PYWHAT_CLI` is set: + +```sh +cargo build --release +python scripts/parity_cli.py +PYWHAT_CLI=target/release/what pytest -o addopts="" tests/test_click.py +``` + +The Rust workflow runs both on every pull request. + +## Measurements + +`hyperfine -N` (hyperfine 2.0.0), with 5 runs for the big file and 10 for the others, on a shared +64-core machine with a load average of 61 to 83, JSON to `/dev/null`. `target/bench.txt` is the +4.2 MB text that `python scripts/bench_corpus.py` writes (see [rust.md](rust.md#measurements)): + +| | `what` (Rust) | `pywhat` (Python 3.10) | +|---|---|---| +| `--json --no-progress target/bench.txt` | 3.05 s wall, 3.64 s user, 366 MiB | 22.87 s wall, 21.72 s user, 632 MiB | +| `--json --no-progress fixtures/file` | 0.14 s wall | 0.36 s wall | +| `--no-progress 'THM{a}'` | 0.12 s wall | 0.30 s wall | + +Both print the same 296,955,554 bytes for `target/bench.txt` (`cmp`). diff --git a/docs/rust.md b/docs/rust.md index 6af1b59..664d67a 100644 --- a/docs/rust.md +++ b/docs/rust.md @@ -43,6 +43,7 @@ assert!(found.iter().any(|m| m.matched == "bee@skerritt.blog" && m.regex.name == | `identifier.identify(text)` | `identify(text)["Regexes"]["text"]` | | `identifier.identify_inputs(&inputs, options, key, reverse)` (feature `files`) | `identify_inputs(inputs, only_text=..., include_filenames=..., key=..., reverse=...)` | | `identifier.iter_identify(&inputs, options, found, error)` (feature `files`) | `iter_identify(...)` | +| `identifier.iter_identify_with_progress(&inputs, options, found, error, progress)` and `Progress` (feature `files`) | `iter_identify(..., progress=...)` and `Progress`: `progress` gets the file, how many of the files and of the file's regex searches are done, and a regex that is searched for. For texts over 4 KB the regexes are searched for in parallel, so it is called from rayon's threads, one call at a time | | `Identified::to_json()` | `json.dumps(identify(...))`, byte for byte | | `Filter::new(tags, exclude_tags, min_rarity, max_rarity)`, `Filter::default()`, `Filter::none()`, `.and()`, `.or()` | `Filter({...})`, `&`, `\|` | | `Processor` (trait), `default_processors()`, `processors::OpenStreetMap`, `processors::Validator` | `Processor`, `default_processors()`, `OpenStreetMapProcessor`, `ValidatorProcessor` | @@ -70,6 +71,8 @@ Rust only: | `files` | yes | `identify_inputs`, `iter_identify`: files, directories (recursive) and file signatures (`file_signatures.json`, under CC BY-SA 4.0) | | `children` | yes | the tables that `Children` read from data files: the locations of phone numbers (`phone_codes.json`) in their descriptions | | `mac-vendors` | yes | `children`, and the vendors of MAC address prefixes (1.6 MB) in the descriptions of EUI-48 matches | +| `cli` | yes | the `what` and `pywhat` commands and `pywhat::cli::main`, see [cli.md](cli.md); it needs `files` and `mac-vendors`, for the output of the Python CLI (not on wasm32) | +| `verify` | yes | `what --verify`: asks Google whether Google API keys are valid (not on wasm32) | | `wasm` | no | the WebAssembly API in `src/wasm.rs` (`identify(text, options_json)`, `tags()`, `version()`) | For wasm32, use `default-features = false` (no threads, no file system), and `features = ["wasm"]` diff --git a/scripts/parity_cli.py b/scripts/parity_cli.py new file mode 100644 index 0000000..3af72ae --- /dev/null +++ b/scripts/parity_cli.py @@ -0,0 +1,695 @@ +""" +Check that the Rust CLI prints what the Python CLI prints: the same stdout and stderr bytes +and the same exit code, for about 290 command lines: every case of tests/test_click.py (read from +it), every option on fixtures/file, the examples of the README and of --help, and corner +cases of the argument parser, of stdin and of files. For --format pretty, whose table +comfy-table draws, the text around the table and the cells of the table must be the same. +Where Python stops with a traceback, its stderr isn't compared. + + cargo build --release + python scripts/parity_cli.py [-k TEXT] [-v] [-j JOBS] + +PYWHAT_CLI is the program to test (default: target/release/what). Python runs as +`COLUMNS=10000 python -m pywhat ...` from the root of the repository, so that rich doesn't +wrap its output, and the Rust CLI with the same environment and the program name "python -m +pywhat", so that the usage lines are the same. + +Python lists the files of a directory in the order of the file system, and Rust in the order +of their names (#291, Python bug 5). So that the output for directories can be compared byte +for byte on any file system, Python runs with glob listing every directory in name order; +nothing else changes. + +KNOWN lists the command lines whose output differs on purpose, with the reason. They run too, +and are reported; they fail the check only if they don't differ. + +The text of `what --help` is src/cli/help.txt, the help of pywhat/what.py's main(). When that +changes, `python scripts/parity_cli.py --update-help` writes the new one. +""" +import argparse +import ast +import inspect +import itertools +import os +import re +import shlex +import subprocess +import sys +import tempfile +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from typing import Any, Dict, Iterator, List, NamedTuple, Optional, Tuple + +ROOT = Path(__file__).resolve().parent.parent +PROG = "python -m pywhat" + +# python -m pywhat, with glob listing every directory in name order (see above) +PYTHON_CLI = """\ +import glob +import runpy + +files_of = glob._iterdir + + +def in_name_order(*args, **kwargs): + yield from sorted(files_of(*args, **kwargs)) + + +glob._iterdir = in_name_order +runpy.run_module("pywhat", run_name="__main__", alter_sys=True) +""" + +ETH = "0x52908400098527886E0F7030069857D2E4169EE7" +F = "fixtures/file" +PNG = b"\x89PNG\r\n\x1a\nTHM{hello}\n" + + +class Case(NamedTuple): + args: Tuple[str, ...] + stdin: bytes = b"" + columns: str = "10000" + # Run in a scratch directory with these files (name, contents) + files: Optional[Tuple[Tuple[str, bytes], ...]] = None + + +def case(*args: str, stdin: bytes = b"", columns: str = "10000", files=None) -> Case: + return Case(tuple(args), stdin, columns, None if files is None else tuple(files)) + + +# The cases of tests/test_click.py + + +class Unsupported(Exception): + """An expression of tests/test_click.py that value() can't work out.""" + + +def value(node: Optional[ast.AST], names: Dict[str, Any]) -> Any: + """ + The value of a simple expression of tests/test_click.py: literals, the names that + parametrize or an assignment of a literal binds, lists and tuples with *starred items, + dicts, and the sum of two lists. Raises Unsupported for anything else. + """ + if isinstance(node, ast.Constant): + return node.value + if isinstance(node, ast.Name) and node.id in names: + return names[node.id] + if isinstance(node, (ast.List, ast.Tuple)): + items: List[Any] = [] + for item in node.elts: + if isinstance(item, ast.Starred): + items.extend(value(item.value, names)) + else: + items.append(value(item, names)) + return items if isinstance(node, ast.List) else tuple(items) + if isinstance(node, ast.Dict): + return { + value(key, names): value(item, names) + for key, item in zip(node.keys, node.values) + } + if isinstance(node, ast.BinOp) and isinstance(node.op, ast.Add): + return value(node.left, names) + value(node.right, names) + raise Unsupported(ast.dump(node) if node is not None else "None") + + +def parametrizations(function: ast.FunctionDef) -> Optional[List[Dict[str, Any]]]: + """ + The parameters of every run of a test function (one run without parameters), or None + if it is skipped. + """ + runs: List[Dict[str, Any]] = [{}] + for decorator in function.decorator_list: + call = decorator if isinstance(decorator, ast.Call) else None + name = getattr(call.func if call else decorator, "attr", None) + if name == "skip": + return None + if name != "parametrize" or call is None: + continue + names = [n.strip() for n in value(call.args[0], {}).split(",")] + sets = [] + for values in value(call.args[1], {}): + values = values if len(names) > 1 else [values] + sets.append(dict(zip(names, values))) + runs = [ + {**run, **parameters} for run, parameters in itertools.product(runs, sets) + ] + return runs + + +def invocations(function: ast.FunctionDef) -> Iterator[ast.Call]: + """The calls of CliRunner.invoke() and run_cli_command() in a test function.""" + for node in ast.walk(function): + if isinstance(node, ast.Call): + name = getattr(node.func, "attr", getattr(node.func, "id", None)) + if name in ("invoke", "run_cli_command"): + yield node + + +def click_case(call: ast.Call, names: Dict[str, Any]) -> Case: + """The command line of a call, with its input and its COLUMNS.""" + is_invoke = getattr(call.func, "attr", None) == "invoke" + args = value(call.args[1] if is_invoke else call.args[0], names) + if isinstance(args, str): + args = shlex.split(args) + keywords = {keyword.arg: keyword.value for keyword in call.keywords} + stdin = value(keywords["input"], names) if "input" in keywords else b"" + env = value(keywords["env"], names) if "env" in keywords else {} + if set(env) - {"COLUMNS"} or not isinstance(stdin, (str, bytes)): + raise Unsupported("environment or input") + if isinstance(stdin, str): + stdin = stdin.encode() + return case(*args, stdin=stdin, columns=env.get("COLUMNS", "10000")) + + +def click_cases() -> List[Case]: + """Every command line that tests/test_click.py runs, as far as value() works it out.""" + tree = ast.parse((ROOT / "tests" / "test_click.py").read_text(encoding="utf-8")) + found: List[Case] = [] + for function in tree.body: + if not isinstance(function, ast.FunctionDef): + continue + if not function.name.startswith("test_"): + continue + runs = parametrizations(function) + if runs is None: + continue + # Names that the test binds to a literal, such as flag = "THM{...}" + local: Dict[str, Any] = {} + for node in ast.walk(function): + if isinstance(node, ast.Assign) and len(node.targets) == 1: + target = node.targets[0] + if isinstance(target, ast.Name): + try: + local[target.id] = value(node.value, {}) + except Unsupported: + pass + for call in invocations(function): + for parameters in runs: + try: + found.append(click_case(call, {**local, **parameters})) + except Unsupported: + pass # e.g. a file in tmp_path, or stdin that is a terminal + return list(dict.fromkeys(found)) + + +# Every option on fixtures/file +OPTIONS = [ + ["-r", "0:", F], + ["--rarity", "0.5:", F], + ["-r", ":0.5", F], + ["-r", "0.2:0.8", F], + ["-i", "Bug Bounty", F], + ["-e", "Credentials,URL", F], + ["-o", F], + ["-k", "name", F], + ["-k", "rarity", F], + ["-k", "matched", F], + ["-k", "likely", F], + ["-k", "NONE", F], + ["-k", "rarity", "--reverse", F], + ["-k", "matched", "--reverse", "--json", F], + ["--top", "10", F], + ["--top", "5%", F], + ["--top", "1.5 %", F], + ["--top", "all", F], + ["--top", "3", "-k", "name", F], + ["--top", "3", "--json", F], + ["-br", "0.6:", F], + ["-bi", "URL", F], + ["-be", "URL", F], + ["-db", F], + ["-dp", F], + ["-db", "-dp", "--json", F], + ["--map", "osm", F], + ["--map", "GOOGLE", F], + ["--json", "--map", "osm", "-dp", F], + ["--json", F], + ["-pt", F], + ["-if", F], + ["-if", "--json", F], + ["--format", "pretty", F], + ["--format", "pretty", "-pt", F], + ["--format", "pretty", "--rarity", "0:", F], + ["--format", "%m\t%n|%r|%t|%l", F], + ["--format", "%d", F], + ["--format", "%e", F], + ["--format", "\\\\%m\\%n\\x", F], + ["--stream", F], + ["--stream", "-pt", F], + ["--stream", "--json", F], + ["--stream", "--format", "%m %n", F], + ["--no-progress", "--json", F], + ["--verify", "THM{no key}"], + ["-o", "--format", "pretty", "fixtures"], + ["--format", "pretty", "fixtures"], +] + +# The examples of the README and of --help +README = [ + [ETH], + ["dQw4w9WgXcQ"], + ["--rarity", "0:", "dQw4w9WgXcQ"], + ["https://youtu.be/dQw4w9WgXcQ"], + ["youtube.com/watch?v=dQw4w9WgXcQ&t=42s"], + ["--rarity", "0.2:0.8", "--include", "Identifiers,URL", "https://skerritt.blog"], + [ + "--include", + "Cryptocurrency Wallet", + "--exclude", + "Ripple Wallet", + "1KFHE7w8BhaENAswwryaoccDb6qcT6DbYY", + ], + ["--include", "BTC Wallet,ETH Wallet", F], + ["-k", "rarity", "--reverse", ETH], + ["--top", "5%", ETH], + ["--top", "1", ETH], + ["-k", "likely", ETH], + ["--json", ETH], + ["--rarity", "0:", "--include", "UNIX Timestamp", "1637093119"], + [ + "--rarity", + "0:", + "--include", + "UNIX Timestamp", + "--format", + "%d", + "1637093119.558717", + ], + ["3F3F3F3F3F3F3F3F3F3F3F3F3F3F3F3F3F"], + ["--disable-processing", "3F3F3F3F3F3F3F3F3F3F3F3F3F3F3F3F3F"], + ["--map", "osm", "52.6169586, -1.9779857"], + ["--format", "pretty", "--map", "osm", "52.6169586, -1.9779857"], + ["--include", "Bug Bounty", F], + ["HTB{this is a flag}"], + ["--", "52.6169586, -1.9779857"], + ["--rarity", "0.6:", "myEmail@host.org"], + [ + "--rarity", + "0:", + "--include", + "credentials", + "--exclude", + "aws", + "James:SecretPassword", + ], + ["-br", "0.6:", "-be", "URL", "123myEmail@host.org456"], + ["--format", "%m - %n", "google.com htb{flag}"], + ["--stream", "fixtures"], + ["--format", "pretty", ETH], +] + +# The argument parser, errors, help and the format string +PARSER = [ + ["-1606586780"], + ["-h"], + ["--nope", "THM{x}"], + ["-ox", "THM{x}"], + ["-kname", "-db", F], + ["--top=1", ETH], + ["--top", "=1", ETH], + ["-r=0.5:", ETH], + ["-br=0.5:", ETH], + ["--json=1", ETH], + ["--rarity"], + ["-k"], + ["-pt=", "x"], + ["--map", "OSM", "52.6169586, -1.9779857"], + ["--map", "o\u017fm", "52.6169586, -1.9779857"], + ["--map", "bing", "x"], + ["--map", "it's", "x"], + ["--", "-db", "--", "THM{x}"], + ["-"], + ["-r", "--", "THM{x}"], + ["--tags", "--version"], + ["--names", "--tags"], + ["--map", "x", "--tags"], + ["--tags", "--map", "x"], + ["--help", "--map", "x"], + ["--map", "x", "--help"], + ["-db", "--help"], + ["--help"], + ["--version"], + ["-v", "THM{x}"], + ["--rarity", "0.5", ETH], + ["--rarity", "a:b", ETH], + ["--rarity", "0.1:0.5:0.9", ETH], + ["--rarity", ":", ETH], + ["--rarity", " : ", ETH], + ["--rarity", "1_0:", ETH], + ["--rarity", "\u0660.\u0665:", ETH], + ["--rarity", "nan:", ETH], + ["--rarity", "-inf:inf", ETH], + ["-br", "x:", ETH], + ["-db", "-br", "x:", ETH], + ["--key", "nope", ETH], + ["--key", "rar\u0131ty", ETH], + ["--top", "0", ETH], + ["--top", "150%", ETH], + ["--top", "abc", ETH], + ["--top", " 2 ", ETH], + ["--top", "99999999999999999999999999999", ETH], + ["--top", "\x1c 5\x1f%\u3000", ETH], + ["--top", "0" * 40 + "1", ETH], + ["--include", "", ETH], + ["--include", "btc wallet", "--exclude", "ETH WALLET", ETH], + ["--stream", "--format", "pretty", "--json", "THM{x}"], + ["--stream", "-k", "name", "--top", "1", "THM{x}"], + ["-db"], + ["--format", "None", "THM{x}"], + ["--format", "", "THM{x}"], + ["--format", " ", "THM{x}"], + ["--format", "%m\\", "THM{x}"], + # Values after "=", which may be empty or contain "=", unknown options with a value, + # and the callbacks of options in the order of their first appearance + ["--format=%m=%n", "THM{x}"], + ["--format=", "THM{x}"], + ["--rarity=", "THM{x}"], + ["--nope=1", "THM{x}"], + ["--tags", "--names", "--tags"], + ["--names", "--tags", "--names"], + # Errors of click's parser have no usage line, invalid values have one + ["-e"], + ["--json=", ETH], + ["-db=", "THM{x}"], + ["--help", "--json=1"], + ["--map"], + # rich markup in the format string and in the data is rendered, as in Python, and + # Python stops at a closing tag that closes nothing (rich's MarkupError) + ["-db", "--format", "[/b]", "THM{x}"], + ["-db", "--format", "[b]%m[/i]", "THM{x}", ETH], + ["-db", "--format", "x[/]y %m", "THM{x}"], + ["--stream", "-db", "--format", "[/b]%m", "THM{x}", ETH], + ["-db", "--format", "[b]%m[/bold] [red bold]%n[/bold red]", "THM{y}"], + ["-db", "--format", "[/b]", "nothing"], + ["-db", "--format", "[@click]%m[/@click]", "THM{x}"], + ["-db", "--format", "[b]%m[/b] [#ff0000]%n[/#ff0000]", "THM{y}"], + ["-db", "--format", "\\[b]%m", "THM{y}"], + ["-db", "--include", "Access-Control-Allow-Header", "Access-Control-Allow: *"], +] + +# The matched text as rich markup and emoji codes (#291, Python bug 4) +BUG_4 = ( + "#291 Python bug 4: rich takes the matched text for markup and emoji codes " + "(Rust prints it as it is)" +) +MARKUP = [ + ["-db", "thm{[bold]x[/bold]}"], + ["-db", "THM{:smile:}"], + ["-db", "--format", "%m", "THM{[i]x[/i]}"], + ["-db", "--format", "pretty", "THM{[b]x[/b]}"], + ["-db", "thm{[/x]}"], +] + +OWN: List[Case] = [case(*args) for args in OPTIONS + README + PARSER + MARKUP] +OWN += [ + case("-db", "--format", ":smile: %m", "THM{y}"), + # A regex whose Exploit is null: the table has an Exploit column (in parts, so that + # secret scanners don't take it for a key) + case("-db", "--format", "pretty", "linkedin_client_id=" + "cp334pjr0lxwmw"), + # stdin, when there are no inputs, or only empty ones; stripped like str.strip() + case("-db", stdin=b"\x1c THM{x}\x1f"), + case(stdin=b""), + case("-db", "", stdin=b"THM{stdin}\n"), + case("-db", "", "", stdin=b"THM{stdin}\n"), + case("-db", "", "THM{a}", stdin=b"THM{stdin}\n"), + case("-db", stdin=b"fixtures/file\n"), + case("--json", stdin="\ufeffTHM{bom}\r\n".encode("utf-16")), + case("--json", stdin=b"\x00\x01" + "THM{wide}".encode("utf-16-le") + b"\xff\xfe"), + case("-db", stdin=b"caf\xe9 THM{latin}"), + # The help and the usage of errors at other widths + case("--help", columns="40"), + case("--help", columns="60"), + case("--help", columns="79"), + case("--help", columns="0"), + case("--help", columns="abc"), + case("--rarity", columns="40"), + # Files in a scratch directory + case( + "-db", + "image.png", + "page.html", + files=[("image.png", PNG), ("page.html", b"https://google.com\n")], + ), + case( + "-db", + "--json", + "image.png", + "page.html", + files=[("image.png", PNG), ("page.html", b"https://google.com\n")], + ), + case("image.png", files=[("image.png", PNG)]), + case("image.png", files=[("image.png", b"\x89PNG\r\n\x1a\n")]), + case("--stream", "image.png", files=[("image.png", PNG)]), + case("--stream", "image.png", files=[("image.png", b"\x89PNG\r\n\x1a\n")]), + case("--stream", "--format", "json", "image.png", files=[("image.png", PNG)]), + case("--format", "pretty", "image.png", files=[("image.png", PNG)]), + case( + "--format", "pretty", "image.png", files=[("image.png", b"\x89PNG\r\n\x1a\n")] + ), + case("-db", "--json", "89504E470D0A1A0A", files=[("89504E470D0A1A0A", b"THM{x}")]), + case( + "--json", + "utf16.txt", + files=[("utf16.txt", "\ufeffTHM{x} bee@skerritt.blog\r\n".encode("utf-16"))], + ), + case("-if", "THM{name}.txt", files=[("THM{name}.txt", b"nothing")]), + case("tabs.txt", "-pt", files=[("tabs.txt", b"x")]), + case( + "--format", + "%m", + "dir", + files=[("dir/a\tb", b"THM{tab}"), ("dir/.hidden", b"THM{hidden}")], + ), + # A tab in a file name, in the File: line and in the table's File column + case("-db", "tab\tname.txt", "THM{x}", files=[("tab\tname.txt", b"THM{t}")]), + case( + "--format", + "pretty", + "tab\tname.txt", + "THM{x}", + files=[("tab\tname.txt", b"THM{t}")], + ), + # Written out of name order, which is the order that some file systems list them in + case( + "--format", + "%m", + "dir", + files=[ + ("dir/b.txt", b"THM{b}"), + ("dir/c/a.txt", b"THM{c}"), + ("dir/a", b"THM{a}"), + ], + ), + case( + "--json", + "dir", + "THM{text}", + files=[("dir/b.txt", b"THM{b}"), ("dir/a.txt", b"THM{a}")], + ), +] + +# Command lines whose output differs on purpose +KNOWN: Dict[Tuple[str, ...], str] = { + (): ( + "python -m pywhat without arguments prints the help (pywhat/__main__.py); the " + "what and pywhat commands read stdin, in Python and in Rust" + ), + ("--version",): ( + "-v prints the version of the crate (6.0.0-alpha.1); Python's __version__ is " + "5.0.0" + ), + ("-v", "THM{x}"): "the version (see --version)", + ("-db", "--format", ":smile: %m", "THM{y}"): ( + "emoji codes in a --format string: rich prints :smile: as an emoji, Rust as it is" + ), +} +KNOWN[("--format", "pretty", "tab\tname.txt", "THM{x}")] = ( + "a tab in the table: rich counts a tab as 0 columns wide when it sizes the column, " + "then wraps the cell at the expanded tab; comfy-table shows the expanded tab" +) +for args in MARKUP: + KNOWN[tuple(args)] = BUG_4 +KNOWN[("-db", "thm{[/x]}")] = BUG_4 + "; Python crashes with a MarkupError" + + +def run( + command: List[str], case: Case, cwd: Path, executable: Optional[str] = None +) -> Tuple[int, bytes, bytes]: + env = dict(os.environ, COLUMNS=case.columns, PYTHONIOENCODING="utf-8") + # python -m pywhat from a scratch directory too + env["PYTHONPATH"] = os.pathsep.join( + filter(None, [str(ROOT), env.get("PYTHONPATH")]) + ) + env.pop("NO_COLOR", None) + result = subprocess.run( + command, + executable=executable, + input=case.stdin, + capture_output=True, + cwd=cwd, + env=env, + timeout=600, + ) + return result.returncode, result.stdout, result.stderr + + +def table_parts(output: str): + """The text around the tables of the output, and the cells of the tables.""" + text: List[str] = [] + tables: List[List[List[str]]] = [] + row: List[List[str]] = [] + in_table = False + for line in output.split("\n"): + if line.startswith("┏"): + in_table, row = True, [] + tables.append([]) + text.append("") + elif in_table and line[:1] in ("┡", "├", "└"): + # The end of a row: its cells, the lines of each cell joined + if row: + cells = [] + for index in range(len(row[0])): + lines = [ + parts[index] if index < len(parts) else "" for parts in row + ] + while lines and not lines[-1]: + lines.pop() + cells.append("\n".join(lines)) + tables[-1].append(cells) + row = [] + in_table = line[:1] != "└" + elif in_table: + row.append([part.strip() for part in re.split("[┃│]", line)[1:-1]]) + else: + text.append(line) + return text, tables + + +def first_difference(a: str, b: str) -> str: + index = next( + (i for i, (x, y) in enumerate(zip(a, b)) if x != y), min(len(a), len(b)) + ) + start = max(0, index - 60) + return ( + f"at {index}: Python {a[start:index + 60]!r}\n" + f" Rust {b[start:index + 60]!r}" + ) + + +def compare(case: Case, what: str, verbose: bool) -> Tuple[str, str]: + """("same" | "same cells" | "different", details) for a case.""" + with tempfile.TemporaryDirectory() as scratch: + cwd = ROOT + if case.files is not None: + cwd = Path(scratch) + for name, data in case.files: + path = cwd / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + python = run([sys.executable, "-c", PYTHON_CLI, *case.args], case, cwd) + rust = run([PROG, *case.args], case, cwd, executable=what) + traceback = b"Traceback (most recent call last)" in python[2] + same_stderr = traceback or python[2] == rust[2] + if python[:2] == rust[:2] and same_stderr: + return "same", "" + details = [] + if python[0] != rust[0]: + details.append(f"exit code: Python {python[0]}, Rust {rust[0]}") + if not same_stderr: + details.append( + "stderr: " + + first_difference( + python[2].decode(errors="replace"), rust[2].decode(errors="replace") + ) + ) + if python[1] != rust[1]: + py_text = python[1].decode(errors="replace") + rs_text = rust[1].decode(errors="replace") + pretty = "pretty" in [arg.strip() for arg in case.args] + if "--format" in case.args and pretty and python[0] == rust[0]: + py_parts, rs_parts = table_parts(py_text), table_parts(rs_text) + if py_parts == rs_parts and same_stderr: + return "same cells", "" + details.append("table: " + first_difference(repr(py_parts), repr(rs_parts))) + else: + details.append("stdout: " + first_difference(py_text, rs_text)) + if verbose: + details.append( + f"Python stderr: {python[2][-300:]!r}\nRust stderr: {rust[2][-300:]!r}" + ) + return "different", "\n ".join(details) + + +HELP_FILE = ROOT / "src" / "cli" / "help.txt" + + +def python_help() -> str: + """The help of pywhat/what.py's main(), as click shows it (`inspect.cleandoc()`).""" + sys.path.insert(0, str(ROOT)) + from pywhat.what import main as what_main + + return inspect.cleandoc(what_main.help or "").partition("\f")[0] + + +def program() -> str: + """The program to test: $PYWHAT_CLI, as an absolute path (cases run elsewhere too).""" + default = ROOT / "target" / "release" / ("what.exe" if os.name == "nt" else "what") + return os.path.abspath(os.environ.get("PYWHAT_CLI") or default) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("-k", help="only the cases whose arguments contain this text") + parser.add_argument("-v", "--verbose", action="store_true") + parser.add_argument("-j", "--jobs", type=int, default=4) + parser.add_argument( + "--update-help", + action="store_true", + help="write the help of pywhat/what.py to src/cli/help.txt, then compare", + ) + args = parser.parse_args() + if args.update_help: + HELP_FILE.write_text(python_help(), encoding="utf-8") + help_is_current = HELP_FILE.read_text(encoding="utf-8") == python_help() + if not help_is_current: + print( + "DIFFERENT src/cli/help.txt is not the help of pywhat/what.py: run " + "python scripts/parity_cli.py --update-help, and build again" + ) + what = program() + from_tests = click_cases() + cases = list(dict.fromkeys(from_tests + OWN)) + if args.k: + cases = [c for c in cases if any(args.k in arg for arg in c.args)] + with ThreadPoolExecutor(args.jobs) as pool: + results = list(pool.map(lambda c: compare(c, what, args.verbose), cases)) + counts: Dict[str, int] = {} + failures = 0 + for case_, (verdict, details) in zip(cases, results): + known = KNOWN.get(case_.args) + if verdict == "different" and known: + verdict = "known difference" + elif known: + # The difference is on purpose: Rust doesn't behave like Python here + verdict = "different" + details = f"expected the known difference: {known}" + counts[verdict] = counts.get(verdict, 0) + 1 + if verdict == "different": + failures += 1 + print( + f"DIFFERENT {list(case_.args)} (COLUMNS={case_.columns})\n {details}" + ) + elif verdict == "known difference": + print(f"KNOWN {list(case_.args)}: {known}") + elif args.verbose: + print(f"{verdict.upper()} {list(case_.args)}") + summary = ", ".join( + f"{count} {verdict}" for verdict, count in sorted(counts.items()) + ) + print( + f"\n{len(cases)} command lines ({len(from_tests)} from tests/test_click.py): " + f"{summary}" + ) + sys.exit(1 if failures or not help_is_current else 0) + + +if __name__ == "__main__": + main() diff --git a/src/bin/pywhat.rs b/src/bin/pywhat.rs new file mode 100644 index 0000000..37a02cc --- /dev/null +++ b/src/bin/pywhat.rs @@ -0,0 +1,5 @@ +//! The `pywhat` command, the same as `what` (see `pywhat --help`). + +fn main() { + std::process::exit(pywhat::cli::main(std::env::args_os())) +} diff --git a/src/bin/what.rs b/src/bin/what.rs new file mode 100644 index 0000000..7f7c966 --- /dev/null +++ b/src/bin/what.rs @@ -0,0 +1,5 @@ +//! The `what` command: identify anything (see `what --help`). + +fn main() { + std::process::exit(pywhat::cli::main(std::env::args_os())) +} diff --git a/src/cli/args.rs b/src/cli/args.rs new file mode 100644 index 0000000..543db48 --- /dev/null +++ b/src/cli/args.rs @@ -0,0 +1,503 @@ +//! The command line, parsed the way click 8.1 parses `pywhat/what.py`'s options: long +//! options with `=` or the next argument as their value, multi-letter "short" options +//! such as `-br` (click treats them as long options), clusters of one-letter options +//! (`-ok name`), `--` before inputs that start with `-`, and unknown options taken as +//! inputs (`ignore_unknown_options`: `what -1606586780` searches the text +//! `-1606586780`). + +use super::python::repr; +use std::collections::HashMap; + +/// An option of `what.py`, in the order in which they are declared there. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub(crate) enum Param { + Tags, + Names, + Rarity, + Include, + Exclude, + OnlyText, + Key, + Reverse, + Top, + BoundarylessRarity, + BoundarylessInclude, + BoundarylessExclude, + DisableBoundaryless, + DisableProcessing, + Verify, + Map, + Json, + Interactive, + Version, + IncludeFilenames, + Format, + PrintTags, + Stream, + NoProgress, + Help, +} + +/// How an option is written and explained in `--help`. +pub(crate) struct OptionSpec { + pub(crate) param: Param, + /// The option strings, e.g. `-t` and `--tags`. + pub(crate) opts: &'static [&'static str], + pub(crate) takes_value: bool, + /// What `--help` shows after the option strings of an option that takes a value. + pub(crate) metavar: &'static str, + pub(crate) help: &'static str, +} + +const fn flag(param: Param, opts: &'static [&'static str], help: &'static str) -> OptionSpec { + OptionSpec { + param, + opts, + takes_value: false, + metavar: "", + help, + } +} + +const fn value( + param: Param, + opts: &'static [&'static str], + metavar: &'static str, + help: &'static str, +) -> OptionSpec { + OptionSpec { + param, + opts, + takes_value: true, + metavar, + help, + } +} + +/// The options, in the order of `what.py` (and of `--help`), with click's `--help` last. +pub(crate) const OPTIONS: &[OptionSpec] = &[ + flag( + Param::Tags, + &["-t", "--tags"], + "Show available tags and exit.", + ), + flag( + Param::Names, + &["--names"], + "Show the names of the regexes, with their alternative names, and exit.", + ), + value( + Param::Rarity, + &["-r", "--rarity"], + "TEXT", + "Filter by rarity. Rarity is how unlikely something is to be a false-positive. The \ + higher the number, the more unlikely. This is in the range of 0:1. To filter only \ + items past 0.5, use 0.5: with the colon on the end. Default 0.1:1", + ), + value( + Param::Include, + &["-i", "--include"], + "TEXT", + "Only show matches with these tags or names.", + ), + value( + Param::Exclude, + &["-e", "--exclude"], + "TEXT", + "Exclude matches with these tags or names.", + ), + flag( + Param::OnlyText, + &["-o", "--only-text"], + "Do not scan files or folders.", + ), + value( + Param::Key, + &["-k", "--key"], + "TEXT", + "Sort by the specified key.", + ), + flag(Param::Reverse, &["--reverse"], "Sort in reverse order."), + value( + Param::Top, + &["--top"], + "N", + "Only show the N most likely matches, or N% of them, e.g. --top 10 or --top 5%.", + ), + value( + Param::BoundarylessRarity, + &["-br", "--boundaryless-rarity"], + "TEXT", + "Same as --rarity but for boundaryless mode (toggles what regexes will not have \ + boundaries).", + ), + value( + Param::BoundarylessInclude, + &["-bi", "--boundaryless-include"], + "TEXT", + "Same as --include but for boundaryless mode.", + ), + value( + Param::BoundarylessExclude, + &["-be", "--boundaryless-exclude"], + "TEXT", + "Same as --exclude but for boundaryless mode.", + ), + flag( + Param::DisableBoundaryless, + &["-db", "--disable-boundaryless"], + "Disable boundaryless mode.", + ), + flag( + Param::DisableProcessing, + &["-dp", "--disable-processing"], + "Disable the processing of matches, e.g. adding dates to Unix timestamps or \ + checking the checksums of Bitcoin addresses.", + ), + flag( + Param::Verify, + &["--verify"], + "Ask the services if the keys found are valid, e.g. Google API keys. This sends \ + the keys to the services.", + ), + value( + Param::Map, + &["--map"], + "[google|osm]", + "The map that coordinates link to: Google Maps or OpenStreetMap (osm). [default: \ + google]", + ), + flag(Param::Json, &["--json"], "Return results in json format."), + flag( + Param::Interactive, + &["--interactive"], + "Load the input into memory and search through the matches interactively.", + ), + flag( + Param::Version, + &["-v", "--version"], + "Display the version of pywhat.", + ), + flag( + Param::IncludeFilenames, + &["-if", "--include-filenames"], + "Search filenames for possible matches.", + ), + value( + Param::Format, + &["--format"], + "TEXT", + "Format output according to specified rules.", + ), + flag( + Param::PrintTags, + &["-pt", "--print-tags"], + "Add flags to output", + ), + flag( + Param::Stream, + &["--stream"], + "Print every match as soon as it is found, not once the search is complete.", + ), + flag( + Param::NoProgress, + &["--no-progress"], + "Do not show progress bars while searching.", + ), + flag(Param::Help, &["--help"], "Show this message and exit."), +]; + +/// The choices of `--map` (`click.Choice(["google", "osm"], case_sensitive=False)`). +pub(crate) const MAP_CHOICES: &[&str] = &["google", "osm"]; + +/// An error of the parser: click prints `Error: ` and exits with 2. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct UsageError(pub(crate) String); + +/// What the command line holds. +#[derive(Debug, Default, Clone)] +pub(crate) struct Parsed { + /// The value of every option that was given (the last one if it was given several + /// times); flags have an empty value. + values: HashMap, + /// The options in the order they were first given, for the order of the callbacks. + order: Vec, + /// The inputs: files, directories or text. + pub(crate) inputs: Vec, +} + +impl Parsed { + pub(crate) fn flag(&self, param: Param) -> bool { + self.values.contains_key(¶m) + } + + pub(crate) fn value(&self, param: Param) -> Option<&str> { + self.values.get(¶m).map(String::as_str) + } + + /// The options that were given, in the order of their first appearance. + pub(crate) fn invoked(&self) -> &[Param] { + &self.order + } + + fn set(&mut self, param: Param, value: String) { + self.values.insert(param, value); + if !self.order.contains(¶m) { + self.order.push(param); + } + } +} + +fn spec(opt: &str) -> Option<&'static OptionSpec> { + OPTIONS.iter().find(|spec| spec.opts.contains(&opt)) +} + +/// click's split: an option string with one letter after one `-` is a short option, any +/// other is a long one (`-br`, `--rarity`). +fn is_short(opt: &str) -> bool { + let mut chars = opt.chars(); + chars.next() == Some('-') && chars.next().is_some_and(|c| c != '-') && chars.next().is_none() +} + +fn long_option(opt: &str) -> Option<&'static OptionSpec> { + spec(opt).filter(|_| !is_short(opt)) +} + +fn short_option(opt: &str) -> Option<&'static OptionSpec> { + spec(opt).filter(|_| is_short(opt)) +} + +/// Parses the arguments after the program name, like click's `OptionParser.parse_args`. +pub(crate) fn parse(args: &[String]) -> Result { + let mut parsed = Parsed::default(); + // rargs: the arguments still to read, largs: the inputs found so far + let mut rargs: std::collections::VecDeque = args.iter().cloned().collect(); + let mut largs: Vec = Vec::new(); + while let Some(arg) = rargs.pop_front() { + if arg == "--" { + break; + } + if arg.starts_with('-') && arg.chars().count() > 1 { + process_opts(&arg, &mut rargs, &mut largs, &mut parsed)?; + } else { + largs.push(arg); + } + } + largs.extend(rargs); + parsed.inputs = largs; + Ok(parsed) +} + +/// The value of an option that takes one: the next argument, whatever it is. +fn take_value( + opt: &str, + rargs: &mut std::collections::VecDeque, +) -> Result { + rargs + .pop_front() + .ok_or_else(|| UsageError(format!("Option {} requires an argument.", repr(opt)))) +} + +/// `_process_opts()`: a long option (possibly with `=value`), else a cluster of short +/// options, else (it starts with `--`) an input. +fn process_opts( + arg: &str, + rargs: &mut std::collections::VecDeque, + largs: &mut Vec, + parsed: &mut Parsed, +) -> Result<(), UsageError> { + let (long_opt, explicit_value) = match arg.split_once('=') { + Some((opt, value)) => (opt, Some(value)), + None => (arg, None), + }; + if let Some(spec) = long_option(long_opt) { + let value = if spec.takes_value { + if let Some(value) = explicit_value { + rargs.push_front(value.to_string()); + } + take_value(long_opt, rargs)? + } else if explicit_value.is_some() { + return Err(UsageError(format!( + "Option {} does not take a value.", + repr(long_opt) + ))); + } else { + String::new() + }; + parsed.set(spec.param, value); + return Ok(()); + } + if arg.starts_with("--") { + // An unknown long option is an input + largs.push(arg.to_string()); + return Ok(()); + } + match_short_opt(arg, rargs, largs, parsed) +} + +/// `_match_short_opt()`: every letter is a short option, and the rest of the argument is +/// the value of the first one that takes a value. Unknown letters are put together into +/// an input, with the `-`. +fn match_short_opt( + arg: &str, + rargs: &mut std::collections::VecDeque, + largs: &mut Vec, + parsed: &mut Parsed, +) -> Result<(), UsageError> { + let mut unknown = String::new(); + for (index, letter) in arg.char_indices().skip(1) { + let opt = format!("-{letter}"); + let Some(spec) = short_option(&opt) else { + unknown.push(letter); + continue; + }; + if spec.takes_value { + let rest = &arg[index + letter.len_utf8()..]; + if !rest.is_empty() { + rargs.push_front(rest.to_string()); + } + let value = take_value(&opt, rargs)?; + parsed.set(spec.param, value); + break; + } + parsed.set(spec.param, String::new()); + } + if !unknown.is_empty() { + largs.push(format!("-{unknown}")); + } + Ok(()) +} + +/// The value of `--map`, like click's `Choice.convert()` without case sensitivity: the +/// choice as it is declared, or the error message. +pub(crate) fn map_choice(value: &str) -> Result<&'static str, UsageError> { + let folded = super::python::casefold_ascii(value); + MAP_CHOICES + .iter() + .find(|choice| folded.as_deref() == Some(**choice)) + .copied() + .ok_or_else(|| { + let choices: Vec = MAP_CHOICES.iter().map(|c| repr(c)).collect(); + UsageError(format!( + "Invalid value for '--map': {} is not one of {}.", + repr(value), + choices.join(", ") + )) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn parse_args(args: &[&str]) -> Result { + parse(&args.iter().map(|a| a.to_string()).collect::>()) + } + + #[test] + fn long_short_and_clusters() { + let parsed = parse_args(&["-db", "--rarity", "0:", "-ok", "name", "x"]).unwrap(); + assert!(parsed.flag(Param::DisableBoundaryless)); + assert!(parsed.flag(Param::OnlyText)); + assert_eq!(parsed.value(Param::Rarity), Some("0:")); + assert_eq!(parsed.value(Param::Key), Some("name")); + assert_eq!(parsed.inputs, ["x"]); + assert_eq!( + parsed.invoked(), + [ + Param::DisableBoundaryless, + Param::Rarity, + Param::OnlyText, + Param::Key + ] + ); + } + + #[test] + fn attached_values() { + let parsed = parse_args(&["-kname", "--top=5%", "-br=0.5:", "-r=0.5:"]).unwrap(); + assert_eq!(parsed.value(Param::Key), Some("name")); + assert_eq!(parsed.value(Param::Top), Some("5%")); + assert_eq!(parsed.value(Param::BoundarylessRarity), Some("0.5:")); + // -r is a short option: the rest of the argument is its value, with the "=" + assert_eq!(parsed.value(Param::Rarity), Some("=0.5:")); + } + + #[test] + fn unknown_options_are_inputs() { + let parsed = parse_args(&["-1606586780", "--nope", "-h", "-o=1", "-tx", "-"]).unwrap(); + assert_eq!( + parsed.inputs, + ["-1606586780", "--nope", "-h", "-=1", "-x", "-"] + ); + assert!(parsed.flag(Param::OnlyText)); + assert!(parsed.flag(Param::Tags)); + } + + #[test] + fn explicit_values() { + // The value is everything after the first "=", and may be empty + let parsed = parse_args(&["--format=%m=%n", "--rarity=", "x"]).unwrap(); + assert_eq!(parsed.value(Param::Format), Some("%m=%n")); + assert_eq!(parsed.value(Param::Rarity), Some("")); + assert_eq!(parsed.inputs, ["x"]); + // An unknown long option is an input, with its "=value" + let parsed = parse_args(&["--nope=1", "--nope=a=b"]).unwrap(); + assert_eq!(parsed.inputs, ["--nope=1", "--nope=a=b"]); + } + + #[test] + fn callbacks_in_the_order_of_first_appearance() { + // click's iter_params_for_processing(): the index of the first appearance + let parsed = parse_args(&["--tags", "--names", "--tags"]).unwrap(); + assert_eq!(parsed.invoked(), [Param::Tags, Param::Names]); + let parsed = parse_args(&["--names", "-t", "--names", "-v"]).unwrap(); + assert_eq!( + parsed.invoked(), + [Param::Names, Param::Tags, Param::Version] + ); + // The last value counts + let parsed = parse_args(&["-k", "name", "-k", "rarity"]).unwrap(); + assert_eq!(parsed.value(Param::Key), Some("rarity")); + } + + #[test] + fn double_dash() { + let parsed = parse_args(&["-db", "--", "-db", "--", "x"]).unwrap(); + assert_eq!(parsed.inputs, ["-db", "--", "x"]); + // A value can be anything, "--" too + let parsed = parse_args(&["-r", "--", "x"]).unwrap(); + assert_eq!(parsed.value(Param::Rarity), Some("--")); + assert_eq!(parsed.inputs, ["x"]); + } + + #[test] + fn errors() { + assert_eq!( + parse_args(&["--rarity"]).unwrap_err().0, + "Option '--rarity' requires an argument." + ); + assert_eq!( + parse_args(&["-k"]).unwrap_err().0, + "Option '-k' requires an argument." + ); + assert_eq!( + parse_args(&["--json=1"]).unwrap_err().0, + "Option '--json' does not take a value." + ); + assert_eq!( + parse_args(&["-pt="]).unwrap_err().0, + "Option '-pt' does not take a value." + ); + } + + #[test] + fn map_choices() { + assert_eq!(map_choice("OSM"), Ok("osm")); + assert_eq!(map_choice("oſm"), Ok("osm")); + assert_eq!( + map_choice("bing").unwrap_err().0, + "Invalid value for '--map': 'bing' is not one of 'google', 'osm'." + ); + } +} diff --git a/src/cli/help.rs b/src/cli/help.rs new file mode 100644 index 0000000..422767d --- /dev/null +++ b/src/cli/help.rs @@ -0,0 +1,448 @@ +//! `--help` and the usage line of errors, formatted like click 8.1 formats them: its +//! `HelpFormatter` (`click/formatting.py`) with Python's `textwrap.TextWrapper`, so the text +//! wraps exactly where Python's does. + +use super::args::OPTIONS; +use super::python; +use std::sync::OnceLock; + +/// The help text of `pywhat/what.py`'s `main()`, as click cleans it (`inspect.cleandoc`). +const HELP: &str = include_str!("help.txt"); + +/// What `--help` shows after the program name in the usage line. +const USAGE_ARGS: &str = "[OPTIONS] [TEXT_INPUT]..."; + +/// The width of click's help: `max(min(columns, 80) - 2, 50)`, where `columns` is what +/// Python's `shutil.get_terminal_size()` returns. +pub(crate) fn help_width(columns: usize) -> usize { + (columns.min(80).saturating_sub(2)).max(50) +} + +/// `--help`, ending with a line break. +pub(crate) fn help(prog: &str, width: usize) -> String { + let mut formatter = HelpFormatter::new(width); + formatter.write_usage(prog, USAGE_ARGS); + formatter.write_paragraph(); + formatter.indent(); + formatter.write_text(HELP); + formatter.dedent(); + formatter.write_paragraph(); + formatter.write_heading("Options"); + formatter.indent(); + let rows: Vec<(String, String)> = OPTIONS + .iter() + .map(|spec| { + // join_options(): sorted by the length of the prefix ("-" before "--") + let mut opts: Vec<&str> = spec.opts.to_vec(); + opts.sort_by_key(|opt| if opt.starts_with("--") { 2 } else { 1 }); + let mut first = opts.join(", "); + if spec.takes_value { + first.push(' '); + first.push_str(spec.metavar); + } + (first, spec.help.to_string()) + }) + .collect(); + formatter.write_dl(&rows); + formatter.dedent(); + let mut text = formatter.buffer.trim_end_matches('\n').to_string(); + text.push('\n'); + text +} + +/// The usage line of a usage error, without a line break. +pub(crate) fn usage(prog: &str, width: usize) -> String { + let mut formatter = HelpFormatter::new(width); + formatter.write_usage(prog, USAGE_ARGS); + formatter.buffer.trim_end_matches('\n').to_string() +} + +/// `len(strip_ansi(text))`: the help has no escape sequences, so the number of characters. +fn term_len(text: &str) -> usize { + text.chars().count() +} + +/// click's `HelpFormatter`. +struct HelpFormatter { + width: usize, + current_indent: usize, + buffer: String, +} + +impl HelpFormatter { + fn new(width: usize) -> Self { + HelpFormatter { + width, + current_indent: 0, + buffer: String::new(), + } + } + + fn write(&mut self, text: &str) { + self.buffer.push_str(text); + } + + fn indent(&mut self) { + self.current_indent += 2; + } + + fn dedent(&mut self) { + self.current_indent -= 2; + } + + fn spaces(count: usize) -> String { + " ".repeat(count) + } + + fn write_usage(&mut self, prog: &str, args: &str) { + let prefix = "Usage: "; + let usage_prefix = format!("{prefix:>width$}{prog} ", width = self.current_indent); + let text_width = self.width.saturating_sub(self.current_indent); + if text_width >= term_len(&usage_prefix) + 20 { + let indent = Self::spaces(term_len(&usage_prefix)); + let wrapped = wrap_text(args, text_width, &usage_prefix, &indent, false); + self.write(&wrapped); + } else { + self.write(&usage_prefix); + self.write("\n"); + let indent = Self::spaces(self.current_indent.max(term_len(prefix)) + 4); + let wrapped = wrap_text(args, text_width, &indent, &indent, false); + self.write(&wrapped); + } + self.write("\n"); + } + + fn write_heading(&mut self, heading: &str) { + let indent = Self::spaces(self.current_indent); + self.write(&format!("{indent}{heading}:\n")); + } + + fn write_paragraph(&mut self) { + if !self.buffer.is_empty() { + self.write("\n"); + } + } + + fn write_text(&mut self, text: &str) { + let indent = Self::spaces(self.current_indent); + let wrapped = wrap_text(text, self.width, &indent, &indent, true); + self.write(&wrapped); + self.write("\n"); + } + + fn write_dl(&mut self, rows: &[(String, String)]) { + let (col_max, col_spacing) = (30, 2); + let widest = rows + .iter() + .map(|(first, _)| term_len(first)) + .max() + .unwrap_or(0); + let first_col = widest.min(col_max) + col_spacing; + for (first, second) in rows { + let indent = Self::spaces(self.current_indent); + self.write(&format!("{indent}{first}")); + if second.is_empty() { + self.write("\n"); + continue; + } + if term_len(first) <= first_col - col_spacing { + self.write(&Self::spaces(first_col - term_len(first))); + } else { + self.write("\n"); + self.write(&Self::spaces(first_col + self.current_indent)); + } + let text_width = (self.width as isize - first_col as isize - 2).max(10) as usize; + let wrapped = wrap_text(second, text_width, "", "", true); + let lines = splitlines(&wrapped); + match lines.split_first() { + Some((first_line, rest)) => { + self.write(&format!("{first_line}\n")); + for line in rest { + let indent = Self::spaces(first_col + self.current_indent); + self.write(&format!("{indent}{line}\n")); + } + } + None => self.write("\n"), + } + } + } +} + +/// `str.splitlines()`. +fn splitlines(text: &str) -> Vec<&str> { + let mut lines = Vec::new(); + let mut start = 0; + let mut chars = text.char_indices().peekable(); + while let Some((index, c)) = chars.next() { + let end = match c { + '\r' => { + if let Some(&(_, '\n')) = chars.peek() { + chars.next(); + index + 2 + } else { + index + 1 + } + } + '\n' | '\x0b' | '\x0c' | '\x1c' | '\x1d' | '\x1e' | '\u{85}' | '\u{2028}' + | '\u{2029}' => index + c.len_utf8(), + _ => continue, + }; + lines.push(&text[start..index]); + start = end; + } + if start < text.len() { + lines.push(&text[start..]); + } + lines +} + +/// `str.expandtabs(8)`. +fn expand_tabs(text: &str) -> String { + let mut out = String::with_capacity(text.len()); + let mut column = 0; + for c in text.chars() { + match c { + '\t' => { + let spaces = 8 - column % 8; + out.push_str(&" ".repeat(spaces)); + column += spaces; + } + '\n' | '\r' => { + out.push(c); + column = 0; + } + _ => { + out.push(c); + column += 1; + } + } + } + out +} + +/// click's `wrap_text()`: wraps `text` to `width` columns, paragraph by paragraph (with +/// the indentation of their first line) if `preserve_paragraphs`. +fn wrap_text( + text: &str, + width: usize, + initial_indent: &str, + subsequent_indent: &str, + preserve_paragraphs: bool, +) -> String { + let text = expand_tabs(text); + let mut wrapper = TextWrapper { + width: width as isize, + initial_indent: initial_indent.to_string(), + subsequent_indent: subsequent_indent.to_string(), + }; + if !preserve_paragraphs { + return wrapper.fill(&text); + } + // (indentation, raw, text) of each paragraph + let mut paragraphs: Vec<(usize, bool, String)> = Vec::new(); + let mut buffer: Vec<&str> = Vec::new(); + let mut indent: Option = None; + let flush = |buffer: &mut Vec<&str>, indent: Option, paragraphs: &mut Vec<_>| { + if buffer.is_empty() { + return; + } + if python::strip(buffer[0]) == "\x08" { + paragraphs.push((indent.unwrap_or(0), true, buffer[1..].join("\n"))); + } else { + paragraphs.push((indent.unwrap_or(0), false, buffer.join(" "))); + } + buffer.clear(); + }; + for line in splitlines(&text) { + if line.is_empty() { + flush(&mut buffer, indent, &mut paragraphs); + indent = None; + } else { + let line = if indent.is_none() { + let stripped = python::lstrip(line); + indent = Some(term_len(line) - term_len(stripped)); + stripped + } else { + line + }; + buffer.push(line); + } + } + flush(&mut buffer, indent, &mut paragraphs); + let mut wrapped = Vec::new(); + for (indent, raw, text) in paragraphs { + let extra = " ".repeat(indent); + let saved = ( + wrapper.initial_indent.clone(), + wrapper.subsequent_indent.clone(), + ); + wrapper.initial_indent.push_str(&extra); + wrapper.subsequent_indent.push_str(&extra); + wrapped.push(if raw { + wrapper.indent_only(&text) + } else { + wrapper.fill(&text) + }); + (wrapper.initial_indent, wrapper.subsequent_indent) = saved; + } + wrapped.join("\n\n") +} + +/// Python's `textwrap.TextWrapper` with click's options (`replace_whitespace=False`) and +/// click's `_handle_long_word`. +struct TextWrapper { + width: isize, + initial_indent: String, + subsequent_indent: String, +} + +/// `TextWrapper.wordsep_re`, with `\Z` written as fancy-regex writes it. +fn wordsep_re() -> &'static fancy_regex::Regex { + static WORDSEP: OnceLock = OnceLock::new(); + WORDSEP.get_or_init(|| { + let ws = r"[\t\n\x0b\x0c\r ]"; + let nws = r"[^\t\n\x0b\x0c\r ]"; + let wp = r#"[\w!"'&.,?]"#; + let lt = r"[^\d\W]"; + let pattern = format!( + r"({ws}+|(?<={wp})-{{2,}}(?=\w)|{nws}+?(?:-(?:(?<={lt}{{2}}-)|(?<={lt}-{lt}-))(?={lt}-?{lt})|(?={ws}|\z)|(?<={wp})(?=-{{2,}}\w)))" + ); + fancy_regex::Regex::new(&pattern).expect("wordsep_re compiles") + }) +} + +impl TextWrapper { + /// `_split_chunks()`: words, whitespace and hyphenated parts of words. + fn split_chunks(text: &str) -> Vec { + let text = expand_tabs(text); + let mut chunks = Vec::new(); + let mut last = 0; + for found in wordsep_re().find_iter(&text) { + let Ok(found) = found else { break }; + chunks.push(text[last..found.start()].to_string()); + chunks.push(found.as_str().to_string()); + last = found.end(); + } + chunks.push(text[last..].to_string()); + chunks.retain(|chunk| !chunk.is_empty()); + chunks + } + + fn fill(&self, text: &str) -> String { + self.wrap(text).join("\n") + } + + fn indent_only(&self, text: &str) -> String { + splitlines(text) + .iter() + .enumerate() + .map(|(index, line)| { + let indent = if index == 0 { + &self.initial_indent + } else { + &self.subsequent_indent + }; + format!("{indent}{line}") + }) + .collect::>() + .join("\n") + } + + /// `_wrap_chunks()`. + fn wrap(&self, text: &str) -> Vec { + let len = |chunk: &str| chunk.chars().count() as isize; + let is_blank = |chunk: &str| python::strip(chunk).is_empty(); + let mut chunks = Self::split_chunks(text); + chunks.reverse(); + let mut lines: Vec = Vec::new(); + while !chunks.is_empty() { + let mut cur_line: Vec = Vec::new(); + let mut cur_len: isize = 0; + let indent = if lines.is_empty() { + &self.initial_indent + } else { + &self.subsequent_indent + }; + let width = self.width - len(indent); + if chunks.last().is_some_and(|chunk| is_blank(chunk)) && !lines.is_empty() { + chunks.pop(); + } + while let Some(chunk) = chunks.last() { + let l = len(chunk); + if cur_len + l <= width { + cur_len += l; + cur_line.push(chunks.pop().expect("a chunk")); + } else { + break; + } + } + if chunks.last().is_some_and(|chunk| len(chunk) > width) { + // click's _handle_long_word: break the word to fill the line + let space_left = (width - cur_len).max(1) as usize; + let last = chunks.last_mut().expect("a chunk"); + let cut_at = last + .char_indices() + .nth(space_left) + .map_or(last.len(), |(index, _)| index); + let rest = last.split_off(cut_at); + cur_line.push(std::mem::replace(last, rest)); + } + cur_line.pop_if(|chunk| is_blank(chunk)); + if !cur_line.is_empty() { + lines.push(format!("{indent}{}", cur_line.concat())); + } + } + lines + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn chunks_like_textwrap() { + // click's TextWrapper()._split_chunks() in Python 3.10 + assert_eq!( + TextWrapper::split_chunks("Hello there -- you goof-ball, use the -b option!"), + [ + "Hello", " ", "there", " ", "--", " ", "you", " ", "goof-", "ball,", " ", "use", + " ", "the", " ", "-b", " ", "option!" + ] + ); + assert_eq!( + TextWrapper::split_chunks("a--b well--done '--rarity 0:' x-1"), + [ + "a", "--", "b", " ", "well", "--", "done", " ", "'", "--", "rarity", " ", "0:'", + " ", "x-1" + ] + ); + } + + #[test] + fn wraps_like_textwrap() { + let wrapper = TextWrapper { + width: 20, + initial_indent: "* ".into(), + subsequent_indent: " ".into(), + }; + // click's TextWrapper(20, initial_indent="* ", subsequent_indent=" ").fill(text) + assert_eq!( + wrapper.fill("The quick brown fox jumps over the lazy dog, supercalifragilistic"), + "* The quick brown\n fox jumps over the\n lazy dog, supercal\n ifragilistic" + ); + } + + #[test] + fn widths() { + assert_eq!(help_width(10_000), 78); + assert_eq!(help_width(60), 58); + assert_eq!(help_width(10), 50); + } + + #[test] + fn usage_line() { + assert_eq!(usage("what", 78), "Usage: what [OPTIONS] [TEXT_INPUT]..."); + assert!(help("what", 78).starts_with("Usage: what [OPTIONS] [TEXT_INPUT]...\n\n pyWhat")); + } +} diff --git a/src/cli/help.txt b/src/cli/help.txt new file mode 100644 index 0000000..3433bc6 --- /dev/null +++ b/src/cli/help.txt @@ -0,0 +1,207 @@ +pyWhat - Identify what something is. + +Made by Bee https://twitter.com/bee_sec_san + +https://github.com/bee-san + +Filtration: + + --rarity min:max + + Rarity is how unlikely something is to be a false-positive. The higher the number, the more unlikely. + + Only print entries with rarity in range [min,max]. min and max can be omitted. + + Note: PyWhat by default has a rarity of 0.1. To see all matches, with many potential false positives use `0:`. For example, a YouTube video ID on its own, such as 'dQw4w9WgXcQ', is only shown with `--rarity 0:`. + + --include list + + Only include entries containing at least one tag in a list. List is a comma separated list. + + --exclude list + + Exclude specified tags. List is a comma separated list. + + The names of the regexes work like tags. Many regexes have alternative names too, such as 'ETH Wallet' for 'Ethereum (ETH) Wallet Address', so --include 'BTC Wallet,ETH Wallet' only shows Bitcoin and Ethereum wallets. + + --names + + Show the name of every regex, followed by its alternative names. + +Sorting: + + --key key_name + + Sort by the given key. + + --reverse + + Sort in reverse order. + + Available keys: + + name - Sort by the name of regex pattern + + rarity - Sort by rarity + + matched - Sort by a matched string + + likely - Sort the most likely matches first, see Top matches + + none - No sorting is done (the default) + +Top matches: + + --top N + + Only show the N most likely matches, e.g. --top 10. --top N% shows the most likely N percent of them, e.g. --top 5%. For a directory or several inputs, these are the most likely matches of all of them. + + The most likely matches are shown first, unless --key sorts them. They have the highest rarity, but fragments of a longer word or match, which boundaryless mode finds, are less likely unless their rarity is 1. For example, the card and social security numbers in '0x52908400098527886E0F7030069857D2E4169EE7' are fragments of the Ethereum address, which is the most likely match. + + How many matches were left out is shown on stderr. Use --interactive to page through all of them. + +Exporting: + + --json + + Return results in json format, on one line, so that it can be piped to other programs such as jq: pywhat --json 'abc@example.com' | jq + +Boundaryless mode: + + CLI tool matches strings like 'abcdTHM{hello}plze' by default because the boundaryless mode is enabled for regexes with a rarity of 0.1 and higher. + + Since boundaryless mode may produce a lot of false-positive matches, it is possible to disable it, either fully or partially. + + '--disable-boundaryless' flag can be used to fully disable this mode. + + In addition, '-br', '-bi', and '-be' options can be used to tweak which regexes should be in boundaryless mode. + + Refer to the Filtration section for more information. + +Processing: + + Matches are processed further once a regex has found them. For example, the date of a Unix timestamp, in UTC, is added to its description: pywhat --rarity 0: --include "UNIX Timestamp" 1637093119 + + Matches whose checksum is wrong are filtered out: Bitcoin, Bitcoin Cash, Litecoin, Dogecoin and XRP wallet addresses (like 3F3F3F3F3F3F3F3F3F3F3F3F3F3F3F3F3F), card numbers (the Luhn check) and Turkish Identification Numbers. + + '--disable-processing' flag can be used to show the matches as the regexes found them. + +Verification: + + --verify + + Ask the services if the keys that are found are valid, and add the answer to the description, e.g. 'Verification: invalid, Google rejected the key (API_KEY_INVALID)'. Currently Google API keys are verified: pywhat --verify 'AIzaSyA00000000000000000000000000000000' + + Only use it for keys that you are allowed to test: it sends the keys to the services (over HTTPS), once per key. It works with '--disable-processing' too. + +Maps: + + --map google|osm + + The map that the links of coordinates go to: Google Maps (the default) or OpenStreetMap. For example, pywhat --map osm '52.6169586, -1.9779857' links to https://www.openstreetmap.org/search?query=52.6169586,-1.9779857 + + It works with '--disable-processing' too. + +Formatting the output: + + --format format_str + + format_str can be equal to: + + pretty - Output data in the table + + json - Output data in json format + + CUSTOM_STRING - Print data in the way you want. For every match CUSTOM_STRING will be printed and '%x' (See below for possible x values) will be substituted with a match value. + + For example: + + pywhat --format '%m - %n' 'google.com htb{flag}' + + will print: + + htb{flag} - HackTheBox Flag Format + google.com - Uniform Resource Locator (URL) + + Possible '%x' values: + + %m - matched text + + %n - name of regex + + %d - description (will not output if absent) + + %e - exploit (will not output if absent) + + %r - rarity + + %l - link (will not output if absent) + + %t - tags (in 'tag1, tag2 ...' format) + + If you want to print '%' or '\' character - escape it: '\%', '\\'. + +Interactive mode: + + --interactive + + Identify the input once, keep the matches in memory and search through them as often as you like, with searches such as 'include:"Bug Bounty", rarity:"0.5:"'. + + The --rarity, --include and --exclude options are the search to start with, the other options work as usual. The input is optional, the 'load' command loads a file, directory or text. Type 'help' in interactive mode to see all commands. + + With --top N or the 'top N' command, searches show the N most likely matches and the 'more' command the next N. + +Progress and streaming: + + When a search takes a while, progress bars show how far it is: one for the regexes that are searched for in a file, with the regex that is searched for now, and one for the files of a directory or of several inputs. They are only shown on a terminal (on stderr if the output is redirected to a file), and they are removed once the search is complete. + + --no-progress + + Do not show the progress bars. + + --stream + + Print every match as soon as it is found, instead of once the search is complete. With --json, every match is a JSON object of its own, on one line, without the "Fragment" key, as whether a match is a fragment is only known once its file has been searched. --stream cannot sort the matches (--key), show only the most likely ones (--top) or print a table (--format pretty), as these need all the matches. + +Unicode: + + Files and text piped to pyWhat can be UTF-8, or UTF-16 or UTF-32 with a byte order mark (BOM). The UTF-16 strings in binary files, which 'strings -el' shows, are searched too. + + Characters that the terminal cannot show are printed as escape sequences such as '\u20bf'. + +Examples: + + * what 'HTB{this is a flag}' + + * what '0x52908400098527886E0F7030069857D2E4169EE7' + + * what -- '52.6169586, -1.9779857' + + * what --rarity 0.6: 'myEmail@host.org' + + * what --rarity 0: --include "credentials" --exclude "aws" 'James:SecretPassword' + + * what -br 0.6: -be URL '123myEmail@host.org456' + + * what --top 1 '0x52908400098527886E0F7030069857D2E4169EE7' + +Your text must either be in quotation marks, or use the POSIX standard of "--" to mean "anything after -- is textual input". + + +pyWhat can also search files or even a whole directory with recursion: + + * what 'secret.txt' + + * what 'this/is/a/path' + + * what --interactive 'this/is/a/path' + + * what --stream 'big.log' + +Several inputs, files, directories or text, can be searched at once. Then the file of every match is shown, and the matches in text are under "text": + + * what 'secret.txt' 'this/is/a/path' 'HTB{this is a flag}' + + * find . -name '*.log' -exec what {} + + +Text with spaces needs quotation marks, as every argument is an input. \ No newline at end of file diff --git a/src/cli/markup.rs b/src/cli/markup.rs new file mode 100644 index 0000000..68b24f8 --- /dev/null +++ b/src/cli/markup.rs @@ -0,0 +1,370 @@ +//! rich's console markup, for the strings of pyWhat's data files: some descriptions in +//! `regex.json` are written for rich (`[#CAE4F1][link=https://...]CORS[/link][/#CAE4F1]`), +//! and rich takes the wiki-style links of `file_signatures.json` (`[https://... name]`) for +//! tags too, and leaves them out. The Python CLI prints all of its output as markup; this +//! one only the data, and prints the matched text and the file names as they are (#291, +//! Python bug 4). + +use super::style::Style; +use std::sync::OnceLock; + +/// rich's `RE_TAGS`: a tag is `[` and a letter, `#`, `/` or `@`, up to the next `]`, +/// possibly escaped with backslashes. +fn tags() -> &'static regex::Regex { + static TAGS: OnceLock = OnceLock::new(); + TAGS.get_or_init(|| regex::Regex::new(r"(\\*)\[([a-z#/@][^\[]*?)\]").expect("valid regex")) +} + +/// A style that rich can parse, as far as the data uses them: `bold`, colours as +/// `#rrggbb` and `link URL`. Anything else is no style, which is what rich makes of a +/// style it can't parse. +fn parse_style(definition: &str) -> Style { + let mut style = Style::default(); + let mut words = definition.split_whitespace(); + while let Some(word) = words.next() { + match word.to_lowercase().as_str() { + "bold" | "b" => style.bold = true, + "link" => match words.next() { + Some(url) => style.link = Some(url.to_string()), + None => return Style::default(), + }, + color if color.len() == 7 && color.starts_with('#') => { + match u32::from_str_radix(&color[1..], 16) { + Ok(rgb) => style.color = Some(((rgb >> 16) as u8, (rgb >> 8) as u8, rgb as u8)), + Err(_) => return Style::default(), + } + } + _ => return Style::default(), + } + } + style +} + +/// An open tag: its name (to match the closing tag) and its style. +struct Open { + name: String, + style: Style, +} + +/// rich's `STYLE_ATTRIBUTES`: the attributes of styles, with their short names. +fn attribute(word: &str) -> Option<&'static str> { + Some(match word { + "dim" | "d" => "dim", + "bold" | "b" => "bold", + "italic" | "i" => "italic", + "underline" | "u" => "underline", + "blink" => "blink", + "blink2" => "blink2", + "reverse" | "r" => "reverse", + "conceal" | "c" => "conceal", + "strike" | "s" => "strike", + "underline2" | "uu" => "underline2", + "frame" => "frame", + "encircle" => "encircle", + "overline" | "o" => "overline", + _ => return None, + }) +} + +/// rich's `Style.normalize()`, which matches closing tags with opening ones: the attributes +/// in rich's order (`[b]` is closed by `[/bold]`), the colour, the background colour and +/// the link. A style that rich can't parse is compared as written, in lower case; this +/// takes every other word for a colour, so it differs from rich only for unknown colours +/// written in another order. +fn normalize(definition: &str) -> String { + const ORDER: [&str; 13] = [ + "bold", + "dim", + "italic", + "underline", + "blink", + "blink2", + "reverse", + "conceal", + "strike", + "underline2", + "frame", + "encircle", + "overline", + ]; + let unparsed = || definition.trim().to_lowercase(); + if definition.is_empty() || definition.trim() == "none" { + return "none".to_string(); + } + let mut attributes: Vec<(&str, bool)> = Vec::new(); + let (mut color, mut background, mut link) = (None, None, None); + let mut words = definition.split_whitespace(); + while let Some(original) = words.next() { + let word = original.to_lowercase(); + match word.as_str() { + "on" => match words.next() { + Some(next) => background = Some(next.to_lowercase()), + None => return unparsed(), + }, + // The word after "not" and "link" as it is written + "not" => match words.next().and_then(attribute) { + Some(name) => attributes.push((name, false)), + None => return unparsed(), + }, + "link" => match words.next() { + Some(next) => link = Some(next.to_string()), + None => return unparsed(), + }, + _ => match attribute(&word) { + Some(name) => attributes.push((name, true)), + None => color = Some(word), + }, + } + } + let mut parts: Vec = Vec::new(); + for name in ORDER { + if let Some(&(_, on)) = attributes.iter().rev().find(|(n, _)| *n == name) { + parts.push(if on { + name.to_string() + } else { + format!("not {name}") + }); + } + } + parts.extend(color); + if let Some(background) = background { + parts.push(format!("on {background}")); + } + if let Some(link) = link { + parts.push(format!("link {link}")); + } + if parts.is_empty() { + "none".to_string() + } else { + parts.join(" ") + } +} + +fn combined(open: &[Open]) -> Style { + let mut style = Style::default(); + for tag in open { + style.bold |= tag.style.bold; + if tag.style.color.is_some() { + style.color = tag.style.color; + } + if tag.style.link.is_some() { + style.link.clone_from(&tag.style.link); + } + } + style +} + +/// The text of `markup` with its styles, as `rich.markup.render()` makes it (without emoji +/// codes, which the data doesn't have). A closing tag that closes nothing is left out. +pub(crate) fn render(markup: &str) -> Vec<(String, Style)> { + match render_or_error(markup, false) { + Ok(spans) => spans, + Err(_) => unreachable!("only the strict rendering fails"), + } +} + +/// The `MarkupError` that rich raises for `markup`, if any: a closing tag that closes +/// nothing, such as the `[/b]` of the format string `[/b]%m`. +pub(crate) fn error(markup: &str) -> Option { + render_or_error(markup, true).err() +} + +fn render_or_error(markup: &str, strict: bool) -> Result, String> { + if !markup.contains('[') { + return Ok(vec![(markup.to_string(), Style::default())]); + } + let mut spans: Vec<(String, Style)> = Vec::new(); + let mut open: Vec = Vec::new(); + let push = |spans: &mut Vec<(String, Style)>, text: &str, open: &[Open]| { + if !text.is_empty() { + // Escaped brackets that aren't part of a tag + spans.push((text.replace("\\[", "["), combined(open))); + } + }; + let mut position = 0; + for captures in tags().captures_iter(markup) { + let whole = captures.get(0).expect("a match"); + let escapes = captures.get(1).map_or("", |m| m.as_str()); + let tag_text = captures.get(2).map_or("", |m| m.as_str()); + push(&mut spans, &markup[position..whole.start()], &open); + position = whole.end(); + if !escapes.is_empty() { + let (backslashes, escaped) = (escapes.len() / 2, escapes.len() % 2 == 1); + if backslashes > 0 { + spans.push(("\\".repeat(backslashes), combined(&open))); + } + if escaped { + // An escaped tag is text, with its brackets + spans.push((whole.as_str()[escapes.len()..].to_string(), combined(&open))); + continue; + } + } + let (name, parameters) = match tag_text.split_once('=') { + Some((name, parameters)) => (name, Some(parameters)), + None => (tag_text, None), + }; + if let Some(closing) = name.strip_prefix('/') { + // Where rich says the tag is: in characters, after the escaped backslashes + let at = || markup[..whole.start()].chars().count() + escapes.len(); + let closing = closing.trim(); + let closed = if closing.is_empty() { + open.pop().is_some() + } else { + let closing = normalize(closing); + match open.iter().rposition(|tag| tag.name == closing) { + Some(index) => { + open.remove(index); + true + } + None => false, + } + }; + if !closed && strict { + return Err(if closing.is_empty() { + format!( + "closing tag '[/]' at position {} has nothing to close", + at() + ) + } else { + let markup = &whole.as_str()[escapes.len()..]; + format!( + "closing tag '{markup}' at position {} doesn't match any open tag", + at() + ) + }); + } + } else { + let definition = match parameters { + Some(parameters) => format!("{name} {parameters}"), + None => name.to_string(), + }; + open.push(Open { + name: normalize(name), + style: parse_style(&definition), + }); + } + } + push(&mut spans, &markup[position..], &open); + Ok(spans) +} + +/// The text of `markup` without its tags. +pub(crate) fn plain(markup: &str) -> String { + render(markup).into_iter().map(|(text, _)| text).collect() +} + +/// `rich.markup.escape()`: `text` as markup that prints `text` as it is. +pub(crate) fn escape(text: &str) -> String { + static ESCAPE: OnceLock = OnceLock::new(); + let escape = + ESCAPE.get_or_init(|| regex::Regex::new(r"(\\*)(\[[a-z#/@].*?\])").expect("valid regex")); + escape + .replace_all(text, |captures: ®ex::Captures<'_>| { + let backslashes = &captures[1]; + format!("{backslashes}{backslashes}\\{}", &captures[2]) + }) + .into_owned() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn data_markup() { + // What rich 12.2.0 prints for these strings of the data files + assert_eq!( + plain("Used for [#CAE4F1][link=https://en.wikipedia.org/wiki/Cross-origin_resource_sharing]Cross-Origin Resource Sharing (CORS)[/link][/#CAE4F1]"), + "Used for Cross-Origin Resource Sharing (CORS)" + ); + assert_eq!( + plain("Compressed file using [https://segaretro.org/Rob_Northen_compression Rob Northen compression] (version 1 and 2) algorithm"), + "Compressed file using (version 1 and 2) algorithm" + ); + assert_eq!(plain("a \\[b] \\\\[bold]c"), "a [b] \\c"); + assert_eq!(plain("[1] [A] x"), "[1] [A] x"); + let spans = render("x [#CAE4F1][link=https://a.b]y[/link][/#CAE4F1] z"); + assert_eq!(spans[1].0, "y"); + assert_eq!(spans[1].1.color, Some((0xCA, 0xE4, 0xF1))); + assert_eq!(spans[1].1.link.as_deref(), Some("https://a.b")); + assert_eq!(spans[2], (" z".to_string(), Style::default())); + } + + #[test] + fn normalize_like_rich() { + // Style.normalize() in rich 12.2.0 + for (definition, normalized) in [ + ("b", "bold"), + ("B", "bold"), + ("bold #D7Afff", "bold #d7afff"), + ("#D7Afff bold", "bold #d7afff"), + ("red bold", "bold red"), + ("on blue red", "red on blue"), + ("link https://A.b", "link https://A.b"), + ("link", "link"), + ("not bold", "not bold"), + ("u r", "underline reverse"), + ("i", "italic"), + ("", "none"), + ] { + assert_eq!(normalize(definition), normalized, "{definition:?}"); + } + } + + #[test] + fn markup_errors_like_rich() { + // The MarkupError messages of rich 12.2.0 + assert_eq!( + error("[/b]THM{x}").as_deref(), + Some("closing tag '[/b]' at position 0 doesn't match any open tag") + ); + assert_eq!( + error("[b]THM{x}[/i]").as_deref(), + Some("closing tag '[/i]' at position 9 doesn't match any open tag") + ); + assert_eq!( + error("x[/]y").as_deref(), + Some("closing tag '[/]' at position 1 has nothing to close") + ); + assert_eq!( + error("[b]THM{x}[/b][/b]").as_deref(), + Some("closing tag '[/b]' at position 13 doesn't match any open tag") + ); + assert_eq!( + error("é[/b]").as_deref(), + Some("closing tag '[/b]' at position 1 doesn't match any open tag") + ); + for valid in [ + "[b]x[/bold]", + "[bold red]x[/red bold]", + "\\[/b]", + "[link=https://a.b]x[/link]", + "[b]x", + "[/b", + "[@click]x[/@click]", + "a \\\\[b]x[/b]", + ] { + assert_eq!(error(valid), None, "{valid:?}"); + } + // Rendering leaves the tag out + assert_eq!(plain("x[/b]y"), "xy"); + // Meta tags (@) are tags too + assert_eq!(plain("[@click]x[/@click] [@click=f()]y[/]"), "x y"); + } + + #[test] + fn escape_like_rich() { + // rich.markup.escape() in rich 12.2.0 + for (text, escaped) in [ + ("a\\", "a\\"), + ("[b]x", "\\[b]x"), + ("\\[b]", "\\\\\\[b]"), + ("x [B] [1]", "x [B] [1]"), + ("[/]", "\\[/]"), + ("a[b]c\\", "a\\[b]c\\"), + ] { + assert_eq!(escape(text), escaped, "{text:?}"); + assert_eq!(plain(&escape(text)), text, "{text:?}"); + } + } +} diff --git a/src/cli/mod.rs b/src/cli/mod.rs new file mode 100644 index 0000000..3de89d0 --- /dev/null +++ b/src/cli/mod.rs @@ -0,0 +1,1077 @@ +//! The `what` and `pywhat` commands, with the options, the output and the exit codes of +//! the Python package's CLI (`pywhat/what.py`): `pywhat::cli::main` is what both binaries +//! run, and what the Python package's commands call. +//! +//! ```no_run +//! // src/bin/what.rs +//! std::process::exit(pywhat::cli::main(std::env::args_os())) +//! ``` +//! +//! What it does differently from the Python CLI: it prints the matched text and the file +//! names as they are, where the Python CLI takes them for rich markup (`thm{[bold]x[/bold]}` +//! or `:smile:`); `--interactive` needs the Python package; and a file that can't be read +//! is reported and skipped. + +mod args; +mod help; +mod markup; +mod output; +mod progress; +mod python; +mod style; +mod table; +pub mod verify; + +use crate::files::{Found, Identified, InputOptions}; +use crate::filter::{split_tags, Filter}; +use crate::processors::{default_processors, OpenStreetMap, Processor}; +use crate::ranking::{Key, Top}; +use crate::Identifier; +use args::Param; +use output::Printer; +use std::collections::HashMap; +use std::ffi::OsString; +use std::io::{self, IsTerminal, Read, Write}; +use style::{Colors, Text, LABEL, VERSION}; + +/// Runs the command line `args`, the program name first (as `std::env::args_os()` gives +/// it), and returns the exit code: 0, 1 for errors (as in Python), 2 for usage errors. +/// The name of the program, without its directory, is the name that `--help` shows. +pub fn main(args: I) -> i32 +where + I: IntoIterator, + T: Into, +{ + let mut args = args.into_iter().map(|arg| { + arg.into() + .into_string() + .unwrap_or_else(|arg| arg.to_string_lossy().into_owned()) + }); + let prog = args + .next() + .map_or_else(|| "what".to_string(), |arg0| program_name(&arg0)); + let args: Vec = args.collect(); + let env = Environment::detect(); + let mut stdin = io::stdin(); + let mut stdout = io::BufWriter::with_capacity(1 << 16, io::stdout()); + let mut stderr = io::stderr(); + let mut streams = Streams { + stdin: &mut stdin, + stdout: &mut stdout, + stderr: &mut stderr, + }; + run(&prog, &args, &env, &mut streams, &GoogleVerifier::default()) +} + +/// The name of the program in `--help`: the file name of `arg0` (without `.exe` on +/// Windows), like click's `_detect_program_name()`. +fn program_name(arg0: &str) -> String { + let separators: &[char] = if cfg!(windows) { &['/', '\\'] } else { &['/'] }; + let name = arg0.rsplit(separators).next().unwrap_or(arg0); + if cfg!(windows) && name.to_ascii_lowercase().ends_with(".exe") { + return name[..name.len() - 4].to_string(); + } + name.to_string() +} + +/// A stream of the process: whether it is a terminal or a regular file, and its width. +#[derive(Debug, Clone, Copy, Default)] +struct StreamInfo { + terminal: bool, + regular_file: bool, + columns: Option, +} + +/// What the command needs to know about where it runs. +#[derive(Debug, Clone, Default)] +struct Environment { + vars: HashMap<&'static str, String>, + stdin: StreamInfo, + stdout: StreamInfo, + stderr: StreamInfo, +} + +/// The environment variables that the Python CLI (click and rich) reads. +const VARIABLES: [&str; 5] = ["COLUMNS", "LINES", "TERM", "COLORTERM", "NO_COLOR"]; + +#[cfg(unix)] +fn stream_info(stream: S) -> StreamInfo { + let regular_file = stream + .as_fd() + .try_clone_to_owned() + .map(std::fs::File::from) + .and_then(|file| file.metadata()) + .is_ok_and(|metadata| metadata.is_file()); + StreamInfo { + terminal: stream.is_terminal(), + regular_file, + columns: terminal_size::terminal_size_of(stream).map(|(width, _)| width.0 as usize), + } +} + +#[cfg(windows)] +fn stream_info(stream: S) -> StreamInfo { + let regular_file = stream + .as_handle() + .try_clone_to_owned() + .map(std::fs::File::from) + .and_then(|file| file.metadata()) + .is_ok_and(|metadata| metadata.is_file()); + StreamInfo { + terminal: stream.is_terminal(), + regular_file, + columns: terminal_size::terminal_size_of(stream).map(|(width, _)| width.0 as usize), + } +} + +#[cfg(not(any(unix, windows)))] +fn stream_info(stream: S) -> StreamInfo { + StreamInfo { + terminal: stream.is_terminal(), + ..StreamInfo::default() + } +} + +impl Environment { + fn detect() -> Environment { + let vars = VARIABLES + .iter() + .filter_map(|&name| { + std::env::var_os(name).map(|value| (name, value.to_string_lossy().into_owned())) + }) + .collect(); + Environment { + vars, + stdin: stream_info(io::stdin()), + stdout: stream_info(io::stdout()), + stderr: stream_info(io::stderr()), + } + } + + fn var(&self, name: &str) -> Option { + self.vars.get(name).cloned() + } + + /// The colours of a stream, as rich detects them. + fn colors(&self, stream: &StreamInfo) -> Colors { + Colors::detect(stream.terminal, &|name| self.var(name)) + } + + /// `shutil.get_terminal_size().columns`, which click's help uses: `COLUMNS` if it is + /// a positive number, else the width of the terminal of stdout, else 80. + fn click_columns(&self) -> usize { + match self + .var("COLUMNS") + .and_then(|columns| python::int(&columns)) + { + Some(columns) if columns > 0 => usize::try_from(columns).unwrap_or(usize::MAX), + _ => self + .stdout + .columns + .filter(|&columns| columns > 0) + .unwrap_or(80), + } + } + + /// The width of a rich `Console` on stdout: `COLUMNS` if it is a number, else the width + /// of the terminal of stdin or (on Windows: only) stdout, else 80. On a dumb terminal, + /// 80 unless `COLUMNS` and `LINES` are both numbers. + fn rich_width(&self) -> usize { + let number = |name: &str| { + self.var(name).filter(|value| { + !value.is_empty() && value.chars().all(|c| python::decimal_value(c).is_some()) + }) + }; + let columns = number("COLUMNS"); + if self.stdout.terminal + && style::is_dumb(self.var("TERM").as_deref()) + && (columns.is_none() || number("LINES").is_none()) + { + return 80; + } + let width = match columns.and_then(|columns| python::int(&columns)) { + Some(columns) => usize::try_from(columns).unwrap_or(usize::MAX), + None if cfg!(windows) => self.stdout.columns.unwrap_or(0), + None => self.stdin.columns.or(self.stdout.columns).unwrap_or(0), + }; + if width == 0 { + 80 + } else { + width + } + } +} + +/// The streams of the process. +struct Streams<'a> { + stdin: &'a mut dyn Read, + stdout: &'a mut dyn Write, + stderr: &'a mut dyn Write, +} + +/// Makes the `--verify` verifier; tests give it a fake Google. +#[derive(Default)] +struct GoogleVerifier { + make: Option verify::GoogleApiKeyVerifier>>, +} + +impl GoogleVerifier { + fn verifier(&self) -> verify::GoogleApiKeyVerifier { + match &self.make { + Some(make) => make(), + None => verify::GoogleApiKeyVerifier::new(), + } + } +} + +/// Runs the command, returns its exit code. A closed output (`what ... | head`) ends it +/// with 1, as click does. +fn run( + prog: &str, + args: &[String], + env: &Environment, + streams: &mut Streams<'_>, + verifier: &GoogleVerifier, +) -> i32 { + let code = match execute(prog, args, env, streams, verifier) { + Ok(code) => streams.stdout.flush().map_or(1, |_| code), + Err(error) if error.kind() == io::ErrorKind::BrokenPipe => 1, + Err(error) => { + let _ = writeln!(streams.stderr, "Error: {error}"); + 1 + } + }; + let _ = streams.stdout.flush(); + code +} + +/// An error of click's parser (an option without its value, or a flag with one): click shows +/// only `Error: `, as these errors have no context, and exits with 2. +fn parse_error(message: &str, streams: &mut Streams<'_>) -> io::Result { + streams.stdout.flush()?; + writeln!(streams.stderr, "Error: {message}")?; + Ok(2) +} + +/// An invalid value of an option (`--map`), as click shows it: the usage line, a hint and +/// the error, exit code 2. +fn usage_error( + prog: &str, + message: &str, + env: &Environment, + streams: &mut Streams<'_>, +) -> io::Result { + streams.stdout.flush()?; + let width = help::help_width(env.click_columns()); + let usage = help::usage(prog, width); + write!( + streams.stderr, + "{usage}\nTry '{prog} --help' for help.\n\nError: {message}\n" + )?; + Ok(2) +} + +/// Prints `message` and a line break on stdout, with `print()`. +fn print_line(streams: &mut Streams<'_>, message: &str) -> io::Result<()> { + writeln!(streams.stdout, "{message}") +} + +/// `sys.exit(message)`: the message on stderr, exit code 1. +fn exit_with(streams: &mut Streams<'_>, message: &str) -> io::Result { + streams.stdout.flush()?; + writeln!(streams.stderr, "{message}")?; + Ok(1) +} + +/// `print_tags()`: every tag, sorted. +fn print_tags(streams: &mut Streams<'_>, colors: Colors) -> io::Result<()> { + let mut text = Text::new(); + for (index, tag) in crate::tags().iter().enumerate() { + if index > 0 { + text.plain("\n"); + } + text.styled(tag, LABEL); + } + let mut printer = Printer { + out: streams.stdout, + colors, + }; + printer.print(&text) +} + +/// `print_names()`: the name of every regex, sorted, with its alternative names. +fn print_names(streams: &mut Streams<'_>, colors: Colors) -> io::Result<()> { + let mut regexes: Vec<_> = crate::regexes().iter().collect(); + regexes.sort_by_cached_key(|regex| regex.name.to_lowercase()); + let mut text = Text::new(); + for (index, regex) in regexes.iter().enumerate() { + if index > 0 { + text.plain("\n"); + } + text.styled(®ex.name, LABEL); + if !regex.alternative_names.is_empty() { + text.plain(": ").plain(®ex.alternative_names.join(", ")); + } + } + let mut printer = Printer { + out: streams.stdout, + colors, + }; + printer.print(&text) +} + +/// `print_version()`. +fn print_version(streams: &mut Streams<'_>, colors: Colors) -> io::Result<()> { + let mut text = Text::new(); + text.plain("PyWhat version ") + .styled(env!("CARGO_PKG_VERSION"), VERSION); + let mut printer = Printer { + out: streams.stdout, + colors, + }; + printer.print(&text) +} + +/// `create_filter()`: the filter of `--rarity`, `--include` and `--exclude` (or of their +/// boundaryless versions), or the message that Python prints before it exits with 1. +fn create_filter( + rarity: Option<&str>, + include: Option<&str>, + exclude: Option<&str>, +) -> Result { + let (mut min_rarity, mut max_rarity) = (0.1, 1.0); + if let Some(rarity) = rarity { + let rarities: Vec<&str> = rarity.split(':').collect(); + if rarities.len() != 2 { + return Err("Invalid rarity range format ('min:max' expected)"); + } + for (rarity, value) in rarities.iter().zip([&mut min_rarity, &mut max_rarity]) { + if !python::is_all_space(rarity) && !rarity.is_empty() { + *value = python::float(rarity).ok_or("Invalid rarity argument (float expected)")?; + } + } + } + let tags = |list: &str| -> Vec { + split_tags(list) + .iter() + .map(|tag| python::strip(tag).to_string()) + .collect() + }; + let include = include.map(tags); + let exclude = exclude.map(tags).unwrap_or_default(); + Filter::new(include.as_deref(), &exclude, min_rarity, max_rarity).map_err(|_| { + "Passed tags are not valid.\n\ + You can check available tags by using: 'pywhat --tags'\n\ + and the names of the regexes, which work like tags, by using: 'pywhat --names'" + }) +} + +/// `parse_top()`, with Python's whitespace: `str.strip()` and the `\s` of `N %` take +/// U+001C to U+001F too. +fn parse_top(text: &str) -> Result { + let text = python::strip(text); + match Top::parse(text) { + Ok(top) => Ok(top), + Err(error) => { + if let Some(number) = text.strip_suffix('%') { + let number = python::rstrip(number); + if let Ok(top) = Top::parse(&format!("{number}%")) { + return Ok(top); + } + } + Err(error) + } + } +} + +/// `read_stdin()`: stdin read like a file (UTF-8, or UTF-16 or UTF-32 with a BOM, and the +/// UTF-16 strings in it), as one text. +fn read_stdin(stdin: &mut dyn Read) -> io::Result { + let mut data = Vec::new(); + stdin.read_to_end(&mut data)?; + Ok(python::strip(&crate::unicode::texts(&data).join("\n")).to_string()) +} + +/// `Identified._collect()` of one thing found. +fn collect(identified: &mut Identified, found: Found) { + match found { + Found::FileSignature { + location, + signature, + } => match identified + .signatures + .iter_mut() + .find(|(existing, _)| *existing == location) + { + Some(existing) => existing.1 = signature, + None => identified.signatures.push((location, signature)), + }, + Found::Match { location, found } => match identified + .regexes + .iter_mut() + .find(|(existing, _)| *existing == location) + { + Some((_, matches)) => matches.push(found), + None => identified.regexes.push((location, vec![found])), + }, + } +} + +/// `progress_bars()`: bars on the terminal of stdout or, if stdout is a regular file +/// (`what ... > out.txt`), on the terminal of stderr. Not into a pipe, as the program that +/// reads it may be writing to the terminal too. +fn progress_bars(env: &Environment, enabled: bool) -> Option<(progress::Bars, progress::Target)> { + let (target, stream) = bars_target(env, enabled)?; + // rich's width (COLUMNS), else the width of the terminal that the bars are on + let width = match env.var("COLUMNS").and_then(|columns| python::int(&columns)) { + Some(columns) if columns > 0 => env.rich_width(), + _ => stream.columns.filter(|&columns| columns > 0).unwrap_or(80), + }; + let bars = progress::Bars::start(target, width, env.colors(stream)); + Some((bars, target)) +} + +/// Where the progress bars go, if anywhere, and the stream that is there. +fn bars_target(env: &Environment, enabled: bool) -> Option<(progress::Target, &StreamInfo)> { + if !enabled { + return None; + } + let (target, stream) = if env.stdout.regular_file { + (progress::Target::Stderr, &env.stderr) + } else { + (progress::Target::Stdout, &env.stdout) + }; + if !stream.terminal || style::is_dumb(env.var("TERM").as_deref()) { + return None; + } + Some((target, stream)) +} + +/// The end of a search: the files that couldn't be read, on stderr (exit code 1, which +/// Python exits with when it can't read a file). +fn finish(streams: &mut Streams<'_>, unreadable: &[String]) -> io::Result { + if unreadable.is_empty() { + return Ok(0); + } + streams.stdout.flush()?; + for message in unreadable { + writeln!(streams.stderr, "{message}")?; + } + Ok(1) +} + +/// The options of the main part of the command. +struct Command<'a> { + parsed: &'a args::Parsed, + inputs: Vec, + map: &'static str, +} + +impl Command<'_> { + fn flag(&self, param: Param) -> bool { + self.parsed.flag(param) + } + + fn value(&self, param: Param) -> Option<&str> { + self.parsed.value(param) + } +} + +fn execute( + prog: &str, + args: &[String], + env: &Environment, + streams: &mut Streams<'_>, + verifier: &GoogleVerifier, +) -> io::Result { + let parsed = match args::parse(args) { + Ok(parsed) => parsed, + Err(args::UsageError(message)) => return parse_error(&message, streams), + }; + // --help is eager: it comes first + if parsed.flag(Param::Help) { + let width = help::help_width(env.click_columns()); + streams + .stdout + .write_all(help::help(prog, width).as_bytes())?; + return Ok(0); + } + // The callbacks of the options, in the order in which they were given + let colors = env.colors(&env.stdout); + let mut map = "google"; + for ¶m in parsed.invoked() { + match param { + Param::Tags => { + print_tags(streams, colors)?; + return Ok(0); + } + Param::Names => { + print_names(streams, colors)?; + return Ok(0); + } + Param::Version => { + print_version(streams, colors)?; + return Ok(0); + } + Param::Map => match args::map_choice(parsed.value(Param::Map).unwrap_or("")) { + Ok(choice) => map = choice, + Err(args::UsageError(message)) => return usage_error(prog, &message, env, streams), + }, + _ => {} + } + } + // get_text(): stdin is the input if there is none, or only empty ones (`not + // any(value)`), unless it is a terminal + let mut inputs = parsed.inputs.clone(); + if inputs.iter().all(String::is_empty) + && !parsed.flag(Param::Interactive) + && !env.stdin.terminal + { + inputs = vec![read_stdin(streams.stdin)?]; + } + let command = Command { + parsed: &parsed, + inputs, + map, + }; + main_command(&command, env, streams, verifier, colors) +} + +/// `main()` of `what.py`, after the options have been parsed. +fn main_command( + command: &Command<'_>, + env: &Environment, + streams: &mut Streams<'_>, + verifier: &GoogleVerifier, + colors: Colors, +) -> io::Result { + let interactive = command.flag(Param::Interactive); + if command.inputs.is_empty() && !interactive { + return exit_with(streams, "Text input expected. Run 'pywhat --help' for help"); + } + let dist = match create_filter( + Some(command.value(Param::Rarity).unwrap_or("0.1:1")), + command.value(Param::Include), + command.value(Param::Exclude), + ) { + Ok(filter) => filter, + Err(message) => { + print_line(streams, message)?; + return Ok(1); + } + }; + let boundaryless = if command.flag(Param::DisableBoundaryless) { + Filter::none() + } else { + match create_filter( + Some(command.value(Param::BoundarylessRarity).unwrap_or("0.1:1")), + command.value(Param::BoundarylessInclude), + command.value(Param::BoundarylessExclude), + ) { + Ok(filter) => filter, + Err(message) => { + print_line(streams, message)?; + return Ok(1); + } + } + }; + // The default processors (unless -dp), and the ones that options ask for + let mut processors: Vec> = if command.flag(Param::DisableProcessing) { + Vec::new() + } else { + default_processors() + }; + if command.flag(Param::Verify) { + processors.push(Box::new(verifier.verifier())); + } + if command.map == "osm" { + processors.push(Box::new(OpenStreetMap)); + } + let key = match command.value(Param::Key) { + None => Key::None, + Some(key) => match key.parse::() { + Ok(key) => key, + Err(()) => { + print_line(streams, "Invalid key")?; + return Ok(1); + } + }, + }; + let top = match command.value(Param::Top) { + None => None, + Some(top) => match parse_top(top) { + Ok(top) => Some(top), + Err(message) => { + print_line(streams, &message)?; + return Ok(1); + } + }, + }; + // str(kwargs["format"]).strip() + let format = command.value(Param::Format); + let format_name = python::strip(format.unwrap_or("None")); + let json_output = command.flag(Param::Json) || format_name == "json"; + let stream = command.flag(Param::Stream); + if stream { + if key != Key::None { + return exit_with( + streams, + "--stream cannot sort the matches, so it cannot be used with --key", + ); + } + if !json_output && format_name == "pretty" { + return exit_with( + streams, + "--stream cannot print a table, so it cannot be used with --format pretty", + ); + } + if top.is_some() { + return exit_with( + streams, + "--stream cannot pick the most likely matches, so it cannot be used with --top", + ); + } + if interactive { + return exit_with(streams, "--stream cannot be used with --interactive"); + } + } + if interactive { + streams.stdout.flush()?; + writeln!( + streams.stderr, + "Error: --interactive is only in the Python package of pyWhat: pip install pywhat" + )?; + return Ok(2); + } + + let search = SearchRun { + identifier: Identifier::new(dist, boundaryless, processors), + inputs: &command.inputs, + options: InputOptions { + only_text: command.flag(Param::OnlyText), + include_filenames: command.flag(Param::IncludeFilenames), + }, + print_tags: command.flag(Param::PrintTags), + bars: progress_bars(env, !command.flag(Param::NoProgress)), + colors, + }; + if stream { + // The matches are printed as soon as they are found (issue #189) + let format = if json_output { None } else { format }; + return stream_search(search, json_output, format, streams); + } + let output = OutputOptions { + json: json_output, + pretty: format_name == "pretty", + format, + key, + reverse: command.flag(Param::Reverse), + top, + width: env.rich_width(), + }; + complete_search(search, &output, streams) +} + +/// A search of the inputs, with its progress bars. +struct SearchRun<'a> { + identifier: Identifier, + inputs: &'a [String], + options: InputOptions, + print_tags: bool, + bars: Option<(progress::Bars, progress::Target)>, + colors: Colors, +} + +impl SearchRun<'_> { + /// Searches the inputs, calling `found` for everything found (it returns `false` to + /// stop), and returns the messages for the files that couldn't be read. + fn run(&self, found: &mut dyn FnMut(Found) -> bool) -> Vec { + let mut unreadable = Vec::new(); + self.identifier.iter_identify_with_progress( + self.inputs, + self.options, + found, + &mut |location, error| unreadable.push(format!("Could not read {location}: {error}")), + &|progress| { + if let Some((bars, _)) = &self.bars { + bars.update(progress); + } + }, + ); + unreadable + } + + /// Removes the progress bars. + fn finish_bars(&mut self) { + if let Some((bars, _)) = self.bars.take() { + bars.finish(); + } + } +} + +/// `print_stream()`: prints everything as soon as it is found. +struct StreamPrinter<'a, 'b> { + printer: Printer<'a>, + inputs: &'b [String], + json: bool, + format: Option<&'b str>, + print_tags: bool, + show_files: bool, + /// Whether what was printed last was a file signature, and its location + previous: Option<(bool, String)>, + found_matches: bool, +} + +impl StreamPrinter<'_, '_> { + fn print(&mut self, found: Found) -> io::Result<()> { + if self.json { + self.printer.stream_json(&found)?; + } else { + match found { + Found::FileSignature { + location, + signature, + } => self.signature(location, signature)?, + Found::Match { location, found } => self.found(location, found)?, + } + } + self.printer.out.flush() + } + + /// A file signature, printed like raw output prints it (not with a format). + fn signature( + &mut self, + location: String, + signature: &serde_json::Map, + ) -> io::Result<()> { + if self.format.is_none() { + let mut text = output::raw_signature(&location, signature); + text.rstrip(); + self.printer.print(&text)?; + self.previous = Some((true, location)); + } + Ok(()) + } + + /// A match, printed like raw output or the format prints it. + fn found(&mut self, location: String, found: crate::Match) -> io::Result<()> { + self.found_matches = true; + let one = Identified { + signatures: Vec::new(), + regexes: vec![(location.clone(), vec![found])], + }; + match self.format { + Some(format) => self.printer.format_print(&one, format)?, + None => { + // An empty line between the matches, but not between a file signature and + // the matches in its file + let after_its_signature = self.previous.as_ref() == Some(&(true, location.clone())); + if self.previous.is_some() && !after_its_signature { + self.printer.print_plain("")?; + } + self.printer.print_raw( + &one, + self.inputs, + self.print_tags, + Some(self.show_files), + )?; + } + } + self.previous = Some((false, location)); + Ok(()) + } +} + +/// `--stream`: everything printed as soon as it is found, the progress bars hidden while +/// it is printed on their terminal. +fn stream_search( + mut search: SearchRun<'_>, + json: bool, + format: Option<&str>, + streams: &mut Streams<'_>, +) -> io::Result { + let bars_on_stdout = matches!(&search.bars, Some((_, progress::Target::Stdout))); + let mut printer = StreamPrinter { + printer: Printer { + out: &mut *streams.stdout, + colors: search.colors, + }, + inputs: search.inputs, + json, + format, + print_tags: search.print_tags, + show_files: output::show_files_of_inputs(search.inputs), + previous: None, + found_matches: false, + }; + let mut failure: Option = None; + let unreadable = search.run(&mut |found| { + let printed = match &search.bars { + Some((bars, _)) if bars_on_stdout => bars.suspend(|| printer.print(found)), + _ => printer.print(found), + }; + printed.map_err(|error| failure = Some(error)).is_ok() + }); + let found_matches = printer.found_matches; + search.finish_bars(); + if let Some(error) = failure { + return Err(error); + } + if !json && !found_matches && format.is_none() { + Printer { + out: streams.stdout, + colors: search.colors, + } + .print_plain("Nothing found!")?; + } + finish(streams, &unreadable) +} + +/// How the matches of a complete search are printed. +struct OutputOptions<'a> { + json: bool, + pretty: bool, + format: Option<&'a str>, + key: Key, + reverse: bool, + top: Option, + /// The width of the terminal, for the table + width: usize, +} + +/// The number of matches found. +fn count(identified: &Identified) -> usize { + identified + .regexes + .iter() + .map(|(_, matches)| matches.len()) + .sum() +} + +/// A search, printed once it is complete: sorted, the most likely matches with `--top` +/// (and a note on stderr about the others), in the output format. +fn complete_search( + mut search: SearchRun<'_>, + output: &OutputOptions<'_>, + streams: &mut Streams<'_>, +) -> io::Result { + let mut identified = Identified::default(); + let unreadable = search.run(&mut |found| { + collect(&mut identified, found); + true + }); + search.finish_bars(); + // top_matches() sorts the top ones itself + let key = if output.top.is_none() { + output.key + } else { + Key::None + }; + identified.sort(key, output.reverse); + let total = count(&identified); + if let Some(top) = &output.top { + identified = identified.top(top, output.key, output.reverse); + } + let mut printer = Printer { + out: streams.stdout, + colors: search.colors, + }; + if output.json { + printer.print_json(&identified)?; + } else if output.pretty { + printer.pretty_print(&identified, search.inputs, search.print_tags, output.width)?; + } else if let Some(format) = output.format { + printer.format_print(&identified, format)?; + } else { + printer.print_raw(&identified, search.inputs, search.print_tags, None)?; + } + let shown = count(&identified); + if shown < total { + // On stderr, so that the output (e.g. JSON) is the same as without --top + streams.stdout.flush()?; + writeln!( + streams.stderr, + "\nShowing the {shown} most likely of {total} matches. Use --top to show more \ + of them, or --interactive to search through all of them." + )?; + } + finish(streams, &unreadable) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn terminal(columns: usize) -> StreamInfo { + StreamInfo { + terminal: true, + regular_file: false, + columns: Some(columns), + } + } + + fn file() -> StreamInfo { + StreamInfo { + terminal: false, + regular_file: true, + columns: None, + } + } + + fn pipe() -> StreamInfo { + StreamInfo::default() + } + + fn env(stdout: StreamInfo, stderr: StreamInfo) -> Environment { + Environment { + vars: HashMap::new(), + stdin: pipe(), + stdout, + stderr, + } + } + + #[test] + fn where_the_bars_go() { + use progress::Target; + let target = |env: &Environment| bars_target(env, true).map(|(target, _)| target); + // On the terminal of the output + assert_eq!( + target(&env(terminal(80), terminal(80))), + Some(Target::Stdout) + ); + // On stderr if the output is redirected to a file + assert_eq!(target(&env(file(), terminal(80))), Some(Target::Stderr)); + assert_eq!(target(&env(file(), file())), None); + // Never into a pipe, even if stderr is a terminal (what --json ... | jq) + assert_eq!(target(&env(pipe(), terminal(80))), None); + // --no-progress, and dumb terminals + assert_eq!( + bars_target(&env(terminal(80), terminal(80)), false).map(|t| t.0), + None + ); + let mut dumb = env(terminal(80), terminal(80)); + dumb.vars.insert("TERM", "dumb".into()); + assert_eq!(target(&dumb), None); + } + + #[test] + fn widths_like_click_and_rich() { + let mut env = env(terminal(120), pipe()); + assert_eq!(env.click_columns(), 120); + assert_eq!(env.rich_width(), 120); + env.vars.insert("COLUMNS", "abc".into()); + assert_eq!((env.click_columns(), env.rich_width()), (120, 120)); + env.vars.insert("COLUMNS", "0".into()); + assert_eq!((env.click_columns(), env.rich_width()), (120, 80)); + env.vars.insert("COLUMNS", " 40".into()); + assert_eq!((env.click_columns(), env.rich_width()), (40, 120)); + env.vars.insert("COLUMNS", "200".into()); + assert_eq!((env.click_columns(), env.rich_width()), (200, 200)); + // A dumb terminal is 80 columns wide for rich, unless COLUMNS and LINES are set + env.vars.insert("TERM", "dumb".into()); + assert_eq!(env.rich_width(), 80); + env.vars.insert("LINES", "50".into()); + assert_eq!(env.rich_width(), 200); + assert_eq!(program_name("/usr/local/bin/what"), "what"); + } + + /// Runs the command with `stdin` and returns (exit code, stdout, stderr). + fn run_with(args: &[&str], stdin: &[u8], verifier: &GoogleVerifier) -> (i32, String, String) { + let args: Vec = args.iter().map(|arg| arg.to_string()).collect(); + let env = env(pipe(), pipe()); + let (mut stdin, mut stdout, mut stderr) = (stdin, Vec::new(), Vec::new()); + let mut streams = Streams { + stdin: &mut stdin, + stdout: &mut stdout, + stderr: &mut stderr, + }; + let code = run("what", &args, &env, &mut streams, verifier); + ( + code, + String::from_utf8(stdout).unwrap(), + String::from_utf8(stderr).unwrap(), + ) + } + + fn what(args: &[&str]) -> (i32, String, String) { + run_with(args, b"", &GoogleVerifier::default()) + } + + #[test] + fn exit_codes_and_messages() { + assert_eq!(what(&["-db", "THM{x}"]).0, 0); + assert_eq!( + what(&["--rarity", "a:", "x"]), + ( + 1, + "Invalid rarity argument (float expected)\n".into(), + String::new() + ) + ); + assert_eq!(what(&["--key", "x", "y"]).1, "Invalid key\n"); + let (code, _, stderr) = what(&["--stream", "--key", "name", "x"]); + assert_eq!(code, 1); + assert_eq!( + stderr, + "--stream cannot sort the matches, so it cannot be used with --key\n" + ); + let (code, stdout, stderr) = what(&["--interactive", "x"]); + assert_eq!((code, stdout.as_str()), (2, "")); + assert!(stderr.contains("pip install pywhat"), "{stderr}"); + let (code, _, stderr) = what(&["--map", "bing", "x"]); + assert_eq!(code, 2); + assert!(stderr.ends_with( + "Error: Invalid value for '--map': 'bing' is not one of 'google', 'osm'.\n" + )); + } + + #[test] + fn stdin_is_the_input_without_inputs() { + let (code, stdout, _) = run_with(&["-db"], b"THM{hello}\n", &GoogleVerifier::default()); + assert_eq!(code, 0); + assert!(stdout.starts_with("Matched on: THM{hello}\n"), "{stdout}"); + let (_, stdout, _) = run_with(&["-db"], b"", &GoogleVerifier::default()); + assert_eq!(stdout, "Nothing found!\n"); + } + + #[test] + fn verify_with_a_fake_google() { + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::Arc; + let requests = Arc::new(AtomicUsize::new(0)); + let counter = requests.clone(); + let verifier = GoogleVerifier { + make: Some(Box::new(move || { + let counter = counter.clone(); + verify::GoogleApiKeyVerifier::with_http(Box::new(move |_| { + counter.fetch_add(1, Ordering::SeqCst); + Ok(( + 400, + br#"{"error": {"details": [{"@type": "type.googleapis.com/google.rpc.ErrorInfo", "reason": "API_KEY_INVALID"}]}}"#.to_vec(), + )) + })) + })), + }; + let key = format!("AIzaSy{}", "A00000000000000000000000000000000"); + let text = format!("{key} {key}"); + let (code, stdout, _) = + run_with(&["--verify", "--format", "%n: %d", &text], b"", &verifier); + assert_eq!(code, 0); + assert_eq!( + stdout + .lines() + .filter(|line| line.starts_with("Google API Key")) + .count(), + 2, + "{stdout}" + ); + assert!(stdout.contains("Verification: invalid, Google rejected the key (API_KEY_INVALID)")); + // Once per key, however many times it is found + assert_eq!(requests.load(Ordering::SeqCst), 1); + // Without --verify, nothing is sent + let (_, stdout, _) = run_with(&["--format", "%d", &key], b"", &verifier); + assert!(!stdout.contains("Verification")); + assert_eq!(requests.load(Ordering::SeqCst), 1); + } +} diff --git a/src/cli/output.rs b/src/cli/output.rs new file mode 100644 index 0000000..7ae04a9 --- /dev/null +++ b/src/cli/output.rs @@ -0,0 +1,406 @@ +//! The output formats, ports of `pywhat/printer.py`: raw output, the `--format pretty` +//! table, custom `--format` strings, `--json`, and `--stream`. + +use super::python; +use super::style::{sanitized, Colors, Text, BOLD}; +use crate::files::{Found, Identified}; +use crate::identify::Match; +use crate::json::dumps; +use serde_json::{Map, Value}; +use std::io::{self, Write}; +use std::path::Path; + +/// `str(value)` of a value of a file signature. +fn py_str(value: Option<&Value>) -> String { + match value { + Some(Value::String(text)) => text.clone(), + Some(Value::Null) | None => "None".to_string(), + Some(Value::Bool(true)) => "True".to_string(), + Some(Value::Bool(false)) => "False".to_string(), + Some(other) => dumps(other), + } +} + +/// `str(match["Regex Pattern"]["Rarity"])`: as the regex has it (`1` or `0.5`), or the +/// float a processor gave it. +fn rarity_text(found: &Match) -> String { + match found.regex.to_json_value().get("Rarity") { + Some(rarity) if found.rarity == found.regex.rarity => dumps(rarity), + _ => dumps(&Value::from(found.rarity)), + } +} + +/// Whether the regex of a match has an `"Exploit"` key, even a null one, which makes +/// `--format pretty` show the Exploit column (`_check_if_exploit_in_json`). +fn has_exploit_key(found: &Match) -> bool { + found.regex.to_json_value().contains_key("Exploit") +} + +fn non_empty(text: &Option) -> Option<&str> { + text.as_deref().filter(|text| !text.is_empty()) +} + +/// `_show_files()`: whether to show the file of every match: the input is a directory, or +/// there are several inputs and some of the matches are in files. +pub(crate) fn show_files(identified: &Identified, inputs: &[String]) -> bool { + if inputs.len() == 1 { + return Path::new(&inputs[0]).is_dir(); + } + identified + .regexes + .iter() + .any(|(location, _)| location != "text") +} + +/// `_show_files_of_inputs()`: like [`show_files`] before anything is found, for +/// `--stream`: the input is a directory, or one of several inputs exists. +pub(crate) fn show_files_of_inputs(inputs: &[String]) -> bool { + if inputs.len() == 1 { + return Path::new(&inputs[0]).is_dir(); + } + inputs.iter().any(|input| Path::new(input).exists()) +} + +/// `_raw_signature()`: a file signature as raw output shows it. +pub(crate) fn raw_signature(location: &str, signature: &Map) -> Text { + let mut text = Text::new(); + text.plain("\n") + .label("File Identified") + .plain(": ") + .styled(location, BOLD) + .plain(" with Magic Numbers ") + .markup(&py_str(signature.get("ISO 8859-1"))) + .plain(".\n") + .label("File Description:") + .plain(" ") + .markup(&py_str(signature.get("Description"))) + .plain(".\n"); + text +} + +/// A match as raw output shows it: "Matched on:", "Name:", "Link:", "Description:", +/// "Exploit:" and "Tags:" lines, then an empty line. +fn raw_match(found: &Match, print_tags: bool) -> Text { + let mut output = Text::new(); + output.label("Matched on: ").plain(&found.matched); + output.plain("\n").label("Name: ").markup(&found.regex.name); + if let Some(link) = found.link() { + output.plain("\n").label("Link: ").plain(" ").plain(&link); + } + if let Some(description) = non_empty(&found.description) { + output + .plain("\n") + .label("Description: ") + .markup(description); + } + if let Some(exploit) = non_empty(&found.exploit) { + output.plain("\n").label("Exploit: ").plain(exploit); + } + if print_tags { + output + .plain("\n") + .label("Tags: ") + .markup(&found.tags.join(", ")); + } + output.plain("\n\n"); + output +} + +/// The Description cell of the table: the link, the description and the tags, each on +/// a line, or "None". +fn description_cell(found: &Match, print_tags: bool) -> String { + let mut lines: Vec = Vec::new(); + if let Some(link) = found.link() { + let mut text = Text::new(); + text.plain("Click here to analyse in the browser\n") + .plain(&link); + lines.push(text); + } + if let Some(description) = non_empty(&found.description) { + let mut text = Text::new(); + text.markup(description); + lines.push(text); + } + if print_tags { + let mut text = Text::new(); + text.plain("Tags: ").markup(&found.tags.join(", ")); + lines.push(text); + } + if lines.is_empty() { + return "None".to_string(); + } + let mut cell = Text::new(); + for (index, line) in lines.into_iter().enumerate() { + if index > 0 { + cell.plain("\n"); + } + cell.append(line); + } + cell.to_plain() +} + +/// Where the output goes, and how its styles are written. +pub(crate) struct Printer<'a> { + pub(crate) out: &'a mut dyn Write, + pub(crate) colors: Colors, +} + +impl Printer<'_> { + /// `console.print(text)`: the text and a line break. + pub(crate) fn print(&mut self, text: &Text) -> io::Result<()> { + let mut rendered = text.render(self.colors); + rendered.push('\n'); + self.out.write_all(rendered.as_bytes()) + } + + /// `console.print(text, markup=False)` of plain text. + pub(crate) fn print_plain(&mut self, text: &str) -> io::Result<()> { + let mut plain = Text::new(); + plain.plain(text); + self.print(&plain) + } + + /// `print_raw()`, with `show_files` given or found out. + pub(crate) fn print_raw( + &mut self, + identified: &Identified, + inputs: &[String], + print_tags: bool, + show_files: Option, + ) -> io::Result<()> { + let mut output = Text::new(); + if !identified.signatures.is_empty() && !identified.regexes.is_empty() { + for (location, signature) in &identified.signatures { + output.append(raw_signature(location, signature)); + } + } + if !identified.regexes.is_empty() { + let show_files = show_files.unwrap_or_else(|| self::show_files(identified, inputs)); + for (location, matches) in &identified.regexes { + for found in matches { + if show_files { + // One span, as in Python's "[bold #D7Afff]File: {key}[/bold #D7Afff]" + output.label(&format!("File: {location}")).plain("\n"); + } + output.append(raw_match(found, print_tags)); + } + } + } + let source = output.source(); + if source.is_empty() { + self.print_plain("Nothing found!")?; + } + if !python::strip(&source).is_empty() { + output.rstrip(); + self.print(&output)?; + } + Ok(()) + } + + /// `format_print()`: `format` for every match, with `%m`, `%n`, `%d`, `%e`, `%r`, `%l` + /// and `%t` replaced, one after another, and `\\` and `\%` for `\` and `%`. The + /// output is rich markup, like in Python; its `MarkupError` is an `InvalidInput` error. + pub(crate) fn format_print(&mut self, identified: &Identified, format: &str) -> io::Result<()> { + if identified.regexes.is_empty() { + return Ok(()); + } + // The format split at its "\\", which become "\" (format_list) + let mut parts: Vec<&str> = Vec::new(); + let mut rest = format; + while let Some(index) = rest.find("\\\\") { + if index > 0 { + parts.push(&rest[..index]); + } + parts.push("\\"); + rest = &rest[index + 2..]; + } + parts.push(rest); + // The output as Python gives it to rich: markup, which the format can use too. The + // values that contain the matched text are escaped, so that they are printed as + // they are (#291, Python bug 4); the names, descriptions and tags are data, whose + // markup rich renders. + let mut source = Vec::new(); + for (_, matches) in &identified.regexes { + for found in matches { + let values = [ + ("%m", super::markup::escape(&found.matched)), + ("%n", found.regex.name.clone()), + ("%d", found.description.clone().unwrap_or_default()), + ( + "%e", + super::markup::escape(found.exploit.as_deref().unwrap_or("")), + ), + ("%r", rarity_text(found)), + ( + "%l", + super::markup::escape(&found.link().unwrap_or_default()), + ), + ("%t", found.tags.join(", ")), + ]; + source.push(substitute(&parts, &values)); + } + } + let source = source.join("\n"); + if !python::strip(&source).is_empty() { + // rich stops at a closing tag that closes nothing, before printing anything + if let Some(error) = super::markup::error(&source) { + return Err(io::Error::new(io::ErrorKind::InvalidInput, error)); + } + let mut text = Text::new(); + text.markup(&source); + self.print(&text)?; + } + Ok(()) + } + + /// `print_json()`: `json.dumps()` of everything found, on one line. + pub(crate) fn print_json(&mut self, identified: &Identified) -> io::Result<()> { + let mut json = identified.to_json(); + json.push('\n'); + self.out.write_all(json.as_bytes()) + } + + /// `print_stream()` with `json_output`: a JSON line for everything found, as soon as it + /// is found, without the "Fragment" of the matches. + pub(crate) fn stream_json(&mut self, found: &Found) -> io::Result<()> { + let mut line = String::new(); + match found { + Found::FileSignature { + location, + signature, + } => { + line.push_str("{\"File Signatures\": {"); + line.push_str(&dumps(&Value::String(location.clone()))); + line.push_str(": "); + line.push_str(&dumps(&Value::Object((*signature).clone()))); + line.push_str("}, \"Regexes\": null}"); + } + Found::Match { location, found } => { + line.push_str("{\"File Signatures\": null, \"Regexes\": {"); + line.push_str(&dumps(&Value::String(location.clone()))); + line.push_str(": ["); + found.write_json(&mut line, false); + line.push_str("]}}"); + } + } + self.print_plain(&line) + } + + /// `--format pretty`: the file signatures, and a table of the matches. + pub(crate) fn pretty_print( + &mut self, + identified: &Identified, + inputs: &[String], + print_tags: bool, + width: usize, + ) -> io::Result<()> { + let mut text = Text::new(); + for (location, signature) in &identified.signatures { + text.plain("\n") + .label("File Identified") + .plain(": ") + .styled(location, BOLD) + .plain(" with Magic Numbers ") + .markup(&py_str(signature.get("ISO 8859-1"))) + .plain(".\n") + .label("File Description: ") + .plain(" ") + .markup(&py_str(signature.get("Description"))) + .plain(".\n\n"); + } + if identified.regexes.is_empty() { + text.plain("\nNothing found!"); + text.lstrip(); + return self.print(&text); + } + text.plain("\n") + .label("Possible Identification") + .plain("\n"); + let show_files = show_files(identified, inputs); + let bug_bounty = identified + .regexes + .iter() + .flat_map(|(_, matches)| matches) + .any(has_exploit_key); + let mut header = vec!["Matched Text", "Identified as", "Description"]; + if show_files { + header.push("File"); + } + if bug_bounty { + header.push("Exploit"); + } + let mut rows: Vec> = Vec::new(); + for (location, matches) in &identified.regexes { + for found in matches { + let mut row = vec![ + sanitized(&found.matched), + sanitized(&super::markup::plain(&found.regex.name)), + sanitized(&description_cell(found, print_tags)), + ]; + if show_files { + row.push(sanitized(location)); + } + if bug_bounty { + row.push(sanitized(non_empty(&found.exploit).unwrap_or(""))); + } + rows.push(row); + } + } + text.strip(); + let mut out = text.render(self.colors); + out.push('\n'); + out.push_str(&super::table::render(&header, &rows, width, self.colors)); + out.push('\n'); + self.out.write_all(out.as_bytes()) + } +} + +/// The substitutions of `format_print()` in each part of the format: `re.sub(r"(? String { + let mut out = String::new(); + for part in parts { + let mut text = part.to_string(); + for (format, value) in values { + text = replace_unescaped(&text, format, value); + } + out.push_str(&text.replace("\\%", "%")); + } + out +} + +/// `re.sub(r"(? String { + let mut out = String::with_capacity(text.len()); + let mut previous: Option = None; + let mut index = 0; + while index < text.len() { + if text[index..].starts_with(pattern) && previous != Some('\\') { + out.push_str(value); + index += pattern.len(); + previous = pattern.chars().last(); + continue; + } + let c = text[index..].chars().next().expect("a character"); + out.push(c); + previous = Some(c); + index += c.len_utf8(); + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn substitutions_like_re_sub() { + assert_eq!(replace_unescaped("%m\\%m%m", "%m", "X"), "X\\%mX"); + // The value of %m is searched for the next formats, as Python does + let values = [("%m", "thm{%n}".to_string()), ("%n", "Name".to_string())]; + assert_eq!(substitute(&["%m"], &values), "thm{Name}"); + assert_eq!(substitute(&["\\%d %m"], &values), "%d thm{Name}"); + assert_eq!(substitute(&["\\", "%m"], &values), "\\thm{Name}"); + } +} diff --git a/src/cli/progress.rs b/src/cli/progress.rs new file mode 100644 index 0000000..f68ea5d --- /dev/null +++ b/src/cli/progress.rs @@ -0,0 +1,460 @@ +//! Progress bars for searches that take a while, like `pywhat/progress.py`: a bar for the +//! files (if there are several) with the file that is searched, and a bar for the regexes +//! of the file with the regex that is searched for. They appear once the search has taken +//! half a second, only on a terminal, and are removed when the search is complete. +//! indicatif draws them, in the look of rich's progress bars. + +use super::style::{write_styled, ColorSystem, Colors, Style, LABEL}; +use crate::files::Progress; +use indicatif::{MultiProgress, ProgressBar, ProgressDrawTarget, ProgressStyle}; +use std::sync::{Arc, Condvar, Mutex}; +use std::thread::JoinHandle; +use std::time::{Duration, Instant}; + +/// How long a search runs before the bars appear, so that they don't flicker for searches +/// that are over in a moment. +pub(crate) const DELAY: Duration = Duration::from_millis(500); +/// How often the bars are drawn. +const REFRESH: Duration = Duration::from_millis(100); +const BAR_WIDTH: usize = 30; + +/// rich's `bar.complete` and `bar.finished` styles. +const COMPLETE: (u8, u8, u8) = (249, 38, 114); +const FINISHED: (u8, u8, u8) = (114, 156, 31); + +/// Where the bars are drawn. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Target { + Stdout, + Stderr, +} + +/// The latest progress. +#[derive(Debug, Default, Clone)] +struct State { + seen: bool, + location: String, + files_done: usize, + files: usize, + regexes_done: usize, + regexes: usize, + regex: Option<&'static str>, +} + +/// Running progress bars, the progress callback of the search. +pub(crate) struct Bars { + state: Arc>, + multi: MultiProgress, + done: Arc<(Mutex, Condvar)>, + ticker: Option>, +} + +/// Control characters (e.g. in file names) would mess up the terminal. +fn printable(text: &str) -> String { + text.chars() + .filter(|&c| !matches!(c, '\u{0}'..='\u{1f}' | '\u{7f}'..='\u{9f}')) + .collect() +} + +/// `str(timedelta(seconds=...))`: H:MM:SS. +fn elapsed(seconds: u64) -> String { + format!( + "{}:{:02}:{:02}", + seconds / 3600, + seconds / 60 % 60, + seconds % 60 + ) +} + +fn styled(text: &str, style: &Style, colors: Colors) -> String { + let mut out = String::new(); + write_styled(&mut out, text, style, colors); + out +} + +/// `rich.progress_bar.ProgressBar` of `completed` of `total`, `width` characters wide. +fn bar(completed: usize, total: usize, width: usize, colors: Colors) -> String { + let completed = completed.min(total); + let halves = (width * 2 * completed) + .checked_div(total) + .unwrap_or(width * 2); + let (bars, half) = (halves / 2, halves % 2); + let color = if completed < total { + COMPLETE + } else { + FINISHED + }; + let complete = Style { + color: Some(color), + ..Style::default() + }; + let mut out = styled(&"━".repeat(bars), &complete, colors); + if half > 0 { + out.push_str(&styled("╸", &complete, colors)); + } + let mut remaining = width - bars - half; + if colors.no_color || colors.is_plain() { + // rich only draws the rest of the bar in colour + out.push_str(&" ".repeat(remaining)); + return out; + } + let mut back = String::new(); + if half == 0 && bars > 0 && remaining > 0 { + back.push('╺'); + remaining -= 1; + } + back.push_str(&"━".repeat(remaining)); + if back.is_empty() { + return out; + } + // grey23, an 8-bit colour + let code = match colors.system { + ColorSystem::Standard => "90".to_string(), + _ => "38;5;237".to_string(), + }; + out.push_str(&format!("\x1b[{code}m{back}\x1b[0m")); + out +} + +/// `text` cut to `width` characters with an ellipsis, like rich's `overflow="ellipsis"`. +fn ellipsis(text: &str, width: usize) -> String { + if text.chars().count() <= width { + return text.to_string(); + } + if width == 0 { + return String::new(); + } + let mut cut: String = text.chars().take(width - 1).collect(); + cut.push('…'); + cut +} + +/// The lines of the bars (`ProgressBars.render()`): a grid of the label, the bar, how far +/// it is, the time since the search started and the file or the regex, one space between +/// the columns, and only the last column cut if the lines are wider than the terminal. +fn lines(state: &State, seconds: u64, width: usize, colors: Colors) -> Vec { + if !state.seen { + return vec![styled("Searching...", &LABEL, colors)]; + } + // Narrower bars on a narrow terminal, to leave room for the names + let bar_width = (width / 4).clamp(10, BAR_WIDTH); + let time = elapsed(seconds); + // One regex is 1% if there are 100 regexes to search for in the file + let total = state.regexes.max(1); + let done = if state.regexes > 0 { + state.regexes_done + } else { + 1 + }; + let mut rows: Vec<(&str, usize, usize, String, String, String)> = Vec::new(); + if state.files > 1 { + rows.push(( + "Files", + state.files_done, + state.files, + format!("{}/{}", state.files_done, state.files), + time.clone(), + printable(&state.location), + )); + } + rows.push(( + "Regexes", + done, + total, + format!("{}%", 100 * done / total), + if state.files > 1 { String::new() } else { time }, + printable(state.regex.unwrap_or("")), + )); + let label_width = rows.iter().map(|row| row.0.len()).max().unwrap_or(0); + let count_width = rows.iter().map(|row| row.3.len()).max().unwrap_or(0); + let time_width = rows.iter().map(|row| row.4.len()).max().unwrap_or(0); + let used = label_width + bar_width + count_width + time_width + 4; + let name_width = width.saturating_sub(used + 1).max(1); + rows.into_iter() + .map(|(label, completed, total, count, time, name)| { + let mut line = styled(&format!("{label:count_width$} {time: Bars { + let draw_target = move || match target { + Target::Stdout => ProgressDrawTarget::stdout(), + Target::Stderr => ProgressDrawTarget::stderr(), + }; + Bars::start_on(draw_target, width, colors) + } + + /// Bars drawn on the target that `draw_target` makes once they appear. + fn start_on( + draw_target: impl FnOnce() -> ProgressDrawTarget + Send + 'static, + width: usize, + colors: Colors, + ) -> Bars { + let state = Arc::new(Mutex::new(State::default())); + let done = Arc::new((Mutex::new(false), Condvar::new())); + let multi = MultiProgress::with_draw_target(ProgressDrawTarget::hidden()); + let ticker = { + let (state, done, multi) = (state.clone(), done.clone(), multi.clone()); + std::thread::spawn(move || { + let (lock, condvar) = &*done; + let wait = |timeout| { + let finished = lock + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let (finished, _) = condvar + .wait_timeout_while(finished, timeout, |finished| !*finished) + .unwrap_or_else(std::sync::PoisonError::into_inner); + *finished + }; + let started = Instant::now(); + if wait(DELAY) { + return; + } + multi.set_draw_target(draw_target()); + let style = ProgressStyle::with_template("{msg}").expect("valid template"); + let mut shown: Vec = Vec::new(); + loop { + let latest = state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone(); + let lines = lines(&latest, started.elapsed().as_secs(), width, colors); + while shown.len() < lines.len() { + // A files bar appears above the regexes bar + let bar = ProgressBar::new(1).with_style(style.clone()); + shown.insert( + 0, + match shown.first() { + Some(first) => multi.insert_before(first, bar), + None => multi.add(bar), + }, + ); + } + let offset = shown.len() - lines.len(); + for (bar, line) in shown[offset..].iter().zip(lines) { + bar.set_message(line); + } + if wait(REFRESH) { + break; + } + } + for bar in &shown { + bar.finish_and_clear(); + } + let _ = multi.clear(); + }) + }; + Bars { + state, + multi, + done, + ticker: Some(ticker), + } + } + + /// The progress callback: keeps the latest progress, which is drawn 10 times a second. + pub(crate) fn update(&self, progress: Progress<'_>) { + let mut state = self + .state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + state.seen = true; + if state.location != progress.location { + state.location = progress.location.to_string(); + } + state.files_done = progress.files_done; + state.files = progress.files; + state.regexes_done = progress.regexes_done; + state.regexes = progress.regexes; + state.regex = progress.regex; + } + + /// Runs `print` with the bars hidden, so that its output goes above them. + pub(crate) fn suspend(&self, print: impl FnOnce() -> R) -> R { + self.multi.suspend(print) + } + + /// Removes the bars. + pub(crate) fn finish(mut self) { + self.stop(); + } + + fn stop(&mut self) { + let (lock, condvar) = &*self.done; + *lock + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = true; + condvar.notify_all(); + if let Some(ticker) = self.ticker.take() { + let _ = ticker.join(); + } + } +} + +impl Drop for Bars { + fn drop(&mut self) { + self.stop(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn state(files: usize, regexes_done: usize, regex: &'static str) -> State { + State { + seen: true, + location: "fixtures/test/file".into(), + files_done: 1, + files, + regexes_done, + regexes: 150, + regex: Some(regex), + } + } + + #[test] + fn lines_like_rich() { + let plain = Colors::PLAIN; + assert_eq!(lines(&State::default(), 0, 80, plain), ["Searching..."]); + // One file: the regexes bar with the time; 80 columns give 20-character bars + assert_eq!( + lines( + &state(1, 75, "Uniform Resource Locator (URL)"), + 3, + 80, + plain + ), + [format!( + "Regexes {}{} 50% 0:00:03 Uniform Resource Locator (URL)", + "━".repeat(10), + " ".repeat(10) + )] + ); + // Several files: the files bar first, with the time, and the regexes bar + let two = lines(&state(2, 150, "URL"), 61, 80, plain); + assert_eq!( + two, + [ + format!( + "Files {}{} 1/2 0:01:01 fixtures/test/file", + "━".repeat(10), + " ".repeat(10) + ), + format!("Regexes {} 100% URL", "━".repeat(20)), + ] + ); + // A name that doesn't fit is cut with an ellipsis, one column short of the width + let narrow = lines(&state(1, 0, "Uniform Resource Locator (URL)"), 0, 50, plain); + assert_eq!(narrow[0].chars().count(), 49); + assert!(narrow[0].ends_with(" Uniform Resource…"), "{narrow:?}"); + } + + #[test] + fn rich_bar() { + let colors = Colors { + system: ColorSystem::TrueColor, + no_color: false, + }; + assert_eq!( + bar(1, 4, 4, colors), + "\x1b[38;2;249;38;114m━\x1b[0m\x1b[38;5;237m╺━━\x1b[0m" + ); + assert_eq!(bar(3, 8, 4, colors).matches('╸').count(), 1); + assert_eq!(bar(4, 4, 4, colors), "\x1b[38;2;114;156;31m━━━━\x1b[0m"); + } + + /// A terminal that records what is written on it. + #[derive(Debug, Clone, Default)] + struct Recorder(Arc>); + + impl indicatif::TermLike for Recorder { + fn width(&self) -> u16 { + 80 + } + fn move_cursor_up(&self, _: usize) -> std::io::Result<()> { + Ok(()) + } + fn move_cursor_down(&self, _: usize) -> std::io::Result<()> { + Ok(()) + } + fn move_cursor_right(&self, _: usize) -> std::io::Result<()> { + Ok(()) + } + fn move_cursor_left(&self, _: usize) -> std::io::Result<()> { + Ok(()) + } + fn write_line(&self, line: &str) -> std::io::Result<()> { + self.0.lock().unwrap().push_str(&format!("{line}\n")); + Ok(()) + } + fn write_str(&self, text: &str) -> std::io::Result<()> { + self.0.lock().unwrap().push_str(text); + Ok(()) + } + fn clear_line(&self) -> std::io::Result<()> { + Ok(()) + } + fn flush(&self) -> std::io::Result<()> { + Ok(()) + } + } + + #[test] + fn bars_appear_after_half_a_second() { + let terminal = Recorder::default(); + let target = terminal.clone(); + let started = Instant::now(); + let bars = Bars::start_on( + move || ProgressDrawTarget::term_like(Box::new(target)), + 80, + Colors::PLAIN, + ); + bars.update(Progress { + location: "big.log", + files_done: 0, + files: 1, + regexes_done: 3, + regexes: 10, + regex: Some("Uniform Resource Locator (URL)"), + }); + std::thread::sleep(Duration::from_millis(300)); + assert_eq!(*terminal.0.lock().unwrap(), "", "nothing before {DELAY:?}"); + while started.elapsed() < DELAY + Duration::from_millis(400) { + std::thread::sleep(Duration::from_millis(50)); + } + let drawn = terminal.0.lock().unwrap().clone(); + assert!(drawn.contains("Regexes"), "{drawn:?}"); + assert!( + drawn.contains("30% 0:00:00 Uniform Resource Locator (URL)"), + "{drawn:?}" + ); + bars.finish(); + } + + #[test] + fn bars_finish_before_the_delay() { + // Nothing is drawn, and finishing doesn't wait for the delay + let started = Instant::now(); + let bars = Bars::start(Target::Stderr, 80, Colors::PLAIN); + bars.update(Progress { + location: "text", + files_done: 0, + files: 1, + regexes_done: 0, + regexes: 10, + regex: Some("URL"), + }); + bars.finish(); + assert!(started.elapsed() < DELAY); + assert_eq!(printable("a\x1b[1mb\u{85}"), "a[1mb"); + } +} diff --git a/src/cli/python.rs b/src/cli/python.rs new file mode 100644 index 0000000..6a38e22 --- /dev/null +++ b/src/cli/python.rs @@ -0,0 +1,320 @@ +//! What Python's built-ins do with strings, where the CLI has to do the same: `str.strip()`, +//! `float()`, `int()`, `repr()` and `str.casefold()`. + +use std::fmt::Write; +use std::sync::OnceLock; + +/// `str.isspace()` of one character: Rust's `char::is_whitespace` and U+001C to U+001F, +/// which Python counts as whitespace too. +pub(crate) fn is_space(c: char) -> bool { + c.is_whitespace() || ('\u{1c}'..='\u{1f}').contains(&c) +} + +/// `str.strip()`. +pub(crate) fn strip(text: &str) -> &str { + text.trim_matches(is_space) +} + +/// `str.rstrip()`. +pub(crate) fn rstrip(text: &str) -> &str { + text.trim_end_matches(is_space) +} + +/// `str.lstrip()`. +pub(crate) fn lstrip(text: &str) -> &str { + text.trim_start_matches(is_space) +} + +/// `str.isspace()`: not empty, and only whitespace. +pub(crate) fn is_all_space(text: &str) -> bool { + !text.is_empty() && text.chars().all(is_space) +} + +fn unicode_class( + pattern: &'static str, + cell: &'static OnceLock, +) -> &'static regex::Regex { + cell.get_or_init(|| regex::Regex::new(pattern).expect("valid class")) +} + +/// The value of a decimal digit (Unicode category Nd), like `unicodedata.decimal()`. +/// Decimal digits are encoded in runs of ten, 0 to 9, so the value is the position in +/// the run. +pub(crate) fn decimal_value(c: char) -> Option { + if c.is_ascii_digit() { + return c.to_digit(10); + } + static DIGIT: OnceLock = OnceLock::new(); + let digit = unicode_class(r"^\p{Nd}$", &DIGIT); + let is_digit = |c: char| digit.is_match(c.encode_utf8(&mut [0; 4])); + if !is_digit(c) { + return None; + } + let mut before = 0; + let mut code = c as u32; + while let Some(previous) = code.checked_sub(1).and_then(char::from_u32) { + if !is_digit(previous) { + break; + } + before += 1; + code -= 1; + } + Some(before % 10) +} + +/// `_PyUnicode_TransformDecimalAndSpaceToASCII()`, which `float()` and `int()` start +/// with: non-ASCII whitespace becomes a space and non-ASCII decimal digits ASCII digits. +/// `None` if there is any other non-ASCII character, which makes the number invalid. +fn transform_decimal_and_space(text: &str) -> Option { + let mut out = String::with_capacity(text.len()); + for c in text.chars() { + if (c as u32) < 127 { + out.push(c); + } else if is_space(c) { + out.push(' '); + } else { + out.push(char::from_digit(decimal_value(c)?, 10)?); + } + } + Some(out) +} + +/// `_Py_string_to_number_with_underscores()`: the text without underscores, which are +/// only allowed between two digits. +fn remove_underscores(text: &str) -> Option { + if !text.contains('_') { + return Some(text.to_string()); + } + let mut out = String::with_capacity(text.len()); + let mut previous = '\0'; + for c in text.chars() { + if c == '_' { + if !previous.is_ascii_digit() { + return None; + } + } else { + if previous == '_' && !c.is_ascii_digit() { + return None; + } + out.push(c); + } + previous = c; + } + (previous != '_').then_some(out) +} + +/// The whitespace that `float()` and `int()` strip (`Py_ISSPACE`), after the transform. +fn is_ascii_space(c: char) -> bool { + matches!(c, ' ' | '\t' | '\n' | '\x0b' | '\x0c' | '\r') +} + +/// `float(text)`, or `None` where Python raises `ValueError`. +pub(crate) fn float(text: &str) -> Option { + let ascii = transform_decimal_and_space(text)?; + let without_underscores = remove_underscores(&ascii)?; + let number = without_underscores.trim_matches(is_ascii_space); + // Python and Rust read the same decimal numbers, infinities and NaNs (in any case), + // both correctly rounded; Rust doesn't take a NUL or anything after the number either + number.parse::().ok() +} + +/// `int(text)`, or `None` where Python raises `ValueError`. Saturates at the limits of +/// `i128`, which is enough for comparisons. +pub(crate) fn int(text: &str) -> Option { + let ascii = transform_decimal_and_space(text)?; + let number = ascii.trim_matches(is_ascii_space); + let (negative, digits) = match number.as_bytes().first()? { + b'-' => (true, &number[1..]), + b'+' => (false, &number[1..]), + _ => (false, number), + }; + if digits.is_empty() || digits.starts_with('_') { + return None; + } + let digits = remove_underscores(digits)?; + if digits.is_empty() || !digits.bytes().all(|b| b.is_ascii_digit()) { + return None; + } + let mut value: i128 = 0; + for b in digits.bytes() { + value = value + .saturating_mul(10) + .saturating_add(i128::from(b - b'0')); + } + Some(if negative { -value } else { value }) +} + +/// `str.isprintable()` of a non-ASCII character: not in the categories Cc, Cf, Cs, Co, +/// Cn, Zl, Zp and Zs. +fn is_printable(c: char) -> bool { + static UNPRINTABLE: OnceLock = OnceLock::new(); + !unicode_class( + r"^[\p{Cc}\p{Cf}\p{Co}\p{Cn}\p{Zl}\p{Zp}\p{Zs}]$", + &UNPRINTABLE, + ) + .is_match(c.encode_utf8(&mut [0; 4])) +} + +/// `repr(text)`: the string in quotes, with escapes for what isn't printable. +pub(crate) fn repr(text: &str) -> String { + let quote = if text.contains('\'') && !text.contains('"') { + '"' + } else { + '\'' + }; + let mut out = String::with_capacity(text.len() + 2); + out.push(quote); + for c in text.chars() { + let code = c as u32; + match c { + '\\' => out.push_str("\\\\"), + '\t' => out.push_str("\\t"), + '\n' => out.push_str("\\n"), + '\r' => out.push_str("\\r"), + _ if c == quote => { + out.push('\\'); + out.push(c); + } + _ if code < 0x20 || code == 0x7f => { + let _ = write!(out, "\\x{code:02x}"); + } + _ if c.is_ascii() || is_printable(c) => out.push(c), + _ if code < 0x100 => { + let _ = write!(out, "\\x{code:02x}"); + } + _ if code < 0x10000 => { + let _ = write!(out, "\\u{code:04x}"); + } + _ => { + let _ = write!(out, "\\U{code:08x}"); + } + } + } + out.push(quote); + out +} + +/// `text.casefold()` if it is ASCII, which is all that a comparison with an ASCII string +/// needs: `None` if the case folded text isn't ASCII. +pub(crate) fn casefold_ascii(text: &str) -> Option { + let mut out = String::with_capacity(text.len()); + for c in text.chars() { + let folded = match c { + _ if c.is_ascii() => { + out.push(c.to_ascii_lowercase()); + continue; + } + 'ß' | 'ẞ' => "ss", + 'ſ' => "s", + 'K' => "k", + 'ff' => "ff", + 'fi' => "fi", + 'fl' => "fl", + 'ffi' => "ffi", + 'ffl' => "ffl", + 'ſt' | 'st' => "st", + _ => return None, + }; + out.push_str(folded); + } + Some(out) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn whitespace() { + assert_eq!(strip("\u{1c} a \u{3000}"), "a"); + assert!(is_all_space(" \u{1f}")); + assert!(!is_all_space("")); + assert_eq!(rstrip("a\n\n"), "a"); + assert_eq!(lstrip("\t a"), "a"); + } + + #[test] + fn floats_like_python() { + // Checked with Python 3.10 + for (text, expected) in [ + ("0.5", Some(0.5)), + (" 1 ", Some(1.0)), + ("\u{2000}1", Some(1.0)), + ("١.٥", Some(1.5)), + ("1_0", Some(10.0)), + ("١_٠", Some(10.0)), + ("1e5", Some(1e5)), + (".5", Some(0.5)), + ("5.", Some(5.0)), + ("-Infinity", Some(f64::NEG_INFINITY)), + ("1e400", Some(f64::INFINITY)), + ("\u{1c}1.5\u{1f}", None), + ("١٫٥", None), + ("1__0", None), + ("_1", None), + ("1_", None), + ("1_.5", None), + ("1._5", None), + (" 1e_5", None), + ("", None), + (" ", None), + ("0x10", None), + ("1,5", None), + ("nan_", None), + ("1\u{0}", None), + ] { + assert_eq!(float(text), expected, "{text:?}"); + } + assert!(float("nAn").unwrap().is_nan()); + assert!(float("+nan").unwrap().is_nan()); + } + + #[test] + fn ints_like_python() { + for (text, expected) in [ + ("٨٠", Some(80)), + (" 80 ", Some(80)), + ("+80", Some(80)), + ("8_0", Some(80)), + ("-5", Some(-5)), + ("080", Some(80)), + ("80.0", None), + ("", None), + ("-", None), + ("_8", None), + ("8_", None), + ("-_8", None), + ] { + assert_eq!(int(text), expected, "{text:?}"); + } + assert_eq!(int(&"9".repeat(60)), Some(i128::MAX)); + } + + #[test] + fn repr_like_python() { + assert_eq!(repr("x"), "'x'"); + assert_eq!(repr("it's"), "\"it's\""); + assert_eq!(repr("'\""), "'\\'\"'"); + assert_eq!(repr("a\\b\n\t\r\x01\x7f"), "'a\\\\b\\n\\t\\r\\x01\\x7f'"); + assert_eq!( + repr("é₿\u{a0}\u{200b}\u{2028}😂"), + "'é₿\\xa0\\u200b\\u2028😂'" + ); + assert_eq!(repr("\u{e000}\u{10ffff}"), "'\\ue000\\U0010ffff'"); + } + + #[test] + fn casefold() { + assert_eq!(casefold_ascii("OſM").as_deref(), Some("osm")); + assert_eq!(casefold_ascii("Google").as_deref(), Some("google")); + assert_eq!(casefold_ascii("goögle"), None); + } + + #[test] + fn decimal_digits() { + assert_eq!(decimal_value('٣'), Some(3)); + assert_eq!(decimal_value('𝟗'), Some(9)); // U+1D7D7, in a run of five runs of ten + assert_eq!(decimal_value('²'), None); + assert_eq!(decimal_value('a'), None); + } +} diff --git a/src/cli/style.rs b/src/cli/style.rs new file mode 100644 index 0000000..f086343 --- /dev/null +++ b/src/cli/style.rs @@ -0,0 +1,502 @@ +//! Styled text and how it is written: plain text when the output isn't a terminal, and +//! the escape sequences of rich's colour systems on a terminal. Like rich, everything that +//! is printed loses its backspaces, vertical tabs, form feeds and carriage returns, and +//! its tabs become spaces (tab stops every 8 characters). + +use std::fmt::Write; + +/// A style: bold, a colour and a link. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub(crate) struct Style { + pub(crate) bold: bool, + pub(crate) color: Option<(u8, u8, u8)>, + pub(crate) link: Option, +} + +/// `bold #D7Afff`, the style of pyWhat's labels ("Matched on: "). +pub(crate) const LABEL: Style = Style { + bold: true, + color: Some((0xD7, 0xAF, 0xFF)), + link: None, +}; + +/// `bold`. +pub(crate) const BOLD: Style = Style { + bold: true, + color: None, + link: None, +}; + +/// `bold #49C3CE`, the style of the version number. +pub(crate) const VERSION: Style = Style { + bold: true, + color: Some((0x49, 0xC3, 0xCE)), + link: None, +}; + +/// The colours that a terminal shows, as rich detects them. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum ColorSystem { + /// Not a terminal: no escape sequences at all. + None, + Standard, + EightBit, + TrueColor, +} + +/// How to write styles. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct Colors { + pub(crate) system: ColorSystem, + /// `NO_COLOR`: bold and links, but no colours. + pub(crate) no_color: bool, +} + +impl Colors { + pub(crate) const PLAIN: Colors = Colors { + system: ColorSystem::None, + no_color: false, + }; + + /// rich's `Console._detect_color_system()` for a stream that is a terminal (or not). + pub(crate) fn detect(is_terminal: bool, env: &dyn Fn(&str) -> Option) -> Colors { + if !is_terminal || is_dumb(env("TERM").as_deref()) { + return Colors::PLAIN; + } + let term = env("TERM").unwrap_or_default().trim().to_lowercase(); + let system = if cfg!(windows) { + // Windows 10 and later understand the escape sequences of 24-bit colours once + // they are enabled; older consoles get none + #[cfg(windows)] + let enabled = anstyle_query::windows::enable_ansi_colors().unwrap_or(false); + #[cfg(not(windows))] + let enabled = true; + if enabled { + ColorSystem::TrueColor + } else { + ColorSystem::None + } + } else { + let color_term = env("COLORTERM").unwrap_or_default().trim().to_lowercase(); + if color_term == "truecolor" || color_term == "24bit" { + ColorSystem::TrueColor + } else { + // term.rpartition("-")[2]: after the last "-", or all of it + match term.rsplit('-').next().unwrap_or("") { + "256color" => ColorSystem::EightBit, + _ => ColorSystem::Standard, + } + } + }; + Colors { + system, + no_color: env("NO_COLOR").is_some(), + } + } + + pub(crate) fn is_plain(self) -> bool { + self.system == ColorSystem::None + } +} + +/// rich's `is_dumb_terminal` for a terminal: `TERM` is "dumb" or "unknown", in any case, +/// without stripping it. +pub(crate) fn is_dumb(term: Option<&str>) -> bool { + let term = term.unwrap_or("").to_lowercase(); + term == "dumb" || term == "unknown" +} + +/// rich's `rgb_to_hls()` saturation and lightness (`colorsys`). +fn lightness_saturation(r: f64, g: f64, b: f64) -> (f64, f64) { + let max = r.max(g).max(b); + let min = r.min(g).min(b); + let lightness = (max + min) / 2.0; + if max == min { + return (lightness, 0.0); + } + let saturation = if lightness <= 0.5 { + (max - min) / (max + min) + } else { + (max - min) / (2.0 - max - min) + }; + (lightness, saturation) +} + +/// rich's `Color.downgrade(EIGHT_BIT)` of a 24-bit colour. +fn eight_bit((r, g, b): (u8, u8, u8)) -> u8 { + let (r, g, b) = (r as f64 / 255.0, g as f64 / 255.0, b as f64 / 255.0); + let (lightness, saturation) = lightness_saturation(r, g, b); + if saturation < 0.1 { + let gray = (lightness * 25.0).round_ties_even() as u8; + return match gray { + 0 => 16, + 25 => 231, + gray => 231 + gray, + }; + } + let level = |value: f64| (value * 5.0).round_ties_even() as u8; + 16 + 36 * level(r) + 6 * level(g) + level(b) +} + +/// rich's `STANDARD_PALETTE.match()`: the closest of the 16 standard colours. +fn standard((r, g, b): (u8, u8, u8)) -> u8 { + const PALETTE: [(i64, i64, i64); 16] = [ + (0, 0, 0), + (170, 0, 0), + (0, 170, 0), + (170, 85, 0), + (0, 0, 170), + (170, 0, 170), + (0, 170, 170), + (170, 170, 170), + (85, 85, 85), + (255, 85, 85), + (85, 255, 85), + (255, 255, 85), + (85, 85, 255), + (255, 85, 255), + (85, 255, 255), + (255, 255, 255), + ]; + let (r, g, b) = (i64::from(r), i64::from(g), i64::from(b)); + let distance = |&(r2, g2, b2): &(i64, i64, i64)| { + let red_mean = (r + r2) / 2; + let (red, green, blue) = (r - r2, g - g2, b - b2); + let squared = (((512 + red_mean) * red * red) >> 8) + + 4 * green * green + + (((767 - red_mean) * blue * blue) >> 8); + (squared as f64).sqrt() + }; + let mut best = 0; + for (index, color) in PALETTE.iter().enumerate() { + if distance(color) < distance(&PALETTE[best]) { + best = index; + } + } + best as u8 +} + +/// Writes `text` in `style`, like rich's `Style.render()`. +pub(crate) fn write_styled(out: &mut String, text: &str, style: &Style, colors: Colors) { + if text.is_empty() || colors.is_plain() || *style == Style::default() { + out.push_str(text); + return; + } + let mut codes = String::new(); + if style.bold { + codes.push('1'); + } + if let (Some(rgb), false) = (style.color, colors.no_color) { + if !codes.is_empty() { + codes.push(';'); + } + let _ = match colors.system { + ColorSystem::TrueColor => write!(codes, "38;2;{};{};{}", rgb.0, rgb.1, rgb.2), + ColorSystem::EightBit => write!(codes, "38;5;{}", eight_bit(rgb)), + ColorSystem::Standard | ColorSystem::None => { + let number = standard(rgb); + if number < 8 { + write!(codes, "{}", 30 + number) + } else { + write!(codes, "{}", 82 + number) + } + } + }; + } + let styled = if codes.is_empty() { + text.to_string() + } else { + format!("\x1b[{codes}m{text}\x1b[0m") + }; + match &style.link { + Some(link) => { + let _ = write!(out, "\x1b]8;;{link}\x1b\\{styled}\x1b]8;;\x1b\\"); + } + None => out.push_str(&styled), + } +} + +/// A piece of text to print: text as it is, or markup of the data files. +#[derive(Debug, Clone)] +enum Piece { + Plain(String, Style), + Markup(String), +} + +/// Text to print, built from pieces. Like the strings that the Python CLI gives rich, +/// except that only the data is markup. +#[derive(Debug, Clone, Default)] +pub(crate) struct Text { + pieces: Vec, +} + +impl Text { + pub(crate) fn new() -> Text { + Text::default() + } + + /// Text as it is. + pub(crate) fn plain(&mut self, text: &str) -> &mut Self { + self.styled(text, Style::default()) + } + + /// Text in a style. + pub(crate) fn styled(&mut self, text: &str, style: Style) -> &mut Self { + if !text.is_empty() { + self.pieces.push(Piece::Plain(text.to_string(), style)); + } + self + } + + /// A label, such as "Matched on: ". + pub(crate) fn label(&mut self, text: &str) -> &mut Self { + self.styled(text, LABEL) + } + + /// A string of the data files, which may contain rich markup. + pub(crate) fn markup(&mut self, markup: &str) -> &mut Self { + if !markup.is_empty() { + self.pieces.push(Piece::Markup(markup.to_string())); + } + self + } + + pub(crate) fn append(&mut self, other: Text) -> &mut Self { + self.pieces.extend(other.pieces); + self + } + + /// The text as the Python CLI writes it before rich renders it, with markup. + pub(crate) fn source(&self) -> String { + self.pieces + .iter() + .map(|piece| match piece { + Piece::Plain(text, _) | Piece::Markup(text) => text.as_str(), + }) + .collect() + } + + /// `str.rstrip()` of the text before rendering (Python strips the markup). + pub(crate) fn rstrip(&mut self) -> &mut Self { + while let Some(last) = self.pieces.last_mut() { + let text = match last { + Piece::Plain(text, _) | Piece::Markup(text) => text, + }; + let kept = super::python::rstrip(text).len(); + text.truncate(kept); + if !text.is_empty() { + break; + } + self.pieces.pop(); + } + self + } + + /// `str.lstrip()` of the text before rendering. + pub(crate) fn lstrip(&mut self) -> &mut Self { + while let Some(first) = self.pieces.first_mut() { + let text = match first { + Piece::Plain(text, _) | Piece::Markup(text) => text, + }; + let removed = text.len() - super::python::lstrip(text).len(); + text.drain(..removed); + if !text.is_empty() { + break; + } + self.pieces.remove(0); + } + self + } + + /// `str.strip()` of the text before rendering. + pub(crate) fn strip(&mut self) -> &mut Self { + self.lstrip().rstrip() + } + + /// The text with its styles, with the markup rendered. + fn spans(&self) -> Vec<(String, Style)> { + let mut spans = Vec::new(); + for piece in &self.pieces { + match piece { + Piece::Plain(text, style) => spans.push((text.clone(), style.clone())), + Piece::Markup(markup) => spans.extend(super::markup::render(markup)), + } + } + spans + } + + /// The plain text, as rich prints it to a file or a pipe. + pub(crate) fn to_plain(&self) -> String { + self.render(Colors::PLAIN) + } + + /// The text as rich prints it: control codes removed, tabs expanded, and styles + /// written for `colors`. + pub(crate) fn render(&self, colors: Colors) -> String { + let mut out = String::new(); + let mut column = 0; + for (text, style) in self.spans() { + let text = sanitize(&text, &mut column); + // A style per line, the line breaks outside of it + for (index, line) in text.split('\n').enumerate() { + if index > 0 { + out.push('\n'); + } + write_styled(&mut out, line, &style, colors); + } + } + out + } +} + +/// What rich does to text it prints: backspaces, vertical tabs, form feeds and carriage +/// returns removed (`strip_control_codes`), tabs expanded to the next multiple of 8 +/// characters on the line (`Text.expand_tabs`). `column` is the number of characters on +/// the line so far. +pub(crate) fn sanitize(text: &str, column: &mut usize) -> String { + let mut out = String::with_capacity(text.len()); + for c in text.chars() { + match c { + '\x08' | '\x0b' | '\x0c' | '\r' => {} + '\n' => { + out.push(c); + *column = 0; + } + '\t' => { + let spaces = 8 - *column % 8; + out.extend(std::iter::repeat_n(' ', spaces)); + *column += spaces; + } + _ => { + out.push(c); + *column += 1; + } + } + } + out +} + +/// `sanitize()` for a text on its own. +pub(crate) fn sanitized(text: &str) -> String { + sanitize(text, &mut 0) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn rich_downgrades() { + // Color.parse("#D7AFFF").downgrade(...) in rich 12.2.0 + assert_eq!(eight_bit((0xD7, 0xAF, 0xFF)), 183); + assert_eq!(eight_bit((0x49, 0xC3, 0xCE)), 80); + assert_eq!(standard((0xD7, 0xAF, 0xFF)), 7); + assert_eq!(standard((0x49, 0xC3, 0xCE)), 6); + assert_eq!(eight_bit((0x80, 0x80, 0x80)), 244); + } + + #[test] + fn styles() { + let true_color = Colors { + system: ColorSystem::TrueColor, + no_color: false, + }; + let mut out = String::new(); + write_styled(&mut out, "x", &LABEL, true_color); + assert_eq!(out, "\x1b[1;38;2;215;175;255mx\x1b[0m"); + let mut out = String::new(); + write_styled( + &mut out, + "x", + &LABEL, + Colors { + no_color: true, + ..true_color + }, + ); + assert_eq!(out, "\x1b[1mx\x1b[0m"); + let mut text = Text::new(); + text.label("Name: ").plain("a\tb\r\n\tc"); + assert_eq!(text.to_plain(), "Name: a b\n c"); + assert_eq!( + text.render(true_color), + "\x1b[1;38;2;215;175;255mName: \x1b[0ma b\n c" + ); + } + + #[test] + fn colour_systems_like_rich() { + // Console._detect_color_system() of rich 12.2.0, on a terminal + let detect = |vars: &[(&str, &str)], terminal: bool| { + let vars: Vec<(String, String)> = vars + .iter() + .map(|(name, value)| (name.to_string(), value.to_string())) + .collect(); + Colors::detect(terminal, &|name| { + vars.iter().find(|(n, _)| n == name).map(|(_, v)| v.clone()) + }) + }; + let system = |vars: &[(&str, &str)]| detect(vars, true).system; + if cfg!(windows) { + return; // Windows terminals get 24-bit colours once they are enabled + } + assert_eq!(system(&[("TERM", "xterm-256color")]), ColorSystem::EightBit); + assert_eq!( + system(&[("TERM", "xterm-256color"), ("COLORTERM", "truecolor")]), + ColorSystem::TrueColor + ); + assert_eq!( + system(&[("TERM", "xterm"), ("COLORTERM", "24bit")]), + ColorSystem::TrueColor + ); + assert_eq!(system(&[("TERM", "xterm")]), ColorSystem::Standard); + assert_eq!(system(&[("TERM", "xterm-16color")]), ColorSystem::Standard); + assert_eq!(system(&[]), ColorSystem::Standard); + for dumb in ["dumb", "unknown", "DUMB"] { + assert_eq!(system(&[("TERM", dumb)]), ColorSystem::None, "{dumb}"); + } + // rich checks for a dumb terminal before it strips TERM, and takes what follows + // the last "-", or all of TERM + assert_eq!(system(&[("TERM", " dumb")]), ColorSystem::Standard); + assert_eq!(system(&[("TERM", "256color")]), ColorSystem::EightBit); + assert_eq!(system(&[("TERM", "a-b-256color")]), ColorSystem::EightBit); + assert_eq!( + system(&[("TERM", "xterm-truecolor")]), + ColorSystem::Standard + ); + // Not a terminal: no escape sequences + assert_eq!(detect(&[("TERM", "xterm-256color")], false), Colors::PLAIN); + // NO_COLOR, even empty, keeps bold and links + let no_color = detect(&[("TERM", "xterm-256color"), ("NO_COLOR", "")], true); + assert!(no_color.no_color); + assert_eq!(no_color.system, ColorSystem::EightBit); + } + + #[test] + fn control_codes_like_rich() { + // strip_control_codes() removes backspace, vertical tab, form feed and carriage + // return; other control characters stay + assert_eq!( + sanitized("a\x08b\x0bc\x0cd\re\x07f\x1bg"), + "abcde\x07f\x1bg" + ); + // Tabs expand to the next multiple of 8 on the line + assert_eq!( + sanitized("\tx\n1234567\tx\n12345678\tx"), + " x\n1234567 x\n12345678 x" + ); + } + + #[test] + fn rstrip_strips_the_source() { + let mut text = Text::new(); + text.plain("a").markup("b [https://x y]").plain("\n\n"); + text.rstrip(); + // rich gets "ab [https://x y]" and prints "ab " + assert_eq!(text.to_plain(), "ab "); + let mut text = Text::new(); + text.plain("a ").markup(" ").plain(" \n"); + text.rstrip(); + assert_eq!(text.to_plain(), "a"); + } +} diff --git a/src/cli/table.rs b/src/cli/table.rs new file mode 100644 index 0000000..dbc06b8 --- /dev/null +++ b/src/cli/table.rs @@ -0,0 +1,63 @@ +//! The table of `--format pretty`, drawn with comfy-table in the look of the Python CLI's +//! rich table (`box.HEAVY_HEAD`, a line between rows, padding of one space). When the +//! table fits, it has the same columns widths as rich's; when it doesn't, comfy-table +//! wraps the cells its own way, but the cells are the same. + +use super::style::{ColorSystem, Colors}; +use comfy_table::{ + Attribute, Cell, Color, ContentArrangement, ContentLineStyle, LineStyle, Table, TableStyle, +}; + +/// rich's `box.HEAVY_HEAD`. +const HEAVY_HEAD: TableStyle = TableStyle::new() + .top_border(LineStyle::new('┏', '━', '┳', '┓')) + .header_lines(ContentLineStyle::new('┃', '┃', '┃')) + .header_separator(LineStyle::new('┡', '━', '╇', '┩')) + .content_lines(ContentLineStyle::new('│', '│', '│')) + .row_separator(LineStyle::new('├', '─', '┼', '┤')) + .bottom_border(LineStyle::new('└', '─', '┴', '┘')); + +/// The style of the header (`header_style="bold #D7Afff"`). +fn header_cell(text: &str, colors: Colors) -> Cell { + let cell = Cell::new(text); + if colors.is_plain() { + return cell; + } + let cell = cell.add_attribute(Attribute::Bold); + if colors.no_color { + return cell; + } + match colors.system { + ColorSystem::TrueColor => cell.fg(Color::Rgb { + r: 0xD7, + g: 0xAF, + b: 0xFF, + }), + ColorSystem::EightBit => cell.fg(Color::AnsiValue(183)), + ColorSystem::Standard | ColorSystem::None => cell.fg(Color::Grey), + } +} + +/// The table, `width` columns wide at most, without a line break at the end. +pub(crate) fn render( + header: &[&str], + rows: &[Vec], + width: usize, + colors: Colors, +) -> String { + let mut table = Table::new(); + table + .load_style(HEAVY_HEAD) + .set_content_arrangement(ContentArrangement::Dynamic) + .set_width(width.min(u16::MAX as usize) as u16); + if colors.is_plain() { + table.force_no_tty(); + } else { + table.enforce_styling(); + } + table.set_header(header.iter().map(|text| header_cell(text, colors))); + for row in rows { + table.add_row(row.iter().map(Cell::new)); + } + table.to_string() +} diff --git a/src/cli/verify.rs b/src/cli/verify.rs new file mode 100644 index 0000000..94b40df --- /dev/null +++ b/src/cli/verify.rs @@ -0,0 +1,968 @@ +//! `--verify`: asks Google whether the Google API keys found are valid, and adds the answer +//! to their description, like `GoogleAPIKeyVerifier` of `pywhat/processors.py`: "Verification: +//! invalid, Google rejected the key (API_KEY_INVALID)". Every key is sent once, however many +//! times it is found. + +use crate::identify::Match; +use crate::processors::Processor; +use std::collections::HashMap; +use std::sync::Mutex; + +/// What a GET request answered: the HTTP status and the body, or why there is no answer. +pub type Answer = Result<(u16, Vec), String>; + +/// Sends a GET request (with the User-Agent `pywhat`) and returns the answer. +pub type HttpGet = dyn Fn(&str) -> Answer + Send + Sync; + +/// Asks Google whether Google API keys are valid (`GoogleAPIKeyVerifier`). +pub struct GoogleApiKeyVerifier { + get: Box, + /// The answer for each key, so that it is only sent once. + results: Mutex>, +} + +impl GoogleApiKeyVerifier { + /// The request: an API that costs nothing (the languages of the YouTube Data API, one + /// unit of quota), with the key. + pub const URL: &str = "https://www.googleapis.com/youtube/v3/i18nLanguages?part=snippet&key="; + + /// The reasons of the `google.rpc.ErrorInfo` of the errors that Google answers when the + /// key is valid but can't be used for the request. + pub const VALID_KEY_REASONS: [&str; 9] = [ + "API_KEY_ANDROID_APP_BLOCKED", + "API_KEY_HTTP_REFERRER_BLOCKED", + "API_KEY_IOS_APP_BLOCKED", + "API_KEY_IP_ADDRESS_BLOCKED", + "API_KEY_SERVICE_BLOCKED", + "BILLING_DISABLED", + "RATE_LIMIT_EXCEEDED", + "RESOURCE_QUOTA_EXCEEDED", + "SERVICE_DISABLED", + ]; + + /// A verifier that asks Google over HTTPS, trusting the operating system's root + /// certificates, with the timeouts of Python's `urlopen(..., timeout=10)`: 10 s to connect + /// and for every read and write. + pub fn new() -> Self { + Self::with_http(Box::new(get)) + } + + /// A verifier that sends its requests with `get`, e.g. a fake Google in tests. + pub fn with_http(get: Box) -> Self { + GoogleApiKeyVerifier { + get, + results: Mutex::new(HashMap::new()), + } + } + + /// Asks Google whether `key` is valid, and returns the answer, e.g. "valid, Google + /// accepted the key". + pub fn verify(&self, key: &str) -> String { + match (self.get)(&format!("{}{}", Self::URL, quote(key))) { + Ok((status, _)) if (200..300).contains(&status) => { + "valid, Google accepted the key".to_string() + } + Ok((status, body)) => Self::describe_error(status, &body), + Err(error) => format!("unknown, could not ask Google ({error})"), + } + } + + /// The answer for a key that Google answered an error for. + pub fn describe_error(status: u16, body: &[u8]) -> String { + let reason = google_error_reason(body); + match &reason { + Some(PyValue::Str(reason)) if reason == "API_KEY_INVALID" => { + format!("invalid, Google rejected the key ({reason})") + } + Some(PyValue::Str(reason)) if Self::VALID_KEY_REASONS.contains(&reason.as_str()) => { + format!("valid, but Google refused the request ({reason})") + } + Some(reason) if reason.truthy() => { + format!("unknown, Google answered HTTP {status} ({})", reason.str()) + } + _ => format!("unknown, Google answered HTTP {status}"), + } + } +} + +impl Default for GoogleApiKeyVerifier { + fn default() -> Self { + Self::new() + } +} + +impl Processor for GoogleApiKeyVerifier { + fn names(&self) -> &[&str] { + &["Google API Key"] + } + + fn process(&self, mut found: Match) -> Option { + let result = { + let mut results = self + .results + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + results + .entry(found.matched.clone()) + .or_insert_with(|| self.verify(&found.matched)) + .clone() + }; + let result = format!("Verification: {result}"); + found.description = Some(match found.description.as_deref() { + Some(description) if !description.is_empty() => format!("{description}. {result}"), + _ => result, + }); + Some(found) + } +} + +/// `google_error_reason()`: the reason of the `google.rpc.ErrorInfo` in the error that a +/// Google API answered, such as "API_KEY_INVALID", or `None`. Like Python, it gives up at +/// the first detail that isn't an object or whose `@type` isn't a string. +fn google_error_reason(body: &[u8]) -> Option { + let PyValue::Dict(json) = pyjson::loads(body)? else { + return None; + }; + let PyValue::Dict(error) = pyjson::get(&json, "error")? else { + return None; + }; + let details = match pyjson::get(error, "details") { + None => return None, + Some(PyValue::List(details)) => details, + Some(_) => return None, + }; + for detail in details { + let PyValue::Dict(detail) = detail else { + return None; + }; + let kind = match pyjson::get(detail, "@type") { + None => "", + Some(PyValue::Str(kind)) => kind, + Some(_) => return None, + }; + if kind.ends_with("google.rpc.ErrorInfo") { + return pyjson::get(detail, "reason").cloned(); + } + } + None +} + +/// A value as Python's `json.loads()` returns it. +#[derive(Debug, Clone, PartialEq)] +enum PyValue { + None, + Bool(bool), + /// An integer, as its digits: Python's integers have no limit. + Int(String), + Float(f64), + Str(String), + List(Vec), + /// The keys in the order of their first appearance, with their last values, as in a dict. + Dict(Vec<(String, PyValue)>), +} + +impl PyValue { + /// Python's truth value. + fn truthy(&self) -> bool { + match self { + PyValue::None => false, + PyValue::Bool(value) => *value, + PyValue::Int(digits) => digits.bytes().any(|b| b.is_ascii_digit() && b != b'0'), + PyValue::Float(value) => *value != 0.0, + PyValue::Str(text) => !text.is_empty(), + PyValue::List(values) => !values.is_empty(), + PyValue::Dict(items) => !items.is_empty(), + } + } + + /// `str()`. + fn str(&self) -> String { + match self { + PyValue::Str(text) => text.clone(), + other => other.repr(), + } + } + + /// `repr()`. + fn repr(&self) -> String { + match self { + PyValue::None => "None".to_string(), + PyValue::Bool(true) => "True".to_string(), + PyValue::Bool(false) => "False".to_string(), + PyValue::Int(digits) => digits.clone(), + PyValue::Float(value) if value.is_nan() => "nan".to_string(), + PyValue::Float(value) if value.is_infinite() => { + if *value > 0.0 { "inf" } else { "-inf" }.to_string() + } + PyValue::Float(value) => { + let mut out = String::new(); + crate::json::write_float(&mut out, *value); + out + } + PyValue::Str(text) => super::python::repr(text), + PyValue::List(values) => { + let items: Vec = values.iter().map(PyValue::repr).collect(); + format!("[{}]", items.join(", ")) + } + PyValue::Dict(items) => { + let items: Vec = items + .iter() + .map(|(key, value)| format!("{}: {}", super::python::repr(key), value.repr())) + .collect(); + format!("{{{}}}", items.join(", ")) + } + } + } +} + +/// Python's `json.loads()` of bytes: the encoding detected like `json.detect_encoding()`, +/// `NaN`, `Infinity` and `-Infinity`, integers of any size. +mod pyjson { + use super::PyValue; + + /// The value of a key of a dict. + pub(super) fn get<'a>(items: &'a [(String, PyValue)], key: &str) -> Option<&'a PyValue> { + items.iter().find(|(k, _)| k == key).map(|(_, value)| value) + } + + /// `json.detect_encoding()` and `bytes.decode(encoding, "surrogatepass")` (lone + /// surrogates become U+FFFD). + fn decode(data: &[u8]) -> Option { + let utf16 = |data: &[u8], little: bool| -> Option { + if !data.len().is_multiple_of(2) { + return None; + } + let units = data.as_chunks::<2>().0.iter().map(|&pair| { + if little { + u16::from_le_bytes(pair) + } else { + u16::from_be_bytes(pair) + } + }); + Some( + char::decode_utf16(units) + .map(|c| c.unwrap_or('\u{fffd}')) + .collect(), + ) + }; + let utf32 = |data: &[u8], little: bool| -> Option { + if !data.len().is_multiple_of(4) { + return None; + } + data.as_chunks::<4>() + .0 + .iter() + .map(|&quad| { + let value = if little { + u32::from_le_bytes(quad) + } else { + u32::from_be_bytes(quad) + }; + match value { + 0xD800..=0xDFFF => Some('\u{fffd}'), + _ => char::from_u32(value), + } + }) + .collect() + }; + if let Some(rest) = data.strip_prefix(&[0xFF, 0xFE, 0, 0]) { + return utf32(rest, true); + } + if let Some(rest) = data.strip_prefix(&[0, 0, 0xFE, 0xFF]) { + return utf32(rest, false); + } + if let Some(rest) = data.strip_prefix(&[0xFF, 0xFE]) { + return utf16(rest, true); + } + if let Some(rest) = data.strip_prefix(&[0xFE, 0xFF]) { + return utf16(rest, false); + } + if let Some(rest) = data.strip_prefix(b"\xEF\xBB\xBF") { + return String::from_utf8(rest.to_vec()).ok(); + } + if data.len() >= 4 { + if data[0] == 0 { + return if data[1] != 0 { + utf16(data, false) + } else { + utf32(data, false) + }; + } + if data[1] == 0 { + return if data[2] != 0 || data[3] != 0 { + utf16(data, true) + } else { + utf32(data, true) + }; + } + } else if data.len() == 2 { + if data[0] == 0 { + return utf16(data, false); + } + if data[1] == 0 { + return utf16(data, true); + } + } + String::from_utf8(data.to_vec()).ok() + } + + /// `json.loads(data)`, or `None` where Python raises an error. + pub(super) fn loads(data: &[u8]) -> Option { + let text = decode(data)?; + let mut parser = Parser { + text: text.as_bytes(), + at: 0, + depth: 0, + }; + parser.whitespace(); + let value = parser.value()?; + parser.whitespace(); + (parser.at == parser.text.len()).then_some(value) + } + + struct Parser<'a> { + text: &'a [u8], + at: usize, + depth: usize, + } + + impl Parser<'_> { + fn peek(&self) -> Option { + self.text.get(self.at).copied() + } + + fn eat(&mut self, literal: &str) -> bool { + if self.text[self.at..].starts_with(literal.as_bytes()) { + self.at += literal.len(); + true + } else { + false + } + } + + fn whitespace(&mut self) { + while matches!(self.peek(), Some(b' ' | b'\t' | b'\n' | b'\r')) { + self.at += 1; + } + } + + fn value(&mut self) -> Option { + // Python stops at about 1,000 levels too + if self.depth > 500 { + return None; + } + match self.peek()? { + b'{' => self.dict(), + b'[' => self.list(), + b'"' => self.string().map(PyValue::Str), + b'n' if self.eat("null") => Some(PyValue::None), + b't' if self.eat("true") => Some(PyValue::Bool(true)), + b'f' if self.eat("false") => Some(PyValue::Bool(false)), + b'N' if self.eat("NaN") => Some(PyValue::Float(f64::NAN)), + b'I' if self.eat("Infinity") => Some(PyValue::Float(f64::INFINITY)), + b'-' if self.eat("-Infinity") => Some(PyValue::Float(f64::NEG_INFINITY)), + b'-' | b'0'..=b'9' => self.number(), + _ => None, + } + } + + fn digits(&mut self) -> usize { + let start = self.at; + while matches!(self.peek(), Some(b'0'..=b'9')) { + self.at += 1; + } + self.at - start + } + + /// `-?(0|[1-9][0-9]*)(\.[0-9]+)?([eE][-+]?[0-9]+)?`: a float with a fraction or an + /// exponent, else an integer. + fn number(&mut self) -> Option { + let start = self.at; + let negative = self.eat("-"); + match self.peek() { + Some(b'0') => self.at += 1, + Some(b'1'..=b'9') => { + self.digits(); + } + _ => return None, + } + let integer_end = self.at; + let mut float = false; + if self.peek() == Some(b'.') && matches!(self.text.get(self.at + 1), Some(b'0'..=b'9')) + { + self.at += 1; + self.digits(); + float = true; + } + if matches!(self.peek(), Some(b'e' | b'E')) { + let save = self.at; + self.at += 1; + if matches!(self.peek(), Some(b'+' | b'-')) { + self.at += 1; + } + if self.digits() == 0 { + self.at = save; + } else { + float = true; + } + } + let text = std::str::from_utf8(&self.text[start..self.at]).ok()?; + if float { + return text.parse().ok().map(PyValue::Float); + } + let digits = + std::str::from_utf8(&self.text[start + negative as usize..integer_end]).ok()?; + Some(PyValue::Int(if negative && digits != "0" { + format!("-{digits}") + } else { + digits.to_string() + })) + } + + fn hex4(&mut self) -> Option { + let hex = std::str::from_utf8(self.text.get(self.at..self.at + 4)?).ok()?; + if !hex.bytes().all(|b| b.is_ascii_hexdigit()) { + return None; + } + self.at += 4; + u32::from_str_radix(hex, 16).ok() + } + + fn string(&mut self) -> Option { + self.at += 1; // the opening quote + let mut out = String::new(); + loop { + let start = self.at; + while let Some(b) = self.peek() { + if b == b'"' || b == b'\\' || b < 0x20 { + break; + } + self.at += 1; + } + out.push_str(std::str::from_utf8(&self.text[start..self.at]).ok()?); + match self.peek()? { + b'"' => { + self.at += 1; + return Some(out); + } + b'\\' => { + self.at += 1; + let escape = self.peek()?; + self.at += 1; + match escape { + b'"' => out.push('"'), + b'\\' => out.push('\\'), + b'/' => out.push('/'), + b'b' => out.push('\u{8}'), + b'f' => out.push('\u{c}'), + b'n' => out.push('\n'), + b'r' => out.push('\r'), + b't' => out.push('\t'), + b'u' => { + let first = self.hex4()?; + let mut code = first; + if (0xD800..0xDC00).contains(&first) && self.eat("\\u") { + let save = self.at; + match self.hex4() { + Some(second) if (0xDC00..0xE000).contains(&second) => { + code = 0x10000 + + ((first - 0xD800) << 10) + + (second - 0xDC00); + } + Some(_) => { + // A lone surrogate, then another escape + self.at = save - 2; + } + None => return None, + } + } + out.push(char::from_u32(code).unwrap_or('\u{fffd}')); + } + _ => return None, + } + } + _ => return None, // a control character + } + } + } + + fn list(&mut self) -> Option { + self.at += 1; + self.depth += 1; + let mut values = Vec::new(); + self.whitespace(); + if self.eat("]") { + self.depth -= 1; + return Some(PyValue::List(values)); + } + loop { + self.whitespace(); + values.push(self.value()?); + self.whitespace(); + if self.eat("]") { + self.depth -= 1; + return Some(PyValue::List(values)); + } + if !self.eat(",") { + return None; + } + } + } + + fn dict(&mut self) -> Option { + self.at += 1; + self.depth += 1; + let mut items: Vec<(String, PyValue)> = Vec::new(); + self.whitespace(); + if self.eat("}") { + self.depth -= 1; + return Some(PyValue::Dict(items)); + } + loop { + self.whitespace(); + if self.peek()? != b'"' { + return None; + } + let key = self.string()?; + self.whitespace(); + if !self.eat(":") { + return None; + } + self.whitespace(); + let value = self.value()?; + match items.iter_mut().find(|(k, _)| *k == key) { + Some(item) => item.1 = value, + None => items.push((key, value)), + } + self.whitespace(); + if self.eat("}") { + self.depth -= 1; + return Some(PyValue::Dict(items)); + } + if !self.eat(",") { + return None; + } + } + } + } +} + +/// `urllib.parse.quote(text, safe="")`. +fn quote(text: &str) -> String { + let mut out = String::with_capacity(text.len()); + for &b in text.as_bytes() { + if b.is_ascii_alphanumeric() || b"_.-~".contains(&b) { + out.push(b as char); + } else { + out.push_str(&format!("%{b:02X}")); + } + } + out +} + +/// A GET request, like Python's `urlopen(Request(url, headers={"User-Agent": "pywhat"}), +/// timeout=10)`: the operating system's root certificates, 10 s to connect and for every read +/// and write (not for the whole request), redirects followed for 301, 302, 303 and 307 (up to +/// 10), and errors described the way `urllib` describes the common ones (`timed out`, +/// ``). +#[cfg(feature = "verify")] +fn get(url: &str) -> Answer { + use std::time::Duration; + // rustls with ring, and the root certificates of the operating system, which Python's + // urllib trusts too (OpenSSL's store on Linux) + let _ = rustls::crypto::ring::default_provider().install_default(); + let tls = ureq::tls::TlsConfig::builder() + .root_certs(ureq::tls::RootCerts::PlatformVerifier) + .build(); + let ten = Some(Duration::from_secs(10)); + let agent: ureq::Agent = ureq::Agent::config_builder() + .tls_config(tls) + .timeout_connect(ten) + .timeout_per_read(ten) + .timeout_per_write(ten) + .http_status_as_error(false) + .max_redirects(0) + .max_redirects_will_error(false) + .user_agent("pywhat") + .build() + .into(); + let mut url = url.to_string(); + let mut redirects = 0; + loop { + let mut response = agent.get(&url).call().map_err(urllib_error)?; + let status = response.status().as_u16(); + let location = response + .headers() + .get("location") + .or_else(|| response.headers().get("uri")) + .and_then(|value| value.to_str().ok()) + .map(str::to_string); + let body = response.body_mut().read_to_vec().map_err(urllib_error)?; + if let (true, Some(location)) = (matches!(status, 301 | 302 | 303 | 307), location) { + let next = urljoin(&url, &location); + let allowed = next.starts_with("http://") || next.starts_with("https://"); + if allowed && redirects < 10 { + redirects += 1; + url = next; + continue; + } + } + return Ok((status, body)); + } +} + +/// `urllib.parse.urljoin(base, location)`, for the forms that a Location header has. +#[cfg(feature = "verify")] +fn urljoin(base: &str, location: &str) -> String { + let (scheme, rest) = base.split_once("://").unwrap_or(("https", base)); + let authority = rest.split(['/', '?', '#']).next().unwrap_or(""); + if location.contains("://") { + location.to_string() + } else if let Some(path) = location.strip_prefix("//") { + format!("{scheme}://{path}") + } else if location.starts_with('/') { + format!("{scheme}://{authority}{location}") + } else { + let path_end = rest.find(['?', '#']).unwrap_or(rest.len()); + let directory = rest[..path_end] + .rfind('/') + .map_or(rest.len().min(path_end), |i| i + 1); + let directory = if directory <= authority.len() { + format!("{authority}/") + } else { + rest[..directory].to_string() + }; + format!("{scheme}://{directory}{location}") + } +} + +/// How Python's `urlopen()` describes an error, as far as it can be told from ureq's: the +/// errors of connecting are ``, OS errors are `[Errno N] message`, and +/// timeouts `timed out`. +#[cfg(feature = "verify")] +fn urllib_error(error: ureq::Error) -> String { + use ureq::Timeout; + let os_error = |error: &std::io::Error| { + let code = error.raw_os_error()?; + let text = error.to_string(); + let message = text + .strip_suffix(&format!(" (os error {code})")) + .unwrap_or(&text) + .to_string(); + Some(format!("[Errno {code}] {message}")) + }; + match &error { + ureq::Error::Timeout(Timeout::Resolve | Timeout::Connect | Timeout::SendRequest) => { + "".to_string() + } + ureq::Error::Timeout(_) => "timed out".to_string(), + ureq::Error::Io(io) => { + use std::io::ErrorKind::*; + let lookup = io.to_string(); + if let Some(detail) = lookup.strip_prefix("failed to lookup address information: ") { + // getaddrinfo()'s errors, which Python shows with their EAI_ code + let code = match detail { + "Name or service not known" => Some(-2), + "Temporary failure in name resolution" => Some(-3), + "No address associated with hostname" => Some(-5), + "nodename nor servname provided, or not known" => Some(8), + _ => None, + }; + return match code { + Some(code) => format!(""), + None => format!(""), + }; + } + match (io.kind(), os_error(io)) { + ( + ConnectionRefused | AddrNotAvailable | NetworkUnreachable | HostUnreachable + | NetworkDown | PermissionDenied | TimedOut, + Some(message), + ) => format!(""), + (_, Some(message)) => message, + (_, None) => error.to_string(), + } + } + _ => error.to_string(), + } +} + +#[cfg(not(feature = "verify"))] +fn get(_url: &str) -> Answer { + Err("this pywhat was built without the verify feature".to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::Arc; + + fn google_error(reason: &str) -> Vec { + serde_json::json!({ + "error": { + "code": 400, + "message": "API key not valid. Please pass a valid API key.", + "details": [ + {"@type": "type.googleapis.com/google.rpc.ErrorInfo", "reason": reason}, + {"@type": "type.googleapis.com/google.rpc.LocalizedMessage"}, + ], + } + }) + .to_string() + .into_bytes() + } + + #[test] + fn answers() { + let invalid = GoogleApiKeyVerifier::with_http(Box::new(|_| { + Ok((400, google_error("API_KEY_INVALID"))) + })); + assert_eq!( + invalid.verify("k"), + "invalid, Google rejected the key (API_KEY_INVALID)" + ); + let disabled = GoogleApiKeyVerifier::with_http(Box::new(|_| { + Ok((403, google_error("SERVICE_DISABLED"))) + })); + assert_eq!( + disabled.verify("k"), + "valid, but Google refused the request (SERVICE_DISABLED)" + ); + let other = GoogleApiKeyVerifier::with_http(Box::new(|_| Ok((403, google_error("OTHER"))))); + assert_eq!( + other.verify("k"), + "unknown, Google answered HTTP 403 (OTHER)" + ); + let html = GoogleApiKeyVerifier::with_http(Box::new(|_| Ok((502, b"".to_vec())))); + assert_eq!(html.verify("k"), "unknown, Google answered HTTP 502"); + let valid = GoogleApiKeyVerifier::with_http(Box::new(|_| Ok((200, b"{}".to_vec())))); + assert_eq!(valid.verify("k"), "valid, Google accepted the key"); + let offline = GoogleApiKeyVerifier::with_http(Box::new(|_| Err("timed out".into()))); + assert_eq!( + offline.verify("k"), + "unknown, could not ask Google (timed out)" + ); + } + + #[test] + fn once_per_key_in_the_description() { + let requests = Arc::new(AtomicUsize::new(0)); + let urls = Arc::new(Mutex::new(Vec::new())); + let (counter, seen) = (requests.clone(), urls.clone()); + let verifier = GoogleApiKeyVerifier::with_http(Box::new(move |url| { + counter.fetch_add(1, Ordering::SeqCst); + seen.lock().unwrap().push(url.to_string()); + Ok((400, google_error("API_KEY_INVALID"))) + })); + let key = format!("AIza{}", "SyA00000000000000000000000000000000"); + let identifier = crate::Identifier::default() + .with_processors(vec![Box::new(verifier)]) + .include_tags(&["Google API Key"]); + let text = format!("{key}\n{key}\n{key}"); + let found = identifier.identify(&text); + assert_eq!(found.len(), 3); + for m in &found { + assert!(m + .description + .as_deref() + .unwrap() + .ends_with("Verification: invalid, Google rejected the key (API_KEY_INVALID)")); + } + assert_eq!(requests.load(Ordering::SeqCst), 1); + assert_eq!( + urls.lock().unwrap()[0], + format!("{}{key}", GoogleApiKeyVerifier::URL) + ); + } + + #[test] + fn error_bodies_like_python() { + // GoogleAPIKeyVerifier().describe_error(code, body) in Python 3.10 + let cases: &[(u16, &[u8], &str)] = &[ + (400, b"{\x22error\x22: {\x22code\x22: 400, \x22message\x22: \x22m\x22, \x22details\x22: [{\x22@type\x22: \x22type.googleapis.com/google.rpc.ErrorInfo\x22, \x22reason\x22: \x22API_KEY_INVALID\x22}]}}", "invalid, Google rejected the key (API_KEY_INVALID)"), + (403, b"{\x22error\x22: {\x22code\x22: 400, \x22message\x22: \x22m\x22, \x22details\x22: [{\x22@type\x22: \x22type.googleapis.com/google.rpc.ErrorInfo\x22, \x22reason\x22: \x22SERVICE_DISABLED\x22}]}}", "valid, but Google refused the request (SERVICE_DISABLED)"), + (429, b"{\x22error\x22: {\x22code\x22: 400, \x22message\x22: \x22m\x22, \x22details\x22: [{\x22@type\x22: \x22type.googleapis.com/google.rpc.ErrorInfo\x22, \x22reason\x22: \x22RATE_LIMIT_EXCEEDED\x22}]}}", "valid, but Google refused the request (RATE_LIMIT_EXCEEDED)"), + (403, b"{\x22error\x22: {\x22code\x22: 400, \x22message\x22: \x22m\x22, \x22details\x22: [{\x22@type\x22: \x22type.googleapis.com/google.rpc.ErrorInfo\x22, \x22reason\x22: \x22OTHER_REASON\x22}]}}", "unknown, Google answered HTTP 403 (OTHER_REASON)"), + (502, b"bad gateway", "unknown, Google answered HTTP 502"), + (500, b"", "unknown, Google answered HTTP 500"), + (403, b"{\x22error\x22: {\x22code\x22: 400, \x22message\x22: \x22m\x22, \x22details\x22: [{\x22@type\x22: \x22type.googleapis.com/google.rpc.ErrorInfo\x22, \x22reason\x22: 5}]}}", "unknown, Google answered HTTP 403 (5)"), + (403, b"{\x22error\x22: {\x22code\x22: 400, \x22message\x22: \x22m\x22, \x22details\x22: [{\x22@type\x22: \x22type.googleapis.com/google.rpc.ErrorInfo\x22, \x22reason\x22: 0}]}}", "unknown, Google answered HTTP 403"), + (403, b"{\x22error\x22: {\x22code\x22: 400, \x22message\x22: \x22m\x22, \x22details\x22: [{\x22@type\x22: \x22type.googleapis.com/google.rpc.ErrorInfo\x22, \x22reason\x22: -0.0}]}}", "unknown, Google answered HTTP 403"), + (403, b"{\x22error\x22: {\x22code\x22: 400, \x22message\x22: \x22m\x22, \x22details\x22: [{\x22@type\x22: \x22type.googleapis.com/google.rpc.ErrorInfo\x22, \x22reason\x22: null}]}}", "unknown, Google answered HTTP 403"), + (403, b"{\x22error\x22: {\x22code\x22: 400, \x22message\x22: \x22m\x22, \x22details\x22: [{\x22@type\x22: \x22type.googleapis.com/google.rpc.ErrorInfo\x22, \x22reason\x22: true}]}}", "unknown, Google answered HTTP 403 (True)"), + (403, b"{\x22error\x22: {\x22code\x22: 400, \x22message\x22: \x22m\x22, \x22details\x22: [{\x22@type\x22: \x22type.googleapis.com/google.rpc.ErrorInfo\x22, \x22reason\x22: false}]}}", "unknown, Google answered HTTP 403"), + (403, b"{\x22error\x22: {\x22code\x22: 400, \x22message\x22: \x22m\x22, \x22details\x22: [{\x22@type\x22: \x22type.googleapis.com/google.rpc.ErrorInfo\x22, \x22reason\x22: 1.5}]}}", "unknown, Google answered HTTP 403 (1.5)"), + (403, b"{\x22error\x22: {\x22code\x22: 400, \x22message\x22: \x22m\x22, \x22details\x22: [{\x22@type\x22: \x22type.googleapis.com/google.rpc.ErrorInfo\x22, \x22reason\x22: 100000000000000000000}]}}", "unknown, Google answered HTTP 403 (100000000000000000000)"), + (403, b"{\x22error\x22: {\x22code\x22: 400, \x22message\x22: \x22m\x22, \x22details\x22: [{\x22@type\x22: \x22type.googleapis.com/google.rpc.ErrorInfo\x22, \x22reason\x22: -100000000000000000000}]}}", "unknown, Google answered HTTP 403 (-100000000000000000000)"), + (403, b"{\x22error\x22: {\x22code\x22: 400, \x22message\x22: \x22m\x22, \x22details\x22: [{\x22@type\x22: \x22type.googleapis.com/google.rpc.ErrorInfo\x22, \x22reason\x22: 1e+300}]}}", "unknown, Google answered HTTP 403 (1e+300)"), + (403, b"{\x22error\x22: {\x22code\x22: 400, \x22message\x22: \x22m\x22, \x22details\x22: [{\x22@type\x22: \x22type.googleapis.com/google.rpc.ErrorInfo\x22, \x22reason\x22: 1e+16}]}}", "unknown, Google answered HTTP 403 (1e+16)"), + (403, b"{\x22error\x22: {\x22code\x22: 400, \x22message\x22: \x22m\x22, \x22details\x22: [{\x22@type\x22: \x22type.googleapis.com/google.rpc.ErrorInfo\x22, \x22reason\x22: 100.0}]}}", "unknown, Google answered HTTP 403 (100.0)"), + (403, b"{\x22error\x22: {\x22code\x22: 400, \x22message\x22: \x22m\x22, \x22details\x22: [{\x22@type\x22: \x22type.googleapis.com/google.rpc.ErrorInfo\x22, \x22reason\x22: \x22\x22}]}}", "unknown, Google answered HTTP 403"), + (403, b"{\x22error\x22: {\x22code\x22: 400, \x22message\x22: \x22m\x22, \x22details\x22: [{\x22@type\x22: \x22type.googleapis.com/google.rpc.ErrorInfo\x22, \x22reason\x22: \x22[bold]X[/bold]\x22}]}}", "unknown, Google answered HTTP 403 ([bold]X[/bold])"), + (403, b"{\x22error\x22: {\x22code\x22: 400, \x22message\x22: \x22m\x22, \x22details\x22: [{\x22@type\x22: \x22type.googleapis.com/google.rpc.ErrorInfo\x22, \x22reason\x22: \x22caf\x5cu00e9 \x5cu20bf\x22}]}}", "unknown, Google answered HTTP 403 (café ₿)"), + (403, b"{\x22error\x22: {\x22code\x22: 400, \x22message\x22: \x22m\x22, \x22details\x22: [{\x22@type\x22: \x22type.googleapis.com/google.rpc.ErrorInfo\x22}]}}", "unknown, Google answered HTTP 403"), + (400, b"{\x22error\x22: {\x22details\x22: {\x22a\x22: 1}}}", "unknown, Google answered HTTP 400"), + (400, b"{\x22error\x22: {\x22details\x22: [\x22x\x22, {\x22@type\x22: \x22type.googleapis.com/google.rpc.ErrorInfo\x22, \x22reason\x22: \x22API_KEY_INVALID\x22}]}}", "unknown, Google answered HTTP 400"), + (400, b"{\x22error\x22: {\x22details\x22: [{\x22@type\x22: 5, \x22reason\x22: \x22API_KEY_INVALID\x22}]}}", "unknown, Google answered HTTP 400"), + (400, b"{\x22error\x22: {\x22details\x22: [{\x22reason\x22: \x22SERVICE_DISABLED\x22}, {\x22@type\x22: \x22type.googleapis.com/google.rpc.ErrorInfo\x22, \x22reason\x22: \x22API_KEY_INVALID\x22}]}}", "invalid, Google rejected the key (API_KEY_INVALID)"), + (400, b"[]", "unknown, Google answered HTTP 400"), + (400, b"{\x22error\x22: \x22x\x22}", "unknown, Google answered HTTP 400"), + (400, b"{\x22error\x22: {}}", "unknown, Google answered HTTP 400"), + (400, b"{\x22error\x22: {\x22details\x22: null}}", "unknown, Google answered HTTP 400"), + (400, b"{\x22error\x22: {\x22details\x22: \x22\x22}}", "unknown, Google answered HTTP 400"), + (400, b"{\x22error\x22: {\x22details\x22: {}}}", "unknown, Google answered HTTP 400"), + (400, b"\xef\xbb\xbf{\x22error\x22: {\x22code\x22: 400, \x22message\x22: \x22m\x22, \x22details\x22: [{\x22@type\x22: \x22type.googleapis.com/google.rpc.ErrorInfo\x22, \x22reason\x22: \x22API_KEY_INVALID\x22}]}}", "invalid, Google rejected the key (API_KEY_INVALID)"), + (400, b"\xff\xfe{\x00\x22\x00e\x00r\x00r\x00o\x00r\x00\x22\x00:\x00 \x00{\x00\x22\x00c\x00o\x00d\x00e\x00\x22\x00:\x00 \x004\x000\x000\x00,\x00 \x00\x22\x00m\x00e\x00s\x00s\x00a\x00g\x00e\x00\x22\x00:\x00 \x00\x22\x00m\x00\x22\x00,\x00 \x00\x22\x00d\x00e\x00t\x00a\x00i\x00l\x00s\x00\x22\x00:\x00 \x00[\x00{\x00\x22\x00@\x00t\x00y\x00p\x00e\x00\x22\x00:\x00 \x00\x22\x00t\x00y\x00p\x00e\x00.\x00g\x00o\x00o\x00g\x00l\x00e\x00a\x00p\x00i\x00s\x00.\x00c\x00o\x00m\x00/\x00g\x00o\x00o\x00g\x00l\x00e\x00.\x00r\x00p\x00c\x00.\x00E\x00r\x00r\x00o\x00r\x00I\x00n\x00f\x00o\x00\x22\x00,\x00 \x00\x22\x00r\x00e\x00a\x00s\x00o\x00n\x00\x22\x00:\x00 \x00\x22\x00A\x00P\x00I\x00_\x00K\x00E\x00Y\x00_\x00I\x00N\x00V\x00A\x00L\x00I\x00D\x00\x22\x00}\x00]\x00}\x00}\x00", "invalid, Google rejected the key (API_KEY_INVALID)"), + (400, b"{\x00\x22\x00e\x00r\x00r\x00o\x00r\x00\x22\x00:\x00 \x00{\x00\x22\x00c\x00o\x00d\x00e\x00\x22\x00:\x00 \x004\x000\x000\x00,\x00 \x00\x22\x00m\x00e\x00s\x00s\x00a\x00g\x00e\x00\x22\x00:\x00 \x00\x22\x00m\x00\x22\x00,\x00 \x00\x22\x00d\x00e\x00t\x00a\x00i\x00l\x00s\x00\x22\x00:\x00 \x00[\x00{\x00\x22\x00@\x00t\x00y\x00p\x00e\x00\x22\x00:\x00 \x00\x22\x00t\x00y\x00p\x00e\x00.\x00g\x00o\x00o\x00g\x00l\x00e\x00a\x00p\x00i\x00s\x00.\x00c\x00o\x00m\x00/\x00g\x00o\x00o\x00g\x00l\x00e\x00.\x00r\x00p\x00c\x00.\x00E\x00r\x00r\x00o\x00r\x00I\x00n\x00f\x00o\x00\x22\x00,\x00 \x00\x22\x00r\x00e\x00a\x00s\x00o\x00n\x00\x22\x00:\x00 \x00\x22\x00A\x00P\x00I\x00_\x00K\x00E\x00Y\x00_\x00I\x00N\x00V\x00A\x00L\x00I\x00D\x00\x22\x00}\x00]\x00}\x00}\x00", "invalid, Google rejected the key (API_KEY_INVALID)"), + (400, b"\x00{\x00\x22\x00e\x00r\x00r\x00o\x00r\x00\x22\x00:\x00 \x00{\x00\x22\x00c\x00o\x00d\x00e\x00\x22\x00:\x00 \x004\x000\x000\x00,\x00 \x00\x22\x00m\x00e\x00s\x00s\x00a\x00g\x00e\x00\x22\x00:\x00 \x00\x22\x00m\x00\x22\x00,\x00 \x00\x22\x00d\x00e\x00t\x00a\x00i\x00l\x00s\x00\x22\x00:\x00 \x00[\x00{\x00\x22\x00@\x00t\x00y\x00p\x00e\x00\x22\x00:\x00 \x00\x22\x00t\x00y\x00p\x00e\x00.\x00g\x00o\x00o\x00g\x00l\x00e\x00a\x00p\x00i\x00s\x00.\x00c\x00o\x00m\x00/\x00g\x00o\x00o\x00g\x00l\x00e\x00.\x00r\x00p\x00c\x00.\x00E\x00r\x00r\x00o\x00r\x00I\x00n\x00f\x00o\x00\x22\x00,\x00 \x00\x22\x00r\x00e\x00a\x00s\x00o\x00n\x00\x22\x00:\x00 \x00\x22\x00A\x00P\x00I\x00_\x00K\x00E\x00Y\x00_\x00I\x00N\x00V\x00A\x00L\x00I\x00D\x00\x22\x00}\x00]\x00}\x00}", "invalid, Google rejected the key (API_KEY_INVALID)"), + (400, b"\xff\xfe\x00\x00{\x00\x00\x00\x22\x00\x00\x00e\x00\x00\x00r\x00\x00\x00r\x00\x00\x00o\x00\x00\x00r\x00\x00\x00\x22\x00\x00\x00:\x00\x00\x00 \x00\x00\x00{\x00\x00\x00\x22\x00\x00\x00c\x00\x00\x00o\x00\x00\x00d\x00\x00\x00e\x00\x00\x00\x22\x00\x00\x00:\x00\x00\x00 \x00\x00\x004\x00\x00\x000\x00\x00\x000\x00\x00\x00,\x00\x00\x00 \x00\x00\x00\x22\x00\x00\x00m\x00\x00\x00e\x00\x00\x00s\x00\x00\x00s\x00\x00\x00a\x00\x00\x00g\x00\x00\x00e\x00\x00\x00\x22\x00\x00\x00:\x00\x00\x00 \x00\x00\x00\x22\x00\x00\x00m\x00\x00\x00\x22\x00\x00\x00,\x00\x00\x00 \x00\x00\x00\x22\x00\x00\x00d\x00\x00\x00e\x00\x00\x00t\x00\x00\x00a\x00\x00\x00i\x00\x00\x00l\x00\x00\x00s\x00\x00\x00\x22\x00\x00\x00:\x00\x00\x00 \x00\x00\x00[\x00\x00\x00{\x00\x00\x00\x22\x00\x00\x00@\x00\x00\x00t\x00\x00\x00y\x00\x00\x00p\x00\x00\x00e\x00\x00\x00\x22\x00\x00\x00:\x00\x00\x00 \x00\x00\x00\x22\x00\x00\x00t\x00\x00\x00y\x00\x00\x00p\x00\x00\x00e\x00\x00\x00.\x00\x00\x00g\x00\x00\x00o\x00\x00\x00o\x00\x00\x00g\x00\x00\x00l\x00\x00\x00e\x00\x00\x00a\x00\x00\x00p\x00\x00\x00i\x00\x00\x00s\x00\x00\x00.\x00\x00\x00c\x00\x00\x00o\x00\x00\x00m\x00\x00\x00/\x00\x00\x00g\x00\x00\x00o\x00\x00\x00o\x00\x00\x00g\x00\x00\x00l\x00\x00\x00e\x00\x00\x00.\x00\x00\x00r\x00\x00\x00p\x00\x00\x00c\x00\x00\x00.\x00\x00\x00E\x00\x00\x00r\x00\x00\x00r\x00\x00\x00o\x00\x00\x00r\x00\x00\x00I\x00\x00\x00n\x00\x00\x00f\x00\x00\x00o\x00\x00\x00\x22\x00\x00\x00,\x00\x00\x00 \x00\x00\x00\x22\x00\x00\x00r\x00\x00\x00e\x00\x00\x00a\x00\x00\x00s\x00\x00\x00o\x00\x00\x00n\x00\x00\x00\x22\x00\x00\x00:\x00\x00\x00 \x00\x00\x00\x22\x00\x00\x00A\x00\x00\x00P\x00\x00\x00I\x00\x00\x00_\x00\x00\x00K\x00\x00\x00E\x00\x00\x00Y\x00\x00\x00_\x00\x00\x00I\x00\x00\x00N\x00\x00\x00V\x00\x00\x00A\x00\x00\x00L\x00\x00\x00I\x00\x00\x00D\x00\x00\x00\x22\x00\x00\x00}\x00\x00\x00]\x00\x00\x00}\x00\x00\x00}\x00\x00\x00", "invalid, Google rejected the key (API_KEY_INVALID)"), + (400, b"\x00\x00\x00{\x00\x00\x00\x22\x00\x00\x00e\x00\x00\x00r\x00\x00\x00r\x00\x00\x00o\x00\x00\x00r\x00\x00\x00\x22\x00\x00\x00:\x00\x00\x00 \x00\x00\x00{\x00\x00\x00\x22\x00\x00\x00c\x00\x00\x00o\x00\x00\x00d\x00\x00\x00e\x00\x00\x00\x22\x00\x00\x00:\x00\x00\x00 \x00\x00\x004\x00\x00\x000\x00\x00\x000\x00\x00\x00,\x00\x00\x00 \x00\x00\x00\x22\x00\x00\x00m\x00\x00\x00e\x00\x00\x00s\x00\x00\x00s\x00\x00\x00a\x00\x00\x00g\x00\x00\x00e\x00\x00\x00\x22\x00\x00\x00:\x00\x00\x00 \x00\x00\x00\x22\x00\x00\x00m\x00\x00\x00\x22\x00\x00\x00,\x00\x00\x00 \x00\x00\x00\x22\x00\x00\x00d\x00\x00\x00e\x00\x00\x00t\x00\x00\x00a\x00\x00\x00i\x00\x00\x00l\x00\x00\x00s\x00\x00\x00\x22\x00\x00\x00:\x00\x00\x00 \x00\x00\x00[\x00\x00\x00{\x00\x00\x00\x22\x00\x00\x00@\x00\x00\x00t\x00\x00\x00y\x00\x00\x00p\x00\x00\x00e\x00\x00\x00\x22\x00\x00\x00:\x00\x00\x00 \x00\x00\x00\x22\x00\x00\x00t\x00\x00\x00y\x00\x00\x00p\x00\x00\x00e\x00\x00\x00.\x00\x00\x00g\x00\x00\x00o\x00\x00\x00o\x00\x00\x00g\x00\x00\x00l\x00\x00\x00e\x00\x00\x00a\x00\x00\x00p\x00\x00\x00i\x00\x00\x00s\x00\x00\x00.\x00\x00\x00c\x00\x00\x00o\x00\x00\x00m\x00\x00\x00/\x00\x00\x00g\x00\x00\x00o\x00\x00\x00o\x00\x00\x00g\x00\x00\x00l\x00\x00\x00e\x00\x00\x00.\x00\x00\x00r\x00\x00\x00p\x00\x00\x00c\x00\x00\x00.\x00\x00\x00E\x00\x00\x00r\x00\x00\x00r\x00\x00\x00o\x00\x00\x00r\x00\x00\x00I\x00\x00\x00n\x00\x00\x00f\x00\x00\x00o\x00\x00\x00\x22\x00\x00\x00,\x00\x00\x00 \x00\x00\x00\x22\x00\x00\x00r\x00\x00\x00e\x00\x00\x00a\x00\x00\x00s\x00\x00\x00o\x00\x00\x00n\x00\x00\x00\x22\x00\x00\x00:\x00\x00\x00 \x00\x00\x00\x22\x00\x00\x00A\x00\x00\x00P\x00\x00\x00I\x00\x00\x00_\x00\x00\x00K\x00\x00\x00E\x00\x00\x00Y\x00\x00\x00_\x00\x00\x00I\x00\x00\x00N\x00\x00\x00V\x00\x00\x00A\x00\x00\x00L\x00\x00\x00I\x00\x00\x00D\x00\x00\x00\x22\x00\x00\x00}\x00\x00\x00]\x00\x00\x00}\x00\x00\x00}", "invalid, Google rejected the key (API_KEY_INVALID)"), + (400, b"{\x22error\x22: {\x22x\x22: NaN, \x22details\x22: [{\x22@type\x22: \x22google.rpc.ErrorInfo\x22, \x22reason\x22: \x22API_KEY_INVALID\x22}]}}", "invalid, Google rejected the key (API_KEY_INVALID)"), + (403, b"{\x22error\x22: {\x22details\x22: [{\x22@type\x22: \x22google.rpc.ErrorInfo\x22, \x22reason\x22: NaN}]}}", "unknown, Google answered HTTP 403 (nan)"), + (403, b"{\x22error\x22: {\x22details\x22: [{\x22@type\x22: \x22google.rpc.ErrorInfo\x22, \x22reason\x22: -Infinity}]}}", "unknown, Google answered HTTP 403 (-inf)"), + (403, b"{\x22error\x22: {\x22details\x22: [{\x22@type\x22: \x22google.rpc.ErrorInfo\x22, \x22reason\x22: 1e400}]}}", "unknown, Google answered HTTP 403 (inf)"), + (400, b"{\x22error\x22: {\x22details\x22: [{\x22@type\x22: \x22google.rpc.ErrorInfo\x22, \x22reason\x22: \x22SERVICE_DISABLED\x22, \x22reason\x22: \x22API_KEY_INVALID\x22}]}}", "invalid, Google rejected the key (API_KEY_INVALID)"), + (400, b"{\x22error\x22: {\x22details\x22: [{\x22@type\x22: \x22google.rpc.ErrorInfo\x22, \x22reason\x22: \x22API_KEY_INVALID\x22}]}} trailing", "unknown, Google answered HTTP 400"), + (400, b"\x80{\x22error\x22: 1}", "unknown, Google answered HTTP 400"), + (400, b" \x09\x0a{\x22error\x22: {\x22details\x22: [{\x22@type\x22: \x22google.rpc.ErrorInfo\x22, \x22reason\x22: \x22\x5cu0041PI_KEY_INVALID\x22}]}} \x0d\x0a", "invalid, Google rejected the key (API_KEY_INVALID)"), + (403, b"{\x22error\x22: {\x22details\x22: [{\x22@type\x22: \x22google.rpc.ErrorInfo\x22, \x22reason\x22: \x22\x5cud83d\x5cude00 x\x22}]}}", "unknown, Google answered HTTP 403 (😀 x)"), + (403, b"{\x22error\x22: {\x22details\x22: [{\x22@type\x22: \x22google.rpc.ErrorInfo\x22, \x22reason\x22: \x22a\x5c\x22b\x5c\x5cc\x5c/d\x5cn\x22}]}}", "unknown, Google answered HTTP 403 (a\"b\\c/d\n)"), + (403, b"{\x22error\x22: {\x22details\x22: [{\x22@type\x22: \x22google.rpc.ErrorInfo\x22, \x22reason\x22: 012}]}}", "unknown, Google answered HTTP 403"), + (403, b"{\x22error\x22: {\x22details\x22: [{\x22@type\x22: \x22google.rpc.ErrorInfo\x22, \x22reason\x22: 1.}]}}", "unknown, Google answered HTTP 403"), + (403, b"{\x22error\x22: {\x22details\x22: [{\x22@type\x22: \x22google.rpc.ErrorInfo\x22, \x22reason\x22: -0}]}}", "unknown, Google answered HTTP 403"), + (403, b"{\x22error\x22: {\x22details\x22: [{\x22@type\x22: \x22google.rpc.ErrorInfo\x22, \x22reason\x22: 2E3}]}}", "unknown, Google answered HTTP 403 (2000.0)"), + (403, b"{\x22error\x22: {\x22details\x22: [{\x22@type\x22: \x22google.rpc.ErrorInfo\x22, \x22reason\x22: \x22x\x22,}]}}", "unknown, Google answered HTTP 403"), + (403, b"{\x22error\x22: {\x22details\x22: [{\x22@type\x22: \x22google.rpc.ErrorInfo\x22, \x22reason\x22: \x22tab\x09here\x22}]}}", "unknown, Google answered HTTP 403"), + ]; + for &(code, body, expected) in cases { + assert_eq!( + GoogleApiKeyVerifier::describe_error(code, body), + expected, + "{}", + String::from_utf8_lossy(body) + ); + } + } + + #[test] + fn any_2xx_is_valid() { + // urlopen() only raises HTTPError outside 200 to 299 + for status in [200, 204, 299] { + let verifier = + GoogleApiKeyVerifier::with_http(Box::new(move |_| Ok((status, Vec::new())))); + assert_eq!( + verifier.verify("k"), + "valid, Google accepted the key", + "{status}" + ); + } + let verifier = GoogleApiKeyVerifier::with_http(Box::new(|_| Ok((300, Vec::new())))); + assert_eq!(verifier.verify("k"), "unknown, Google answered HTTP 300"); + } + + /// A local HTTP server that answers each request with the response of its path, and + /// records the requests. + #[cfg(feature = "verify")] + fn serve(responses: Vec<(&'static str, String)>) -> (String, Arc>>) { + use std::io::{BufRead, BufReader, Write}; + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let base = format!("http://{}", listener.local_addr().unwrap()); + let requests = Arc::new(Mutex::new(Vec::new())); + let seen = requests.clone(); + std::thread::spawn(move || { + for stream in listener.incoming() { + let Ok(mut stream) = stream else { break }; + let mut reader = BufReader::new(stream.try_clone().unwrap()); + let mut request = String::new(); + loop { + let mut line = String::new(); + if reader.read_line(&mut line).unwrap_or(0) == 0 || line == "\r\n" { + break; + } + request.push_str(&line); + } + let path = request.split_whitespace().nth(1).unwrap_or("").to_string(); + seen.lock().unwrap().push(request); + let response = responses + .iter() + .find(|(prefix, _)| path.starts_with(prefix)) + .map_or_else( + || "HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\n\r\n".to_string(), + |(_, r)| r.clone(), + ); + let _ = stream.write_all(response.as_bytes()); + } + }); + (base, requests) + } + + #[test] + #[cfg(feature = "verify")] + fn requests_like_urlopen() { + let invalid = String::from_utf8(google_error("API_KEY_INVALID")).unwrap(); + let (base, requests) = serve(vec![ + ("/invalid", format!("HTTP/1.1 400 Bad Request\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{invalid}", invalid.len())), + ("/moved", "HTTP/1.1 301 Moved\r\nLocation: /valid?key=K\r\nContent-Length: 0\r\nConnection: close\r\n\r\n".to_string()), + ("/nowhere", "HTTP/1.1 302 Found\r\nContent-Length: 5\r\nConnection: close\r\n\r\nmoved".to_string()), + ("/valid", "HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\n{}".to_string()), + ]); + let answer = get(&format!("{base}/invalid?key=K")).unwrap(); + assert_eq!(answer, (400, invalid.into_bytes())); + // The User-Agent of the Python package + let first = requests.lock().unwrap()[0].clone(); + assert!( + first.starts_with("GET /invalid?key=K HTTP/1.1\r\n"), + "{first}" + ); + assert!( + first.to_lowercase().contains("\r\nuser-agent: pywhat\r\n"), + "{first}" + ); + // A redirect is followed, a redirect without a Location is the answer + assert_eq!( + get(&format!("{base}/moved?key=K")).unwrap(), + (200, b"{}".to_vec()) + ); + assert_eq!( + get(&format!("{base}/nowhere?key=K")).unwrap(), + (302, b"moved".to_vec()) + ); + // Nothing listening: urllib's description of the error + let closed = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let port = closed.local_addr().unwrap().port(); + drop(closed); + let error = get(&format!("http://127.0.0.1:{port}/x")).unwrap_err(); + assert!(error.starts_with("") || cfg!(windows), + "{error}" + ); + } + + #[test] + #[cfg(feature = "verify")] + fn redirect_targets() { + // urllib.parse.urljoin(base, location) + let base = "https://www.googleapis.com/youtube/v3/i18nLanguages?part=snippet&key=K"; + assert_eq!( + urljoin(base, "https://example.com/x"), + "https://example.com/x" + ); + assert_eq!(urljoin(base, "//example.com/x"), "https://example.com/x"); + assert_eq!( + urljoin(base, "/x?key=K"), + "https://www.googleapis.com/x?key=K" + ); + assert_eq!( + urljoin(base, "languages?key=K"), + "https://www.googleapis.com/youtube/v3/languages?key=K" + ); + assert_eq!(urljoin("http://h:8080", "x"), "http://h:8080/x"); + } + + #[test] + fn quoting() { + assert_eq!(quote("a-b_c.d~e/f g"), "a-b_c.d~e%2Ff%20g"); + } +} diff --git a/src/files.rs b/src/files.rs index 6824eed..8883cff 100644 --- a/src/files.rs +++ b/src/files.rs @@ -2,7 +2,7 @@ //! `Identifier._files()`, `Identifier._collect()` and `pywhat/magic_numbers.py`. use crate::data::FILE_SIGNATURES_JSON; -use crate::identify::{check, to_identify_json, Identifier, Match}; +use crate::identify::{check_with, to_identify_json, Identifier, Match}; use crate::ranking::{sort_matches, Key}; use crate::unicode; use serde_json::{Map, Value}; @@ -201,6 +201,27 @@ pub struct InputOptions { pub include_filenames: bool, } +/// How far [`Identifier::iter_identify_with_progress`] is, like `Progress` in Python. The +/// files (a text is a file too) are searched one after another, and every regex is +/// searched for in every text of a file: its contents, the UTF-16 strings in it and, with +/// `include_filenames`, its name. The regexes of a large text are searched for in +/// parallel. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Progress<'a> { + /// The file that is searched, or "text", like the locations of the matches. + pub location: &'a str, + /// How many files have been searched, of how many. + pub files_done: usize, + pub files: usize, + /// How many of the regex searches of the file are done, of how many (the number of + /// regexes times the number of texts in the file). + pub regexes_done: usize, + pub regexes: usize, + /// The name of a regex that is searched for now (the one searched for the longest), + /// `None` once the file has been searched. + pub regex: Option<&'static str>, +} + /// What [`Identifier::identify_inputs`] found, like the dict that Python returns. #[derive(Debug, Clone, Default)] pub struct Identified { @@ -328,9 +349,38 @@ impl Identifier { options: InputOptions, found: &mut dyn FnMut(Found) -> bool, error: &mut dyn FnMut(&str, std::io::Error), + ) { + self.identify_sources(inputs, options, found, error, None); + } + + /// [`Identifier::iter_identify`], calling `progress` when a regex search starts in a + /// file (and when one ends while others still run), and once the file has been + /// searched, like the `progress` callback of Python's `iter_identify()`. For a large + /// text the regexes are searched for in parallel, so `progress` is called from rayon's + /// threads. + pub fn iter_identify_with_progress( + &self, + inputs: &[String], + options: InputOptions, + found: &mut dyn FnMut(Found) -> bool, + error: &mut dyn FnMut(&str, std::io::Error), + progress: &(dyn Fn(Progress<'_>) + Sync), + ) { + self.identify_sources(inputs, options, found, error, Some(progress)); + } + + fn identify_sources( + &self, + inputs: &[String], + options: InputOptions, + found: &mut dyn FnMut(Found) -> bool, + error: &mut dyn FnMut(&str, std::io::Error), + progress: Option<&(dyn Fn(Progress<'_>) + Sync)>, ) { let search = self.search(); - for source in sources(inputs, options.only_text) { + let sources = sources(inputs, options.only_text); + let files = sources.len(); + for (files_done, source) in sources.into_iter().enumerate() { let (location, texts) = match source { Source::Text(text) => ("text".to_string(), vec![text]), Source::File { location, path } => { @@ -353,8 +403,41 @@ impl Identifier { } } }; + let regexes = search.regexes.len() * texts.len(); + // The regexes whose search has started and isn't done, oldest first, and how + // many searches are done + let running: std::sync::Mutex<(Vec<&'static str>, usize)> = + std::sync::Mutex::new((Vec::new(), 0)); + let on_regex = |entry: &'static crate::data::RegexEntry, done: bool| { + let Some(progress) = progress else { return }; + let mut running = running + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let (names, finished) = &mut *running; + if done { + if let Some(index) = names.iter().position(|name| *name == entry.name) { + names.remove(index); + } + *finished += 1; + if names.is_empty() { + return; // the next regex reports when it starts + } + } else { + names.push(&entry.name); + } + progress(Progress { + location: &location, + files_done, + files, + regexes_done: *finished, + regexes, + regex: names.first().copied(), + }); + }; + let on_regex: Option> = + progress.is_some().then_some(&on_regex); for text in &texts { - let matches = match check(text, &search) { + let matches = match check_with(text, &search, on_regex) { Ok(matches) => matches, Err(_) => self.identify(text), }; @@ -367,6 +450,16 @@ impl Identifier { } } } + if let Some(progress) = progress { + progress(Progress { + location: &location, + files_done: files_done + 1, + files, + regexes_done: regexes, + regexes, + regex: None, + }); + } } } @@ -415,6 +508,164 @@ mod tests { ); } + /// A [`Progress`] that was reported. + #[derive(Debug, Clone, PartialEq, Eq)] + struct Reported { + location: String, + files_done: usize, + files: usize, + regexes_done: usize, + regexes: usize, + regex: Option<&'static str>, + } + + /// What `iter_identify_with_progress` found in `inputs`, and the progress it reported. + fn with_progress(inputs: &[String]) -> (Vec, Vec) { + let reported = std::sync::Mutex::new(Vec::new()); + let mut found = Vec::new(); + Identifier::default().iter_identify_with_progress( + inputs, + InputOptions::default(), + &mut |item| { + found.push(format!("{item:?}")); + true + }, + &mut |_, _| {}, + &|progress| { + reported.lock().unwrap().push(Reported { + location: progress.location.to_string(), + files_done: progress.files_done, + files: progress.files, + regexes_done: progress.regexes_done, + regexes: progress.regexes, + regex: progress.regex, + }) + }, + ); + (found, reported.into_inner().unwrap()) + } + + /// The names of the regexes that `Identifier::default()` searches for, in order. + fn default_regexes() -> Vec<&'static str> { + Identifier::default() + .search() + .regexes + .iter() + .map(|&entry| entry.name.as_str()) + .collect() + } + + #[test] + fn progress_of_a_search() { + let inputs = vec!["THM{a}".to_string(), "bee@skerritt.blog".to_string()]; + let mut without = Vec::new(); + Identifier::default().iter_identify( + &inputs, + InputOptions::default(), + &mut |item| { + without.push(format!("{item:?}")); + true + }, + &mut |_, _| {}, + ); + let (found, reported) = with_progress(&inputs); + // Progress doesn't change what is found + assert_eq!(found, without); + assert!(!found.is_empty()); + // Short texts: every regex in database order, then the text is done + let names = default_regexes(); + let regexes = names.len(); + let mut expected = Vec::new(); + for file in 0..2 { + for (regexes_done, &name) in names.iter().enumerate() { + expected.push(Reported { + location: "text".to_string(), + files_done: file, + files: 2, + regexes_done, + regexes, + regex: Some(name), + }); + } + expected.push(Reported { + location: "text".to_string(), + files_done: file + 1, + files: 2, + regexes_done: regexes, + regexes, + regex: None, + }); + } + assert_eq!(reported, expected); + } + + #[test] + fn progress_of_a_large_text() { + // Over 4 KB, the regexes are searched for in parallel: the progress comes from + // several threads, one report at a time, and its counts only go up + let text = "THM{a} bee@skerritt.blog\n".repeat(400); + let (_, reported) = with_progress(&[text]); + let regexes = default_regexes().len(); + let searching = reported.iter().filter(|r| r.regex.is_some()).count(); + assert!( + searching >= regexes, + "{searching} reports, {regexes} regexes" + ); + assert!(reported + .windows(2) + .all(|pair| pair[0].regexes_done <= pair[1].regexes_done)); + assert_eq!( + reported.last(), + Some(&Reported { + location: "text".to_string(), + files_done: 1, + files: 1, + regexes_done: regexes, + regexes, + regex: None, + }) + ); + } + + #[test] + fn progress_counts_every_text_of_a_file() { + // The contents, the UTF-16 strings in them and, with include_filenames, the name: + // every regex is searched for in each (`len(dist.get_regexes()) * len(contents)`) + let dir = std::env::temp_dir().join(format!("pywhat-progress-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let file = dir.join("THM{name}.bin"); + let mut data = b"THM{a}\n\x00".to_vec(); + data.extend("THM{wide}".encode_utf16().flat_map(u16::to_le_bytes)); + std::fs::write(&file, &data).unwrap(); + let regexes = default_regexes().len(); + let identifier = Identifier::default(); + let reported = std::sync::Mutex::new(Vec::new()); + let options = InputOptions { + only_text: false, + include_filenames: true, + }; + identifier.iter_identify_with_progress( + &[file.to_string_lossy().into_owned()], + options, + &mut |_| true, + &mut |_, _| {}, + &|progress| { + reported.lock().unwrap().push(( + progress.regexes_done, + progress.regexes, + progress.regex, + )) + }, + ); + std::fs::remove_dir_all(&dir).unwrap(); + let reported = reported.into_inner().unwrap(); + assert_eq!(reported.last(), Some(&(3 * regexes, 3 * regexes, None))); + assert_eq!( + reported.iter().filter(|r| r.2.is_some()).count(), + 3 * regexes + ); + } + #[test] fn join_like_os_path() { let sep = MAIN_SEPARATOR; diff --git a/src/identify.rs b/src/identify.rs index 3476251..188e445 100644 --- a/src/identify.rs +++ b/src/identify.rs @@ -429,7 +429,21 @@ pub(crate) struct Search<'a> { /// The matches of every regex of `search` in `text`, in database order and then in the /// order of the text, with their fragments marked (`iter_check()` for one string). pub(crate) fn check(text: &str, search: &Search<'_>) -> Result, MatchError> { - let spans = find_spans(text, search)?; + check_with(text, search, None) +} + +/// Called with each regex when its search starts (`false`) and when it is done (`true`), +/// for progress bars. For a large text the regexes are searched for in parallel, so it is +/// called from rayon's threads. +pub(crate) type OnRegex<'a> = &'a (dyn Fn(&'static RegexEntry, bool) + Sync); + +/// [`check`], calling `on_regex` when each regex search starts and ends. +pub(crate) fn check_with( + text: &str, + search: &Search<'_>, + on_regex: Option>, +) -> Result, MatchError> { + let spans = find_spans(text, search, on_regex)?; let mut found = Vec::new(); for (entry, entry_spans) in search.regexes.iter().zip(spans) { if entry_spans.is_empty() { @@ -451,17 +465,31 @@ pub(crate) fn check(text: &str, search: &Search<'_>) -> Result, Match } /// The spans of the matches of each regex of `search` in `text`. -fn find_spans(text: &str, search: &Search<'_>) -> Result>, MatchError> { +fn find_spans( + text: &str, + search: &Search<'_>, + on_regex: Option>, +) -> Result>, MatchError> { let spans_of = |(entry, &boundaryless): (&&'static RegexEntry, &bool)| { + if let Some(on_regex) = on_regex { + on_regex(entry, false); + } let form = if boundaryless { Form::Boundaryless } else { Form::Anchored }; - let compiled = compiled(entry, form).map_err(MatchError)?; - let mut spans = Vec::new(); - compiled.find_iter(text, &mut |start, end| spans.push((start, end)))?; - Ok(spans) + let search = || -> Result, MatchError> { + let compiled = compiled(entry, form).map_err(MatchError)?; + let mut spans = Vec::new(); + compiled.find_iter(text, &mut |start, end| spans.push((start, end)))?; + Ok(spans) + }; + let spans = search(); + if let Some(on_regex) = on_regex { + on_regex(entry, true); + } + spans }; #[cfg(feature = "parallel")] if text.len() > 4096 { diff --git a/src/lib.rs b/src/lib.rs index f070e3c..59e92c5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -19,6 +19,8 @@ mod bitcoin; mod charclass; +#[cfg(all(feature = "cli", not(target_arch = "wasm32")))] +pub mod cli; pub mod data; pub mod engine; #[cfg(feature = "files")] @@ -37,7 +39,7 @@ pub use bitcoin::is_bitcoin_address; pub use data::{regexes, tags, RegexEntry, CHARSET_ONLY}; pub use engine::{EngineKind, MatchError}; #[cfg(feature = "files")] -pub use files::{Found, Identified, InputOptions}; +pub use files::{Found, Identified, InputOptions, Progress}; pub use filter::{split_tags, Filter, InvalidTag}; pub use identify::{compile_all, engine_of, Identifier, Match}; pub use processors::{default_processors, Processor}; diff --git a/tests/cli.rs b/tests/cli.rs new file mode 100644 index 0000000..5382ee0 --- /dev/null +++ b/tests/cli.rs @@ -0,0 +1,632 @@ +//! Tests of the `what` command: its argument parser and its output formats. They run the +//! binary that cargo built, without a terminal, so there are no colours and no progress +//! bars. `scripts/parity_cli.py` compares the output with the Python CLI's. + +use std::io::Write; +use std::path::Path; +use std::process::{Command, Stdio}; + +/// The exit code, stdout and stderr of `what args` with `stdin`. +fn what_with(args: &[&str], stdin: &[u8], env: &[(&str, &str)]) -> (i32, String, String) { + let mut command = Command::new(env!("CARGO_BIN_EXE_what")); + command + .args(args) + .current_dir(env!("CARGO_MANIFEST_DIR")) + .env_remove("NO_COLOR") + .env_remove("COLUMNS") + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + for (name, value) in env { + command.env(name, value); + } + let mut child = command.spawn().expect("what runs"); + child + .stdin + .take() + .expect("stdin") + .write_all(stdin) + .expect("stdin is written"); + let output = child.wait_with_output().expect("what finishes"); + ( + output.status.code().expect("an exit code"), + String::from_utf8(output.stdout).expect("UTF-8 stdout"), + String::from_utf8(output.stderr).expect("UTF-8 stderr"), + ) +} + +fn what(args: &[&str]) -> (i32, String, String) { + what_with(args, b"", &[]) +} + +/// The names of the matches in raw output. +fn names(output: &str) -> Vec<&str> { + output + .lines() + .filter_map(|line| line.strip_prefix("Name: ")) + .collect() +} + +// The argument parser + +#[test] +fn unknown_options_are_inputs() { + // click's ignore_unknown_options: -1606586780 is the text to search, as after -- + let (code, stdout, _) = what(&["--rarity", "0:", "--format", "%m|%n", "-1606586780"]); + assert_eq!(code, 0); + assert_eq!(stdout, "160658678|American Social Security Number\n"); + let (_, after_dashes, _) = what(&["--rarity", "0:", "--format", "%m|%n", "--", "-1606586780"]); + assert_eq!(stdout, after_dashes); + let (code, stdout, _) = what(&["-db", "--nope", "THM{x}"]); + assert_eq!(code, 0); + assert_eq!(names(&stdout), ["TryHackMe Flag Format"]); +} + +#[test] +fn multi_letter_short_options() { + let (_, with, _) = what(&["abc118.103.238.230abc"]); + let (_, without, _) = what(&["-db", "abc118.103.238.230abc"]); + assert!(names(&with).contains(&"Internet Protocol (IP) Address Version 4")); + assert_eq!(without, "Nothing found!\n"); + // -bi takes a value, -pt is a flag + let (_, stdout, _) = what(&["-bi", "IPv4 Address", "-pt", "abc118.103.238.230abc"]); + assert!( + stdout.starts_with("Matched on: 118.103.238.230\n"), + "{stdout}" + ); + assert!(stdout.contains("\nTags: "), "{stdout}"); + // The email address has a rarity below 0.6, so -br 0.6: searches it with boundaries + let (_, stdout, _) = what(&["-br", "0.6:", "-be", "URL", "123myEmail@host.org456"]); + assert_eq!(stdout, "Nothing found!\n"); +} + +#[test] +fn clusters_and_attached_values() { + // -o and -k in one argument, -k with its value attached + let (code, stdout, _) = what(&["-okname", "--json", "fixtures/file", "THM{x}"]); + assert_eq!(code, 0); + assert!( + stdout + .starts_with(r#"{"File Signatures": null, "Regexes": {"text": [{"Matched": "THM{x}""#), + "{stdout}" + ); + let (_, a, _) = what(&[ + "--top=1", + "-db", + "0x52908400098527886E0F7030069857D2E4169EE7", + ]); + let (_, b, _) = what(&[ + "--top", + "1", + "-db", + "0x52908400098527886E0F7030069857D2E4169EE7", + ]); + assert_eq!(a, b); +} + +#[test] +fn double_dash() { + let (code, stdout, _) = what(&["--format", "%m", "--", "-db", "THM{x}"]); + assert_eq!(code, 0); + assert!(stdout.lines().any(|line| line == "THM{x}"), "{stdout}"); +} + +#[test] +fn usage_errors_exit_with_2() { + // click's parser errors have no context: no usage line + let (code, stdout, stderr) = what(&["--rarity"]); + assert_eq!((code, stdout.as_str()), (2, "")); + assert_eq!(stderr, "Error: Option '--rarity' requires an argument.\n"); + let (code, _, stderr) = what(&["--json=1", "x"]); + assert_eq!(code, 2); + assert_eq!(stderr, "Error: Option '--json' does not take a value.\n"); + // An invalid value: the usage line and a hint first + let (code, _, stderr) = what(&["--map", "bing", "x"]); + assert_eq!(code, 2); + assert_eq!( + stderr, + "Usage: what [OPTIONS] [TEXT_INPUT]...\nTry 'what --help' for help.\n\n\ + Error: Invalid value for '--map': 'bing' is not one of 'google', 'osm'.\n" + ); +} + +#[test] +fn markup_errors_in_the_format() { + // rich's MarkupError, where Python stops with a traceback before printing anything + let (code, stdout, stderr) = what(&["-db", "--format", "[b]%m[/i]", "THM{x}"]); + assert_eq!((code, stdout.as_str()), (1, "")); + assert_eq!( + stderr, + "Error: closing tag '[/i]' at position 9 doesn't match any open tag\n" + ); + let (code, stdout, _) = what(&["-db", "--stream", "--format", "[/]%m", "THM{x}"]); + assert_eq!((code, stdout.as_str()), (1, "")); + // Valid markup is rendered, and nothing is checked when nothing is found + assert_eq!( + what(&["-db", "--format", "[b]%m[/bold]", "THM{x}"]).1, + "THM{x}\n" + ); + assert_eq!(what(&["-db", "--format", "[/b]", "nothing"]).0, 0); +} + +#[test] +fn errors_exit_with_1() { + for (args, message) in [ + (&["--rarity", "1", "x"][..], "Invalid rarity range format ('min:max' expected)\n"), + (&["--rarity", "a:", "x"], "Invalid rarity argument (float expected)\n"), + (&["--key", "nope", "x"], "Invalid key\n"), + ( + &["--top", "0", "x"], + "Invalid top '0', use a number of matches such as 10, a percentage of them such as 5% or all\n", + ), + ] { + let (code, stdout, _) = what(args); + assert_eq!((code, stdout.as_str()), (1, message), "{args:?}"); + } + let (code, stdout, _) = what(&["--include", "ETH Walet", "x"]); + assert_eq!(code, 1); + assert!(stdout.contains("Passed tags are not valid")); + assert!(stdout.contains("'pywhat --names'")); + for (option, value, message) in [ + ( + "--key", + "name", + "--stream cannot sort the matches, so it cannot be used with --key\n", + ), + ( + "--top", + "1", + "--stream cannot pick the most likely matches, so it cannot be used with --top\n", + ), + ] { + let (code, _, stderr) = what(&["--stream", option, value, "x"]); + assert_eq!((code, stderr.as_str()), (1, message)); + } +} + +#[test] +fn interactive_needs_python() { + let (code, stdout, stderr) = what(&["--interactive", "x"]); + assert_eq!((code, stdout.as_str()), (2, "")); + assert_eq!(stderr.lines().count(), 1); + assert!(stderr.contains("pip install pywhat")); +} + +#[test] +fn eager_options() { + let (code, stdout, _) = what(&["--tags", "x"]); + assert_eq!(code, 0); + assert!(stdout.lines().any(|line| line == "Bug Bounty")); + let (_, stdout, _) = what(&["--names"]); + assert!(stdout + .lines() + .any(|line| line == "Ethereum (ETH) Wallet Address: Ethereum Wallet, ETH Wallet")); + let (_, stdout, _) = what(&["-v"]); + assert_eq!( + stdout, + format!("PyWhat version {}\n", env!("CARGO_PKG_VERSION")) + ); + // --help comes first, whatever the order + let (code, stdout, _) = what(&["--map", "bing", "--help"]); + assert_eq!(code, 0); + assert!(stdout.starts_with("Usage: what [OPTIONS] [TEXT_INPUT]...\n\n pyWhat - Identify")); +} + +#[test] +fn help_wraps_at_the_width_of_click() { + let (_, wide, _) = what_with(&["--help"], b"", &[("COLUMNS", "10000")]); + assert!(wide.lines().all(|line| line.chars().count() <= 78)); + assert!(wide.contains(" -br, --boundaryless-rarity TEXT\n")); + let (_, narrow, _) = what_with(&["--help"], b"", &[("COLUMNS", "30")]); + assert!( + narrow.lines().all(|line| line.chars().count() <= 50), + "at least 50 columns" + ); + assert!(narrow.lines().count() > wide.lines().count()); +} + +#[test] +fn stdin_without_inputs() { + let (code, stdout, _) = what_with(&["-db"], b"THM{hello}\nfixtures/file\n", &[]); + assert_eq!(code, 0); + // One text, not a list of inputs + assert_eq!(names(&stdout), ["TryHackMe Flag Format"]); + assert!(!stdout.contains("File:")); + let (_, stdout, _) = what_with(&["--json"], &utf16_with_bom("THM{wide}"), &[]); + assert!(stdout.contains(r#""Matched": "THM{wide}""#), "{stdout}"); + let (code, stdout, _) = what_with(&[], b"", &[]); + assert_eq!((code, stdout.as_str()), (0, "Nothing found!\n")); + // Only empty inputs are no inputs too (get_text()'s `not any(value)`) + for args in [&["-db", ""][..], &["-db", "", ""]] { + let (_, stdout, _) = what_with(args, b"THM{stdin}\n", &[]); + assert_eq!(names(&stdout), ["TryHackMe Flag Format"], "{args:?}"); + } + let (_, stdout, _) = what_with(&["-db", "", "THM{a}"], b"THM{stdin}\n", &[]); + assert!(stdout.starts_with("Matched on: THM{a}\n"), "{stdout}"); + // Stripped like str.strip(), which takes U+001C to U+001F too + let (_, stdout, _) = what_with(&["-db"], b"\x1c THM{x}\x1f", &[]); + assert_eq!(names(&stdout), ["TryHackMe Flag Format"]); +} + +fn utf16_with_bom(text: &str) -> Vec { + let mut data = vec![0xFF, 0xFE]; + data.extend(text.encode_utf16().flat_map(u16::to_le_bytes)); + data +} + +// The formats + +#[test] +fn raw_output() { + let (code, stdout, _) = what(&["-db", "THM{hello}", "github@skerritt.blog"]); + assert_eq!(code, 0); + assert_eq!( + stdout, + "Matched on: THM{hello}\nName: TryHackMe Flag Format\n\ + Description: Used for Capture The Flags at https://tryhackme.com\n\n\ + Matched on: github@skerritt.blog\nName: Email Address\n" + ); + let (_, stdout, _) = what(&["-db", "-pt", "thm{2}"]); + assert!(stdout.contains("Tags: CTF Flag")); + let (_, stdout, _) = what(&["-db", "fixtures/test/file", "THM{hello}"]); + assert!(stdout.contains("File: fixtures/test/file\nMatched on: https://google.com")); + assert!(stdout.contains("File: text\nMatched on: THM{hello}")); +} + +#[test] +fn matched_text_is_not_markup() { + // The Python CLI prints "thm{x}" (#291, Python bug 4) + let (_, stdout, _) = what(&["-db", "thm{[bold]x[/bold]}"]); + assert!( + stdout.starts_with("Matched on: thm{[bold]x[/bold]}\n"), + "{stdout}" + ); + // In --format strings and tables too, where the format's own markup is rendered + let (_, stdout, _) = what(&["-db", "--format", "[b]%m[/b]", "THM{[i]x[/i]}"]); + assert_eq!(stdout, "THM{[i]x[/i]}\n"); + let (_, stdout, _) = what_with( + &["-db", "--format", "pretty", "THM{[b]x[/b]}"], + b"", + &[("COLUMNS", "200")], + ); + assert_eq!(cells(&stdout)[1][0], "THM{[b]x[/b]}"); + // The markup of the data is rendered, like rich does + let (_, stdout, _) = what(&["-db", "Access-Control-Allow: *"]); + assert!( + stdout.contains("Description: Used for Cross-Origin Resource Sharing (CORS)\n"), + "{stdout}" + ); +} + +#[test] +fn nothing_found() { + assert_eq!(what(&["-db", ""]).1, "Nothing found!\n"); + assert_eq!(what(&["-db", "nothing", "here"]).1, "Nothing found!\n"); + assert_eq!(what(&["-db", "--format", "%m", "nothing"]).1, ""); + assert_eq!( + what(&["-db", "--json", ""]).1, + "{\"File Signatures\": null, \"Regexes\": null}\n" + ); +} + +#[test] +fn custom_format() { + let (_, stdout, _) = what(&[ + "-db", + "--format", + r"%m 2%n %d --- -%e%r %l %t \%d", + "rBPAQmwMrt7FDDPNyjwFgwSqbWZPf6SLkk", + ]); + assert_eq!( + stdout, + "rBPAQmwMrt7FDDPNyjwFgwSqbWZPf6SLkk 2Ripple (XRP) Wallet Address --- -0.3 \ + https://xrpscan.com/account/rBPAQmwMrt7FDDPNyjwFgwSqbWZPf6SLkk Finance, \ + Cryptocurrency Wallet, Ripple Wallet, Ripple, XRP %d\n" + ); + // \\ is a backslash, and a backslash in the match is not an escape (Python bug 2) + let (code, stdout, _) = what(&["-db", "--format", r"\\%m - %n", r"thm{a\1}"]); + assert_eq!(code, 0); + assert_eq!(stdout, "\\thm{a\\1} - TryHackMe Flag Format\n"); + // An absent exploit is nothing + assert_eq!(what(&["-db", "--format", "%e", "thm{2}"]).1, ""); +} + +#[test] +fn json_is_one_line_of_ascii() { + let (code, stdout, _) = what(&[ + "--json", + "fixtures/test", + "fixtures/file", + "THM{caf\u{e9} \u{20bf}}", + ]); + assert_eq!(code, 0); + assert!(stdout.is_ascii()); + assert_eq!(stdout.matches('\n').count(), 1); + assert!(stdout.contains(r#""Matched": "THM{caf\u00e9 \u20bf}""#)); + let json: serde_json::Value = serde_json::from_str(&stdout).unwrap(); + let locations: Vec<&String> = json["Regexes"].as_object().unwrap().keys().collect(); + let test_file = Path::new("fixtures/test").join("file"); + assert!(locations.contains(&&test_file.to_string_lossy().into_owned())); + assert!(locations.contains(&&"text".to_string())); + // --format json is --json + assert_eq!( + what(&["--format", " json ", "abc@example.com"]).1, + what(&["--json", "abc@example.com"]).1 + ); +} + +#[test] +fn stream_json_lines_without_fragment() { + let (code, stdout, _) = what(&["--stream", "--json", "fixtures/file"]); + assert_eq!(code, 0); + let (_, all, _) = what(&["--json", "fixtures/file"]); + let all: serde_json::Value = serde_json::from_str(&all).unwrap(); + let mut expected = all["Regexes"]["file"].as_array().unwrap().clone(); + for found in &mut expected { + found.as_object_mut().unwrap().remove("Fragment"); + } + let mut streamed = Vec::new(); + for line in stdout.lines() { + let found: serde_json::Value = serde_json::from_str(line).unwrap(); + assert!(found["File Signatures"].is_null()); + let matches = found["Regexes"]["file"].as_array().unwrap(); + assert_eq!(matches.len(), 1); + assert!(matches[0].get("Fragment").is_none()); + streamed.push(matches[0].clone()); + } + assert_eq!(streamed, expected); + // Nothing found: no lines at all + assert_eq!(what(&["--stream", "--json", "-db", ""]).1, ""); +} + +/// The output with every run of empty lines as one empty line (`squeeze()` of +/// tests/test_click.py). +fn squeeze(output: &str) -> String { + let mut squeezed = output.to_string(); + while squeezed.contains("\n\n\n") { + squeezed = squeezed.replace("\n\n\n", "\n\n"); + } + squeezed +} + +#[test] +fn stream_raw_is_the_complete_output() { + for args in [ + &["-db", "fixtures/file"][..], + &["fixtures/file", "fixtures/test/file", "THM{hello}"], + &["--format", "%m - %n", "fixtures"], + &["-db", ""], + ] { + let (_, complete, _) = what(args); + let (code, streamed, _) = what(&[&["--stream"], args].concat()); + assert_eq!(code, 0); + assert_eq!(squeeze(&streamed), squeeze(&complete), "{args:?}"); + } +} + +#[test] +fn top_note_on_stderr() { + let eth = "0x52908400098527886E0F7030069857D2E4169EE7"; + let (code, stdout, stderr) = what(&["--top", "1", "--json", eth]); + assert_eq!(code, 0); + let json: serde_json::Value = serde_json::from_str(&stdout).unwrap(); + let matches = json["Regexes"]["text"].as_array().unwrap(); + assert_eq!(matches.len(), 1); + assert_eq!( + matches[0]["Regex Pattern"]["Name"], + "Ethereum (ETH) Wallet Address" + ); + assert!( + stderr.starts_with("\nShowing the 1 most likely of "), + "{stderr}" + ); + assert!(stderr.ends_with( + " matches. Use --top to show more of them, or --interactive to search through all of them.\n" + )); + // No note when nothing is left out + assert_eq!(what(&["--top", "all", eth]).2, ""); +} + +/// The cells of the tables in `output`, one row per line, with the lines of a cell +/// joined by spaces. +fn cells(output: &str) -> Vec> { + let mut rows: Vec> = Vec::new(); + let mut row: Vec> = Vec::new(); + for line in output.lines() { + if line.starts_with(['┡', '├', '└']) { + if let Some(first) = row.first() { + let columns = first.len(); + rows.push( + (0..columns) + .map(|i| { + let parts: Vec<&str> = row + .iter() + .map(|parts| parts[i].as_str()) + .filter(|part| !part.is_empty()) + .collect(); + parts.join(" ") + }) + .collect(), + ); + } + row.clear(); + } else if line.starts_with(['┃', '│']) { + // "┃ a ┃ b ┃" is "", " a ", " b ", "" + let parts: Vec<&str> = line.split(['┃', '│']).collect(); + row.push( + parts[1..parts.len() - 1] + .iter() + .map(|cell| cell.trim().to_string()) + .collect(), + ); + } + } + rows +} + +#[test] +fn pretty_table() { + let (code, stdout, _) = what_with( + &[ + "-db", + "--format", + "pretty", + "fixtures/test/file", + "THM{hello}", + ], + b"", + &[("COLUMNS", "200")], + ); + assert_eq!(code, 0); + assert!(stdout.starts_with("Possible Identification\n┏"), "{stdout}"); + assert_eq!( + cells(&stdout), + [ + ["Matched Text", "Identified as", "Description", "File"], + [ + "https://google.com", + "Uniform Resource Locator (URL)", + "None", + "fixtures/test/file" + ], + [ + "THM{hello}", + "TryHackMe Flag Format", + "Used for Capture The Flags at https://tryhackme.com", + "text" + ], + ] + ); + // An Exploit column when a regex has one, and the tags + let heroku = "heroku00000000-0000-0000-0000-000000000000"; + let (_, stdout, _) = what_with( + &[ + "-db", + "--include", + "Bug Bounty", + "--format", + "pretty", + "-pt", + heroku, + ], + b"", + &[("COLUMNS", "1000")], + ); + let rows = cells(&stdout); + assert_eq!( + rows[0], + ["Matched Text", "Identified as", "Description", "Exploit"] + ); + assert_eq!( + rows[1][2], + "Tags: Bug Bounty, API Keys, Credentials, Heroku" + ); + assert!(rows[1][3].contains("curl -X POST https://api.heroku.com/apps")); + // A regex whose Exploit is null has the column too, empty + let client_id = format!("linkedin_client_id={}", "cp334pjr0lxwmw"); + let (_, stdout, _) = what_with( + &["-db", "--format", "pretty", &client_id], + b"", + &[("COLUMNS", "200")], + ); + let rows = cells(&stdout); + assert_eq!(rows[0].last().map(String::as_str), Some("Exploit")); + assert_eq!(rows[1].last().map(String::as_str), Some("")); + // Nothing found + assert_eq!( + what(&["-db", "--format", "pretty", "nothing"]).1, + "Nothing found!\n" + ); +} + +#[test] +fn pretty_table_fits_narrow_terminals() { + let (code, stdout, _) = what_with( + &["-db", "--format", "pretty", "fixtures/file"], + b"", + &[("COLUMNS", "60")], + ); + assert_eq!(code, 0); + assert!(stdout.lines().all(|line| line.chars().count() <= 60)); +} + +#[test] +fn processing_and_maps() { + let (_, stdout, _) = what(&[ + "--rarity", + "0:", + "--include", + "UNIX Timestamp", + "--format", + "%d", + "1637093119", + ]); + assert!( + stdout.contains("Date: November 16, 2021 8:05:19 PM UTC"), + "{stdout}" + ); + // Bitcoin addresses with a wrong checksum are filtered out, unless -dp + let wrong = "3F3F3F3F3F3F3F3F3F3F3F3F3F3F3F3F3F"; + assert!(!names(&what(&["-db", wrong]).1).contains(&"Bitcoin (₿) Wallet Address")); + assert!(names(&what(&["-db", "-dp", wrong]).1).contains(&"Bitcoin (₿) Wallet Address")); + let (_, stdout, _) = what(&["--map", "osm", "--format", "%l", "52.6169586, -1.9779857"]); + assert!(stdout + .lines() + .any(|line| line == "https://www.openstreetmap.org/search?query=52.6169586,-1.9779857")); +} + +#[test] +fn files_and_signatures() { + let dir = std::env::temp_dir().join(format!("pywhat-cli-test-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let image = dir.join("image.png"); + std::fs::write(&image, b"\x89PNG\r\n\x1a\nTHM{hello}\n").unwrap(); + let image = image.to_string_lossy().into_owned(); + let (code, stdout, _) = what(&["-db", &image]); + assert_eq!(code, 0); + assert!( + stdout.starts_with("\nFile Identified: image.png with Magic Numbers .PNG.....\n"), + "{stdout}" + ); + assert!(stdout.contains("Matched on: THM{hello}")); + let (_, streamed, _) = what(&["-db", "--stream", &image]); + assert_eq!(streamed, stdout); + // A file signature without matches: raw output shows only "Nothing found!", --stream + // shows the signature as soon as it is found + let empty = dir.join("empty.png"); + std::fs::write(&empty, b"\x89PNG\r\n\x1a\n").unwrap(); + let empty = empty.to_string_lossy().into_owned(); + assert_eq!(what(&["-db", &empty]).1, "Nothing found!\n"); + let (_, streamed, _) = what(&["-db", "--stream", &empty]); + assert!( + streamed.starts_with("\nFile Identified: empty.png"), + "{streamed}" + ); + assert!(streamed.ends_with("\nNothing found!\n"), "{streamed}"); + // -if searches the name of the file too + let named = dir.join("THM{name}.txt"); + std::fs::write(&named, b"nothing").unwrap(); + let (_, stdout, _) = what(&["-if", "--format", "%m", &named.to_string_lossy()]); + assert_eq!(stdout, "THM{name}\n"); + std::fs::remove_dir_all(&dir).unwrap(); +} + +#[test] +fn closed_output_ends_quietly() { + // what ... | head: the rest of the output can't be written + let mut child = Command::new(env!("CARGO_BIN_EXE_what")) + .args(["--rarity", "0:", "fixtures/file"]) + .current_dir(env!("CARGO_MANIFEST_DIR")) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + drop(child.stdout.take()); + let output = child.wait_with_output().unwrap(); + assert_eq!(output.status.code(), Some(1)); + assert!( + output.stderr.is_empty(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); +} diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..93c3732 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,146 @@ +""" +Run tests/test_click.py against another program, such as the Rust CLI: + + cargo build --release + PYWHAT_CLI=target/release/what pytest -o addopts="" tests/test_click.py + +With PYWHAT_CLI set, CliRunner.invoke() runs $PYWHAT_CLI with the arguments in a +subprocess, with the input on stdin and the environment changed like CliRunner changes +it, and returns its exit code and its output (stdout and stderr together, like +CliRunner's). The tests that call or patch the Python CLI's internals are skipped. +Without PYWHAT_CLI, nothing changes. +""" +import os +import shlex +import shutil +import subprocess +import sys +from typing import IO, Any, Mapping, Optional, Sequence, Union + +import pytest +from click.testing import CliRunner, Result + +# The program to test, as an absolute path, as tests may change the directory +_cli = os.environ.get("PYWHAT_CLI") +PYWHAT_CLI = ( + None + if not _cli + else os.path.abspath(_cli) + if os.path.dirname(_cli) + else shutil.which(_cli) or _cli +) + +# The tests of tests/test_click.py that call or patch the Python CLI's internals, so +# that they can only test the Python CLI +PYTHON_ONLY = { + "test_stream_prints_before_the_search_is_complete": "patches Identifier.iter_identify", + "test_progress_bars": "patches pywhat.what.progress_bars", + "test_no_progress": "patches pywhat.what.progress_bars", + "test_print_stream_api": "calls Printing.print_stream() and Identifier", +} + + +def _is_click_test(item: Any) -> bool: + module = getattr(item, "module", None) + return module is not None and module.__name__.rsplit(".", 1)[-1] == "test_click" + + +def pytest_collection_modifyitems(config: Any, items: Sequence[Any]) -> None: + if not PYWHAT_CLI: + return + for item in items: + name = getattr(item, "originalname", None) or item.name + if _is_click_test(item) and name in PYTHON_ONLY: + reason = f"PYWHAT_CLI is set, and this test {PYTHON_ONLY[name]}" + item.add_marker(pytest.mark.skip(reason=reason)) + + +def _input_bytes(input: Optional[Union[str, bytes, IO]], charset: str) -> bytes: + """The input as CliRunner gives it to the command (make_input_stream()).""" + if input is None: + return b"" + if isinstance(input, str): + return input.encode(charset) + if isinstance(input, bytes): + return input + data = input.read() + return data.encode(charset) if isinstance(data, str) else data + + +def _run_on_a_terminal( + command: Sequence[str], data: bytes, env: Mapping[str, str], stderr: Any +) -> "subprocess.CompletedProcess[bytes]": + """Run command with a terminal (a pseudo-terminal) as stdin, holding data.""" + # A check of sys.platform, so that mypy doesn't check the Unix part on Windows + if sys.platform == "win32": + pytest.skip("PYWHAT_CLI: no pseudo-terminal for stdin on Windows") + import pty + + leader, follower = pty.openpty() + try: + os.write(leader, data) + return subprocess.run( + command, + stdin=follower, + stdout=subprocess.PIPE, + stderr=stderr, + env=dict(env), + timeout=600, + ) + finally: + os.close(follower) + os.close(leader) + + +def invoke( + self: CliRunner, + cli: Any, + args: Optional[Union[str, Sequence[str]]] = None, + input: Optional[Union[str, bytes, IO]] = None, + env: Optional[Mapping[str, Optional[str]]] = None, + catch_exceptions: bool = True, + color: bool = False, + **extra: Any, +) -> Result: + """CliRunner.invoke(), running $PYWHAT_CLI instead of cli.""" + assert PYWHAT_CLI is not None + if isinstance(args, str): + args = shlex.split(args) + command = [PYWHAT_CLI, *(args or ())] + environment = dict(os.environ) + for name, value in {**self.env, **(env or {})}.items(): + if value is None: + environment.pop(name, None) + else: + environment[name] = value + # stderr in the output, in the order they were written, like CliRunner's + stderr = subprocess.STDOUT if self.mix_stderr else subprocess.PIPE + data = _input_bytes(input, self.charset) + isatty = getattr(input, "isatty", None) + if callable(isatty) and isatty(): + completed = _run_on_a_terminal(command, data, environment, stderr) + else: + completed = subprocess.run( + command, + input=data, + stdout=subprocess.PIPE, + stderr=stderr, + env=environment, + timeout=600, + ) + exit_code = completed.returncode + return Result( + runner=self, + stdout_bytes=completed.stdout, + stderr_bytes=None if self.mix_stderr else completed.stderr, + return_value=None, + exit_code=exit_code, + exception=None if exit_code == 0 else SystemExit(exit_code), + ) + + +@pytest.fixture(autouse=True) +def pywhat_cli(request: Any, monkeypatch: Any) -> None: + """With PYWHAT_CLI set, the tests of test_click.py invoke $PYWHAT_CLI.""" + if PYWHAT_CLI and _is_click_test(request.node): + monkeypatch.setattr(CliRunner, "invoke", invoke)