diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index fb97180..a234783 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -77,6 +77,10 @@ jobs:
python-version: "3.8"
- name: Build wheels
uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1
+ env:
+ # ring (the TLS of `what --verify`) needs __ARM_ARCH, which the GCC 4.8 of the
+ # manylinux2014 aarch64 cross compiler doesn't define (briansmith/ring#1728)
+ CFLAGS_aarch64_unknown_linux_gnu: -D__ARM_ARCH=8
with:
target: ${{ matrix.platform.target }}
args: --release --out dist
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
index 39a6690..a06a98c 100644
--- a/.github/workflows/rust.yml
+++ b/.github/workflows/rust.yml
@@ -100,3 +100,23 @@ jobs:
env:
PYWHAT_GOLDEN: target/golden.jsonl
PYWHAT_GOLDEN_FILES: target/golden-files.jsonl
+
+ cli:
+ name: Rust CLI prints what Python prints (parity)
+ runs-on: ubuntu-latest
+ env:
+ PYWHAT_CLI: target/release/what
+ steps:
+ - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
+ - uses: dtolnay/rust-toolchain@b725eda525362e7cf34af9a7243cac1e3f01a7ec # stable
+ - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2
+ - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
+ with:
+ python-version: '3.10'
+ # The versions in poetry.lock
+ - run: 'python -m pip install --only-binary :all: click==8.1.2 rich==12.2.0 pytest==7.1.2'
+ - run: cargo build --locked --release --bin what
+ # The stdout bytes and exit codes of ~270 command lines, against python -m pywhat
+ - run: python scripts/parity_cli.py
+ # pyWhat's CLI tests, run against the binary
+ - run: python -m pytest -o addopts="" tests/test_click.py
diff --git a/Cargo.lock b/Cargo.lock
index c088587..fb684fb 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -11,6 +11,21 @@ dependencies = [
"memchr",
]
+[[package]]
+name = "anstyle-query"
+version = "1.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc"
+dependencies = [
+ "windows-sys 0.61.2",
+]
+
+[[package]]
+name = "base64"
+version = "0.23.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5"
+
[[package]]
name = "bit-set"
version = "0.5.3"
@@ -41,6 +56,12 @@ version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7"
+[[package]]
+name = "bitflags"
+version = "2.13.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06"
+
[[package]]
name = "block-buffer"
version = "0.12.1"
@@ -56,12 +77,51 @@ version = "3.20.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649"
+[[package]]
+name = "bytes"
+version = "1.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04"
+
+[[package]]
+name = "cc"
+version = "1.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "375989decec1a0fa424442bed520fa12ede43a48475ce8c7417d2c69b32b5f6b"
+dependencies = [
+ "find-msvc-tools",
+ "shlex",
+]
+
[[package]]
name = "cfg-if"
version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600"
+[[package]]
+name = "comfy-table"
+version = "8.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b96a46ad395a5e00676eddaca4a45ed78b29c08be5a878d35e4b071c625637dd"
+dependencies = [
+ "crossterm",
+ "unicode-segmentation",
+ "unicode-width",
+]
+
+[[package]]
+name = "console"
+version = "0.16.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e96a4956774c13c126a8b5af4daa79384f4d826534c95a02d76afb39e2ab64e3"
+dependencies = [
+ "encode_unicode",
+ "libc",
+ "unicode-width",
+ "windows-sys 0.61.2",
+]
+
[[package]]
name = "const-oid"
version = "0.10.2"
@@ -102,6 +162,29 @@ version = "0.8.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6"
+[[package]]
+name = "crossterm"
+version = "0.29.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d8b9f2e4c67f833b660cdb0a3523065869fb35570177239812ed4c905aeff87b"
+dependencies = [
+ "bitflags",
+ "crossterm_winapi",
+ "document-features",
+ "parking_lot",
+ "rustix",
+ "winapi",
+]
+
+[[package]]
+name = "crossterm_winapi"
+version = "0.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "acdd7c62a3665c7f6830a51635d9ac9b23ed385797f70a83bb8bafe9c572ab2b"
+dependencies = [
+ "winapi",
+]
+
[[package]]
name = "crypto-common"
version = "0.2.2"
@@ -122,18 +205,43 @@ dependencies = [
"crypto-common",
]
+[[package]]
+name = "document-features"
+version = "0.2.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d4b8a88685455ed29a21542a33abd9cb6510b6b129abadabdcef0f4c55bc8f61"
+dependencies = [
+ "litrs",
+]
+
[[package]]
name = "either"
version = "1.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be"
+[[package]]
+name = "encode_unicode"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0"
+
[[package]]
name = "equivalent"
version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826"
+[[package]]
+name = "errno"
+version = "0.3.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
+dependencies = [
+ "libc",
+ "windows-sys 0.61.2",
+]
+
[[package]]
name = "fancy-regex"
version = "0.11.0"
@@ -155,12 +263,29 @@ dependencies = [
"regex-syntax",
]
+[[package]]
+name = "find-msvc-tools"
+version = "0.1.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1c66915cc5391097de7a2b0b972e8505aa33c0df401fd3a6811dbdb32881ba48"
+
[[package]]
name = "foldhash"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb"
+[[package]]
+name = "getrandom"
+version = "0.2.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "wasi",
+]
+
[[package]]
name = "hashbrown"
version = "0.17.1"
@@ -173,6 +298,22 @@ version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
+[[package]]
+name = "http"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0"
+dependencies = [
+ "bytes",
+ "itoa",
+]
+
+[[package]]
+name = "httparse"
+version = "1.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87"
+
[[package]]
name = "hybrid-array"
version = "0.4.15"
@@ -192,6 +333,18 @@ dependencies = [
"hashbrown",
]
+[[package]]
+name = "indicatif"
+version = "0.18.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9433806cd6b4ec1aba79c021c7e4c58fb4c3b9977c085062e611ac929998fb0c"
+dependencies = [
+ "console",
+ "portable-atomic",
+ "unicode-width",
+ "unit-prefix",
+]
+
[[package]]
name = "itoa"
version = "1.0.18"
@@ -218,6 +371,33 @@ version = "0.2.191"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a"
+[[package]]
+name = "linux-raw-sys"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53"
+
+[[package]]
+name = "litrs"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4744e383959f0db86ede514b809b1c53251889093803c05267acc7d4e7030d70"
+
+[[package]]
+name = "lock_api"
+version = "0.4.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965"
+dependencies = [
+ "scopeguard",
+]
+
+[[package]]
+name = "log"
+version = "0.4.35"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5d15f86d12da44cd8d4ec3c73984537eb7f4cadcb9776a575a0a8a05ca11005c"
+
[[package]]
name = "memchr"
version = "2.8.3"
@@ -230,6 +410,35 @@ version = "1.21.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
+[[package]]
+name = "parking_lot"
+version = "0.12.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a"
+dependencies = [
+ "lock_api",
+ "parking_lot_core",
+]
+
+[[package]]
+name = "parking_lot_core"
+version = "0.9.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "redox_syscall",
+ "smallvec",
+ "windows-link",
+]
+
+[[package]]
+name = "percent-encoding"
+version = "2.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220"
+
[[package]]
name = "portable-atomic"
version = "1.15.0"
@@ -306,7 +515,10 @@ dependencies = [
name = "pywhat"
version = "6.0.0-alpha.1"
dependencies = [
+ "anstyle-query",
+ "comfy-table",
"fancy-regex 0.20.0",
+ "indicatif",
"lemmeknow",
"rayon",
"regex",
@@ -314,6 +526,8 @@ dependencies = [
"regex-syntax",
"serde_json",
"sha2",
+ "terminal_size",
+ "ureq",
"walkdir",
"wasm-bindgen",
]
@@ -355,6 +569,15 @@ dependencies = [
"crossbeam-utils",
]
+[[package]]
+name = "redox_syscall"
+version = "0.5.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d"
+dependencies = [
+ "bitflags",
+]
+
[[package]]
name = "regex"
version = "1.13.1"
@@ -384,6 +607,68 @@ version = "0.8.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4"
+[[package]]
+name = "ring"
+version = "0.17.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7"
+dependencies = [
+ "cc",
+ "cfg-if",
+ "getrandom",
+ "libc",
+ "untrusted",
+ "windows-sys 0.52.0",
+]
+
+[[package]]
+name = "rustix"
+version = "1.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e5e76742e5294153d4dde1207dbec5bdd81eef6fabdc0d2118babadabfd858aa"
+dependencies = [
+ "bitflags",
+ "errno",
+ "libc",
+ "linux-raw-sys",
+ "windows-sys 0.61.2",
+]
+
+[[package]]
+name = "rustls"
+version = "0.23.46"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf"
+dependencies = [
+ "log",
+ "once_cell",
+ "ring",
+ "rustls-pki-types",
+ "rustls-webpki",
+ "subtle",
+ "zeroize",
+]
+
+[[package]]
+name = "rustls-pki-types"
+version = "1.15.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96"
+dependencies = [
+ "zeroize",
+]
+
+[[package]]
+name = "rustls-webpki"
+version = "0.103.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4443b1b6d39d7c663139942cc4558262562d2c84123bb6282ef177009ca8b281"
+dependencies = [
+ "ring",
+ "rustls-pki-types",
+ "untrusted",
+]
+
[[package]]
name = "rustversion"
version = "1.0.23"
@@ -399,6 +684,12 @@ dependencies = [
"winapi-util",
]
+[[package]]
+name = "scopeguard"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
+
[[package]]
name = "serde"
version = "1.0.229"
@@ -455,6 +746,24 @@ dependencies = [
"digest",
]
+[[package]]
+name = "shlex"
+version = "2.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba"
+
+[[package]]
+name = "smallvec"
+version = "1.16.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d0d5d4b1981b8bbbb7aed204cae5726d683d5d6ac7e74ec54122ea28f315e7c8"
+
+[[package]]
+name = "subtle"
+version = "2.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
+
[[package]]
name = "syn"
version = "2.0.119"
@@ -483,6 +792,16 @@ version = "0.13.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
+[[package]]
+name = "terminal_size"
+version = "0.4.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "230a1b821ccbd75b185820a1f1ff7b14d21da1e442e22c0863ea5f08771a8874"
+dependencies = [
+ "rustix",
+ "windows-sys 0.61.2",
+]
+
[[package]]
name = "typenum"
version = "1.20.1"
@@ -495,6 +814,64 @@ version = "1.0.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563"
+[[package]]
+name = "unicode-segmentation"
+version = "1.13.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8"
+
+[[package]]
+name = "unicode-width"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ca718471876992059f251e33aafc334346ada08042b7661dd1fe313a9bced557"
+
+[[package]]
+name = "unit-prefix"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "81e544489bf3d8ef66c953931f56617f423cd4b5494be343d9b9d3dda037b9a3"
+
+[[package]]
+name = "untrusted"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
+
+[[package]]
+name = "ureq"
+version = "3.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e7be76686bdf7198002062423ca0d8cf9b75a27c8e1de804513b97e97b555d9d"
+dependencies = [
+ "base64",
+ "log",
+ "percent-encoding",
+ "rustls",
+ "rustls-pki-types",
+ "ureq-proto",
+ "utf8-zero",
+ "webpki-roots",
+]
+
+[[package]]
+name = "ureq-proto"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "622dce33c252115466d83f63e9744a18f12b4406aae51a01c15ad8fcf1185eb3"
+dependencies = [
+ "base64",
+ "http",
+ "httparse",
+ "log",
+]
+
+[[package]]
+name = "utf8-zero"
+version = "0.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b8c0a043c9540bae7c578c88f91dda8bd82e59ae27c21baca69c8b191aaf5a6e"
+
[[package]]
name = "walkdir"
version = "2.5.0"
@@ -505,6 +882,12 @@ dependencies = [
"winapi-util",
]
+[[package]]
+name = "wasi"
+version = "0.11.1+wasi-snapshot-preview1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b"
+
[[package]]
name = "wasm-bindgen"
version = "0.2.130"
@@ -549,21 +932,61 @@ dependencies = [
"unicode-ident",
]
+[[package]]
+name = "webpki-roots"
+version = "1.0.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f5f280bbb2b287bc8e7de7e3852945a0ff50d1bbcb5ed42b6c7a568c546df171"
+dependencies = [
+ "rustls-pki-types",
+]
+
+[[package]]
+name = "winapi"
+version = "0.3.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
+dependencies = [
+ "winapi-i686-pc-windows-gnu",
+ "winapi-x86_64-pc-windows-gnu",
+]
+
+[[package]]
+name = "winapi-i686-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
+
[[package]]
name = "winapi-util"
version = "0.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
dependencies = [
- "windows-sys",
+ "windows-sys 0.61.2",
]
+[[package]]
+name = "winapi-x86_64-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
+
[[package]]
name = "windows-link"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
+[[package]]
+name = "windows-sys"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
+dependencies = [
+ "windows-targets",
+]
+
[[package]]
name = "windows-sys"
version = "0.61.2"
@@ -573,6 +996,76 @@ dependencies = [
"windows-link",
]
+[[package]]
+name = "windows-targets"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
+dependencies = [
+ "windows_aarch64_gnullvm",
+ "windows_aarch64_msvc",
+ "windows_i686_gnu",
+ "windows_i686_gnullvm",
+ "windows_i686_msvc",
+ "windows_x86_64_gnu",
+ "windows_x86_64_gnullvm",
+ "windows_x86_64_msvc",
+]
+
+[[package]]
+name = "windows_aarch64_gnullvm"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
+
+[[package]]
+name = "windows_aarch64_msvc"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
+
+[[package]]
+name = "windows_i686_gnu"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
+
+[[package]]
+name = "windows_i686_gnullvm"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
+
+[[package]]
+name = "windows_i686_msvc"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
+
+[[package]]
+name = "windows_x86_64_gnu"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
+
+[[package]]
+name = "windows_x86_64_gnullvm"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
+
+[[package]]
+name = "windows_x86_64_msvc"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
+
+[[package]]
+name = "zeroize"
+version = "1.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e13084392c5e4bc371903e2935a5eaeed24905a7511356b883835e18a78f6879"
+
[[package]]
name = "zmij"
version = "1.0.23"
diff --git a/Cargo.toml b/Cargo.toml
index 6f8a424..5ea457c 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -31,7 +31,21 @@ categories = ["command-line-utilities", "text-processing", "wasm"]
include = ["/src", "/pywhat/Data/*.json", "/pywhat/Data/regexes/*.json", "/LICENSE", "/NOTICE", "/README.md"]
[features]
-default = ["parallel", "files", "children", "mac-vendors"]
+default = ["parallel", "files", "children", "mac-vendors", "cli", "verify"]
+# The `what` and `pywhat` commands (src/cli/), with the options and the output of the
+# Python package's CLI: tables, colours and progress bars. pywhat::cli::main is what they
+# run, and what the Python package's `pywhat` and `what` commands call. They need the file
+# signatures and every Children table to print what the Python CLI prints.
+cli = [
+ "files",
+ "mac-vendors",
+ "dep:comfy-table",
+ "dep:indicatif",
+ "dep:anstyle-query",
+ "dep:terminal_size",
+]
+# `what --verify`: asks Google whether the Google API keys found are valid (over HTTPS).
+verify = ["cli", "dep:ureq"]
# Search the regexes of a large text, and compile them, on rayon's thread pool. Not on
# wasm32, which has no threads.
parallel = ["dep:rayon"]
@@ -60,6 +74,17 @@ rayon = { version = "1.12.0", optional = true }
walkdir = { version = "2.5.0", optional = true }
wasm-bindgen = { version = "=0.2.130", optional = true }
+# The CLI (feature `cli`): the --format pretty table, progress bars, colours on Windows
+# consoles, and the terminal sizes that rich and click use. Not on wasm32, which has no
+# terminal: there the features build the library only.
+[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
+comfy-table = { version = "8.0.1", optional = true }
+indicatif = { version = "0.18.6", optional = true, default-features = false, features = ["unicode-width"] }
+anstyle-query = { version = "1.1.5", optional = true }
+terminal_size = { version = "0.4.4", optional = true }
+# --verify (feature `verify`), without gzip, as Python's urllib
+ureq = { version = "3.5.1", optional = true, default-features = false, features = ["rustls"] }
+
[dev-dependencies]
serde_json = { version = "1.0.154", features = ["preserve_order"] }
# Only for benches/whole.rs, which compares the speed with lemmeknow, the crate that Ciphey
@@ -68,6 +93,16 @@ lemmeknow = { version = "=0.8.0", default-features = false }
# tests/charset_only.rs builds the DFA of every regex
regex-automata = { version = "0.4.18", features = ["dfa-build"] }
+[[bin]]
+name = "what"
+path = "src/bin/what.rs"
+required-features = ["cli"]
+
+[[bin]]
+name = "pywhat"
+path = "src/bin/pywhat.rs"
+required-features = ["cli"]
+
[[example]]
name = "bench_file"
required-features = ["files"]
diff --git a/docs/cli.md b/docs/cli.md
new file mode 100644
index 0000000..955f523
--- /dev/null
+++ b/docs/cli.md
@@ -0,0 +1,69 @@
+# The `what` command in Rust
+
+`cargo install pywhat` installs `what` and `pywhat`: the command line of the Python package in
+Rust, with the same options, defaults, help, error messages and exit codes (`what --help`), and the
+same output. `pywhat::cli::main(std::env::args_os())` runs it and returns the exit code. Both
+binaries do just that, and the Python package's commands can call it too. It is the crate's `cli`
+feature, and `--verify` is its `verify` feature (both are default features, and add nothing on
+wasm32).
+
+- The arguments are parsed the way click parses them: `-br`, `-db` and the other options with
+ several letters after one `-`, clusters such as `-ok name`, `--top=5`, `--` before inputs that
+ start with `-`, and unknown options are inputs (`what -1606586780` searches `-1606586780`).
+- Raw output, custom `--format` strings, `--json`, `--stream` (with `--json`: a JSON object per
+ line, without `"Fragment"`) and `Nothing found!` are byte for byte what the Python CLI prints.
+ `--format pretty` has the same cells, in a table that comfy-table draws.
+- Colours (bold `#D7AFFF` labels) only on a terminal, without colours if `NO_COLOR` is set.
+ Progress bars follow the Python CLI's rules: only on a terminal, after half a second, on stderr
+ if stdout is a file, never into a pipe, and `--no-progress` turns them off.
+- `--verify` sends each Google API key to Google once, like `GoogleAPIKeyVerifier`, with the same
+ messages. The verifier is `pywhat::cli::verify::GoogleApiKeyVerifier`, a `Processor` whose
+ `with_http` takes the HTTP call, so that tests don't send anything.
+- `--interactive` is only in the Python package: `what --interactive` says so and exits with 2.
+
+## Differences from the Python CLI
+
+- The matched text and the file names are printed as they are. The Python CLI takes them for rich
+ markup and emoji codes: `pywhat 'thm{[bold]x[/bold]}'` prints `thm{x}`, and `pywhat 'thm{[/x]}'`
+ crashes with a `MarkupError` (#291, Python bug 4). The markup in the data files (some
+ descriptions) and in `--format` strings is rendered as rich renders it; emoji codes in a
+ `--format` string are not.
+- `-v` prints the version of the crate.
+- Directories are searched in the order of their names, where Python takes the order of the file
+ system (#291, Python bug 5). A file that can't be read is skipped and reported on stderr, with
+ exit code 1, where Python stops with a traceback.
+- On Windows, output redirected to a file is UTF-8, where Python writes the ANSI code page with
+ escapes such as `\u20bf` for what it can't encode. Arguments that aren't valid UTF-8 get U+FFFD
+ for the invalid bytes.
+- Without arguments, `what` reads stdin, as `pywhat` does; `python -m pywhat` shows the help.
+
+## Parity tests
+
+`scripts/parity_cli.py` runs 272 command lines with both CLIs and compares their stdout and exit
+codes: every case of `tests/test_click.py` (read from it), every option on `fixtures/file`, the
+examples of the README and of the help, and corner cases of the argument parser, of stdin and of
+files. Python runs with glob listing directories in name order, so that the output of directories
+can be compared byte for byte. `tests/conftest.py` runs `tests/test_click.py` against the binary
+when `PYWHAT_CLI` is set:
+
+```sh
+cargo build --release
+python scripts/parity_cli.py
+PYWHAT_CLI=target/release/what pytest -o addopts="" tests/test_click.py
+```
+
+The Rust workflow runs both on every pull request.
+
+## Measurements
+
+`hyperfine -N` (hyperfine 2.0.0), with 5 runs for the big file and 10 for the others, on a shared
+64-core machine with a load average of 61 to 83, JSON to `/dev/null`. `target/bench.txt` is the
+4.2 MB text that `python scripts/bench_corpus.py` writes (see [rust.md](rust.md#measurements)):
+
+| | `what` (Rust) | `pywhat` (Python 3.10) |
+|---|---|---|
+| `--json --no-progress target/bench.txt` | 3.05 s wall, 3.64 s user, 366 MiB | 22.87 s wall, 21.72 s user, 632 MiB |
+| `--json --no-progress fixtures/file` | 0.14 s wall | 0.36 s wall |
+| `--no-progress 'THM{a}'` | 0.12 s wall | 0.30 s wall |
+
+Both print the same 296,955,554 bytes for `target/bench.txt` (`cmp`).
diff --git a/docs/rust.md b/docs/rust.md
index 58c06b2..b1cb63a 100644
--- a/docs/rust.md
+++ b/docs/rust.md
@@ -43,6 +43,7 @@ assert!(found.iter().any(|m| m.matched == "bee@skerritt.blog" && m.regex.name ==
| `identifier.identify(text)` | `identify(text)["Regexes"]["text"]` |
| `identifier.identify_inputs(&inputs, options, key, reverse)` (feature `files`) | `identify_inputs(inputs, only_text=..., include_filenames=..., key=..., reverse=...)` |
| `identifier.iter_identify(&inputs, options, found, error)` (feature `files`) | `iter_identify(...)` |
+| `identifier.iter_identify_with_progress(&inputs, options, found, error, progress)` and `Progress` (feature `files`) | `iter_identify(..., progress=...)` and `Progress`: `progress` gets the file, how many of the files and of the file's regex searches are done, and a regex that is searched for. For texts over 4 KB the regexes are searched for in parallel, so it is called from rayon's threads, one call at a time |
| `Identified::to_json()` | `json.dumps(identify(...))`, byte for byte |
| `Filter::new(tags, exclude_tags, min_rarity, max_rarity)`, `Filter::default()`, `Filter::none()`, `.and()`, `.or()` | `Filter({...})`, `&`, `\|` |
| `Processor` (trait), `default_processors()`, `processors::OpenStreetMap`, `processors::Validator` | `Processor`, `default_processors()`, `OpenStreetMapProcessor`, `ValidatorProcessor` |
@@ -70,6 +71,8 @@ Rust only:
| `files` | yes | `identify_inputs`, `iter_identify`: files, directories (recursive) and file signatures (`file_signatures.json`, under CC BY-SA 4.0) |
| `children` | yes | the tables that `Children` read from data files: the locations of phone numbers (`phone_codes.json`) in their descriptions |
| `mac-vendors` | yes | `children`, and the vendors of MAC address prefixes (1.6 MB) in the descriptions of EUI-48 matches |
+| `cli` | yes | the `what` and `pywhat` commands and `pywhat::cli::main`, see [cli.md](cli.md); it needs `files` and `mac-vendors`, for the output of the Python CLI (not on wasm32) |
+| `verify` | yes | `what --verify`: asks Google whether Google API keys are valid (not on wasm32) |
| `wasm` | no | the WebAssembly API in `src/wasm.rs` (`identify(text, options_json)`, `tags()`, `version()`) |
For wasm32, use `default-features = false` (no threads, no file system), and `features = ["wasm"]`
diff --git a/scripts/parity_cli.py b/scripts/parity_cli.py
new file mode 100644
index 0000000..812e7b0
--- /dev/null
+++ b/scripts/parity_cli.py
@@ -0,0 +1,610 @@
+"""
+Check that the Rust CLI prints what the Python CLI prints: the same stdout bytes and the
+same exit code, for about 270 command lines: every case of tests/test_click.py (read from
+it), every option on fixtures/file, the examples of the README and of --help, and corner
+cases of the argument parser, of stdin and of files. For --format pretty, whose table
+comfy-table draws, the text around the table and the cells of the table must be the same.
+
+ cargo build --release
+ python scripts/parity_cli.py [-k TEXT] [-v] [-j JOBS]
+
+PYWHAT_CLI is the program to test (default: target/release/what). Python runs as
+`COLUMNS=10000 python -m pywhat ...` from the root of the repository, so that rich doesn't
+wrap its output, and the Rust CLI with the same environment and the program name "python -m
+pywhat", so that the usage lines are the same.
+
+Python lists the files of a directory in the order of the file system, and Rust in the order
+of their names (#291, Python bug 5). So that the output for directories can be compared byte
+for byte on any file system, Python runs with glob listing every directory in name order;
+nothing else changes.
+
+KNOWN lists the command lines whose output differs on purpose, with the reason. They run too,
+and are reported, but they don't fail the check.
+"""
+import argparse
+import ast
+import itertools
+import os
+import re
+import shlex
+import subprocess
+import sys
+import tempfile
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+from typing import Any, Dict, Iterator, List, NamedTuple, Optional, Tuple
+
+ROOT = Path(__file__).resolve().parent.parent
+PROG = "python -m pywhat"
+
+# python -m pywhat, with glob listing every directory in name order (see above)
+PYTHON_CLI = """\
+import glob
+import runpy
+
+files_of = glob._iterdir
+
+
+def in_name_order(*args, **kwargs):
+ yield from sorted(files_of(*args, **kwargs))
+
+
+glob._iterdir = in_name_order
+runpy.run_module("pywhat", run_name="__main__", alter_sys=True)
+"""
+
+ETH = "0x52908400098527886E0F7030069857D2E4169EE7"
+F = "fixtures/file"
+PNG = b"\x89PNG\r\n\x1a\nTHM{hello}\n"
+
+
+class Case(NamedTuple):
+ args: Tuple[str, ...]
+ stdin: bytes = b""
+ columns: str = "10000"
+ # Run in a scratch directory with these files (name, contents)
+ files: Optional[Tuple[Tuple[str, bytes], ...]] = None
+
+
+def case(*args: str, stdin: bytes = b"", columns: str = "10000", files=None) -> Case:
+ return Case(tuple(args), stdin, columns, None if files is None else tuple(files))
+
+
+# The cases of tests/test_click.py
+
+
+class Unsupported(Exception):
+ """An expression of tests/test_click.py that value() can't work out."""
+
+
+def value(node: Optional[ast.AST], names: Dict[str, Any]) -> Any:
+ """
+ The value of a simple expression of tests/test_click.py: literals, the names that
+ parametrize or an assignment of a literal binds, lists and tuples with *starred items,
+ dicts, and the sum of two lists. Raises Unsupported for anything else.
+ """
+ if isinstance(node, ast.Constant):
+ return node.value
+ if isinstance(node, ast.Name) and node.id in names:
+ return names[node.id]
+ if isinstance(node, (ast.List, ast.Tuple)):
+ items: List[Any] = []
+ for item in node.elts:
+ if isinstance(item, ast.Starred):
+ items.extend(value(item.value, names))
+ else:
+ items.append(value(item, names))
+ return items if isinstance(node, ast.List) else tuple(items)
+ if isinstance(node, ast.Dict):
+ return {
+ value(key, names): value(item, names)
+ for key, item in zip(node.keys, node.values)
+ }
+ if isinstance(node, ast.BinOp) and isinstance(node.op, ast.Add):
+ return value(node.left, names) + value(node.right, names)
+ raise Unsupported(ast.dump(node) if node is not None else "None")
+
+
+def parametrizations(function: ast.FunctionDef) -> Optional[List[Dict[str, Any]]]:
+ """
+ The parameters of every run of a test function (one run without parameters), or None
+ if it is skipped.
+ """
+ runs: List[Dict[str, Any]] = [{}]
+ for decorator in function.decorator_list:
+ call = decorator if isinstance(decorator, ast.Call) else None
+ name = getattr(call.func if call else decorator, "attr", None)
+ if name == "skip":
+ return None
+ if name != "parametrize" or call is None:
+ continue
+ names = [n.strip() for n in value(call.args[0], {}).split(",")]
+ sets = []
+ for values in value(call.args[1], {}):
+ values = values if len(names) > 1 else [values]
+ sets.append(dict(zip(names, values)))
+ runs = [
+ {**run, **parameters} for run, parameters in itertools.product(runs, sets)
+ ]
+ return runs
+
+
+def invocations(function: ast.FunctionDef) -> Iterator[ast.Call]:
+ """The calls of CliRunner.invoke() and run_cli_command() in a test function."""
+ for node in ast.walk(function):
+ if isinstance(node, ast.Call):
+ name = getattr(node.func, "attr", getattr(node.func, "id", None))
+ if name in ("invoke", "run_cli_command"):
+ yield node
+
+
+def click_case(call: ast.Call, names: Dict[str, Any]) -> Case:
+ """The command line of a call, with its input and its COLUMNS."""
+ is_invoke = getattr(call.func, "attr", None) == "invoke"
+ args = value(call.args[1] if is_invoke else call.args[0], names)
+ if isinstance(args, str):
+ args = shlex.split(args)
+ keywords = {keyword.arg: keyword.value for keyword in call.keywords}
+ stdin = value(keywords["input"], names) if "input" in keywords else b""
+ env = value(keywords["env"], names) if "env" in keywords else {}
+ if set(env) - {"COLUMNS"} or not isinstance(stdin, (str, bytes)):
+ raise Unsupported("environment or input")
+ if isinstance(stdin, str):
+ stdin = stdin.encode()
+ return case(*args, stdin=stdin, columns=env.get("COLUMNS", "10000"))
+
+
+def click_cases() -> List[Case]:
+ """Every command line that tests/test_click.py runs, as far as value() works it out."""
+ tree = ast.parse((ROOT / "tests" / "test_click.py").read_text(encoding="utf-8"))
+ found: List[Case] = []
+ for function in tree.body:
+ if not isinstance(function, ast.FunctionDef):
+ continue
+ if not function.name.startswith("test_"):
+ continue
+ runs = parametrizations(function)
+ if runs is None:
+ continue
+ # Names that the test binds to a literal, such as flag = "THM{...}"
+ local: Dict[str, Any] = {}
+ for node in ast.walk(function):
+ if isinstance(node, ast.Assign) and len(node.targets) == 1:
+ target = node.targets[0]
+ if isinstance(target, ast.Name):
+ try:
+ local[target.id] = value(node.value, {})
+ except Unsupported:
+ pass
+ for call in invocations(function):
+ for parameters in runs:
+ try:
+ found.append(click_case(call, {**local, **parameters}))
+ except Unsupported:
+ pass # e.g. a file in tmp_path, or stdin that is a terminal
+ return list(dict.fromkeys(found))
+
+
+# Every option on fixtures/file
+OPTIONS = [
+ ["-r", "0:", F],
+ ["--rarity", "0.5:", F],
+ ["-r", ":0.5", F],
+ ["-r", "0.2:0.8", F],
+ ["-i", "Bug Bounty", F],
+ ["-e", "Credentials,URL", F],
+ ["-o", F],
+ ["-k", "name", F],
+ ["-k", "rarity", F],
+ ["-k", "matched", F],
+ ["-k", "likely", F],
+ ["-k", "NONE", F],
+ ["-k", "rarity", "--reverse", F],
+ ["-k", "matched", "--reverse", "--json", F],
+ ["--top", "10", F],
+ ["--top", "5%", F],
+ ["--top", "1.5 %", F],
+ ["--top", "all", F],
+ ["--top", "3", "-k", "name", F],
+ ["--top", "3", "--json", F],
+ ["-br", "0.6:", F],
+ ["-bi", "URL", F],
+ ["-be", "URL", F],
+ ["-db", F],
+ ["-dp", F],
+ ["-db", "-dp", "--json", F],
+ ["--map", "osm", F],
+ ["--map", "GOOGLE", F],
+ ["--json", "--map", "osm", "-dp", F],
+ ["--json", F],
+ ["-pt", F],
+ ["-if", F],
+ ["-if", "--json", F],
+ ["--format", "pretty", F],
+ ["--format", "pretty", "-pt", F],
+ ["--format", "pretty", "--rarity", "0:", F],
+ ["--format", "%m\t%n|%r|%t|%l", F],
+ ["--format", "%d", F],
+ ["--format", "%e", F],
+ ["--format", "\\\\%m\\%n\\x", F],
+ ["--stream", F],
+ ["--stream", "-pt", F],
+ ["--stream", "--json", F],
+ ["--stream", "--format", "%m %n", F],
+ ["--no-progress", "--json", F],
+ ["--verify", "THM{no key}"],
+ ["-o", "--format", "pretty", "fixtures"],
+ ["--format", "pretty", "fixtures"],
+]
+
+# The examples of the README and of --help
+README = [
+ [ETH],
+ ["dQw4w9WgXcQ"],
+ ["--rarity", "0:", "dQw4w9WgXcQ"],
+ ["https://youtu.be/dQw4w9WgXcQ"],
+ ["youtube.com/watch?v=dQw4w9WgXcQ&t=42s"],
+ ["--rarity", "0.2:0.8", "--include", "Identifiers,URL", "https://skerritt.blog"],
+ [
+ "--include",
+ "Cryptocurrency Wallet",
+ "--exclude",
+ "Ripple Wallet",
+ "1KFHE7w8BhaENAswwryaoccDb6qcT6DbYY",
+ ],
+ ["--include", "BTC Wallet,ETH Wallet", F],
+ ["-k", "rarity", "--reverse", ETH],
+ ["--top", "5%", ETH],
+ ["--top", "1", ETH],
+ ["-k", "likely", ETH],
+ ["--json", ETH],
+ ["--rarity", "0:", "--include", "UNIX Timestamp", "1637093119"],
+ [
+ "--rarity",
+ "0:",
+ "--include",
+ "UNIX Timestamp",
+ "--format",
+ "%d",
+ "1637093119.558717",
+ ],
+ ["3F3F3F3F3F3F3F3F3F3F3F3F3F3F3F3F3F"],
+ ["--disable-processing", "3F3F3F3F3F3F3F3F3F3F3F3F3F3F3F3F3F"],
+ ["--map", "osm", "52.6169586, -1.9779857"],
+ ["--format", "pretty", "--map", "osm", "52.6169586, -1.9779857"],
+ ["--include", "Bug Bounty", F],
+ ["HTB{this is a flag}"],
+ ["--", "52.6169586, -1.9779857"],
+ ["--rarity", "0.6:", "myEmail@host.org"],
+ [
+ "--rarity",
+ "0:",
+ "--include",
+ "credentials",
+ "--exclude",
+ "aws",
+ "James:SecretPassword",
+ ],
+ ["-br", "0.6:", "-be", "URL", "123myEmail@host.org456"],
+ ["--format", "%m - %n", "google.com htb{flag}"],
+ ["--stream", "fixtures"],
+ ["--format", "pretty", ETH],
+]
+
+# The argument parser, errors, help and the format string
+PARSER = [
+ ["-1606586780"],
+ ["-h"],
+ ["--nope", "THM{x}"],
+ ["-ox", "THM{x}"],
+ ["-kname", "-db", F],
+ ["--top=1", ETH],
+ ["--top", "=1", ETH],
+ ["-r=0.5:", ETH],
+ ["-br=0.5:", ETH],
+ ["--json=1", ETH],
+ ["--rarity"],
+ ["-k"],
+ ["-pt=", "x"],
+ ["--map", "OSM", "52.6169586, -1.9779857"],
+ ["--map", "o\u017fm", "52.6169586, -1.9779857"],
+ ["--map", "bing", "x"],
+ ["--map", "it's", "x"],
+ ["--", "-db", "--", "THM{x}"],
+ ["-"],
+ ["-r", "--", "THM{x}"],
+ ["--tags", "--version"],
+ ["--names", "--tags"],
+ ["--map", "x", "--tags"],
+ ["--tags", "--map", "x"],
+ ["--help", "--map", "x"],
+ ["--map", "x", "--help"],
+ ["-db", "--help"],
+ ["--help"],
+ ["--version"],
+ ["-v", "THM{x}"],
+ ["--rarity", "0.5", ETH],
+ ["--rarity", "a:b", ETH],
+ ["--rarity", "0.1:0.5:0.9", ETH],
+ ["--rarity", ":", ETH],
+ ["--rarity", " : ", ETH],
+ ["--rarity", "1_0:", ETH],
+ ["--rarity", "\u0660.\u0665:", ETH],
+ ["--rarity", "nan:", ETH],
+ ["--rarity", "-inf:inf", ETH],
+ ["-br", "x:", ETH],
+ ["-db", "-br", "x:", ETH],
+ ["--key", "nope", ETH],
+ ["--key", "rar\u0131ty", ETH],
+ ["--top", "0", ETH],
+ ["--top", "150%", ETH],
+ ["--top", "abc", ETH],
+ ["--top", " 2 ", ETH],
+ ["--top", "99999999999999999999999999999", ETH],
+ ["--top", "\x1c 5\x1f%\u3000", ETH],
+ ["--top", "0" * 40 + "1", ETH],
+ ["--include", "", ETH],
+ ["--include", "btc wallet", "--exclude", "ETH WALLET", ETH],
+ ["--stream", "--format", "pretty", "--json", "THM{x}"],
+ ["--stream", "-k", "name", "--top", "1", "THM{x}"],
+ ["-db"],
+ ["--format", "None", "THM{x}"],
+ ["--format", "", "THM{x}"],
+ ["--format", " ", "THM{x}"],
+ ["--format", "%m\\", "THM{x}"],
+ # rich markup in the format string and in the data is rendered, as in Python
+ ["-db", "--format", "[b]%m[/b] [#ff0000]%n[/#ff0000]", "THM{y}"],
+ ["-db", "--format", "\\[b]%m", "THM{y}"],
+ ["-db", "--include", "Access-Control-Allow-Header", "Access-Control-Allow: *"],
+]
+
+# The matched text as rich markup and emoji codes (#291, Python bug 4)
+BUG_4 = (
+ "#291 Python bug 4: rich takes the matched text for markup and emoji codes "
+ "(Rust prints it as it is)"
+)
+MARKUP = [
+ ["-db", "thm{[bold]x[/bold]}"],
+ ["-db", "THM{:smile:}"],
+ ["-db", "--format", "%m", "THM{[i]x[/i]}"],
+ ["-db", "--format", "pretty", "THM{[b]x[/b]}"],
+ ["-db", "thm{[/x]}"],
+]
+
+OWN: List[Case] = [case(*args) for args in OPTIONS + README + PARSER + MARKUP]
+OWN += [
+ case("-db", "--format", ":smile: %m", "THM{y}"),
+ # stdin, when there are no inputs
+ case(stdin=b""),
+ case("-db", stdin=b"fixtures/file\n"),
+ case("--json", stdin="\ufeffTHM{bom}\r\n".encode("utf-16")),
+ case("--json", stdin=b"\x00\x01" + "THM{wide}".encode("utf-16-le") + b"\xff\xfe"),
+ case("-db", stdin=b"caf\xe9 THM{latin}"),
+ # The help and the usage of errors at other widths
+ case("--help", columns="40"),
+ case("--help", columns="60"),
+ case("--help", columns="79"),
+ case("--help", columns="0"),
+ case("--help", columns="abc"),
+ case("--rarity", columns="40"),
+ # Files in a scratch directory
+ case(
+ "-db",
+ "image.png",
+ "page.html",
+ files=[("image.png", PNG), ("page.html", b"https://google.com\n")],
+ ),
+ case(
+ "-db",
+ "--json",
+ "image.png",
+ "page.html",
+ files=[("image.png", PNG), ("page.html", b"https://google.com\n")],
+ ),
+ case("image.png", files=[("image.png", PNG)]),
+ case("--stream", "image.png", files=[("image.png", PNG)]),
+ case("--stream", "image.png", files=[("image.png", b"\x89PNG\r\n\x1a\n")]),
+ case("--stream", "--format", "json", "image.png", files=[("image.png", PNG)]),
+ case("--format", "pretty", "image.png", files=[("image.png", PNG)]),
+ case(
+ "--format", "pretty", "image.png", files=[("image.png", b"\x89PNG\r\n\x1a\n")]
+ ),
+ case("-db", "--json", "89504E470D0A1A0A", files=[("89504E470D0A1A0A", b"THM{x}")]),
+ case(
+ "--json",
+ "utf16.txt",
+ files=[("utf16.txt", "\ufeffTHM{x} bee@skerritt.blog\r\n".encode("utf-16"))],
+ ),
+ case("-if", "THM{name}.txt", files=[("THM{name}.txt", b"nothing")]),
+ case("tabs.txt", "-pt", files=[("tabs.txt", b"x")]),
+ case(
+ "--format",
+ "%m",
+ "dir",
+ files=[("dir/a\tb", b"THM{tab}"), ("dir/.hidden", b"THM{hidden}")],
+ ),
+ # Written out of name order, which is the order that some file systems list them in
+ case(
+ "--format",
+ "%m",
+ "dir",
+ files=[
+ ("dir/b.txt", b"THM{b}"),
+ ("dir/c/a.txt", b"THM{c}"),
+ ("dir/a", b"THM{a}"),
+ ],
+ ),
+ case(
+ "--json",
+ "dir",
+ "THM{text}",
+ files=[("dir/b.txt", b"THM{b}"), ("dir/a.txt", b"THM{a}")],
+ ),
+]
+
+# Command lines whose output differs on purpose
+KNOWN: Dict[Tuple[str, ...], str] = {
+ (): (
+ "python -m pywhat without arguments prints the help (pywhat/__main__.py); the "
+ "what and pywhat commands read stdin, in Python and in Rust"
+ ),
+ ("--version",): (
+ "-v prints the version of the crate (6.0.0-alpha.1); Python's __version__ is "
+ "5.0.0"
+ ),
+ ("-v", "THM{x}"): "the version (see --version)",
+ ("-db", "--format", ":smile: %m", "THM{y}"): (
+ "emoji codes in a --format string: rich prints :smile: as an emoji, Rust as it is"
+ ),
+}
+for args in MARKUP:
+ KNOWN[tuple(args)] = BUG_4
+KNOWN[("-db", "thm{[/x]}")] = BUG_4 + "; Python crashes with a MarkupError"
+
+
+def run(
+ command: List[str], case: Case, cwd: Path, executable: Optional[str] = None
+) -> Tuple[int, bytes, bytes]:
+ env = dict(os.environ, COLUMNS=case.columns, PYTHONIOENCODING="utf-8")
+ # python -m pywhat from a scratch directory too
+ env["PYTHONPATH"] = os.pathsep.join(
+ filter(None, [str(ROOT), env.get("PYTHONPATH")])
+ )
+ env.pop("NO_COLOR", None)
+ result = subprocess.run(
+ command,
+ executable=executable,
+ input=case.stdin,
+ capture_output=True,
+ cwd=cwd,
+ env=env,
+ timeout=600,
+ )
+ return result.returncode, result.stdout, result.stderr
+
+
+def table_parts(output: str):
+ """The text around the tables of the output, and the cells of the tables."""
+ text: List[str] = []
+ tables: List[List[List[str]]] = []
+ row: List[List[str]] = []
+ in_table = False
+ for line in output.split("\n"):
+ if line.startswith("┏"):
+ in_table, row = True, []
+ tables.append([])
+ text.append("
")
+ elif in_table and line[:1] in ("┡", "├", "└"):
+ # The end of a row: its cells, the lines of each cell joined
+ if row:
+ cells = []
+ for index in range(len(row[0])):
+ lines = [
+ parts[index] if index < len(parts) else "" for parts in row
+ ]
+ while lines and not lines[-1]:
+ lines.pop()
+ cells.append("\n".join(lines))
+ tables[-1].append(cells)
+ row = []
+ in_table = line[:1] != "└"
+ elif in_table:
+ row.append([part.strip() for part in re.split("[┃│]", line)[1:-1]])
+ else:
+ text.append(line)
+ return text, tables
+
+
+def first_difference(a: str, b: str) -> str:
+ index = next(
+ (i for i, (x, y) in enumerate(zip(a, b)) if x != y), min(len(a), len(b))
+ )
+ start = max(0, index - 60)
+ return (
+ f"at {index}: Python {a[start:index + 60]!r}\n"
+ f" Rust {b[start:index + 60]!r}"
+ )
+
+
+def compare(case: Case, what: str, verbose: bool) -> Tuple[str, str]:
+ """("same" | "same cells" | "different", details) for a case."""
+ with tempfile.TemporaryDirectory() as scratch:
+ cwd = ROOT
+ if case.files is not None:
+ cwd = Path(scratch)
+ for name, data in case.files:
+ path = cwd / name
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_bytes(data)
+ python = run([sys.executable, "-c", PYTHON_CLI, *case.args], case, cwd)
+ rust = run([PROG, *case.args], case, cwd, executable=what)
+ if python[:2] == rust[:2]:
+ return "same", ""
+ details = []
+ if python[0] != rust[0]:
+ details.append(f"exit code: Python {python[0]}, Rust {rust[0]}")
+ if python[1] != rust[1]:
+ py_text = python[1].decode(errors="replace")
+ rs_text = rust[1].decode(errors="replace")
+ pretty = "pretty" in [arg.strip() for arg in case.args]
+ if "--format" in case.args and pretty and python[0] == rust[0]:
+ py_parts, rs_parts = table_parts(py_text), table_parts(rs_text)
+ if py_parts == rs_parts:
+ return "same cells", ""
+ details.append("table: " + first_difference(repr(py_parts), repr(rs_parts)))
+ else:
+ details.append("stdout: " + first_difference(py_text, rs_text))
+ if verbose:
+ details.append(
+ f"Python stderr: {python[2][-300:]!r}\nRust stderr: {rust[2][-300:]!r}"
+ )
+ return "different", "\n ".join(details)
+
+
+def program() -> str:
+ """The program to test: $PYWHAT_CLI, as an absolute path (cases run elsewhere too)."""
+ default = ROOT / "target" / "release" / ("what.exe" if os.name == "nt" else "what")
+ return os.path.abspath(os.environ.get("PYWHAT_CLI") or default)
+
+
+def main() -> None:
+ parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
+ parser.add_argument("-k", help="only the cases whose arguments contain this text")
+ parser.add_argument("-v", "--verbose", action="store_true")
+ parser.add_argument("-j", "--jobs", type=int, default=4)
+ args = parser.parse_args()
+ what = program()
+ from_tests = click_cases()
+ cases = list(dict.fromkeys(from_tests + OWN))
+ if args.k:
+ cases = [c for c in cases if any(args.k in arg for arg in c.args)]
+ with ThreadPoolExecutor(args.jobs) as pool:
+ results = list(pool.map(lambda c: compare(c, what, args.verbose), cases))
+ counts: Dict[str, int] = {}
+ failures = 0
+ for case_, (verdict, details) in zip(cases, results):
+ known = KNOWN.get(case_.args) if case_.files is None else None
+ if verdict == "different" and known:
+ verdict = "known difference"
+ counts[verdict] = counts.get(verdict, 0) + 1
+ if verdict == "different":
+ failures += 1
+ print(
+ f"DIFFERENT {list(case_.args)} (COLUMNS={case_.columns})\n {details}"
+ )
+ elif verdict == "known difference":
+ print(f"KNOWN {list(case_.args)}: {known}")
+ elif args.verbose:
+ print(f"{verdict.upper()} {list(case_.args)}")
+ summary = ", ".join(
+ f"{count} {verdict}" for verdict, count in sorted(counts.items())
+ )
+ print(
+ f"\n{len(cases)} command lines ({len(from_tests)} from tests/test_click.py): "
+ f"{summary}"
+ )
+ sys.exit(1 if failures else 0)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/src/bin/pywhat.rs b/src/bin/pywhat.rs
new file mode 100644
index 0000000..37a02cc
--- /dev/null
+++ b/src/bin/pywhat.rs
@@ -0,0 +1,5 @@
+//! The `pywhat` command, the same as `what` (see `pywhat --help`).
+
+fn main() {
+ std::process::exit(pywhat::cli::main(std::env::args_os()))
+}
diff --git a/src/bin/what.rs b/src/bin/what.rs
new file mode 100644
index 0000000..7f7c966
--- /dev/null
+++ b/src/bin/what.rs
@@ -0,0 +1,5 @@
+//! The `what` command: identify anything (see `what --help`).
+
+fn main() {
+ std::process::exit(pywhat::cli::main(std::env::args_os()))
+}
diff --git a/src/cli/args.rs b/src/cli/args.rs
new file mode 100644
index 0000000..fd2d4e1
--- /dev/null
+++ b/src/cli/args.rs
@@ -0,0 +1,476 @@
+//! The command line, parsed the way click 8.1 parses `pywhat/what.py`'s options: long
+//! options with `=` or the next argument as their value, multi-letter "short" options
+//! such as `-br` (click treats them as long options), clusters of one-letter options
+//! (`-ok name`), `--` before inputs that start with `-`, and unknown options taken as
+//! inputs (`ignore_unknown_options`: `what -1606586780` searches the text
+//! `-1606586780`).
+
+use super::python::repr;
+use std::collections::HashMap;
+
+/// An option of `what.py`, in the order in which they are declared there.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub(crate) enum Param {
+ Tags,
+ Names,
+ Rarity,
+ Include,
+ Exclude,
+ OnlyText,
+ Key,
+ Reverse,
+ Top,
+ BoundarylessRarity,
+ BoundarylessInclude,
+ BoundarylessExclude,
+ DisableBoundaryless,
+ DisableProcessing,
+ Verify,
+ Map,
+ Json,
+ Interactive,
+ Version,
+ IncludeFilenames,
+ Format,
+ PrintTags,
+ Stream,
+ NoProgress,
+ Help,
+}
+
+/// How an option is written and explained in `--help`.
+pub(crate) struct OptionSpec {
+ pub(crate) param: Param,
+ /// The option strings, e.g. `-t` and `--tags`.
+ pub(crate) opts: &'static [&'static str],
+ pub(crate) takes_value: bool,
+ /// What `--help` shows after the option strings of an option that takes a value.
+ pub(crate) metavar: &'static str,
+ pub(crate) help: &'static str,
+}
+
+const fn flag(param: Param, opts: &'static [&'static str], help: &'static str) -> OptionSpec {
+ OptionSpec {
+ param,
+ opts,
+ takes_value: false,
+ metavar: "",
+ help,
+ }
+}
+
+const fn value(
+ param: Param,
+ opts: &'static [&'static str],
+ metavar: &'static str,
+ help: &'static str,
+) -> OptionSpec {
+ OptionSpec {
+ param,
+ opts,
+ takes_value: true,
+ metavar,
+ help,
+ }
+}
+
+/// The options, in the order of `what.py` (and of `--help`), with click's `--help` last.
+pub(crate) const OPTIONS: &[OptionSpec] = &[
+ flag(
+ Param::Tags,
+ &["-t", "--tags"],
+ "Show available tags and exit.",
+ ),
+ flag(
+ Param::Names,
+ &["--names"],
+ "Show the names of the regexes, with their alternative names, and exit.",
+ ),
+ value(
+ Param::Rarity,
+ &["-r", "--rarity"],
+ "TEXT",
+ "Filter by rarity. Rarity is how unlikely something is to be a false-positive. The \
+ higher the number, the more unlikely. This is in the range of 0:1. To filter only \
+ items past 0.5, use 0.5: with the colon on the end. Default 0.1:1",
+ ),
+ value(
+ Param::Include,
+ &["-i", "--include"],
+ "TEXT",
+ "Only show matches with these tags or names.",
+ ),
+ value(
+ Param::Exclude,
+ &["-e", "--exclude"],
+ "TEXT",
+ "Exclude matches with these tags or names.",
+ ),
+ flag(
+ Param::OnlyText,
+ &["-o", "--only-text"],
+ "Do not scan files or folders.",
+ ),
+ value(
+ Param::Key,
+ &["-k", "--key"],
+ "TEXT",
+ "Sort by the specified key.",
+ ),
+ flag(Param::Reverse, &["--reverse"], "Sort in reverse order."),
+ value(
+ Param::Top,
+ &["--top"],
+ "N",
+ "Only show the N most likely matches, or N% of them, e.g. --top 10 or --top 5%.",
+ ),
+ value(
+ Param::BoundarylessRarity,
+ &["-br", "--boundaryless-rarity"],
+ "TEXT",
+ "Same as --rarity but for boundaryless mode (toggles what regexes will not have \
+ boundaries).",
+ ),
+ value(
+ Param::BoundarylessInclude,
+ &["-bi", "--boundaryless-include"],
+ "TEXT",
+ "Same as --include but for boundaryless mode.",
+ ),
+ value(
+ Param::BoundarylessExclude,
+ &["-be", "--boundaryless-exclude"],
+ "TEXT",
+ "Same as --exclude but for boundaryless mode.",
+ ),
+ flag(
+ Param::DisableBoundaryless,
+ &["-db", "--disable-boundaryless"],
+ "Disable boundaryless mode.",
+ ),
+ flag(
+ Param::DisableProcessing,
+ &["-dp", "--disable-processing"],
+ "Disable the processing of matches, e.g. adding dates to Unix timestamps or \
+ checking the checksums of Bitcoin addresses.",
+ ),
+ flag(
+ Param::Verify,
+ &["--verify"],
+ "Ask the services if the keys found are valid, e.g. Google API keys. This sends \
+ the keys to the services.",
+ ),
+ value(
+ Param::Map,
+ &["--map"],
+ "[google|osm]",
+ "The map that coordinates link to: Google Maps or OpenStreetMap (osm). [default: \
+ google]",
+ ),
+ flag(Param::Json, &["--json"], "Return results in json format."),
+ flag(
+ Param::Interactive,
+ &["--interactive"],
+ "Load the input into memory and search through the matches interactively.",
+ ),
+ flag(
+ Param::Version,
+ &["-v", "--version"],
+ "Display the version of pywhat.",
+ ),
+ flag(
+ Param::IncludeFilenames,
+ &["-if", "--include-filenames"],
+ "Search filenames for possible matches.",
+ ),
+ value(
+ Param::Format,
+ &["--format"],
+ "TEXT",
+ "Format output according to specified rules.",
+ ),
+ flag(
+ Param::PrintTags,
+ &["-pt", "--print-tags"],
+ "Add flags to output",
+ ),
+ flag(
+ Param::Stream,
+ &["--stream"],
+ "Print every match as soon as it is found, not once the search is complete.",
+ ),
+ flag(
+ Param::NoProgress,
+ &["--no-progress"],
+ "Do not show progress bars while searching.",
+ ),
+ flag(Param::Help, &["--help"], "Show this message and exit."),
+];
+
+/// The choices of `--map` (`click.Choice(["google", "osm"], case_sensitive=False)`).
+pub(crate) const MAP_CHOICES: &[&str] = &["google", "osm"];
+
+/// A usage error: click prints the usage, a hint and `Error: `, and exits with 2.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub(crate) struct UsageError(pub(crate) String);
+
+/// What the command line holds.
+#[derive(Debug, Default, Clone)]
+pub(crate) struct Parsed {
+ /// The value of every option that was given (the last one if it was given several
+ /// times); flags have an empty value.
+ values: HashMap,
+ /// The options in the order they were first given, for the order of the callbacks.
+ order: Vec,
+ /// The inputs: files, directories or text.
+ pub(crate) inputs: Vec,
+}
+
+impl Parsed {
+ pub(crate) fn flag(&self, param: Param) -> bool {
+ self.values.contains_key(¶m)
+ }
+
+ pub(crate) fn value(&self, param: Param) -> Option<&str> {
+ self.values.get(¶m).map(String::as_str)
+ }
+
+ /// The options that were given, in the order of their first appearance.
+ pub(crate) fn invoked(&self) -> &[Param] {
+ &self.order
+ }
+
+ fn set(&mut self, param: Param, value: String) {
+ self.values.insert(param, value);
+ if !self.order.contains(¶m) {
+ self.order.push(param);
+ }
+ }
+}
+
+fn spec(opt: &str) -> Option<&'static OptionSpec> {
+ OPTIONS.iter().find(|spec| spec.opts.contains(&opt))
+}
+
+/// click's split: an option string with one letter after one `-` is a short option, any
+/// other is a long one (`-br`, `--rarity`).
+fn is_short(opt: &str) -> bool {
+ let mut chars = opt.chars();
+ chars.next() == Some('-') && chars.next().is_some_and(|c| c != '-') && chars.next().is_none()
+}
+
+fn long_option(opt: &str) -> Option<&'static OptionSpec> {
+ spec(opt).filter(|_| !is_short(opt))
+}
+
+fn short_option(opt: &str) -> Option<&'static OptionSpec> {
+ spec(opt).filter(|_| is_short(opt))
+}
+
+/// Parses the arguments after the program name, like click's `OptionParser.parse_args`.
+pub(crate) fn parse(args: &[String]) -> Result {
+ let mut parsed = Parsed::default();
+ // rargs: the arguments still to read, largs: the inputs found so far
+ let mut rargs: std::collections::VecDeque = args.iter().cloned().collect();
+ let mut largs: Vec = Vec::new();
+ while let Some(arg) = rargs.pop_front() {
+ if arg == "--" {
+ break;
+ }
+ if arg.starts_with('-') && arg.chars().count() > 1 {
+ process_opts(&arg, &mut rargs, &mut largs, &mut parsed)?;
+ } else {
+ largs.push(arg);
+ }
+ }
+ largs.extend(rargs);
+ parsed.inputs = largs;
+ Ok(parsed)
+}
+
+/// The value of an option that takes one: the next argument, whatever it is.
+fn take_value(
+ opt: &str,
+ rargs: &mut std::collections::VecDeque,
+) -> Result {
+ rargs
+ .pop_front()
+ .ok_or_else(|| UsageError(format!("Option {} requires an argument.", repr(opt))))
+}
+
+/// `_process_opts()`: a long option (possibly with `=value`), else a cluster of short
+/// options, else (it starts with `--`) an input.
+fn process_opts(
+ arg: &str,
+ rargs: &mut std::collections::VecDeque,
+ largs: &mut Vec,
+ parsed: &mut Parsed,
+) -> Result<(), UsageError> {
+ let (long_opt, explicit_value) = match arg.split_once('=') {
+ Some((opt, value)) => (opt, Some(value)),
+ None => (arg, None),
+ };
+ if let Some(spec) = long_option(long_opt) {
+ let value = if spec.takes_value {
+ if let Some(value) = explicit_value {
+ rargs.push_front(value.to_string());
+ }
+ take_value(long_opt, rargs)?
+ } else if explicit_value.is_some() {
+ return Err(UsageError(format!(
+ "Option {} does not take a value.",
+ repr(long_opt)
+ )));
+ } else {
+ String::new()
+ };
+ parsed.set(spec.param, value);
+ return Ok(());
+ }
+ if arg.starts_with("--") {
+ // An unknown long option is an input
+ largs.push(arg.to_string());
+ return Ok(());
+ }
+ match_short_opt(arg, rargs, largs, parsed)
+}
+
+/// `_match_short_opt()`: every letter is a short option, and the rest of the argument is
+/// the value of the first one that takes a value. Unknown letters are put together into
+/// an input, with the `-`.
+fn match_short_opt(
+ arg: &str,
+ rargs: &mut std::collections::VecDeque,
+ largs: &mut Vec,
+ parsed: &mut Parsed,
+) -> Result<(), UsageError> {
+ let mut unknown = String::new();
+ for (index, letter) in arg.char_indices().skip(1) {
+ let opt = format!("-{letter}");
+ let Some(spec) = short_option(&opt) else {
+ unknown.push(letter);
+ continue;
+ };
+ if spec.takes_value {
+ let rest = &arg[index + letter.len_utf8()..];
+ if !rest.is_empty() {
+ rargs.push_front(rest.to_string());
+ }
+ let value = take_value(&opt, rargs)?;
+ parsed.set(spec.param, value);
+ break;
+ }
+ parsed.set(spec.param, String::new());
+ }
+ if !unknown.is_empty() {
+ largs.push(format!("-{unknown}"));
+ }
+ Ok(())
+}
+
+/// The value of `--map`, like click's `Choice.convert()` without case sensitivity: the
+/// choice as it is declared, or the error message.
+pub(crate) fn map_choice(value: &str) -> Result<&'static str, UsageError> {
+ let folded = super::python::casefold_ascii(value);
+ MAP_CHOICES
+ .iter()
+ .find(|choice| folded.as_deref() == Some(**choice))
+ .copied()
+ .ok_or_else(|| {
+ let choices: Vec = MAP_CHOICES.iter().map(|c| repr(c)).collect();
+ UsageError(format!(
+ "Invalid value for '--map': {} is not one of {}.",
+ repr(value),
+ choices.join(", ")
+ ))
+ })
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn parse_args(args: &[&str]) -> Result {
+ parse(&args.iter().map(|a| a.to_string()).collect::>())
+ }
+
+ #[test]
+ fn long_short_and_clusters() {
+ let parsed = parse_args(&["-db", "--rarity", "0:", "-ok", "name", "x"]).unwrap();
+ assert!(parsed.flag(Param::DisableBoundaryless));
+ assert!(parsed.flag(Param::OnlyText));
+ assert_eq!(parsed.value(Param::Rarity), Some("0:"));
+ assert_eq!(parsed.value(Param::Key), Some("name"));
+ assert_eq!(parsed.inputs, ["x"]);
+ assert_eq!(
+ parsed.invoked(),
+ [
+ Param::DisableBoundaryless,
+ Param::Rarity,
+ Param::OnlyText,
+ Param::Key
+ ]
+ );
+ }
+
+ #[test]
+ fn attached_values() {
+ let parsed = parse_args(&["-kname", "--top=5%", "-br=0.5:", "-r=0.5:"]).unwrap();
+ assert_eq!(parsed.value(Param::Key), Some("name"));
+ assert_eq!(parsed.value(Param::Top), Some("5%"));
+ assert_eq!(parsed.value(Param::BoundarylessRarity), Some("0.5:"));
+ // -r is a short option: the rest of the argument is its value, with the "="
+ assert_eq!(parsed.value(Param::Rarity), Some("=0.5:"));
+ }
+
+ #[test]
+ fn unknown_options_are_inputs() {
+ let parsed = parse_args(&["-1606586780", "--nope", "-h", "-o=1", "-tx", "-"]).unwrap();
+ assert_eq!(
+ parsed.inputs,
+ ["-1606586780", "--nope", "-h", "-=1", "-x", "-"]
+ );
+ assert!(parsed.flag(Param::OnlyText));
+ assert!(parsed.flag(Param::Tags));
+ }
+
+ #[test]
+ fn double_dash() {
+ let parsed = parse_args(&["-db", "--", "-db", "--", "x"]).unwrap();
+ assert_eq!(parsed.inputs, ["-db", "--", "x"]);
+ // A value can be anything, "--" too
+ let parsed = parse_args(&["-r", "--", "x"]).unwrap();
+ assert_eq!(parsed.value(Param::Rarity), Some("--"));
+ assert_eq!(parsed.inputs, ["x"]);
+ }
+
+ #[test]
+ fn errors() {
+ assert_eq!(
+ parse_args(&["--rarity"]).unwrap_err().0,
+ "Option '--rarity' requires an argument."
+ );
+ assert_eq!(
+ parse_args(&["-k"]).unwrap_err().0,
+ "Option '-k' requires an argument."
+ );
+ assert_eq!(
+ parse_args(&["--json=1"]).unwrap_err().0,
+ "Option '--json' does not take a value."
+ );
+ assert_eq!(
+ parse_args(&["-pt="]).unwrap_err().0,
+ "Option '-pt' does not take a value."
+ );
+ }
+
+ #[test]
+ fn map_choices() {
+ assert_eq!(map_choice("OSM"), Ok("osm"));
+ assert_eq!(map_choice("oſm"), Ok("osm"));
+ assert_eq!(
+ map_choice("bing").unwrap_err().0,
+ "Invalid value for '--map': 'bing' is not one of 'google', 'osm'."
+ );
+ }
+}
diff --git a/src/cli/help.rs b/src/cli/help.rs
new file mode 100644
index 0000000..422767d
--- /dev/null
+++ b/src/cli/help.rs
@@ -0,0 +1,448 @@
+//! `--help` and the usage line of errors, formatted like click 8.1 formats them: its
+//! `HelpFormatter` (`click/formatting.py`) with Python's `textwrap.TextWrapper`, so the text
+//! wraps exactly where Python's does.
+
+use super::args::OPTIONS;
+use super::python;
+use std::sync::OnceLock;
+
+/// The help text of `pywhat/what.py`'s `main()`, as click cleans it (`inspect.cleandoc`).
+const HELP: &str = include_str!("help.txt");
+
+/// What `--help` shows after the program name in the usage line.
+const USAGE_ARGS: &str = "[OPTIONS] [TEXT_INPUT]...";
+
+/// The width of click's help: `max(min(columns, 80) - 2, 50)`, where `columns` is what
+/// Python's `shutil.get_terminal_size()` returns.
+pub(crate) fn help_width(columns: usize) -> usize {
+ (columns.min(80).saturating_sub(2)).max(50)
+}
+
+/// `--help`, ending with a line break.
+pub(crate) fn help(prog: &str, width: usize) -> String {
+ let mut formatter = HelpFormatter::new(width);
+ formatter.write_usage(prog, USAGE_ARGS);
+ formatter.write_paragraph();
+ formatter.indent();
+ formatter.write_text(HELP);
+ formatter.dedent();
+ formatter.write_paragraph();
+ formatter.write_heading("Options");
+ formatter.indent();
+ let rows: Vec<(String, String)> = OPTIONS
+ .iter()
+ .map(|spec| {
+ // join_options(): sorted by the length of the prefix ("-" before "--")
+ let mut opts: Vec<&str> = spec.opts.to_vec();
+ opts.sort_by_key(|opt| if opt.starts_with("--") { 2 } else { 1 });
+ let mut first = opts.join(", ");
+ if spec.takes_value {
+ first.push(' ');
+ first.push_str(spec.metavar);
+ }
+ (first, spec.help.to_string())
+ })
+ .collect();
+ formatter.write_dl(&rows);
+ formatter.dedent();
+ let mut text = formatter.buffer.trim_end_matches('\n').to_string();
+ text.push('\n');
+ text
+}
+
+/// The usage line of a usage error, without a line break.
+pub(crate) fn usage(prog: &str, width: usize) -> String {
+ let mut formatter = HelpFormatter::new(width);
+ formatter.write_usage(prog, USAGE_ARGS);
+ formatter.buffer.trim_end_matches('\n').to_string()
+}
+
+/// `len(strip_ansi(text))`: the help has no escape sequences, so the number of characters.
+fn term_len(text: &str) -> usize {
+ text.chars().count()
+}
+
+/// click's `HelpFormatter`.
+struct HelpFormatter {
+ width: usize,
+ current_indent: usize,
+ buffer: String,
+}
+
+impl HelpFormatter {
+ fn new(width: usize) -> Self {
+ HelpFormatter {
+ width,
+ current_indent: 0,
+ buffer: String::new(),
+ }
+ }
+
+ fn write(&mut self, text: &str) {
+ self.buffer.push_str(text);
+ }
+
+ fn indent(&mut self) {
+ self.current_indent += 2;
+ }
+
+ fn dedent(&mut self) {
+ self.current_indent -= 2;
+ }
+
+ fn spaces(count: usize) -> String {
+ " ".repeat(count)
+ }
+
+ fn write_usage(&mut self, prog: &str, args: &str) {
+ let prefix = "Usage: ";
+ let usage_prefix = format!("{prefix:>width$}{prog} ", width = self.current_indent);
+ let text_width = self.width.saturating_sub(self.current_indent);
+ if text_width >= term_len(&usage_prefix) + 20 {
+ let indent = Self::spaces(term_len(&usage_prefix));
+ let wrapped = wrap_text(args, text_width, &usage_prefix, &indent, false);
+ self.write(&wrapped);
+ } else {
+ self.write(&usage_prefix);
+ self.write("\n");
+ let indent = Self::spaces(self.current_indent.max(term_len(prefix)) + 4);
+ let wrapped = wrap_text(args, text_width, &indent, &indent, false);
+ self.write(&wrapped);
+ }
+ self.write("\n");
+ }
+
+ fn write_heading(&mut self, heading: &str) {
+ let indent = Self::spaces(self.current_indent);
+ self.write(&format!("{indent}{heading}:\n"));
+ }
+
+ fn write_paragraph(&mut self) {
+ if !self.buffer.is_empty() {
+ self.write("\n");
+ }
+ }
+
+ fn write_text(&mut self, text: &str) {
+ let indent = Self::spaces(self.current_indent);
+ let wrapped = wrap_text(text, self.width, &indent, &indent, true);
+ self.write(&wrapped);
+ self.write("\n");
+ }
+
+ fn write_dl(&mut self, rows: &[(String, String)]) {
+ let (col_max, col_spacing) = (30, 2);
+ let widest = rows
+ .iter()
+ .map(|(first, _)| term_len(first))
+ .max()
+ .unwrap_or(0);
+ let first_col = widest.min(col_max) + col_spacing;
+ for (first, second) in rows {
+ let indent = Self::spaces(self.current_indent);
+ self.write(&format!("{indent}{first}"));
+ if second.is_empty() {
+ self.write("\n");
+ continue;
+ }
+ if term_len(first) <= first_col - col_spacing {
+ self.write(&Self::spaces(first_col - term_len(first)));
+ } else {
+ self.write("\n");
+ self.write(&Self::spaces(first_col + self.current_indent));
+ }
+ let text_width = (self.width as isize - first_col as isize - 2).max(10) as usize;
+ let wrapped = wrap_text(second, text_width, "", "", true);
+ let lines = splitlines(&wrapped);
+ match lines.split_first() {
+ Some((first_line, rest)) => {
+ self.write(&format!("{first_line}\n"));
+ for line in rest {
+ let indent = Self::spaces(first_col + self.current_indent);
+ self.write(&format!("{indent}{line}\n"));
+ }
+ }
+ None => self.write("\n"),
+ }
+ }
+ }
+}
+
+/// `str.splitlines()`.
+fn splitlines(text: &str) -> Vec<&str> {
+ let mut lines = Vec::new();
+ let mut start = 0;
+ let mut chars = text.char_indices().peekable();
+ while let Some((index, c)) = chars.next() {
+ let end = match c {
+ '\r' => {
+ if let Some(&(_, '\n')) = chars.peek() {
+ chars.next();
+ index + 2
+ } else {
+ index + 1
+ }
+ }
+ '\n' | '\x0b' | '\x0c' | '\x1c' | '\x1d' | '\x1e' | '\u{85}' | '\u{2028}'
+ | '\u{2029}' => index + c.len_utf8(),
+ _ => continue,
+ };
+ lines.push(&text[start..index]);
+ start = end;
+ }
+ if start < text.len() {
+ lines.push(&text[start..]);
+ }
+ lines
+}
+
+/// `str.expandtabs(8)`.
+fn expand_tabs(text: &str) -> String {
+ let mut out = String::with_capacity(text.len());
+ let mut column = 0;
+ for c in text.chars() {
+ match c {
+ '\t' => {
+ let spaces = 8 - column % 8;
+ out.push_str(&" ".repeat(spaces));
+ column += spaces;
+ }
+ '\n' | '\r' => {
+ out.push(c);
+ column = 0;
+ }
+ _ => {
+ out.push(c);
+ column += 1;
+ }
+ }
+ }
+ out
+}
+
+/// click's `wrap_text()`: wraps `text` to `width` columns, paragraph by paragraph (with
+/// the indentation of their first line) if `preserve_paragraphs`.
+fn wrap_text(
+ text: &str,
+ width: usize,
+ initial_indent: &str,
+ subsequent_indent: &str,
+ preserve_paragraphs: bool,
+) -> String {
+ let text = expand_tabs(text);
+ let mut wrapper = TextWrapper {
+ width: width as isize,
+ initial_indent: initial_indent.to_string(),
+ subsequent_indent: subsequent_indent.to_string(),
+ };
+ if !preserve_paragraphs {
+ return wrapper.fill(&text);
+ }
+ // (indentation, raw, text) of each paragraph
+ let mut paragraphs: Vec<(usize, bool, String)> = Vec::new();
+ let mut buffer: Vec<&str> = Vec::new();
+ let mut indent: Option = None;
+ let flush = |buffer: &mut Vec<&str>, indent: Option, paragraphs: &mut Vec<_>| {
+ if buffer.is_empty() {
+ return;
+ }
+ if python::strip(buffer[0]) == "\x08" {
+ paragraphs.push((indent.unwrap_or(0), true, buffer[1..].join("\n")));
+ } else {
+ paragraphs.push((indent.unwrap_or(0), false, buffer.join(" ")));
+ }
+ buffer.clear();
+ };
+ for line in splitlines(&text) {
+ if line.is_empty() {
+ flush(&mut buffer, indent, &mut paragraphs);
+ indent = None;
+ } else {
+ let line = if indent.is_none() {
+ let stripped = python::lstrip(line);
+ indent = Some(term_len(line) - term_len(stripped));
+ stripped
+ } else {
+ line
+ };
+ buffer.push(line);
+ }
+ }
+ flush(&mut buffer, indent, &mut paragraphs);
+ let mut wrapped = Vec::new();
+ for (indent, raw, text) in paragraphs {
+ let extra = " ".repeat(indent);
+ let saved = (
+ wrapper.initial_indent.clone(),
+ wrapper.subsequent_indent.clone(),
+ );
+ wrapper.initial_indent.push_str(&extra);
+ wrapper.subsequent_indent.push_str(&extra);
+ wrapped.push(if raw {
+ wrapper.indent_only(&text)
+ } else {
+ wrapper.fill(&text)
+ });
+ (wrapper.initial_indent, wrapper.subsequent_indent) = saved;
+ }
+ wrapped.join("\n\n")
+}
+
+/// Python's `textwrap.TextWrapper` with click's options (`replace_whitespace=False`) and
+/// click's `_handle_long_word`.
+struct TextWrapper {
+ width: isize,
+ initial_indent: String,
+ subsequent_indent: String,
+}
+
+/// `TextWrapper.wordsep_re`, with `\Z` written as fancy-regex writes it.
+fn wordsep_re() -> &'static fancy_regex::Regex {
+ static WORDSEP: OnceLock = OnceLock::new();
+ WORDSEP.get_or_init(|| {
+ let ws = r"[\t\n\x0b\x0c\r ]";
+ let nws = r"[^\t\n\x0b\x0c\r ]";
+ let wp = r#"[\w!"'&.,?]"#;
+ let lt = r"[^\d\W]";
+ let pattern = format!(
+ r"({ws}+|(?<={wp})-{{2,}}(?=\w)|{nws}+?(?:-(?:(?<={lt}{{2}}-)|(?<={lt}-{lt}-))(?={lt}-?{lt})|(?={ws}|\z)|(?<={wp})(?=-{{2,}}\w)))"
+ );
+ fancy_regex::Regex::new(&pattern).expect("wordsep_re compiles")
+ })
+}
+
+impl TextWrapper {
+ /// `_split_chunks()`: words, whitespace and hyphenated parts of words.
+ fn split_chunks(text: &str) -> Vec {
+ let text = expand_tabs(text);
+ let mut chunks = Vec::new();
+ let mut last = 0;
+ for found in wordsep_re().find_iter(&text) {
+ let Ok(found) = found else { break };
+ chunks.push(text[last..found.start()].to_string());
+ chunks.push(found.as_str().to_string());
+ last = found.end();
+ }
+ chunks.push(text[last..].to_string());
+ chunks.retain(|chunk| !chunk.is_empty());
+ chunks
+ }
+
+ fn fill(&self, text: &str) -> String {
+ self.wrap(text).join("\n")
+ }
+
+ fn indent_only(&self, text: &str) -> String {
+ splitlines(text)
+ .iter()
+ .enumerate()
+ .map(|(index, line)| {
+ let indent = if index == 0 {
+ &self.initial_indent
+ } else {
+ &self.subsequent_indent
+ };
+ format!("{indent}{line}")
+ })
+ .collect::>()
+ .join("\n")
+ }
+
+ /// `_wrap_chunks()`.
+ fn wrap(&self, text: &str) -> Vec {
+ let len = |chunk: &str| chunk.chars().count() as isize;
+ let is_blank = |chunk: &str| python::strip(chunk).is_empty();
+ let mut chunks = Self::split_chunks(text);
+ chunks.reverse();
+ let mut lines: Vec = Vec::new();
+ while !chunks.is_empty() {
+ let mut cur_line: Vec = Vec::new();
+ let mut cur_len: isize = 0;
+ let indent = if lines.is_empty() {
+ &self.initial_indent
+ } else {
+ &self.subsequent_indent
+ };
+ let width = self.width - len(indent);
+ if chunks.last().is_some_and(|chunk| is_blank(chunk)) && !lines.is_empty() {
+ chunks.pop();
+ }
+ while let Some(chunk) = chunks.last() {
+ let l = len(chunk);
+ if cur_len + l <= width {
+ cur_len += l;
+ cur_line.push(chunks.pop().expect("a chunk"));
+ } else {
+ break;
+ }
+ }
+ if chunks.last().is_some_and(|chunk| len(chunk) > width) {
+ // click's _handle_long_word: break the word to fill the line
+ let space_left = (width - cur_len).max(1) as usize;
+ let last = chunks.last_mut().expect("a chunk");
+ let cut_at = last
+ .char_indices()
+ .nth(space_left)
+ .map_or(last.len(), |(index, _)| index);
+ let rest = last.split_off(cut_at);
+ cur_line.push(std::mem::replace(last, rest));
+ }
+ cur_line.pop_if(|chunk| is_blank(chunk));
+ if !cur_line.is_empty() {
+ lines.push(format!("{indent}{}", cur_line.concat()));
+ }
+ }
+ lines
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn chunks_like_textwrap() {
+ // click's TextWrapper()._split_chunks() in Python 3.10
+ assert_eq!(
+ TextWrapper::split_chunks("Hello there -- you goof-ball, use the -b option!"),
+ [
+ "Hello", " ", "there", " ", "--", " ", "you", " ", "goof-", "ball,", " ", "use",
+ " ", "the", " ", "-b", " ", "option!"
+ ]
+ );
+ assert_eq!(
+ TextWrapper::split_chunks("a--b well--done '--rarity 0:' x-1"),
+ [
+ "a", "--", "b", " ", "well", "--", "done", " ", "'", "--", "rarity", " ", "0:'",
+ " ", "x-1"
+ ]
+ );
+ }
+
+ #[test]
+ fn wraps_like_textwrap() {
+ let wrapper = TextWrapper {
+ width: 20,
+ initial_indent: "* ".into(),
+ subsequent_indent: " ".into(),
+ };
+ // click's TextWrapper(20, initial_indent="* ", subsequent_indent=" ").fill(text)
+ assert_eq!(
+ wrapper.fill("The quick brown fox jumps over the lazy dog, supercalifragilistic"),
+ "* The quick brown\n fox jumps over the\n lazy dog, supercal\n ifragilistic"
+ );
+ }
+
+ #[test]
+ fn widths() {
+ assert_eq!(help_width(10_000), 78);
+ assert_eq!(help_width(60), 58);
+ assert_eq!(help_width(10), 50);
+ }
+
+ #[test]
+ fn usage_line() {
+ assert_eq!(usage("what", 78), "Usage: what [OPTIONS] [TEXT_INPUT]...");
+ assert!(help("what", 78).starts_with("Usage: what [OPTIONS] [TEXT_INPUT]...\n\n pyWhat"));
+ }
+}
diff --git a/src/cli/help.txt b/src/cli/help.txt
new file mode 100644
index 0000000..3433bc6
--- /dev/null
+++ b/src/cli/help.txt
@@ -0,0 +1,207 @@
+pyWhat - Identify what something is.
+
+Made by Bee https://twitter.com/bee_sec_san
+
+https://github.com/bee-san
+
+Filtration:
+
+ --rarity min:max
+
+ Rarity is how unlikely something is to be a false-positive. The higher the number, the more unlikely.
+
+ Only print entries with rarity in range [min,max]. min and max can be omitted.
+
+ Note: PyWhat by default has a rarity of 0.1. To see all matches, with many potential false positives use `0:`. For example, a YouTube video ID on its own, such as 'dQw4w9WgXcQ', is only shown with `--rarity 0:`.
+
+ --include list
+
+ Only include entries containing at least one tag in a list. List is a comma separated list.
+
+ --exclude list
+
+ Exclude specified tags. List is a comma separated list.
+
+ The names of the regexes work like tags. Many regexes have alternative names too, such as 'ETH Wallet' for 'Ethereum (ETH) Wallet Address', so --include 'BTC Wallet,ETH Wallet' only shows Bitcoin and Ethereum wallets.
+
+ --names
+
+ Show the name of every regex, followed by its alternative names.
+
+Sorting:
+
+ --key key_name
+
+ Sort by the given key.
+
+ --reverse
+
+ Sort in reverse order.
+
+ Available keys:
+
+ name - Sort by the name of regex pattern
+
+ rarity - Sort by rarity
+
+ matched - Sort by a matched string
+
+ likely - Sort the most likely matches first, see Top matches
+
+ none - No sorting is done (the default)
+
+Top matches:
+
+ --top N
+
+ Only show the N most likely matches, e.g. --top 10. --top N% shows the most likely N percent of them, e.g. --top 5%. For a directory or several inputs, these are the most likely matches of all of them.
+
+ The most likely matches are shown first, unless --key sorts them. They have the highest rarity, but fragments of a longer word or match, which boundaryless mode finds, are less likely unless their rarity is 1. For example, the card and social security numbers in '0x52908400098527886E0F7030069857D2E4169EE7' are fragments of the Ethereum address, which is the most likely match.
+
+ How many matches were left out is shown on stderr. Use --interactive to page through all of them.
+
+Exporting:
+
+ --json
+
+ Return results in json format, on one line, so that it can be piped to other programs such as jq: pywhat --json 'abc@example.com' | jq
+
+Boundaryless mode:
+
+ CLI tool matches strings like 'abcdTHM{hello}plze' by default because the boundaryless mode is enabled for regexes with a rarity of 0.1 and higher.
+
+ Since boundaryless mode may produce a lot of false-positive matches, it is possible to disable it, either fully or partially.
+
+ '--disable-boundaryless' flag can be used to fully disable this mode.
+
+ In addition, '-br', '-bi', and '-be' options can be used to tweak which regexes should be in boundaryless mode.
+
+ Refer to the Filtration section for more information.
+
+Processing:
+
+ Matches are processed further once a regex has found them. For example, the date of a Unix timestamp, in UTC, is added to its description: pywhat --rarity 0: --include "UNIX Timestamp" 1637093119
+
+ Matches whose checksum is wrong are filtered out: Bitcoin, Bitcoin Cash, Litecoin, Dogecoin and XRP wallet addresses (like 3F3F3F3F3F3F3F3F3F3F3F3F3F3F3F3F3F), card numbers (the Luhn check) and Turkish Identification Numbers.
+
+ '--disable-processing' flag can be used to show the matches as the regexes found them.
+
+Verification:
+
+ --verify
+
+ Ask the services if the keys that are found are valid, and add the answer to the description, e.g. 'Verification: invalid, Google rejected the key (API_KEY_INVALID)'. Currently Google API keys are verified: pywhat --verify 'AIzaSyA00000000000000000000000000000000'
+
+ Only use it for keys that you are allowed to test: it sends the keys to the services (over HTTPS), once per key. It works with '--disable-processing' too.
+
+Maps:
+
+ --map google|osm
+
+ The map that the links of coordinates go to: Google Maps (the default) or OpenStreetMap. For example, pywhat --map osm '52.6169586, -1.9779857' links to https://www.openstreetmap.org/search?query=52.6169586,-1.9779857
+
+ It works with '--disable-processing' too.
+
+Formatting the output:
+
+ --format format_str
+
+ format_str can be equal to:
+
+ pretty - Output data in the table
+
+ json - Output data in json format
+
+ CUSTOM_STRING - Print data in the way you want. For every match CUSTOM_STRING will be printed and '%x' (See below for possible x values) will be substituted with a match value.
+
+ For example:
+
+ pywhat --format '%m - %n' 'google.com htb{flag}'
+
+ will print:
+
+ htb{flag} - HackTheBox Flag Format
+ google.com - Uniform Resource Locator (URL)
+
+ Possible '%x' values:
+
+ %m - matched text
+
+ %n - name of regex
+
+ %d - description (will not output if absent)
+
+ %e - exploit (will not output if absent)
+
+ %r - rarity
+
+ %l - link (will not output if absent)
+
+ %t - tags (in 'tag1, tag2 ...' format)
+
+ If you want to print '%' or '\' character - escape it: '\%', '\\'.
+
+Interactive mode:
+
+ --interactive
+
+ Identify the input once, keep the matches in memory and search through them as often as you like, with searches such as 'include:"Bug Bounty", rarity:"0.5:"'.
+
+ The --rarity, --include and --exclude options are the search to start with, the other options work as usual. The input is optional, the 'load' command loads a file, directory or text. Type 'help' in interactive mode to see all commands.
+
+ With --top N or the 'top N' command, searches show the N most likely matches and the 'more' command the next N.
+
+Progress and streaming:
+
+ When a search takes a while, progress bars show how far it is: one for the regexes that are searched for in a file, with the regex that is searched for now, and one for the files of a directory or of several inputs. They are only shown on a terminal (on stderr if the output is redirected to a file), and they are removed once the search is complete.
+
+ --no-progress
+
+ Do not show the progress bars.
+
+ --stream
+
+ Print every match as soon as it is found, instead of once the search is complete. With --json, every match is a JSON object of its own, on one line, without the "Fragment" key, as whether a match is a fragment is only known once its file has been searched. --stream cannot sort the matches (--key), show only the most likely ones (--top) or print a table (--format pretty), as these need all the matches.
+
+Unicode:
+
+ Files and text piped to pyWhat can be UTF-8, or UTF-16 or UTF-32 with a byte order mark (BOM). The UTF-16 strings in binary files, which 'strings -el' shows, are searched too.
+
+ Characters that the terminal cannot show are printed as escape sequences such as '\u20bf'.
+
+Examples:
+
+ * what 'HTB{this is a flag}'
+
+ * what '0x52908400098527886E0F7030069857D2E4169EE7'
+
+ * what -- '52.6169586, -1.9779857'
+
+ * what --rarity 0.6: 'myEmail@host.org'
+
+ * what --rarity 0: --include "credentials" --exclude "aws" 'James:SecretPassword'
+
+ * what -br 0.6: -be URL '123myEmail@host.org456'
+
+ * what --top 1 '0x52908400098527886E0F7030069857D2E4169EE7'
+
+Your text must either be in quotation marks, or use the POSIX standard of "--" to mean "anything after -- is textual input".
+
+
+pyWhat can also search files or even a whole directory with recursion:
+
+ * what 'secret.txt'
+
+ * what 'this/is/a/path'
+
+ * what --interactive 'this/is/a/path'
+
+ * what --stream 'big.log'
+
+Several inputs, files, directories or text, can be searched at once. Then the file of every match is shown, and the matches in text are under "text":
+
+ * what 'secret.txt' 'this/is/a/path' 'HTB{this is a flag}'
+
+ * find . -name '*.log' -exec what {} +
+
+Text with spaces needs quotation marks, as every argument is an input.
\ No newline at end of file
diff --git a/src/cli/markup.rs b/src/cli/markup.rs
new file mode 100644
index 0000000..a79d35a
--- /dev/null
+++ b/src/cli/markup.rs
@@ -0,0 +1,179 @@
+//! rich's console markup, for the strings of pyWhat's data files: some descriptions in
+//! `regex.json` are written for rich (`[#CAE4F1][link=https://...]CORS[/link][/#CAE4F1]`),
+//! and rich takes the wiki-style links of `file_signatures.json` (`[https://... name]`) for
+//! tags too, and leaves them out. The Python CLI prints all of its output as markup; this
+//! one only the data, and prints the matched text and the file names as they are (#291,
+//! Python bug 4).
+
+use super::style::Style;
+use std::sync::OnceLock;
+
+/// rich's `RE_TAGS`: a tag is `[` and a letter, `#`, `/` or `@`, up to the next `]`,
+/// possibly escaped with backslashes.
+fn tags() -> &'static regex::Regex {
+ static TAGS: OnceLock = OnceLock::new();
+ TAGS.get_or_init(|| regex::Regex::new(r"(\\*)\[([a-z#/@][^\[]*?)\]").expect("valid regex"))
+}
+
+/// A style that rich can parse, as far as the data uses them: `bold`, colours as
+/// `#rrggbb` and `link URL`. Anything else is no style, which is what rich makes of a
+/// style it can't parse.
+fn parse_style(definition: &str) -> Style {
+ let mut style = Style::default();
+ let mut words = definition.split_whitespace();
+ while let Some(word) = words.next() {
+ match word.to_lowercase().as_str() {
+ "bold" | "b" => style.bold = true,
+ "link" => match words.next() {
+ Some(url) => style.link = Some(url.to_string()),
+ None => return Style::default(),
+ },
+ color if color.len() == 7 && color.starts_with('#') => {
+ match u32::from_str_radix(&color[1..], 16) {
+ Ok(rgb) => style.color = Some(((rgb >> 16) as u8, (rgb >> 8) as u8, rgb as u8)),
+ Err(_) => return Style::default(),
+ }
+ }
+ _ => return Style::default(),
+ }
+ }
+ style
+}
+
+/// An open tag: its name (to match the closing tag) and its style.
+struct Open {
+ name: String,
+ style: Style,
+}
+
+fn combined(open: &[Open]) -> Style {
+ let mut style = Style::default();
+ for tag in open {
+ style.bold |= tag.style.bold;
+ if tag.style.color.is_some() {
+ style.color = tag.style.color;
+ }
+ if tag.style.link.is_some() {
+ style.link.clone_from(&tag.style.link);
+ }
+ }
+ style
+}
+
+/// The text of `markup` with its styles, as `rich.markup.render()` makes it (without emoji
+/// codes, which the data doesn't have).
+pub(crate) fn render(markup: &str) -> Vec<(String, Style)> {
+ if !markup.contains('[') {
+ return vec![(markup.to_string(), Style::default())];
+ }
+ let mut spans: Vec<(String, Style)> = Vec::new();
+ let mut open: Vec = Vec::new();
+ let push = |spans: &mut Vec<(String, Style)>, text: &str, open: &[Open]| {
+ if !text.is_empty() {
+ // Escaped brackets that aren't part of a tag
+ spans.push((text.replace("\\[", "["), combined(open)));
+ }
+ };
+ let mut position = 0;
+ for captures in tags().captures_iter(markup) {
+ let whole = captures.get(0).expect("a match");
+ let escapes = captures.get(1).map_or("", |m| m.as_str());
+ let tag_text = captures.get(2).map_or("", |m| m.as_str());
+ push(&mut spans, &markup[position..whole.start()], &open);
+ position = whole.end();
+ if !escapes.is_empty() {
+ let (backslashes, escaped) = (escapes.len() / 2, escapes.len() % 2 == 1);
+ if backslashes > 0 {
+ spans.push(("\\".repeat(backslashes), combined(&open)));
+ }
+ if escaped {
+ // An escaped tag is text, with its brackets
+ spans.push((whole.as_str()[escapes.len()..].to_string(), combined(&open)));
+ continue;
+ }
+ }
+ let (name, parameters) = match tag_text.split_once('=') {
+ Some((name, parameters)) => (name, Some(parameters)),
+ None => (tag_text, None),
+ };
+ if let Some(closing) = name.strip_prefix('/') {
+ let closing = closing.trim().to_lowercase();
+ if closing.is_empty() {
+ open.pop();
+ } else if let Some(index) = open.iter().rposition(|tag| tag.name == closing) {
+ open.remove(index);
+ }
+ // rich raises an error for a closing tag that closes nothing; the data has none
+ } else {
+ let definition = match parameters {
+ Some(parameters) => format!("{name} {parameters}"),
+ None => name.to_string(),
+ };
+ open.push(Open {
+ name: name.trim().to_lowercase(),
+ style: parse_style(&definition),
+ });
+ }
+ }
+ push(&mut spans, &markup[position..], &open);
+ spans
+}
+
+/// The text of `markup` without its tags.
+pub(crate) fn plain(markup: &str) -> String {
+ render(markup).into_iter().map(|(text, _)| text).collect()
+}
+
+/// `rich.markup.escape()`: `text` as markup that prints `text` as it is.
+pub(crate) fn escape(text: &str) -> String {
+ static ESCAPE: OnceLock = OnceLock::new();
+ let escape =
+ ESCAPE.get_or_init(|| regex::Regex::new(r"(\\*)(\[[a-z#/@].*?\])").expect("valid regex"));
+ escape
+ .replace_all(text, |captures: ®ex::Captures<'_>| {
+ let backslashes = &captures[1];
+ format!("{backslashes}{backslashes}\\{}", &captures[2])
+ })
+ .into_owned()
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn data_markup() {
+ // What rich 12.2.0 prints for these strings of the data files
+ assert_eq!(
+ plain("Used for [#CAE4F1][link=https://en.wikipedia.org/wiki/Cross-origin_resource_sharing]Cross-Origin Resource Sharing (CORS)[/link][/#CAE4F1]"),
+ "Used for Cross-Origin Resource Sharing (CORS)"
+ );
+ assert_eq!(
+ plain("Compressed file using [https://segaretro.org/Rob_Northen_compression Rob Northen compression] (version 1 and 2) algorithm"),
+ "Compressed file using (version 1 and 2) algorithm"
+ );
+ assert_eq!(plain("a \\[b] \\\\[bold]c"), "a [b] \\c");
+ assert_eq!(plain("[1] [A] x"), "[1] [A] x");
+ let spans = render("x [#CAE4F1][link=https://a.b]y[/link][/#CAE4F1] z");
+ assert_eq!(spans[1].0, "y");
+ assert_eq!(spans[1].1.color, Some((0xCA, 0xE4, 0xF1)));
+ assert_eq!(spans[1].1.link.as_deref(), Some("https://a.b"));
+ assert_eq!(spans[2], (" z".to_string(), Style::default()));
+ }
+
+ #[test]
+ fn escape_like_rich() {
+ // rich.markup.escape() in rich 12.2.0
+ for (text, escaped) in [
+ ("a\\", "a\\"),
+ ("[b]x", "\\[b]x"),
+ ("\\[b]", "\\\\\\[b]"),
+ ("x [B] [1]", "x [B] [1]"),
+ ("[/]", "\\[/]"),
+ ("a[b]c\\", "a\\[b]c\\"),
+ ] {
+ assert_eq!(escape(text), escaped, "{text:?}");
+ assert_eq!(plain(&escape(text)), text, "{text:?}");
+ }
+ }
+}
diff --git a/src/cli/mod.rs b/src/cli/mod.rs
new file mode 100644
index 0000000..9fecb18
--- /dev/null
+++ b/src/cli/mod.rs
@@ -0,0 +1,953 @@
+//! The `what` and `pywhat` commands, with the options, the output and the exit codes of
+//! the Python package's CLI (`pywhat/what.py`): `pywhat::cli::main` is what both binaries
+//! run, and what the Python package's commands call.
+//!
+//! ```no_run
+//! // src/bin/what.rs
+//! std::process::exit(pywhat::cli::main(std::env::args_os()))
+//! ```
+//!
+//! What it does differently from the Python CLI: it prints the matched text and the file
+//! names as they are, where the Python CLI takes them for rich markup (`thm{[bold]x[/bold]}`
+//! or `:smile:`); `--interactive` needs the Python package; and a file that can't be read
+//! is reported and skipped.
+
+mod args;
+mod help;
+mod markup;
+mod output;
+mod progress;
+mod python;
+mod style;
+mod table;
+pub mod verify;
+
+use crate::files::{Found, Identified, InputOptions, Progress};
+use crate::filter::{split_tags, Filter};
+use crate::processors::{default_processors, OpenStreetMap, Processor};
+use crate::ranking::{Key, Top};
+use crate::Identifier;
+use args::Param;
+use output::Printer;
+use std::collections::HashMap;
+use std::ffi::OsString;
+use std::io::{self, IsTerminal, Read, Write};
+use style::{Colors, Text, LABEL, VERSION};
+
+/// Runs the command line `args`, the program name first (as `std::env::args_os()` gives
+/// it), and returns the exit code: 0, 1 for errors (as in Python), 2 for usage errors.
+/// The name of the program, without its directory, is the name that `--help` shows.
+pub fn main(args: I) -> i32
+where
+ I: IntoIterator- ,
+ T: Into,
+{
+ let mut args = args.into_iter().map(|arg| {
+ arg.into()
+ .into_string()
+ .unwrap_or_else(|arg| arg.to_string_lossy().into_owned())
+ });
+ let prog = args
+ .next()
+ .map_or_else(|| "what".to_string(), |arg0| program_name(&arg0));
+ let args: Vec = args.collect();
+ let env = Environment::detect();
+ let mut stdin = io::stdin();
+ let mut stdout = io::BufWriter::with_capacity(1 << 16, io::stdout());
+ let mut stderr = io::stderr();
+ let mut streams = Streams {
+ stdin: &mut stdin,
+ stdout: &mut stdout,
+ stderr: &mut stderr,
+ };
+ run(&prog, &args, &env, &mut streams, &GoogleVerifier::default())
+}
+
+/// The name of the program in `--help`: the file name of `arg0` (without `.exe` on
+/// Windows), like click's `_detect_program_name()`.
+fn program_name(arg0: &str) -> String {
+ let separators: &[char] = if cfg!(windows) { &['/', '\\'] } else { &['/'] };
+ let name = arg0.rsplit(separators).next().unwrap_or(arg0);
+ if cfg!(windows) && name.to_ascii_lowercase().ends_with(".exe") {
+ return name[..name.len() - 4].to_string();
+ }
+ name.to_string()
+}
+
+/// A stream of the process: whether it is a terminal or a regular file, and its width.
+#[derive(Debug, Clone, Copy, Default)]
+struct StreamInfo {
+ terminal: bool,
+ regular_file: bool,
+ columns: Option,
+}
+
+/// What the command needs to know about where it runs.
+#[derive(Debug, Clone, Default)]
+struct Environment {
+ vars: HashMap<&'static str, String>,
+ stdin: StreamInfo,
+ stdout: StreamInfo,
+ stderr: StreamInfo,
+}
+
+/// The environment variables that the Python CLI (click and rich) reads.
+const VARIABLES: [&str; 5] = ["COLUMNS", "LINES", "TERM", "COLORTERM", "NO_COLOR"];
+
+#[cfg(unix)]
+fn stream_info(stream: S) -> StreamInfo {
+ let regular_file = stream
+ .as_fd()
+ .try_clone_to_owned()
+ .map(std::fs::File::from)
+ .and_then(|file| file.metadata())
+ .is_ok_and(|metadata| metadata.is_file());
+ StreamInfo {
+ terminal: stream.is_terminal(),
+ regular_file,
+ columns: terminal_size::terminal_size_of(stream).map(|(width, _)| width.0 as usize),
+ }
+}
+
+#[cfg(windows)]
+fn stream_info(stream: S) -> StreamInfo {
+ let regular_file = stream
+ .as_handle()
+ .try_clone_to_owned()
+ .map(std::fs::File::from)
+ .and_then(|file| file.metadata())
+ .is_ok_and(|metadata| metadata.is_file());
+ StreamInfo {
+ terminal: stream.is_terminal(),
+ regular_file,
+ columns: terminal_size::terminal_size_of(stream).map(|(width, _)| width.0 as usize),
+ }
+}
+
+#[cfg(not(any(unix, windows)))]
+fn stream_info(stream: S) -> StreamInfo {
+ StreamInfo {
+ terminal: stream.is_terminal(),
+ ..StreamInfo::default()
+ }
+}
+
+impl Environment {
+ fn detect() -> Environment {
+ let vars = VARIABLES
+ .iter()
+ .filter_map(|&name| {
+ std::env::var_os(name).map(|value| (name, value.to_string_lossy().into_owned()))
+ })
+ .collect();
+ Environment {
+ vars,
+ stdin: stream_info(io::stdin()),
+ stdout: stream_info(io::stdout()),
+ stderr: stream_info(io::stderr()),
+ }
+ }
+
+ fn var(&self, name: &str) -> Option {
+ self.vars.get(name).cloned()
+ }
+
+ /// The colours of a stream, as rich detects them.
+ fn colors(&self, stream: &StreamInfo) -> Colors {
+ Colors::detect(stream.terminal, &|name| self.var(name))
+ }
+
+ /// `shutil.get_terminal_size().columns`, which click's help uses: `COLUMNS` if it is
+ /// a positive number, else the width of the terminal of stdout, else 80.
+ fn click_columns(&self) -> usize {
+ match self
+ .var("COLUMNS")
+ .and_then(|columns| python::int(&columns))
+ {
+ Some(columns) if columns > 0 => usize::try_from(columns).unwrap_or(usize::MAX),
+ _ => self
+ .stdout
+ .columns
+ .filter(|&columns| columns > 0)
+ .unwrap_or(80),
+ }
+ }
+
+ /// The width of a rich `Console`: `COLUMNS` if it is a number, else the width of the
+ /// terminal of stdin or (on Windows: only) stdout, else 80.
+ fn rich_width(&self) -> usize {
+ let columns = self.var("COLUMNS").filter(|columns| {
+ !columns.is_empty() && columns.chars().all(|c| python::decimal_value(c).is_some())
+ });
+ let width = match columns.and_then(|columns| python::int(&columns)) {
+ Some(columns) => usize::try_from(columns).unwrap_or(usize::MAX),
+ None if cfg!(windows) => self.stdout.columns.unwrap_or(0),
+ None => self.stdin.columns.or(self.stdout.columns).unwrap_or(0),
+ };
+ if width == 0 {
+ 80
+ } else {
+ width
+ }
+ }
+}
+
+/// The streams of the process.
+struct Streams<'a> {
+ stdin: &'a mut dyn Read,
+ stdout: &'a mut dyn Write,
+ stderr: &'a mut dyn Write,
+}
+
+/// Makes the `--verify` verifier; tests give it a fake Google.
+#[derive(Default)]
+struct GoogleVerifier {
+ make: Option verify::GoogleApiKeyVerifier>>,
+}
+
+impl GoogleVerifier {
+ fn verifier(&self) -> verify::GoogleApiKeyVerifier {
+ match &self.make {
+ Some(make) => make(),
+ None => verify::GoogleApiKeyVerifier::new(),
+ }
+ }
+}
+
+/// Runs the command, returns its exit code. A closed output (`what ... | head`) ends it
+/// with 1, as click does.
+fn run(
+ prog: &str,
+ args: &[String],
+ env: &Environment,
+ streams: &mut Streams<'_>,
+ verifier: &GoogleVerifier,
+) -> i32 {
+ let code = match execute(prog, args, env, streams, verifier) {
+ Ok(code) => streams.stdout.flush().map_or(1, |_| code),
+ Err(error) if error.kind() == io::ErrorKind::BrokenPipe => 1,
+ Err(error) => {
+ let _ = writeln!(streams.stderr, "Error: {error}");
+ 1
+ }
+ };
+ let _ = streams.stdout.flush();
+ code
+}
+
+/// A usage error, as click shows it: the usage line, a hint and the error, exit code 2.
+fn usage_error(
+ prog: &str,
+ message: &str,
+ env: &Environment,
+ streams: &mut Streams<'_>,
+) -> io::Result {
+ streams.stdout.flush()?;
+ let width = help::help_width(env.click_columns());
+ let usage = help::usage(prog, width);
+ write!(
+ streams.stderr,
+ "{usage}\nTry '{prog} --help' for help.\n\nError: {message}\n"
+ )?;
+ Ok(2)
+}
+
+/// Prints `message` and a line break on stdout, with `print()`.
+fn print_line(streams: &mut Streams<'_>, message: &str) -> io::Result<()> {
+ writeln!(streams.stdout, "{message}")
+}
+
+/// `sys.exit(message)`: the message on stderr, exit code 1.
+fn exit_with(streams: &mut Streams<'_>, message: &str) -> io::Result {
+ streams.stdout.flush()?;
+ writeln!(streams.stderr, "{message}")?;
+ Ok(1)
+}
+
+/// `print_tags()`: every tag, sorted.
+fn print_tags(streams: &mut Streams<'_>, colors: Colors) -> io::Result<()> {
+ let mut text = Text::new();
+ for (index, tag) in crate::tags().iter().enumerate() {
+ if index > 0 {
+ text.plain("\n");
+ }
+ text.styled(tag, LABEL);
+ }
+ let mut printer = Printer {
+ out: streams.stdout,
+ colors,
+ };
+ printer.print(&text)
+}
+
+/// `print_names()`: the name of every regex, sorted, with its alternative names.
+fn print_names(streams: &mut Streams<'_>, colors: Colors) -> io::Result<()> {
+ let mut regexes: Vec<_> = crate::regexes().iter().collect();
+ regexes.sort_by_cached_key(|regex| regex.name.to_lowercase());
+ let mut text = Text::new();
+ for (index, regex) in regexes.iter().enumerate() {
+ if index > 0 {
+ text.plain("\n");
+ }
+ text.styled(®ex.name, LABEL);
+ if !regex.alternative_names.is_empty() {
+ text.plain(": ").plain(®ex.alternative_names.join(", "));
+ }
+ }
+ let mut printer = Printer {
+ out: streams.stdout,
+ colors,
+ };
+ printer.print(&text)
+}
+
+/// `print_version()`.
+fn print_version(streams: &mut Streams<'_>, colors: Colors) -> io::Result<()> {
+ let mut text = Text::new();
+ text.plain("PyWhat version ")
+ .styled(env!("CARGO_PKG_VERSION"), VERSION);
+ let mut printer = Printer {
+ out: streams.stdout,
+ colors,
+ };
+ printer.print(&text)
+}
+
+/// `create_filter()`: the filter of `--rarity`, `--include` and `--exclude` (or of their
+/// boundaryless versions), or the message that Python prints before it exits with 1.
+fn create_filter(
+ rarity: Option<&str>,
+ include: Option<&str>,
+ exclude: Option<&str>,
+) -> Result {
+ let (mut min_rarity, mut max_rarity) = (0.1, 1.0);
+ if let Some(rarity) = rarity {
+ let rarities: Vec<&str> = rarity.split(':').collect();
+ if rarities.len() != 2 {
+ return Err("Invalid rarity range format ('min:max' expected)");
+ }
+ for (rarity, value) in rarities.iter().zip([&mut min_rarity, &mut max_rarity]) {
+ if !python::is_all_space(rarity) && !rarity.is_empty() {
+ *value = python::float(rarity).ok_or("Invalid rarity argument (float expected)")?;
+ }
+ }
+ }
+ let tags = |list: &str| -> Vec {
+ split_tags(list)
+ .iter()
+ .map(|tag| python::strip(tag).to_string())
+ .collect()
+ };
+ let include = include.map(tags);
+ let exclude = exclude.map(tags).unwrap_or_default();
+ Filter::new(include.as_deref(), &exclude, min_rarity, max_rarity).map_err(|_| {
+ "Passed tags are not valid.\n\
+ You can check available tags by using: 'pywhat --tags'\n\
+ and the names of the regexes, which work like tags, by using: 'pywhat --names'"
+ })
+}
+
+/// `parse_top()`, with Python's whitespace: `str.strip()` and the `\s` of `N %` take
+/// U+001C to U+001F too.
+fn parse_top(text: &str) -> Result {
+ let text = python::strip(text);
+ match Top::parse(text) {
+ Ok(top) => Ok(top),
+ Err(error) => {
+ if let Some(number) = text.strip_suffix('%') {
+ let number = python::rstrip(number);
+ if let Ok(top) = Top::parse(&format!("{number}%")) {
+ return Ok(top);
+ }
+ }
+ Err(error)
+ }
+ }
+}
+
+/// `read_stdin()`: stdin read like a file (UTF-8, or UTF-16 or UTF-32 with a BOM, and the
+/// UTF-16 strings in it), as one text.
+fn read_stdin(stdin: &mut dyn Read) -> io::Result {
+ let mut data = Vec::new();
+ stdin.read_to_end(&mut data)?;
+ Ok(python::strip(&crate::unicode::texts(&data).join("\n")).to_string())
+}
+
+/// `Identified._collect()` of one thing found.
+fn collect(identified: &mut Identified, found: Found) {
+ match found {
+ Found::FileSignature {
+ location,
+ signature,
+ } => match identified
+ .signatures
+ .iter_mut()
+ .find(|(existing, _)| *existing == location)
+ {
+ Some(existing) => existing.1 = signature,
+ None => identified.signatures.push((location, signature)),
+ },
+ Found::Match { location, found } => match identified
+ .regexes
+ .iter_mut()
+ .find(|(existing, _)| *existing == location)
+ {
+ Some((_, matches)) => matches.push(found),
+ None => identified.regexes.push((location, vec![found])),
+ },
+ }
+}
+
+/// `progress_bars()`: bars on the terminal of stdout or, if stdout is a regular file
+/// (`what ... > out.txt`), on the terminal of stderr. Not into a pipe, as the program that
+/// reads it may be writing to the terminal too.
+fn progress_bars(env: &Environment, enabled: bool) -> Option<(progress::Bars, progress::Target)> {
+ let (target, stream) = bars_target(env, enabled)?;
+ // rich's width (COLUMNS), else the width of the terminal that the bars are on
+ let width = match env.var("COLUMNS").and_then(|columns| python::int(&columns)) {
+ Some(columns) if columns > 0 => env.rich_width(),
+ _ => stream.columns.filter(|&columns| columns > 0).unwrap_or(80),
+ };
+ let bars = progress::Bars::start(target, width, env.colors(stream));
+ Some((bars, target))
+}
+
+/// Where the progress bars go, if anywhere, and the stream that is there.
+fn bars_target(env: &Environment, enabled: bool) -> Option<(progress::Target, &StreamInfo)> {
+ if !enabled {
+ return None;
+ }
+ let (target, stream) = if env.stdout.regular_file {
+ (progress::Target::Stderr, &env.stderr)
+ } else {
+ (progress::Target::Stdout, &env.stdout)
+ };
+ let term = env.var("TERM").unwrap_or_default().to_lowercase();
+ if !stream.terminal || term == "dumb" || term == "unknown" {
+ return None;
+ }
+ Some((target, stream))
+}
+
+/// The end of a search: the files that couldn't be read, on stderr (exit code 1, which
+/// Python exits with when it can't read a file).
+fn finish(streams: &mut Streams<'_>, unreadable: &[String]) -> io::Result {
+ if unreadable.is_empty() {
+ return Ok(0);
+ }
+ streams.stdout.flush()?;
+ for message in unreadable {
+ writeln!(streams.stderr, "{message}")?;
+ }
+ Ok(1)
+}
+
+/// The options of the main part of the command.
+struct Command<'a> {
+ parsed: &'a args::Parsed,
+ inputs: Vec,
+ map: &'static str,
+}
+
+impl Command<'_> {
+ fn flag(&self, param: Param) -> bool {
+ self.parsed.flag(param)
+ }
+
+ fn value(&self, param: Param) -> Option<&str> {
+ self.parsed.value(param)
+ }
+}
+
+fn execute(
+ prog: &str,
+ args: &[String],
+ env: &Environment,
+ streams: &mut Streams<'_>,
+ verifier: &GoogleVerifier,
+) -> io::Result {
+ let parsed = match args::parse(args) {
+ Ok(parsed) => parsed,
+ Err(args::UsageError(message)) => return usage_error(prog, &message, env, streams),
+ };
+ // --help is eager: it comes first
+ if parsed.flag(Param::Help) {
+ let width = help::help_width(env.click_columns());
+ streams
+ .stdout
+ .write_all(help::help(prog, width).as_bytes())?;
+ return Ok(0);
+ }
+ // The callbacks of the options, in the order in which they were given
+ let colors = env.colors(&env.stdout);
+ let mut map = "google";
+ for ¶m in parsed.invoked() {
+ match param {
+ Param::Tags => {
+ print_tags(streams, colors)?;
+ return Ok(0);
+ }
+ Param::Names => {
+ print_names(streams, colors)?;
+ return Ok(0);
+ }
+ Param::Version => {
+ print_version(streams, colors)?;
+ return Ok(0);
+ }
+ Param::Map => match args::map_choice(parsed.value(Param::Map).unwrap_or("")) {
+ Ok(choice) => map = choice,
+ Err(args::UsageError(message)) => return usage_error(prog, &message, env, streams),
+ },
+ _ => {}
+ }
+ }
+ // get_text(): stdin is the input if there is none, unless it is a terminal
+ let mut inputs = parsed.inputs.clone();
+ if inputs.is_empty() && !parsed.flag(Param::Interactive) && !env.stdin.terminal {
+ inputs = vec![read_stdin(streams.stdin)?];
+ }
+ let command = Command {
+ parsed: &parsed,
+ inputs,
+ map,
+ };
+ main_command(&command, env, streams, verifier, colors)
+}
+
+/// `main()` of `what.py`, after the options have been parsed.
+fn main_command(
+ command: &Command<'_>,
+ env: &Environment,
+ streams: &mut Streams<'_>,
+ verifier: &GoogleVerifier,
+ colors: Colors,
+) -> io::Result {
+ let interactive = command.flag(Param::Interactive);
+ if command.inputs.is_empty() && !interactive {
+ return exit_with(streams, "Text input expected. Run 'pywhat --help' for help");
+ }
+ let dist = match create_filter(
+ Some(command.value(Param::Rarity).unwrap_or("0.1:1")),
+ command.value(Param::Include),
+ command.value(Param::Exclude),
+ ) {
+ Ok(filter) => filter,
+ Err(message) => {
+ print_line(streams, message)?;
+ return Ok(1);
+ }
+ };
+ let boundaryless = if command.flag(Param::DisableBoundaryless) {
+ Filter::none()
+ } else {
+ match create_filter(
+ Some(command.value(Param::BoundarylessRarity).unwrap_or("0.1:1")),
+ command.value(Param::BoundarylessInclude),
+ command.value(Param::BoundarylessExclude),
+ ) {
+ Ok(filter) => filter,
+ Err(message) => {
+ print_line(streams, message)?;
+ return Ok(1);
+ }
+ }
+ };
+ // The default processors (unless -dp), and the ones that options ask for
+ let mut processors: Vec> = if command.flag(Param::DisableProcessing) {
+ Vec::new()
+ } else {
+ default_processors()
+ };
+ if command.flag(Param::Verify) {
+ processors.push(Box::new(verifier.verifier()));
+ }
+ if command.map == "osm" {
+ processors.push(Box::new(OpenStreetMap));
+ }
+ let key = match command.value(Param::Key) {
+ None => Key::None,
+ Some(key) => match key.parse::() {
+ Ok(key) => key,
+ Err(()) => {
+ print_line(streams, "Invalid key")?;
+ return Ok(1);
+ }
+ },
+ };
+ let top = match command.value(Param::Top) {
+ None => None,
+ Some(top) => match parse_top(top) {
+ Ok(top) => Some(top),
+ Err(message) => {
+ print_line(streams, &message)?;
+ return Ok(1);
+ }
+ },
+ };
+ // str(kwargs["format"]).strip()
+ let format = command.value(Param::Format);
+ let format_name = python::strip(format.unwrap_or("None"));
+ let json_output = command.flag(Param::Json) || format_name == "json";
+ let stream = command.flag(Param::Stream);
+ if stream {
+ if key != Key::None {
+ return exit_with(
+ streams,
+ "--stream cannot sort the matches, so it cannot be used with --key",
+ );
+ }
+ if !json_output && format_name == "pretty" {
+ return exit_with(
+ streams,
+ "--stream cannot print a table, so it cannot be used with --format pretty",
+ );
+ }
+ if top.is_some() {
+ return exit_with(
+ streams,
+ "--stream cannot pick the most likely matches, so it cannot be used with --top",
+ );
+ }
+ if interactive {
+ return exit_with(streams, "--stream cannot be used with --interactive");
+ }
+ }
+ if interactive {
+ streams.stdout.flush()?;
+ writeln!(
+ streams.stderr,
+ "Error: --interactive is only in the Python package of pyWhat: pip install pywhat"
+ )?;
+ return Ok(2);
+ }
+
+ let identifier = Identifier::new(dist, boundaryless, processors);
+ let options = InputOptions {
+ only_text: command.flag(Param::OnlyText),
+ include_filenames: command.flag(Param::IncludeFilenames),
+ };
+ let reverse = command.flag(Param::Reverse);
+ let print_tags = command.flag(Param::PrintTags);
+ let bars = progress_bars(env, !command.flag(Param::NoProgress));
+ let mut unreadable: Vec = Vec::new();
+ let mut report = |location: &str, error: io::Error| {
+ unreadable.push(format!("Could not read {location}: {error}"));
+ };
+ let on_progress = |progress: Progress<'_>| {
+ if let Some((bars, _)) = &bars {
+ bars.update(progress);
+ }
+ };
+
+ if stream {
+ // The matches are printed as soon as they are found (issue #189)
+ let format = if json_output { None } else { format };
+ let show_files = output::show_files_of_inputs(&command.inputs);
+ let bars_on_stdout = matches!(&bars, Some((_, progress::Target::Stdout)));
+ let mut previous: Option<(bool, String)> = None;
+ let mut found_matches = false;
+ let mut failure: Option = None;
+ let mut printer = Printer {
+ out: &mut *streams.stdout,
+ colors,
+ };
+ let mut print_found = |found: Found| -> io::Result<()> {
+ if json_output {
+ printer.stream_json(&found)?;
+ return printer.out.flush();
+ }
+ match found {
+ Found::FileSignature {
+ location,
+ signature,
+ } => {
+ if format.is_none() {
+ let mut text = output::raw_signature(&location, signature);
+ text.rstrip();
+ printer.print(&text)?;
+ previous = Some((true, location));
+ }
+ }
+ Found::Match { location, found } => {
+ found_matches = true;
+ let one = Identified {
+ signatures: Vec::new(),
+ regexes: vec![(location.clone(), vec![found])],
+ };
+ match format {
+ Some(format) => printer.format_print(&one, format)?,
+ None => {
+ // An empty line between the matches, but not between a file
+ // signature and the matches in its file
+ if previous.is_some()
+ && previous.as_ref() != Some(&(true, location.clone()))
+ {
+ printer.print_plain("")?;
+ }
+ printer.print_raw(
+ &one,
+ &command.inputs,
+ print_tags,
+ Some(show_files),
+ )?;
+ }
+ }
+ previous = Some((false, location));
+ }
+ }
+ printer.out.flush()
+ };
+ identifier.iter_identify_with_progress(
+ &command.inputs,
+ options,
+ &mut |found| {
+ let printed = match &bars {
+ Some((bars, _)) if bars_on_stdout => bars.suspend(|| print_found(found)),
+ _ => print_found(found),
+ };
+ match printed {
+ Ok(()) => true,
+ Err(error) => {
+ failure = Some(error);
+ false
+ }
+ }
+ },
+ &mut report,
+ &on_progress,
+ );
+ if let Some((bars, _)) = bars {
+ bars.finish();
+ }
+ if let Some(error) = failure {
+ return Err(error);
+ }
+ if !json_output && !found_matches && format.is_none() {
+ Printer {
+ out: streams.stdout,
+ colors,
+ }
+ .print_plain("Nothing found!")?;
+ }
+ return finish(streams, &unreadable);
+ }
+
+ let mut identified = Identified::default();
+ identifier.iter_identify_with_progress(
+ &command.inputs,
+ options,
+ &mut |found| {
+ collect(&mut identified, found);
+ true
+ },
+ &mut report,
+ &on_progress,
+ );
+ if let Some((bars, _)) = bars {
+ bars.finish();
+ }
+ // top_matches() sorts the top ones itself
+ identified.sort(if top.is_none() { key } else { Key::None }, reverse);
+ let total = identified.located_matches().len();
+ if let Some(top) = &top {
+ identified = identified.top(top, key, reverse);
+ }
+ let mut printer = Printer {
+ out: streams.stdout,
+ colors,
+ };
+ if json_output {
+ printer.print_json(&identified)?;
+ } else if format_name == "pretty" {
+ printer.pretty_print(&identified, &command.inputs, print_tags, env.rich_width())?;
+ } else if let Some(format) = format {
+ printer.format_print(&identified, format)?;
+ } else {
+ printer.print_raw(&identified, &command.inputs, print_tags, None)?;
+ }
+ let shown = identified.located_matches().len();
+ if shown < total {
+ // On stderr, so that the output (e.g. JSON) is the same as without --top
+ streams.stdout.flush()?;
+ writeln!(
+ streams.stderr,
+ "\nShowing the {shown} most likely of {total} matches. Use --top to show more \
+ of them, or --interactive to search through all of them."
+ )?;
+ }
+ finish(streams, &unreadable)
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn terminal(columns: usize) -> StreamInfo {
+ StreamInfo {
+ terminal: true,
+ regular_file: false,
+ columns: Some(columns),
+ }
+ }
+
+ fn file() -> StreamInfo {
+ StreamInfo {
+ terminal: false,
+ regular_file: true,
+ columns: None,
+ }
+ }
+
+ fn pipe() -> StreamInfo {
+ StreamInfo::default()
+ }
+
+ fn env(stdout: StreamInfo, stderr: StreamInfo) -> Environment {
+ Environment {
+ vars: HashMap::new(),
+ stdin: pipe(),
+ stdout,
+ stderr,
+ }
+ }
+
+ #[test]
+ fn where_the_bars_go() {
+ use progress::Target;
+ let target = |env: &Environment| bars_target(env, true).map(|(target, _)| target);
+ // On the terminal of the output
+ assert_eq!(
+ target(&env(terminal(80), terminal(80))),
+ Some(Target::Stdout)
+ );
+ // On stderr if the output is redirected to a file
+ assert_eq!(target(&env(file(), terminal(80))), Some(Target::Stderr));
+ assert_eq!(target(&env(file(), file())), None);
+ // Never into a pipe, even if stderr is a terminal (what --json ... | jq)
+ assert_eq!(target(&env(pipe(), terminal(80))), None);
+ // --no-progress, and dumb terminals
+ assert_eq!(
+ bars_target(&env(terminal(80), terminal(80)), false).map(|t| t.0),
+ None
+ );
+ let mut dumb = env(terminal(80), terminal(80));
+ dumb.vars.insert("TERM", "dumb".into());
+ assert_eq!(target(&dumb), None);
+ }
+
+ #[test]
+ fn widths_like_click_and_rich() {
+ let mut env = env(terminal(120), pipe());
+ assert_eq!(env.click_columns(), 120);
+ assert_eq!(env.rich_width(), 120);
+ env.vars.insert("COLUMNS", "abc".into());
+ assert_eq!((env.click_columns(), env.rich_width()), (120, 120));
+ env.vars.insert("COLUMNS", "0".into());
+ assert_eq!((env.click_columns(), env.rich_width()), (120, 80));
+ env.vars.insert("COLUMNS", " 40".into());
+ assert_eq!((env.click_columns(), env.rich_width()), (40, 120));
+ env.vars.insert("COLUMNS", "200".into());
+ assert_eq!((env.click_columns(), env.rich_width()), (200, 200));
+ assert_eq!(program_name("/usr/local/bin/what"), "what");
+ }
+
+ /// Runs the command with `stdin` and returns (exit code, stdout, stderr).
+ fn run_with(args: &[&str], stdin: &[u8], verifier: &GoogleVerifier) -> (i32, String, String) {
+ let args: Vec = args.iter().map(|arg| arg.to_string()).collect();
+ let env = env(pipe(), pipe());
+ let (mut stdin, mut stdout, mut stderr) = (stdin, Vec::new(), Vec::new());
+ let mut streams = Streams {
+ stdin: &mut stdin,
+ stdout: &mut stdout,
+ stderr: &mut stderr,
+ };
+ let code = run("what", &args, &env, &mut streams, verifier);
+ (
+ code,
+ String::from_utf8(stdout).unwrap(),
+ String::from_utf8(stderr).unwrap(),
+ )
+ }
+
+ fn what(args: &[&str]) -> (i32, String, String) {
+ run_with(args, b"", &GoogleVerifier::default())
+ }
+
+ #[test]
+ fn exit_codes_and_messages() {
+ assert_eq!(what(&["-db", "THM{x}"]).0, 0);
+ assert_eq!(
+ what(&["--rarity", "a:", "x"]),
+ (
+ 1,
+ "Invalid rarity argument (float expected)\n".into(),
+ String::new()
+ )
+ );
+ assert_eq!(what(&["--key", "x", "y"]).1, "Invalid key\n");
+ let (code, _, stderr) = what(&["--stream", "--key", "name", "x"]);
+ assert_eq!(code, 1);
+ assert_eq!(
+ stderr,
+ "--stream cannot sort the matches, so it cannot be used with --key\n"
+ );
+ let (code, stdout, stderr) = what(&["--interactive", "x"]);
+ assert_eq!((code, stdout.as_str()), (2, ""));
+ assert!(stderr.contains("pip install pywhat"), "{stderr}");
+ let (code, _, stderr) = what(&["--map", "bing", "x"]);
+ assert_eq!(code, 2);
+ assert!(stderr.ends_with(
+ "Error: Invalid value for '--map': 'bing' is not one of 'google', 'osm'.\n"
+ ));
+ }
+
+ #[test]
+ fn stdin_is_the_input_without_inputs() {
+ let (code, stdout, _) = run_with(&["-db"], b"THM{hello}\n", &GoogleVerifier::default());
+ assert_eq!(code, 0);
+ assert!(stdout.starts_with("Matched on: THM{hello}\n"), "{stdout}");
+ let (_, stdout, _) = run_with(&["-db"], b"", &GoogleVerifier::default());
+ assert_eq!(stdout, "Nothing found!\n");
+ }
+
+ #[test]
+ fn verify_with_a_fake_google() {
+ use std::sync::atomic::{AtomicUsize, Ordering};
+ use std::sync::Arc;
+ let requests = Arc::new(AtomicUsize::new(0));
+ let counter = requests.clone();
+ let verifier = GoogleVerifier {
+ make: Some(Box::new(move || {
+ let counter = counter.clone();
+ verify::GoogleApiKeyVerifier::with_http(Box::new(move |_| {
+ counter.fetch_add(1, Ordering::SeqCst);
+ Ok((
+ 400,
+ br#"{"error": {"details": [{"@type": "type.googleapis.com/google.rpc.ErrorInfo", "reason": "API_KEY_INVALID"}]}}"#.to_vec(),
+ ))
+ }))
+ })),
+ };
+ let key = format!("AIzaSy{}", "A00000000000000000000000000000000");
+ let text = format!("{key} {key}");
+ let (code, stdout, _) =
+ run_with(&["--verify", "--format", "%n: %d", &text], b"", &verifier);
+ assert_eq!(code, 0);
+ assert_eq!(
+ stdout
+ .lines()
+ .filter(|line| line.starts_with("Google API Key"))
+ .count(),
+ 2,
+ "{stdout}"
+ );
+ assert!(stdout.contains("Verification: invalid, Google rejected the key (API_KEY_INVALID)"));
+ // Once per key, however many times it is found
+ assert_eq!(requests.load(Ordering::SeqCst), 1);
+ // Without --verify, nothing is sent
+ let (_, stdout, _) = run_with(&["--format", "%d", &key], b"", &verifier);
+ assert!(!stdout.contains("Verification"));
+ assert_eq!(requests.load(Ordering::SeqCst), 1);
+ }
+}
diff --git a/src/cli/output.rs b/src/cli/output.rs
new file mode 100644
index 0000000..7ea2c83
--- /dev/null
+++ b/src/cli/output.rs
@@ -0,0 +1,382 @@
+//! The output formats, ports of `pywhat/printer.py`: raw output, the `--format pretty`
+//! table, custom `--format` strings, `--json`, and `--stream`.
+
+use super::python;
+use super::style::{sanitized, Colors, Text, BOLD, LABEL};
+use crate::files::{Found, Identified};
+use crate::identify::Match;
+use crate::json::dumps;
+use serde_json::{Map, Value};
+use std::io::{self, Write};
+use std::path::Path;
+
+/// `str(value)` of a value of a file signature.
+fn py_str(value: Option<&Value>) -> String {
+ match value {
+ Some(Value::String(text)) => text.clone(),
+ Some(Value::Null) | None => "None".to_string(),
+ Some(Value::Bool(true)) => "True".to_string(),
+ Some(Value::Bool(false)) => "False".to_string(),
+ Some(other) => dumps(other),
+ }
+}
+
+/// `str(match["Regex Pattern"]["Rarity"])`: as the regex has it (`1` or `0.5`), or the
+/// float a processor gave it.
+fn rarity_text(found: &Match) -> String {
+ match found.regex.to_json_value().get("Rarity") {
+ Some(rarity) if found.rarity == found.regex.rarity => dumps(rarity),
+ _ => dumps(&Value::from(found.rarity)),
+ }
+}
+
+/// Whether the regex of a match has an `"Exploit"` key, even a null one, which makes
+/// `--format pretty` show the Exploit column (`_check_if_exploit_in_json`).
+fn has_exploit_key(found: &Match) -> bool {
+ found.regex.to_json_value().contains_key("Exploit")
+}
+
+fn non_empty(text: &Option) -> Option<&str> {
+ text.as_deref().filter(|text| !text.is_empty())
+}
+
+/// `_show_files()`: whether to show the file of every match: the input is a directory, or
+/// there are several inputs and some of the matches are in files.
+pub(crate) fn show_files(identified: &Identified, inputs: &[String]) -> bool {
+ if inputs.len() == 1 {
+ return Path::new(&inputs[0]).is_dir();
+ }
+ identified
+ .regexes
+ .iter()
+ .any(|(location, _)| location != "text")
+}
+
+/// `_show_files_of_inputs()`: like [`show_files`] before anything is found, for
+/// `--stream`: the input is a directory, or one of several inputs exists.
+pub(crate) fn show_files_of_inputs(inputs: &[String]) -> bool {
+ if inputs.len() == 1 {
+ return Path::new(&inputs[0]).is_dir();
+ }
+ inputs.iter().any(|input| Path::new(input).exists())
+}
+
+/// `_raw_signature()`: a file signature as raw output shows it.
+pub(crate) fn raw_signature(location: &str, signature: &Map) -> Text {
+ let mut text = Text::new();
+ text.plain("\n")
+ .label("File Identified")
+ .plain(": ")
+ .styled(location, BOLD)
+ .plain(" with Magic Numbers ")
+ .markup(&py_str(signature.get("ISO 8859-1")))
+ .plain(".\n")
+ .label("File Description:")
+ .plain(" ")
+ .markup(&py_str(signature.get("Description")))
+ .plain(".\n");
+ text
+}
+
+/// Where the output goes, and how its styles are written.
+pub(crate) struct Printer<'a> {
+ pub(crate) out: &'a mut dyn Write,
+ pub(crate) colors: Colors,
+}
+
+impl Printer<'_> {
+ /// `console.print(text)`: the text and a line break.
+ pub(crate) fn print(&mut self, text: &Text) -> io::Result<()> {
+ let mut rendered = text.render(self.colors);
+ rendered.push('\n');
+ self.out.write_all(rendered.as_bytes())
+ }
+
+ /// `console.print(text, markup=False)` of plain text.
+ pub(crate) fn print_plain(&mut self, text: &str) -> io::Result<()> {
+ let mut plain = Text::new();
+ plain.plain(text);
+ self.print(&plain)
+ }
+
+ /// `print_raw()`, with `show_files` given or found out.
+ pub(crate) fn print_raw(
+ &mut self,
+ identified: &Identified,
+ inputs: &[String],
+ print_tags: bool,
+ show_files: Option,
+ ) -> io::Result<()> {
+ let mut output = Text::new();
+ if !identified.signatures.is_empty() && !identified.regexes.is_empty() {
+ for (location, signature) in &identified.signatures {
+ output.append(raw_signature(location, signature));
+ }
+ }
+ if !identified.regexes.is_empty() {
+ let show_files = show_files.unwrap_or_else(|| self::show_files(identified, inputs));
+ for (location, matches) in &identified.regexes {
+ for found in matches {
+ if show_files {
+ output.label("File: ").styled(location, LABEL).plain("\n");
+ }
+ output.label("Matched on: ").plain(&found.matched);
+ output.plain("\n").label("Name: ").markup(&found.regex.name);
+ if let Some(link) = found.link() {
+ output.plain("\n").label("Link: ").plain(" ").plain(&link);
+ }
+ if let Some(description) = non_empty(&found.description) {
+ output
+ .plain("\n")
+ .label("Description: ")
+ .markup(description);
+ }
+ if let Some(exploit) = non_empty(&found.exploit) {
+ output.plain("\n").label("Exploit: ").plain(exploit);
+ }
+ if print_tags {
+ output
+ .plain("\n")
+ .label("Tags: ")
+ .markup(&found.tags.join(", "));
+ }
+ output.plain("\n\n");
+ }
+ }
+ }
+ let source = output.source();
+ if source.is_empty() {
+ self.print_plain("Nothing found!")?;
+ }
+ if !python::strip(&source).is_empty() {
+ output.rstrip();
+ self.print(&output)?;
+ }
+ Ok(())
+ }
+
+ /// `format_print()`: `format` for every match, with `%m`, `%n`, `%d`, `%e`, `%r`, `%l`
+ /// and `%t` replaced, one after another, and `\\` and `\%` for `\` and `%`.
+ pub(crate) fn format_print(&mut self, identified: &Identified, format: &str) -> io::Result<()> {
+ if identified.regexes.is_empty() {
+ return Ok(());
+ }
+ // The format split at its "\\", which become "\" (format_list)
+ let mut parts: Vec<&str> = Vec::new();
+ let mut rest = format;
+ while let Some(index) = rest.find("\\\\") {
+ if index > 0 {
+ parts.push(&rest[..index]);
+ }
+ parts.push("\\");
+ rest = &rest[index + 2..];
+ }
+ parts.push(rest);
+ // The output as Python gives it to rich: markup, which the format can use too. The
+ // values that contain the matched text are escaped, so that they are printed as
+ // they are (#291, Python bug 4); the names, descriptions and tags are data, whose
+ // markup rich renders.
+ let mut source = Vec::new();
+ for (_, matches) in &identified.regexes {
+ for found in matches {
+ let values = [
+ ("%m", super::markup::escape(&found.matched)),
+ ("%n", found.regex.name.clone()),
+ ("%d", found.description.clone().unwrap_or_default()),
+ (
+ "%e",
+ super::markup::escape(found.exploit.as_deref().unwrap_or("")),
+ ),
+ ("%r", rarity_text(found)),
+ (
+ "%l",
+ super::markup::escape(&found.link().unwrap_or_default()),
+ ),
+ ("%t", found.tags.join(", ")),
+ ];
+ source.push(substitute(&parts, &values));
+ }
+ }
+ let source = source.join("\n");
+ if !python::strip(&source).is_empty() {
+ let mut text = Text::new();
+ text.markup(&source);
+ self.print(&text)?;
+ }
+ Ok(())
+ }
+
+ /// `print_json()`: `json.dumps()` of everything found, on one line.
+ pub(crate) fn print_json(&mut self, identified: &Identified) -> io::Result<()> {
+ let mut json = identified.to_json();
+ json.push('\n');
+ self.out.write_all(json.as_bytes())
+ }
+
+ /// `print_stream()` with `json_output`: a JSON line for everything found, as soon as it
+ /// is found, without the "Fragment" of the matches.
+ pub(crate) fn stream_json(&mut self, found: &Found) -> io::Result<()> {
+ let mut line = String::new();
+ match found {
+ Found::FileSignature {
+ location,
+ signature,
+ } => {
+ line.push_str("{\"File Signatures\": {");
+ line.push_str(&dumps(&Value::String(location.clone())));
+ line.push_str(": ");
+ line.push_str(&dumps(&Value::Object((*signature).clone())));
+ line.push_str("}, \"Regexes\": null}");
+ }
+ Found::Match { location, found } => {
+ line.push_str("{\"File Signatures\": null, \"Regexes\": {");
+ line.push_str(&dumps(&Value::String(location.clone())));
+ line.push_str(": [");
+ found.write_json(&mut line, false);
+ line.push_str("]}}");
+ }
+ }
+ self.print_plain(&line)
+ }
+
+ /// `--format pretty`: the file signatures, and a table of the matches.
+ pub(crate) fn pretty_print(
+ &mut self,
+ identified: &Identified,
+ inputs: &[String],
+ print_tags: bool,
+ width: usize,
+ ) -> io::Result<()> {
+ let mut text = Text::new();
+ for (location, signature) in &identified.signatures {
+ text.plain("\n")
+ .label("File Identified")
+ .plain(": ")
+ .styled(location, BOLD)
+ .plain(" with Magic Numbers ")
+ .markup(&py_str(signature.get("ISO 8859-1")))
+ .plain(".\n")
+ .label("File Description: ")
+ .plain(" ")
+ .markup(&py_str(signature.get("Description")))
+ .plain(".\n\n");
+ }
+ if identified.regexes.is_empty() {
+ text.plain("\nNothing found!");
+ text.lstrip();
+ return self.print(&text);
+ }
+ text.plain("\n")
+ .label("Possible Identification")
+ .plain("\n");
+ let show_files = show_files(identified, inputs);
+ let bug_bounty = identified
+ .regexes
+ .iter()
+ .flat_map(|(_, matches)| matches)
+ .any(has_exploit_key);
+ let mut header = vec!["Matched Text", "Identified as", "Description"];
+ if show_files {
+ header.push("File");
+ }
+ if bug_bounty {
+ header.push("Exploit");
+ }
+ let mut rows: Vec> = Vec::new();
+ for (location, matches) in &identified.regexes {
+ for found in matches {
+ let mut description = Text::new();
+ if let Some(link) = found.link() {
+ description
+ .plain("Click here to analyse in the browser\n")
+ .plain(&link);
+ }
+ if let Some(text) = non_empty(&found.description) {
+ if !description.source().is_empty() {
+ description.plain("\n");
+ }
+ description.markup(text);
+ }
+ if print_tags {
+ if !description.source().is_empty() {
+ description.plain("\n");
+ }
+ description.plain("Tags: ").markup(&found.tags.join(", "));
+ }
+ let description = if description.source().is_empty() {
+ "None".to_string()
+ } else {
+ description.to_plain()
+ };
+ let mut row = vec![
+ sanitized(&found.matched),
+ sanitized(&super::markup::plain(&found.regex.name)),
+ sanitized(&description),
+ ];
+ if show_files {
+ row.push(sanitized(location));
+ }
+ if bug_bounty {
+ row.push(sanitized(non_empty(&found.exploit).unwrap_or("")));
+ }
+ rows.push(row);
+ }
+ }
+ text.strip();
+ let mut out = text.render(self.colors);
+ out.push('\n');
+ out.push_str(&super::table::render(&header, &rows, width, self.colors));
+ out.push('\n');
+ self.out.write_all(out.as_bytes())
+ }
+}
+
+/// The substitutions of `format_print()` in each part of the format: `re.sub(r"(? String {
+ let mut out = String::new();
+ for part in parts {
+ let mut text = part.to_string();
+ for (format, value) in values {
+ text = replace_unescaped(&text, format, value);
+ }
+ out.push_str(&text.replace("\\%", "%"));
+ }
+ out
+}
+
+/// `re.sub(r"(? String {
+ let mut out = String::with_capacity(text.len());
+ let mut previous: Option = None;
+ let mut index = 0;
+ while index < text.len() {
+ if text[index..].starts_with(pattern) && previous != Some('\\') {
+ out.push_str(value);
+ index += pattern.len();
+ previous = pattern.chars().last();
+ continue;
+ }
+ let c = text[index..].chars().next().expect("a character");
+ out.push(c);
+ previous = Some(c);
+ index += c.len_utf8();
+ }
+ out
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn substitutions_like_re_sub() {
+ assert_eq!(replace_unescaped("%m\\%m%m", "%m", "X"), "X\\%mX");
+ // The value of %m is searched for the next formats, as Python does
+ let values = [("%m", "thm{%n}".to_string()), ("%n", "Name".to_string())];
+ assert_eq!(substitute(&["%m"], &values), "thm{Name}");
+ assert_eq!(substitute(&["\\%d %m"], &values), "%d thm{Name}");
+ assert_eq!(substitute(&["\\", "%m"], &values), "\\thm{Name}");
+ }
+}
diff --git a/src/cli/progress.rs b/src/cli/progress.rs
new file mode 100644
index 0000000..3d715f8
--- /dev/null
+++ b/src/cli/progress.rs
@@ -0,0 +1,372 @@
+//! Progress bars for searches that take a while, like `pywhat/progress.py`: a bar for the
+//! files (if there are several) with the file that is searched, and a bar for the regexes
+//! of the file with the regex that is searched for. They appear once the search has taken
+//! half a second, only on a terminal, and are removed when the search is complete.
+//! indicatif draws them, in the look of rich's progress bars.
+
+use super::style::{write_styled, ColorSystem, Colors, Style, LABEL};
+use crate::files::Progress;
+use indicatif::{MultiProgress, ProgressBar, ProgressDrawTarget, ProgressStyle};
+use std::sync::{Arc, Condvar, Mutex};
+use std::thread::JoinHandle;
+use std::time::{Duration, Instant};
+
+/// How long a search runs before the bars appear, so that they don't flicker for searches
+/// that are over in a moment.
+pub(crate) const DELAY: Duration = Duration::from_millis(500);
+/// How often the bars are drawn.
+const REFRESH: Duration = Duration::from_millis(100);
+const BAR_WIDTH: usize = 30;
+
+/// rich's `bar.complete` and `bar.finished` styles.
+const COMPLETE: (u8, u8, u8) = (249, 38, 114);
+const FINISHED: (u8, u8, u8) = (114, 156, 31);
+
+/// Where the bars are drawn.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub(crate) enum Target {
+ Stdout,
+ Stderr,
+}
+
+/// The latest progress.
+#[derive(Debug, Default, Clone)]
+struct State {
+ seen: bool,
+ location: String,
+ files_done: usize,
+ files: usize,
+ regexes_done: usize,
+ regexes: usize,
+ regex: Option<&'static str>,
+}
+
+/// Running progress bars, the progress callback of the search.
+pub(crate) struct Bars {
+ state: Arc>,
+ multi: MultiProgress,
+ done: Arc<(Mutex, Condvar)>,
+ ticker: Option>,
+}
+
+/// Control characters (e.g. in file names) would mess up the terminal.
+fn printable(text: &str) -> String {
+ text.chars()
+ .filter(|&c| !matches!(c, '\u{0}'..='\u{1f}' | '\u{7f}'..='\u{9f}'))
+ .collect()
+}
+
+/// `str(timedelta(seconds=...))`: H:MM:SS.
+fn elapsed(seconds: u64) -> String {
+ format!(
+ "{}:{:02}:{:02}",
+ seconds / 3600,
+ seconds / 60 % 60,
+ seconds % 60
+ )
+}
+
+fn styled(text: &str, style: &Style, colors: Colors) -> String {
+ let mut out = String::new();
+ write_styled(&mut out, text, style, colors);
+ out
+}
+
+/// `rich.progress_bar.ProgressBar` of `completed` of `total`, `width` characters wide.
+fn bar(completed: usize, total: usize, width: usize, colors: Colors) -> String {
+ let completed = completed.min(total);
+ let halves = (width * 2 * completed)
+ .checked_div(total)
+ .unwrap_or(width * 2);
+ let (bars, half) = (halves / 2, halves % 2);
+ let color = if completed < total {
+ COMPLETE
+ } else {
+ FINISHED
+ };
+ let complete = Style {
+ color: Some(color),
+ ..Style::default()
+ };
+ let mut out = styled(&"━".repeat(bars), &complete, colors);
+ if half > 0 {
+ out.push_str(&styled("╸", &complete, colors));
+ }
+ let mut remaining = width - bars - half;
+ if colors.no_color || colors.is_plain() {
+ // rich only draws the rest of the bar in colour
+ out.push_str(&" ".repeat(remaining));
+ return out;
+ }
+ let mut back = String::new();
+ if half == 0 && bars > 0 && remaining > 0 {
+ back.push('╺');
+ remaining -= 1;
+ }
+ back.push_str(&"━".repeat(remaining));
+ if back.is_empty() {
+ return out;
+ }
+ // grey23, an 8-bit colour
+ let code = match colors.system {
+ ColorSystem::Standard => "90".to_string(),
+ _ => "38;5;237".to_string(),
+ };
+ out.push_str(&format!("\x1b[{code}m{back}\x1b[0m"));
+ out
+}
+
+/// `text` cut to `width` characters with an ellipsis, like rich's `overflow="ellipsis"`.
+fn ellipsis(text: &str, width: usize) -> String {
+ if text.chars().count() <= width {
+ return text.to_string();
+ }
+ if width == 0 {
+ return String::new();
+ }
+ let mut cut: String = text.chars().take(width - 1).collect();
+ cut.push('…');
+ cut
+}
+
+/// The lines of the bars (`ProgressBars.render()`): a grid of the label, the bar, how far
+/// it is, the time since the search started and the file or the regex, one space between
+/// the columns, and only the last column cut if the lines are wider than the terminal.
+fn lines(state: &State, seconds: u64, width: usize, colors: Colors) -> Vec {
+ if !state.seen {
+ return vec![styled("Searching...", &LABEL, colors)];
+ }
+ // Narrower bars on a narrow terminal, to leave room for the names
+ let bar_width = (width / 4).clamp(10, BAR_WIDTH);
+ let time = elapsed(seconds);
+ // One regex is 1% if there are 100 regexes to search for in the file
+ let total = state.regexes.max(1);
+ let done = if state.regexes > 0 {
+ state.regexes_done
+ } else {
+ 1
+ };
+ let mut rows: Vec<(&str, usize, usize, String, String, String)> = Vec::new();
+ if state.files > 1 {
+ rows.push((
+ "Files",
+ state.files_done,
+ state.files,
+ format!("{}/{}", state.files_done, state.files),
+ time.clone(),
+ printable(&state.location),
+ ));
+ }
+ rows.push((
+ "Regexes",
+ done,
+ total,
+ format!("{}%", 100 * done / total),
+ if state.files > 1 { String::new() } else { time },
+ printable(state.regex.unwrap_or("")),
+ ));
+ let label_width = rows.iter().map(|row| row.0.len()).max().unwrap_or(0);
+ let count_width = rows.iter().map(|row| row.3.len()).max().unwrap_or(0);
+ let time_width = rows.iter().map(|row| row.4.len()).max().unwrap_or(0);
+ let used = label_width + bar_width + count_width + time_width + 4;
+ let name_width = width.saturating_sub(used + 1).max(1);
+ rows.into_iter()
+ .map(|(label, completed, total, count, time, name)| {
+ let mut line = styled(&format!("{label:count_width$} {time: Bars {
+ let state = Arc::new(Mutex::new(State::default()));
+ let done = Arc::new((Mutex::new(false), Condvar::new()));
+ let multi = MultiProgress::with_draw_target(ProgressDrawTarget::hidden());
+ let ticker = {
+ let (state, done, multi) = (state.clone(), done.clone(), multi.clone());
+ std::thread::spawn(move || {
+ let (lock, condvar) = &*done;
+ let wait = |timeout| {
+ let finished = lock.lock().unwrap_or_else(|e| e.into_inner());
+ let (finished, _) = condvar
+ .wait_timeout_while(finished, timeout, |finished| !*finished)
+ .unwrap_or_else(|e| e.into_inner());
+ *finished
+ };
+ let started = Instant::now();
+ if wait(DELAY) {
+ return;
+ }
+ multi.set_draw_target(match target {
+ Target::Stdout => ProgressDrawTarget::stdout(),
+ Target::Stderr => ProgressDrawTarget::stderr(),
+ });
+ let style = ProgressStyle::with_template("{msg}").expect("valid template");
+ let mut shown: Vec = Vec::new();
+ loop {
+ let latest = state.lock().unwrap_or_else(|e| e.into_inner()).clone();
+ let lines = lines(&latest, started.elapsed().as_secs(), width, colors);
+ while shown.len() < lines.len() {
+ // A files bar appears above the regexes bar
+ let bar = ProgressBar::new(1).with_style(style.clone());
+ shown.insert(
+ 0,
+ match shown.first() {
+ Some(first) => multi.insert_before(first, bar),
+ None => multi.add(bar),
+ },
+ );
+ }
+ let offset = shown.len() - lines.len();
+ for (bar, line) in shown[offset..].iter().zip(lines) {
+ bar.set_message(line);
+ }
+ if wait(REFRESH) {
+ break;
+ }
+ }
+ for bar in &shown {
+ bar.finish_and_clear();
+ }
+ let _ = multi.clear();
+ })
+ };
+ Bars {
+ state,
+ multi,
+ done,
+ ticker: Some(ticker),
+ }
+ }
+
+ /// The progress callback: keeps the latest progress, which is drawn 10 times a second.
+ pub(crate) fn update(&self, progress: Progress<'_>) {
+ let mut state = self.state.lock().unwrap_or_else(|e| e.into_inner());
+ state.seen = true;
+ if state.location != progress.location {
+ state.location = progress.location.to_string();
+ }
+ state.files_done = progress.files_done;
+ state.files = progress.files;
+ state.regexes_done = progress.regexes_done;
+ state.regexes = progress.regexes;
+ state.regex = progress.regex;
+ }
+
+ /// Runs `print` with the bars hidden, so that its output goes above them.
+ pub(crate) fn suspend(&self, print: impl FnOnce() -> R) -> R {
+ self.multi.suspend(print)
+ }
+
+ /// Removes the bars.
+ pub(crate) fn finish(mut self) {
+ self.stop();
+ }
+
+ fn stop(&mut self) {
+ let (lock, condvar) = &*self.done;
+ *lock.lock().unwrap_or_else(|e| e.into_inner()) = true;
+ condvar.notify_all();
+ if let Some(ticker) = self.ticker.take() {
+ let _ = ticker.join();
+ }
+ }
+}
+
+impl Drop for Bars {
+ fn drop(&mut self) {
+ self.stop();
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn state(files: usize, regexes_done: usize, regex: &'static str) -> State {
+ State {
+ seen: true,
+ location: "fixtures/test/file".into(),
+ files_done: 1,
+ files,
+ regexes_done,
+ regexes: 150,
+ regex: Some(regex),
+ }
+ }
+
+ #[test]
+ fn lines_like_rich() {
+ let plain = Colors::PLAIN;
+ assert_eq!(lines(&State::default(), 0, 80, plain), ["Searching..."]);
+ // One file: the regexes bar with the time; 80 columns give 20-character bars
+ assert_eq!(
+ lines(
+ &state(1, 75, "Uniform Resource Locator (URL)"),
+ 3,
+ 80,
+ plain
+ ),
+ [format!(
+ "Regexes {}{} 50% 0:00:03 Uniform Resource Locator (URL)",
+ "━".repeat(10),
+ " ".repeat(10)
+ )]
+ );
+ // Several files: the files bar first, with the time, and the regexes bar
+ let two = lines(&state(2, 150, "URL"), 61, 80, plain);
+ assert_eq!(
+ two,
+ [
+ format!(
+ "Files {}{} 1/2 0:01:01 fixtures/test/file",
+ "━".repeat(10),
+ " ".repeat(10)
+ ),
+ format!("Regexes {} 100% URL", "━".repeat(20)),
+ ]
+ );
+ // A name that doesn't fit is cut with an ellipsis, one column short of the width
+ let narrow = lines(&state(1, 0, "Uniform Resource Locator (URL)"), 0, 50, plain);
+ assert_eq!(narrow[0].chars().count(), 49);
+ assert!(narrow[0].ends_with(" Uniform Resource…"), "{narrow:?}");
+ }
+
+ #[test]
+ fn rich_bar() {
+ let colors = Colors {
+ system: ColorSystem::TrueColor,
+ no_color: false,
+ };
+ assert_eq!(
+ bar(1, 4, 4, colors),
+ "\x1b[38;2;249;38;114m━\x1b[0m\x1b[38;5;237m╺━━\x1b[0m"
+ );
+ assert_eq!(bar(3, 8, 4, colors).matches('╸').count(), 1);
+ assert_eq!(bar(4, 4, 4, colors), "\x1b[38;2;114;156;31m━━━━\x1b[0m");
+ }
+
+ #[test]
+ fn bars_finish_before_the_delay() {
+ // Nothing is drawn, and finishing doesn't wait for the delay
+ let started = Instant::now();
+ let bars = Bars::start(Target::Stderr, 80, Colors::PLAIN);
+ bars.update(Progress {
+ location: "text",
+ files_done: 0,
+ files: 1,
+ regexes_done: 0,
+ regexes: 10,
+ regex: Some("URL"),
+ });
+ bars.finish();
+ assert!(started.elapsed() < DELAY);
+ assert_eq!(printable("a\x1b[1mb\u{85}"), "a[1mb");
+ }
+}
diff --git a/src/cli/python.rs b/src/cli/python.rs
new file mode 100644
index 0000000..6a38e22
--- /dev/null
+++ b/src/cli/python.rs
@@ -0,0 +1,320 @@
+//! What Python's built-ins do with strings, where the CLI has to do the same: `str.strip()`,
+//! `float()`, `int()`, `repr()` and `str.casefold()`.
+
+use std::fmt::Write;
+use std::sync::OnceLock;
+
+/// `str.isspace()` of one character: Rust's `char::is_whitespace` and U+001C to U+001F,
+/// which Python counts as whitespace too.
+pub(crate) fn is_space(c: char) -> bool {
+ c.is_whitespace() || ('\u{1c}'..='\u{1f}').contains(&c)
+}
+
+/// `str.strip()`.
+pub(crate) fn strip(text: &str) -> &str {
+ text.trim_matches(is_space)
+}
+
+/// `str.rstrip()`.
+pub(crate) fn rstrip(text: &str) -> &str {
+ text.trim_end_matches(is_space)
+}
+
+/// `str.lstrip()`.
+pub(crate) fn lstrip(text: &str) -> &str {
+ text.trim_start_matches(is_space)
+}
+
+/// `str.isspace()`: not empty, and only whitespace.
+pub(crate) fn is_all_space(text: &str) -> bool {
+ !text.is_empty() && text.chars().all(is_space)
+}
+
+fn unicode_class(
+ pattern: &'static str,
+ cell: &'static OnceLock,
+) -> &'static regex::Regex {
+ cell.get_or_init(|| regex::Regex::new(pattern).expect("valid class"))
+}
+
+/// The value of a decimal digit (Unicode category Nd), like `unicodedata.decimal()`.
+/// Decimal digits are encoded in runs of ten, 0 to 9, so the value is the position in
+/// the run.
+pub(crate) fn decimal_value(c: char) -> Option {
+ if c.is_ascii_digit() {
+ return c.to_digit(10);
+ }
+ static DIGIT: OnceLock = OnceLock::new();
+ let digit = unicode_class(r"^\p{Nd}$", &DIGIT);
+ let is_digit = |c: char| digit.is_match(c.encode_utf8(&mut [0; 4]));
+ if !is_digit(c) {
+ return None;
+ }
+ let mut before = 0;
+ let mut code = c as u32;
+ while let Some(previous) = code.checked_sub(1).and_then(char::from_u32) {
+ if !is_digit(previous) {
+ break;
+ }
+ before += 1;
+ code -= 1;
+ }
+ Some(before % 10)
+}
+
+/// `_PyUnicode_TransformDecimalAndSpaceToASCII()`, which `float()` and `int()` start
+/// with: non-ASCII whitespace becomes a space and non-ASCII decimal digits ASCII digits.
+/// `None` if there is any other non-ASCII character, which makes the number invalid.
+fn transform_decimal_and_space(text: &str) -> Option {
+ let mut out = String::with_capacity(text.len());
+ for c in text.chars() {
+ if (c as u32) < 127 {
+ out.push(c);
+ } else if is_space(c) {
+ out.push(' ');
+ } else {
+ out.push(char::from_digit(decimal_value(c)?, 10)?);
+ }
+ }
+ Some(out)
+}
+
+/// `_Py_string_to_number_with_underscores()`: the text without underscores, which are
+/// only allowed between two digits.
+fn remove_underscores(text: &str) -> Option {
+ if !text.contains('_') {
+ return Some(text.to_string());
+ }
+ let mut out = String::with_capacity(text.len());
+ let mut previous = '\0';
+ for c in text.chars() {
+ if c == '_' {
+ if !previous.is_ascii_digit() {
+ return None;
+ }
+ } else {
+ if previous == '_' && !c.is_ascii_digit() {
+ return None;
+ }
+ out.push(c);
+ }
+ previous = c;
+ }
+ (previous != '_').then_some(out)
+}
+
+/// The whitespace that `float()` and `int()` strip (`Py_ISSPACE`), after the transform.
+fn is_ascii_space(c: char) -> bool {
+ matches!(c, ' ' | '\t' | '\n' | '\x0b' | '\x0c' | '\r')
+}
+
+/// `float(text)`, or `None` where Python raises `ValueError`.
+pub(crate) fn float(text: &str) -> Option {
+ let ascii = transform_decimal_and_space(text)?;
+ let without_underscores = remove_underscores(&ascii)?;
+ let number = without_underscores.trim_matches(is_ascii_space);
+ // Python and Rust read the same decimal numbers, infinities and NaNs (in any case),
+ // both correctly rounded; Rust doesn't take a NUL or anything after the number either
+ number.parse::().ok()
+}
+
+/// `int(text)`, or `None` where Python raises `ValueError`. Saturates at the limits of
+/// `i128`, which is enough for comparisons.
+pub(crate) fn int(text: &str) -> Option {
+ let ascii = transform_decimal_and_space(text)?;
+ let number = ascii.trim_matches(is_ascii_space);
+ let (negative, digits) = match number.as_bytes().first()? {
+ b'-' => (true, &number[1..]),
+ b'+' => (false, &number[1..]),
+ _ => (false, number),
+ };
+ if digits.is_empty() || digits.starts_with('_') {
+ return None;
+ }
+ let digits = remove_underscores(digits)?;
+ if digits.is_empty() || !digits.bytes().all(|b| b.is_ascii_digit()) {
+ return None;
+ }
+ let mut value: i128 = 0;
+ for b in digits.bytes() {
+ value = value
+ .saturating_mul(10)
+ .saturating_add(i128::from(b - b'0'));
+ }
+ Some(if negative { -value } else { value })
+}
+
+/// `str.isprintable()` of a non-ASCII character: not in the categories Cc, Cf, Cs, Co,
+/// Cn, Zl, Zp and Zs.
+fn is_printable(c: char) -> bool {
+ static UNPRINTABLE: OnceLock = OnceLock::new();
+ !unicode_class(
+ r"^[\p{Cc}\p{Cf}\p{Co}\p{Cn}\p{Zl}\p{Zp}\p{Zs}]$",
+ &UNPRINTABLE,
+ )
+ .is_match(c.encode_utf8(&mut [0; 4]))
+}
+
+/// `repr(text)`: the string in quotes, with escapes for what isn't printable.
+pub(crate) fn repr(text: &str) -> String {
+ let quote = if text.contains('\'') && !text.contains('"') {
+ '"'
+ } else {
+ '\''
+ };
+ let mut out = String::with_capacity(text.len() + 2);
+ out.push(quote);
+ for c in text.chars() {
+ let code = c as u32;
+ match c {
+ '\\' => out.push_str("\\\\"),
+ '\t' => out.push_str("\\t"),
+ '\n' => out.push_str("\\n"),
+ '\r' => out.push_str("\\r"),
+ _ if c == quote => {
+ out.push('\\');
+ out.push(c);
+ }
+ _ if code < 0x20 || code == 0x7f => {
+ let _ = write!(out, "\\x{code:02x}");
+ }
+ _ if c.is_ascii() || is_printable(c) => out.push(c),
+ _ if code < 0x100 => {
+ let _ = write!(out, "\\x{code:02x}");
+ }
+ _ if code < 0x10000 => {
+ let _ = write!(out, "\\u{code:04x}");
+ }
+ _ => {
+ let _ = write!(out, "\\U{code:08x}");
+ }
+ }
+ }
+ out.push(quote);
+ out
+}
+
+/// `text.casefold()` if it is ASCII, which is all that a comparison with an ASCII string
+/// needs: `None` if the case folded text isn't ASCII.
+pub(crate) fn casefold_ascii(text: &str) -> Option {
+ let mut out = String::with_capacity(text.len());
+ for c in text.chars() {
+ let folded = match c {
+ _ if c.is_ascii() => {
+ out.push(c.to_ascii_lowercase());
+ continue;
+ }
+ 'ß' | 'ẞ' => "ss",
+ 'ſ' => "s",
+ 'K' => "k",
+ 'ff' => "ff",
+ 'fi' => "fi",
+ 'fl' => "fl",
+ 'ffi' => "ffi",
+ 'ffl' => "ffl",
+ 'ſt' | 'st' => "st",
+ _ => return None,
+ };
+ out.push_str(folded);
+ }
+ Some(out)
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn whitespace() {
+ assert_eq!(strip("\u{1c} a \u{3000}"), "a");
+ assert!(is_all_space(" \u{1f}"));
+ assert!(!is_all_space(""));
+ assert_eq!(rstrip("a\n\n"), "a");
+ assert_eq!(lstrip("\t a"), "a");
+ }
+
+ #[test]
+ fn floats_like_python() {
+ // Checked with Python 3.10
+ for (text, expected) in [
+ ("0.5", Some(0.5)),
+ (" 1 ", Some(1.0)),
+ ("\u{2000}1", Some(1.0)),
+ ("١.٥", Some(1.5)),
+ ("1_0", Some(10.0)),
+ ("١_٠", Some(10.0)),
+ ("1e5", Some(1e5)),
+ (".5", Some(0.5)),
+ ("5.", Some(5.0)),
+ ("-Infinity", Some(f64::NEG_INFINITY)),
+ ("1e400", Some(f64::INFINITY)),
+ ("\u{1c}1.5\u{1f}", None),
+ ("١٫٥", None),
+ ("1__0", None),
+ ("_1", None),
+ ("1_", None),
+ ("1_.5", None),
+ ("1._5", None),
+ (" 1e_5", None),
+ ("", None),
+ (" ", None),
+ ("0x10", None),
+ ("1,5", None),
+ ("nan_", None),
+ ("1\u{0}", None),
+ ] {
+ assert_eq!(float(text), expected, "{text:?}");
+ }
+ assert!(float("nAn").unwrap().is_nan());
+ assert!(float("+nan").unwrap().is_nan());
+ }
+
+ #[test]
+ fn ints_like_python() {
+ for (text, expected) in [
+ ("٨٠", Some(80)),
+ (" 80 ", Some(80)),
+ ("+80", Some(80)),
+ ("8_0", Some(80)),
+ ("-5", Some(-5)),
+ ("080", Some(80)),
+ ("80.0", None),
+ ("", None),
+ ("-", None),
+ ("_8", None),
+ ("8_", None),
+ ("-_8", None),
+ ] {
+ assert_eq!(int(text), expected, "{text:?}");
+ }
+ assert_eq!(int(&"9".repeat(60)), Some(i128::MAX));
+ }
+
+ #[test]
+ fn repr_like_python() {
+ assert_eq!(repr("x"), "'x'");
+ assert_eq!(repr("it's"), "\"it's\"");
+ assert_eq!(repr("'\""), "'\\'\"'");
+ assert_eq!(repr("a\\b\n\t\r\x01\x7f"), "'a\\\\b\\n\\t\\r\\x01\\x7f'");
+ assert_eq!(
+ repr("é₿\u{a0}\u{200b}\u{2028}😂"),
+ "'é₿\\xa0\\u200b\\u2028😂'"
+ );
+ assert_eq!(repr("\u{e000}\u{10ffff}"), "'\\ue000\\U0010ffff'");
+ }
+
+ #[test]
+ fn casefold() {
+ assert_eq!(casefold_ascii("OſM").as_deref(), Some("osm"));
+ assert_eq!(casefold_ascii("Google").as_deref(), Some("google"));
+ assert_eq!(casefold_ascii("goögle"), None);
+ }
+
+ #[test]
+ fn decimal_digits() {
+ assert_eq!(decimal_value('٣'), Some(3));
+ assert_eq!(decimal_value('𝟗'), Some(9)); // U+1D7D7, in a run of five runs of ten
+ assert_eq!(decimal_value('²'), None);
+ assert_eq!(decimal_value('a'), None);
+ }
+}
diff --git a/src/cli/style.rs b/src/cli/style.rs
new file mode 100644
index 0000000..6658be7
--- /dev/null
+++ b/src/cli/style.rs
@@ -0,0 +1,431 @@
+//! Styled text and how it is written: plain text when the output isn't a terminal, and
+//! the escape sequences of rich's colour systems on a terminal. Like rich, everything that
+//! is printed loses its backspaces, vertical tabs, form feeds and carriage returns, and
+//! its tabs become spaces (tab stops every 8 characters).
+
+use std::fmt::Write;
+
+/// A style: bold, a colour and a link.
+#[derive(Debug, Clone, Default, PartialEq, Eq)]
+pub(crate) struct Style {
+ pub(crate) bold: bool,
+ pub(crate) color: Option<(u8, u8, u8)>,
+ pub(crate) link: Option,
+}
+
+/// `bold #D7Afff`, the style of pyWhat's labels ("Matched on: ").
+pub(crate) const LABEL: Style = Style {
+ bold: true,
+ color: Some((0xD7, 0xAF, 0xFF)),
+ link: None,
+};
+
+/// `bold`.
+pub(crate) const BOLD: Style = Style {
+ bold: true,
+ color: None,
+ link: None,
+};
+
+/// `bold #49C3CE`, the style of the version number.
+pub(crate) const VERSION: Style = Style {
+ bold: true,
+ color: Some((0x49, 0xC3, 0xCE)),
+ link: None,
+};
+
+/// The colours that a terminal shows, as rich detects them.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub(crate) enum ColorSystem {
+ /// Not a terminal: no escape sequences at all.
+ None,
+ Standard,
+ EightBit,
+ TrueColor,
+}
+
+/// How to write styles.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub(crate) struct Colors {
+ pub(crate) system: ColorSystem,
+ /// `NO_COLOR`: bold and links, but no colours.
+ pub(crate) no_color: bool,
+}
+
+impl Colors {
+ pub(crate) const PLAIN: Colors = Colors {
+ system: ColorSystem::None,
+ no_color: false,
+ };
+
+ /// rich's `Console._detect_color_system()` for a stream that is a terminal (or not).
+ pub(crate) fn detect(is_terminal: bool, env: &dyn Fn(&str) -> Option) -> Colors {
+ let term = env("TERM").unwrap_or_default().trim().to_lowercase();
+ if !is_terminal || term == "dumb" || term == "unknown" {
+ return Colors::PLAIN;
+ }
+ let system = if cfg!(windows) {
+ // Windows 10 and later understand the escape sequences of 24-bit colours once
+ // they are enabled; older consoles get none
+ #[cfg(windows)]
+ let enabled = anstyle_query::windows::enable_ansi_colors().unwrap_or(false);
+ #[cfg(not(windows))]
+ let enabled = true;
+ if enabled {
+ ColorSystem::TrueColor
+ } else {
+ ColorSystem::None
+ }
+ } else {
+ let color_term = env("COLORTERM").unwrap_or_default().trim().to_lowercase();
+ if color_term == "truecolor" || color_term == "24bit" {
+ ColorSystem::TrueColor
+ } else {
+ match term.rsplit_once('-').map_or("", |(_, colors)| colors) {
+ "256color" => ColorSystem::EightBit,
+ _ => ColorSystem::Standard,
+ }
+ }
+ };
+ Colors {
+ system,
+ no_color: env("NO_COLOR").is_some(),
+ }
+ }
+
+ pub(crate) fn is_plain(self) -> bool {
+ self.system == ColorSystem::None
+ }
+}
+
+/// rich's `rgb_to_hls()` saturation and lightness (`colorsys`).
+fn lightness_saturation(r: f64, g: f64, b: f64) -> (f64, f64) {
+ let max = r.max(g).max(b);
+ let min = r.min(g).min(b);
+ let lightness = (max + min) / 2.0;
+ if max == min {
+ return (lightness, 0.0);
+ }
+ let saturation = if lightness <= 0.5 {
+ (max - min) / (max + min)
+ } else {
+ (max - min) / (2.0 - max - min)
+ };
+ (lightness, saturation)
+}
+
+/// rich's `Color.downgrade(EIGHT_BIT)` of a 24-bit colour.
+fn eight_bit((r, g, b): (u8, u8, u8)) -> u8 {
+ let (r, g, b) = (r as f64 / 255.0, g as f64 / 255.0, b as f64 / 255.0);
+ let (lightness, saturation) = lightness_saturation(r, g, b);
+ if saturation < 0.1 {
+ let gray = (lightness * 25.0).round_ties_even() as u8;
+ return match gray {
+ 0 => 16,
+ 25 => 231,
+ gray => 231 + gray,
+ };
+ }
+ let level = |value: f64| (value * 5.0).round_ties_even() as u8;
+ 16 + 36 * level(r) + 6 * level(g) + level(b)
+}
+
+/// rich's `STANDARD_PALETTE.match()`: the closest of the 16 standard colours.
+fn standard((r, g, b): (u8, u8, u8)) -> u8 {
+ const PALETTE: [(i64, i64, i64); 16] = [
+ (0, 0, 0),
+ (170, 0, 0),
+ (0, 170, 0),
+ (170, 85, 0),
+ (0, 0, 170),
+ (170, 0, 170),
+ (0, 170, 170),
+ (170, 170, 170),
+ (85, 85, 85),
+ (255, 85, 85),
+ (85, 255, 85),
+ (255, 255, 85),
+ (85, 85, 255),
+ (255, 85, 255),
+ (85, 255, 255),
+ (255, 255, 255),
+ ];
+ let (r, g, b) = (i64::from(r), i64::from(g), i64::from(b));
+ let distance = |&(r2, g2, b2): &(i64, i64, i64)| {
+ let red_mean = (r + r2) / 2;
+ let (red, green, blue) = (r - r2, g - g2, b - b2);
+ let squared = (((512 + red_mean) * red * red) >> 8)
+ + 4 * green * green
+ + (((767 - red_mean) * blue * blue) >> 8);
+ (squared as f64).sqrt()
+ };
+ let mut best = 0;
+ for (index, color) in PALETTE.iter().enumerate() {
+ if distance(color) < distance(&PALETTE[best]) {
+ best = index;
+ }
+ }
+ best as u8
+}
+
+/// Writes `text` in `style`, like rich's `Style.render()`.
+pub(crate) fn write_styled(out: &mut String, text: &str, style: &Style, colors: Colors) {
+ if text.is_empty() || colors.is_plain() || *style == Style::default() {
+ out.push_str(text);
+ return;
+ }
+ let mut codes = String::new();
+ if style.bold {
+ codes.push('1');
+ }
+ if let (Some(rgb), false) = (style.color, colors.no_color) {
+ if !codes.is_empty() {
+ codes.push(';');
+ }
+ let _ = match colors.system {
+ ColorSystem::TrueColor => write!(codes, "38;2;{};{};{}", rgb.0, rgb.1, rgb.2),
+ ColorSystem::EightBit => write!(codes, "38;5;{}", eight_bit(rgb)),
+ ColorSystem::Standard | ColorSystem::None => {
+ let number = standard(rgb);
+ if number < 8 {
+ write!(codes, "{}", 30 + number)
+ } else {
+ write!(codes, "{}", 82 + number)
+ }
+ }
+ };
+ }
+ let styled = if codes.is_empty() {
+ text.to_string()
+ } else {
+ format!("\x1b[{codes}m{text}\x1b[0m")
+ };
+ match &style.link {
+ Some(link) => {
+ let _ = write!(out, "\x1b]8;;{link}\x1b\\{styled}\x1b]8;;\x1b\\");
+ }
+ None => out.push_str(&styled),
+ }
+}
+
+/// A piece of text to print: text as it is, or markup of the data files.
+#[derive(Debug, Clone)]
+enum Piece {
+ Plain(String, Style),
+ Markup(String),
+}
+
+/// Text to print, built from pieces. Like the strings that the Python CLI gives rich,
+/// except that only the data is markup.
+#[derive(Debug, Clone, Default)]
+pub(crate) struct Text {
+ pieces: Vec,
+}
+
+impl Text {
+ pub(crate) fn new() -> Text {
+ Text::default()
+ }
+
+ /// Text as it is.
+ pub(crate) fn plain(&mut self, text: &str) -> &mut Self {
+ self.styled(text, Style::default())
+ }
+
+ /// Text in a style.
+ pub(crate) fn styled(&mut self, text: &str, style: Style) -> &mut Self {
+ if !text.is_empty() {
+ self.pieces.push(Piece::Plain(text.to_string(), style));
+ }
+ self
+ }
+
+ /// A label, such as "Matched on: ".
+ pub(crate) fn label(&mut self, text: &str) -> &mut Self {
+ self.styled(text, LABEL)
+ }
+
+ /// A string of the data files, which may contain rich markup.
+ pub(crate) fn markup(&mut self, markup: &str) -> &mut Self {
+ if !markup.is_empty() {
+ self.pieces.push(Piece::Markup(markup.to_string()));
+ }
+ self
+ }
+
+ pub(crate) fn append(&mut self, other: Text) -> &mut Self {
+ self.pieces.extend(other.pieces);
+ self
+ }
+
+ /// The text as the Python CLI writes it before rich renders it, with markup.
+ pub(crate) fn source(&self) -> String {
+ self.pieces
+ .iter()
+ .map(|piece| match piece {
+ Piece::Plain(text, _) | Piece::Markup(text) => text.as_str(),
+ })
+ .collect()
+ }
+
+ /// `str.rstrip()` of the text before rendering (Python strips the markup).
+ pub(crate) fn rstrip(&mut self) -> &mut Self {
+ while let Some(last) = self.pieces.last_mut() {
+ let text = match last {
+ Piece::Plain(text, _) | Piece::Markup(text) => text,
+ };
+ let kept = super::python::rstrip(text).len();
+ text.truncate(kept);
+ if !text.is_empty() {
+ break;
+ }
+ self.pieces.pop();
+ }
+ self
+ }
+
+ /// `str.lstrip()` of the text before rendering.
+ pub(crate) fn lstrip(&mut self) -> &mut Self {
+ while let Some(first) = self.pieces.first_mut() {
+ let text = match first {
+ Piece::Plain(text, _) | Piece::Markup(text) => text,
+ };
+ let removed = text.len() - super::python::lstrip(text).len();
+ text.drain(..removed);
+ if !text.is_empty() {
+ break;
+ }
+ self.pieces.remove(0);
+ }
+ self
+ }
+
+ /// `str.strip()` of the text before rendering.
+ pub(crate) fn strip(&mut self) -> &mut Self {
+ self.lstrip().rstrip()
+ }
+
+ /// The text with its styles, with the markup rendered.
+ fn spans(&self) -> Vec<(String, Style)> {
+ let mut spans = Vec::new();
+ for piece in &self.pieces {
+ match piece {
+ Piece::Plain(text, style) => spans.push((text.clone(), style.clone())),
+ Piece::Markup(markup) => spans.extend(super::markup::render(markup)),
+ }
+ }
+ spans
+ }
+
+ /// The plain text, as rich prints it to a file or a pipe.
+ pub(crate) fn to_plain(&self) -> String {
+ self.render(Colors::PLAIN)
+ }
+
+ /// The text as rich prints it: control codes removed, tabs expanded, and styles
+ /// written for `colors`.
+ pub(crate) fn render(&self, colors: Colors) -> String {
+ let mut out = String::new();
+ let mut column = 0;
+ for (text, style) in self.spans() {
+ let text = sanitize(&text, &mut column);
+ // A style per line, the line breaks outside of it
+ for (index, line) in text.split('\n').enumerate() {
+ if index > 0 {
+ out.push('\n');
+ }
+ write_styled(&mut out, line, &style, colors);
+ }
+ }
+ out
+ }
+}
+
+/// What rich does to text it prints: backspaces, vertical tabs, form feeds and carriage
+/// returns removed (`strip_control_codes`), tabs expanded to the next multiple of 8
+/// characters on the line (`Text.expand_tabs`). `column` is the number of characters on
+/// the line so far.
+pub(crate) fn sanitize(text: &str, column: &mut usize) -> String {
+ let mut out = String::with_capacity(text.len());
+ for c in text.chars() {
+ match c {
+ '\x08' | '\x0b' | '\x0c' | '\r' => {}
+ '\n' => {
+ out.push(c);
+ *column = 0;
+ }
+ '\t' => {
+ let spaces = 8 - *column % 8;
+ out.extend(std::iter::repeat_n(' ', spaces));
+ *column += spaces;
+ }
+ _ => {
+ out.push(c);
+ *column += 1;
+ }
+ }
+ }
+ out
+}
+
+/// `sanitize()` for a text on its own.
+pub(crate) fn sanitized(text: &str) -> String {
+ sanitize(text, &mut 0)
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn rich_downgrades() {
+ // Color.parse("#D7AFFF").downgrade(...) in rich 12.2.0
+ assert_eq!(eight_bit((0xD7, 0xAF, 0xFF)), 183);
+ assert_eq!(eight_bit((0x49, 0xC3, 0xCE)), 80);
+ assert_eq!(standard((0xD7, 0xAF, 0xFF)), 7);
+ assert_eq!(standard((0x49, 0xC3, 0xCE)), 6);
+ assert_eq!(eight_bit((0x80, 0x80, 0x80)), 244);
+ }
+
+ #[test]
+ fn styles() {
+ let true_color = Colors {
+ system: ColorSystem::TrueColor,
+ no_color: false,
+ };
+ let mut out = String::new();
+ write_styled(&mut out, "x", &LABEL, true_color);
+ assert_eq!(out, "\x1b[1;38;2;215;175;255mx\x1b[0m");
+ let mut out = String::new();
+ write_styled(
+ &mut out,
+ "x",
+ &LABEL,
+ Colors {
+ no_color: true,
+ ..true_color
+ },
+ );
+ assert_eq!(out, "\x1b[1mx\x1b[0m");
+ let mut text = Text::new();
+ text.label("Name: ").plain("a\tb\r\n\tc");
+ assert_eq!(text.to_plain(), "Name: a b\n c");
+ assert_eq!(
+ text.render(true_color),
+ "\x1b[1;38;2;215;175;255mName: \x1b[0ma b\n c"
+ );
+ }
+
+ #[test]
+ fn rstrip_strips_the_source() {
+ let mut text = Text::new();
+ text.plain("a").markup("b [https://x y]").plain("\n\n");
+ text.rstrip();
+ // rich gets "ab [https://x y]" and prints "ab "
+ assert_eq!(text.to_plain(), "ab ");
+ let mut text = Text::new();
+ text.plain("a ").markup(" ").plain(" \n");
+ text.rstrip();
+ assert_eq!(text.to_plain(), "a");
+ }
+}
diff --git a/src/cli/table.rs b/src/cli/table.rs
new file mode 100644
index 0000000..dbc06b8
--- /dev/null
+++ b/src/cli/table.rs
@@ -0,0 +1,63 @@
+//! The table of `--format pretty`, drawn with comfy-table in the look of the Python CLI's
+//! rich table (`box.HEAVY_HEAD`, a line between rows, padding of one space). When the
+//! table fits, it has the same columns widths as rich's; when it doesn't, comfy-table
+//! wraps the cells its own way, but the cells are the same.
+
+use super::style::{ColorSystem, Colors};
+use comfy_table::{
+ Attribute, Cell, Color, ContentArrangement, ContentLineStyle, LineStyle, Table, TableStyle,
+};
+
+/// rich's `box.HEAVY_HEAD`.
+const HEAVY_HEAD: TableStyle = TableStyle::new()
+ .top_border(LineStyle::new('┏', '━', '┳', '┓'))
+ .header_lines(ContentLineStyle::new('┃', '┃', '┃'))
+ .header_separator(LineStyle::new('┡', '━', '╇', '┩'))
+ .content_lines(ContentLineStyle::new('│', '│', '│'))
+ .row_separator(LineStyle::new('├', '─', '┼', '┤'))
+ .bottom_border(LineStyle::new('└', '─', '┴', '┘'));
+
+/// The style of the header (`header_style="bold #D7Afff"`).
+fn header_cell(text: &str, colors: Colors) -> Cell {
+ let cell = Cell::new(text);
+ if colors.is_plain() {
+ return cell;
+ }
+ let cell = cell.add_attribute(Attribute::Bold);
+ if colors.no_color {
+ return cell;
+ }
+ match colors.system {
+ ColorSystem::TrueColor => cell.fg(Color::Rgb {
+ r: 0xD7,
+ g: 0xAF,
+ b: 0xFF,
+ }),
+ ColorSystem::EightBit => cell.fg(Color::AnsiValue(183)),
+ ColorSystem::Standard | ColorSystem::None => cell.fg(Color::Grey),
+ }
+}
+
+/// The table, `width` columns wide at most, without a line break at the end.
+pub(crate) fn render(
+ header: &[&str],
+ rows: &[Vec],
+ width: usize,
+ colors: Colors,
+) -> String {
+ let mut table = Table::new();
+ table
+ .load_style(HEAVY_HEAD)
+ .set_content_arrangement(ContentArrangement::Dynamic)
+ .set_width(width.min(u16::MAX as usize) as u16);
+ if colors.is_plain() {
+ table.force_no_tty();
+ } else {
+ table.enforce_styling();
+ }
+ table.set_header(header.iter().map(|text| header_cell(text, colors)));
+ for row in rows {
+ table.add_row(row.iter().map(Cell::new));
+ }
+ table.to_string()
+}
diff --git a/src/cli/verify.rs b/src/cli/verify.rs
new file mode 100644
index 0000000..51ffb29
--- /dev/null
+++ b/src/cli/verify.rs
@@ -0,0 +1,286 @@
+//! `--verify`: asks Google whether the Google API keys found are valid, and adds the answer
+//! to their description, like `GoogleAPIKeyVerifier` of `pywhat/processors.py`: "Verification:
+//! invalid, Google rejected the key (API_KEY_INVALID)". Every key is sent once, however many
+//! times it is found.
+
+use crate::identify::Match;
+use crate::processors::Processor;
+use serde_json::Value;
+use std::collections::HashMap;
+use std::sync::Mutex;
+
+/// What a GET request answered: the HTTP status and the body, or why there is no answer.
+pub type Answer = Result<(u16, Vec), String>;
+
+/// Sends a GET request (with the User-Agent `pywhat`) and returns the answer.
+pub type HttpGet = dyn Fn(&str) -> Answer + Send + Sync;
+
+/// Asks Google whether Google API keys are valid (`GoogleAPIKeyVerifier`).
+pub struct GoogleApiKeyVerifier {
+ get: Box,
+ /// The answer for each key, so that it is only sent once.
+ results: Mutex>,
+}
+
+impl GoogleApiKeyVerifier {
+ /// The request: an API that costs nothing (the languages of the YouTube Data API, one
+ /// unit of quota), with the key.
+ pub const URL: &'static str =
+ "https://www.googleapis.com/youtube/v3/i18nLanguages?part=snippet&key=";
+
+ /// The reasons of the `google.rpc.ErrorInfo` of the errors that Google answers when the
+ /// key is valid but can't be used for the request.
+ pub const VALID_KEY_REASONS: [&'static str; 9] = [
+ "API_KEY_ANDROID_APP_BLOCKED",
+ "API_KEY_HTTP_REFERRER_BLOCKED",
+ "API_KEY_IOS_APP_BLOCKED",
+ "API_KEY_IP_ADDRESS_BLOCKED",
+ "API_KEY_SERVICE_BLOCKED",
+ "BILLING_DISABLED",
+ "RATE_LIMIT_EXCEEDED",
+ "RESOURCE_QUOTA_EXCEEDED",
+ "SERVICE_DISABLED",
+ ];
+
+ /// A verifier that asks Google over HTTPS (10 s timeout).
+ pub fn new() -> Self {
+ Self::with_http(Box::new(get))
+ }
+
+ /// A verifier that sends its requests with `get`, e.g. a fake Google in tests.
+ pub fn with_http(get: Box) -> Self {
+ GoogleApiKeyVerifier {
+ get,
+ results: Mutex::new(HashMap::new()),
+ }
+ }
+
+ /// Asks Google whether `key` is valid, and returns the answer, e.g. "valid, Google
+ /// accepted the key".
+ pub fn verify(&self, key: &str) -> String {
+ match (self.get)(&format!("{}{}", Self::URL, quote(key))) {
+ Ok((status, _)) if (200..300).contains(&status) => {
+ "valid, Google accepted the key".to_string()
+ }
+ Ok((status, body)) => Self::describe_error(status, &body),
+ Err(error) => format!("unknown, could not ask Google ({error})"),
+ }
+ }
+
+ /// The answer for a key that Google answered an error for.
+ pub fn describe_error(status: u16, body: &[u8]) -> String {
+ let reason = google_error_reason(body);
+ match &reason {
+ Some(Value::String(reason)) if reason == "API_KEY_INVALID" => {
+ format!("invalid, Google rejected the key ({reason})")
+ }
+ Some(Value::String(reason)) if Self::VALID_KEY_REASONS.contains(&reason.as_str()) => {
+ format!("valid, but Google refused the request ({reason})")
+ }
+ Some(reason) if truthy(reason) => {
+ format!(
+ "unknown, Google answered HTTP {status} ({})",
+ python_str(reason)
+ )
+ }
+ _ => format!("unknown, Google answered HTTP {status}"),
+ }
+ }
+}
+
+impl Default for GoogleApiKeyVerifier {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+impl Processor for GoogleApiKeyVerifier {
+ fn names(&self) -> &[&str] {
+ &["Google API Key"]
+ }
+
+ fn process(&self, mut found: Match) -> Option {
+ let result = {
+ let mut results = self.results.lock().unwrap_or_else(|e| e.into_inner());
+ results
+ .entry(found.matched.clone())
+ .or_insert_with(|| self.verify(&found.matched))
+ .clone()
+ };
+ let result = format!("Verification: {result}");
+ found.description = Some(match found.description.as_deref() {
+ Some(description) if !description.is_empty() => format!("{description}. {result}"),
+ _ => result,
+ });
+ Some(found)
+ }
+}
+
+/// Python's truth value of a JSON value.
+fn truthy(value: &Value) -> bool {
+ match value {
+ Value::Null => false,
+ Value::Bool(value) => *value,
+ Value::Number(number) => number.as_f64() != Some(0.0),
+ Value::String(text) => !text.is_empty(),
+ Value::Array(values) => !values.is_empty(),
+ Value::Object(map) => !map.is_empty(),
+ }
+}
+
+/// `str()` of a JSON value as Python loads it.
+fn python_str(value: &Value) -> String {
+ match value {
+ Value::String(text) => text.clone(),
+ Value::Bool(true) => "True".to_string(),
+ Value::Bool(false) => "False".to_string(),
+ other => crate::json::dumps(other),
+ }
+}
+
+/// `google_error_reason()`: the reason of the `google.rpc.ErrorInfo` in the error that a
+/// Google API answered, such as "API_KEY_INVALID", or `None`.
+fn google_error_reason(body: &[u8]) -> Option {
+ let json: Value = serde_json::from_slice(body).ok()?;
+ let error = json.as_object()?.get("error")?.as_object()?;
+ let details = error.get("details")?;
+ for detail in details.as_array()? {
+ let detail = detail.as_object()?;
+ let kind = match detail.get("@type") {
+ None => "",
+ Some(Value::String(kind)) => kind,
+ Some(_) => return None,
+ };
+ if kind.ends_with("google.rpc.ErrorInfo") {
+ return detail.get("reason").cloned();
+ }
+ }
+ None
+}
+
+/// `urllib.parse.quote(text, safe="")`.
+fn quote(text: &str) -> String {
+ let mut out = String::with_capacity(text.len());
+ for &b in text.as_bytes() {
+ if b.is_ascii_alphanumeric() || b"_.-~".contains(&b) {
+ out.push(b as char);
+ } else {
+ out.push_str(&format!("%{b:02X}"));
+ }
+ }
+ out
+}
+
+/// A GET request over HTTPS.
+#[cfg(feature = "verify")]
+fn get(url: &str) -> Answer {
+ use std::time::Duration;
+ let agent: ureq::Agent = ureq::Agent::config_builder()
+ .timeout_global(Some(Duration::from_secs(10)))
+ .http_status_as_error(false)
+ .user_agent("pywhat")
+ .build()
+ .into();
+ let mut response = agent.get(url).call().map_err(|error| error.to_string())?;
+ let status = response.status().as_u16();
+ let body = response
+ .body_mut()
+ .read_to_vec()
+ .map_err(|error| error.to_string())?;
+ Ok((status, body))
+}
+
+#[cfg(not(feature = "verify"))]
+fn get(_url: &str) -> Answer {
+ Err("this pywhat was built without the verify feature".to_string())
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use std::sync::atomic::{AtomicUsize, Ordering};
+ use std::sync::Arc;
+
+ fn google_error(reason: &str) -> Vec {
+ serde_json::json!({
+ "error": {
+ "code": 400,
+ "message": "API key not valid. Please pass a valid API key.",
+ "details": [
+ {"@type": "type.googleapis.com/google.rpc.ErrorInfo", "reason": reason},
+ {"@type": "type.googleapis.com/google.rpc.LocalizedMessage"},
+ ],
+ }
+ })
+ .to_string()
+ .into_bytes()
+ }
+
+ #[test]
+ fn answers() {
+ let invalid = GoogleApiKeyVerifier::with_http(Box::new(|_| {
+ Ok((400, google_error("API_KEY_INVALID")))
+ }));
+ assert_eq!(
+ invalid.verify("k"),
+ "invalid, Google rejected the key (API_KEY_INVALID)"
+ );
+ let disabled = GoogleApiKeyVerifier::with_http(Box::new(|_| {
+ Ok((403, google_error("SERVICE_DISABLED")))
+ }));
+ assert_eq!(
+ disabled.verify("k"),
+ "valid, but Google refused the request (SERVICE_DISABLED)"
+ );
+ let other = GoogleApiKeyVerifier::with_http(Box::new(|_| Ok((403, google_error("OTHER")))));
+ assert_eq!(
+ other.verify("k"),
+ "unknown, Google answered HTTP 403 (OTHER)"
+ );
+ let html = GoogleApiKeyVerifier::with_http(Box::new(|_| Ok((502, b"".to_vec()))));
+ assert_eq!(html.verify("k"), "unknown, Google answered HTTP 502");
+ let valid = GoogleApiKeyVerifier::with_http(Box::new(|_| Ok((200, b"{}".to_vec()))));
+ assert_eq!(valid.verify("k"), "valid, Google accepted the key");
+ let offline = GoogleApiKeyVerifier::with_http(Box::new(|_| Err("timed out".into())));
+ assert_eq!(
+ offline.verify("k"),
+ "unknown, could not ask Google (timed out)"
+ );
+ }
+
+ #[test]
+ fn once_per_key_in_the_description() {
+ let requests = Arc::new(AtomicUsize::new(0));
+ let urls = Arc::new(Mutex::new(Vec::new()));
+ let (counter, seen) = (requests.clone(), urls.clone());
+ let verifier = GoogleApiKeyVerifier::with_http(Box::new(move |url| {
+ counter.fetch_add(1, Ordering::SeqCst);
+ seen.lock().unwrap().push(url.to_string());
+ Ok((400, google_error("API_KEY_INVALID")))
+ }));
+ let key = format!("AIza{}", "SyA00000000000000000000000000000000");
+ let identifier = crate::Identifier::default()
+ .with_processors(vec![Box::new(verifier)])
+ .include_tags(&["Google API Key"]);
+ let text = format!("{key}\n{key}\n{key}");
+ let found = identifier.identify(&text);
+ assert_eq!(found.len(), 3);
+ for m in &found {
+ assert!(m
+ .description
+ .as_deref()
+ .unwrap()
+ .ends_with("Verification: invalid, Google rejected the key (API_KEY_INVALID)"));
+ }
+ assert_eq!(requests.load(Ordering::SeqCst), 1);
+ assert_eq!(
+ urls.lock().unwrap()[0],
+ format!("{}{key}", GoogleApiKeyVerifier::URL)
+ );
+ }
+
+ #[test]
+ fn quoting() {
+ assert_eq!(quote("a-b_c.d~e/f g"), "a-b_c.d~e%2Ff%20g");
+ }
+}
diff --git a/src/files.rs b/src/files.rs
index 6824eed..9db7687 100644
--- a/src/files.rs
+++ b/src/files.rs
@@ -2,7 +2,7 @@
//! `Identifier._files()`, `Identifier._collect()` and `pywhat/magic_numbers.py`.
use crate::data::FILE_SIGNATURES_JSON;
-use crate::identify::{check, to_identify_json, Identifier, Match};
+use crate::identify::{check_with, to_identify_json, Identifier, Match};
use crate::ranking::{sort_matches, Key};
use crate::unicode;
use serde_json::{Map, Value};
@@ -201,6 +201,27 @@ pub struct InputOptions {
pub include_filenames: bool,
}
+/// How far [`Identifier::iter_identify_with_progress`] is, like `Progress` in Python. The
+/// files (a text is a file too) are searched one after another, and every regex is
+/// searched for in every text of a file: its contents, the UTF-16 strings in it and, with
+/// `include_filenames`, its name. The regexes of a large text are searched for in
+/// parallel.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct Progress<'a> {
+ /// The file that is searched, or "text", like the locations of the matches.
+ pub location: &'a str,
+ /// How many files have been searched, of how many.
+ pub files_done: usize,
+ pub files: usize,
+ /// How many of the regex searches of the file are done, of how many (the number of
+ /// regexes times the number of texts in the file).
+ pub regexes_done: usize,
+ pub regexes: usize,
+ /// The name of a regex that is searched for now (the one searched for the longest),
+ /// `None` once the file has been searched.
+ pub regex: Option<&'static str>,
+}
+
/// What [`Identifier::identify_inputs`] found, like the dict that Python returns.
#[derive(Debug, Clone, Default)]
pub struct Identified {
@@ -328,9 +349,38 @@ impl Identifier {
options: InputOptions,
found: &mut dyn FnMut(Found) -> bool,
error: &mut dyn FnMut(&str, std::io::Error),
+ ) {
+ self.identify_sources(inputs, options, found, error, None);
+ }
+
+ /// [`Identifier::iter_identify`], calling `progress` when a regex search starts in a
+ /// file (and when one ends while others still run), and once the file has been
+ /// searched, like the `progress` callback of Python's `iter_identify()`. For a large
+ /// text the regexes are searched for in parallel, so `progress` is called from rayon's
+ /// threads.
+ pub fn iter_identify_with_progress(
+ &self,
+ inputs: &[String],
+ options: InputOptions,
+ found: &mut dyn FnMut(Found) -> bool,
+ error: &mut dyn FnMut(&str, std::io::Error),
+ progress: &(dyn Fn(Progress<'_>) + Sync),
+ ) {
+ self.identify_sources(inputs, options, found, error, Some(progress));
+ }
+
+ fn identify_sources(
+ &self,
+ inputs: &[String],
+ options: InputOptions,
+ found: &mut dyn FnMut(Found) -> bool,
+ error: &mut dyn FnMut(&str, std::io::Error),
+ progress: Option<&(dyn Fn(Progress<'_>) + Sync)>,
) {
let search = self.search();
- for source in sources(inputs, options.only_text) {
+ let sources = sources(inputs, options.only_text);
+ let files = sources.len();
+ for (files_done, source) in sources.into_iter().enumerate() {
let (location, texts) = match source {
Source::Text(text) => ("text".to_string(), vec![text]),
Source::File { location, path } => {
@@ -353,8 +403,39 @@ impl Identifier {
}
}
};
+ let regexes = search.regexes.len() * texts.len();
+ // The regexes whose search has started and isn't done, oldest first, and how
+ // many searches are done
+ let running: std::sync::Mutex<(Vec<&'static str>, usize)> =
+ std::sync::Mutex::new((Vec::new(), 0));
+ let on_regex = |entry: &'static crate::data::RegexEntry, done: bool| {
+ let Some(progress) = progress else { return };
+ let mut running = running.lock().unwrap_or_else(|e| e.into_inner());
+ let (names, finished) = &mut *running;
+ if done {
+ if let Some(index) = names.iter().position(|name| *name == entry.name) {
+ names.remove(index);
+ }
+ *finished += 1;
+ if names.is_empty() {
+ return; // the next regex reports when it starts
+ }
+ } else {
+ names.push(&entry.name);
+ }
+ progress(Progress {
+ location: &location,
+ files_done,
+ files,
+ regexes_done: *finished,
+ regexes,
+ regex: names.first().copied(),
+ });
+ };
+ let on_regex: Option> =
+ progress.is_some().then_some(&on_regex);
for text in &texts {
- let matches = match check(text, &search) {
+ let matches = match check_with(text, &search, on_regex) {
Ok(matches) => matches,
Err(_) => self.identify(text),
};
@@ -367,6 +448,16 @@ impl Identifier {
}
}
}
+ if let Some(progress) = progress {
+ progress(Progress {
+ location: &location,
+ files_done: files_done + 1,
+ files,
+ regexes_done: regexes,
+ regexes,
+ regex: None,
+ });
+ }
}
}
@@ -415,6 +506,125 @@ mod tests {
);
}
+ /// A [`Progress`] that was reported.
+ #[derive(Debug, Clone, PartialEq, Eq)]
+ struct Reported {
+ location: String,
+ files_done: usize,
+ files: usize,
+ regexes_done: usize,
+ regexes: usize,
+ regex: Option<&'static str>,
+ }
+
+ /// What `iter_identify_with_progress` found in `inputs`, and the progress it reported.
+ fn with_progress(inputs: &[String]) -> (Vec, Vec) {
+ let reported = std::sync::Mutex::new(Vec::new());
+ let mut found = Vec::new();
+ Identifier::default().iter_identify_with_progress(
+ inputs,
+ InputOptions::default(),
+ &mut |item| {
+ found.push(format!("{item:?}"));
+ true
+ },
+ &mut |_, _| {},
+ &|progress| {
+ reported.lock().unwrap().push(Reported {
+ location: progress.location.to_string(),
+ files_done: progress.files_done,
+ files: progress.files,
+ regexes_done: progress.regexes_done,
+ regexes: progress.regexes,
+ regex: progress.regex,
+ })
+ },
+ );
+ (found, reported.into_inner().unwrap())
+ }
+
+ /// The names of the regexes that `Identifier::default()` searches for, in order.
+ fn default_regexes() -> Vec<&'static str> {
+ Identifier::default()
+ .search()
+ .regexes
+ .iter()
+ .map(|&entry| entry.name.as_str())
+ .collect()
+ }
+
+ #[test]
+ fn progress_of_a_search() {
+ let inputs = vec!["THM{a}".to_string(), "bee@skerritt.blog".to_string()];
+ let mut without = Vec::new();
+ Identifier::default().iter_identify(
+ &inputs,
+ InputOptions::default(),
+ &mut |item| {
+ without.push(format!("{item:?}"));
+ true
+ },
+ &mut |_, _| {},
+ );
+ let (found, reported) = with_progress(&inputs);
+ // Progress doesn't change what is found
+ assert_eq!(found, without);
+ assert!(!found.is_empty());
+ // Short texts: every regex in database order, then the text is done
+ let names = default_regexes();
+ let regexes = names.len();
+ let mut expected = Vec::new();
+ for file in 0..2 {
+ for (regexes_done, &name) in names.iter().enumerate() {
+ expected.push(Reported {
+ location: "text".to_string(),
+ files_done: file,
+ files: 2,
+ regexes_done,
+ regexes,
+ regex: Some(name),
+ });
+ }
+ expected.push(Reported {
+ location: "text".to_string(),
+ files_done: file + 1,
+ files: 2,
+ regexes_done: regexes,
+ regexes,
+ regex: None,
+ });
+ }
+ assert_eq!(reported, expected);
+ }
+
+ #[test]
+ fn progress_of_a_large_text() {
+ // Over 4 KB, the regexes are searched for in parallel: the progress comes from
+ // several threads, one report at a time, and its counts only go up
+ let text = "THM{a} bee@skerritt.blog\n".repeat(400);
+ let (_, reported) = with_progress(&[text]);
+ let regexes = default_regexes().len();
+ let searching = reported.iter().filter(|r| r.regex.is_some()).count();
+ assert!(
+ searching >= regexes,
+ "{searching} reports, {regexes} regexes"
+ );
+ assert!(reported
+ .windows(2)
+ .all(|pair| pair[0].regexes_done <= pair[1].regexes_done));
+ assert_eq!(
+ reported.last(),
+ Some(&Reported {
+ location: "text".to_string(),
+ files_done: 1,
+ files: 1,
+ regexes_done: regexes,
+ regexes,
+ regex: None,
+ })
+ );
+ }
+
#[test]
fn join_like_os_path() {
let sep = MAIN_SEPARATOR;
diff --git a/src/identify.rs b/src/identify.rs
index 3476251..188e445 100644
--- a/src/identify.rs
+++ b/src/identify.rs
@@ -429,7 +429,21 @@ pub(crate) struct Search<'a> {
/// The matches of every regex of `search` in `text`, in database order and then in the
/// order of the text, with their fragments marked (`iter_check()` for one string).
pub(crate) fn check(text: &str, search: &Search<'_>) -> Result, MatchError> {
- let spans = find_spans(text, search)?;
+ check_with(text, search, None)
+}
+
+/// Called with each regex when its search starts (`false`) and when it is done (`true`),
+/// for progress bars. For a large text the regexes are searched for in parallel, so it is
+/// called from rayon's threads.
+pub(crate) type OnRegex<'a> = &'a (dyn Fn(&'static RegexEntry, bool) + Sync);
+
+/// [`check`], calling `on_regex` when each regex search starts and ends.
+pub(crate) fn check_with(
+ text: &str,
+ search: &Search<'_>,
+ on_regex: Option>,
+) -> Result, MatchError> {
+ let spans = find_spans(text, search, on_regex)?;
let mut found = Vec::new();
for (entry, entry_spans) in search.regexes.iter().zip(spans) {
if entry_spans.is_empty() {
@@ -451,17 +465,31 @@ pub(crate) fn check(text: &str, search: &Search<'_>) -> Result, Match
}
/// The spans of the matches of each regex of `search` in `text`.
-fn find_spans(text: &str, search: &Search<'_>) -> Result>, MatchError> {
+fn find_spans(
+ text: &str,
+ search: &Search<'_>,
+ on_regex: Option>,
+) -> Result>, MatchError> {
let spans_of = |(entry, &boundaryless): (&&'static RegexEntry, &bool)| {
+ if let Some(on_regex) = on_regex {
+ on_regex(entry, false);
+ }
let form = if boundaryless {
Form::Boundaryless
} else {
Form::Anchored
};
- let compiled = compiled(entry, form).map_err(MatchError)?;
- let mut spans = Vec::new();
- compiled.find_iter(text, &mut |start, end| spans.push((start, end)))?;
- Ok(spans)
+ let search = || -> Result, MatchError> {
+ let compiled = compiled(entry, form).map_err(MatchError)?;
+ let mut spans = Vec::new();
+ compiled.find_iter(text, &mut |start, end| spans.push((start, end)))?;
+ Ok(spans)
+ };
+ let spans = search();
+ if let Some(on_regex) = on_regex {
+ on_regex(entry, true);
+ }
+ spans
};
#[cfg(feature = "parallel")]
if text.len() > 4096 {
diff --git a/src/lib.rs b/src/lib.rs
index f070e3c..59e92c5 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -19,6 +19,8 @@
mod bitcoin;
mod charclass;
+#[cfg(all(feature = "cli", not(target_arch = "wasm32")))]
+pub mod cli;
pub mod data;
pub mod engine;
#[cfg(feature = "files")]
@@ -37,7 +39,7 @@ pub use bitcoin::is_bitcoin_address;
pub use data::{regexes, tags, RegexEntry, CHARSET_ONLY};
pub use engine::{EngineKind, MatchError};
#[cfg(feature = "files")]
-pub use files::{Found, Identified, InputOptions};
+pub use files::{Found, Identified, InputOptions, Progress};
pub use filter::{split_tags, Filter, InvalidTag};
pub use identify::{compile_all, engine_of, Identifier, Match};
pub use processors::{default_processors, Processor};
diff --git a/tests/cli.rs b/tests/cli.rs
new file mode 100644
index 0000000..a5ae7f6
--- /dev/null
+++ b/tests/cli.rs
@@ -0,0 +1,576 @@
+//! Tests of the `what` command: its argument parser and its output formats. They run the
+//! binary that cargo built, without a terminal, so there are no colours and no progress
+//! bars. `scripts/parity_cli.py` compares the output with the Python CLI's.
+//!
+//! Without the `cli` feature there is no binary, and nothing to test. (A `[[test]]` entry with
+//! `required-features` would do the same, but the sdist of the Python package has no `tests/`,
+//! and cargo can't read a manifest whose test file is missing.)
+#![cfg(feature = "cli")]
+
+use std::io::Write;
+use std::path::Path;
+use std::process::{Command, Stdio};
+
+/// The exit code, stdout and stderr of `what args` with `stdin`.
+fn what_with(args: &[&str], stdin: &[u8], env: &[(&str, &str)]) -> (i32, String, String) {
+ let mut command = Command::new(env!("CARGO_BIN_EXE_what"));
+ command
+ .args(args)
+ .current_dir(env!("CARGO_MANIFEST_DIR"))
+ .env_remove("NO_COLOR")
+ .env_remove("COLUMNS")
+ .stdin(Stdio::piped())
+ .stdout(Stdio::piped())
+ .stderr(Stdio::piped());
+ for (name, value) in env {
+ command.env(name, value);
+ }
+ let mut child = command.spawn().expect("what runs");
+ child
+ .stdin
+ .take()
+ .expect("stdin")
+ .write_all(stdin)
+ .expect("stdin is written");
+ let output = child.wait_with_output().expect("what finishes");
+ (
+ output.status.code().expect("an exit code"),
+ String::from_utf8(output.stdout).expect("UTF-8 stdout"),
+ String::from_utf8(output.stderr).expect("UTF-8 stderr"),
+ )
+}
+
+fn what(args: &[&str]) -> (i32, String, String) {
+ what_with(args, b"", &[])
+}
+
+/// The names of the matches in raw output.
+fn names(output: &str) -> Vec<&str> {
+ output
+ .lines()
+ .filter_map(|line| line.strip_prefix("Name: "))
+ .collect()
+}
+
+// The argument parser
+
+#[test]
+fn unknown_options_are_inputs() {
+ // click's ignore_unknown_options: -1606586780 is the text to search, as after --
+ let (code, stdout, _) = what(&["--rarity", "0:", "--format", "%m|%n", "-1606586780"]);
+ assert_eq!(code, 0);
+ assert_eq!(stdout, "160658678|American Social Security Number\n");
+ let (_, after_dashes, _) = what(&["--rarity", "0:", "--format", "%m|%n", "--", "-1606586780"]);
+ assert_eq!(stdout, after_dashes);
+ let (code, stdout, _) = what(&["-db", "--nope", "THM{x}"]);
+ assert_eq!(code, 0);
+ assert_eq!(names(&stdout), ["TryHackMe Flag Format"]);
+}
+
+#[test]
+fn multi_letter_short_options() {
+ let (_, with, _) = what(&["abc118.103.238.230abc"]);
+ let (_, without, _) = what(&["-db", "abc118.103.238.230abc"]);
+ assert!(names(&with).contains(&"Internet Protocol (IP) Address Version 4"));
+ assert_eq!(without, "Nothing found!\n");
+ // -bi takes a value, -pt is a flag
+ let (_, stdout, _) = what(&["-bi", "IPv4 Address", "-pt", "abc118.103.238.230abc"]);
+ assert!(
+ stdout.starts_with("Matched on: 118.103.238.230\n"),
+ "{stdout}"
+ );
+ assert!(stdout.contains("\nTags: "), "{stdout}");
+ // The email address has a rarity below 0.6, so -br 0.6: searches it with boundaries
+ let (_, stdout, _) = what(&["-br", "0.6:", "-be", "URL", "123myEmail@host.org456"]);
+ assert_eq!(stdout, "Nothing found!\n");
+}
+
+#[test]
+fn clusters_and_attached_values() {
+ // -o and -k in one argument, -k with its value attached
+ let (code, stdout, _) = what(&["-okname", "--json", "fixtures/file", "THM{x}"]);
+ assert_eq!(code, 0);
+ assert!(
+ stdout
+ .starts_with(r#"{"File Signatures": null, "Regexes": {"text": [{"Matched": "THM{x}""#),
+ "{stdout}"
+ );
+ let (_, a, _) = what(&[
+ "--top=1",
+ "-db",
+ "0x52908400098527886E0F7030069857D2E4169EE7",
+ ]);
+ let (_, b, _) = what(&[
+ "--top",
+ "1",
+ "-db",
+ "0x52908400098527886E0F7030069857D2E4169EE7",
+ ]);
+ assert_eq!(a, b);
+}
+
+#[test]
+fn double_dash() {
+ let (code, stdout, _) = what(&["--format", "%m", "--", "-db", "THM{x}"]);
+ assert_eq!(code, 0);
+ assert!(stdout.lines().any(|line| line == "THM{x}"), "{stdout}");
+}
+
+#[test]
+fn usage_errors_exit_with_2() {
+ let (code, stdout, stderr) = what(&["--rarity"]);
+ assert_eq!((code, stdout.as_str()), (2, ""));
+ assert_eq!(
+ stderr,
+ "Usage: what [OPTIONS] [TEXT_INPUT]...\nTry 'what --help' for help.\n\n\
+ Error: Option '--rarity' requires an argument.\n"
+ );
+ let (code, _, stderr) = what(&["--json=1", "x"]);
+ assert_eq!(code, 2);
+ assert!(stderr.ends_with("Error: Option '--json' does not take a value.\n"));
+ let (code, _, stderr) = what(&["--map", "bing", "x"]);
+ assert_eq!(code, 2);
+ assert!(stderr
+ .ends_with("Error: Invalid value for '--map': 'bing' is not one of 'google', 'osm'.\n"));
+}
+
+#[test]
+fn errors_exit_with_1() {
+ for (args, message) in [
+ (&["--rarity", "1", "x"][..], "Invalid rarity range format ('min:max' expected)\n"),
+ (&["--rarity", "a:", "x"], "Invalid rarity argument (float expected)\n"),
+ (&["--key", "nope", "x"], "Invalid key\n"),
+ (
+ &["--top", "0", "x"],
+ "Invalid top '0', use a number of matches such as 10, a percentage of them such as 5% or all\n",
+ ),
+ ] {
+ let (code, stdout, _) = what(args);
+ assert_eq!((code, stdout.as_str()), (1, message), "{args:?}");
+ }
+ let (code, stdout, _) = what(&["--include", "ETH Walet", "x"]);
+ assert_eq!(code, 1);
+ assert!(stdout.contains("Passed tags are not valid"));
+ assert!(stdout.contains("'pywhat --names'"));
+ for (option, value, message) in [
+ (
+ "--key",
+ "name",
+ "--stream cannot sort the matches, so it cannot be used with --key\n",
+ ),
+ (
+ "--top",
+ "1",
+ "--stream cannot pick the most likely matches, so it cannot be used with --top\n",
+ ),
+ ] {
+ let (code, _, stderr) = what(&["--stream", option, value, "x"]);
+ assert_eq!((code, stderr.as_str()), (1, message));
+ }
+}
+
+#[test]
+fn interactive_needs_python() {
+ let (code, stdout, stderr) = what(&["--interactive", "x"]);
+ assert_eq!((code, stdout.as_str()), (2, ""));
+ assert_eq!(stderr.lines().count(), 1);
+ assert!(stderr.contains("pip install pywhat"));
+}
+
+#[test]
+fn eager_options() {
+ let (code, stdout, _) = what(&["--tags", "x"]);
+ assert_eq!(code, 0);
+ assert!(stdout.lines().any(|line| line == "Bug Bounty"));
+ let (_, stdout, _) = what(&["--names"]);
+ assert!(stdout
+ .lines()
+ .any(|line| line == "Ethereum (ETH) Wallet Address: Ethereum Wallet, ETH Wallet"));
+ let (_, stdout, _) = what(&["-v"]);
+ assert_eq!(
+ stdout,
+ format!("PyWhat version {}\n", env!("CARGO_PKG_VERSION"))
+ );
+ // --help comes first, whatever the order
+ let (code, stdout, _) = what(&["--map", "bing", "--help"]);
+ assert_eq!(code, 0);
+ assert!(stdout.starts_with("Usage: what [OPTIONS] [TEXT_INPUT]...\n\n pyWhat - Identify"));
+}
+
+#[test]
+fn help_wraps_at_the_width_of_click() {
+ let (_, wide, _) = what_with(&["--help"], b"", &[("COLUMNS", "10000")]);
+ assert!(wide.lines().all(|line| line.chars().count() <= 78));
+ assert!(wide.contains(" -br, --boundaryless-rarity TEXT\n"));
+ let (_, narrow, _) = what_with(&["--help"], b"", &[("COLUMNS", "30")]);
+ assert!(
+ narrow.lines().all(|line| line.chars().count() <= 50),
+ "at least 50 columns"
+ );
+ assert!(narrow.lines().count() > wide.lines().count());
+}
+
+#[test]
+fn stdin_without_inputs() {
+ let (code, stdout, _) = what_with(&["-db"], b"THM{hello}\nfixtures/file\n", &[]);
+ assert_eq!(code, 0);
+ // One text, not a list of inputs
+ assert_eq!(names(&stdout), ["TryHackMe Flag Format"]);
+ assert!(!stdout.contains("File:"));
+ let (_, stdout, _) = what_with(&["--json"], &utf16_with_bom("THM{wide}"), &[]);
+ assert!(stdout.contains(r#""Matched": "THM{wide}""#), "{stdout}");
+ let (code, stdout, _) = what_with(&[], b"", &[]);
+ assert_eq!((code, stdout.as_str()), (0, "Nothing found!\n"));
+}
+
+fn utf16_with_bom(text: &str) -> Vec {
+ let mut data = vec![0xFF, 0xFE];
+ data.extend(text.encode_utf16().flat_map(u16::to_le_bytes));
+ data
+}
+
+// The formats
+
+#[test]
+fn raw_output() {
+ let (code, stdout, _) = what(&["-db", "THM{hello}", "github@skerritt.blog"]);
+ assert_eq!(code, 0);
+ assert_eq!(
+ stdout,
+ "Matched on: THM{hello}\nName: TryHackMe Flag Format\n\
+ Description: Used for Capture The Flags at https://tryhackme.com\n\n\
+ Matched on: github@skerritt.blog\nName: Email Address\n"
+ );
+ let (_, stdout, _) = what(&["-db", "-pt", "thm{2}"]);
+ assert!(stdout.contains("Tags: CTF Flag"));
+ let (_, stdout, _) = what(&["-db", "fixtures/test/file", "THM{hello}"]);
+ assert!(stdout.contains("File: fixtures/test/file\nMatched on: https://google.com"));
+ assert!(stdout.contains("File: text\nMatched on: THM{hello}"));
+}
+
+#[test]
+fn matched_text_is_not_markup() {
+ // The Python CLI prints "thm{x}" (#291, Python bug 4)
+ let (_, stdout, _) = what(&["-db", "thm{[bold]x[/bold]}"]);
+ assert!(
+ stdout.starts_with("Matched on: thm{[bold]x[/bold]}\n"),
+ "{stdout}"
+ );
+ // The markup of the data is rendered, like rich does
+ let (_, stdout, _) = what(&["-db", "Access-Control-Allow: *"]);
+ assert!(
+ stdout.contains("Description: Used for Cross-Origin Resource Sharing (CORS)\n"),
+ "{stdout}"
+ );
+}
+
+#[test]
+fn nothing_found() {
+ assert_eq!(what(&["-db", ""]).1, "Nothing found!\n");
+ assert_eq!(what(&["-db", "nothing", "here"]).1, "Nothing found!\n");
+ assert_eq!(what(&["-db", "--format", "%m", "nothing"]).1, "");
+ assert_eq!(
+ what(&["-db", "--json", ""]).1,
+ "{\"File Signatures\": null, \"Regexes\": null}\n"
+ );
+}
+
+#[test]
+fn custom_format() {
+ let (_, stdout, _) = what(&[
+ "-db",
+ "--format",
+ r"%m 2%n %d --- -%e%r %l %t \%d",
+ "rBPAQmwMrt7FDDPNyjwFgwSqbWZPf6SLkk",
+ ]);
+ assert_eq!(
+ stdout,
+ "rBPAQmwMrt7FDDPNyjwFgwSqbWZPf6SLkk 2Ripple (XRP) Wallet Address --- -0.3 \
+ https://xrpscan.com/account/rBPAQmwMrt7FDDPNyjwFgwSqbWZPf6SLkk Finance, \
+ Cryptocurrency Wallet, Ripple Wallet, Ripple, XRP %d\n"
+ );
+ // \\ is a backslash, and a backslash in the match is not an escape (Python bug 2)
+ let (code, stdout, _) = what(&["-db", "--format", r"\\%m - %n", r"thm{a\1}"]);
+ assert_eq!(code, 0);
+ assert_eq!(stdout, "\\thm{a\\1} - TryHackMe Flag Format\n");
+ // An absent exploit is nothing
+ assert_eq!(what(&["-db", "--format", "%e", "thm{2}"]).1, "");
+}
+
+#[test]
+fn json_is_one_line_of_ascii() {
+ let (code, stdout, _) = what(&[
+ "--json",
+ "fixtures/test",
+ "fixtures/file",
+ "THM{caf\u{e9} \u{20bf}}",
+ ]);
+ assert_eq!(code, 0);
+ assert!(stdout.is_ascii());
+ assert_eq!(stdout.matches('\n').count(), 1);
+ assert!(stdout.contains(r#""Matched": "THM{caf\u00e9 \u20bf}""#));
+ let json: serde_json::Value = serde_json::from_str(&stdout).unwrap();
+ let locations: Vec<&String> = json["Regexes"].as_object().unwrap().keys().collect();
+ let test_file = Path::new("fixtures/test").join("file");
+ assert!(locations.contains(&&test_file.to_string_lossy().into_owned()));
+ assert!(locations.contains(&&"text".to_string()));
+ // --format json is --json
+ assert_eq!(
+ what(&["--format", " json ", "abc@example.com"]).1,
+ what(&["--json", "abc@example.com"]).1
+ );
+}
+
+#[test]
+fn stream_json_lines_without_fragment() {
+ let (code, stdout, _) = what(&["--stream", "--json", "fixtures/file"]);
+ assert_eq!(code, 0);
+ let (_, all, _) = what(&["--json", "fixtures/file"]);
+ let all: serde_json::Value = serde_json::from_str(&all).unwrap();
+ let mut expected = all["Regexes"]["file"].as_array().unwrap().clone();
+ for found in &mut expected {
+ found.as_object_mut().unwrap().remove("Fragment");
+ }
+ let mut streamed = Vec::new();
+ for line in stdout.lines() {
+ let found: serde_json::Value = serde_json::from_str(line).unwrap();
+ assert!(found["File Signatures"].is_null());
+ let matches = found["Regexes"]["file"].as_array().unwrap();
+ assert_eq!(matches.len(), 1);
+ assert!(matches[0].get("Fragment").is_none());
+ streamed.push(matches[0].clone());
+ }
+ assert_eq!(streamed, expected);
+ // Nothing found: no lines at all
+ assert_eq!(what(&["--stream", "--json", "-db", ""]).1, "");
+}
+
+/// The output with every run of empty lines as one empty line (`squeeze()` of
+/// tests/test_click.py).
+fn squeeze(output: &str) -> String {
+ let mut squeezed = output.to_string();
+ while squeezed.contains("\n\n\n") {
+ squeezed = squeezed.replace("\n\n\n", "\n\n");
+ }
+ squeezed
+}
+
+#[test]
+fn stream_raw_is_the_complete_output() {
+ for args in [
+ &["-db", "fixtures/file"][..],
+ &["fixtures/file", "fixtures/test/file", "THM{hello}"],
+ &["--format", "%m - %n", "fixtures"],
+ &["-db", ""],
+ ] {
+ let (_, complete, _) = what(args);
+ let (code, streamed, _) = what(&[&["--stream"], args].concat());
+ assert_eq!(code, 0);
+ assert_eq!(squeeze(&streamed), squeeze(&complete), "{args:?}");
+ }
+}
+
+#[test]
+fn top_note_on_stderr() {
+ let eth = "0x52908400098527886E0F7030069857D2E4169EE7";
+ let (code, stdout, stderr) = what(&["--top", "1", "--json", eth]);
+ assert_eq!(code, 0);
+ let json: serde_json::Value = serde_json::from_str(&stdout).unwrap();
+ let matches = json["Regexes"]["text"].as_array().unwrap();
+ assert_eq!(matches.len(), 1);
+ assert_eq!(
+ matches[0]["Regex Pattern"]["Name"],
+ "Ethereum (ETH) Wallet Address"
+ );
+ assert!(
+ stderr.starts_with("\nShowing the 1 most likely of "),
+ "{stderr}"
+ );
+ assert!(stderr.ends_with(
+ " matches. Use --top to show more of them, or --interactive to search through all of them.\n"
+ ));
+ // No note when nothing is left out
+ assert_eq!(what(&["--top", "all", eth]).2, "");
+}
+
+/// The cells of the tables in `output`, one row per line, with the lines of a cell
+/// joined by spaces.
+fn cells(output: &str) -> Vec> {
+ let mut rows: Vec> = Vec::new();
+ let mut row: Vec> = Vec::new();
+ for line in output.lines() {
+ if line.starts_with(['┡', '├', '└']) {
+ if let Some(first) = row.first() {
+ let columns = first.len();
+ rows.push(
+ (0..columns)
+ .map(|i| {
+ let parts: Vec<&str> = row
+ .iter()
+ .map(|parts| parts[i].as_str())
+ .filter(|part| !part.is_empty())
+ .collect();
+ parts.join(" ")
+ })
+ .collect(),
+ );
+ }
+ row.clear();
+ } else if line.starts_with(['┃', '│']) {
+ // "┃ a ┃ b ┃" is "", " a ", " b ", ""
+ let parts: Vec<&str> = line.split(['┃', '│']).collect();
+ row.push(
+ parts[1..parts.len() - 1]
+ .iter()
+ .map(|cell| cell.trim().to_string())
+ .collect(),
+ );
+ }
+ }
+ rows
+}
+
+#[test]
+fn pretty_table() {
+ let (code, stdout, _) = what_with(
+ &[
+ "-db",
+ "--format",
+ "pretty",
+ "fixtures/test/file",
+ "THM{hello}",
+ ],
+ b"",
+ &[("COLUMNS", "200")],
+ );
+ assert_eq!(code, 0);
+ assert!(stdout.starts_with("Possible Identification\n┏"), "{stdout}");
+ assert_eq!(
+ cells(&stdout),
+ [
+ ["Matched Text", "Identified as", "Description", "File"],
+ [
+ "https://google.com",
+ "Uniform Resource Locator (URL)",
+ "None",
+ "fixtures/test/file"
+ ],
+ [
+ "THM{hello}",
+ "TryHackMe Flag Format",
+ "Used for Capture The Flags at https://tryhackme.com",
+ "text"
+ ],
+ ]
+ );
+ // An Exploit column when a regex has one, and the tags
+ let heroku = "heroku00000000-0000-0000-0000-000000000000";
+ let (_, stdout, _) = what_with(
+ &[
+ "-db",
+ "--include",
+ "Bug Bounty",
+ "--format",
+ "pretty",
+ "-pt",
+ heroku,
+ ],
+ b"",
+ &[("COLUMNS", "1000")],
+ );
+ let rows = cells(&stdout);
+ assert_eq!(
+ rows[0],
+ ["Matched Text", "Identified as", "Description", "Exploit"]
+ );
+ assert_eq!(
+ rows[1][2],
+ "Tags: Bug Bounty, API Keys, Credentials, Heroku"
+ );
+ assert!(rows[1][3].contains("curl -X POST https://api.heroku.com/apps"));
+ // Nothing found
+ assert_eq!(
+ what(&["-db", "--format", "pretty", "nothing"]).1,
+ "Nothing found!\n"
+ );
+}
+
+#[test]
+fn pretty_table_fits_narrow_terminals() {
+ let (code, stdout, _) = what_with(
+ &["-db", "--format", "pretty", "fixtures/file"],
+ b"",
+ &[("COLUMNS", "60")],
+ );
+ assert_eq!(code, 0);
+ assert!(stdout.lines().all(|line| line.chars().count() <= 60));
+}
+
+#[test]
+fn processing_and_maps() {
+ let (_, stdout, _) = what(&[
+ "--rarity",
+ "0:",
+ "--include",
+ "UNIX Timestamp",
+ "--format",
+ "%d",
+ "1637093119",
+ ]);
+ assert!(
+ stdout.contains("Date: November 16, 2021 8:05:19 PM UTC"),
+ "{stdout}"
+ );
+ // Bitcoin addresses with a wrong checksum are filtered out, unless -dp
+ let wrong = "3F3F3F3F3F3F3F3F3F3F3F3F3F3F3F3F3F";
+ assert!(!names(&what(&["-db", wrong]).1).contains(&"Bitcoin (₿) Wallet Address"));
+ assert!(names(&what(&["-db", "-dp", wrong]).1).contains(&"Bitcoin (₿) Wallet Address"));
+ let (_, stdout, _) = what(&["--map", "osm", "--format", "%l", "52.6169586, -1.9779857"]);
+ assert!(stdout
+ .lines()
+ .any(|line| line == "https://www.openstreetmap.org/search?query=52.6169586,-1.9779857"));
+}
+
+#[test]
+fn files_and_signatures() {
+ let dir = std::env::temp_dir().join(format!("pywhat-cli-test-{}", std::process::id()));
+ std::fs::create_dir_all(&dir).unwrap();
+ let image = dir.join("image.png");
+ std::fs::write(&image, b"\x89PNG\r\n\x1a\nTHM{hello}\n").unwrap();
+ let image = image.to_string_lossy().into_owned();
+ let (code, stdout, _) = what(&["-db", &image]);
+ assert_eq!(code, 0);
+ assert!(
+ stdout.starts_with("\nFile Identified: image.png with Magic Numbers .PNG.....\n"),
+ "{stdout}"
+ );
+ assert!(stdout.contains("Matched on: THM{hello}"));
+ let (_, streamed, _) = what(&["-db", "--stream", &image]);
+ assert_eq!(streamed, stdout);
+ // -if searches the name of the file too
+ let named = dir.join("THM{name}.txt");
+ std::fs::write(&named, b"nothing").unwrap();
+ let (_, stdout, _) = what(&["-if", "--format", "%m", &named.to_string_lossy()]);
+ assert_eq!(stdout, "THM{name}\n");
+ std::fs::remove_dir_all(&dir).unwrap();
+}
+
+#[test]
+fn closed_output_ends_quietly() {
+ // what ... | head: the rest of the output can't be written
+ let mut child = Command::new(env!("CARGO_BIN_EXE_what"))
+ .args(["--rarity", "0:", "fixtures/file"])
+ .current_dir(env!("CARGO_MANIFEST_DIR"))
+ .stdout(Stdio::piped())
+ .stderr(Stdio::piped())
+ .spawn()
+ .unwrap();
+ drop(child.stdout.take());
+ let output = child.wait_with_output().unwrap();
+ assert_eq!(output.status.code(), Some(1));
+ assert!(
+ output.stderr.is_empty(),
+ "{}",
+ String::from_utf8_lossy(&output.stderr)
+ );
+}
diff --git a/tests/conftest.py b/tests/conftest.py
new file mode 100644
index 0000000..93c3732
--- /dev/null
+++ b/tests/conftest.py
@@ -0,0 +1,146 @@
+"""
+Run tests/test_click.py against another program, such as the Rust CLI:
+
+ cargo build --release
+ PYWHAT_CLI=target/release/what pytest -o addopts="" tests/test_click.py
+
+With PYWHAT_CLI set, CliRunner.invoke() runs $PYWHAT_CLI with the arguments in a
+subprocess, with the input on stdin and the environment changed like CliRunner changes
+it, and returns its exit code and its output (stdout and stderr together, like
+CliRunner's). The tests that call or patch the Python CLI's internals are skipped.
+Without PYWHAT_CLI, nothing changes.
+"""
+import os
+import shlex
+import shutil
+import subprocess
+import sys
+from typing import IO, Any, Mapping, Optional, Sequence, Union
+
+import pytest
+from click.testing import CliRunner, Result
+
+# The program to test, as an absolute path, as tests may change the directory
+_cli = os.environ.get("PYWHAT_CLI")
+PYWHAT_CLI = (
+ None
+ if not _cli
+ else os.path.abspath(_cli)
+ if os.path.dirname(_cli)
+ else shutil.which(_cli) or _cli
+)
+
+# The tests of tests/test_click.py that call or patch the Python CLI's internals, so
+# that they can only test the Python CLI
+PYTHON_ONLY = {
+ "test_stream_prints_before_the_search_is_complete": "patches Identifier.iter_identify",
+ "test_progress_bars": "patches pywhat.what.progress_bars",
+ "test_no_progress": "patches pywhat.what.progress_bars",
+ "test_print_stream_api": "calls Printing.print_stream() and Identifier",
+}
+
+
+def _is_click_test(item: Any) -> bool:
+ module = getattr(item, "module", None)
+ return module is not None and module.__name__.rsplit(".", 1)[-1] == "test_click"
+
+
+def pytest_collection_modifyitems(config: Any, items: Sequence[Any]) -> None:
+ if not PYWHAT_CLI:
+ return
+ for item in items:
+ name = getattr(item, "originalname", None) or item.name
+ if _is_click_test(item) and name in PYTHON_ONLY:
+ reason = f"PYWHAT_CLI is set, and this test {PYTHON_ONLY[name]}"
+ item.add_marker(pytest.mark.skip(reason=reason))
+
+
+def _input_bytes(input: Optional[Union[str, bytes, IO]], charset: str) -> bytes:
+ """The input as CliRunner gives it to the command (make_input_stream())."""
+ if input is None:
+ return b""
+ if isinstance(input, str):
+ return input.encode(charset)
+ if isinstance(input, bytes):
+ return input
+ data = input.read()
+ return data.encode(charset) if isinstance(data, str) else data
+
+
+def _run_on_a_terminal(
+ command: Sequence[str], data: bytes, env: Mapping[str, str], stderr: Any
+) -> "subprocess.CompletedProcess[bytes]":
+ """Run command with a terminal (a pseudo-terminal) as stdin, holding data."""
+ # A check of sys.platform, so that mypy doesn't check the Unix part on Windows
+ if sys.platform == "win32":
+ pytest.skip("PYWHAT_CLI: no pseudo-terminal for stdin on Windows")
+ import pty
+
+ leader, follower = pty.openpty()
+ try:
+ os.write(leader, data)
+ return subprocess.run(
+ command,
+ stdin=follower,
+ stdout=subprocess.PIPE,
+ stderr=stderr,
+ env=dict(env),
+ timeout=600,
+ )
+ finally:
+ os.close(follower)
+ os.close(leader)
+
+
+def invoke(
+ self: CliRunner,
+ cli: Any,
+ args: Optional[Union[str, Sequence[str]]] = None,
+ input: Optional[Union[str, bytes, IO]] = None,
+ env: Optional[Mapping[str, Optional[str]]] = None,
+ catch_exceptions: bool = True,
+ color: bool = False,
+ **extra: Any,
+) -> Result:
+ """CliRunner.invoke(), running $PYWHAT_CLI instead of cli."""
+ assert PYWHAT_CLI is not None
+ if isinstance(args, str):
+ args = shlex.split(args)
+ command = [PYWHAT_CLI, *(args or ())]
+ environment = dict(os.environ)
+ for name, value in {**self.env, **(env or {})}.items():
+ if value is None:
+ environment.pop(name, None)
+ else:
+ environment[name] = value
+ # stderr in the output, in the order they were written, like CliRunner's
+ stderr = subprocess.STDOUT if self.mix_stderr else subprocess.PIPE
+ data = _input_bytes(input, self.charset)
+ isatty = getattr(input, "isatty", None)
+ if callable(isatty) and isatty():
+ completed = _run_on_a_terminal(command, data, environment, stderr)
+ else:
+ completed = subprocess.run(
+ command,
+ input=data,
+ stdout=subprocess.PIPE,
+ stderr=stderr,
+ env=environment,
+ timeout=600,
+ )
+ exit_code = completed.returncode
+ return Result(
+ runner=self,
+ stdout_bytes=completed.stdout,
+ stderr_bytes=None if self.mix_stderr else completed.stderr,
+ return_value=None,
+ exit_code=exit_code,
+ exception=None if exit_code == 0 else SystemExit(exit_code),
+ )
+
+
+@pytest.fixture(autouse=True)
+def pywhat_cli(request: Any, monkeypatch: Any) -> None:
+ """With PYWHAT_CLI set, the tests of test_click.py invoke $PYWHAT_CLI."""
+ if PYWHAT_CLI and _is_click_test(request.node):
+ monkeypatch.setattr(CliRunner, "invoke", invoke)