diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 05dc516e190..175473f4080 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -24,7 +24,7 @@ repos: hooks: - id: check-added-large-files # Retain the exact lossless cross-platform performance evidence archives. - exclude: ^docs/benchmarks/((executor-copy-audit|bounded-read-ahead)-20261101|(directory-task-slots|packed-visit-snapshots|bounded-match-results|useful-match-workers|direct-evaluator-workers|derived-content-handoff|immutable-default-catalogs|prepared-diff-configuration)-20261102)\.json\.gz$ + exclude: ^docs/benchmarks/((executor-copy-audit|bounded-read-ahead)-20261101|(directory-task-slots|packed-visit-snapshots|bounded-match-results|useful-match-workers|direct-evaluator-workers|derived-content-handoff|immutable-default-catalogs|prepared-diff-configuration|prepared-hash-operands)-20261102)\.json\.gz$ - id: check-merge-conflict - id: check-yaml # BINARY_TYPES below: archive containers are binary but can START with text (a phar begins with diff --git a/CHANGELOG.md b/CHANGELOG.md index deb037f0277..a1e64f80677 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,8 @@ # 0.11.0 +- Prepare explicit hash-primary algorithms and encodings once while preserving per-context defaults and conditional expected-hash templates. + - Prepare file-diff normalization once per action style during a run instead of recompiling ignored-line regexes for every file pair. - Reuse immutable default language and MIME catalogs across repeated library runs while preserving custom configurations and later registrations. diff --git a/TODO.md b/TODO.md index dcf48ea25f9..577e04e3b95 100644 --- a/TODO.md +++ b/TODO.md @@ -423,8 +423,12 @@ EP/JC/P work; candidates are not speedup claims until measured on both platforms - [x] O15a: Qualify invariant file-diff configuration without moving conditional peer/content observations. [Native, in-memory and slower controls](docs/benchmarks/prepared-diff-configuration-20261102.md) retain once-per-style options/regex records, direct-context fallback and cross-platform costs. -- [ ] O15b: Qualify remaining hash-spec/default binding and template-family gaps against actual - production preparation; preserve dynamic defaults, conditional observations and EP03/EP07 gates. +- [x] O15b: Qualify compact hash-primary syntax/default binding against actual production execution. + [Paired native/kernel/preparation evidence](docs/benchmarks/prepared-hash-operands-20261102.md) + retains hot explicit-spec gains, native controls and additional cold preparation costs. + Keep live direct-context defaults, expected-field observations and verification bookkeeping. + Field-hash qualifier binding and broader template-family qualification remain under EP03/EP07 + and O22; this does not mark those wider plans complete. - [ ] O16-O22: Reconcile paired streaming compare, reusable stage pools, byte-bounded overlap, archive indexing/admission, optional bulk metadata, similarity representations and output churn with EP/JC/P acceptance. Keep macrobenchmark, kernel and diagnostic contracts separate. diff --git a/docs/benchmarks/prepared-hash-operands-20261102.json.gz b/docs/benchmarks/prepared-hash-operands-20261102.json.gz new file mode 100644 index 00000000000..8f233153998 Binary files /dev/null and b/docs/benchmarks/prepared-hash-operands-20261102.json.gz differ diff --git a/docs/benchmarks/prepared-hash-operands-20261102.md b/docs/benchmarks/prepared-hash-operands-20261102.md new file mode 100644 index 00000000000..5e64c1bb69e --- /dev/null +++ b/docs/benchmarks/prepared-hash-operands-20261102.md @@ -0,0 +1,100 @@ +# Prepare compact hash operands (2026-11-02) + +## Decision + +Retain compact prepared records for `-hash` and `-hasheq`: resolve explicit algorithm/encoding syntax once using the existing hash registry, and resolve only requested defaults from each reached context. Keep the expected-hash template under its existing single unique owner. No algorithm spelling list, public hash API, flag, lock, thread, shared owner, per-file clock or new mutation route is introduced. + +A prepared record stores a checked index into `AlgorithmNames()`, encoding/default-use bits, validity and the optional expected-template owner. A compile-time assertion keeps it within the existing 24-byte operand alternative, so unrelated operands do not grow. Preparation callbacks receive the expression through existing constexpr dispatch metadata; argument-only families use one registered adapter and preserve their previous missing-argument guard. A boolean in existing dispatch-record padding permits bare hash preparation without another callback pointer. + +Direct tree callbacks remain the reference. Expected-field rendering, conditional content reads, malformed/missing operands and hash-verification bookkeeping retain their order. Reused/moved plans observe each direct caller's current defaults; no view into temporary spec/default strings is cached. Explicit algorithms use the static registry index, while default-dependent algorithms still validate the current direct context. + +Preparation is not free: bare hash now owns an inline pool record/capacity, and decoding explicit syntax once increases preparation work. Hash digests, snapshot cache keys, field rendering and output construction still allocate/copy as before. O22 remains separate. The narrower operation does not close the entire EP03/EP07 qualification plan. + +## Measurement contracts + +Parent `77abeacdcc8fad1028ae5485e66208dc63ec5c7b` (#1055), clean `clang_release`/ThinLTO variants. Runtime changes only `evaluate.cc`; common support sources are identical between builds. Build manifests capture compiler version/binary hash, actual build/toolchain configuration, complete CLI link actions including parameter contents, source and executable hashes. Copied benchmark binaries load only stable system libraries. No build, lint or sanitizer overlaps timings on its host. + +Per host: 56 native cases x three sessions, independent complete stdout/hashlib digest/zero-exit oracles, discarded warmups and nine rotating rounds/fastest-seven. Inputs include 1/10/1,000/10,000 seven-byte files and 1,000 4-KiB files. Linux tmpfs/affinity and macOS warm APFS are separate host contracts. Native requested workers 1/3 do not imply parallel hash actions. Globals after primaries are exercised deliberately. + +Separate in-memory populations cover 36 whole-engine cases, 16 hot kernels and eight preparation-only cases, in three sessions per variant, with nine repetitions/fastest-seven. Whole engine includes run preflight/preparation/traversal/hash/output/teardown, excluding parser/fixture construction; preparation and hot kernels retain the existing expression-benchmark contracts. Do not compare their absolute values to native process startup or apply external-tool normalization. + +Confirmation adds a byte-identical baseline: eight native cases x five sessions and 12 memory cases x three sessions. The initial private Linux pilot and missing-duplicate-executable setup failure remain separate from final qualification. Unrelated name/diff controls and slower cases remain visible. + +## Native command controls + +Median paired session change, negative is faster. File I/O dominates these commands, so hot-loop savings need not produce a large native change. + +| Task | Files | Workers | Linux | macOS | +| :-------------- | -----: | ------: | -----: | -----: | +| hash-md5 | 1 | 1 | +0.34% | -0.41% | +| hash-md5 | 10,000 | 1 | -0.69% | -0.21% | +| hash-global | 1 | 1 | +2.27% | -0.51% | +| hash-global | 10,000 | 1 | +1.28% | -0.02% | +| hash-sha | 1 | 1 | +1.15% | -0.06% | +| hash-sha | 10,000 | 1 | -0.09% | -0.03% | +| hasheq-md5 | 1 | 1 | +1.35% | -0.24% | +| hasheq-md5 | 10,000 | 1 | -0.80% | +0.18% | +| hasheq-template | 1 | 1 | +0.04% | -0.11% | +| hasheq-template | 10,000 | 1 | +0.34% | -0.41% | +| hasheq-global | 1 | 1 | +0.68% | +0.10% | +| hasheq-global | 10,000 | 1 | +0.60% | -0.11% | + +## In-memory whole-engine timing + +Median session fastest-seven means; microseconds per complete run. Delta is derived from the displayed population estimates, not a native-command estimate. + +| Task | Files | Linux baseline us | Linux prepared us | Linux delta | macOS baseline us | macOS prepared us | macOS delta | +| :-------------- | -----: | ----------------: | ----------------: | ----------: | ----------------: | ----------------: | ----------: | +| hash-explicit | 10 | 12.55 | 12.47 | -0.64% | 20.58 | 20.07 | -2.46% | +| hash-explicit | 10,000 | 3,078.92 | 2,877.30 | -6.55% | 6,234.00 | 5,808.54 | -6.82% | +| hasheq-explicit | 10 | 13.33 | 13.20 | -0.94% | 21.47 | 21.03 | -2.03% | +| hasheq-explicit | 10,000 | 3,353.68 | 3,061.88 | -8.70% | 6,493.84 | 6,058.14 | -6.71% | +| hash-default | 10 | 13.84 | 14.17 | +2.41% | 21.61 | 21.42 | -0.92% | +| hash-default | 10,000 | 4,330.08 | 4,236.57 | -2.16% | 7,019.34 | 6,978.74 | -0.58% | +| hasheq-default | 10 | 15.38 | 15.29 | -0.59% | 23.30 | 23.49 | +0.81% | +| hasheq-default | 10,000 | 5,227.51 | 5,089.05 | -2.65% | 8,427.21 | 8,423.33 | -0.05% | +| name | 10 | 10.22 | 10.21 | -0.08% | 15.37 | 15.35 | -0.11% | +| name | 10,000 | 1,163.09 | 1,113.81 | -4.24% | 1,658.41 | 1,622.85 | -2.14% | +| diff-normalized | 10 | 19.99 | 19.94 | -0.25% | 29.11 | 29.60 | +1.67% | +| diff-normalized | 10,000 | 3,942.05 | 3,769.13 | -4.39% | 5,145.95 | 5,169.39 | +0.46% | + +## Preparation costs + +Nanoseconds per one-primary preparation under the existing benchmark contract. Preserve absolute costs alongside ratios; do not project these percentages onto full CLI startup. + +| Primary family | Linux baseline ns | Linux prepared ns | Linux delta | macOS baseline ns | macOS prepared ns | macOS delta | +| :------------- | ----------------: | ----------------: | ----------: | ----------------: | ----------------: | ----------: | +| hash-explicit | 168.97 | 229.03 | +35.54% | 363.39 | 436.19 | +20.03% | +| hash-default | 163.24 | 222.27 | +36.16% | 360.20 | 408.31 | +13.36% | +| hash-template | 252.89 | 321.58 | +27.16% | 501.68 | 545.17 | +8.67% | +| name | 196.06 | 222.91 | +13.70% | 418.04 | 421.50 | +0.83% | + +## Native confirmation and slower controls + +Each confirmation contains the candidate and an identical baseline executable, both relative to the baseline participant. These results supplement the initial samples. + +| Fixture / task | Workers | Initial Linux | Confirm Linux | Repeat Linux | Initial macOS | Confirm macOS | Repeat macOS | +| :------------------------ | ------: | ------------: | ------------: | -----------: | ------------: | ------------: | -----------: | +| flat10 / hash-global | 3 | +2.27% | +1.34% | -0.45% | -0.74% | +0.87% | +1.48% | +| flat10000 / listing | 1 | +2.95% | +2.46% | -0.77% | -0.17% | +0.20% | -0.01% | +| flat10000 / hash-md5 | 1 | -0.69% | -0.26% | +0.76% | -0.21% | +0.12% | +0.25% | +| flat10000 / hasheq-global | 3 | +1.18% | +1.63% | -0.38% | -0.52% | +0.98% | +0.66% | +| flat10000 / hasheq-md5 | 3 | -0.11% | +0.70% | +0.43% | +0.76% | +0.44% | +0.34% | +| flat1 / hash-sha | 3 | +2.45% | +1.48% | +0.17% | -0.23% | +1.32% | +0.98% | +| large1000 / hash-md5 | 1 | -0.21% | +0.13% | +0.02% | -0.12% | -0.38% | -0.04% | +| flat10 / listing | 3 | +2.79% | +1.08% | -0.91% | +0.11% | +0.73% | +0.02% | + +The raw artifact also retains memory confirmation, hot-kernel results, all non-displayed counts/workers, owned-byte counters and individual samples. A faster unrelated name/diff control is not evidence that its algorithm was improved by hash binding. No universal speedup or reduction in content I/O is claimed. + +## Correctness and footprint + +Tests cover all registered algorithms and both encodings, changing valid/invalid/empty direct defaults across reused and moved plans, malformed specs with content-dependent/empty expected templates, empty/binary input, read failures, short-circuiting and missing arguments. Existing evaluator/whole-run suites retain final CLI defaults, archive/VFS sources, exactly-once effects, summaries and snapshot behavior. The new fixture cannot mutate the host. + +All 322 extension-enabled release tests pass per host, with required Linux FUSE; six focused ASan suites pass per host and six Linux TSan suites pass. Scoped local lint covers all modified translation units. Normal enforcing hooks and current pushed-head CI remain required before merging. + +- linux: unstripped CLI 12,360,488 -> 12,387,664 bytes (+27,176). +- macos: unstripped CLI 4,071,248 -> 4,078,832 bytes (+7,584). + +## Lossless evidence + +[prepared-hash-operands-20261102.json.gz](prepared-hash-operands-20261102.json.gz): 1,069,985 bytes; SHA-256 `21fa9fb0387b4ae60123cc64632f12ed731451258fc6c673862a41da26dd8661`. Final, confirmation and private-pilot populations retain separate identities. diff --git a/docs/design-expression-program.md b/docs/design-expression-program.md index 79eac4a4d5f..83a5c6dd8c9 100644 --- a/docs/design-expression-program.md +++ b/docs/design-expression-program.md @@ -867,5 +867,7 @@ rendered peer paths and failures. Native CI adds literal and transformed peer, s expected-hash preparation/kernels plus three whole-engine scenarios. No speedup or final retention claim precedes those measurements. The later [O15a qualification](benchmarks/prepared-diff-configuration-20261102.md) retains complete file-diff options/ignore-regex preparation at run scope, preserving direct-context -fallback and conditional observations. Hash-spec/default resolution remains separate invariant work; -this does not close EP03/EP07 or requalify every template family. +fallback and conditional observations. The later [O15b qualification](benchmarks/prepared-hash-operands-20261102.md) +retains compact hash-primary syntax/default binding, with explicit cold preparation costs and live +context defaults. Field-hash rendering and broader template-family qualification remain separate; +these results do not close EP03/EP07. diff --git a/docs/expression-qualification-audit.md b/docs/expression-qualification-audit.md index d22fdc1570e..fec7e75e17d 100644 --- a/docs/expression-qualification-audit.md +++ b/docs/expression-qualification-audit.md @@ -79,3 +79,13 @@ dedicated direct-context tests separately compare prepared and unprepared option Retain the documented native costs as well as normalized-pair gains. This closes that subset of acceptance item 1; hash specification/default binding and remaining template-family qualification stay open. It is not evidence for an additional executor representation, optimizer or shared pool. + +## Later invariant-family qualification: O15b + +[O15b's paired report](benchmarks/prepared-hash-operands-20261102.md) qualifies compact hash-primary +algorithm/encoding preparation in the existing dispatch/operand representation. Expected fields +retain their existing template owner and conditional rendering; direct contexts retain live defaults +and malformed/missing-operand behavior. Explicit-spec hot-loop gains coexist with extra preparation +work, and native commands remain dominated by I/O. The record fits the existing operand maximum; +its creation still needs pool capacity. No new per-entry ownership wrapper or hash vocabulary is +introduced. Field-hash rendering/borrowing and the wider template/executor qualifications remain open. diff --git a/docs/performance-analysis.md b/docs/performance-analysis.md index 71547d81afe..0efebc8cbba 100644 --- a/docs/performance-analysis.md +++ b/docs/performance-analysis.md @@ -3390,3 +3390,30 @@ Linux TSan. Shipping size, raw samples and development-oracle failures are recor This is only the diff subset of O15. Peer/expected-hash templates already bind through the existing prepared operand pool. Hash specification/default resolution remains a separate candidate; EP03 and EP07 are not marked complete by this result. + +### O15b decision: compact prepared hash-primary operands + +[The final paired report](benchmarks/prepared-hash-operands-20261102.md) qualifies explicit +algorithm/encoding binding for hash primaries. Compact records index the existing algorithm registry, +retain encoding/default-use/validity bits, and reuse the expected template's unique owner. A compile-time +size check keeps the record within the existing 24-byte operand alternative. The constexpr dispatch +registers expression-aware preparation directly; argument-only families retain their missing-argument +guard through one preparation adapter. No new spelling table, per-file lock, shared owner or clock +is introduced. + +Explicit hash/equality hot work improves roughly 6-9% in confirmation on both platforms. Native +commands remain near baseline because file reads dominate. Retain the actual preparation cost: +roughly 50-70 ns additional work for a one-primary hash case, with separate percentages and absolute +values in the report. The Linux name-preparation control also costs about 27 ns; do not hide it or +project its percentage onto a full command. Duplicate baseline participants, unrelated name/diff +controls and the initial private pilot have separate provenance. Faster unrelated controls are not +credited to hash parsing. + +Direct contexts observe their current defaults when reached, including across moved/reused plans; +expected-field reads and verification bookkeeping retain their prior order. Hash actions and +verification remain coordinator work, so requested workers do not establish parallel hashing. +All 322 extension-enabled release tests pass per host; scoped lint and focused sanitizer evidence +are retained with compiler/link-action provenance. No toolchain/LTO change is part of this candidate. + +This closes the hash-primary subset of O15. Field-hash qualifier preparation and digest/output-copy +work remain associated with O22, and broader EP03/EP07 template-family qualification stays open. diff --git a/release-site.json b/release-site.json index 865fd70bc81..6fedecd359a 100644 --- a/release-site.json +++ b/release-site.json @@ -73,7 +73,8 @@ "docs/benchmarks/direct-evaluator-workers-20261102.md": "docs/benchmarks/direct-evaluator-workers-20261102.html", "docs/benchmarks/derived-content-handoff-20261102.md": "docs/benchmarks/derived-content-handoff-20261102.html", "docs/benchmarks/immutable-default-catalogs-20261102.md": "docs/benchmarks/immutable-default-catalogs-20261102.html", - "docs/benchmarks/prepared-diff-configuration-20261102.md": "docs/benchmarks/prepared-diff-configuration-20261102.html" + "docs/benchmarks/prepared-diff-configuration-20261102.md": "docs/benchmarks/prepared-diff-configuration-20261102.html", + "docs/benchmarks/prepared-hash-operands-20261102.md": "docs/benchmarks/prepared-hash-operands-20261102.html" }, "generated_html": { "XFF.md": { diff --git a/xff/engine/BUILD.bazel b/xff/engine/BUILD.bazel index 2d9d7ffb280..e8eb07932a1 100644 --- a/xff/engine/BUILD.bazel +++ b/xff/engine/BUILD.bazel @@ -1049,3 +1049,22 @@ cc_test( "@mboworks_mbo//mbo/testing:status_cc", ], ) + +cc_test( + name = "hash_pattern_test", + size = "small", + srcs = ["hash_pattern_test.cc"], + deps = [ + ":evaluate_cc", + ":walk_cc", + "//xff/hash:hash_cc", + "//xff/parser:parser_cc", + "//xff/vfs:vfs_cc", + "@abseil-cpp//absl/status", + "@abseil-cpp//absl/status:statusor", + "@abseil-cpp//absl/time", + "@googletest//:gtest_main", + "@mboworks_mbo//mbo/testing:matchers_cc", + "@mboworks_mbo//mbo/testing:status_cc", + ], +) diff --git a/xff/engine/evaluate.cc b/xff/engine/evaluate.cc index d7ab37e9fdb..780074eeaef 100644 --- a/xff/engine/evaluate.cc +++ b/xff/engine/evaluate.cc @@ -1619,6 +1619,46 @@ bool EvalDiff(const parser::Expr& expr, EvalContext& ctx) { return EvalDiffWithTemplate(expr, ctx, {}); } +// Compact IDs borrow the single hash registry, never source/default string storage. +// The expected template reuses the one owner already needed by -hasheq preparation. +struct HashPattern { + std::size_t algorithm = 0; + hash::Encoding encoding = hash::Encoding::kHex; + hash::DefaultUsage defaults; + bool valid = false; + std::unique_ptr expected; +}; + +std::optional ResolveHashPattern(const HashPattern& pattern, const EvalContext& context) { + if (!pattern.valid) { + return std::nullopt; + } + const std::string_view algorithm = pattern.defaults.algorithm + ? (context.hash_algorithm.empty() ? "sha256" : context.hash_algorithm) + : hash::AlgorithmNames().at(pattern.algorithm); + if (pattern.defaults.algorithm && !hash::IsAlgorithm(algorithm)) { + return std::nullopt; + } + return hash::AlgoEncoding{ + .algo = algorithm, + .encoding = pattern.defaults.encoding ? hash::ParseEncoding(context.hash_encoding).value_or(hash::Encoding::kHex) + : pattern.encoding, + .defaults = pattern.defaults, + }; +} + +std::optional ResolveHashRequest( + const parser::Expr& expression, + const EvalContext& context, + mbo::types::OptionalRef prepared) { + if (prepared) { + return ResolveHashPattern(*prepared, context); + } + const std::string_view algorithm = context.hash_algorithm.empty() ? "sha256" : context.hash_algorithm; + const auto encoding = hash::ParseEncoding(context.hash_encoding).value_or(hash::Encoding::kHex); + return hash::ParseSpec(expression.hash_spec, algorithm, encoding); +} + // The entry's digest, read through the filesystem the entry came FROM. That indirection is what // lets -hash / -hasheq work on an archive member: hashing by path would look for `a.tar!x` on the // real filesystem and find nothing. Both routes read the whole entry anyway. @@ -1632,11 +1672,11 @@ std::optional DigestOfEntry(const EvalContext& ctx, const hash::Alg // --hash-encoding (sha256 / hex). Reads the file (Cost::kExpensive); an unreadable file emits // nothing. Like -print it is always true. The spec was validated before the walk // (ValidateHashArgs), so a parse failure here defensively no-ops. -// NOLINTNEXTLINE(misc-const-correctness): EvalFn requires a mutable context. -bool EvalHash(const parser::Expr& expr, EvalContext& ctx) { - const std::string_view default_algo = ctx.hash_algorithm.empty() ? "sha256" : ctx.hash_algorithm; - const hash::Encoding default_encoding = hash::ParseEncoding(ctx.hash_encoding).value_or(hash::Encoding::kHex); - const std::optional spec = hash::ParseSpec(expr.hash_spec, default_algo, default_encoding); +bool EvalHashWithOperand( + const parser::Expr& expr, + const EvalContext& ctx, + mbo::types::OptionalRef prepared) { + const auto spec = ResolveHashRequest(expr, ctx, prepared); if (!spec.has_value()) { return true; } @@ -1647,6 +1687,11 @@ bool EvalHash(const parser::Expr& expr, EvalContext& ctx) { return true; } +// NOLINTNEXTLINE(misc-const-correctness): EvalFn requires a mutable context. +bool EvalHash(const parser::Expr& expression, EvalContext& context) { + return EvalHashWithOperand(expression, context, {}); +} + // xff -hasheq EXPECTED: TRUE when the entry's digest equals EXPECTED, the manifest-verification // companion of -hash. EXPECTED is a field template rendered per entry (so `-hasheq {def.SUMS}` // checks against a sidecar value, and `! -hasheq {def.SUMS}` lists drift). The algorithm / encoding @@ -1655,10 +1700,11 @@ bool EvalHash(const parser::Expr& expr, EvalContext& ctx) { // EXPECTED, an unreadable file, or a bad spec is FALSE (no match), so drift-selection is safe. // Cost::kExpensive (reads the whole file). // NOLINTNEXTLINE(misc-const-correctness): EvalFn requires a mutable context. -bool EvalHasheqWithTemplate( +bool EvalHasheqWithOperand( const parser::Expr& expr, EvalContext& ctx, - mbo::types::OptionalRef prepared) { + mbo::types::OptionalRef prepared, + mbo::types::OptionalRef pattern) { const auto verdict = [&](bool matched) { if (ctx.hash_verification.has_value()) { *ctx.hash_verification = matched; @@ -1672,9 +1718,7 @@ bool EvalHasheqWithTemplate( if (expected.empty()) { return verdict(false); // no expected hash resolved (e.g. an unset {def.X}) -> mismatch } - const std::string_view default_algo = ctx.hash_algorithm.empty() ? "sha256" : ctx.hash_algorithm; - const hash::Encoding default_encoding = hash::ParseEncoding(ctx.hash_encoding).value_or(hash::Encoding::kHex); - const std::optional spec = hash::ParseSpec(expr.hash_spec, default_algo, default_encoding); + const auto spec = ResolveHashRequest(expr, ctx, pattern); if (!spec.has_value()) { return verdict(false); // defensively no-op; ValidateHashArgs rejects a bad spec before the walk } @@ -1689,7 +1733,7 @@ bool EvalHasheqWithTemplate( } bool EvalHasheq(const parser::Expr& expr, EvalContext& ctx) { - return EvalHasheqWithTemplate(expr, ctx, {}); + return EvalHasheqWithOperand(expr, ctx, {}, {}); } // A view over one explicit grep pattern or the union used by default match output. @@ -2978,9 +3022,66 @@ using PreparedOperand = std::variant< FirstLimit, MimePattern, GlobPattern, - FieldTemplate>; + FieldTemplate, + HashPattern>; + +static_assert(sizeof(HashPattern) <= sizeof(MimePattern)); using PreparedEvalFn = bool (*)(const parser::Expr&, const PreparedOperand&, MatcherSlots, EvalContext&); -using PrepareOperandFn = PreparedOperand (*)(std::string_view); +using PrepareOperandFn = PreparedOperand (*)(const parser::Expr&); + +template +PreparedOperand PrepareArgument(const parser::Expr& expression) { + return Prepare(expression.args.front()); +} + +HashPattern ParseHashPattern(std::string_view specification) { + const auto spec = hash::ParseSpec(specification, "sha256"); + if (!spec) { + return {}; + } + const auto names = hash::AlgorithmNames(); + return HashPattern{ + .algorithm = static_cast(std::ranges::find(names, spec->algo) - names.begin()), + .encoding = spec->encoding, + .defaults = spec->defaults, + .valid = true, + }; +} + +PreparedOperand PrepareHashOperand(const parser::Expr& expression) { + return ParseHashPattern(expression.hash_spec); +} + +PreparedOperand PrepareExpectedHashOperand(const parser::Expr& expression) { + auto pattern = ParseHashPattern(expression.hash_spec); + pattern.expected = std::make_unique(fields::Template::Compile(expression.args.front())); + return pattern; +} + +bool EvalPreparedHash( + const parser::Expr& expression, + const PreparedOperand& operand, + MatcherSlots, + // NOLINTNEXTLINE(misc-const-correctness): PreparedEvalFn requires a mutable context. + EvalContext& context) { + return EvalHashWithOperand( + expression, context, + std::holds_alternative(operand) + ? mbo::types::OptionalRef{std::get(operand)} + : mbo::types::OptionalRef{}); +} + +bool EvalPreparedExpectedHash( + const parser::Expr& expression, + const PreparedOperand& operand, + MatcherSlots, + EvalContext& context) { + if (!std::holds_alternative(operand)) { + return EvalHasheq(expression, context); + } + const auto& pattern = std::get(operand); + return EvalHasheqWithOperand(expression, context, *pattern.expected, pattern); +} PreparedOperand PrepareGlobOperand(std::string_view argument) { // Match the original C-string boundary, including callers that supply embedded NUL bytes. @@ -3292,6 +3393,7 @@ struct EvalEntry { PrepareOperandFn prepare = nullptr; PreparedEvalFn prepared_eval = nullptr; bool indexed_matcher = false; + bool prepare_without_argument = false; }; template @@ -3303,12 +3405,14 @@ template consteval EvalEntry MakeEvalEntry( PrepareOperandFn prepare = nullptr, PreparedEvalFn prepared_eval = nullptr, - bool indexed_matcher = false) { + bool indexed_matcher = false, + bool prepare_without_argument = false) { return { .eval = Function, .prepare = prepare, .prepared_eval = prepared_eval == nullptr ? &EvalOriginalOperand : prepared_eval, .indexed_matcher = indexed_matcher, + .prepare_without_argument = prepare_without_argument, }; } @@ -3316,32 +3420,40 @@ using DispatchPair = std::pair; constexpr auto kDispatch = mbo::container::MakeLimitedMap( DispatchPair{ "-Bmin", - MakeEvalEntry<&EvalBmin>(&PrepareTimeOperand<60, false>, &EvalPreparedTime<'B'>), + MakeEvalEntry<&EvalBmin>(&PrepareArgument<&PrepareTimeOperand<60, false>>, &EvalPreparedTime<'B'>), }, DispatchPair{ "-Btime", - MakeEvalEntry<&EvalBtime>(&PrepareTimeOperand<86'400, true>, &EvalPreparedTime<'B'>), + MakeEvalEntry<&EvalBtime>(&PrepareArgument<&PrepareTimeOperand<86'400, true>>, &EvalPreparedTime<'B'>), }, - DispatchPair{"-amin", MakeEvalEntry<&EvalAmin>(&PrepareTimeOperand<60, false>, &EvalPreparedTime<'a'>)}, + DispatchPair{ + "-amin", MakeEvalEntry<&EvalAmin>(&PrepareArgument<&PrepareTimeOperand<60, false>>, &EvalPreparedTime<'a'>)}, DispatchPair{"-anewer", MakeEvalEntry<&EvalAnewer>()}, - DispatchPair{"-atime", MakeEvalEntry<&EvalAtime>(&PrepareTimeOperand<86'400, true>, &EvalPreparedTime<'a'>)}, + DispatchPair{ + "-atime", + MakeEvalEntry<&EvalAtime>(&PrepareArgument<&PrepareTimeOperand<86'400, true>>, &EvalPreparedTime<'a'>)}, DispatchPair{"-binary", MakeEvalEntry<&EvalBinary>()}, - DispatchPair{"-blocks", MakeEvalEntry<&EvalBlocks>(&PrepareSizeOperand, &EvalPreparedBlocks)}, + DispatchPair{"-blocks", MakeEvalEntry<&EvalBlocks>(&PrepareArgument<&PrepareSizeOperand>, &EvalPreparedBlocks)}, DispatchPair{"-capture", MakeEvalEntry<&EvalCapture>()}, DispatchPair{"-capturedir", MakeEvalEntry<&EvalCapturedir>()}, - DispatchPair{"-cmin", MakeEvalEntry<&EvalCmin>(&PrepareTimeOperand<60, false>, &EvalPreparedTime<'c'>)}, + DispatchPair{ + "-cmin", MakeEvalEntry<&EvalCmin>(&PrepareArgument<&PrepareTimeOperand<60, false>>, &EvalPreparedTime<'c'>)}, DispatchPair{ "-cmp", - MakeEvalEntry<&EvalCmp>(&PrepareTemplateOperand, &EvalPreparedTemplate<&EvalCmpWithTemplate>), + MakeEvalEntry<&EvalCmp>(&PrepareArgument<&PrepareTemplateOperand>, &EvalPreparedTemplate<&EvalCmpWithTemplate>), }, DispatchPair{"-cnewer", MakeEvalEntry<&EvalCnewer>()}, DispatchPair{"-collect", MakeEvalEntry<&EvalCollect>()}, DispatchPair{"-content", MakeEvalEntry<&EvalContent>()}, - DispatchPair{"-ctime", MakeEvalEntry<&EvalCtime>(&PrepareTimeOperand<86'400, true>, &EvalPreparedTime<'c'>)}, + DispatchPair{ + "-ctime", + MakeEvalEntry<&EvalCtime>(&PrepareArgument<&PrepareTimeOperand<86'400, true>>, &EvalPreparedTime<'c'>)}, DispatchPair{"-delete", MakeEvalEntry<&EvalDelete>()}, DispatchPair{ "-diff", - MakeEvalEntry<&EvalDiff>(&PrepareTemplateOperand, &EvalPreparedTemplate<&EvalDiffWithTemplate>), + MakeEvalEntry<&EvalDiff>( + &PrepareArgument<&PrepareTemplateOperand>, + &EvalPreparedTemplate<&EvalDiffWithTemplate>), }, DispatchPair{"-empty", MakeEvalEntry<&EvalEmpty>()}, DispatchPair{"-eofcr", MakeEvalEntry<&EvalEofcr>()}, @@ -3358,36 +3470,45 @@ constexpr auto kDispatch = mbo::container::MakeLimitedMap( DispatchPair{"-fprintfln", MakeEvalEntry<&EvalFprintfln>()}, DispatchPair{"-fprintln", MakeEvalEntry<&EvalFprintln>()}, DispatchPair{"-fstype", MakeEvalEntry<&EvalFstype>()}, - DispatchPair{"-gid", MakeEvalEntry<&EvalGid>(&PrepareNumericOperand, &EvalPreparedNumeric<&vfs::Metadata::gid>)}, + DispatchPair{ + "-gid", + MakeEvalEntry<&EvalGid>(&PrepareArgument<&PrepareNumericOperand>, &EvalPreparedNumeric<&vfs::Metadata::gid>)}, DispatchPair{"-grep", MakeEvalEntry<&EvalGrep>()}, DispatchPair{"-group", MakeEvalEntry<&EvalGroup>()}, - DispatchPair{"-hash", MakeEvalEntry<&EvalHash>()}, + DispatchPair{"-hash", MakeEvalEntry<&EvalHash>(&PrepareHashOperand, &EvalPreparedHash, false, true)}, DispatchPair{ "-hasheq", - MakeEvalEntry<&EvalHasheq>(&PrepareTemplateOperand, &EvalPreparedTemplate<&EvalHasheqWithTemplate>), + MakeEvalEntry<&EvalHasheq>(&PrepareExpectedHashOperand, &EvalPreparedExpectedHash), }, DispatchPair{"-icontent", MakeEvalEntry<&EvalContent>()}, DispatchPair{"-ilname", MakeEvalEntry<&EvalLname>()}, - DispatchPair{"-iname", MakeEvalEntry<&EvalName>(&PrepareGlobOperand, &EvalPreparedName)}, - DispatchPair{"-inum", MakeEvalEntry<&EvalInum>(&PrepareNumericOperand, &EvalPreparedNumeric<&vfs::Metadata::ino>)}, - DispatchPair{"-ipath", MakeEvalEntry<&EvalPath>(&PrepareGlobOperand, &EvalPreparedPath)}, + DispatchPair{"-iname", MakeEvalEntry<&EvalName>(&PrepareArgument<&PrepareGlobOperand>, &EvalPreparedName)}, + DispatchPair{ + "-inum", + MakeEvalEntry<&EvalInum>(&PrepareArgument<&PrepareNumericOperand>, &EvalPreparedNumeric<&vfs::Metadata::ino>)}, + DispatchPair{"-ipath", MakeEvalEntry<&EvalPath>(&PrepareArgument<&PrepareGlobOperand>, &EvalPreparedPath)}, DispatchPair{"-iregex", MakeEvalEntry<&EvalRegex>(nullptr, &EvalPreparedRegex, true)}, DispatchPair{"-irxc", MakeEvalEntry<&EvalRxc>(nullptr, &EvalPreparedRxc, true)}, - DispatchPair{"-iwholename", MakeEvalEntry<&EvalPath>(&PrepareGlobOperand, &EvalPreparedPath)}, + DispatchPair{"-iwholename", MakeEvalEntry<&EvalPath>(&PrepareArgument<&PrepareGlobOperand>, &EvalPreparedPath)}, DispatchPair{"-lang", MakeEvalEntry<&EvalLang>()}, DispatchPair{ - "-links", MakeEvalEntry<&EvalLinks>(&PrepareNumericOperand, &EvalPreparedNumeric<&vfs::Metadata::nlink>)}, + "-links", MakeEvalEntry<&EvalLinks>( + &PrepareArgument<&PrepareNumericOperand>, + &EvalPreparedNumeric<&vfs::Metadata::nlink>)}, DispatchPair{"-lname", MakeEvalEntry<&EvalLname>()}, DispatchPair{"-ls", MakeEvalEntry<&EvalLs>()}, - DispatchPair{"-mime", MakeEvalEntry<&EvalMime>(&PrepareMimeOperand, &EvalPreparedMime)}, - DispatchPair{"-mmin", MakeEvalEntry<&EvalMmin>(&PrepareTimeOperand<60, false>, &EvalPreparedTime<'m'>)}, - DispatchPair{"-mtime", MakeEvalEntry<&EvalMtime>(&PrepareTimeOperand<86'400, true>, &EvalPreparedTime<'m'>)}, - DispatchPair{"-first", MakeEvalEntry<&EvalFirst>(&PrepareFirstOperand, &EvalPreparedFirst)}, + DispatchPair{"-mime", MakeEvalEntry<&EvalMime>(&PrepareArgument<&PrepareMimeOperand>, &EvalPreparedMime)}, + DispatchPair{ + "-mmin", MakeEvalEntry<&EvalMmin>(&PrepareArgument<&PrepareTimeOperand<60, false>>, &EvalPreparedTime<'m'>)}, + DispatchPair{ + "-mtime", + MakeEvalEntry<&EvalMtime>(&PrepareArgument<&PrepareTimeOperand<86'400, true>>, &EvalPreparedTime<'m'>)}, + DispatchPair{"-first", MakeEvalEntry<&EvalFirst>(&PrepareArgument<&PrepareFirstOperand>, &EvalPreparedFirst)}, DispatchPair{"-fuzzy", MakeEvalEntry<&EvalFuzzy>()}, DispatchPair{"-fuzzypath", MakeEvalEntry<&EvalFuzzyPath>()}, DispatchPair{"-ifuzzy", MakeEvalEntry<&EvalFuzzy>()}, DispatchPair{"-ifuzzypath", MakeEvalEntry<&EvalFuzzyPath>()}, - DispatchPair{"-name", MakeEvalEntry<&EvalName>(&PrepareGlobOperand, &EvalPreparedName)}, + DispatchPair{"-name", MakeEvalEntry<&EvalName>(&PrepareArgument<&PrepareGlobOperand>, &EvalPreparedName)}, DispatchPair{"-newer", MakeEvalEntry<&EvalNewer>()}, DispatchPair{"-newerBB", MakeEvalEntry<&EvalNewerXY>()}, // birthtime -newerXY combos (BSD-compat) DispatchPair{"-newerBa", MakeEvalEntry<&EvalNewerXY>()}, @@ -3413,8 +3534,9 @@ constexpr auto kDispatch = mbo::container::MakeLimitedMap( DispatchPair{"-nouser", MakeEvalEntry<&EvalNouser>()}, DispatchPair{"-ok", MakeEvalEntry<&EvalOk>()}, DispatchPair{"-okdir", MakeEvalEntry<&EvalOkdir>()}, - DispatchPair{"-path", MakeEvalEntry<&EvalPath>(&PrepareGlobOperand, &EvalPreparedPath)}, - DispatchPair{"-perm", MakeEvalEntry<&EvalPerm>(&PreparePermissionOperand, &EvalPreparedPermission)}, + DispatchPair{"-path", MakeEvalEntry<&EvalPath>(&PrepareArgument<&PrepareGlobOperand>, &EvalPreparedPath)}, + DispatchPair{ + "-perm", MakeEvalEntry<&EvalPerm>(&PrepareArgument<&PreparePermissionOperand>, &EvalPreparedPermission)}, DispatchPair{"-print", MakeEvalEntry<&EvalPrint>()}, DispatchPair{"-print0", MakeEvalEntry<&EvalPrint0>()}, DispatchPair{"-printf", MakeEvalEntry<&EvalPrintf>()}, @@ -3424,7 +3546,9 @@ constexpr auto kDispatch = mbo::container::MakeLimitedMap( DispatchPair{"-quit", MakeEvalEntry<&EvalQuit>()}, DispatchPair{ "-rdiff", - MakeEvalEntry<&EvalDiff>(&PrepareTemplateOperand, &EvalPreparedTemplate<&EvalDiffWithTemplate>), + MakeEvalEntry<&EvalDiff>( + &PrepareArgument<&PrepareTemplateOperand>, + &EvalPreparedTemplate<&EvalDiffWithTemplate>), }, DispatchPair{"-readable", MakeEvalEntry<&EvalReadable>()}, DispatchPair{"-regex", MakeEvalEntry<&EvalRegex>(nullptr, &EvalPreparedRegex, true)}, @@ -3432,19 +3556,23 @@ constexpr auto kDispatch = mbo::container::MakeLimitedMap( DispatchPair{"-samefile", MakeEvalEntry<&EvalSamefile>()}, DispatchPair{ "-similar", - MakeEvalEntry<&EvalSimilar>(&PrepareTemplateOperand, &EvalPreparedTemplate<&EvalSimilarWithTemplate>), + MakeEvalEntry<&EvalSimilar>( + &PrepareArgument<&PrepareTemplateOperand>, + &EvalPreparedTemplate<&EvalSimilarWithTemplate>), }, - DispatchPair{"-size", MakeEvalEntry<&EvalSize>(&PrepareSizeOperand, &EvalPreparedSize)}, + DispatchPair{"-size", MakeEvalEntry<&EvalSize>(&PrepareArgument<&PrepareSizeOperand>, &EvalPreparedSize)}, DispatchPair{"-sparse", MakeEvalEntry<&EvalSparse>()}, DispatchPair{"-text", MakeEvalEntry<&EvalText>()}, DispatchPair{"-true", MakeEvalEntry<&EvalTrue>()}, - DispatchPair{"-type", MakeEvalEntry<&EvalType>(&PrepareTypeOperand, &EvalPreparedType)}, - DispatchPair{"-uid", MakeEvalEntry<&EvalUid>(&PrepareNumericOperand, &EvalPreparedNumeric<&vfs::Metadata::uid>)}, - DispatchPair{"-used", MakeEvalEntry<&EvalUsed>(&PrepareSignedNumericOperand, &EvalPreparedUsed)}, + DispatchPair{"-type", MakeEvalEntry<&EvalType>(&PrepareArgument<&PrepareTypeOperand>, &EvalPreparedType)}, + DispatchPair{ + "-uid", + MakeEvalEntry<&EvalUid>(&PrepareArgument<&PrepareNumericOperand>, &EvalPreparedNumeric<&vfs::Metadata::uid>)}, + DispatchPair{"-used", MakeEvalEntry<&EvalUsed>(&PrepareArgument<&PrepareSignedNumericOperand>, &EvalPreparedUsed)}, DispatchPair{"-user", MakeEvalEntry<&EvalUser>()}, - DispatchPair{"-wholename", MakeEvalEntry<&EvalPath>(&PrepareGlobOperand, &EvalPreparedPath)}, + DispatchPair{"-wholename", MakeEvalEntry<&EvalPath>(&PrepareArgument<&PrepareGlobOperand>, &EvalPreparedPath)}, DispatchPair{"-writable", MakeEvalEntry<&EvalWritable>()}, - DispatchPair{"-xtype", MakeEvalEntry<&EvalXtype>(&PrepareTypeOperand, &EvalPreparedXtype)}); + DispatchPair{"-xtype", MakeEvalEntry<&EvalXtype>(&PrepareArgument<&PrepareTypeOperand>, &EvalPreparedXtype)}); absl::StatusOr ResolveEvaluation(const parser::Expr& expression) { if (expression.kind != parser::Expr::Kind::kPredicate) { @@ -3916,7 +4044,8 @@ absl::StatusOr PreparedExpression::Prepare(const parser::Exp .expression = source.expression, .evaluate = binding.prepared_eval, }; - if (binding.indexed_matcher || (binding.prepare != nullptr && !expr.args.empty())) { + if (binding.indexed_matcher + || (binding.prepare != nullptr && (!expr.args.empty() || binding.prepare_without_argument))) { if (data->operands.size() == 1) { data->operands.reserve(1 + static_cast(predicates)); } @@ -3925,7 +4054,7 @@ absl::StatusOr PreparedExpression::Prepare(const parser::Exp data->operands.emplace_back(MatcherSlotId{data->matcher_sources.size()}); data->matcher_sources.push_back(source.expression); } else { - data->operands.push_back(binding.prepare(expr.args.front())); + data->operands.push_back(binding.prepare(expr)); } } data->nodes.push_back(node); @@ -3986,6 +4115,11 @@ std::size_t PreparedExpression::StorageBytes() const { text_bytes += std::get(operand).bytes.capacity(); } else if (std::holds_alternative(operand)) { text_bytes += std::get(operand).value->StorageBytes(); + } else if (std::holds_alternative(operand)) { + const auto& pattern = std::get(operand); + if (pattern.expected) { + text_bytes += pattern.expected->StorageBytes(); + } } } return sizeof(*this) + sizeof(Data) + (data_->nodes.capacity() * sizeof(Data::Node)) diff --git a/xff/engine/expression_benchmark.cc b/xff/engine/expression_benchmark.cc index 9de8c81daf1..6371eb67859 100644 --- a/xff/engine/expression_benchmark.cc +++ b/xff/engine/expression_benchmark.cc @@ -133,6 +133,29 @@ constexpr auto kCases = std::to_array({ {.name = "compare-template", .primary = "-cmp", .argument = "peer/{name:s/file/copy/}"}, {.name = "similar-template", .primary = "-similar", .argument = "peer/{name}"}, {.name = "diff-template", .primary = "-diff", .argument = "peer/{name}"}, + { + .name = "hash-explicit", + .primary = "-hash:md5/base64", + }, + { + .name = "hash-default", + .primary = "-hash", + }, + { + .name = "hasheq-explicit", + .primary = "-hasheq:md5/base64", + .argument = "V2pFZbcPWkwaCSXKvbWHpg==", + }, + { + .name = "hasheq-encoding-default", + .primary = "-hasheq:md5", + .argument = "576a4565b70f5a4c1a0925cabdb587a6", + }, + { + .name = "hasheq-algorithm-default", + .primary = "-hasheq:/base64", + .argument = "0pIQd3d32sCz0S9qZWoHPJunF89pMtvAGwzG3B53ebg=", + }, {.name = "hash-template", .primary = "-hasheq", .argument = "{hash}"}, {.name = "first-limit", .primary = "-first", .argument = "2147483647"}, {.name = "mime", .primary = "-mime", .argument = "TEXT/*"}, diff --git a/xff/engine/expression_run_benchmark.cc b/xff/engine/expression_run_benchmark.cc index 248447e7df6..45ba042c4ff 100644 --- a/xff/engine/expression_run_benchmark.cc +++ b/xff/engine/expression_run_benchmark.cc @@ -171,6 +171,22 @@ int main(int argc, char** argv) { .arguments = {"--diff-ignore-matching=^(needle|absent|reference)$", "-false", "-diff", "peer/diff-control"}, }, {.name = "diff-template", .arguments = {"-type", "f", "-diff", "peer/{relpath}"}}, + { + .name = "hash-explicit", + .arguments = {"-type", "f", "-hash:md5/base64"}, + }, + { + .name = "hash-default", + .arguments = {"-type", "f", "-hash"}, + }, + { + .name = "hasheq-explicit", + .arguments = {"-type", "f", "-hasheq:md5/base64", "V2pFZbcPWkwaCSXKvbWHpg=="}, + }, + { + .name = "hasheq-default", + .arguments = {"-type", "f", "-hasheq", "{hash}"}, + }, {.name = "hash-template", .arguments = {"-type", "f", "-hasheq", "{hash}"}}, {.name = "first-limit", .arguments = {"-type", "f", "-first", "3", "-print"}}, {.name = "mime", .arguments = {"-type", "f", "-mime", "TEXT/*"}}, diff --git a/xff/engine/hash_pattern_test.cc b/xff/engine/hash_pattern_test.cc new file mode 100644 index 00000000000..440226d10ea --- /dev/null +++ b/xff/engine/hash_pattern_test.cc @@ -0,0 +1,224 @@ +// SPDX-FileCopyrightText: Copyright (c) M. Boerger, the MBO Works authors +// SPDX-License-Identifier: Apache-2.0 + +#include +#include +#include +#include +#include +#include +#include + +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/time/time.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "mbo/testing/matchers.h" +#include "mbo/testing/status.h" +#include "xff/engine/evaluate.h" +#include "xff/hash/hash.h" +#include "xff/parser/parser.h" +#include "xff/vfs/filesystem.h" + +namespace xff::engine { +namespace { +using ::mbo::testing::EqualsText; +using ::testing::Eq; +using ::testing::IsEmpty; +using ::testing::IsFalse; +using ::testing::IsTrue; +using ::testing::Optional; + +struct HashFs final : vfs::FileSystem { + std::string bytes = "a\nb\n"; + bool fail = false; + mutable std::size_t reads = 0; + + absl::StatusOr> ReadDir(std::string_view) const override { return std::vector{}; } + + absl::StatusOr Stat(std::string_view, bool) const override { + return vfs::Metadata{.type = vfs::FileType::kRegular, .size = bytes.size()}; + } + + absl::StatusOr ReadContent(std::string_view) const override { + ++reads; + return fail ? absl::StatusOr{absl::PermissionDeniedError("denied fixture")} : bytes; + } + + absl::Status Remove(std::string_view) const override { + return absl::PermissionDeniedError("isolated read-only input"); + } + + bool Access(std::string_view, vfs::AccessMode) const override { return true; } + + absl::StatusOr ReadLink(std::string_view) const override { return absl::NotFoundError("not a symlink"); } + + absl::StatusOr FsType(std::string_view) const override { return "memory"; } + + absl::StatusOr IsCaseSensitive(std::string_view) const override { return true; } +}; + +struct Observation { + bool matched = false; + std::string text; + std::size_t reads = 0; + std::optional verified; +}; + +struct Defaults { + std::string_view algorithm; + std::string_view encoding; +}; + +struct Input { + std::string_view bytes = "a\nb\n"; + bool fail = false; +}; + +struct HashPatternTest : ::testing::Test { + static Observation Observe( + const parser::Expr& expression, + mbo::types::OptionalRef prepared, + Defaults defaults, + Input input = {}) { + HashFs filesystem; + filesystem.bytes = input.bytes; + filesystem.fail = input.fail; + Observation result; + Control control; + const auto sink = [&](std::string_view text) { result.text.append(text); }; + const Visit visit{ + .path = "root/file", + .name = "file", + .root = "root", + .metadata = {.type = vfs::FileType::kRegular, .size = input.bytes.size()}, + }; + EvalContext context{ + .visit = visit, + .emit = sink, + .fs = filesystem, + .now = absl::UnixEpoch(), + .tz = absl::UTCTimeZone(), + .hash_verification = result.verified, + .hash_algorithm = defaults.algorithm, + .hash_encoding = defaults.encoding, + .control = control, + }; + result.matched = prepared ? prepared->Evaluate(context).matched : Evaluate(expression, context); + result.reads = filesystem.reads; + return result; + } +}; + +TEST_F(HashPatternTest, ReusedMovedPlansObserveEachContextsDefaultsWithoutOwningViews) { + constexpr auto kDefaults = std::to_array({ + {}, + {.algorithm = "md5", .encoding = "base64"}, + {.algorithm = "sha1", .encoding = "hex"}, + {.algorithm = "garbage", .encoding = "base64"}, + {.algorithm = "sha512", .encoding = "garbage"}, + }); + constexpr auto kRequests = std::to_array({ + "-hash", + "-hash:md5/base64", + "-hash:/base64", + "-hash:sha1/", + }); + for (const auto request : kRequests) { + SCOPED_TRACE(request); + ASSERT_OK_AND_ASSIGN(const auto command, parser::Parse({"root", std::string(request)})); + ASSERT_OK_AND_ASSIGN(auto initial, PreparedExpression::Prepare(*command.expression)); + const auto prepared = std::move(initial); + for (const auto defaults : kDefaults) { + SCOPED_TRACE(defaults.algorithm); + SCOPED_TRACE(defaults.encoding); + const auto reference = Observe(*command.expression, {}, defaults); + const auto current = Observe(*command.expression, prepared, defaults); + EXPECT_THAT(current.matched, Eq(reference.matched)); + EXPECT_THAT(current.text, EqualsText(reference.text)); + EXPECT_THAT(current.reads, Eq(reference.reads)); + } + } +} + +TEST_F(HashPatternTest, EveryRegisteredAlgorithmAndEncodingRetainsItsDigestOutput) { + constexpr auto kEncodings = std::to_array({ + hash::Encoding::kHex, + hash::Encoding::kBase64, + }); + for (const auto algorithm : hash::AlgorithmNames()) { + for (const auto encoding : kEncodings) { + const std::string option = + "-hash:" + std::string(algorithm) + (encoding == hash::Encoding::kHex ? "/hex" : "/base64"); + SCOPED_TRACE(option); + ASSERT_OK_AND_ASSIGN(const auto command, parser::Parse({"root", option})); + ASSERT_OK_AND_ASSIGN(const auto prepared, PreparedExpression::Prepare(*command.expression)); + const auto result = Observe(*command.expression, prepared, {.algorithm = "garbage", .encoding = "garbage"}); + const auto digest = hash::HashData(algorithm, "a\nb\n", encoding); + EXPECT_THAT( + digest.transform([](const std::string& value) { return value + " root/file\n"; }), + Optional(EqualsText(result.text))); + EXPECT_THAT(result.reads, Eq(1)); + EXPECT_THAT(result.matched, IsTrue()); + } + } +} + +TEST_F(HashPatternTest, ByteInputsFailuresAndSkippedHashActionsRetainObservations) { + const auto inputs = std::to_array({ + {.bytes = ""}, + {.bytes = std::string_view("a\0b", 3)}, + {.bytes = "unused", .fail = true}, + }); + ASSERT_OK_AND_ASSIGN(const auto command, parser::Parse({"root", "-hash:md5/base64"})); + ASSERT_OK_AND_ASSIGN(const auto prepared, PreparedExpression::Prepare(*command.expression)); + for (const auto input : inputs) { + const auto reference = Observe(*command.expression, {}, {}, input); + const auto current = Observe(*command.expression, prepared, {}, input); + EXPECT_THAT(current.matched, Eq(reference.matched)); + EXPECT_THAT(current.text, EqualsText(reference.text)); + EXPECT_THAT(current.reads, Eq(reference.reads)); + } + ASSERT_OK_AND_ASSIGN(const auto skipped, parser::Parse({"root", "-false", "-hash:md5/base64"})); + ASSERT_OK_AND_ASSIGN(const auto skip_plan, PreparedExpression::Prepare(*skipped.expression)); + const auto result = Observe(*skipped.expression, skip_plan, {}); + EXPECT_THAT(result.matched, IsFalse()); + EXPECT_THAT(result.text, IsEmpty()); + EXPECT_THAT(result.reads, Eq(0)); + EXPECT_THAT(result.verified, Eq(std::nullopt)); +} + +TEST_F(HashPatternTest, MissingExpectedArgumentRemainsFalseWithoutPreparingAnOperand) { + ASSERT_OK_AND_ASSIGN(auto command, parser::Parse({"root", "-hasheq", "unused"})); + command.expression->args.clear(); + ASSERT_OK_AND_ASSIGN(const auto prepared, PreparedExpression::Prepare(*command.expression)); + const auto result = Observe(*command.expression, prepared, {}); + EXPECT_THAT(result.matched, IsFalse()); + EXPECT_THAT(result.text, IsEmpty()); + EXPECT_THAT(result.reads, Eq(0)); + EXPECT_THAT(result.verified, Optional(IsFalse())); + EXPECT_THAT(prepared.OperandCount(), Eq(0)); +} + +TEST_F(HashPatternTest, MalformedSpecificationsAndExpectedTemplatesRetainConditionalReads) { + constexpr auto kExpected = std::to_array({ + "{lines}", + "{def.MISSING}", + "", + }); + for (const auto expected : kExpected) { + ASSERT_OK_AND_ASSIGN(auto command, parser::Parse({"root", "-hasheq", std::string(expected)})); + command.expression->hash_spec = "bad-algorithm"; + ASSERT_OK_AND_ASSIGN(const auto prepared, PreparedExpression::Prepare(*command.expression)); + const auto reference = Observe(*command.expression, {}, {}); + const auto current = Observe(*command.expression, prepared, {}); + EXPECT_THAT(current.matched, IsFalse()); + EXPECT_THAT(current.text, IsEmpty()); + EXPECT_THAT(current.reads, Eq(reference.reads)); + EXPECT_THAT(current.verified, Optional(IsFalse())); + EXPECT_THAT(reference.verified, Optional(IsFalse())); + } +} +} // namespace +} // namespace xff::engine