diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a5448ab0ae0..1c9714707e8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -81,7 +81,7 @@ repos: description: Use ASCII outside explicitly listed Unicode test files and graphical output locations. language: pygrep entry: '[^\x00-\x7f]' - exclude: ^xff/content/word_unicode_test\.cc$|^xff/engine/rg_unicode_test\.cc$|^tools/release_smoke_unicode\.py$|^xff/cli/csv_unicode_test\.sh$|^xff/presentation/render/(render|scoped_table)_unicode_test\.cc$|^xff/presentation/format/text_width_unicode_test\.cc$|^README\.md$|^(docs/history-roadmap\.md|xff/cli/histogram_test\.sh|xff/engine/run\.cc|xff/presentation/render/render(_test)?\.cc)$|\.(phar|tar|tgz|gz|bz2|xz|zst|br|zip|jar|war|ear|apk|aab|whl|egg|nupkg|vsix|xpi|crx|jmod|deb|rpm|gem|crate|asar|7z|sqfs|png|ico)$ + exclude: ^xff/engine/glob_unicode_test\.cc$|^xff/content/word_unicode_test\.cc$|^xff/engine/rg_unicode_test\.cc$|^tools/release_smoke_unicode\.py$|^xff/cli/csv_unicode_test\.sh$|^xff/presentation/render/(render|scoped_table)_unicode_test\.cc$|^xff/presentation/format/text_width_unicode_test\.cc$|^README\.md$|^(docs/history-roadmap\.md|xff/cli/histogram_test\.sh|xff/engine/run\.cc|xff/presentation/render/render(_test)?\.cc)$|\.(phar|tar|tgz|gz|bz2|xz|zst|br|zip|jar|war|ear|apk|aab|whl|egg|nupkg|vsix|xpi|crx|jmod|deb|rpm|gem|crate|asar|7z|sqfs|png|ico)$ types: [text] - id: histogram-doc-glyphs name: Histogram text uses only approved Unicode block glyphs diff --git a/CHANGELOG.md b/CHANGELOG.md index 3662bb86aa8..316baf38437 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,8 @@ # 0.11.0 +- Prepare simple case-sensitive name/path globs once and avoid copying owned native glob patterns; preserve native matching for complex and case-folded patterns. + - Share collection basename and root text with the owned path where possible, reducing retained memory without changing buffer limits. - Use lazy fixed 1 MiB blocks for retained collection text to reduce allocation overhead. diff --git a/TODO.md b/TODO.md index 869e0779939..10e22a1ef43 100644 --- a/TODO.md +++ b/TODO.md @@ -368,7 +368,10 @@ EP/JC/P work; candidates are not speedup claims until measured on both platforms - [ ] O01: Adopt measured path-owned basename/root views with fallback copies, unchanged logical budgets and ownership/growth/move coverage. Retain small regressions and Linux/macOS evidence in [the collection-sharing report](docs/benchmarks/collection-text-sharing-20261101.md). -- [ ] O02-O06: Independently qualify native glob argument-copy removal, fuzzy scratch/self-score +- [ ] O02: Qualify owned-pattern references and prepared simple ASCII glob operands. + [Cross-platform measurements](docs/benchmarks/native-glob-preparation-20261101.md) preserve + case-folded/complex fallbacks, native controls and preparation costs. +- [ ] O03-O06: Independently qualify fuzzy scratch/self-score reuse, lexical field views, heterogeneous capture/definition lookup, and remaining read-buffer zeroing. Prefer existing values and interfaces before adding prepared wrappers. - [ ] O07-O12: Profile and qualify caller-owned stream buffers, typed reusable directory task slots, diff --git a/docs/benchmarks/native-glob-confirmation-20261101.json.gz b/docs/benchmarks/native-glob-confirmation-20261101.json.gz new file mode 100644 index 00000000000..77539c602cf Binary files /dev/null and b/docs/benchmarks/native-glob-confirmation-20261101.json.gz differ diff --git a/docs/benchmarks/native-glob-preparation-20261101.json.gz b/docs/benchmarks/native-glob-preparation-20261101.json.gz new file mode 100644 index 00000000000..8f6f2153429 Binary files /dev/null and b/docs/benchmarks/native-glob-preparation-20261101.json.gz differ diff --git a/docs/benchmarks/native-glob-preparation-20261101.md b/docs/benchmarks/native-glob-preparation-20261101.md new file mode 100644 index 00000000000..efe16826fb0 --- /dev/null +++ b/docs/benchmarks/native-glob-preparation-20261101.md @@ -0,0 +1,129 @@ +# Native glob preparation (2026-11-01) + +## Decision + +Retain the existing owned pattern string instead of copying it through a view. Prepare literal, prefix, suffix, contains and one-interior-star ASCII patterns once in the existing operand pool; evaluate them with byte slices when no case folding applies. Preserve libc `fnmatch` for folded case, multibyte/invalid bytes, escapes, character classes, question marks and other complex patterns. There is no new pool, lock, flag, wrapper hierarchy or user-visible grammar. + +The reference is #1041 head `cb441b7d8dee9893f4b76ebef0a5613d46679f59`, whose runtime incorporates #1040 plus owned collection text sharing. All three release variants share that source tree except `evaluate.cc`: baseline, owned-pattern-only, and owned-pattern plus prepared simple patterns. This isolates the cheap reference change from the larger prepared strategy. + +## Correctness and costs + +The immutable AST owns the pattern. A prepared operand borrows it and survives plan moves; its worker and AST lifetime contract is unchanged. Inputs remain length-delimited views. Match the existing first-NUL C-string boundary explicitly, and never assume a view ends in a NUL. Prefix/suffix checks require enough input bytes to avoid overlapping literals. Every unsupported case returns to the platform implementation. Native case policy, safety, metadata observations and short-circuit behavior are unchanged. + +The final prepared record stores the logical pattern length, a full-width star offset and a kind in the existing operand variant; the AST remains the pattern owner. It adds preparation/pool storage for name/path predicates that formerly needed no prepared operand. It does not add mutable per-worker glob storage. Native binary growth is recorded below; preparation costs are reported separately from repeated matching. + +`glob_evaluate_test` differentially checks all six name/path/wholename variants, explicit folding, native folding, simple/complex/escaped/malformed patterns, nonterminated input slices, NUL boundaries and plan moves with two independent workers. `glob_unicode_test` separately checks accented multibyte strings and deliberately malformed bytes against native locale behavior; ordinary fixtures remain ASCII. Existing evaluation, execution and full-engine tests remain applicable. + +## Measurement contract + +Local M5 Pro: warm APFS, one/three worker allowances, no affinity. Linux drogon: Ryzen 9950X KVM guest, tmpfs fixtures, child affinity limited to one/three CPUs. Both use release LLVM 23 and ThinLTO. Builds finish before timing. Preserve the host distinction rather than comparing absolute times across machines. + +Each native case has three sessions, a discarded warmup per variant/session, nine interleaved measurements and the fastest seven retained. Exact filename output is checked independently after every run. Counts are 10/1,000/10,000/100,000. Cases include a simple suffix, a literal selecting one file, a complex bracket pattern retaining libc, and a no-name-predicate control. Linux and macOS default series have 32 cases/session. Two macOS supplemental series have 24 cases/session: `--case=sensitive` alone, then `--case=sensitive --exact`. Across all series this is 10,080 successful fresh invocations including warmups. + +The first supplemental series is an explicit control, not byte-exact matching: documented filesystem-native folding still applies without `--exact`. In XFF mode the runner also requests full metadata and keeps matching serial while filesystem-native probing is active. The exact series changes those existing policies for all three binaries; compare variants within each series, never interpret cross-series differences as the glob optimization alone. + +The existing in-memory expression benchmark separately measures preparation and prepared execution for name/type/constant-true predicates, chains of 1/16/64 predicates and batches of 10/1,000 entries. Nine rounds retain seven, with three shuffled variant sessions. Host I/O is absent; correctness is checked before kernel timing. Percent changes below are median within-session ratios; negative is faster. + +## Initial prototype: native 100,000-file results + +| Series | Case | Workers | Baseline ms | Owned-only change | Prepared ms | Prepared change | +| ------------- | ------- | ------: | ----------: | ----------------: | ----------: | --------------: | +| linux-default | suffix | 1 | 103.257 | -0.00% | 101.944 | -1.26% | +| linux-default | suffix | 3 | 37.732 | -0.42% | 37.194 | -1.77% | +| linux-default | literal | 1 | 97.751 | +0.37% | 97.144 | -0.36% | +| linux-default | literal | 3 | 35.642 | -0.46% | 35.278 | -1.02% | +| linux-default | control | 1 | 113.272 | +0.20% | 114.197 | +0.82% | +| linux-default | control | 3 | 90.439 | +1.10% | 91.881 | +1.58% | +| macos-default | suffix | 1 | 248.826 | -0.35% | 248.913 | -0.05% | +| macos-default | suffix | 3 | 108.380 | +0.86% | 110.371 | +1.20% | +| macos-default | literal | 1 | 230.520 | +0.37% | 229.872 | -0.35% | +| macos-default | literal | 3 | 108.045 | +0.14% | 110.079 | +2.21% | +| macos-default | control | 1 | 233.033 | -0.35% | 233.150 | -0.81% | +| macos-default | control | 3 | 192.807 | +0.03% | 190.701 | -1.09% | +| macos-exact | suffix | 1 | 250.326 | -0.43% | 235.959 | -5.07% | +| macos-exact | suffix | 3 | 210.744 | -0.02% | 196.678 | -6.84% | +| macos-exact | literal | 1 | 71.383 | -0.25% | 68.573 | -3.94% | +| macos-exact | literal | 3 | 49.299 | -0.42% | 49.275 | +0.10% | +| macos-exact | control | 1 | 232.772 | -0.38% | 232.404 | -0.15% | +| macos-exact | control | 3 | 195.425 | +0.18% | 194.890 | -1.06% | + +The default macOS series mostly uses the native fallback and is not a claimed case-sensitive speedup. No-name-predicate controls also change slightly; retain these code-layout/system-noise controls instead of attributing them to matching. The exact series is useful in find-compatible or explicitly exact XFF workflows. + +## Initial prototype: kernel, preparation and binary size + +| Host | Case | Baseline ns | Owned-only change | Prepared ns | Prepared change | +| ------------- | ------------------- | ----------: | ----------------: | ----------: | --------------: | +| linux-default | kernel/name/1/1000 | 23444.46 | -9.09% | 13604.80 | -41.96% | +| linux-default | kernel/name/16/1000 | 504453.38 | -4.02% | 351124.72 | -30.26% | +| linux-default | prepare/name/1 | 179.58 | -2.90% | 205.01 | +12.89% | +| linux-default | prepare/name/64 | 9109.75 | -0.74% | 9711.85 | +6.90% | +| macos-default | kernel/name/1/1000 | 77201.27 | +0.30% | 19737.90 | -74.43% | +| macos-default | kernel/name/16/1000 | 1358585.49 | -2.25% | 412661.24 | -69.63% | +| macos-default | prepare/name/1 | 396.49 | -1.77% | 437.81 | +5.91% | +| macos-default | prepare/name/64 | 16162.58 | +1.95% | 17122.16 | +5.56% | + +| Host | Baseline binary bytes | Owned-only bytes | Prepared bytes | +| ------------- | --------------------: | ---------------: | -------------: | +| linux-default | 12293344 | 12291624 | 12307056 | +| macos-default | 4065104 | 4065136 | 4069136 | + +These executable sizes include the build output debug sections and are not final compressed release sizes. A fast repeated kernel does not imply a similarly large whole-command gain: traversal, metadata, formatting and process startup remain real costs. + +## Initial prototype: counterexamples and remaining work + +- `linux-default`: worst paired median +3.66% at (10, 3, 'literal'); best -1.77% at (100000, 3, 'suffix'). All raw samples, including slower cases and controls, are retained. +- `macos-default`: worst paired median +2.21% at (100000, 3, 'literal'); best -1.86% at (1000, 3, 'control'). All raw samples, including slower cases and controls, are retained. +- `macos-case-sensitive-flag`: worst paired median +2.39% at (10, 3, 'control'); best -1.68% at (1000, 1, 'control'). All raw samples, including slower cases and controls, are retained. +- `macos-exact`: worst paired median +1.25% at (10, 1, 'control'); best -6.84% at (100000, 3, 'suffix'). All raw samples, including slower cases and controls, are retained. + +Preparation is slightly slower and reserves operand capacity once; it must not be described as free. The simple ASCII path deliberately scans for multibyte input to keep locale-dependent behavior native. Case folding and complex patterns still pay for input materialization. No claim is made for eliminating all glob allocations or speeding every small invocation. + +The metadata/worker follow-up (O11/O20) should examine the existing filesystem-native case probe: `RunFindCore` includes it in `full_metadata` and excludes it from parallel matching. Propagating validated directory/filesystem case policy may remove regular-file metadata demand without changing matching semantics. Root files, mounted volumes, symlinks and virtual filesystems need explicit tests; this is a new decision experiment within the retained metadata/worker work, not a shortcut changing defaults. + +## Validation + +Release differential glob/Unicode, evaluator, expression-execution and full-engine targets passed on both hosts. The complete Linux release/extension suite passed 304/304 with its existing FUSE-required configuration. The complete macOS release/extension suite also passed 304/304. Scoped clang-tidy passed the runtime and Unicode translation units. Its two test-only findings (const helper and trace concatenation) were corrected; that translation unit then passed focused release tests on both hosts and its scoped clang-tidy recheck. Normal hooks and published-head CI remain required before merge. + +## Raw evidence + +[Lossless JSON evidence](native-glob-preparation-20261101.json.gz): 483,174 bytes; SHA-256 `304977003bf522eff9e5c52922a46c647b828a97097da03e0929d9b59cd6b4a0`. Raw native/kernel samples, source variants, build manifests, scripts and verification logs are retained. + +## Final representation confirmation + +The final record stores two full-width positions (logical pattern length and star offset) plus the +kind. The pattern remains AST-owned and the evaluator constructs a borrowed view from its stored +length. This retains the same operand-record size without an arbitrary 32-bit argument limit. +A missing-argument regression preserves the non-match behavior of in-process malformed expressions. +The fallback case now runs once through the switch; the exhaustive enum's impossible fallthrough is +explicitly unreachable, with the repository's existing coverage annotation. No threshold was changed. + +A second complete three-session matrix compares baseline, the first prepared prototype and the +exact revised source. Each host again has 32 native cases/session, discarded warmups and nine +interleaved rounds retaining seven, plus the existing expression kernel/preparation controls. +Linux uses defaults; macOS uses `--case=sensitive --exact`. This adds 5,760 successful native +invocations including warmups. The earlier tables and counterexamples describe the first prototype; +this confirmation is the qualification for the final representation. + +| Host | Case | Workers | Baseline ms | First prepared ms | Final ms | Final change | +| ----------- | ------- | ------: | ----------: | ----------------: | -------: | -----------: | +| linux | suffix | 1 | 103.779 | 101.568 | 102.077 | -1.69% | +| linux | suffix | 3 | 38.159 | 37.738 | 37.761 | -1.04% | +| linux | literal | 1 | 99.305 | 98.039 | 98.038 | -1.28% | +| linux | literal | 3 | 35.836 | 35.503 | 35.539 | -0.73% | +| linux | complex | 1 | 104.458 | 104.123 | 103.562 | -0.43% | +| linux | complex | 3 | 38.003 | 38.067 | 38.499 | +1.26% | +| linux | control | 1 | 113.969 | 114.418 | 114.387 | +0.37% | +| linux | control | 3 | 93.839 | 94.477 | 93.597 | -0.26% | +| macos-exact | suffix | 1 | 247.142 | 233.717 | 233.475 | -6.02% | +| macos-exact | suffix | 3 | 207.530 | 194.426 | 192.726 | -7.11% | +| macos-exact | literal | 1 | 70.917 | 67.520 | 67.407 | -4.70% | +| macos-exact | literal | 3 | 49.039 | 48.875 | 48.973 | -0.19% | +| macos-exact | complex | 1 | 247.671 | 247.171 | 246.003 | -0.67% | +| macos-exact | complex | 3 | 207.178 | 205.397 | 206.221 | -0.46% | +| macos-exact | control | 1 | 230.822 | 229.841 | 227.539 | -1.44% | +| macos-exact | control | 3 | 188.509 | 188.111 | 188.495 | -0.38% | + +[Lossless confirmation evidence](native-glob-confirmation-20261101.json.gz): 405,657 bytes; +SHA-256 `65dca91d9a40943d1e487188f69445607f2bc381eab1edefff132b3c7021c86b`. It retains every raw sample, all three source +variants, exact runtime hash, scripts and final focused regression logs. The first matrix remains +available independently rather than being overwritten. diff --git a/docs/design-expression-program.md b/docs/design-expression-program.md index 2bfe387a7c4..5588c69d3f7 100644 --- a/docs/design-expression-program.md +++ b/docs/design-expression-program.md @@ -568,7 +568,7 @@ ordinary dispatcher; parser/traversal-only primaries preserve their existing tru | :------------------------------------------------------------- | :---------------------------------------------- | :----------------------------------------------------------------------------- | | `-true`, `-false` | None | Whole-command effects of surrounding syntax still apply | | `-type`, `-xtype` | Type-list decoding | Entry type; `-xtype` may observe the link target and fail | -| `-name`, `-iname`, path/wholename variants | Pattern bytes; case selection | Entry name/path and filesystem case policy | +| `-name`, `-iname`, path/wholename variants | ASCII simple-glob shape; case selection | Entry name/path and filesystem case policy | | `-lname`, `-ilname` | Link glob and case policy | VFS link read and failure | | `-regex`, `-iregex`, `-rxc`, `-irxc` | Matcher already compiled | Entry bytes, optional captures, backend worker scratch | | `-content`, `-icontent`, `-text`, `-binary`, `-eof*` | Literal/flavor interpretation | Content reads, cache invalidation, unsupported/error outcomes | diff --git a/docs/performance-analysis.md b/docs/performance-analysis.md index e1d7ffd8fc4..55c2c37e975 100644 --- a/docs/performance-analysis.md +++ b/docs/performance-analysis.md @@ -3070,3 +3070,25 @@ Small native costs remain (up to +1.33% Linux and +1.64% macOS); kernel countere checks, source/binary identities and separate untimed accounting. Both source variants pass collection and full-engine release tests on both platforms. This is qualified local evidence; a subsequent published PR head still needs its normal hooks and CI. + +## Native glob preparation and filesystem-native case cost + +O02 now has [separate owned-pattern and prepared-glob measurements](benchmarks/native-glob-preparation-20261101.md). +Reuse the AST-owned pattern string for libc calls. In the existing operand pool, prepare simple +ASCII literal/prefix/suffix/contains/interior-star patterns once. Byte-slice matching is restricted +to case-sensitive ASCII input; folded case, multibyte/invalid bytes, escapes, classes and complex +patterns retain libc behavior. No new mutable shared state or wrapper hierarchy is needed. + +The one-predicate/1,000-entry in-memory kernel improves about 42% on Linux and 74% on macOS. +Whole commands improve much less: at 100,000 files, Linux default suffix matching improves +1.3-1.8%; macOS `--case=sensitive --exact` improves 5.1-6.8%. Default APFS folding keeps the +native fallback, so those runs are controls rather than claimed case-sensitive gains. Preparation +is slower, operand storage is added once, and slower native cases remain in the report. + +Keep the documented difference between `--case=sensitive` and `--exact`: the former alone does +not disable filesystem-native folding. Existing `RunFindCore` uses filesystem-native probing as +a reason for full entry metadata and serial matching. O11/O20 should examine carrying a validated +per-directory filesystem case decision instead of forcing full regular-file metadata. Test roots, +mount boundaries, followed links and virtual filesystems before changing that policy; cross-series +exact/default timings conflate metadata, matching admission and case behavior and cannot isolate +one of them. diff --git a/release-site.json b/release-site.json index 62c17cdc26d..fe9443e3f54 100644 --- a/release-site.json +++ b/release-site.json @@ -5,6 +5,7 @@ "CONTRIBUTING.md": "CONTRIBUTING.html", "docs/benchmarks/execution-resources.md": "docs/benchmarks/execution-resources.html", "docs/benchmarks/executor-copy-audit-20261101.md": "docs/benchmarks/executor-copy-audit-20261101.html", + "docs/benchmarks/native-glob-preparation-20261101.md": "docs/benchmarks/native-glob-preparation-20261101.html", "docs/benchmarks/collection-text-sharing-20261101.md": "docs/benchmarks/collection-text-sharing-20261101.html", "docs/benchmarks/collection-arena-20261101.md": "docs/benchmarks/collection-arena-20261101.html", "docs/benchmarks/bounded-read-ahead-20261101.md": "docs/benchmarks/bounded-read-ahead-20261101.html", diff --git a/xff/engine/BUILD.bazel b/xff/engine/BUILD.bazel index 10dac8690dc..8eae0bfe34c 100644 --- a/xff/engine/BUILD.bazel +++ b/xff/engine/BUILD.bazel @@ -290,6 +290,36 @@ cc_test( ], ) +cc_test( + name = "glob_evaluate_test", + size = "small", + srcs = ["glob_evaluate_test.cc"], + deps = [ + ":evaluate_cc", + ":walk_cc", + "//xff/parser:parser_cc", + "//xff/vfs:local_fs_cc", + "@abseil-cpp//absl/time", + "@googletest//:gtest_main", + "@mboworks_mbo//mbo/testing:status_cc", + ], +) + +cc_test( + name = "glob_unicode_test", + size = "small", + srcs = ["glob_unicode_test.cc"], + deps = [ + ":evaluate_cc", + ":walk_cc", + "//xff/parser:parser_cc", + "//xff/vfs:local_fs_cc", + "@abseil-cpp//absl/time", + "@googletest//:gtest_main", + "@mboworks_mbo//mbo/testing:status_cc", + ], +) + # Parse arbitrary newline-framed argv and evaluate accepted, non-destructive expressions against a # read-only synthetic entry. Ordinary tests replay the corpus; the bounded CI campaign mutates it. cc_fuzz_test( diff --git a/xff/engine/evaluate.cc b/xff/engine/evaluate.cc index 5a4b0f4d2d6..c3e8aa22e0b 100644 --- a/xff/engine/evaluate.cc +++ b/xff/engine/evaluate.cc @@ -115,8 +115,8 @@ mbo::types::OptionalRef AsConstOptionalRef(const mbo::types::OptionalRe // agree and the platform choice lives in one place. constexpr std::string_view kOsLineEnding = "\n"; -bool Fnmatch(std::string_view pattern, std::string_view text, int flags) { - return ::fnmatch(std::string(pattern).c_str(), std::string(text).c_str(), flags) == 0; +bool Fnmatch(const std::string& pattern, std::string_view text, int flags) { + return ::fnmatch(pattern.c_str(), std::string(text).c_str(), flags) == 0; } // The (letter, type) pairs find's -type/-xtype letters denote, alphabetical by @@ -2935,6 +2935,16 @@ struct MimePattern { std::vector bytes; }; +enum class GlobKind : std::uint8_t { kFallback, kLiteral, kPrefix, kSuffix, kContains, kPrefixSuffix }; + +struct GlobPattern { + // The immutable AST owns the pattern. Store its C-string length and star position, with + // no duplicated pointer and no artificial size restriction on an in-process argument. + std::size_t length = 0; + std::size_t star = 0; + GlobKind kind = GlobKind::kFallback; +}; + struct FieldTemplate { // Keep the compiled program out of the hot operand variant. No shared ownership or // reference counting: workers borrow this immutable program through the owning plan. @@ -2953,10 +2963,98 @@ using PreparedOperand = std::variant< CalendarAge, FirstLimit, MimePattern, + GlobPattern, FieldTemplate>; using PreparedEvalFn = bool (*)(const parser::Expr&, const PreparedOperand&, MatcherSlots, EvalContext&); using PrepareOperandFn = PreparedOperand (*)(std::string_view); +PreparedOperand PrepareGlobOperand(std::string_view argument) { + // Match the original C-string boundary, including callers that supply embedded NUL bytes. + argument = argument.substr(0, argument.find('\0')); + GlobPattern result{.length = argument.size()}; + for (const unsigned char byte : argument) { + if (byte >= 128 || byte == '\\' || byte == '?' || byte == '[') { + return result; + } + } + const auto first = argument.find('*'); + if (first == std::string_view::npos) { + result.kind = GlobKind::kLiteral; + return result; + } + result.star = first; + const auto second = argument.find('*', first + 1); + if (second == std::string_view::npos) { + result.kind = first == 0 ? GlobKind::kSuffix + : first + 1 == argument.size() ? GlobKind::kPrefix + : GlobKind::kPrefixSuffix; + } else if (first == 0 && second + 1 == argument.size()) { + result.kind = GlobKind::kContains; + } + return result; +} + +bool MatchesPreparedGlob( + const parser::Expr& expression, + const PreparedOperand& operand, + std::string_view text, + int flags) { + if (expression.args.empty()) { + return false; + } + if (flags != 0 || !std::holds_alternative(operand)) { + return Fnmatch(expression.args.front(), text, flags); + } + const auto& pattern = std::get(operand); + if (pattern.kind != GlobKind::kFallback) { + // Locale-sensitive/multibyte behavior remains with libc. ASCII and zero flags use byte slices. + for (std::size_t index = 0; index < text.size(); ++index) { + const auto byte = static_cast(text[index]); + if (byte == 0) { + text = text.substr(0, index); + break; + } + if (byte >= 128) { + return Fnmatch(expression.args.front(), text, flags); + } + } + } + const std::string_view literal(expression.args.front().data(), pattern.length); + switch (pattern.kind) { + case GlobKind::kLiteral: return text == literal; + case GlobKind::kPrefix: return text.starts_with(literal.substr(0, pattern.star)); + case GlobKind::kSuffix: return text.ends_with(literal.substr(1)); + case GlobKind::kContains: return text.contains(literal.substr(1, literal.size() - 2)); + case GlobKind::kPrefixSuffix: + return text.size() >= literal.size() - 1 && text.starts_with(literal.substr(0, pattern.star)) + && text.ends_with(literal.substr(pattern.star + 1)); + case GlobKind::kFallback: return Fnmatch(expression.args.front(), text, flags); + } + std::unreachable(); // LCOV_EXCL_LINE: every internally prepared glob kind is handled above. +} + +bool EvalPreparedName( + const parser::Expr& expression, + const PreparedOperand& operand, + MatcherSlots, + // NOLINTNEXTLINE(misc-const-correctness): PreparedEvalFn requires a mutable context. + EvalContext& context) { + const int flags = + (expression.descriptor->fold_case || context.fold_name_case || expression.case_fold) ? FNM_CASEFOLD : 0; + return MatchesPreparedGlob(expression, operand, context.visit.name, flags); +} + +bool EvalPreparedPath( + const parser::Expr& expression, + const PreparedOperand& operand, + MatcherSlots, + // NOLINTNEXTLINE(misc-const-correctness): PreparedEvalFn requires a mutable context. + EvalContext& context) { + const int flags = + (expression.descriptor->fold_case || context.fold_name_case || expression.case_fold) ? FNM_CASEFOLD : 0; + return MatchesPreparedGlob(expression, operand, context.visit.path, flags); +} + PreparedOperand PrepareTypeOperand(std::string_view argument) { TypeSpec result; // Use the existing decoder, including its complete-list validation. Seven preparation-only @@ -3256,12 +3354,12 @@ constexpr auto kDispatch = mbo::container::MakeLimitedMap( }, DispatchPair{"-icontent", MakeEvalEntry<&EvalContent>()}, DispatchPair{"-ilname", MakeEvalEntry<&EvalLname>()}, - DispatchPair{"-iname", MakeEvalEntry<&EvalName>()}, + DispatchPair{"-iname", MakeEvalEntry<&EvalName>(&PrepareGlobOperand, &EvalPreparedName)}, DispatchPair{"-inum", MakeEvalEntry<&EvalInum>(&PrepareNumericOperand, &EvalPreparedNumeric<&vfs::Metadata::ino>)}, - DispatchPair{"-ipath", MakeEvalEntry<&EvalPath>()}, + DispatchPair{"-ipath", MakeEvalEntry<&EvalPath>(&PrepareGlobOperand, &EvalPreparedPath)}, DispatchPair{"-iregex", MakeEvalEntry<&EvalRegex>(nullptr, &EvalPreparedRegex, true)}, DispatchPair{"-irxc", MakeEvalEntry<&EvalRxc>(nullptr, &EvalPreparedRxc, true)}, - DispatchPair{"-iwholename", MakeEvalEntry<&EvalPath>()}, + DispatchPair{"-iwholename", MakeEvalEntry<&EvalPath>(&PrepareGlobOperand, &EvalPreparedPath)}, DispatchPair{"-lang", MakeEvalEntry<&EvalLang>()}, DispatchPair{ "-links", MakeEvalEntry<&EvalLinks>(&PrepareNumericOperand, &EvalPreparedNumeric<&vfs::Metadata::nlink>)}, @@ -3275,7 +3373,7 @@ constexpr auto kDispatch = mbo::container::MakeLimitedMap( DispatchPair{"-fuzzypath", MakeEvalEntry<&EvalFuzzyPath>()}, DispatchPair{"-ifuzzy", MakeEvalEntry<&EvalFuzzy>()}, DispatchPair{"-ifuzzypath", MakeEvalEntry<&EvalFuzzyPath>()}, - DispatchPair{"-name", MakeEvalEntry<&EvalName>()}, + DispatchPair{"-name", MakeEvalEntry<&EvalName>(&PrepareGlobOperand, &EvalPreparedName)}, DispatchPair{"-newer", MakeEvalEntry<&EvalNewer>()}, DispatchPair{"-newerBB", MakeEvalEntry<&EvalNewerXY>()}, // birthtime -newerXY combos (BSD-compat) DispatchPair{"-newerBa", MakeEvalEntry<&EvalNewerXY>()}, @@ -3301,7 +3399,7 @@ constexpr auto kDispatch = mbo::container::MakeLimitedMap( DispatchPair{"-nouser", MakeEvalEntry<&EvalNouser>()}, DispatchPair{"-ok", MakeEvalEntry<&EvalOk>()}, DispatchPair{"-okdir", MakeEvalEntry<&EvalOkdir>()}, - DispatchPair{"-path", MakeEvalEntry<&EvalPath>()}, + DispatchPair{"-path", MakeEvalEntry<&EvalPath>(&PrepareGlobOperand, &EvalPreparedPath)}, DispatchPair{"-perm", MakeEvalEntry<&EvalPerm>(&PreparePermissionOperand, &EvalPreparedPermission)}, DispatchPair{"-print", MakeEvalEntry<&EvalPrint>()}, DispatchPair{"-print0", MakeEvalEntry<&EvalPrint0>()}, @@ -3330,7 +3428,7 @@ constexpr auto kDispatch = mbo::container::MakeLimitedMap( DispatchPair{"-uid", MakeEvalEntry<&EvalUid>(&PrepareNumericOperand, &EvalPreparedNumeric<&vfs::Metadata::uid>)}, DispatchPair{"-used", MakeEvalEntry<&EvalUsed>(&PrepareSignedNumericOperand, &EvalPreparedUsed)}, DispatchPair{"-user", MakeEvalEntry<&EvalUser>()}, - DispatchPair{"-wholename", MakeEvalEntry<&EvalPath>()}, + DispatchPair{"-wholename", MakeEvalEntry<&EvalPath>(&PrepareGlobOperand, &EvalPreparedPath)}, DispatchPair{"-writable", MakeEvalEntry<&EvalWritable>()}, DispatchPair{"-xtype", MakeEvalEntry<&EvalXtype>(&PrepareTypeOperand, &EvalPreparedXtype)}); diff --git a/xff/engine/glob_evaluate_test.cc b/xff/engine/glob_evaluate_test.cc new file mode 100644 index 00000000000..a6015ea9d69 --- /dev/null +++ b/xff/engine/glob_evaluate_test.cc @@ -0,0 +1,188 @@ +// SPDX-FileCopyrightText: Copyright (c) M. Boerger, the MBO Works authors +// SPDX-License-Identifier: Apache-2.0 + +#include + +#include +#include +#include +#include +#include + +#include "absl/time/time.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "mbo/testing/status.h" +#include "xff/engine/evaluate.h" +#include "xff/engine/walk.h" +#include "xff/parser/parser.h" +#include "xff/vfs/local_fs.h" + +namespace xff::engine { +namespace { + +using ::testing::Eq; +using ::testing::IsFalse; + +struct GlobEvaluateTest : ::testing::Test { + void Check(const std::vector& patterns, const std::vector& texts) const { + static constexpr auto kPrimaries = std::to_array({ + "-name", + "-iname", + "-path", + "-ipath", + "-wholename", + "-iwholename", + }); + for (const std::string_view primary : kPrimaries) { + for (const std::string& pattern : patterns) { + MBO_ASSERT_OK_AND_ASSIGN(const auto command, parser::Parse({".", std::string(primary), pattern})); + const auto& expression = *command.expression; + MBO_ASSERT_OK_AND_ASSIGN(auto prepared, PreparedExpression::Prepare(expression)); + const auto worker = prepared.MakeWorker(); + // Operands borrow the immutable AST, not the movable prepared-plan object. + const auto moved = std::move(prepared); + const auto second_worker = moved.MakeWorker(); + for (const std::string& text : texts) { + for (const bool fold : {false, true}) { + SCOPED_TRACE(primary); + SCOPED_TRACE(pattern); + SCOPED_TRACE(text); + // Neither view is NUL-terminated at its end. The suffix must never be matched. + const std::string backing = text + "unrelated suffix"; + const std::string_view slice(backing.data(), text.size()); + const vfs::Metadata metadata{.type = vfs::FileType::kRegular}; + const Visit visit{.path = slice, .name = slice, .metadata = metadata}; + // NOLINTNEXTLINE(misc-const-correctness): EvalContext mutates this control. + Control control; + const auto emit = [](std::string_view) {}; + EvalContext context{ + .visit = visit, + .emit = emit, + .fs = fs, + .now = absl::UnixEpoch(), + .tz = absl::UTCTimeZone(), + .fold_name_case = fold, + .control = control, + }; + const int flags = fold || expression.descriptor->fold_case ? FNM_CASEFOLD : 0; + const bool expected = ::fnmatch(pattern.c_str(), text.c_str(), flags) == 0; + EXPECT_THAT(Evaluate(expression, context), Eq(expected)); + EXPECT_THAT(worker.Evaluate(context).matched, Eq(expected)); + EXPECT_THAT(worker.EvaluateIterative(context).matched, Eq(expected)); + EXPECT_THAT(second_worker.Evaluate(context).matched, Eq(expected)); + } + } + } + } + } + + vfs::LocalFs fs; +}; + +TEST_F(GlobEvaluateTest, SimpleAndComplexPatternsAgreeWithNativeFnmatch) { + const std::vector patterns{ + "", + "*", + "**", + "***", + "a", + "abc", + "a*", + "*c", + "*b*", + "a*c", + "ab*bc", + "a**c", + "*a*b*", + "?", + "a?c", + "[ab]", + "[!a]", + "[a-z]*", + "[[:alpha:]]*", + "[", + "[]", + "a]", + R"(a\*)", + R"(a\?)", + R"(a\[)", + R"(a\)", + "*/a", + "a/*", + "a/*/b", + "*/.*", + "*.txt", + "a*b*c", + "*abc*", + std::string(160, 'a') + "*", + }; + const std::vector texts{ + "", + "a", + "A", + "b", + "abc", + "abBc", + "abbc", + "ababc", + "a*", + "a?", + "a[", + "a]", + "a\\", + ".", + "..", + ".abc", + "a/b", + "a/.b", + "a/x/b", + "a/x/y/b", + "/a", + "file.txt", + "FILE.TXT", + "a b", + "a\nb", + std::string(160, 'a'), + std::string(170, 'a'), + }; + Check(patterns, texts); +} + +TEST_F(GlobEvaluateTest, MissingPatternRemainsANonMatchInAnInProcessExpression) { + const vfs::Metadata metadata{.type = vfs::FileType::kRegular}; + const Visit visit{.path = "file.txt", .name = "file.txt", .metadata = metadata}; + // NOLINTNEXTLINE(misc-const-correctness): EvalContext mutates this control. + Control control; + const auto emit = [](std::string_view) {}; + EvalContext context{ + .visit = visit, + .emit = emit, + .fs = fs, + .now = absl::UnixEpoch(), + .tz = absl::UTCTimeZone(), + .control = control, + }; + for (const std::string_view primary : {"-name", "-path"}) { + MBO_ASSERT_OK_AND_ASSIGN(auto command, parser::Parse({".", std::string(primary), "unused"})); + command.expression->args.clear(); + MBO_ASSERT_OK_AND_ASSIGN(const auto prepared, PreparedExpression::Prepare(*command.expression)); + EXPECT_THAT(Evaluate(*command.expression, context), IsFalse()); + EXPECT_THAT(prepared.Evaluate(context).matched, IsFalse()); + } +} + +TEST_F(GlobEvaluateTest, EmbeddedNulUsesTheExistingCStringBoundary) { + const std::vector patterns{ + std::string("a\0ignored", 9), + std::string("*\0ignored", 9), + std::string("?\0ignored", 9), + }; + const std::vector texts{ + "", "a", "ab", std::string("a\0tail", 6), std::string("b\0tail", 6), + }; + Check(patterns, texts); +} + +} // namespace +} // namespace xff::engine diff --git a/xff/engine/glob_unicode_test.cc b/xff/engine/glob_unicode_test.cc new file mode 100644 index 00000000000..1aba13c51ec --- /dev/null +++ b/xff/engine/glob_unicode_test.cc @@ -0,0 +1,76 @@ +// SPDX-FileCopyrightText: Copyright (c) M. Boerger, the MBO Works authors +// SPDX-License-Identifier: Apache-2.0 + +#include + +#include +#include +#include + +#include "absl/time/time.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "mbo/testing/status.h" +#include "xff/engine/evaluate.h" +#include "xff/engine/walk.h" +#include "xff/parser/parser.h" +#include "xff/vfs/local_fs.h" + +namespace xff::engine { +namespace { + +using ::testing::Eq; + +struct GlobUnicodeTest : ::testing::Test {}; + +TEST_F(GlobUnicodeTest, MultibyteAndInvalidBytesRetainNativeLocaleSemantics) { + // Accented Latin letters exercise multibyte input/patterns. The invalid byte deliberately + // exercises native fnmatch's behavior for malformed input, without imposing UTF-8 semantics. + static constexpr auto kPatterns = std::to_array({ + "*", + "*.txt", + "c*", + "*é*", + "café.txt", + "CAFÉ.TXT", + "?*.txt", + "[[:alpha:]]*", + }); + static constexpr auto kTexts = std::to_array({ + "café.txt", + "CAFÉ.TXT", + "é", + "\377.txt", + }); + const vfs::LocalFs fs; + const vfs::Metadata metadata{.type = vfs::FileType::kRegular}; + const auto emit = [](std::string_view) {}; + for (const std::string_view pattern : kPatterns) { + MBO_ASSERT_OK_AND_ASSIGN(const auto command, parser::Parse({".", "-name", std::string(pattern)})); + MBO_ASSERT_OK_AND_ASSIGN(const auto prepared, PreparedExpression::Prepare(*command.expression)); + const auto worker = prepared.MakeWorker(); + for (const std::string_view text : kTexts) { + for (const bool fold : {false, true}) { + SCOPED_TRACE(std::string(pattern) + " / " + std::string(text)); + const Visit visit{.path = text, .name = text, .metadata = metadata}; + // NOLINTNEXTLINE(misc-const-correctness): EvalContext mutates this control. + Control control; + EvalContext context{ + .visit = visit, + .emit = emit, + .fs = fs, + .now = absl::UnixEpoch(), + .tz = absl::UTCTimeZone(), + .fold_name_case = fold, + .control = control, + }; + const bool expected = + ::fnmatch(std::string(pattern).c_str(), std::string(text).c_str(), fold ? FNM_CASEFOLD : 0) == 0; + EXPECT_THAT(worker.Evaluate(context).matched, Eq(expected)); + } + } + } +} + +} // namespace +} // namespace xff::engine