diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a9643d666d..7059d41a90 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -24,7 +24,7 @@ repos: hooks: - id: check-added-large-files # Retain the exact lossless cross-platform performance evidence archives. - exclude: ^docs/benchmarks/((executor-copy-audit|bounded-read-ahead)-20261101|(directory-task-slots|packed-visit-snapshots|bounded-match-results|useful-match-workers|direct-evaluator-workers|derived-content-handoff|immutable-default-catalogs|prepared-diff-configuration|prepared-hash-operands|aggregate-comparison-totals|byte-shingle-representations|archive-input-buffers)-20261102|(digest-and-record-storage|caller-owned-streams|direct-source-materialization|hash-field-binding|hash-field-binding-tag|forward-cursor-skipping|raw-record-reuse|similarity-prefix-view|similarity-reference-reuse|owned-word-moves|smaller-shingle-probes|native-zip-seek-evaluation|encoded-record-reuse|threshold-aware-similarity)-20261103|(column-row-reuse|word-set-insertion|field-context-owned-results|selected-summary-output|fixed-summary-metrics|direct-summary-cells|compact-comparison-entries|comparison-category-sequence|native-name-paths|uniform-native-case|compact-traversal-metadata|eager-native-paths|native-case-experiments)-20261104|(folded-ascii-globs|bulk-directory-metadata|comparison-retention-followups|comparison-storage-followups|archive-prefix-reuse|archive-directory-range|archive-directory-allocation|archive-directory-observations|native-symlink-observations|native-metadata-followups|symlink-target-demand|native-pgo|readlink-short-target|cctz-empty-format|readlink-input-path|readlink-owned-transfer|archive-lookup-trials|archive-member-keys)-20261105|archive-directory-child-capacity-20261106|shared-stage-workers-20261107|archive-index-insertion-20261107|transactional-rg-records-20261107|immutable-zip-locator-evaluation-20261107|queued-read-cancellation-20261107|archive-btree-historical-decision-20261107)\.json\.gz$ + exclude: ^docs/benchmarks/((executor-copy-audit|bounded-read-ahead)-20261101|(directory-task-slots|packed-visit-snapshots|bounded-match-results|useful-match-workers|direct-evaluator-workers|derived-content-handoff|immutable-default-catalogs|prepared-diff-configuration|prepared-hash-operands|aggregate-comparison-totals|byte-shingle-representations|archive-input-buffers)-20261102|(digest-and-record-storage|caller-owned-streams|direct-source-materialization|hash-field-binding|hash-field-binding-tag|forward-cursor-skipping|raw-record-reuse|similarity-prefix-view|similarity-reference-reuse|owned-word-moves|smaller-shingle-probes|native-zip-seek-evaluation|encoded-record-reuse|threshold-aware-similarity)-20261103|(column-row-reuse|word-set-insertion|field-context-owned-results|selected-summary-output|fixed-summary-metrics|direct-summary-cells|compact-comparison-entries|comparison-category-sequence|native-name-paths|uniform-native-case|compact-traversal-metadata|eager-native-paths|native-case-experiments)-20261104|(folded-ascii-globs|bulk-directory-metadata|comparison-retention-followups|comparison-storage-followups|archive-prefix-reuse|archive-directory-range|archive-directory-allocation|archive-directory-observations|native-symlink-observations|native-metadata-followups|symlink-target-demand|native-pgo|readlink-short-target|cctz-empty-format|readlink-input-path|readlink-owned-transfer|archive-lookup-trials|archive-member-keys)-20261105|archive-directory-child-capacity-20261106|shared-stage-workers-20261107|archive-index-insertion-20261107|transactional-rg-records-20261107|immutable-zip-locator-evaluation-20261107|queued-read-cancellation-20261107|archive-btree-historical-decision-20261107|compiled-field-stat-demand-20261107)\.json\.gz$ - id: check-merge-conflict - id: check-yaml # BINARY_TYPES below: archive containers are binary but can START with text (a phar begins with diff --git a/CHANGELOG.md b/CHANGELOG.md index 356b5f7f04..209f43b866 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,9 @@ # 0.11.0 +- Avoid unnecessary per-entry stat requests for lexical listing templates while preserving required + metadata, fuzzy scores and conservative column output. + - Skip a fifth main-branch benchmark measurement once four candidates provide a compatible quorum. - Reduce temporary allocations and copies when lazily decoding the embedded language database. diff --git a/TODO.md b/TODO.md index 4590d093e7..1ed8887c20 100644 --- a/TODO.md +++ b/TODO.md @@ -601,6 +601,14 @@ EP/JC/P work; candidates are not speedup claims until measured on both platforms Separately assess skipping bulk reads when prepared expression metadata proves no eligible regular files or non-followed symlinks can contribute; preserve directory/link/mount stat semantics and avoid ad hoc flag-name, count or timer gates. Publish only measured improvements. +- [x] O31: Qualify compiled stat demand for implicit output and select template-only admission. + [Both-host raw evidence, all-cell results and explicit costs](docs/benchmarks/compiled-field-stat-demand-20261107.md) + reject the combined columns policy; preserve column eager behavior, fuzzy scores, mixed/ + unknown/denied metadata semantics and all independent traversal/safety requirements. +- [ ] O31 follow-ups: Identify the Linux many-directory TSV pipeline cost before admitting lexical + columns without metadata. Separately qualify score-consumer-specific demand and conditional + stat-bearing field admission; preserve error ordering and existing traversal requirements. + No ad hoc file-count, worker, timer or operating-system gate; retain current control costs. - [x] O20k: Qualify complete native non-followed symlink observations against every ordinary stat field, including dangling, long and hardlinked links. Preserve returned-attribute requirements, other special-type fallback, followed-target behavior and unchanged backend worker preference. diff --git a/docs/benchmarks/compiled-field-stat-demand-20261107.json.gz b/docs/benchmarks/compiled-field-stat-demand-20261107.json.gz new file mode 100644 index 0000000000..43a95af78f Binary files /dev/null and b/docs/benchmarks/compiled-field-stat-demand-20261107.json.gz differ diff --git a/docs/benchmarks/compiled-field-stat-demand-20261107.md b/docs/benchmarks/compiled-field-stat-demand-20261107.md new file mode 100644 index 0000000000..cb35fa1157 --- /dev/null +++ b/docs/benchmarks/compiled-field-stat-demand-20261107.md @@ -0,0 +1,204 @@ +# Compiled output field stat demand + +The combined template-and-column policy is not selected. It removes unnecessary metadata +requests and improves many lexical-output cases, but it repeatedly slows Linux TSV output +over many directories. The separate template-only policy is proposed after both-host development qualification against +a fresh main baseline. It leaves columns conservative on every platform and worker count. +This is a targeted improvement with measured costs, not a universal faster-path claim. + +## Requirement ownership and semantics + +The field registry owns a conservative stat requirement beside each bound renderer. Literal +segments, audited lexical path fields and supplied context values do not require stat metadata. +Unclassified bindings, size/time/identity fields, content fields, symlink targets and type fields +remain conservative. Compilation copies the requirement into the segment; command planning +queries the compiled requirements once. There is no per-entry field-name dispatch, new worker, +shared owner or synchronization primitive. + +Expression demand, colors, reductions, collections, deferred evaluation, native-case handling, +root/directory/unknown-type/followed-link traversal and safety action requirements remain +independent. A mixed `{name}:{size}` template still needs metadata. Lexical output follows the +same enumeration semantics as ordinary metadata-free listing: an attribute it never reads does +not cause a new stat failure. Required metadata errors still stop that entry. + +Fuzzy-score observation is a separate requirement. The original implementation coupled it to +full metadata. The first full test runs caught empty `{fuzzy}` output after the new admission +avoided eager stats. The corrected implementation preserves the original template/column score +collection independently of stat demand. Both failed suites and the initial, corrected and final +cross-batch regression sources stay in the evidence. The final test uses 1,000 entries, both one +and three workers, a successful score and a non-fuzzy OR result whose score must be empty. + +## Protocol and populations + +The combined policy uses base `d8cf61cccd4b0efed5058bf6fa45228e47162b48`. Its first bounded +screens retain three raw repetitions and the fastest two. Its initial full and corrected-image +full populations each have three sessions, nine raw repetitions and the fastest seven averaged +per case. Do not mix observations across these source/image identities. The correction was +required by a demonstrated correctness failure; it was not another attempt to select favorable +timings. Original images, controls and slower observations remain separate. + +Each full host population has 80 memory cases: broad/deep trees, 10 through 100,000 files, +one/three workers, plain listing, lexical template, lexical TSV columns and required-size +template. The native CLI population has 64 cases: flat/many-directory layouts, 10 through +10,000 files, the same four output tasks and CPU allowances. Each native command verifies its +complete independently generated output. Each memory case verifies complete output before +timing production `RunFind`; this does not substitute for the engine/CLI regression suites. + +Participants are unchanged parent, candidate and an exact same-image parent repeat. Native +participants rotate between rounds and discard warmups; the memory library randomly interleaves +cases/repetitions and participant order rotates across sessions and CPU groups. Linux pins one +or three distinct physical cores. macOS requests one/three workers without affinity enforcement. Linux native fixtures live in +`/dev/shm`; macOS uses warmed `/private/tmp` filesystem fixtures, not a claimed RAM disk. +Per case, divide candidate by parent within each session and take the median of those three +elapsed-time changes. A cohort median describes its included cases, not a weighted throughput +claim; inspect the per-case rows too. Negative elapsed-time change means faster. + +## Combined-policy counterexample + +The corrected cohort median for Linux three-worker lexical columns is about -1.00%, but the +many-directory 10,000-file TSV case is +24.36%. Its three session changes are -0.53%, +24.36% +and +35.24%; its same-image repeat median is -5.87%. Candidate elapsed times remain about +5.2-5.5 ms. The initial full image has the same case at +24.30% median, with session changes ++43.83%, +24.30% and +15.65%. This is an adoption blocker for the combined policy, despite large +flat-directory wins. No cause is claimed from these timings alone. + +Other costs remain visible: corrected macOS serial memory cohort medians are +2.21% for plain +listing and +2.31% for required-size templates. Individual required-size cells reach +6.31% on +macOS and +5.67% on Linux. Native ordinary-control medians are generally within about 1%, but +that does not erase individual costs or establish a universal no-regression result. + +## Separate template-only qualification + +O31b starts from fresh main `1786340680499f50b08dee5b1ceffbc20fd7b533`, after the independent +rg transaction change. Only implicit templates use the compiled requirement for admission; +column output keeps eager metadata on every platform and worker count. There is no count, +timer, CPU-specific or operating-system gate. The full protocol, all controls and the earlier +regressing TSV case were declared before measuring this source. Its measurements cannot be +added to earlier gains or used to infer a benchmark change caused solely by the base update. + +Both-host focused suites and all 323 targets pass; seven focused ASan targets pass on both +hosts, and seven focused MSan/TSan targets pass on Linux. The unchanged-parent counter test +produces exactly the expected +eight template failures: 1,001 stats instead of one for four output forms at one/three workers. +Columns still satisfy their unchanged 1,001-stat expectations. These development results cover the stated main baseline and variant; the actual synchronized +public head still needs its own required CI and integration review. + +The shipping CLI images use `clang_release` and `xff_full`; actual macOS main compile and link +actions both contain `-flto=thin`. The memory API harness has its own target configuration and +is not claimed to be the shipping LTO binary. The temporary measurement source is restored +after each build; its exact source and command/image receipts are retained for reproduction. + +## Template-only measured results + +Each table cell is a cohort median of the per-case three-session changes; its adjacent repeat +column uses the exact unchanged parent image. All individual rows and session signs are in +`analysis-final/all-case-comparisons.json` and CSV in the lossless capsule. + +### Native whole-command CLI + +8 cases per output/CPU cohort. Negative change is faster. + +| Output | CPUs | Mac change | Mac repeat | Linux change | Linux repeat | +| --------------------- | ---: | ---------: | ---------: | -----------: | -----------: | +| Lexical template | 1 | -4.34% | +0.64% | -11.18% | +0.44% | +| Lexical template | 3 | -1.06% | +0.91% | -2.43% | +0.62% | +| Columns control | 1 | +0.91% | +0.78% | +0.75% | +0.05% | +| Columns control | 3 | +0.74% | +0.21% | +0.64% | -0.23% | +| Plain control | 1 | +0.29% | -0.08% | +1.09% | +0.31% | +| Plain control | 3 | +1.12% | +0.29% | +1.22% | +0.12% | +| Size template control | 1 | +1.25% | +0.50% | +1.49% | +0.36% | +| Size template control | 3 | +0.68% | +0.64% | +0.89% | +1.71% | + +### Production RunFind in memory + +10 cases per output/CPU cohort. Negative change is faster. + +| Output | CPUs | Mac change | Mac repeat | Linux change | Linux repeat | +| --------------------- | ---: | ---------: | ---------: | -----------: | -----------: | +| Lexical template | 1 | -10.55% | -0.17% | -17.41% | -1.04% | +| Lexical template | 3 | -5.63% | +6.95% | -12.04% | -0.06% | +| Columns control | 1 | +1.92% | -0.18% | -0.65% | -0.30% | +| Columns control | 3 | +0.48% | +3.25% | +0.28% | +0.34% | +| Plain control | 1 | +3.24% | +0.31% | -1.24% | -0.44% | +| Plain control | 3 | +0.02% | +3.10% | -1.13% | +0.19% | +| Size template control | 1 | +4.04% | -0.68% | +1.14% | -0.33% | +| Size template control | 3 | +0.24% | +5.60% | +0.04% | +0.04% | + +The former Linux 10,000-file/many-directory/three-CPU TSV case is now +1.08% at its median, +versus -0.33% repeat. Session changes are +1.08%, -7.50% and +10.25%; do not describe it as +uniformly unchanged. Column controls overall remain close to their repeat cohorts. + +Costs are explicit. macOS serial memory plain/size cohorts are +3.24%/+4.04%, with repeats ++0.31%/-0.68%. A deep 100,000-file size case is +4.88% with -1.41% repeat; some small/medium +memory controls reach about +6%. Linux native flat/ten-file size is +7.69% (+3.10% repeat), +and many-directory/10,000-file serial size is +5.60% (+1.27% repeat). The cause is not established; +these are not removed as noise. Mac parallel memory repeats vary about +3% to +7% at cohort +level, limiting precision of those comparisons. No unchanged-image rerun was selected to erase +costs or enlarge a favorable result. + +The recommendation favors the lexical-template gain and removal of unnecessary required I/O +on both hosts, while retaining those bounded costs for review and further work. Native ordinary +control medians are about +0.3% to +1.5%; individual results need the qualifications above. +Do not multiply gains by earlier optimization percentages or apply them to different roots, +archive workloads, operating systems, CPU allocations or source versions. + +### Frozen shipping-config image sizes + +| Host | Parent bytes | Template-only bytes | Change bytes | +| ------------ | -----------: | ------------------: | -----------: | +| macOS arm64 | 8277640 | 8277528 | -112 | +| Linux x86_64 | 18454496 | 18451888 | -2608 | + +These are the exact experimental full CLI images, not downloaded release assets or compressed +packages. Image hashes and actual main compile/link action arguments are retained. The source +and configuration difference from the older combined-policy base is explicit; do not attribute +cross-base image differences to this single policy. + +## Verification and limits + +The corrected combined prototype passes all 323 targets on each host, seven focused ASan suites +on both hosts and seven focused MSan/TSan suites on Linux. Strengthening only the cross-batch +regression afterward passes its optimized and applicable sanitizer targets on both hosts. +These are development results on the stated prototype base, not final checks of a later public +head. Runtime adoption requires the actual synchronized public context to pass its own checks. +No local clang-tidy, peak RSS, hardware-counter, cold-cache or archive-specific speedup claim is +made. No experimental population is uploaded as an official merge/release benchmark series. + +Preparation failures are retained: an initially deprecated benchmark barrier, an unchanged failed +replacement attempt, the original fuzzy-score coupling failure, a missing workflow-input setup +failure and the corrected TSV-header expectation. The source completeness repair restores two +workflow action files without changing runtime inputs. A later cross-host check covers all +1,302 tracked files per role: the combined parent is identical, and candidate differences are +limited to two working prose files; runtime, tests and build inputs agree. + +## Remaining opportunities + +The combined column policy needs a causal pipeline investigation before adoption; do not add +an unexplained file-count, worker-count or platform threshold to conceal the counterexample. +A separate score-demand capability could eventually restrict fuzzy ranking work to consumers +that actually read scores. This trial deliberately preserves the old broader score-observation +contract for every template/column and does not claim that further optimization. Stat-bearing +field output also remains eager; conditional rendered-field metadata admission needs its own +error-ordering and traversal analysis rather than being silently included here. + +## Download and reproduce + +[Lossless evidence capsule](compiled-field-stat-demand-20261107.json.gz) retains +629 individually hashed records, 73,562,227 decoded record bytes and 6,954,149 compressed bytes. +SHA-256: `b52cc8687dd213cda89b958b31ca912c4846d8f9d00ae32714227d7afdbbd8be`. + +Restore each `records` entry according to `encoding`, then verify `bytes` and `sha256`. +The roots are `combined/` and `template_only/`. They retain the original/corrected source and +image identities, plans, all raw timing runs, all-cell analysis, failed and successful tests, +source agreement, build logs/actions and private runner programs. Some stage snapshots retain +identical records; copies are not independent observations. Executables and large transport +archives remain private; exact image hashes and reproducible source/configuration inputs are +included instead. + +`combined/analyze-final.py` recomputes the corrected-image case/cohort report. +`template_only/analyze-final.py` computes the separate template-only report. Adjust only their +input-root paths after restoring; neither rebuilds or times a binary. Native fixtures have an +explicit relative-path/byte-length specification and a constructor that requires an empty root. +Those tasks read paths and sizes, not file contents. The native launcher source is included; +its paths and output manifests must be adjusted for the new machine. Preserve the declared CPU, +participant, warmup and retention protocol when reproducing timing, and retain every result. diff --git a/docs/design-parallel.md b/docs/design-parallel.md index f8103ebdec..c252dc3897 100644 --- a/docs/design-parallel.md +++ b/docs/design-parallel.md @@ -337,6 +337,16 @@ without an additional existence check, just as a name-only directory listing can New descriptors conservatively require basic metadata until explicitly audited; birth-time consumers also declare `needs_birth_time`. +Implicit `--template` output uses stat requirements from compiled field bindings. Literal text, +lexical path components and supplied context values do not force full metadata; mixed output +such as `{name}:{size}` still does. New or unclassified bindings, content, symlink-target and type +fields remain conservative. Expression, reduction, color and traversal requirements stay independent. +Lexical templates have the same directory-entry semantics as ordinary metadata-free listing. +Fuzzy-score collection is independent of stat demand, so `{fuzzy}` keeps its evaluated score. +Column output remains eager: [qualification found a Linux multi-directory cost](benchmarks/compiled-field-stat-demand-20261107.md) +when lexical columns used the same admission policy. The template-only policy avoids that change; +its measured gains and remaining costs are documented there too. + Field selection is independent of eager versus lazy demand. Ordinary traversal and size, owner, mode and modification-time consumers request portable stat fields. Birth-time predicates and compiled `{btime}` fields request creation time too; arbitrary entry callbacks retain complete metadata. diff --git a/docs/performance-analysis.md b/docs/performance-analysis.md index a6eba9ebea..95e4b8b8f2 100644 --- a/docs/performance-analysis.md +++ b/docs/performance-analysis.md @@ -4903,3 +4903,32 @@ archive ownership/scanning and release linking; original exact-image/compiler re incomplete. Later captures are explicitly later. This is not current-main runtime qualification. A duplicate fresh draft was stopped before compilation/timing. Retain future directory-specific index designs separately and do not repeat this unchanged container policy until favorable. + +### O31: compiled field demand, template-only admission and retained costs + +Implicit output previously forced eager metadata for every compiled template or column, even +literal/path-only output. The field registry now owns a conservative stat requirement; compiled +segments retain it and planning queries it once. Only implicit templates use it for admission. +Columns remain eager after a repeated Linux many-directory/10,000-file/three-CPU TSV cost of +about +24% rejected the combined policy. There is no OS, timer, count or CPU threshold. +Fuzzy-score observation stays independent, preserving template/column scores and per-entry reset. + +[The complete decision, raw capsule and all-cell analysis](benchmarks/compiled-field-stat-demand-20261107.md) +retain initial screens, the original fuzzy correctness failure, corrected combined runs and the +separate template-only current-main experiment. Template-only native cohort medians improve +about 1-4% on Mac and 2-11% on Linux; memory targets improve about 6-17%. Costs are not hidden: +Mac serial memory plain/size cohorts are +3.24%/+4.04%, individual Linux native size controls +reach about +5.6%/+7.7%, and Mac parallel memory repeat variability limits precision. This is a +targeted optimization with documented tradeoffs, not a universal no-regression claim. + +The separate variant is based on `1786340680499f50b08dee5b1ceffbc20fd7b533`. Both-host seven +focused suites and all 323 targets pass, as do ASan on both hosts and focused MSan/TSan on Linux. +Parent proofs reproduce 1,001 stat requests rather than one for four template forms at one/three +workers; +columns retain their old eager expectations. Actual shipping-config compile/link action and +frozen-image receipts are retained; the measurement-only harness is restored. These development +results do not substitute for current synchronized public-head checks. + +Column pipeline causes, score-consumer-specific demand and conditional stat-bearing field +admission remain separate future work. Do not reopen the combined policy unchanged or use a +cohort median to hide its slower case. diff --git a/release-site.json b/release-site.json index ce64044c1c..259c05a339 100644 --- a/release-site.json +++ b/release-site.json @@ -140,7 +140,8 @@ "docs/benchmarks/archive-ancestor-hint-20261107.md": "docs/benchmarks/archive-ancestor-hint-20261107.html", "docs/benchmarks/immutable-zip-locator-evaluation-20261107.md": "docs/benchmarks/immutable-zip-locator-evaluation-20261107.html", "docs/benchmarks/queued-read-cancellation-20261107.md": "docs/benchmarks/queued-read-cancellation-20261107.html", - "docs/benchmarks/archive-btree-historical-decision-20261107.md": "docs/benchmarks/archive-btree-historical-decision-20261107.html" + "docs/benchmarks/archive-btree-historical-decision-20261107.md": "docs/benchmarks/archive-btree-historical-decision-20261107.html", + "docs/benchmarks/compiled-field-stat-demand-20261107.md": "docs/benchmarks/compiled-field-stat-demand-20261107.html" }, "generated_html": { "XFF.md": { @@ -168,6 +169,7 @@ "docs/benchmarks/archive-ancestor-hint-20261107.json.gz": "docs/benchmarks/archive-ancestor-hint-20261107.json.gz", "docs/benchmarks/immutable-zip-locator-evaluation-20261107.json.gz": "docs/benchmarks/immutable-zip-locator-evaluation-20261107.json.gz", "docs/benchmarks/queued-read-cancellation-20261107.json.gz": "docs/benchmarks/queued-read-cancellation-20261107.json.gz", - "docs/benchmarks/archive-btree-historical-decision-20261107.json.gz": "docs/benchmarks/archive-btree-historical-decision-20261107.json.gz" + "docs/benchmarks/archive-btree-historical-decision-20261107.json.gz": "docs/benchmarks/archive-btree-historical-decision-20261107.json.gz", + "docs/benchmarks/compiled-field-stat-demand-20261107.json.gz": "docs/benchmarks/compiled-field-stat-demand-20261107.json.gz" } } diff --git a/xff/engine/BUILD.bazel b/xff/engine/BUILD.bazel index b13db9a300..d708ea7451 100644 --- a/xff/engine/BUILD.bazel +++ b/xff/engine/BUILD.bazel @@ -527,6 +527,7 @@ cc_test( "@abseil-cpp//absl/status:statusor", "@abseil-cpp//absl/strings", "@googletest//:gtest_main", + "@mboworks_mbo//mbo/testing:matchers_cc", "@mboworks_mbo//mbo/testing:status_cc", ], ) diff --git a/xff/engine/metadata_test.cc b/xff/engine/metadata_test.cc index 63aa441b1c..fbbfc74657 100644 --- a/xff/engine/metadata_test.cc +++ b/xff/engine/metadata_test.cc @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) M. Boerger, the MBO Works authors // SPDX-License-Identifier: Apache-2.0 +#include #include #include #include @@ -14,6 +15,7 @@ #include "absl/time/time.h" #include "gmock/gmock.h" #include "gtest/gtest.h" +#include "mbo/testing/matchers.h" #include "mbo/testing/status.h" #include "xff/engine/run.h" #include "xff/parser/parser.h" @@ -22,6 +24,7 @@ namespace xff::engine { namespace { +using ::mbo::testing::EqualsText; using ::mbo::testing::IsOk; using ::testing::Eq; using ::testing::IsEmpty; @@ -107,6 +110,95 @@ TEST_F(MetadataTest, BasicConsumersDoNotRequestBirthTime) { EXPECT_THAT(fs.birth_requests.load(), Eq(0)); } +TEST_F(MetadataTest, LexicalTemplatesAvoidPerEntryStatsAndColumnsStayConservative) { + Populate(1'000); + constexpr auto kWorkers = std::to_array({1, 3}); + const auto arguments = std::to_array>({ + {"--template={path}:{name}:{ext}:{root}:{depth}"}, + {"--template={name:s/txt/log/}"}, + {"--template={{size}}:{def.label}", "--define=label=value"}, + {"--template=literal"}, + }); + for (const auto workers : kWorkers) { + for (const auto& argument : arguments) { + SCOPED_TRACE(argument.front()); + Check(argument, workers, 1); + } + Check({"--columns=path,name,ext", "--format=csv"}, workers, 1'001); + Check({"--columns=name:s/txt/log/", "--format=tsv"}, workers, 1'001); + } +} + +TEST_F(MetadataTest, LexicalOutputKeepsExpressionAndMixedFieldStatRequirements) { + Populate(1'000); + constexpr auto kWorkers = std::to_array({1, 3}); + for (const auto workers : kWorkers) { + Check({"--template={name}", "-name", "file0.txt", "-size", "7c"}, workers, 2); + Check({"--template={name}", "-name", "absent", "-size", "7c"}, workers, 1); + Check({"--template={name}:{size}"}, workers, 1'001); + Check({"--columns=path,size", "--format=csv"}, workers, 1'001); + Check({"--template={name}", "--summary=ext"}, workers, 1'001); + } +} + +TEST_F(MetadataTest, LexicalOutputWithUnknownTypesStillResolvesEntries) { + Populate(10, vfs::FileType::kUnknown); + constexpr auto kWorkers = std::to_array({1, 3}); + for (const auto workers : kWorkers) { + Check({"--template={name}"}, workers, 11); + Check({"--columns=path", "--format=csv"}, workers, 11); + } +} + +TEST_F(MetadataTest, LexicalOutputDoesNotRequireDeniedFileMetadata) { + Populate(2); + fs.fail_stats = true; + MBO_ASSERT_OK_AND_ASSIGN( + const auto command, parser::Parse({"--exact", "--color=never", "--template={name}", "root", "-type", "f"})); + std::string output; + std::vector errors; + const auto result = RunFind( + command, fs, [&](std::string_view value) { output += value; }, + [&](std::string_view, absl::Status status) { errors.push_back(status); }); + EXPECT_THAT(result.errors, Eq(0)); + EXPECT_THAT(errors, IsEmpty()); + EXPECT_THAT(output, EqualsText("file0.txt\nfile1.txt\n")); + EXPECT_THAT(fs.stats.load(), Eq(1)); +} + +TEST_F(MetadataTest, FuzzyOutputCollectsScoresWithoutPerEntryStats) { + Populate(1'000); + constexpr auto kWorkers = std::to_array({1, 3}); + constexpr auto kFormats = std::to_array({ + "--template={name}={fuzzy}", + "--columns=name,fuzzy", + }); + for (const auto workers : kWorkers) { + for (const auto format : kFormats) { + SCOPED_TRACE(workers); + SCOPED_TRACE(format); + MBO_ASSERT_OK_AND_ASSIGN( + const auto command, + parser::Parse( + {"--exact", "--color=never", absl::StrCat("--jobs=", workers), std::string(format), "--format=tsv", + "root", "-type", "f", "(", "-fuzzy:100%", "file0.txt", "-o", "-name", "file999.txt", ")"})); + fs.stats = 0; + std::string output; + std::vector errors; + const auto result = RunFind( + command, fs, [&](std::string_view value) { output += value; }, + [&](std::string_view, absl::Status status) { errors.push_back(status); }); + EXPECT_THAT(result.errors, Eq(0)); + EXPECT_THAT(errors, IsEmpty()); + EXPECT_THAT( + output, EqualsText( + format == kFormats.front() ? "path\nfile0.txt=100\nfile999.txt=\n" + : "name\tfuzzy\nfile0.txt\t100\nfile999.txt\t\n")); + EXPECT_THAT(fs.stats.load(), Eq(format == kFormats.front() ? 1 : 1'001)); + } + } +} + TEST_F(MetadataTest, ReferenceMetadataRequestsOnlyTheComparedFields) { Populate(10); Check({"-newer", "reference"}, 1, 22); diff --git a/xff/engine/run.cc b/xff/engine/run.cc index 45baeec097..9594503c5e 100644 --- a/xff/engine/run.cc +++ b/xff/engine/run.cc @@ -6142,9 +6142,16 @@ RunResult RunFindCore( for (std::size_t index = 0; index < deferred_nodes.size(); ++index) { deferred_node_order.emplace(deferred_nodes[index], index); } + // Keep columns conservative: metadata-free tabular output still has an unresolved + // multi-directory Linux cost. Qualify template admission separately. + const bool listing_stat_metadata = !column_templates.empty() || (compiled_tmpl && compiled_tmpl->NeedsStatMetadata()); const bool full_metadata = (fs_native_case && !uniform_native_case) || colorize || any_reduction - || matched_entry.has_value() || !column_templates.empty() || compiled_tmpl.has_value() - || collections.Active() || !deferred_nodes.empty(); + || matched_entry.has_value() || listing_stat_metadata || collections.Active() + || !deferred_nodes.empty(); + // Template score collection is independent of filesystem metadata. Preserve the + // previous output contract even when lexical/context fields avoid eager stats. + const bool collect_fuzzy_score = + full_metadata || rank_by_score || compiled_tmpl.has_value() || !column_templates.empty(); options.metadata = full_metadata ? MetadataDemand::kAlways : expression.has_value() ? ExpressionMetadata(*expression) : MetadataDemand::kNever; @@ -6238,8 +6245,8 @@ RunResult RunFindCore( } } parallel_match.emplace( - parallel_expression, options.workers, full_metadata || rank_by_score, std::move(content_output), - executions->Parallel(), std::move(derivation)); + parallel_expression, options.workers, collect_fuzzy_score, std::move(content_output), executions->Parallel(), + std::move(derivation)); } std::vector pending_matches; // Decision-only batches retain no file bytes or rendered line records. Give them more work @@ -6414,8 +6421,8 @@ RunResult RunFindCore( .zone_suffix = zone_suffix, .block_size = block_size, .fold_name_case = fold_name_case, - .fuzzy_score = full_metadata || rank_by_score ? mbo::types::OptionalRef{fuzzy_score} - : mbo::types::OptionalRef>{}, + .fuzzy_score = collect_fuzzy_score ? mbo::types::OptionalRef{fuzzy_score} + : mbo::types::OptionalRef>{}, .fuzzy_workspace = fuzzy_workspace, .hash_verification = hash_verification_summary ? mbo::types::OptionalRef{hash_verification} : mbo::types::OptionalRef>{}, diff --git a/xff/presentation/fields/fields.cc b/xff/presentation/fields/fields.cc index fd0a5030e8..b789c051ef 100644 --- a/xff/presentation/fields/fields.cc +++ b/xff/presentation/fields/fields.cc @@ -502,66 +502,316 @@ StringOrView EmptyField(std::string_view, std::string_view, const RenderContext& // Constexpr field-name -> renderer table, built once at compile time via mbo's // LimitedMap. Aliases (file/name, ext/extension, mode/perm) share a renderer; // the empty name backs {}, find's full-path placeholder (an alias for {path}). -using FieldEntry = std::pair; +struct FieldBinding { + FieldFn fn = nullptr; + // New/unclassified renderers conservatively require full stat metadata. + bool needs_stat_metadata = true; +}; + +using FieldEntry = std::pair; constexpr auto kFieldTable = mbo::container::MakeLimitedMap( - FieldEntry{"", &PathField}, // {} -> full path (find's -exec placeholder) - FieldEntry{"access", &AccessField}, - FieldEntry{"atime", &AtimeField}, - FieldEntry{"blocks", &BlocksField}, - FieldEntry{"btime", &BtimeField}, - FieldEntry{"column", &ColumnField}, - FieldEntry{"core", &CoreField}, - FieldEntry{"ctime", &CtimeField}, - FieldEntry{"depth", &DepthField}, - FieldEntry{"dev", &DevField}, - FieldEntry{"dir", &DirField}, - FieldEntry{"ext", &ExtField}, - FieldEntry{"extension", &ExtField}, - FieldEntry{"file", &NameField}, - FieldEntry{"fuzzy", &FuzzyField}, - FieldEntry{"gid", &GidField}, - FieldEntry{"group", &GroupField}, - FieldEntry{"hash", nullptr}, // Bound by HashBinding; no generic per-entry parser. - FieldEntry{"inode", &InodeField}, - FieldEntry{"lang", &LanguageField}, - FieldEntry{"lang-color", &LanguageColorField}, - FieldEntry{"lang-group", &LanguageGroupField}, - FieldEntry{"lang-source", &LanguageSourceField}, - FieldEntry{"lang-type", &LanguageTypeField}, - FieldEntry{"language", &LanguageField}, - FieldEntry{"line", &LineField}, - FieldEntry{"lines", &LinesField}, - FieldEntry{"links", &LinksField}, - FieldEntry{"match", &MatchField}, - FieldEntry{"mime", &MimeField}, - FieldEntry{"mime-category", &MimeCategoryField}, - FieldEntry{"mime-charset", &MimeCharsetField}, - FieldEntry{"mime-compressible", &MimeCompressibleField}, - FieldEntry{"mime-description", &MimeDescriptionField}, - FieldEntry{"mime-source", &MimeSourceField}, - FieldEntry{"mode", &ModeField}, - FieldEntry{"mtime", &MtimeField}, - FieldEntry{"name", &NameField}, - FieldEntry{"owner", &UserField}, - FieldEntry{"path", &PathField}, - FieldEntry{"perm", &ModeField}, - FieldEntry{"relpath", &RelpathField}, - FieldEntry{"root", &RootField}, - FieldEntry{"shard", &ShardField}, - FieldEntry{"size", &SizeField}, - FieldEntry{"stem", &StemField}, - FieldEntry{"suffix", &SuffixField}, - FieldEntry{"suffixes", &SuffixesField}, - FieldEntry{"target", &TargetField}, - FieldEntry{"text", &TextField}, - FieldEntry{"type", &TypeField}, - FieldEntry{"uid", &UidField}, - FieldEntry{"user", &UserField}); + FieldEntry{ + "", + { + .fn = &PathField, + .needs_stat_metadata = false, + }}, // {} -> full path (find's -exec placeholder) + FieldEntry{ + "access", + { + .fn = &AccessField, + }}, + FieldEntry{ + "atime", + { + .fn = &AtimeField, + }}, + FieldEntry{ + "blocks", + { + .fn = &BlocksField, + }}, + FieldEntry{ + "btime", + { + .fn = &BtimeField, + }}, + FieldEntry{ + "column", + { + .fn = &ColumnField, + .needs_stat_metadata = false, + }}, + FieldEntry{ + "core", + { + .fn = &CoreField, + .needs_stat_metadata = false, + }}, + FieldEntry{ + "ctime", + { + .fn = &CtimeField, + }}, + FieldEntry{ + "depth", + { + .fn = &DepthField, + .needs_stat_metadata = false, + }}, + FieldEntry{ + "dev", + { + .fn = &DevField, + }}, + FieldEntry{ + "dir", + { + .fn = &DirField, + .needs_stat_metadata = false, + }}, + FieldEntry{ + "ext", + { + .fn = &ExtField, + .needs_stat_metadata = false, + }}, + FieldEntry{ + "extension", + { + .fn = &ExtField, + .needs_stat_metadata = false, + }}, + FieldEntry{ + "file", + { + .fn = &NameField, + .needs_stat_metadata = false, + }}, + FieldEntry{ + "fuzzy", + { + .fn = &FuzzyField, + .needs_stat_metadata = false, + }}, + FieldEntry{ + "gid", + { + .fn = &GidField, + }}, + FieldEntry{ + "group", + { + .fn = &GroupField, + }}, + FieldEntry{ + "hash", + { + .fn = nullptr, + }}, // Bound by HashBinding; no generic per-entry parser. + FieldEntry{ + "inode", + { + .fn = &InodeField, + }}, + FieldEntry{ + "lang", + { + .fn = &LanguageField, + .needs_stat_metadata = false, + }}, + FieldEntry{ + "lang-color", + { + .fn = &LanguageColorField, + .needs_stat_metadata = false, + }}, + FieldEntry{ + "lang-group", + { + .fn = &LanguageGroupField, + .needs_stat_metadata = false, + }}, + FieldEntry{ + "lang-source", + { + .fn = &LanguageSourceField, + .needs_stat_metadata = false, + }}, + FieldEntry{ + "lang-type", + { + .fn = &LanguageTypeField, + .needs_stat_metadata = false, + }}, + FieldEntry{ + "language", + { + .fn = &LanguageField, + .needs_stat_metadata = false, + }}, + FieldEntry{ + "line", + { + .fn = &LineField, + .needs_stat_metadata = false, + }}, + FieldEntry{ + "lines", + { + .fn = &LinesField, + }}, + FieldEntry{ + "links", + { + .fn = &LinksField, + }}, + FieldEntry{ + "match", + { + .fn = &MatchField, + .needs_stat_metadata = false, + }}, + FieldEntry{ + "mime", + { + .fn = &MimeField, + .needs_stat_metadata = false, + }}, + FieldEntry{ + "mime-category", + { + .fn = &MimeCategoryField, + .needs_stat_metadata = false, + }}, + FieldEntry{ + "mime-charset", + { + .fn = &MimeCharsetField, + .needs_stat_metadata = false, + }}, + FieldEntry{ + "mime-compressible", + { + .fn = &MimeCompressibleField, + .needs_stat_metadata = false, + }}, + FieldEntry{ + "mime-description", + { + .fn = &MimeDescriptionField, + .needs_stat_metadata = false, + }}, + FieldEntry{ + "mime-source", + { + .fn = &MimeSourceField, + .needs_stat_metadata = false, + }}, + FieldEntry{ + "mode", + { + .fn = &ModeField, + }}, + FieldEntry{ + "mtime", + { + .fn = &MtimeField, + }}, + FieldEntry{ + "name", + { + .fn = &NameField, + .needs_stat_metadata = false, + }}, + FieldEntry{ + "owner", + { + .fn = &UserField, + }}, + FieldEntry{ + "path", + { + .fn = &PathField, + .needs_stat_metadata = false, + }}, + FieldEntry{ + "perm", + { + .fn = &ModeField, + }}, + FieldEntry{ + "relpath", + { + .fn = &RelpathField, + .needs_stat_metadata = false, + }}, + FieldEntry{ + "root", + { + .fn = &RootField, + .needs_stat_metadata = false, + }}, + FieldEntry{ + "shard", + { + .fn = &ShardField, + .needs_stat_metadata = false, + }}, + FieldEntry{ + "size", + { + .fn = &SizeField, + }}, + FieldEntry{ + "stem", + { + .fn = &StemField, + .needs_stat_metadata = false, + }}, + FieldEntry{ + "suffix", + { + .fn = &SuffixField, + .needs_stat_metadata = false, + }}, + FieldEntry{ + "suffixes", + { + .fn = &SuffixesField, + .needs_stat_metadata = false, + }}, + FieldEntry{ + "target", + { + .fn = &TargetField, + }}, + FieldEntry{ + "text", + { + .fn = &TextField, + .needs_stat_metadata = false, + }}, + FieldEntry{ + "type", + { + .fn = &TypeField, + }}, + FieldEntry{ + "uid", + { + .fn = &UidField, + }}, + FieldEntry{ + "user", + { + .fn = &UserField, + }}); // Resolves a field name to its renderer; an unknown name renders empty. FieldFn LookupField(std::string_view name) { const auto it = kFieldTable.find(name); - return it == kFieldTable.end() ? &EmptyField : it->second; + return it == kFieldTable.end() ? &EmptyField : it->second.fn; } struct ParsedField { @@ -670,20 +920,44 @@ StringOrView OutputField(std::string_view key, std::string_view, const RenderCon // {0}..{N} -> a regex capture; the {env.NAME} namespace -> the environment; // otherwise a builtin field from the table (empty key). New namespaces // ({def.*}, {capture.*}) slot in here. -std::pair ResolveName(std::string_view name) { +struct ResolvedField { + FieldFn fn; + std::string key; + bool needs_stat_metadata = true; +}; + +ResolvedField ResolveName(std::string_view name) { if (CaptureIndex(name) >= 0) { - return {&CaptureField, std::string(name)}; + return { + .fn = &CaptureField, + .key = std::string(name), + .needs_stat_metadata = false, + }; } if (name.starts_with(kEnvironmentNamespace)) { - return {&EnvField, std::string(name.substr(kEnvironmentNamespace.size()))}; + return { + .fn = &EnvField, + .key = std::string(name.substr(kEnvironmentNamespace.size())), + .needs_stat_metadata = false, + }; } if (name.starts_with(kDefinitionNamespace)) { - return {&DefField, std::string(name.substr(kDefinitionNamespace.size()))}; + return { + .fn = &DefField, + .key = std::string(name.substr(kDefinitionNamespace.size())), + .needs_stat_metadata = false, + }; } if (name.starts_with(kCaptureNamespace)) { - return {&OutputField, std::string(name.substr(kCaptureNamespace.size()))}; + return { + .fn = &OutputField, + .key = std::string(name.substr(kCaptureNamespace.size())), + .needs_stat_metadata = false, + }; } - return {LookupField(name), std::string()}; + const auto found = kFieldTable.find(name); + return found == kFieldTable.end() ? ResolvedField{.fn = &EmptyField,} + : ResolvedField{.fn = found->second.fn, .needs_stat_metadata = found->second.needs_stat_metadata,}; } // A qualifier is a sed-style rewrite when it is `s` followed by a punctuation @@ -1270,7 +1544,7 @@ Template Template::Compile(std::string_view tmpl) { } flush_literal(); compiled.validation_.Update(ValidateFieldName(field->name)); - auto [fn, key] = ResolveName(field->name); // builtin field, {0}..{N} capture, or {env.NAME} + auto [fn, key, needs_stat_metadata] = ResolveName(field->name); // builtin field, {0}..{N} capture, or {env.NAME} std::string& qualifier = field->qualifier; // Classify the qualifier: an s/// rewrite, an m/// per-line extraction, or a path-component // extraction is a post-render transform; anything else is the field's own format argument. @@ -1288,6 +1562,7 @@ Template Template::Compile(std::string_view tmpl) { .qualifier = std::move(qualifier), .post = post, .hash_binding = binding, + .needs_stat_metadata = needs_stat_metadata, .transform = transform, }); i = field->next; @@ -1313,6 +1588,10 @@ bool Template::ReferencesCapture(std::string_view name) const { }); } +bool Template::NeedsStatMetadata() const { + return std::ranges::any_of(segments_, [](const Segment& segment) { return segment.needs_stat_metadata; }); +} + bool Template::NeedsBirthTime() const { return absl::c_any_of(segments_, [](const Segment& segment) { return segment.fn == &BtimeField; }); } diff --git a/xff/presentation/fields/fields.h b/xff/presentation/fields/fields.h index 64fc93969a..ca88205152 100644 --- a/xff/presentation/fields/fields.h +++ b/xff/presentation/fields/fields.h @@ -157,6 +157,10 @@ class Template { // Inspect parsed hash fields, including post-processing, without reading any entry content. hash::DefaultUsage HashDefaultsUsed() const; + // Whether any compiled field requires full stat metadata. Literal, lexical path and + // supplied-context values do not. Unclassified/content/link/type fields stay conservative. + bool NeedsStatMetadata() const; + // Whether a compiled field consumes birth time, including transformed fields. bool NeedsBirthTime() const; @@ -211,6 +215,7 @@ class Template { std::string qualifier; PostProcess post = PostProcess::kNone; HashBinding hash_binding; + bool needs_stat_metadata = false; std::shared_ptr transform; bool IsField() const { return fn != nullptr || hash_binding.algorithm != 0; } diff --git a/xff/presentation/fields/fields_test.cc b/xff/presentation/fields/fields_test.cc index 7a3a275079..ce95bfdc11 100644 --- a/xff/presentation/fields/fields_test.cc +++ b/xff/presentation/fields/fields_test.cc @@ -226,6 +226,89 @@ TEST_F(FieldsTest, TimeZoneDemandIncludesTransformedTimeFields) { EXPECT_THAT(Template::Compile("{{mtime}} {name} {size} {text}").NeedsTimeZone(), IsFalse()); } +TEST_F(FieldsTest, StatDemandComesFromCompiledFieldBindingsIncludingAliasesAndTransforms) { + constexpr auto kStatFields = std::to_array({ + "{access}", + "{atime}", + "{blocks:h}", + "{btime:epoch}", + "{ctime}", + "{dev}", + "{gid}", + "{group}", + "{hash}", + "{hash:md5}", + "{inode}", + "{lines}", + "{links}", + "{mode}", + "{mtime}", + "{owner}", + "{perm}", + "{size}", + "{target}", + "{type}", + "{uid}", + "{user}", + "{name}:{size:s/7/seven/}", + }); + constexpr auto kContextFields = std::to_array({ + "", + "literal", + "{{size}}", + "{}", + "{path}", + "{relpath}", + "{name}", + "{file}", + "{root}", + "{dir}", + "{ext}", + "{extension}", + "{stem}", + "{core}", + "{suffix}", + "{suffixes}", + "{depth}", + "{lang}", + "{language}", + "{lang-type}", + "{lang-color}", + "{lang-group}", + "{lang-source}", + "{mime}", + "{mime-category}", + "{mime-charset}", + "{mime-compressible}", + "{mime-description}", + "{mime-source}", + "{line}", + "{text}", + "{match}", + "{column}", + "{fuzzy}", + "{shard}", + "{0}", + "{def.size}", + "{env.XFF_TEST_UNUSED}", + "{capture.size}", + "{name:s/txt/log/}", + "{def.value:m/x/y/;join(,)}", + }); + for (const auto text : kStatFields) { + SCOPED_TRACE(text); + const auto field = Template::Compile(text); + ASSERT_THAT(field.Validate(), IsOk()); + EXPECT_THAT(field.NeedsStatMetadata(), IsTrue()); + } + for (const auto text : kContextFields) { + SCOPED_TRACE(text); + const auto field = Template::Compile(text); + ASSERT_THAT(field.Validate(), IsOk()); + EXPECT_THAT(field.NeedsStatMetadata(), IsFalse()); + } +} + TEST_F(FieldsTest, LinkTargetDemandUsesCompiledBindingsIncludingTransforms) { static constexpr auto kTargetFields = std::to_array({ "{target}",