diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 13a1ccf1c7..21d432e18e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -24,7 +24,7 @@ repos: hooks: - id: check-added-large-files # Retain the exact lossless cross-platform performance evidence archives. - exclude: ^docs/benchmarks/((executor-copy-audit|bounded-read-ahead)-20261101|(directory-task-slots|packed-visit-snapshots|bounded-match-results|useful-match-workers|direct-evaluator-workers|derived-content-handoff|immutable-default-catalogs|prepared-diff-configuration|prepared-hash-operands|aggregate-comparison-totals|byte-shingle-representations|archive-input-buffers)-20261102|(digest-and-record-storage|caller-owned-streams|direct-source-materialization|hash-field-binding|hash-field-binding-tag|forward-cursor-skipping|raw-record-reuse|similarity-prefix-view|similarity-reference-reuse|owned-word-moves|smaller-shingle-probes|native-zip-seek-evaluation|encoded-record-reuse|threshold-aware-similarity)-20261103|(column-row-reuse|word-set-insertion|field-context-owned-results|selected-summary-output|fixed-summary-metrics|direct-summary-cells|compact-comparison-entries|comparison-category-sequence|native-name-paths|uniform-native-case|compact-traversal-metadata|eager-native-paths|native-case-experiments)-20261104|(folded-ascii-globs|bulk-directory-metadata|comparison-retention-followups|comparison-storage-followups|archive-prefix-reuse|archive-directory-range|archive-directory-allocation|archive-directory-observations|native-symlink-observations|native-metadata-followups|symlink-target-demand|native-pgo|readlink-short-target|cctz-empty-format|readlink-input-path|readlink-owned-transfer|archive-lookup-trials|archive-member-keys)-20261105|archive-directory-child-capacity-20261106|shared-stage-workers-20261107|archive-index-insertion-20261107|transactional-rg-records-20261107|immutable-zip-locator-evaluation-20261107|queued-read-cancellation-20261107|archive-btree-historical-decision-20261107|compiled-field-stat-demand-20261107|unchanged-site-payloads-20261107|archive-read-buffer-clear-20261107|archive-read-buffer-requalification-20261108)\.json\.gz$ + exclude: ^docs/benchmarks/((executor-copy-audit|bounded-read-ahead)-20261101|(directory-task-slots|packed-visit-snapshots|bounded-match-results|useful-match-workers|direct-evaluator-workers|derived-content-handoff|immutable-default-catalogs|prepared-diff-configuration|prepared-hash-operands|aggregate-comparison-totals|byte-shingle-representations|archive-input-buffers)-20261102|(digest-and-record-storage|caller-owned-streams|direct-source-materialization|hash-field-binding|hash-field-binding-tag|forward-cursor-skipping|raw-record-reuse|similarity-prefix-view|similarity-reference-reuse|owned-word-moves|smaller-shingle-probes|native-zip-seek-evaluation|encoded-record-reuse|threshold-aware-similarity)-20261103|(column-row-reuse|word-set-insertion|field-context-owned-results|selected-summary-output|fixed-summary-metrics|direct-summary-cells|compact-comparison-entries|comparison-category-sequence|native-name-paths|uniform-native-case|compact-traversal-metadata|eager-native-paths|native-case-experiments)-20261104|(folded-ascii-globs|bulk-directory-metadata|comparison-retention-followups|comparison-storage-followups|archive-prefix-reuse|archive-directory-range|archive-directory-allocation|archive-directory-observations|native-symlink-observations|native-metadata-followups|symlink-target-demand|native-pgo|readlink-short-target|cctz-empty-format|readlink-input-path|readlink-owned-transfer|archive-lookup-trials|archive-member-keys)-20261105|archive-directory-child-capacity-20261106|shared-stage-workers-20261107|archive-index-insertion-20261107|transactional-rg-records-20261107|immutable-zip-locator-evaluation-20261107|queued-read-cancellation-20261107|archive-btree-historical-decision-20261107|compiled-field-stat-demand-20261107|unchanged-site-payloads-20261107|archive-read-buffer-clear-20261107|archive-read-buffer-requalification-20261108|language-list-reserve-20261107|language-list-reserve-requalification-20261108)\.json\.gz$ - id: check-merge-conflict - id: check-yaml # BINARY_TYPES below: archive containers are binary but can START with text (a phar begins with diff --git a/TODO.md b/TODO.md index 7a0e7dab18..a19e5aa89d 100644 --- a/TODO.md +++ b/TODO.md @@ -1532,3 +1532,10 @@ this as intentional accounting. - [ ] Fix the legacy ReadCompressedSingleFile API's rejection of a valid empty gzip. Retain the unchanged-parent availability control in O32; do not turn its missing content result into a zero-time speed ratio or silently remove it from the measured matrix. + +- [x] Evaluate exact reservation for known-length language JSON lists on both hosts. + Retain the current runtime: small linked cold-load gains do not establish a useful + practical command-level improvement alongside the complete controls. Preserve both + harness versions, slower cases and all raw observations in + [the decision report](docs/benchmarks/language-list-reserve-20261107.md). + Generated immutable catalog representation remains a separate O14 candidate. diff --git a/docs/benchmarks/language-list-reserve-20261107.json.gz b/docs/benchmarks/language-list-reserve-20261107.json.gz new file mode 100644 index 0000000000..63938ec87d Binary files /dev/null and b/docs/benchmarks/language-list-reserve-20261107.json.gz differ diff --git a/docs/benchmarks/language-list-reserve-20261107.md b/docs/benchmarks/language-list-reserve-20261107.md new file mode 100644 index 0000000000..780ee327f2 --- /dev/null +++ b/docs/benchmarks/language-list-reserve-20261107.md @@ -0,0 +1,113 @@ +# Exact language JSON list reservation, 2026-11-07 + +Retain the current runtime. The one-line prototype reserves `found->size()` for the +`StringList` result before appending borrowed JSON strings. Cold linked-language +loads improve modestly, but the complete population also contains slower controls +and substantial variation between identical images. This screen does not establish +a useful practical command-level gain. Keep the evidence and do not add the runtime +or the research executable to production. + +## Scope and source + +The baseline is merged main `77aa9ef9bc9475adcbdd37cf3004143a96f03060`. +The candidate changes only `xff/matching/language/language.cc` by adding +`result.reserve(found->size());`. Parsing, validation, claims, precedence, +normalization, registration, immutable snapshot ownership and fallback behavior +remain unchanged. This is not a generated-table implementation or an allocation +counting measurement. No library, wrapper, cache, flag or worker policy is added. + +A source-only transfer to an isolated Linux workspace verifies all 1,023 source +hashes. Parent and candidate C++ bytes match across hosts. Each host uses separate +immutable parent, candidate and byte-identical parent-copy images. Linux images +link only the system C/math libraries and loader; Mac images link only system +libc++/System. Rebuilding the candidate cannot change a library loaded by its parent. +Hermetic optimized diagnostics use the recorded `clang_release` configuration; +these executable sizes and timings are not published CLI size or ThinLTO claims. + +## Measurement contract + +Two harness versions remain separate. The initial Google Benchmark driver completes +all 16 cases on Linux. Mac observations reveal about one second of per-process +bookkeeping outside the measured API call; that would cost about 24 minutes for the +population. The private Mac run is intentionally stopped, preserving every completed +and partial artifact. The direct-clock replacement repeats the complete population +on both hosts; no initial sample is overwritten, relabeled or used as a later sample. + +The final driver starts a fresh process for every observation, prepares fixtures +before the clock, then measures the first `ActiveSnapshot` and optionally +`ActiveCatalog`. A warm control initializes the snapshot before the clock and +measures 1,000 subsequent acquisitions. All synthetic list values in all five fields +are verified independently outside timing, along with the ordinary C++ filename +classification. A failure returns nonzero; no observation fails those checks. + +The lean image covers cold snapshot/catalog, warm snapshot, and 8/128-language +fixtures with 0/1/4/16/64 values per list. The full image adds the actual linked +Linguist database for cold snapshot/catalog and warm controls. Each of three sessions +randomizes case order and rotates participant order, with nine measured rounds, +fastest-seven means, and one fresh-process warmup per participant/case/session. +Each host has 1,296 timed observations and 144 warmups. Linux pins every measurement +to one allowed CPU; no macOS pinning is claimed. Both hosts avoid concurrent local +builds/hooks during timings. Absolute API timing excludes process startup, fixture +construction, validation and printing. + +## Complete final population + +Negative elapsed change means faster. Each cell is the median of its three session +ratios, using each session's fastest-seven mean. The copied-parent values remain +independent controls; they are not subtracted to manufacture a corrected result. + +| Image | Case | Mac candidate | Mac copy | Linux candidate | Linux copy | +| :---- | :-------------- | ------------: | -------: | --------------: | ---------: | +| lean | `cold/snapshot` | +0.877% | +2.255% | -1.346% | -0.188% | +| lean | `cold/catalog` | +4.992% | +4.882% | +0.228% | -1.382% | +| lean | `warm/snapshot` | +0.217% | +0.727% | +0.000% | +0.052% | +| lean | `lists/8/0` | +2.081% | +0.175% | -1.417% | -0.738% | +| lean | `lists/8/1` | -0.255% | -1.237% | -2.357% | -1.273% | +| lean | `lists/8/4` | -2.972% | -0.187% | -3.702% | -4.227% | +| lean | `lists/8/16` | -0.806% | +2.040% | +0.741% | +0.171% | +| lean | `lists/8/64` | -1.890% | -0.638% | +0.094% | +0.412% | +| lean | `lists/128/0` | +0.161% | +0.351% | +1.792% | +0.248% | +| lean | `lists/128/1` | -0.623% | -0.093% | +0.349% | +0.197% | +| lean | `lists/128/4` | -1.976% | +0.095% | -0.757% | -0.345% | +| lean | `lists/128/16` | -1.781% | +0.684% | -1.078% | +0.499% | +| lean | `lists/128/64` | -0.134% | -0.317% | -0.039% | +0.117% | +| full | `cold/snapshot` | -1.121% | +0.600% | -0.627% | +0.073% | +| full | `cold/catalog` | -0.474% | +0.558% | -0.556% | -0.036% | +| full | `warm/snapshot` | -0.504% | +1.084% | -0.111% | -0.202% | + +The linked cold snapshot/catalog medians are -1.121%/-0.474% on Mac and +-0.627%/-0.556% on Linux. Every final session for those four cases improves, which +is retained rather than dismissed. However, the earlier complete Linux driver gives +cold linked changes near its copied-parent drift, the final warm unchanged controls +also move, and the overall isolated signal remains small. No end-to-end CLI or +filesystem result establishes its practical value. + +Keep the costs too: the Mac empty eight-language list case is +2.081% overall and +positive in all three sessions (+2.081%, +3.129%, +1.225%). Linux's empty +128-language case is +1.792%, with sessions +1.296%, +6.654%, +1.792%. +The unaffected lean Mac catalog median is +4.992%, versus its copied parent +4.882%. +Do not assign an allocator/cache/layout explanation or an empty-list threshold to +these observations. A removed growth operation alone does not prove a command gain. + +## Verification and limits + +Six focused release targets pass on each host: language, catalog, default cache, +language-database integration, shared language API, and the linked language extension. +The runtime source tested there is exactly the one-line candidate. Every v2 raw +output, iteration count, session mean, copy ratio, candidate ratio and final median +is independently recomputed from the retained per-process records. Linux transport +verifies 8,682 evidence hashes before analysis. + +This is an exploratory rejection screen, not a final runtime qualification. No full +release suite, sanitizer, local clang-tidy, public-head CI, allocation count, peak RSS, +startup or whole-command speedup is claimed. No additional final-runtime cycle is +justified for this unselected implementation. Existing published PR heads remain +unchanged; the decision can accompany the candidate-inventory audit. + +## Evidence + +[Lossless source, configuration, logs and observations](language-list-reserve-20261107.json.gz) +contain both harness versions and all raw populations. Executable identities are +retained; binaries and repeated full source archives are private. The public baseline +commit plus the source hash manifest and exact experimental sources reconstruct the +trial. The capsule identity is recorded alongside the handoff. diff --git a/docs/benchmarks/language-list-reserve-requalification-20261108.json.gz b/docs/benchmarks/language-list-reserve-requalification-20261108.json.gz new file mode 100644 index 0000000000..8f2b1a3baa Binary files /dev/null and b/docs/benchmarks/language-list-reserve-requalification-20261108.json.gz differ diff --git a/docs/benchmarks/language-list-reserve-requalification-20261108.md b/docs/benchmarks/language-list-reserve-requalification-20261108.md new file mode 100644 index 0000000000..9418c64c1f --- /dev/null +++ b/docs/benchmarks/language-list-reserve-requalification-20261108.md @@ -0,0 +1,63 @@ +# Language list reservation requalification after #1143 + +Keep the current runtime. Repeating the exact reservation prototype on current main still +shows small and mixed results, without a demonstrated practical command-level improvement. + +## Measurement + +The baseline is main `50fe2426d690156e5c388eb879b6295c4e123801`, after #1140 and #1144. +The unchanged language source matches the original baseline byte for byte. The candidate +adds only the original `result.reserve(found->size());` statement. The original private +diagnostic and direct-clock driver are used unchanged; no runtime change enters this PR. + +Each host builds lean and linked-database variants under `--config=clang_release`, copies +each image immediately after its build, and retains exact baseline-copy controls. Linux +pins measurements to one allowed CPU; macOS is unpinned. Three sessions each measure all +16 cases with nine observations, retain the fastest seven and include a fresh-process +warmup per participant/case. Each host produces 1,296 timed observations and 144 warmups. +Every invocation validates complete generated lists and normal classification outside the +clock. All 2,880 observations, case means, session ratios and medians were independently +recomputed. All 4,345 transferred Linux files were verified by SHA-256; both hosts used +identical baseline, candidate, diagnostic, rule and driver bytes. + +## Results + +Negative is faster. Per-case values are medians of the three session ratios, with the +exact-image copy as a separate control; they are descriptive and are not significance tests. + +| Image | Case | macOS candidate | macOS copy | Linux candidate | Linux copy | +| :---- | :-------------- | --------------: | ---------: | --------------: | ---------: | +| lean | `cold/snapshot` | +4.718% | +2.226% | -1.735% | -3.599% | +| lean | `cold/catalog` | +2.015% | +0.442% | -0.016% | +2.088% | +| lean | `warm/snapshot` | -0.146% | -0.215% | +0.181% | +0.012% | +| lean | `lists/8/0` | -0.971% | +1.633% | +1.694% | +0.079% | +| lean | `lists/8/1` | -0.202% | +1.367% | +1.892% | +0.416% | +| lean | `lists/8/4` | -2.278% | +0.722% | -1.717% | -1.368% | +| lean | `lists/8/16` | -1.004% | +0.758% | -0.958% | +1.574% | +| lean | `lists/8/64` | -1.796% | -1.126% | -0.829% | -0.463% | +| lean | `lists/128/0` | -0.803% | +0.126% | +1.683% | -0.827% | +| lean | `lists/128/1` | -0.609% | +0.661% | +1.168% | +0.211% | +| lean | `lists/128/4` | -2.863% | -0.030% | -0.960% | +0.367% | +| lean | `lists/128/16` | -2.105% | -0.693% | -0.467% | -1.125% | +| lean | `lists/128/64` | -0.448% | +0.864% | -0.131% | -1.791% | +| full | `cold/snapshot` | -0.909% | +0.333% | -0.420% | +0.003% | +| full | `cold/catalog` | -0.483% | +0.062% | +0.296% | +0.104% | +| full | `warm/snapshot` | +0.292% | +0.297% | +0.124% | -0.063% | + +The full linked cold snapshot changes -0.909% on macOS and -0.420% on Linux; +cold catalog changes -0.483% and +0.296%, respectively. Linux 8-language lists with +zero or one value cost +1.694%/+1.892%, and the 128-language empty case costs +1.683% +(copy -0.827%). The lean macOS cold snapshot costs +4.718% (copy +2.226%). Keep all +cases and controls visible. The isolated results do not establish a general CLI speedup; +no allocation-count, RSS or production-binary claim follows from this screen. + +The decision remains to retain the current runtime, with the audit and negative result +available for future work. The earlier qualification remains separate and unchanged. + +## Evidence + +[The lossless capsule](language-list-reserve-requalification-20261108.json.gz) retains 8,680 individually hashed records +(542,618 compressed bytes; SHA-256 `7ecdc251f181d979cc1ca839f725dc25343027588bca740923d895e4b82205fc`). It includes all raw process output, +ordering, summaries, source/driver snapshots, build and linkage logs, image hashes and +Linux transport manifest. Executables remain private. Both host builds and all 2,880 +observations completed successfully. diff --git a/docs/performance-analysis.md b/docs/performance-analysis.md index 47dc668253..39f7107d24 100644 --- a/docs/performance-analysis.md +++ b/docs/performance-analysis.md @@ -5028,3 +5028,25 @@ research benchmark target/source is published. Current public integration CI rem [Fresh post-#1143 requalification](benchmarks/archive-read-buffer-requalification-20261108.md) retains the memory-TAR benefit (-8.47% macOS/-3.78% Linux) with narrower native-TAR gains and every slower gzip case and copy control. The original report/capsule stays frozen; exact-head CI remains required. + +### O14 follow-up: exact JSON list reservation not selected + +A one-line `StringList` prototype reserves the already-known JSON array length. +Six focused release suites pass on each host, and complete fresh-process controls +preserve every list value. Three balanced sessions per host (16 cases, nine rounds, +fastest seven, identical parent copy) show modest linked cold snapshot/catalog gains: +Mac -1.121%/-0.474%, Linux -0.627%/-0.556%. The complete earlier Linux driver and +unchanged/empty-list controls retain variability and costs; no convincing practical +whole-command gain is established. Keep the current runtime and preserve both +harness versions, the complete final 1,296 timed observations plus 144 warmups per +host, and the partial first Mac attempt in +[the decision report](benchmarks/language-list-reserve-20261107.md). +This does not qualify generated immutable catalogs or close the broader O14 design. + +The [candidate completion audit](performance-candidate-completion-audit.md) preserves +the original 22families, distinguishes rejected trials from unimplemented parent +requirements, and identifies the next ownership/admission gates for O18. It does +not substitute historical or child checks for current integration and main health. + +[Current-main requalification](benchmarks/language-list-reserve-requalification-20261108.md) repeats all 16 cases on both +hosts. The linked gains remain small and mixed; the exact reservation stays unselected. diff --git a/docs/performance-candidate-completion-audit.md b/docs/performance-candidate-completion-audit.md new file mode 100644 index 0000000000..eab52fcc63 --- /dev/null +++ b/docs/performance-candidate-completion-audit.md @@ -0,0 +1,152 @@ +# Performance candidate completion audit, 2026-11-07 + +The performance goal is not complete. A completed experiment, an accepted private +handoff, a published PR, a successful PR run and a successful merged-main run are +different evidence states. Preserve all candidates and rejected implementations; +do not replace unfinished parent requirements with whichever child has passed. + +## Audit boundary + +GitHub's main ref was read directly for this audit and is +`77aa9ef9bc9475adcbdd37cf3004143a96f03060` (#1129). Its actual Test workflow +`42359504322` succeeds. The table reconciles that tree's canonical TODO and +performance analysis with the frozen, separately published follow-ups through +#1140. Linked reports document the original populations and their limits. This +audit does not rerun every historical benchmark, independently reprove every past +performance estimate, or turn old green checks into current PR-context checks. + +The dedicated orchestration ledger and GitHub remain authoritative for the moving +PR graph. A snapshot here is not a merge schedule, check substitution or readiness +claim. Required current-head integration and resulting main health are still +required for every published implementation. + +## Original candidate families + +The original O01-O22 inventory is retained. A selected subset in a partially +completed family is useful progress, not completion of the full parent design. +An evaluated, rejected concrete implementation need not be reimplemented just to +obtain a favorable result; a new mechanism needs a new hypothesis and population. + +| Candidate | Documented decision | Evidence | Remaining boundary | +| :-------- | :-------------------------------------------------------------------------- | :--------------------------------------------------------------------------------------------------------------------------------------------------- | :----------------------------------------------------------------------------------------- | +| O01 | Selected: path-owned collection text | [Report](benchmarks/collection-text-sharing-20261101.md) | Owned paths, fallback copies and budgets remain part of the contract. | +| O02 | Selected: prepared owned/simple/folded glob paths | [Report](benchmarks/folded-ascii-globs-20261105.md) | Native/complex/locale fallbacks and their measured costs remain. | +| O03 | Selected: fuzzy ceiling and worker scratch | [Report](benchmarks/fuzzy-score-workspace-20261102.md) | Initial exact capacity chosen; the 64-character minimum was rejected. | +| O04 | Selected: lexical field views | [Report](benchmarks/lexical-field-views-20261102.md) | Preserve native spelling and lifetime/fallback behavior. | +| O05 | Evaluated: generic field lookup changes rejected | [Report](benchmarks/field-lookup-controls-20261102.md) | Future prepared bindings need a different measured mechanism. | +| O06/O07 | Selected: range overwrite and cursor-owned reuse | [Report](benchmarks/cursor-owned-read-buffers-20261102.md) | One initial clear remains selected; O32 is a distinct legacy API. | +| O08 | Selected: typed atomic task slots; cleanup variant rejected | [Report](benchmarks/directory-task-slots-20261102.md) | No new admission threshold or global executor is implied. | +| O09 | Selected: packed owned snapshots | [Report](benchmarks/packed-visit-snapshots-20261102.md) | No dangling views into movable short strings. | +| O10 | Selected: bounded retained result capacity | [Report](benchmarks/bounded-match-results-20261102.md) | A retention limit is not an in-flight output-byte limit. | +| O11 | Selected: useful warm participation | [Report](benchmarks/useful-match-workers-20261102.md) | Worker creation and failed growth-throttling policies stay separate. | +| O12 | Evaluated: direct evaluator storage rejected | [Report](benchmarks/direct-evaluator-workers-20261102.md) | A different typed plan remains an EP design, not an adopted win. | +| O13 | Selected: requested derived-value handoff | [Report](benchmarks/derived-content-handoff-20261102.md) | Only requested existing data crosses the stage boundary. | +| O14 | Partial: default snapshot reuse selected | [Report](benchmarks/immutable-default-catalogs-20261102.md) | First-parse generated flat/offset tables and per-run MIME binding are not qualified. | +| O15 | Partial: diff/hash invariant operands selected | [Report](benchmarks/prepared-hash-operands-20261102.md) | Remaining EP03/EP07 operand/template acceptance is not proved by these subsets. | +| O16 | Evaluated: compact retention selected; paired-frontier prototype rejected | [Report](benchmarks/comparison-frontier-rejection-20261106.md) | A completed-frontier redesign remains open; both production inventories are retained. | +| O17 | Evaluated: all six shared-stage executor variants rejected | [Report](benchmarks/shared-stage-workers-20261107.md) | Keep specialized persistent pools; old draft PRs are not qualified replacements. | +| O18 | Open: full byte-bounded stage overlap not implemented/qualified | [Report](https://github.com/mboworks/xff/blob/3e50f36bde466ef576e03510780c3515c2c474ca/docs/benchmarks/queued-read-cancellation-20261107.md) | Queued-read cancellation trials are separate and do not close pipeline overlap. | +| O19 | Partial: buffers/ranges/observations selected; seek/locator trials rejected | [Report](https://github.com/mboworks/xff/blob/912386016f1e42874b2060537bc1e92bea03e611/docs/benchmarks/immutable-zip-locator-evaluation-20261107.md) | Indexed/serial decoder admission and streamed ZIP correctness remain open. | +| O20 | Selected: optional backend bulk metadata and native observations | [Report](benchmarks/native-metadata-followups-20261105.md) | Direct-vector/scan/cleanup trials rejected; further demand planning requires new evidence. | +| O21 | Partial: literal-reference and exact shingle verification selected | [Report](benchmarks/similarity-reference-reuse-20261103.md) | Dynamic source identity, invalidation and exact candidate indexes remain unqualified. | +| O22 | Partial: record/cell/template/rg transaction reuse selected | [Report](benchmarks/transactional-rg-records-20261107.md) | Remaining reusable structured storage and streaming machine representations stay open. | + +## Later concrete candidates and integration + +- O23's native PGO experiment is retained as research. Portable training, + source/compiler/architecture provenance, trust, regeneration and source-only + release fallback are required before adoption. +- O24's actual-library stream synchronization audit establishes no useful runtime + switch for the currently pinned libc++; do not add a placebo call. +- O25 direct final-string language decoding is merged in #1128. This does not remove + first-language JSON parsing or build the O14 generated-table representation. +- O26 empty deferred time formats, O28 observation-mode specialization, O29 borrowed + immediate archive keys and O30 owned archive input have separate documented + decisions. O28 is rejected; O29/O30 ownership changes do not license a global + unbudgeted archive index. +- O27's three insertion proposals are measured and unselected, including the hinted + insertion report merged in #1129. Do not describe the hinted form as unmeasured + after that report or treat an older empty-map microbenchmark as full parsing cost. +- O31's full policy is unselected. The template-only subset is published in #1134 + and still requires current-base integration and eventual main verification. +- O32's legacy extraction-buffer proposal is published in #1140. Its one-line + change, both-host focused release/sanitizer qualification and raw matrices are + preserved. Mac gzip's repeatable 64 KiB cost remains explicit. This audit does + not count an open PR as merged or a legacy API result as a general CLI gain. +- The exact known-length JSON list reservation is also measured on both hosts and + unselected. Keep [both harness versions and all observations](benchmarks/language-list-reserve-20261107.md). + Its six focused release suites and final 16-case matrix do not establish first-load + generated catalogs, allocation counts, a full suite or whole-command gains. + +## Requirements that remain genuinely open + +1. **O18 pipeline ownership and output admission.** The shipping matcher waits for + an entire batch; O22c2h transfers a validated per-file rg transaction but does not + cap in-flight output or overlap the next batch. Implement an ordered completion + protocol and actual admission before output allocation. A delayed head must not + lose credit to later entries. Define one explicitly oversized head transaction + or another transactional storage contract; do not truncate results, spill through + an unauthorized write path, or count worker-held strings as free memory. +2. **O19 archive admission and correctness.** The seek and cached-locator prototypes + were actually implemented and measured; they are not unfinished trial runs. + Retain their repeatable first-read costs. A different immutable-identity/index + design must preserve all visible ZIP members, methods, exact sizes, shared + budgets and sequential-source fallbacks. The streamed descriptor/bzip2 defects + in #1139 are still defects; a decoder-to-count shortcut fails method/corruption + coverage and is not a completed fix. +3. **O14 first construction and O15 remaining preparation.** Default restoration and + direct Brotli string output reduce different costs. Generated immutable tables + must preserve append-only registration, user layers, aliases/overlaps, errors, + old views, binary size and loader relocation costs. The original EP03/EP07 + acceptance inventory remains authoritative for operands not covered by the + selected diff/hash implementations. +4. **O21 dynamic representations.** A fresh source observation and an immutable + literal reference have different contracts. Caches require explicit source + identity/invalidation and retained-byte ownership; indexes must preserve exact + verification and the requested grouping semantics. +5. **O22 and metadata follow-ups.** Existing output buffers, callback lifetimes and + cheap metadata paths are real implementations. Further storage or bulk-read + changes need an observed cost, with reentrant callbacks, view lifetimes, directory, + followed-link and mount behavior preserved. Do not repeat rejected vector or + batching wrappers without a changed mechanism. +6. **Publication and merged qualification.** Current public implementations must + still finish their own checks, synchronize and merge in the authorized sequence. + Verify resulting main jobs and actual hosted results. Queue retention is bounded; + already canceled main campaigns need explicit reconciliation/backfill, and the + publication snapshot/compaction fixes still need hosted evidence. Preserve all + release/main benchmark data and coverage aggregate history. + +## Next O18 implementation and decision gates + +The source audit identifies `ParallelMatch::Match`, its owned entry/result banks, +`RenderRgOutput`'s validated transaction, and the coordinator's `flush_matches` as +the concrete boundaries. Replacing the existing pools with another generic executor +is not a prerequisite; O17's tested regression must not be reintroduced. + +- Separate batch submission from ordered entry completion. Keep persistent leaf + workers and coordinator publication. Two explicitly owned banks can let traversal + prepare a bounded next input while workers finish the prior one; callbacks must + not observe a result or filesystem owner after its bank is recycled. +- Account for retained input text, entry/result slots and output storage before + admitting work. Bounded entry counts or capacity retained between batches are not + a byte-bound proof. Track the peak and oversized-head contract independently of + active decoder/read budgets. +- Preserve per-file late read/binary suppression. A completed unvalidated partial + string cannot be emitted just because it is first. Any chunked transaction must + retain that validation contract and avoid a compulsory full-size flattening copy. +- Drain before incompatible archive/mount work, effects, deferred/capture barriers, + traversal error publication and owner destruction. Early result limits and stops + need exact read/effect traces; do not speculate beyond the old behavior silently. +- Wake blocked producers and consumers on failure/cancellation before joining. + A leaf worker must never await work on its own saturated executor. Test delayed + heads, full later records, giant/empty/no-output files, preparation and read errors, + tiny tails, root/mount lifetime and repeated reuse. +- Measure unchanged output and worker-only controls on both hosts first, then + native/in-memory small/large one/three-worker workloads, first output, elapsed, + actual queue bytes and peak memory. Preserve every slower case and identical-image + controls. Reject a design that merely shifts retention or adds dispatch overhead; + do not hide costs with arbitrary count/time thresholds. + +These are next implementation gates, not claims that a pipeline exists. The goal +remains active until every required candidate disposition and promised integration +is proved against current source and external state. diff --git a/release-site.json b/release-site.json index a9c7fb7b29..98cf5f58cd 100644 --- a/release-site.json +++ b/release-site.json @@ -149,7 +149,10 @@ "docs/benchmarks/lazy-content-state-20261107.md": "docs/benchmarks/lazy-content-state-20261107.html", "docs/benchmarks/deferred-zip-size-investigation-20261107.md": "docs/benchmarks/deferred-zip-size-investigation-20261107.html", "docs/benchmarks/archive-read-buffer-clear-20261107.md": "docs/benchmarks/archive-read-buffer-clear-20261107.html", - "docs/benchmarks/archive-read-buffer-requalification-20261108.md": "docs/benchmarks/archive-read-buffer-requalification-20261108.html" + "docs/benchmarks/archive-read-buffer-requalification-20261108.md": "docs/benchmarks/archive-read-buffer-requalification-20261108.html", + "docs/benchmarks/language-list-reserve-20261107.md": "docs/benchmarks/language-list-reserve-20261107.html", + "docs/performance-candidate-completion-audit.md": "docs/performance-candidate-completion-audit.html", + "docs/benchmarks/language-list-reserve-requalification-20261108.md": "docs/benchmarks/language-list-reserve-requalification-20261108.html" }, "generated_html": { "XFF.md": { @@ -190,6 +193,8 @@ "docs/benchmarks/lazy-content-state-20261107.json": "docs/benchmarks/lazy-content-state-20261107.json", "docs/benchmarks/deferred-zip-size-investigation-20261107.json.gz": "docs/benchmarks/deferred-zip-size-investigation-20261107.json.gz", "docs/benchmarks/archive-read-buffer-clear-20261107.json.gz": "docs/benchmarks/archive-read-buffer-clear-20261107.json.gz", - "docs/benchmarks/archive-read-buffer-requalification-20261108.json.gz": "docs/benchmarks/archive-read-buffer-requalification-20261108.json.gz" + "docs/benchmarks/archive-read-buffer-requalification-20261108.json.gz": "docs/benchmarks/archive-read-buffer-requalification-20261108.json.gz", + "docs/benchmarks/language-list-reserve-20261107.json.gz": "docs/benchmarks/language-list-reserve-20261107.json.gz", + "docs/benchmarks/language-list-reserve-requalification-20261108.json.gz": "docs/benchmarks/language-list-reserve-requalification-20261108.json.gz" } }