diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7059d41a90..48168432d5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -24,7 +24,7 @@ repos: hooks: - id: check-added-large-files # Retain the exact lossless cross-platform performance evidence archives. - exclude: ^docs/benchmarks/((executor-copy-audit|bounded-read-ahead)-20261101|(directory-task-slots|packed-visit-snapshots|bounded-match-results|useful-match-workers|direct-evaluator-workers|derived-content-handoff|immutable-default-catalogs|prepared-diff-configuration|prepared-hash-operands|aggregate-comparison-totals|byte-shingle-representations|archive-input-buffers)-20261102|(digest-and-record-storage|caller-owned-streams|direct-source-materialization|hash-field-binding|hash-field-binding-tag|forward-cursor-skipping|raw-record-reuse|similarity-prefix-view|similarity-reference-reuse|owned-word-moves|smaller-shingle-probes|native-zip-seek-evaluation|encoded-record-reuse|threshold-aware-similarity)-20261103|(column-row-reuse|word-set-insertion|field-context-owned-results|selected-summary-output|fixed-summary-metrics|direct-summary-cells|compact-comparison-entries|comparison-category-sequence|native-name-paths|uniform-native-case|compact-traversal-metadata|eager-native-paths|native-case-experiments)-20261104|(folded-ascii-globs|bulk-directory-metadata|comparison-retention-followups|comparison-storage-followups|archive-prefix-reuse|archive-directory-range|archive-directory-allocation|archive-directory-observations|native-symlink-observations|native-metadata-followups|symlink-target-demand|native-pgo|readlink-short-target|cctz-empty-format|readlink-input-path|readlink-owned-transfer|archive-lookup-trials|archive-member-keys)-20261105|archive-directory-child-capacity-20261106|shared-stage-workers-20261107|archive-index-insertion-20261107|transactional-rg-records-20261107|immutable-zip-locator-evaluation-20261107|queued-read-cancellation-20261107|archive-btree-historical-decision-20261107|compiled-field-stat-demand-20261107)\.json\.gz$ + exclude: ^docs/benchmarks/((executor-copy-audit|bounded-read-ahead)-20261101|(directory-task-slots|packed-visit-snapshots|bounded-match-results|useful-match-workers|direct-evaluator-workers|derived-content-handoff|immutable-default-catalogs|prepared-diff-configuration|prepared-hash-operands|aggregate-comparison-totals|byte-shingle-representations|archive-input-buffers)-20261102|(digest-and-record-storage|caller-owned-streams|direct-source-materialization|hash-field-binding|hash-field-binding-tag|forward-cursor-skipping|raw-record-reuse|similarity-prefix-view|similarity-reference-reuse|owned-word-moves|smaller-shingle-probes|native-zip-seek-evaluation|encoded-record-reuse|threshold-aware-similarity)-20261103|(column-row-reuse|word-set-insertion|field-context-owned-results|selected-summary-output|fixed-summary-metrics|direct-summary-cells|compact-comparison-entries|comparison-category-sequence|native-name-paths|uniform-native-case|compact-traversal-metadata|eager-native-paths|native-case-experiments)-20261104|(folded-ascii-globs|bulk-directory-metadata|comparison-retention-followups|comparison-storage-followups|archive-prefix-reuse|archive-directory-range|archive-directory-allocation|archive-directory-observations|native-symlink-observations|native-metadata-followups|symlink-target-demand|native-pgo|readlink-short-target|cctz-empty-format|readlink-input-path|readlink-owned-transfer|archive-lookup-trials|archive-member-keys)-20261105|archive-directory-child-capacity-20261106|shared-stage-workers-20261107|archive-index-insertion-20261107|transactional-rg-records-20261107|immutable-zip-locator-evaluation-20261107|queued-read-cancellation-20261107|archive-btree-historical-decision-20261107|compiled-field-stat-demand-20261107|unchanged-site-payloads-20261107)\.json\.gz$ - id: check-merge-conflict - id: check-yaml # BINARY_TYPES below: archive containers are binary but can START with text (a phar begins with diff --git a/TODO.md b/TODO.md index 26886713ab..85711dc9ee 100644 --- a/TODO.md +++ b/TODO.md @@ -171,6 +171,13 @@ scope or declaring new flags implemented. See or dropping release/main observations. Old queued definitions retain their deployment steps. - [ ] Reduce full historical rendering inside the writer lock; preserve all raw measurements and coverage aggregate history when designing coalesced rendering or ingestion replay. +- [x] Reuse unchanged raw JSON archives during repeated publication after byte-exact comparison; + qualify complete expand/compact cycles on Mac/Linux and retain changed/new-input controls + and the rejected broader chart-reuse trial in + [the storage qualification](docs/benchmarks/unchanged-site-payloads-20261107.md). +- [ ] Review the now-observed 728 MB staged site (about 499 MB benchmarks and 202 MB coverage), + above the 200 MB review threshold. Profile historical rendering/compaction and preserve all + release/main performance observations and coverage aggregate history when reducing storage. - [x] Redirect byte-identical stable coverage HTML to preserved run archives in all publishers; check the complete staged payload before upload and verify the served benchmark index. - [ ] Decide long-term coverage history hosting or retention before immutable reports alone diff --git a/docs/benchmark-history.md b/docs/benchmark-history.md index 47a7c33ff5..89316e541a 100644 --- a/docs/benchmark-history.md +++ b/docs/benchmark-history.md @@ -55,6 +55,12 @@ publication, preserving their pages and every incoming measurement. After retent notifications coalesce; source-artifact ingestion remains serialized without cancellation. A Pages timeout no longer holds the writer lock. Rendering remains inside the writer queue. +Repeated compaction reuses a retained non-interned JSON archive only when its decompressed bytes +exactly match the expanded input. Changed or damaged archives are rebuilt; packed charts keep +their existing path. This preserves original observation bytes and avoids recompressing unchanged +data. See the [cross-platform qualification](benchmarks/unchanged-site-payloads-20261107.md), +including changed-input costs and the rejected broader chart-reuse experiment. + Local uploads can advance the retained branch while a publisher is rendering it. A rejected push fetches and rebases onto that update, then performs a full benchmark refresh before retrying, with at most three push attempts. Newly uploaded measurements remain in the branch and enter the rebuilt diff --git a/docs/benchmarks/unchanged-site-payloads-20261107.json.gz b/docs/benchmarks/unchanged-site-payloads-20261107.json.gz new file mode 100644 index 0000000000..9af18aa2b2 Binary files /dev/null and b/docs/benchmarks/unchanged-site-payloads-20261107.json.gz differ diff --git a/docs/benchmarks/unchanged-site-payloads-20261107.md b/docs/benchmarks/unchanged-site-payloads-20261107.md new file mode 100644 index 0000000000..1acf77a8bf --- /dev/null +++ b/docs/benchmarks/unchanged-site-payloads-20261107.md @@ -0,0 +1,82 @@ +# Reuse unchanged raw publication payloads + +Select byte-exact reuse for non-interned JSON archives. Keep chart packing unchanged. +The complete two-report expand/compact cycle improves roughly 52% on macOS and 47% +on Linux when the retained raw observations are unchanged. Changed-input costs remain +visible: up to 0.99% Mac and 3.77% Linux; fresh-input controls stay within 1%. +These are fixture-cycle results, not total hosted publication-time measurements. + +## Mechanism and decision + +Publication expands existing reports, artifacts, normalization records and charts, then +compacts them again. Previously every expanded file was recompressed even when its +original archive remained alongside it. For non-interned JSON, decompress that retained +archive and compare exact bytes with the authoritative expanded file. On equality, +keep the original compressed bytes and remove only the expanded copy. Changed, missing, +or corrupt retained archives use the existing compression path. Numeric spellings, +whitespace, booleans, signed zero and original observation bytes remain significant. + +The first experiment also reused packed charts. Unchanged charts improved 67-72%, but +modified charts regressed 27-33% because comparison decoded the old packed structure +before repacking the new one. Reject that broader implementation. The selected code +never adds that comparison to chart packing. This does not change the storage schema, +retention policy, chart values, renderer, source identity, or official measurement series. + +## Paired measurements + +The fixtures are the exact public Linux and macOS report, packed chart, and HTML payloads +from main run `42254867488`, attempt 1, revision `2487927cc6`. Original compressed inputs, +source URLs/hashes, source variants, scripts, host details and every timing sample are in +the [lossless evidence capsule](unchanged-site-payloads-20261107.json.gz). +The raw report files expand to 8,714,448 and 8,806,698 bytes. Their charts expand to +5,096,770 and 4,310,810 bytes. + +Each full-cycle case processes both reports through the real `expand_benchmarks` and +`compact_benchmarks` functions. Three sessions alternate parent/candidate order, discard +one warm-up round, and retain the mean fastest seven of nine measured rounds per role. +The table shows median session times and each session's percentage change; negative +means less elapsed time. Setup and output verification are outside the timer. Every +iteration verifies exact raw bytes, chart values/types, removal of expanded files, and +the packed HTML entry point. Unchanged candidate reports also retain identical gzip bytes. + +| Host | Input state | Parent ms | Candidate ms | Session 1 | Session 2 | Session 3 | +| ----- | ----------- | --------: | -----------: | --------: | --------: | --------: | +| macos | unchanged | 742.66 | 354.10 | -52.39% | -52.27% | -52.10% | +| macos | changed | 818.86 | 825.93 | +0.45% | +0.99% | +0.86% | +| macos | new | 684.37 | 684.27 | +0.08% | +0.07% | -0.01% | +| linux | unchanged | 497.43 | 263.08 | -47.02% | -47.11% | -46.89% | +| linux | changed | 556.75 | 575.20 | +3.32% | +1.91% | +3.77% | +| linux | new | 445.53 | 448.34 | +0.84% | +0.44% | +0.19% | + +The changed case appends raw JSON whitespace and changes chart metadata; both variants +must produce the changed values. The fresh case starts without retained gzip files. +Only the unchanged and changed cases include expansion in the timer, identically for +both roles; fresh inputs are prepared expanded. Compare roles within each row. +Mac uses native temporary storage and Python 3.13; Linux uses native home-directory +scratch and Python 3.14. Linux's first attempted full-cycle run hit the `/tmp` tmpfs user +quota before producing a case; its final three sessions all use the same home scratch. +No prior experiments or evidence were deleted. Timings ran without concurrent builds. +Absolute times across hosts are not machine-normalized comparisons. + +## Correctness and practical limits + +All five relevant Bazel targets pass on Mac: storage, coverage storage, complete-site +compaction, benchmark history, and PR previews. The corresponding 48 direct tests pass +on both hosts; final malformed-DEFLATE coverage also passes separately on Linux and in +Mac's hermetic storage target. Linux verification checked all 198 transferred source +hashes before testing; final source and test hashes were checked again after that case. + +Committed tests cover exact gzip reuse, changed scalar types/signed zero, missing and +stale archives, invalid gzip headers, truncated streams, invalid DEFLATE, and existing +expand/compact idempotence and chart-disagreement checks. A damaged retained cache never +replaces authoritative expanded input. Filesystem permission or write failures still fail. +The comparison holds one additional decompressed file temporarily; peak RSS was not +measured. No entire-site cache or additional persistent checksum files are introduced. + +A sampled publisher log had 80.8 seconds expanding inputs, 172.1 seconds before reporting +zero decorated pages, and another 444.3 seconds in the subsequent artwork/compaction +interval. That interval is not an isolated compactor profile. The staged site was about +728 MB (499 MB benchmarks, 202 MB coverage), already above the 200 MB review threshold. +The separate deployment repair does not remove these ingestion costs. This change +addresses repeated compression only; historical rendering, the publication backlog, +recovery of rejected source events and storage reduction remain open. diff --git a/docs/performance-analysis.md b/docs/performance-analysis.md index 95e4b8b8f2..e1aa8e1fa8 100644 --- a/docs/performance-analysis.md +++ b/docs/performance-analysis.md @@ -4932,3 +4932,23 @@ results do not substitute for current synchronized public-head checks. Column pipeline causes, score-consumer-specific demand and conditional stat-bearing field admission remain separate future work. Do not reopen the combined policy unchanged or use a cohort median to hide its slower case. + +## Repeated publication compression, 2026-11-07 + +[The retained qualification](benchmarks/unchanged-site-payloads-20261107.md) separates +publication-tool performance from executable measurements. Select byte-exact reuse of +unchanged non-interned JSON gzip archives; preserve chart packing. Three balanced +sessions of the real two-report expand/compact cycle improve 52.1-52.4% on Mac and +46.9-47.1% on Linux. Changed inputs cost up to 0.99%/3.77%; fresh controls remain within +1%. Exact raw bytes and chart values are checked every iteration. Five Mac Bazel suites +and all 48 direct tests on both hosts pass, with explicit malformed-cache coverage. + +Reject the initial broad chart-reuse candidate: modified charts cost 27-33%, despite +67-72% unchanged-chart gains. Both variants, exact public fixtures, source/hash identity, +scripts and every measured round remain in the lossless capsule. Comparing retained +raw bytes temporarily holds one additional decompressed file; no peak-RSS improvement +or full-hosted-publication speedup is claimed. A provider Pages timeout and queue-full +rejections require the separate deployment/recovery work; this optimization cannot +recover missing source events. A sampled 728 MB site also exceeds the existing 200 MB +storage-review threshold. Keep historical rendering and storage reduction open without +dropping release/main observations or coverage aggregates. diff --git a/release-site.json b/release-site.json index 259c05a339..b70355fb84 100644 --- a/release-site.json +++ b/release-site.json @@ -141,7 +141,8 @@ "docs/benchmarks/immutable-zip-locator-evaluation-20261107.md": "docs/benchmarks/immutable-zip-locator-evaluation-20261107.html", "docs/benchmarks/queued-read-cancellation-20261107.md": "docs/benchmarks/queued-read-cancellation-20261107.html", "docs/benchmarks/archive-btree-historical-decision-20261107.md": "docs/benchmarks/archive-btree-historical-decision-20261107.html", - "docs/benchmarks/compiled-field-stat-demand-20261107.md": "docs/benchmarks/compiled-field-stat-demand-20261107.html" + "docs/benchmarks/compiled-field-stat-demand-20261107.md": "docs/benchmarks/compiled-field-stat-demand-20261107.html", + "docs/benchmarks/unchanged-site-payloads-20261107.md": "docs/benchmarks/unchanged-site-payloads-20261107.html" }, "generated_html": { "XFF.md": { @@ -170,6 +171,7 @@ "docs/benchmarks/immutable-zip-locator-evaluation-20261107.json.gz": "docs/benchmarks/immutable-zip-locator-evaluation-20261107.json.gz", "docs/benchmarks/queued-read-cancellation-20261107.json.gz": "docs/benchmarks/queued-read-cancellation-20261107.json.gz", "docs/benchmarks/archive-btree-historical-decision-20261107.json.gz": "docs/benchmarks/archive-btree-historical-decision-20261107.json.gz", - "docs/benchmarks/compiled-field-stat-demand-20261107.json.gz": "docs/benchmarks/compiled-field-stat-demand-20261107.json.gz" + "docs/benchmarks/compiled-field-stat-demand-20261107.json.gz": "docs/benchmarks/compiled-field-stat-demand-20261107.json.gz", + "docs/benchmarks/unchanged-site-payloads-20261107.json.gz": "docs/benchmarks/unchanged-site-payloads-20261107.json.gz" } } diff --git a/tools/site_storage.py b/tools/site_storage.py index 75b2651b22..2163de8733 100644 --- a/tools/site_storage.py +++ b/tools/site_storage.py @@ -7,6 +7,7 @@ import gzip import json from pathlib import Path +import zlib def packed(value): @@ -53,9 +54,18 @@ def unpacked(value): def compact_json(path, intern=False): original = path.read_bytes() + destination = path.with_suffix(path.suffix + '.gz') + if not intern and destination.exists(): + try: + retained = gzip.decompress(destination.read_bytes()) + except (gzip.BadGzipFile, EOFError, zlib.error): + # The expanded input remains authoritative when an old cache is damaged. + retained = None + if retained == original: + path.unlink() + return data = (json.dumps(packed(json.loads(original)), ensure_ascii=False, separators=(',', ':'), allow_nan=False).encode() if intern else original) - destination = path.with_suffix(path.suffix + '.gz') destination.write_bytes(gzip.compress(data, mtime=0)) path.unlink() diff --git a/tools/site_storage_test.py b/tools/site_storage_test.py index 26c28d1251..21234b9e58 100644 --- a/tools/site_storage_test.py +++ b/tools/site_storage_test.py @@ -12,6 +12,46 @@ class SiteStorageTest(unittest.TestCase): + def test_unchanged_expanded_measurements_reuse_exact_compressed_bytes(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / 'report.json' + original = b'{ "samples": [1.000, true, -0.0, null] }\n' + retained = gzip.compress(original, compresslevel=1, mtime=123) + path.with_suffix('.json.gz').write_bytes(retained) + path.write_bytes(original) + storage.compact_json(path) + self.assertFalse(path.exists()) + self.assertEqual(path.with_suffix('.json.gz').read_bytes(), retained) + + def test_changed_expanded_inputs_replace_retained_bytes_including_scalar_types_and_signs(self): + for before, after in ((1, True), (1, 1.0), (-0.0, 0.0), (['before'], ['after'])): + for intern in (False, True): + with self.subTest(before=before, after=after, intern=intern), tempfile.TemporaryDirectory() as directory: + path = Path(directory) / 'landscape.json' + path.write_text(json.dumps(before)) + storage.compact_json(path, intern) + original = json.dumps(after).encode() + path.write_bytes(original) + storage.compact_json(path, intern) + actual = gzip.decompress(path.with_suffix('.json.gz').read_bytes()) + if intern: + actual = json.dumps(storage.unpacked(json.loads(actual))).encode() + self.assertEqual(actual, original) + + def test_missing_or_corrupt_retained_payload_is_rebuilt_from_expanded_input(self): + invalid_deflate = bytearray(gzip.compress(b'payload')) + invalid_deflate[10] = 7 # Reserved DEFLATE block type, after the ten-byte gzip header. + for previous in (None, b'not gzip', gzip.compress(b'stale payload'), gzip.compress(b'truncated')[:-2], + invalid_deflate): + with self.subTest(previous=previous), tempfile.TemporaryDirectory() as directory: + path = Path(directory) / 'report.json' + if previous is not None: + path.with_suffix('.json.gz').write_bytes(previous) + original = b'{ "valid": [1, true] }\n' + path.write_bytes(original) + storage.compact_json(path) + self.assertEqual(gzip.decompress(path.with_suffix('.json.gz').read_bytes()), original) + def test_shared_json_values_round_trip_without_type_collisions(self): row = {'sample': [1, True, 1.0, None, False, 0, -0.0], 'label': '', '__proto__': 'literal'} value = {'a': row, 'b': row, 'nested': [[row] * 200]}