diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9447747..267c0ae 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,6 +12,12 @@ jobs: verify: name: Windows Python ${{ matrix.python }} runs-on: windows-latest + # Freeze native CPU pools before Torch/NumPy import, as in server witnesses. + env: + OMP_NUM_THREADS: "1" + OPENBLAS_NUM_THREADS: "1" + MKL_NUM_THREADS: "1" + PYTHONHASHSEED: "0" strategy: fail-fast: false matrix: @@ -44,7 +50,9 @@ jobs: - name: Lint run: python -m ruff check src tests scripts - name: Test - run: python -B -m pytest -p no:cacheprovider -q ${{ matrix.pytest_args }} + run: | + python -B -c "import torch; print({'torch': torch.__version__, 'cpu_capability': torch.backends.cpu.get_cpu_capability(), 'num_threads': torch.get_num_threads(), 'num_interop_threads': torch.get_num_interop_threads(), 'mkldnn_available': torch.backends.mkldnn.is_available(), 'mkldnn_enabled': torch.backends.mkldnn.enabled}); assert torch.get_num_threads() == 1" + python -B -m pytest -p no:cacheprovider -q ${{ matrix.pytest_args }} - name: Build wheel run: python -m pip wheel --no-deps --wheel-dir dist . - name: Audit built wheel bytes diff --git a/.gitignore b/.gitignore index 987f065..5d29fd8 100644 --- a/.gitignore +++ b/.gitignore @@ -36,6 +36,7 @@ paper/*.bbl paper/*.blg paper/*.log paper/*.out +paper/.venue/ figures/*_preview.png recommend paper/ diff --git a/LICENSE_TMR.md b/LICENSE_TMR.md new file mode 100644 index 0000000..3b80d28 --- /dev/null +++ b/LICENSE_TMR.md @@ -0,0 +1,28 @@ +# TMR components attribution + +The ACTOR-style components and objective in +`src/phaseset_core/tmr_components.py` are adapted from +[Mathux/TMR](https://github.com/Mathux/TMR/tree/6d74688730d15d43b0a755ce2b0e1f2d76138fc1). +No pretrained weights, datasets, or third-party paper PDFs are included. + +MIT License + +Copyright (c) 2023 Mathis Petrovich + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md index 011c8c0..096f1c8 100644 --- a/README.md +++ b/README.md @@ -37,9 +37,130 @@ multi-person skeleton sequence an independently hashed 20-Hz formula bank after resampling. This is a compatibility boundary, not a claim that the two kernel banks or full models are bitwise identical. See the - [20-Hz rate contract](docs/PHASESET_20HZ_MORLET_CONTRACT.md). + [20-Hz rate contract](docs/PHASESET_20HZ_MORLET_CONTRACT.md) and + [canonical-byte portability policy](docs/MORLET_PORTABILITY.md). -## Current milestone +## PhaseSet-V2 research upgrade — 2026-09-26 + +The user-selected target is now [ACM ICMR 2027](docs/PHASESET_V2_ICMR_TARGET.md). + +The subsequent [relation-supervision amendment](docs/PHASESET_V2_RELATION_SUPERVISION_AMENDMENT.md) +uses existing official human retrieval annotations and permits separately +disclosed weak machine supervision; no new human annotators are awaited. +Machine labels are not independent motion ground truth or human-challenge +success. The [separate weak-CF loss/backward seam](docs/PHASESET_V2_WEAK_CF_TRAINING_SEAM.md) +has passed 37 scoped server checks, and the +[cached training-only weak text pool](docs/PHASESET_V2_WEAK_CLIP_POOL.md) has passed +24 additional scoped checks. Actual generation, encoded weak text and private +provenance admission remain pending. Neither is a completed native training run. + +A [complete-native-parent CPU profile](docs/PHASESET_V2_NATIVE_PROCESS_PROFILE.md) +has now passed through the production lifecycle and journal: both 512-D systems, +all 55 accepted windows and 10 official human rows, full-negative backward +replay, unchanged weights, zero optimizer/GPU use. This is not formal training, +GPU throughput or a paper retrieval result; formal stages remain **0/87**. +The venue amendment changes submission format, not the scientific matrix or +compute budget. The existing ICASSP manuscript is historical, not an ICMR-ready +paper or a submitted result. + +The new [V2 amendment](docs/PHASESET_V2_AMENDMENT.md) introduces signed local +phase, time-varying actor–edge incidence, chronological capture readout, and +verified counterfactual coordination supervision. The +[87-stage matrix](configs/phaseset_v2_experiment_matrix.json) supersedes the +affected legacy 33-run plan. New modules are independent research APIs; the +production V2 training-host integration and converged literature reproductions +are not yet qualified. +The V2 A6 comparator now has a data-free, whole-capture relation seam that +forms actual `(a+b)/2` and `a-b` velocity signals *before* six-band DCT, +retaining their cross term. This is distinct from legacy system 03's separate +endpoint self-power control and is not an InterEdit reproduction. A6 requires +its own typed, training-population DCT-floor receipt. Five real training-only +floor populations and the [native source-binding qualification](docs/PHASESET_V2_A6_SOURCE_QUALIFICATION.md) +have private server receipts; future run-bound GPU, cost, and checkpoint +admission remains separate. Neither the source check nor its CPU tests +constitute an A6 training run. +The [A4 incidence-shuffle control](docs/PHASESET_V2_A4_INCIDENCE.md) preserves +physical pair packets, edge support and model capacity while deterministically +reassigning supported half-edge values at the actor-incidence node input. Its +shuffle is independent of private participant commitments. Software review +and tests do not count as formal A4 training or retrieval results. +The [A2 order-free control](docs/PHASESET_V2_A2_ORDER_FREE.md) replaces all +four learned temporal GRU paths with near-equal-capacity independent patch +MLPs and masked means while retaining the same physical phase input. It tests +the contribution of learned order, not the existence of physical phase. +The [A3 capacity-matched pair-bag control](docs/PHASESET_V2_A3_PAIR_BAG.md) +keeps the full head's active node and edge modules but pools relation packets +without assigning them to actor nodes. It is distinct from the legacy +topology-off pair-only diagnostic and has no reported retrieval result yet. +The [matrix-bound V2 mechanism factory](docs/PHASESET_V2_MECHANISM_FACTORY.md) +constructs the implemented main B2 and full/A1/A2/A3/A4/A5/A6/A7/A8 host paths +with registered seeds, predecessor/terminal and source bindings. The A7 +focused registered-server CPU qualification passed 42 tests with one +CUDA-only skip; it grants no pilot or +formal launch, and A9, fold hosts and Inter-X remain held. A7 is a +[generic local relation control](docs/PHASESET_V2_A7_GENERIC_LOCAL.md): its +scores ignore phase features, while its shared input transport still incurs +phase-cache preparation cost. +[TMR-Set](docs/TMR_SET_ADAPTATION.md) now implements the +complete original VAE objectives with disclosed full-parent group adaptation; +its pinned frozen language loader is not a substitute for native learning. +[WaMo-Set](docs/WAMO_SET_ADAPTATION.md) implements learnable stationary analysis +and inverse wavelets, both reconstruction branches and original/shuffled-frame +ordering, with explicit full-parent group adaptation and unpublished-detail +choices. Its software qualification is not converged baseline training. +[MIME](docs/MIME_SET_ADAPTATION.md) keeps separate role-aware original dyadic +and shared complete-parent group APIs, full-time bidirectional co-attention, +and a genuinely trainable pinned CLIP text tower. Group XYZ/long-text adapters +are disclosed; equation/software checks are not original-task convergence. +Their [full-negative parent backward seam](docs/PHASESET_V2_LITERATURE_PARENT_TRAINING.md) +replays one complete capture graph at a time without dropping original WaMo +auxiliary objectives or freezing MIME text. Dense-gradient software oracles +are not native optimizer, profile, checkpoint-host or convergence evidence. +The separate [TMR VAE parent backward seam](docs/PHASESET_V2_TMR_PARENT_TRAINING.md) +preserves the original stochastic order, both full-track reconstructions, +all four KL terms, latent alignment and filtered full-batch negatives; it does +not route TMR through a generic contrastive-only objective. +The [literature optimizer host](docs/PHASESET_V2_LITERATURE_PARENT_HOST.md) +shares the full-parent AdamW, validation selection and checkpoint/resume loop, +while keeping each baseline's original objective. Analytic optimizer/resume +checks and native dense-gradient replay are distinct from a native training +pilot or converged baseline result. +The [fixed TMR/WaMo language-row cache](docs/PHASESET_V2_FROZEN_LANGUAGE_ROWS.md) +retains every human caption occurrence and content token, reusing only frozen +features; MIME still trains its own genuine CLIP tower. +The [development disk source](docs/PHASESET_V2_PARENT_DISK_SOURCE.md) streams +complete body/phase timelines and uses source/seed/epoch group yaw independent +of model RNG. Caption eligibility and run-specific floors remain explicit +private inputs. The [frozen human CLIP row source](docs/PHASESET_V2_FROZEN_CLIP_ROWS.md) +connects base/V2 human-only text using exact cached preparation rows with a +distinct derived receipt, not a new encoder invocation. The genuine verified-CF +pool and production admission remain pending. +The [parent operation monitor](docs/PHASESET_V2_PARENT_OPERATION_MONITOR.md) +holds a process-released lease, emits independent long-operation heartbeats and +supports explicit recent/best checkpoint retention without deleting predecessor +or historical evidence. It does not grant formal starts or global compute budget. +The [persistent study budget](docs/PHASESET_V2_STUDY_BUDGET.md) preserves actual +historical GPU charges, pending wall-time envelopes, fixed pilot allocations, +and rooted formal predecessors. Bookkeeping alone does not qualify process +execution or scientific admission. +The [Linux process controller](docs/PHASESET_V2_STUDY_PROCESS.md) connects +reservations to owned child sessions, verified exit and actual wall-time charges. +Its 23 focused tests ran on the registered Linux server with actual stdlib +children, not native training. Native GPU integration and scientific admission +still need their own evidence. + +[Server mechanism verification](docs/PHASESET_V2_MECHANISM_VERIFICATION.md) +records 117 focused CPU tests and one full-width CUDA forward/backward witness. +There are still **0/87 formal V2 stages** and no native-data retrieval result. +Approved main-data and licensed body-model assets have been obtained privately, +and the handoff fold assignments have been recovered. Complete development-parent +preprocessing and five training-only physical calibrations have closed private +receipts. Run-specific admission, faithful baseline training, resource profiling, +and real human relational verification still require actual execution evidence. +The descriptions and score equation below document the preserved V1 core, +not completed V2 experiments or a submission-ready paper. + +## Preserved V1 milestone This source snapshot declares `v0.2.1`, the portable-verification patch for the stable data-free multi-person core and execution-contract milestone. It is a diff --git a/RELEASE_FILES.sha256 b/RELEASE_FILES.sha256 index 873a4d2..a04da97 100644 --- a/RELEASE_FILES.sha256 +++ b/RELEASE_FILES.sha256 @@ -1,24 +1,49 @@ # sha256\tbytes\tgit_index_path; self excluded ca4262d34e1553bf562d8ef5d3ef15b666b8f12214d5ff9345a8573baf9dd0f3 327 .gitattributes 2c5a644e7b812eb516fedeee66218e87a3ee9a0e6206d0e37a383b24eb1f1522 129 .github/CODEOWNERS -6e6cf3cba1ad55fb25e1aff2a0037d9791d45f0ca757c33f4f2335d75ec061c7 3019 .github/workflows/ci.yml -4bed921bd4f4cdd73c5533ccf30fd765c426da1835e397c7e7ec90ae6d5f9302 378 .gitignore +582a8f85592c529b6638397936cf0b7d1cad32f5192d90e6812db0bdb4e04d07 3607 .github/workflows/ci.yml +e9938bfe4bbd1ee2357396d1a6a8788e511389e297c28d7247d9e16a9a8137fc 392 .gitignore 4c20bbdbd002e6f52d43e0d4e5b2dd826a577cf45c817582ae3ed896e6d9f075 6957 CHANGELOG.md 34ad6f564d314e436426109fb8e7e67a2473e8dedf200d45a766d27d030cd080 780 CITATION.cff a9cb2d366c1a36a48dc840c697b844f173f1947eb6c022c399d2d4725ce6ed93 791 CONTRIBUTING.md cac1d22ba72c8bec57d01dab993e001f5d002eb73346e2a666dedd996186e3bb 1096 LICENSE +761307ccabbb869559bc7b9a22c0df72a5521ff1d1278eed77b564cfe5870e31 1367 LICENSE_TMR.md b5613fee419a712d8909e243c73547a7d0a5915900365e801cc3840abdbc50ab 2199 MIGRATION.md -d44f41127431cee3838a5fb9a6b7b903ccc06a050609c7f843a60da17cf90f97 9139 README.md +b0a88c6f90e097550acc7dc2053642eb488b958b58711a9d62ac97b0272c5619 17553 README.md adaa9b29f0b73606055b727affb459418f5e986c932b330c1b93ef1007a4aedb 7828 RELEASE_MANIFEST.md 876add20b16390ac8329efd2f94039f0304508d75995af1b556b70203027d8c2 579 SECURITY.md -c3166a0ca82dc07053c9ef67788f610ea21d6fe72ae8df2306ffcc742b0e6014 7975 STATUS.md +e126fea9f0f73e87bab2a596c08192271336a95b065218a5003180a029c52498 31798 STATUS.md 78d3b78d74ee222c1a34f157223cc56107d1dad0861f4a8d432c69ecd62a8ad7 7112 configs/phaseset/embody3d_split_v1.json 5ed598e51799ac33bc4d9aef6e88aa4640cdfcd97a7f4ae4f589a27ece838144 2683 configs/phaseset/experiment_matrix.json 195ae5151917647795224c1437faebd37e78b31ec62e92793919a6058f9bebfd 1476 configs/phaseset/training.json +7282eeba660484d1ae2df18e5e8d80772d755e4d3f7d9df41e35e0a59dcfeff3 7631 configs/phaseset_v2_analysis_rules.json +482a4432ac95cfbd21d5b6363893377ab61f6c84373bcf07b7441c4bfa8891dd 26630 configs/phaseset_v2_experiment_matrix.json +97a0296e84dfdf3255a52041e50e873664e3493edc2e47bb3f787accc3442911 2710 configs/phaseset_v2_relation_supervision.json +59916f216450b4d79817f6c7f0574243e1092496e53a07a4f1a518047e427acd 994 configs/phaseset_v2_venue.json +2970469ad0787e74ab4fabce2275486eb50c30c38f3cb7cf8f2800551d16991f 9540 docs/BASE_COHORT_QUALIFICATION.md +3824afe584e31b5765ebcedfae64b26d2b12a3b20ee9936e0cd5bda4e57493d4 5573 docs/BASE_COHORT_VALIDATION.md +e32c97a223c2b04278b9ce3c5d3b25590b390bc82dbf29e99b06d08d6c96b268 3054 docs/CACHED_RESIDUAL_TRAINING.md +81ecc1cc742604f47833a0aa5b4c567165aa01e9c500d05157e325bd3b4ccdb3 4281 docs/CAPTION_ROLE_PROVENANCE.md +62f6077a4d1cf0142e85ac28455a090d732b550874e47ea5644403566500f784 4644 docs/CAPTURE_PREPARED_STORAGE.md +e66e0530b02a222d0b3c46590290d8ebcdafa21d7cd3acf2d751355cc1f8e558 9492 docs/CAPTURE_VALIDATION.md +e9331b64297d6c4586ccb22022c972fad657ac2b3f9202203be62b067a38ca82 3686 docs/CUDA_LIFECYCLE_OBSERVATION.md e91eb48eea7b2b4db91184825f012c62301fc649cfd656ab1723ddfddf634da7 14306 docs/DATA_PROTOCOL.md +7246495a01908041d7a3d56268f92a5cfadc44655292a2d93f50a94748338f33 9755 docs/EMBODY_ADAPTER_INTEGRATION.md 3633e37c3eae3289d552a363e565bee027ebd83661f1c44ea136f9498de9e5af 19059 docs/EXECUTION_PROTOCOL.md 75c1180867d7f9e527212dcbb543e6b5d91a687fdbba70a7e3c75d97a5aa7104 8069 docs/EXPERIMENT_CODE_REVIEW_20260824_220209.md dc015b9fb4bba90e3f69759d77f3940ee2a364391ec1a3f429a0cbb881b77990 6989 docs/EXPERIMENT_IMPLEMENTATION_REPORT_20260824_220209.md +97b7b7e68b7896220b38661b094505b47c711a431f546e6da9772cfedcf6beaa 14238 docs/FROZEN_CLIP_TEXT_ADAPTER.md +f2c59540bfc4dcdbb5013b76eb20090a7dd61c805523121995ed9206c843f9c8 3711 docs/FROZEN_MHA_RUNTIME.md +d03a1569556beb1b3ad7f7f501d7bdc40e271f6912c173e403f75458fd6ed238 6158 docs/HOST_BASE_QUALIFICATION.md +e38d6f254f71d0027a0b82e1dc3ec95d23a9041831d21d41e96acf10b81e155f 19223 docs/HOST_RUNTIME_INTEGRATION.md +5dd898347d3646cde4a74e10afa8ba1d12703e3d0c6df51e5113659a31808086 3606 docs/LATENCY_PROGRESS_JOURNAL.md +f734981b1e98813719b5debe769dfeed0c63c127a35632cc54399f46e4b80e16 2760 docs/LEGACY_TEST_STABILITY.md +646f1fc5e4cb7ee423891ad6d212c83ce2eae604754b738ff8a78a3870d0bc7a 4607 docs/MIME_SET_ADAPTATION.md +0fd1d1289229788be8bf8b4e6c81d807d8e32a223a90717f4a27d64b35c5d6ad 3005 docs/MORLET_PORTABILITY.md +1cbb857ba962b29eae75d206aa49657f481e559085e062a2ef51ab4a6d87a220 3440 docs/PERIODIC_CACHE_EXECUTION.md +a9e215279d354999c1f949f776a131b8cf4cec850abd81459c5c0d8cee0c8b03 4356 docs/PERIODIC_CAPTURE_TRAINING_CACHE.md +6e9b9513bcfdbe76fbd28472225cb1fa185029a2b272e0b3f7d45e4a581f57c6 7886 docs/PERIODIC_DESCRIPTOR_CACHE_V2.md +00794af0ae3a3cf00c8d276183a34c3f0bca08e00f749d1f71e6444248a7ed0d 3353 docs/PERIODIC_TRAINING_CACHE_PLAN.md e19e4a6f20b5263faede0001581210885ffbb72c684fdf30482d3058d7937bbb 4448 docs/PHASEPAIR_ANNOTATION_CAPTION_LINEAGE_V2_PUBLIC_20260825_144456.json f94d7d7a270ad9c4055ee024693d8a20a9e916c9624771754db580ac963e6d53 1305 docs/PHASEPAIR_ANNOTATION_CAPTION_LINEAGE_V2_REQUIRED_CODE_SUCCESSOR_PUBLIC_20260825_144222.json 69b5e67cbb59c9c658e5feb11d3f6c01afbc5e2d26a1acd2d6dba8f81f9e67ff 4296 docs/PHASEPAIR_B0_ANNOTATION_CAPTION_CLOSURE_SUCCESSOR_PUBLIC_V4_20260825_145013.json @@ -42,44 +67,87 @@ d588d52a1d2674ba5dde7543915dc77299457a840af35aad83f96191f9913e24 37391 docs/PHAS 1da1df6293c4f77f42ce0b91e62b8884b0c0055b199e99e7ebbac46760db1c27 9930 docs/PHASEPAIR_SIGNAL_SPEC_20260824_165840.md cc7106ec61da31a6ccbea1ac5b151b6275a8d8ccf8dc6cce9377680ce30e6b55 21155 docs/PHASEPAIR_TRAINING_EXECUTION_SCHEMA_AMENDMENT_20260824_234612.md c7a113407b0cf6c60ddaa6e8aa786cc3652fda0dcb6330fa870abfecc71c4f8d 3316 docs/PHASESET_20HZ_MORLET_CONTRACT.md -846ffa812d14c5e7652a503d19230e7954bdedbc630af33560cd99380124d86b 11188 docs/PHASESET_EXPERIMENT_PLAN.md +0ae55c2d5142c294e0915dc8c64a9919a11d6218b437f4f90b4ecc903599a4d6 11413 docs/PHASESET_EXPERIMENT_PLAN.md 81c52ad4a5d6391a57988b5cab5d29f869c5069b0962bca5577ed9c5bedfea39 1568 docs/PHASESET_GITHUB_MIGRATION_RECEIPT.json f98ebe2bfed932c63725f88522f3b98f25ce092daeeb61e92578b7931376e3a0 27672 docs/PHASESET_LITERATURE_REVIEW.md -ecb85faa3b442450f6ecc8d6f2fff77baf19e833254fa8a715127d9127742f63 13825 docs/PHASESET_RESEARCH_CONTRACT.md +cffee3955d5134b007d8ea29b52f92b13a284d8347950698a082cad2659e5045 14130 docs/PHASESET_RESEARCH_CONTRACT.md +1a3dbb04f7750f6ca6107a4ddfa2c30b073bfcf1e74e01418c9e5cb879f54f72 1935 docs/PHASESET_V2_A1_SPEED_CONTROL.md +f98b8d0c2a4e9f75e65d46556951472cd4838f32f2561cc00934c2fc9f810bb8 1647 docs/PHASESET_V2_A2_ORDER_FREE.md +794bcdc4fe9d2c7cf6014157a7958851c3e86256006b74844fb97013117a4bbf 1920 docs/PHASESET_V2_A3_PAIR_BAG.md +09c14876d4e0055f4ea19c710523f31ec6aab505ee22ca6f21a2ada6414e03bc 1534 docs/PHASESET_V2_A4_INCIDENCE.md +cb44019ac3459bb2e26f52e1be65d80d3ff065edd8d39aa0d84155772e0ee258 1893 docs/PHASESET_V2_A6_SOURCE_QUALIFICATION.md +fdcf40e6f0f30729808c66c55d9a0817b1c6b71032751e0baa5e1a287771794e 2252 docs/PHASESET_V2_A7_GENERIC_LOCAL.md +4f6373298ef558489e33a48ee7cf31090095cf19508c92b40ace682f461518bc 6458 docs/PHASESET_V2_AMENDMENT.md +9a2067d2412fb619408af75cf0d7644c24a4086b7bb3a15dfca9f13ad77a3b44 21120 docs/PHASESET_V2_CALIBRATION_AND_CACHE.md +6d50263826636377615082911b0de5afe871b8887f9419f95bbe4fb4db7f914d 3439 docs/PHASESET_V2_FROZEN_CLIP_ROWS.md +b5313996cc1e80e4b9fabc58b2d306efd0a51d05288b4cc856b77d7f438529b3 2526 docs/PHASESET_V2_FROZEN_LANGUAGE_ROWS.md +0cf908e9dfff51ed307f9f09e9e96144c0445bfff8c35ec258f8e3f3f7d094be 6867 docs/PHASESET_V2_HANDOFF_INTEGRATION.md +1e0d2fb0b35a6900dfb03af4b9bfe1701eb2c843f566c0f842b47b03b11b645e 1276 docs/PHASESET_V2_ICMR_TARGET.md +5e180412adf601b10d5f9f3c05ca0140b1d93bbd31439af95745f26b48294b2e 3282 docs/PHASESET_V2_LEGACY_PARENT_SCORE.md +b5042e5fa002fa057ca4af61f4a06953662708d221016c460acd9cd0278c5863 4606 docs/PHASESET_V2_LITERATURE_PARENT_HOST.md +ed094b856390f21490b60c9aa84a4d866112b2c5001f459fae07b2c7d6befcea 2668 docs/PHASESET_V2_LITERATURE_PARENT_TRAINING.md +932cc27f1e4c53f3ee09042514949cf9b2132758cee2d1dd71d709dda4a08884 4435 docs/PHASESET_V2_MECHANISM_FACTORY.md +4d6b023e98c96116fa338db02005200716e0f85a1e5417228b5f375492abddd9 4269 docs/PHASESET_V2_MECHANISM_VERIFICATION.md +b1bed8feb97cb7ca379c1952bb2201fc94a2810f48e59d0d8096fc31aac15be9 3302 docs/PHASESET_V2_MIME_CURRICULUM.md +9ead6b89eacf034b52dcc15544e25be56e225c1b0432475b66c1a300adfa562d 3961 docs/PHASESET_V2_NATIVE_PROCESS_PROFILE.md +1c077c87ee2de81e6afd9f0a6541f7940a64d1496d8db3eddd10e0d0028acd7d 3475 docs/PHASESET_V2_PARENT_DISK_SOURCE.md +667eba26ecbbe11016064e9068764a1a86320e47d68b8e1eb44b422936721e8e 9609 docs/PHASESET_V2_PARENT_HOST.md +1eef1de33e2f1d193a6a02a0a982df0c9dea7b1f9406b4cbd155d22ef7378f51 3314 docs/PHASESET_V2_PARENT_OPERATION_MONITOR.md +a4ff32606557f35ade891fbf0786b96cb544bea9ceca8c1d4e58a47836415d4f 3325 docs/PHASESET_V2_PARENT_TASK.md +3d13f9b828c5deb62ff50e75dc6bd6c7663d3174ac636b4794c62412490a095b 2496 docs/PHASESET_V2_PARENT_TRAINING.md +bab08ac05ce49d686cf19b64cf19b2dc05a4928d24e9e5b19ef122ac5c732632 3567 docs/PHASESET_V2_RELATION_SUPERVISION_AMENDMENT.md +8944faef4cfe50dfdccf0443abf739a510a94291184b2374392e0c039762f93a 5607 docs/PHASESET_V2_STUDY_BUDGET.md +0e7c300498c28e83f8df3c3d78e2be44e8d9270e0005e79f4ec9847cbfc8cc16 4079 docs/PHASESET_V2_STUDY_PROCESS.md +81b47eff03531162dd2ab6b3d29a0ca9909326e7b13dec34776551d171b182c7 1727 docs/PHASESET_V2_TMR_PARENT_TRAINING.md +8ed656e8f4c6e1c9513bf69795275cba9948643cdc0c3ba47043602b30ece251 4370 docs/PHASESET_V2_WEAK_CF_TRAINING_SEAM.md +53f55d1554cc9e1262a11f0cbe69e2d41526f2aeca38e063082b0439346db8c9 5726 docs/PHASESET_V2_WEAK_CLIP_POOL.md +a05236e99b1607555a4d45dc99b35a8b0a13b69257ef9ddb3a3f434cac576d4a 8323 docs/PREPARED_DATA_V2.md 2a78ffbc8db0040747f95935d4bb77f40bdbf7755f3a4619a63e153fc4a6a7b1 6393 docs/RELEASE_VERIFICATION.md +abf26d1ae851ea941df4045c9f97ae04cc1aee75d233c166116ed71fed0e1170 4012 docs/RESIDUAL_RESUME.md +c2c47d122f77ff5976d4f148a1a39c3089e78c47213b071427fd203e638e676c 2807 docs/TMR_COMPONENTS.md +c310ce446533d9d0bb2ab1bebc30104c7ac17cb5cc84e34d1fa7bef2ad9eed19 5047 docs/TMR_SET_ADAPTATION.md +b4723657c23ca109b8c4ee21862ba7826dd9c2e1229eec648d072bec48436c3a 4935 docs/WAMO_SET_ADAPTATION.md 8d689f49b2704ff7c5988eff8a7e3e062f209b85d18425ccdba1e1099581079b 10658 docs/phasepair_research_contract_20260824_165840.md -d55d8e47040e9cf99d360f9e58df0e83bf2e24eddd8c165e40e09e1084dbd119 1057 figures/README.md -5a715d557ddb0a87236960e5af0142c6448e801d17e2a161d614948e78c3e203 13879 figures/phaseset_architecture.drawio -e544b64654355f01055eaf0806a66b6ab547a34354f62b97b24cbfc20104d652 76877 figures/phaseset_architecture.pdf -429a9052468f3ecc5038a2f1d5bc5bbe072537e14a83d29085c91f22fcbcab81 743516 figures/phaseset_architecture.png -4f5208d7e123c29079e86239e1248ec050b06cffe47f5b33901bc7436842e1d5 742318 figures/phaseset_architecture.pptx -3074af3aaf26f5782eebd3f64dbba3a6c183d5071af32315234a3a8ab08953f7 786654 figures/phaseset_architecture.svg -dd5fd26c085c8fc9143ffe8d116d23a816c4360eccb1747abeeb0803284738d8 9839 figures/phaseset_motivation.drawio -61bf4a503c8126b9df1be29303ef25955babd6b6bbdaa881414c487396013827 64923 figures/phaseset_motivation.pdf -a8d257fce7d54cfbce67ff2f5aeb17bf081e10f70d3c5e7c5b02f21278baeb96 476421 figures/phaseset_motivation.png -248ce77fc05ebd370a907b2c470bfe0fc50d6fa050759f5694e5ae081c8513a1 489547 figures/phaseset_motivation.pptx -a028dddf74489baaade772b5881004abb27de38f4ab631a529693cd9badce75a 363524 figures/phaseset_motivation.svg -304330ae853b1119ca0dd140603ed205f25c50b64aa83eece9a32e77228d3fc9 3472 paper/README.md -47cf3be3daafde0286b78e0bd61338e32f799ccba90e7a5cf92f710b70004a7a 709369 paper/main.pdf -1da98a709968867742e385a891deee8b50193618c2219654641b1715c0df4b53 22151 paper/main.tex +8d4b03326029b2a06478295b7328d61181d022604b1465bccd3a6cfcc8ce9704 2155 figures/EDITABLE_FIGURES.json +efb48515925d5412bc840a5ae07415cda4645591deaa1ab6aca90e1e4f528591 4208 figures/README.md +1a61a503c935a72cbfbe1ce175ab54bf975eb3e0a2164018aeda80448f472a1a 16465 figures/phaseset_architecture.drawio +7efb6fd4964cce91163a419fc5a312c2844e1e686c044b29502ff381c793c35b 82774 figures/phaseset_architecture.pdf +2dea358d457f9f12483af22e1f66c022b923cbc97d8c5ec12fd3b98be7027faf 385686 figures/phaseset_architecture.png +9793fcafb8dccbd1d19a75ee7cfe23a654ec03bf193d84574a9dcbc38260b046 147344 figures/phaseset_architecture.pptx +83b9c2c7464cf53e79e1a79dffbd15965506dd6cab1f1f019b967ada752492ab 902946 figures/phaseset_architecture.svg +a0a25fbc8c55575d50c6ffa8d6add01ce780cc8842328bb3f6bba20c6509d18a 10601 figures/phaseset_motivation.drawio +2c55a3103950187740f0200d1d4497d393c0c93c16d0f0722f7410065141d4ba 58231 figures/phaseset_motivation.pdf +a43fbeb379f7357af12deddc8b837f81218716ea30428ca320bfd5467968ca6e 209581 figures/phaseset_motivation.png +4c7ef9150eddd60f9ee75adbd4d17f2755c8144aad796251a0310a6dd545b048 118539 figures/phaseset_motivation.pptx +5ff14d17ee8dc393047131558e147f76b7037c903bed835d763a7a1363ccd7d9 369531 figures/phaseset_motivation.svg +6c3079e72f159eb737e336f64a776c6b19753dd25ff6a86649b782f6ce06b3a1 741 paper/BUILD_RECEIPT.json +492a2f33f90446fcc53c855bb1d91170740306612fe4f2fe64ef41d2bfa02213 4162 paper/README.md +e6589c0e35da06b3b7b8f72c3fbacc99eaa44adf6be9c20465b4cb03fe3961a8 514458 paper/main.pdf +21f99fb638bce1d6451bf58fa75ce01a7f0d081c8573c4d82aaf1ec502c33f50 21449 paper/main.tex 7db5537b9c85545e3fff89b29e17438936f0bcfb3e7d1ee2cd8de24f4de1a279 8419 paper/references.bib -2da20c2ffd05c4d6419531d802aaf6f276c2575ad05d99b9c753850ad938cd90 1283 pyproject.toml +3c86acb1ec972ef65b2dd2e950bd617a027802a91b44cc8a8ff64906ea6ac538 1408 pyproject.toml a54cab7df5f76e32387eba55757a4bd8fa18d9f2cdc8e5b0c6725ceb00af1e78 9580 scripts/audit_all_refs.py b4184b377fc1dbde1bf4a14ff002c5b13be09dbe6a545cee30eb878ee017fdd3 13507 scripts/audit_release_artifact.py +f6447b2870b24c657e3ca6f18d7330834826b6e5ec284392d94a82d25f5b180e 4422 scripts/build_paper.py ffb5b9e798fac11d7ec0f9cb13f1fe9ffd71dc197361795dd1a2c043d37d8080 3168 scripts/check_local_links.py +e8f61209a8be7bdae537b5d21e0108f6c6e6bbec0ab932b957703c3dee011ba3 4232 scripts/export_morlet_oracles.py +4a3396142a7e012143ccac8e2df4c45bee39778f1565a8f1a395e3b8e7fab6b7 3286 scripts/fetch_icassp2027_template.py +cc60aca15d425b1f684bf9555ad127bd37d0072d0967839f00e2efc41fb26ac2 4460 scripts/probe_morlet_portability.py 58c33c281a02a928556d0a25d87bdb0c893e99667d40d2c1c0326d2a899b0137 38002 scripts/public_release_audit.py 79e3e6160ef210fa03f1c5d8a3eb17c9298ba4b38e910d93e609ce41c1d27ec7 32001 scripts/release_asset_inventory.py -678b7a73a29360f5b87f525f9445ff05758d4f3e423a0e5ee215cbeb98900363 2769 scripts/release_manifest.py -a006b46722e3e711793b657b6a7c7c452deae39fb806b5b05420227279f0c337 10202 scripts/release_tree.py +8c99175b3f115ba9e8d63870988a6ed3d7cf8009d272b5c5daaefb85887ca495 2950 scripts/release_manifest.py +013dd2d7e7bbb7c07d86f49b49a400ad7c92a818a82cd98cbafee6d5a91d112a 14705 scripts/release_tree.py +18babb8ff4042dc091bb426e6679186f7427b3b01695d82e56d4468de2740222 33666 scripts/render_editable_figures.py 4db71f21e11539c4a1493ba7ea3d8127a56f1faba522c2813c61927e40ea9239 24981 scripts/verify_wheel_against_index.py 7d0eb6a71f0c0964a653c2b536ba232728885a9a526186835ba8896137c4ea11 30359 slides/PHASEPAIR_EDITABLE_PAPER_FIGURES_DATA_FREE_DRAFT.pptx 5ebe1cd2d83702888f77c642b8537d3544580e05fc73bf2e6baab9253e1f4dc6 464 src/phasepair_core/__init__.py c01ce63e8b55b4b58b14366cf971fe7683f627ea285a2dd22a140a3230b8bbf8 3025 src/phasepair_core/_identity_registry.py +deeadb80176998bc0c40eae598a8e4d292b8fac57867a900b6281bf5302db2a1 12186 src/phasepair_core/_morlet_oracle_bytes.py a74cdf1587509da10f5e787027e441cc810d1ab2efd3d7e062f58ed86157b9c7 17459 src/phasepair_core/base_qualification.py 3221b572ef4dc2edae832cf26e05f07c2b69eb2e1d4808bec5f61e0c3c3e69a6 19672 src/phasepair_core/batching.py 13a002c6146c3307639c84c6aba439dd0b3fbf216caea5ef49b06918b221f210 22962 src/phasepair_core/bootstrap.py efaa503b3d37653dfc790f2309d935b130a0dd0eaeec08ecfee74459d3da4123 20690 src/phasepair_core/caption_processing.py -40735f9c3bfe68a761552402ba779c53ca89da63667bf52b411647049424cb5c 33986 src/phasepair_core/checkpoint.py +80a4460126886bd3c7452a433c647331907af76d6bbb26bb86317b2dcb548297 34594 src/phasepair_core/checkpoint.py a444199a52f3b914d7ca8843ac2c95c288ca2728cd340d4d51f9523b0cd411c0 96389 src/phasepair_core/clip_resolution.py c9d9e5d114beae556da2e0fc0a729562edb57cf63f666b05d609288ff05c5cd8 36371 src/phasepair_core/contracts.py 74e55c77d3c70204fcf13b29ca167eb31dc81fc62d7b0e71a8e1be72b2c984a2 29847 src/phasepair_core/dropout.py @@ -95,7 +163,7 @@ cadc323c74c8f12467cdb64fbb2214f2012d23a8911eb803ee170c56ba088329 60050 src/phase c66b76b92d9b0b2a5747e0571a67f848e0dea9d7de99f16c70428e9df8b1818e 25036 src/phasepair_core/runtime_dropout.py 2957ecffb82d8e347d1bc37b41d902cd64858071bedaf51c719efb949cf4e0cc 10549 src/phasepair_core/sampler.py 9d4b27d08f04b9a4fa23b69c1594ff00945b11d6ab23c4efc8760a347db87806 17567 src/phasepair_core/sampler_manifest.py -135fa34813d27f14ceda731bf1515acbc42c496284fb3bfce4fd1f1d07c3fdbd 40708 src/phasepair_core/signal.py +1db8b593c0e5987c26c866a5d6f91a03b8574a0d298330460bf0dcf8c1ed1c82 46827 src/phasepair_core/signal.py 6d638065aa167f693cab53b074f385d8568b2baf4c84e9e9c75f6a486f54b988 25113 src/phasepair_core/statistical_reporting.py 2fbb5f4ece73f97037ac77dcfa75722f5ebf0f6d864412061218ebcf869a0585 26606 src/phasepair_core/text_runtime.py 067161d15b8f247e555df601a2b2f7221fb10d03b91997e3a40e06d719f877a7 42561 src/phasepair_core/torch_models.py @@ -103,33 +171,129 @@ c66b76b92d9b0b2a5747e0571a67f848e0dea9d7de99f16c70428e9df8b1818e 25036 src/phase fa68bcab33d09588a8653151ad8e5741d5e6530263bcb633d871d7744598b194 10364 src/phasepair_core/training_batch.py dbdb210202b1e2f5ade401f9fea488d948cc0f3d861fb8ab94a6fb3b50f68e33 20534 src/phasepair_core/validation_gallery.py 2e5fb5280ed27a46c587d07a60ce663fce676cbdd30b059ec34882b4507d157f 604 src/phaseset_core/__init__.py +9668a89535264a3ed6b58ffc7ac31b4904a4d8f68636d405e15cbf7368096716 8437 src/phaseset_core/_morlet_oracle_bytes.py +d1aa8d6837a40ed062dc941a5052c25c6304b2f60e97782af306a83ef4078842 72640 src/phaseset_core/base_cohort_latency.py +ecda879d3f531ebb747b561612fa34df37f1ecbd8e1977d5089d38b01ec63131 29076 src/phaseset_core/base_cohort_qualification.py +d2d71766c1c6af95480d507960d7186bbec9ff8a424f6e79ce6611fd0aa2e611 38885 src/phaseset_core/base_cohort_resolver.py +3d4f31b6a773081af41f1d9d45ee98986ea33ef6f954070564373f995af1f2e9 40593 src/phaseset_core/base_cohort_validation.py +ba59fcfa317aae5d0bf8e5330ede0eea2f399c5a4ebe7700b2f49029bdb53ff9 46345 src/phaseset_core/base_qualification_host.py +622082f9660a4cc766573edb3bb2ce0548ef5f03bd515aca002f1b190e89c249 3669 src/phaseset_core/calibration_populations.py b8c239437873367ce2880d460c17f51f58bf94d800b8087aa1a811fc0d9e678d 37144 src/phaseset_core/caption_fusion.py -0ddb4070d6af6a3221afda66aef84261255af3008ca8da0a2cc3b471d9b682fa 15837 src/phaseset_core/cli.py +121dcb6b948b9a3a4878ed376b3692dcbbd4b19f7ea147a7d28341c64b77d296 36926 src/phaseset_core/caption_role_provenance.py +d8125f420db0ec8d72faab80557f3a4c1ed983793d7ecdb042065829b5e73714 8686 src/phaseset_core/capture_pooling.py +0eaf0057ccf6ab6a4b83c1fa6e649399c4f783a79fab301e0778bafb5699daa9 47086 src/phaseset_core/capture_prepared_storage.py +213df61614f99afc5936dc42f897466945f1979339f7008fc23414e026b89324 45428 src/phaseset_core/capture_validation.py +c0bcf4fa676310a82564b7edd8a2b9ba78edfd01aceb751ee6f2d5b99120228f 16007 src/phaseset_core/cli.py +80e6d497ff9bb4354664cb1a054cdb550ab222e92c841813afc27af6d3759495 5603 src/phaseset_core/continuous_base_retrieval.py +ad2f6cbeddafa5531adeb949e419ee948744a8e97bf793a62ad0660a77de36e4 12556 src/phaseset_core/continuous_capture.py +50335890a5f78ffd320ad5533a5c589817bf4401f1dede769d7146563d97b4d9 3062 src/phaseset_core/continuous_capture_io.py +22fcbb5182e88c06eca1052e398c1ce67bb17ddb350f423f52bc13dec408453d 37857 src/phaseset_core/continuous_parent_host.py +0330c2050dfcfebe8da08eb5e848bd74f063fc9b9df7710bd20a1972fb602846 19635 src/phaseset_core/continuous_parent_training.py +564bfe9b1717ac6e539e62726a245b27fc84103e235a1ed1ab1bf04569b36f7a 7483 src/phaseset_core/continuous_phase_cache.py +a0bfc58b51d6ac65432f0d4a45ba7df78836fe0fc9b5613b1a23a2974bea264d 18760 src/phaseset_core/continuous_retrieval.py +d66eb2fd534a31701c7dcb8b4fdb634b8db6186b423e83fcc3af5c60152aa8c8 7381 src/phaseset_core/continuous_training_input.py 2e5531954e3c589bec79d8c3246182001782ddfef7b0409f060e2440b5ad49e6 21701 src/phaseset_core/contracts.py -9636324cff46e2a89bb93b71f8c3a4cc07544f8dd0e10e2da60d5a2ffbc7b356 25892 src/phaseset_core/controls.py +bfdfba8797aff6412e5b2c2d46d5cc0473a39980e1fb3c7b68f6299dbbc2c92a 27143 src/phaseset_core/controls.py f0f50300b6d209af8211872b17de86bd445f9f239f8d369220bc3236e51e03e2 17054 src/phaseset_core/data.py +a03bc907307f82452469e38d1b39c97c6c39febd4ad64f19d6ff320fd70f3171 11969 src/phaseset_core/dct_calibration.py +727d85b652932cb2b8a6e76b2c470ab009c0f30a723011c2ce581e8b2e548fe0 13070 src/phaseset_core/dct_relations.py +78ee596a0a7ae34e686acf6e0c1da46d1e15d70466d54f5df4e18ffdefdf753f 7776 src/phaseset_core/directional_calibration.py +4c7fa6c88bba905035365a236b32e7203679cf1d9f8eba321b9d72c7d1210f93 23779 src/phaseset_core/directional_phase.py +e1617f38d4ff751cf5d736da3eac99af678286ee934eb0ad4defa0f28da3e81a 27430 src/phaseset_core/embody_adapter.py 72f47efa82da668a950e79a2ece53ece60736361b55390dd7c89f48fd24208fd 57367 src/phaseset_core/evaluation.py -0c6cae2999670ce59e5a48f2ed33f7c89eca941cc35da338977c7d551e136024 77642 src/phaseset_core/execution.py +eb3149dbb20156168051442acae158e6009b92ce41f705f974f4a6f232129857 83444 src/phaseset_core/execution.py 05175adc59f341f4f59da3b18f8e834e8ac537424c9df3d8aa78ba6253e5b235 34026 src/phaseset_core/experiments.py -47cff521b4f43ce185bb99326fe4b579dd630fc493daaeb85768e9a5475d7a62 33917 src/phaseset_core/models.py -98f6c1ecbd29398c201b9595cea7f792e5f2a370ade124d01b24f19506bb4261 7640 src/phaseset_core/morlet.py +7c491ef7d65cddffa264d68d0a9fdd9216180bb741f85dd662cb570ca6e872f1 77027 src/phaseset_core/frozen_clip_text.py +6d600f0affbea8e69cb2e1df8589ff53acf785eae34cc16f742b0d2176554644 107360 src/phaseset_core/host.py +3d4bfeebd0940ef11d0b2253c187fa6e716de369d4f273c03bf7f55a598dc400 44734 src/phaseset_core/latency_progress_journal.py +84c7052b74cb8be4cddcfb11fea633b9bb46d3e3f2690761db3134ddb1c3f6b7 3982 src/phaseset_core/legacy_capture.py +f9edf6694574b9b40ba920d44a34fb7de5ba06f3e18dd1634c43b908448e9d19 9679 src/phaseset_core/legacy_continuous_retrieval.py +4c64fe3390f5921a1fe3f06c70abd8228e30ee1c6a64b7e53ac2c2bf2c829579 11981 src/phaseset_core/legacy_scalar_calibration.py +1fb8e81b89b58e995a08a016aa3efad5074a269907e5d51da040fb0918d64335 11164 src/phaseset_core/literature_parent_host.py +02a11d20ac299c4886479b863f2f614512365a9f1fe800861b2f261dbdad1c7a 9101 src/phaseset_core/literature_parent_training.py +f0d7c36ecda4326ca2b1f9461fdc812039cae33739595c60d40a436f0051edb7 6976 src/phaseset_core/mime_language.py +07092fa015f73af9da2b23dfa26c6e19e4c7a8b6a5cc53c316acb1d2d4b356e5 7245 src/phaseset_core/mime_parent_curriculum.py +c86e8e54de9f1e5f592d41b59bb5342b7fb68400188d8c8af61417f459943a42 17290 src/phaseset_core/mime_set.py +9e7fa84554d5676f8ef75049b17b53580395c3c60be0f75fcb55996ccc8ef36d 37720 src/phaseset_core/models.py +57bccd26419918de128839516955263a4823a2ead5c3465901c1c5278fbf5e43 14128 src/phaseset_core/morlet.py 95091a9d8cff83917832578a36478e8219d2f4b245c207cc2d432ef24cf461dc 10503 src/phaseset_core/objectives.py +0f0309c54a5b48a0493bb24f97eda39ff93be657295e1b0ebcc3dd6e9a8ecd76 6565 src/phaseset_core/parent_capture.py +be88afc2d543a9ceb227eda07046cc8e9221faf7af85888c1b303632da642dc8 3366 src/phaseset_core/parent_clip_rows.py +c15e1436000b3fc6eba22ba900e27372f30f30dd02534b8713729255504c11a8 12685 src/phaseset_core/parent_disk_source.py +be883315aa773e6c2d535ea981853ae92a791187338fd2b310b574f956174a82 7326 src/phaseset_core/parent_retrieval_task.py +46baa360084180121c16e1ce4d0c0a21841c6ad91c462afd272d5b92b0a603ad 9528 src/phaseset_core/parent_run_monitor.py +52b22af181cb52cf3c9c3584d15ea02a604e5c29d8abeaa0de217baf07d8af06 7027 src/phaseset_core/parent_weak_clip_rows.py 7a616be629078df58b4289be72e389261c11e1fa53637eadcecb3967824dcb93 30197 src/phaseset_core/periodic.py +e69b0d27d479b4499bae73c96f7854f9d07539043a0b66f0f75f6bc3e23c1f51 55415 src/phaseset_core/periodic_cache_execution.py +21ab159e434cc504867d22a22894a8e6a82afa77dde88a466773b5549a004850 42190 src/phaseset_core/periodic_capture_training_cache.py +10a3dcde2653e4ffa3eeaba06426736d47c9ebb2caa565cc0698df744658d249 55700 src/phaseset_core/periodic_descriptor_cache_v2.py +8e36e1a62d55962e255d1d08d3608de0fcc31dbacc879051b8e1d88d8d3988c8 35602 src/phaseset_core/periodic_training_cache.py a4dddc01470e2c2f6596733c338527da74eb5b4254dd0599c39a3deb0ee1a860 25500 src/phaseset_core/pipeline.py +5bd46d884f0fa3b8f022d1684ab5ffa1799ded3b970649753b0dd0510350216d 47614 src/phaseset_core/prepared_data_v2.py 273225e6696351ccb16a3a9e7a653b55af18c64b316a4e5d80c7b265625cc5c1 17929 src/phaseset_core/preprocessing.py -044b381ba73e73f476918d353f29c7cbee38cdca32ee612826549fef60671bdc 73146 src/phaseset_core/production.py +ad2ce56325f09d2efb17847ef3f947535f81f7099019275fdd074c12efeeb559 73588 src/phaseset_core/production.py 6f60feada83ebf1a55635bb657a5a3998e9fc77fa85012aad46e4f8d90d344db 37785 src/phaseset_core/publication.py +ff3b8022698bd2c1357c3dd11df2babf10e4d4ccbc00f135dab227ae34b5e1cd 4237 src/phaseset_core/speed_calibration.py 2292bc9f142c3b84804845d2682d52ba211609d5a54f976555db1ba4b886aae0 23917 src/phaseset_core/split_audit.py eba60052653bec6631a9e5d28828858802315b0d242e2eb610db2b8a4f45b2ce 54164 src/phaseset_core/statistics.py -0c8f52f8822f4e3822e5124ce794d482bb09d0545c73f5c6b341c1bbd7bc86f2 35557 src/phaseset_core/streaming_autograd.py +a0d56f1ddc63782c43b909125227c5f0be34bd16ed0a651a4a403e0a53c3de8b 36845 src/phaseset_core/streaming_autograd.py +47a2aa0806af373e5df40c611bdad425efd672588fe912d4470ee27d8985181a 21832 src/phaseset_core/study_budget.py +78cbb47380af76db33acb04350f2307ffae290b6db06b9edd9f1866a2d876bfd 19877 src/phaseset_core/study_process.py +934b4933795a2971d83811dc36865ddd43d0fd1670e8c2caf9ec53755d362c9b 8297 src/phaseset_core/study_storage.py +bddd2124cac00d4d6b193cdcec1c2f801b958720434b98a8600bb187e45ed471 36851 src/phaseset_core/temporal_coordination.py +47458d07e0c2132b0c4451195d99bf15ff754766453e840284ffc06617b88ae0 14095 src/phaseset_core/tmr_components.py +974fa0a326fb7e3b6b4897551d5aacdac9220a935025689437b2d7b7b84072d4 5566 src/phaseset_core/tmr_feature_cache.py +cf8f38b22fc2f5b2cff3ea39a8512892b5e1d6e739e199b5e4b7b5e49c4e6c9a 6550 src/phaseset_core/tmr_language.py +d1b92f7b65d750572a8447a48cd2a09cf12d7ec0fc3edd2f33c4088d0d77a14d 9442 src/phaseset_core/tmr_parent_training.py +b4bc9462c0b9a05ff3c26a4395c6be1f9df0c07bb64103a3ba8805dfe3187d56 16713 src/phaseset_core/tmr_set.py 2c43fa274a5b10040b8df2283585424204f3380030213b0f85a9e2c18cbffc3c 20463 src/phaseset_core/torch_periodic.py -e5de1fee169341f56721b198d0708fd21f56bd26d2771fad2ea88bb03bbb2d94 153786 src/phaseset_core/training.py +168d94a87ba328ddccbb86595e62fdbdbe7910d59cd4f1c10b6089ccab54827c 175962 src/phaseset_core/training.py +c2501a1753f2b0ce0d87e0a695f9a16c6fbe9eb0d6440e59051c69728361a68b 6693 src/phaseset_core/v2_legacy_factory.py +7ed3ca9cc8e534230b63d46395c63bd7c8f552f8004470b97bc1e9878c5004f1 19714 src/phaseset_core/v2_mechanism_factory.py +cbeb8553a7b0b3bb68a092a98e394cb25917410dc4b13d3d7212cee91b456aae 5583 src/phaseset_core/v2_score_tables.py +e8e4fa5cf988c11bc7aecbff241f3c5c253533652b1a31ea2313ee78626c3371 11472 src/phaseset_core/v2_statistics.py +c1b145691b73cb3fe56925de75b114dac6ba8a84fcae6ffeec282010fc36fa77 15762 src/phaseset_core/wamo_set.py 7d93120cff7411fd98ea0b15dc0102d98d8e47a9d013270f11af079fa6201e03 3217 tests/test_all_refs_audit.py +ae334bee8e2762f022020942141cb52f3285d3d9cb07d53574b6c37f57b2a556 20274 tests/test_cached_residual_lifecycle.py +358c3ac4470be9c1199a85ab18cadfc977690359875f226fbb4c591b323997bf 2082 tests/test_calibration_populations.py +d5d69092cc271059400773ef68447e06068aa802d5e9c82193704c6b65c71f59 18827 tests/test_caption_role_provenance.py +ae0b1cb7b79703de8cdd1406267a60101b05f191255da30fadaec6e6fbdde5d4 8598 tests/test_capture_validation_source_lineage_v3.py +c1115e9afed9fc3811c48439cc12a89e4b8bf9c7a6cb134d867aee0fc95ca6e6 6925 tests/test_checkpoint_observer_integration.py +9a6d69be46d3df730acabdb7e2b7fd23a9f4a10ecb6895c013a6fc9d712f14e5 13066 tests/test_continuous_base_retrieval.py +b0f9a8d920640d3c5455e9eacdeb87feecbaca87a8b1b06affe8ca297c78d075 3593 tests/test_continuous_capture_io.py +c0124a85e3154881a41c3d49706f81667663d5bfa996066e73ea299a72d586d0 17101 tests/test_continuous_parent_host.py +8b91250de082536ab1d8b0118977e1d569e4f2698e5073ebce44c6ab4bd8324f 11609 tests/test_continuous_parent_training.py +5ca49ce282fc8cff3654af8af56ade66cfa6b732afd7dcca3e92ffe06d4ce64a 6030 tests/test_continuous_phase_cache.py +e48d9f9eb92180b18f9b9dcb159e8da451697835d54fec4d00c86e737a531dff 12747 tests/test_continuous_retrieval.py +2128be288d7b7ed74d9c84acec4a22b5f6d5399a0b21d31b4c9aaa4061abe0e3 8878 tests/test_continuous_training_input.py +d27df99ab4a6d36bde033e4b5a3e41ba804d7304282fe5237fb1e4ed7cc3d603 5428 tests/test_dct_calibration.py +1564143a1f26f535805d0238ff035f036deb775bdf975e54ec4ef90a9b3da55e 14813 tests/test_dct_relations.py +5ed2bd1ec476a41ab1601bf8aab96a4da0447e671c617e1289a048109cd670df 7454 tests/test_directional_calibration.py +af12c2c1920f14cb9ea018f39178e91c47457f3008976dd8eab02fc1a164aa1d 4542 tests/test_editable_figures.py +ad4fb11844e43b26566803a0805dacf20e0a6b6cdf4e9741dcc98739141d6b2a 10614 tests/test_literature_parent_host.py +4b4193cc100b1076e31fe8c9f3fd1801b0270368afd80eca93c6cce0e671482a 6224 tests/test_literature_parent_training.py +7af87221b239a46d80e160341b4b4b3d7ccab85fe03b6e0368de463ee78c2b63 6575 tests/test_mime_curriculum_host.py +b65a2178d7faf25372adeccdfcd19ee7804544665c6059f2d684801ccff83e11 6050 tests/test_mime_parent_curriculum.py +773430e42927751a5b07ab20da7ddd716d13853f9496033476b2926198148ef8 14947 tests/test_mime_set.py +d75b28bf730f483b8980aea1aa1398c127e10e33ec901012db77d04ec2d3b79a 6775 tests/test_morlet_portability.py +3b23c2b08ed81029ca6239cbb63b98093077786163aaf5bb47ccfefb21c6ef24 7117 tests/test_parent_capture.py +fc0c27fb68c7d119d1cc7a48afeeff5a707cab9ec206eb5ad85c606e31656490 9443 tests/test_parent_clip_rows.py +b3b3473c7925d3d9b4307be4539e809b06e3f7cca64da0a0db938ee1e2a6a152 8056 tests/test_parent_disk_source.py +a070e5ce27d454531125639eea5353a222fe01227c0fabee6d8be6dcdffc5e36 7126 tests/test_parent_optimizer_config.py +32b783f094f7b770623294f49a5c0ef960d1277daf29d9210a8858ec6d3d192a 4624 tests/test_parent_retrieval_task.py +cdf4d6dcea5c29e44ebb3f2f91945fd944a8026a7abe91eaa47beb59428f159c 12935 tests/test_parent_run_monitor.py +d189e29ef4ef81b9604a352a7828ff97c789a79cafda18e8928056faf08a7825 13424 tests/test_parent_weak_cf.py +d61fcc5a3f5ca3428300e49bcdc138a6483edd82fed11aac9d06de79293bfd65 15534 tests/test_parent_weak_clip_rows.py +deb4200c8161fefd5c9b0d992fa81cb2448545b419bf60f8f238f9a3efdcac8b 18014 tests/test_periodic_cache_model_integration.py +9a05b2a025a694f94e158d2de74242d2a820efbc8973920001895114e7a894fb 14880 tests/test_periodic_cache_source_lineage.py +8dccac5551ec4d519f98b4b81d8f10ba47bd1f19ece6168ce137c51dc2ca2f0e 18170 tests/test_periodic_capture_training_cache.py +9c8ff961404bb8d5da86a30c3f533fe8d1909cdfbd76ebb496262ff222121d51 24769 tests/test_periodic_descriptor_cache_v2.py +0397dbfdfc5f3cd356d437dfec762bcc498e9847eb1c30ae8f71f4139b4e9dd7 22302 tests/test_periodic_training_cache.py af9b5adc42d82b9a0aa4ed744bb069e36b7dd041c1957fdc97d17eb814a0fc0c 17284 tests/test_phasepair_core_base_qualification.py ad630675cc642350a988d92de6176053d1bfab5f81cbc49548cb0b5dbeb679d9 23596 tests/test_phasepair_core_batching_readiness.py 3b4b3654d38d9b99983e924d838b7236b6d2e8b27554298762be1e8c4fcd00c4 10736 tests/test_phasepair_core_caption_processing.py 0f247c766ce4ff3ea8be87429696f4b7e021d5ea9965fe352542e3ca1371f7ea 19115 tests/test_phasepair_core_checkpoint.py -d1f96da586cdc53c5036fb9d105e57e4be0f9580ab55924351ff23b36823a58d 33016 tests/test_phasepair_core_clip_resolution.py +c61ce0ab46726427abe866e61bbd1cf450cf28063072909b5d80718a23c4b645 33787 tests/test_phasepair_core_clip_resolution.py 09e9f151cc5c23a50434bf5db8d27823f97e260e03263c056ac8e1e06306fbc3 28076 tests/test_phasepair_core_contracts.py 046b55ea6c6daaff89f278850b0093deb27c9c4cfeabd0831bc589986cb8698f 20812 tests/test_phasepair_core_evaluation.py 394950bc154f69faa859236fafab4f9591d6ab738695a0dea2bc34396fddfbf9 29559 tests/test_phasepair_core_execution_schema.py @@ -143,37 +307,93 @@ d834b7bd9dbb40f42e5f1b977a120493f028b5c77309434282c997eae8ee5d6e 333 tests/test_ b950ab7ab0f02c31e4c645a752c549312eb0ed2022f113feb87784aa8ccfffe4 24468 tests/test_phasepair_core_runtime_dropout.py fd7aae8ba8fafffc7809eb8a6c0873ae7c93f345127d5d32a8d2776cf5a17d98 7564 tests/test_phasepair_core_sampler.py 09648239fc50127698cc007e4335de0dfcc257ac789339b773a7acf4d512a818 19014 tests/test_phasepair_core_sampler_manifest.py -561cc6b8b79802d34a9f0c2f1fa3561080c8a3ff8e13d8b5b589fc86c11abad7 32382 tests/test_phasepair_core_signal.py +5ba6dab4543d61a0beeddd55a799fe2ccbdc3f816214fd4e549cd89b75bfdd24 33280 tests/test_phasepair_core_signal.py afb529c3a4e333a9159f196e877deeb8075381502949964fa355fb5e17b957bb 16079 tests/test_phasepair_core_statistical_reporting.py 08dc2ec64b422542deb5383eae67afbc0dbedb845a4ce44fe7a6e5953d153b54 24793 tests/test_phasepair_core_text_runtime.py b801d46fb184d99c65825d7f9bf91cbced022418600a95d421c5d9244dbded13 19373 tests/test_phasepair_core_torch_models.py 4044d65faf31bf5762c725f6103f561d7f915361c324baf938b449aedc65ea88 45570 tests/test_phasepair_core_trainer.py 59dde3593b46a2a84453a722adb6b8748cec8246c50699e49087193af47da182 8604 tests/test_phasepair_core_training_batch.py 8f31d46a9ff2d4ca96fcd34dc6fd617933f2e604c0dd6e95e0d10ec9db487074 10255 tests/test_phasepair_core_validation_gallery.py +4da1d6c1b05bc485b9afb78776264ae7757744f104c37b947cdce0280cdfb172 2347 tests/test_phasepair_posix_checkpoint_ownership.py +6462129abd78cce74c4a2b21c50cefc7220b9092511dab13f5abb9b6277ea621 8467 tests/test_phaseset_base_cohort_checkpoint_state.py +69a7e034f81c381d5eb4ce6caa02f98950975ebaaeed6ff3e212e4d4fe80d03f 15361 tests/test_phaseset_base_cohort_latency.py +c34ac09fa80b4809db91f383b4f1ea79a609aa75e852aba91298fa2ca97e618c 6277 tests/test_phaseset_base_cohort_latency_uuid.py +e1e944d235aa1e91965805edb522486edea8ce850bf89e6cb05d8678d62e5938 6119 tests/test_phaseset_base_cohort_latency_workspace.py +1379f5e8d8bb11443615d14e6a14e4091319563cc4bfe67de0c4621f1e9476cb 13454 tests/test_phaseset_base_cohort_qualification.py +41535795e088b5374bf59b95491afa4d1aac08622cddf1aa7d76c5fb0af990fa 23471 tests/test_phaseset_base_cohort_resolver.py +2cd1a40ebf6c10c83dae4f72bf41821d25ee153c8b3f05819ac664eb0a8f611e 14477 tests/test_phaseset_base_cohort_score_witness.py +e4b3ebf50a8f0971431905cebd12829b5799f677333ec573bf5809b90209018a 19405 tests/test_phaseset_base_cohort_validation.py +1fd40ded07adcd93a3eb0bc85d3167991702a39b5b3fe72f9291a46fe35e234f 17163 tests/test_phaseset_base_latency_journal_integration.py fa34950fb26f63d5d1ba169499e6e4af49c9b691f33c5cb3aac6e961bdac9e55 18384 tests/test_phaseset_caption_fusion.py +c88e5b91823b22ebb7a4019acb328040c6b42be7acc03738949439e4efbe443f 10126 tests/test_phaseset_capture_pooling.py +ec0609bf54766ffec53a1d7e4af7e778e2c2efbd9f4d2c4c06984814a066ca7f 21782 tests/test_phaseset_capture_prepared_storage.py +a02d460a28bd6aeb372fa66e779cb4a65c86536ab951ccd3b6c30ed8091cbd15 8371 tests/test_phaseset_capture_training.py +beacbd78c68771001b60ff7c6271e6d99c92d24d83f251e1074fc4ce50d1216c 15511 tests/test_phaseset_capture_validation.py +0281be4e47421dc8c52839ea99a45d32531f6ce21a6f726a6ec32ea5ca6a3d4b 8300 tests/test_phaseset_capture_validation_cached.py 5dbbb81d6725f1185f4c12dc85653b6eac1e47ae2ffb7fb36f5b951eb8dea632 8848 tests/test_phaseset_cli.py -ac57f0ea3950b7ac6795f6f9973cf68ece3d370b686835dc4f8a2892c7233fdb 12603 tests/test_phaseset_cli_production_bridge.py +c3060c5567214d195e024152f2d427ff3a63f3cc5fef1ded01e73d6887a3431e 13653 tests/test_phaseset_cli_production_bridge.py +02b7700b556d31b9d49ebcaaa3500278ea7e46f47314f9414896abf73ba5b74c 7046 tests/test_phaseset_continuous_capture.py a53fe5c6b53adb5c80421842f806fc45e8305360348d7065aa0ddfb0b9316dcb 6546 tests/test_phaseset_contracts.py a74143036a41dd9d8779dc4a9b70fe20e5b38d013ef23778d13e73c11273dbd8 26066 tests/test_phaseset_controls.py a17a50e801c389b4e24b2840f2aafcb7b476cbef86e2e142ad0af6245a54718f 4515 tests/test_phaseset_data.py -8bd6e95a35523d108153f668f8919e6f22858e8283e6308041751c6c92d0f904 18605 tests/test_phaseset_evaluation.py -aea41df712a25be78841e655a6fbd91fc96217ce92e60d3ab842f185fc39a58c 41553 tests/test_phaseset_execution.py +2f2fa0bf4d10771573896cf5d46eaaeffaaa6d9c52ca647a9ab9aa32a1766c87 14442 tests/test_phaseset_embody_adapter.py +b9944ab5eb45854df49a003c867278afeac28bab10dea2c39aad1e519abbcdd3 20468 tests/test_phaseset_evaluation.py +28ae2930b6e953909d8ff422284d8c2c580f94b65e022b8b2d9101b2c9f3a454 49018 tests/test_phaseset_execution.py +e79f2cb54fc18bccef9b94adeaf8c1811b913d8ba2a22f77e4f35ad9669ad6db 1324 tests/test_phaseset_execution_chain_bound.py dfaca3506375cf25fef77f572b00709bd72c25d6c613340a95d66c6bb15aa57c 13749 tests/test_phaseset_experiments.py +04df5b4a8d75bed006f16b56910e4082d4104a45cc8aeb11d6932ede645777cb 14111 tests/test_phaseset_frozen_clip_cache_rehydrate.py +937a48b65ab6fa2e91239b823ef528edb0af143f30c9c1c5298c3fc01e3c5327 18396 tests/test_phaseset_frozen_clip_text.py +6d738b5ab6f2bdda6d5bdca2604105c84a688d5adf4daa16cb761b96f2fe04cd 18798 tests/test_phaseset_host.py +841bb5bd72b98885cdc50c95a92022bacd5d9ba63f9f3776ccaaf9df06ac6de6 62702 tests/test_phaseset_host_base_qualification.py +14f6d4b54d78876822e6f96440a32360ea4d4654c1d18d8d8662fe0f4dc42bf8 23309 tests/test_phaseset_host_capture_storage.py +6eb892149a9b3dee02f48e816acfe0026a0efaed8985a32eedf412bad1edd141 11008 tests/test_phaseset_host_intent_binding.py +d133ca3a4df71baaea1cc40f68d049b56a7fcfc0d0861a87e5916c9637bdadb2 16345 tests/test_phaseset_latency_progress_journal.py a75f583fd0a233760b1a421ca51680afe4195c33f9ef4bce6e9a1737828f207e 14907 tests/test_phaseset_models.py -ec4cdc33ea31a0d4a2d4b0f3a529f69e458f1a9b1af9c991c8e78511d49d7131 4925 tests/test_phaseset_morlet.py +e55318c9bb4b265a1c174f20dacf3590f2743ea74ca4f6f4694770eb249f8dc5 5351 tests/test_phaseset_morlet.py 6d7005585882e25be9479542aac96a5b2aeb1bddc5619a0f37c57f61f2cbe6c3 6348 tests/test_phaseset_objectives.py 52ccd9d4ea8351d5aaaa82eb99d84c6bd240c8b6147a506fd65dbca3d1394331 11547 tests/test_phaseset_periodic.py +709e51151dc6e5a6e05de49987e303f9118c30f159f19513245a6d4f1793977b 14857 tests/test_phaseset_periodic_cache_execution.py 6ebbf2ece33ff2390036e5ce55dcd34a0459211b79922219971c27bee37844a0 10278 tests/test_phaseset_pipeline.py +82c15cd86cd1ac918d4cec678814ad61509ae7405173a73871c2f2172e82faeb 17328 tests/test_phaseset_prepared_data_v2.py +aae917cd95bde322f4ec7cea693fd5d0c926d51f0a366e7be160426472ad7973 15770 tests/test_phaseset_prepared_host_integration.py f6837c3ac23f4209cb3ec3eadb899d0bdcd4b4259c901b021254ffe7b3688492 8482 tests/test_phaseset_preprocessing.py -64016f9369540dae8b87654f43d09df9b9c07a3c4af06fa45567008b57140193 45251 tests/test_phaseset_production.py +7b6fc1ff9ddab99a41fc38fd7a44b21bfb51ff6057b9d8b8c227244ba83de52b 45585 tests/test_phaseset_production.py a1fbdfe55e50f67bb6568d5eba276e24beeb5a44373c516f0bd7758bd5742127 12486 tests/test_phaseset_publication.py +44552bf3de846bc0ad98e4e4185cc9a19ae2f3172ac8047f4875644afb1010d5 10587 tests/test_phaseset_residual_cached_encode.py +005a91d472a715e98b3a733cde102ecffc70222e1572d6100bb18f87ce9e3361 33889 tests/test_phaseset_residual_resume.py 3f7e4d75e8bd29e6b90c8742d8b5aeccdbe08f454257c43a05c14165ae01d327 16439 tests/test_phaseset_scaling.py 9147a58a884062ca0b549e33aa7141c9e3d8729db1132f328673d59e93b13177 6762 tests/test_phaseset_split_audit.py d632ae495d5e8cc7863bebc4b32f1a582eb65ea4a40ce87fb6551373ab0e40c5 16496 tests/test_phaseset_statistics.py -e6e9600a244af88f731e9cc981f19478b5760db8c254dfd8455960ffe789ef1a 12321 tests/test_phaseset_streaming_autograd.py +11727c78e08d02dd433abf66ca75e7f3aafd77c10a98d94ea98f17560d6e86ba 13200 tests/test_phaseset_streaming_autograd.py ae349d5c5f678a0bf3ff30ac868aec70d14bf5113b97e55ffe56aeb25de305f4 7350 tests/test_phaseset_torch_periodic.py -0e7f9c3b6b04fb94826cc1005460a291f92a6cbe385e80004ff50f169f0ddbee 52009 tests/test_phaseset_training.py +280daf77e254abd08f9f06d7bc71f1e3fbb9dcdd17b15d4569f7ad0934504e1e 52526 tests/test_phaseset_training.py +d7b1d991814eed83c888e1e62a88b5bbf877b1c92401f616d10bc1574c7275d0 8593 tests/test_phaseset_v2_a1_speed.py +4ee992cf41d66cf7e2cae98f4ec00e69b080bf05a9d9f005e29099b92ee6ab4d 5099 tests/test_phaseset_v2_a2_order_free.py +4679d7fb1fc5e19f88ade47039c413adb51dbe5473a4206947338b2b7a70ab60 7217 tests/test_phaseset_v2_a3_pair_bag.py +fa1c43ea09bea11d4e43aac6fc70f789679083852fc23d9be527076802952c15 4154 tests/test_phaseset_v2_a4_incidence.py +6747a55d9c743766491ce62db6562b3e077bca67da1b8104cfdf894f94aff326 3559 tests/test_phaseset_v2_a9_legacy_score.py +c855095323e7c6c68cdd071508b83829ec5b71878f5a0aeedd13686d7f7751c1 2964 tests/test_phaseset_v2_analysis_rules.py +1ff21b1e8133cf14b0672d984a061cfbe71c58c155e1e53f4e40c5280f3f74fe 21461 tests/test_phaseset_v2_coordination.py +2aae2397f6c40d929b0eb9b61d340d3ece6d48a17da01d6b3bd2767c2b6945b3 7513 tests/test_phaseset_v2_legacy_capture.py +09ddbdb38d2c48c08ea09be0618a462b399c1cffe971cae9ea019cef003d8425 10128 tests/test_phaseset_v2_legacy_factory.py +4dbd3fa91ebda8402edb81af281ffdbb1be8d730119e9e461b350471e464dd04 10100 tests/test_phaseset_v2_legacy_parent_score.py +278279ce18dd0f74c397bc69f99828afcacf9c1c6b5466fe34d1071df3bf99e8 4075 tests/test_phaseset_v2_legacy_scalar_calibration.py +90ede69f8695413a5cbc38c45b67602e5b3ed26212ac09e7f33c568b0129f719 15608 tests/test_phaseset_v2_mechanism_factory.py +1df27542e87ae6d3d47ac0e6a865a6e9a5be49ea8a9bc5af61758b18e830b64a 6261 tests/test_phaseset_v2_score_tables.py +08d27f7109d678ca011ef63dd99429185eb7b8c16d5f22e266dc05fbd58ea383 9778 tests/test_phaseset_v2_statistics.py 8f5f02d335d1aca0a260f4748f5a0d33c6770f09a0f5d12435956597832c6a3f 15874 tests/test_public_release_tooling.py 45b42266bc1c42c36af7eb7d07be2fc85d8b1d0a539ffb3674f98b47e1327254 4973 tests/test_release_artifact_audit.py 801b38ae89732e692ce42a7928278cdc14b87d78e32d459e700ab60246606106 14458 tests/test_release_asset_inventory.py +8727e87f1a0d44db6a39b1e5cfd8e22e3bed05544ddaed07b66b759adee66d84 5587 tests/test_release_tree_batch.py +eb643d3d63d0833eef895fec42ca35095b7adb8874a6dcdf89ef7cb0bc36b198 7401 tests/test_render_editable_figures.py +9a971a607eba6e35479fa599b7363f22a7b08f7d2e5b6a780467cf8f75d904ed 20049 tests/test_study_budget.py +0dac3611a3c8641524dd43c216771f0f8f2f6e031dcdf586872a226bb28a67d0 20709 tests/test_study_process.py +5ae2b2baa3addc74ac4914fde10fab635f949c8f84b478ce764d4ffc71fbc32b 5128 tests/test_study_storage.py +96fd7d990fb5446f8ae435dcc61d0339caa8f5503c547cbee6b74b4527ce7477 8462 tests/test_tmr_components.py +18c5c10980d8e0d81eb1a6b351c0e0e770325c95bcca59e15763e5c86967ce78 4260 tests/test_tmr_feature_cache.py +726ee82009ecb29ce87208749fb1780365640a52f2e2d6e55399e22119838b89 2988 tests/test_tmr_language.py +9441ab431bc210814bbc5c94a353fc5178d133e729ffb75c67456cf07ed38d79 4559 tests/test_tmr_parent_training.py +2ae860859c1f9282a02cea929e54fa950433ac8fba81c7510c2b900321d5ff29 9606 tests/test_tmr_set.py +b3eaa9fc4160833e9bc8229af4980030ceabe8c7316885d2637de489e035f44e 5806 tests/test_true_mean_difference_dct.py +342cea54b04c8b6fe0faae0cf3ae84a1abac9142792acf5fd042ae44937c62f4 11334 tests/test_wamo_set.py 3aef8f056ef38ecbe5af51c283285b132db97f175c181018d8a4278d82131915 8150 tests/test_wheel_index_binding.py diff --git a/STATUS.md b/STATUS.md index ca85f07..1a1da76 100644 --- a/STATUS.md +++ b/STATUS.md @@ -1,8 +1,9 @@ # PhaseSet project status -**Evidence snapshot:** 2026-08-26 (Asia/Shanghai) +**Evidence snapshot:** 2026-09-08 (UTC) **Research state:** `ACTIVE / AUTHORITY0 / NO_REAL_DATA_RESULT / NO_CLAIM` -**Release target:** `v0.2.1 / PUBLIC_DATA_FREE_STABLE_CORE_PORTABLE_TEST_HOTFIX` +**Published release:** `v0.2.1 / PUBLIC_DATA_FREE_STABLE_CORE_PORTABLE_TEST_HOTFIX` +**Final target:** `v1.0.0 / REAL_EXPERIMENTS_AND_PAPER / NOT_COMPLETE` PhaseSet is the active multi-person successor to PhasePair. The in-place public migration and `v0.2.0-alpha.1` pre-release are complete; the immutable @@ -10,6 +11,339 @@ annotated PhasePair `v0.1.0` tag remains the dyadic compatibility baseline. Implemented code, passing synthetic tests, and frozen plans are software evidence, not training results or scientific claims. +## Server continuation + +- Complete two-store descriptor-cache construction and cached residual + checkpoint/resume now pass one combined 270-test Linux server cohort in + 429.82 seconds, with source unchanged. It also includes the qualification + timer/runtime corrections and Windows readonly-shard fault-fixture fix. + The actual official CLIP outputs retain their two exact embedding goldens + and unchanged RNG under the new training-source binding. The same source + subsequently passed 1360 full Linux tests, 2 existing skips and 39 subtests + in 743.26 seconds, with source unchanged. Cross-platform CI acceptance is + tracked separately; no formal run is + implied. See [cache construction](docs/PERIODIC_CACHE_EXECUTION.md) and + [cached training/resume](docs/CACHED_RESIDUAL_TRAINING.md). + +- The cached complete-capture evaluator and private host qualification + controller now pass one combined Linux server regression: 1334 tests, + 2 existing skips and 39 subtests in 643.90 seconds, with source unchanged. + This closes software integration only. Native data access, all formal + training, actual cohort qualification and sealed scores remain absent. + The current successor now implements early POSIX/main-thread timer + admission, exception-safe timer restoration, and qualify-base-only runtime + classification for post-admission source/output/lease failures. Static review + passed, and its focused Linux server run passed 145 tests in 64.24 seconds + with source unchanged. That successor is not part of the observed 1334-test + source; it is included in the newer 1360-test cohort above. Windows CI + acceptance remains separate. See + [host runtime boundaries](docs/HOST_BASE_QUALIFICATION.md). + +- The combined capture-lineage, prepared-window cache-plan and CUDA-workspace + correction passed 157 focused and 1285 full Linux server tests, two existing + skips and 39 subtests, with source unchanged. A separate actual K=32 GPU + retry completed B0/B1/B2 at full registered width and T=200: valid outputs, + unchanged model state, and exact zero allocated/reserved memory after each + release. Peak allocations were 81,548,800 / 94,160,384 / 315,323,392 bytes, + below the unchanged 2 GiB cap. The probe used untrained models and one warmup + plus one observed forward per model; it is not the formal 81-visit latency + session, a trained cohort, retrieval evidence or a performance benchmark. + Earlier failed attempts remain retained. See + [CUDA lifecycle correction](docs/BASE_COHORT_QUALIFICATION.md). + +- Capture-storage v2 now derives every descriptor context from the consumed + manifest/NPZ and canonical global window ordinal while keeping v1 and full + holistic-gallery semantics intact. It passed 92 focused and 1265 full + server tests, two existing skips and 39 subtests, source unchanged. + A separate prepared-window periodic plan passed 76 combined tests, admitting + the exact twenty-epoch source/cache census for each registered seed and + reopening each selected shard without fallback. A separate complete-capture + plan now binds the holistic capture source and its cache without relabeling + prepared-window auxiliary validation. At that predecessor milestone the plans + were not yet integrated into training/checkpoint/resume; the current explicit + cached lifecycle is described above. + See [capture storage](docs/CAPTURE_PREPARED_STORAGE.md) and + [window cache plan](docs/PERIODIC_TRAINING_CACHE_PLAN.md). + +- The complete-capture plan enumerates the actual twenty-epoch prepared source + and every holistic validation window for one registered seed, requiring its + closed cache census before returning explicit sealed streams. The residual + runtime now has a separate cached encoding entry, and a cached holistic + evaluator shares the original complete pooling/gallery oracle while changing + only periodic descriptor acquisition. The original uncached APIs remain. + A focused server suite covering the cached residual model, frozen-text source + binding and complete-capture plan passed 121 tests in 271.25 seconds with + source unchanged. The complete combined regression then passed 1305 tests, + 2 existing skips and 39 subtests in 613.86 seconds, again with source + unchanged. Three actual official CLIP forwards retained the established + outputs and caller RNG. The later cached holistic evaluator passed 108 + focused server tests in 151.03 seconds with source unchanged; its subsequent + combined 1334-test regression is reported above. These were + software-source enumeration and text observations, not twenty epochs of + training. At that predecessor milestone, training-loop, checkpoint/resume + and host cache-plan consumption were unfinished. The newer core cached + lifecycle is reported above; host end-to-end injection remains separate. See + [complete-capture cache plan](docs/PERIODIC_CAPTURE_TRAINING_CACHE.md) and + [capture validation](docs/CAPTURE_VALIDATION.md). + +- Prepared-v2 batches now retain their exact descriptor source/window/epoch/yaw + contexts. Repeated rotation of an already contextualized batch is rejected. + The live CLIP training-source pin was updated and actual official text + embedding bytes remained unchanged. A separate actual diagnostic localized + a Torch/NVIDIA UUID-prefix mismatch; strict full-UUID matching now supports + the observed representation without bypassing device or resource checks. + The combined Linux server suite passed 136 focused tests and 1257 full + tests, 2 existing skips and 39 subtests in 413.42 s, source unchanged. + Complete training cache-plan/checkpoint/resume integration was still pending + at that predecessor milestone; + these observations are not real-data training or a K32 performance result. + See [source lineage](docs/PREPARED_DATA_V2.md) and + [CUDA identity](docs/BASE_COHORT_QUALIFICATION.md). + +- Explicit reader-backed cached model execution binds the actual complete + batch, energy floors and stream family, and enforces the live encoder edge + budget before consuming descriptors. The same stream is + replayed through forward incidence, backward moments and parameter VJP, + without an uncached fallback. The model/cache/control suite passed 98 Linux + server tests; full regression passed 1228 tests, 2 existing skips and + 39 subtests in 401.07 s, source unchanged. These are software fixtures, + not real data or measured acceleration. At that predecessor milestone plan + admission covered training rows and the complete epoch census, while the + training-loop/checkpoint/resume consumption reported above was unfinished. + See [descriptor model integration](docs/PERIODIC_DESCRIPTOR_CACHE_V2.md). + The host now retains the canonical admitted plan/matrix/config instead of + reading fields absent from the actual command intent. Commands and handler + identity are checked before retaining the admission snapshot. Real + CLI-to-backend fixture regression passed 94 focused tests; full regression + passed 1232 tests, 2 existing skips and 39 subtests in 401.58 s, source + unchanged. The new CLI fixtures intentionally stop at the model-runtime + seam and retain FAILED terminals; they are not completed training runs. + See [host integration](docs/HOST_RUNTIME_INTEGRATION.md). + +- The complete frozen base-latency runner, exact qualification assembler, + typed progress observer and durable POSIX journal are implemented. Separately, + a weight-independent three-stream periodic descriptor cache now preserves + exact canonical batches and bounded immutable shards. The merged Linux + server source passed 1211 tests, 2 existing skips and 39 subtests in 362.75 s + with source unchanged. Journal integration passed 73 combined tests; + descriptor storage/streaming then passed 19 focused tests after a test-only + hardlink/oversize case split. These are explicit + software fixtures, not a real nine-run cohort, formal CUDA latency session, + winner or licensed-data cache. The later explicit cache-model API is reported + above. Current host end-to-end cached training/resume and host qualification + command execution with real cohort inputs remain unfinished. See + [base qualification](docs/BASE_COHORT_QUALIFICATION.md), + [progress journal](docs/LATENCY_PROGRESS_JOURNAL.md) and + [descriptor cache](docs/PERIODIC_DESCRIPTOR_CACHE_V2.md). + +- The window-caption provenance component now checks the complete chain from + same-window fusion inputs/outputs through ordered frozen text rows to the + materialized training census. An actual official-CLIP probe exposed and fixed + a prepared-data bug: inference chunk size had been mistaken for total caption + count. Exact chunk coverage and all existing row/tensor/digest checks are now + retained together. The unchanged probe subsequently passed three actual + encoding chunks, prepared-tree reload and exact repeated provenance checks, + without re-encoding or observed CPU Torch RNG change. The original failure + remains retained. The source passed 53 focused / 1155 complete Linux server + tests, two existing skips and 39 subtests; complete regression took 316.45 s + with source unchanged. Fusion receipts, captions and motion in this probe + were explicit software fixtures. No fusion backend, licensed data, motion + training or formal supervision selection is claimed. See + [caption-role and text-chunk binding](docs/CAPTION_ROLE_PROVENANCE.md). + +- The completed-base resolver now reconstructs the exact nine registered + successful chains, preserves failed predecessors, authenticates terminal-bound + latest and distinct validation-best checkpoint bytes, and revalidates the + complete registered state. The complete-capture scorer strictly loads those + selected states, derives live parameter censuses and recomputes full-gallery + score identities and exact capture-macro fractions. Their integration passed + 119 focused / 1119 complete server tests with 2 existing skips and 39 subtests. + A separate unmocked full-width B0/B1/B2 selected-state-to-gallery witness + passed all three lanes in 19.36 s on analytic K2/K3, T200 captures. It uses + explicit checkpoint-cursor and text fixtures, not completed formal epochs, + real captions or an actual nine-run cohort. The merged source at that stage + passed 1124 complete Linux server tests, 2 existing skips and 39 subtests in + 315.67 s with the source census unchanged. Latency and qualification assembly + were unfinished at that snapshot; their newer software verification is + reported above. A private host qualification controller is now implemented, + and its corrected focused server suite passed 131 tests in 37.07 seconds + with source unchanged. The first attempt's seven fixture API failures and + 123 passes remain retained; only the test file changed for the successful + retry. These software fixtures do not constitute actual qualification of a + trained cohort. See + [completed-base capture scoring](docs/BASE_COHORT_VALIDATION.md). + +- Strict known-host SSH login to the registered host now succeeds. The initial + read-only inventory is complete. Existing workloads were not interrupted. +- A separate data-volume Python 3.12.12 / NumPy 2.4.6 / Torch 2.12.0+cu126 + environment is installed with a clean dependency check. The complete Linux + CPU regression including capture host composition, storage portability and + frozen-text restoration passed: 1077 tests, 2 skipped, and 39 subtests, in + 296.55 s. Its focused integration passed 103 tests. The preceding capture + host integration passed 96 focused / 1070 complete tests; the earlier storage + integration passed 110 focused / 1043 complete tests. The preceding capture-training integration passed 1012 + complete / 84 focused tests and 6 focused subtests. Source bytes were unchanged before and after + both suites. The prior prepared-data v2 host integration passed 94 focused / + 964 complete tests. The prior residual-resume integration + passed 948 tests and 39 subtests. + The earlier frozen-CLIP integration passed 932 tests and 39 subtests. + The earlier host-only suite passed 917 tests and its 25 focused checks. + This includes synthetic model/lifecycle checks, not main-data training. +- A separately bounded shared-GPU functionality probe passed FP64 forward and + FP32 forward/backward using 8x8 tensors. It did not stop existing workloads + and does not qualify BF16, model execution, performance, or the full runtime. +- A subsequent actual-width PhaseSet periodic-core CUDA observation passed + with 512-dimensional tokens and a 256-dimensional hidden layer: K2/K3 + forward/backward, actor permutations, K2 topology output/gradients as exact + positive zero, and K13 (78 edges) bitwise equality at chunks 64/128/256. + Maximum observed CPU/CUDA token difference was 1.431e-6 within the frozen + FP32 tolerance. The engineering fixture peaked at 80,722,944 allocated bytes + under a 2 GiB allocator cap; this is not a performance benchmark. Its first + attempt stopped before model construction because physical FB total and + CUDA context-usable total were incorrectly equated; that failed receipt is + retained. The corrected memory instrumentation did not relax a model gate. + This does not qualify BF16, the learned group base, or the full training runtime. +- Separate actual-width B0/B1/B2 FP32 CUDA forward/backward lanes passed the + registered parameter census, physical permutation/padding equality, finite + gradients, frozen-text preservation, and CPU/CUDA tolerance checks. Their + small analytic fixtures are not training or performance results. A subsequent + residual retrieval lane failed its frozen-base CPU/CUDA embedding comparison + after zero-residual scoring checks. That failed receipt is preserved. A + same-weight diagnostic isolated an observed native-MHA-fastpath dependency. + The frozen runtime now disables that fastpath on CPU and CUDA and restores + caller flags. With unchanged tolerance, the new CUDA retrieval-head seam + passed: frozen-base maximum difference 7.153e-7, score difference 4.471e-8, + and finite nonzero exercised head gradients. This did not run the periodic + core or an optimizer. A subsequent B0 repeat stopped before model creation + for insufficient free memory. After memory recovered, a distinct retry chain + passed retrieval/B0/B1/B2 with the same reviewed source and tolerance; the + original resource failure remains preserved. Formal-training and BF16 + qualification remain pending. See [runtime policy](docs/FROZEN_MHA_RUNTIME.md). +- Full-width FP32 CUDA lifecycle checks subsequently passed for B0/B1/B2 and + system 08: width 512, 8 heads, FFN 2048, full 200-frame body22 windows and analytic + K2/K3 groups. Three uninterrupted optimizer updates exactly matched a newly + constructed runtime restoring update 2 and completing update 3, including + model, optimizer, scheduler, RNG, cursor and manifests. All four lanes stayed + within a 2 GiB allocator cap; B2 retained all four social-temporal layers. + These bounded fixtures used effective batch 2, completed no epoch, selected + no validation checkpoint, and are not formal runs or performance benchmarks. + See [observed lifecycle scope](docs/CUDA_LIFECYCLE_OBSERVATION.md). +- The native Embody loader and concrete licensed SMPL-X body-22 evaluator are + implemented. Seven adapter contract tests passed on Linux/Python 3.12.12/ + NumPy 2.4.6; these use explicit fixtures, not licensed assets or main data. +- The immutable prepared-data v2 writer/reader now connects canonical body-22 + windows and frozen variable-count text features to the private-host factory. + It verifies the exact consumed index/manifest/payload bytes, rejects split + overlap, and applies reproducible shared-group yaw from original bytes each + training epoch. Validation is unrotated. Window-positive families support the + auxiliary task; a separate combined index now connects the private-host + holistic capture source without relabeling auxiliary validation data. + Actor-set commitments must not be used as unique capture/window row IDs. + See [prepared-data contract](docs/PREPARED_DATA_V2.md). +- The complete-window capture evaluator now pools unnormalized embeddings + before one full-gallery scoring call and preserves exact capture-macro R@1 + fractions. The training API uses this concrete val-only source for checkpoint + selection and binds its full census on resume. The standalone caller passed + 42 focused / 997 complete server tests; the training integration passed 54 + focused tests including actual optimizer/checkpoint/resume equality, followed + by complete 1002- and 1012-test regressions and actual CLIP requalifications. Two + legacy fixture failures from the first full attempt are retained and explained + in [test stability](docs/LEGACY_TEST_STABILITY.md). These are analytic lifecycle + checks, not formal runs. Actual scored-census consistency and scoped live + capture-function drift checks were added after provisional composition review. + The private host now composes authenticated window training and capture + validation from disk, checking actual actor/caption/window overlap before + backend construction and forwarding the exact capture source through both + base/residual and resume paths. Licensed source/caption provenance and + participant-disjoint input preparation remain incomplete. See + [capture validation](docs/CAPTURE_VALIDATION.md). +- A bounded validation-only capture storage writer/reader now preserves the + complete window plan, original padding, variable caption counts and original + frozen text receipts. Receipt-bound restoration performs no CLIP inference. + An actual official-model cache probe caught a snapshot-file ordering bug that + the initial structural fixtures had missed. The failed attempt is preserved; + the corrected source passed the 110 / 1043 tests above plus three official + forwards and two exact disk-restoration roundtrips, with no extra forward. + The original output goldens and observed CPU Torch RNG were unchanged. + Storage is not a rights, annotation, split or anonymization proof. See + [capture storage](docs/CAPTURE_PREPARED_STORAGE.md). +- The preceding capture-storage commit failed 14 tests on each Windows CI + runtime while its Ubuntu audit succeeded. A read-only filesystem diagnostic + found different path-stat and descriptor-stat `ctime` values for the same + unchanged file. The correction preserves complete before/after checks within + each interface, compares stable identity fields across interfaces, and keeps + all content-digest and bounded-read checks. Seven added cases cover this + distinction and actual drift. The 103 / 1077 server tests above and the + unchanged real CLIP cache observation passed. Windows acceptance of this + correction requires its own GitHub CI result; no failed check was skipped. +- The next Windows run passed those storage-stat cases but exposed a separate + test helper's default CRLF translation. The test-only LF correction and two + authenticated CRLF negatives passed 71 focused server tests. Subsequent + GitHub Actions for commit `db71a83` passed both Windows Python versions and + Ubuntu's public-tree/wheel audit. Both preceding failed runs remain retained; + production canonical-byte checks were not relaxed. See + [fixture stability](docs/LEGACY_TEST_STABILITY.md). +- Neither Embody approval nor a licensed neutral model has been observed. + Open-license Multi-TPC acquisition completed with its official byte count and + MD5 verified. Its archive inventory has 322 files. The bounded audit confirms + 19 Euler rotation triplets plus three translation anchors, not body22 joint + positions, and unresolved modality-alignment/group-identity details. It is + supplementary conversation data, never replacement + confirmatory data. The pinned CLIP snapshot's eight files also match the + previously acquired official SHA-256 values. Actual frozen CLIP text CPU + loading and two public-caption forwards passed, with bitwise repeat outputs + and unchanged RNG. The production-shaped frozen text adapter subsequently + passed 15 server tests and three real-model forwards: the unchanged repeat + golden plus a separate 226-to-77-token long-caption observation. It verifies + the exact retained prefix and EOS, restores caller RNG, and emits text-free + receipts. See [frozen CLIP text adapter](docs/FROZEN_CLIP_TEXT_ADAPTER.md). + This does not qualify GPU text execution or real-data training. + The MHA-policy successor separately passed 33 focused/949 complete server + tests and three official CLIP forwards, preserving the original output + golden and RNG under correctly changed code/runtime/receipt identities. + The capture-training successor also passed three official CLIP forwards with + both prior output digests and RNG unchanged under its new source identity. +- The earlier text-adapter commit was downloaded from anonymous codeload, + manifest/content/link checked, built offline into a wheel, and installed into + a separate server target. All 56 wheel Python modules match the public Git + index. Three real CLIP forwards from that installed wheel produced a complete + observation byte-identical to the previously verified source copy. +- The subsequent public capture-storage commit was also anonymously downloaded, + checked against its 223-file manifest, and built offline into a separate + wheel target. All 60 Python modules matched the source. Three actual CLIP + forwards and two disk restorations from the installed wheel produced the + same complete observation as the server source. This Linux packaging check + does not close the Windows filesystem-portability failure described above. +- Native Linux last-bit Morlet formula drift is repaired using strictly + verified canonical coefficient bytes, while preserving both original hashes. + Windows still requires its original exact formula digest; Linux requires its + measured exact native formula fingerprint plus a bounded tap comparison. + See [Morlet portability](docs/MORLET_PORTABILITY.md). POSIX checkpoint cleanup + also now pins its inode so a concurrently replaced file is not deleted. +- The current no-result manuscript now builds using the hash-pinned official + ICASSP 2027 template: five pages, technical content through page four, + references-only page five, all fonts embedded, no unresolved citations or + overfull boxes. All result cells and pending author metadata remain explicit. +- The concrete private-host bridge now connects base/residual training to + live checkpoint receipts, a periodic heartbeat, and a whole-attempt OS lease. + Base and residual resume verify the predecessor checkpoint chain, including + process-loss recovery and authenticated latest/older-best materialization. + Fifty-one focused server tests passed, including actual tiny residual + optimization/checkpoint/resume equality and zero execution on invalid prior + checkpoint bytes. See [resume details](docs/RESIDUAL_RESUME.md). + Preparation and sealed evaluation host commands remain unwired. The separate + base-qualification controller passed 131 focused server tests after a + test-only correction; the first attempt's seven fixture API failures and + 123 passes remain retained. No actual nine-run completion is claimed; see + [host base qualification](docs/HOST_BASE_QUALIFICATION.md). + Four actual tiny server fits verified observer behavior without changing + checkpoint/model/loss/validation outputs. These are engineering fixtures, + not formal training attempts. GitHub CI passed both Windows runtimes and the + Ubuntu public-tree/wheel audit for the integration commit. + +This continuation is not a completed scientific release. Private receipts, +data, model assets, connection details, and download material are not public. + ## Completed in this migration - Full encrypted snapshots of the public repository and private research @@ -55,6 +389,7 @@ evidence, not training results or scientific claims. - A frozen numerical runtime with deterministic-algorithm errors, highest float32 matmul precision, TF32/reduced-precision reductions disabled, deterministic cuDNN, a mandatory deterministic CUDA workspace setting, and + disabled native MHA fastpath on CPU/CUDA, an environment-v2 hardware/runtime/build/thread inventory. Live drift is rejected and ambient host flags are restored on exit. - A differentiable small-batch Torch CPU skeleton/Morlet oracle for input- @@ -96,11 +431,11 @@ Embody 3D requires a real applicant to submit the official release form with true identity, institution, and email. No automation may invent those facts. Private download URLs and licensed assets stay outside Git. -At `2026-08-26 03:17:33 +08:00`, the registered experiment server again -actively refused the registered non-default SSH port before authentication. -The next attempt may probe only that port, must verify the host key, and must -begin with a read-only system/GPU inventory. No default-port scan or substitute -host is authorized by the frozen contract. +The earlier connection-refused state was superseded by successful strict +known-host login on 2026-09-08 UTC. Only the registered port and host remain +in scope. Resource availability, numerical qualification, and actual data +access are separate observed requirements; SSH success alone is not a passed +training environment. The GitHub repository was renamed in place to `appleweiping/phaseset-multiperson-motion-language`. Its numeric repository ID, diff --git a/configs/phaseset_v2_analysis_rules.json b/configs/phaseset_v2_analysis_rules.json new file mode 100644 index 0000000..23745cd --- /dev/null +++ b/configs/phaseset_v2_analysis_rules.json @@ -0,0 +1,130 @@ +{ + "schema": "phaseset-v2-pretest-analysis-rules-v1", + "status": "DRAFT_PRE_PILOT_NO_SCORES_REVIEW_REVISION", + "decision_source": "2026-09-26 PhaseSet-V2 amendment plus 2026-09-27 relation-supervision amendment", + "scope": "rules only; not the final data, checkpoint, comparator, gallery, or score manifest", + "task": { + "primary_text_source": "original_official_human_holistic_rows_only", + "primary_motion_unit": "complete_parent_capture", + "test_parent_count_expected": 40, + "test_human_caption_rows_expected": 200, + "test_parent_group_size_counts_expected": {"3": 16, "4": 24}, + "test_parent_component_counts_expected": {"C09": 16, "C11": 8, "C15": 16}, + "positive_rule": "each official caption row is positive only for its sealed parent family", + "primary_metric": "0.5 * (capture_macro_T2M_R1 + capture_macro_M2T_any_positive_R1)", + "query_scoring": "rank each trained seed separately; never average logits across seeds", + "statistical_unit": "parent_capture_not_released_episode_or_caption", + "sealed_test_uses_machine_weak_text": false + }, + "pilot_comparator": { + "train_components": ["C01", "C02"], + "validation_components": ["C03"], + "pilot_training_seed": 1729, + "population": "all_34_C01_C02_complete_train_parents_and_all_15_C03_complete_validation_parents_official_human_rows_only", + "families": ["TMR-Set", "WaMo-Set", "MIME-Set"], + "candidate_budget_per_family": 3, + "candidate_metric": "C03_complete_parent_human_only_primary_metric", + "checkpoint_tie": "earliest_validation_checkpoint", + "within_family_exact_tie": "paper_learning_rate_then_half_then_double", + "between_family_exact_tie": "fewer_trainable_parameters_then_lower_measured_parent_latency_then_lexicographic_system_id", + "fidelity_qualification": [ + "native_complete_parent_input_and_official_human_targets", + "source_to_paper_and_original_reference_input_output_loss_crosswalk_recorded_per_family", + "original_task_sanity_fixture_runs_and_expected_input_output_loss_semantics_match_original_reference_or_method_is_unqualified", + "all_paper_essential_objective_terms_present_with_nonzero_gradients_and_documented_multiperson_adaptation", + "tiny_original_task_overfit_or_equivalent_original_reference_sanity_passes_before_pilot_selection", + "finite_losses_and_validation_scores", + "successful_terminal_or_predeclared_early_stop", + "no_training_or_selection_on_sealed_test" + ], + "selection_rule": "best_C03_primary_metric_among_qualified_candidates_then_fixed_ties", + "all_unqualified_action": "no_B_star_and_no_main_superiority_claim", + "selected_B_star": null, + "selected_candidate": null, + "selection_status": "PENDING_PILOT_NOT_CHOSEN" + }, + "v2_head_pilot_selection": { + "predecessor": "same_selected_B2_anchor_checkpoint_for_both_candidates", + "candidate_ids": ["v2-cf-half", "v2-cf-fixed"], + "candidate_metric": "C03_complete_parent_human_only_primary_metric", + "checkpoint_tie": "earliest_validation_checkpoint", + "candidate_exact_tie": "v2-cf-fixed_then_v2-cf-half", + "selection_rule": "best_C03_primary_metric_among_qualified_candidates_then_fixed_tie", + "qualification": "same_predecessor_population_steps_human_validation_and_finite_training_terminal", + "failed_both_action": "no_V2_configuration_and_no_formal_V2_launch", + "selected_candidate": null, + "status": "PENDING_PILOT_NOT_CHOSEN" + }, + "formal_comparison": { + "training_seeds": [1729, 2718, 31415], + "one_comparator_identity_for_all_seeds": true, + "missing_or_invalid_seed": "keep_terminal_and_report_incomplete; three_seed_inference_and_corresponding_claim_UNASSESSABLE; do_not_drop_replace_or_recalculate_with_remaining_seeds", + "failed_comparator": "do_not_substitute_weaker_test_chosen_method; main_claim_unassessable", + "checkpoint_selection": "development_validation_only_before_any_formal_test_score", + "terminal_exclusions": "must_be_named_and_frozen_before_formal_test; no_post_score_exclusion", + "test_passes": 1 + }, + "inference": { + "resampling_unit": "parent_capture", + "seed_reduction": "mean_three_seed_paired_parent_effects_before_bootstrap", + "draws": 100000, + "rng": "numpy_PCG64", + "resampling_seed": 20260928, + "same_parent_index_stream_for_all_comparisons": true, + "confidence": "two_sided_95_percent_percentile", + "lower_sorted_zero_based_index": 2499, + "upper_sorted_zero_based_index": 97499, + "raw_p_h1": "one_sided_bootstrap_tail_(1+count(delta_draw<=0.02))/100001", + "h1_decision_calibration": "intentional_conjunctive_conservative_gate_one_sided_p_at_0.05_and_two_sided_95_percent_lower_above_0.02; interval_condition_is_stricter_than_the_p_threshold", + "raw_p_a1_a9": "two_sided_bootstrap_tail_min(1,2*min((1+count(delta_draw<=0))/100001,(1+count(delta_draw>=0))/100001))", + "holm_family": ["A1", "A2", "A3", "A4", "A5", "A6", "A7", "A8", "A9"], + "holm_alpha": 0.05, + "scope_limit": "conditional_on_fixed_three_participant_components_models_gallery_and_seeds_not_population_generalization" + }, + "claim_gates": { + "H1": { + "comparison": "PhaseSet-V2_minus_single_pilot_frozen_B_star", + "direction": "positive", + "null_margin_pp": 2, + "point_estimate_at_least_pp": 2, + "one_sided_raw_p_at_most": 0.05, + "confidence_lower_strictly_above_margin": true, + "rule": "all_conditions_required_for_superiority_language" + }, + "A1_to_A9": { + "comparison": "PhaseSet-V2_minus_named_ablation_on_same_B2_and_human_task", + "null_margin_pp": 0, + "direction": "positive", + "raw_p_sidedness": "two_sided", + "holm_adjusted_p_at_most": 0.05, + "confidence_lower_strictly_positive": true, + "all_nine_reported_whether_or_not_any_pass": true + }, + "topology_sensitive_package": "requires_A3_and_A4_gates; does_not_isolate_incidence_from_endpoint_binding", + "phase_family": "requires_A5_gate; no_claim_of_causal_delay_or_human_relation_truth", + "weak_counterfactual_training_utility": "A8_gate_may_support_retrieval_effect_only_not_verified_motion_falsehood", + "generic_local_relation": "A7_gate_required_before_claiming_more_than_generic_local_capacity", + "calibration_only": "A9_gate_required_before_claiming_more_than_scoring_calibration", + "temporal_order": "A2_does_not_isolate_local_from_capture_order", + "relation_challenge": "UNASSESSABLE_without_matching_independent_human_labels" + }, + "secondary": { + "metrics": ["R@3", "R@5", "R@10", "Hard-32", "K_strata", "component_strata", "three_development_folds", "Inter-X"], + "three_fold_target": "positive_effect_in_each_prespecified_development_fold_descriptive_only", + "inter_x_scope": "dyadic_trained_applicability_not_Embody_zero_shot", + "nonperiodic_regression_target_pp": 1, + "nonperiodic_stratum_definition": "PENDING_PRETEST_FREEZE_OR_MARK_UNASSESSABLE", + "inference_cost_ratio_target": 2, + "cost_protocol": "PENDING_ACTUAL_GPU_PROFILE_THEN_FREEZE_HARDWARE_WORKLOAD_BATCH_REPETITIONS_P50_P95_PEAK_MEMORY_COLD_WARM", + "no_secondary_metric_replaces_primary": true + }, + "negative_result_policy": { + "H1_fails": "no_superiority_claim; deliver_honest_negative_paper_or_technical_report", + "A3_or_A4_fails": "no_topology_sensitive_package_attribution", + "A5_fails": "no_phase_family_attribution", + "A7_or_A9_fails": "no_more_than_generic_local_or_calibration_claim_respectively", + "A8_fails": "no_weak_counterfactual_utility_claim", + "machine_relation_labels": "never_independent_human_truth", + "no_new_seeds_data_or_test_tuning": true + } +} diff --git a/configs/phaseset_v2_experiment_matrix.json b/configs/phaseset_v2_experiment_matrix.json new file mode 100644 index 0000000..ffa82e4 --- /dev/null +++ b/configs/phaseset_v2_experiment_matrix.json @@ -0,0 +1,1156 @@ +{ + "schema": "phaseset-v2-training-matrix-v1", + "amendment_date": "2026-09-26", + "status": "PLANNED_NOT_LAUNCHABLE", + "supersedes": "legacy_33_run_matrix", + "seeds": [ + 1729, + 2718, + 31415 + ], + "budget": { + "gpu_hours": 300, + "formal_stages": 87, + "max_formal_starts": 90, + "max_pilot_stages": 12, + "max_architecture_revisions": 2, + "max_root_cause_restarts": 3, + "base_max_epochs": 30, + "residual_max_epochs": 20, + "pilot_hyperparameter_adjustments": 1, + "pilot_max_formal_step_fraction": 0.2, + "allowed_uniform_epoch_expansion": {"base": 60, "residual": 40}, + "epoch_expansion_must_precede_formal_freeze": true + }, + "primary_split": { + "train_released_episodes": 400, + "validation_released_episodes": 96, + "test_released_episodes": 76, + "effective_holistic_task_released_episodes": {"train": 398, "validation": 96, "test": 75}, + "sealed_test_parent_captures": 40, + "sealed_test_official_human_rows": 200, + "final_test_components": 3, + "validation_component_labels": ["C00"], + "sealed_test_component_labels": ["C09", "C11", "C15"] + }, + "pilot_and_group_fold_components": { + "pilot_train": ["C01", "C02"], + "pilot_validation": ["C03"], + "development": ["C00", "C01", "C02", "C03", "C04", "C05", "C06", "C07", "C08", "C10", "C12", "C13", "C14"], + "fold_validation": ["C03"], + "folds": [ + {"id": "fold_0", "run_split": "group_fold_1", "held_out": ["C00", "C06", "C10", "C14"], "train": ["C01", "C02", "C04", "C05", "C07", "C08", "C12", "C13"]}, + {"id": "fold_1", "run_split": "group_fold_2", "held_out": ["C04", "C07", "C12"], "train": ["C00", "C01", "C02", "C05", "C06", "C08", "C10", "C13", "C14"]}, + {"id": "fold_2", "run_split": "group_fold_3", "held_out": ["C05", "C08", "C13"], "train": ["C00", "C01", "C02", "C04", "C06", "C07", "C10", "C12", "C14"]} + ], + "reuse_main_checkpoints": false + }, + "group_fold_components_source": "User-supplied PhaseSet_Codex_Handoff_2026-09-26.md, section 5.2", + "pilot_allocation": {"TMR-Set": 3, "WaMo-Set": 3, "MIME-Set": 3, "B2": 1, "PhaseSet-V2-head": 2}, + "mechanism_contract": { + "residual_base": "B2", + "temporal_layers": 2, + "width": 512, + "local_statistics_frames": 40, + "hop_frames": 20, + "initial_relation_mixture": 0.1, + "relation_language_readout": "0.5 * ordered_capture_cosine + 0.5 * max_whole_directed_packet_cosine", + "relation_language_edge_block": 64, + "coordination_text_adapter": "shared_512_GELU_512_on_frozen_CLIP", + "human_holistic_caption_field": "all_scene_explained_rows_no_questionnaire_fusion", + "retrieval_positive_identity": "admitted_parent_annotation_family_not_released_segment_or_actor_set", + "parent_capture_task_freeze": "PRELEARNING_PARENT_SOURCE_AND_HUMAN_POSITIVE_FAMILY_SEALED_PHYSICAL_GALLERIES_AND_STATISTICS_PENDING", + "no_periodic_support": "global_branch_fallback", + "head_parameter_relative_tolerance": 0.05, + "dummy_capacity_parameters": false, + "dct_mean_signal": "(a+b)/2", + "dct_difference_signal": "a-b", + "capture_temporal_state": "consistent_actor_and_edge_tracks_before_set_readout" + }, + "frozen_strong_baseline": null, + "success_thresholds": { + "retrieval_mean_r1_gain_pp": 2, + "positive_group_folds": 3, + "positive_group_folds_role": "descriptive_development_target_not_independent_inference", + "nonperiodic_max_regression_pp": 1, + "inference_cost_ratio": 2 + }, + "historical_superseded_relation_challenge": { + "status": "UNASSESSABLE_WITHOUT_MATCHING_INDEPENDENT_HUMAN_LABELS_NOT_AN_ACTIVE_SUCCESS_GATE", + "historical_gain_pp": 5, + "minimum_verified_text_pairs": 200, + "minimum_real_captures": 100, + "minimum_participant_components": 8, + "blind_human_annotators": 2 + }, + "ablations": { + "A1": "speed_only_frontend", + "A2": "remove_local_and_capture_order", + "A3": "pair_bag", + "A4": "shuffle_explicit_incidence", + "A5": "remove_explicit_phase_fields", + "A6": "mean_difference_signal_DCT", + "A7": "generic_local_relation_matching_same_counterfactual_data", + "A8": "no_counterfactual_objective", + "A9": "legacy_representation_calibrated_score_only" + }, + "terminal_states": [ + "READY_FOR_SUBMISSION", + "COMPLETE_NOT_SUPPORTED", + "BLOCKED_DATA", + "BLOCKED_RIGHTS", + "BLOCKED_RUNTIME", + "BLOCKED_RELATIONAL_EVIDENCE", + "BLOCKED_BUDGET", + "FAILED_IMPLEMENTATION" + ], + "runs": [ + { + "id": "V2-001", + "group": "internal_base", + "split": "main", + "system": "B0", + "seed": 1729, + "max_epochs": 30, + "max_steps": null, + "depends_on": [], + "status": "PLANNED" + }, + { + "id": "V2-002", + "group": "internal_base", + "split": "main", + "system": "B0", + "seed": 2718, + "max_epochs": 30, + "max_steps": null, + "depends_on": [], + "status": "PLANNED" + }, + { + "id": "V2-003", + "group": "internal_base", + "split": "main", + "system": "B0", + "seed": 31415, + "max_epochs": 30, + "max_steps": null, + "depends_on": [], + "status": "PLANNED" + }, + { + "id": "V2-004", + "group": "internal_base", + "split": "main", + "system": "B1", + "seed": 1729, + "max_epochs": 30, + "max_steps": null, + "depends_on": [], + "status": "PLANNED" + }, + { + "id": "V2-005", + "group": "internal_base", + "split": "main", + "system": "B1", + "seed": 2718, + "max_epochs": 30, + "max_steps": null, + "depends_on": [], + "status": "PLANNED" + }, + { + "id": "V2-006", + "group": "internal_base", + "split": "main", + "system": "B1", + "seed": 31415, + "max_epochs": 30, + "max_steps": null, + "depends_on": [], + "status": "PLANNED" + }, + { + "id": "V2-007", + "group": "internal_base", + "split": "main", + "system": "B2", + "seed": 1729, + "max_epochs": 30, + "max_steps": null, + "depends_on": [], + "status": "PLANNED" + }, + { + "id": "V2-008", + "group": "internal_base", + "split": "main", + "system": "B2", + "seed": 2718, + "max_epochs": 30, + "max_steps": null, + "depends_on": [], + "status": "PLANNED" + }, + { + "id": "V2-009", + "group": "internal_base", + "split": "main", + "system": "B2", + "seed": 31415, + "max_epochs": 30, + "max_steps": null, + "depends_on": [], + "status": "PLANNED" + }, + { + "id": "V2-010", + "group": "literature_baseline", + "split": "main", + "system": "TMR-Set", + "seed": 1729, + "max_epochs": 30, + "max_steps": null, + "depends_on": [], + "status": "PLANNED" + }, + { + "id": "V2-011", + "group": "literature_baseline", + "split": "main", + "system": "TMR-Set", + "seed": 2718, + "max_epochs": 30, + "max_steps": null, + "depends_on": [], + "status": "PLANNED" + }, + { + "id": "V2-012", + "group": "literature_baseline", + "split": "main", + "system": "TMR-Set", + "seed": 31415, + "max_epochs": 30, + "max_steps": null, + "depends_on": [], + "status": "PLANNED" + }, + { + "id": "V2-013", + "group": "literature_baseline", + "split": "main", + "system": "WaMo-Set", + "seed": 1729, + "max_epochs": 30, + "max_steps": null, + "depends_on": [], + "status": "PLANNED" + }, + { + "id": "V2-014", + "group": "literature_baseline", + "split": "main", + "system": "WaMo-Set", + "seed": 2718, + "max_epochs": 30, + "max_steps": null, + "depends_on": [], + "status": "PLANNED" + }, + { + "id": "V2-015", + "group": "literature_baseline", + "split": "main", + "system": "WaMo-Set", + "seed": 31415, + "max_epochs": 30, + "max_steps": null, + "depends_on": [], + "status": "PLANNED" + }, + { + "id": "V2-016", + "group": "literature_baseline", + "split": "main", + "system": "MIME-Set", + "seed": 1729, + "max_epochs": 30, + "max_steps": null, + "depends_on": [], + "status": "PLANNED" + }, + { + "id": "V2-017", + "group": "literature_baseline", + "split": "main", + "system": "MIME-Set", + "seed": 2718, + "max_epochs": 30, + "max_steps": null, + "depends_on": [], + "status": "PLANNED" + }, + { + "id": "V2-018", + "group": "literature_baseline", + "split": "main", + "system": "MIME-Set", + "seed": 31415, + "max_epochs": 30, + "max_steps": null, + "depends_on": [], + "status": "PLANNED" + }, + { + "id": "V2-019", + "group": "main_residual", + "split": "main", + "system": "PhaseSet-v0.2-on-B2", + "seed": 1729, + "max_epochs": 20, + "max_steps": null, + "depends_on": [ + "V2-007" + ], + "status": "PLANNED" + }, + { + "id": "V2-020", + "group": "main_residual", + "split": "main", + "system": "PhaseSet-v0.2-on-B2", + "seed": 2718, + "max_epochs": 20, + "max_steps": null, + "depends_on": [ + "V2-008" + ], + "status": "PLANNED" + }, + { + "id": "V2-021", + "group": "main_residual", + "split": "main", + "system": "PhaseSet-v0.2-on-B2", + "seed": 31415, + "max_epochs": 20, + "max_steps": null, + "depends_on": [ + "V2-009" + ], + "status": "PLANNED" + }, + { + "id": "V2-022", + "group": "main_residual", + "split": "main", + "system": "PhaseSet-V2", + "seed": 1729, + "max_epochs": 20, + "max_steps": null, + "depends_on": [ + "V2-007" + ], + "status": "PLANNED" + }, + { + "id": "V2-023", + "group": "main_residual", + "split": "main", + "system": "PhaseSet-V2", + "seed": 2718, + "max_epochs": 20, + "max_steps": null, + "depends_on": [ + "V2-008" + ], + "status": "PLANNED" + }, + { + "id": "V2-024", + "group": "main_residual", + "split": "main", + "system": "PhaseSet-V2", + "seed": 31415, + "max_epochs": 20, + "max_steps": null, + "depends_on": [ + "V2-009" + ], + "status": "PLANNED" + }, + { + "id": "V2-025", + "group": "mechanism_ablation", + "split": "main", + "system": "A1", + "seed": 1729, + "max_epochs": 20, + "max_steps": null, + "depends_on": [ + "V2-007" + ], + "status": "PLANNED" + }, + { + "id": "V2-026", + "group": "mechanism_ablation", + "split": "main", + "system": "A1", + "seed": 2718, + "max_epochs": 20, + "max_steps": null, + "depends_on": [ + "V2-008" + ], + "status": "PLANNED" + }, + { + "id": "V2-027", + "group": "mechanism_ablation", + "split": "main", + "system": "A1", + "seed": 31415, + "max_epochs": 20, + "max_steps": null, + "depends_on": [ + "V2-009" + ], + "status": "PLANNED" + }, + { + "id": "V2-028", + "group": "mechanism_ablation", + "split": "main", + "system": "A2", + "seed": 1729, + "max_epochs": 20, + "max_steps": null, + "depends_on": [ + "V2-007" + ], + "status": "PLANNED" + }, + { + "id": "V2-029", + "group": "mechanism_ablation", + "split": "main", + "system": "A2", + "seed": 2718, + "max_epochs": 20, + "max_steps": null, + "depends_on": [ + "V2-008" + ], + "status": "PLANNED" + }, + { + "id": "V2-030", + "group": "mechanism_ablation", + "split": "main", + "system": "A2", + "seed": 31415, + "max_epochs": 20, + "max_steps": null, + "depends_on": [ + "V2-009" + ], + "status": "PLANNED" + }, + { + "id": "V2-031", + "group": "mechanism_ablation", + "split": "main", + "system": "A3", + "seed": 1729, + "max_epochs": 20, + "max_steps": null, + "depends_on": [ + "V2-007" + ], + "status": "PLANNED" + }, + { + "id": "V2-032", + "group": "mechanism_ablation", + "split": "main", + "system": "A3", + "seed": 2718, + "max_epochs": 20, + "max_steps": null, + "depends_on": [ + "V2-008" + ], + "status": "PLANNED" + }, + { + "id": "V2-033", + "group": "mechanism_ablation", + "split": "main", + "system": "A3", + "seed": 31415, + "max_epochs": 20, + "max_steps": null, + "depends_on": [ + "V2-009" + ], + "status": "PLANNED" + }, + { + "id": "V2-034", + "group": "mechanism_ablation", + "split": "main", + "system": "A4", + "seed": 1729, + "max_epochs": 20, + "max_steps": null, + "depends_on": [ + "V2-007" + ], + "status": "PLANNED" + }, + { + "id": "V2-035", + "group": "mechanism_ablation", + "split": "main", + "system": "A4", + "seed": 2718, + "max_epochs": 20, + "max_steps": null, + "depends_on": [ + "V2-008" + ], + "status": "PLANNED" + }, + { + "id": "V2-036", + "group": "mechanism_ablation", + "split": "main", + "system": "A4", + "seed": 31415, + "max_epochs": 20, + "max_steps": null, + "depends_on": [ + "V2-009" + ], + "status": "PLANNED" + }, + { + "id": "V2-037", + "group": "mechanism_ablation", + "split": "main", + "system": "A5", + "seed": 1729, + "max_epochs": 20, + "max_steps": null, + "depends_on": [ + "V2-007" + ], + "status": "PLANNED" + }, + { + "id": "V2-038", + "group": "mechanism_ablation", + "split": "main", + "system": "A5", + "seed": 2718, + "max_epochs": 20, + "max_steps": null, + "depends_on": [ + "V2-008" + ], + "status": "PLANNED" + }, + { + "id": "V2-039", + "group": "mechanism_ablation", + "split": "main", + "system": "A5", + "seed": 31415, + "max_epochs": 20, + "max_steps": null, + "depends_on": [ + "V2-009" + ], + "status": "PLANNED" + }, + { + "id": "V2-040", + "group": "mechanism_ablation", + "split": "main", + "system": "A6", + "seed": 1729, + "max_epochs": 20, + "max_steps": null, + "depends_on": [ + "V2-007" + ], + "status": "PLANNED" + }, + { + "id": "V2-041", + "group": "mechanism_ablation", + "split": "main", + "system": "A6", + "seed": 2718, + "max_epochs": 20, + "max_steps": null, + "depends_on": [ + "V2-008" + ], + "status": "PLANNED" + }, + { + "id": "V2-042", + "group": "mechanism_ablation", + "split": "main", + "system": "A6", + "seed": 31415, + "max_epochs": 20, + "max_steps": null, + "depends_on": [ + "V2-009" + ], + "status": "PLANNED" + }, + { + "id": "V2-043", + "group": "mechanism_ablation", + "split": "main", + "system": "A7", + "seed": 1729, + "max_epochs": 20, + "max_steps": null, + "depends_on": [ + "V2-007" + ], + "status": "PLANNED" + }, + { + "id": "V2-044", + "group": "mechanism_ablation", + "split": "main", + "system": "A7", + "seed": 2718, + "max_epochs": 20, + "max_steps": null, + "depends_on": [ + "V2-008" + ], + "status": "PLANNED" + }, + { + "id": "V2-045", + "group": "mechanism_ablation", + "split": "main", + "system": "A7", + "seed": 31415, + "max_epochs": 20, + "max_steps": null, + "depends_on": [ + "V2-009" + ], + "status": "PLANNED" + }, + { + "id": "V2-046", + "group": "mechanism_ablation", + "split": "main", + "system": "A8", + "seed": 1729, + "max_epochs": 20, + "max_steps": null, + "depends_on": [ + "V2-007" + ], + "status": "PLANNED" + }, + { + "id": "V2-047", + "group": "mechanism_ablation", + "split": "main", + "system": "A8", + "seed": 2718, + "max_epochs": 20, + "max_steps": null, + "depends_on": [ + "V2-008" + ], + "status": "PLANNED" + }, + { + "id": "V2-048", + "group": "mechanism_ablation", + "split": "main", + "system": "A8", + "seed": 31415, + "max_epochs": 20, + "max_steps": null, + "depends_on": [ + "V2-009" + ], + "status": "PLANNED" + }, + { + "id": "V2-049", + "group": "mechanism_ablation", + "split": "main", + "system": "A9", + "seed": 1729, + "max_epochs": 20, + "max_steps": null, + "depends_on": [ + "V2-007" + ], + "status": "PLANNED" + }, + { + "id": "V2-050", + "group": "mechanism_ablation", + "split": "main", + "system": "A9", + "seed": 2718, + "max_epochs": 20, + "max_steps": null, + "depends_on": [ + "V2-008" + ], + "status": "PLANNED" + }, + { + "id": "V2-051", + "group": "mechanism_ablation", + "split": "main", + "system": "A9", + "seed": 31415, + "max_epochs": 20, + "max_steps": null, + "depends_on": [ + "V2-009" + ], + "status": "PLANNED" + }, + { + "id": "V2-052", + "group": "group_held_out", + "split": "group_fold_1", + "system": "B2", + "seed": 1729, + "max_epochs": 30, + "max_steps": null, + "depends_on": [], + "status": "PLANNED" + }, + { + "id": "V2-053", + "group": "group_held_out", + "split": "group_fold_1", + "system": "PhaseSet-V2", + "seed": 1729, + "max_epochs": 20, + "max_steps": null, + "depends_on": [ + "V2-052" + ], + "status": "PLANNED" + }, + { + "id": "V2-054", + "group": "group_held_out", + "split": "group_fold_1", + "system": "pilot_frozen_strong_baseline", + "seed": 1729, + "max_epochs": 30, + "max_steps": null, + "depends_on": [], + "status": "PLANNED" + }, + { + "id": "V2-055", + "group": "group_held_out", + "split": "group_fold_1", + "system": "B2", + "seed": 2718, + "max_epochs": 30, + "max_steps": null, + "depends_on": [], + "status": "PLANNED" + }, + { + "id": "V2-056", + "group": "group_held_out", + "split": "group_fold_1", + "system": "PhaseSet-V2", + "seed": 2718, + "max_epochs": 20, + "max_steps": null, + "depends_on": [ + "V2-055" + ], + "status": "PLANNED" + }, + { + "id": "V2-057", + "group": "group_held_out", + "split": "group_fold_1", + "system": "pilot_frozen_strong_baseline", + "seed": 2718, + "max_epochs": 30, + "max_steps": null, + "depends_on": [], + "status": "PLANNED" + }, + { + "id": "V2-058", + "group": "group_held_out", + "split": "group_fold_1", + "system": "B2", + "seed": 31415, + "max_epochs": 30, + "max_steps": null, + "depends_on": [], + "status": "PLANNED" + }, + { + "id": "V2-059", + "group": "group_held_out", + "split": "group_fold_1", + "system": "PhaseSet-V2", + "seed": 31415, + "max_epochs": 20, + "max_steps": null, + "depends_on": [ + "V2-058" + ], + "status": "PLANNED" + }, + { + "id": "V2-060", + "group": "group_held_out", + "split": "group_fold_1", + "system": "pilot_frozen_strong_baseline", + "seed": 31415, + "max_epochs": 30, + "max_steps": null, + "depends_on": [], + "status": "PLANNED" + }, + { + "id": "V2-061", + "group": "group_held_out", + "split": "group_fold_2", + "system": "B2", + "seed": 1729, + "max_epochs": 30, + "max_steps": null, + "depends_on": [], + "status": "PLANNED" + }, + { + "id": "V2-062", + "group": "group_held_out", + "split": "group_fold_2", + "system": "PhaseSet-V2", + "seed": 1729, + "max_epochs": 20, + "max_steps": null, + "depends_on": [ + "V2-061" + ], + "status": "PLANNED" + }, + { + "id": "V2-063", + "group": "group_held_out", + "split": "group_fold_2", + "system": "pilot_frozen_strong_baseline", + "seed": 1729, + "max_epochs": 30, + "max_steps": null, + "depends_on": [], + "status": "PLANNED" + }, + { + "id": "V2-064", + "group": "group_held_out", + "split": "group_fold_2", + "system": "B2", + "seed": 2718, + "max_epochs": 30, + "max_steps": null, + "depends_on": [], + "status": "PLANNED" + }, + { + "id": "V2-065", + "group": "group_held_out", + "split": "group_fold_2", + "system": "PhaseSet-V2", + "seed": 2718, + "max_epochs": 20, + "max_steps": null, + "depends_on": [ + "V2-064" + ], + "status": "PLANNED" + }, + { + "id": "V2-066", + "group": "group_held_out", + "split": "group_fold_2", + "system": "pilot_frozen_strong_baseline", + "seed": 2718, + "max_epochs": 30, + "max_steps": null, + "depends_on": [], + "status": "PLANNED" + }, + { + "id": "V2-067", + "group": "group_held_out", + "split": "group_fold_2", + "system": "B2", + "seed": 31415, + "max_epochs": 30, + "max_steps": null, + "depends_on": [], + "status": "PLANNED" + }, + { + "id": "V2-068", + "group": "group_held_out", + "split": "group_fold_2", + "system": "PhaseSet-V2", + "seed": 31415, + "max_epochs": 20, + "max_steps": null, + "depends_on": [ + "V2-067" + ], + "status": "PLANNED" + }, + { + "id": "V2-069", + "group": "group_held_out", + "split": "group_fold_2", + "system": "pilot_frozen_strong_baseline", + "seed": 31415, + "max_epochs": 30, + "max_steps": null, + "depends_on": [], + "status": "PLANNED" + }, + { + "id": "V2-070", + "group": "group_held_out", + "split": "group_fold_3", + "system": "B2", + "seed": 1729, + "max_epochs": 30, + "max_steps": null, + "depends_on": [], + "status": "PLANNED" + }, + { + "id": "V2-071", + "group": "group_held_out", + "split": "group_fold_3", + "system": "PhaseSet-V2", + "seed": 1729, + "max_epochs": 20, + "max_steps": null, + "depends_on": [ + "V2-070" + ], + "status": "PLANNED" + }, + { + "id": "V2-072", + "group": "group_held_out", + "split": "group_fold_3", + "system": "pilot_frozen_strong_baseline", + "seed": 1729, + "max_epochs": 30, + "max_steps": null, + "depends_on": [], + "status": "PLANNED" + }, + { + "id": "V2-073", + "group": "group_held_out", + "split": "group_fold_3", + "system": "B2", + "seed": 2718, + "max_epochs": 30, + "max_steps": null, + "depends_on": [], + "status": "PLANNED" + }, + { + "id": "V2-074", + "group": "group_held_out", + "split": "group_fold_3", + "system": "PhaseSet-V2", + "seed": 2718, + "max_epochs": 20, + "max_steps": null, + "depends_on": [ + "V2-073" + ], + "status": "PLANNED" + }, + { + "id": "V2-075", + "group": "group_held_out", + "split": "group_fold_3", + "system": "pilot_frozen_strong_baseline", + "seed": 2718, + "max_epochs": 30, + "max_steps": null, + "depends_on": [], + "status": "PLANNED" + }, + { + "id": "V2-076", + "group": "group_held_out", + "split": "group_fold_3", + "system": "B2", + "seed": 31415, + "max_epochs": 30, + "max_steps": null, + "depends_on": [], + "status": "PLANNED" + }, + { + "id": "V2-077", + "group": "group_held_out", + "split": "group_fold_3", + "system": "PhaseSet-V2", + "seed": 31415, + "max_epochs": 20, + "max_steps": null, + "depends_on": [ + "V2-076" + ], + "status": "PLANNED" + }, + { + "id": "V2-078", + "group": "group_held_out", + "split": "group_fold_3", + "system": "pilot_frozen_strong_baseline", + "seed": 31415, + "max_epochs": 30, + "max_steps": null, + "depends_on": [], + "status": "PLANNED" + }, + { + "id": "V2-079", + "group": "external_dyadic", + "split": "Inter-X", + "system": "B2", + "seed": 1729, + "max_epochs": 30, + "max_steps": null, + "depends_on": [], + "status": "PLANNED" + }, + { + "id": "V2-080", + "group": "external_dyadic", + "split": "Inter-X", + "system": "PhaseSet-V2", + "seed": 1729, + "max_epochs": 20, + "max_steps": null, + "depends_on": [ + "V2-079" + ], + "status": "PLANNED" + }, + { + "id": "V2-081", + "group": "external_dyadic", + "split": "Inter-X", + "system": "MIME-original", + "seed": 1729, + "max_epochs": 30, + "max_steps": null, + "depends_on": [], + "status": "PLANNED" + }, + { + "id": "V2-082", + "group": "external_dyadic", + "split": "Inter-X", + "system": "B2", + "seed": 2718, + "max_epochs": 30, + "max_steps": null, + "depends_on": [], + "status": "PLANNED" + }, + { + "id": "V2-083", + "group": "external_dyadic", + "split": "Inter-X", + "system": "PhaseSet-V2", + "seed": 2718, + "max_epochs": 20, + "max_steps": null, + "depends_on": [ + "V2-082" + ], + "status": "PLANNED" + }, + { + "id": "V2-084", + "group": "external_dyadic", + "split": "Inter-X", + "system": "MIME-original", + "seed": 2718, + "max_epochs": 30, + "max_steps": null, + "depends_on": [], + "status": "PLANNED" + }, + { + "id": "V2-085", + "group": "external_dyadic", + "split": "Inter-X", + "system": "B2", + "seed": 31415, + "max_epochs": 30, + "max_steps": null, + "depends_on": [], + "status": "PLANNED" + }, + { + "id": "V2-086", + "group": "external_dyadic", + "split": "Inter-X", + "system": "PhaseSet-V2", + "seed": 31415, + "max_epochs": 20, + "max_steps": null, + "depends_on": [ + "V2-085" + ], + "status": "PLANNED" + }, + { + "id": "V2-087", + "group": "external_dyadic", + "split": "Inter-X", + "system": "MIME-original", + "seed": 31415, + "max_epochs": 30, + "max_steps": null, + "depends_on": [], + "status": "PLANNED" + } + ] +} diff --git a/configs/phaseset_v2_relation_supervision.json b/configs/phaseset_v2_relation_supervision.json new file mode 100644 index 0000000..7ee6df1 --- /dev/null +++ b/configs/phaseset_v2_relation_supervision.json @@ -0,0 +1,48 @@ +{ + "schema": "phaseset-v2-relation-supervision-amendment-v1", + "amendment_date": "2026-09-27", + "status": "WEAK_LOSS_AND_POOL_SOFTWARE_QUALIFIED_GENERATION_PENDING_NOT_TRAINING_ADMISSION", + "reason": "No two new human annotators are available; the project owner requested existing annotations or LLM assistance instead of waiting.", + "supersedes": "The requirement to obtain two new human annotators before independent development or training preparation can proceed.", + "primary_retrieval": { + "positive_source": "original_official_human_holistic_rows", + "machine_text_in_primary_gallery": false, + "identity": "admitted_parent_annotation_family", + "sealed_test_unchanged": true + }, + "counterfactual_training": { + "preferred_source": "existing_explicit_relation_annotations_if_their_label_semantics_and_rights_match", + "fallback_source": "machine_caption_contradiction", + "fallback_role": "weak_training_supervision_not_verified_motion_truth", + "model": "gpt-5.6-sol", + "inputs": "official_human_text_of_the_same_training_parent_only", + "forbidden_inputs": ["model_scores", "motion_features", "phase", "held_out_or_final_test_text"], + "same_pool_for_full_and_applicable_controls": true, + "no_new_data_or_training_slots": true, + "human_verified_false_from_machine_labels": false, + "implementation_requirement": "Separate weak-label admission and counts; never set the existing human-only verified_false bit merely because an LLM supplies a label.", + "prompt_and_selection_freeze": "before_the_relevant_pilot_or_training_start" + }, + "relation_evaluation": { + "two_new_human_annotators_required_to_continue": false, + "machine_assisted_challenge_role": "exploratory_separate_from_primary_retrieval", + "old_200_pair_100_capture_8_component_target_met_by_llm": false, + "old_human_challenge_5pp_criterion": "UNASSESSABLE_WITHOUT_MATCHING_INDEPENDENT_LABELS", + "existing_positive_caption_is_proof_of_false_counterfactual": false, + "machine_accuracy_is_independent_ground_truth": false + }, + "claim_limits": [ + "Report primary retrieval against the unchanged official human labels.", + "CF-versus-no-CF retrieval effects may support the utility of weak training supervision, not verified relation understanding.", + "Do not claim human-verified relational challenge success or grounding from machine labels.", + "Do not mark the historical all-mechanism success target achieved by substituting an LLM for the missing humans." + ], + "unchanged": { + "formal_stages": 87, + "max_formal_starts": 90, + "pilot_stages": 12, + "seeds": [1729, 2718, 31415], + "gpu_hours": 300, + "component_splits_and_budget_history": true + } +} diff --git a/configs/phaseset_v2_venue.json b/configs/phaseset_v2_venue.json new file mode 100644 index 0000000..4fdb4c2 --- /dev/null +++ b/configs/phaseset_v2_venue.json @@ -0,0 +1,22 @@ +{ + "schema": "phaseset-v2-venue-amendment-v1", + "amendment_date": "2026-09-27", + "user_selected_venue": "ACM ICMR 2027", + "track": "Regular long research paper", + "status": "PLANNED_NOT_SUBMITTED", + "deadline": "2027-01-31T23:59:59-12:00", + "review": "double-blind", + "latex_class": "acmart", + "latex_options": ["sigconf", "review", "anonymous"], + "maximum_body_pages": 8, + "maximum_reference_only_pages": 2, + "appendices_allowed": false, + "supplementary_material_allowed": false, + "official_submission_portal_status_at_check": "More details will be announced soon", + "checked_on": "2026-09-27", + "official_sources": ["https://www.icmr2027.org/date", "https://www.icmr2027.org/cfc"], + "scientific_matrix_changed": false, + "compute_budget_changed": false, + "public_repository_anonymity": "Public development continues; the anonymous submission must not identify authors or link to identifying websites. Keep a separate submission artifact.", + "submission_authority": false +} diff --git a/docs/BASE_COHORT_QUALIFICATION.md b/docs/BASE_COHORT_QUALIFICATION.md new file mode 100644 index 0000000..6707f41 --- /dev/null +++ b/docs/BASE_COHORT_QUALIFICATION.md @@ -0,0 +1,162 @@ +# Base cohort latency and qualification + +The latency runner and deterministic assembler extend the +[completed-base resolver and capture scorer](BASE_COHORT_VALIDATION.md). +They implement a software path from nine completed training chains to the +existing qualification selector. No real nine-run cohort, winner or latency +benchmark has yet been observed. + +## One complete latency session + +`run_base_cohort_latency_session` reuses the admitted, validation-selected +checkpoint bytes for all three bases and all three seeds. It checks the +installed source, selected state, runtime, validation census and live parameter +census against the resolver and scorer. It does not accept supplied timings +or parameter counts. + +The frozen workload is an analytic K=32, T=200, body-22 group at the registered +512-dimensional model width. It is an engineering timing workload, not a +replacement dataset. Nine cyclic rounds visit all nine rows: 81 visits in +total. Each visit has five warmups and eleven synchronized timed forwards, +giving 99 samples per row. Output copying and finite-value checking are inside +the defined forward interval; progress-journal I/O is outside it. The statistic +is the exact integer median of all 99 positive nanosecond samples. + +The runner records the actual CUDA UUID, resolved device, runtime and precision. +Its shared-device policy requires at least 8 GiB free and a 2 GiB allocator cap. +An admission, resource, identity or execution failure holds the entire session. +It does not discard slow rows, lower K, remove layers, switch checkpoints or +select a quiet subset of visits. The controller must predeclare and freeze an +appropriate outer wall timeout before the actual session. A short functionality +probe's timeout is not the formal session timeout. Existing unrelated device +workloads are not interrupted. + +The injected runtime seam is explicitly nonformal and exists for contract +tests. It cannot produce an accepted actual-runtime qualification merely by +finishing the same number of loops. + +An actual registered-server diagnostic found that NVIDIA returned the complete +`GPU-`-prefixed UUID while Torch's CUDA UUID object rendered the same payload +without that prefix. The runtime now accepts only those two complete canonical +lowercase representations from Torch and compares the resulting full UUID +exactly against both registration and the NVIDIA observation. Short forms, +whitespace, uppercase payloads, repeated prefixes, MIG identifiers and changed +payloads are rejected; this conversion cannot select or change the device. +Tests prove invalid identities fail before allocator setup or tensor creation, +and accepted representations preserve the original allocator cap. The original +failed K32 observations remain retained; this identity correction alone is not +a completed K32 model run or a formal latency session. + +The following actual K32 retry passed device initialization but failed the +strict zero-allocation check when releasing B0. No completed model row was +returned. Its failure and unchanged-source receipts remain retained; the +following diagnosis and correction did not change the workload or limit. + +A separate actual diagnostic then found that B0's warmup and observed forward +returned without an earlier exception. All model parameters and buffers were +on CPU, but one 32 MiB process-local cuBLAS workspace remained. Garbage +collection did not remove it; the installed cuBLAS workspace-clear API reduced +both allocated and reserved memory to zero. That diagnostic still rethrew the +original failure and did not produce a completed latency row. + +The release path now explicitly synchronizes and clears this process's cuBLAS +workspaces before emptying the allocator cache and applying the original exact +zero guard. Missing, noncallable or failing cleanup APIs raise a typed error; +real remaining allocations still reject release. Device selection, the 2 GiB +cap, the 8 GiB threshold, warmups, timing boundaries and visit order are +unchanged. This cleanup does not stop or alter any other process. + +The corrected release path, capture lineage and window cache plan passed 157 +focused and 1285 full server tests, two existing skips and 39 subtests in +512.46 seconds, with source unchanged. A subsequent actual K32 GPU retry +completed B0, B1 and B2 using untrained registered-width models, T=200, one +warmup and one observed FP32 forward each. All outputs were finite CUDA +`[1,512]` tensors, model state was unchanged and each production release passed +the unchanged exact-zero allocated/reserved guard. + +| Model | Peak allocated bytes | Peak reserved bytes | +| --- | ---: | ---: | +| B0 ActorMean | 81,548,800 | 102,760,448 | +| B1 SetPMA | 94,160,384 | 119,537,664 | +| B2 SocialTemporal | 315,323,392 | 375,390,208 | + +These are bounded diagnostic resource observations under the original 2 GiB +cap and 8 GiB free-memory admission. They do not constitute the formal +81-visit/99-samples-per-row session, latency qualification rows, trained +checkpoints or retrieval-quality results. No diagnostic duration is used as a +qualification tie-break. Every earlier failed attempt remains retained. + +The integrated UUID/source-lineage/text-boundary CPU regression passed 136 +focused tests and 1257 full tests, two existing skips and 39 subtests in +413.42 seconds with source unchanged. Its CUDA API stand-ins are explicitly +software tests, not a substitute for the separate actual device observation. + +## Qualification assembly + +`assemble_base_cohort_qualification` accepts the resolved cohort, admission, +complete capture-scoring result and complete actual latency session. It accepts +no submitted score rows, winner, timing samples or evaluator digest. It checks +the complete row order and binds each terminal, selected checkpoint, selected +state, query census, precision, runtime and installed-source identity across +the three observations. + +Each `BaseScore` obtains its exact `Fraction` metric from the capture scorer, +its parameter count from matching scorer/latency live censuses, and its latency +from recomputing the retained 99-sample median. Its combined artifact digest +binds both the score and latency observations. The evaluator identity is the +installed production qualification artifact, not an unrelated module digest. + +The assembler calls the existing `experiments.qualify_base` and production +canonical verifier. Selection remains: highest exact three-seed mean primary +score; then fewer parameters, lower three-seed median latency, and smaller base +ID. It does not implement a second competing selection rule. Selected bytes +and scoped source dependencies are checked for stability around assembly. + +The result remains a software artifact with `authority=0`, `production=False` +and `result_claimed=False`. The trusted private controller must use actual +independently retained execution evidence through the existing backend flow. +This does not introduce an extra user-signature requirement, turn hashes into +data rights, or authorize access to sealed test data. + +## Private host composition + +The [host qualification controller](HOST_BASE_QUALIFICATION.md) implements the +private orchestration seam from the existing `qualify-base` CLI request to this +resolver, scorer, latency runner, journal and assembler. The CLI still supplies +only the attempt root; it cannot submit a winner, score, timing, parameter +count, checkpoint choice or authorization object. + +Before the scorer places a model on CUDA, the controller applies the same +registered-device identity, minimum free-memory and allocator-cap policy used +by latency. After scoring it releases scorer references, synchronizes, clears +this process's cuBLAS workspaces, empties the allocator cache and retains the +exact-zero guard before starting the independent latency session. A scoring +failure remains primary even if cleanup also fails, and resource causes remain +HOLD outcomes rather than being converted into model-quality failures. + +After a complete journal-agreeing latency result, the controller calls the +existing assembler, retains its exact bytes and lets the existing production +adapter recompute them before the backend callback. The controller creates no +new scientific selector or human-authorization mechanism. + +## Progress, verification and remaining work + +The optional typed observer connects to the +[durable progress journal](LATENCY_PROGRESS_JOURNAL.md). A journal terminal is +not a substitute for the actual latency call outcome and its receipt. + +The latency contract passed 45 focused server tests; qualification assembly +passed a 53-test combined suite. The final observer/journal integration passed +73 tests in 22.63 s. The merged source, also containing the descriptor cache, +passed 1211 tests, two existing skips and 39 subtests in 362.75 s on the Linux +experiment server, with its complete source census unchanged. These counts +overlap and must not be added as independent experiments. + +Tests use explicit software fixtures and injected observations. No actual +81-visit CUDA timing session, real completed training cohort, qualified winner +or data-dependent host `qualify-base` completion is claimed. The host controller +passed 131 focused server tests in 37.07 seconds with source unchanged, +including the real CLI-to-backend path and a structurally supplied qualification +callback. The first attempt's 123 passes and seven fixture API failures remain +retained. Its correction changed only the test file, not production guards. +These software tests did not produce real nine-run inputs or a qualified winner. diff --git a/docs/BASE_COHORT_VALIDATION.md b/docs/BASE_COHORT_VALIDATION.md new file mode 100644 index 0000000..6d2cce9 --- /dev/null +++ b/docs/BASE_COHORT_VALIDATION.md @@ -0,0 +1,90 @@ +# Completed-base cohort and capture validation + +The completed-base resolver and capture scorer connect the registered training +ledger to actual validation-selected model states. They do not select a winner +without the separately required complete latency session and qualification +assembly, and they do not establish data rights. + +## Completed attempts, latest checkpoints and selected checkpoints + +`resolve_completed_base_cohort` consumes the trusted host's attempt registry +and admitted plan, matrix, installed-source, runtime and data bindings. It +requires exactly one successful chain for each registered B0/B1/B2 by seed +row. Failed and held predecessors remain in the validated resume chain rather +than becoming additional score rows. + +The terminal-bound latest checkpoint is not automatically the best checkpoint. +The resolver authenticates and decodes that payload, follows its exact best +filename and digest, and independently validates the selected payload. It +retains immutable owned bytes rather than returning a mutable checkpoint path. +Latest and selected checkpoints must agree on best metric and schedule; the +selected validation cursor and cumulative counters cannot exceed the latest +state. Selected-best-at-write evidence alone is not a completed-run receipt. + +The checks reuse the actual registered constructors and checkpoint validators: +closed headers, full model state, optimizer, scheduler, RNG, cursor, seed-bound +initialization, code, environment and behavior. Bounded no-follow regular-file +reads precede restricted CPU checkpoint decoding. The recursive attempt chain +is rejected before a 65th entry, and checkpoint directory iteration has a fixed +100,000-entry bound. A single payload is bounded at 2 GiB and all retained +selected payloads at 4 GiB. Resource rejection never silently drops a base row. + +## Actual complete-capture scoring + +`score_resolved_base_cohort` accepts those resolved bytes and one concrete +validation-only `CaptureValidationSource`. Its full census must equal the +checkpoint's admitted validation binding. This census includes the upstream +manifest, complete window plan, numeric inputs, text receipts and identities; +the upstream manifest digest remains separately identified. + +For every registered row, the scorer revalidates the selected bytes, constructs +the real seed-bound base, strictly loads its complete state, verifies the live +state and behavior, and derives the full trainable parameter census including +`logit_scale`. There is no random-checkpoint fallback, caller-supplied factory, +parameter count or score input. Each architecture's parameter rows must agree +across all three seeds. + +The existing audited [capture evaluator](CAPTURE_VALIDATION.md) encodes every +accepted window, pools unnormalized embeddings per capture and evaluates one +complete motion/text gallery. The scorer independently checks gallery IDs, +positive relations, group sizes, component labels, score bytes and exact +capture-macro R@1 fractions. It checks source census and selected-byte stability +before and after scoring. One failed row prevents a complete cohort result. + +The complete float64 score matrix has a 256 MiB preflight bound. No capture, +caption, window, actor or edge is sampled to fit this bound. Returned score +bytes are immutable little-endian float64; canonical observation bytes bind +their digest to the actual selected checkpoint, terminal, runtime and query +census. These private query identities and scores are not automatically safe +to publish. + +## Acceptance boundary + +These APIs emit software observations, not a completed qualification or new +execution authority. The trusted controller must still combine actual nine-run +terminals, scores and parameter censuses with one complete frozen latency +session, then call the existing selector and production evidence assembly. +No caller may authenticate a qualification merely by submitting matching +hashes. The exact formal training-caption policy must also be clarified before +real training; the scorer does not choose that policy or access sealed test. + +Server contract tests cover restricted checkpoint state validation, exact row +ordering, latest/best linkage, immutable inputs, actual score-derived metrics, +live parameter counts and malformed dependencies. Explicit analytic fixtures +are not real training attempts or evidence of retrieval quality. In the +observed integrated candidate, 119 focused and 1119 complete Linux server +tests passed, with two existing skips and 39 subtests; source comparisons +passed. A separate three-lane server witness subsequently passed for actual +registered width-512 B0/B1/B2. Without replacing constructors, checkpoint +validators, encoders, pooling or scoring, it loaded complete selected model +states and scored two analytic T200 captures with K2/K3. The checkpoint cursor +and single initialized optimizer-state row were explicit software fixtures, +not formal epochs or a completed attempt ledger. Its text embeddings were +analytic unit vectors, not captions encoded by CLIP. All three tests passed in +19.36 seconds with source unchanged. This tests numeric integration only; +later cross-platform/public-package checks retain their independent scope. +The merged source, including that actual-width witness and the previously +verified LF-only host fixtures, subsequently passed 1124 complete Linux server +tests, two existing skips and 39 subtests in 315.67 seconds. The full source +census was unchanged across the run. This merged regression does not turn the +analytic checkpoints into completed formal training attempts. diff --git a/docs/CACHED_RESIDUAL_TRAINING.md b/docs/CACHED_RESIDUAL_TRAINING.md new file mode 100644 index 0000000..59872d6 --- /dev/null +++ b/docs/CACHED_RESIDUAL_TRAINING.md @@ -0,0 +1,55 @@ +# Cached residual training and resume + +The residual lifecycle accepts an explicitly admitted +`PeriodicDescriptorCaptureTrainingPlan` for systems 02/03/04/06/07/08. It does +not autodetect caches, substitute a window-level validation task for the full +capture gallery, or fall back to uncached descriptors on a miss. + +Pass the same exact plan to both +`construct_registered_residual_seed_bound_system(..., descriptor_plan=plan)` +and `PhaseSetTrainingRuntime(..., descriptor_plan=plan)`. Admission checks its +system, official seed, formal twenty-epoch configuration, edge budget, energy +floors, train manifest and complete holistic capture census. Systems 01/05 and +base systems do not accept descriptor plans. + +Every gradient-cache first pass and differentiable replay reopens the current +batch through `plan.open_training_batch(batch)` and calls +`encode_trainable_cached`. Readers, streams and learned activations are not +stored in checkpoints. A replay error restores the post-objective RNG state +and propagates before an optimizer or scheduler step; no failed cache read is +replaced by recomputation. Holistic validation calls the shared +`run_capture_validation_cached` implementation for every accepted window. + +## Serialized compatibility + +Uncached initialization, checkpoint and report schemas and bytes remain v1; +they do not gain nullable cache fields. Cached execution uses additive schemas: + +- `phaseset-training-initialization-binding-v2-cached-descriptor-plan` +- `phaseset-training-checkpoint-v2-cached-descriptor-plan` +- `phaseset-training-report-v2-cached-descriptor-plan` + +The cached checkpoint binds the complete plan digest and a dataloader digest +over the train manifest, holistic capture census and that plan. Resume must +reconstruct the same live plan and revalidate those identities before restoring +state. Cached and uncached checkpoints are not implicitly migrated into one +another. Paths, descriptor payloads, reader objects and stream cursors are +excluded from serialization. + +## Evidence and remaining integration + +The combined registered-server software cohort passed 270 focused tests in +429.82 seconds with source unchanged. It includes real analytic cache writers, +readers and plan admission, small-model lifecycle tests, failure injection and +the platform corrections. The official frozen CLIP adapter also retained its +two exact embedding goldens and unchanged RNG after the training-source pin +was updated. The same source then passed 1360 full Linux tests, 2 existing +skips and 39 subtests in 743.26 seconds, with source unchanged. These +observations are not formal model-quality results. + +Full-width cached numerical equivalence and host-level plan/configuration and +checkpoint-ledger injection are separate work. The current CLI host does not +yet dispatch this cached lifecycle end to end. Native data, fitted scientific +floors, the nine qualified bases and the formal residual runs remain required. +See [cache construction](PERIODIC_CACHE_EXECUTION.md) and +[complete plan admission](PERIODIC_CAPTURE_TRAINING_CACHE.md). diff --git a/docs/CAPTION_ROLE_PROVENANCE.md b/docs/CAPTION_ROLE_PROVENANCE.md new file mode 100644 index 0000000..0d2bd28 --- /dev/null +++ b/docs/CAPTION_ROLE_PROVENANCE.md @@ -0,0 +1,72 @@ +# Window-caption provenance and frozen-text chunk coverage + +`phaseset_core.caption_role_provenance` joins the content records that already +exist during private preparation: same-window human actor descriptions, +machine-fused description/paraphrase outputs, ordered frozen CLIP receipt rows, +and the materialized prepared-data v2 training census. It does not run a fusion +model, select a scientific supervision policy, authorize data, or train motion +models. + +## Content binding + +`validate_candidate_machine_fused_window_training(...)` checks the proposed +window-supervision mapping while original text is available. Each admitted +window has its own positive family, with the fused description followed by +its paraphrase. Human input text/source digests, actor membership, group/source +lineage, window start, fusion provenance, exact UTF-8 output digests, CLIP row +order and embedding bytes must agree. Canonical actor ordering after storage +is accepted without changing membership. The complete reloaded train census +must contain each bound motion and caption exactly once. + +The returned private digest-only receipt retains machine-fused/non-human roles. +It is not a claim that a language model preserved semantics, nor a data-rights +certificate. Commitments can remain linkable and must stay private. A trusted +preparation process supplies already verified source summaries; caller-supplied +digests alone are not authentication. Runtime source bytes must be independently +verified before import, not inferred from a static review record. + +The candidate is bounded to 16,384 windows, 32,768 caption rows, and 64 MiB of +inspected float32 text embeddings per representation. This is accounting after +objects or batches are materialized, not an aggregate allocator cap. The +validator never samples the census to fit a bound. + +## Actual CLIP integration correction + +The frozen adapter's `receipt.batch_size` is its inference chunk size, not its +total caption count. Prepared-data v2 now checks an exact integer chunk size +within the existing adapter bound and reconstructs the exact contiguous +`chunk_ranges` over every caption row. A chunk size larger than the census is +valid and yields one shorter chunk. Missing, reordered, overlapping or oversized +ranges and Boolean/malformed sizes are rejected. Existing row, shape, stride, +byte, full/individual embedding digest and positive-family checks remain. + +A registered-server probe using the actual official CLIP weights first exposed +the old false equality after encoding four texts in two chunks. Its failure +was retained. After the correction, the unchanged probe encoded four training +fixture texts and two separate auxiliary-validation fixture texts in three +actual chunks, wrote and reloaded the prepared tree, and reproduced the exact +provenance receipt without another encode call. Observed CPU Torch RNG was +unchanged. All 53 focused server tests passed with source bytes unchanged. +A subsequent complete Linux server regression passed 1,155 tests, two existing +skips and 39 subtests in 316.45 seconds; its source census was also unchanged. + +The texts, fusion receipts and analytic motion in this probe were explicit +software fixtures. No fusion backend, licensed dataset, motion-model training, +real cohort, retrieval-quality metric or formal supervision choice was involved. + +## Scientific boundary + +This validator implements a candidate hybrid: machine-fused same-window +description/paraphrase supervision for window InfoNCE, with separate official +human holistic capture validation/test. The candidate has not amended the +frozen experiment contract. It must not silently turn auxiliary captions into +human primary labels or copy whole-capture descriptions onto individual windows. +All systems/seeds must share whichever policy is explicitly preregistered +before real training. Primary capture pooling and checkpoint selection remain +the existing [capture-validation contract](CAPTURE_VALIDATION.md). + +This is a preparation component, not the complete `prepare-data` command. +Licensed native inputs, participant-disjoint source preparation, actual fusion +execution and full private command composition remain required. See +[prepared-data v2](PREPARED_DATA_V2.md) and +[the frozen text adapter](FROZEN_CLIP_TEXT_ADAPTER.md). diff --git a/docs/CAPTURE_PREPARED_STORAGE.md b/docs/CAPTURE_PREPARED_STORAGE.md new file mode 100644 index 0000000..61eed6f --- /dev/null +++ b/docs/CAPTURE_PREPARED_STORAGE.md @@ -0,0 +1,88 @@ +# Prepared capture-validation storage + +`phaseset_core.capture_prepared_storage` persists an already admitted, +validation-only `CaptureValidationSource` without discarding windows or +re-encoding text. It preserves full chronological 200-frame body22 windows, +original actor padding, variable caption counts, and each original frozen-CLIP +feature batch and canonical encoding receipt. + +```python +from phaseset_core.capture_prepared_storage import ( + load_capture_validation_source, + write_capture_validation_source, +) + +written = write_capture_validation_source(new_absolute_private_root, source) +restored = load_capture_validation_source( + written.manifest_path, + expected_manifest_sha256=written.manifest_sha256, +) +assert restored.census_sha256 == source.census_sha256 +``` + +The caller must retain the expected manifest digest through its independently +authenticated preparation record. Reading a digest from the same untrusted +artifact is not authentication. The writer requires a new root; existing +artifacts are not overwritten. + +## Descriptor source lineage in v2 + +`write_capture_validation_source_v2` writes the additive v2 schema. The same +loader accepts both versions. V1 remains the default writer and restores no +descriptor lineage; it rejects attempts to discard existing v2 lineage. + +For each v2 window, the loader derives `CaptureDescriptorWindowSource` from +the exact consumed storage-manifest digest, the verified window NPZ digest, +and its canonical global ordinal. Global ordinals cover all windows in sorted +capture-commitment and capture-plan order. They are neither capture-local +positions nor source-frame starts. + +`window.descriptor_context(seed=...)` produces that window's val/epoch-zero/ +positive-zero-yaw context. The source rejects mixed v1/v2 lineage, multiple +storage manifests and any noncanonical ordinal census. The upstream scientific +manifest and complete source census retain their existing meanings; the +storage-manifest binding is separate to avoid a circular digest. + +The server integration passed 92 focused and 1265 full tests, two existing +skips and 39 subtests, with source unchanged. These are storage/software +fixtures, not access approval, human annotation verification or a real cache. + +## Existing storage and provenance guarantees + +The manifest binds exact capture/window/caption counts, complete window plans, +the upstream manifest and source census, every numeric artifact, original text +receipts, caption lineage, output feature hashes and embedding cache keys. +Window arrays are deterministic uncompressed NPZ/NPY1.0, with no pickle or +object arrays. Text features and receipt JSON are stored separately. + +The reader hashes and decodes the same bounded byte buffers. It checks closed +canonical JSON, file and directory census, regular non-symlink file identity, +stable opened-descriptor metadata, ZIP metadata and CRC, NPY headers and payload +lengths, exact dtypes/shapes/masks, and final reconstructed source census. +Header-based logical materialized-byte limits are enforced before NumPy loads +the arrays. Resource failures reject the complete source; they never sample +windows, people or captions. These limits are not a total-process-RSS bound. +Root ancestors must be trusted and stable; the loader does not claim +descriptor-relative protection against hostile writable ancestors. + +Frozen text restoration calls only +`rehydrate_frozen_clip_text_batch(embeddings, receipt_json_bytes=..., +expected_receipt_sha256=...)` in the owning text module. That API verifies the +complete original canonical receipt, full/per-row feature bytes, fixed model +and tokenizer identity, historical encoding source/runtime manifests, +caption lineage, chunk ranges and cache key before reconstructing the owned +batch. It does not load CLIP or perform inference. Historical training/adapter +source identities remain historical; live adapter loading retains its own +strict current source pins. + +This is private data storage, not an anonymization or data-release format. +Capture, actor, group, caption and component commitments can remain linkable. +The upstream controller must ensure component labels contain no identifying +text. Stored bodies, features, captions, operational paths and receipts do not +belong in the public repository. + +Storage integrity does not establish data rights, participant-disjointness, +official holistic annotation identity, model quality, or permission to open +the sealed test split. This API accepts validation only. Cross-source training +and validation admission belongs to the host and split-audit layer; a pair of +individually valid sources must not be assumed disjoint. diff --git a/docs/CAPTURE_VALIDATION.md b/docs/CAPTURE_VALIDATION.md new file mode 100644 index 0000000..29e1099 --- /dev/null +++ b/docs/CAPTURE_VALIDATION.md @@ -0,0 +1,156 @@ +# Holistic capture validation and training selection + +`capture_pooling.py` and `capture_validation.py` implement the main task's +numeric model-to-gallery path. `PhaseSetTrainingRuntime.fit(train_source, +capture_validation_source)` uses it for validation checkpoint selection. The +private host can load the on-disk capture source, but upstream licensed-data +preparation and provenance remain separate work; this is not a declaration of +data readiness. + +## Complete capture inputs + +`CaptureWindowPlan` binds a separate capture identity, distinct accepted-window +identities, and chronological source starts on the 300-frame/30-fps grid. +Upstream preparation must freeze the complete accepted nonoverlapping window +census before model outputs exist. Rejected windows and tails belong in its +audit; a plan cannot be assembled from whichever outputs happened to finish. + +`CaptureValidationWindow` supplies one complete body-22 window as B=1, +200-frame `PreparedGroupBatch`. All windows of a capture must preserve its +valid actor set, group commitment, and K. Padding and input actor positions may +differ. Several distinct captures may legitimately share the same actor set; +actor-set group commitments are never used as unique capture or window row IDs. + +`CaptureValidationCapture` combines the complete plan/windows with an exact +`FrozenClipTextBatch` and its participant-component label. Its text receipt +binds the actual feature bytes, but cannot establish that unseen text came +from an official human holistic annotation. That provenance and data rights +must be supplied by the private source audit. Variable positive text counts are +supported; no fixed three-caption assumption is introduced. + +`CaptureValidationSource` is val-only and requires at least two captures. +It canonicalizes capture and window order and checks duplicate/missing/foreign +identities. Test cannot be relabeled through a split argument. Source plans, +commitments, embeddings, captions, and component membership remain private. + +Optional [v2 storage lineage](CAPTURE_PREPARED_STORAGE.md) binds every window +to its actual consumed manifest/NPZ and canonical global ordinal. A source +must contain either no such lineage or a complete, single-manifest sequence. +It does not change capture pooling, text order or gallery semantics, and it +does not by itself connect descriptor caching to this evaluator. The separate +prepared-window cache plan cannot replace holistic capture validation. The +[complete-capture descriptor plan](PERIODIC_CAPTURE_TRAINING_CACHE.md) provides +that distinct typed connection. + +## Encoding and pooling + +The evaluator encodes every admitted window at B=1 with a registered retrieval +system interface and width 512. It uses the existing frozen numerical context, +eval/no-grad, and the configured qualified precision. CPU/CUDA device aliases +are resolved to actual tensor devices. Caller training modes and numerical +flags are restored even when an encoder fails. + +Base embeddings are summed chronologically with the fixed adjacent float64 +binary tree and divided by the complete accepted-window count. Periodic tokens +use the same tree but divide by each band's supported-window count. Results +are cast once to float32; unsupported bands remain exact positive zero. + +Only after pooling does the existing retrieval scorer normalize features and +compute the full capture/text gallery in FP32. The implementation does not +average normalized window embeddings, window scores, or window recall values. + +An explicit byte budget covers supplied encoded arrays, and the configured +edge budget admits every pair in every window. CUDA also checks a static +model/gallery tensor estimate; this is not a total allocator/RSS guarantee. +Resource failures reject the complete operation, never sample windows, +captures, people, or edges. CUDA OOM remains an explicit resource failure. + +## Explicit cached descriptor path + +`run_capture_validation_cached` uses the same complete source, fixed-tree +pooling, full gallery, objective and capture-macro metric implementation as the +uncached runner. It accepts an admitted +`PeriodicDescriptorCaptureTrainingPlan`, binds it to the residual system and +complete training configuration, and asks the plan to rebuild and validate the +whole capture source before any model forward or score. + +For each actual window in that rebuilt source, the plan reopens one verified +`CachedPairChunkStream`; `ResidualRetrievalSystem.encode_trainable_cached` +passes it explicitly to the registered periodic encoder. There is no silent +cache-miss fallback and no cache of frozen-base or learned residual outputs. +The attached frozen CLIP feature batch is validated and consumed exactly as in +the uncached path; cached validation does not re-encode text. Prepared-window +auxiliary validation is not accepted as a substitute for the holistic source. + +The original `run_capture_validation` API remains unchanged. Both entry points +share the same core, and the cached wrapper/core are included in the scoped +live-function identity checks. These checks detect in-process drift within the +declared runtime; they are not arbitrary-Python isolation. + +## Metrics and checkpoints + +The gallery contains one motion row per capture and every admitted holistic +text positive. The existing evaluator produces integer R@1 hits. Exact +`Fraction` arithmetic computes each capture's T2M and M2T contributions and +their equally weighted capture mean. The primary metric is their half-sum; +captions are not independent capture clusters. Variable-positive symmetric +InfoNCE is computed on this same full gallery. + +Training admission snapshots the complete validation census, including the +upstream manifest and window/text feature identities. The existing checkpoint +`val_manifest_sha256` stores that combined census digest. A changed gallery +is rejected on resume, and a capture source cannot become a training iterator. +At each epoch the training loop uses this evaluator's capture primary metric +for the normal best-checkpoint selection and records all encoded window edges. +It first creates an owned snapshot and computes edge counts from that snapshot, +then verifies the evaluator's returned scored census before consuming its metric. +Formal runtime checks also bind the capture class/runner and the key imported +pooling, evaluation, and objective functions. This is scoped live-function drift +detection, not a security guarantee against arbitrary Python execution. + +The checkpoint schema retains its historical float metric field; it is not +presented as an exact qualification artifact. Qualification must recompute +exact fractions from the selected checkpoint and full capture gallery. The +window-oriented validation source remains available for the separately +declared auxiliary/legacy task and is not main-task evidence. + +## Observed scope and remaining work + +The standalone caller's focused server suite passed 42 tests, including actual +512-wide B0 and PhaseSet encoders; its full regression passed 997 tests, +2 skipped, and 39 subtests. The subsequent training integration passed 54 +focused tests, including actual tiny optimizer/checkpoint/new-runtime resume +equality and rejection of a changed capture census. Full regression also +exposed two legacy fixture-timing/lifetime failures +in a complete run (1000 passed, two failed), preserved as a failed attempt. +The [test-only stability repair](LEGACY_TEST_STABILITY.md) subsequently passed +74 focused tests and 6 subtests, then 1002 complete tests, 2 skipped, and +39 subtests in 297.31 seconds, with unchanged source bytes. Three actual +official CLIP forwards requalified the changed training source while retaining +the pre-existing output golden, long-caption observation, and caller RNG. +An available same-author provisional composition review then identified the +scored-census and live-function consistency gaps described above. Their narrow +repair passed 84 focused tests and 6 subtests, then 1012 complete tests, +2 skipped, and 39 subtests in 286.46 seconds. Three more actual CLIP forwards +retained the original output values and caller RNG with correctly updated +source identities. Earlier attempts are not overwritten. Initial review used +the ARIS local-only fallback; the subsequent same-author review remains +provisional, not independent scientific acceptance. + +These tests use labeled analytic fixtures. They do not substitute for native +Embody captures, official holistic descriptions, a rights grant, complete host +command composition, the nine-run base qualification, or sealed evaluation. +See [host integration](HOST_RUNTIME_INTEGRATION.md) and +[prepared window storage](PREPARED_DATA_V2.md). + +The complete capture descriptor plan and cached residual model entry later +passed a 121-test focused server suite in 271.25 seconds with source unchanged; +the complete combined regression passed 1305 tests, 2 existing skips and 39 +subtests in 613.86 seconds, also with source unchanged. Three official CLIP +forwards retained the established outputs and caller RNG. Those suites +enumerated the twenty-epoch software source census but did not train for twenty +epochs or produce a checkpoint. The cached capture runner then passed 108 +focused server tests in 151.03 seconds with source unchanged. Its complete +combined regression, including host qualification integration, passed 1334 +tests, 2 existing skips and 39 subtests in 643.90 seconds, source unchanged. Training, +checkpoint/resume and host lifecycle integration for the plan remain pending. diff --git a/docs/CUDA_LIFECYCLE_OBSERVATION.md b/docs/CUDA_LIFECYCLE_OBSERVATION.md new file mode 100644 index 0000000..cf3429d --- /dev/null +++ b/docs/CUDA_LIFECYCLE_OBSERVATION.md @@ -0,0 +1,62 @@ +# Bounded full-width CUDA lifecycle observation + +This is engineering evidence for public commit +`e3c73c031c0034701277870ca99fbbcbdd037294`, not a real-data result or completed +formal training qualification. No dataset, external checkpoint or licensed +body-model asset was used. The four lanes ran on an NVIDIA RTX A6000 using +Python3.12.12, NumPy2.4.6, Torch2.12.0+cu126 and CUDA12.6, with FP32, +deterministic backend settings and the frozen disabled-MHA-fastpath policy. + +Each lane used the actual registered architecture: width512,8 attention heads, +FFN2048, full200-frame body22 windows, analytic K2/K3 groups, and frozen analytic +text embeddings. The effective batch was2 for this engineering fixture, not +the formal128-window batch. System08 retained its six-band periodic encoder, +incidence topology and text-band head. No reduced-width substitute was used. + +The lifecycle was:3 uninterrupted optimizer updates; an independent2-update +run; a third newly constructed runtime restoring its update2 checkpoint and +performing update3. Three updates allow initially zero residual/topology gates +to begin passing nonzero topology gradients. Every lane used fresh work and +checkpoint directories. No epoch was completed and no validation metric or +selected checkpoint was produced. + +| Observation | B0 | B1 | B2 | System08 | +|---|---:|---:|---:|---:| +| Parameter elements, including frozen base | 6351361 | 9504257 | 20014593 | 8197639 | +| Peak allocated CUDA bytes | 259429888 | 326669824 | 641133568 | 138454528 | +| Checkpoint files retained | 4 | 4 | 4 | 4 | +| Checkpoint bytes retained | 305078852 | 456479908 | 961194980 | 190464852 | +| Uninterrupted vs resumed decoded payload/model/optimizer/scheduler/RNG/cursor/manifests | exact | exact | exact | exact | + +For B0/B1/B2, the group encoder and logit scale had finite nonzero gradients and +changed state. For08, the topology encoder, six topology-gate elements, +text-band MLP and residual score gate had finite nonzero gradients and changed +state. Its frozen base retained exactly the same state with no gradients. +Frozen fixture text bytes and non-gradient state were preserved in every lane. +All source files remained unchanged, and all canonical observation validators +and all local receipt-member checks passed. + +Private receipt SHA-256 commitments (operational files and checkpoints are not +public release assets): + +- B0: `6517734b854e74ea42982a9055de08256d14fe4876cdd91b38c091a8a92346b0` +- B1: `a0df9f67b5bb47fa957f36b8adc3ea68c412a5dae2a5c950daeaa2decd7f5e8f` +- B2: `fe0914cac54e6ae88f01f104489c44086d3d6f2cde8dd281cc1afe17f0fea671` +- System08: `4815471fc85c3dbbbf8afd2798295fee5327b437a82f6fadfa6cb0c259b70d1d` + +B1/B2 were a distinct reviewed extension with explicit PMA and four-layer +social-temporal parameter censuses. Their wrapper revalidated the successful +B0/08 receipts before execution. Both preserved the full200-frame fixture and +the same2GiB cap; no architecture was reduced to fit. All60 local receipt +members across the four lanes were checked after their transfers completed. + +Raw checkpoint archive bytes also happened to match for the uninterrupted and +resumed runs in each lane. Only decoded-state equality was required; this does +not promise equal serialization bytes across independent hardware or runtimes. + +These figures are not throughput, latency or memory-efficiency benchmarks: +the device was shared, a2GiB allocator cap was imposed, and only this bounded +fixture was observed. BF16, formal initialization/base +qualification, real-data overfit, convergence, native-data scores and empirical +paper readiness remain separate requirements. The synthetic losses are not +reported as retrieval quality. diff --git a/docs/EMBODY_ADAPTER_INTEGRATION.md b/docs/EMBODY_ADAPTER_INTEGRATION.md new file mode 100644 index 0000000..5f5ed6f --- /dev/null +++ b/docs/EMBODY_ADAPTER_INTEGRATION.md @@ -0,0 +1,169 @@ +# Embody SMPL-X adapter integration + +Status: implemented integration seam. A caller-licensed official locked-head +neutral model passed a server loader/batch qualification on 2026-09-26. +This is not native-capture training or retrieval evidence. Contract fixtures +remain nonempirical and are not experimental results. + +## Actual asset qualification, 2026-09-26 + +The official Embody example uses locked-head SMPL-X. The SMPL-X download page +warns that its shape space differs from v1.1; do not silently substitute that +model. The observed `lockedhead_20230207` neutral NPZ matched its official +archive's checksum. With the existing evaluator and isolated server runtime, +the `3 → 1 → 3` frame-batch loader check passed in 4.865 seconds, returning +finite float32 `[3,22,3]` joints and repeatable outputs. This qualifies loading +and short-tail compatibility, not a learned model or a dataset benchmark. + +The downloaded human-text archives were independently matched to the existing +participant-disjoint component manifest: 572 eligible captures, 16 components, +and 400/96/76 captures with no participant/capture overlap. Native holistic +file presence is **398/96/76**: two training captures lack holistic text. +As preregistered, those are excluded from the holistic task before any model +scores, with no split reassignment. The 400-capture training universe remains +unchanged for tasks whose required annotations are present. Test caption files +were sealed by digest without decoding their semantic contents. + +Dataset archives, body models, identifiers, signed URLs, private receipts and +paths remain private. These observations do not grant access to another user; +each reproducer must obtain their own official dataset/model authorization. +The next server check decoded one actual four-actor pilot-training window: +native tracks had 3600 frames per actor, the first accepted 300-frame source +window converted to finite float32 `[4,200,22,3]` in 11.048 seconds with zero +interpolated actor frames. This uses real Embody parameters and the official +body model, not fixture arrays. It does not read caption semantics, create +retrieval scores, persist a prepared training cache, or train a model. +Full-corpus motion auditing and the formal training matrix remain separate gates. + +## Native boundary + +For every synchronized actor, supply exactly these local `.npy` tracks and an +expected SHA-256 for each file: + +- `smplx_mesh_body_pose`: float `[T,63]` +- `smplx_mesh_global_orient`: float `[T,3]` +- `smplx_mesh_transl`: float `[T,3]` +- `smplx_mesh_betas`: float `[T,300]` +- `smplx_mesh_left_hand_pose`: float `[T,45]` +- `smplx_mesh_right_hand_pose`: float `[T,45]` +- `missing`: binary `[T]` or `[T,1]`; upstream defines 0 as corrupted and 1 as + good tracking + +The adapter retains corrupted frames in place, marks all 22 body joints +untracked, and uses exact positive zero for their coordinates. It rejects a +missing track, incomplete file provenance, non-finite observed parameters, +inconsistent actor lengths, duplicate actor commitments, and invalid evaluator +output. It has no dataset discovery, download, metadata-ID, caption, or audio +surface. + +`LicensedSMPLXEvaluator(model_path, device=..., expected_model_sha256=...)` is the concrete conversion +path. It lazy-imports the caller-installed official `smplx` package and calls +`smplx.create` with Embody's fixed configuration: neutral SMPL-X, +`flat_hand_mean=True`, `num_betas=300`, `num_expression_coeffs=100`, and +`use_pca=False`. It loads only a local `SMPLX_NEUTRAL.npz`, observes that exact +file's SHA-256 before and after model construction, and derives its evaluator +provenance from the observed digest and frozen configuration. It neither +downloads the model nor represents that the caller has a license. + +The expected model digest is mandatory and must be obtained from the caller's +verified licensed-asset manifest. A mismatch stops before optional imports or +model construction. This is important because the upstream loader enables NPZ +pickle support. Hashing an arbitrary local file and supplying that same hash +does not establish trust, provenance, or a license; do not use that shortcut. + +Install the optional software dependency with `pip install '.[embody]'`; +it pins `smplx==0.1.28`. Model assets are not included in this extra. The +environment lock and the actual licensed-model witness must also be recorded +before treating any conversion as qualified. Unit fixtures are not that witness. + +The evaluator disables constructor-owned pose, shape, translation, hand, jaw, +eye, and expression parameters. Every call supplies the six Embody tensors and +explicit zero jaw/eye/expression tensors at the actual chunk length. It also +sets the owned model's `batch_size` under a lock for each chunk because the +official regular `SMPLX.forward` uses that member when repeating landmark +barycentric coordinates. This prevents a short final chunk from inheriting a +fixed constructor length. It runs in evaluation mode under `torch.no_grad`, +does not return vertices or shaped vertices, and exports the first 22 official +joints (pelvis through right wrist) as finite float32 NumPy data. + +A custom evaluator remains possible for integration testing, but then the +caller must explicitly supply its provenance SHA-256. For the exact concrete +evaluator, `embody_to_private_capture` reads the observed model binding itself +and rejects any conflicting supplied value. All float64 input and output is +checked again after float32 conversion so overflow cannot enter the PhaseSet +seam. + +Pass the resulting `PrivateCaptureArrays` to the existing +`prepare_capture(...)`. That existing path, rather than this adapter, owns +body-22 extraction defense in depth, 300-frame windows, 30-to-20 Hz resampling, +missing-run rejection/interpolation, group canonicalization, augmentation, and +dynamic collation. + +For the V2 continuous path, pass the same complete numeric capture to +`prepare_continuous_capture`, then `continuous_directional_phase_field`. +This preserves one coordinate frame, absolute window positions, anti-alias +and Morlet halos, and actor/edge trajectories across ten-second boundaries. +It does not replace the original window API or authorize data access. + +The licensed native qualification now also covers one complete four-person +120-second training capture: official SMPL-X conversion to `[4,2400,22,3]`, +12 accepted windows, 119 patches and an untrained 512D temporal forward, in +63.55 seconds on server CPU. No trained scores or captions were consumed. +The entire main-corpus conversion, calibration and training are not implied. + +## Full native main-corpus input audit, 2026-09-26 + +Server-only inspection of both complete licensed motion archives passed in +339.93 seconds: **572 captures, 2,261 native actor records and 15,827 selected +NPY members**. Every selected member was CRC-checked, and all seven native +tracks were validated for exact dimensions/frame counts, binary tracking masks +and observed finite/float32-representable values. This is an input audit, not +SMPL-X conversion or model evaluation of the full corpus. + +The original participant-disjoint universe remains **400/96/76**. The frozen +nonoverlapping ten-second rule accepts **4,894/1,276/1,055** windows and rejects +**18/11/2** in train/validation/test. One test capture has no accepted complete +window; no motion is padded, repeated or borrowed to keep it. After additionally +requiring the existing human holistic description, the confirmatory task has +**398/96/75** eligible captures. Exclusions precede all model scores; there is +no split reassignment. Original K counts, 545 four-person and 27 three-person, +describe the universe, not an assertion of task-eligible K subgroup counts. + +Caption semantics and retrieval scores were not inspected. Full development +body-model conversion, training-only calibration, baselines and scientific +evaluation remain separate stages; input-audit success is not scientific +acceptance. + +## Access and license gates not implemented in code + +- Embody data access is not established by this adapter. The official repository says a user + must complete the release form before downloads work, that approved access + supplies 21 links, and that the dataset is governed by the XRCIA license. + This adapter neither requests nor proves that approval. +- The official Meta release-form page did not render in the documentation + fetch used for this review, so its current identity fields and review timing + are not asserted here. +- SMPL-X model assets are a separate prerequisite. The official SMPL-X site + offers independent account registration (email/password) and requires + acceptance of its privacy statement, Model license, and Body license for + noncommercial research. SMPL-X access does not grant Embody access. +- No licensed dataset array, SMPL-X model asset, credential, signed link, + private path, participant identifier, or generated output may be committed + to the public repository. + +## Official sources checked 2026-09-07 + +- Embody README (native folders, 30 fps, missing convention, access and XRCIA): + https://github.com/facebookresearch/embody-3d/blob/main/README.md +- Embody feature names: + https://github.com/facebookresearch/embody-3d/blob/main/src/enums.py +- Embody official SMPL-X invocation: + https://github.com/facebookresearch/embody-3d/blob/main/src/run.py +- Embody local-array loading behavior: + https://github.com/facebookresearch/embody-3d/blob/main/src/dataloader.py +- Official SMPL-X model API: + https://github.com/vchoutas/smplx/blob/main/smplx/body_models.py +- Official SMPL-X joint ordering: + https://github.com/vchoutas/smplx/blob/main/smplx/joint_names.py +- Official SMPL-X registration and licensing: + https://smpl-x.is.tue.mpg.de/register.php diff --git a/docs/FROZEN_CLIP_TEXT_ADAPTER.md b/docs/FROZEN_CLIP_TEXT_ADAPTER.md new file mode 100644 index 0000000..9bb0362 --- /dev/null +++ b/docs/FROZEN_CLIP_TEXT_ADAPTER.md @@ -0,0 +1,265 @@ +# PhaseSet frozen CLIP text adapter + +The frozen CPU feature path passed its registered Linux server observation on +2026-09-08. Fifteen adapter tests passed; the actual pretrained model completed +the two-forward golden repeat and one separate overlength forward. This is a +feature-path engineering result, not a dataset, training, or quality result. + +Implementation: `src/phaseset_core/frozen_clip_text.py`; tests: +`tests/test_phaseset_frozen_clip_text.py`. The server probe imported the canonical +`phaseset_core.frozen_clip_text` module from a separately verified source copy. +Model files and private execution receipts are not distributed in this tree. + +## Selection made for PhaseSet + +PhaseSet consumes the official frozen pretrained projection from +`CLIPTextModelWithProjection`: + +```text +caption + -> fixed CLIP tokenizer (77 tokens) + -> frozen CLIP text_model + -> final-LayerNorm pooled EOS representation + -> frozen official text_projection + -> raw finite contiguous CPU float32 [Q,512] +``` + +The adapter does not L2-normalize this output. The existing retrieval systems +normalize embeddings at scoring time. It also does not add the legacy +PhasePair bridge-owned trainable `text.project` layer. That old layer belonged +to a different training design and is not needed to place the official CLIP +projection in PhaseSet's registered 512-dimensional space. + +This choice does not eliminate PhaseSet's downstream `TextBandMLP` in residual +retrieval. `TextBandMLP` remains part of the trainable PhaseSet retrieval head, +where it converts an already-frozen global text embedding into band queries. +It is not part of the CLIP tower and is not run by this adapter. + +The public `RetrievalTrainingBatch` boundary already enforces finite contiguous +CPU float32 `[Q,D]`, detaches/clones it, requires one commitment per row, and +uses commitments only for positive-family/lineage logic. This adapter tightens +the producer width to exactly 512 and calls the installed training embedding +validator before returning. + +## API + +```python +adapter = load_frozen_clip_text_adapter( + verified_snapshot_path, + max_batch_size=64, +) +batch = adapter.encode( + ordered_caption_tuple, + ordered_caption_commitment_tuple, + batch_size=32, +) +text_embeddings = batch.embeddings +receipt_bytes = batch.receipt.canonical_json_bytes() +``` + +`Q` is variable and preserves input order. The caller must choose an explicit +batch size no larger than the load-time maximum; both are capped at 256. +Caption count, individual UTF-8 bytes, and total UTF-8 bytes are bounded. +Commitments must be unique exact `bytes[32]` rows. They are never passed to the +tokenizer/model, never influence embeddings, and appear only as hex lineage in +the path-free, text-free receipt. + +Long text uses the single frozen `clip_native_truncate_77` policy already used +by the legacy loader. Each batch is first tokenized with special tokens and no +truncation solely to measure the complete token count. The formal tokenizer +call then uses `max_length=77`, `padding="max_length"`, and `truncation=True`; +an overlength row retains CLIP's native prefix and tokenizer EOS token 49407. +The adapter separately preserves the model configuration's pinned legacy +`eos_token_id=2` compatibility branch; those two identities are intentionally +not conflated. The receipt exposes the original and encoded token counts plus +`truncated=true`, +and binds the full caption UTF-8 digest and complete padded input-ID/attention- +mask tensor digests. It never prints raw text or token IDs, so truncation cannot +be silent even though the text itself remains private. + +The formal token tensor is also compared element-for-element with a local +expectation constructed from the complete untruncated token row: short rows are +padded without changing their tokens, while long rows retain the first 76 +native tokens and place tokenizer EOS 49407 at position 77. The attention mask +must exactly match that expected row. A count/EOS-only match is insufficient. + +The authorized prepare-data host retains the complete caption. The 512-vector +represents only CLIP's native truncated-prefix semantics when `truncated=true`; +it is not claimed to preserve the full caption's meaning. Segmenting, LLM +summarization, or multi-window aggregation would be a different scientific +contract and is not performed here. + +Batch size and ordered chunk ranges are provenance. This implementation does +not promise bitwise equality across different CPU batch sizes, so a different +batch size produces a different frozen-cache key even when caption order is +unchanged. The cache key binds method, source, runtime, snapshot, batch/chunk +order, full-text digests, and tokenization digests, but excludes lineage +commitments and every model/system identifier downstream of CLIP. B0/B1/B2 and +all residual systems must reuse the same receipt-bound frozen embedding cache; +they must not run separate text conversions. + +`FrozenClipTextBatch.embeddings` returns a fresh owned clone on every access so +a caller cannot mutate the receipt-bound stored value through the public +property. `RetrievalTrainingBatch` takes another snapshot at its boundary. + +## Snapshot and runtime trust + +The only accepted snapshot is `openai/clip-vit-base-patch32` revision +`3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268`. The module contains literal name, +size, and SHA-256 pins for its eight official files. It requires an absolute, +resolved, non-reparse directory containing exactly those eight regular, +single-link files—no subdirectories, metadata, cache files, aliases, or extras. + +All eight files are hashed before and after model loading. They are hashed +again before and after every complete `encode` call. This is a bounded +best-effort stability check, not an operating-system atomic snapshot. + +The loader is exact and text-only: + +- `CLIPTokenizerFast.from_pretrained(local_path, local_files_only=True, + trust_remote_code=False)`; +- `CLIPTextModelWithProjection.from_pretrained` with the same offline flags, + `low_cpu_mem_usage=False`, and loading information returned; +- a temporary empty cache plus offline environment and blocked common network + connection entry points; +- exact loading-info census: no missing/mismatched/error rows and only the + expected vision/logit keys discarded from the full checkpoint; +- exact model config, 197 live text/projection parameters, one buffer, no vision + module, all parameters finite CPU float32 and permanently `requires_grad=False`; +- `eval()` plus `torch.inference_mode()` for every forward; +- live parameter/buffer content manifest checked before and after each encode; +- caller Torch RNG preserved across load and every success or failure of + encode; any RNG advance during encode is a closed hold after the caller state + is restored. + +The implementation is deliberately pinned to the newly observed CPU identity: +CPython 3.12.12, Linux x86-64 little-endian, Torch 2.12.0+cu126 with CUDA hidden, +Transformers 4.57.3, tokenizers 0.22.2, huggingface-hub 0.36.2, and NumPy 2.4.6. +It requires one intra-op/inter-op thread, deterministic algorithms, and highest +float32 matmul precision, and native MHA fastpath disabled. The caller must +install that false flag; the adapter checks and records it without changing it. +A different runtime is a new qualification lane, not +a permissive fallback. + +The exact Transformers model/tokenizer sources and installed +`phaseset_core.training` source are pinned. The adapter's own source is +hashed before/after load and encode and recorded in every receipt. The receipt +cross-binds runtime, source, snapshot, ordered caption text digests, lineage +commitments, per-row embedding digests, and the full output digest without +including captions or filesystem paths. + +## Prepare-data integration boundary + +The eventual prepare-data host, not this module, owns: + +1. authorized caption access and deterministic caption selection; +2. split and positive-family assignment; +3. caption commitments and manifest provenance; +4. secure storage of raw text and frozen arrays; +5. checking that the receipt and array digest still agree before constructing + `RetrievalTrainingBatch`. + +The adapter accepts captions already selected and authorized by the host. It +cannot discover data, read caption files, infer splits, issue commitments, +authorize training, or convert a receipt into authority. Its receipt always +contains `authority=0`, `production=false`, and `training_authorized=false`. +Successful feature preparation is not a scientific result or a data-rights +witness. + +## Tested adapter coverage + +`test_phaseset_frozen_clip_text.py` uses a deterministic fake tokenizer/model +with real Torch tensors; it never imports Transformers or loads CLIP. It covers: + +- variable ordered `Q`, explicit chunking, exact `[Q,512]` output; +- native truncation above 77 tokens with original/encoded counts, a visible + truncation flag, exact retained-prefix tokens/mask, terminal EOS, and no raw + caption in the receipt; +- commitments affecting lineage/receipt but never embedding values; +- input-order preservation; +- text/path-free authority-zero receipts; +- literal offline loader arguments and frozen eval state; +- extra snapshot files and post-load snapshot mutation; +- non-finite forward output and live parameter mutation; +- exact bounded integer batch size; +- cache identity changing with batch/chunk order but not lineage commitments; +- returned tensor ownership. +- formal-token body mutation before forward and model RNG advance with caller- + state restoration. + +Additional coverage can include mutation tests for missing files, aliases/hardlinks, +source/runtime drift, tokenizer shape/key drift, loading-info/config drift, +caption UTF-8/size/count limits, duplicate commitments, output dtype/device/ +layout drift, network attempts, warning emission, and receipt canonicalization. + +## Observed server qualification + +The fifteen adapter tests passed in 11.45 seconds. The separate real-model +observation completed with exit zero and empty stderr; source hashes were equal +before and after execution. The two original public fixture rows produced +bitwise-equal embeddings and receipts across repeated forwards, with the same +previously observed combined output SHA-256: +`4a2ab93dab393a38bd0e78ee737fa31813c8e35ef3fcb5f4f3108705a502f58e`. + +A third forward used a separate public overlength fixture. The original 226 +tokens became exactly 77, with the complete expected prefix, attention mask, +and terminal tokenizer EOS 49407 checked before forwarding. Its observed output +SHA-256 was +`3f89662e5f6c06d4c7f9c1e2e77a78ad73dbe4da0a7be4a56a35df27060ebcba`. +That value is an observation, not a self-created acceptance golden. Caller RNG +state was preserved across all three calls. Independent same-family code review +remains provisional; these actual tests close only the measured feature-path +checks. Prepare-data integration, data rights, and formal experiments are separate. + +The full Linux regression with this adapter then passed 932 tests, one skipped, +and 39 subtests in 287.60 seconds, with unchanged source bytes. The model weights +are not required for that unit/regression suite; actual loading is verified by +the separate private snapshot-bound observation above. + +After the [frozen MHA policy](FROZEN_MHA_RUNTIME.md) changed the pinned training +source and runtime identity, a separate server requalification passed 33 +focused tests and 949 full-suite tests, 2 skipped, and 39 subtests. Three actual +official CLIP forwards preserved both output values above and caller RNG. +The new receipt contains the new source/runtime identity and false MHA flag; +it is intentionally not byte-identical to the historical receipt. These checks +do not qualify CUDA text, full training, or real-data experiments. + +The later capture-training integration changed the exact training source to +157160 bytes, SHA-256 +`9f6c21e354a17facda89a542d53e39013477c5e1b1a259781e2446cd394f6d06`. +After its 1012-test complete server regression passed, another three official +CLIP forwards passed. Both output digests above, repeated embedding equality, +and caller RNG were unchanged. The adapter receipt correctly records the new +training and adapter source identities; no golden was regenerated. This is a +source-bound CPU feature-path requalification, not a real-data training result. + +## Receipt-bound disk restoration + +`rehydrate_frozen_clip_text_batch` restores an already encoded batch from its +owned CPU float32 features and complete original canonical JSON receipt. The +caller supplies the independently retained expected receipt digest. Closed +field/type validation checks all official file pins, historical source/runtime +manifests, caption lineage, chunk ranges, cache identity, per-row feature hashes +and the complete feature hash. It does not load CLIP or run inference. A +historical receipt is not rewritten to claim today's encoding source. + +The first separate real-model restoration probe found a snapshot-order bug: +the live encoder records files in filename order, whereas restoration compared +against pin declaration order. Unit fixtures had repeated that wrong ordering. +The failed attempt is retained. Restoration and fixtures were corrected to the +encoder's actual ordering; a negative case now rejects declaration-order rows. +No official pin, live encoding behavior or output golden was changed. + +The corrected integration passed110 focused tests and1043 complete tests, +2 existing skips and39 subtests. Three subsequent actual official forwards +preserved both original output digests above. Two separately persisted cache +batches then restored exact features, lineage, complete receipts and cache +keys, with no extra forward and unchanged observed CPU Torch RNG state. +Source comparisons passed, and all44 receipt members were independently +rehashed after transfer. The actual cache-probe private receipt commitment is +`b8dfe62f3773f81ff117d31da703cf60130eaf0b6890407bc203cdff35bbd2ac`. +Private cache payloads and raw operational receipts are not public assets. + +See [prepared capture storage](CAPTURE_PREPARED_STORAGE.md) for the bounded +on-disk consumer. Neither restoration nor a matching receipt proves caption +rights, official annotation identity, participant-disjointness or model quality. diff --git a/docs/FROZEN_MHA_RUNTIME.md b/docs/FROZEN_MHA_RUNTIME.md new file mode 100644 index 0000000..7439bc3 --- /dev/null +++ b/docs/FROZEN_MHA_RUNTIME.md @@ -0,0 +1,63 @@ +# Frozen attention runtime policy + +PhaseSet's numerical context disables PyTorch's native MHA fastpath on both +CPU and CUDA. It captures the caller's boolean, installs and checks `False`, +and restores the original value on normal and exceptional exit. The training +environment digest includes this policy; older code/environment-bound +checkpoints and qualifications are not evidence for the new runtime. + +This is a runtime change, not a change to model layers, weights, masks, loss, +optimizer, or the registered CPU/CUDA tolerance (`rtol=1e-4, atol=1e-5`). It is +applied around the actual training/validation context, not just a test branch. + +## Observed reason + +A preserved same-weight B0 inference diagnostic found 336 group-embedding +elements outside that tolerance with native fastpath enabled (maximum absolute +difference about 2.26e-4). Disabling it reduced the exceedance count to zero +(maximum about 7.15e-7). The direct temporal comparison exhibited the same +pattern with both an all-false padding mask and no mask. This establishes the +observed runtime dependency; it is not a claim about all hardware or a general +vendor defect. + +The original residual-retrieval failure remains a failed historical attempt. +The diagnostic did not overwrite it or change its acceptance threshold. + +## Subsequent registered-server observations + +- Focused CPU regression: 33 tests passed. A separate full suite passed 949 + tests, 2 skipped, and 39 subtests, with unchanged source bytes. +- Actual official CLIP CPU requalification: three forwards passed. The old + two-row output golden and separate long-caption output were unchanged, + and caller RNG was preserved. Source/runtime/receipt identities changed + correctly; complete old and new receipts are not byte-identical. +- A new actual-width FP32 CUDA residual-retrieval engineering observation + passed. Frozen-base embedding maximum CPU/CUDA difference was 7.153e-7; + score difference was 4.471e-8; loss difference was zero. The zero-initialized + residual equaled the frozen base and the exercised residual text-head + parameters received finite nonzero gradients. The frozen text fixture was + unchanged. Peak allocated memory was 108,373,504 bytes under a 2 GiB cap. + +The last observation exercises the closed retrieval-head seam with analytic +features, not the periodic core or an optimizer. It does not qualify full +training, BF16, performance, data access, or a winning base checkpoint. A +subsequent B0 requalification stopped before model construction because no GPU +met the fixed 8 GiB free-memory requirement. Once memory became available, a +separate resource-retry chain repeated retrieval and passed B0, B1, and B2 with +the same reviewed source and unchanged numerical checks. That chain used new +attempt directories; the failed resource receipt was not overwritten. These +are bounded FP32 forward/backward observations, not optimizer, BF16, throughput, +or full-training qualification. Existing workloads were not interrupted. + +## CLIP and empirical boundaries + +The frozen text adapter pins the entire installed training module, records +`mha_fastpath_enabled=false`, and rejects a live true flag. Its caller must +configure the validated CPU runtime before loading it; the adapter does not +silently alter global flags. See [the text adapter](FROZEN_CLIP_TEXT_ADAPTER.md). + +The [capture evaluator](CAPTURE_VALIDATION.md) now aggregates every admitted +window into one capture embedding and uses true capture identities in the +training API. Its private-host disk composition remains unfinished. Window +scores and repeating actor-set lineage IDs are not a substitute. This runtime +repair is not a completed empirical qualification or a scientific result. diff --git a/docs/HOST_BASE_QUALIFICATION.md b/docs/HOST_BASE_QUALIFICATION.md new file mode 100644 index 0000000..489c74a --- /dev/null +++ b/docs/HOST_BASE_QUALIFICATION.md @@ -0,0 +1,108 @@ +# Private host base qualification + +The private host controller composes the already defined base-cohort resolver, +holistic capture scorer, CUDA latency session, durable progress journal, +qualification assembler and production adapter. It does not change the public +selection rule and does not let a CLI caller submit scientific results. + +## Request and input boundary + +The public command remains `qualify-base --terminal-root`. The authenticated +host configuration supplies the closed attempt registry, output and journal +locations, registered device identity, frozen protocol/launcher identities and +latency in-process wall-time bound. Concrete host names, ports, device identifiers, private paths, +receipts and data stay outside the public tree. + +The controller requires the exact registered nine-run history: each run keeps +failed predecessors and ends in its only successful terminal. The resolver +then verifies terminal-bound latest checkpoint bytes and the distinct selected +validation-best checkpoint where applicable. Missing or inconsistent rows stop +before scoring. A registry digest records consumed bytes; it is not proof of +permission or data provenance. + +## Closed execution order + +For a formal call, the controller performs this sequence: + +1. Load the canonical registry and resolve all nine completed chains and their + selected checkpoint payloads. +2. Before capture scoring, observe the registered CUDA device, require the + registered minimum free memory, establish the process allocator cap and + require an initially clean allocator. +3. Recompute the complete holistic validation gallery for every selected base + checkpoint. Submitted scores and parameter counts are not accepted. +4. Release scorer objects, clear this process's cuBLAS workspaces, empty the + allocator cache and require zero allocated and reserved bytes. The device + identity and free-memory observation are checked again. +5. Run the complete cyclic latency session with its typed write-once observer. + The returned session and durable journal terminal must agree. +6. Assemble the qualification with the existing selector, retain the exact + qualification and dependency bytes, and pass them to the existing + production adapter for independent recomputation and backend callback. + +Scoring and latency share the same physical-device and precision binding, but +latency still performs its own admission. The scoring preflight does not weaken +the latency gate. Cleanup affects only allocations and workspaces owned by the +current process and does not stop other jobs. + +## Failure and timeout behavior + +CUDA identity, free-memory, allocator, workspace and OOM failures are retained +as HOLD outcomes. If scoring and cleanup both fail, the original scoring error +and its resource cause remain primary while the cleanup type is recorded +separately. A successful score cannot enter latency unless cleanup reaches the +exact-zero boundary. + +The in-process interval timer is supported only where the required POSIX timer +API exists. The current controller checks that API, the Python main thread +and the absence of an inherited deadline before registry resolution or scoring, +then rechecks immediately before latency timer installation. Its cleanup always +attempts both disarm and prior-handler restoration, and a cleanup error cannot +replace the primary scoring or latency error. Timer capability or restoration +drift is a runtime HOLD, not a scientific failure. +Deployment must also provide its separately frozen hard timeout because a +process signal cannot guarantee that a blocked native call returns. A hard kill +may leave only a valid prefix of the progress journal and the start record; +that prefix is not a qualification. + +An injected runtime exists only for software contract tests. Even a complete +synthetic journal from that seam returns a nonformal HOLD and cannot reach +qualification assembly or authentication. + +## Verification status + +The controller passed 131 focused server tests in 37.07 seconds with source +unchanged. Tests cover resource admission/release, failure preservation, timer +portability, the actual CLI-to-backend path and a structurally supplied +host-to-production-adapter callback. The first attempt's 123 passes and seven +fixture API failures in 35.21 seconds remain retained. A test-only successor +fixed journal bindings, CLI stream arguments, fake-CUDA test isolation and a +class-name assertion; production controller and host bytes were unchanged. +These software fixtures do not establish a real nine-run cohort, formal +81-visit CUDA latency session, qualified winner or data-dependent completion. + +The combined source, including cached complete-capture validation, subsequently +passed 1334 tests, 2 existing skips and 39 subtests in 643.90 seconds on the +Linux server, with source unchanged. No formal cohort was produced by this +regression. + +Follow-up code now implements the two reviewed runtime-boundary +corrections: the early and exception-safe timer behavior described above, and +qualify-base-only classification of post-admission source/output/lease runtime +failures as HOLD outcomes rather than CLI input errors. Other commands and the +v1 private-host schema retain their prior behavior. Fault-injection tests use a +typed portable timer facade so Windows does not need to emulate POSIX signals; +a separate test exercises the real POSIX handler and timer restoration. + +The successor passed static checks, fresh static review, and 145 focused Linux +server tests in 64.24 seconds with source unchanged. Those code and test changes +are not included in the observed 1334-test source, and cross-platform CI +acceptance remains pending, so this paragraph does not claim the portability +defect is empirically closed. Formal qualification remains unexecuted; this +feature-branch milestone is not an empirical release. + +The earlier K=32 B0/B1/B2 diagnostic verified the corrected per-process CUDA +release mechanics at registered width, but used untrained models, one warmup +and one observed forward per model. It is not the formal latency session or a +host qualification result. Earlier failed observations remain part of the +history. diff --git a/docs/HOST_RUNTIME_INTEGRATION.md b/docs/HOST_RUNTIME_INTEGRATION.md new file mode 100644 index 0000000..853ba0e --- /dev/null +++ b/docs/HOST_RUNTIME_INTEGRATION.md @@ -0,0 +1,329 @@ +# PhaseSet private-host runtime integration + +The composition root is `src/phaseset_core/host.py`. It contains no server endpoint, +credential, dataset root, or receipt payload. + +## Implemented production chain + +`host.RuntimeAdapterFactory(private_host_config)` is passed directly to +`phaseset_core.cli.main(..., runtime_adapter_factory=factory)`. The factory: + +1. loads an existing private receipt record; +2. hashes every receipt artifact named by that record, including the rights, + runtime, execution, source/prepared-data, split, caption, scoring-census, + and hard-gallery records required by `PrivateReceiptAssertions`; +3. reads every prepared NPZ once into a bounded buffer, authenticates that exact + buffer, and parses the same bytes without loading pickle data; +4. constructs the repository's `ProductionRuntimeAdapter`. + +It never creates authority, rights, runtime, or execution receipts. If an +artifact is absent or changed, the factory raises the existing +`ExecutionHold` codes and the public CLI remains authority zero. + +Admission and command execution have distinct typed contracts. +`RuntimeAdmissionRequest` carries the frozen plan, matrix, training-config, +command census and handler digest; it has no per-command `command` field. +`CommandIntent` carries the selected command and run, not matrix/config fields. +The backend checks all five canonical admission bindings before retaining its +one-shot snapshot. New base/residual attempt records obtain their three +experiment digests from that retained admission, while dispatch checks the +actual intent against the parsed request. A rejected admission cannot poison +the snapshot and prevent a later canonical admission. Resume continues to +inherit its verified predecessor binding. No extra operator hash approval is +introduced. + +The factory dispatches v1 and v2 prepared indexes using the actual index bytes +bound to the private prepared-data receipt. The v2 loader also verifies each +consumed split manifest before decoding its numeric batches. Malformed batch +semantics are translated into the existing execution hold before a training +backend is constructed. The legacy v1 loader now checks its consumed manifest +against the index digest rather than relying on a separate earlier path read. + +`PrivatePreparedDataSource` maps a v1 immutable split manifest to the public +`TrainingDataSource` protocol. Each NPZ contains exactly: + +- `skeletons`: float32 `[B,K,200,22,3]` +- `actor_mask`: bool `[B,K]` +- `frame_mask`: bool `[B,200]` +- `track_mask`: bool `[B,K,200,22]` +- `actor_commitments`: uint8 `[B,K,32]` (ignored where actor_mask is false) +- `group_commitments`: uint8 `[B,32]` +- `text_embeddings`: float32 `[Q,D]` +- `motion_positive_ids`: uint8 `[B,32]` +- `text_positive_ids`: uint8 `[Q,32]` +- `text_commitments`: uint8 `[Q,32]` + +The v2 contract fixes the text width at 512, accepts variable caption counts, +and supplies epoch-specific deterministic whole-group yaw from immutable +unrotated training bytes. Its validation source is unrotated. See +[prepared-data v2](PREPARED_DATA_V2.md) for the writer, bounded numeric format, +split checks, and provenance limits. Neither loader establishes data rights. + +## Main holistic capture validation + +The additional closed `phaseset-host-capture-validation-index-v1` composes an +existing prepared training index with validation-only +[capture storage](CAPTURE_PREPARED_STORAGE.md). The combined index bytes are +bound to the same private prepared-data receipt. Both the outer index and the +referenced training index must equal canonical ASCII JSON bytes, not merely +parse to equivalent objects. Duplicate keys and noncanonical encodings are +rejected even when a supplied digest matches them. + +Its `train` row has exactly `kind/path/sha256`. The kind is +`prepared-training-index-v1` or `prepared-training-index-v2` and must match the +actual referenced schema. That complete old index is validated, then only its +training source is used. Its auxiliary window-validation source is never +relabeled as holistic capture data. + +Its `val` row has exactly `kind/path/sha256/source_census_sha256/` +`upstream_manifest_sha256`, with kind `capture-validation-storage-v1`. The host +loads the actual storage bytes with the retained expected digest and checks +the reconstructed source type, `val` split, complete census and upstream +manifest. The runtime receives this `CaptureValidationSource` unchanged. +Base and residual runs, including both resume paths, use the same composition. +Checkpoint and scoring-time census binding remain in the existing training API. + +Before constructing the backend, the host streams every actual training batch +and rejects actor/track or caption commitments shared with capture validation. +Prepared v2 additionally compares its documented window-positive IDs with +capture window commitments; v1 arbitrary positive families are not falsely +treated as window IDs. Source manifests/censuses are checked before and after +the scan. The fixed scan order uses local deterministic source RNG and preserves +Python, NumPy and Torch global RNG. Actor commitments are not participant IDs: +this check complements, but never replaces, the authenticated participant split +audit. + +Preflight materializes the bounded capture source and streams the training +tree once. The single-batch byte limit still applies; train scan I/O is linear +in total artifact bytes, and there is no new aggregate train-byte limit. +Capture identity sets are linear in the materialized validation census. This +operation neither loads CLIP nor opens the sealed test split. Direct legacy +v1/v2 window-task dispatch retains its prior behavior. + +`run-base` uses the formal seed-bound B0--B2 constructor and +`PhaseSetTrainingRuntime`. Its checkpoint directory must be +`//model-checkpoints`, allowing the harness to build +the public write-once attempt/checkpoint/terminal ledger around the runtime +checkpoint. A controlled interruption is terminalized as +`CONTROLLED_INTERRUPTION`, retaining a resumable checkpoint. + +The optional `checkpoint_observer` adds a post-atomic-write +observer to the public runtime without changing model, optimizer, sampler, or +numerical behavior. The host uses it to write live checkpoint receipts and a +30-second heartbeat chain while `fit()` is active. The final update checkpoint +of an epoch is deliberately deferred because the runtime immediately writes a +validation checkpoint at the same global step, while `AttemptStore` requires +strictly increasing receipt steps. Python failures and controlled interrupts +terminalize against the latest live receipt. A report checkpoint restored from +the predecessor is explicitly receipted into the new attempt even when `fit()` +makes no progress and emits no observer event. + +Each active attempt holds an OS-released exclusive lock at +`/.lifetime.lock` from attempt creation through successful or failed +terminal persistence. Heartbeats are immutable files under +`/heartbeats/`; checkpoint receipts are immutable files under +`/checkpoints/`; runtime payloads remain under +`/model-checkpoints/`. If a process is lost before it writes a +terminal, `resume` may close the orphan only after nonblocking acquisition of +the lifetime lock and a second complete attempt-chain verification while that +lock remains held. It never infers death from PID identity or heartbeat age. + +An ordinary runtime or checkpoint-observer exception writes `failure.json` and +a `HOST_EXECUTION_FAILED` terminal, then returns a `FAILED BackendExecution` +whose artifacts are those actual files. `BaseException` paths attempt the same +terminalization but are re-raised. If the failure or terminal cannot be +persisted (for example, disk failure), the host propagates the error and does +not manufacture a backend artifact. + +`resume` accepts base and residual attempts. The predecessor checkpoint +must be under the same attempt root; the harness verifies its terminal and +checkpoint receipt, creates a distinct resume-linked attempt with +`AttemptStore.create_resumed`, and passes that exact `ResumeRecord` to the +training runtime. Residual resume also requires `--base-checkpoint` and +`--periodic-cache`; base resume rejects those arguments. The host verifies and +copies the latest and any older best checkpoint into the new attempt before +decoding owned authenticated bytes. Failed partial copies are retained, not +deleted by a path-based cleanup race. See [resume details](RESIDUAL_RESUME.md). + +## Executable invocation + +Install the package first. The +private-host module accepts its private host configuration before `--` and +passes everything after `--` through the public parser: + +```bash +python -m phaseset_core.host --host-config private-run/host.json -- \ + preflight --config configs/phaseset/training.json + +python -m phaseset_core.host --host-config private-run/host.json -- \ + run-base \ + --run-id phaseset-run-v1/BASE_QUALIFICATION/1729/B0 \ + --checkpoint-dir private-run/attempts/base-1729-b0-a/model-checkpoints \ + --config configs/phaseset/training.json + +python -m phaseset_core.host --host-config private-run/host.json -- \ + run-residual \ + --run-id phaseset-run-v1/RESIDUAL_TRAIN/1729/08 \ + --base-checkpoint private-run/base/1729/selected-validation.pt \ + --periodic-cache private-run/cache/1729 \ + --checkpoint-dir private-run/attempts/residual-1729-08-a/model-checkpoints \ + --config configs/phaseset/training.json +``` + +A trusted Python composition root can equivalently call: + +```python +from phaseset_core.host import main + +raise SystemExit(main(PUBLIC_CLI_ARGUMENTS, config_path=PRIVATE_HOST_CONFIG)) +``` + +The original private host config schema is `phaseset-private-host-v1`. It points to a +prepared index, an existing receipt record plus the exact artifact file for +each receipt digest, the installed source-tree digest, and runtime facts +(device, BF16 qualification decision, edge budget, checkpoint cadence, and +resume metadata). Paths may be absolute or relative to the private config. +They are never serialized by the public CLI. + +The backward-compatible `phaseset-private-host-v2` requires the +closed `base_qualification` configuration. Its `qualify-base` implementation +resolves the actual nine terminal histories, rescoring their selected +checkpoints on complete validation captures before an independent CUDA +latency observation and qualification assembly. See +[host base qualification](HOST_BASE_QUALIFICATION.md) for the exact command, +latency in-process wall-time bound, runtime admission, evidence and failure semantics. Existing +v1 configurations are not silently upgraded or assigned a qualification. + +For residual runs, the optional `residual` object points to the one canonical +base-qualification artifact and one canonical capacity-audit artifact per +registered seed. The host process derives and verifies their SHA-256 values; +the operator does not transcribe individual hashes. `--periodic-cache` names a +seed directory containing canonical `record.json` (`PeriodicCacheRecord`) and +the digest-bound `cache-content.npz`. The payload must expose float64 +`energy_floors[6]`. The host reads the payload once into a bounded buffer and +hashes and parses those same bytes. It verifies seed, winning-base run, base +terminal, qualification, and cache-content bindings before the public residual +factory is called. + +The execution controller is responsible for creating the single private +receipt record from the user's already granted execution scope and the actual +rights/runtime/data measurements. This harness consumes and verifies that +record; it does not ask the user to sign each digest and does not create a +rights assertion when Embody approval is unknown. + +## Deliberate gaps (no placeholder completion) + +- Embody rights are not known. Until a real rights artifact participates in + the receipt record, preflight/run-base/run-residual/resume remain held. +- The Embody-format conversion is outside this harness. It must produce the + exact prepared index/split/NPZ seam above and bind it to the authenticated + prepared-data receipt. +- The main capture disk composition above is implemented. Producing its inputs + from licensed captures and verified official holistic captions remains the + upstream preparation responsibility. Auxiliary window positive families and + repeating actor-set commitments never substitute for actual capture row IDs. +- `run-residual` is wired through the strict qualified-base loader, canonical + periodic-cache record, energy floors, canonical all-system capacity audit, + closed residual constructor, and the same attempt ledger. Residual resume + reconstructs those same dependencies and passes the verified predecessor + record to the unchanged runtime. Historical cache-file SHA provenance is not + retroactively claimed when only energy floors were consumed. +- `prepare-data`, `audit-split`, `build-periodic-cache`, + `evaluate`, `bootstrap`, and `render-paper` are not + implemented here. Calling one raises before a `BackendExecution` exists. +- The public `resume` parser has no stop-after-step option. It accepts B0/B1/B2 + and registered final system IDs; the host implements base and residual resume + and recovers identity from the verified attempt ledger. +- The residual encoder and complete-capture evaluator now expose explicit + reader-backed descriptor-cache APIs. The training loop now consumes explicit + admitted plans, including cached checkpoint/resume. This host's residual + construction/checkpoint/resume path does not yet inject those plans. + The existing host cache argument authenticates the complete cache payload + and consumes its energy floors, while its model path still recomputes + descriptors. That historical behavior must not be represented as descriptor + cache consumption in reports. See the + [complete-capture cache plan](PERIODIC_CAPTURE_TRAINING_CACHE.md). +- A hard power loss before the first atomic runtime checkpoint has no resumable + payload. The host refuses such a resume instead of manufacturing a checkpoint + receipt. SIGKILL/power loss cannot write a terminal synchronously; terminal + recovery occurs on the authorized resume operation. +- Runtime checkpoint fields do not expose separate sampler, dataloader, and + dropout blobs. The host receipt records the exact cursor/manifests and RNG + payload under those ledger identities; a future public checkpoint-to-ledger + helper would remove this local encoding convention. + +`score_execution_census_sha256` is treated as the preregistered census of score +executions required by admission. It is never interpreted as a post-training +score table and therefore does not make base training depend on future scores. + +## Observed tests and limits + +On the Linux server, 12 host contract tests passed, including a genuine child +process lifetime-lock test and injected persistence failures. Four separate +tiny CPU fits passed checkpoint-observer integration: durable update/validation +callbacks, propagation of observer failure with a resumable checkpoint, +unchanged model/checkpoint/loss/validation values with an attached observer, +and a real checkpoint payload receipted by the attempt ledger. These use +`synthetic_contract=True` and do not assert production admission or results. + +Report/failure JSON is file-fsynced and, on POSIX, parent-directory-fsynced +before a terminal is persisted. A failed fsync cannot create a success terminal. +Actual Embody rights, full model/runtime qualification, all registered runs, +and sealed test remain separate requirements. + +The subsequent v2 prepared-source integration passed 94 focused server tests +and a separate complete suite of 964 tests, 2 skipped, and 39 subtests. These +include actual malformed numeric payloads and manifest/index replacement +cases; they do not assert a completed preparation command or a real-data run. + +The complete-capture host composition subsequently passed 96 focused tests +and 1070 complete Linux server tests, with 2 skips and 39 subtests. A later +storage-portability correction passed 103 focused / 1077 complete tests, +again with 2 skips and 39 subtests, plus the unchanged official CLIP forward +and disk-restoration observation. Source comparisons passed throughout. +The new tests include canonical index bytes, real train/validation commitment +overlap, unchanged ambient RNG, and exact dispatch through base/residual/resume. +They use explicit analytic fixtures and do not establish data access or a +completed real training run. + +The portability correction follows observed failures on both Windows CI +versions at the preceding capture-storage commit. On the inspected Windows +runtime, path stat and descriptor stat returned different `ctime` semantics +for one unchanged file. Cross-interface identity checks now compare device, +inode, size and modification time; each interface separately retains its full +before/after identity check including `ctime`. Content digests, bounded reads, +regular-file and symlink checks remain mandatory. Tests simulate the stable +cross-interface difference and reject drift within either interface. Windows +CI for the correction is a separate external acceptance check. + +The later admission/intent correction passed 94 focused Linux server tests +and 1232 full tests, two existing skips and 39 subtests in 401.58 seconds. +Source censuses remained unchanged. New tests enter the real public CLI, +request-aware factory, production admission and backend with real typed +requests. They create actual attempt ledgers, then deliberately raise at the +typed base/residual model-runtime seam. Correct experiment bindings and +durable FAILED terminals must exist; no runtime checkpoint is manufactured. +Negative admissions with changed matrix, command census or handler digest +leave the snapshot unbound and still allow the subsequent canonical request. +These tests close the concrete missing-field/poisoning defects, not a complete +licensed-data CLI training run or the unimplemented commands listed above. + +The host qualification integration passed 131 focused Linux server tests in +37.07 seconds with source unchanged. Tests enter the real CLI/factory/adapter/ +backend path, validate typed holds and independently recheck completion +artifacts. A preceding run with seven fixture/API failures is retained; only +the tests changed for this rerun, not production code. These are software +fixtures, not nine trained bases, formal CUDA latency rows or a selected base. + +The subsequent controller boundary correction passed 145 focused tests and was +included in a 270-test combined cached-training/cache-construction cohort. +Both observed runs kept the source unchanged. Early main-thread/POSIX timer +checks, cleanup preservation and qualify-base-only runtime HOLD classification +are implemented; Windows CI acceptance is a separate observation. + +The [descriptor cache builder](PERIODIC_CACHE_EXECUTION.md) and +[cached residual lifecycle](CACHED_RESIDUAL_TRAINING.md) are callable core +interfaces, not yet an end-to-end cached CLI host. Typed build evidence, +private configuration, direct plan injection and ledger/resume linkage remain +explicit follow-up integration. The existing uncached host does not silently +select a cache based on files found on disk. diff --git a/docs/LATENCY_PROGRESS_JOURNAL.md b/docs/LATENCY_PROGRESS_JOURNAL.md new file mode 100644 index 0000000..7674610 --- /dev/null +++ b/docs/LATENCY_PROGRESS_JOURNAL.md @@ -0,0 +1,67 @@ +# Durable latency progress journal + +This optional journal preserves a diagnostic prefix when a latency session +fails or stops. It is POSIX-only; importing the module is portable, while +creating or reading a journal on an unsupported runtime fails explicitly. +Journal integration tests are correspondingly POSIX-scoped. + +## Files and ordering + +The controller exclusively creates a fresh session below trusted, stable +private ancestors. The writer uses mode 0700 for the session directory and +0600, exclusive no-follow creation for canonical event files. Events and +sequence markers are flushed and synchronized in a fixed order. Every event +binds its sequence, predecessor digest and the external cohort, scorer, +installed-source, protocol, wrapper and registered-device identities. + +The typed interface allows only session start, context ready, visit start, +warmup completion, timed-sample completion, visit end and terminal events. It +does not offer arbitrary metric insertion. A successful fixed session has +1461 committed events. Enumeration stops at fixed bounds before collecting an +unbounded directory; event count, leaf count and individual byte sizes are +separately bounded. Reads require stable, single-link regular files. + +These checks assume trusted stable ancestors and are not a sandbox against +hostile concurrent filesystem writers. Read-only flags and digests are not +operating-system immutability or external authorization. + +## Failure semantics + +The latency runner owns all actual visit/sample observations. Observer calls +occur after the corresponding operation; timed-sample callbacks run after the +timer has stopped. One callback failure latches observation failure, holds the +latency session and prevents later callbacks. Previously committed files are +never erased or rewritten to make the attempt appear successful. + +A callback can finish durable terminal commit and then throw. The committed +terminal cannot be rolled back: + +- After a COMPLETED terminal, the API can conservatively hold the session even + though the reader sees a structurally complete, nonformal terminal. The + complete-session digest differs from the subsequently held-session digest. +- After a HELD terminal, the API retains the original held receipt and chains + the observer exception. The reader can retain that matching nonformal HELD + terminal. + +The controller must cross-check the actual API outcome, session receipt and +terminal digest. Neither terminal grants formal status on its own. An +incomplete committed prefix or one physically uncommitted tail is preserved; +the reader does not infer whether a hard kill, interruption or I/O failure +caused it. `KeyboardInterrupt`, `SystemExit` and hard kills are not recast as +successful sessions. + +## Observed scope + +The journal dependency passed a 68-test Linux suite in 14.13 s with source +unchanged. A previous attempt passed its software assertions but failed its +source-stability check because an isolated SIGKILL child generated bytecode. +That attempt remains failed. Adding explicit `-B` to that test child fixed the +harness without deleting the original files or weakening the comparison. + +The final latency-to-journal integration passed 73 combined tests in 22.63 s. +It includes all 81 fake-runtime visits, real durable filesystem writes, +admission failure, active-visit failure, post-sample callback failure, and a +post-commit COMPLETED-terminal exception. Those fixtures establish software +mechanics, not real GPU timings or formal training. The merged 1211-test +server regression is reported in +[base qualification](BASE_COHORT_QUALIFICATION.md). diff --git a/docs/LEGACY_TEST_STABILITY.md b/docs/LEGACY_TEST_STABILITY.md new file mode 100644 index 0000000..656b8d6 --- /dev/null +++ b/docs/LEGACY_TEST_STABILITY.md @@ -0,0 +1,45 @@ +# Legacy CLIP test determinism + +A complete server run during capture-training integration reported 1000 passed +tests and two failures in the existing legacy CLIP-resolution tests. That +failed attempt is retained. The production scanner, lease implementation, +canonical model pins, and old immutable release are unchanged. + +The late equal-length file-mutation test specifically exercises the closing +metadata sweep after the changed file has already been hashed in pass two. +It previously assumed that a very fast same-length write necessarily changes +the filesystem's observed timestamps. The fixture now explicitly advances its +owned temporary file's mtime and verifies that change. The real byte mutation, +16-open expectation, and required `HOLD_FILE_TOCTOU` result are unchanged. This +does not turn the production scanner's documented best-effort observations +into an atomic snapshot guarantee. + +The weak-lease registry test now collects unreachable earlier test cycles +before measuring its baseline, just as it already collects them after removing +the 256 test assessments. Otherwise automatic collection during those 256 +allocations can remove a pre-existing unreachable lease and invalidate the +relative-count assertion. All retention and release checks remain in place. + +The corrected focused server integration passed 74 tests and 6 subtests. +The subsequent complete suite passed 1002 tests, 2 skipped, and 39 subtests +in 297.31 seconds; source bytes were unchanged across execution. +These are test-fixture changes, not relaxed model tolerances or weaker data +admission checks. Review was local-only under ARIS's unavailable-delegation +fallback; actual server outcomes remain the deterministic evidence. + +## Canonical capture-index fixture newlines + +The later complete-capture host integration passed its Linux tests but exposed +18 failures on each Windows CI runtime. Its test-only JSON writer used the +platform default newline translation: canonical text ending in LF became CRLF +on Windows. The production byte-equality check correctly rejected the fixture +before the intended dispatch or overlap assertion. This is distinct from the +earlier storage-stat issue, whose tests passed in that same CI run. + +The fixture now supplies an explicit LF newline policy. Production code and +canonical-byte acceptance are unchanged. Two additional authenticated cases +insert actual CRLF at the outer and referenced training-index seams; both must +still be rejected even with recomputed matching digests. The corrected focused +Linux server suite passed 71 tests in 11.08 seconds with source bytes unchanged. +The failed Windows logs are retained privately. Acceptance on Windows requires +the subsequent actual CI result, not inference from this Linux observation. diff --git a/docs/MIME_SET_ADAPTATION.md b/docs/MIME_SET_ADAPTATION.md new file mode 100644 index 0000000..446815e --- /dev/null +++ b/docs/MIME_SET_ADAPTATION.md @@ -0,0 +1,67 @@ +# MIME: original dyadic and complete-parent group implementations + +Independent implementation of [MIME v1](https://arxiv.org/html/2607.22702v1), +not released author code or a claim of converged original-task reproduction. +The original dyadic API and the group adapter are distinct; neither converts +XYZ into fictitious SMPL rotations or silently truncates a capture. + +`OriginalMIMEMotion` takes true 135D root-displacement/22-joint 6D-rotation +features plus common-frame root positions. It retains distinct role projections, +normalizations and type embeddings, root distance/signed displacement, +relative configuration and root-motion magnitude ratio, early relation injection, +four layers of pre-LN independent self-attention, simultaneous bidirectional +full-time cross-attention and GELU FFNs, ordered frame fusion and learned-query +temporal pooling. The original Inter-X host must enforce its 30Hz/300-frame +protocol separately and supply real rotation inputs; body22 is not a substitute. + +`MIMESetMotion` explicitly adapts these interaction mechanisms to shared body22 +XYZ inputs. All actor weights and endpoint block weights are shared; input-role +embeddings are absent. Each endpoint receives its own directed relation encoding. +Sum/absolute-difference/product fusion makes the pair readout symmetric. +Every unordered pair is visited, checkpointed independently, and accumulated +in canonical commitment order as float64 before the window mean. No actor +ordinal or commitment is a neural feature; no dense K-by-K-by-time activation +is constructed and no edge is silently sampled. All accepted windows enter an +extra four-layer temporal hierarchy with actual absolute starts, including gaps, +and learned-query full-parent pooling. The per-window hierarchy and 20Hz XYZ +feature adapter differ from the original single 30Hz dyadic sequence. + +Missing pelvis gates root-relative features, rather than inventing zero distance. +Pair pooling uses jointly observed pelvis frames. Per-actor self-attention sees +valid observed frames; root displacement does not bridge a missing frame and is +zero at each window start. These mask/boundary conventions are disclosed choices. + +Both use *fine-tuned* local CLIP text, a learned projection and positive learned +logit scale, then bidirectional-average variable-known-positive InfoNCE. This +is not the immutable frozen CLIP feature adapter; that adapter remains unchanged. +The existing official asset revision/hashes are verified before local-only loading. +No vision module is instantiated or optimized. The original dyadic text API +rejects captions exceeding 77 tokens. The complete-parent group adaptation +preserves every content BPE token in consecutive 75-content-token segments, +adds BOS/EOS to each, and averages trainable pooled segment features per caption. +This long-text adaptation is disclosed, not purported original MIME behavior. + +Paper settings are width 512, four layers, four heads, dropout 0.1, AdamW 1e-4, +weight decay 1e-4 and batch 128; curriculum is the separately implemented +[training-only parent sampler](PHASESET_V2_MIME_CURRICULUM.md). FFN 2048, +two-layer GELU frame fusion, ratio epsilon 1e-8 and initial logit scale 1/0.07 +are disclosed implementation choices where author code is not available. +The existing three MIME short-pilot slots govern eligible choices, not added HPO. + +`ParentHostConfig.for_literature("MIME-Set", ...)` declares AdamW, learning rate +1e-4 and weight decay 1e-4 rather than inheriting the internal AdamW/0.01 decay. +These settings are explicit in the run/checkpoint config and must match on +resume. The common group host's 5% warmup/cosine schedule and gradient clipping +at 1.0 are disclosed project adaptations; they are not asserted author defaults. + +Tests include original-input dimensions, independent full-time attention +equations, all parameter families, endpoint swap, all K=3 permutations and +input-gradient equivariance, dropout/checkpoint/RNG equality, missing root steps, +full timeline and complete text segmentation. Tiny language seams are labeled +software fixtures; only a real pinned CLIP/native attempt proves that execution. +When Transformers is installed, a random small actual CLIP component also +checks the extracted-transformer API and checkpoint/direct gradient/RNG equality; +its outer model constructor initializes the attention backend, as in asset loading. +That component is not a pretrained-language or real-data qualification. +Neither software nor native backward qualification proves original-task learning, +production-host readiness, retrieval performance, scientific claims or release. diff --git a/docs/MORLET_PORTABILITY.md b/docs/MORLET_PORTABILITY.md new file mode 100644 index 0000000..2e8dc5f --- /dev/null +++ b/docs/MORLET_PORTABILITY.md @@ -0,0 +1,56 @@ +# Frozen Morlet bytes and native formula verification + +The physical 30 Hz PhasePair and 20 Hz PhaseSet kernels are immutable serialized +coefficients, not learned weights. Their original SHA-256 values remain: + +- PhasePair: `4f75ab687d333af8b436feea371dbaee56c55256e91c42c5e1ce45935beed5d7`. +- PhaseSet: `633d754bba12dab0555722f4b8963409a110c8681796cc907d163096e2e42655`. + +The generated Python byte modules contain the original big-endian float64 +serialization. The loader validates provenance, digest, support lengths, +finiteness and trailing bytes, then returns read-only canonical kernels. +The build-only exporter preserves the original CPython 3.14.5 / NumPy 2.4.6 / +Windows provenance. It neither trains a model nor creates experimental results. + +## Runtime policy + +The package pins NumPy 2.4.6. Within the advertised CPython 3.12--3.14 range, +x86-64 Windows and Linux select different strict native-formula checks: + +- Windows must reproduce the original exact serialized digest and original + residual checks. Even a one-ULP within-tolerance mutation is rejected. +- Linux must reproduce the observed native-formula digest below and remain + within `2**-52` maximum complex-tap distance from the canonical bank. +- Other interpreter families, platforms, architectures, or NumPy versions are + not qualified by this implementation. Unrecognized formula bytes fail closed. + +Matching the exact formula fingerprint is required; an arbitrary Python patch +number is not a rejection reason. This does not declare every accepted machine +fully qualified for training. Each actual run still needs its own frozen +environment and numerical/model/CUDA qualification records. + +Measured verifier lanes so far are Windows Python 3.12.0 and 3.14.5, and the +Linux Python 3.12.12 server, all with NumPy 2.4.6. Simulated identity tests only +check dispatch and do not count as execution on other interpreters. + +Linux native-formula fingerprints: + +- PhasePair: `3b4cec820d8202d4576d829d37626acc2414f0aeecf430e17c751d9bd1731f6c`. +- PhaseSet: `926c06aa57a27139d1ed1a9e0d9c3da191717b8a9495d6fff900bda44459d9a3`. + +The measured maximum complex-tap differences are respectively +`6.206335383118183e-17` and `3.1031676915590914e-17`, both below `2**-52`. +The native diagnostic's DC/energy rounding residuals need not equal the +Windows construction receipt. Physical output is always the original canonical +bank; its original DC and unit-energy guarantees are preserved and tested. + +## Regression boundaries + +The unchanged five-reduction construction spy, 13D relation descriptor, +endpoint-swap algebra and known-delay checks remain. Separate tests reject +artifact mutation, Windows one-ULP drift, unqualified hardware and the native +left-to-right reduction mutant. The latter must fail the Linux formula digest +gate even when its tap distance alone is within tolerance. + +These checks qualify kernel software behavior, not empirical retrieval quality, +BF16 precision, GPU performance, licensed data, or a completed scientific study. diff --git a/docs/PERIODIC_CACHE_EXECUTION.md b/docs/PERIODIC_CACHE_EXECUTION.md new file mode 100644 index 0000000..2702ed5 --- /dev/null +++ b/docs/PERIODIC_CACHE_EXECUTION.md @@ -0,0 +1,61 @@ +# Complete periodic descriptor cache construction + +`periodic_cache_execution` builds and consumes one seed-specific private cache +tree. It composes the actual v2 descriptor writers/readers and complete plan +admission; it does not estimate energy floors, choose a base, run a model or +authorize data. + +The entry points are `load_qualified_energy_floor_input`, +`build_periodic_descriptor_cache_for_seed` and +`consume_periodic_descriptor_cache_build`. They require one official seed, its +formal twenty-epoch residual configuration, canonical selected-base +qualification, the exact winning checkpoint, supplied qualified floor bytes, +an admitted prepared-v2 train source, a complete holistic capture-val source, +source/environment identities and explicit resource bounds. + +The writer enumerates every train batch in epochs 0..19 and every accepted +capture-validation window once, then admits the complete plans for +02/03/04/06/07/08. Systems 01/05 do not receive fabricated descriptor plans. +No person, edge or window is silently dropped to fit a resource limit. + +The train writer is constrained by both its per-cache and combined limits; +the capture writer receives only the positive remaining shard/byte allowance. +An exceeding shard is rejected before it is written. File reads compare +portable path/descriptor identity and independently verify full before/after +metadata on each interface, including the Windows ctime distinction. + +## Immutable output and consumption + +The output root must be new. It contains `train/`, `capture-validation/`, six +`plans/.json` files, and a final `build.json`. The last file binds the +seed/configuration, selected base/terminal/checkpoint, consumed floor +record/payload/vector, both source and cache identities/censuses, six admitted +plans, code/environment and resource bounds. It is published only after the +complete build and admission succeed. Partial failed roots are preserved and +are not resumed or overwritten by this version. + +Consumption requires a separately retained expected build-manifest digest. +It reopens both readers, repeats all six complete admissions, compares stored +plan bytes, checks the live checkpoint and training-code identities, and +reconstructs the exact canonical manifest. A self-consistent replacement JSON +file or an asserted success boolean is insufficient. + +Cache contents remain private: commitments can still be linkable, and +descriptor shards must not be treated as automatically publishable data. + +## Verification boundary + +The combined 270-test registered-server cohort includes actual complete +twenty-epoch and full-capture analytic construction/consumption, no-overwrite, +wrong split/checkpoint, mutated plan, extra member, strict floor payload, +aggregate disk limits and canonical type-drift tests. It passed with unchanged +source. Full regression on the same source passed 1360 tests, 2 existing skips +and 39 subtests in 743.26 seconds. These are software tests, not licensed-data +construction or training. + +The existing floor-only record proves linkage to supplied seed/base bytes; +it does not prove how the six values were scientifically fitted. A separately +specified actor-once training-only floor collector and provenance are still +required for formal use. Diagnostic constants are not a substitute. Host CLI +dispatch and typed production evidence also remain separate integration work. +See [cached residual lifecycle](CACHED_RESIDUAL_TRAINING.md). diff --git a/docs/PERIODIC_CAPTURE_TRAINING_CACHE.md b/docs/PERIODIC_CAPTURE_TRAINING_CACHE.md new file mode 100644 index 0000000..475196c --- /dev/null +++ b/docs/PERIODIC_CAPTURE_TRAINING_CACHE.md @@ -0,0 +1,76 @@ +# Complete-capture periodic descriptor plan + +`PeriodicDescriptorCaptureTrainingPlan` is the authority-zero planning seam +for one residual system and one registered seed. It combines the existing +twenty-epoch prepared-training request census with the main task's complete +holistic capture-validation source. It does not run training, select a +checkpoint, grant data access or claim a result. + +## Complete admission + +`admit_periodic_descriptor_capture_training_plan` accepts an exact residual +`TrainingConfig`, a prepared-v2 training source, a holistic +`CaptureValidationSource`, and separate authenticated descriptor-cache readers +for training and capture validation. It accepts no caller-authored plan digest, +partial key list or cache-miss fallback. + +Admission enumerates every actual training row for epochs 0 through 19 using +the selected seed. It separately enumerates every window in every holistic +validation capture at validation epoch zero and positive-zero yaw. Capture +requests derive their source NPZ, storage manifest and canonical global window +ordinal from the loaded capture source; a loop index or source start frame is +not substituted for lineage. + +The required training and capture key sets must each equal the complete census +of their corresponding cache reader. Source manifests, full source censuses, +capture plans, window and caption counts, numeric batches including padding, +energy floors, edge limits and descriptor families remain bound. Admission +preserves Python, NumPy and CPU Torch RNG, and preserves CUDA RNG when CUDA was +already initialized. These checks establish consistency of supplied artifacts, +not data rights or protection from arbitrary code in the process. + +## Explicit selection + +`plan.open_training_batch(batch)` revalidates one exact contextualized training +row and returns its sealed descriptor stream. Before cached holistic validation, +`plan.validate_capture_validation_source(source)` rebuilds and verifies the +entire capture source and cache census. The evaluator then calls +`plan.open_capture_validation_window(capture, window_position)` only for windows +from that rebuilt source. + +Systems 02 and 03 select marginal-power and mean/difference-DCT descriptors; +systems 04, 06, 07 and 08 select full-relation descriptors. Systems 01 and 05, +base models, missing shards, wrong batches, foreign families and changed reader +censuses are rejected rather than routed to an uncached fallback. + +`ResidualRetrievalSystem.encode_trainable_cached` is the corresponding model +entry. It supplies the verified stream to the registered PhaseSet encoder while +leaving the frozen base, residual head and learned outputs uncached. The +uncached model method remains available for existing callers. + +`run_capture_validation_cached` shares the original holistic evaluator's +complete pooling, gallery, objective and capture-macro metric core. Its only +alternate branch is periodic descriptor acquisition. It validates the whole +capture source before any model forward or gallery score and consumes the +already attached frozen text features without invoking CLIP again. The original +`run_capture_validation` signature and uncached behavior remain available. + +## Verification and remaining integration + +A registered-server focused software suite covering the cached residual model, +frozen-text source binding and complete capture plan passed 121 tests in +271.25 seconds with source unchanged. The complete combined regression then +passed 1305 tests, 2 existing skips and 39 subtests in 613.86 seconds, again +with source unchanged. Three actual official CLIP forwards retained the +established output values and caller RNG. Those observations used software +fixtures and did not perform optimization, finish an epoch, load licensed data +or produce a checkpoint. + +The later cached holistic runner passed 108 focused server tests in 151.03 +seconds with source unchanged. Its complete combined regression with host +qualification integration passed 1334 tests, 2 existing skips and 39 subtests +in 643.90 seconds, source unchanged. The complete training lifecycle still needs a typed +plan handoff for initialization, all epochs, checkpoint/report binding and +resume. Host storage loading and standalone plan admission are not that lifecycle +integration. A real-data cache, acceleration claim, formal score and +qualification remain absent. diff --git a/docs/PERIODIC_DESCRIPTOR_CACHE_V2.md b/docs/PERIODIC_DESCRIPTOR_CACHE_V2.md new file mode 100644 index 0000000..53d9e81 --- /dev/null +++ b/docs/PERIODIC_DESCRIPTOR_CACHE_V2.md @@ -0,0 +1,138 @@ +# Immutable periodic descriptor cache + +`periodic_descriptor_cache_v2` stores weight-independent physical pair +descriptors, not learned tokens or activations. The existing small periodic +cache stores energy-floor metadata; this separate format stores all three +descriptor streams: full relations, marginal power, and mean/difference DCT. +No licensed motion payload or private cache artifact is included here. + +## Exact cache identity + +A cache key binds the complete canonical numeric group batch, masks, padding, +actor/group lineage commitments, fixed Morlet oracle and energy floors. It +also binds explicit source/environment, manifest, batch, split, window ordinal, +seed, epoch and exact yaw identities. Commitments determine canonical reduction +order but do not become neural-network inputs. + +These caller-supplied digests provide integrity and equality binding only; +cache construction does not authenticate their origin or grant dataset access, +consent, split, training or execution authority. + +Reordering actors preserves canonical identity; changing dynamic batch packing +is a distinct key, not an inferred cache hit. Shared yaw includes floating-point +rotation and rounding, so another epoch's apparently rotation-invariant motion +is not assumed bitwise interchangeable. There is no approximate cross-epoch +reuse or silent uncached fallback for a configured wrong cache. + +The bounded format preserves directed descriptor arrays, validity and endpoint +indices in canonical order. Re-iteration can yield requested runtime chunks +without changing the logical stream, including chunks spanning batch groups. +The same physical full-relation stream can underlie the controls that transform +it; marginal and DCT require their separately identified streams. + +## Storage and resource boundaries + +The writer checks expected pair count and decoded byte requirements from actor +counts before materializing a stream. The fixed decoded census is 1914 bytes +per edge for all stored arrays. Per-shard, decoded, total-byte and edge bounds +are explicit; an additional exact serialized-size check follows encoding. +Resource rejection does not drop actors, sample edges or truncate an artifact. + +Artifacts are exclusively created and the index is published last. The reader +performs bounded owned reads, raw digest checks, strict schema checks and exact +dtype/shape/offset/payload digest validation. It checks static symlinks, +single-link payloads and byte bounds; loaded arrays are non-writeable. On +platforms providing `O_NOFOLLOW`, the owned read also requests no-follow +opening. On Windows the module is import-safe, but cache use requires a trusted, +stable, non-concurrently-mutated tree and does not claim POSIX-equivalent +no-follow or junction protection. Static symlink checks, single-link checks, +hashes and read-only flags are integrity hardening, not filesystem isolation +or data authorization. + +## Descriptor storage verification + +Nineteen server tests passed in 36.21 s with source unchanged. They cover all +three streams, a 186-edge K12/K16 mixed batch, chunks 64/128/256 including a +cross-group boundary, malformed authenticated artifacts, bounds, and unchanged +CPU Torch RNG. A small test-only encoder seam also checks exact cached versus +uncached parameter gradients through the existing custom backward replay. +The hardlink and oversize checks are independent: only an explicit hardlink +creation capability error can skip that case; unknown I/O errors fail, and the +oversize check still runs. All nineteen ran successfully in this Linux +observation. Its private receipt digest is +`8edbb675a64773fe104a1eae7b4bf398262b61827199152833da1e0765237f41`; +the receipt itself and its private paths are not published. + +That earlier test seam deliberately supplied fixture chunks and was not a +production cache provider. The unchanged-source 1211-test regression used the +preceding production module and eighteen-test file; the test-only case split +was separately verified by the nineteen-test observation above. + +## Explicit model entry and backward replay + +`PhaseSetEncoder.forward_cached` and `PhaseSetSystem.forward_cached` now +accept an exact reader-backed `CachedPairChunkStream`. They revalidate the +actual complete `PreparedGroupBatch`, its canonical numeric digest, the +reader-bound six-band energy-floor digest, the expected stream family and the +complete unordered-edge census. Configured cached execution never silently +falls back to a physical recomputation. + +| Consumer | Required cached stream | +| --- | --- | +| Plain encoder; systems 04, 06, 07, 08 | FULL_RELATION | +| System 02 | MARGINAL_POWER | +| System 03 | MEAN_DIFFERENCE_DCT | +| Systems 00, 01, 05 | Cached execution rejected | + +Systems 06 and 07 still apply the registered incidence and phase-stripping +transformations. System 04 retains the shared postprocess with topology +disabled. No learned token, intermediate activation or gradient is stored in +the descriptor cache. + +The same re-iterable stream is explicitly passed through forward reduction, +system-06 incidence census, no-grad backward moment reconstruction, and +differentiable microblock VJP replay. It is retained on that autograd call's +context, not installed globally or as mutable model state. Legacy `forward` +and `forward_activity` remain uncached. + +Cached execution preserves the encoder's exact edge-budget contract. It +rejects E greater than the budget with the existing `ResourceLimitError` and +`RESOURCE_LIMIT` fields before consuming descriptors; equality is accepted. +This checks a real early integration defect: bypassing the uncached iterator +must not also bypass its resource admission. + +## Observed model integration and remaining work + +The combined descriptor/model/control/custom-autograd suite passed 98 tests +in 90.90 s on the Linux experiment server, with source unchanged. It includes +the 186-edge two-group stream at chunks 64/128/256, exact output and parameter +gradient comparisons, all six registered cached consumers, actor-storage +permutation, wrong batch/floors/family, caller RNG preservation, and cached +forward/backward with an instance-local forbidden uncached iterator, including +system 06. The 185/186 edge-budget boundary is independently exercised. + +The complete integrated source then passed 1228 tests, two existing skips and +39 subtests in 401.07 s with source unchanged. These are CPU software fixtures, +not licensed-data training, CUDA performance measurements or retrieval-quality +evidence. + +The [prepared-v2 source](PREPARED_DATA_V2.md) now derives immutable descriptor +contexts from the actual consumed NPZ, split manifest, global window ordinal, +seed, epoch and applied yaw. Legacy batches retain `None`, and a contextualized +batch cannot be rotated again while keeping stale yaw metadata. + +The [prepared-window cache plan](PERIODIC_TRAINING_CACHE_PLAN.md) now admits +an exact complete 20-epoch training census and one val/epoch-zero census for +each official seed. Its selector reopens the authenticated reader on every +request; the reader exposes only a read-only complete key-census tuple. +This remains an engineering window-validation seam, not holistic capture +validation. A separate [complete-capture plan](PERIODIC_CAPTURE_TRAINING_CACHE.md) +now binds the same complete training census to every actual holistic +validation window. The residual system has an explicit reader-backed cached +encoding method, and [capture validation](CAPTURE_VALIDATION.md) can consume +that plan through its shared complete-gallery evaluator. Training-loop, +initialization and checkpoint/resume consumption still require integration. +The model checks actual batch bytes but does not invent source window ordinals +or silently select a different epoch/yaw context. Host preparation must supply +the correct already verified reader handle. No real-data cache, acceleration, +measured throughput or model-quality result is claimed. diff --git a/docs/PERIODIC_TRAINING_CACHE_PLAN.md b/docs/PERIODIC_TRAINING_CACHE_PLAN.md new file mode 100644 index 0000000..478b7b1 --- /dev/null +++ b/docs/PERIODIC_TRAINING_CACHE_PLAN.md @@ -0,0 +1,62 @@ +# Prepared-window periodic descriptor plans + +`admit_periodic_descriptor_training_plan` derives a sealed plan from exact +formal-residual configuration, prepared-v2 train/val sources and authenticated +`PeriodicDescriptorCacheV2` readers. It accepts no caller-authored plan digest, +key subset or cache-miss fallback. + +This API remains an engineering prepared-window seam. It does not implement the +main task's holistic capture validation, start training, restore a checkpoint +or establish data rights. A separate +[complete-capture plan](PERIODIC_CAPTURE_TRAINING_CACHE.md) now covers the +holistic validation source without relabeling this auxiliary validation input. + +## Complete requests, one seed at a time + +Admission enumerates the actual training source for epochs 0 through 19 and +the actual prepared validation source once at epoch zero, using one of the +three registered seeds. Every request binds its consumed manifest/NPZ, +window identities and ordinals, exact yaw bits, full numeric batch including +padding, edge count, energy floors and descriptor family. + +The keys actually required by those requests must exactly equal each reader's +complete `cache_key_census`. Missing, additional, duplicate or corrupt rows +reject admission. The edge budget is checked before shard opening, with the +existing `ResourceLimitError`; no person or edge is silently removed. + +System 02 uses marginal-power descriptors, 03 uses mean/difference DCT, and +04/06/07/08 use full relations. The registered model still applies incidence +shuffling or phase stripping where required. Base systems and residual +systems 01/05 reject this cache-plan API. + +## Stateless selection + +`plan.open_training_batch(batch)` and `plan.open_validation_batch(batch)` +recompute the actual request identity, match one admitted row, revalidate the +reader and complete key census, enforce the edge budget, and reopen the real +shard before returning its sealed stream. Repeated selection is repeated +verification, not an execution cursor or a cached learned output. + +The plan binds actual source contexts, not unobservable iterator call +arguments. Prepared validation contexts are always epoch zero; a future +training integration must explicitly request validation epoch zero as well. + +Admission preserves Python, NumPy, CPU Torch and already-initialized CUDA RNG +states. It does not initialize CUDA to inspect its RNG. If an input source +unexpectedly consumes randomness, the saved states are restored and admission +fails. These checks are ordinary API integrity, not arbitrary-Python isolation. + +## Observed scope + +The combined plan/source/cache/model suite passed 76 tests in 184.61 seconds +on the Linux experiment server, with source unchanged. Tests build actual +descriptor artifacts from explicitly analytic prepared-data fixtures and +exercise all three seeds, twenty epochs, family mapping, closed census, +resource order, request mutants and repeated reader opens. + +No optimization, trained checkpoint, real-data cache, speedup, formal result or +holistic validation is claimed by that observation. The complete-capture plan +and explicit cached model entry are implemented as separate typed APIs. Their +training-loop, initialization/report, checkpoint and resume integration remains +unfinished; this prepared-window plan is not upgraded into main-task evidence +by those later components. diff --git a/docs/PHASESET_EXPERIMENT_PLAN.md b/docs/PHASESET_EXPERIMENT_PLAN.md index f20e3a3..3fc0e0e 100644 --- a/docs/PHASESET_EXPERIMENT_PLAN.md +++ b/docs/PHASESET_EXPERIMENT_PLAN.md @@ -1,5 +1,9 @@ # PhaseSet frozen experiment and statistics plan +> Historical V1 plan. The affected 33-run census is superseded by the +> [2026-09-26 V2 amendment](PHASESET_V2_AMENDMENT.md) and its 87-stage matrix. +> Preserve this file for legacy reproducibility; do not run both matrices. + **Plan ID:** `phaseset-multiperson-20260825` **Frozen at:** 2026-08-25 20:43:38 +08:00 **State:** `FROZEN / NOT_EXECUTED / SEALED_TEST_CLOSED` diff --git a/docs/PHASESET_RESEARCH_CONTRACT.md b/docs/PHASESET_RESEARCH_CONTRACT.md index 083fe2f..171ef0a 100644 --- a/docs/PHASESET_RESEARCH_CONTRACT.md +++ b/docs/PHASESET_RESEARCH_CONTRACT.md @@ -1,5 +1,10 @@ # PhaseSet frozen research contract +> Historical V1 contract. Signed local phase, temporal incidence, calibrated +> scoring, counterfactual supervision, and the formal census are amended by +> [PhaseSet-V2](PHASESET_V2_AMENDMENT.md). The original contract and code remain +> preserved; V2 has no real-data result or scientific acceptance yet. + **Contract ID:** `phaseset-multiperson-20260825` **Frozen at:** 2026-08-25 20:43:38 +08:00 **Target:** ICASSP 2027, Multimedia Signal Processing primary topic diff --git a/docs/PHASESET_V2_A1_SPEED_CONTROL.md b/docs/PHASESET_V2_A1_SPEED_CONTROL.md new file mode 100644 index 0000000..4886867 --- /dev/null +++ b/docs/PHASESET_V2_A1_SPEED_CONTROL.md @@ -0,0 +1,30 @@ +# A1: speed-only periodic-input control + +A1 is the matrix-registered speed-only ablation, not a replacement for the +full PhaseSet-V2 model or a retrieval result. The common B2 checkpoint, +captions, weak counterfactual branch, text adapter, six Morlet filters, +relation encoder, scoring module, run seed and trainable capacity are retained. +Only the periodic frontend changes. For each observed pelvis or pelvis-relative +joint velocity vector `v`, A1 broadcasts `||v||₂ / sqrt(3)` into its three XYZ +channels *before* convolution. This removes sign and direction while +preserving each vector's squared energy. Taking the magnitude of a complex +Morlet response would be a different control; removing its phase fields is A5. + +The full model's geometric root branch and frozen B2 global branch still see +the original group geometry. Therefore full-versus-A1 can only support a claim +about signed direction in the **periodic input**, not an assertion that the +entire A1 system is spatially direction-blind. + +Speed responses have their own cache schema, and cache loading declares the +expected frontend. Main, pilot and each external fold need separate floors +fitted from their complete training actor/patch/band population. The fit +rejects mixed frontends or physical configurations. A typed speed-floor +receipt binds population, training-source manifest, actual fit config and six +floor values; source loading, model scoring and checkpoint extra-state reject +identity drift. Signed-vector floors or caches are not interchangeable. + +The software constructor does not authorize a formal run. It does not fit +real-data floors, prove storage/GPU capacity, run a pilot, touch sealed test, +or establish an ablation effect. Those steps require private admission and +immutable server receipts. The official human-only evaluation gallery is +unchanged by the user's [relation-supervision amendment](PHASESET_V2_RELATION_SUPERVISION_AMENDMENT.md). diff --git a/docs/PHASESET_V2_A2_ORDER_FREE.md b/docs/PHASESET_V2_A2_ORDER_FREE.md new file mode 100644 index 0000000..4bebb46 --- /dev/null +++ b/docs/PHASESET_V2_A2_ORDER_FREE.md @@ -0,0 +1,28 @@ +# PhaseSet-V2 A2 order-free control + +A2 is the registered `remove_local_and_capture_order` control. It receives +the same physical six-band phase field, frozen B2 anchor, text embeddings, +losses, complete captures and actor–edge incidence as the full system. It +removes the *learned temporal order* in the actor, incident-node, directed-edge +and group-readout paths. It does not strip phase, subsample frames, shuffle +actors, or change the physical Morlet input; A1 and A5 test other questions. + +Each ordered two-layer GRU stack is replaced by a shared pointwise +`width → 6 × width → width` MLP with GELU and a masked mean. The MLP is +independent at every patch and has no state, timestamp, position embedding or +convolution. The final mean is over all supported capture patches. Its +trainable parameter count is `12w² + 7w`, versus `12w² + 12w` for the two +GRU cells; at the fixed `w=512`, this is approximately 0.081% smaller per +replacement and introduces no inactive capacity padding. + +All invalid patch outputs are hard-gated to exact zero after the MLP. A2 +cannot be combined with the separate A3/A4/A5/A6 controls, and its model +checkpoint records the order-free identity. The same tensor-only checkpoint +identity also distinguishes A3 pair-bag, the legacy topology-off switch and A5 +phase-stripped switches from the full model. The full model's previous state +payload remains unchanged. + +This is a mechanism control, not evidence that time order helps retrieval. +That conclusion requires the frozen three-seed comparison on the official +human-only task and the registered statistics; software tests alone are not +the paper result. diff --git a/docs/PHASESET_V2_A3_PAIR_BAG.md b/docs/PHASESET_V2_A3_PAIR_BAG.md new file mode 100644 index 0000000..57d9bb0 --- /dev/null +++ b/docs/PHASESET_V2_A3_PAIR_BAG.md @@ -0,0 +1,31 @@ +# PhaseSet-V2 A3 capacity-matched pair-bag control + +A3 is the registered `pair_bag` mechanism control. It uses the same physical +phase field, frozen B2 anchor, text adapter, training captions, objective, +complete-capture timeline and parameterized head as V2 full. It removes the +explicit assignment of relation half-edges to actor nodes. It does **not** +discard relations, phase, temporal order or the shared incident-node and +edge-update modules. + +The pair-bag streams all valid unordered pairs in 64-edge blocks. Each block +contributes the sum, square sum and count of both directed half-edge packets +to global per-patch moments. A symmetric masked mean of actor histories and +those global packet moments form one context vector per patch, broadcast to +identical node streams. The existing node MLP, two-layer node GRU, +node-conditioned edge update, two-layer edge GRU and ordered group GRU remain +active. The same pair packets then enter the normal unordered readout and +directed text score; only the explicit actor–edge incidence map is absent. +No dense `K × K` adjacency is built. + +Consequently, rerouting an unchanged multiset of pair packets between actor +endpoint slots leaves A3 unchanged while it can change V2 full. A3 and full +have exactly the same trainable parameter count. `use_topology=False` remains +the distinct legacy pair-only/K=2 diagnostic; it is **not** the registered A3 +because it leaves node and edge-update capacity inactive. A3 is mutually +exclusive with A2/A4/A5/A6. Its identity is bound in tensor-only checkpoints. + +With exactly two valid actors, the context gate remains off, the topology +values and gradients are positive zero, and A3 is bitwise equal to V2 full on +the same weights. This implementation and its tests are software qualification, +not a claim of retrieval superiority. The registered three-seed, human-only +comparison and parent-cluster statistics are still required. diff --git a/docs/PHASESET_V2_A4_INCIDENCE.md b/docs/PHASESET_V2_A4_INCIDENCE.md new file mode 100644 index 0000000..a8fa12f --- /dev/null +++ b/docs/PHASESET_V2_A4_INCIDENCE.md @@ -0,0 +1,25 @@ +# PhaseSet-V2 A4 incidence-shuffle control + +A4 is the registered `shuffle_explicit_incidence` control in the frozen V2 +matrix. It tests whether the assignment of physical pair relations to actors +contributes beyond the multiset of pair packets. It is not a claim that every +form of actor topology has been erased. + +The control uses the same trainable `ContinuousRetrievalSystem` and +`TemporalIncidenceEncoder` as the full phase system. For each capture, local +patch and canonical 64-edge block, it permutes only supported half-edge values +entering `_node_block`. The endpoint slots, support mask, actor degree, +coverage, physical pair-token stream, directed text packets and all parameters +remain unchanged. A fixed nonnegative `incidence_shuffle_seed` determines the +reassignment; private actor/group commitments do not enter its hash or neural +features. This makes the ablation independent of participant identity and +keeps the same data and capacity as the full arm. The seed is part of +checkpoint extra state and mismatched checkpoint loads fail closed. + +The assignment is deterministic under canonical actor/edge order. Invalid +half-edge positions are untouched; supported values are bijectively gathered, +so their gradients remain connected. The control is phase-only and cannot be +combined with the separately calibrated A6 DCT comparator. No training or +paper result follows from implementation or software tests alone; formal +A4 runs still require the frozen pilot, server GPU admission and their own +immutable run receipts. diff --git a/docs/PHASESET_V2_A6_SOURCE_QUALIFICATION.md b/docs/PHASESET_V2_A6_SOURCE_QUALIFICATION.md new file mode 100644 index 0000000..791eb0c --- /dev/null +++ b/docs/PHASESET_V2_A6_SOURCE_QUALIFICATION.md @@ -0,0 +1,30 @@ +# PhaseSet-V2 A6 source qualification (not training authority) + +The A6 comparator forms actual signed `(a+b)/2` and `a-b` joint-velocity +signals before its six-band DCT. It does not use the Morlet response cache or +Morlet energy floors. The code-level DCT-only disk path and receipt binding +were qualified by 95 scoped CPU tests against source commit `5506bec`. + +A separate private, read-only qualification on the registered research server +checked all 253 native development parent captures against their admitted +body22 bytes and rebuilt the training-source manifest for each of the five +registered DCT-floor populations. Their parent counts were main 202, pilot +34, fold 0 134, and folds 1 and 2 188 each. The three sealed test components +were absent. It checked the fitted DCT receipts against those exact population, +configuration, calibration, and source-manifest identities, then repeated the +body-byte audit after a native sanity computation. + +For that sanity computation, one deterministic four-person, 5,800-frame +*training* capture ran the full-capture A6 relation encoder at width 512. +Forward and backward completed with 49 finite gradient tensors. The run used +one CPU, no CUDA, no captions, no test-gallery motion, and no optimizer step. +The private immutable receipt bundle is identified by SHA-256 +`496bfa65844d252ac25dcb97094547cc627c30be3f48cf991ec8868eaf1f814e`. +Neither the restricted body files, participant/source identifiers, floor +values, nor the private receipt bundle are distributed here. + +The terminal status is `PASS_SOURCE_BINDING_ONLY`: it is not an A6 learning +pilot, GPU resource profile, formal training run, retrieval score, or paper +claim. Future attempts must bind their own run inputs and checkpoint to the +verified receipt and satisfy the study-wide GPU, cost, and storage gates. +Formal V2 training remains 0/87 and the sealed test has not been scored. diff --git a/docs/PHASESET_V2_A7_GENERIC_LOCAL.md b/docs/PHASESET_V2_A7_GENERIC_LOCAL.md new file mode 100644 index 0000000..bc724b9 --- /dev/null +++ b/docs/PHASESET_V2_A7_GENERIC_LOCAL.md @@ -0,0 +1,35 @@ +# A7: generic local relation control + +A7 is the registered `V2-043`–`V2-045` residual control. It keeps the same +frozen B2 global anchor, adapted whole-sentence text vectors, complete-capture +timeline, actor-incidence topology, directed local-packet matching, calibrated +score and weak counterfactual training objective as PhaseSet-V2. The only +mechanism change is the input to each directed half-edge: shared per-actor +signed-velocity patch mean/RMS, the endpoint actor histories and their relative +pelvis position. There is no six-band relation descriptor, Morlet response, +phase mask or energy-floor input to the A7 neural path. A valid local packet +requires both actor tracks, not physical-band support. + +Each edge is still generated in canonical 64-edge blocks; the half-edge input +width is `2 * width + 3`. At width 512, the A7 encoder has 21,504 fewer +active parameters than the full physical encoder, approximately 0.13% of the +full encoder. No dummy trainable parameters are added. A7 cannot be combined +with topology-off, A2/A3/A4/A5, DCT or speed-only controls. Checkpoint extra +state records `generic_local=true`, so a full-model checkpoint cannot resume +as A7 or vice versa. The matrix-bound host retains the same nonzero CF weight +and training-text eligibility; unlike A8, it does not suppress CF loss. + +This is a *phase-independent score* control, not yet a phase-free end-to-end +data pipeline. The current common `ContinuousTrainingInput` transport loads +and verifies a shared physical cache for all residual rows and recomputes it +after eligible yaw augmentation. A7 ignores its physical response arrays when +scoring, but cache generation, storage and yaw recomputation still incur that +cost. Any later runtime/FLOPs table must report (i) active A7 neural work and +(ii) the shared phase-cache frontend separately, or qualify a genuinely +phase-free input route before making an end-to-end cost claim. A7 must not be +credited with a lower data-preparation cost merely because its score is +independent of phase. + +Construction and passing software tests are not training results. The public +factory has `launch_authority=false`; pilot, formal runs and sealed-test +evaluation require independent private admissions and terminal receipts. diff --git a/docs/PHASESET_V2_AMENDMENT.md b/docs/PHASESET_V2_AMENDMENT.md new file mode 100644 index 0000000..edb3d81 --- /dev/null +++ b/docs/PHASESET_V2_AMENDMENT.md @@ -0,0 +1,103 @@ +# PhaseSet-V2 研究修订 — 2026-09-26 + +2026-09-27 后续用户指令已形成 +[关系监督修订](PHASESET_V2_RELATION_SUPERVISION_AMENDMENT.md)：不再等待两位新 +标注者，已有人工主检索标签保持不变；机器关系监督只作披露的弱监督/探索性 +证据，不能替代下文历史的人类挑战成功条件。其接入仍须实际实现和资格。 + +此修订执行用户指定的“升级至顶会程度”聊天。研究目标是区分 +“个体动作相近、协同关系不同”的原生多人场景。旧协议和历史证据保留； +受影响的正式训练使用本修订的87阶段矩阵，不再叠加旧33-run矩阵。 + +## 固定的三个升级 + +1. 方向保持的局部相位：保留带符号的XYZ关节速度，区分pelvis运动与 + 身体相对pelvis运动。沿用既有20Hz六频带核，提取局部复交叉响应、 + coherence、端点能量和支持。低能量相位标记为不可观测。 +2. 时间关系：保留actor–edge incidence，沿同一个人物及同一条边的时间轨迹 + 编码，再做集合读出；完整capture按实际窗口起始时间做有序读出。 +3. 协同语言监督：匹配完整、端点绑定的关系表示，使用variable-positive + InfoNCE及有核实假标签的反事实softplus损失。人物重编号不是负例； + 泛泛动作描述在相位扰动后仍可能为真，不能自动标假。 + +评分单独校准：两分支先变为cosine，在共同学习尺度上以sigmoid权重混合。 +A9使用旧表示、只改变校准，不能把尺度修复归为新机制贡献。 +A6对平均与差分信号先做DCT，保留交叉项；不以端点自功率平均冒充它。 + +坐标约定为共享世界坐标。世界方向反相不自动等于面对面镜像手势的 +语义反相。所有相位操作仍由motion产生，文本不能更改物理字段。 +全人对时间复杂度为O(K²)；流式显存不代表线性时间。 + +## 实验与停止边界 + +| 阶段 | 正式训练阶段 | +|---|---:| +| B0/B1/B2，三seed | 9 | +| TMR-Set/WaMo-Set/MIME-Set，三seed | 9 | +| 旧PhaseSet与V2 residual，三seed | 6 | +| A1–A9，各三seed | 27 | +| 三group-held-out folds，各三seed的B2、V2、冻结强对手 | 27 | +| Inter-X：三seed的B2、V2、原MIME | 9 | +| 合计 | 87 | + +机器矩阵见 [配置](../configs/phaseset_v2_experiment_matrix.json)。 +seeds为1729/2718/31415。base最多30epoch、residual最多20epoch； +最多12短pilot、两轮结构返工、全局三次有根因的故障重启， +正式启动最多90，总保护预算300 GPU-hours。先profile代表性任务，再冻结 +max_steps；每个pilot不超过对应正式阶段20%的optimizer steps。 +默认pilot为三个文献家族各3次、B2一次、V2 head两次，总计12次。 +只允许冻结前一次有依据的统一调整；epoch可统一扩大至base60/residual40， +但仍受300 GPU-hours和已冻结max_steps约束。达到限制后交付真实终态。 + +性能比较保留文献原架构及辅助损失的合理多人适配；机制比较固定B2、 +文本来源及相近有效容量（head参数默认差异不超过5%，不计无用dummy参数）。 +TMR普通双塔不能替代完整TMR，WaMo功率头不能 +替代原方法，MIME多人适配及收敛情况须明确。外折独立训练，权重不跨折复用。 +Inter-X如在其训练集训练，只能称外部双人适用性验证。 + +A1速度输入、A2无时间顺序、A3pair-bag、A4错误incidence、A5无显式phase、 +A6真实mean/difference DCT、A7相同反事实数据的普通局部关系匹配、 +A8无反事实目标、A9只校准旧表示。每一贡献必须有对应可推翻它的对照。 + +## 数据和科学验收 + +保留400/96/76原split，主validation为C00，最终test封存C09/C11/C15。 +用户提供的`PhaseSet_Codex_Handoff_2026-09-26.md`已接入：pilot训练C01/C02、 +验证C03；fold0留出C00/C06/C10/C14，fold1留出C04/C07/C12， +fold2留出C05/C08/C13。各折训练开发组件中除C03及本折held-out的所有组件， +checkpoint selection统一C03，不复用主实验checkpoint。 +冻结强对手B*只能按pilot验证从TMR-Set/WaMo-Set/MIME-Set选择。 +没有观察到真实数据时，不执行正式训练或生成主表数字。 + +真实关系挑战至少200核实最小文本对、100capture、8参与者组件， +由两名不知道模型分数的人工标注者审核。不能以机器改写凑数。 +先确认原生数据中的可观察关系是否足够；不自动追加Multi-TPC。 + +内部科学目标：主双向R@1三seed均值比最强合格文献baseline至少+2pp； +关系挑战至少+5pp；三个外折平均效应为正；方向/结构/反事实对应消融 +支持并进行预登记多重比较；超过A9；非周期相对B2退化不超过1pp； +相同硬件和缓存规则下端到端推理默认不超过强对手2倍。报告不确定性、 +组件级结果及效率trade-off；这些目标不保证会议录用。 + +终态采用READY_FOR_SUBMISSION、COMPLETE_NOT_SUPPORTED、 +BLOCKED_DATA/RIGHTS/RUNTIME/RELATIONAL_EVIDENCE/BUDGET或FAILED_IMPLEMENTATION。 +软件测试、PDF编译和同模型家族审查不替代真实科学验收。 +结论与正负结果一致；不追加seed、epoch、数据集或更换test追逐结果。 + +## 本次实现的边界 + +两个新模块实现固定物理前端、checkpointed时间incidence、 +有序capture读出、共同尺度评分和核实负例损失。它们作为独立研究API， +尚未接入现有host正式训练、文献适配或数据标注。 +handoff确认40帧统计/20帧hop，低频核支持约4秒，不能声称1秒精确定位。 +起始时间编码为两层512D；两层状态可跨chunk连续，但host层的整capture +actor/edge轨迹、halo卷积和关系packet读出尚待接入，不能以窗口embedding +的有序读出冒充它们。评分起始alpha=0.1，无周期支持时显式回退global。 +本次精确组件配置不等于pilot超参已冻结或正式矩阵可启动。 +新前端的训练能量阈值需重新拟合； +旧五速度通道阈值不能直接套用。机制检查使用分析运动，不产生检索成绩。 + +正文与图表最终来自同一冻结aggregate，提供可编辑源、真实限制和失败案例。 +本日查证 [ICASSP2027官方征稿](https://2027.ieeeicassp.org/call-for-papers/) +显示full-paper截止2026-09-23，已过。研究实现继续，投稿目标需按真实开放的 +会议及当届模板/页数/披露规则确认。 diff --git a/docs/PHASESET_V2_CALIBRATION_AND_CACHE.md b/docs/PHASESET_V2_CALIBRATION_AND_CACHE.md new file mode 100644 index 0000000..c03781d --- /dev/null +++ b/docs/PHASESET_V2_CALIBRATION_AND_CACHE.md @@ -0,0 +1,335 @@ +# Signed-vector calibration and continuous physical cache + +This extends the V2 continuous numeric seam. It is not the completed training +runner, literature-baseline reproduction, or a retrieval-results release. + +## Calibration population and arithmetic + +`actor_patch_power_observations` collects each actor/local-patch/band once, +without enumerating edges, reading captions, or consulting energy thresholds. +It uses the same response centers, scalar masks and complex squared-energy +units as `local_pair_chunk`. Marginal actor support is not replaced by the +smaller intersection with any particular partner. + +The estimator remains the previously specified linear fifth percentile: +`h=(n-1)/20`, with exact integer rank and fractional interpolation. Observed +exact-zero power is included; missing kernel support is excluded and counted. +This policy is not tuned on scores. The V2 population is **signed-vector +actor/local-patch power**, not the legacy five-speed-channel window population; +legacy fitted values are not reusable here. + +`fit_directional_energy_floors` requires the full expected capture census and +that invocation's training components. Repeated, unexpected or incomplete +captures and non-training components fail without a returned fit. Runtime +limits fail explicitly rather than sample observations. Final C09/C11/C15 +are never calibration components. C00 can be training in appropriate external +folds; it is excluded from main calibration. C03 is excluded from pilot/fold +calibration but is part of the main training split. Every fit must use its own +declared training census, never reuse another fold's thresholds. + +## Response storage and reuse + +`write_continuous_phase_cache` writes a fresh **private** directory containing +complete complex128 actor/band responses and bool masks as NPY memory maps. +Convolution keeps the original full kernel halo and response arithmetic. +Centers, signed actor features, root features, masks and absolute intervals +are preserved. There is no actor, edge, time or missing-interval sampling. +Disk bytes/free space are admitted before the directory is created, existing +attempts are not overwritten, and `metadata.json` is written last. + +The physical response cache does not fit or bake in thresholds. Its zero-floor +field is collection-only. `load_continuous_phase_cache` requires explicit +fitted floors and expected capture source lineage; thresholds are applied when +constructing local relations, without reconvolving motion. Main/pilot/fold +fits may reuse fixed physical responses but cannot share learned checkpoints +or calibration populations. The existing execution manifest remains +responsible for preprocessing/source admission and file-integrity checks. + +The original RAM API and its 1 GiB response gate remain unchanged. The disk +backend avoids retaining the complete response arrays as heap allocations; +linear velocity/features, OS page cache and neural activations still consume +memory. It does **not** establish constant resident memory or finish long- +capture neural backward scheduling, augmentation, production CLI integration, +or full-corpus training. All real-data artifacts remain private. + +Software and native qualification results are recorded only after server +execution. Synthetic correctness and a one-capture native fit do not amount +to full-training calibration, formal optimization or scientific acceptance. + +## Observed server qualification, 2026-09-27 + +The calibration/cache tests together with continuous preprocessing, V2 +coordination, pipeline and legacy signal/Morlet regression passed **133 tests, +1 CUDA-only skip in 22.17 seconds** on server CPU. Source before/after matched. +Checks include exact RAM/mapped-response parity, pair features, immutable +thresholds, unequal actor masks, calibration isolation and cached gradients. + +The same previously qualified **real four-actor, 120-second training capture** +then passed disk-cache creation, one-capture calibration, read-only reload and +**untrained 512D forward/backward** in **12.42 seconds**, using one CPU thread. +The cache contains 64,038,532 bytes; measured cache write was 1.51 seconds, +calibration 0.15 seconds, forward 2.82 seconds and backward 5.67 seconds. +There are 2,400 frames, 119 patches, six edges and 49 finite gradient tensors, +with nonzero half-edge/topology gradients. These are single qualification +measurements, not p50/p95 benchmarks or evidence of retrieval improvement. + +The one-capture thresholds are **not** the complete training/fold fit. No +captions, final-test input, optimizer steps or retrieval scores were used. +Formal training remains **0/87**; production host integration, complete fits, +faithful literature baselines, pilot and scientific evaluation are unfinished. + +## Full-development storage and split routing + +`continuous_capture_io.load_prepared_continuous_capture` restores a complete +body22 timeline from the preparation manifest's completed record and its +three private arrays. The record, array hashes, headers, canonical actor +lineage and complete window decisions are checked; mmap arrays stay read-only. +This does not replace dataset rights/split admission by the execution manifest. + +`calibration_populations.calibration_populations` reads the V2 matrix's exact +user-supplied component assignments. It returns five distinct training routes: + +- Main: the twelve registered training components; C00 remains validation. +- Pilot: C01/C02 train, C03 selects. +- Fold 0: development excluding C03 and held-out C00/C06/C10/C14. +- Fold 1: development excluding C03 and held-out C04/C07/C12. +- Fold 2: development excluding C03 and held-out C05/C08/C13. + +Thus C00 is training data in folds 1/2, not in main, pilot or fold 0. C03 is +main training data but never pilot/fold training data. C09/C11/C15 cannot be +admitted. Distinct folds refit floors and models; no main-checkpoint reuse. +This helper does not change `PLANNED_NOT_LAUNCHABLE` or launch training. + +The private development consumer admits only completed body records, checks +the frozen eligible census, creates whole-capture physical caches, and waits +for the producer's successful source-bound receipt before fitting any floors. +If the producer finishes while a cache snapshot is being processed, it must +re-read and drain all newly completed records before verifying the terminal +census. No partial sample can masquerade as complete main/fold calibration. +The full body/caption/model inputs, physical arrays and fitted population +files remain private; the public release contains APIs and data-free tests. + +The extended server CPU qualification passed **146 tests, one CUDA-only skip +in 21.06 seconds**, including three private orchestration regressions. The +producer-completion race is tested with mocks explicitly marked as sequencing +tests, not as research data or results. The public storage/population tests +are data-free and included in the package's source release. + +Three already prepared **real native development captures** separately passed +admitted-record restoration, complete physical-cache creation, read-only +reload and artifact recording in **2.94 seconds** (94,609,294 cache bytes). +This is a bounded input-preparation qualification, not a 512D model profile, +training-population fit or retrieval result. No captions, final-test input, +optimizer steps or fitted floors were used. The complete 494-capture consumer +has been launched; it is not yet declared complete, and formal training is 0/87. + +The disk cache is an input-preparation stage, not a completed V2 training +host. CLIP/counterfactual integration, faithful baselines, actual profile/pilot +and the 87-stage formal matrix remain separate required work. + +## Whole-capture training input and shared yaw + +`continuous_training_input.ContinuousTrainingInput.load` binds the body22 and +physical records through the consumer's `prepared_record`, verifies their +fixed artifact files, and restores one complete, immutable capture. Load once +per admitted capture rather than rehashing every response on every step. +Rights, split admission and the fitted calibration population still belong to +the frozen execution manifest. A positive key is `source_sha256`, never the +possibly reused actor-set/group commitment. + +`view(yaw_delta=0, allow_shared_yaw=False)` reuses the read-only physical maps. +A nonzero view rotates **all actors and all frames together** about the same +existing group origin; it never recenters on a person or resets at a window. +Original masks, rejected intervals, absolute times and capture identity stay. +The transform explicitly operates on already prepared float32 body22; it does +not claim bitwise equivalence to rotating raw parameters before resampling. + +Nonzero yaw recomputes complete signed velocity, Morlet responses, actor +signed-mean/per-channel-RMS and root statistics using the same frontend and +the unchanged fitted floors. It does **not** rotate the RMS vector, reuse stale +zero-yaw actor/root values, or assume floating-point phase invariance. The +RAM/edge resource gates apply to the whole capture; an oversized augmentation +fails explicitly, with no zero-yaw fallback, dropped actors or shortened time. +This correctness-first route reconvolves motion and its cost must be measured +in the actual host profile; augmentation is not advertised as free or mmap-only. + +The caller must explicitly decide text yaw eligibility. Captions describing +world directions cannot silently receive incompatible motion augmentation; +this data-loading module neither reads nor rewrites any caption. A rejected +yaw request fails rather than silently proceeding without augmentation. +The frozen patch/hop/epsilon configuration cannot change between views. + +Data-free tests cover exact masks and whole timeline, rotation about the +shared origin, correct RMS rather than the rotated-RMS shortcut, complete-field +parity with direct recomputation, full-trajectory gradients, cache drift and +resource/text-eligibility failures. These are implementation checks, not +retrieval improvements or formal optimizer runs. + +The extended server CPU suite passed **154 tests, one CUDA-only skip in +26.21 seconds**, with unchanged source and invocation files. The same fixed +real four-actor, 120-second training capture then passed the admitted +zero-yaw cache view, full nonzero-yaw recomputation and two untrained **512D +forward/backward** evaluations in **20.49 seconds** total. Complete fields, +embeddings and all parameter gradients match direct augmented-field execution +bitwise in this frozen runtime; 49 gradient tensors are finite and the +half-edge/topology branches have nonzero gradients. Peak RSS was 1,180,908 KiB. + +Measured admission was 0.49 seconds and yaw reconvolution 1.69 seconds. These +single-capture qualification timings do not establish end-to-end throughput, +p50/p95 latency or a GPU cost bound. Floors came from the prior fixed +one-capture QA fit, **not** the full main/pilot/fold population. No captions, +final-test input, optimizer or retrieval scores were used. The production +optimizer/CLIP/counterfactual/baseline host and 87 formal stages remain undone. + +## Complete-capture sentence score and loss seam + +`continuous_retrieval.ContinuousRetrievalSystem` reuses the fixed 512D B2 +anchor and admitted `FrozenClipTextBatch` receipts. B2 stays frozen and in +evaluation mode even when the residual system trains. Every accepted window +is encoded in the same capture frame; its fixed mean supplies only the global +branch. No legacy base temperature is added to the bounded coordination cosine. + +The new shared text adapter is 512 -> GELU -> 512. The coordination encoder +keeps its original complete ordered actor/node/pair/group histories. For +language, each **directed** packet retains its ordered half-edge and both +ordered endpoint node contexts; both orientations share update/temporal +weights. Complete trajectories are streamed in 64-edge blocks and recomputed +in backward. Tokens are matched to whole sentence vectors, not independently +maximized subject/action/object fields. No actor ordinal or lineage ID is a +neural feature. + +The pre-pilot rule is fixed in the experiment matrix: +`r = 0.5 * ordered_capture_cosine + 0.5 * max_directed_packet_cosine`. +This bounded weighted local cosine is a scoring implementation, not a fourth +novelty claim. Group temporal parameters are effective, not dummy capacity. +Language maxima include only energy-observable patches; zero-energy track +support can retain history but cannot enable the periodic score. Low coherence +still does not remove an edge. A capture with no observable periodic evidence +has exact-zero coordination cosine and falls back to the global branch. + +No all-pair packet tensor is retained. A block score matrix over text rows has +an explicit 512 MiB storage gate; callers must batch a larger gallery, never +drop actors, edges or time to fit it. Activation, physical-cache and complete +capture host memory are not constant-memory claims. Actual timing and cost +must be established in the later hardware profile. + +The official holistic schema's **all `scene_explained` human sentence rows** +are used, without concatenating mood/atmosphere questionnaire fields or +machine-fusing annotations. Row count is variable. Source capture SHA identifies +the actual motion input, not its positive-target family. Released siblings can +share the same parent holistic file. `forward` requires separately admitted +motion/text positive-family keys and labels all siblings as positives; repeated +actor sets still do not create positives. Schema selection is not proof of motion observability; +official file provenance and human relation truth remain separate requirements. + +`score` accepts arbitrary gallery/CF sentence batches without assigning any +positive labels; `forward` adds admitted target-family positives only for a valid +contrastive batch. A gallery chunk need not contain all matching motion rows, +and false-CF text is never forced to carry a fabricated positive capture key. + +The loss interface requires explicit counterfactual scores and verified-false +masks. It cannot certify those human labels, and no false negative is invented +from generic captions. Empty explicit tensors cover implementation-only +contrastive checks or the no-CF control, not a completed full-CF training run. +The existing low-level atomic checkpoint and RNG helpers can be reused without +relaxing the old qualified-winner/execution guards. This new seam does not yet +claim a formal optimizer scheduler, full corpus calibration, faithful TMR/WaMo/ +MIME adaptations, a trained B2 checkpoint, GPU profile/pilot, verified-CF +dataset, sealed evaluation or any of the 87 formal training stages. + +The expanded server CPU regression passed **164 tests, one CUDA-only skip in +46.50 seconds**, with unchanged source and invocation files. This includes the +small synthetic optimizer/save/resume sequence and all prior continuous, +Morlet and legacy pair checks. The first immutable attempt had 163 passes and +one overly strict stationary-gradient sign-bit assertion: zero derivatives +included IEEE negative zero. Only that test was corrected to finite numerical +zero; the exact-positive-zero score and separate K=2 topology value/gradient +bit-pattern checks remain strict. The failed attempt is retained, not rewritten +or counted as a formal training restart. + +A real development-only audit found **494 released segments but 253 parent +capture/holistic annotation families**. Exactly 138 families repeat across +segments; each file family maps to one parent stem, with no cross-component or +train/validation family duplication. The 494 files contain 2,613 sentence rows +including these copies (2,144 train, 469 validation); their per-segment row-count +histogram is 1:2, 4:20, 5:437, 9:4, 10:31. These are not 2,613 independent human +annotations. Final-test captions were not read. + +The first native qualification's two segments shared a holistic file; its +finite backward was an execution check, not valid evidence of independent +retrieval targets. This prompted the explicit positive-family interface before +formal training. Parent-level motion/task aggregation, deduplicated gallery, +positive mapping, independent sampling/statistics and effective-test sealing +still require a unified contract freeze. Existing per-segment caches are input +artifacts, not proof that that task is closed. Do not treat sibling segments as +false negatives, bootstrap them as independent parent captures, invent a split +leak, change the participant-disjoint split, or concatenate missing time away. + +After separating source and positive-family identities, the fresh server suite +again passed **164 tests, one CUDA-only skip in 46.98 seconds** with unchanged +source/invocation. Two **different** real C01 annotation families then passed +actual frozen-CLIP, complete **120/260-second** motion, all **12/26** accepted +base windows, **119/259** ordered patches, 10 human sentences and untrained +512D forward/backward in **151.26 seconds** total. All 55 trainable gradient +tensors were finite, required branches had nonzero gradients, frozen B2 stayed +unchanged, and the original group readout remained bitwise identical to the +previous qualified source in this runtime. Forward/backward were 51.69/71.94 +seconds; peak RSS was 1,856,512 KiB. This is CPU execution qualification, not a +GPU throughput profile or retrieval result. + +The same real-file audit counted **1,334 human answer-row slots when each +annotation family is counted once**, not 2,613 independent annotations and not +1,334 independently verified relation examples. No text truncation occurred in +this two-family check. It used one-capture QA floors and random untrained B2: +zero optimizer, pilot, formal, verified-CF and final-test runs. Complete parent +task admission, true human relation evidence and the 87-stage study remain +unfinished; their pending state must not be replaced by this qualification. + +## Raw parent assembly, not concatenated preprocessed children + +`parent_capture.ParentCaptureEpisode` accepts caller-licensed raw world body-22 +tracks with native source provenance and their actual half-open 30 Hz frame +intervals. `assemble_parent_capture` aligns the same real actors across releases +and preserves the complete parent timeline. Unreleased intervals retain false +masks and exact positive-zero coordinates; they are not stationary motion, +interpolated poses or deleted time. Overlap or changed actor membership needs +an explicit data-contract resolution. Lineage is not a neural input. + +Only after assembly is the existing capture-wide origin, shared yaw, FIR and +30-to-20 Hz grid applied once. Individually recentered/resampled child caches +cannot reproduce this raw-input operation. The assembly budget is explicit +and linear in actors and frames, not a claim about combined neural peak memory. +The existing complete-10-second window policy is unchanged: a shorter final +source tail is declared separately, not hidden or padded with invented poses. + +The expanded registered-server CPU regression passed **170 tests, one CUDA-only +skip in 44.21 seconds**. It covers actor/episode permutation, direct full-world +preprocessing, original absolute intervals, 15/150-second holes, short native +tracking loss, tail accounting and input/resource rejection. Exact K=2 topology, +legacy pair checks and same-runtime bitwise invariance remain strict. An +independent directed-packet oracle uses frozen `rtol=2e-6, atol=2e-7`: separately +executed and batched GRUs produced at most 2.98e-8 observed score rounding +differences on Windows. This is not tolerance for actor permutation or old +readout equality, and no model/runtime guard was changed. + +A preselected real development parent then passed raw licensed geometry, +assembly, physical-cache loading, native frozen CLIP and untrained 512D +forward/backward in **248.26 seconds**. Its three native releases contain +**7,650 observed frames** within an **8,550-frame (285-second)** parent span, +including **900 missing frames (30 seconds)**. The existing policy retains +**5,600 target frames**, declares the remaining 150 source frames, accepts +24 ten-second windows and rejects four. All 28 gap-only patches have no +actor/edge physical evidence. Raw actor/episode permutation is bitwise equal; +all 55 gradient tensors are finite and frozen random B2 remains unchanged. +Five real human sentence rows have no CLIP truncation. Forward/backward took +35.31/48.90 seconds; peak RSS was 1,969,516 KiB. + +This is a CPU execution qualification using one-parent QA floors and scalar +score differentiation, **not** an all-positive one-parent InfoNCE experiment, +trained checkpoint, GPU profile, pilot, verified-counterfactual dataset or +formal run. Complete development-parent artifacts, five full train-only +calibrations, the unified parent gallery/sampling/statistics contract, faithful +literature controls, human relation verification and all 87 stages are still +required before final research delivery. The original failed invocation is +preserved; its existing SMPL-X library-path omission was corrected without +installing packages or changing the qualified model source. diff --git a/docs/PHASESET_V2_FROZEN_CLIP_ROWS.md b/docs/PHASESET_V2_FROZEN_CLIP_ROWS.md new file mode 100644 index 0000000..c4855df --- /dev/null +++ b/docs/PHASESET_V2_FROZEN_CLIP_ROWS.md @@ -0,0 +1,54 @@ +# Frozen human CLIP rows for complete-parent training + +`ParentHumanClipRows` takes one original sealed CLIP preparation batch and the +complete caller-admitted development `ParentRetrievalTask`. All official human +occurrences must be present, in canonical task order, with their exact UTF-8 +text digest and source/family/ordinal commitment. Equal sentence strings are +not deduplicated. No motion-derived information or fitted learning statistics +enter these frozen text features. Final-test parents are rejected. + +Prepare with the existing pinned offline CPU-only CLIP adapter, save its +original receipt and raw projected FP32 embeddings, and authenticate them with +the existing original-batch rehydrator. Select rows in the actual parent batch +order using `select_frozen_clip_text_rows`. This does not construct a CLIP tower +in the training process or relax the preparation runtime contract. + +Each selection has the distinct `phaseset-frozen-clip-row-selection-v1` schema +and `ROW_SELECTION_AUTHORITY0` status. It records original row indices, the +original receipt identity and unchanged token/embedding/commitment provenance. +It is explicitly a cached-row operation, not a new encoder invocation. Original +runtime/model/source identities describe preparation, not the selection host. +No caption or commitment can be supplied anew to the selection factory. +Nested selection and duplicate indices are rejected. Persist the original +batch; the original rehydrator deliberately rejects derived selection receipts. + +Only the continuous base/V2 scoring and parent-training validators opt into +this derived schema. Legacy prepared-capture validation/storage admission is +unchanged. Returned embeddings remain snapshot-owned contiguous frozen CPU +FP32; the scoring module moves these numbers to its model device as usual. +This promises exact reuse of the stored preparation rows, not universally +bitwise-identical Transformer outputs under different inference batch shapes +or hardware. + +The existing CLIP-native 77-token truncation policy is retained, with each +original token count and truncation flag preserved. Actual private preparation +reports how many captions exceed it. This differs openly from the untruncated +[TMR/WaMo language rows](PHASESET_V2_FROZEN_LANGUAGE_ROWS.md). MIME still trains +its own genuine CLIP tower; this cache is not its text training path. + +`DevelopmentParentDiskSource(..., human_clip_rows=...)` supplies its human-only +`text(labels, training=...)` interface with empty counterfactual indices. Empty +CF rows mean no evidence: neither a negative truth label nor permission to run +the formal V2 CF objective without the two real blind human verifications. +The full human-verified CF pool, semantic yaw policy, run-specific floors, +production locks, heartbeat, budget, hardware profile and study freeze remain +separate inputs and obligations. No pilot or formal stage is authorized here. + +Analytic tests cover ordered exact reuse, ownership, RNG preservation, altered +ancestry/text/task rejection, original-schema exclusion and base/V2 optimizer, +full validation and bitwise interruption/resume. Actual native preparation must +separately verify every admitted human occurrence and all three seed orderings. +Neither software tests nor feature preparation are native learning results. + +See the [development disk source](PHASESET_V2_PARENT_DISK_SOURCE.md) and +[complete-parent training host](PHASESET_V2_PARENT_HOST.md). diff --git a/docs/PHASESET_V2_FROZEN_LANGUAGE_ROWS.md b/docs/PHASESET_V2_FROZEN_LANGUAGE_ROWS.md new file mode 100644 index 0000000..97a0baa --- /dev/null +++ b/docs/PHASESET_V2_FROZEN_LANGUAGE_ROWS.md @@ -0,0 +1,41 @@ +# Fixed TMR/WaMo human-language features + +`phaseset_core.tmr_feature_cache` stores actual frozen TMR language outputs +for private complete-parent development data. It is not an encoder, a new +annotation, a label filter, or evidence that arbitrary input files are official. +The private preparation caller uses the admitted pinned DistilBERT and MPNet +snapshots through `FrozenTMRLanguage`, without training or content truncation. +WaMo reuses the actual DistilBERT CLS token, not a TMR projection or MPNet/CLIP +substitute. MIME's own genuinely trainable CLIP is **not** cached here. + +Only contiguous masked right padding is removed. Every content token and +special token is retained. Reload dynamically restores exact positive-zero +padding and masks; valid tokens, CLS and sentence features match the stored +preparation outputs exactly. The preparation batch shape and frozen runtime +are part of that feature snapshot; this is not a universal bitwise claim +across different Transformer batching layouts or hardware. + +Exact duplicate UTF-8 text can reuse a fixed feature file. Annotation +occurrences, caption commitments, complete-parent sources, families and all +variable positive rows remain distinct in `ParentRetrievalTask`. The cache +never samples captions or changes their whitespace, wording or order. + +Files are written once into a fresh private directory. Loading verifies +closed manifest entries, complete array headers, finite FP32 values, token +lengths and file identity. Unknown rows or changed artifacts fail explicitly; +they do not trigger a substitute model, online download or regeneration. +The private caller additionally verifies the admitted development population, +official human-file lineage, exact feature reload, unchanged model weights +and RNG, and source/asset identity before and after preparation. The final +test population is excluded, and no learned training statistic is fitted. + +Analytic tests cover ordering, repeated occurrences, masks, padding, CLS, +file drift, unknown rows, and TMR text distribution/gradient preservation. +They are not proof of pretrained assets. That requires the separate actual +private generation receipt. Neither a cache, these tests, nor a same-family +review is optimization, a pilot, convergence, a hardware profile, a retrieval +result or completion of the research project. No data or model-derived +feature files belong in this public repository. + +See the [literature optimizer host](PHASESET_V2_LITERATURE_PARENT_HOST.md) +and [TMR adaptation](TMR_SET_ADAPTATION.md). diff --git a/docs/PHASESET_V2_HANDOFF_INTEGRATION.md b/docs/PHASESET_V2_HANDOFF_INTEGRATION.md new file mode 100644 index 0000000..abdd2be --- /dev/null +++ b/docs/PHASESET_V2_HANDOFF_INTEGRATION.md @@ -0,0 +1,116 @@ +# 2026-09-26 handoff integration + +The user-supplied Markdown handoff resolves the missing component definitions. +It is preserved privately; no ZIP, licensed data, body model or download grant +is asserted to have been supplied with it. + +The 87-stage JSON now encodes the exact C01/C02 pilot learning, C03 selection, +ten out-of-fold components in three folds, and sealed C09/C11/C15 exclusion. +Handoff fold indices 0/1/2 map explicitly to existing run split labels 1/2/3. +The total remains 87 planned stages, not 87 completed experiments. Strong +comparator selection and exact step limits still require the real pilot. + +The bounded implementation changes are: + +- Mean/difference DCT uses `(a+b)/2` and `a-b`, not half the difference. +- Two masked recurrent layers carry both track states across temporal chunks. +- Shared-cosine fusion starts at alpha 0.1 and accepts an explicit support mask + to fall back to the global branch for unsupported motion rows. +- Pilot allocation includes B2, each pilot is capped at 20% of its formal + optimizer steps, and mechanism head capacity tolerance is 5% without dummy + parameters. Any permitted 60/40 epoch expansion precedes formal freeze and + remains subject to the 300 GPU-hour cap. + +## Observed software verification + +Fresh GPT-5.6-Sol review found no blocking issue; semantic review remains +provisional. Server-only execution of the V2 coordination, Morlet and legacy +PhasePair signal suites: **65 passed, 1 CUDA-only skip in 9.89 s**. The separate +512D, three-actor CUDA forward/backward witness: **1 passed in 5.17 s**, RTX +A6000, peak allocated **180,931,072 bytes**, reserved **205,520,896 bytes**. +These are analytic mechanism checks, not motion-language retrieval results. + +No real native-data training, human relational challenge, sealed-test scores, +paper aggregate or scientific acceptance is implied. Whole-capture actor/edge +state host orchestration, disk-backed response streaming, relation-packet integration +with the host runner, and legitimate literature adaptations remain to finish. +The prior exact-commit CI and anonymous-codeload receipts retain their original +scope and are not relabeled as verification of this new revision. + +## Asset discovery correction + +The current [official Embody page](https://www.meta.com/emerging-tech/codec-avatars/embody-3d/) +exposes an organization-email field and XRCIA acceptance/download form. +Do not insist on an approval email as the only possible access evidence. +The [official repository](https://github.com/facebookresearch/embody-3d#download-data) +documents the resulting download links. Neither this discovery nor the +repository's code license establishes that restricted captures may be used or +deployed anywhere; the actual data license and grant must be respected. +The [SMPL-X model instructions](https://github.com/vchoutas/smplx#downloading-the-model) +require registration and the model license. A Python package or pytest fixture +with a model-like filename is not the licensed body-model asset. + +## Capture-core follow-up, 2026-09-26 + +Shared two-layer actor temporal states now precede half-edge construction; +actor histories are not pooled or reset between local patches in one field. +Morlet convolution uses complete kernel halos around each central response +block (default 1024 frames), computing each response once. The resulting +complete response cache still has an explicit 1 GiB limit: this is bounded +temporary convolution, **not** a completed disk-streaming host. + +The updated modules passed server CPU checks: **70 passed, 1 CUDA-only skip** +in 13.64 seconds, including chunk lengths 1/7/31/64 versus unchunked responses, +gap-mask equality, actor-state continuity, K=2 exact-zero regression, and the +legacy signal/Morlet tests. Before/after source snapshots are identical. +This is software qualification; formal V2 training remains 0/87. + +## Continuous native capture follow-up, 2026-09-26 + +`prepare_continuous_capture` now preserves one group origin/yaw and the complete +absolute timeline across accepted ten-second intervals. The original 300-frame +missing-data decisions are unchanged; rejected windows remain zero/masked on +the timeline, rather than concatenating their neighbors. Anti-alias FIR chunks +include the full halo, and targets touching rejected FIR support are masked. +Original short-gap observation masks remain false. Complete windows are used; +any trailing source frames are explicitly recorded. + +`continuous_directional_phase_field` consumes the unbatched capture seam, +preserving signed velocity and six-band support across window boundaries. The +legacy short-window `PreparedGroupBatch` contract has not been relaxed. +Actor, node, edge and group temporal encoders therefore see one whole field, +not separately initialized ten-second fields. Base-window views share the +same capture-wide coordinates and retain their absolute source starts. + +Server-only checks passed **106 tests, 1 CUDA-only skip in 18.24 s**, including +FIR chunk lengths 1/7/31/300/1024, actor permutation bit equality, rejected +intervals, retained short gaps, shared yaw, absolute window offsets, cross-seam +Morlet support, the original preprocessing/pipeline and legacy signal suites. +Source snapshots before and after execution are identical. + +A real native four-actor, 120-second training capture passed the complete +official-model conversion and **untrained** 512D temporal forward in **63.55 s**: +3600 source frames, `[4,2400,22,3]` at 20 Hz, 12 accepted windows, 119 local +patches and six complete-graph edges. There was no caption semantic reading, +trained retrieval score, optimization or test evaluation. Zero energy floors +were solely a numerical fixture, not production calibration. + +This closes the continuous numeric seam and a native full-capture forward, +not the full-corpus training host, disk-backed cache/backward scheduling, +training-only floor fit, literature baselines or scientific experiments. +The response cache still enforces its 1 GiB RAM limit without sampling actors +or edges. Formal training remains **0/87**. + +## Updated CUDA mechanism witness, 2026-09-26 + +The same continuous-capture source passed a fresh server-only 512D, three-actor +CUDA forward/backward check: **1 passed in 11.30 s**, NVIDIA RTX A6000, peak +allocated **206,455,808 bytes**, reserved **249,561,088 bytes**. Admission +required observed device use below 500 MiB; the allocator was bounded to 2 GiB. +Source snapshots match and the final attempt exit is zero. This supersedes no +earlier receipt: the earlier peaks above belong to their earlier source. + +The witness uses an analytic input and no optimizer or retrieval scores. It +does not qualify full-corpus training, establish accuracy, or count as one of +the 87 formal stages. Actual development body-model conversion is a separate +private server job; final-test captures are excluded from that conversion. diff --git a/docs/PHASESET_V2_ICMR_TARGET.md b/docs/PHASESET_V2_ICMR_TARGET.md new file mode 100644 index 0000000..77ec1b9 --- /dev/null +++ b/docs/PHASESET_V2_ICMR_TARGET.md @@ -0,0 +1,17 @@ +# 用户选择的 ICMR 2027 目标 + +2026-09-27 用户将目标从 ICASSP 改为 **ACM ICMR 2027 regular long paper**。 +这改变投稿包装，不修改三seed/87阶段/12pilot/300GPUh及科学主张/negative branch。 +旧ICASSP模板/记录保留为历史，不移动旧tag，不宣称曾提交或已获录用。 + +当前官方[日期](https://www.icmr2027.org/date)：regular paper截止2027-01-31 +23:59:59 AoE。官方[征稿](https://www.icmr2027.org/cfc)要求ACM双栏、double-blind， +`acmart`选项`sigconf,review,anonymous`，long paper正文≤8页、纯参考文献≤2页； +不允许附录或supplementary。投稿入口细节尚待官方公布，提交前须再次核验。 + +公开开发仓库继续保留；匿名投稿稿件不可直接链接识别作者的网站或公开owner。 +实验完整证据/可复现release继续作为项目交付，但不冒充大会允许的投稿附件。 +关系证据按后续[监督修订](PHASESET_V2_RELATION_SUPERVISION_AMENDMENT.md)披露： +不等两位新标注者，机器不冒充真人挑战；数据权利、真实训练/统计不以此省略。 +新ICMR论文须由同一冻结aggregate生成，尚无真实结果时不填写结果数字。 +机器可读目标见[venue contract](../configs/phaseset_v2_venue.json)。 diff --git a/docs/PHASESET_V2_LEGACY_PARENT_SCORE.md b/docs/PHASESET_V2_LEGACY_PARENT_SCORE.md new file mode 100644 index 0000000..e48623d --- /dev/null +++ b/docs/PHASESET_V2_LEGACY_PARENT_SCORE.md @@ -0,0 +1,54 @@ +# Old PhaseSet and A9 complete-parent score seam + +`LegacyWholeCaptureRetrievalSystem` scores every accepted ten-second window of +an already prepared capture, with no actor or edge sampling. Its legacy motion +branch remains the old system-08 Morlet/13D/six-band encoder; each band token is +the masked mean over all supporting windows. The frozen B2 branch uses the same +complete-parent mean and normalization as the V2 base readout. + +Two score modes share this representation: + +- `old`: frozen B2 temperature-scaled logits plus the original bounded + `PhaseSetRetrievalHead` relation residual. +- `A9`: frozen B2 cosine and the same old relation cosine under the common + learned cosine calibration. This changes score scale, not relation features + or the old text-band projection architecture. + +The mode, window batching, edge chunk, scalar-floor values and typed floor +receipt are bound to the scorer state. The receipt's method is the unchanged +old 20 Hz five-speed-channel Morlet actor-local power, with a linear fifth +percentile per band over every accepted actor-window in the declared training +parent census. Observed exact-zero powers count; absent Morlet support does +not. Each band's observed and missing counts must total the actor-window +census. The receipt binds the private source-manifest digest, population, +physical kernel identity, floor bytes and diagnostics. The data-free test uses +a synthetic receipt and does not imply that native floors were fitted. + +The complete-parent training host admits this scorer only when the source, +scorer and host bindings name the same typed scalar-floor receipt, its training +population matches the host, and the run ID/seed/system/B2 predecessor is one +of the six old/A9 rows in the V2 matrix. The current host candidate also pins +main training/validation components, 20 epochs and its optimizer/CF defaults; +these defaults are not a completed pilot freeze. A short synthetic run cannot +be labeled as a formal row. The base and V2 host paths reject a +legacy receipt. The host still does not establish data rights, +provenance or run budget by itself; the private operator must verify those +inputs and the real server qualification before formal training. No final-test +interface is added. + +`build_v2_legacy_control` constructs only registered V2-019/020/021 (old) +and V2-049/050/051 (A9) rows from the exact frozen matrix. It reuses the +same validation-selected B2 checkpoint/terminal verifier as the V2 residual +factory, requires the same-seed B2 predecessor and main population, and +checks that the typed old scalar floor names the admitted training source. +`make_v2_legacy_parent_host` then binds that build, floor, predecessor and +source to the complete-parent host. The build explicitly carries +`launch_authority=False`: construction and synthetic tests cannot admit a +native optimizer step, decide the pilot schedule, or turn an unverified floor +receipt into real-data evidence. + +Data-free tests check both modes' score decomposition, frozen B2 gradients, +floor mutation rejection, unbound-host rejection, guarded score/VJP replay and +short-run mislabeling rejection, and six registered factory rows on synthetic +inputs. Passing those checks is +software qualification, not a retrieval result or evidence for a paper claim. diff --git a/docs/PHASESET_V2_LITERATURE_PARENT_HOST.md b/docs/PHASESET_V2_LITERATURE_PARENT_HOST.md new file mode 100644 index 0000000..7194917 --- /dev/null +++ b/docs/PHASESET_V2_LITERATURE_PARENT_HOST.md @@ -0,0 +1,78 @@ +# Complete-parent literature optimizer host + +`phaseset_core.literature_parent_host.LiteratureParentTrainingHost` uses the +same explicit Adam(W), warmup/cosine, gradient clipping, validation checkpoint selector, +immutable attempt and checkpoint/resume loop as the complete-parent base and +residual host. The original constructor still admits only its actual base/V2 +scorers; literature models enter through their own strict constructor. +`ParentHostConfig.for_literature` supplies the disclosed method-specific +optimizer/decay defaults; those settings are exact resume inputs. + +Actual MIME group training uses `MIMEParentTrainingHost`, not the generic +uniform-batch literature host. Its `_batches` hook invokes the qualified +training-only MIME curriculum at every epoch, including resume/census checks. +The caller prepares anchors once from all learning-parent human CLIP rows; +neighbor width and an owned snapshot of anchor values join the checkpoint manifest. The genuinely +trainable MIME CLIP tower remains separate from these frozen sampling anchors. + +## Objectives and inputs + +- TMR-Set: complete stochastic VAE objective, both full-track reconstructions, + four KL terms, latent alignment and MPNet-filtered full-batch contrastive loss. +- WaMo-Set: both reconstructions, original/shuffled temporal classification, + and the original **sum** of contrastive directions. +- MIME-Set: genuine trainable CLIP language tower and learned score scale; + the original **average** of contrastive directions. Original role-aware + Inter-X MIME is a distinct interface and is not admitted by this group host. + +The host dispatches the already-qualified per-parent cache/VJP/replay seams, +not a common replacement loss. Every effective batch retains its entire +rectangular motion-by-human-caption negative gallery, variable positives, +and every accepted window at its absolute time, including windows after gaps. +Source reload augmentation must depend on source/seed/epoch, independently of +model dropout and VAE sampling RNG. It must reproduce the same arrays each time. + +`LiteratureParentSource` supplies entire `PreparedContinuousCapture` objects, +frozen TMR token/sentence features, and frozen WaMo DistilBERT CLS features. +MIME uses its own actual trainable tokenizer/tower. Language asset identities, +source adapter, preprocessing and complete census belong in the operator's +private input/code bindings; no private download URLs or captions are supplied +by this public package. + +## Validation and restart + +All validation human rows are encoded once; motion parents are streamed one +at a time without auxiliary loss or augmentation. Scores follow the methods' +public `score` equations, with TMR distribution means at evaluation. The +selector remains capture-macro bidirectional R@1 against registered family +positives, deterministic lineage tie breaks, and first-checkpoint retention +on exact ties. Model mode and dropout/sampling RNG are restored afterwards. + +The manifest additionally records the method and its loss configuration; +resume rejects changed objective coefficients as well as execution bindings, +initial state, task, scalar/module configuration and optimizer settings. +The shared checkpoint loop retains optimizer/scheduler/RNG state, source census, +cursor checks, predecessor evidence and terminal failures. + +## Qualification is not research completion + +The analytic tests exercise real optimizer updates, method-specific losses, +full validation endpoint, dropout/VAE bitwise interrupted resume, objective +drift refusal, and exclusion of held-out/test rows. They are expressly not +native optimization, a pilot, original-task convergence or a hardware profile. +The existing base/residual host and original model/seam tests must pass too; +the real Transformers CLIP component must execute, not skip. + +Native default-model replay qualification separately uses complete real +development captures, all human rows, and each original dense objective as +the gradient oracle. It proves objective preservation, not baseline accuracy. + +No constructor, software test or code review authorizes formal training. +The dated whole-parent training-unit amendment, hardware profile, bounded +pilot, study/budget freeze and rights remain private admission steps. No +final-test interface is provided here. Same-family semantic review remains +provisional; research metrics, statistics and paper results are still pending. + +See the [WaMo/MIME replay contract](PHASESET_V2_LITERATURE_PARENT_TRAINING.md), +[TMR replay contract](PHASESET_V2_TMR_PARENT_TRAINING.md), and +[complete-parent host](PHASESET_V2_PARENT_HOST.md). diff --git a/docs/PHASESET_V2_LITERATURE_PARENT_TRAINING.md b/docs/PHASESET_V2_LITERATURE_PARENT_TRAINING.md new file mode 100644 index 0000000..f6ed871 --- /dev/null +++ b/docs/PHASESET_V2_LITERATURE_PARENT_TRAINING.md @@ -0,0 +1,40 @@ +# Full-negative WaMo and MIME parent backward + +`literature_parent_training.py` preserves each model's complete batch objective +while loading and replaying one entire prepared capture graph at a time. +This is a numerical seam, not the production optimizer/supervisor, a study +freeze, a converged baseline, or an authorization to open final-test data. +TMR is deliberately absent from this callable: its VAE samples, global filtering +and track-consistent set reconstruction cannot be replaced by either of these +objectives. Its separate faithful path is described in +[TMR parent backward](PHASESET_V2_TMR_PARENT_TRAINING.md). + +For WaMo, every capture's complete motion representation, both reconstruction +branches and both original/shuffled ordering branches are cached without an +activation graph. All exact human rows receive admitted frozen DistilBERT CLS +features once. The original summed bidirectional variable-positive InfoNCE and +equal-parent auxiliary means are differentiated over the entire B-by-Q gallery. +Its VJPs replay every complete capture, including original auxiliary branches. + +For MIME, all complete motion rows are cached, then all exact human captions +are tokenized once and encoded by the genuinely trainable CLIP tower. The +original averaged bidirectional known-positive objective differentiates text, +learned logit scale and motion leaves jointly. Motion VJPs are replayed one +parent at a time. No text gradients are frozen and no microbatch negatives +are dropped. Role-aware original dyadic Inter-X inputs are a separate protocol. + +Only explicit learning components are admitted; historical development C00 may +be a learning component in its authorized outer fold, but final-test rows may +never enter. Loaders must return the same admitted complete physical source +on cache and replay. Exact representation/auxiliary equality is checked before +backward. The private host still binds official data/asset identity, augmentation, +component census, runtime, budget and real human provenance; callbacks do not +invent those authorities. It must not retain the entire physical population. + +RNG snapshots include each physical load and stochastic encoding; replay restores +the next update's post-text-forward state. Shared-parameter gradient accumulation +order can differ from dense autograd, so independent FP32 gradient oracles use +frozen tolerances while loss and cached representation equality remain exact. +FP32 only; hardware/BF16/resource qualification and full optimizer/checkpoint/ +resume/native learning remain separate. Software fixtures are explicitly analytic, +including rejected timeline gaps, and cannot populate any paper result table. diff --git a/docs/PHASESET_V2_MECHANISM_FACTORY.md b/docs/PHASESET_V2_MECHANISM_FACTORY.md new file mode 100644 index 0000000..728a26a --- /dev/null +++ b/docs/PHASESET_V2_MECHANISM_FACTORY.md @@ -0,0 +1,66 @@ +# PhaseSet-V2 matrix-bound mechanism construction + +`build_v2_mechanism` reads the exact registered 87-row matrix bytes and one +run ID, an admitted training-source digest, and a validation-selected complete-parent +B2 checkpoint with its exact file digest. It constructs the currently implemented complete-parent residual +systems only: PhaseSet-V2 full and A1/A2/A3/A4/A5/A6/A7/A8. It verifies the row's +fixed seed, split and same-seed B2 predecessor, and refuses A9, the old +head, base and literature rows until their distinct implementations exist. +There is no fallback to the full model for an unsupported system. + +| Row | Constructor difference | +|---|---| +| PhaseSet-V2 | full signed-vector phase and incidence model | +| A1 | speed-magnitude input before the same six Morlet convolutions, with an independent typed training-floor receipt and physical cache | +| A2 | order-free temporal MLP/mean paths | +| A3 | capacity-matched pair-bag context | +| A4 | explicit incidence shuffle seeded by the registered training seed | +| A5 | explicit phase fields stripped; energy/support retained | +| A6 | true mean/difference-signal DCT with a typed, main-population floor receipt | +| A7 | signed local actor kinematics and relative pelvis relation, without explicit phase; same CF pool/loss | +| A8 | full model; `bind_v2_parent_host_config` sets only the CF loss weight to zero | + +For main rows, host config binding also checks the exact twelve development +training components, C00 validation, fixed seed and residual stage. A8 is +derived from the same nonzero-CF candidate, changing only its loss weight; +the caller must still present the same admitted weak training text pool as +full. Residual initialization uses a forked CPU RNG seeded by the registered +row, so same-seed full/A8 begin from the same trainable weights without +mutating the caller's RNG. The A1/A6 receipts must match the admitted +training-source manifest digest in addition to their physical config and +main population. A1 also requires a speed-only source and checks its receipt +against the model at host binding; it cannot silently use signed-vector cached +responses. The periodic transform and its claim boundary are specified in +[A1 speed control](PHASESET_V2_A1_SPEED_CONTROL.md). +A7's score does not consume phase responses, but the current common transport +still loads the phase cache; see [A7 cost boundary](PHASESET_V2_A7_GENERIC_LOCAL.md). + +`make_v2_registered_base_host` constructs each of the nine main B0/B1/B2 +qualification hosts from its exact row, including the registered architecture, +official seed, twelve training components, C00 validation and 30-epoch +schedule. It writes the system/run identity into the checkpoint manifest and +does not assign formal launch authority. The prior `make_v2_b2_base_host` +entry point remains B2-only and delegates to this same checked path; a short +pilot cannot carry a formal row ID. All nine row/seed combinations are tested. +The residual builder verifies that row identity, stage, seed, population, +checkpoint file/state digests, a completed 30-epoch terminal and its final +validation-selected checkpoint digest, then loads strict B2 weights into the +registered architecture. `make_v2_parent_host` +is the single row-bound residual host path: it compares checkpoint/source +bindings, applies A8's loss transformation and persists the residual row, +system and predecessor in every checkpoint manifest. It also requires the +registered 20-epoch residual schedule. A direct host carrying +registered A8 identity rejects a positive CF weight. + +This factory does not independently prove that the private nine-run B2 +qualification selected the supplied checkpoint, authenticate private assets, +freeze pilot hyperparameters or max steps, claim storage/GPU capacity, or +launch training. That selection and the admitted source digest remain private +admission responsibilities. Fold model identity can be constructed only with +its registered fold-specific B2 training/validation population; its residual +host binding is still held. Inter-X remains unimplemented and fails closed. +The returned `launch_authority` is always false. +The public CLI likewise remains fail-closed until a qualified private +production adapter is attached. The host factories are executable row-bound +construction, but no formal run is admitted by calling them. This is executable mechanism selection, +not a completed 87-stage runner or a retrieval result. diff --git a/docs/PHASESET_V2_MECHANISM_VERIFICATION.md b/docs/PHASESET_V2_MECHANISM_VERIFICATION.md new file mode 100644 index 0000000..6136d4f --- /dev/null +++ b/docs/PHASESET_V2_MECHANISM_VERIFICATION.md @@ -0,0 +1,79 @@ +# PhaseSet-V2 mechanism verification — 2026-09-26 + +This is implementation evidence, not a retrieval benchmark, convergence claim, +or submission-ready paper. The scientific upgrade is described in +[the V2 amendment](PHASESET_V2_AMENDMENT.md); its +[87-stage matrix](../configs/phaseset_v2_experiment_matrix.json) supersedes the +affected legacy 33-run plan. + +## What was implemented + +- Signed world-coordinate pelvis velocity and pelvis-relative joint velocity; + six unchanged 20-Hz Morlet kernels; local complex cross-response, endpoint + energies, coherence, support, and explicit phase observability. +- Shared actor and edge temporal encoders with time-varying incidence moments, + fixed 64-edge checkpoint/recompute blocks, and an ordered capture readout. +- A common-scale cosine mixture, complete endpoint-bound relation matching, + variable-positive symmetric InfoNCE, and verified-negative-only softplus loss. +- A true mean/difference signal DCT control, preserving its cross terms. +- A portable structural test journal using the real state machine. Production + POSIX journal durability and runtime admission were not relaxed. + +These are independent research APIs, not yet a formal V2 host execution path. +No TMR/WaMo/MIME reproduction, V2 text annotation, training-only energy-floor +fit, or native-data training is claimed here. + +## Observed server checks + +The existing isolated environment was reused: Python 3.12.12, NumPy 2.4.6, +Torch 2.12.0+cu126, CUDA 12.6. Numerical checks ran on the registered experiment +server, not on the local workstation. + +| Check | Observed result | +|---|---| +| Focused V2, Morlet, legacy signal, host controller, and journal regressions | 117 passed, 1 CUDA-only test skipped; 32.47 s | +| Width-512 CUDA forward/backward, K=3 | 1 passed; 7.43 s | +| CUDA device | NVIDIA RTX A6000 | +| Peak allocated / reserved memory in that witness | 143,108,608 / 165,675,008 bytes | +| Source changes during either attempt | none; exact digest comparison passed | + +Mechanism tests include speed-only in-phase/antiphase collision versus signed +phase separation; endpoint conjugation; observed stationary support with +unobservable phase; no fabricated missing-joint velocity; non-incident edge +stability; actor permutations and padding; exact K=2 positive-zero topology +outputs and gradients; pair-only equivalence; checkpoint/eager forward and +parameter gradients at K=3 and K=13 (78 edges, crossing the 64-edge boundary); +chronological capture order; calibration; and verified-counterfactual exclusion. + +The width-512 GPU witness is **not** an arbitrary-K stress study, a complete +end-to-end training/resume run, or a resource comparison with literature methods. +The same-family GPT-5.6-Sol code review remains provisional scientific evidence. +Both execution receipts and raw logs stay private; their closed receipt digests +are: + +- CPU: `e1a9a5efa7089c0a5ae8263a712492ba57032dc6b2d440794ee25d73dbefc42f` +- CUDA: `2952fdbe06d81afb23ae89389a52bbe747c1cb37ca58d11cb2580fd7dc895e1c` + +Windows CI is required for the portable structural journal fix. A Linux server +check alone is not evidence of Windows execution; use the commit-bound CI result. + +## Scientific execution state + +Formal V2 stages completed: **0 / 87**. Sealed test remains closed. + +The project-owned data directory was inspected read-only on this date: only +the supplemental Multi-TPC data was observed. Licensed Embody main motion data, +neutral SMPL-X model assets, and a corresponding access approval were not found +in the checked project locations. This is not a claim that approval could not +exist elsewhere. + +The referenced chat's ZIP/Markdown handoff was not attached to the accessible +conversation. Exact pilot and three-fold component assignments remain null; +they must be recovered, not invented. Two blinded human reviewers and the real +relational challenge are still outstanding. No synthetic result is promoted +to main-data evidence, and no auxiliary dataset silently replaces Embody. + +Current execution state is **BLOCKED_DATA**, with rights, handoff, and human +relational-evidence dependencies recorded separately. Once those inputs exist, +the next step is the bounded native-data audit/pilot, then the frozen formal +matrix—not additional data-free audit scaffolding. diff --git a/docs/PHASESET_V2_MIME_CURRICULUM.md b/docs/PHASESET_V2_MIME_CURRICULUM.md new file mode 100644 index 0000000..2451b00 --- /dev/null +++ b/docs/PHASESET_V2_MIME_CURRICULUM.md @@ -0,0 +1,51 @@ +# MIME-Set curriculum component + +This is one paper-defined sampler component, **not a qualified MIME baseline**. +It does not implement original role-aware/co-attention motion encoding, original +fine-tuned CLIP, original Inter-X evaluation, training or scientific task seal. + +The [MIME v1 paper](https://arxiv.org/html/2607.22702v1) specifies uniform warmup, +training-anchor cosine neighbors, no-replacement batch coverage and a cosine +hardness ramp. This component uses three uniform epochs and a ten-epoch ramp +to 0.25. The zero-based convention is explicit: epochs 0..2 are uniform, epoch +3 starts at zero, epoch 12 reaches maximum. For descending-cosine eligible +neighbors, the window center is `floor((1-hardness)*(N_i-1))`. + +Adaptations to the complete-parent group task are explicit: one anchor is the +normalized mean of **all** admitted frozen CLIP human sentence embeddings of +that parent, not a chosen caption or validation/held-out neighbor. This is not +original MIME's fine-tuned language tower. The private bounded pilot freezes +the disclosed neighbor-window width, which the paper does not specify, and +the same effective-parent batch policy as the other systems. A singleton tail +is rebalanced without dropping/repeating parents; all human rows remain +retrieval positives. Historical C00 can enter an outer fold's learning set, +but final-test parents cannot. The host supplies exact learning components. + +The whole-parent anchor function now accepts the authenticated row selection +returned by `ParentHumanClipRows`, instead of requiring the complete original +development preparation batch to be re-encoded for every training subset. +It still checks exactly all learning-human commitments and original caption +digests: validation, held-out or appended weak columns cannot become anchors. +This changes the group sampling seam only, not original dyadic MIME text +admission or the legacy capture/storage receipt interfaces. + +`MIMEParentTrainingHost` connects this sampler to real method-specific updates, +validation and exact resume. Anchor identity and explicit neighbor width are +resume-bound. For a pilot population smaller than the batch ceiling, the +curriculum cannot change the all-parent negative set; it can change order only. +Report that limitation rather than claiming a curriculum benefit from pilot +scores. The main population can form multiple no-replacement batches. + +The scoped registered-server CPU qualification executed ten cases with no +skips: three actual host/row-selection cases plus seven existing sampler +cases. It checks independent all-human anchor means, held-out/changed-text +rejection, post-construction caller tensor mutation isolation, actual epoch +hooks and optimizer updates, and bitwise model/optimizer/scheduler/RNG resume. +Neighbor-width and anchor drift are refused before training data/text loads. +Production budget history is unchanged. This is analytic software evidence, +not native pilot learning, curriculum benefit or retrieval performance. + +Tests use analytic text/motion-free fixtures; no resulting number is baseline +performance or human verification. Original dyadic MIME and the shared group +adapter, original auxiliary/input fidelity checks, actual profile, convergence, +fair pilot selection and the registered formal matrix remain required. diff --git a/docs/PHASESET_V2_NATIVE_PROCESS_PROFILE.md b/docs/PHASESET_V2_NATIVE_PROCESS_PROFILE.md new file mode 100644 index 0000000..37a37dd --- /dev/null +++ b/docs/PHASESET_V2_NATIVE_PROCESS_PROFILE.md @@ -0,0 +1,67 @@ +# Complete native parent CPU profile through the production journal + +One real, bounded CPU invocation completed on 2026-09-27, 16:54:29–17:03:37 UTC. +It joined the already qualified [complete-parent backward interface](PHASESET_V2_PARENT_TRAINING.md), +development disk source, original human CLIP rows and +[StudyProcess](PHASESET_V2_STUDY_PROCESS.md) to the existing production budget. +This is **native CPU seam qualification**, not a trained result or GPU profile. + +## Actual input and scope + +Two native training parents were selected in immutable byte-key order before +model scores, without timeline cropping or window sampling. They contain four +actors each, 5,800 and 6,000 frames, all 26 and 29 accepted windows, and all five +official human holistic rows each. The other development parents remained in +the admitted task metadata; final-test inputs were absent. Main-training-only +physical floors and the already closed body/physical/language caches were used. + +Full-width 512-D B2 and V2 performed a complete rectangular 2-motion × 10-text +negative-gallery cache followed by RNG replay and vector–Jacobian backward. +Each system loaded each entire parent once for caching and again for replay. +The untrained seeded B2 state was reused; it was frozen for the V2 head. +There was no optimizer, weight update, CF pair or training-yaw admission. + +| System | Complete cache + VJP seconds | Trainable gradient tensors | +|---|---:|---:| +| B2 | 264.5908735031262 | 88 | +| V2 | 266.6414572298527 | 55 | + +Every trainable gradient was present and finite; at least one gradient element +per system was nonzero. This does **not** assert every individual parameter +tensor had a nonzero element. All model state bytes remained unchanged and the +V2 frozen base received no gradients. The recorded peak process RSS was +4,113,476 KiB. These single complete-batch CPU timings are not pure forward +latencies, p50/p95, GPU throughput, a 128-parent batch cost, inference ratio or +a prediction that all stages fit the 300-GPU-hour envelope. + +## Production lifecycle and costs + +FP32, seed 1729, CPU/interop one thread, no visible CUDA, offline warm environment, +nice 10 and a 16-GiB address-space bound were fixed. The child had a 1,800-second +watchdog and the production reservation included its full cleanup envelope. +It exited normally: all three wrapper/numerical/source-check exits were zero, +both stderr files empty, and source/invocation bytes unchanged. The controller +verified its own process group had exited; no timeout or forced kill occurred. + +The existing production journal appended one real `RESERVE` and `SETTLE`, +without rebootstrap or alteration of its original historical import. Actual +controller wall time was 543.8673342310358 seconds, GPU cost zero, optimizer +cursor zero, and no pending/unknown-cursor attempt remained. Historical cost +stayed 34 conservatively accounted GPU seconds. Pilot and formal counts stayed +**0/12 and 0/87**; dated formal step limits remain unfrozen. + +The private metadata-only archive was independently downloaded and verified: +28 hashed files plus the receipt itself, 29 flat members total. Raw inputs, +caption text, arrays, model assets and checkpoints were not in that archive or +the public repository. + +- Receipt SHA-256: `d243b5a8547f97636d04d0ade3b521aa3b16573ee0ba824381d5bed766271a0b`. +- Archive SHA-256: `43df64c735d5dca3fbdd3bac8a8201ea13c17937ba410cde5f03174ab7808e5a`. + +Fresh GPT-5.6-Sol/xhigh review of the three new private callers found no blocking +or non-blocking issues; same-family semantic review remains provisional. The +actual server evidence closes only the scope above. Real GPU profile, admitted +training transforms, bounded pilot, 87-stage learning, sealed evaluation, +statistics and paper delivery still require their own actual execution. +The later [label-source amendment](PHASESET_V2_RELATION_SUPERVISION_AMENDMENT.md) +cancels waiting for new annotators, but does not fabricate human CF evidence. diff --git a/docs/PHASESET_V2_PARENT_DISK_SOURCE.md b/docs/PHASESET_V2_PARENT_DISK_SOURCE.md new file mode 100644 index 0000000..26a8d7c --- /dev/null +++ b/docs/PHASESET_V2_PARENT_DISK_SOURCE.md @@ -0,0 +1,56 @@ +# Complete-parent disk source + +`DevelopmentParentDiskSource` consumes caller-admitted `ParentRetrievalTask` +and `ParentDiskRecord` entries. Every family must have exactly one complete +source and an explicit caption-yaw eligibility bit. Unknown or changed human +records fail before loading. The development source rejects final-test rows; +it does not discover datasets or infer rights from matching files. + +The literature interface supplies complete body timelines and fixed TMR/WaMo +human-language rows. The body view loads verified prepared arrays without +reading the physical cache. The phase view additionally loads the entire +verified physical cache with the supplied training-only floors. It recomputes +all motion-derived phase, actor and root fields for a nonzero rotation; it +does not approximate a rotation of cached RMS vectors. Accepted windows, +absolute positions on the timeline, rejected intervals, masks and trailing +frame declarations stay intact. There is no dataset-sized in-memory cache. + +One shared group yaw is determined solely by source, one of the three fixed +seeds, and epoch. A fixed SHA-to-angle mapping makes replay independent of +model dropout, VAE sampling and global NumPy/Python/Torch RNG state. Every +reload of that source/seed/epoch receives the same physical arrays. Validation +always uses zero yaw. Disallowed captions also receive zero yaw at training; +the private caller must bind a real, frozen text-eligibility rule and its +population, not assume all world-direction captions allow arbitrary rotation. + +The physical floors must come from the correct run's learning population. +This source does not fit them or authorize reusing main-experiment floors in +a held-out fold. Source, annotation, floor, feature, augmentation-rule and +runtime identities belong in the private execution inputs. + +For A1, the source requires a separate speed-only cache and a typed floor +receipt. It checks the loaded physical config and floor values against that +receipt before yielding a view. Shared-yaw recomputation preserves the +selected speed-only frontend rather than falling back to signed vectors. +See the [A1 control contract](PHASESET_V2_A1_SPEED_CONTROL.md). + +The optional [frozen human CLIP rows](PHASESET_V2_FROZEN_CLIP_ROWS.md) connect +the base/V2 human-only text interface without loading a tower at training. +The legacy verified-CF branch still needs real human provenance. Under the +user's [supervision amendment](PHASESET_V2_RELATION_SUPERVISION_AMENDMENT.md), +the separate [weak CLIP pool](PHASESET_V2_WEAK_CLIP_POOL.md) supplies disclosed +machine-caption training targets without granting human verification. Neither +branch changes the human-only primary gallery. The source supplies no +production lock, heartbeat, budget, retention, hardware, +pilot, formal-stage or final-test authority. + +Analytic tests exercise full timelines with a rejected gap, RNG-independent +replay, shared rotation, actual phase recomputation, exact-zero validation, +caption multiplicity, task drift and final-test exclusion. Native validation +must separately load the complete admitted development population and its +closed fixed text features. An evaluation-only source qualification does not +verify a training yaw policy or demonstrate retrieval performance. + +See the [fixed language rows](PHASESET_V2_FROZEN_LANGUAGE_ROWS.md), +[literature host](PHASESET_V2_LITERATURE_PARENT_HOST.md) and +[physical cache and calibration](PHASESET_V2_CALIBRATION_AND_CACHE.md). diff --git a/docs/PHASESET_V2_PARENT_HOST.md b/docs/PHASESET_V2_PARENT_HOST.md new file mode 100644 index 0000000..3915ca4 --- /dev/null +++ b/docs/PHASESET_V2_PARENT_HOST.md @@ -0,0 +1,151 @@ +# Complete-parent base and residual training host + +`continuous_parent_host.py` connects the qualified full-gallery gradient replay +to real FP32 Adam(W) updates, checkpoint selection and restart. It is not the old +window-unit training recipe, a formal-study authorization, a trained B2 anchor, +or a literature-baseline implementation. A software qualification is not a +native learning result. The formal study still requires the common bounded +pilot, explicit parent-unit amendment, floors, hardware profile and 87-stage +freeze. The legacy human relationship audit cannot be supplied by this code; +the [user amendment](PHASESET_V2_RELATION_SUPERVISION_AMENDMENT.md) now permits +a separate weak-training branch, not replacement of independent motion truth. + +## Inputs and updates + +The private `ParentTrainingSource` loads one admitted **whole** capture at a +time from disk. Every accepted window remains in the frozen global branch; +every physical timeline interval remains in the ordered coordination branch. +The source must reproduce the same shared-yaw transform during replay, and +respect the pre-admitted language/yaw rule. No shorter timeline or physical +batch materialization is introduced by the host. + +For a **base** stage, the source returns `PreparedContinuousCapture` directly, +not a fabricated residual view. `ContinuousBaseRetrievalSystem` reuses B0/B1/B2 +without shrinking their registered architectures. Every accepted window is +encoded in absolute order, then pooled with the same fixed float64 mean and +float32 L2 normalization as the residual's global anchor. The base encoder and +its bounded logit temperature are trainable; no text adapter, phase cache, +physical floor, coordination evidence or learned window pool is added. +Non-reentrant window checkpointing preserves dropout RNG and binds each actual +window to its own replay closure. The checkpoint invocation includes an +explicit tensor on the model's device so +CUDA RNG is discoverable; closure-only parameters would not provide that. +CPU invocation checks do not qualify CUDA numerical gradients. It bounds +stored neural activations, not the +size of the complete physical skeleton input. Native memory/cost qualification +is still required. The outer full-gallery parent-score replay is shared by +base and residual stages; the architectures and input types are not conflated. + +The effective batch is selected by explicit learning components and includes +all official human holistic rows of every parent. Only the B-by-Q score matrix +and RNG snapshots are cached. The full variable-positive symmetric objective +is differentiated once; each complete capture graph is replayed separately. +The physical loader is called separately for cache and replay. It must not +silently keep the entire learning population in memory. Loading and neural +activation storage are separate from the total disk-cache footprint. + +Optional CF sentences follow the human retrieval prefix. Their columns have no +fabricated retrieval positive. A verified-false bit is not human provenance: +the private manifest must bind the actual blind two-human records. Unverified +CFs contribute no CF term. No label generation or data discovery occurs here. +`ParentWeakCounterfactualRows` instead uses `included_weak` and the separate +weak loss/count fields. Its source is disclosed machine-caption contradiction, +never human `verified_false`. The original retrieval prefix and all validation +rows stay human-only. Base stages reject CF rows, extra text columns and any CF weight. They optimize +only the existing variable-positive symmetric retrieval InfoNCE. + +Internal default optimizer: AdamW, weight decay 0.01, clipping 1.0, 5% warmup followed by cosine +decay, default residual learning rate 3e-4 and 20 complete epochs. Seeds remain +1729, 2718 and 31415. Batch size is explicitly **parents**, not 128 windows. +The short qualification schedules are not the formal/pilot freeze. There is +no automatic BF16 fallback; FP32 is the only supported path in this host. +`ParentHostConfig.for_base` explicitly sets the distinct base defaults: +30 epochs, learning rate 2e-4, zero CF loss and `stage="base"`. The stage must +match the actual scorer. A base has no frozen-anchor checkpoint binding; +residual stages still require that exact frozen B2 binding and enforce its +unchanged state. No checkpoint produced by a fixture is a qualified base. + +## Validation and selection + +Learning and checkpoint-selection components are explicit, disjoint and +non-test. Historical main-validation components can be learning components in +an authorized outer fold; held-out components are never opened by the host. +Validation loads every parent and every exact human row of its complete +gallery, with no CF extras or augmentation. Scores are computed against the +same full text pool, one whole motion at a time. Evaluation consumes no training +RNG and restores the previous model mode. + +The reused endpoint is capture-macro bidirectional R@1: motion-to-text counts +any positive; text-to-motion is averaged first within each capture family, +then equally across captures. Ties are broken by fixed lineage commitments, +not input order. This is explicitly not caption-count-weighted R@1. Only a +strictly better validation endpoint replaces the selected checkpoint; exact +ties preserve the first. There is no final-test method or test-based selector. + +## Checkpoints and attempts + +Each `fit` owns a fresh attempt directory with an immutable run manifest, +numbered progress events, atomic write-once checkpoints and terminal evidence. +The private run manifest contains the full **active learning/selection** +parent/text census and must remain private. Unused held-out/test rows of a +larger supplied task are not serialized or made part of this host's resume +identity. It binds configuration, input/physical-floor/CLIP/CF +manifest, code including the source adapter, qualified runtime and frozen +base identity. This caller-provided binding is not a substitute for real +qualification or permission. + +Checkpoints contain model, configured optimizer, scheduler, Python/NumPy/Torch RNG, complete +epoch/batch cursor, parents seen, validation history and selected artifact. +Resume verifies the exact predecessor digest, payload, same execution inputs, +cursor/census, frozen anchor and optimizer/scheduler progress. A resumed host +must be built from the same seed-bound **initial** model. Resume uses a new +attempt; it never overwrites or mutates the predecessor. Interruption at an +epoch's last update still completes that epoch's validation before returning. +Zero-update and already-completed resume are explicit cases. + +These event/terminal records intentionally carry no formal-study authority. +The private production adapter remains responsible for resource admission, +locks, budget accounting, attempt-ledger linkage, signal/error classification, +long-operation heartbeat and publication-safe result export. Failed disk or +process writes cannot guarantee an on-disk terminal; the remote supervisor +must close that failure from its own evidence. Never publish private host +manifests, checkpoints, caption text or licensed physical cache files. + +## Qualification scope + +Optimizer family and weight decay are explicit `ParentHostConfig` settings, +serialized in the run/checkpoint manifest and checked on resume. Internal +base/residual defaults remain AdamW and 0.01 decay. `for_literature` supplies +declared paper-optimizer starting points: TMR AdamW/0.01, WaMo Adam/0, +MIME AdamW/1e-4, each at lr1e-4. Explicit bounded-pilot overrides remain +visible; they are not hidden author defaults. The shared 5%-warmup/cosine, +clipping, complete-parent schedule and validation endpoint are group-study +adaptations, not claims of original single-/two-person training reproduction. +The earlier hard-coded host could not express WaMo Adam or MIME decay; no +native/formal learning used that host before this correction. + +The scoped server CPU qualification executed 17 cases with no skips. It +includes actual analytic TMR/WaMo/MIME optimizer loops, independent coupled +Adam versus decoupled AdamW first-update oracles, the unchanged legacy-default +oracle, configuration-drift rejection before loading training inputs, and +dropout interruption/resume equality for Adam/0 and AdamW/1e-4 across model, +optimizer, scheduler and RNG. Production budget history was unchanged. +These are software execution checks, not native training or retrieval results. + +Tests exercise actual registered 512D B0/B1/B2 forward and all-parameter +checkpointed backward with active dropout, plus an independent per-window +pooling oracle. Separate, clearly marked analytic fixtures exercise the real +base/residual host optimizer loop, +fixtures, full gallery/caption census, first-best selection, bitwise +uninterrupted-versus-resumed model/optimizer/scheduler/RNG, zero/mid/end/completed +resume, concrete input drift, truncated checkpoints and wrong progress cursor. +They also retain the existing independent dense-versus-replay gradient tests. +None is evidence of real native learning, a paper metric, GPU cost, human CF +truth, or successful completion of any of the 87 formal stages. + +A separate native CPU qualification has now replayed two complete four-person +captures through the actual 512D V2 score/loss backward with ten original human +rows and six disclosed machine weak negatives. All 55 trainable gradient +tensors were finite and weights stayed unchanged; no optimizer was created. +See the [weak-pool qualification scope](PHASESET_V2_WEAK_CLIP_POOL.md). +This closes that input/loss seam, not native learning or a hardware cost claim. diff --git a/docs/PHASESET_V2_PARENT_OPERATION_MONITOR.md b/docs/PHASESET_V2_PARENT_OPERATION_MONITOR.md new file mode 100644 index 0000000..b77583c --- /dev/null +++ b/docs/PHASESET_V2_PARENT_OPERATION_MONITOR.md @@ -0,0 +1,41 @@ +# 完整 parent host 的运行监测和 checkpoint 保留 + +`parent_run_monitor.py` 复用现有 `_AttemptLease`，不改旧 33-run 的 +`AttemptStore`/run-ID 合同。`ContinuousParentTrainingHost.fit`（包含继承它的 +TMR/WaMo/MIME host）在数值循环前持有 lifetime lease；终止/异常均释放。 +每 30 秒写一次独立线程 heartbeat，包含当前操作、step、epoch、最近已发布 +checkpoint SHA、单进程 monotonic elapsed。线程只读 scalar snapshot，不碰 +模型、张量或 RNG，因此完整 capture forward/backward、数据读取和保存期间 +没有新 update，也仍可判断进程存活。它不是每 30 秒必须完成一步的 watchdog。 + +现有不可变 run/events/checkpoint/resume 继续使用，新增不可变 heartbeat、 +checkpoint metadata receipts 和 retention receipts。终止记录补充失败分类： +磁盘不足、checkpoint 无效、资源上限、进程中断、输入/合同错误或未分类。 +原始 traceback 应留在私有日志；分类不授予自动重试。监测/终止写入同时失败时 +尽量保留主异常并追加 note，不用清理错误替换原始训练根因。突然 kill/power loss +可能没有 terminal，最后 heartbeat 和 shell exit 收据需由操作员据实核对。 + +生产调用显式使用 `fit(..., checkpoint_keep_recent=2)`。默认 `None` 保留原先 +keep-all 行为，不回收旧资格目录。保留最新两个 payload（回滚候选）、它们 +各自依赖的 selected-best 以及当前 best；resume predecessor 在其他目录，不删除。 +仅从本 attempt 自己发布并已记录的 checkpoint 中回收旧且不再被保留候选引用的 +payload，逐文件确认目录/regular file/原摘要，先写 planned、删除后写 retired。 +事件、摘要、历史 checkpoint receipts、旧失败目录均永久保留。不递归、不 glob +删除，不按未验证的磁盘占用猜测删除资产。已回收 payload 不能直接恢复，只能从 +保留候选重新计算；planned 无 retired 明确是中途中断，不冒充已完成回收。 + +该监测器不批准正式实验、不实现全局预算或硬墙钟 timeout，也不把单进程 elapsed +冒充多 GPU-hours。私有执行器仍须对所有分配设备（包括失败计算）累计 GPU 时间， +在冻结的 87 阶段、最多 90 starts/12 pilots/3 rooted restarts/2 structural reworks/ +300 GPU-hours 内预留和结算预算，并用有界 shell timeout 处理卡死操作。 +监测器的存在不关闭真实 GPU profile、正式 freeze、逐 family yaw 或双人类 CF 证据。 + +资格测试只在登记服务器执行。分析性软件 optimizer/resume 通过不是 native +学习结果、短程 pilot 或论文效果；本实现也不修改 loss、优化器、调度器、 +完整 parent/gallery 单位、验证选择或冻结 base 的算法。 + +登记服务器 CPU-only 资格于 2026-09-27 关闭：**19 PASS / 0 skip / 0 xfail**， +103.72 秒；包含新 16 case（base/residual 实际分析性 optimizer + bitwise resume、 +初始化/heartbeat/terminal/release 故障、保留 best 依赖）及 3 个直接相关回归。 +所有 numerical/source/invocation 检查 exit0，stderr 空；旧失败、旧资格不重启。 +它不证明 GPU training profile、真实数据学习、正式预算或研究假设成立。 diff --git a/docs/PHASESET_V2_PARENT_TASK.md b/docs/PHASESET_V2_PARENT_TASK.md new file mode 100644 index 0000000..86539f7 --- /dev/null +++ b/docs/PHASESET_V2_PARENT_TASK.md @@ -0,0 +1,59 @@ +# Complete-parent retrieval labels + +`phaseset_core.parent_retrieval_task` is a provenance/label index, not a data +license, physical admission gate, final task seal, or trained model. It reads +no paths and opens no test data on its own. + +## Unit of retrieval + +Released episodes may share one native parent and one human holistic annotation +file. They must first be placed on the actual parent world-coordinate timeline, +including unreleased gaps, and preprocessed once. They are not independent +motion rows or negative examples of each other. The host separately admits +the complete body/cache, declared tail, source lineage, and population-specific +training-only physical floors. + +The development annotation audit found 494 eligible released episodes belonging +to 253 native parent/holistic-file families: 202 training and 51 validation +parents. Reading each official `scene_explained` list once yields 1,334 human +answer rows. An exact UTF-8 text audit found no repeated rows within or across +these families. This is not semantic-equivalence or relation-truth verification; +test annotations were not read by that diagnostic. + +## Interface + +The private host supplies `ParentCaptionRecord.from_official_bytes(...)` with +the complete-parent source identity, admitted annotation-family digest, +component, split, original annotation bytes and expected native human row +count. Every `scene_explained` row is retained verbatim. Other questionnaire +fields are not fused into motion captions. A digest cannot by itself prove that +an annotation is official, licensed or skeleton-observable. + +`ParentRetrievalTask(records, expected_family_keys=...)` verifies that the +explicit complete population is present before selecting any subset. Repeated +source/family records, missing or extra families, and a participant component +crossing splits are errors. It does not silently deduplicate released siblings +or infer which parent should be retained. + +`gallery(split=...)` returns one motion row per admitted parent and every human +caption row for that split. `batch(family_keys)` preserves the supplied parent +order and rejects repeated or unadmitted parents. Positive keys use the parent +annotation family; source keys remain separate. The resulting rectangular +variable-positive mask has no fixed three-caption assumption. Text commitments +follow the existing source/family/row-ordinal CLIP receipt convention and are +lineage only, never tokenizer or neural features. + +Sampling schedules and full-population physical admission remain the training +host's responsibility. All bases, literature controls and V2 variants must use +the same admitted parent task. Capture-cluster statistics use independent +parents, not released episodes, windows or individual captions as replicates. + +## Evidence boundary + +Data-free metadata tests are not human truth. A two-real-parent frozen-random-B2 +forward/loss/backward witness, if passed, establishes execution and label +plumbing only; it is not an optimizer run, pilot, formal stage or retrieval +result. Two real blinded human annotators are still required for verified +relation counterfactuals. Complete-population floors, task freeze, faithful +literature baselines, the fixed 87-stage matrix and sealed evaluation remain +separate gates. No final-test discovery or unsealing is performed by this API. diff --git a/docs/PHASESET_V2_PARENT_TRAINING.md b/docs/PHASESET_V2_PARENT_TRAINING.md new file mode 100644 index 0000000..0b3d58a --- /dev/null +++ b/docs/PHASESET_V2_PARENT_TRAINING.md @@ -0,0 +1,37 @@ +# Complete-parent numerical training seam + +This module is the numerical seam used by the private V2 host, not a scientific +task seal, production attempt scheduler, dataset download or human-CF verifier. +It does not launch an optimizer, choose checkpoints, open final-test data or +declare that the 87-stage plan is launchable. + +`parent_epoch_batches` visits every parent in the explicit learning components +once, uses one of seeds 1729/2718/31415, and includes every human caption row. +Its batch-size unit is **complete parents**, not released segments or windows. +The common batch/epoch/max-step settings are still to be frozen by the bounded +pilot. A singleton tail is rebalanced without dropping/repeating a parent; +an impossible ceiling fails explicitly. Historical main validation C00 may be +a learning component in the appropriate independent outer fold; final-test +parents may not. The host takes the precise component sets from the dated study +matrix, not from a sampler-selected score or fallback split. + +`backward_parent_batch` binds complete physical source order, annotation-family +positives, all official human caption commitments and exact text bytes to the +frozen CLIP pool. CF sentences may follow the human rows without acquiring fake +retrieval positives. Explicit source/column/verified masks select their losses; +only real blind-human provenance admitted by the private host can establish a +true `verified_false` bit. Synthetic test bits are fixtures, not human evidence. + +The full B-by-Q objective includes cross-microbatch negatives and variable +positives. A no-grad score pass plus a leaf-score VJP replays one complete +capture graph at a time, including the shared text adapter and calibration. +When extra CF columns follow the human prefix, that prefix is made contiguous +at the legacy loss boundary; the copy remains differentiable into all scores. +Within each capture, the existing ordered temporal/edge checkpointing remains. +This bounds live *multi-capture neural activation graphs*; it does not make +all-pair compute linear, eliminate physical-cache storage or bound all host RSS. +RNG snapshots preserve replay and the next update's stochastic state. B2 is +frozen, and gradients must be finite before the caller clips/steps/checkpoints. +This seam is FP32-qualified; hardware BF16 needs separate qualification and +integration. The complete private run host, registered budget accounting, +faithful TMR/WaMo/MIME, validation selection and science freeze remain required. diff --git a/docs/PHASESET_V2_RELATION_SUPERVISION_AMENDMENT.md b/docs/PHASESET_V2_RELATION_SUPERVISION_AMENDMENT.md new file mode 100644 index 0000000..b4aaf1a --- /dev/null +++ b/docs/PHASESET_V2_RELATION_SUPERVISION_AMENDMENT.md @@ -0,0 +1,50 @@ +# PhaseSet-V2：已有标注优先，机器关系监督单独披露 + +2026-09-27，项目负责人明确表示没有两位新标注者，要求使用 LLM 或已有标注 +继续研究。本修订取代“必须等到两位新标注者才能继续”的执行要求，不追溯 +改写原 handoff，不伪造人工记录，也不改变当前 87 阶段、12 pilot、三 seed、 +300 GPU-hours 上限或已有成本。对应机器可读规则见 +[supervision contract](../configs/phaseset_v2_relation_supervision.json)。 + +## 主结果不换 ground truth + +确认性 full-gallery motion–language retrieval 继续使用官方原始人工 holistic +文本、既定 parent 正例身份和组件划分。机器改写、机器融合、伪负例不进入 +human-only 主 gallery。已有人工正例不自动证明一个反事实句子为假：同一长 +capture 内可能同时发生它们，缺少一句描述也不等于事件不存在。 + +优先复用已有、许可适配且语义确实匹配的显式关系标注；本修订不声称官方 +holistic 描述已经提供两人独立的真/假关系标签、actor/time grounding，或已经 +找到额外资产。不得为补标签无声增加数据集或训练阶段。 + +## LLM 可提供弱监督，不是独立真值 + +可用 GPT-5.6-Sol 读取同一训练 parent 的官方人工文本，提出带来源的最小 +关系对比。它不读取模型分数、骨架/频谱/phase、held-out 或最终 test 文本。 +固定 prompt、模型与解码参数、候选数及不依赖分数的纳入/拒绝规则，在相应 +pilot/训练前一次冻结；保留机器来源、原人工文本依据、输出及排除记录。 +无法判断的项不强行打真/假标签。不通过反复改写刷挑战集数量或模型成绩。 + +若实施机器 CF 训练，其标签身份为 `machine_caption_contradiction`，只提供 +明确披露的弱训练监督。full 与适用的机制对照使用相同弱监督池和预算，A8 +仍关闭 CF 目标；不可将机器推断塞进原 human-only `verified_false=True` +接口。独立弱标签类型、loss、完整 parent backward 及事件计数已经实现， +并通过 [37 项服务器软件检查](PHASESET_V2_WEAK_CF_TRAINING_SEAM.md)；缓存 +训练-only文本池通过 [另外 24 项检查](PHASESET_V2_WEAK_CLIP_POOL.md)，原真实 +人工接口与旧测试保留。**实际弱 CF 生成、真实文本编码池和来源准入仍待完成**， +不能据此宣称正式训练已启动。无 CF 的基础模型与独立开发工作不再等新真人。 + +## 评估与结论边界 + +机器辅助关系挑战只能单独标为探索性：报告其标签来源和覆盖/排除情况， +不能称为人工核实的关系准确率，更不能以模型对机器标签的一致性证明模型 +理解真实动作。具有真正匹配的已有独立标注时，才按其真实适用范围另报结果。 + +历史“200 对、100 capture、8 组件、两名盲人类审核”和挑战提升 5pp 目标不由 +机器数量替代；没有匹配独立标签时记为 **UNASSESSABLE**。这取消继续等待 +真人的执行阻碍，不把原全部机制成功标准偷偷降格为已通过。官方人工检索 +结果、预登记的结构/方向消融仍可支持各自的实证结论；CF 对 A8 的检索增益 +只能支持弱监督训练的效用，不单独支持人类核实的关系理解或 grounding。 + +最终论文必须反映实际证据：弱关系监督和没有独立真人挑战是明确限制。 +负结果和证据不足仍交付完整代码、真实结果、统计与论文，不刷 seed/test。 diff --git a/docs/PHASESET_V2_STUDY_BUDGET.md b/docs/PHASESET_V2_STUDY_BUDGET.md new file mode 100644 index 0000000..7c5260e --- /dev/null +++ b/docs/PHASESET_V2_STUDY_BUDGET.md @@ -0,0 +1,70 @@ +# V2 持久全局预算账本 + +`StudyBudget` 是单个研究的 append-only accounting journal，复用原 `_AttemptLease` +做短事务锁，不启动模型、SSH、GPU 或授权系统。不改 legacy33run 解析器。所有 +生产任务必须使用同一个私有根目录；另建空账本或忽略历史违反研究合同。 + +初始化必须显式提供完整87行/三seed矩阵、每一个历史计数器、历史证据及每个 +pilot家族对应的 nominal formal optimizer-step cap，不会默认历史为0。已有目录 +不能重新初始化。GPU预算300h、formal reservations90、pilot12、rooted formal +restart3、结构返工2，以及五个pilot家族3/3/3/1/2的分配固定；不能通过传参放大。 +Reservations是实际启动数的保守上界，spawn失败也消费名额，不伪装成成功训练。 +当前研究尚无formal/pilot启动，因此bootstrap只允许显式确认的zero-training历史， +另导入真实既往GPU检查用量；若找到已有训练，必须沿用详细原journal，不能把 +非零训练仅总结成几个计数后丢掉run/seed/predecessor身份重新初始化。 + +启动子进程前预约所有分配GPU乘以完整wall envelope（包括timeout和kill grace）。 +未结算任务一直占用整份预约，进程缺失/过时heartbeat不自动返还。生产wrapper +仍须实际设置相同有界timeout，并确认自己的子进程已终止/不再持GPU之后才结算。 +按实际分配wall time×GPU数收费，失败/中断也收费；早结束归还未用时长，但不 +归还pilot/formal名额。超过预估/300h的实际用量照实结算，不能为使数字好看拒记。 +Optimizer-step超cap也保留成本并标出，不能冒充合同通过；操作员须修复根因。 +实际硬中断未留下完整terminal时，FAILED/INTERRUPTED可显式记录未知cursor=null， +成本仍照实计入；单列unknown cursor，不伪造0步，也不允许以未知cursor称COMPLETED +或NOT_STARTED。原整数cursor接口不变，重启仍需实际原始证据和checkpoint恢复。 +账本不会自动kill进程、判断关系真值、选择seed或关闭科学acceptance gate。 + +profile必须0 optimizer updates；真实训练不能藏在未计数profile中。若完整training +profile执行真实优化器更新，就占既定pilot名额，同一次pilot可同时提供profile +证据，不再偷偷额外训练。每pilot `5*planned_steps <= nominal_formal_steps`，不以 +epoch单位转换逃过20%限制。pilot编号唯一，失败名额不重置。正式run只匹配矩阵 +ID和seed；先记录**已在外部真实冻结**的全87个max_steps，只可记录一次。 +记录step limits本身不是研究freeze或训练许可；当前真实研究仍未完成该freeze。 + +Formal重复启动必须关联同run最新已结算失败/中断前驱，写真实原始日志根因， +全球只允许3次，不分配置额外给三次，也不从旧前驱分叉或重跑已完成seed。 +GPU收费、starts/restarts、结构返工跨重新打开账本保存，不因新host或中断而清零。 +达到三次formal rooted restart额度后，不再新增长formal任务；先交付实际故障证据 +和预算缺口，不扩大额度。90 reservation上限也独立保护，未用seed不因此删掉。 +写盘截断/gap等不完整journal会显式拒绝继续；保留证据，不能另建零账本重试。 + +JSON中的`scientific_or_launch_authority:false`明确是成本证据，不替代数据权利、 +完整输入/环境/代码binding、true two-human CF、真实GPU空闲和dated科学合同。 +生产外部controller的实际spawn/timeout/settlement仍要有真实收据，软件模拟账本 +和测试中模拟90 reservations不等于启动了90个实验。所有数值/模型/资格仍只在 +登记服务器；公开只含源代码、合成bookkeeping测试和不敏感说明。 +新[Linux process controller](PHASESET_V2_STUDY_PROCESS.md)提供实际执行接入； +其本身不授科学admission，真实nativeGPU训练还需独立资格和数据/关系合同。 + +## 本次实际限定资格 + +Fresh ARIS GPT-5.6-Sol/xhigh发现首次建账的parent-directory durability缺口， +已修正为已有parent、event-file fsync → journal-directory fsync → parent fsync。 +两个新增case验证POSIX顺序及sync失败时保留账本/拒绝重建；Windows不冒称同样 +directory-fsync保证。最终限定复核none，same-family仍provisional。 + +登记服务器warmenv、CPU1/interop1/noCUDA/nice10/16GiB有界执行，新26项全部 +PASS/0skip/0xfail，1.79秒；numerical/source-check/exit全0，源/invocation前后相同。 +测试含并发最后额度保护、失败成本、pending保持、90合成reservation、三rooted +restart/两rework、12pilot分配/20%steps、截断journal与目录同步故障，不运行模型。 + +随后独立CPU-only历史导入真实核验四份既有CUDA收据：三个单卡mechanism检查 +整段秒级receipt wall各加1秒rounding，保守合计34GPU秒；另一个profile在Torch/ +CUDA分配前HOLD，计0。这是保守历史成本，不是精确allocation计时或新profile。 +唯一私有生产journal现在history34、pending0、formal0、pilot0、formalstepfreeze +未记录。旧失败/资产/收据保留，未新增GPU分配/优化器步骤、重启或结构返工。 + +这些bookkeeping测试和一次历史bootstrap本身不替代production controller资格。 +新controller另已完成[23项实际Linux软件资格](PHASESET_V2_STUDY_PROCESS.md)， +仍不替代native GPU接入、完整training profile、既定pilot/87阶段学习、two-human +关系核实、结果统计或论文交付。 diff --git a/docs/PHASESET_V2_STUDY_PROCESS.md b/docs/PHASESET_V2_STUDY_PROCESS.md new file mode 100644 index 0000000..8b3da03 --- /dev/null +++ b/docs/PHASESET_V2_STUDY_PROCESS.md @@ -0,0 +1,53 @@ +# Linux owned-process执行与真实成本结算 + +后续 [完整 native CPU profile](PHASESET_V2_NATIVE_PROCESS_PROFILE.md) 已实际接入 +同一生产 journal 并闭合；下文旧23项软件资格记录保留，不能代替GPU/训练结果。 + +`run_budgeted_process`把已实现的单一StudyBudget接到实际子进程生命周期：先读 +GPU占用/其他compute，取得本研究per-GPU lease，再检查；完整wall envelope预约 +后spawn前又查一次。不启动SSH、不使用shell=True、不自动重试，也不授科学许可。 +UUID、visible cards、GPU数量和timeout+两份TERM grace+exit-verification envelope必须 +相同；CPU1/PYTHONHASHSEED=0固定。CPU profile可无GPU；训练仍需真实外部admission。 + +每child新POSIX session，只有这个session被TERM/KILL。Linux WNOWAIT保留leader +PID直到所有同组live进程/CUDA持有者退出验证后再reap，避免PID过早回收；他人的 +进程不终止。已登记训练程序不得daemonize/setsid逃出session；这是当前实际Python +parent host调用约定，不声称支持任意daemon或Windows生产服务。 +子session由已存在的GNU timeout独立计时/TERM/KILL，父controller被SIGKILL也 +不留下无限运行的child；账本仍保留其pending，不能因父死亡自动退款。controller +自身SIGINT/SIGTERM由main-thread暂存，拿到childhandle后先清理/结算，再恢复 +原handler并报告中断；不在spawn中途因signal故意丢失handle。 + +正常退出后也检查遗留descendant；有遗留则清理并FAILED，不假称成功。忽略TERM +会在固定grace后KILL；不能证实退出、startup handle丢失或证据写盘失败时保留整 +份pending预约，不猜PID/不自动退款。未来repair必须读原日志和实际状态，不盲重启。 +只有确认termination之后，实测monotonic分配wall×全GPU数结算，失败/中断也收费。 + +`process-terminal.json`保存实际child exit、cleanup、host cursor和成本；单独 +`settlement.json`只能在journal真的settle后出现。若结算后第二份收据写失败， +必须以实际journal为准，不重复收费。原primary错误不被cleanup/terminal secondary +覆盖。stderr/stdout和所有失败目录保留；没有动作/数据/模型资产公开。 + +`ContinuousParentTrainingHost`及继承它的`LiteratureParentTrainingHost`的 +terminal `outcome/global_step`直接消费，不兼容被替代的legacy `Host` schema，不加载Torch +checkpoint猜optimizer进度。无/截断terminal记录cursor=null而非0、不得COMPLETED； +StudyBudget仍记录已知实际成本，并单列unknown cursor。它不把未知进度转换成 +真实结果，也不为restart挑seed、设置floor/yaw或自动构造两真人CF许可。 + +软件验证只用真实stdlib子进程和合成journal；fake-card参数只测试账务，非实际 +CUDA分配/硬件资格。实际GPU profile、完整native host/pilot、87datedfreeze、真人 +关系核实及真实论文结果仍需要各自外部证据。当前没有新增正式/pilot训练。 + +## 登记服务器实际软件资格 + +2026-09-27一次CPU1/interop1/noCUDA/nice10/16GiB/warmenv有界执行， +23项全部PASS，10.90秒，0skip/0xfail；numeric/source-check/exit全0、stderr0。 +19项新process cases及4项定向budget cases，真实stdlib child和信号，不是 +CUDA训练或模拟日志替代真实进程。覆盖父controller被SIGKILL后的独立watchdog、 +自身SIGTERM、忽略TERM、残留descendant、WNOWAIT、缺失/截断terminal、 +三次GPU gate/lease碰撞、spawn歧义、PID/terminal/settle写盘及cleanup失败。 +两个已有实际base/residual host terminal均固定SHA并成功读出真实global_step=4； +未重跑旧模型。整个资格前后同一生产journal摘要/原event SHA一致，无新增预约、 +成本或native optimizer；所有故障仅发生于隔离合成fixture，原失败证据保留。 +Fresh ARIS限定Sol/xhigh复核none，same-family仍provisional。Linux生命周期软件 +资格已闭合；实际native训练仍0/87、pilot0/12，其接入/科学资格不因此闭合。 diff --git a/docs/PHASESET_V2_TMR_PARENT_TRAINING.md b/docs/PHASESET_V2_TMR_PARENT_TRAINING.md new file mode 100644 index 0000000..99ec8ab --- /dev/null +++ b/docs/PHASESET_V2_TMR_PARENT_TRAINING.md @@ -0,0 +1,27 @@ +# Full-negative whole-parent TMR VAE backward + +`tmr_parent_training.py` is a numerical seam, not a production experiment or +completed original-task replication. It uses the actual `TMRSet`, frozen +DistilBERT token features, separate MPNet negative-filter features, family-derived +positives, and every accepted window with its absolute chronology and gaps. + +Stochastic order matches `TMRSet.compute_loss`: encode/sample all texts, encode +all parents, draw all motion latents together, then reconstruct each parent's +positive captions and motion. Reconstruction uses the full-parent Hungarian +assignment, never independent per-window identities. Both reconstructions, four +KL terms, sampled latent alignment, and filtered full-batch contrastive loss +retain their coefficients and equal-parent weighting. + +Only Gaussian parameters, sampled latents and scalar reconstruction values are +cached. Decoder replay differentiates both sampled latent paths and decoder +parameters; the global objective then differentiates text and cached motion +distributions, followed by one-parent-at-a-time encoder replay. This bounds the +number of resident motion/decoder graphs, not the work within one complete +parent or the full text batch. It does not promise arbitrary-size memory use. + +Tests compare the original dense VAE loss, every parameter gradient, RNG, full +gallery, and physical drift rejection. Loss/RNG are exact in the frozen runtime; +shared FP32 gradient accumulation order may differ (rtol 3e-5, atol 3e-6). +These analytic oracles are not native optimizer, CUDA/BF16, convergence, pilot, +profile, production-host, or scientific-result evidence. Those gates remain +separate and require actual server receipts before formal training. diff --git a/docs/PHASESET_V2_WEAK_CF_TRAINING_SEAM.md b/docs/PHASESET_V2_WEAK_CF_TRAINING_SEAM.md new file mode 100644 index 0000000..4da6c42 --- /dev/null +++ b/docs/PHASESET_V2_WEAK_CF_TRAINING_SEAM.md @@ -0,0 +1,72 @@ +# PhaseSet-V2：独立弱监督训练接口资格 + +2026-09-27。按 [用户关系监督修订](PHASESET_V2_RELATION_SUPERVISION_AMENDMENT.md) +实现软件接口，不再等待两位新标注者。不改变官方人工主检索 gallery、最终 +test、87 阶段矩阵、三 seed 或 300 GPU-hours 上限。 + +## 已实现的边界 + +`ParentWeakCounterfactualRows` 使用独立 `included_weak` mask，没有 +`verified_false` 字段。正例必须是同一 parent 的原始人工行，负例必须追加在 +训练 text pool 的人工 gallery 前缀之后；与该 parent 原人工正例完全相同的 +句子不能作为弱负例。索引和 mask 检查不证明一个句子在真实视频中为假。 + +`weak_counterfactual_loss` 保留已登记 softplus margin 公式，独立于原人工 +verified-false 入口。`weak_coordination_objective` 的 variable-positive +symmetric InfoNCE 只使用原人工前缀；额外弱句子不成为检索正例或主 gallery。 +基础模型仍拒绝 CF 训练输入。 + +完整 parent cache/VJP/backward 和原 replay/RNG 路径支持独立弱行类型。 +`ParentBackwardResult.weak_counterfactual_count` 和主机 `weak_cf` 事件与原 +`verified_cf` 分开。它统计纳入的弱行数，不是独立真值数量；A8 的 CF weight +为 0 时仍记录相同输入池的纳入数量，但不贡献 CF loss。旧六字段位置构造 +保持兼容，TMR/WaMo/MIME 的新 weak count 默认 0。 + +## 一次真实服务器软件检查 + +Fresh ARIS GPT-5.6-Sol/xhigh 受限复核无 blocking/non-blocking，仅同家族 +provisional。随后只运行一次隔离尝试，2026-09-27 17:48:16–17:51:30 UTC： +CPU FP32、单计算线程/单 interop、nice 10、离线、无 CUDA、16 GiB 上限和 +900 秒 timeout。warm environment 没有重建，不分配或抢占其他 GPU。 + +**37 passed、0 skipped、0 failures/errors，pytest 189.61 秒。** 31 项新测试 +加 6 项显式原接口检查，覆盖： + +- margin/mask 公式和梯度、全排除 exact zero、错误 source/column/type 拒绝； +- 原人工前缀 InfoNCE、弱句子隔离、完整 dense-vs-replay 参数梯度一致； +- 真正 analytic AdamW 主机更新、中断/resume 的 state/optimizer/scheduler/RNG + 一致、弱与人工计数分离，以及 validation gallery 污染产生 FAILED terminal； +- TMR/WaMo/MIME 既有 analytic 主机端点对新增事件字段的兼容性。 + +包含一次 Torch nested-tensor prototype 警告，无异常；三 exit code 均为 0， +stderr 空，source/invocation 前后逐字一致。19 个 hashed metadata 成员及 +receipt 共 20 个 flat 文件双端核验；仅元数据私有留存，没有公开数据或权重。 +Receipt SHA-256： +`37ec17cb43d3c7706942cf7861c4a3753a415205dd9b9ab2789e30346e7f4ac7`。 + +生产预算三个原事件及 summary 前后不变。正式 **0/87**、pilot **0/12**，历史 +GPU 使用仍 34 秒，本次新增 0 秒。测试中的 analytic optimizer 是实际执行， +不是 native 训练、合格 pilot、收敛或论文检索成绩。 + +## 未完成与不能声称的内容 + +该接口不生成弱句子、不验证它们的来源/语义，也不把现有人工正例当成 false +CF 的证明。实际训练池、固定生成 prompt/选择规则、私有生成记录和 CLIP +编码准入仍待完成；当前 `ParentHumanClipRows` 继续只返回原人工行和空 human +CF，没有偷偷注入机器标签。 + +这些检查不关闭真实 native CF/yaw、GPU training profile、预算内 pilot 和 +87 阶段冻结/训练 gate，不证明任何关系理解或原人类挑战成功。 + +公开重现本页的 37 项选择（在合适的独立 CPU 环境中）： + +```bash +OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 MKL_NUM_THREADS=1 CUDA_VISIBLE_DEVICES='' \ +PYTHONPATH=src:tests python -m pytest -q tests/test_parent_weak_cf.py \ + tests/test_continuous_parent_training.py::test_complete_gallery_replay_matches_dense_loss_and_all_parameter_gradients \ + tests/test_continuous_parent_training.py::test_rng_advances_once_and_mode_restores_after_complete_capture_replays \ + tests/test_continuous_parent_host.py::test_host_runs_optimizer_selects_validation_and_preserves_full_parent_census \ + tests/test_continuous_base_retrieval.py::test_base_host_stage_checkpoint_and_cf_contracts_are_explicit +``` + +上述命令不包含私有预算 caller，不会重新建立真实训练准入或数据来源证据。 diff --git a/docs/PHASESET_V2_WEAK_CLIP_POOL.md b/docs/PHASESET_V2_WEAK_CLIP_POOL.md new file mode 100644 index 0000000..f8654d7 --- /dev/null +++ b/docs/PHASESET_V2_WEAK_CLIP_POOL.md @@ -0,0 +1,89 @@ +# PhaseSet-V2：训练-only缓存弱文本池 + +2026-09-27。按 [关系监督修订](PHASESET_V2_RELATION_SUPERVISION_AMENDMENT.md) +继续使用原官方人工主检索文本，机器句子只能作为单独披露的弱训练监督。 + +## 缓存与阶段隔离 + +`pool_frozen_clip_text_rows` 精确选择两个 original CLIP batch，再追加机器 +向量。人工前缀按位不变，不重新编码，不消耗 RNG。派生 receipt 保存两个 +original receipts、各自原行索引及人工前缀长度，明确不是一次新的 CLIP 编码。 +不同模型/snapshot/runtime、重复 commitment 和篡改 ancestry 被拒绝；原始 +rehydrator 不接收派生池。两个 original 的 source provenance 均保留，不 +虚构共同 encoder source。commitment 只用于谱系，不进入模型。 + +`ParentWeakClipRows` 绑定同一完整开发任务的人工 provider，显式接收当前 +阶段 training components、原机器编码 receipt 和每句来源记录。检查 source/ +family/人工 ordinal、政策/生成记录摘要及编码文本相符；机器文本不能是同一 +parent 的原人工正例。阶段 held-out 不进入训练，外层 fold 的训练组件可以 +包含旧主验证组件，不混淆这些不同任务。**这些检查不证明机器语义正确。** + +训练只选择当前 batch 对应的弱句子并追加于全部原人工 occurrence 后。 +没有候选的 parent 返回空 weak rows，不补造标签。evaluation 直接走原 +`ParentHumanClipRows`，不读机器池；基础模型在读取 motion 前拒绝 weak CF。 +既有 disk source 接入为显式可选项，原 human-only 路径保留。 + +## 一次真实服务器软件资格 + +Fresh ARIS GPT-5.6-Sol/xhigh 部署前受限复核 PASS/no findings，同家族 +provisional。2026-09-27 18:32:25–18:33:47 UTC，仅运行一次：单线程 CPU +FP32/interop 1、nice 10、离线/无 CUDA、16 GiB/900 秒限额，warm env 未改变。 + +**24 passed，0 skipped/failures/errors，76.80 秒。** 21 项新用例加 3 项 +相关原人工接口用例，覆盖缓存前缀、honest receipts、阶段过滤、独立计数、 +全人工 occurrence、重排、排除 mask、空候选、disk source、validation +不污染，以及 analytic AdamW 完整主机中断/resume bitwise 一致。 + +三 exit code 均 0、stderr 空、source/invocation 前后不变。20 个 flat +metadata 文件（19 hashed 加 receipt）双端核验，receipt SHA-256： +`6074d028be82ab5564f3577044190f9f0496769ae1d657f33e028d5c34b464a0`。 +原生产三个预算事件与 summary 不变；正式 **0/87**、pilot **0/12**，本次 +GPU 0 秒。analytic optimizer 用例不是原生学习或论文结果。 + +## 后续真实训练文本准备（不是训练结果） + +同日完成一次固定 pilot-learning 文本批次：34 个训练 parent、169 条原官方 +人工 occurrence，GPT-5.6-Sol 纳入 57 条弱句子、排除 112 条。仅读取这些 +parent 的人工 holistic 文本；不读取 motion、phase、模型分数、held-out +文本或 test。每 occurrence 只允许一次候选或排除，不按配额补齐，不做 +语义重生成。一次无语义输出的真实网络超时保留证据，修复传输后单次恢复； +首个成功 job 在完整批次复用。原始私有生成记录和完整批次均已冻结。 + +真实 CPU CLIP 准备于 19:40:37–19:42:15 UTC 完成：只编码这 57 条新增 +机器句子，未截断、最大未截断长度 71 tokens。原 1,334 条人工向量没有 +重新编码；当前 169 条训练人工前缀按位不变。全部 253 个开发 parent 的 +evaluation 仍只返回原人工文本，stage held-out 不进入 weak training。 +模型、RNG、源码、资产及生产预算前后不变，所有退出码 0、stderr 空。 + +这部分是数据准备，不是 native learning、pilot 或检索增益。 + +## 完整原生 capture 的实际 weak-CF backward + +同日 19:54:32–19:59:08 UTC，一次 CPU FP32/单线程检查使用两个完整 +四人 capture（各 5,800 帧、共 54 个有效窗口），不裁短、不拼接。 +完整 512D V2、随机未训练且冻结的 B2，使用 10 条原人工正例和 6 条 +实际生成弱负例，进行全 capture score cache、RNG replay 和 VJP。 +55 个可训练梯度张量全部存在且有限，有非零梯度；模型权重按位未改变。 +cache/replay/VJP 耗时 264.08 秒；退出码均 0、stderr 空。 +receipt SHA-256： +`5e5889cdf7666b2f592dde5c81db4f6e536d236187ae1018203b45fb54f4b50d`。 + +没有 optimizer、GPU、非零训练 yaw、正式学习或检索成绩；这不是完整训练 +成本或性能结论。生产预算保留历史 GPU 用量，新增两个 CPU profile +记账事件已闭合，正式 **0/87**、pilot **0/12**。 + +一次训练文本 yaw 分类在固定时限内没有返回，失败证据保留，不重生成。 +该调用没有提供任何资格决定。后续诊断对未知资格显式使用零 yaw，并记录 +“资格未知”，不能称为语义否决、非零 yaw 验收或全关增强政策通过。 +GPU profile 和正式学习仍待实际可用资源。机器标签永不升级为人工 +`verified_false`、独立运动真值或历史两人挑战成功。 + +公开选择重现（独立合适 CPU 环境，不含私有预算 caller）： + +```bash +OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 MKL_NUM_THREADS=1 CUDA_VISIBLE_DEVICES='' \ +PYTHONPATH=src:tests python -m pytest -q tests/test_parent_weak_clip_rows.py \ + tests/test_parent_clip_rows.py::test_selection_is_exact_owned_and_honest_without_encoder_runtime_or_rng \ + tests/test_parent_clip_rows.py::test_every_human_occurrence_and_reordered_parent_batch_are_preserved \ + tests/test_parent_clip_rows.py::test_disk_source_binds_the_same_task_and_returns_explicitly_no_cf +``` diff --git a/docs/PREPARED_DATA_V2.md b/docs/PREPARED_DATA_V2.md new file mode 100644 index 0000000..7270b6a --- /dev/null +++ b/docs/PREPARED_DATA_V2.md @@ -0,0 +1,145 @@ +# Prepared motion-text batches and epoch augmentation + +`phaseset_core.prepared_data_v2` connects already prepared body-22 motion +windows to already frozen CLIP text features. It writes immutable numeric +training artifacts and reads them as `TrainingDataSource` objects. It does not +download or authorize data, decode SMPL-X, generate captions, or run CLIP. + +## Interface + +`PreparedMotionTextBlock` contains canonical `PreparedGroupSample` values, +one `FrozenClipTextBatch`, variable per-motion `text_counts`, and independent +uint64 `window_ordinals`. Every motion receives at least one text row; no fixed +three-caption or two-caption assumption is made. The supplied frozen text +receipt must bind the actual embedding bytes, row order, and commitments. +Its `batch_size` denotes the CLIP inference chunk size, not total text rows. +The exact contiguous `chunk_ranges` must cover the complete row census; all +existing tensor and individual-row checks remain. See the +[actual CLIP composition correction](CAPTION_ROLE_PROVENANCE.md). + +`write_prepared_training_tree_v2(...)` takes separate train/val blocks, an edge +budget, optional microbatch-size bound, and explicit artifact-size bounds. It +creates a new tree and returns its index and census digests. Existing output +roots cannot be reused. Failed partial builds are preserved, not admitted as +completed datasets. + +`load_prepared_training_sources_v2(...)` requires the expected index digest +and returns train and val sources. Each split constructor verifies the exact +manifest bytes it consumes against the digest in that index. Test cannot be +inserted into this training-only index. + +## Numeric artifact contract + +Every uncompressed NPZ has exactly ten arrays: + +| Name | Type and shape | +|---|---| +| `skeletons` | float32 `[B,K_pad,200,22,3]` | +| `actor_mask` | bool `[B,K_pad]` | +| `frame_mask` | bool `[B,200]` | +| `track_mask` | bool `[B,K_pad,200,22]` | +| `actor_commitments` | uint8 `[B,K_pad,32]` | +| `group_commitments` | uint8 `[B,32]` | +| `text_embeddings` | float32 `[Q,512]` | +| `motion_positive_ids` | uint8 `[B,32]` | +| `text_positive_ids` | uint8 `[Q,32]` | +| `text_commitments` | uint8 `[Q,32]` | + +Padding is exact positive zero. Output creation is write-once with read-only +file permissions; this is not operating-system immutability. Authenticated +digests bind the bytes actually consumed. Storage uses fixed-order ZIP_STORED members +and NPY v1 headers. Loading checks member identity, shape/type, declared byte +length, CRC, and decoded-size bounds before array allocation; pickle is not +accepted. The writer rejects train/val overlap in window, ordinal, caption, +and actor commitments. Upstream participant/capture split verification remains +necessary; these numeric checks do not establish participant identity. + +## Epoch behavior + +Stored motion always has exact positive-zero augmentation yaw. Each training +epoch rereads the original authenticated bytes and applies the registered +`deterministic_group_yaw(seed, epoch, window_ordinal)` once, synchronously to +all actors. The angle is not derived from any lineage commitment. Rotation is +computed in float64 and cast to float32, then all invalid positions are reset +to exact positive zero. No augmented sample becomes the next epoch's input. + +Validation is unrotated. Masks, lineage, positive families, caption identity, +and frozen text values do not change. This storage-based path is not claimed +bitwise equal to rotating earlier, before preprocessing's final float32 cast. + +## Descriptor source context + +Actual returned `RetrievalTrainingBatch` values now carry an optional immutable +`descriptor_contexts` tuple, one exact context per motion row. Prepared v2 +derives it from the same verified split-manifest bytes, consumed NPZ digest, +global ordinal and motion-window identity used to load that row. Train contexts +record the actual seed, epoch and exact float64 yaw bits. Validation keeps its +historical unrotated iteration behavior while contexts use epoch zero and +exact positive-zero yaw. Legacy batches default to `None`. + +Batch validation requires matching row identities/split, one common source, +seed and epoch, and unique window identities/ordinals. The raw rotation helper +rejects an already contextualized batch before converting arrays, preventing +second rotation with stale first-rotation lineage. Numeric or padding changes +remain distinct complete-batch cache digests even when lineage is unchanged. + +These contexts are integrity metadata, not cache admission, data rights or +proof of execution. Epoch-complete cache lookup, training replay and checkpoint +binding are separate unfinished layers; main holistic capture validation has +its own source lineage and is not replaced by this window source. + +## Provenance and integration boundary + +The private build census binds index/batch/window/source/text digests and the +independent ordinals. It contains no raw captions, source paths, or participant +raw identifiers and is not a data-rights certificate. Stable commitments may +still be linkable; they are not an anonymity guarantee. Keep the entire +prepared tree, its commitments, and its census private. + +The private-host factory recognizes the v2 index and passes the actual +receipt-bound index digest into this loader. A replaced index or manifest, or +invalid decoded training batch, is rejected before backend construction. The +legacy v1 loader remains supported and also binds the manifest bytes consumed. + +Positive families in this storage format identify individual windows. They +support the auxiliary window-caption task, not the main holistic capture task. +Actor-set group commitments are lineage, not unique capture/window row IDs. +Main validation must separately encode every admitted window in a capture, +aggregate embeddings with the fixed capture mean, and score one capture gallery +against official holistic text. The [capture validation API](CAPTURE_VALIDATION.md) +now implements that operation and training checkpoint selection. The separate +capture-storage route is connected through the private host's combined index; +it does not relabel this auxiliary window source as the primary task. + +This module alone is not the complete `prepare-data` command. Native capture +loading, licensed body-model conversion, preprocessing rejection/tail census, +caption provenance, and the preparation command composition must also be wired +and verified before real training. See [the frozen text adapter](FROZEN_CLIP_TEXT_ADAPTER.md) +for text feature semantics and [host integration](HOST_RUNTIME_INTEGRATION.md) +for the execution lifecycle. + +Server tests exercise analytic numeric artifacts, variable caption counts, +epoch replay, padding zeros, validation invariance, split isolation, immutable +creation, and invalid/replaced manifests and ZIP payloads. They are regression +evidence, not real-data experiments or scientific results. The final combined +host integration passed 94 focused tests and a separate complete Linux server +suite of 964 tests, 2 skipped, and 39 subtests, with unchanged source bytes. + +Later source-lineage tests cover all three official seeds and all 20 residual +epochs against the actual returned rotated values, canonical validation, +legacy behavior and altered lineage/padding. The 89-test focused integration +passed on the Linux server. A full integration exposed one old negative +fixture whose deliberately changed family was now rejected earlier by the +new batch invariant. Its context family was changed consistently to exercise +the original caption-role rejection; the production validator and expected +error were unchanged. The subsequent combined suite, including text-boundary +and CUDA-identity contracts, passed 136 focused tests, then 1257 full tests, +two existing skips and 39 subtests in 413.42 seconds, source unchanged. + +The live official CLIP adapter's imported training-boundary source pin was +updated to the reviewed source bytes. An actual server call reproduced the +earlier embedding bytes exactly, using four/two text rows in three chunks; +CPU Torch RNG was unchanged and repeated provenance validation made no new +encode call. Receipts retain new source identities without relabeling old +receipts. Analytic motion and explicitly fabricated fusion fixtures in that +check do not establish semantic caption quality or licensed main data. diff --git a/docs/RESIDUAL_RESUME.md b/docs/RESIDUAL_RESUME.md new file mode 100644 index 0000000..81e909a --- /dev/null +++ b/docs/RESIDUAL_RESUME.md @@ -0,0 +1,73 @@ +# Base and residual host resume + +The private host supports both registered base-qualification and residual +training attempts through `phaseset resume`. The public CLI still requires the +existing authenticated host adapter; this feature does not manufacture data +rights, a qualified base, or an experiment result. + +## Command contract + +Base resume accepts an existing predecessor checkpoint and a new attempt +directory. Residual resume additionally requires `--base-checkpoint` and +`--periodic-cache`, the same artifact inputs as `run-residual`. Base resume +rejects these residual-only arguments. Resume has no stop-after-step option. + +The verified predecessor run ID determines the seed, system, and legal role. +An explicit system argument must agree. Both attempts must share their private +attempt root. Symlinked paths, absent artifacts, incompatible roles, or a +checkpoint other than the terminal-selected checkpoint are rejected. + +The residual host reconstructs, in order, the base qualification, selected +frozen base, validated periodic-cache energy floors, seed capacity audit, and +registered residual constructor. The unchanged training runtime then verifies +its configuration, data, code/environment, initialization, optimizer, RNG, and +resume-chain bindings before continuing. + +## Immutable attempts and checkpoint copies + +Resume creates a new attempt rather than rewriting its predecessor. A process +lease excludes a simultaneous writer. Process-loss recovery acquires the same +lease and uses already persisted checkpoint evidence; it does not guess from +PID age or invent a missing checkpoint. + +The host copies the authenticated latest checkpoint into the new attempt with +exclusive creation, a regular-file descriptor check, streaming SHA-256, and +fsync. It reads and rehashes the bytes from the same open destination descriptor +before decoding an owned byte buffer with `weights_only=True`. Unverified +checkpoint bytes are never passed to the decoder. + +If that payload refers to an older best-validation checkpoint, only the named, +digest-bound best file is copied too. This allows the unchanged runtime to find +its best checkpoint under the new attempt directory. The copies do not replace +the predecessor ledger or alter checkpoint payloads. + +A failed copy is retained inside the new failed attempt for diagnosis. The host +does not use a check-then-unlink sequence that could delete a replacement file. +Failure artifacts and a truthful FAILED terminal are written; retained partial +bytes are not accepted as a resumable checkpoint. The predecessor is unchanged. + +## Cache provenance boundary + +Existing checkpoints bind the consumed six float64 energy floors through the +registered residual initialization, together with the selected frozen base, +qualification, capacity audit, and factory/code identity. They do not bind the +historical periodic-cache filename, complete file SHA, or descriptors that the +runtime did not consume. Resume validates the supplied cache and reconstructs +the same consumed state; it does not retroactively claim file provenance. + +## Observed verification + +On the registered Linux server, 51 focused tests passed in 50.29 seconds with +unchanged source bytes. Coverage includes role/artifact rejection, real attempt +ledgers, latest and older-best materialization, changed checkpoint bytes rejected +before decoding, zero runtime calls on invalid prior-best input, failure +terminalization, and a tiny genuine optimization/checkpoint/interruption/resume +comparison against uninterrupted execution. + +Two earlier test-fixture failures were retained and corrected without changing +the host implementation: an incomplete fake report and an outdated deletion +expectation. The tiny actual runtime test already passed in that first attempt. +These are engineering tests, not formal PhaseSet training or scientific results. + +The complete server regression with this integration then passed 948 tests, +two skipped, and 39 subtests in 376.00 seconds, again with unchanged source bytes. diff --git a/docs/TMR_COMPONENTS.md b/docs/TMR_COMPONENTS.md new file mode 100644 index 0000000..d15498c --- /dev/null +++ b/docs/TMR_COMPONENTS.md @@ -0,0 +1,44 @@ +# TMR implementation components — not a trained TMR-Set result + +The [component implementation](../src/phaseset_core/tmr_components.py) follows +the [official pinned TMR source](https://github.com/Mathux/TMR/tree/6d74688730d15d43b0a755ce2b0e1f2d76138fc1), +under its [MIT license](../LICENSE_TMR.md). Defaults are the actual +training YAML: latent 256, FFN 1024, six layers, four heads, dropout 0.1, GELU. +The complete objective keeps both SmoothL1 motion reconstructions, all four +element-mean KL divergences, sampled latent SmoothL1 alignment and filtered +contrastive loss. Weights are 1, 1e-5, 1e-5, 0.1; temperature is 0.1. + +The encoder prepends two learned distribution tokens. The decoder attends to +one sampled latent using zero time queries plus sinusoidal positions. Retrieval +uses distribution means. The caller supplies frozen DistilBERT token features +(default dimension 768), and separate normalized MPNet sentence features for +negative filtering; global CLIP vectors are not substituted for either. + +The original diagonal, one-caption objective is available through +`TMRSinglePerson`. This is an architecture/loss equivalence tool, not evidence +that the original dataset preprocessing or converged original-task replication +has been completed. It does not implement the group decoder, language asset +loader, complete-parent training host, sampler, or pilot admission. + +Two explicit extensions are provided for later adaptation: + +- `filtered_contrastive_loss` accepts a rectangular known-positive matrix. A + motion–text negative is filtered when that text exceeds cosine **0.6** against + any known caption of that motion (official threshold 0.8 maps to 2t−1). + Known positives are always preserved. Similarity never creates a positive. + With a diagonal positive matrix this reduces to the original filter and + symmetric cross-entropy, up to frozen floating-point tolerance. +- Reconstruction defaults to the original mean over the padded target tensor. + An explicit feature-level `observed` mask instead computes the mean only over + observed values, preserving genuine capture gaps. This changes normalization + and must be disclosed/frozen for the group adapter; gaps are not zero-valued + reconstruction targets. No set reconstruction assignment is implied here. + +Position capacity defaults to the upstream 5000. Longer sequences require an +explicit larger `max_len` using the same analytic encoding; there is no silent +truncation. Encoder capacity includes its two prefix tokens. Dense attention +over whole native parents has not been profiled or approved for training. + +Training-only pool selection remains the host's responsibility. Semantic +filtering must not inspect held-out/test captions and must not be applied to the +study's common human-only full-gallery retrieval metrics. diff --git a/docs/TMR_SET_ADAPTATION.md b/docs/TMR_SET_ADAPTATION.md new file mode 100644 index 0000000..11c18b2 --- /dev/null +++ b/docs/TMR_SET_ADAPTATION.md @@ -0,0 +1,75 @@ +# Full-parent TMR-Set adaptation + +[The implementation](../src/phaseset_core/tmr_set.py) retains the +[qualified TMR components](TMR_COMPONENTS.md), including the actual six-layer +ACTOR VAE, two reconstructions, four KL terms, sampled-latent alignment and +semantic-negative-filtered contrastive training. It is explicitly a group +adaptation, not the original single-person TMR or a converged replication. + +Each actor's body22 coordinates use the existing capture-wide shared origin. +A shared ACTOR encoder processes each accepted window; unordered PMA pools its +actor states. A second ACTOR encoder reads every window summary in chronological +order, with its absolute start time. Real gaps remain gaps: later windows are +not reassigned earlier timestamps. Text uses the original 768D DistilBERT token +features and separate normalized MPNet sentence features, not sentence CLIP. +Group, text and decoder defaults are 256D, FFN 1024, six layers, four heads, +dropout 0.1. The capture hierarchy/PMA are additional disclosed capacity. + +The declared optimizer starting point is lr1e-4 from the +[pinned training YAML](https://github.com/Mathux/TMR/blob/6d74688730d15d43b0a755ce2b0e1f2d76138fc1/configs/model/tmr.yaml), +with AdamW from the +[pinned optimizer implementation](https://github.com/Mathux/TMR/blob/6d74688730d15d43b0a755ce2b0e1f2d76138fc1/src/model/temos.py). +That call leaves weight decay at the PyTorch AdamW default, made explicit as +0.01 in this host. `ParentHostConfig.for_literature` exposes these values; +bounded pilot overrides must be declared. The common group-study warmup/cosine +schedule is an adaptation, not the upstream training loop. + +[The language loader](../src/phaseset_core/tmr_language.py) checks every +tokenizer/config/checkpoint file against pinned public digests before loading +from private directories, with local-only safetensors and no remote code. +DistilBERT is fixed at `12040accade4e8a0f71eabdb258fecc2e7e948be`; MPNet at +`e8c3b32edf5434bc2275fc9bab85f82640a19130`. Both are Apache-2.0 pretrained +assets. The latter uses the upstream attention-masked mean plus L2 normalization. +Language models stay frozen/eval even when the trainable group model is trained. +Any caption exceeding the actual token bound fails explicitly instead of being +silently truncated. No dataset, model weights, download receipts or private +asset paths are packaged. Asset download alone is not runtime qualification. + +Both sampled motion and text latents reconstruct the entire observed group. +The original six-layer decoder is reused over bounded windows, conditioned on +the global latent, absolute window time, and analytic output-slot codes. These +are **prediction slots**, not embeddings of input actor ordinal/identity. +There is no learned finite actor-slot bank or maximum semantic K. Known K sets +the auxiliary output cardinality only; K is not appended to retrieval features. + +For each reconstruction, SmoothL1 costs are accumulated over every window and +observed body coordinate. A single minimum-cost slot-to-track assignment is +then solved over the entire parent. This one assignment prevents a decoder from +changing a slot's actor identity at each window to lower its reconstruction +loss. The selected summed loss is divided by the parent's total observed +coordinate count, retaining both reconstruction branches and ignoring gaps. +No first actor is chosen as the target. Matching uses a K-by-K scalar table and +O(K^3) Hungarian solver; no dense K-by-K-by-time feature tensor is allocated. +The matching cost computation is O(K^2 T); these costs must be profiled. + +Known captions remain variable in number. Reconstruction/KL/latent terms are +averaged within each parent's captions, then across parents, so caption-rich +parents do not multiply auxiliary weight. The full rectangular contrastive +gallery preserves all known positives; sentence similarity only filters +training negatives against that run's learning captions, not evaluation labels. +Retrieval uses distribution means and the common unfiltered human-only gallery. + +Actor commitment bytes canonicalize calculation and tie-breaking order only; +they are never embedded. Shared input encoders and invariant set pooling do +not use actor ordinal. Whole-window encoder and cost checkpointing preserve +dropout/sampling RNG and pass explicit on-device tensors for CUDA discovery. +Capture inputs themselves are not O(1) memory; no such claim is made. + +`TMRGroupCapture.from_prepared` includes every accepted window of the supplied +admitted complete capture. A hand-built window tuple is a software seam, not +proof of native provenance/completeness. Runtime caller responsibilities still +include fixed asset hashes, correct body/text provenance, split/budget/seed +admission, full-parent optimizer/checkpoint/resume host, and actual hardware +qualification. GPU/BF16, native learning, convergence, and fair bounded pilot +evidence are not granted by component tests. Temporal decoder conditioning and +whole-track assignment are adaptations, not a claim of upstream bitwise identity. diff --git a/docs/WAMO_SET_ADAPTATION.md b/docs/WAMO_SET_ADAPTATION.md new file mode 100644 index 0000000..a64d3ed --- /dev/null +++ b/docs/WAMO_SET_ADAPTATION.md @@ -0,0 +1,74 @@ +# WaMo-Set: paper-defined complete-parent adaptation + +This is an independently written implementation of the methods in +[WaMo v2](https://arxiv.org/html/2508.03343v2), not released author code or a +claim of original-benchmark reproduction. The audited +[official repository](https://github.com/3DAgentWorld/WaMo) at +`cef15e9c75e736e3351f052e39e0aee12b13ad0e` contains no implementation; +the latest read-only tree check also found only README and attributes. + +The retained components are learnable, dilated stationary wavelet analysis +without temporal decimation; separate intra-frequency temporal encoders; +concatenation and inter-frequency temporal encoding; additive frame attention; +independent learnable inverse SWT plus intra-band reconstruction MLPs; +inter-feature reconstruction; and original/partially shuffled temporal-group +classification. Both reconstructions use SmoothL1; both ordering branches use +cross-entropy. Text is projected frozen DistilBERT CLS, not CLIP or an MPNet +sentence replacement. The caller supplies verified local frozen language. + +Paper-specified settings: decomposition level 3, latent width 256, temperature +0.07, 16 temporal groups, shuffled fraction 0.25, Haar/db1 initialization +(Table 9), Adam learning rate 1e-4 and cosine schedule. The model implements the +paper's *sum* of directional contrastive losses, not a halved objective. +For variable numbers of human captions, known positives use logsumexp in the +numerator and all gallery candidates in the denominator. No semantic filtering +or pseudo-positive mining is introduced. + +Unpublished details are disclosed project choices: circular SWT boundaries; +two-tap Haar analysis `[1,1]/sqrt(2)` and `[-1,1]/sqrt(2)`; independent synthesis +initialized to half those taps to account for undecimated redundancy; depthwise +coordinate convolutions of width 9 (low) and 3 (high); two Transformer layers, +four heads, FFN 1024, GELU and dropout 0.1; two-layer projection/decoder MLPs; +unit reconstruction and ordering weights. These are configurable choices, +not purported author defaults. Learnable filters are not constrained to remain +orthogonal or perfectly reconstructing. Initial reconstruction is tested. +The existing three WaMo short-pilot slots, not an additional search, govern +any permitted configuration selection before the common contract freezes. + +`ParentHostConfig.for_literature("WaMo-Set", ...)` declares Adam, learning rate +1e-4 and zero weight decay (no decay is specified in the paper), instead of +the internal host's AdamW/0.01 default. The settings are checkpoint/resume-bound; +pilot overrides must be explicit. The common group host adds 5% warmup before +cosine decay and clips gradients at 1.0. These are disclosed shared training +adaptations, not extra purported author settings or convergence evidence. + +Multi-person/whole-parent extensions: + +- Reuse the admitted complete body22 capture contract; shared per-actor + processing of *every* accepted 200-frame window, unordered actor PMA, + followed by a capture temporal encoder and additive pooling. +- The capture hierarchy receives actual absolute window starts; omitted + invalid windows leave their temporal gaps rather than being collapsed. + This bounded-window wavelet hierarchy differs from a single-person transform + across an entire continuous trajectory and must be reported as such. +- Actor commitments choose a deterministic reduction order only; no ordinal, + actor-specific weights or commitment features enter the model. Dynamic K + has no fixed slot bank. Reconstruction is actor-aligned autoencoding, not + text-to-motion set reconstruction, so no Hungarian matching is appropriate. +- DMSP groups are defined within each accepted window, approximately evenly + by `floor(frame_index * 16 / window_length)`. A uniform subset of + `floor(0.25*T)` frames is permuted; positions outside the subset are untouched. + The same permutation moves all actors, masks and original labels together. + This concrete frame-subset realization and window-local objective are + disclosed adaptations. Labels never enter the encoders; positional codes + refer to current sequence positions, not shuffled origin positions. +- Unobserved coordinates remain zero and do not contribute reconstruction; + entirely unobserved actor frames do not contribute either ordering loss. + Window losses accumulate coordinate/frame sums, normalize over the complete + parent's observed coordinates/actor frames, then average parents, so a longer + parent or more captions do not multiply its auxiliary weight. Masks are + still visible through the input and must be covered by missingness controls. +- The model is not a production trainer: provenance, split selection, + optimizer/schedule, full effective negative gallery, budgets and immutable + attempt admission remain the host's responsibility. Qualification alone + does not establish learning, original-task reproduction or paper results. diff --git a/figures/EDITABLE_FIGURES.json b/figures/EDITABLE_FIGURES.json new file mode 100644 index 0000000..364d80e --- /dev/null +++ b/figures/EDITABLE_FIGURES.json @@ -0,0 +1,81 @@ +{ + "figures": [ + { + "empirical_result": false, + "external_relationships": 0, + "native_connector_segments": 18, + "native_container_headers": 2, + "native_edge_labels": 0, + "native_node_shapes": 16, + "native_shapes": 18, + "package_parts": 36, + "pptx": "phaseset_motivation.pptx", + "pptx_sha256": "4c7ef9150eddd60f9ee75adbd4d17f2755c8144aad796251a0310a6dd545b048", + "raster_media": 0, + "source": "phaseset_motivation.drawio", + "source_edges": 14, + "source_sha256": "a0a25fbc8c55575d50c6ffa8d6add01ce780cc8842328bb3f6bba20c6509d18a" + }, + { + "empirical_result": false, + "external_relationships": 0, + "native_connector_segments": 41, + "native_container_headers": 2, + "native_edge_labels": 3, + "native_node_shapes": 26, + "native_shapes": 31, + "package_parts": 36, + "pptx": "phaseset_architecture.pptx", + "pptx_sha256": "9793fcafb8dccbd1d19a75ee7cfe23a654ec03bf193d84574a9dcbc38260b046", + "raster_media": 0, + "source": "phaseset_architecture.drawio", + "source_edges": 19, + "source_sha256": "1a61a503c935a72cbfbe1ce175ab54bf975eb3e0a2164018aeda80448f472a1a" + } + ], + "package_policy": { + "compression": "ZIP_STORED", + "create_system": 0, + "external_attr": 25165824, + "retained_part_count": 36, + "timestamp": [ + 1980, + 1, + 1, + 0, + 0, + 0 + ] + }, + "renderer": { + "bytes": 33666, + "name": "render_editable_figures.py", + "sha256": "18babb8ff4042dc091bb426e6679186f7427b3b01695d82e56d4468de2740222" + }, + "runtime": { + "libxml2": [ + 2, + 11, + 9 + ], + "libxslt": [ + 1, + 1, + 45 + ], + "lxml": "6.1.1.0", + "machine": "AMD64", + "python": "3.12.14", + "python_implementation": "CPython", + "python_pptx": "1.0.2", + "system": "Windows", + "zlib_build": "1.3.2", + "zlib_runtime": "1.3.2" + }, + "schema": "phaseset-editable-figure-census-v2", + "template": { + "bytes": 34030, + "name": "python-pptx-default.pptx", + "sha256": "e10cc9e120961f6bd4074a373c9c80d2a06c497157e8f4972977b7bea83a8f34" + } +} diff --git a/figures/README.md b/figures/README.md index a74756f..ecc281c 100644 --- a/figures/README.md +++ b/figures/README.md @@ -1,19 +1,74 @@ # PhaseSet paper figures -The figure sources are data-free and contain no empirical result. +These are data-free conceptual/method figures, not empirical results. -| Figure | Editable source | Publication exports | Slide version | +| Figure | Editable source | Publication exports | Native editable slide | |---|---|---|---| | Pair-bag incidence counterexample | `phaseset_motivation.drawio` | PDF, SVG, PNG | `phaseset_motivation.pptx` | | PhaseSet architecture | `phaseset_architecture.drawio` | PDF, SVG, PNG | `phaseset_architecture.pptx` | -The `.drawio` files are the semantic/editable authority. PPTX files contain a -lossless full-slide rendering for convenient placement; edit the matching -draw.io source and regenerate all exports rather than manually changing one -format. Paper captions and numerical annotations must be rendered from the -frozen aggregate once real results exist. +The draw.io files are the shared semantic/layout source. The PPTX files now +contain native PowerPoint shapes, text runs, and connector segments, with **no +slide-wide raster image**. Individual labels and nodes are editable. Regenerate +all formats from draw.io for a consistent publication update; a PowerPoint-only +edit does not update the source or other exports automatically. -The motivation figure deliberately constructs two four-actor complete graphs -with the same actor marginals and the same multiset of six pair tokens (two of -each relation type) but different actor–edge incidence. Pair-bag averaging is -therefore identical, while incident moments can distinguish the structures. +The motivation figure contains 18 native shapes/text boxes and 18 connector +segments; the architecture contains 31 and 41. `EDITABLE_FIGURES.json` binds each +source and PPTX by SHA-256 and records the native-object census. The renderer +uses an allowlisted, uncompressed ZIP package with fixed metadata and records +the renderer, document runtime, and default template hashes. Repeat rendering +in that recorded runtime produced byte-identical PPTX files. Its fixed creation date is +not an assertion about when the research or figure was created. + +## Rebuild + +Canonical native-PPTX rebuilding requires the recorded Windows AMD64 CPython +3.12.14 document runtime, python-pptx 1.0.2, lxml 6.1.1.0 / libxml2 2.11.9 / +libxslt 1.1.45, and zlib 1.3.2. The renderer rejects other tuples and altered +source filenames/bytes rather than issuing a canonical-build receipt. Opening +or editing the provided PPTX does not require that Python runtime. The portable +artifact tests verify the delivered hashes and native objects without it. + +Within the registered document runtime, run the closed project renderer: + +```bash +python -m pip install 'python-pptx==1.0.2' +python scripts/render_editable_figures.py --figure-dir figures --output-dir figure-build +``` + +This renderer supports only the limited shapes/styles used in these two pages; +it is not a general draw.io importer. Draw.io remains the native publication +exporter. With draw.io 31.1.8, export PDF with `--export --format pdf --embed-diagram +--crop`, and SVG/PNG with `--export --format svg` or `png` plus `--embed-diagram`. +Use explicit output files in a new build directory. Check the PNG IEND trailer +after export; the known truncated-IEND exporter defect must be repaired before +strict PNG validation. Embedded diagram metadata is scanned with the rendered +content before public release. + +On 2026-09-08 both final native slide files were opened in Microsoft PowerPoint +without a repair prompt and visually inspected; their hashes were unchanged +after inspection. Further renderer gate hardening reproduced the same PPTX +bytes. Both packages pass exact object-identity, relationship, XML, and ZIP +checks and reopen in python-pptx. Unused thumbnail/printer parts were removed. +This is not a claim about arbitrary future input. +PDF exports use embedded Arial TrueType fonts; +Unicode math subscripts that caused Type 3 fallback have been replaced by +native subscript text runs. The `*_preview.png` files are display conveniences, +not independent figure sources. + +## Scientific correspondence + +The motivation figure constructs two four-actor complete graphs with identical +actor marginals and the same six-token pair multiset, but different actor-edge +incidence. Pair-bag averaging is identical; incident moments can distinguish +the constructed structures. It is not an observed motion sample. + +The architecture depicts the current implementation: fixed mask-aware body +activity feeds the physical Morlet branch, in parallel with the qualified +learned B0/B1/B2 group base. The learned base embedding does not feed Morlet. +For residual training the selected seed-specific base is frozen. Both its +global text score and the six-band text alignment contribute to retrieval. +This drawing correction does not change the model or claim encoder-to-Morlet +end-to-end gradients. Empirical annotations and paper values must come from +the same frozen aggregate when actual results exist. diff --git a/figures/phaseset_architecture.drawio b/figures/phaseset_architecture.drawio index a3fd630..6eb5b96 100644 --- a/figures/phaseset_architecture.drawio +++ b/figures/phaseset_architecture.drawio @@ -5,91 +5,103 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - + + + + + + + + + + + + - + - + - + - + - - - - - + + + + + - - - - - + + + + + + + + + + + + + + + + + diff --git a/figures/phaseset_architecture.pdf b/figures/phaseset_architecture.pdf index 883a603..a48e202 100644 Binary files a/figures/phaseset_architecture.pdf and b/figures/phaseset_architecture.pdf differ diff --git a/figures/phaseset_architecture.png b/figures/phaseset_architecture.png index 4e533b7..eac5b4f 100644 Binary files a/figures/phaseset_architecture.png and b/figures/phaseset_architecture.png differ diff --git a/figures/phaseset_architecture.pptx b/figures/phaseset_architecture.pptx index 398f308..df5eeb7 100644 Binary files a/figures/phaseset_architecture.pptx and b/figures/phaseset_architecture.pptx differ diff --git a/figures/phaseset_architecture.svg b/figures/phaseset_architecture.svg index 760bb37..87885b9 100644 --- a/figures/phaseset_architecture.svg +++ b/figures/phaseset_architecture.svg @@ -1,4 +1,4 @@ -
PhaseSet: streamed permutation-invariant group periodic tokens
Motion path
Group skeletons
(B,K,T,22,3)
dynamic actor mask
Shared actor
motion encoder
no actor IDs
Six fixed Morlet bands
compute each actor once
Stream all unordered pairs
canonical 64-edge blocks
no dense K×K tensor
Directed 13D relations
uᵢⱼ, uⱼᵢ
Symmetric PairMLP
sum |diff| product
Incident moments
Δmean, M2, coverage, missing
Pair component
canonical masked mean
NodeMLP + hard gate
K=2: exact +0 and zero grad
CommonPostprocess
pair + tanh(λ) · topology
Six group tokens
(B,6,512)
Permutation invariant
O(K²) streamed edges
RESOURCE_LIMIT, never sampling
Language path and retrieval
Human group text
Frozen CLIP
text tower
Shared TextBandMLP
+ learned band ID
Six semantic
text-band tokens
Frozen base score
+ bounded mean band cosine
motion bands
Input / output
Signal / relation
Attention / topology
Projection / reduction
Gate / score
Text is not SVG - cannot display \ No newline at end of file +
PhaseSet: streamed permutation-invariant group periodic tokens
Motion path
Group skeletons
(B,K,T,22,3)
dynamic actor mask
Shared body-motion
activity extraction
fixed, mask-aware
Six fixed Morlet bands
compute each actor once
Stream all unordered pairs
canonical 64-edge blocks
no dense K×K tensor
Directed 13D relations
uij, uji
Symmetric PairMLP
sum |diff| product
Incident moments
Δmean, M2, coverage, missing
Pair component
canonical masked mean
NodeMLP + hard gate
K=2: exact +0 and zero grad
CommonPostprocess
pair + tanh(λ) · topology
Six group tokens
(B,6,512)
Permutation invariant
O(K²) streamed edges
RESOURCE_LIMIT, never sampling
Qualified B0 / B1 / B2 base
shared actor encoding + group pooling
frozen per seed for residual systems
Language path and retrieval
Human group text
Frozen CLIP
text tower
Shared TextBandMLP
+ learned band ID
Six semantic
text-band tokens
Frozen base score
+ bounded mean band cosine
global text embedding
motion bands
Input / output
Signal / relation
Attention / topology
Projection / reduction
Gate / score
frozen group embedding
Text is not SVG - cannot display \ No newline at end of file diff --git a/figures/phaseset_motivation.drawio b/figures/phaseset_motivation.drawio index 035ba27..4913d6d 100644 --- a/figures/phaseset_motivation.drawio +++ b/figures/phaseset_motivation.drawio @@ -5,94 +5,94 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + diff --git a/figures/phaseset_motivation.pdf b/figures/phaseset_motivation.pdf index 3d017e8..ef4e6c7 100644 Binary files a/figures/phaseset_motivation.pdf and b/figures/phaseset_motivation.pdf differ diff --git a/figures/phaseset_motivation.png b/figures/phaseset_motivation.png index 086c6f9..2132da8 100644 Binary files a/figures/phaseset_motivation.png and b/figures/phaseset_motivation.png differ diff --git a/figures/phaseset_motivation.pptx b/figures/phaseset_motivation.pptx index c8b4c2b..8ca0107 100644 Binary files a/figures/phaseset_motivation.pptx and b/figures/phaseset_motivation.pptx differ diff --git a/figures/phaseset_motivation.svg b/figures/phaseset_motivation.svg index c76d06a..6d5ed4f 100644 --- a/figures/phaseset_motivation.svg +++ b/figures/phaseset_motivation.svg @@ -1,4 +1,4 @@ -
Why pair-bagging is not enough
Group A: balanced incidence
A
m₁
B
m₂
C
m₃
D
m₄
Every actor: 1 red + 1 blue + 1 gray incident relation
Group B: concentrated incidence
A
m₁
B
m₂
C
m₃
D
m₄
A: 2 red; D: 2 gray; B/C: mixed incident relations
Identical actor marginals: {m₁,m₂,m₃,m₄}
Identical pair-token multiset: {red×2, blue×2, gray×2}
Pair-bag mean
identical
PhaseSet incidence moments
distinguishable
Text is not SVG - cannot display \ No newline at end of file +
Why pair-bagging is not enough
Group A: balanced incidence
A
m1
B
m2
C
m3
D
m4
Every actor: 1 red + 1 blue + 1 gray incident relation
Group B: concentrated incidence
A
m1
B
m2
C
m3
D
m4
A: 2 red; D: 2 gray; B/C: mixed incident relations
Identical actor marginals: {m1,m2,m3,m4}
Identical pair-token multiset: {red×2, blue×2, gray×2}
Pair-bag mean
identical
PhaseSet incidence moments
distinguishable
Text is not SVG - cannot display \ No newline at end of file diff --git a/paper/BUILD_RECEIPT.json b/paper/BUILD_RECEIPT.json new file mode 100644 index 0000000..e47b3c0 --- /dev/null +++ b/paper/BUILD_RECEIPT.json @@ -0,0 +1,16 @@ +{ + "bibliography_sha256": "7db5537b9c85545e3fff89b29e17438936f0bcfb3e7d1ee2cd8de24f4de1a279", + "created_at_utc": "20260908T055718662230Z", + "ieeebib_sha256": "7e6ca0c8b72158d504a12bb091f817c07032a021ba41ca783cca4c2dd80d570b", + "overfull_hbox_count": 0, + "overfull_vbox_count": 0, + "pdf_bytes": 514458, + "pdf_sha256": "e6589c0e35da06b3b7b8f72c3fbacc99eaa44adf6be9c20465b4cb03fe3961a8", + "schema": "phaseset-paper-build-v1", + "scientific_results_created": false, + "source_sha256": "21f99fb638bce1d6451bf58fa75ce01a7f0d081c8573c4d82aaf1ec502c33f50", + "spconf_sha256": "dc5d632639040cb183f2ab62780f314845aa021be73048c8a9ec9c2072d64a86", + "submission_ready_asserted": false, + "underfull_hbox_count": 2, + "underfull_vbox_count": 1 +} diff --git a/paper/README.md b/paper/README.md index ecd6d95..e9d91f5 100644 --- a/paper/README.md +++ b/paper/README.md @@ -28,34 +28,45 @@ single sealed-test execution and frozen aggregate. ## Template status -The [official ICASSP 2027 publishing page](https://2027.ieeeicassp.org/publishing-and-paper-presentation-options/) -confirms four technical pages plus an optional references-only fifth page, but -its current author section does not expose a downloadable 2027 paper kit in a -form available to this repository. The source therefore deliberately retains -the engineering `article` template and labels that fact in `main.tex`. Page -count in this template is only a drafting constraint. Re-template and re-audit -page limits, fonts, figures, bibliography, and anonymization against the -official kit before any submission claim. The official full-paper deadline is -[September 16, 2026](https://2027.ieeeicassp.org/call-for-papers/). +The source uses the [official ICASSP 2027 paper kit](https://cmsworkshops.com/ICASSP2027/papers/paper_kit.php), +unmodified `spconf.sty`, its documented 9 pt mode, and `IEEEbib.bst`. The +hash-pinned fetch script obtains only those two style files into ignored +`paper/.venue/`; no example paper or third-party illustration is republished. +The limit is four technical pages plus an optional fifth page restricted to +references, acknowledgments, and ethical-compliance statements. This draft's +fifth page contains references only. + +ICASSP 2027 is non-blind. The explicit author/affiliation metadata-pending +fields must be replaced by the real author list before a submission build. +Formatting success is not submission readiness. The official full-paper +deadline is [September 16, 2026](https://2027.ieeeicassp.org/call-for-papers/). ## Local build -From this directory: +From the repository root, with Python, `pdflatex`, and `bibtex` installed: ```text -pdflatex -interaction=nonstopmode -halt-on-error main.tex -bibtex main -pdflatex -interaction=nonstopmode -halt-on-error main.tex -pdflatex -interaction=nonstopmode -halt-on-error main.tex +python scripts/build_paper.py ``` -Verified locally on 2026-08-25 with MiKTeX pdfTeX/BibTeX: the build succeeds as -a five-page PDF, pages 1–4 contain manuscript content, and page 5 contains only -references. The final LaTeX log has no undefined citation/reference, multiply -defined label, overfull box, underfull box, or package warning. The command-line -tools emit the environment-level advisory “MiKTeX updates have not been -checked”; this is a toolchain-maintenance notice, not a manuscript warning. -A generated PDF remains a `NO_RESULT / NOT_FOR_SUBMISSION` engineering artifact. +For an offline build, add `--template-archive `. Its exact +SHA-256 is checked; a different archive is not silently accepted. Each build +gets a new ignored directory, four pass logs, and a source/style/PDF digest +receipt. The script needs no `latexmk` or Perl and disables TeX shell escape. + +Verified on 2026-09-08 with MiKTeX pdfTeX/BibTeX: five pages, technical content +ends on page 4, page 5 contains only references, no undefined references or +overfull boxes, and all 18 font entries embedded with no Type 3 fallback. Two underfull horizontal +boxes, one underfull vertical box, and a nonfatal `balance` warning remain +reported in the logs. All 57 `HOLD` result cells remain visible. Inspect page +boundaries, fonts, figures, citations, and claims after every content change. +A generated PDF remains a `NO_RESULT / NOT_FOR_SUBMISSION` artifact. + +`BUILD_RECEIPT.json` binds the checked source, bibliography, official styles, +and PDF by SHA-256. Both diagrams now have native editable PowerPoint objects, +alongside draw.io/PDF/SVG/PNG exports. The architecture and method text show the +implemented parallel fixed-activity periodic branch and learned group base; +they do not assert that Morlet reads learned base features. ## Non-negotiable language diff --git a/paper/main.pdf b/paper/main.pdf index d518eee..f2406c1 100644 Binary files a/paper/main.pdf and b/paper/main.pdf differ diff --git a/paper/main.tex b/paper/main.tex index 125a5e5..ee26a4c 100644 --- a/paper/main.tex +++ b/paper/main.tex @@ -1,44 +1,29 @@ % DRAFT / AUTHORITY0 / NO_RESULT / NOT_FOR_SUBMISSION -% Engineering article template retained because no usable official conference -% paper kit is available in the repository. Re-template before submission. +% ICASSP 2027 official-template migration. Author metadata remains pending. \pdfminorversion=7 -\documentclass[10pt,twocolumn]{article} -\usepackage[letterpaper,margin=0.58in,columnsep=0.21in]{geometry} -\usepackage{amsmath,amssymb,booktabs,array,balance,graphicx,microtype,xcolor} +\documentclass{article} +\usepackage{spconf,amsmath,amssymb,booktabs,array,balance,graphicx,microtype,xcolor} \usepackage[T1]{fontenc} -\usepackage{lmodern} \usepackage[hidelinks]{hyperref} \definecolor{holdred}{RGB}{160,22,22} \newcommand{\DraftState}{\textcolor{holdred}{\textbf{DRAFT / AUTHORITY0 / NO\_RESULT / NOT\_FOR\_SUBMISSION}}} \newcommand{\ResultHold}{\textcolor{holdred}{\textsf{HOLD}}} \newcommand{\K}{K} -\setlength{\parindent}{0.9em} -\setlength{\parskip}{0pt} -\setlength{\textfloatsep}{5pt plus 1pt minus 2pt} -\setlength{\floatsep}{4pt plus 1pt minus 1pt} -\setlength{\intextsep}{4pt plus 1pt minus 1pt} -\setlength{\abovedisplayskip}{4pt plus 1pt minus 2pt} -\setlength{\belowdisplayskip}{4pt plus 1pt minus 2pt} -\setlength{\abovecaptionskip}{2pt} -\setlength{\belowcaptionskip}{0pt} -\renewcommand{\arraystretch}{0.92} -\raggedbottom - -\title{\vspace{-1.5em}PhaseSet: Permutation-Invariant Periodic Relation Tokens\\ + +\title{PhaseSet: Permutation-Invariant Periodic Relation Tokens\\ for Multi-Person Motion--Language Retrieval} -\author{Anonymous Authors\\[-1pt]\small\DraftState} -\date{} +\name{Author metadata pending --- NOT FOR SUBMISSION} +\address{Affiliation metadata pending\\\DraftState} \begin{document} +\ninept \maketitle -\vspace{-1.4em} \begin{center} -\fbox{\parbox{0.93\linewidth}{\centering\small +\fbox{\parbox{0.93\linewidth}{\centering This manuscript registers a method and evaluation protocol only. No training run, test score, confidence interval, or submission authority is represented.}} \end{center} -\vspace{-0.55em} \begin{abstract} Group motions can share actor marginals and the same multiset of pair relations @@ -54,6 +39,10 @@ is a \textbf{NO\_RESULT} technical draft: every empirical cell is held. \end{abstract} +\begin{keywords} +multi-person motion, text--motion retrieval, periodic relations, set representation +\end{keywords} + \section{Problem and claim boundary} Single-person retrieval already includes contrastive motion synthesis and fine-grained wavelet encoders \cite{tmr2023,wamo2026}. Dyadic motion work covers @@ -107,8 +96,11 @@ \section{Related evidence} \section{PhaseSet} \subsection{Input and deterministic periodic relations} An input is a dynamically padded tensor of $K\!\geq\!2$ actors, 200 frames, -and 22 body joints, with actor/joint/frame masks. A shared 512D actor encoder -uses no actor ID. Motion is resampled to 20 Hz; an independently hashed 20-Hz +and 22 body joints, with actor/joint/frame masks. The group base uses a shared +512D actor encoder with no actor ID. In parallel, the periodic branch extracts +fixed, mask-aware body-motion activity from skeleton coordinates; it does not +apply Morlet kernels to learned base features. Motion is resampled to 20 Hz; +an independently hashed 20-Hz Morlet bank retains PhasePair's six physical center frequencies while its legacy 30-Hz tap bytes remain unchanged. Each response is computed once per actor. For every valid unordered pair $\{i,j\}$ and band $b$, the inherited directed 13D @@ -145,7 +137,8 @@ \subsection{Pair and incidence paths} \begin{figure*}[t] \centering \includegraphics[width=0.96\textwidth]{../figures/phaseset_architecture.pdf} -\caption{Registered architecture (conceptual, no-result). Directed relations +\caption{Implementation architecture (no-result). Fixed motion activity and +the qualified learned group base are parallel paths. Directed relations feed a symmetric pair mean and incident-moment topology path; a hard $K{=}2$ gate preserves dyadic equivalence. Six motion tokens align with six semantic text-band tokens on top of the frozen qualified base score.} @@ -157,9 +150,9 @@ \subsection{Pair and incidence paths} g_b=m_b\,C\!\left(\bar p_b+\tanh(\lambda_b)q_b\right), \label{eq:token} \end{equation} -where $m_b$ is the band-valid mask and $C_b$ is the common postprocessor. The +where $m_b$ is the band-valid mask and $C$ is the common postprocessor. The capacity-matched pair-only control passes exact positive zero for $q_b$ through -the same $C_b$. Fixed-tree float64 microblock reduction, compensated +the same $C$. Fixed-tree float64 microblock reduction, compensated microblock accumulation, and fixed-order Chan merges make the output invariant to registered chunk sizes. @@ -219,10 +212,8 @@ \section{Registered experiments: all results held} uses four alternating blocks: temporal self-attention is applied independently with shared weights, then each frame performs set attention across the valid actors, followed by PMA. No candidate receives an actor index, actor-specific -tower, or actor-0 reference. The architecture decision uses the three-seed -validation mean, not per-seed winners; an exact tie is resolved only by the -registered resource order. Thus every residual system for a seed reads the -same immutable winning-base checkpoint and cache. +tower, or actor-0 reference. Every residual system reads the same immutable +winning-base checkpoint and cache for its seed. Systems 01--08 each run at the same three seeds (24 more attempts), for a formal total of \textbf{33 training attempts}. Bases use the registered AdamW @@ -265,18 +256,11 @@ \subsection{Execution and scaling gates} insufficient edge budget yields an explicit resource failure rather than a different graph. -\clearpage - -\twocolumn[{% -\begin{minipage}{\textwidth} -\begin{minipage}[t]{0.475\textwidth} +\begin{table*}[t] \centering -\refstepcounter{table} -\textbf{Table \thetable:} Frozen full-gallery matrix; every result is -\ResultHold.\label{tab:hold}\\[2pt] -\scriptsize -\resizebox{\linewidth}{!}{% -\begin{tabular}{@{}clcccc@{}} +\caption{Frozen full-gallery matrix; every result is \ResultHold.} +\label{tab:hold} +\begin{tabular}{@{}c p{49mm} c c c c@{}} \toprule ID & System & 1729 & 2718 & 31415 & Aggregate \\ \midrule @@ -291,16 +275,13 @@ \subsection{Execution and scaling gates} 08 & PhaseSet full & \ResultHold & \ResultHold & \ResultHold & \ResultHold \\ \bottomrule \end{tabular} -} -\end{minipage}\hfill -\begin{minipage}[t]{0.505\textwidth} +\end{table*} + +\begin{table*}[t] \centering -\refstepcounter{table} -\textbf{Table \thetable:} Frozen slice/scaling report; dash is out of -scope.\label{tab:scaling}\\[2pt] -\scriptsize -\resizebox{\linewidth}{!}{% -\begin{tabular}{@{}clcccc@{}} +\caption{Frozen slice/scaling report; dash is out of scope.} +\label{tab:scaling} +\begin{tabular}{@{}c p{45mm} c c c c@{}} \toprule $K$ & Source / role & Full & Hard-$\leq32$ & Memory & Throughput \\ \midrule @@ -314,11 +295,7 @@ \subsection{Execution and scaling gates} 256 & synthetic / resource limit & -- & -- & \ResultHold & \ResultHold \\ \bottomrule \end{tabular} -} -\end{minipage} -\end{minipage} -\vspace{4pt} -}] +\end{table*} \balance @@ -401,10 +378,6 @@ \section{Conclusion and limits} publication review; this repository exposes no raw motion, captions, participant identities, reversible memberships, or private model artifacts. -\clearpage -\sloppy -\small -\raggedright -\bibliographystyle{unsrt} +\bibliographystyle{IEEEbib} \bibliography{references} \end{document} diff --git a/pyproject.toml b/pyproject.toml index 922d111..a5ca605 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,9 +9,10 @@ description = "Permutation-invariant periodic relation tokens for multi-person m readme = "README.md" requires-python = ">=3.12,<3.15" license = "MIT" +license-files = ["LICENSE", "LICENSE_TMR.md"] authors = [{ name = "PhaseSet contributors" }] dependencies = [ - "numpy>=1.26,<3", + "numpy==2.4.6", "torch>=2.10,<2.13", ] @@ -23,8 +24,8 @@ Repository = "https://github.com/appleweiping/phaseset-multiperson-motion-langua Documentation = "https://github.com/appleweiping/phaseset-multiperson-motion-language/tree/main/docs" [project.optional-dependencies] -# The exact signal oracle is presently qualified on Windows x86-64 only. -# Linux installs remain useful for audits/builds but are not an oracle PASS. +# Canonical bytes are shared by the strict Windows and Linux formula paths. +# A matching formula bank alone is not full model/CUDA runtime qualification. signal = [ "numpy==2.4.6", "torch==2.12.0", @@ -33,6 +34,12 @@ clip = [ "requests==2.34.2", "transformers==4.57.3", ] +tmr-language = [ + "transformers==4.57.3", +] +embody = [ + "smplx==0.1.28", +] test = [ "pytest==9.0.3", "ruff==0.15.22", diff --git a/scripts/build_paper.py b/scripts/build_paper.py new file mode 100644 index 0000000..b2bcb65 --- /dev/null +++ b/scripts/build_paper.py @@ -0,0 +1,95 @@ +"""Build the current manuscript with the hash-pinned official ICASSP 2027 kit. + +This compiles existing content. It never creates scientific results or grants +submission readiness. Logs and receipts stay in a separate ignored build tree. +""" +from __future__ import annotations + +import argparse +from datetime import datetime, timezone +import hashlib +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys + +from fetch_icassp2027_template import main as fetch_template + + +def file_sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--template-archive", type=Path) + parser.add_argument("--output-directory", type=Path) + args = parser.parse_args(argv) + root = Path(__file__).resolve().parents[1] + paper = root / "paper" + venue = paper / ".venue" + template_args = ["--destination", str(venue)] + if args.template_archive is not None: + template_args.extend(["--archive", str(args.template_archive)]) + fetch_template(template_args) + executables = {name: shutil.which(name) for name in ("pdflatex", "bibtex")} + if any(value is None for value in executables.values()): + raise RuntimeError("pdflatex and bibtex must be installed and on PATH") + stamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%S%fZ") + output = (args.output_directory or paper / "build" / stamp).resolve() + # Never mix this attempt with another build or overwrite tracked paper files. + if output.exists() or output.is_symlink(): + raise FileExistsError("paper build output must be a new directory") + output.mkdir(parents=True) + env = dict(os.environ) + env["TEXINPUTS"] = str(venue) + os.pathsep + env["BSTINPUTS"] = str(venue) + os.pathsep + env["BIBINPUTS"] = str(paper) + os.pathsep + tex_command = [ + executables["pdflatex"], "-no-shell-escape", "-interaction=nonstopmode", + "-halt-on-error", "-output-directory", str(output), "main.tex", + ] + commands = [tex_command, [executables["bibtex"], str(output / "main")], + tex_command, tex_command] + for index, command in enumerate(commands, start=1): + with (output / f"pass-{index}.log").open("xb") as log: + result = subprocess.run(command, cwd=paper, env=env, stdout=log, + stderr=subprocess.STDOUT, timeout=180, check=False) + if result.returncode: + raise RuntimeError(f"paper pass {index} failed; inspect its private build log") + pdf = output / "main.pdf" + if not pdf.is_file() or pdf.stat().st_size == 0: + raise RuntimeError("paper build produced no PDF") + log_text = (output / "main.log").read_text(encoding="utf-8", errors="replace") + for marker in ("There were undefined references", "There were multiply-defined labels"): + if marker in log_text: + raise RuntimeError("paper has unresolved references or duplicate labels") + receipt = { + "schema": "phaseset-paper-build-v1", "created_at_utc": stamp, + "scientific_results_created": False, "submission_ready_asserted": False, + "source_sha256": file_sha256(paper / "main.tex"), + "bibliography_sha256": file_sha256(paper / "references.bib"), + "spconf_sha256": file_sha256(venue / "spconf.sty"), + "ieeebib_sha256": file_sha256(venue / "IEEEbib.bst"), + "pdf_sha256": file_sha256(pdf), "pdf_bytes": pdf.stat().st_size, + "overfull_hbox_count": log_text.count("Overfull \\hbox"), + "overfull_vbox_count": log_text.count("Overfull \\vbox"), + "underfull_hbox_count": log_text.count("Underfull \\hbox"), + "underfull_vbox_count": log_text.count("Underfull \\vbox"), + } + with (output / "build-receipt.json").open("x", encoding="utf-8") as stream: + json.dump(receipt, stream, sort_keys=True, indent=2) + stream.write("\n") + print("PAPER_BUILD_COMPLETED " + receipt["pdf_sha256"]) + print("Inspect page boundaries, embedded fonts, figures, and claims before publication.") + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except (OSError, RuntimeError, ValueError, subprocess.TimeoutExpired) as exc: + print(f"PAPER_BUILD_FAILED: {type(exc).__name__}", file=sys.stderr) + raise SystemExit(1) from exc diff --git a/scripts/export_morlet_oracles.py b/scripts/export_morlet_oracles.py new file mode 100644 index 0000000..a35873d --- /dev/null +++ b/scripts/export_morlet_oracles.py @@ -0,0 +1,123 @@ +"""Export the two frozen Morlet formula banks from the registered Windows oracle. + +This build-only script imports the unchanged public baseline, replays its +diagnostic formulas, verifies the already-frozen bank digests, and writes only +Python modules containing the canonical serialized bytes and source-runtime +provenance. It consumes no model, dataset, or experiment input. +""" + +from __future__ import annotations + +import argparse +import hashlib +import importlib +from pathlib import Path +import platform +import struct +import sys +from types import ModuleType + +import numpy as np + + +EXPECTED_PYTHON = (3, 14, 5) +EXPECTED_NUMPY = "2.4.6" +EXPECTED_PLATFORM = "Windows" +EXPECTED_DIGESTS = { + "phasepair_core.signal": "4f75ab687d333af8b436feea371dbaee56c55256e91c42c5e1ce45935beed5d7", + "phaseset_core.morlet": "633d754bba12dab0555722f4b8963409a110c8681796cc907d163096e2e42655", +} +OUTPUTS = { + "phasepair_core.signal": ("phasepair_core", "_morlet_oracle_bytes.py"), + "phaseset_core.morlet": ("phaseset_core", "_morlet_oracle_bytes.py"), +} + + +def _serialize(module: ModuleType) -> bytes: + bands = module.morlet_formula_bank_diagnostic() + output = bytearray() + for band in bands: + output.extend(struct.pack(">H", band.length)) + for value in band.kernel: + output.extend(struct.pack(">dd", float(value.real), float(value.imag))) + return bytes(output) + + +def _render(*, module_name: str, raw: bytes, digest: str) -> str: + wrapped = "\n".join( + f' "{raw.hex()[offset:offset + 96]}"' + for offset in range(0, len(raw.hex()), 96) + ) + return f'''"""Generated frozen Morlet bytes; do not hand-edit. + +Source formula module: {module_name} +Exporter runtime: CPython 3.14.5 / NumPy 2.4.6 / Windows +Canonical serialized SHA-256: {digest} +""" + +from __future__ import annotations + +from typing import Final + + +ORACLE_EXPORT_SCHEMA: Final = "portable-morlet-oracle-bytes-v1" +ORACLE_EXPORT_RUNTIME: Final = "CPython 3.14.5 / NumPy 2.4.6 / Windows" +ORACLE_EXPORT_FORMULA_MODULE: Final = "{module_name}" +ORACLE_EXPORT_SHA256: Final = "{digest}" +CANONICAL_BANK_BYTES: Final = bytes.fromhex( +{wrapped} +) + + +__all__ = [ + "CANONICAL_BANK_BYTES", + "ORACLE_EXPORT_FORMULA_MODULE", + "ORACLE_EXPORT_RUNTIME", + "ORACLE_EXPORT_SCHEMA", + "ORACLE_EXPORT_SHA256", +] +''' + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--source-root", type=Path, required=True) + parser.add_argument("--output-root", type=Path, required=True) + args = parser.parse_args() + + if sys.version_info[:3] != EXPECTED_PYTHON: + raise SystemExit(f"wrong Python oracle: {sys.version}") + if np.__version__ != EXPECTED_NUMPY: + raise SystemExit(f"wrong NumPy oracle: {np.__version__}") + if platform.system() != EXPECTED_PLATFORM: + raise SystemExit(f"wrong platform oracle: {platform.system()}") + source_root = args.source_root.resolve(strict=True) + output_root = args.output_root.resolve(strict=True) + sys.path.insert(0, str(source_root / "src")) + + for module_name, expected_digest in EXPECTED_DIGESTS.items(): + module = importlib.import_module(module_name) + module_file = Path(module.__file__).resolve(strict=True) + if source_root not in module_file.parents: + raise SystemExit(f"module escaped baseline source root: {module_name}") + raw = _serialize(module) + digest = hashlib.sha256(raw).hexdigest() + if digest != expected_digest or module.morlet_bank_sha256( + module.morlet_formula_bank_diagnostic() + ) != expected_digest: + raise SystemExit(f"frozen formula mismatch: {module_name} {digest}") + package, filename = OUTPUTS[module_name] + destination = output_root / "src" / package / filename + if not destination.parent.is_dir(): + raise SystemExit(f"missing staging package directory: {destination.parent}") + destination.write_text( + _render(module_name=module_name, raw=raw, digest=digest), + encoding="ascii", + newline="\n", + ) + print(f"EXPORTED {module_name} bytes={len(raw)} sha256={digest}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/fetch_icassp2027_template.py b/scripts/fetch_icassp2027_template.py new file mode 100644 index 0000000..2069d16 --- /dev/null +++ b/scripts/fetch_icassp2027_template.py @@ -0,0 +1,76 @@ +"""Fetch only the hash-pinned official conference style, never example papers.""" +from __future__ import annotations + +import argparse +import hashlib +import io +from pathlib import Path +from urllib.parse import urlparse +from urllib.request import Request, urlopen +import zipfile + + +TEMPLATE_URL = ( + "https://cmsworkshops.com/ICASSP2027/papers/PaperFormat/" + "ICASSP2027_Paper_Templates.zip" +) +ARCHIVE_SHA256 = "fd1cc4102c4ad9a3e85eba96ee5ed261a45fab9c2630875eb2566812230c91a8" +MAX_ARCHIVE_BYTES = 32 * 1024 * 1024 +STYLE_SHA256 = { + "spconf.sty": "dc5d632639040cb183f2ab62780f314845aa021be73048c8a9ec9c2072d64a86", + "IEEEbib.bst": "7e6ca0c8b72158d504a12bb091f817c07032a021ba41ca783cca4c2dd80d570b", +} + + +def install_template(raw: bytes, destination: Path) -> None: + if len(raw) > MAX_ARCHIVE_BYTES or hashlib.sha256(raw).hexdigest() != ARCHIVE_SHA256: + raise ValueError("official ICASSP 2027 template archive digest mismatch") + selected = {} + with zipfile.ZipFile(io.BytesIO(raw)) as archive: + for name, expected in STYLE_SHA256.items(): + matches = [entry for entry in archive.infolist() if entry.filename == name] + if len(matches) != 1 or matches[0].file_size > 1024 * 1024: + raise ValueError("official style member is absent, repeated, or oversized") + content = archive.read(matches[0]) + if hashlib.sha256(content).hexdigest() != expected: + raise ValueError("official style member digest mismatch") + selected[name] = content + destination.mkdir(parents=True, exist_ok=True) + # Validate the complete target set before writing; preserve differing local files. + for name, content in selected.items(): + path = destination / name + if path.is_symlink() or (path.exists() and path.read_bytes() != content): + raise ValueError("refusing to replace an existing different style file") + for name, content in selected.items(): + path = destination / name + if not path.exists(): + with path.open("xb") as handle: + handle.write(content) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--archive", type=Path, help="optional offline official template archive") + parser.add_argument( + "--destination", type=Path, + default=Path(__file__).resolve().parents[1] / "paper" / ".venue", + ) + args = parser.parse_args(argv) + if args.archive: + if args.archive.stat().st_size > MAX_ARCHIVE_BYTES: + raise ValueError("template archive exceeds the byte bound") + raw = args.archive.read_bytes() + else: + request = Request(TEMPLATE_URL, headers={"User-Agent": "PhaseSet-paper-build/1"}) + with urlopen(request, timeout=60) as response: + final = urlparse(response.geturl()) + if final.scheme != "https" or final.hostname != "cmsworkshops.com": + raise ValueError("template download left the official HTTPS host") + raw = response.read(MAX_ARCHIVE_BYTES + 1) + install_template(raw, args.destination) + print("ICASSP2027_TEMPLATE_VERIFIED " + ARCHIVE_SHA256) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/probe_morlet_portability.py b/scripts/probe_morlet_portability.py new file mode 100644 index 0000000..842d4e8 --- /dev/null +++ b/scripts/probe_morlet_portability.py @@ -0,0 +1,118 @@ +"""Bounded, NumPy-only cross-runtime probe for the two Morlet banks.""" + +from __future__ import annotations + +import argparse +import hashlib +import importlib +import importlib.util +import json +from pathlib import Path +import struct +import sys + +import numpy as np + + +MODULES = ( + ("phasepair_core.signal", "phasepair_core/_morlet_oracle_bytes.py"), + ("phaseset_core.morlet", "phaseset_core/_morlet_oracle_bytes.py"), +) + + +def _load_artifact(path: Path) -> bytes: + spec = importlib.util.spec_from_file_location("_private_morlet_artifact", path) + if spec is None or spec.loader is None: + raise RuntimeError("artifact import specification failed") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module.CANONICAL_BANK_BYTES + + +def _parse(raw: bytes, lengths: tuple[int, ...]) -> tuple[np.ndarray, ...]: + cursor = 0 + kernels = [] + for expected_length in lengths: + (observed_length,) = struct.unpack_from(">H", raw, cursor) + cursor += 2 + if observed_length != expected_length: + raise RuntimeError("artifact support mismatch") + values = np.frombuffer(raw, dtype=">f8", count=2 * expected_length, offset=cursor) + cursor += 16 * expected_length + pairs = values.reshape(expected_length, 2) + kernel = np.empty(expected_length, dtype=np.complex128) + kernel.real = pairs[:, 0] + kernel.imag = pairs[:, 1] + kernels.append(kernel) + if cursor != len(raw): + raise RuntimeError("artifact trailing bytes") + return tuple(kernels) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--source-root", type=Path, required=True) + parser.add_argument("--artifact-root", type=Path, required=True) + args = parser.parse_args() + source_root = args.source_root.resolve(strict=True) + artifact_root = args.artifact_root.resolve(strict=True) + sys.path.insert(0, str(source_root / "src")) + observations = [] + for module_name, relative_artifact in MODULES: + module = importlib.import_module(module_name) + if module_name == "phasepair_core.signal": + formula_receipt = module._build_morlet_formula_bank() + else: + formula_receipt = module._build_formula_bank() + formula = formula_receipt.bands + raw = _load_artifact(artifact_root / relative_artifact) + canonical = _parse(raw, module.MORLET_LENGTHS) + formula_sha = module.morlet_bank_sha256(formula) + differences = [ + np.abs(band.kernel - kernel) for band, kernel in zip(formula, canonical, strict=True) + ] + observations.append( + { + "module": module_name, + "python": sys.version.split()[0], + "numpy": np.__version__, + "artifact_sha256": hashlib.sha256(raw).hexdigest(), + "formula_sha256": formula_sha, + "max_complex_abs_error": max(float(np.max(value)) for value in differences), + "max_component_abs_error": max( + max( + float(np.max(np.abs(band.kernel.real - kernel.real))), + float(np.max(np.abs(band.kernel.imag - kernel.imag))), + ) + for band, kernel in zip(formula, canonical, strict=True) + ), + "formula_zero_dc_errors": [ + float(value) for value in formula_receipt.zero_dc_errors + ], + "formula_unit_energy_errors": [ + float(value) for value in formula_receipt.unit_energy_errors + ], + "canonical_zero_dc_errors": [ + float(np.abs(np.sum(kernel, dtype=np.complex128))) + for kernel in canonical + ], + "canonical_unit_energy_errors": [ + float( + np.abs( + np.sum( + np.square(np.abs(kernel), dtype=np.float64), + dtype=np.float64, + ) + - np.float64(1.0) + ) + ) + for kernel in canonical + ], + } + ) + print(json.dumps(observations, sort_keys=True, separators=(",", ":"))) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/release_manifest.py b/scripts/release_manifest.py index 1ba3729..1fe01f7 100644 --- a/scripts/release_manifest.py +++ b/scripts/release_manifest.py @@ -13,14 +13,14 @@ from release_tree import ( # noqa: E402 MANIFEST_HEADER, MANIFEST_RELATIVE, + GitIndexEntry, ReleaseTreeError, - clean_git_environment, - git_tracked_paths, + git_index_blobs, + git_index_entries, is_exact_git_checkout, load_archive_manifest, ) - ROOT = Path(__file__).resolve().parents[1] @@ -28,32 +28,29 @@ class ManifestError(RuntimeError): """Raised when the tracked tree cannot be represented or verified exactly.""" -def _tracked_paths() -> tuple[PurePosixPath, ...]: - return tuple( - path for path in git_tracked_paths(ROOT) if path != MANIFEST_RELATIVE - ) +def _row(path: PurePosixPath, raw: bytes) -> str: + return f"{hashlib.sha256(raw).hexdigest()}\t{len(raw)}\t{path.as_posix()}" -def _index_blob(path: PurePosixPath) -> bytes: - completed = subprocess.run( - ["git", "cat-file", "blob", f":{path.as_posix()}"], - cwd=ROOT, - check=True, - env=clean_git_environment(), - stdout=subprocess.PIPE, - ) - return completed.stdout +def _render_manifest( + entries: tuple[GitIndexEntry, ...], content: dict[PurePosixPath, bytes] +) -> bytes: + header = MANIFEST_HEADER.decode("ascii") + rows = "\n".join(_row(entry.path, content[entry.path]) for entry in entries) + return (header + rows + ("\n" if rows else "")).encode("utf-8") -def _row(path: PurePosixPath) -> str: - raw = _index_blob(path) - return f"{hashlib.sha256(raw).hexdigest()}\t{len(raw)}\t{path.as_posix()}" +def _index_snapshot() -> tuple[ + tuple[GitIndexEntry, ...], dict[PurePosixPath, bytes] +]: + entries = git_index_entries(ROOT) + return entries, git_index_blobs(ROOT, entries) def render_manifest() -> bytes: - header = MANIFEST_HEADER.decode("ascii") - rows = "\n".join(_row(path) for path in _tracked_paths()) - return (header + rows + ("\n" if rows else "")).encode("utf-8") + entries, content = _index_snapshot() + tracked = tuple(entry for entry in entries if entry.path != MANIFEST_RELATIVE) + return _render_manifest(tracked, content) def build_manifest() -> None: @@ -69,8 +66,10 @@ def verify_manifest() -> None: if not is_exact_git_checkout(ROOT): load_archive_manifest(ROOT) return - expected = render_manifest() - actual = _index_blob(MANIFEST_RELATIVE) + entries, content = _index_snapshot() + tracked = tuple(entry for entry in entries if entry.path != MANIFEST_RELATIVE) + expected = _render_manifest(tracked, content) + actual = content[MANIFEST_RELATIVE] if actual != expected: raise ManifestError("release manifest does not exactly match the tracked tree") diff --git a/scripts/release_tree.py b/scripts/release_tree.py index de83448..54407c6 100644 --- a/scripts/release_tree.py +++ b/scripts/release_tree.py @@ -6,10 +6,9 @@ import os import re import subprocess +from collections.abc import Mapping from dataclasses import dataclass from pathlib import Path, PurePosixPath -from typing import Mapping - MANIFEST_RELATIVE = PurePosixPath("RELEASE_FILES.sha256") MANIFEST_HEADER = b"# sha256\\tbytes\\tgit_index_path; self excluded\n" @@ -29,6 +28,15 @@ class ManifestEntry: path: PurePosixPath +@dataclass(frozen=True) +class GitIndexEntry: + """One validated stage-zero regular file in the Git index.""" + + mode: str + object_id: str + path: PurePosixPath + + @dataclass(frozen=True) class ReleaseTree: """One resolved tree whose paths and bytes share the same authority.""" @@ -37,19 +45,17 @@ class ReleaseTree: paths: tuple[PurePosixPath, ...] authority: str archive_content: Mapping[PurePosixPath, bytes] | None = None + index_content: Mapping[PurePosixPath, bytes] | None = None def read_bytes(self, path: PurePosixPath) -> bytes: if path not in self.paths: raise ReleaseTreeError(f"path is outside the release tree: {path.as_posix()}") if self.authority == "git-index": - completed = subprocess.run( - ["git", "cat-file", "blob", f":{path.as_posix()}"], - cwd=self.root, - check=True, - env=clean_git_environment(), - stdout=subprocess.PIPE, - ) - return completed.stdout + if self.index_content is None or path not in self.index_content: + raise ReleaseTreeError( + f"Git-index snapshot lacks release path: {path.as_posix()}" + ) + return self.index_content[path] if self.authority == "archive-manifest": if self.archive_content is None or path not in self.archive_content: raise ReleaseTreeError( @@ -106,8 +112,8 @@ def is_exact_git_checkout(root: Path) -> bool: return False -def git_tracked_paths(root: Path) -> tuple[PurePosixPath, ...]: - """Return safe index paths, rejecting an empty or non-PhasePair index.""" +def git_index_entries(root: Path) -> tuple[GitIndexEntry, ...]: + """Return validated regular-file entries from the stage-zero Git index.""" if not is_exact_git_checkout(root): raise ReleaseTreeError("root is not an exact Git checkout") @@ -118,7 +124,7 @@ def git_tracked_paths(root: Path) -> tuple[PurePosixPath, ...]: env=clean_git_environment(), stdout=subprocess.PIPE, ) - paths: list[PurePosixPath] = [] + entries: list[GitIndexEntry] = [] for raw in completed.stdout.split(b"\0"): if not raw: continue @@ -133,15 +139,119 @@ def git_tracked_paths(root: Path) -> tuple[PurePosixPath, ...]: raise ReleaseTreeError("release index contains a non-regular or unmerged item") if re.fullmatch(rb"(?:[0-9a-f]{40}|[0-9a-f]{64})", raw_object) is None: raise ReleaseTreeError("release index contains an invalid object id") - paths.append(_safe_relative_path(raw_path.decode("utf-8", errors="strict"))) - ordered = tuple(sorted(paths, key=lambda value: value.as_posix().encode("utf-8"))) - if not ordered or MANIFEST_RELATIVE not in ordered: + entries.append( + GitIndexEntry( + raw_mode.decode("ascii"), + raw_object.decode("ascii"), + _safe_relative_path(raw_path.decode("utf-8", errors="strict")), + ) + ) + ordered = tuple( + sorted(entries, key=lambda value: value.path.as_posix().encode("utf-8")) + ) + paths = tuple(entry.path for entry in ordered) + if not ordered or MANIFEST_RELATIVE not in paths: raise ReleaseTreeError("Git index is empty or lacks RELEASE_FILES.sha256") - if len(ordered) != len(set(ordered)): + if len(paths) != len(set(paths)): raise ReleaseTreeError("duplicate Git-index release path") return ordered +def git_tracked_paths(root: Path) -> tuple[PurePosixPath, ...]: + """Return safe regular-file index paths in canonical byte order.""" + + return tuple(entry.path for entry in git_index_entries(root)) + + +def _parse_git_batch_output( + raw: bytes, entries: tuple[GitIndexEntry, ...] +) -> dict[PurePosixPath, bytes]: + """Bind one strict ``git cat-file --batch`` response to index entries.""" + + content: dict[PurePosixPath, bytes] = {} + cursor = 0 + for entry in entries: + header_end = raw.find(b"\n", cursor) + if header_end < 0: + raise ReleaseTreeError( + f"Git batch header is truncated: {entry.path.as_posix()}" + ) + header = raw[cursor:header_end] + cursor = header_end + 1 + expected_object = entry.object_id.encode("ascii") + if header == expected_object + b" missing": + raise ReleaseTreeError( + f"Git batch object is missing: {entry.path.as_posix()}" + ) + fields = header.split(b" ") + if len(fields) != 3 or any(not field for field in fields): + raise ReleaseTreeError( + f"Git batch header is malformed: {entry.path.as_posix()}" + ) + raw_object, raw_type, raw_size = fields + if raw_object != expected_object: + raise ReleaseTreeError( + f"Git batch object ID mismatch: {entry.path.as_posix()}" + ) + if raw_type != b"blob": + raise ReleaseTreeError( + f"Git index object is not a blob: {entry.path.as_posix()}" + ) + if re.fullmatch(rb"(?:0|[1-9][0-9]*)", raw_size) is None: + raise ReleaseTreeError( + f"Git batch byte count is malformed: {entry.path.as_posix()}" + ) + size = int(raw_size) + payload_end = cursor + size + if payload_end > len(raw) or raw[payload_end : payload_end + 1] != b"\n": + raise ReleaseTreeError( + f"Git batch payload is truncated: {entry.path.as_posix()}" + ) + content[entry.path] = raw[cursor:payload_end] + cursor = payload_end + 1 + if cursor != len(raw): + raise ReleaseTreeError("Git batch output has unexpected trailing bytes") + return content + + +def git_index_blobs( + root: Path, entries: tuple[GitIndexEntry, ...] +) -> dict[PurePosixPath, bytes]: + """Read all validated index blobs in one strict Git batch transaction.""" + + paths = tuple(entry.path for entry in entries) + canonical = tuple(sorted(paths, key=lambda value: value.as_posix().encode("utf-8"))) + if ( + not entries + or paths != canonical + or len(paths) != len(set(paths)) + or MANIFEST_RELATIVE not in paths + ): + raise ReleaseTreeError("Git index entries are incomplete or not canonical") + for entry in entries: + if entry.mode not in {"100644", "100755"}: + raise ReleaseTreeError("release index contains a non-regular item") + if re.fullmatch(r"(?:[0-9a-f]{40}|[0-9a-f]{64})", entry.object_id) is None: + raise ReleaseTreeError("release index contains an invalid object id") + if _safe_relative_path(entry.path.as_posix()) != entry.path: + raise ReleaseTreeError("release index contains an invalid path") + requests = b"".join(entry.object_id.encode("ascii") + b"\n" for entry in entries) + completed = subprocess.run( + ["git", "cat-file", "--batch"], + cwd=root, + check=False, + env=clean_git_environment(), + input=requests, + capture_output=True, + ) + if completed.returncode != 0: + raise ReleaseTreeError("Git batch object stream failed") + content = _parse_git_batch_output(completed.stdout, entries) + if git_index_entries(root) != entries: + raise ReleaseTreeError("Git index changed while reading release objects") + return content + + def _filesystem_files(root: Path) -> dict[PurePosixPath, Path]: files: dict[PurePosixPath, Path] = {} for absolute in root.rglob("*"): @@ -242,7 +352,14 @@ def resolve_release_tree(root: Path) -> ReleaseTree: """Resolve paths and content reads through one consistent authority.""" if is_exact_git_checkout(root): - return ReleaseTree(root, git_tracked_paths(root), "git-index") + entries = git_index_entries(root) + content = git_index_blobs(root, entries) + return ReleaseTree( + root, + tuple(entry.path for entry in entries), + "git-index", + index_content=content, + ) entries, content = _load_archive_snapshot(root) paths = tuple(entry.path for entry in entries) + (MANIFEST_RELATIVE,) ordered = tuple(sorted(paths, key=lambda value: value.as_posix().encode("utf-8"))) diff --git a/scripts/render_editable_figures.py b/scripts/render_editable_figures.py new file mode 100644 index 0000000..a01aae7 --- /dev/null +++ b/scripts/render_editable_figures.py @@ -0,0 +1,823 @@ +"""Render the two PhaseSet draw.io pages as native editable PowerPoint shapes. + +This is a deliberately small renderer for the closed shapes used by this +project, not a generic draw.io importer. Publication PDF/SVG exports remain +the responsibility of the native draw.io CLI. No slide-wide raster is used. +""" +from __future__ import annotations + +import argparse +from datetime import datetime, timezone +import hashlib +from html.parser import HTMLParser +import io +import json +import math +import platform +import posixpath +from pathlib import Path +import re +import xml.etree.ElementTree as ET +import zipfile +import zlib + +import lxml.etree +import pptx +import pptx.api +from pptx import Presentation +from pptx.dml.color import RGBColor +from pptx.enum.dml import MSO_LINE_DASH_STYLE +from pptx.enum.shapes import MSO_AUTO_SHAPE_TYPE, MSO_CONNECTOR_TYPE +from pptx.enum.text import MSO_ANCHOR, PP_ALIGN +from pptx.oxml.xmlchemy import OxmlElement +from pptx.util import Emu, Inches, Pt + + +PYTHON_PPTX_VERSION = "1.0.2" +DEFAULT_TEMPLATE_BYTES = 34_030 +DEFAULT_TEMPLATE_SHA256 = "e10cc9e120961f6bd4074a373c9c80d2a06c497157e8f4972977b7bea83a8f34" +REGISTERED_SOURCES = { + "phaseset_motivation.drawio": ( + "a0a25fbc8c55575d50c6ffa8d6add01ce780cc8842328bb3f6bba20c6509d18a" + ), + "phaseset_architecture.drawio": ( + "1a61a503c935a72cbfbe1ce175ab54bf975eb3e0a2164018aeda80448f472a1a" + ), +} +REGISTERED_RUNTIME: dict[str, object] = { + "libxml2": [2, 11, 9], + "libxslt": [1, 1, 45], + "lxml": "6.1.1.0", + "machine": "AMD64", + "python": "3.12.14", + "python_implementation": "CPython", + "python_pptx": PYTHON_PPTX_VERSION, + "system": "Windows", + "zlib_build": "1.3.2", + "zlib_runtime": "1.3.2", +} +ZIP_TIMESTAMP = (1980, 1, 1, 0, 0, 0) +ZIP_CREATE_SYSTEM = 0 +ZIP_EXTERNAL_ATTR = 0o600 << 16 + +REL_NS = "http://schemas.openxmlformats.org/package/2006/relationships" +CONTENT_TYPES_NS = "http://schemas.openxmlformats.org/package/2006/content-types" +APP_NS = "http://schemas.openxmlformats.org/officeDocument/2006/extended-properties" +PML_NS = "http://schemas.openxmlformats.org/presentationml/2006/main" +DML_NS = "http://schemas.openxmlformats.org/drawingml/2006/main" + +THUMBNAIL_REL = "http://schemas.openxmlformats.org/package/2006/relationships/metadata/thumbnail" +PRINTER_REL = "http://schemas.openxmlformats.org/officeDocument/2006/relationships/printerSettings" +REMOVED_TEMPLATE_PARTS = frozenset( + {"docProps/thumbnail.jpeg", "ppt/printerSettings/printerSettings1.bin"} +) +RETAINED_PACKAGE_PARTS = frozenset( + { + "[Content_Types].xml", + "_rels/.rels", + "docProps/app.xml", + "docProps/core.xml", + "ppt/_rels/presentation.xml.rels", + "ppt/presProps.xml", + "ppt/presentation.xml", + "ppt/slideMasters/_rels/slideMaster1.xml.rels", + "ppt/slideMasters/slideMaster1.xml", + "ppt/slides/_rels/slide1.xml.rels", + "ppt/slides/slide1.xml", + "ppt/tableStyles.xml", + "ppt/theme/theme1.xml", + "ppt/viewProps.xml", + } + | { + f"ppt/slideLayouts/slideLayout{index}.xml" + for index in range(1, 12) + } + | { + f"ppt/slideLayouts/_rels/slideLayout{index}.xml.rels" + for index in range(1, 12) + } +) +TEMPLATE_OUTPUT_PARTS = RETAINED_PACKAGE_PARTS | REMOVED_TEMPLATE_PARTS + +ALLOWED_RELATIONSHIP_TYPES = frozenset( + { + "http://schemas.openxmlformats.org/officeDocument/2006/relationships/extended-properties", + "http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument", + "http://schemas.openxmlformats.org/officeDocument/2006/relationships/presProps", + "http://schemas.openxmlformats.org/officeDocument/2006/relationships/slide", + "http://schemas.openxmlformats.org/officeDocument/2006/relationships/slideLayout", + "http://schemas.openxmlformats.org/officeDocument/2006/relationships/slideMaster", + "http://schemas.openxmlformats.org/officeDocument/2006/relationships/tableStyles", + "http://schemas.openxmlformats.org/officeDocument/2006/relationships/theme", + "http://schemas.openxmlformats.org/officeDocument/2006/relationships/viewProps", + "http://schemas.openxmlformats.org/package/2006/relationships/metadata/core-properties", + } +) +EXPECTED_RELATIONSHIPS = frozenset( + { + ( + "", + "rId1", + "http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument", + "ppt/presentation.xml", + ), + ( + "", + "rId3", + "http://schemas.openxmlformats.org/package/2006/relationships/metadata/core-properties", + "docProps/core.xml", + ), + ( + "", + "rId4", + "http://schemas.openxmlformats.org/officeDocument/2006/relationships/extended-properties", + "docProps/app.xml", + ), + ( + "ppt/presentation.xml", + "rId1", + "http://schemas.openxmlformats.org/officeDocument/2006/relationships/slideMaster", + "slideMasters/slideMaster1.xml", + ), + ( + "ppt/presentation.xml", + "rId3", + "http://schemas.openxmlformats.org/officeDocument/2006/relationships/presProps", + "presProps.xml", + ), + ( + "ppt/presentation.xml", + "rId4", + "http://schemas.openxmlformats.org/officeDocument/2006/relationships/viewProps", + "viewProps.xml", + ), + ( + "ppt/presentation.xml", + "rId5", + "http://schemas.openxmlformats.org/officeDocument/2006/relationships/theme", + "theme/theme1.xml", + ), + ( + "ppt/presentation.xml", + "rId6", + "http://schemas.openxmlformats.org/officeDocument/2006/relationships/tableStyles", + "tableStyles.xml", + ), + ( + "ppt/presentation.xml", + "rId7", + "http://schemas.openxmlformats.org/officeDocument/2006/relationships/slide", + "slides/slide1.xml", + ), + ( + "ppt/slides/slide1.xml", + "rId1", + "http://schemas.openxmlformats.org/officeDocument/2006/relationships/slideLayout", + "../slideLayouts/slideLayout7.xml", + ), + ( + "ppt/slideMasters/slideMaster1.xml", + "rId12", + "http://schemas.openxmlformats.org/officeDocument/2006/relationships/theme", + "../theme/theme1.xml", + ), + } + | { + ( + f"ppt/slideLayouts/slideLayout{index}.xml", + "rId1", + "http://schemas.openxmlformats.org/officeDocument/2006/relationships/slideMaster", + "../slideMasters/slideMaster1.xml", + ) + for index in range(1, 12) + } + | { + ( + "ppt/slideMasters/slideMaster1.xml", + f"rId{index}", + "http://schemas.openxmlformats.org/officeDocument/2006/relationships/slideLayout", + f"../slideLayouts/slideLayout{index}.xml", + ) + for index in range(1, 12) + } +) +DEFAULT_CONTENT_TYPES = { + "rels": "application/vnd.openxmlformats-package.relationships+xml", + "xml": "application/xml", +} +OVERRIDE_CONTENT_TYPES = { + "/docProps/app.xml": "application/vnd.openxmlformats-officedocument.extended-properties+xml", + "/docProps/core.xml": "application/vnd.openxmlformats-package.core-properties+xml", + "/ppt/presProps.xml": "application/vnd.openxmlformats-officedocument.presentationml.presProps+xml", + "/ppt/presentation.xml": "application/vnd.openxmlformats-officedocument.presentationml.presentation.main+xml", + "/ppt/slideMasters/slideMaster1.xml": "application/vnd.openxmlformats-officedocument.presentationml.slideMaster+xml", + "/ppt/slides/slide1.xml": "application/vnd.openxmlformats-officedocument.presentationml.slide+xml", + "/ppt/tableStyles.xml": "application/vnd.openxmlformats-officedocument.presentationml.tableStyles+xml", + "/ppt/theme/theme1.xml": "application/vnd.openxmlformats-officedocument.theme+xml", + "/ppt/viewProps.xml": "application/vnd.openxmlformats-officedocument.presentationml.viewProps+xml", + **{ + f"/ppt/slideLayouts/slideLayout{index}.xml": ( + "application/vnd.openxmlformats-officedocument.presentationml.slideLayout+xml" + ) + for index in range(1, 12) + }, +} +PRIVATE_PATH_RE = re.compile( + rb"(?:(? str: + return hashlib.sha256(raw).hexdigest() + + +def xml_bytes(root: ET.Element, default_namespace: str) -> bytes: + ET.register_namespace("", default_namespace) + return ET.tostring(root, encoding="utf-8", xml_declaration=True) + + +def runtime_receipt() -> tuple[dict[str, object], Path]: + renderer = Path(__file__).resolve(strict=True) + template = Path(pptx.api.__file__).resolve(strict=True).parent / "templates" / "default.pptx" + template = template.resolve(strict=True) + template_raw = template.read_bytes() + if len(template_raw) != DEFAULT_TEMPLATE_BYTES or sha256_bytes(template_raw) != DEFAULT_TEMPLATE_SHA256: + raise RuntimeError("python-pptx default template identity mismatch") + runtime: dict[str, object] = { + "libxml2": list(lxml.etree.LIBXML_VERSION), + "libxslt": list(lxml.etree.LIBXSLT_VERSION), + "lxml": ".".join(str(value) for value in lxml.etree.LXML_VERSION), + "machine": platform.machine(), + "python": platform.python_version(), + "python_implementation": platform.python_implementation(), + "python_pptx": pptx.__version__, + "system": platform.system(), + "zlib_build": zlib.ZLIB_VERSION, + "zlib_runtime": zlib.ZLIB_RUNTIME_VERSION, + } + if runtime != REGISTERED_RUNTIME: + raise RuntimeError("registered canonical-build runtime identity mismatch") + receipt = { + "renderer": { + "bytes": renderer.stat().st_size, + "name": renderer.name, + "sha256": sha256_bytes(renderer.read_bytes()), + }, + "runtime": runtime, + "template": { + "bytes": len(template_raw), + "name": "python-pptx-default.pptx", + "sha256": sha256_bytes(template_raw), + }, + } + return receipt, template + + +class LabelParser(HTMLParser): + def __init__(self, label: str) -> None: + super().__init__(convert_charrefs=True) + self.parts: list[tuple[str, bool]] = [] + self.subscript = False + self.feed(label) + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + del attrs + if tag == "sub": + self.subscript = True + elif tag == "br": + self.parts.append(("\n", False)) + else: + raise ValueError(f"unsupported label tag: {tag}") + + def handle_endtag(self, tag: str) -> None: + if tag != "sub": + raise ValueError(f"unsupported closing label tag: {tag}") + self.subscript = False + + def handle_data(self, data: str) -> None: + self.parts.append((data, self.subscript)) + + +def style(cell: ET.Element) -> dict[str, str]: + result = {} + for field in cell.get("style", "").split(";"): + if field: + key, _, value = field.partition("=") + result[key] = value + return result + + +def read_zip_parts(raw: bytes) -> dict[str, bytes]: + with zipfile.ZipFile(io.BytesIO(raw)) as archive: + names = archive.namelist() + if len(names) != len(set(names)) or set(names) != TEMPLATE_OUTPUT_PARTS: + raise ValueError("python-pptx template part census mismatch") + if archive.comment: + raise ValueError("source package comment is not permitted") + return {name: archive.read(name) for name in names} + + +def remove_relationship(parts: dict[str, bytes], rels_name: str, relationship_type: str) -> None: + root = ET.fromstring(parts[rels_name]) + matches = [ + item + for item in list(root) + if item.tag == f"{{{REL_NS}}}Relationship" + and item.get("Type") == relationship_type + ] + if len(matches) != 1: + raise ValueError("template relationship census mismatch") + root.remove(matches[0]) + parts[rels_name] = xml_bytes(root, REL_NS) + + +def sanitize_package(raw: bytes) -> dict[str, bytes]: + parts = read_zip_parts(raw) + remove_relationship(parts, "_rels/.rels", THUMBNAIL_REL) + remove_relationship(parts, "ppt/_rels/presentation.xml.rels", PRINTER_REL) + for name in REMOVED_TEMPLATE_PARTS: + parts.pop(name) + + content_root = ET.fromstring(parts["[Content_Types].xml"]) + removed_defaults: dict[str, str] = {} + for item in list(content_root): + if item.tag != f"{{{CONTENT_TYPES_NS}}}Default": + continue + extension = item.get("Extension", "") + if extension in {"bin", "jpeg"}: + removed_defaults[extension] = item.get("ContentType", "") + content_root.remove(item) + if removed_defaults != { + "bin": "application/vnd.openxmlformats-officedocument.presentationml.printerSettings", + "jpeg": "image/jpeg", + }: + raise ValueError("template content-type census mismatch") + parts["[Content_Types].xml"] = xml_bytes(content_root, CONTENT_TYPES_NS) + + app_root = ET.fromstring(parts["docProps/app.xml"]) + stale_fields = {"TotalTime", "Words", "Paragraphs", "HeadingPairs", "TitlesOfParts"} + for item in list(app_root): + if item.tag.rsplit("}", 1)[-1] in stale_fields: + app_root.remove(item) + truthful = { + "Application": "PhaseSet native figure renderer", + "AppVersion": "2.0000", + "PresentationFormat": "Custom", + "Slides": "1", + "Notes": "0", + "HiddenSlides": "0", + "MMClips": "0", + "HyperlinkBase": "", + "HyperlinksChanged": "false", + "LinksUpToDate": "false", + "SharedDoc": "false", + } + for name, value in truthful.items(): + item = app_root.find(f"{{{APP_NS}}}{name}") + if item is None: + item = ET.SubElement(app_root, f"{{{APP_NS}}}{name}") + item.text = value + parts["docProps/app.xml"] = xml_bytes(app_root, APP_NS) + if set(parts) != RETAINED_PACKAGE_PARTS: + raise ValueError("retained package part census mismatch") + return parts + + +def normalized_zip(parts: dict[str, bytes]) -> bytes: + output = io.BytesIO() + with zipfile.ZipFile(output, "w", compression=zipfile.ZIP_STORED) as target: + for name in sorted(parts): + info = zipfile.ZipInfo(name, ZIP_TIMESTAMP) + info.compress_type = zipfile.ZIP_STORED + info.create_system = ZIP_CREATE_SYSTEM + info.create_version = 20 + info.extract_version = 20 + info.flag_bits = 0 + info.external_attr = ZIP_EXTERNAL_ATTR + info.internal_attr = 0 + info.extra = b"" + info.comment = b"" + target.writestr(info, parts[name], compress_type=zipfile.ZIP_STORED) + return output.getvalue() + + +def relationship_owner(rels_name: str) -> str: + if rels_name == "_rels/.rels": + return "" + prefix, local_name = rels_name.rsplit("/_rels/", 1) + if not local_name.endswith(".rels"): + raise ValueError("malformed relationship part name") + return f"{prefix}/{local_name[:-5]}" + + +def validate_content_types(parts: dict[str, bytes]) -> None: + root = ET.fromstring(parts["[Content_Types].xml"]) + if root.tag != f"{{{CONTENT_TYPES_NS}}}Types" or any( + item.tag + not in { + f"{{{CONTENT_TYPES_NS}}}Default", + f"{{{CONTENT_TYPES_NS}}}Override", + } + for item in root + ): + raise ValueError("retained content-type schema mismatch") + default_rows = root.findall(f"{{{CONTENT_TYPES_NS}}}Default") + override_rows = root.findall(f"{{{CONTENT_TYPES_NS}}}Override") + defaults = { + item.get("Extension", ""): item.get("ContentType", "") + for item in default_rows + } + overrides = { + item.get("PartName", ""): item.get("ContentType", "") + for item in override_rows + } + if ( + len(defaults) != len(default_rows) + or len(overrides) != len(override_rows) + or defaults != DEFAULT_CONTENT_TYPES + or overrides != OVERRIDE_CONTENT_TYPES + ): + raise ValueError("retained content-type allowlist mismatch") + + +def validate_relationships(parts: dict[str, bytes]) -> None: + relationship_rows: set[tuple[str, str, str, str]] = set() + for name in sorted(item for item in parts if item.endswith(".rels")): + root = ET.fromstring(parts[name]) + if root.tag != f"{{{REL_NS}}}Relationships" or any( + item.tag != f"{{{REL_NS}}}Relationship" for item in root + ): + raise ValueError("retained relationship schema mismatch") + identifiers: set[str] = set() + for relationship in root.findall(f"{{{REL_NS}}}Relationship"): + identifier = relationship.get("Id", "") + relation_type = relationship.get("Type", "") + target = relationship.get("Target", "") + if ( + not re.fullmatch(r"rId[1-9][0-9]*", identifier) + or identifier in identifiers + or relation_type not in ALLOWED_RELATIONSHIP_TYPES + or relationship.get("TargetMode") is not None + or not target + or "\\" in target + or ":" in target + or target.startswith("/") + ): + raise ValueError("retained relationship allowlist mismatch") + identifiers.add(identifier) + owner = relationship_owner(name) + relationship_rows.add((owner, identifier, relation_type, target)) + resolved = posixpath.normpath(posixpath.join(posixpath.dirname(owner), target)) + if resolved.startswith("../") or resolved not in parts: + raise ValueError("retained relationship target escapes package") + if relationship_rows != EXPECTED_RELATIONSHIPS: + raise ValueError("retained relationship tuple census mismatch") + + +def validate_app_properties(parts: dict[str, bytes]) -> None: + root = ET.fromstring(parts["docProps/app.xml"]) + expected = { + "Application": "PhaseSet native figure renderer", + "AppVersion": "2.0000", + "PresentationFormat": "Custom", + "Slides": "1", + "Notes": "0", + "HiddenSlides": "0", + "MMClips": "0", + } + for name, value in expected.items(): + item = root.find(f"{{{APP_NS}}}{name}") + if item is None or (item.text or "") != value: + raise ValueError("extended property truthfulness mismatch") + for name in ("TotalTime", "Words", "Paragraphs", "HeadingPairs", "TitlesOfParts"): + if root.find(f"{{{APP_NS}}}{name}") is not None: + raise ValueError("stale extended property retained") + + +def validate_hardened_package( + raw: bytes, + expected_shape_names: set[str], + expected_connector_names: set[str], +) -> tuple[int, int]: + with zipfile.ZipFile(io.BytesIO(raw)) as archive: + names = archive.namelist() + if names != sorted(RETAINED_PACKAGE_PARTS) or len(names) != len(set(names)): + raise ValueError("normalized package part order or census mismatch") + if archive.comment: + raise ValueError("normalized package comment is not permitted") + for info in archive.infolist(): + if ( + info.date_time != ZIP_TIMESTAMP + or info.compress_type != zipfile.ZIP_STORED + or info.create_system != ZIP_CREATE_SYSTEM + or info.create_version != 20 + or info.extract_version != 20 + or info.flag_bits != 0 + or info.external_attr != ZIP_EXTERNAL_ATTR + or info.internal_attr != 0 + or info.extra + or info.comment + ): + raise ValueError("normalized ZIP metadata mismatch") + parts = {name: archive.read(name) for name in names} + for name, part in parts.items(): + upper = part.upper() + if b" dict[str, object]: + raw = source.read_bytes() + source_sha256 = sha256_bytes(raw) + if REGISTERED_SOURCES.get(source.name) != source_sha256: + raise ValueError("registered figure source identity mismatch") + if ( + len(raw) > 1024 * 1024 + or b" tuple[float, float, float, float]: + if identifier in boxes: + return boxes[identifier] + cell = cells[identifier] + geometry = cell.find("mxGeometry") + if geometry is None: + return (0, 0, 0, 0) + parent = cell.get("parent", "1") + px, py, _, _ = box(parent) if parent in cells and parent != identifier else (0, 0, 0, 0) + result = ( + px + float(geometry.get("x", "0")), + py + float(geometry.get("y", "0")), + float(geometry.get("width", "0")), + float(geometry.get("height", "0")), + ) + boxes[identifier] = result + return result + + def emu(value: float) -> Emu: + return Emu(round(value * scale)) + + def font_points(value: float) -> Pt: + return Pt(value * scale / 12700) + + def format_line(shape, settings: dict[str, str]) -> None: + shape.line.color.rgb = RGBColor.from_string(settings.get("strokeColor", "#000000").lstrip("#")) + shape.line.width = font_points(float(settings.get("strokeWidth", "1"))) + if settings.get("dashed") == "1": + shape.line.dash_style = MSO_LINE_DASH_STYLE.DASH + + def label(shape, text: str, settings: dict[str, str]) -> None: + frame = shape.text_frame + frame.clear() + frame.word_wrap = True + frame.margin_left = frame.margin_right = emu(3) + frame.margin_top = frame.margin_bottom = emu(1) + frame.vertical_anchor = MSO_ANCHOR.MIDDLE + paragraph = frame.paragraphs[0] + paragraph.alignment = PP_ALIGN.CENTER + paragraph.space_before = paragraph.space_after = Pt(0) + for text_part, subscript in LabelParser(text).parts: + for index, segment in enumerate(text_part.split("\n")): + if index: + paragraph = frame.add_paragraph() + paragraph.alignment = PP_ALIGN.CENTER + paragraph.space_before = paragraph.space_after = Pt(0) + if segment: + run = paragraph.add_run() + run.text = segment + run.font.name = "Arial" + size = float(settings.get("fontSize", "14")) + run.font.size = font_points(size * (0.75 if subscript else 1)) + run.font.bold = bool(int(settings.get("fontStyle", "0")) & 1) + run.font.color.rgb = RGBColor.from_string(settings.get("fontColor", "#000000").lstrip("#")) + if subscript: + run._r.get_or_add_rPr().set("baseline", "-25000") + + vertices = [cell for cell in cells.values() if cell.get("vertex") == "1"] + edges = [cell for cell in cells.values() if cell.get("edge") == "1"] + containers = [cell for cell in vertices if "swimlane" in style(cell)] + expected_shape_names = { + *("drawio-node-" + cell.attrib["id"] for cell in vertices), + *("drawio-header-" + cell.attrib["id"] for cell in containers), + *( + "drawio-edge-label-" + cell.attrib["id"] + for cell in edges + if cell.get("value") + ), + } + expected_connector_names: set[str] = set() + edge_segment_counts = {cell.attrib["id"]: 0 for cell in edges} + + def vertex(cell: ET.Element, *, container: bool = False) -> None: + identifier = cell.attrib["id"] + settings = style(cell) + x, y, w, h = box(identifier) + if "text" in settings: + shape = slide.shapes.add_textbox(emu(x), emu(y), emu(w), emu(h)) + else: + kind = ( + MSO_AUTO_SHAPE_TYPE.OVAL if "ellipse" in settings else + MSO_AUTO_SHAPE_TYPE.ROUNDED_RECTANGLE if settings.get("rounded") == "1" else + MSO_AUTO_SHAPE_TYPE.RECTANGLE + ) + shape = slide.shapes.add_shape(kind, emu(x), emu(y), emu(w), emu(h)) + shape.fill.solid() + shape.fill.fore_color.rgb = RGBColor.from_string(settings.get("fillColor", "#FFFFFF").lstrip("#")) + format_line(shape, settings) + if kind == MSO_AUTO_SHAPE_TYPE.ROUNDED_RECTANGLE: + shape.adjustments[0] = 0.08 + shape.name = "drawio-node-" + identifier + text = cell.get("value", "") + if container: + header = float(settings.get("startSize", "36")) + title = slide.shapes.add_textbox(emu(x), emu(y), emu(w), emu(header)) + title.name = "drawio-header-" + identifier + label(title, text, settings) + separator = slide.shapes.add_connector(MSO_CONNECTOR_TYPE.STRAIGHT, emu(x), emu(y + header), emu(x + w), emu(y + header)) + separator.name = "drawio-container-separator-" + identifier + expected_connector_names.add(separator.name) + format_line(separator, settings) + else: + label(shape, text, settings) + + for cell in containers: + vertex(cell, container=True) + + def endpoint(identifier: str, other: str, edge_style: dict[str, str], prefix: str) -> tuple[float, float]: + x, y, w, h = box(identifier) + if prefix + "X" in edge_style: + return x + float(edge_style[prefix + "X"]) * w, y + float(edge_style[prefix + "Y"]) * h + ox, oy, ow, oh = box(other) + dx, dy = ox + ow / 2 - x - w / 2, oy + oh / 2 - y - h / 2 + if "ellipse" in style(cells[identifier]): + factor = 1 / math.sqrt((2 * dx / w) ** 2 + (2 * dy / h) ** 2) + else: + factor = 1 / max(abs(2 * dx / w), abs(2 * dy / h)) + return x + w / 2 + dx * factor, y + h / 2 + dy * factor + + for cell in edges: + settings = style(cell) + source_id, target_id = cell.attrib["source"], cell.attrib["target"] + start = endpoint(source_id, target_id, settings, "exit") + end = endpoint(target_id, source_id, settings, "entry") + geometry = cell.find("mxGeometry") + px, py, _, _ = box(cell.get("parent", "1")) + points = [] if geometry is None else [ + (px + float(point.attrib["x"]), py + float(point.attrib["y"])) + for point in geometry.findall("./Array/mxPoint") + ] + if not points and settings.get("edgeStyle") == "orthogonalEdgeStyle" and start[0] != end[0] and start[1] != end[1]: + middle = (start[0] + end[0]) / 2 + points = [(middle, start[1]), (middle, end[1])] + route = [start, *points, end] + for index, (a, b) in enumerate(zip(route, route[1:])): + if a == b: + continue + connector = slide.shapes.add_connector(MSO_CONNECTOR_TYPE.STRAIGHT, emu(a[0]), emu(a[1]), emu(b[0]), emu(b[1])) + connector.name = f"drawio-edge-{cell.attrib['id']}-segment-{index}" + expected_connector_names.add(connector.name) + edge_segment_counts[cell.attrib["id"]] += 1 + format_line(connector, settings) + if index == len(route) - 2 and settings.get("endArrow", "none") != "none": + tail = OxmlElement("a:tailEnd") + tail.set("type", "triangle") + connector._element.spPr.get_or_add_ln().append(tail) + if cell.get("value"): + a, b = route[len(route) // 2 - 1:len(route) // 2 + 1] + tx, ty = (a[0] + b[0]) / 2, (a[1] + b[1]) / 2 + tag = slide.shapes.add_textbox(emu(tx - 115), emu(ty - 12), emu(230), emu(24)) + tag.name = "drawio-edge-label-" + cell.attrib["id"] + tag.fill.solid() + tag.fill.fore_color.rgb = RGBColor(255, 255, 255) + label(tag, cell.attrib["value"], settings) + for cell in vertices: + if cell not in containers: + vertex(cell) + if any(count < 1 for count in edge_segment_counts.values()): + raise ValueError("an editable edge produced no native connector segment") + output = io.BytesIO() + prs.save(output) + normalized = normalized_zip(sanitize_package(output.getvalue())) + shapes, connectors = validate_hardened_package( + normalized, + expected_shape_names, + expected_connector_names, + ) + destination.write_bytes(normalized) + return { + "source": source.name, "source_sha256": source_sha256, + "pptx": destination.name, "pptx_sha256": hashlib.sha256(normalized).hexdigest(), + "native_shapes": shapes, "native_connector_segments": connectors, + "native_node_shapes": len(vertices), + "native_container_headers": len(containers), + "native_edge_labels": sum(bool(cell.get("value")) for cell in edges), + "source_edges": len(edges), + "package_parts": len(RETAINED_PACKAGE_PARTS), + "external_relationships": 0, + "raster_media": 0, + "empirical_result": False, + } + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--figure-dir", type=Path, required=True) + parser.add_argument("--output-dir", type=Path, required=True) + args = parser.parse_args() + args.output_dir.mkdir(parents=True, exist_ok=True) + destinations = tuple( + args.output_dir / name + for name in ( + "phaseset_motivation.pptx", + "phaseset_architecture.pptx", + "EDITABLE_FIGURES.json", + ) + ) + if any(path.exists() for path in destinations): + raise FileExistsError("refusing to overwrite an existing figure receipt") + receipt, template = runtime_receipt() + rows = [] + for stem in ("phaseset_motivation", "phaseset_architecture"): + rows.append( + render( + args.figure_dir / (stem + ".drawio"), + args.output_dir / (stem + ".pptx"), + template, + ) + ) + result = { + "schema": "phaseset-editable-figure-census-v2", + "figures": rows, + **receipt, + "package_policy": { + "compression": "ZIP_STORED", + "create_system": ZIP_CREATE_SYSTEM, + "external_attr": ZIP_EXTERNAL_ATTR, + "retained_part_count": len(RETAINED_PACKAGE_PARTS), + "timestamp": list(ZIP_TIMESTAMP), + }, + } + (args.output_dir / "EDITABLE_FIGURES.json").write_text(json.dumps(result, sort_keys=True, indent=2) + "\n", encoding="ascii") + print(json.dumps(result, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/src/phasepair_core/_morlet_oracle_bytes.py b/src/phasepair_core/_morlet_oracle_bytes.py new file mode 100644 index 0000000..7014809 --- /dev/null +++ b/src/phasepair_core/_morlet_oracle_bytes.py @@ -0,0 +1,138 @@ +"""Generated frozen Morlet bytes; do not hand-edit. + +Source formula module: phasepair_core.signal +Exporter runtime: CPython 3.14.5 / NumPy 2.4.6 / Windows +Canonical serialized SHA-256: 4f75ab687d333af8b436feea371dbaee56c55256e91c42c5e1ce45935beed5d7 +""" + +from __future__ import annotations + +from typing import Final + + +ORACLE_EXPORT_SCHEMA: Final = "portable-morlet-oracle-bytes-v1" +ORACLE_EXPORT_RUNTIME: Final = "CPython 3.14.5 / NumPy 2.4.6 / Windows" +ORACLE_EXPORT_FORMULA_MODULE: Final = "phasepair_core.signal" +ORACLE_EXPORT_SHA256: Final = "4f75ab687d333af8b436feea371dbaee56c55256e91c42c5e1ce45935beed5d7" +CANONICAL_BANK_BYTES: Final = bytes.fromhex( + "0078bf6093072bf7e6d13f24b0c3aeb73183bf62bd59b2ea393a3f41d628c4f891c9bf6497cf9ec8d2c93f50e69d38a4" + "5f7ebf65eecff02123b23f5a9c17d13b8129bf6686f72dad07533f6305ccc67de421bf661e625b738fed3f698e423433" + "ecdbbf646eb65f791faa3f70660d1fe13d1fbf612ff2b29709e73f744946630afa91bf583828cf571c7a3f7850a973aa" + "c548bf43ce0858a5b63b3f7c51a1ba1d37363f41f756912c62873f800b745a0f7a0e3f605ad201d0dcae3f81b06a321c" + "1fa83f6eaf22f74860e93f82f3284f96f0603f77b4a1b4ffdbc33f83ab4cfceeb85c3f8095315944aa063f83ade4d009" + "83db3f85c55629cda09f3f82cf1ba9e52a163f8b48bf5681a9233f80e45536f5d1ab3f9078fb11c905c73f7b8d445e26" + "ac843f9344039bac68a33f72aaff77b1abf93f95e38dc776e8b43f5bce32ae06b4fa3f9830e27d57c880bf5ebad34c51" + "3e283f9a0185cd4fdd59bf7932f8f16d4cc03f9b28a037567788bf86b66f8b19239f3f9b78c6e9fa0e64bf9102118a06" + "a43b3f9ac61915e1e292bf9721a09c3ac5553f98e8a1c6d0a9c5bf9d8f55ae6347a73f95bee70987d724bfa20b00c7f8" + "f80b3f913088f3fa032fbfa53b5847dfe2403f86619bcb422a1fbfa835137d71e4f63f6e07ac1f043b6fbfaad1807619" + "68d9bf743757ed1f1e88bface837304547e6bf8e3230ae034607bfae50bd3d34d4f2bf9a2c4ae484339bbfaee4614d69" + "7ebabfa300798df4ba2abfae803c0d5ba02fbfa91fcfe65442c4bfad073f58160c54bfaf40ed1f3048d7bfaa64394bd4" + "ff46bfb2954304d22672bfa68bae8ce174a5bfb54f933c5353d3bfa17d6e4356ce96bfb7afbf212b2397bf968b86aa52" + "9279bfb9967fcdebbc03bf7ff14910425902bfbae659bb40734b3f80ca459afd5335bfbb85118536fb343f9a318e7c26" + "dac5bfbb5d18de42e81a3fa6752284d8011ebfba5ee0365ee7173faffe68099051a9bfb881f99e540da73fb4b1ef1f07" + "0830bfb5c5fa97749e5e3fb9281db3c0faa5bfb2330cd12fd4f83fbd367efe62eeb7bfabb4427cf4bdc63fc059829c15" + "36d5bfa1a97a905df12d3fc1bb58e6943aaebf8a22f7c11447173fc2afb3c521e2c43f85742bdeed2bd93fc328cd5ea1" + "07ff3fa1abcd62e823743fc31ce3f63ae76d3fadffb02900f3203fc286cded466a8a3fb4f87a5abec54c3fc1665d757f" + "e0313fba8c13fac5fe163fbf811a84ff08123fbf8938363789243fbb3ee488d4e1213fc1e170e5715ea83fb623e30092" + "d8cc3fc389332d60ea933fb056788dfbe3313fc4ac94904515163fa40860a751db973fc540fb43378bcf3f8affaea6de" + "f54e3fc540fb43378bcfbf8affaea6def54b3fc4ac9490451516bfa40860a751db953fc389332d60ea93bfb056788dfb" + "e3313fc1e170e5715ea8bfb623e30092d8cc3fbf893836378924bfbb3ee488d4e1213fba8c13fac5fe16bfbf811a84ff" + "08123fb4f87a5abec54cbfc1665d757fe0313fadffb02900f320bfc286cded466a8a3fa1abcd62e82374bfc31ce3f63a" + "e76d3f85742bdeed2bd9bfc328cd5ea107ffbf8a22f7c1144717bfc2afb3c521e2c4bfa1a97a905df12dbfc1bb58e694" + "3aaebfabb4427cf4bdc6bfc059829c1536d5bfb2330cd12fd4f8bfbd367efe62eeb7bfb5c5fa97749e5ebfb9281db3c0" + "faa5bfb881f99e540da7bfb4b1ef1f070830bfba5ee0365ee717bfaffe68099051a9bfbb5d18de42e81abfa6752284d8" + "011ebfbb85118536fb34bf9a318e7c26dac5bfbae659bb40734bbf80ca459afd5332bfb9967fcdebbc033f7ff1491042" + "5907bfb7afbf212b23973f968b86aa529279bfb54f933c5353d33fa17d6e4356ce96bfb2954304d226723fa68bae8ce1" + "74a5bfaf40ed1f3048d73faa64394bd4ff46bfa91fcfe65442c43fad073f58160c54bfa300798df4ba2a3fae803c0d5b" + "a02fbf9a2c4ae484339b3faee4614d697ebabf8e3230ae0346073fae50bd3d34d4f2bf743757ed1f1e883face8373045" + "47e63f6e07ac1f043b6f3faad180761968d93f86619bcb422a1f3fa835137d71e4f63f913088f3fa032f3fa53b5847df" + "e2403f95bee70987d7243fa20b00c7f8f80b3f98e8a1c6d0a9c53f9d8f55ae6347a73f9ac61915e1e2923f9721a09c3a" + "c5553f9b78c6e9fa0e643f9102118a06a43b3f9b28a0375677883f86b66f8b19239f3f9a0185cd4fdd593f7932f8f16d" + "4cc03f9830e27d57c8803f5ebad34c513e2b3f95e38dc776e8b4bf5bce32ae06b4f73f9344039bac68a3bf72aaff77b1" + "abf73f9078fb11c905c7bf7b8d445e26ac843f8b48bf5681a923bf80e45536f5d1ab3f85c55629cda09fbf82cf1ba9e5" + "2a163f8095315944aa06bf83ade4d00983db3f77b4a1b4ffdbc3bf83ab4cfceeb85c3f6eaf22f74860e9bf82f3284f96" + "f0603f605ad201d0dcaebf81b06a321c1fa83f41f756912c6287bf800b745a0f7a0ebf43ce0858a5b63bbf7c51a1ba1d" + "3736bf583828cf571c7abf7850a973aac548bf612ff2b29709e7bf744946630afa91bf646eb65f791faabf70660d1fe1" + "3d1fbf661e625b738fedbf698e423433ecdbbf6686f72dad0753bf6305ccc67de421bf65eecff02123b2bf5a9c17d13b" + "8129bf6497cf9ec8d2c9bf50e69d38a45f7ebf62bd59b2ea393abf41d628c4f891c8bf6093072bf7e6d1bf24b0c3aeb7" + "31800050bf64fc6e545b2b003f33b2d76b62ddebbf68b3c74dffad183f520f1fed772314bf6b2436a253885a3f61f2a5" + "4ae99cfbbf6b5915d1def0813f6d3608b5fa9e65bf6839207cdbef983f753fbf2ab3a469bf609d10f773c7ab3f7c8a4c" + "cac9f960bf3b8b088f19b7393f81f01839579af03f601d7eabab09633f8532b2a181df023f753780b050a8b23f877fbd" + "539d56543f82cdc2b78af4aa3f88311fb37d2cec3f8c5278e1ac0a013f8693be4e6f72b63f934e7f50a576383f81f90e" + "3bb8dcfd3f986cd97d5c0ead3f739b88a9d5382f3f9cf880d1845cd4bf525f35cc0b76c13fa024e5f1a5ff06bf8260da" + "f726750e3fa0d44375cc6c59bf930a036c6939133fa02e378a85210dbf9e45ba461b489c3f9bc2029d7b5266bfa519e9" + "054bad8e3f936206c6ad1cabbfaaf2357a171a993f7c258cfcb57464bfb00c9cb90d7c93bf8243442224efa6bfb1f5d5" + "7c2be471bf9c9005ba49ca9cbfb2e1b62a7a178ebfa921d07c5413d5bfb28452e86c99d5bfb23510d9c47d88bfb0a245" + "83fdc1d7bfb7a0711c2e0ec5bfaa33cbec9aebc5bfbc54c4e0abb9bdbf9facc88cf5f07cbfbfd027cdcc67c3bf73d3a1" + "a118d86cbfc0ccd69dfdde783f9a7fd5fd216e62bfc0a75f4a882be03fae6b4733f88835bfbeafc6019466633fb7ed3d" + "21c35e03bfb9aa937d37f6ab3fc00db41b5943fbbfb26105f1eb7a573fc387436af77948bfa255a4d67ab7163fc60faa" + "175a1fea3f76e3d4877ffc763fc760f285f304793fa96f358e5254233fc74a9122db5b373fb7e42a8235b2293fc5b76f" + "874e75003fc10f0b5f4f9bef3fc2b17b8140b6bb3fc54d37d8957e6e3fbcc462edb9fa043fc8563e3042283b3fb21fe0" + "5f0e90ba3fc9eb4a23f8de303f98c225e911c41b3fc9eb4a23f8de30bf98c225e911c41b3fc8563e3042283bbfb21fe0" + "5f0e90ba3fc54d37d8957e6ebfbcc462edb9fa043fc10f0b5f4f9befbfc2b17b8140b6bb3fb7e42a8235b229bfc5b76f" + "874e75003fa96f358e525423bfc74a9122db5b373f76e3d4877ffc76bfc760f285f30479bfa255a4d67ab716bfc60faa" + "175a1feabfb26105f1eb7a57bfc387436af77948bfb9aa937d37f6abbfc00db41b5943fbbfbeafc601946663bfb7ed3d" + "21c35e03bfc0a75f4a882be0bfae6b4733f88835bfc0ccd69dfdde78bf9a7fd5fd216e62bfbfd027cdcc67c33f73d3a1" + "a118d86cbfbc54c4e0abb9bd3f9facc88cf5f07cbfb7a0711c2e0ec53faa33cbec9aebc5bfb23510d9c47d883fb0a245" + "83fdc1d7bfa921d07c5413d53fb28452e86c99d5bf9c9005ba49ca9c3fb2e1b62a7a178ebf8243442224efa63fb1f5d5" + "7c2be4713f7c258cfcb574643fb00c9cb90d7c933f936206c6ad1cab3faaf2357a171a993f9bc2029d7b52663fa519e9" + "054bad8e3fa02e378a85210d3f9e45ba461b489c3fa0d44375cc6c593f930a036c6939133fa024e5f1a5ff063f8260da" + "f726750e3f9cf880d1845cd43f525f35cc0b76c13f986cd97d5c0eadbf739b88a9d5382f3f934e7f50a57638bf81f90e" + "3bb8dcfd3f8c5278e1ac0a01bf8693be4e6f72b63f82cdc2b78af4aabf88311fb37d2cec3f753780b050a8b2bf877fbd" + "539d56543f601d7eabab0963bf8532b2a181df02bf3b8b088f19b739bf81f01839579af0bf609d10f773c7abbf7c8a4c" + "cac9f960bf6839207cdbef98bf753fbf2ab3a469bf6b5915d1def081bf6d3608b5fa9e65bf6b2436a253885abf61f2a5" + "4ae99cfbbf68b3c74dffad18bf520f1fed772314bf64fc6e545b2b00bf33b2d76b62ddeb0036bf686d558ffed44f3f26" + "db7143b55609bf6f2d79c97a3d0a3f5b0730bca77706bf70df16e84f27253f70091b6a3a5381bf6be2b5b82175723f7c" + "349eb4db20c3bf51d8a83b3d1d483f84de8996a85c8d3f6b0d76e596f83a3f8ace327cf519eb3f845c5593c8e2773f8d" + "a5a53d5a24273f92e614fe1aa7cc3f8a9d5a49f96d773f9c64b1355285563f7de517e03488633fa257730ef6fadbbf6d" + "26143d5a478c3fa48b8230d7ca29bf941df2fbe6eaf03fa352fb6413597fbfa457ff2f67db073f9ac72513422e38bfaf" + "43107d5ecdcf3f7075e60a346b10bfb4553946b3958dbf9caedc9e58bda1bfb6fa1c3e724c67bfb11a3daf87d7abbfb6" + "48caaee04f02bfbb52eda6f774a7bfb151ba0834e93ebfc1f53bb2d100e3bf9f40d36f9a1ad5bfc468102ca02b113f96" + "6437a17ac7b4bfc41ff8323dc3d43fb54fd7c64f732dbfc09ceae83031523fc27678db243c2dbfb3dec97a2610cf3fc8" + "d094d12dedd1bf796e8c0125e1043fcc6210a09ab39a3fb31185feb6dac93fcc36e75417cf0d3fc36b21411bdb573fc7" + "fa3e7106fda03fcb2e3ceb717b293fc016350e0a97f73fcf72944f29afa53fa6a7de0eaf7f273fcf72944f29afa5bfa6" + "a7de0eaf7f273fcb2e3ceb717b29bfc016350e0a97f73fc36b21411bdb57bfc7fa3e7106fda03fb31185feb6dac9bfcc" + "36e75417cf0dbf796e8c0125e104bfcc6210a09ab39abfb3dec97a2610cfbfc8d094d12dedd1bfc09ceae8303152bfc2" + "7678db243c2dbfc41ff8323dc3d4bfb54fd7c64f732dbfc468102ca02b11bf966437a17ac7b4bfc1f53bb2d100e33f9f" + "40d36f9a1ad5bfbb52eda6f774a73fb151ba0834e93ebfb11a3daf87d7ab3fb648caaee04f02bf9caedc9e58bda13fb6" + "fa1c3e724c673f7075e60a346b103fb4553946b3958d3f9ac72513422e383faf43107d5ecdcf3fa352fb6413597f3fa4" + "57ff2f67db073fa48b8230d7ca293f941df2fbe6eaf03fa257730ef6fadb3f6d26143d5a478c3f9c64b135528556bf7d" + "e517e03488633f92e614fe1aa7ccbf8a9d5a49f96d773f845c5593c8e277bf8da5a53d5a24273f6b0d76e596f83abf8a" + "ce327cf519ebbf51d8a83b3d1d48bf84de8996a85c8dbf6be2b5b8217572bf7c349eb4db20c3bf70df16e84f2725bf70" + "091b6a3a5381bf6f2d79c97a3d0abf5b0730bca77706bf686d558ffed44fbf26db7143b556090024bf701cfbe82b3f16" + "3f42f6203cdd874abf74a4ad7e734a263f706ddaaec14849bf6e11484f7506903f835255762286733f63ff4852dd8b28" + "3f8f3eb6b7cc7d8b3f8de900e25068bb3f9218f71db1e4653f9fe5033b8dac543f87073742bf6be33fa76ead61b7a56c" + "bf81d734453aa4943fa8879cb9d3e5f8bfa59c093dd5733c3f9b1d886766ab2bbfb4badc2ffb21dbbf98146bb8064fb1" + "bfbba84103b58595bfb8204f01899d3ebfba4ae2190b2983bfc4d77989c2408abfa9f55faae4a569bfc940e48b26809e" + "3fa702068a87547abfc5cd1b6d6f8bb33fc45204fbcc754bbfb333bcb72ac8ae3fcfdc2d3c32f4103fb0f7b14722db7a" + "3fd18967204286203fca7a9979e1b5743fcb54654cac49a43fd2d6bdfd00823e3fb4a2638880961e3fd2d6bdfd00823e" + "bfb4a2638880961e3fca7a9979e1b574bfcb54654cac49a43fb0f7b14722db7abfd1896720428620bfb333bcb72ac8ae" + "bfcfdc2d3c32f410bfc5cd1b6d6f8bb3bfc45204fbcc754bbfc940e48b26809ebfa702068a87547abfc4d77989c2408a" + "3fa9f55faae4a569bfb8204f01899d3e3fba4ae2190b2983bf98146bb8064fb13fbba84103b585953f9b1d886766ab2b" + "3fb4badc2ffb21db3fa8879cb9d3e5f83fa59c093dd5733c3fa76ead61b7a56c3f81d734453aa4943f9fe5033b8dac54" + "bf87073742bf6be33f8de900e25068bbbf9218f71db1e4653f63ff4852dd8b28bf8f3eb6b7cc7d8bbf6e11484f750690" + "bf83525576228673bf74a4ad7e734a26bf706ddaaec14849bf701cfbe82b3f16bf42f6203cdd874a0018bf75b4249004" + "3d913f58bb3b002132a8bf75ed4b917c24183f83ec6e9fd469873f78a5b4a997c2db3f94899b85ed02bc3fa0d7f509e1" + "1dc63f91897e348166ad3fae100466d35555bf9409afe98ef03c3fa5a219e749f7dabfb672881dd8b82ebfa904e8786d" + "b087bfc1528e7c5fbe82bfc722904b76dcd4bfb5691fed769cc1bfcee3289a42dc303fb6ee86c5db8287bfc0629236fa" + "bf5e3fd21c6cf3d7c80d3fc03f32398967b13fd4edf595d343733fd5efd6038313863fc297e43c37e6403fd5efd60383" + "1386bfc297e43c37e6403fc03f32398967b1bfd4edf595d34373bfc0629236fabf5ebfd21c6cf3d7c80dbfcee3289a42" + "dc30bfb6ee86c5db8287bfc722904b76dcd43fb5691fed769cc1bfa904e8786db0873fc1528e7c5fbe823fa5a219e749" + "f7da3fb672881dd8b82e3fae100466d355553f9409afe98ef03c3fa0d7f509e11dc6bf91897e348166ad3f78a5b4a997" + "c2dbbf94899b85ed02bcbf75ed4b917c2418bf83ec6e9fd46987bf75b42490043d91bf58bb3b002132a80010bf7d7928" + "3573ab8d3f6e4db67ca7230d3f60a736214ee5013f9672f58dd7462c3fa98cb39eb301553f8d5818f461f7433fb08f69" + "99212b1cbfb5d6cc15d1b45fbfb9818f10f3c216bfc50f0bce04b08dbfd2ce36cddb45623fa801dbd731d6ccbfb6ac7c" + "325e21a63fd89ac77b52c9193fd7d8def92444433fd059faa509c17c3fd7d8def9244443bfd059faa509c17cbfb6ac7c" + "325e21a6bfd89ac77b52c919bfd2ce36cddb4562bfa801dbd731d6d0bfb9818f10f3c2163fc50f0bce04b08d3fb08f69" + "99212b1c3fb5d6cc15d1b45f3fa98cb39eb30155bf8d5818f461f7453f60a736214ee501bf9672f58dd7462cbf7d7928" + "3573ab8dbf6e4db67ca7230d" +) + + +__all__ = [ + "CANONICAL_BANK_BYTES", + "ORACLE_EXPORT_FORMULA_MODULE", + "ORACLE_EXPORT_RUNTIME", + "ORACLE_EXPORT_SCHEMA", + "ORACLE_EXPORT_SHA256", +] diff --git a/src/phasepair_core/checkpoint.py b/src/phasepair_core/checkpoint.py index deed470..f8e26d3 100644 --- a/src/phasepair_core/checkpoint.py +++ b/src/phasepair_core/checkpoint.py @@ -589,6 +589,7 @@ def write_checkpoint_atomic(value: ValidatedCheckpoint, path: Path) -> bytes: if _path_entry_exists(target, "checkpoint target availability is unavailable"): raise CheckpointPathError("checkpoint target already exists") descriptor = -1 + ownership_descriptor = -1 temp_name: str | None = None committed = False succeeded = False @@ -637,6 +638,12 @@ def remove_committed_target_if_owned() -> None: raise CheckpointPathError("checkpoint temp short write") handle.flush() os.fsync(handle.fileno()) + if os.name == "posix": + # Keep the original inode alive even if every pathname is + # concurrently removed. Otherwise POSIX may immediately reuse + # its (device,inode) for a replacement owned by another writer, + # and failure cleanup could mistake that replacement for ours. + ownership_descriptor = os.dup(handle.fileno()) temp_path = Path(temp_name) retirement_path = Path(temp_name + ".committed-retirement") if _path_entry_exists( @@ -709,8 +716,12 @@ def remove_committed_target_if_owned() -> None: Path(temp_name).unlink(missing_ok=True) except OSError: pass - if committed and not succeeded: - remove_committed_target_if_owned() + try: + if committed and not succeeded: + remove_committed_target_if_owned() + finally: + if ownership_descriptor >= 0: + os.close(ownership_descriptor) def _stable_read(path: Path) -> bytes: @@ -839,11 +850,13 @@ def _install_sealed_api() -> None: ) os_proxy = types.SimpleNamespace( close=os.close, + dup=os.dup, fdopen=os.fdopen, fspath=os.fspath, fstat=os.fstat, fsync=os.fsync, link=os.link, + name=os.name, rename=os.rename, path=types.SimpleNamespace(abspath=os.path.abspath), ) diff --git a/src/phasepair_core/signal.py b/src/phasepair_core/signal.py index 51ef6f3..fabdd09 100644 --- a/src/phasepair_core/signal.py +++ b/src/phasepair_core/signal.py @@ -15,11 +15,21 @@ from dataclasses import dataclass import hashlib import math +import platform import struct +import sys from typing import Final, Protocol import numpy as np +from phasepair_core._morlet_oracle_bytes import ( + CANONICAL_BANK_BYTES as _MORLET_CANONICAL_BANK_BYTES, + ORACLE_EXPORT_FORMULA_MODULE as _MORLET_EXPORT_FORMULA_MODULE, + ORACLE_EXPORT_RUNTIME as MORLET_ORACLE_EXPORT_RUNTIME, + ORACLE_EXPORT_SCHEMA as MORLET_PORTABLE_SCHEMA, + ORACLE_EXPORT_SHA256 as _MORLET_EXPORT_SHA256, +) + SIGNAL_CONTRACT_FAMILY_ID: Final[str] = ( "phasepair-scientific-contract-successor-v7/20260824_165840" @@ -51,6 +61,12 @@ MORLET_ORACLE_SHA256: Final[str] = ( "4f75ab687d333af8b436feea371dbaee56c55256e91c42c5e1ce45935beed5d7" ) +MORLET_FORMULA_MAX_ABS_ERROR_BOUND: Final[float] = float.fromhex( + "0x1.0000000000000p-52" +) +MORLET_PORTABLE_LINUX_FORMULA_SHA256: Final[str] = ( + "3b4cec820d8202d4576d829d37626acc2414f0aeecf430e17c751d9bd1731f6c" +) MORLET_ZERO_DC_BOUND: Final[float] = 2.22e-16 MORLET_UNIT_ENERGY_BOUND: Final[float] = 2.220446049250313e-16 MORLET_EXPECTED_UNIT_ENERGY_ERRORS: Final[tuple[float, ...]] = ( @@ -473,19 +489,150 @@ def morlet_formula_bank_diagnostic() -> tuple[MorletBand, ...]: return _build_morlet_formula_bank().bands +def _canonical_morlet_bank() -> tuple[MorletBand, ...]: + """Decode the Windows-oracle bytes without replaying platform libm calls.""" + + if ( + MORLET_PORTABLE_SCHEMA != "portable-morlet-oracle-bytes-v1" + or MORLET_ORACLE_EXPORT_RUNTIME != "CPython 3.14.5 / NumPy 2.4.6 / Windows" + or _MORLET_EXPORT_FORMULA_MODULE != "phasepair_core.signal" + or _MORLET_EXPORT_SHA256 != MORLET_ORACLE_SHA256 + or hashlib.sha256(_MORLET_CANONICAL_BANK_BYTES).hexdigest() != MORLET_ORACLE_SHA256 + ): + raise SignalContractHold("HOLD_MORLET_PORTABLE_PROVENANCE_MISMATCH") + cursor = 0 + bands: list[MorletBand] = [] + for index, (frequency, length, sigma) in enumerate( + zip(MORLET_FREQUENCIES_HZ, MORLET_LENGTHS, MORLET_SIGMAS), start=1 + ): + if cursor + 2 > len(_MORLET_CANONICAL_BANK_BYTES): + raise SignalContractHold("HOLD_MORLET_PORTABLE_BYTES_TRUNCATED") + (encoded_length,) = struct.unpack_from(">H", _MORLET_CANONICAL_BANK_BYTES, cursor) + cursor += 2 + if encoded_length != length: + raise SignalContractHold("HOLD_MORLET_PORTABLE_SUPPORT_MISMATCH") + stop = cursor + 16 * length + if stop > len(_MORLET_CANONICAL_BANK_BYTES): + raise SignalContractHold("HOLD_MORLET_PORTABLE_BYTES_TRUNCATED") + values = np.frombuffer( + _MORLET_CANONICAL_BANK_BYTES, + dtype=">f8", + count=2 * length, + offset=cursor, + ).reshape(length, 2) + kernel = np.empty(length, dtype=np.complex128) + kernel.real = values[:, 0] + kernel.imag = values[:, 1] + kernel = np.ascontiguousarray(kernel, dtype=np.complex128) + if not bool(np.isfinite(kernel).all()): + raise SignalContractHold("HOLD_MORLET_PORTABLE_NONFINITE") + kernel.setflags(write=False) + bands.append(MorletBand(index, frequency, length, sigma, kernel)) + cursor = stop + if cursor != len(_MORLET_CANONICAL_BANK_BYTES): + raise SignalContractHold("HOLD_MORLET_PORTABLE_TRAILING_BYTES") + result = tuple(bands) + if morlet_bank_sha256(result) != MORLET_ORACLE_SHA256: + raise SignalContractHold("HOLD_MORLET_ORACLE_DIGEST_MISMATCH") + return result + + +def _formula_max_abs_error( + formula: tuple[MorletBand, ...], canonical: tuple[MorletBand, ...] +) -> float: + if len(formula) != len(canonical): + raise SignalContractHold("HOLD_MORLET_FORMULA_EQUIVALENCE_SHAPE") + maximum = 0.0 + for expected_index, (observed, frozen) in enumerate( + zip(formula, canonical), start=1 + ): + if ( + observed.index != expected_index + or frozen.index != expected_index + or observed.frequency_hz != frozen.frequency_hz + or observed.length != frozen.length + or observed.sigma != frozen.sigma + or observed.kernel.shape != frozen.kernel.shape + ): + raise SignalContractHold("HOLD_MORLET_FORMULA_EQUIVALENCE_SCHEMA") + difference = np.abs(observed.kernel - frozen.kernel) + if not bool(np.isfinite(difference).all()): + raise SignalContractHold("HOLD_MORLET_FORMULA_EQUIVALENCE_NONFINITE") + maximum = max(maximum, float(np.max(difference))) + return maximum + + +def morlet_formula_max_abs_error() -> float: + """Return the native-formula distance from the immutable canonical bytes.""" + + return _formula_max_abs_error( + morlet_formula_bank_diagnostic(), + _canonical_morlet_bank(), + ) + + +def _morlet_runtime_identity() -> tuple[str, tuple[int, int, int], str, str, str]: + """Return only the fields used to qualify a native formula replay.""" + + machine = platform.machine().lower() + if machine == "amd64": + machine = "x86_64" + return ( + platform.python_implementation(), + sys.version_info[:3], + np.__version__, + platform.system(), + machine, + ) + + +def _validate_morlet_formula_receipt( + receipt: _MorletBuildReceipt, + canonical: tuple[MorletBand, ...], +) -> None: + """Accept only a registered exact replay or the qualified Linux replay.""" + + identity = _morlet_runtime_identity() + implementation, python_version, numpy_version, system, machine = identity + supported_family = ( + implementation == "CPython" + and python_version[:2] in {(3, 12), (3, 13), (3, 14)} + and numpy_version == "2.4.6" + and machine == "x86_64" + ) + formula_digest = morlet_bank_sha256(receipt.bands) + if _formula_max_abs_error( + receipt.bands, canonical + ) > MORLET_FORMULA_MAX_ABS_ERROR_BOUND: + raise SignalContractHold("HOLD_MORLET_FORMULA_EQUIVALENCE_FAIL") + if supported_family and system == "Windows": + # Runtime identity, rather than an accidentally matching digest, selects + # the original exact-oracle branch. Any one-ULP receipt drift must HOLD. + if formula_digest != MORLET_ORACLE_SHA256: + raise SignalContractHold("HOLD_MORLET_ORACLE_DIGEST_MISMATCH") + if receipt.unit_energy_errors != MORLET_EXPECTED_UNIT_ENERGY_ERRORS: + raise SignalNumericError("MORLET_UNIT_ENERGY_ORACLE_FAIL") + if any(error > MORLET_ZERO_DC_BOUND for error in receipt.zero_dc_errors): + raise SignalNumericError("MORLET_ZERO_DC_FAIL") + if any(error > MORLET_UNIT_ENERGY_BOUND for error in receipt.unit_energy_errors): + raise SignalNumericError("MORLET_UNIT_ENERGY_FAIL") + return + if supported_family and system == "Linux": + if formula_digest != MORLET_PORTABLE_LINUX_FORMULA_SHA256: + raise SignalContractHold("HOLD_MORLET_PORTABLE_FORMULA_DIGEST_MISMATCH") + return + raise SignalContractHold("HOLD_MORLET_PORTABLE_RUNTIME_UNQUALIFIED") + + def morlet_kernel_bank() -> tuple[MorletBand, ...]: - """Construct the canonical bank and enforce the complete KERNEL01 oracle.""" + """Return portable oracle bytes after replaying the unchanged formula.""" receipt = _build_morlet_formula_bank() - if morlet_bank_sha256(receipt.bands) != MORLET_ORACLE_SHA256: - raise SignalContractHold("HOLD_MORLET_ORACLE_DIGEST_MISMATCH") - if receipt.unit_energy_errors != MORLET_EXPECTED_UNIT_ENERGY_ERRORS: - raise SignalNumericError("MORLET_UNIT_ENERGY_ORACLE_FAIL") - if any(error > MORLET_ZERO_DC_BOUND for error in receipt.zero_dc_errors): - raise SignalNumericError("MORLET_ZERO_DC_FAIL") - if any(error > MORLET_UNIT_ENERGY_BOUND for error in receipt.unit_energy_errors): - raise SignalNumericError("MORLET_UNIT_ENERGY_FAIL") - return receipt.bands + canonical = _canonical_morlet_bank() + _validate_morlet_formula_receipt(receipt, canonical) + # The returned physical bank is canonical, not the native diagnostic. Its + # verified original digest binds the original zero-DC and unit-energy bytes. + return canonical def morlet_bank_sha256(bands: tuple[MorletBand, ...] | None = None) -> str: @@ -758,21 +905,19 @@ def pooled_relation_descriptor_diagnostic( *, energy_floors: np.ndarray | None = None, ) -> RelationDescriptor: - """Compute a synthetic formula replay explicitly marked ``NO_RESULT``. + """Compute a synthetic portable-oracle replay marked ``NO_RESULT``. This diagnostic is not a canonical-bank PASS and cannot grant data, cache, - training, or scientific eligibility. It constructs the inherited formula - bank internally and checks the pinned byte digest; callers cannot supply a - substitute bank. + training, or scientific eligibility. It replays and bounds the inherited + native formula internally, then computes with the canonical frozen bytes; + callers cannot supply a substitute bank. """ checked = validate_paired_activity( pair.actor_a, pair.actor_b, pair.valid_mask, pair.valid_length ) floors = _validate_energy_floors(energy_floors) - bands = morlet_formula_bank_diagnostic() - if morlet_bank_sha256(bands) != MORLET_ORACLE_SHA256: - raise SignalNumericError("MORLET_DIAGNOSTIC_ORACLE_DIGEST_FAIL") + bands = morlet_kernel_bank() return _pooled_relation_descriptor_from_bank(checked, floors, bands) @@ -1034,7 +1179,11 @@ def swap_relation_descriptor(descriptor: RelationDescriptor) -> RelationDescript "MORLET_FREQUENCIES_HZ", "MORLET_LENGTHS", "MORLET_EXPECTED_UNIT_ENERGY_ERRORS", + "MORLET_FORMULA_MAX_ABS_ERROR_BOUND", "MORLET_ORACLE_SHA256", + "MORLET_ORACLE_EXPORT_RUNTIME", + "MORLET_PORTABLE_LINUX_FORMULA_SHA256", + "MORLET_PORTABLE_SCHEMA", "MORLET_SIGMAS", "MORLET_UNIT_ENERGY_BOUND", "MORLET_ZERO_DC_BOUND", @@ -1055,6 +1204,7 @@ def swap_relation_descriptor(descriptor: RelationDescriptor) -> RelationDescript "interedit_descriptor_from_source", "morlet_bank_sha256", "morlet_formula_bank_diagnostic", + "morlet_formula_max_abs_error", "morlet_kernel_bank", "morlet_length_mask", "morlet_unit_energy_errors", diff --git a/src/phaseset_core/_morlet_oracle_bytes.py b/src/phaseset_core/_morlet_oracle_bytes.py new file mode 100644 index 0000000..51e6bf4 --- /dev/null +++ b/src/phaseset_core/_morlet_oracle_bytes.py @@ -0,0 +1,101 @@ +"""Generated frozen Morlet bytes; do not hand-edit. + +Source formula module: phaseset_core.morlet +Exporter runtime: CPython 3.14.5 / NumPy 2.4.6 / Windows +Canonical serialized SHA-256: 633d754bba12dab0555722f4b8963409a110c8681796cc907d163096e2e42655 +""" + +from __future__ import annotations + +from typing import Final + + +ORACLE_EXPORT_SCHEMA: Final = "portable-morlet-oracle-bytes-v1" +ORACLE_EXPORT_RUNTIME: Final = "CPython 3.14.5 / NumPy 2.4.6 / Windows" +ORACLE_EXPORT_FORMULA_MODULE: Final = "phaseset_core.morlet" +ORACLE_EXPORT_SHA256: Final = "633d754bba12dab0555722f4b8963409a110c8681796cc907d163096e2e42655" +CANONICAL_BANK_BYTES: Final = bytes.fromhex( + "0050bf64fc6e545b2b013f33b2d76b62ddecbf68b3c74dffad1e3f520f1fed7722fcbf6b2436a253885b3f61f2a54ae9" + "9cfcbf6b5915d1def0823f6d3608b5fa9e66bf6839207cdbefae3f753fbf2ab3a463bf609d10f773c7ac3f7c8a4ccac9" + "f961bf3b8b088f19b73c3f81f01839579af13f601d7eabab09643f8532b2a181df033f753780b050a8a63f877fbd539d" + "56583f82cdc2b78af4ab3f88311fb37d2ced3f8c5278e1ac09f73f8693be4e6f72c53f934e7f50a576393f81f90e3bb8" + "dcfe3f986cd97d5c0eb03f739b88a9d538173f9cf880d1845cd5bf525f35cc0b764d3fa024e5f1a5ff06bf8260daf726" + "750f3fa0d44375cc6c5cbf930a036c69390c3fa02e378a852111bf9e45ba461b48943f9bc2029d7b5267bfa519e9054b" + "ad8e3f936206c6ad1cacbfaaf2357a171a9a3f7c258cfcb574a4bfb00c9cb90d7c93bf8243442224efa7bfb1f5d57c2b" + "e472bf9c9005ba49ca8bbfb2e1b62a7a178fbfa921d07c5413d6bfb28452e86c99d6bfb23510d9c47d87bfb0a24583fd" + "c1dabfb7a0711c2e0ec6bfaa33cbec9aebc6bfbc54c4e0abb9bebf9facc88cf5f07dbfbfd027cdcc67c5bf73d3a1a118" + "d86cbfc0ccd69dfdde783f9a7fd5fd216e52bfc0a75f4a882be13fae6b4733f8882cbfbeafc6019466653fb7ed3d21c3" + "5e04bfb9aa937d37f6ac3fc00db41b5943fcbfb26105f1eb7a583fc387436af77949bfa255a4d67ab71b3fc60faa175a" + "1feb3f76e3d4877ffc763fc760f285f304793fa96f358e52541e3fc74a9122db5b373fb7e42a8235b22a3fc5b76f874e" + "75003fc10f0b5f4f9bef3fc2b17b8140b6bc3fc54d37d8957e6f3fbcc462edb9fa053fc8563e3042283c3fb21fe05f0e" + "90ba3fc9eb4a23f8de313f98c225e911c41c3fc9eb4a23f8de31bf98c225e911c41c3fc8563e3042283cbfb21fe05f0e" + "90ba3fc54d37d8957e6fbfbcc462edb9fa053fc10f0b5f4f9befbfc2b17b8140b6bc3fb7e42a8235b22abfc5b76f874e" + "75003fa96f358e52541ebfc74a9122db5b373f76e3d4877ffc76bfc760f285f30479bfa255a4d67ab71bbfc60faa175a" + "1febbfb26105f1eb7a58bfc387436af77949bfb9aa937d37f6acbfc00db41b5943fcbfbeafc601946665bfb7ed3d21c3" + "5e04bfc0a75f4a882be1bfae6b4733f8882cbfc0ccd69dfdde78bf9a7fd5fd216e52bfbfd027cdcc67c53f73d3a1a118" + "d86cbfbc54c4e0abb9be3f9facc88cf5f07dbfb7a0711c2e0ec63faa33cbec9aebc6bfb23510d9c47d873fb0a24583fd" + "c1dabfa921d07c5413d63fb28452e86c99d6bf9c9005ba49ca8b3fb2e1b62a7a178fbf8243442224efa73fb1f5d57c2b" + "e4723f7c258cfcb574a43fb00c9cb90d7c933f936206c6ad1cac3faaf2357a171a9a3f9bc2029d7b52673fa519e9054b" + "ad8e3fa02e378a8521113f9e45ba461b48943fa0d44375cc6c5c3f930a036c69390c3fa024e5f1a5ff063f8260daf726" + "750f3f9cf880d1845cd53f525f35cc0b764d3f986cd97d5c0eb0bf739b88a9d538173f934e7f50a57639bf81f90e3bb8" + "dcfe3f8c5278e1ac09f7bf8693be4e6f72c53f82cdc2b78af4abbf88311fb37d2ced3f753780b050a8a6bf877fbd539d" + "56583f601d7eabab0964bf8532b2a181df03bf3b8b088f19b73cbf81f01839579af1bf609d10f773c7acbf7c8a4ccac9" + "f961bf6839207cdbefaebf753fbf2ab3a463bf6b5915d1def082bf6d3608b5fa9e66bf6b2436a253885bbf61f2a54ae9" + "9cfcbf68b3c74dffad1ebf520f1fed7722fcbf64fc6e545b2b01bf33b2d76b62ddec0036bf686d558ffed44f3f26db71" + "43b55546bf6f2d79c97a3d073f5b0730bca77704bf70df16e84f272c3f70091b6a3a5377bf6be2b5b821756f3f7c349e" + "b4db20c2bf51d8a83b3d1d463f84de8996a85c8c3f6b0d76e596f8023f8ace327cf519ee3f845c5593c8e2683f8da5a5" + "3d5a24303f92e614fe1aa7c93f8a9d5a49f96d7f3f9c64b1355285533f7de517e034887d3fa257730ef6fadbbf6d2614" + "3d5a473f3fa48b8230d7ca2abf941df2fbe6eae53fa352fb6413597ebfa457ff2f67db063f9ac72513422e37bfaf4310" + "7d5ecdcd3f7075e60a346b60bfb4553946b3958cbf9caedc9e58bd88bfb6fa1c3e724c66bfb11a3daf87d7aabfb648ca" + "aee04f01bfbb52eda6f774a4bfb151ba0834e941bfc1f53bb2d100e2bf9f40d36f9a1ae4bfc468102ca02b103f966437" + "a17ac7b2bfc41ff8323dc3d33fb54fd7c64f732cbfc09ceae83031513fc27678db243c2cbfb3dec97a2610d73fc8d094" + "d12dedcebf796e8c0125e1003fcc6210a09ab3983fb31185feb6dac53fcc36e75417cf0c3fc36b21411bdb563fc7fa3e" + "7106fd9f3fcb2e3ceb717b273fc016350e0a97f63fcf72944f29afa33fa6a7de0eaf7f253fcf72944f29afa3bfa6a7de" + "0eaf7f253fcb2e3ceb717b27bfc016350e0a97f63fc36b21411bdb56bfc7fa3e7106fd9f3fb31185feb6dac5bfcc36e7" + "5417cf0cbf796e8c0125e100bfcc6210a09ab398bfb3dec97a2610d7bfc8d094d12dedcebfc09ceae8303151bfc27678" + "db243c2cbfc41ff8323dc3d3bfb54fd7c64f732cbfc468102ca02b10bf966437a17ac7b2bfc1f53bb2d100e23f9f40d3" + "6f9a1ae4bfbb52eda6f774a43fb151ba0834e941bfb11a3daf87d7aa3fb648caaee04f01bf9caedc9e58bd883fb6fa1c" + "3e724c663f7075e60a346b603fb4553946b3958c3f9ac72513422e373faf43107d5ecdcd3fa352fb6413597e3fa457ff" + "2f67db063fa48b8230d7ca2a3f941df2fbe6eae53fa257730ef6fadb3f6d26143d5a473f3f9c64b135528553bf7de517" + "e034887d3f92e614fe1aa7c9bf8a9d5a49f96d7f3f845c5593c8e268bf8da5a53d5a24303f6b0d76e596f802bf8ace32" + "7cf519eebf51d8a83b3d1d46bf84de8996a85c8cbf6be2b5b821756fbf7c349eb4db20c2bf70df16e84f272cbf70091b" + "6a3a5377bf6f2d79c97a3d07bf5b0730bca77704bf686d558ffed44fbf26db7143b555460024bf701cfbe82b3f163f42" + "f6203cdd874abf74a4ad7e734a2f3f706ddaaec1483fbf6e11484f7506903f835255762286733f63ff4852dd8b0a3f8f" + "3eb6b7cc7d8d3f8de900e25068b23f9218f71db1e4683f9fe5033b8dac543f87073742bf6be33fa76ead61b7a56dbf81" + "d734453aa47d3fa8879cb9d3e5f8bfa59c093dd5733c3f9b1d886766ab2bbfb4badc2ffb21dbbf98146bb8064fb1bfbb" + "a84103b58595bfb8204f01899d3ebfba4ae2190b2983bfc4d77989c24089bfa9f55faae4a573bfc940e48b2680a03fa7" + "02068a87546cbfc5cd1b6d6f8bb63fc45204fbcc7547bfb333bcb72ac8aa3fcfdc2d3c32f4113fb0f7b14722db7a3fd1" + "8967204286203fca7a9979e1b5743fcb54654cac49a43fd2d6bdfd00823e3fb4a2638880961e3fd2d6bdfd00823ebfb4" + "a2638880961e3fca7a9979e1b574bfcb54654cac49a43fb0f7b14722db7abfd1896720428620bfb333bcb72ac8aabfcf" + "dc2d3c32f411bfc5cd1b6d6f8bb6bfc45204fbcc7547bfc940e48b2680a0bfa702068a87546cbfc4d77989c240893fa9" + "f55faae4a573bfb8204f01899d3e3fba4ae2190b2983bf98146bb8064fb13fbba84103b585953f9b1d886766ab2b3fb4" + "badc2ffb21db3fa8879cb9d3e5f83fa59c093dd5733c3fa76ead61b7a56d3f81d734453aa47d3f9fe5033b8dac54bf87" + "073742bf6be33f8de900e25068b2bf9218f71db1e4683f63ff4852dd8b0abf8f3eb6b7cc7d8dbf6e11484f750690bf83" + "525576228673bf74a4ad7e734a2fbf706ddaaec1483fbf701cfbe82b3f16bf42f6203cdd874a0018bf75b42490043d92" + "3f58bb3b002132a9bf75ed4b917c24193f83ec6e9fd469883f78a5b4a997c2dc3f94899b85ed02bd3fa0d7f509e11dc4" + "3f91897e348166b63fae100466d35557bf9409afe98ef03d3fa5a219e749f7e5bfb672881dd8b82dbfa904e8786db088" + "bfc1528e7c5fbe82bfc722904b76dcd5bfb5691fed769cc2bfcee3289a42dc323fb6ee86c5db8288bfc0629236fabf5d" + "3fd21c6cf3d7c80e3fc03f32398967ae3fd4edf595d343743fd5efd6038313873fc297e43c37e6413fd5efd603831387" + "bfc297e43c37e6413fc03f32398967aebfd4edf595d34374bfc0629236fabf5dbfd21c6cf3d7c80ebfcee3289a42dc32" + "bfb6ee86c5db8288bfc722904b76dcd53fb5691fed769cc2bfa904e8786db0883fc1528e7c5fbe823fa5a219e749f7e5" + "3fb672881dd8b82d3fae100466d355573f9409afe98ef03d3fa0d7f509e11dc4bf91897e348166b63f78a5b4a997c2dc" + "bf94899b85ed02bdbf75ed4b917c2419bf83ec6e9fd46988bf75b42490043d92bf58bb3b002132a90010bf7d79283573" + "ab943f6e4db67ca722ef3f60a736214ee4d73f9672f58dd7462d3fa98cb39eb301503f8d5818f461f7783fb08f699921" + "2b27bfb5d6cc15d1b457bfb9818f10f3c215bfc50f0bce04b08dbfd2ce36cddb45623fa801dbd731d6aabfb6ac7c325e" + "21ab3fd89ac77b52c9193fd7d8def92444443fd059faa509c17d3fd7d8def9244444bfd059faa509c17dbfb6ac7c325e" + "21abbfd89ac77b52c919bfd2ce36cddb4562bfa801dbd731d6aabfb9818f10f3c2153fc50f0bce04b08d3fb08f699921" + "2b273fb5d6cc15d1b4573fa98cb39eb30150bf8d5818f461f7783f60a736214ee4d7bf9672f58dd7462dbf7d79283573" + "ab94bf6e4db67ca722ef000bbf8207ef2a11c5b43f728e4a7c3556d83f9b8379076c43043fa09f5000cf8c623fb44ee3" + "08696ffebfbabed75017eaa6bfd155244ee7eda5bfc00e884a66bef3bfbbaac8ab267df43fde21e15942d9973fe20425" + "a070fb213c71cfa5b8022987bfbbaac8ab267df4bfde21e15942d997bfd155244ee7eda53fc00e884a66bef43fb44ee3" + "08696ffe3fbabed75017eaa63f9b8379076c4304bfa09f5000cf8c62bf8207ef2a11c5b4bf728e4a7c3556d8" +) + + +__all__ = [ + "CANONICAL_BANK_BYTES", + "ORACLE_EXPORT_FORMULA_MODULE", + "ORACLE_EXPORT_RUNTIME", + "ORACLE_EXPORT_SCHEMA", + "ORACLE_EXPORT_SHA256", +] diff --git a/src/phaseset_core/base_cohort_latency.py b/src/phaseset_core/base_cohort_latency.py new file mode 100644 index 0000000..3fcae0a --- /dev/null +++ b/src/phaseset_core/base_cohort_latency.py @@ -0,0 +1,1719 @@ +"""Run the frozen nine-row PhaseSet base latency session on one CUDA device. + +The public entry point derives every timed sample itself. Resolver-owned +selected-best checkpoint bytes and the separately recomputed capture-score +cohort are comparison inputs; neither a latency nor a parameter count can be +submitted by a caller. Results remain authority zero and stop before +``BaseScore`` construction, winner selection, or production authorization. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass, field +import hashlib +import json +import math +import re +import struct +import subprocess +import time +from typing import Final, Protocol, runtime_checkable + +import numpy as np +import torch + +from . import base_cohort_resolver as resolver_module +from . import base_cohort_validation as scoring_module +from . import training as training_module +from .contracts import PreparedGroupBatch, group_commitment +from .experiments import base_run_ids, parse_run_id + + +AUTHORITY: Final = 0 +PRODUCTION: Final = False +RESULT_CLAIMED: Final = False +SESSION_SCHEMA: Final = "phaseset-base-cohort-latency-session-v1" +VISIT_SCHEMA: Final = "phaseset-base-cohort-latency-visit-v1" +ROW_SCHEMA: Final = "phaseset-base-cohort-latency-row-v1" +WORKLOAD_SCHEMA: Final = "phaseset-base-cohort-latency-workload-v1" +RUNTIME_SCHEMA: Final = "phaseset-base-cohort-latency-runtime-v1" +SNAPSHOT_SCHEMA: Final = "phaseset-shared-cuda-observation-v1" +ACTUAL_COMPLETE_STATUS: Final = "ACTUAL_NINE_ROW_LATENCY_SESSION_COMPLETE_NO_QUALIFICATION" +FAKE_COMPLETE_STATUS: Final = "ENGINEERING_FAKE_RUNTIME_COMPLETE_NOT_FORMAL" +HOLD_STATUS: Final = "LATENCY_SESSION_HOLD_NO_FORMAL_ROW_LATENCY" + +ROUND_COUNT: Final = 9 +ROWS_PER_ROUND: Final = 9 +WARMUPS_PER_VISIT: Final = 5 +TIMED_PER_VISIT: Final = 11 +SAMPLES_PER_ROW: Final = ROUND_COUNT * TIMED_PER_VISIT +FIXTURE_BATCH: Final = 1 +FIXTURE_ACTORS: Final = 32 +FIXTURE_FRAMES: Final = 200 +FIXTURE_JOINTS: Final = 22 +FIXTURE_COORDINATES: Final = 3 +FIXTURE_EMBEDDING_DIM: Final = 512 +MINIMUM_FREE_MIB: Final = 8192 +CUDA_ALLOCATOR_LIMIT_BYTES: Final = 2 * 1024**3 +_ACTOR_DOMAIN: Final = b"phaseset-base-latency-actor-v1\n" +_CUDA_UUID_PAYLOAD: Final = re.compile( + r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}" +) + + +class BaseCohortLatencyError(ValueError): + """The cohort, score binding, runtime, or measured sample is invalid.""" + + +class BaseCohortLatencyHold(RuntimeError): + """A started session failed; ``receipt`` retains every observed sample.""" + + def __init__(self, failure_code: str, receipt: BaseCohortLatencySession) -> None: + super().__init__(failure_code) + self.failure_code = failure_code + self.receipt = receipt + + +def _sha256(raw: bytes) -> str: + return hashlib.sha256(raw).hexdigest() + + +def _lower_sha256(value: object, label: str) -> str: + if ( + type(value) is not str + or len(value) != 64 + or value.lower() != value + or any(character not in "0123456789abcdef" for character in value) + ): + raise BaseCohortLatencyError(f"{label} must be lowercase SHA-256 hex") + return value + + +def _prefixed_cuda_uuid(value: object, label: str) -> str: + if ( + type(value) is not str + or not value.startswith("GPU-") + or _CUDA_UUID_PAYLOAD.fullmatch(value[4:]) is None + ): + raise BaseCohortLatencyError( + f"{label} must be GPU- plus one canonical lowercase CUDA UUID" + ) + return value + + +def _torch_cuda_uuid(value: object) -> str: + try: + rendered = str(value) + except Exception as error: + raise BaseCohortLatencyError( + "Torch device UUID cannot be rendered as a canonical CUDA UUID" + ) from error + payload = rendered[4:] if rendered.startswith("GPU-") else rendered + if _CUDA_UUID_PAYLOAD.fullmatch(payload) is None: + raise BaseCohortLatencyError( + "Torch device UUID is not a canonical lowercase CUDA UUID" + ) + return f"GPU-{payload}" + + +def _positive_int(value: object, label: str) -> int: + if type(value) is not int or not 1 <= value <= (1 << 63) - 1: + raise BaseCohortLatencyError(f"{label} must be an exact positive int") + return value + + +def _nonnegative_int(value: object, label: str) -> int: + if type(value) is not int or not 0 <= value <= (1 << 63) - 1: + raise BaseCohortLatencyError(f"{label} must be an exact nonnegative int") + return value + + +def _canonical_json_bytes(value: object) -> bytes: + return ( + json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + allow_nan=False, + ).encode("ascii") + + b"\n" + ) + + +def _canonical_digest(value: object) -> str: + return _sha256(_canonical_json_bytes(value)) + + +@dataclass(frozen=True, slots=True) +class SharedCudaObservation: + captured_unix_ns: int + device_uuid: str + memory_total_mib: int + memory_free_mib: int + memory_used_mib: int + gpu_utilization_percent: int + memory_utilization_percent: int + compute_mode: str + driver_version: str + schema: str = SNAPSHOT_SCHEMA + + def __post_init__(self) -> None: + _positive_int(self.captured_unix_ns, "captured_unix_ns") + _prefixed_cuda_uuid(self.device_uuid, "CUDA observation UUID") + _positive_int(self.memory_total_mib, "memory_total_mib") + _nonnegative_int(self.memory_free_mib, "memory_free_mib") + _nonnegative_int(self.memory_used_mib, "memory_used_mib") + if ( + self.memory_free_mib > self.memory_total_mib + or self.memory_used_mib > self.memory_total_mib + ): + raise BaseCohortLatencyError("CUDA memory observation exceeds physical total") + for name in ("gpu_utilization_percent", "memory_utilization_percent"): + value = getattr(self, name) + if type(value) is not int or not 0 <= value <= 100: + raise BaseCohortLatencyError(f"{name} must be an exact percent") + if type(self.compute_mode) is not str or not self.compute_mode: + raise BaseCohortLatencyError("CUDA compute mode is absent") + if type(self.driver_version) is not str or not self.driver_version: + raise BaseCohortLatencyError("CUDA driver version is absent") + if self.schema != SNAPSHOT_SCHEMA: + raise BaseCohortLatencyError("CUDA observation schema changed") + + def payload(self) -> dict[str, object]: + return { + "captured_unix_ns": self.captured_unix_ns, + "compute_mode": self.compute_mode, + "device_uuid": self.device_uuid, + "driver_version": self.driver_version, + "gpu_utilization_percent": self.gpu_utilization_percent, + "memory_free_mib": self.memory_free_mib, + "memory_total_mib": self.memory_total_mib, + "memory_used_mib": self.memory_used_mib, + "memory_utilization_percent": self.memory_utilization_percent, + "schema": self.schema, + } + + +@dataclass(frozen=True, slots=True) +class BaseLatencyRuntimeIdentity: + device: str + device_uuid: str + device_name: str + torch_version: str + cuda_runtime_version: str + cudnn_version: int + driver_version: str + nvidia_total_memory_mib: int + compute_mode: str + torch_total_memory_bytes: int + initial_allocated_bytes: int + initial_reserved_bytes: int + environment_sha256: str + precision_mode: str + allocator_limit_bytes: int = CUDA_ALLOCATOR_LIMIT_BYTES + schema: str = RUNTIME_SCHEMA + + def __post_init__(self) -> None: + try: + device = torch.device(self.device) + except (RuntimeError, ValueError) as error: + raise BaseCohortLatencyError("latency runtime device is invalid") from error + if device.type != "cuda": + raise BaseCohortLatencyError("formal latency runtime must be CUDA") + _prefixed_cuda_uuid(self.device_uuid, "latency runtime UUID") + for name in ( + "device_name", + "torch_version", + "cuda_runtime_version", + "driver_version", + ): + if type(getattr(self, name)) is not str or not getattr(self, name): + raise BaseCohortLatencyError(f"latency runtime {name} is absent") + _positive_int(self.cudnn_version, "cudnn_version") + _positive_int(self.nvidia_total_memory_mib, "nvidia_total_memory_mib") + if type(self.compute_mode) is not str or not self.compute_mode: + raise BaseCohortLatencyError("latency runtime compute mode is absent") + _positive_int(self.torch_total_memory_bytes, "torch_total_memory_bytes") + _nonnegative_int(self.initial_allocated_bytes, "initial_allocated_bytes") + _nonnegative_int(self.initial_reserved_bytes, "initial_reserved_bytes") + if self.initial_allocated_bytes != 0 or self.initial_reserved_bytes != 0: + raise BaseCohortLatencyError("latency process has preexisting CUDA allocations") + _lower_sha256(self.environment_sha256, "environment_sha256") + if self.precision_mode not in ("FP32", "BF16"): + raise BaseCohortLatencyError("latency precision mode is invalid") + if self.allocator_limit_bytes != CUDA_ALLOCATOR_LIMIT_BYTES: + raise BaseCohortLatencyError("CUDA allocator limit must remain exact 2 GiB") + if self.schema != RUNTIME_SCHEMA: + raise BaseCohortLatencyError("latency runtime schema changed") + + @property + def sha256(self) -> str: + return _canonical_digest( + { + "allocator_limit_bytes": self.allocator_limit_bytes, + "cuda_runtime_version": self.cuda_runtime_version, + "cudnn_version": self.cudnn_version, + "compute_mode": self.compute_mode, + "device": self.device, + "device_name": self.device_name, + "device_uuid": self.device_uuid, + "driver_version": self.driver_version, + "environment_sha256": self.environment_sha256, + "initial_allocated_bytes": self.initial_allocated_bytes, + "initial_reserved_bytes": self.initial_reserved_bytes, + "nvidia_total_memory_mib": self.nvidia_total_memory_mib, + "precision_mode": self.precision_mode, + "schema": self.schema, + "torch_total_memory_bytes": self.torch_total_memory_bytes, + "torch_version": self.torch_version, + } + ) + + +@dataclass(frozen=True, slots=True) +class BaseLatencyWorkloadReceipt: + array_rows: tuple[tuple[str, str, tuple[int, ...], str], ...] + raw_skeletons_sha256: str + actor_commitments: tuple[bytes, ...] = field(repr=False) + group_commitment: bytes = field(repr=False) + schema: str = WORKLOAD_SCHEMA + + def __post_init__(self) -> None: + expected_rows = ( + ( + "skeletons", + ( + FIXTURE_BATCH, + FIXTURE_ACTORS, + FIXTURE_FRAMES, + FIXTURE_JOINTS, + FIXTURE_COORDINATES, + ), + " str: + return _canonical_digest( + { + "actor_commitments": [value.hex() for value in self.actor_commitments], + "array_rows": [ + { + "dtype": dtype, + "name": name, + "sha256": digest, + "shape": list(shape), + } + for name, digest, shape, dtype in self.array_rows + ], + "group_commitment": self.group_commitment.hex(), + "raw_skeletons_sha256": self.raw_skeletons_sha256, + "schema": self.schema, + } + ) + + +@dataclass(frozen=True, slots=True) +class BaseLatencyVisit: + round_index: int + ordinal_in_round: int + run_id: str + before: SharedCudaObservation + after: SharedCudaObservation | None + samples_ns: tuple[int, ...] + peak_allocated_bytes: int | None + peak_reserved_bytes: int | None + completed: bool + failure_code: str | None + warmups_completed: int = WARMUPS_PER_VISIT + schema: str = VISIT_SCHEMA + + def __post_init__(self) -> None: + if type(self.round_index) is not int or not 0 <= self.round_index < ROUND_COUNT: + raise BaseCohortLatencyError("visit round index is invalid") + if ( + type(self.ordinal_in_round) is not int + or not 0 <= self.ordinal_in_round < ROWS_PER_ROUND + ): + raise BaseCohortLatencyError("visit ordinal is invalid") + if self.run_id not in base_run_ids(): + raise BaseCohortLatencyError("visit run ID is unregistered") + if type(self.before) is not SharedCudaObservation: + raise BaseCohortLatencyError("visit before observation is missing") + if type(self.samples_ns) is not tuple or len(self.samples_ns) > TIMED_PER_VISIT: + raise BaseCohortLatencyError("visit sample census exceeds eleven") + for value in self.samples_ns: + _positive_int(value, "latency sample") + if ( + type(self.warmups_completed) is not int + or not 0 <= self.warmups_completed <= WARMUPS_PER_VISIT + or self.schema != VISIT_SCHEMA + ): + raise BaseCohortLatencyError("visit protocol header changed") + if self.completed: + if ( + type(self.after) is not SharedCudaObservation + or len(self.samples_ns) != TIMED_PER_VISIT + or type(self.peak_allocated_bytes) is not int + or type(self.peak_reserved_bytes) is not int + or self.warmups_completed != WARMUPS_PER_VISIT + or self.failure_code is not None + ): + raise BaseCohortLatencyError("completed visit evidence is incomplete") + elif self.failure_code is None: + raise BaseCohortLatencyError("incomplete visit requires a failure code") + for name in ("peak_allocated_bytes", "peak_reserved_bytes"): + value = getattr(self, name) + if value is not None: + _nonnegative_int(value, name) + + @property + def sha256(self) -> str: + return _canonical_digest( + { + "after": None if self.after is None else self.after.payload(), + "before": self.before.payload(), + "completed": self.completed, + "failure_code": self.failure_code, + "ordinal_in_round": self.ordinal_in_round, + "peak_allocated_bytes": self.peak_allocated_bytes, + "peak_reserved_bytes": self.peak_reserved_bytes, + "round_index": self.round_index, + "run_id": self.run_id, + "samples_ns": list(self.samples_ns), + "schema": self.schema, + "warmups_completed": self.warmups_completed, + } + ) + + +def _nearest_rank(sorted_values: tuple[int, ...], percentile: int) -> int: + rank = math.ceil(percentile * len(sorted_values) / 100) + return sorted_values[max(1, rank) - 1] + + +@dataclass(frozen=True, slots=True) +class BaseLatencyRowObservation: + run_id: str + system_id: str + seed: int + selected_checkpoint_sha256: str + selected_checkpoint_state_digest: str + parameter_census_sha256: str + parameter_count: int + runtime_sha256: str + workload_sha256: str + samples_ns: tuple[int, ...] + median_ns: int + median_absolute_deviation_ns: int + minimum_ns: int + maximum_ns: int + p05_nearest_rank_ns: int + p95_nearest_rank_ns: int + schema: str = ROW_SCHEMA + + def __post_init__(self) -> None: + role, seed, system_id = parse_run_id(self.run_id) + if role != "BASE_QUALIFICATION" or self.seed != seed or self.system_id != system_id: + raise BaseCohortLatencyError("latency row identity is inconsistent") + for name in ( + "selected_checkpoint_sha256", + "selected_checkpoint_state_digest", + "parameter_census_sha256", + "runtime_sha256", + "workload_sha256", + ): + _lower_sha256(getattr(self, name), name) + _positive_int(self.parameter_count, "parameter_count") + if type(self.samples_ns) is not tuple or len(self.samples_ns) != SAMPLES_PER_ROW: + raise BaseCohortLatencyError("latency row must retain exact 99 samples") + for value in self.samples_ns: + _positive_int(value, "latency sample") + sorted_values = tuple(sorted(self.samples_ns)) + median = sorted_values[len(sorted_values) // 2] + deviations = tuple(sorted(abs(value - median) for value in self.samples_ns)) + expected = ( + median, + deviations[len(deviations) // 2], + sorted_values[0], + sorted_values[-1], + _nearest_rank(sorted_values, 5), + _nearest_rank(sorted_values, 95), + ) + observed = ( + self.median_ns, + self.median_absolute_deviation_ns, + self.minimum_ns, + self.maximum_ns, + self.p05_nearest_rank_ns, + self.p95_nearest_rank_ns, + ) + if observed != expected or self.schema != ROW_SCHEMA: + raise BaseCohortLatencyError("latency row statistics differ from raw samples") + + @property + def frozen_runtime_latency_ns(self) -> int: + return self.median_ns + + @property + def sha256(self) -> str: + return _canonical_digest( + { + "maximum_ns": self.maximum_ns, + "median_absolute_deviation_ns": self.median_absolute_deviation_ns, + "median_ns": self.median_ns, + "minimum_ns": self.minimum_ns, + "p05_nearest_rank_ns": self.p05_nearest_rank_ns, + "p95_nearest_rank_ns": self.p95_nearest_rank_ns, + "parameter_census_sha256": self.parameter_census_sha256, + "parameter_count": self.parameter_count, + "run_id": self.run_id, + "runtime_sha256": self.runtime_sha256, + "samples_ns": list(self.samples_ns), + "schema": self.schema, + "seed": self.seed, + "selected_checkpoint_sha256": self.selected_checkpoint_sha256, + "selected_checkpoint_state_digest": self.selected_checkpoint_state_digest, + "system_id": self.system_id, + "workload_sha256": self.workload_sha256, + } + ) + + +@dataclass(frozen=True, slots=True, repr=False) +class BaseCohortLatencySession: + registered_cuda_uuid: str + resolved_cohort_sha256: str + scored_cohort_sha256: str + workload_receipt: BaseLatencyWorkloadReceipt + admission_observation: SharedCudaObservation + post_context_observation: SharedCudaObservation | None + runtime_identity: BaseLatencyRuntimeIdentity | None + visits: tuple[BaseLatencyVisit, ...] + rows: tuple[BaseLatencyRowObservation, ...] + actual_runtime: bool + complete: bool + formal_samples_complete: bool + failure_code: str | None + authority: int = AUTHORITY + production: bool = PRODUCTION + result_claimed: bool = RESULT_CLAIMED + schema: str = SESSION_SCHEMA + status: str = HOLD_STATUS + + def __post_init__(self) -> None: + _prefixed_cuda_uuid(self.registered_cuda_uuid, "session CUDA UUID") + _lower_sha256(self.resolved_cohort_sha256, "resolved_cohort_sha256") + _lower_sha256(self.scored_cohort_sha256, "scored_cohort_sha256") + if type(self.workload_receipt) is not BaseLatencyWorkloadReceipt: + raise BaseCohortLatencyError("session workload receipt is invalid") + if type(self.admission_observation) is not SharedCudaObservation: + raise BaseCohortLatencyError("session admission observation is invalid") + if type(self.visits) is not tuple or any( + type(value) is not BaseLatencyVisit for value in self.visits + ): + raise BaseCohortLatencyError("session visit census is malformed") + if type(self.rows) is not tuple or any( + type(value) is not BaseLatencyRowObservation for value in self.rows + ): + raise BaseCohortLatencyError("session row census is malformed") + if type(self.actual_runtime) is not bool or type(self.complete) is not bool: + raise BaseCohortLatencyError("session runtime/completion flags are malformed") + if type(self.formal_samples_complete) is not bool: + raise BaseCohortLatencyError("session formal sample flag is malformed") + if self.complete: + expected_visits = tuple( + (round_index, ordinal, run_id) + for round_index, round_rows in enumerate(cyclic_visit_schedule()) + for ordinal, run_id in enumerate(round_rows) + ) + observed_visits = tuple( + (visit.round_index, visit.ordinal_in_round, visit.run_id) for visit in self.visits + ) + if ( + observed_visits != expected_visits + or any(not visit.completed for visit in self.visits) + or tuple(row.run_id for row in self.rows) != base_run_ids() + or type(self.runtime_identity) is not BaseLatencyRuntimeIdentity + or type(self.post_context_observation) is not SharedCudaObservation + or self.failure_code is not None + ): + raise BaseCohortLatencyError("completed latency session census is incomplete") + if any( + visit.before.device_uuid != self.registered_cuda_uuid + or visit.after is None + or visit.after.device_uuid != self.registered_cuda_uuid + for visit in self.visits + ): + raise BaseCohortLatencyError("completed session visit changed CUDA UUID") + if any( + row.runtime_sha256 != self.runtime_identity.sha256 + or row.workload_sha256 != self.workload_receipt.sha256 + for row in self.rows + ): + raise BaseCohortLatencyError("completed latency rows lost runtime/workload binding") + expected_formal = self.actual_runtime + expected_status = ( + ACTUAL_COMPLETE_STATUS if self.actual_runtime else FAKE_COMPLETE_STATUS + ) + if ( + self.formal_samples_complete is not expected_formal + or self.status != expected_status + ): + raise BaseCohortLatencyError("completed latency session status is inconsistent") + elif ( + self.rows + or self.formal_samples_complete + or self.failure_code is None + or self.status != HOLD_STATUS + ): + raise BaseCohortLatencyError("held latency session must not expose formal rows") + if ( + type(self.authority) is not int + or self.authority != AUTHORITY + or self.production is not PRODUCTION + or self.result_claimed is not RESULT_CLAIMED + or self.schema != SESSION_SCHEMA + ): + raise BaseCohortLatencyError("latency session result header is invalid") + + @property + def sha256(self) -> str: + return _canonical_digest( + { + "actual_runtime": self.actual_runtime, + "admission_observation": self.admission_observation.payload(), + "authority": self.authority, + "complete": self.complete, + "failure_code": self.failure_code, + "formal_samples_complete": self.formal_samples_complete, + "post_context_observation": ( + None + if self.post_context_observation is None + else self.post_context_observation.payload() + ), + "production": self.production, + "registered_cuda_uuid": self.registered_cuda_uuid, + "resolved_cohort_sha256": self.resolved_cohort_sha256, + "result_claimed": self.result_claimed, + "row_sha256s": [row.sha256 for row in self.rows], + "runtime_sha256": None + if self.runtime_identity is None + else self.runtime_identity.sha256, + "schema": self.schema, + "scored_cohort_sha256": self.scored_cohort_sha256, + "status": self.status, + "visit_sha256s": [visit.sha256 for visit in self.visits], + "workload_sha256": self.workload_receipt.sha256, + } + ) + + def __repr__(self) -> str: + return ( + "BaseCohortLatencySession(" + f"complete={self.complete}, actual_runtime={self.actual_runtime}, " + f"visits={len(self.visits)}, sha256={self.sha256!r})" + ) + + +class _LatencyRuntime(Protocol): + def snapshot(self) -> SharedCudaObservation: ... + + def initialize( + self, + *, + device: torch.device, + precision_mode: str, + admission_observation: SharedCudaObservation, + ) -> BaseLatencyRuntimeIdentity: ... + + def synchronize(self, device: torch.device) -> None: ... + + def perf_counter_ns(self) -> int: ... + + def reset_peak_memory(self, device: torch.device) -> None: ... + + def peak_memory(self, device: torch.device) -> tuple[int, int]: ... + + def place(self, system: training_module.BaseRetrievalSystem, device: torch.device) -> None: ... + + def encode( + self, + system: training_module.BaseRetrievalSystem, + groups: PreparedGroupBatch, + ) -> object: ... + + def validate_output(self, output: object, device: torch.device) -> None: ... + + def release( + self, system: training_module.BaseRetrievalSystem, device: torch.device + ) -> None: ... + + +@runtime_checkable +class LatencySessionObserver(Protocol): + """Typed post-operation observer for the private latency controller. + + Implementations receive only values already derived by this module. They + cannot submit samples, metrics, checkpoints, or completion state. + """ + + def session_start( + self, + *, + admission_observation_sha256: str, + started_unix_ns: int, + ) -> object: ... + + def context_ready( + self, + *, + runtime_sha256: str, + post_context_observation_sha256: str, + ) -> object: ... + + def visit_started( + self, + *, + round_index: int, + ordinal_in_round: int, + run_id: str, + before_observation_sha256: str, + ) -> object: ... + + def warmup_completed( + self, + *, + round_index: int, + ordinal_in_round: int, + run_id: str, + warmup_index: int, + ) -> object: ... + + def timed_sample_completed( + self, + *, + round_index: int, + ordinal_in_round: int, + run_id: str, + sample_index: int, + sample_ns: int, + ) -> object: ... + + def visit_ended( + self, + *, + round_index: int, + ordinal_in_round: int, + run_id: str, + outcome: str, + after_observation_sha256: str | None, + peak_allocated_bytes: int | None, + peak_reserved_bytes: int | None, + failure_code: str | None, + ) -> object: ... + + def terminal( + self, + *, + outcome: str, + latency_session_sha256: str, + failure_code: str | None, + ) -> object: ... + + +class _LatencyObserverFailure(RuntimeError): + """An observer callback failed; no later callback may be attempted.""" + + +class _NvidiaSmiTorchRuntime: + __slots__ = ("_uuid",) + + def __init__(self, registered_cuda_uuid: str) -> None: + self._uuid = _prefixed_cuda_uuid( + registered_cuda_uuid, + "registered CUDA UUID", + ) + + def snapshot(self) -> SharedCudaObservation: + query = ( + "uuid,memory.total,memory.free,memory.used,utilization.gpu," + "utilization.memory,compute_mode,driver_version" + ) + try: + completed = subprocess.run( + [ + "nvidia-smi", + f"--id={self._uuid}", + f"--query-gpu={query}", + "--format=csv,noheader,nounits", + ], + check=True, + capture_output=True, + text=True, + timeout=10, + ) + except (OSError, subprocess.SubprocessError) as error: + raise BaseCohortLatencyError("NVIDIA device observation failed") from error + rows = tuple(line.strip() for line in completed.stdout.splitlines() if line.strip()) + if len(rows) != 1: + raise BaseCohortLatencyError("NVIDIA device observation is not one exact row") + fields = tuple(value.strip() for value in rows[0].split(",")) + if len(fields) != 8: + raise BaseCohortLatencyError("NVIDIA device observation field count changed") + try: + return SharedCudaObservation( + captured_unix_ns=time.time_ns(), + device_uuid=fields[0], + memory_total_mib=int(fields[1]), + memory_free_mib=int(fields[2]), + memory_used_mib=int(fields[3]), + gpu_utilization_percent=int(fields[4]), + memory_utilization_percent=int(fields[5]), + compute_mode=fields[6], + driver_version=fields[7], + ) + except (TypeError, ValueError) as error: + raise BaseCohortLatencyError("NVIDIA device observation is malformed") from error + + def initialize( + self, + *, + device: torch.device, + precision_mode: str, + admission_observation: SharedCudaObservation, + ) -> BaseLatencyRuntimeIdentity: + if device.type != "cuda" or not torch.cuda.is_available(): + raise BaseCohortLatencyError("formal latency CUDA device is unavailable") + index = torch.cuda.current_device() if device.index is None else device.index + properties = torch.cuda.get_device_properties(index) + observed_uuid = _torch_cuda_uuid(getattr(properties, "uuid", None)) + _prefixed_cuda_uuid(admission_observation.device_uuid, "admission CUDA UUID") + if observed_uuid != self._uuid or admission_observation.device_uuid != self._uuid: + raise BaseCohortLatencyError("Torch and NVIDIA device UUIDs differ") + fraction = CUDA_ALLOCATOR_LIMIT_BYTES / int(properties.total_memory) + if not 0.0 < fraction <= 1.0: + raise BaseCohortLatencyError("CUDA allocator limit is incompatible with device") + torch.cuda.set_per_process_memory_fraction(fraction, device=index) + probe = torch.empty((1,), dtype=torch.float32, device=device) + del probe + torch.cuda.synchronize(device) + torch.cuda.empty_cache() + initial_allocated = int(torch.cuda.memory_allocated(index)) + initial_reserved = int(torch.cuda.memory_reserved(index)) + if initial_allocated != 0 or initial_reserved != 0: + raise BaseCohortLatencyError("latency process has preexisting CUDA allocations") + cudnn = torch.backends.cudnn.version() + if type(cudnn) is not int or cudnn < 1 or type(torch.version.cuda) is not str: + raise BaseCohortLatencyError("Torch CUDA runtime identity is incomplete") + return BaseLatencyRuntimeIdentity( + device=str(torch.device("cuda", index)), + device_uuid=self._uuid, + device_name=str(properties.name), + torch_version=str(torch.__version__), + cuda_runtime_version=torch.version.cuda, + cudnn_version=cudnn, + driver_version=admission_observation.driver_version, + nvidia_total_memory_mib=admission_observation.memory_total_mib, + compute_mode=admission_observation.compute_mode, + torch_total_memory_bytes=int(properties.total_memory), + initial_allocated_bytes=initial_allocated, + initial_reserved_bytes=initial_reserved, + environment_sha256=training_module.training_environment_sha256(), + precision_mode=precision_mode, + ) + + def synchronize(self, device: torch.device) -> None: + torch.cuda.synchronize(device) + + def perf_counter_ns(self) -> int: + return time.perf_counter_ns() + + def reset_peak_memory(self, device: torch.device) -> None: + torch.cuda.reset_peak_memory_stats(device) + + def peak_memory(self, device: torch.device) -> tuple[int, int]: + return ( + int(torch.cuda.max_memory_allocated(device)), + int(torch.cuda.max_memory_reserved(device)), + ) + + def place(self, system: training_module.BaseRetrievalSystem, device: torch.device) -> None: + system.to(device) + + def encode( + self, + system: training_module.BaseRetrievalSystem, + groups: PreparedGroupBatch, + ) -> object: + return system.encode_trainable(groups) + + def validate_output(self, output: object, device: torch.device) -> None: + _validate_output(output, device) + + def release(self, system: training_module.BaseRetrievalSystem, device: torch.device) -> None: + system.to(torch.device("cpu")) + torch.cuda.synchronize(device) + try: + clear_cublas_workspaces = torch._C._cuda_clearCublasWorkspaces + except AttributeError as error: + raise BaseCohortLatencyError( + "Torch CUDA cuBLAS workspace clear API is unavailable" + ) from error + if not callable(clear_cublas_workspaces): + raise BaseCohortLatencyError( + "Torch CUDA cuBLAS workspace clear API is unavailable" + ) + try: + clear_cublas_workspaces() + except Exception as error: + raise BaseCohortLatencyError( + "Torch CUDA cuBLAS workspace clear failed" + ) from error + torch.cuda.synchronize(device) + torch.cuda.empty_cache() + if torch.cuda.memory_allocated(device) != 0 or torch.cuda.memory_reserved(device) != 0: + raise BaseCohortLatencyError( + "measured model CUDA allocation remained after visit release" + ) + + +def cyclic_visit_schedule() -> tuple[tuple[str, ...], ...]: + rows = base_run_ids() + if len(rows) != ROWS_PER_ROUND: + raise AssertionError("registered base row count changed") + return tuple( + tuple(rows[(round_index + ordinal) % len(rows)] for ordinal in range(len(rows))) + for round_index in range(ROUND_COUNT) + ) + + +def _array_row(name: str, value: np.ndarray) -> tuple[str, str, tuple[int, ...], str]: + return ( + name, + _sha256(value.tobytes(order="C")), + tuple(int(axis) for axis in value.shape), + value.dtype.str, + ) + + +def _source_actor_commitments() -> tuple[bytes, ...]: + return tuple( + hashlib.sha256(_ACTOR_DOMAIN + struct.pack(">I", actor)).digest() + for actor in range(FIXTURE_ACTORS) + ) + + +def build_frozen_latency_workload() -> tuple[PreparedGroupBatch, BaseLatencyWorkloadReceipt]: + actors = np.arange(FIXTURE_ACTORS, dtype=np.int64).reshape(1, FIXTURE_ACTORS, 1, 1, 1) + frames = np.arange(FIXTURE_FRAMES, dtype=np.int64).reshape(1, 1, FIXTURE_FRAMES, 1, 1) + joints = np.arange(FIXTURE_JOINTS, dtype=np.int64).reshape(1, 1, 1, FIXTURE_JOINTS, 1) + coordinates = np.arange(FIXTURE_COORDINATES, dtype=np.int64).reshape( + 1, 1, 1, 1, FIXTURE_COORDINATES + ) + raw = ((actors * 131 + frames * 17 + joints * 7 + coordinates * 3) % 2048) - 1024 + skeletons = np.ascontiguousarray(raw.astype(np.float32) * np.float32(2**-10)) + actor_mask = np.ones((FIXTURE_BATCH, FIXTURE_ACTORS), dtype=np.bool_) + frame_mask = np.ones((FIXTURE_BATCH, FIXTURE_FRAMES), dtype=np.bool_) + track_mask = np.ones( + (FIXTURE_BATCH, FIXTURE_ACTORS, FIXTURE_FRAMES, FIXTURE_JOINTS), + dtype=np.bool_, + ) + commitments = _source_actor_commitments() + group = group_commitment(commitments) + batch = PreparedGroupBatch( + skeletons, + actor_mask, + frame_mask, + track_mask, + (commitments,), + (group,), + ) + receipt = BaseLatencyWorkloadReceipt( + array_rows=tuple( + _array_row(name, value) + for name, value in ( + ("skeletons", batch.skeletons), + ("actor_mask", batch.actor_mask), + ("frame_mask", batch.frame_mask), + ("track_mask", batch.track_mask), + ) + ), + raw_skeletons_sha256=_sha256(skeletons.tobytes(order="C")), + actor_commitments=tuple(value for value in batch.actor_commitments[0] if value is not None), + group_commitment=batch.group_commitments[0], + ) + return batch, receipt + + +def _workload_receipt(groups: PreparedGroupBatch) -> BaseLatencyWorkloadReceipt: + commitments = tuple(value for value in groups.actor_commitments[0] if value is not None) + source_commitments = _source_actor_commitments() + if set(commitments) != set(source_commitments): + raise BaseCohortLatencyError("latency workload actor census changed") + source_order = tuple(commitments.index(value) for value in source_commitments) + raw_skeletons = np.ascontiguousarray(groups.skeletons[:, source_order]) + return BaseLatencyWorkloadReceipt( + array_rows=tuple( + _array_row(name, value) + for name, value in ( + ("skeletons", groups.skeletons), + ("actor_mask", groups.actor_mask), + ("frame_mask", groups.frame_mask), + ("track_mask", groups.track_mask), + ) + ), + raw_skeletons_sha256=_sha256(raw_skeletons.tobytes(order="C")), + actor_commitments=commitments, + group_commitment=groups.group_commitments[0], + ) + + +def _resolved_cohort_sha256(cohort: resolver_module.ResolvedBaseCohort) -> str: + return _canonical_digest( + { + "matrix_sha256": cohort.matrix_sha256, + "plan_sha256": cohort.plan_sha256, + "rows": [ + { + "attempt_id": row.attempt_id, + "failed_predecessors": [ + { + "attempt_id": failed.attempt_id, + "failure_code": failed.failure_code, + "terminal_outcome": failed.terminal_outcome, + "terminal_sha256": failed.terminal_sha256, + } + for failed in row.failed_predecessors + ], + "latest_checkpoint_name": row.latest_checkpoint_name, + "latest_checkpoint_sha256": row.latest_checkpoint_sha256, + "run_id": row.run_id, + "selected_checkpoint_epoch": row.selected_checkpoint.epoch, + "selected_checkpoint_global_step": row.selected_checkpoint.global_step, + "selected_checkpoint_name": row.selected_checkpoint.name, + "selected_checkpoint_sha256": row.selected_checkpoint.sha256, + "selected_checkpoint_state_digest": row.selected_checkpoint.state_digest, + "terminal_sha256": row.terminal_sha256, + } + for row in cohort.rows + ], + "source_tree_sha256": cohort.source_tree_sha256, + "training_config_sha256": cohort.training_config_sha256, + "train_manifest_sha256": cohort.train_manifest_sha256, + "val_manifest_sha256": cohort.val_manifest_sha256, + } + ) + + +def _checked_inputs( + cohort: object, + admission: object, + scored_cohort: object, +) -> tuple[ + resolver_module.ResolvedBaseCohort, + resolver_module.BaseCohortAdmission, + scoring_module.BaseCohortValidationResult, +]: + try: + checked_admission = scoring_module._checked_admission(admission) + checked_cohort = scoring_module._checked_cohort(cohort, checked_admission) + except (TypeError, ValueError, RuntimeError) as error: + raise BaseCohortLatencyError("resolved base cohort admission is invalid") from error + if type(scored_cohort) is not scoring_module.BaseCohortValidationResult: + raise BaseCohortLatencyError("score input must be exact BaseCohortValidationResult") + if ( + type(scored_cohort.authority) is not int + or scored_cohort.authority != scoring_module.AUTHORITY + or scored_cohort.production is not scoring_module.PRODUCTION + or scored_cohort.result_claimed is not scoring_module.RESULT_CLAIMED + or scored_cohort.status != scoring_module.COHORT_STATUS + ): + raise BaseCohortLatencyError("score cohort header is invalid") + expected = { + "plan_sha256": checked_cohort.plan_sha256, + "matrix_sha256": checked_cohort.matrix_sha256, + "training_config_sha256": checked_cohort.training_config_sha256, + "source_tree_sha256": checked_cohort.source_tree_sha256, + "train_manifest_sha256": checked_cohort.train_manifest_sha256, + "query_census_sha256": checked_cohort.val_manifest_sha256, + } + if any(getattr(scored_cohort, name) != value for name, value in expected.items()): + raise BaseCohortLatencyError("score cohort differs from resolved cohort admission") + if tuple(row.run_id for row in scored_cohort.observations) != base_run_ids(): + raise BaseCohortLatencyError("score cohort lost the registered nine-row order") + for resolved, scored in zip(checked_cohort.rows, scored_cohort.observations, strict=True): + if ( + scored.run_id != resolved.run_id + or scored.attempt_id != resolved.attempt_id + or scored.system_id != resolved.system_id + or scored.seed != resolved.seed + or scored.terminal_sha256 != resolved.terminal_sha256 + or scored.selected_checkpoint_sha256 != resolved.selected_checkpoint.sha256 + or scored.selected_checkpoint_state_digest != resolved.selected_checkpoint.state_digest + ): + raise BaseCohortLatencyError("score row differs from resolved selected-best row") + return checked_cohort, checked_admission, scored_cohort + + +def _checked_scorer_runtime_device( + *, + config_device: str, + score_device: str, + runtime_device: torch.device, +) -> None: + try: + configured_device = torch.device(config_device) + except (TypeError, ValueError, RuntimeError) as error: + raise BaseCohortLatencyError("configured latency device is invalid") from error + if configured_device.type == "cuda" and configured_device.index is None: + configured_device = torch.device("cuda", torch.cuda.current_device()) + if ( + configured_device.type != "cuda" + or runtime_device.type != "cuda" + or configured_device != runtime_device + or type(score_device) is not str + or score_device != str(configured_device) + ): + raise BaseCohortLatencyError( + "score device differs from the resolved physical latency device" + ) + + +def _load_selected_system( + row: resolver_module.ResolvedBaseRun, + score: scoring_module.BaseValidationScoreObservation, + admission: resolver_module.BaseCohortAdmission, + *, + runtime_device: torch.device, +) -> tuple[ + training_module.BaseRetrievalSystem, + training_module.TrainingConfig, + training_module.PrecisionDecision, + scoring_module.BaseParameterCensus, +]: + payload = scoring_module._validate_selected_payload(row, admission) + config = training_module.TrainingConfig( + stage="base", + seed=row.seed, + device=admission.device, + request_bf16=admission.request_bf16, + bf16_runtime_qualified=admission.bf16_runtime_qualified, + edge_budget=admission.edge_budget, + checkpoint_every_updates=admission.checkpoint_every_updates, + synthetic_contract=False, + ) + try: + system, binding = training_module.construct_registered_base_seed_bound_system( + row.system_id, + config, + ) + except (TypeError, ValueError, RuntimeError) as error: + raise BaseCohortLatencyError("registered base reconstruction failed") from error + expected_bindings = { + "behavior_sha256": binding.behavior_sha256, + "code_artifact_sha256": binding.code_artifact_sha256, + "environment_sha256": binding.environment_sha256, + "factory_sha256": binding.factory_sha256, + "initial_optimizable_state_sha256": binding.initial_optimizable_state_sha256, + "initialization_binding_sha256": binding.sha256, + } + precision = training_module.resolve_precision(config) + _checked_scorer_runtime_device( + config_device=config.device, + score_device=score.device, + runtime_device=runtime_device, + ) + if any(payload.get(name) != expected for name, expected in expected_bindings.items()): + raise BaseCohortLatencyError("selected checkpoint initialization binding changed") + if ( + score.initialization_binding_sha256 != binding.sha256 + or score.factory_sha256 != binding.factory_sha256 + or score.code_artifact_sha256 != binding.code_artifact_sha256 + or score.environment_sha256 != binding.environment_sha256 + or score.behavior_sha256 != binding.behavior_sha256 + or score.runtime_config_sha256 != config.sha256 + or score.precision_mode != precision.mode + ): + raise BaseCohortLatencyError("score runtime binding differs from latency reconstruction") + model = payload.get("model") + if not isinstance(model, Mapping): + raise BaseCohortLatencyError("selected checkpoint model is malformed") + try: + system.load_state_dict(model, strict=True) + except (KeyError, TypeError, ValueError, RuntimeError) as error: + raise BaseCohortLatencyError("selected model state could not be loaded") from error + if ( + training_module.training_system_state_sha256(system) != training_module._stable_hash(model) + or training_module.training_system_behavior_sha256(system) != binding.behavior_sha256 + ): + raise BaseCohortLatencyError("loaded selected model state or behavior changed") + census = scoring_module._parameter_census(system) + if census != score.parameter_census: + raise BaseCohortLatencyError("latency live parameter census differs from score census") + return system, config, precision, census + + +def _validate_snapshot( + snapshot: SharedCudaObservation, + *, + registered_uuid: str, + runtime: BaseLatencyRuntimeIdentity | None, +) -> None: + if type(snapshot) is not SharedCudaObservation or snapshot.device_uuid != registered_uuid: + raise BaseCohortLatencyError("shared CUDA observation changed device UUID") + if runtime is not None and ( + snapshot.driver_version != runtime.driver_version + or snapshot.memory_total_mib != runtime.nvidia_total_memory_mib + or snapshot.compute_mode != runtime.compute_mode + ): + raise BaseCohortLatencyError("shared CUDA runtime identity changed") + + +def _validate_output(output: object, device: torch.device) -> torch.Tensor: + if ( + type(output) is not torch.Tensor + or output.shape != (FIXTURE_BATCH, FIXTURE_EMBEDDING_DIM) + or output.dtype != torch.float32 + or output.device != device + or not output.is_contiguous() + or output.requires_grad + or output.grad_fn is not None + or not bool(torch.isfinite(output).all().item()) + ): + raise BaseCohortLatencyError("timed base embedding is malformed or nonfinite") + return output + + +def _forward_once( + system: training_module.BaseRetrievalSystem, + groups: PreparedGroupBatch, + *, + precision: training_module.PrecisionDecision, + device: torch.device, + runtime: _LatencyRuntime, + timed: bool, +) -> int | None: + runtime.synchronize(device) + start = runtime.perf_counter_ns() if timed else None + with training_module._autocast_context(precision, device): + output = runtime.encode(system, groups) + runtime.validate_output(output, device) + runtime.synchronize(device) + stop = runtime.perf_counter_ns() if timed else None + del output + if start is None or stop is None: + return None + if type(start) is not int or type(stop) is not int or stop <= start: + raise BaseCohortLatencyError("host monotonic latency timer is invalid") + return stop - start + + +def _row_observations( + visits: tuple[BaseLatencyVisit, ...], + cohort: resolver_module.ResolvedBaseCohort, + scored: scoring_module.BaseCohortValidationResult, + runtime: BaseLatencyRuntimeIdentity, + workload: BaseLatencyWorkloadReceipt, +) -> tuple[BaseLatencyRowObservation, ...]: + rows: list[BaseLatencyRowObservation] = [] + for resolved, score in zip(cohort.rows, scored.observations, strict=True): + samples = tuple( + sample + for visit in visits + if visit.run_id == resolved.run_id + for sample in visit.samples_ns + ) + ordered = tuple(sorted(samples)) + median = ordered[len(ordered) // 2] + deviations = tuple(sorted(abs(value - median) for value in samples)) + rows.append( + BaseLatencyRowObservation( + run_id=resolved.run_id, + system_id=resolved.system_id, + seed=resolved.seed, + selected_checkpoint_sha256=resolved.selected_checkpoint.sha256, + selected_checkpoint_state_digest=resolved.selected_checkpoint.state_digest, + parameter_census_sha256=score.parameter_census.sha256, + parameter_count=score.parameter_census.total, + runtime_sha256=runtime.sha256, + workload_sha256=workload.sha256, + samples_ns=samples, + median_ns=median, + median_absolute_deviation_ns=deviations[len(deviations) // 2], + minimum_ns=ordered[0], + maximum_ns=ordered[-1], + p05_nearest_rank_ns=_nearest_rank(ordered, 5), + p95_nearest_rank_ns=_nearest_rank(ordered, 95), + ) + ) + return tuple(rows) + + +def _failure_code(error: BaseException) -> str: + chain: list[BaseException] = [] + seen: set[int] = set() + current: BaseException | None = error + while current is not None and id(current) not in seen: + seen.add(id(current)) + chain.append(current) + current = current.__cause__ + if any(isinstance(item, torch.cuda.OutOfMemoryError) for item in chain): + return "CUDA_OUT_OF_MEMORY" + if any(isinstance(item, (TimeoutError, subprocess.TimeoutExpired)) for item in chain): + return "SESSION_TIMEOUT" + if isinstance(error, BaseCohortLatencyError): + message = str(error) + if "UUID" in message or "runtime identity" in message or "precision" in message: + return "RUNTIME_DRIFT" + if "allocator" in message or "memory" in message or "free" in message: + return "RESOURCE_LIMIT" + if "checkpoint" in message or "selected" in message or "score" in message: + return "CHECKPOINT_OR_SCORE_DRIFT" + if "embedding" in message or "timer" in message: + return "MALFORMED_MEASUREMENT" + return "UNEXPECTED_SESSION_FAILURE" + + +def run_base_cohort_latency_session( + cohort: resolver_module.ResolvedBaseCohort, + *, + admission: resolver_module.BaseCohortAdmission, + scored_cohort: scoring_module.BaseCohortValidationResult, + registered_cuda_uuid: str, + observer: LatencySessionObserver | None = None, + _runtime: _LatencyRuntime | None = None, +) -> BaseCohortLatencySession: + """Measure all nine selected-best rows; no latency value is caller supplied. + + ``_runtime`` exists solely for deterministic software contract tests. Any + injected runtime is permanently marked non-formal in the returned object. + """ + + checked_cohort, checked_admission, checked_scores = _checked_inputs( + cohort, + admission, + scored_cohort, + ) + _prefixed_cuda_uuid(registered_cuda_uuid, "registered_cuda_uuid") + if observer is not None and not isinstance(observer, LatencySessionObserver): + raise BaseCohortLatencyError("latency observer does not implement the typed interface") + device = torch.device(checked_admission.device) + if device.type != "cuda": + raise BaseCohortLatencyError("formal base latency requires the admitted CUDA device") + decisions = tuple( + training_module.resolve_precision( + training_module.TrainingConfig( + stage="base", + seed=row.seed, + device=checked_admission.device, + request_bf16=checked_admission.request_bf16, + bf16_runtime_qualified=checked_admission.bf16_runtime_qualified, + edge_budget=checked_admission.edge_budget, + checkpoint_every_updates=checked_admission.checkpoint_every_updates, + synthetic_contract=False, + ) + ) + for row in checked_cohort.rows + ) + if len({decision.mode for decision in decisions}) != 1: + raise BaseCohortLatencyError("latency precision must be session-wide") + precision_mode = decisions[0].mode + groups, workload = build_frozen_latency_workload() + if checked_admission.edge_budget < FIXTURE_ACTORS * (FIXTURE_ACTORS - 1) // 2: + raise BaseCohortLatencyError("admitted edge budget cannot contain the K=32 workload") + actual_runtime = _runtime is None + runtime: _LatencyRuntime = ( + _NvidiaSmiTorchRuntime(registered_cuda_uuid) if _runtime is None else _runtime + ) + resolved_sha = _resolved_cohort_sha256(checked_cohort) + scored_sha = checked_scores.sha256 + admission_observation = runtime.snapshot() + if type(admission_observation) is not SharedCudaObservation: + raise BaseCohortLatencyError("pre-session CUDA observation is malformed") + visits: list[BaseLatencyVisit] = [] + runtime_identity: BaseLatencyRuntimeIdentity | None = None + post_context: SharedCudaObservation | None = None + observer_failed = False + + def observe(method_name: str, **payload: object) -> None: + nonlocal observer_failed + if observer is None: + return + try: + getattr(observer, method_name)(**payload) + except Exception as error: + observer_failed = True + raise _LatencyObserverFailure( + f"latency observer failed during {method_name}" + ) from error + + def held(error: BaseException) -> BaseCohortLatencyHold: + code = _failure_code(error) + receipt = BaseCohortLatencySession( + registered_cuda_uuid=registered_cuda_uuid, + resolved_cohort_sha256=resolved_sha, + scored_cohort_sha256=scored_sha, + workload_receipt=workload, + admission_observation=admission_observation, + post_context_observation=post_context, + runtime_identity=runtime_identity, + visits=tuple(visits), + rows=(), + actual_runtime=actual_runtime, + complete=False, + formal_samples_complete=False, + failure_code=code, + status=HOLD_STATUS, + ) + return BaseCohortLatencyHold(code, receipt) + + def observed_hold(error: BaseException) -> BaseCohortLatencyHold: + session_hold = held(error) + if not observer_failed: + try: + observe( + "terminal", + outcome="HELD", + latency_session_sha256=session_hold.receipt.sha256, + failure_code=session_hold.failure_code, + ) + except _LatencyObserverFailure as observer_error: + raise session_hold from observer_error + return session_hold + + try: + observe( + "session_start", + admission_observation_sha256=_canonical_digest(admission_observation.payload()), + started_unix_ns=time.time_ns(), + ) + _validate_snapshot( + admission_observation, + registered_uuid=registered_cuda_uuid, + runtime=None, + ) + if admission_observation.memory_free_mib < MINIMUM_FREE_MIB: + raise BaseCohortLatencyError("shared CUDA admission has less than 8 GiB free") + runtime_identity = runtime.initialize( + device=device, + precision_mode=precision_mode, + admission_observation=admission_observation, + ) + if type(runtime_identity) is not BaseLatencyRuntimeIdentity: + raise BaseCohortLatencyError("latency runtime initialization receipt is invalid") + device = torch.device(runtime_identity.device) + post_context = runtime.snapshot() + _validate_snapshot( + post_context, + registered_uuid=registered_cuda_uuid, + runtime=runtime_identity, + ) + if post_context.memory_free_mib < MINIMUM_FREE_MIB: + raise BaseCohortLatencyError("shared CUDA context has less than 8 GiB free") + if runtime_identity.precision_mode != precision_mode: + raise BaseCohortLatencyError("latency runtime precision changed") + score_environments = {score.environment_sha256 for score in checked_scores.observations} + if score_environments != {runtime_identity.environment_sha256}: + raise BaseCohortLatencyError("score cohort and latency runtime environment differ") + observe( + "context_ready", + runtime_sha256=runtime_identity.sha256, + post_context_observation_sha256=_canonical_digest(post_context.payload()), + ) + with training_module._frozen_numerical_runtime(device), torch.no_grad(): + for round_index, round_rows in enumerate(cyclic_visit_schedule()): + for ordinal, run_id in enumerate(round_rows): + row_index = base_run_ids().index(run_id) + row = checked_cohort.rows[row_index] + score = checked_scores.observations[row_index] + before = runtime.snapshot() + _validate_snapshot( + before, + registered_uuid=registered_cuda_uuid, + runtime=runtime_identity, + ) + observe( + "visit_started", + round_index=round_index, + ordinal_in_round=ordinal, + run_id=run_id, + before_observation_sha256=_canonical_digest(before.payload()), + ) + samples: list[int] = [] + warmups_completed = 0 + peaks: tuple[int, int] | None = None + after: SharedCudaObservation | None = None + system: training_module.BaseRetrievalSystem | None = None + visit_error: BaseException | None = None + try: + if ( + _resolved_cohort_sha256(checked_cohort) != resolved_sha + or checked_scores.sha256 != scored_sha + or _workload_receipt(groups) != workload + or _sha256(row.selected_checkpoint.raw) + != row.selected_checkpoint.sha256 + ): + raise BaseCohortLatencyError("session input changed between visits") + system, _config, precision, census = _load_selected_system( + row, + score, + checked_admission, + runtime_device=device, + ) + if precision.mode != precision_mode or census != score.parameter_census: + raise BaseCohortLatencyError( + "visit precision or parameter census changed" + ) + system.eval() + for parameter in system.parameters(): + parameter.grad = None + parameter.requires_grad_(False) + if any(parameter.requires_grad for parameter in system.parameters()): + raise BaseCohortLatencyError("measured model was not completely frozen") + runtime.reset_peak_memory(device) + runtime.place(system, device) + for warmup_index in range(WARMUPS_PER_VISIT): + _forward_once( + system, + groups, + precision=precision, + device=device, + runtime=runtime, + timed=False, + ) + warmups_completed += 1 + observe( + "warmup_completed", + round_index=round_index, + ordinal_in_round=ordinal, + run_id=run_id, + warmup_index=warmup_index, + ) + for sample_index in range(TIMED_PER_VISIT): + sample = _forward_once( + system, + groups, + precision=precision, + device=device, + runtime=runtime, + timed=True, + ) + if type(sample) is not int: + raise AssertionError("timed forward did not produce an integer") + samples.append(sample) + observe( + "timed_sample_completed", + round_index=round_index, + ordinal_in_round=ordinal, + run_id=run_id, + sample_index=sample_index, + sample_ns=sample, + ) + peaks = runtime.peak_memory(device) + if ( + type(peaks) is not tuple + or len(peaks) != 2 + or any(type(value) is not int or value < 0 for value in peaks) + or max(peaks) > CUDA_ALLOCATOR_LIMIT_BYTES + ): + raise BaseCohortLatencyError("CUDA allocator peak exceeded 2 GiB") + if any(parameter.grad is not None for parameter in system.parameters()): + raise BaseCohortLatencyError("latency measurement created a gradient") + if _sha256(row.selected_checkpoint.raw) != row.selected_checkpoint.sha256: + raise BaseCohortLatencyError("selected checkpoint changed during visit") + except Exception as error: + visit_error = error + finally: + if system is not None: + if peaks is None: + try: + peaks = runtime.peak_memory(device) + except Exception as error: + if visit_error is None: + visit_error = error + try: + runtime.release(system, device) + except Exception as error: + if visit_error is None: + visit_error = error + try: + after = runtime.snapshot() + except Exception as error: + if visit_error is None: + visit_error = error + else: + try: + _validate_snapshot( + after, + registered_uuid=registered_cuda_uuid, + runtime=runtime_identity, + ) + except Exception as error: + if visit_error is None: + visit_error = error + if visit_error is not None: + code = _failure_code(visit_error) + valid_peaks = ( + type(peaks) is tuple + and len(peaks) == 2 + and all(type(value) is int and value >= 0 for value in peaks) + ) + failed_visit = BaseLatencyVisit( + round_index=round_index, + ordinal_in_round=ordinal, + run_id=run_id, + before=before, + after=after, + samples_ns=tuple(samples), + peak_allocated_bytes=peaks[0] if valid_peaks else None, + peak_reserved_bytes=peaks[1] if valid_peaks else None, + completed=False, + failure_code=code, + warmups_completed=warmups_completed, + ) + visits.append(failed_visit) + if not observer_failed: + observe( + "visit_ended", + round_index=round_index, + ordinal_in_round=ordinal, + run_id=run_id, + outcome="HELD", + after_observation_sha256=( + None if after is None else _canonical_digest(after.payload()) + ), + peak_allocated_bytes=failed_visit.peak_allocated_bytes, + peak_reserved_bytes=failed_visit.peak_reserved_bytes, + failure_code=code, + ) + raise visit_error + completed_visit = BaseLatencyVisit( + round_index=round_index, + ordinal_in_round=ordinal, + run_id=run_id, + before=before, + after=after, + samples_ns=tuple(samples), + peak_allocated_bytes=peaks[0], + peak_reserved_bytes=peaks[1], + completed=True, + failure_code=None, + warmups_completed=warmups_completed, + ) + visits.append(completed_visit) + observe( + "visit_ended", + round_index=round_index, + ordinal_in_round=ordinal, + run_id=run_id, + outcome="COMPLETED", + after_observation_sha256=_canonical_digest(after.payload()), + peak_allocated_bytes=completed_visit.peak_allocated_bytes, + peak_reserved_bytes=completed_visit.peak_reserved_bytes, + failure_code=None, + ) + except BaseCohortLatencyHold: + raise + except Exception as error: + raise observed_hold(error) from error + if runtime_identity is None or post_context is None: + raise AssertionError("completed session lost its runtime admission") + visit_tuple = tuple(visits) + try: + rows = _row_observations( + visit_tuple, + checked_cohort, + checked_scores, + runtime_identity, + workload, + ) + if ( + _resolved_cohort_sha256(checked_cohort) != resolved_sha + or checked_scores.sha256 != scored_sha + or _workload_receipt(groups) != workload + ): + raise BaseCohortLatencyError("session input changed after measurement") + except Exception as error: + raise observed_hold(error) from error + status = ACTUAL_COMPLETE_STATUS if actual_runtime else FAKE_COMPLETE_STATUS + result = BaseCohortLatencySession( + registered_cuda_uuid=registered_cuda_uuid, + resolved_cohort_sha256=resolved_sha, + scored_cohort_sha256=scored_sha, + workload_receipt=workload, + admission_observation=admission_observation, + post_context_observation=post_context, + runtime_identity=runtime_identity, + visits=visit_tuple, + rows=rows, + actual_runtime=actual_runtime, + complete=True, + formal_samples_complete=actual_runtime, + failure_code=None, + status=status, + ) + try: + observe( + "terminal", + outcome="COMPLETED", + latency_session_sha256=result.sha256, + failure_code=None, + ) + except _LatencyObserverFailure as error: + raise held(error) from error + return result + + +__all__ = [ + "ACTUAL_COMPLETE_STATUS", + "AUTHORITY", + "BaseCohortLatencyError", + "BaseCohortLatencyHold", + "BaseCohortLatencySession", + "BaseLatencyRowObservation", + "BaseLatencyRuntimeIdentity", + "BaseLatencyVisit", + "BaseLatencyWorkloadReceipt", + "CUDA_ALLOCATOR_LIMIT_BYTES", + "FAKE_COMPLETE_STATUS", + "HOLD_STATUS", + "MINIMUM_FREE_MIB", + "LatencySessionObserver", + "ROUND_COUNT", + "SAMPLES_PER_ROW", + "SharedCudaObservation", + "TIMED_PER_VISIT", + "WARMUPS_PER_VISIT", + "build_frozen_latency_workload", + "cyclic_visit_schedule", + "run_base_cohort_latency_session", +] diff --git a/src/phaseset_core/base_cohort_qualification.py b/src/phaseset_core/base_cohort_qualification.py new file mode 100644 index 0000000..7e35aed --- /dev/null +++ b/src/phaseset_core/base_cohort_qualification.py @@ -0,0 +1,666 @@ +"""Assemble one verified nine-row base cohort into the existing selector. + +This module performs no scoring, latency measurement, training, winner override, +or authorization. It accepts only the resolver, scorer, and formal-latency +objects, closes their shared bindings, derives every :class:`BaseScore` field, +and invokes the already installed deterministic qualification path. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from fractions import Fraction +import hashlib +import json +from pathlib import Path +from types import ModuleType +from typing import Final + +from . import base_cohort_latency as latency_module +from . import base_cohort_resolver as resolver_module +from . import base_cohort_validation as scoring_module +from . import experiments as experiments_module +from . import production as production_module + + +AUTHORITY: Final = 0 +PRODUCTION: Final = False +RESULT_CLAIMED: Final = False +ROW_BINDING_SCHEMA: Final = "phaseset-base-qualification-combined-row-v1" +DEPENDENCY_SCHEMA: Final = "phaseset-base-qualification-dependencies-v1" +ASSEMBLY_SCHEMA: Final = "phaseset-base-cohort-qualification-assembly-v1" +STATUS: Final = "BASE_QUALIFICATION_ASSEMBLED_NO_EXTERNAL_AUTHORIZATION" + + +class BaseCohortQualificationError(ValueError): + """The three input evidence families cannot form one qualification.""" + + +def _sha256(raw: bytes) -> str: + return hashlib.sha256(raw).hexdigest() + + +def _lower_sha256(value: object, label: str) -> str: + if ( + type(value) is not str + or len(value) != 64 + or value.lower() != value + or any(character not in "0123456789abcdef" for character in value) + ): + raise BaseCohortQualificationError(f"{label} must be lowercase SHA-256 hex") + return value + + +def _positive_int(value: object, label: str) -> int: + if type(value) is not int or not 1 <= value <= (1 << 63) - 1: + raise BaseCohortQualificationError(f"{label} must be an exact positive int") + return value + + +def _canonical_json_bytes(value: object) -> bytes: + return ( + json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + allow_nan=False, + ).encode("ascii") + + b"\n" + ) + + +def _canonical_digest(value: object) -> str: + return _sha256(_canonical_json_bytes(value)) + + +def _fraction_payload(value: Fraction) -> dict[str, int]: + if type(value) is not Fraction or not Fraction(0, 1) <= value <= Fraction(1, 1): + raise BaseCohortQualificationError("base validation score must be an exact fraction") + return {"denominator": value.denominator, "numerator": value.numerator} + + +def _source_bytes(module: ModuleType, label: str) -> bytes: + module_file = getattr(module, "__file__", None) + if type(module_file) is not str: + raise BaseCohortQualificationError(f"installed {label} source path is unavailable") + path = Path(module_file) + if path.suffix in {".pyc", ".pyo"}: + path = path.with_suffix(".py") + if path.is_symlink() or not path.is_file(): + raise BaseCohortQualificationError(f"installed {label} source is irregular") + raw = path.read_bytes() + if not raw: + raise BaseCohortQualificationError(f"installed {label} source is empty") + return raw + + +def _admission_sha256(value: resolver_module.BaseCohortAdmission) -> str: + return _canonical_digest( + { + "bf16_runtime_qualified": value.bf16_runtime_qualified, + "checkpoint_every_updates": value.checkpoint_every_updates, + "device": value.device, + "edge_budget": value.edge_budget, + "matrix_sha256": value.matrix_sha256, + "plan_sha256": value.plan_sha256, + "request_bf16": value.request_bf16, + "source_tree_sha256": value.source_tree_sha256, + "train_manifest_sha256": value.train_manifest_sha256, + "training_config_sha256": value.training_config_sha256, + "val_manifest_sha256": value.val_manifest_sha256, + } + ) + + +def _samples_sha256(samples: tuple[int, ...]) -> str: + if type(samples) is not tuple or len(samples) != latency_module.SAMPLES_PER_ROW: + raise BaseCohortQualificationError("latency row must contain exact 99 samples") + for value in samples: + _positive_int(value, "latency sample") + return _canonical_digest( + { + "samples_ns": list(samples), + "schema": "phaseset-base-latency-raw-samples-v1", + } + ) + + +def _median_from_samples(samples: tuple[int, ...]) -> int: + _samples_sha256(samples) + ordered = tuple(sorted(samples)) + return ordered[len(ordered) // 2] + + +def _visit_samples_for_run( + latency: latency_module.BaseCohortLatencySession, + run_id: str, +) -> tuple[int, ...]: + matching = tuple(visit for visit in latency.visits if visit.run_id == run_id) + if len(matching) != latency_module.ROUND_COUNT or any( + visit.completed is not True or len(visit.samples_ns) != latency_module.TIMED_PER_VISIT + for visit in matching + ): + raise BaseCohortQualificationError( + "latency visit census is incomplete for a qualification row" + ) + samples = tuple(sample for visit in matching for sample in visit.samples_ns) + _samples_sha256(samples) + return samples + + +@dataclass(frozen=True, slots=True) +class BaseQualificationDependencyBinding: + admission_sha256: str + resolved_cohort_sha256: str + scored_cohort_sha256: str + latency_session_sha256: str + plan_sha256: str + matrix_sha256: str + training_config_sha256: str + source_tree_sha256: str + train_manifest_sha256: str + admitted_val_census_sha256: str + validation_source_manifest_sha256: str + query_census_sha256: str + resolver_source_sha256: str + scorer_source_sha256: str + latency_source_sha256: str + assembler_source_sha256: str + production_source_sha256: str + selector_artifact_sha256: str + runtime_sha256: str + environment_sha256: str + precision_mode: str + schema: str = DEPENDENCY_SCHEMA + + def __post_init__(self) -> None: + for name in ( + "admission_sha256", + "resolved_cohort_sha256", + "scored_cohort_sha256", + "latency_session_sha256", + "plan_sha256", + "matrix_sha256", + "training_config_sha256", + "source_tree_sha256", + "train_manifest_sha256", + "admitted_val_census_sha256", + "validation_source_manifest_sha256", + "query_census_sha256", + "resolver_source_sha256", + "scorer_source_sha256", + "latency_source_sha256", + "assembler_source_sha256", + "production_source_sha256", + "selector_artifact_sha256", + "runtime_sha256", + "environment_sha256", + ): + _lower_sha256(getattr(self, name), name) + if self.admitted_val_census_sha256 != self.query_census_sha256: + raise BaseCohortQualificationError( + "admitted validation census differs from scored query census" + ) + if self.precision_mode not in ("FP32", "BF16"): + raise BaseCohortQualificationError("qualification precision mode is invalid") + if self.schema != DEPENDENCY_SCHEMA: + raise BaseCohortQualificationError("qualification dependency schema changed") + + @property + def sha256(self) -> str: + return _canonical_digest({name: getattr(self, name) for name in self.__dataclass_fields__}) + + +@dataclass(frozen=True, slots=True) +class CombinedBaseScoreBinding: + run_id: str + attempt_id: str + system_id: str + seed: int + terminal_sha256: str + selected_checkpoint_sha256: str + selected_checkpoint_state_digest: str + validation_score_artifact_sha256: str + latency_row_sha256: str + latency_samples_sha256: str + parameter_census_sha256: str + parameter_count: int + frozen_runtime_latency_ns: int + bidirectional_capture_r1: Fraction + validation_manifest_sha256: str + validation_source_manifest_sha256: str + query_census_sha256: str + evaluator_sha256: str + runtime_sha256: str + environment_sha256: str + precision_mode: str + schema: str = ROW_BINDING_SCHEMA + + def __post_init__(self) -> None: + role, seed, system_id = experiments_module.parse_run_id(self.run_id) + if role != "BASE_QUALIFICATION" or seed != self.seed or system_id != self.system_id: + raise BaseCohortQualificationError("combined base row identity is inconsistent") + if type(self.attempt_id) is not str or not self.attempt_id: + raise BaseCohortQualificationError("combined base attempt ID is invalid") + for name in ( + "terminal_sha256", + "selected_checkpoint_sha256", + "selected_checkpoint_state_digest", + "validation_score_artifact_sha256", + "latency_row_sha256", + "latency_samples_sha256", + "parameter_census_sha256", + "validation_manifest_sha256", + "validation_source_manifest_sha256", + "query_census_sha256", + "evaluator_sha256", + "runtime_sha256", + "environment_sha256", + ): + _lower_sha256(getattr(self, name), name) + _positive_int(self.parameter_count, "parameter_count") + _positive_int(self.frozen_runtime_latency_ns, "frozen_runtime_latency_ns") + _fraction_payload(self.bidirectional_capture_r1) + if self.precision_mode not in ("FP32", "BF16"): + raise BaseCohortQualificationError("combined row precision mode is invalid") + if self.schema != ROW_BINDING_SCHEMA: + raise BaseCohortQualificationError("combined row schema changed") + + def canonical_bytes(self) -> bytes: + return _canonical_json_bytes( + { + "attempt_id": self.attempt_id, + "bidirectional_capture_r1": _fraction_payload(self.bidirectional_capture_r1), + "environment_sha256": self.environment_sha256, + "evaluator_sha256": self.evaluator_sha256, + "frozen_runtime_latency_ns": self.frozen_runtime_latency_ns, + "latency_row_sha256": self.latency_row_sha256, + "latency_samples_sha256": self.latency_samples_sha256, + "parameter_census_sha256": self.parameter_census_sha256, + "parameter_count": self.parameter_count, + "precision_mode": self.precision_mode, + "query_census_sha256": self.query_census_sha256, + "run_id": self.run_id, + "runtime_sha256": self.runtime_sha256, + "schema": self.schema, + "seed": self.seed, + "selected_checkpoint_sha256": self.selected_checkpoint_sha256, + "selected_checkpoint_state_digest": (self.selected_checkpoint_state_digest), + "system_id": self.system_id, + "terminal_sha256": self.terminal_sha256, + "validation_manifest_sha256": self.validation_manifest_sha256, + "validation_score_artifact_sha256": (self.validation_score_artifact_sha256), + "validation_source_manifest_sha256": (self.validation_source_manifest_sha256), + } + ) + + @property + def sha256(self) -> str: + return _sha256(self.canonical_bytes()) + + +@dataclass(frozen=True, slots=True, repr=False) +class BaseCohortQualificationAssembly: + qualification: experiments_module.BaseQualification + execution_evidence: production_module.BaseQualificationExecutionEvidence + artifact: bytes = field(repr=False) + row_bindings: tuple[CombinedBaseScoreBinding, ...] + dependency_binding: BaseQualificationDependencyBinding + authority: int = AUTHORITY + production: bool = PRODUCTION + result_claimed: bool = RESULT_CLAIMED + schema: str = ASSEMBLY_SCHEMA + status: str = STATUS + + def __post_init__(self) -> None: + if type(self.qualification) is not experiments_module.BaseQualification: + raise BaseCohortQualificationError("assembly qualification type is invalid") + if ( + type(self.execution_evidence) + is not production_module.BaseQualificationExecutionEvidence + ): + raise BaseCohortQualificationError("assembly execution evidence type is invalid") + if type(self.artifact) is not bytes or not self.artifact: + raise BaseCohortQualificationError("assembly artifact must be nonempty bytes") + if ( + type(self.row_bindings) is not tuple + or len(self.row_bindings) != len(experiments_module.base_run_ids()) + or any(type(row) is not CombinedBaseScoreBinding for row in self.row_bindings) + ): + raise BaseCohortQualificationError("assembly combined row census is invalid") + if type(self.dependency_binding) is not BaseQualificationDependencyBinding: + raise BaseCohortQualificationError("assembly dependency binding is invalid") + if tuple(row.run_id for row in self.row_bindings) != experiments_module.base_run_ids(): + raise BaseCohortQualificationError("assembly combined rows changed order") + if self.execution_evidence.score_rows != self.qualification.score_rows: + raise BaseCohortQualificationError("execution evidence differs from qualification rows") + if tuple(row.sha256 for row in self.row_bindings) != tuple( + row.score_artifact_sha256 for row in self.qualification.score_rows + ): + raise BaseCohortQualificationError("combined evidence differs from BaseScore rows") + if _sha256(self.execution_evidence.selector_artifact) != ( + self.dependency_binding.selector_artifact_sha256 + ): + raise BaseCohortQualificationError("execution selector differs from dependencies") + try: + canonical = experiments_module.canonical_base_qualification_bytes(self.qualification) + except (TypeError, ValueError, RuntimeError) as error: + raise BaseCohortQualificationError("assembled qualification is invalid") from error + if self.artifact != canonical: + raise BaseCohortQualificationError("assembly artifact is not canonical qualification") + if ( + type(self.authority) is not int + or self.authority != AUTHORITY + or self.production is not PRODUCTION + or self.result_claimed is not RESULT_CLAIMED + or self.schema != ASSEMBLY_SCHEMA + or self.status != STATUS + ): + raise BaseCohortQualificationError("assembly result header is invalid") + + @property + def sha256(self) -> str: + return _canonical_digest( + { + "artifact_sha256": _sha256(self.artifact), + "authority": self.authority, + "dependency_binding_sha256": self.dependency_binding.sha256, + "production": self.production, + "result_claimed": self.result_claimed, + "row_binding_sha256s": [row.sha256 for row in self.row_bindings], + "schema": self.schema, + "status": self.status, + } + ) + + def __repr__(self) -> str: + return ( + "BaseCohortQualificationAssembly(" + f"winner={self.qualification.winner_system_id!r}, " + f"sha256={self.sha256!r})" + ) + + +def _installed_code_snapshot() -> tuple[dict[str, str], bytes]: + selector_artifact = production_module.current_base_qualification_artifact_bytes() + if type(selector_artifact) is not bytes or not selector_artifact: + raise BaseCohortQualificationError("installed selector artifact is invalid") + sources = { + "resolver_source_sha256": _sha256(_source_bytes(resolver_module, "base cohort resolver")), + "scorer_source_sha256": _sha256(_source_bytes(scoring_module, "base cohort validation")), + "latency_source_sha256": _sha256(_source_bytes(latency_module, "base cohort latency")), + "assembler_source_sha256": _sha256( + _source_bytes( + __import__(__name__, fromlist=["*"]), + "base cohort qualification", + ) + ), + "production_source_sha256": _sha256(_source_bytes(production_module, "production adapter")), + "selector_artifact_sha256": _sha256(selector_artifact), + } + return sources, selector_artifact + + +def _checked_inputs( + cohort: object, + admission: object, + scored_cohort: object, + latency_session: object, +) -> tuple[ + resolver_module.ResolvedBaseCohort, + resolver_module.BaseCohortAdmission, + scoring_module.BaseCohortValidationResult, + latency_module.BaseCohortLatencySession, +]: + try: + checked_cohort, checked_admission, checked_scores = latency_module._checked_inputs( + cohort, + admission, + scored_cohort, + ) + except (TypeError, ValueError, RuntimeError) as error: + raise BaseCohortQualificationError( + "resolver and scorer inputs do not form one admitted cohort" + ) from error + if type(latency_session) is not latency_module.BaseCohortLatencySession: + raise BaseCohortQualificationError("latency input must be exact formal session") + if ( + latency_session.authority != latency_module.AUTHORITY + or latency_session.production is not latency_module.PRODUCTION + or latency_session.result_claimed is not latency_module.RESULT_CLAIMED + or latency_session.status != latency_module.ACTUAL_COMPLETE_STATUS + or latency_session.actual_runtime is not True + or latency_session.complete is not True + or latency_session.formal_samples_complete is not True + or latency_session.failure_code is not None + ): + raise BaseCohortQualificationError("latency session is not one actual complete session") + if ( + latency_session.resolved_cohort_sha256 + != latency_module._resolved_cohort_sha256(checked_cohort) + or latency_session.scored_cohort_sha256 != checked_scores.sha256 + ): + raise BaseCohortQualificationError("latency session differs from resolver or scorer") + runtime = latency_session.runtime_identity + if type(runtime) is not latency_module.BaseLatencyRuntimeIdentity: + raise BaseCohortQualificationError("latency session runtime identity is absent") + if ( + runtime.device_uuid != latency_session.registered_cuda_uuid + or latency_session.admission_observation.device_uuid != runtime.device_uuid + or latency_session.post_context_observation is None + or latency_session.post_context_observation.device_uuid != runtime.device_uuid + ): + raise BaseCohortQualificationError("latency session CUDA identity is inconsistent") + expected_runs = experiments_module.base_run_ids() + if tuple(row.run_id for row in latency_session.rows) != expected_runs: + raise BaseCohortQualificationError("latency session row order is incomplete") + return checked_cohort, checked_admission, checked_scores, latency_session + + +def _build_row_bindings( + cohort: resolver_module.ResolvedBaseCohort, + scored: scoring_module.BaseCohortValidationResult, + latency: latency_module.BaseCohortLatencySession, + evaluator_sha256: str, +) -> tuple[CombinedBaseScoreBinding, ...]: + runtime = latency.runtime_identity + if type(runtime) is not latency_module.BaseLatencyRuntimeIdentity: + raise BaseCohortQualificationError("formal latency runtime is absent") + rows: list[CombinedBaseScoreBinding] = [] + for resolved, score, measured in zip( + cohort.rows, + scored.observations, + latency.rows, + strict=True, + ): + visit_samples = _visit_samples_for_run(latency, resolved.run_id) + samples_sha256 = _samples_sha256(visit_samples) + measured_median = _median_from_samples(visit_samples) + if ( + resolved.run_id != score.run_id + or resolved.run_id != measured.run_id + or resolved.attempt_id != score.attempt_id + or resolved.system_id != score.system_id + or resolved.system_id != measured.system_id + or resolved.seed != score.seed + or resolved.seed != measured.seed + or resolved.terminal_sha256 != score.terminal_sha256 + or resolved.selected_checkpoint.sha256 != score.selected_checkpoint_sha256 + or resolved.selected_checkpoint.sha256 != measured.selected_checkpoint_sha256 + or resolved.selected_checkpoint.state_digest != score.selected_checkpoint_state_digest + or resolved.selected_checkpoint.state_digest + != measured.selected_checkpoint_state_digest + ): + raise BaseCohortQualificationError("qualification row identity or checkpoint changed") + if _sha256(resolved.selected_checkpoint.raw) != resolved.selected_checkpoint.sha256: + raise BaseCohortQualificationError("selected-best checkpoint bytes changed") + if ( + measured.parameter_census_sha256 != score.parameter_census.sha256 + or measured.parameter_count != score.parameter_census.total + or measured.runtime_sha256 != runtime.sha256 + or measured.samples_ns != visit_samples + or measured_median != measured.median_ns + or score.environment_sha256 != runtime.environment_sha256 + or score.precision_mode != runtime.precision_mode + or score.device != runtime.device + or score.admitted_val_census_sha256 != cohort.val_manifest_sha256 + or score.query_census_sha256 != cohort.val_manifest_sha256 + or score.source_manifest_sha256 != scored.source_manifest_sha256 + ): + raise BaseCohortQualificationError( + "qualification row score, parameter, runtime, or census changed" + ) + rows.append( + CombinedBaseScoreBinding( + run_id=resolved.run_id, + attempt_id=resolved.attempt_id, + system_id=resolved.system_id, + seed=resolved.seed, + terminal_sha256=resolved.terminal_sha256, + selected_checkpoint_sha256=resolved.selected_checkpoint.sha256, + selected_checkpoint_state_digest=(resolved.selected_checkpoint.state_digest), + validation_score_artifact_sha256=score.score_artifact_sha256, + latency_row_sha256=measured.sha256, + latency_samples_sha256=samples_sha256, + parameter_census_sha256=score.parameter_census.sha256, + parameter_count=score.parameter_census.total, + frozen_runtime_latency_ns=measured_median, + bidirectional_capture_r1=score.primary_capture_r1, + validation_manifest_sha256=cohort.val_manifest_sha256, + validation_source_manifest_sha256=score.source_manifest_sha256, + query_census_sha256=score.query_census_sha256, + evaluator_sha256=evaluator_sha256, + runtime_sha256=runtime.sha256, + environment_sha256=runtime.environment_sha256, + precision_mode=runtime.precision_mode, + ) + ) + if tuple(row.run_id for row in rows) != experiments_module.base_run_ids(): + raise BaseCohortQualificationError("combined qualification rows are incomplete") + return tuple(rows) + + +def _base_scores( + row_bindings: tuple[CombinedBaseScoreBinding, ...], +) -> tuple[experiments_module.BaseScore, ...]: + return tuple( + experiments_module.BaseScore( + run_id=row.run_id, + bidirectional_r1_numerator=row.bidirectional_capture_r1.numerator, + bidirectional_r1_denominator=row.bidirectional_capture_r1.denominator, + parameter_count=row.parameter_count, + frozen_runtime_latency_ns=row.frozen_runtime_latency_ns, + terminal_sha256=row.terminal_sha256, + selected_checkpoint_sha256=row.selected_checkpoint_sha256, + split="validation", + validation_manifest_sha256=row.validation_manifest_sha256, + query_census_sha256=row.query_census_sha256, + evaluator_sha256=row.evaluator_sha256, + score_artifact_sha256=row.sha256, + ) + for row in row_bindings + ) + + +def assemble_base_cohort_qualification( + cohort: resolver_module.ResolvedBaseCohort, + *, + admission: resolver_module.BaseCohortAdmission, + scored_cohort: scoring_module.BaseCohortValidationResult, + latency_session: latency_module.BaseCohortLatencySession, +) -> BaseCohortQualificationAssembly: + """Derive all nine BaseScore rows and invoke the existing selector once.""" + + checked_cohort, checked_admission, checked_scores, checked_latency = _checked_inputs( + cohort, + admission, + scored_cohort, + latency_session, + ) + before_sources, selector_artifact = _installed_code_snapshot() + evaluator_sha256 = before_sources["selector_artifact_sha256"] + row_bindings = _build_row_bindings( + checked_cohort, + checked_scores, + checked_latency, + evaluator_sha256, + ) + score_rows = _base_scores(row_bindings) + try: + qualification = experiments_module.qualify_base(score_rows) + evidence = production_module.BaseQualificationExecutionEvidence( + score_rows=score_rows, + selector_artifact=selector_artifact, + ) + artifact = production_module.build_base_qualification_execution_artifact(evidence) + canonical = experiments_module.canonical_base_qualification_bytes(qualification) + except (TypeError, ValueError, RuntimeError) as error: + raise BaseCohortQualificationError( + "installed base qualification recomputation failed" + ) from error + if artifact != canonical: + raise BaseCohortQualificationError( + "production evidence and deterministic selector artifacts differ" + ) + runtime = checked_latency.runtime_identity + if type(runtime) is not latency_module.BaseLatencyRuntimeIdentity: + raise AssertionError("checked formal latency lost runtime identity") + dependency = BaseQualificationDependencyBinding( + admission_sha256=_admission_sha256(checked_admission), + resolved_cohort_sha256=latency_module._resolved_cohort_sha256(checked_cohort), + scored_cohort_sha256=checked_scores.sha256, + latency_session_sha256=checked_latency.sha256, + plan_sha256=checked_cohort.plan_sha256, + matrix_sha256=checked_cohort.matrix_sha256, + training_config_sha256=checked_cohort.training_config_sha256, + source_tree_sha256=checked_cohort.source_tree_sha256, + train_manifest_sha256=checked_cohort.train_manifest_sha256, + admitted_val_census_sha256=checked_cohort.val_manifest_sha256, + validation_source_manifest_sha256=checked_scores.source_manifest_sha256, + query_census_sha256=checked_scores.query_census_sha256, + resolver_source_sha256=before_sources["resolver_source_sha256"], + scorer_source_sha256=before_sources["scorer_source_sha256"], + latency_source_sha256=before_sources["latency_source_sha256"], + assembler_source_sha256=before_sources["assembler_source_sha256"], + production_source_sha256=before_sources["production_source_sha256"], + selector_artifact_sha256=evaluator_sha256, + runtime_sha256=runtime.sha256, + environment_sha256=runtime.environment_sha256, + precision_mode=runtime.precision_mode, + ) + after_sources, after_selector = _installed_code_snapshot() + if after_sources != before_sources or after_selector != selector_artifact: + raise BaseCohortQualificationError("installed dependency bytes changed during assembly") + if ( + latency_module._resolved_cohort_sha256(checked_cohort) != dependency.resolved_cohort_sha256 + or checked_scores.sha256 != dependency.scored_cohort_sha256 + or checked_latency.sha256 != dependency.latency_session_sha256 + or _admission_sha256(checked_admission) != dependency.admission_sha256 + or any( + _sha256(row.selected_checkpoint.raw) != row.selected_checkpoint.sha256 + for row in checked_cohort.rows + ) + ): + raise BaseCohortQualificationError("qualification inputs changed during assembly") + return BaseCohortQualificationAssembly( + qualification=qualification, + execution_evidence=evidence, + artifact=artifact, + row_bindings=row_bindings, + dependency_binding=dependency, + ) + + +__all__ = [ + "ASSEMBLY_SCHEMA", + "AUTHORITY", + "BaseCohortQualificationAssembly", + "BaseCohortQualificationError", + "BaseQualificationDependencyBinding", + "CombinedBaseScoreBinding", + "PRODUCTION", + "RESULT_CLAIMED", + "STATUS", + "assemble_base_cohort_qualification", +] diff --git a/src/phaseset_core/base_cohort_resolver.py b/src/phaseset_core/base_cohort_resolver.py new file mode 100644 index 0000000..af7e65f --- /dev/null +++ b/src/phaseset_core/base_cohort_resolver.py @@ -0,0 +1,1004 @@ +"""Resolve the completed registered base cohort without scoring it. + +This private controller helper consumes an already authenticated attempt-registry +snapshot. It validates the exact nine successful terminal chains through +``execution.verified_completed_attempt_evidence``, retains their failed resume +ancestry, and returns owned bytes for each validation-selected checkpoint. + +The result is deliberately authority zero. It is neither a score artifact nor +a qualification, and a registry row or digest supplied to this API is never +treated as proof that an external registry is authentic. +""" + +from __future__ import annotations + +from dataclasses import dataclass +import hashlib +import io +import json +import os +from pathlib import Path +import re +import stat +from typing import Literal, Mapping + +import torch +from torch import nn + +from phaseset_core import execution as execution_module +from phaseset_core import training as training_module +from phaseset_core.experiments import ( + base_run_ids, + build_experiment_plan, + experiment_plan_sha256, + parse_run_id, +) + + +AUTHORITY = 0 +PRODUCTION = False +RESULT_CLAIMED = False +STATUS = "BASE_COHORT_CHECKPOINTS_RESOLVED_NO_SCORE_NO_QUALIFICATION" +MAX_CHECKPOINT_BYTES = 2 * 1024**3 +MAX_SELECTED_COHORT_BYTES = 4 * 1024**3 +MAX_LEDGER_ARTIFACT_BYTES = 1024**2 +MAX_CHECKPOINT_DIRECTORY_ENTRIES = 100_000 +MAX_ATTEMPTS_PER_RUN = 64 + +_ATTEMPT_ID = re.compile(r"[a-z0-9][a-z0-9._-]{0,63}\Z") +_CHECKPOINT_NAME = re.compile( + r"checkpoint-(?P[0-9]{12})-(?P[0-9]{6})-" + r"(?Pupdate|validation)\.pt\Z" +) +_CHECKPOINT_KEYS = frozenset( + { + "authority", + "behavior_sha256", + "best_checkpoint_name", + "best_checkpoint_sha256", + "best_validation_metric", + "checkpoint_sequence", + "code_artifact_sha256", + "config_sha256", + "environment_sha256", + "epoch", + "external_receipt_verified", + "factory_sha256", + "frozen_base_checkpoint_sha256", + "frozen_base_state_sha256", + "global_step", + "initial_optimizable_state_sha256", + "initialization_binding_sha256", + "last_train_loss", + "max_microbatch_edges", + "model", + "optimizer", + "precision_mode", + "production", + "qualified_base_selection_sha256", + "residual_capacity_audit_sha256", + "result_claimed", + "rng", + "scheduler", + "schema", + "seed", + "stage", + "state_digest", + "system_id", + "total_steps", + "train_edges_seen", + "train_manifest_sha256", + "update_index", + "updates_per_epoch", + "val_manifest_sha256", + "validation_edges_seen", + } +) + + +class BaseCohortResolutionError(ValueError): + """The registry, terminal cohort, or selected checkpoint is inconsistent.""" + + +@dataclass(frozen=True, slots=True) +class BaseCohortAdmission: + """Expected bindings retained by the trusted private controller. + + These values are comparison inputs only. Constructing this object does not + authenticate their origin and does not confer execution or result authority. + """ + + plan_sha256: str + matrix_sha256: str + training_config_sha256: str + source_tree_sha256: str + train_manifest_sha256: str + val_manifest_sha256: str + device: str + request_bf16: bool + bf16_runtime_qualified: bool + edge_budget: int + checkpoint_every_updates: int + + +@dataclass(frozen=True, slots=True) +class BaseAttemptRegistryRow: + """One row from an externally authenticated cohort registry snapshot. + + Rows for one run are ordered oldest-to-newest and must describe exactly one + resume chain. A digest here is checked against actual terminal bytes but is + not accepted as evidence that the registry itself was authenticated. + """ + + attempt_id: str + run_id: str + terminal_outcome: Literal["FAILED", "HELD", "SUCCEEDED"] + terminal_sha256: str + + +@dataclass(frozen=True, slots=True, repr=False) +class FailedBaseAttempt: + attempt_id: str + terminal_sha256: str + terminal_outcome: Literal["FAILED", "HELD"] + failure_code: str + + +@dataclass(frozen=True, slots=True, repr=False) +class SelectedBaseCheckpoint: + """Owned immutable bytes for one fully checked validation-best payload.""" + + name: str + sha256: str + raw: bytes + epoch: int + global_step: int + state_digest: str + + +@dataclass(frozen=True, slots=True, repr=False) +class ResolvedBaseRun: + run_id: str + attempt_id: str + system_id: str + seed: int + terminal_sha256: str + latest_checkpoint_sha256: str + latest_checkpoint_name: str + selected_checkpoint: SelectedBaseCheckpoint + failed_predecessors: tuple[FailedBaseAttempt, ...] + + +@dataclass(frozen=True, slots=True, repr=False) +class ResolvedBaseCohort: + """Exact registered row order with no score, latency, or winner.""" + + rows: tuple[ResolvedBaseRun, ...] + plan_sha256: str + matrix_sha256: str + training_config_sha256: str + source_tree_sha256: str + train_manifest_sha256: str + val_manifest_sha256: str + authority: int = AUTHORITY + production: bool = PRODUCTION + result_claimed: bool = RESULT_CLAIMED + status: str = STATUS + + +@dataclass(frozen=True, slots=True, repr=False) +class _CheckedCheckpoint: + raw: bytes + sha256: str + name: str + epoch: int + update_index: int + global_step: int + checkpoint_sequence: int + updates_per_epoch: int + total_steps: int + train_edges_seen: int + validation_edges_seen: int + max_microbatch_edges: int + best_validation_metric: float | None + state_digest: str + best_name: str + best_sha256: str + + +def _lower_sha256(value: object, label: str) -> str: + if type(value) is not str or len(value) != 64 or any( + character not in "0123456789abcdef" for character in value + ): + raise BaseCohortResolutionError(f"{label} must be lowercase SHA-256 hex") + return value + + +def _attempt_id(value: object) -> str: + if ( + type(value) is not str + or _ATTEMPT_ID.fullmatch(value) is None + or value in {".", ".."} + ): + raise BaseCohortResolutionError("attempt_id is not one safe path component") + return value + + +def _exact_positive_int(value: object, label: str) -> int: + if type(value) is not int or value < 1 or value > (1 << 63) - 1: + raise BaseCohortResolutionError(f"{label} must be an exact positive int") + return value + + +def _checked_admission(value: object) -> BaseCohortAdmission: + if type(value) is not BaseCohortAdmission: + raise TypeError("admission must be exact BaseCohortAdmission") + plan = build_experiment_plan() + expected_plan = experiment_plan_sha256(plan) + if value.plan_sha256 != expected_plan or value.matrix_sha256 != plan.matrix_sha256: + raise BaseCohortResolutionError("admission differs from the installed experiment plan") + for label in ( + "training_config_sha256", + "source_tree_sha256", + "train_manifest_sha256", + "val_manifest_sha256", + ): + _lower_sha256(getattr(value, label), f"admission.{label}") + if type(value.device) is not str or not value.device: + raise BaseCohortResolutionError("admission.device must be a nonempty exact string") + if ( + type(value.request_bf16) is not bool + or type(value.bf16_runtime_qualified) is not bool + ): + raise TypeError("admission precision switches must be exact bool values") + _exact_positive_int(value.edge_budget, "admission.edge_budget") + _exact_positive_int( + value.checkpoint_every_updates, + "admission.checkpoint_every_updates", + ) + try: + # This closes all formal runtime knobs not represented as fields above. + training_module.TrainingConfig( + stage="base", + seed=1729, + device=value.device, + request_bf16=value.request_bf16, + bf16_runtime_qualified=value.bf16_runtime_qualified, + edge_budget=value.edge_budget, + checkpoint_every_updates=value.checkpoint_every_updates, + synthetic_contract=False, + ) + except (TypeError, ValueError, RuntimeError) as error: + raise BaseCohortResolutionError("admission training runtime is invalid") from error + return value + + +def _checked_registry_rows(value: object) -> tuple[BaseAttemptRegistryRow, ...]: + if type(value) is not tuple: + raise TypeError("registry_rows must be an exact tuple") + if not value: + raise BaseCohortResolutionError("attempt registry snapshot is empty") + registered = frozenset(base_run_ids()) + if len(value) > MAX_ATTEMPTS_PER_RUN * len(registered): + raise BaseCohortResolutionError("attempt registry exceeds its closed row bound") + seen_attempts: set[str] = set() + checked: list[BaseAttemptRegistryRow] = [] + for ordinal, row in enumerate(value): + if type(row) is not BaseAttemptRegistryRow: + raise TypeError(f"registry_rows[{ordinal}] has the wrong exact type") + attempt = _attempt_id(row.attempt_id) + if attempt in seen_attempts: + raise BaseCohortResolutionError("attempt registry contains a duplicate attempt_id") + seen_attempts.add(attempt) + if type(row.run_id) is not str or row.run_id not in registered: + raise BaseCohortResolutionError("attempt registry contains an unregistered base run") + role, _seed, _system = parse_run_id(row.run_id) + if role != "BASE_QUALIFICATION": + raise BaseCohortResolutionError("attempt registry row is not a base run") + if row.terminal_outcome not in ("FAILED", "HELD", "SUCCEEDED"): + raise BaseCohortResolutionError("registry terminal outcome is outside the vocabulary") + _lower_sha256(row.terminal_sha256, "registry terminal_sha256") + checked.append(row) + successes = [row for row in checked if row.terminal_outcome == "SUCCEEDED"] + expected = base_run_ids() + if len(successes) != len(expected): + raise BaseCohortResolutionError("registry must contain exactly nine successful terminals") + for run_id in expected: + if sum(row.run_id == run_id for row in successes) != 1: + raise BaseCohortResolutionError( + "registry must contain exactly one successful terminal per base run" + ) + if sum(row.run_id == run_id for row in checked) > MAX_ATTEMPTS_PER_RUN: + raise BaseCohortResolutionError( + "registry run history exceeds its closed attempt bound" + ) + return tuple(checked) + + +def _checked_directory(path: Path, label: str) -> Path: + try: + metadata = path.lstat() + except OSError as error: + raise BaseCohortResolutionError(f"{label} is absent") from error + if not stat.S_ISDIR(metadata.st_mode) or path.is_symlink(): + raise BaseCohortResolutionError(f"{label} must be a non-symlink directory") + return path + + +def _read_owned_file( + path: Path, + *, + label: str, + maximum_bytes: int, + expected_sha256: str | None = None, +) -> tuple[bytes, str]: + """Read one stable regular leaf from one descriptor without following it.""" + + flags = os.O_RDONLY + for name in ("O_BINARY", "O_CLOEXEC", "O_NOFOLLOW"): + flags |= int(getattr(os, name, 0)) + before_link = None + if not hasattr(os, "O_NOFOLLOW"): + try: + before_link = path.lstat() + except OSError as error: + raise BaseCohortResolutionError(f"{label} is absent") from error + if stat.S_ISLNK(before_link.st_mode): + raise BaseCohortResolutionError(f"{label} cannot be a symlink") + try: + descriptor = os.open(path, flags) + except OSError as error: + raise BaseCohortResolutionError(f"{label} could not be opened safely") from error + try: + before = os.fstat(descriptor) + if not stat.S_ISREG(before.st_mode): + raise BaseCohortResolutionError(f"{label} must be a regular file") + if before.st_size < 1 or before.st_size > maximum_bytes: + raise BaseCohortResolutionError(f"{label} exceeds its closed byte bound") + remaining = before.st_size + chunks: list[bytes] = [] + while remaining: + chunk = os.read(descriptor, min(1024 * 1024, remaining)) + if not chunk: + raise BaseCohortResolutionError(f"{label} ended before its stated size") + chunks.append(chunk) + remaining -= len(chunk) + if os.read(descriptor, 1): + raise BaseCohortResolutionError(f"{label} grew during its bounded read") + after = os.fstat(descriptor) + finally: + os.close(descriptor) + stable_fields = ("st_dev", "st_ino", "st_size", "st_mtime_ns", "st_ctime_ns") + if any(getattr(before, field) != getattr(after, field) for field in stable_fields): + raise BaseCohortResolutionError(f"{label} changed during its bounded read") + if before_link is not None: + try: + after_link = path.lstat() + except OSError as error: + raise BaseCohortResolutionError(f"{label} changed after its read") from error + if stat.S_ISLNK(after_link.st_mode) or any( + getattr(before_link, field) != getattr(after_link, field) + for field in stable_fields + ): + raise BaseCohortResolutionError(f"{label} changed around its fallback read") + raw = b"".join(chunks) + digest = hashlib.sha256(raw).hexdigest() + if expected_sha256 is not None and digest != _lower_sha256( + expected_sha256, + f"expected {label} sha256", + ): + raise BaseCohortResolutionError(f"{label} digest mismatch") + return raw, digest + + +def _decode_checkpoint(raw: bytes, label: str) -> dict[str, object]: + try: + payload = torch.load(io.BytesIO(raw), map_location="cpu", weights_only=True) + except Exception as error: + raise BaseCohortResolutionError(f"{label} is not a restricted checkpoint") from error + if type(payload) is not dict or frozenset(payload) != _CHECKPOINT_KEYS: + raise BaseCohortResolutionError(f"{label} checkpoint keys are not closed") + return payload + + +def _canonical_json_bytes(value: object) -> bytes: + return ( + json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + allow_nan=False, + ).encode("ascii") + + b"\n" + ) + + +def _torch_value_sha256(value: object) -> str: + stream = io.BytesIO() + torch.save(value, stream) + return hashlib.sha256(stream.getvalue()).hexdigest() + + +def _exact_payload_int( + payload: Mapping[str, object], + name: str, + *, + minimum: int = 0, +) -> int: + value = payload.get(name) + if type(value) is not int or value < minimum or value > (1 << 63) - 1: + raise BaseCohortResolutionError(f"checkpoint {name} is not an exact bounded int") + return value + + +def _validate_rng_without_consuming(payload: Mapping[str, object]) -> None: + rng = payload.get("rng") + if not isinstance(rng, Mapping): + raise BaseCohortResolutionError("checkpoint RNG state is malformed") + ambient = training_module._capture_rng() + try: + training_module._restore_rng(rng) + except training_module.TrainingCheckpointError as error: + raise BaseCohortResolutionError("checkpoint RNG state is malformed") from error + finally: + training_module._restore_rng(ambient) + + +def _validate_model_and_optimizer( + payload: Mapping[str, object], + *, + config: training_module.TrainingConfig, + system_id: str, + global_step: int, + total_steps: int, +) -> None: + try: + system, binding = training_module.construct_registered_base_seed_bound_system( + system_id, + config, + ) + except (TypeError, ValueError, RuntimeError) as error: + raise BaseCohortResolutionError("registered base reconstruction failed") from error + expected_bindings = { + "behavior_sha256": binding.behavior_sha256, + "code_artifact_sha256": binding.code_artifact_sha256, + "environment_sha256": binding.environment_sha256, + "factory_sha256": binding.factory_sha256, + "initial_optimizable_state_sha256": binding.initial_optimizable_state_sha256, + "initialization_binding_sha256": binding.sha256, + } + for name, expected in expected_bindings.items(): + if payload.get(name) != expected: + raise BaseCohortResolutionError(f"checkpoint {name} binding mismatch") + + model = payload.get("model") + if not isinstance(model, Mapping): + raise BaseCohortResolutionError("checkpoint model state is malformed") + parameters = [parameter for parameter in system.parameters() if parameter.requires_grad] + if not parameters or any(not isinstance(parameter, nn.Parameter) for parameter in parameters): + raise BaseCohortResolutionError("registered base parameter census is malformed") + optimizer = torch.optim.AdamW( + parameters, + lr=config.learning_rate, + weight_decay=training_module.WEIGHT_DECAY, + ) + + def multiplier(step: int) -> float: + return training_module._learning_rate_multiplier(step, total_steps) + + scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, multiplier) + try: + training_module._validate_restored_optimizer_scheduler( + payload, + optimizer=optimizer, + scheduler=scheduler, + config=config, + global_step=global_step, + total_steps=total_steps, + ) + system.load_state_dict(model, strict=True) + except (KeyError, TypeError, ValueError, RuntimeError) as error: + raise BaseCohortResolutionError("checkpoint model/optimizer state is malformed") from error + if training_module.training_system_state_sha256(system) != training_module._stable_hash( + model + ): + raise BaseCohortResolutionError("loaded registered base differs from checkpoint bytes") + if training_module.training_system_behavior_sha256(system) != binding.behavior_sha256: + raise BaseCohortResolutionError("loaded registered base behavior changed") + _validate_rng_without_consuming(payload) + + +def _validate_ledger_payload_binding( + payload: Mapping[str, object], + record: execution_module.CheckpointRecord, +) -> None: + epoch = _exact_payload_int(payload, "epoch") + global_step = _exact_payload_int(payload, "global_step") + if record.epoch_index != epoch or record.global_step != global_step: + raise BaseCohortResolutionError("ledger/runtime checkpoint cursor mismatch") + required = { + "model_state_sha256": _torch_value_sha256(payload["model"]), + "optimizer_state_sha256": _torch_value_sha256(payload["optimizer"]), + "rng_state_sha256": _torch_value_sha256(payload["rng"]), + "dropout_state_sha256": _torch_value_sha256(payload["rng"]), + "sampler_state_sha256": hashlib.sha256( + _canonical_json_bytes( + { + "epoch": epoch, + "global_step": global_step, + "update_index": _exact_payload_int(payload, "update_index"), + } + ) + ).hexdigest(), + "dataloader_state_sha256": hashlib.sha256( + _canonical_json_bytes( + { + "train_manifest_sha256": payload["train_manifest_sha256"], + "val_manifest_sha256": payload["val_manifest_sha256"], + } + ) + ).hexdigest(), + "validation_state_sha256": hashlib.sha256( + _canonical_json_bytes( + { + "best_checkpoint_name": payload["best_checkpoint_name"], + "best_checkpoint_sha256": payload["best_checkpoint_sha256"], + "best_validation_metric": payload["best_validation_metric"], + } + ) + ).hexdigest(), + } + for name, expected in required.items(): + if getattr(record, name) != expected: + raise BaseCohortResolutionError(f"ledger/runtime checkpoint {name} mismatch") + + +def _validate_checkpoint_payload( + raw: bytes, + *, + sha256: str, + name: str, + run_id: str, + admission: BaseCohortAdmission, + ledger_record: execution_module.CheckpointRecord | None, + require_completed_latest: bool, + require_selected_best: bool, +) -> _CheckedCheckpoint: + match = _CHECKPOINT_NAME.fullmatch(name) + if match is None: + raise BaseCohortResolutionError("checkpoint filename is outside the runtime grammar") + role, seed, system_id = parse_run_id(run_id) + if role != "BASE_QUALIFICATION": + raise BaseCohortResolutionError("checkpoint run is not a registered base run") + payload = _decode_checkpoint(raw, name) + state_digest = payload.get("state_digest") + without_digest = dict(payload) + without_digest.pop("state_digest") + if ( + type(state_digest) is not str + or state_digest != training_module._stable_hash(without_digest) + ): + raise BaseCohortResolutionError("checkpoint state digest mismatch") + if ( + type(payload["authority"]) is not int + or payload["authority"] != 0 + or payload["production"] is not False + or payload["result_claimed"] is not False + or payload["external_receipt_verified"] is not False + ): + raise BaseCohortResolutionError("checkpoint authority/result header is malformed") + if any( + payload[name_] is not None + for name_ in ( + "frozen_base_checkpoint_sha256", + "frozen_base_state_sha256", + "qualified_base_selection_sha256", + "residual_capacity_audit_sha256", + ) + ): + raise BaseCohortResolutionError("base checkpoint contains residual-only bindings") + config = training_module.TrainingConfig( + stage="base", + seed=seed, + device=admission.device, + request_bf16=admission.request_bf16, + bf16_runtime_qualified=admission.bf16_runtime_qualified, + edge_budget=admission.edge_budget, + checkpoint_every_updates=admission.checkpoint_every_updates, + synthetic_contract=False, + ) + expected = { + "config_sha256": config.sha256, + "precision_mode": training_module.resolve_precision(config).mode, + "schema": training_module.CHECKPOINT_SCHEMA, + "seed": seed, + "stage": "base", + "system_id": system_id, + "train_manifest_sha256": admission.train_manifest_sha256, + "val_manifest_sha256": admission.val_manifest_sha256, + } + for field, value in expected.items(): + if payload.get(field) != value: + raise BaseCohortResolutionError(f"checkpoint {field} binding mismatch") + updates_per_epoch = _exact_payload_int(payload, "updates_per_epoch", minimum=1) + total_steps = _exact_payload_int(payload, "total_steps", minimum=1) + if total_steps != training_module.BASE_EPOCHS * updates_per_epoch: + raise BaseCohortResolutionError("checkpoint total schedule is not the formal base schedule") + try: + cursor = training_module._validate_checkpoint_progress( + payload, + epochs=training_module.BASE_EPOCHS, + updates_per_epoch=updates_per_epoch, + total_steps=total_steps, + artifact_path=Path(name), + artifact_sha256=sha256, + ) + except training_module.TrainingCheckpointError as error: + raise BaseCohortResolutionError("checkpoint cursor/state is malformed") from error + if int(match.group("step")) != cursor.global_step: + raise BaseCohortResolutionError("checkpoint filename step differs from payload") + if int(match.group("sequence")) != cursor.checkpoint_sequence: + raise BaseCohortResolutionError("checkpoint filename sequence differs from payload") + if require_completed_latest and ( + match.group("reason") != "validation" + or cursor.epoch != training_module.BASE_EPOCHS + or cursor.update_index != 0 + or cursor.global_step != total_steps + or cursor.best_validation_metric is None + ): + raise BaseCohortResolutionError("successful terminal does not point to a completed base checkpoint") + if cursor.best_checkpoint_name is None or cursor.best_checkpoint_sha256 is None: + raise BaseCohortResolutionError("checkpoint has no immutable validation-best pointer") + if require_selected_best and ( + match.group("reason") != "validation" + or cursor.epoch < 1 + or cursor.update_index != 0 + or cursor.best_validation_metric is None + or cursor.best_checkpoint_name != name + or cursor.best_checkpoint_sha256 != sha256 + ): + raise BaseCohortResolutionError("selected payload is not its best-at-write checkpoint") + _validate_model_and_optimizer( + payload, + config=config, + system_id=system_id, + global_step=cursor.global_step, + total_steps=total_steps, + ) + if ledger_record is not None: + _validate_ledger_payload_binding(payload, ledger_record) + return _CheckedCheckpoint( + raw=raw, + sha256=sha256, + name=name, + epoch=cursor.epoch, + update_index=cursor.update_index, + global_step=cursor.global_step, + checkpoint_sequence=cursor.checkpoint_sequence, + updates_per_epoch=updates_per_epoch, + total_steps=total_steps, + train_edges_seen=cursor.train_edges_seen, + validation_edges_seen=cursor.validation_edges_seen, + max_microbatch_edges=cursor.max_microbatch_edges, + best_validation_metric=cursor.best_validation_metric, + state_digest=state_digest, + best_name=cursor.best_checkpoint_name, + best_sha256=cursor.best_checkpoint_sha256, + ) + + +def _validate_latest_selected_link( + latest: _CheckedCheckpoint, + selected: _CheckedCheckpoint, +) -> None: + """Close invariants shared by the terminal and its validation-best payload.""" + + if latest.best_name != selected.name or latest.best_sha256 != selected.sha256: + raise BaseCohortResolutionError("latest checkpoint best pointer changed") + if ( + latest.updates_per_epoch != selected.updates_per_epoch + or latest.total_steps != selected.total_steps + ): + raise BaseCohortResolutionError("latest/selected checkpoint schedule mismatch") + if ( + latest.best_validation_metric is None + or selected.best_validation_metric is None + or latest.best_validation_metric != selected.best_validation_metric + ): + raise BaseCohortResolutionError("latest/selected validation-best metric mismatch") + if ( + selected.epoch > latest.epoch + or selected.global_step > latest.global_step + or selected.checkpoint_sequence > latest.checkpoint_sequence + ): + raise BaseCohortResolutionError("selected checkpoint is later than terminal latest") + if ( + selected.train_edges_seen > latest.train_edges_seen + or selected.validation_edges_seen > latest.validation_edges_seen + or selected.max_microbatch_edges > latest.max_microbatch_edges + ): + raise BaseCohortResolutionError("selected checkpoint counters exceed terminal latest") + + +def _latest_checkpoint_file( + checkpoint_directory: Path, + *, + global_step: int, +) -> Path: + candidates: list[Path] = [] + try: + with os.scandir(checkpoint_directory) as entries: + for ordinal, entry in enumerate(entries, start=1): + if ordinal > MAX_CHECKPOINT_DIRECTORY_ENTRIES: + raise BaseCohortResolutionError( + "checkpoint directory exceeds its closed entry bound" + ) + match = _CHECKPOINT_NAME.fullmatch(entry.name) + if ( + match is not None + and int(match.group("step")) == global_step + and match.group("reason") == "validation" + ): + candidates.append(checkpoint_directory / entry.name) + except BaseCohortResolutionError: + raise + except OSError as error: + raise BaseCohortResolutionError("checkpoint directory could not be enumerated") from error + if len(candidates) != 1: + raise BaseCohortResolutionError( + "terminal checkpoint step must have exactly one validation payload" + ) + return candidates[0] + + +def _owned_failed_predecessors( + attempt_root: Path, + evidence: execution_module.CompletedAttemptEvidence, + *, + admission: BaseCohortAdmission, +) -> tuple[FailedBaseAttempt, ...]: + rows: list[FailedBaseAttempt] = [] + resume = evidence.resume + visited = {evidence.attempt.attempt_id} + while resume is not None: + if len(rows) >= MAX_ATTEMPTS_PER_RUN - 1: + raise BaseCohortResolutionError("resume chain exceeds its closed attempt bound") + predecessor_id = _attempt_id(resume.predecessor_attempt_id) + if predecessor_id in visited: + raise BaseCohortResolutionError("resume chain contains a cycle") + visited.add(predecessor_id) + directory = _checked_directory(attempt_root / predecessor_id, "predecessor attempt") + attempt_raw, _ = _read_owned_file( + directory / "attempt.json", + label="predecessor attempt receipt", + maximum_bytes=MAX_LEDGER_ARTIFACT_BYTES, + ) + terminal_raw, terminal_sha = _read_owned_file( + directory / "terminal.json", + label="predecessor terminal", + maximum_bytes=MAX_LEDGER_ARTIFACT_BYTES, + expected_sha256=resume.predecessor_terminal_sha256, + ) + try: + attempt = execution_module.parse_attempt_bytes(attempt_raw) + terminal = execution_module.parse_terminal_bytes(terminal_raw) + except (TypeError, ValueError) as error: + raise BaseCohortResolutionError("predecessor ledger artifact is malformed") from error + if ( + attempt.attempt_id != predecessor_id + or attempt.run_id != evidence.attempt.run_id + or attempt.plan_sha256 != admission.plan_sha256 + or attempt.matrix_sha256 != admission.matrix_sha256 + or attempt.training_config_sha256 != admission.training_config_sha256 + or attempt.source_tree_sha256 != admission.source_tree_sha256 + or terminal.attempt_id != predecessor_id + or terminal.run_id != evidence.attempt.run_id + or terminal.outcome == "SUCCEEDED" + or terminal.failure_code is None + or terminal.attempt_receipt_sha256 != hashlib.sha256(attempt_raw).hexdigest() + or terminal.latest_checkpoint_receipt_sha256 != resume.checkpoint_receipt_sha256 + ): + raise BaseCohortResolutionError("failed predecessor identity/binding is inconsistent") + rows.append( + FailedBaseAttempt( + attempt_id=predecessor_id, + terminal_sha256=terminal_sha, + terminal_outcome=terminal.outcome, + failure_code=terminal.failure_code, + ) + ) + resume_path = directory / "resume.json" + if resume_path.exists() or resume_path.is_symlink(): + resume_raw, _ = _read_owned_file( + resume_path, + label="predecessor resume receipt", + maximum_bytes=MAX_LEDGER_ARTIFACT_BYTES, + ) + try: + predecessor_resume = execution_module.parse_resume_bytes(resume_raw) + except (TypeError, ValueError) as error: + raise BaseCohortResolutionError("predecessor resume receipt is malformed") from error + if ( + predecessor_resume.new_attempt_id != predecessor_id + or predecessor_resume.run_id != evidence.attempt.run_id + or predecessor_resume.created_at_utc != attempt.created_at_utc + ): + raise BaseCohortResolutionError("predecessor resume identity is inconsistent") + resume = predecessor_resume + else: + resume = None + rows.reverse() + return tuple(rows) + + +def _resolve_registered_run( + attempt_root: Path, + run_id: str, + registry_history: tuple[BaseAttemptRegistryRow, ...], + admission: BaseCohortAdmission, + *, + selected_maximum_bytes: int = MAX_CHECKPOINT_BYTES, +) -> ResolvedBaseRun: + success = [row for row in registry_history if row.terminal_outcome == "SUCCEEDED"] + if len(success) != 1 or registry_history[-1] != success[0]: + raise BaseCohortResolutionError("successful registry row must close its run history") + success_row = success[0] + try: + evidence = execution_module.verified_completed_attempt_evidence( + attempt_root, + success_row.attempt_id, + ) + except (TypeError, ValueError) as error: + raise BaseCohortResolutionError("successful attempt chain did not verify") from error + attempt = evidence.attempt + if ( + attempt.run_id != run_id + or attempt.plan_sha256 != admission.plan_sha256 + or attempt.matrix_sha256 != admission.matrix_sha256 + or attempt.training_config_sha256 != admission.training_config_sha256 + or attempt.source_tree_sha256 != admission.source_tree_sha256 + or evidence.terminal_sha256 != success_row.terminal_sha256 + ): + raise BaseCohortResolutionError("successful attempt differs from cohort admission/registry") + failed = _owned_failed_predecessors(attempt_root, evidence, admission=admission) + actual_history = tuple( + (row.attempt_id, row.terminal_outcome, row.terminal_sha256) for row in failed + ) + ((success_row.attempt_id, "SUCCEEDED", evidence.terminal_sha256),) + expected_history = tuple( + (row.attempt_id, row.terminal_outcome, row.terminal_sha256) + for row in registry_history + ) + if actual_history != expected_history: + raise BaseCohortResolutionError("registry history differs from the verified resume chain") + + checkpoint_directory = _checked_directory( + attempt_root / attempt.attempt_id / "model-checkpoints", + "successful checkpoint directory", + ) + latest_path = _latest_checkpoint_file( + checkpoint_directory, + global_step=evidence.latest_checkpoint.global_step, + ) + latest_raw, latest_sha = _read_owned_file( + latest_path, + label="terminal-bound latest checkpoint", + maximum_bytes=MAX_CHECKPOINT_BYTES, + expected_sha256=evidence.latest_checkpoint_payload_sha256, + ) + latest = _validate_checkpoint_payload( + latest_raw, + sha256=latest_sha, + name=latest_path.name, + run_id=run_id, + admission=admission, + ledger_record=evidence.latest_checkpoint, + require_completed_latest=True, + require_selected_best=False, + ) + selected_path = checkpoint_directory / latest.best_name + selected_raw, selected_sha = _read_owned_file( + selected_path, + label="validation-selected checkpoint", + maximum_bytes=min(MAX_CHECKPOINT_BYTES, selected_maximum_bytes), + expected_sha256=latest.best_sha256, + ) + selected = _validate_checkpoint_payload( + selected_raw, + sha256=selected_sha, + name=selected_path.name, + run_id=run_id, + admission=admission, + ledger_record=None, + require_completed_latest=False, + require_selected_best=True, + ) + _validate_latest_selected_link(latest, selected) + _role, seed, system_id = parse_run_id(run_id) + return ResolvedBaseRun( + run_id=run_id, + attempt_id=attempt.attempt_id, + system_id=system_id, + seed=seed, + terminal_sha256=evidence.terminal_sha256, + latest_checkpoint_sha256=latest.sha256, + latest_checkpoint_name=latest.name, + selected_checkpoint=SelectedBaseCheckpoint( + name=selected.name, + sha256=selected.sha256, + raw=selected.raw, + epoch=selected.epoch, + global_step=selected.global_step, + state_digest=selected.state_digest, + ), + failed_predecessors=failed, + ) + + +def resolve_completed_base_cohort( + attempt_root: str | Path, + *, + registry_rows: tuple[BaseAttemptRegistryRow, ...], + admission: BaseCohortAdmission, +) -> ResolvedBaseCohort: + """Resolve exactly nine terminal chains and their validation-best payloads. + + ``registry_rows`` must come from the controller's separately authenticated, + stable private registry snapshot. This function validates the snapshot's + claims against ledger/checkpoint bytes, but cannot authenticate the source + of the snapshot and never treats its hashes as a qualification grant. + """ + + checked_admission = _checked_admission(admission) + checked_registry = _checked_registry_rows(registry_rows) + root = _checked_directory(Path(attempt_root), "attempt root") + rows: list[ResolvedBaseRun] = [] + selected_bytes = 0 + for run_id in base_run_ids(): + history = tuple(row for row in checked_registry if row.run_id == run_id) + remaining = MAX_SELECTED_COHORT_BYTES - selected_bytes + if remaining < 1: + raise BaseCohortResolutionError( + "selected checkpoint cohort exceeds its closed byte bound" + ) + resolved = _resolve_registered_run( + root, + run_id, + history, + checked_admission, + selected_maximum_bytes=remaining, + ) + selected_bytes += len(resolved.selected_checkpoint.raw) + if selected_bytes > MAX_SELECTED_COHORT_BYTES: + raise AssertionError("selected checkpoint byte accounting exceeded its bound") + rows.append(resolved) + canonical = tuple(rows) + if tuple(row.run_id for row in canonical) != base_run_ids(): + raise AssertionError("resolved base cohort lost the registered order") + return ResolvedBaseCohort( + rows=canonical, + plan_sha256=checked_admission.plan_sha256, + matrix_sha256=checked_admission.matrix_sha256, + training_config_sha256=checked_admission.training_config_sha256, + source_tree_sha256=checked_admission.source_tree_sha256, + train_manifest_sha256=checked_admission.train_manifest_sha256, + val_manifest_sha256=checked_admission.val_manifest_sha256, + ) + + +__all__ = [ + "AUTHORITY", + "BaseAttemptRegistryRow", + "BaseCohortAdmission", + "BaseCohortResolutionError", + "FailedBaseAttempt", + "MAX_CHECKPOINT_BYTES", + "MAX_SELECTED_COHORT_BYTES", + "PRODUCTION", + "RESULT_CLAIMED", + "ResolvedBaseCohort", + "ResolvedBaseRun", + "STATUS", + "SelectedBaseCheckpoint", + "resolve_completed_base_cohort", +] diff --git a/src/phaseset_core/base_cohort_validation.py b/src/phaseset_core/base_cohort_validation.py new file mode 100644 index 0000000..3e5d1a3 --- /dev/null +++ b/src/phaseset_core/base_cohort_validation.py @@ -0,0 +1,968 @@ +"""Score a resolved nine-base cohort on one frozen capture validation census. + +This private seam starts from ``base_cohort_resolver`` owned selected-best +checkpoint bytes. It reconstructs each registered seed-bound B0/B1/B2 system, +revalidates the selected payload, loads exactly that model state, and delegates +the complete capture gallery to the audited capture-validation runtime. + +The output is authority zero and deliberately stops before latency measurement, +``BaseScore`` construction, winner selection, or production authorization. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass, field +from fractions import Fraction +import hashlib +import json +from typing import Final + +import numpy as np +import torch +from torch import nn + +from . import base_cohort_resolver as resolver_module +from . import capture_validation as capture_module +from . import evaluation as evaluation_module +from . import training as training_module +from .experiments import base_run_ids, parse_run_id + + +AUTHORITY: Final = 0 +PRODUCTION: Final = False +RESULT_CLAIMED: Final = False +OBSERVATION_SCHEMA: Final = "phaseset-base-capture-score-observation-v1" +COHORT_SCHEMA: Final = "phaseset-base-cohort-validation-v1" +OBSERVATION_STATUS: Final = "ACTUAL_GALLERY_SCORED_NO_LATENCY_NO_QUALIFICATION" +COHORT_STATUS: Final = "NINE_BASE_GALLERIES_SCORED_NO_WINNER_NO_QUALIFICATION" +MAX_SCORE_MATRIX_BYTES: Final = capture_module.DEFAULT_MAX_ENCODED_BYTES + + +class BaseCohortValidationError(ValueError): + """The resolved cohort, selected model, or scored gallery is inconsistent.""" + + +def _sha256(raw: bytes) -> str: + return hashlib.sha256(raw).hexdigest() + + +def _lower_sha256(value: object, label: str) -> str: + if ( + type(value) is not str + or len(value) != 64 + or value.lower() != value + or any(character not in "0123456789abcdef" for character in value) + ): + raise BaseCohortValidationError(f"{label} must be lowercase SHA-256 hex") + return value + + +def _raw32(value: object, label: str) -> bytes: + if type(value) is not bytes or len(value) != 32: + raise BaseCohortValidationError(f"{label} must be exact bytes[32]") + return value + + +def _positive_int(value: object, label: str) -> int: + if type(value) is not int or value < 1 or value > (1 << 63) - 1: + raise BaseCohortValidationError(f"{label} must be an exact positive int") + return value + + +def _canonical_json_bytes(value: object) -> bytes: + return ( + json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + allow_nan=False, + ).encode("ascii") + + b"\n" + ) + + +@dataclass(frozen=True, slots=True) +class BaseParameterRow: + """One live named parameter in canonical module order.""" + + name: str + shape: tuple[int, ...] + dtype: str + numel: int + + def __post_init__(self) -> None: + if type(self.name) is not str or not self.name: + raise BaseCohortValidationError("parameter name must be a nonempty string") + if ( + type(self.shape) is not tuple + or any(type(axis) is not int or axis < 0 for axis in self.shape) + ): + raise BaseCohortValidationError("parameter shape must be an exact int tuple") + if type(self.dtype) is not str or self.dtype != "torch.float32": + raise BaseCohortValidationError("registered base parameter dtype must be float32") + expected = 1 + for axis in self.shape: + expected *= axis + if _positive_int(self.numel, "parameter numel") != expected: + raise BaseCohortValidationError("parameter numel differs from its shape") + + +@dataclass(frozen=True, slots=True) +class BaseParameterCensus: + """Complete live parameter census derived after selected-state loading.""" + + system_id: str + rows: tuple[BaseParameterRow, ...] + total: int + loaded_state_sha256: str + + def __post_init__(self) -> None: + if self.system_id not in training_module.REGISTERED_BASE_SYSTEM_IDS: + raise BaseCohortValidationError("parameter census system_id is not registered") + if type(self.rows) is not tuple or not self.rows: + raise BaseCohortValidationError("parameter census must be a nonempty tuple") + if any(type(row) is not BaseParameterRow for row in self.rows): + raise BaseCohortValidationError("parameter census row type is invalid") + names = tuple(row.name for row in self.rows) + if names != tuple(sorted(names)) or len(set(names)) != len(names): + raise BaseCohortValidationError( + "parameter census names must be unique and lexically ordered" + ) + if _positive_int(self.total, "parameter total") != sum( + row.numel for row in self.rows + ): + raise BaseCohortValidationError("parameter total differs from its rows") + _lower_sha256(self.loaded_state_sha256, "loaded_state_sha256") + + @property + def sha256(self) -> str: + return _sha256( + _canonical_json_bytes( + { + "loaded_state_sha256": self.loaded_state_sha256, + "rows": [ + { + "dtype": row.dtype, + "name": row.name, + "numel": row.numel, + "shape": list(row.shape), + } + for row in self.rows + ], + "schema": "phaseset-base-parameter-census-v1", + "system_id": self.system_id, + "total": self.total, + } + ) + ) + + +def _fraction_payload(value: Fraction) -> dict[str, int]: + if type(value) is not Fraction or not Fraction(0, 1) <= value <= Fraction(1, 1): + raise BaseCohortValidationError("capture R1 must be an exact fraction in [0,1]") + return {"denominator": value.denominator, "numerator": value.numerator} + + +def _dataset_from_observation( + *, + scores_float64le: bytes, + score_shape: tuple[int, int], + motion_commitments: tuple[bytes, ...], + caption_commitments: tuple[bytes, ...], + positive_motion_indices: tuple[tuple[int, ...], ...], + group_sizes: tuple[int, ...], + component_labels: tuple[str, ...], +) -> evaluation_module.RetrievalDataset: + if type(scores_float64le) is not bytes: + raise BaseCohortValidationError("score bytes must be exact immutable bytes") + if ( + type(score_shape) is not tuple + or len(score_shape) != 2 + or any(type(axis) is not int or axis < 2 for axis in score_shape) + ): + raise BaseCohortValidationError("score shape must be exact [M>=2,Q>=2]") + expected = score_shape[0] * score_shape[1] * np.dtype(" MAX_SCORE_MATRIX_BYTES: + raise BaseCohortValidationError( + "complete score matrix exceeds its fixed observation byte bound" + ) + if len(scores_float64le) != expected: + raise BaseCohortValidationError("score bytes differ from declared float64 shape") + try: + scores = ( + np.frombuffer(scores_float64le, dtype=" tuple[Fraction, Fraction, Fraction]: + rows = evaluation_module.capture_r1_contributions(dataset) + if not rows: + raise BaseCohortValidationError("capture score contribution census is empty") + count = len(rows) + text = sum( + ( + Fraction(sum(row.text_to_motion_hits), len(row.text_to_motion_hits)) + for row in rows + ), + start=Fraction(0, 1), + ) / count + motion = sum( + ( + Fraction(sum(row.motion_to_text_hits), len(row.motion_to_text_hits)) + for row in rows + ), + start=Fraction(0, 1), + ) / count + return text, motion, (text + motion) / 2 + + +@dataclass(frozen=True, slots=True, repr=False) +class BaseValidationScoreObservation: + """Immutable actual-score bytes plus checkpoint/runtime/query bindings.""" + + run_id: str + attempt_id: str + system_id: str + seed: int + terminal_sha256: str + selected_checkpoint_sha256: str + selected_checkpoint_state_digest: str + initialization_binding_sha256: str + runtime_config_sha256: str + factory_sha256: str + code_artifact_sha256: str + environment_sha256: str + behavior_sha256: str + source_manifest_sha256: str + admitted_val_census_sha256: str + query_census_sha256: str + parameter_census: BaseParameterCensus + scores_float64le: bytes = field(repr=False) + score_shape: tuple[int, int] + motion_commitments: tuple[bytes, ...] = field(repr=False) + caption_commitments: tuple[bytes, ...] = field(repr=False) + positive_motion_indices: tuple[tuple[int, ...], ...] = field(repr=False) + group_sizes: tuple[int, ...] + component_labels: tuple[str, ...] + text_to_motion_capture_r1: Fraction + motion_to_text_capture_r1: Fraction + primary_capture_r1: Fraction + device: str + precision_mode: str + encoded_bytes: int + authority: int = AUTHORITY + production: bool = PRODUCTION + result_claimed: bool = RESULT_CLAIMED + schema: str = OBSERVATION_SCHEMA + status: str = OBSERVATION_STATUS + + def __post_init__(self) -> None: + role, seed, system_id = parse_run_id(self.run_id) + if ( + role != "BASE_QUALIFICATION" + or self.seed != seed + or self.system_id != system_id + ): + raise BaseCohortValidationError("observation run identity is inconsistent") + if type(self.attempt_id) is not str or not self.attempt_id: + raise BaseCohortValidationError("observation attempt_id is invalid") + for label in ( + "terminal_sha256", + "selected_checkpoint_sha256", + "selected_checkpoint_state_digest", + "initialization_binding_sha256", + "runtime_config_sha256", + "factory_sha256", + "code_artifact_sha256", + "environment_sha256", + "behavior_sha256", + "source_manifest_sha256", + "admitted_val_census_sha256", + "query_census_sha256", + ): + _lower_sha256(getattr(self, label), label) + if self.admitted_val_census_sha256 != self.query_census_sha256: + raise BaseCohortValidationError( + "checkpoint-admitted validation census differs from scored query census" + ) + if ( + type(self.parameter_census) is not BaseParameterCensus + or self.parameter_census.system_id != self.system_id + ): + raise BaseCohortValidationError("observation parameter census is incompatible") + motions = tuple( + _raw32(value, f"motion_commitments[{index}]") + for index, value in enumerate(self.motion_commitments) + ) + captions = tuple( + _raw32(value, f"caption_commitments[{index}]") + for index, value in enumerate(self.caption_commitments) + ) + if type(self.group_sizes) is not tuple or any( + type(value) is not int for value in self.group_sizes + ): + raise BaseCohortValidationError("group_sizes must be an exact int tuple") + if type(self.component_labels) is not tuple: + raise BaseCohortValidationError("component_labels must be an exact tuple") + dataset = _dataset_from_observation( + scores_float64le=self.scores_float64le, + score_shape=self.score_shape, + motion_commitments=motions, + caption_commitments=captions, + positive_motion_indices=self.positive_motion_indices, + group_sizes=self.group_sizes, + component_labels=self.component_labels, + ) + expected_fractions = _capture_fractions(dataset) + observed_fractions = ( + self.text_to_motion_capture_r1, + self.motion_to_text_capture_r1, + self.primary_capture_r1, + ) + for value in observed_fractions: + _fraction_payload(value) + if observed_fractions != expected_fractions: + raise BaseCohortValidationError("observation capture R1 differs from score bytes") + if type(self.device) is not str or not self.device: + raise BaseCohortValidationError("observation device is invalid") + if self.precision_mode not in ("FP32", "BF16"): + raise BaseCohortValidationError("observation precision mode is invalid") + _positive_int(self.encoded_bytes, "encoded_bytes") + if ( + type(self.authority) is not int + or self.authority != AUTHORITY + or self.production is not PRODUCTION + or self.result_claimed is not RESULT_CLAIMED + or self.schema != OBSERVATION_SCHEMA + or self.status != OBSERVATION_STATUS + ): + raise BaseCohortValidationError("observation result header is invalid") + object.__setattr__(self, "motion_commitments", motions) + object.__setattr__(self, "caption_commitments", captions) + + @property + def scores_float64_sha256(self) -> str: + return _sha256(self.scores_float64le) + + def canonical_bytes(self) -> bytes: + return _canonical_json_bytes( + { + "admitted_val_census_sha256": self.admitted_val_census_sha256, + "attempt_id": self.attempt_id, + "authority": self.authority, + "behavior_sha256": self.behavior_sha256, + "caption_commitments": [value.hex() for value in self.caption_commitments], + "code_artifact_sha256": self.code_artifact_sha256, + "component_labels": list(self.component_labels), + "device": self.device, + "encoded_bytes": self.encoded_bytes, + "environment_sha256": self.environment_sha256, + "factory_sha256": self.factory_sha256, + "group_sizes": list(self.group_sizes), + "initialization_binding_sha256": self.initialization_binding_sha256, + "motion_commitments": [value.hex() for value in self.motion_commitments], + "motion_to_text_capture_r1": _fraction_payload( + self.motion_to_text_capture_r1 + ), + "parameter_census_sha256": self.parameter_census.sha256, + "parameter_count": self.parameter_census.total, + "positive_motion_indices": [ + list(value) for value in self.positive_motion_indices + ], + "precision_mode": self.precision_mode, + "primary_capture_r1": _fraction_payload(self.primary_capture_r1), + "production": self.production, + "query_census_sha256": self.query_census_sha256, + "result_claimed": self.result_claimed, + "run_id": self.run_id, + "runtime_config_sha256": self.runtime_config_sha256, + "schema": self.schema, + "score_dtype": " str: + return _sha256(self.canonical_bytes()) + + def __repr__(self) -> str: + return ( + f"BaseValidationScoreObservation(run_id={self.run_id!r}, " + f"primary={self.primary_capture_r1}, " + f"score_artifact_sha256={self.score_artifact_sha256!r})" + ) + + +@dataclass(frozen=True, slots=True, repr=False) +class BaseCohortValidationResult: + """All nine score observations, still without latency or a winner.""" + + observations: tuple[BaseValidationScoreObservation, ...] + plan_sha256: str + matrix_sha256: str + training_config_sha256: str + source_tree_sha256: str + train_manifest_sha256: str + source_manifest_sha256: str + query_census_sha256: str + authority: int = AUTHORITY + production: bool = PRODUCTION + result_claimed: bool = RESULT_CLAIMED + schema: str = COHORT_SCHEMA + status: str = COHORT_STATUS + + def __post_init__(self) -> None: + if type(self.observations) is not tuple or any( + type(value) is not BaseValidationScoreObservation + for value in self.observations + ): + raise BaseCohortValidationError("cohort observations have an invalid type") + if tuple(value.run_id for value in self.observations) != base_run_ids(): + raise BaseCohortValidationError( + "cohort observations must cover the exact registered nine-row order" + ) + for label in ( + "plan_sha256", + "matrix_sha256", + "training_config_sha256", + "source_tree_sha256", + "train_manifest_sha256", + "source_manifest_sha256", + "query_census_sha256", + ): + _lower_sha256(getattr(self, label), label) + if any( + row.source_manifest_sha256 != self.source_manifest_sha256 + or row.query_census_sha256 != self.query_census_sha256 + for row in self.observations + ): + raise BaseCohortValidationError("cohort rows do not share one validation census") + if len({row.terminal_sha256 for row in self.observations}) != 9: + raise BaseCohortValidationError("cohort terminal digests must be unique") + if len({row.selected_checkpoint_sha256 for row in self.observations}) != 9: + raise BaseCohortValidationError("cohort selected checkpoints must be unique") + if len({row.score_artifact_sha256 for row in self.observations}) != 9: + raise BaseCohortValidationError("cohort score artifacts must be unique") + for system_id in training_module.REGISTERED_BASE_SYSTEM_IDS: + counts = { + row.parameter_census.total + for row in self.observations + if row.system_id == system_id + } + census_rows = { + tuple((item.name, item.shape, item.dtype, item.numel) for item in row.parameter_census.rows) + for row in self.observations + if row.system_id == system_id + } + if len(counts) != 1 or len(census_rows) != 1: + raise BaseCohortValidationError( + "base parameter census must agree across each system's three seeds" + ) + if ( + type(self.authority) is not int + or self.authority != AUTHORITY + or self.production is not PRODUCTION + or self.result_claimed is not RESULT_CLAIMED + or self.schema != COHORT_SCHEMA + or self.status != COHORT_STATUS + ): + raise BaseCohortValidationError("cohort result header is invalid") + + @property + def sha256(self) -> str: + return _sha256( + _canonical_json_bytes( + { + "authority": self.authority, + "matrix_sha256": self.matrix_sha256, + "observation_sha256s": [ + value.score_artifact_sha256 for value in self.observations + ], + "plan_sha256": self.plan_sha256, + "production": self.production, + "query_census_sha256": self.query_census_sha256, + "result_claimed": self.result_claimed, + "schema": self.schema, + "source_manifest_sha256": self.source_manifest_sha256, + "source_tree_sha256": self.source_tree_sha256, + "status": self.status, + "training_config_sha256": self.training_config_sha256, + "train_manifest_sha256": self.train_manifest_sha256, + } + ) + ) + + +def _checked_admission( + value: object, +) -> resolver_module.BaseCohortAdmission: + try: + return resolver_module._checked_admission(value) + except (TypeError, ValueError, RuntimeError) as error: + raise BaseCohortValidationError("base cohort admission is invalid") from error + + +def _checked_cohort( + value: object, + admission: resolver_module.BaseCohortAdmission, +) -> resolver_module.ResolvedBaseCohort: + if type(value) is not resolver_module.ResolvedBaseCohort: + raise BaseCohortValidationError("cohort must be exact ResolvedBaseCohort") + if ( + type(value.authority) is not int + or value.authority != resolver_module.AUTHORITY + or value.production is not resolver_module.PRODUCTION + or value.result_claimed is not resolver_module.RESULT_CLAIMED + or value.status != resolver_module.STATUS + ): + raise BaseCohortValidationError("resolved cohort header is invalid") + expected_bindings = { + "plan_sha256": admission.plan_sha256, + "matrix_sha256": admission.matrix_sha256, + "training_config_sha256": admission.training_config_sha256, + "source_tree_sha256": admission.source_tree_sha256, + "train_manifest_sha256": admission.train_manifest_sha256, + "val_manifest_sha256": admission.val_manifest_sha256, + } + for name, expected in expected_bindings.items(): + if getattr(value, name) != expected: + raise BaseCohortValidationError(f"resolved cohort {name} binding mismatch") + if type(value.rows) is not tuple or tuple(row.run_id for row in value.rows) != base_run_ids(): + raise BaseCohortValidationError("resolved cohort lost the registered nine-row order") + terminal_digests: set[str] = set() + checkpoint_digests: set[str] = set() + selected_bytes = 0 + for index, row in enumerate(value.rows): + if type(row) is not resolver_module.ResolvedBaseRun: + raise BaseCohortValidationError(f"resolved cohort row {index} type is invalid") + role, seed, system_id = parse_run_id(row.run_id) + if role != "BASE_QUALIFICATION" or row.seed != seed or row.system_id != system_id: + raise BaseCohortValidationError("resolved row identity is inconsistent") + if type(row.attempt_id) is not str or not row.attempt_id: + raise BaseCohortValidationError("resolved row attempt_id is invalid") + terminal = _lower_sha256(row.terminal_sha256, "terminal_sha256") + _lower_sha256(row.latest_checkpoint_sha256, "latest_checkpoint_sha256") + if type(row.latest_checkpoint_name) is not str or not row.latest_checkpoint_name: + raise BaseCohortValidationError("latest checkpoint name is invalid") + selected = row.selected_checkpoint + if type(selected) is not resolver_module.SelectedBaseCheckpoint: + raise BaseCohortValidationError("selected checkpoint type is invalid") + if type(selected.raw) is not bytes or not selected.raw: + raise BaseCohortValidationError("selected checkpoint bytes are invalid") + selected_digest = _lower_sha256(selected.sha256, "selected checkpoint SHA-256") + if _sha256(selected.raw) != selected_digest: + raise BaseCohortValidationError("selected owned checkpoint digest mismatch") + if type(selected.name) is not str or not selected.name: + raise BaseCohortValidationError("selected checkpoint name is invalid") + _positive_int(selected.epoch, "selected checkpoint epoch") + _positive_int(selected.global_step, "selected checkpoint global_step") + _lower_sha256(selected.state_digest, "selected checkpoint state_digest") + if type(row.failed_predecessors) is not tuple or any( + type(item) is not resolver_module.FailedBaseAttempt + for item in row.failed_predecessors + ): + raise BaseCohortValidationError("failed predecessor census is invalid") + terminal_digests.add(terminal) + checkpoint_digests.add(selected_digest) + selected_bytes += len(selected.raw) + if selected_bytes > resolver_module.MAX_SELECTED_COHORT_BYTES: + raise BaseCohortValidationError("selected cohort exceeds its owned-byte bound") + if len(terminal_digests) != 9 or len(checkpoint_digests) != 9: + raise BaseCohortValidationError( + "resolved cohort terminals and selected checkpoints must be unique" + ) + return value + + +def _snapshot_validation_source(value: object) -> capture_module.CaptureValidationSource: + try: + source_type, _runner = capture_module._verified_capture_validation_bindings() + if type(value) is not source_type: + raise BaseCohortValidationError( + "validation source must be exact CaptureValidationSource" + ) + return source_type(value.split, value.manifest_sha256, value.captures) + except BaseCohortValidationError: + raise + except (TypeError, ValueError, RuntimeError) as error: + raise BaseCohortValidationError("capture validation source is invalid") from error + + +def _parameter_census( + system: training_module.BaseRetrievalSystem, +) -> BaseParameterCensus: + rows: list[BaseParameterRow] = [] + for name, parameter in sorted(system.named_parameters()): + if ( + type(parameter) is not nn.Parameter + or not parameter.requires_grad + or parameter.dtype != torch.float32 + or parameter.numel() < 1 + or not bool(torch.isfinite(parameter.detach()).all().item()) + ): + raise BaseCohortValidationError( + "loaded registered base parameter census is invalid" + ) + rows.append( + BaseParameterRow( + name=name, + shape=tuple(int(axis) for axis in parameter.shape), + dtype=str(parameter.dtype), + numel=int(parameter.numel()), + ) + ) + state_sha256 = training_module.training_system_state_sha256(system) + return BaseParameterCensus( + system_id=system.system_id, + rows=tuple(rows), + total=sum(row.numel for row in rows), + loaded_state_sha256=state_sha256, + ) + + +def _validate_selected_payload( + row: resolver_module.ResolvedBaseRun, + admission: resolver_module.BaseCohortAdmission, +) -> dict[str, object]: + selected = row.selected_checkpoint + try: + checked = resolver_module._validate_checkpoint_payload( + selected.raw, + sha256=selected.sha256, + name=selected.name, + run_id=row.run_id, + admission=admission, + ledger_record=None, + require_completed_latest=False, + require_selected_best=True, + ) + if ( + checked.raw != selected.raw + or checked.sha256 != selected.sha256 + or checked.name != selected.name + or checked.epoch != selected.epoch + or checked.global_step != selected.global_step + or checked.state_digest != selected.state_digest + ): + raise BaseCohortValidationError( + "revalidated selected checkpoint differs from resolved owned bytes" + ) + payload = resolver_module._decode_checkpoint(selected.raw, selected.name) + except BaseCohortValidationError: + raise + except (TypeError, ValueError, RuntimeError) as error: + raise BaseCohortValidationError( + "selected checkpoint failed scoring-time revalidation" + ) from error + return payload + + +def _expected_dataset_identity( + source: capture_module.CaptureValidationSource, +) -> tuple[ + tuple[bytes, ...], + tuple[bytes, ...], + tuple[tuple[int, ...], ...], + tuple[int, ...], + tuple[str, ...], +]: + motions = tuple(capture.capture_commitment for capture in source.captures) + captions: list[bytes] = [] + positives: list[tuple[int, ...]] = [] + for motion_index, capture in enumerate(source.captures): + commitments = capture.holistic_text.caption_commitments + order = sorted(range(len(commitments)), key=lambda index: commitments[index]) + captions.extend(commitments[index] for index in order) + positives.extend((motion_index,) for _ in order) + return ( + motions, + tuple(captions), + tuple(positives), + tuple(capture.group_size for capture in source.captures), + tuple(capture.component_label for capture in source.captures), + ) + + +def _preflight_score_matrix(source: capture_module.CaptureValidationSource) -> int: + motion_count = len(source.captures) + caption_count = sum( + len(capture.holistic_text.caption_commitments) + for capture in source.captures + ) + expected = motion_count * caption_count * np.dtype(" MAX_SCORE_MATRIX_BYTES: + raise BaseCohortValidationError( + "complete score matrix exceeds its fixed observation byte bound" + ) + return expected + + +def _validated_capture_result( + result: object, + *, + row: resolver_module.ResolvedBaseRun, + source: capture_module.CaptureValidationSource, + config: training_module.TrainingConfig, +) -> tuple[ + evaluation_module.RetrievalDataset, + tuple[Fraction, Fraction, Fraction], + bytes, +]: + if type(result) is not capture_module.CaptureValidationResult: + raise BaseCohortValidationError( + "capture scorer did not return exact CaptureValidationResult" + ) + try: + dataset = evaluation_module.validate_retrieval_dataset(result.dataset) + except (TypeError, ValueError) as error: + raise BaseCohortValidationError("capture scorer dataset is invalid") from error + expected_identity = _expected_dataset_identity(source) + observed_identity = ( + dataset.motion_commitments, + dataset.caption_commitments, + dataset.positive_motion_indices, + tuple(int(value) for value in dataset.group_sizes), + dataset.component_labels, + ) + if observed_identity != expected_identity: + raise BaseCohortValidationError("scored gallery identity differs from frozen source") + public_score_sha256 = _sha256(dataset.scores.tobytes(order="C")) + scores = np.ascontiguousarray(dataset.scores, dtype=" BaseValidationScoreObservation: + before_census = source.census_sha256 + if before_census != admission.val_manifest_sha256: + raise BaseCohortValidationError( + "validation source differs from checkpoint-admitted val census" + ) + payload = _validate_selected_payload(row, admission) + config = training_module.TrainingConfig( + stage="base", + seed=row.seed, + device=admission.device, + request_bf16=admission.request_bf16, + bf16_runtime_qualified=admission.bf16_runtime_qualified, + edge_budget=admission.edge_budget, + checkpoint_every_updates=admission.checkpoint_every_updates, + synthetic_contract=False, + ) + try: + system, binding = training_module.construct_registered_base_seed_bound_system( + row.system_id, + config, + ) + except (TypeError, ValueError, RuntimeError) as error: + raise BaseCohortValidationError("registered base reconstruction failed") from error + expected_bindings = { + "behavior_sha256": binding.behavior_sha256, + "code_artifact_sha256": binding.code_artifact_sha256, + "environment_sha256": binding.environment_sha256, + "factory_sha256": binding.factory_sha256, + "initial_optimizable_state_sha256": binding.initial_optimizable_state_sha256, + "initialization_binding_sha256": binding.sha256, + } + if any(payload.get(name) != value for name, value in expected_bindings.items()): + raise BaseCohortValidationError( + "selected checkpoint differs from live seed-bound initialization" + ) + model = payload.get("model") + if not isinstance(model, Mapping): + raise BaseCohortValidationError("selected checkpoint model state is malformed") + try: + system.load_state_dict(model, strict=True) + except (KeyError, TypeError, ValueError, RuntimeError) as error: + raise BaseCohortValidationError("selected model state could not be loaded") from error + loaded_state_sha256 = training_module.training_system_state_sha256(system) + if ( + loaded_state_sha256 != training_module._stable_hash(model) + or training_module.training_system_behavior_sha256(system) + != binding.behavior_sha256 + ): + raise BaseCohortValidationError( + "loaded selected model differs from its checkpoint/runtime binding" + ) + parameter_census = _parameter_census(system) + if parameter_census.loaded_state_sha256 != loaded_state_sha256: + raise AssertionError("parameter census lost the selected model state") + try: + system.to(torch.device(config.device)) + source_type, runner = capture_module._verified_capture_validation_bindings() + if type(source) is not source_type: + raise BaseCohortValidationError("capture source type changed before scoring") + result = runner(system, config, source) + except BaseCohortValidationError: + raise + except (TypeError, ValueError, RuntimeError) as error: + raise BaseCohortValidationError("complete capture gallery scoring failed") from error + dataset, fractions, score_bytes = _validated_capture_result( + result, + row=row, + source=source, + config=config, + ) + after_census = source.census_sha256 + if after_census != before_census or result.source_census_sha256 != before_census: + raise BaseCohortValidationError("validation census changed during scoring") + if _sha256(row.selected_checkpoint.raw) != row.selected_checkpoint.sha256: + raise BaseCohortValidationError("selected checkpoint bytes changed during scoring") + return BaseValidationScoreObservation( + run_id=row.run_id, + attempt_id=row.attempt_id, + system_id=row.system_id, + seed=row.seed, + terminal_sha256=row.terminal_sha256, + selected_checkpoint_sha256=row.selected_checkpoint.sha256, + selected_checkpoint_state_digest=row.selected_checkpoint.state_digest, + initialization_binding_sha256=binding.sha256, + runtime_config_sha256=config.sha256, + factory_sha256=binding.factory_sha256, + code_artifact_sha256=binding.code_artifact_sha256, + environment_sha256=binding.environment_sha256, + behavior_sha256=binding.behavior_sha256, + source_manifest_sha256=source.manifest_sha256, + admitted_val_census_sha256=admission.val_manifest_sha256, + query_census_sha256=result.source_census_sha256, + parameter_census=parameter_census, + scores_float64le=score_bytes, + score_shape=tuple(int(axis) for axis in dataset.scores.shape), + motion_commitments=dataset.motion_commitments, + caption_commitments=dataset.caption_commitments, + positive_motion_indices=dataset.positive_motion_indices, + group_sizes=tuple(int(value) for value in dataset.group_sizes), + component_labels=dataset.component_labels, + text_to_motion_capture_r1=fractions[0], + motion_to_text_capture_r1=fractions[1], + primary_capture_r1=fractions[2], + device=result.device, + precision_mode=result.precision_mode, + encoded_bytes=result.encoded_bytes, + ) + + +def score_resolved_base_cohort( + cohort: resolver_module.ResolvedBaseCohort, + *, + admission: resolver_module.BaseCohortAdmission, + validation_source: capture_module.CaptureValidationSource, +) -> BaseCohortValidationResult: + """Score all nine selected-best bases without latency or winner selection.""" + + checked_admission = _checked_admission(admission) + checked_cohort = _checked_cohort(cohort, checked_admission) + source = _snapshot_validation_source(validation_source) + expected_census = source.census_sha256 + if expected_census != checked_admission.val_manifest_sha256: + raise BaseCohortValidationError( + "capture census differs from the cohort validation binding" + ) + _preflight_score_matrix(source) + observations: list[BaseValidationScoreObservation] = [] + for row in checked_cohort.rows: + if source.census_sha256 != expected_census: + raise BaseCohortValidationError("validation source changed between cohort rows") + observations.append( + _score_resolved_run( + row, + admission=checked_admission, + source=source, + ) + ) + if source.census_sha256 != expected_census: + raise BaseCohortValidationError("validation source changed after cohort scoring") + return BaseCohortValidationResult( + observations=tuple(observations), + plan_sha256=checked_cohort.plan_sha256, + matrix_sha256=checked_cohort.matrix_sha256, + training_config_sha256=checked_cohort.training_config_sha256, + source_tree_sha256=checked_cohort.source_tree_sha256, + train_manifest_sha256=checked_cohort.train_manifest_sha256, + source_manifest_sha256=source.manifest_sha256, + query_census_sha256=expected_census, + ) + + +__all__ = [ + "AUTHORITY", + "BaseCohortValidationError", + "BaseCohortValidationResult", + "BaseParameterCensus", + "BaseParameterRow", + "BaseValidationScoreObservation", + "COHORT_SCHEMA", + "COHORT_STATUS", + "MAX_SCORE_MATRIX_BYTES", + "OBSERVATION_SCHEMA", + "OBSERVATION_STATUS", + "PRODUCTION", + "RESULT_CLAIMED", + "score_resolved_base_cohort", +] diff --git a/src/phaseset_core/base_qualification_host.py b/src/phaseset_core/base_qualification_host.py new file mode 100644 index 0000000..e80838c --- /dev/null +++ b/src/phaseset_core/base_qualification_host.py @@ -0,0 +1,1137 @@ +"""Private host controller for one actual nine-row base qualification. + +The public CLI supplies only an attempt root. This module consumes a frozen +private registry, resolves and scores the actual selected checkpoints, runs +one complete latency session with a durable observer, and invokes the existing +qualification assembler. It never accepts submitted scores, latencies, +parameter counts, or a winner. +""" + +from __future__ import annotations + +from contextlib import contextmanager +from dataclasses import asdict, dataclass, field +from datetime import datetime, timezone +import gc +import hashlib +import json +import os +from pathlib import Path +import re +import signal +import stat +import threading +from typing import Final, Iterator + +from . import base_cohort_latency as latency_module +from . import base_cohort_qualification as assembly_module +from . import base_cohort_resolver as resolver_module +from . import base_cohort_validation as scoring_module +from . import latency_progress_journal as journal_module +from . import production as production_module +from . import training as training_module +from .capture_validation import CaptureValidationResourceLimit, CaptureValidationSource +from .experiments import base_run_ids + + +CONFIG_SCHEMA: Final = "phaseset-private-base-qualification-controller-v2" +REGISTRY_SCHEMA: Final = "phaseset-private-base-attempt-registry-v1" +START_SCHEMA: Final = "phaseset-private-base-qualification-start-v1" +FAILURE_SCHEMA: Final = "phaseset-private-base-qualification-failure-v2" +GRANT_BASIS_SCHEMA: Final = "phaseset-private-base-qualification-grant-basis-v2" +AUTHORIZATION_SCHEMA: Final = "phaseset-private-base-qualification-authorization-v1" +COMPLETION_SCHEMA: Final = "phaseset-private-base-qualification-completion-v2" +SCORING_CUDA_SCHEMA: Final = "phaseset-private-base-scoring-cuda-v1" +MAX_REGISTRY_BYTES: Final = 1024 * 1024 +MAX_OUTPUT_BYTES: Final = 8 * 1024 * 1024 +MAX_WALL_TIMEOUT_SECONDS: Final = 7 * 24 * 60 * 60 +_SESSION_ID: Final = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,95}") +_CUDA_UUID: Final = re.compile( + r"GPU-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}" +) + + +class HostBaseQualificationError(ValueError): + """The private controller configuration or retained evidence is invalid.""" + + +class _NonformalRuntimeHold(RuntimeError): + """An explicitly injected engineering runtime reached its closed boundary.""" + + +class _ScoringCudaHold(RuntimeError): + """The actual scorer CUDA admission or zero-allocation boundary held.""" + + def __init__(self, failure_code: str, message: str) -> None: + super().__init__(message) + self.failure_code = failure_code + + +class _FormalLatencyTimerHold(RuntimeError): + """The required process-local latency timer is unavailable or drifted.""" + + +@dataclass(frozen=True, slots=True) +class _ScoringCudaState: + runtime: object + admission_observation: latency_module.SharedCudaObservation + runtime_identity: latency_module.BaseLatencyRuntimeIdentity + post_context_observation: latency_module.SharedCudaObservation + + +@dataclass(frozen=True, slots=True) +class HostBaseQualificationConfig: + registry_path: Path + registry_sha256: str + output_root: Path + journal_parent: Path + session_id: str + registered_cuda_uuid: str + latency_protocol_sha256: str + launcher_sha256: str + wall_timeout_seconds: int + schema: str = CONFIG_SCHEMA + + def __post_init__(self) -> None: + for name in ("registry_path", "output_root", "journal_parent"): + if not isinstance(getattr(self, name), Path): + raise TypeError(f"{name} must be pathlib.Path") + for name in ( + "registry_sha256", + "latency_protocol_sha256", + "launcher_sha256", + ): + _lower_sha256(getattr(self, name), name) + if type(self.session_id) is not str or _SESSION_ID.fullmatch(self.session_id) is None: + raise HostBaseQualificationError("qualification session_id is not one safe component") + if ( + type(self.registered_cuda_uuid) is not str + or _CUDA_UUID.fullmatch(self.registered_cuda_uuid) is None + ): + raise HostBaseQualificationError("registered CUDA UUID is invalid") + if ( + type(self.wall_timeout_seconds) is not int + or not 1 <= self.wall_timeout_seconds <= MAX_WALL_TIMEOUT_SECONDS + ): + raise HostBaseQualificationError("formal latency wall timeout is outside its bound") + if self.schema != CONFIG_SCHEMA: + raise HostBaseQualificationError("qualification controller schema changed") + + +@dataclass(frozen=True, slots=True, repr=False) +class HostBaseQualificationAuthentication: + authorization: production_module.PrivateBaseQualificationAuthorization + qualification_sha256: str + adapter_sha256: str + grant_path: Path + qualification_path: Path + grant_raw: bytes = field(repr=False) + qualification_raw: bytes = field(repr=False) + + def __post_init__(self) -> None: + if type(self.authorization) is not production_module.PrivateBaseQualificationAuthorization: + raise HostBaseQualificationError("qualification authorization type is invalid") + for name in ("qualification_sha256", "adapter_sha256"): + _lower_sha256(getattr(self, name), name) + if not isinstance(self.grant_path, Path) or not isinstance( + self.qualification_path, Path + ): + raise TypeError("qualification authentication paths must be pathlib.Path") + if type(self.grant_raw) is not bytes or type(self.qualification_raw) is not bytes: + raise TypeError("qualification authentication bytes must be exact bytes") + if _sha256(self.grant_raw) != self.authorization.external_grant_sha256: + raise HostBaseQualificationError("authorization does not bind the grant bytes") + if _sha256(self.qualification_raw) != self.qualification_sha256: + raise HostBaseQualificationError("authentication does not bind qualification bytes") + + def verifies( + self, + authorization: object, + *, + qualification_sha256: object, + adapter_sha256: object, + ) -> bool: + if ( + authorization != self.authorization + or qualification_sha256 != self.qualification_sha256 + or adapter_sha256 != self.adapter_sha256 + ): + return False + try: + grant, grant_sha = resolver_module._read_owned_file( + self.grant_path, + label="qualification grant", + maximum_bytes=MAX_OUTPUT_BYTES, + expected_sha256=self.authorization.external_grant_sha256, + ) + qualification, qualification_sha = resolver_module._read_owned_file( + self.qualification_path, + label="qualification artifact", + maximum_bytes=MAX_OUTPUT_BYTES, + expected_sha256=self.qualification_sha256, + ) + except (OSError, TypeError, ValueError, RuntimeError): + return False + return ( + grant == self.grant_raw + and grant_sha == self.authorization.external_grant_sha256 + and qualification == self.qualification_raw + and qualification_sha == self.qualification_sha256 + ) + + +@dataclass(frozen=True, slots=True, repr=False) +class HostBaseQualificationRun: + execution: production_module.BackendExecution + authentication: HostBaseQualificationAuthentication | None + journal_path: Path | None + registry_sha256: str + formal: bool + + def __post_init__(self) -> None: + if type(self.execution) is not production_module.BackendExecution: + raise HostBaseQualificationError("controller execution type is invalid") + _lower_sha256(self.registry_sha256, "registry_sha256") + if type(self.formal) is not bool: + raise TypeError("formal must be an exact bool") + if self.execution.outcome == "COMPLETED": + if not self.formal or type(self.authentication) is not HostBaseQualificationAuthentication: + raise HostBaseQualificationError("only a formal run may return completion") + elif self.authentication is not None or self.execution.semantic_evidence is not None: + raise HostBaseQualificationError("non-completion cannot expose qualification evidence") + + +def _sha256(raw: bytes) -> str: + return hashlib.sha256(raw).hexdigest() + + +def _utc_now() -> str: + return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def _lower_sha256(value: object, label: str) -> str: + if ( + type(value) is not str + or len(value) != 64 + or value.lower() != value + or any(character not in "0123456789abcdef" for character in value) + ): + raise HostBaseQualificationError(f"{label} must be lowercase SHA-256 hex") + return value + + +def _canonical_json_bytes(value: object) -> bytes: + try: + return ( + json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + allow_nan=False, + ).encode("ascii") + + b"\n" + ) + except (TypeError, ValueError, OverflowError, RecursionError) as error: + raise HostBaseQualificationError("controller value is not canonical JSON") from error + + +def _without_duplicates(pairs: list[tuple[str, object]]) -> dict[str, object]: + result: dict[str, object] = {} + for key, value in pairs: + if key in result: + raise HostBaseQualificationError("registry JSON repeats a key") + result[key] = value + return result + + +def _parse_canonical_object(raw: bytes, label: str) -> dict[str, object]: + try: + value = json.loads( + raw, + object_pairs_hook=_without_duplicates, + parse_float=lambda _value: (_ for _ in ()).throw( + HostBaseQualificationError(f"{label} cannot contain floats") + ), + parse_constant=lambda _value: (_ for _ in ()).throw( + HostBaseQualificationError(f"{label} cannot contain constants") + ), + ) + except HostBaseQualificationError: + raise + except (UnicodeDecodeError, json.JSONDecodeError, RecursionError) as error: + raise HostBaseQualificationError(f"{label} is not valid bounded JSON") from error + if type(value) is not dict or raw != _canonical_json_bytes(value): + raise HostBaseQualificationError(f"{label} is not one canonical object") + return value + + +def _write_once(path: Path, raw: bytes) -> Path: + if type(raw) is not bytes or not raw or len(raw) > MAX_OUTPUT_BYTES: + raise HostBaseQualificationError("controller output bytes exceed their bound") + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL + for name in ("O_BINARY", "O_CLOEXEC", "O_NOFOLLOW"): + flags |= int(getattr(os, name, 0)) + try: + fd = os.open(path, flags, 0o600) + except OSError as error: + raise HostBaseQualificationError("controller output already exists or cannot be created") from error + try: + offset = 0 + while offset < len(raw): + written = os.write(fd, raw[offset:]) + if written <= 0: + raise OSError("short controller artifact write") + offset += written + os.fsync(fd) + finally: + os.close(fd) + if os.name == "posix": + directory_fd = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY | os.O_CLOEXEC) + try: + os.fsync(directory_fd) + finally: + os.close(directory_fd) + return path + + +def _write_json_once(path: Path, value: object) -> tuple[Path, bytes]: + raw = _canonical_json_bytes(value) + return _write_once(path, raw), raw + + +def _checked_directory(path: Path, label: str) -> Path: + if not isinstance(path, Path): + raise TypeError(f"{label} must be pathlib.Path") + try: + metadata = path.lstat() + except OSError as error: + raise HostBaseQualificationError(f"{label} is absent") from error + if not stat.S_ISDIR(metadata.st_mode) or path.is_symlink(): + raise HostBaseQualificationError(f"{label} must be a non-symlink directory") + return path + + +def load_base_attempt_registry( + path: Path, + *, + expected_sha256: str, +) -> tuple[tuple[resolver_module.BaseAttemptRegistryRow, ...], bytes, str]: + """Read one canonical private registry into the resolver's exact rows.""" + + expected = _lower_sha256(expected_sha256, "registry expected SHA-256") + try: + raw, digest = resolver_module._read_owned_file( + path, + label="base attempt registry", + maximum_bytes=MAX_REGISTRY_BYTES, + expected_sha256=expected, + ) + except (OSError, TypeError, ValueError, RuntimeError) as error: + raise HostBaseQualificationError("base attempt registry bytes are invalid") from error + value = _parse_canonical_object(raw, "base attempt registry") + if set(value) != { + "authority", + "production", + "result_claimed", + "rows", + "schema", + }: + raise HostBaseQualificationError("base attempt registry keys are not closed") + if ( + type(value["authority"]) is not int + or value["authority"] != 0 + or type(value["production"]) is not bool + or value["production"] is not False + or type(value["result_claimed"]) is not bool + or value["result_claimed"] is not False + or value["schema"] != REGISTRY_SCHEMA + or type(value["rows"]) is not list + ): + raise HostBaseQualificationError("base attempt registry header is invalid") + rows: list[resolver_module.BaseAttemptRegistryRow] = [] + for index, item in enumerate(value["rows"]): + if type(item) is not dict or set(item) != { + "attempt_id", + "run_id", + "terminal_outcome", + "terminal_sha256", + }: + raise HostBaseQualificationError(f"base attempt registry row {index} is invalid") + try: + rows.append(resolver_module.BaseAttemptRegistryRow(**item)) + except (TypeError, ValueError, RuntimeError) as error: + raise HostBaseQualificationError( + f"base attempt registry row {index} is invalid" + ) from error + try: + checked = resolver_module._checked_registry_rows(tuple(rows)) + except (TypeError, ValueError, RuntimeError) as error: + raise HostBaseQualificationError("base attempt registry census is invalid") from error + run_order = {run_id: index for index, run_id in enumerate(base_run_ids())} + if tuple(run_order[row.run_id] for row in checked) != tuple( + sorted(run_order[row.run_id] for row in checked) + ): + raise HostBaseQualificationError("base attempt registry rows changed registered run order") + for run_id in base_run_ids(): + history = tuple(row for row in checked if row.run_id == run_id) + if history[-1].terminal_outcome != "SUCCEEDED" or any( + row.terminal_outcome == "SUCCEEDED" for row in history[:-1] + ): + raise HostBaseQualificationError( + "each base registry history must end in its only successful attempt" + ) + return checked, raw, digest + + +def _validate_formal_latency_timer_support( + *, + _os_name: str | None = None, + _signal_api: object | None = None, + _threading_api: object | None = None, +) -> None: + """Check the real process timer boundary without mutating timer state.""" + + os_name = os.name if _os_name is None else _os_name + signal_api = signal if _signal_api is None else _signal_api + threading_api = threading if _threading_api is None else _threading_api + if os_name != "posix" or not all( + hasattr(signal_api, name) + for name in ( + "SIGALRM", + "ITIMER_REAL", + "getitimer", + "getsignal", + "setitimer", + "signal", + ) + ): + raise _FormalLatencyTimerHold( + "formal latency timeout requires POSIX interval timers" + ) + if not all( + callable(getattr(threading_api, name, None)) + for name in ("current_thread", "main_thread") + ): + raise _FormalLatencyTimerHold("formal latency timer thread API is unavailable") + try: + if threading_api.current_thread() is not threading_api.main_thread(): + raise _FormalLatencyTimerHold( + "formal latency timeout requires the Python main thread" + ) + prior_timer = signal_api.getitimer(signal_api.ITIMER_REAL) + signal_api.getsignal(signal_api.SIGALRM) + except _FormalLatencyTimerHold: + raise + except (AttributeError, OSError, RuntimeError, TypeError, ValueError) as error: + raise _FormalLatencyTimerHold( + "formal latency timer capability check failed" + ) from error + if prior_timer != (0.0, 0.0): + raise _FormalLatencyTimerHold( + "formal latency refuses an inherited process timer" + ) + + +@contextmanager +def _formal_latency_timeout( + seconds: int, + *, + _os_name: str | None = None, + _signal_api: object | None = None, + _threading_api: object | None = None, +) -> Iterator[None]: + """Install the process-local formal timeout; the outer hard-kill is separate.""" + + signal_api = signal if _signal_api is None else _signal_api + if type(seconds) is not int or not 1 <= seconds <= MAX_WALL_TIMEOUT_SECONDS: + raise HostBaseQualificationError("formal latency timeout is outside its bound") + _validate_formal_latency_timer_support( + _os_name=_os_name, + _signal_api=signal_api, + _threading_api=_threading_api, + ) + try: + prior_handler = signal_api.getsignal(signal_api.SIGALRM) + except (AttributeError, OSError, RuntimeError, TypeError, ValueError) as error: + raise _FormalLatencyTimerHold( + "formal latency timer handler could not be read" + ) from error + + def timeout_handler(_signum: int, _frame: object) -> None: + raise TimeoutError("formal base latency wall timeout expired") + + primary_error: BaseException | None = None + try: + try: + signal_api.signal(signal_api.SIGALRM, timeout_handler) + signal_api.setitimer(signal_api.ITIMER_REAL, float(seconds), 0.0) + except (AttributeError, OSError, RuntimeError, TypeError, ValueError) as error: + raise _FormalLatencyTimerHold( + "formal latency timer could not be installed" + ) from error + yield + except BaseException as error: + primary_error = error + raise + finally: + cleanup_error: BaseException | None = None + try: + signal_api.setitimer(signal_api.ITIMER_REAL, 0.0, 0.0) + except BaseException as error: + cleanup_error = error + try: + signal_api.signal(signal_api.SIGALRM, prior_handler) + except BaseException as error: + if cleanup_error is None: + cleanup_error = error + if primary_error is None and cleanup_error is not None: + if isinstance(cleanup_error, Exception): + raise _FormalLatencyTimerHold( + "formal latency timer could not be restored" + ) from cleanup_error + raise cleanup_error + + +def _journal_terminal_payload( + read: journal_module.LatencyJournalReadResult, +) -> dict[str, object]: + if not read.records: + raise HostBaseQualificationError("latency journal has no committed records") + value = _parse_canonical_object(read.records[-1].raw, "latency terminal event") + if value.get("event_type") != "TERMINAL" or type(value.get("payload")) is not dict: + raise HostBaseQualificationError("latency journal did not end in a terminal event") + return value["payload"] + + +def _scoring_precision(admission: resolver_module.BaseCohortAdmission) -> str: + try: + decision = training_module.resolve_precision( + training_module.TrainingConfig( + stage="base", + seed=1729, + device=admission.device, + request_bf16=admission.request_bf16, + bf16_runtime_qualified=admission.bf16_runtime_qualified, + edge_budget=admission.edge_budget, + checkpoint_every_updates=admission.checkpoint_every_updates, + synthetic_contract=False, + ) + ) + except (TypeError, ValueError, RuntimeError) as error: + raise HostBaseQualificationError( + "base scoring CUDA precision could not be resolved" + ) from error + return decision.mode + + +def _admit_scoring_cuda( + config: HostBaseQualificationConfig, + admission: resolver_module.BaseCohortAdmission, + *, + _runtime: object | None = None, +) -> _ScoringCudaState: + """Apply the latency UUID/free/cap admission before any capture scoring.""" + + runtime = ( + latency_module._NvidiaSmiTorchRuntime(config.registered_cuda_uuid) + if _runtime is None + else _runtime + ) + try: + device = latency_module.torch.device(admission.device) + if device.type != "cuda": + raise HostBaseQualificationError("base capture scoring requires CUDA") + observed = runtime.snapshot() + if type(observed) is not latency_module.SharedCudaObservation: + raise _ScoringCudaHold( + "RUNTIME_DRIFT", + "base scoring admission snapshot is malformed", + ) + latency_module._validate_snapshot( + observed, + registered_uuid=config.registered_cuda_uuid, + runtime=None, + ) + if observed.memory_free_mib < latency_module.MINIMUM_FREE_MIB: + raise _ScoringCudaHold( + "RESOURCE_LIMIT", + "base scoring admission has less than 8 GiB free" + ) + identity = runtime.initialize( + device=device, + precision_mode=_scoring_precision(admission), + admission_observation=observed, + ) + if type(identity) is not latency_module.BaseLatencyRuntimeIdentity: + raise _ScoringCudaHold( + "RUNTIME_DRIFT", + "base scoring CUDA identity is malformed", + ) + post_context = runtime.snapshot() + if type(post_context) is not latency_module.SharedCudaObservation: + raise _ScoringCudaHold( + "RUNTIME_DRIFT", + "base scoring post-context snapshot is malformed", + ) + latency_module._validate_snapshot( + post_context, + registered_uuid=config.registered_cuda_uuid, + runtime=identity, + ) + if post_context.memory_free_mib < latency_module.MINIMUM_FREE_MIB: + raise _ScoringCudaHold( + "RESOURCE_LIMIT", + "base scoring CUDA context has less than 8 GiB free" + ) + except (HostBaseQualificationError, _ScoringCudaHold): + raise + except (TypeError, ValueError, RuntimeError) as error: + raise _ScoringCudaHold( + latency_module._failure_code(error), + "base scoring CUDA admission failed", + ) from error + return _ScoringCudaState( + runtime=runtime, + admission_observation=observed, + runtime_identity=identity, + post_context_observation=post_context, + ) + + +def _strip_exception_tracebacks(error: BaseException) -> None: + """Release scorer frames while retaining exact exception objects and chaining.""" + + pending: list[BaseException] = [error] + seen: set[int] = set() + while pending: + current = pending.pop() + if id(current) in seen: + continue + seen.add(id(current)) + current.__traceback__ = None + for linked in (current.__cause__, current.__context__): + if linked is not None: + pending.append(linked) + + +def _release_scoring_cuda( + state: _ScoringCudaState, + *, + _torch_api: object | None = None, +) -> latency_module.SharedCudaObservation: + """Clear scorer objects/workspaces, then retain the exact-zero latency gate.""" + + torch_api = latency_module.torch if _torch_api is None else _torch_api + gc.collect() + try: + try: + device = torch_api.device(state.runtime_identity.device) + except (AttributeError, TypeError, ValueError, RuntimeError) as error: + raise _ScoringCudaHold( + "RUNTIME_DRIFT", + "base scoring CUDA release device is malformed", + ) from error + state.runtime.synchronize(device) + clear_workspaces = getattr( + getattr(torch_api, "_C", None), + "_cuda_clearCublasWorkspaces", + None, + ) + if not callable(clear_workspaces): + raise _ScoringCudaHold( + "RUNTIME_DRIFT", + "Torch CUDA cuBLAS workspace clear API is unavailable after scoring" + ) + clear_workspaces() + state.runtime.synchronize(device) + torch_api.cuda.empty_cache() + allocated = int(torch_api.cuda.memory_allocated(device)) + reserved = int(torch_api.cuda.memory_reserved(device)) + if allocated != 0 or reserved != 0: + raise _ScoringCudaHold( + "RESOURCE_LIMIT", + "base scoring CUDA allocation remained before latency" + ) + after = state.runtime.snapshot() + if type(after) is not latency_module.SharedCudaObservation: + raise _ScoringCudaHold( + "RUNTIME_DRIFT", + "base scoring release snapshot is malformed", + ) + latency_module._validate_snapshot( + after, + registered_uuid=state.runtime_identity.device_uuid, + runtime=state.runtime_identity, + ) + if after.memory_free_mib < latency_module.MINIMUM_FREE_MIB: + raise _ScoringCudaHold( + "RESOURCE_LIMIT", + "base scoring release has less than 8 GiB free" + ) + except (HostBaseQualificationError, _ScoringCudaHold): + raise + except (AttributeError, TypeError, ValueError, RuntimeError) as error: + raise _ScoringCudaHold( + latency_module._failure_code(error), + "base scoring CUDA release failed", + ) from error + return after + + +def _failure_code(error: BaseException, stage: str) -> tuple[str, str]: + if isinstance(error, _NonformalRuntimeHold): + return "HELD", "NONFORMAL_TEST_RUNTIME" + if isinstance(error, _ScoringCudaHold): + return "HELD", f"BASE_SCORING_CUDA_{error.failure_code}" + if isinstance(error, latency_module.BaseCohortLatencyHold): + return "HELD", f"LATENCY_{error.failure_code}" + if isinstance(error, _FormalLatencyTimerHold): + return "HELD", "LATENCY_RUNTIME_DRIFT" + if isinstance(error, (TimeoutError, journal_module.LatencyJournalWriteError)): + return "HELD", "LATENCY_OBSERVATION_INCOMPLETE" + if isinstance(error, resolver_module.BaseCohortResolutionError): + return "HELD", "BASE_COHORT_INPUT_INVALID" + if isinstance(error, HostBaseQualificationError) and stage in { + "REGISTRY", + "RESOLVE", + }: + return "HELD", "BASE_COHORT_INPUT_INVALID" + chain: list[BaseException] = [] + current: BaseException | None = error + seen: set[int] = set() + while current is not None and id(current) not in seen: + seen.add(id(current)) + chain.append(current) + current = current.__cause__ + if any( + isinstance(item, (CaptureValidationResourceLimit, latency_module.torch.cuda.OutOfMemoryError)) + for item in chain + ): + return "HELD", "BASE_CAPTURE_SCORE_RESOURCE_LIMIT" + if isinstance(error, scoring_module.BaseCohortValidationError): + return "FAILED", "BASE_CAPTURE_SCORE_FAILED" + if isinstance(error, assembly_module.BaseCohortQualificationError): + return "FAILED", "BASE_QUALIFICATION_ASSEMBLY_FAILED" + return "FAILED", "BASE_QUALIFICATION_CONTROLLER_FAILED" + + +def _persist_failure( + output_directory: Path, + *, + error: BaseException, + stage: str, + registry_sha256: str, + journal_path: Path | None, + journal_sha256: str | None, + secondary_cleanup_error_type: str | None, + completed_at_utc: str, +) -> tuple[production_module.BackendExecution, Path]: + outcome, failure_code = _failure_code(error, stage) + path, _raw = _write_json_once( + output_directory / "failure.json", + { + "authority": 0, + "completed_at_utc": completed_at_utc, + "error_type": type(error).__qualname__, + "failure_code": failure_code, + "formal_qualification_completed": False, + "journal_present": journal_path is not None, + "journal_read_sha256": journal_sha256, + "production": False, + "registry_sha256": registry_sha256, + "result_claimed": False, + "schema": FAILURE_SCHEMA, + "secondary_cleanup_error_type": secondary_cleanup_error_type, + "stage": stage, + "status": outcome, + }, + ) + return ( + production_module.BackendExecution( + outcome=outcome, + completed_at_utc=completed_at_utc, + artifacts=(path,), + ), + path, + ) + + +def run_host_base_qualification( + attempt_root: Path, + *, + config: HostBaseQualificationConfig, + admission: resolver_module.BaseCohortAdmission, + validation_source: CaptureValidationSource, + output_directory: Path, + adapter_sha256: str, + _test_runtime: object | None = None, +) -> HostBaseQualificationRun: + """Execute the actual chain; an injected runtime can only return nonformal HOLD.""" + + if type(config) is not HostBaseQualificationConfig: + raise TypeError("config must be exact HostBaseQualificationConfig") + if type(admission) is not resolver_module.BaseCohortAdmission: + raise TypeError("admission must be exact BaseCohortAdmission") + if type(validation_source) is not CaptureValidationSource: + raise TypeError("validation_source must be exact CaptureValidationSource") + _lower_sha256(adapter_sha256, "adapter_sha256") + root = _checked_directory(attempt_root, "base attempt root") + output = _checked_directory(output_directory, "qualification output directory") + output_root = _checked_directory( + config.output_root, + "base qualification output root", + ) + journal_parent = _checked_directory( + config.journal_parent, + "latency journal parent", + ) + if len({root.resolve(), output_root.resolve(), journal_parent.resolve()}) != 3: + raise HostBaseQualificationError( + "attempt, qualification-output, and journal roots must be distinct" + ) + if output.resolve() != (output_root / config.session_id).resolve(): + raise HostBaseQualificationError( + "qualification output directory differs from its frozen session path" + ) + formal = _test_runtime is None + stage = "LATENCY_PREFLIGHT" if formal else "REGISTRY" + registry_digest = config.registry_sha256 + journal_path: Path | None = None + journal_sha256: str | None = None + scoring_cuda_path: Path | None = None + scoring_cuda_raw: bytes | None = None + secondary_cleanup_error_type: str | None = None + start_path: Path | None = None + try: + if formal: + _validate_formal_latency_timer_support() + stage = "REGISTRY" + registry_rows, registry_raw, registry_digest = load_base_attempt_registry( + config.registry_path, + expected_sha256=config.registry_sha256, + ) + start_path, _ = _write_json_once( + output / "controller-start.json", + { + "adapter_sha256": adapter_sha256, + "authority": 0, + "formal_execution_requested": formal, + "journal_binding": { + "latency_protocol_sha256": config.latency_protocol_sha256, + "launcher_sha256": config.launcher_sha256, + "registered_cuda_uuid": config.registered_cuda_uuid, + }, + "production": False, + "registry_sha256": registry_digest, + "result_claimed": False, + "schema": START_SCHEMA, + "session_id": config.session_id, + "started_at_utc": _utc_now(), + "source_tree_sha256": admission.source_tree_sha256, + "train_manifest_sha256": admission.train_manifest_sha256, + "val_census_sha256": admission.val_manifest_sha256, + "wall_timeout_seconds": config.wall_timeout_seconds, + }, + ) + stage = "RESOLVE" + cohort = resolver_module.resolve_completed_base_cohort( + root, + registry_rows=registry_rows, + admission=admission, + ) + stage = "SCORE" + if formal: + scoring_state = _admit_scoring_cuda(config, admission) + score_error: Exception | None = None + scored = None + try: + scored = scoring_module.score_resolved_base_cohort( + cohort, + admission=admission, + validation_source=validation_source, + ) + except Exception as error: + score_error = error + _strip_exception_tracebacks(error) + release_error: Exception | None = None + try: + post_score_observation = _release_scoring_cuda(scoring_state) + except Exception as error: + release_error = error + _strip_exception_tracebacks(error) + if score_error is not None: + if release_error is not None: + secondary_cleanup_error_type = type(release_error).__qualname__ + raise score_error + if release_error is not None: + raise release_error + if type(scored) is not scoring_module.BaseCohortValidationResult: + raise HostBaseQualificationError( + "base scorer returned an invalid cohort result" + ) + if any( + row.device != scoring_state.runtime_identity.device + or row.precision_mode != scoring_state.runtime_identity.precision_mode + for row in scored.observations + ): + raise HostBaseQualificationError( + "base scorer device or precision differs from CUDA admission" + ) + scoring_cuda_path, scoring_cuda_raw = _write_json_once( + output / "scoring-cuda.json", + { + "admission_observation": scoring_state.admission_observation.payload(), + "allocator_limit_bytes": latency_module.CUDA_ALLOCATOR_LIMIT_BYTES, + "authority": 0, + "post_context_observation": ( + scoring_state.post_context_observation.payload() + ), + "post_score_observation": post_score_observation.payload(), + "production": False, + "registered_cuda_uuid": config.registered_cuda_uuid, + "result_claimed": False, + "runtime_identity": asdict(scoring_state.runtime_identity), + "schema": SCORING_CUDA_SCHEMA, + "scored_cohort_sha256": scored.sha256, + "status": "RESOURCE_BOUND_SCORING_COMPLETED", + }, + ) + else: + scored = scoring_module.score_resolved_base_cohort( + cohort, + admission=admission, + validation_source=validation_source, + ) + bindings = journal_module.LatencyJournalBindings( + resolved_cohort_sha256=latency_module._resolved_cohort_sha256(cohort), + scored_cohort_sha256=scored.sha256, + source_tree_sha256=cohort.source_tree_sha256, + latency_protocol_sha256=config.latency_protocol_sha256, + wrapper_sha256=config.launcher_sha256, + registered_cuda_uuid=config.registered_cuda_uuid, + ) + stage = "LATENCY" + with journal_module.create_latency_progress_journal( + config.journal_parent, + session_id=config.session_id, + bindings=bindings, + ) as writer: + journal_path = writer.path + with _formal_latency_timeout(config.wall_timeout_seconds): + if _test_runtime is None: + latency = latency_module.run_base_cohort_latency_session( + cohort, + admission=admission, + scored_cohort=scored, + registered_cuda_uuid=config.registered_cuda_uuid, + observer=writer, + ) + else: + latency = latency_module.run_base_cohort_latency_session( + cohort, + admission=admission, + scored_cohort=scored, + registered_cuda_uuid=config.registered_cuda_uuid, + observer=writer, + _runtime=_test_runtime, + ) + read = journal_module.read_latency_progress_journal( + journal_path, + expected_bindings=bindings, + ) + journal_sha256 = read.sha256 + terminal_payload = _journal_terminal_payload(read) + if ( + read.status != journal_module.COMPLETED_STATUS + or read.terminal_outcome != "COMPLETED" + or terminal_payload.get("outcome") != "COMPLETED" + or terminal_payload.get("failure_code") is not None + or terminal_payload.get("latency_session_sha256") != latency.sha256 + ): + raise HostBaseQualificationError( + "latency call result and durable terminal do not form one completion" + ) + if not formal: + raise _NonformalRuntimeHold( + "injected latency runtime is an engineering observation, not qualification" + ) + if ( + type(latency) is not latency_module.BaseCohortLatencySession + or latency.actual_runtime is not True + or latency.complete is not True + or latency.formal_samples_complete is not True + or latency.failure_code is not None + or latency.status != latency_module.ACTUAL_COMPLETE_STATUS + ): + raise HostBaseQualificationError("formal latency session is incomplete") + stage = "ASSEMBLY" + assembled = assembly_module.assemble_base_cohort_qualification( + cohort, + admission=admission, + scored_cohort=scored, + latency_session=latency, + ) + if _sha256(registry_raw) != registry_digest: + raise HostBaseQualificationError("owned registry bytes changed in memory") + registry_after, registry_after_digest = resolver_module._read_owned_file( + config.registry_path, + label="base attempt registry", + maximum_bytes=MAX_REGISTRY_BYTES, + expected_sha256=registry_digest, + ) + if registry_after != registry_raw or registry_after_digest != registry_digest: + raise HostBaseQualificationError("base attempt registry changed during qualification") + if ( + validation_source.census_sha256 != admission.val_manifest_sha256 + or validation_source.manifest_sha256 != scored.source_manifest_sha256 + or latency.sha256 != assembled.dependency_binding.latency_session_sha256 + ): + raise HostBaseQualificationError("qualification source or latency changed after assembly") + stage = "PERSIST" + if scoring_cuda_path is None or scoring_cuda_raw is None: + raise AssertionError("formal qualification lost scoring CUDA evidence") + qualification_path = _write_once(output / "base-qualification.json", assembled.artifact) + dependency_path, dependency_raw = _write_json_once( + output / "dependency-binding.json", + asdict(assembled.dependency_binding), + ) + grant_path, grant_raw = _write_json_once( + output / "grant-basis.json", + { + "adapter_sha256": adapter_sha256, + "assembly_sha256": assembled.sha256, + "authority": 0, + "dependency_binding_sha256": assembled.dependency_binding.sha256, + "journal_read_sha256": journal_sha256, + "latency_session_sha256": latency.sha256, + "production": False, + "qualification_sha256": _sha256(assembled.artifact), + "registry_sha256": registry_digest, + "result_claimed": False, + "row_binding_sha256s": [row.sha256 for row in assembled.row_bindings], + "schema": GRANT_BASIS_SCHEMA, + "scoring_cuda_sha256": _sha256(scoring_cuda_raw), + }, + ) + authorization = production_module.PrivateBaseQualificationAuthorization( + authorized_at_utc=_utc_now(), + external_grant_sha256=_sha256(grant_raw), + validation_manifest_sha256=assembled.qualification.validation_manifest_sha256, + query_census_sha256=assembled.qualification.query_census_sha256, + evaluator_sha256=assembled.qualification.evaluator_sha256, + score_rows_sha256=assembled.qualification.score_rows_sha256, + ) + authorization_path, authorization_raw = _write_json_once( + output / "authorization.json", + { + **asdict(authorization), + "authority": 0, + "production": False, + "result_claimed": False, + "schema": AUTHORIZATION_SCHEMA, + }, + ) + qualification_sha256 = _sha256(assembled.artifact) + completed_at_utc = _utc_now() + completion_path, _completion_raw = _write_json_once( + output / "completion.json", + { + "artifact_sha256s": { + "authorization": _sha256(authorization_raw), + "dependency_binding": _sha256(dependency_raw), + "grant_basis": _sha256(grant_raw), + "qualification": qualification_sha256, + "scoring_cuda": _sha256(scoring_cuda_raw), + }, + "assembly_sha256": assembled.sha256, + "authority": 0, + "completed_at_utc": completed_at_utc, + "journal_read_sha256": journal_sha256, + "production": False, + "registry_sha256": registry_digest, + "result_claimed": False, + "schema": COMPLETION_SCHEMA, + "status": "COMPLETED_PENDING_EXISTING_ADAPTER_RECOMPUTATION", + }, + ) + authentication = HostBaseQualificationAuthentication( + authorization=authorization, + qualification_sha256=qualification_sha256, + adapter_sha256=adapter_sha256, + grant_path=grant_path, + qualification_path=qualification_path, + grant_raw=grant_raw, + qualification_raw=assembled.artifact, + ) + execution = production_module.BackendExecution( + outcome="COMPLETED", + completed_at_utc=completed_at_utc, + artifacts=( + qualification_path, + scoring_cuda_path, + dependency_path, + grant_path, + authorization_path, + completion_path, + ), + semantic_evidence=assembled.execution_evidence, + ) + return HostBaseQualificationRun( + execution=execution, + authentication=authentication, + journal_path=journal_path, + registry_sha256=registry_digest, + formal=True, + ) + except Exception as error: + if journal_path is not None and journal_path.exists(): + try: + # A structurally complete terminal remains nonformal unless it + # agrees with the actual call outcome consumed above. + if "bindings" in locals(): + observed = journal_module.read_latency_progress_journal( + journal_path, + expected_bindings=bindings, + ) + journal_sha256 = observed.sha256 + except (OSError, TypeError, ValueError, RuntimeError): + journal_sha256 = None + execution, _failure_path = _persist_failure( + output, + error=error, + stage=stage, + registry_sha256=registry_digest, + journal_path=journal_path, + journal_sha256=journal_sha256, + secondary_cleanup_error_type=secondary_cleanup_error_type, + completed_at_utc=_utc_now(), + ) + prefix = tuple( + path for path in (start_path, scoring_cuda_path) if path is not None + ) + artifacts = (*prefix, *execution.artifacts) + return HostBaseQualificationRun( + execution=production_module.BackendExecution( + outcome=execution.outcome, + completed_at_utc=execution.completed_at_utc, + artifacts=artifacts, + ), + authentication=None, + journal_path=journal_path, + registry_sha256=registry_digest, + formal=formal, + ) + + +__all__ = [ + "CONFIG_SCHEMA", + "HostBaseQualificationAuthentication", + "HostBaseQualificationConfig", + "HostBaseQualificationError", + "HostBaseQualificationRun", + "MAX_WALL_TIMEOUT_SECONDS", + "REGISTRY_SCHEMA", + "load_base_attempt_registry", + "run_host_base_qualification", +] diff --git a/src/phaseset_core/calibration_populations.py b/src/phaseset_core/calibration_populations.py new file mode 100644 index 0000000..f887aae --- /dev/null +++ b/src/phaseset_core/calibration_populations.py @@ -0,0 +1,91 @@ +"""Training populations for V2 main, pilot and independent outer folds. + +Read the user-supplied matrix rather than inventing component allocations. +This does not make that planned training matrix launchable or fit any data. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +from .directional_calibration import SEALED_EMBODY_COMPONENTS + + +@dataclass(frozen=True) +class CalibrationPopulation: + name: str + training_components: tuple[str, ...] + validation_components: tuple[str, ...] + held_out_components: tuple[str, ...] + + +def calibration_populations(matrix: dict[str, Any]) -> tuple[CalibrationPopulation, ...]: + """Derive five exact populations, rejecting inconsistent held-out routes. + + The complete eligible capture census must additionally be established + before fitting; this function only translates the registered components. + Main validation C00 is training data in folds 1/2, never in main/pilot. + """ + if matrix["schema"] != "phaseset-v2-training-matrix-v1": + raise ValueError("V2 matrix schema differs") + groups = matrix["pilot_and_group_fold_components"] + + def components(values): + if ( + not values + or any(not isinstance(value, str) or not value for value in values) + or len(set(values)) != len(values) + or set(values) & SEALED_EMBODY_COMPONENTS + ): + raise ValueError("development components must be unique and exclude sealed test") + return set(values) + + development = components(groups["development"]) + main_validation = components(matrix["primary_split"]["validation_component_labels"]) + pilot_train = components(groups["pilot_train"]) + pilot_validation = components(groups["pilot_validation"]) + fold_validation = components(groups["fold_validation"]) + if ( + not (main_validation | pilot_train | pilot_validation | fold_validation) <= development + or pilot_train & pilot_validation + or fold_validation != pilot_validation + or groups["reuse_main_checkpoints"] is not False + ): + raise ValueError("pilot/main/fold development routes differ") + populations = [ + CalibrationPopulation( + "main", tuple(sorted(development - main_validation)), tuple(sorted(main_validation)), () + ), + CalibrationPopulation( + "pilot", tuple(sorted(pilot_train)), tuple(sorted(pilot_validation)), () + ), + ] + held_union: set[str] = set() + folds = groups["folds"] + if len(folds) != 3 or {fold["id"] for fold in folds} != {"fold_0", "fold_1", "fold_2"}: + raise ValueError("exactly three distinct registered outer folds are required") + for fold in sorted(folds, key=lambda value: value["id"]): + held, training = components(fold["held_out"]), components(fold["train"]) + if ( + not held <= development + or held & (fold_validation | pilot_train | held_union) + or training != development - held - fold_validation + ): + raise ValueError( + "outer fold training overlaps or differs from the registered complement" + ) + held_union |= held + populations.append( + CalibrationPopulation( + fold["id"], + tuple(sorted(training)), + tuple(sorted(fold_validation)), + tuple(sorted(held)), + ) + ) + if held_union != development - pilot_train - fold_validation: + raise ValueError( + "outer held-out components do not cover the registered evaluation population" + ) + return tuple(populations) diff --git a/src/phaseset_core/caption_role_provenance.py b/src/phaseset_core/caption_role_provenance.py new file mode 100644 index 0000000..905e8a1 --- /dev/null +++ b/src/phaseset_core/caption_role_provenance.py @@ -0,0 +1,871 @@ +"""Authority-zero binding for a candidate machine-fused window training policy. + +This private candidate is a consistency validator, not a policy selection, +rights authenticator, semantic-equivalence review, or training authorization. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +import hashlib +import json +from typing import Final + +import torch + +from phaseset_core.caption_fusion import ( + ANNOTATION_KIND, + TASK_SCOPE, + CaptionFusionInput, + CaptionFusionReceipt, + MachineFusedCaption, + _input_digest as _caption_input_digest, + _output_digest as _caption_output_digest, + canonical_caption_fusion_receipt_bytes, +) +from phaseset_core.contracts import group_commitment +from phaseset_core.prepared_data_v2 import ( + PreparedMotionTextBlock, + PreparedTrainingDataSourceV2, +) +from phaseset_core.pipeline import PreparedGroupSample +from phaseset_core.training import RetrievalTrainingBatch + + +SCHEMA: Final = "phaseset-private-caption-role-validation-v2" +WINDOW_SUMMARY_SCHEMA: Final = "phaseset-private-window-manifest-content-summary-v1" +CANDIDATE_POLICY_ID: Final = "candidate-machine-fused-window-supervision-v1" +STATUS: Final = "VALIDATED_CANDIDATE_NOT_SELECTED_AUTHORITY0" +CAPTION_ROLES: Final = ("group_description", "semantic_paraphrase") +INSPECTION_SEED: Final = 1729 +MAX_WINDOWS: Final = 16_384 +MAX_CAPTIONS: Final = 32_768 +MAX_MICROBATCHES: Final = 16_384 +MAX_EMBEDDING_BYTES: Final = 64 * 1024 * 1024 + + +class CaptionRoleProvenanceError(ValueError): + """The proposed private supervision binding is inconsistent.""" + + +def _sha256_bytes(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def _lower_sha256(value: object, label: str) -> str: + if ( + type(value) is not str + or len(value) != 64 + or value.lower() != value + or any(character not in "0123456789abcdef" for character in value) + ): + raise CaptionRoleProvenanceError(f"{label} must be lowercase SHA-256 hex") + return value + + +def _raw32(value: object, label: str) -> bytes: + if type(value) is not bytes or len(value) != 32: + raise CaptionRoleProvenanceError(f"{label} must be exact bytes[32]") + return value + + +def _canonical_json(value: object) -> bytes: + return ( + json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + allow_nan=False, + ).encode("ascii") + + b"\n" + ) + + +def _manifest_digest(domain: bytes, values: tuple[str, ...]) -> str: + digest = hashlib.sha256(domain) + digest.update(len(values).to_bytes(8, "big")) + for value in values: + raw = value.encode("ascii") + digest.update(len(raw).to_bytes(8, "big")) + digest.update(raw) + return digest.hexdigest() + + +def _tensor_bytes(value: torch.Tensor) -> bytes: + return value.detach().cpu().contiguous().view(torch.uint8).numpy().tobytes() + + +@dataclass(frozen=True, slots=True, repr=False) +class ActorCaptionContentSummary: + """Verified-manifest content digest for one same-window actor caption.""" + + actor_commitment: bytes = field(repr=False) + source_record_sha256: str = field(repr=False) + text_utf8_sha256: str = field(repr=False) + + def __post_init__(self) -> None: + object.__setattr__( + self, + "actor_commitment", + _raw32(self.actor_commitment, "actor caption commitment"), + ) + _lower_sha256(self.source_record_sha256, "actor caption source record sha256") + _lower_sha256(self.text_utf8_sha256, "actor caption text sha256") + + def _private_dict(self) -> dict[str, str]: + return { + "actor_commitment": self.actor_commitment.hex(), + "source_record_sha256": self.source_record_sha256, + "text_utf8_sha256": self.text_utf8_sha256, + } + + def __repr__(self) -> str: + return "ActorCaptionContentSummary()" + + +@dataclass(frozen=True, slots=True, repr=False) +class HolisticContextContentSummary: + """Verified-manifest content digest for optional same-window context.""" + + source_record_sha256: str = field(repr=False) + text_utf8_sha256: str = field(repr=False) + + def __post_init__(self) -> None: + _lower_sha256(self.source_record_sha256, "holistic source record sha256") + _lower_sha256(self.text_utf8_sha256, "holistic text sha256") + + def _private_dict(self) -> dict[str, str]: + return { + "source_record_sha256": self.source_record_sha256, + "text_utf8_sha256": self.text_utf8_sha256, + } + + def __repr__(self) -> str: + return "HolisticContextContentSummary()" + + +@dataclass(frozen=True, slots=True, repr=False) +class WindowManifestRow: + """Private content summary of one already-verified train-window row.""" + + window_sha256: str + window_commitment: bytes = field(repr=False) + actor_commitments: tuple[bytes, ...] = field(repr=False) + group_commitment: bytes = field(repr=False) + source_sha256: str = field(repr=False) + source_start_frame: int + actor_caption_summaries: tuple[ActorCaptionContentSummary, ...] = field(repr=False) + holistic_context_summary: HolisticContextContentSummary | None = field( + default=None, + repr=False, + ) + + def __post_init__(self) -> None: + _lower_sha256(self.window_sha256, "window_sha256") + window = _raw32(self.window_commitment, "window_commitment") + if type(self.actor_commitments) is not tuple or len(self.actor_commitments) < 3: + raise CaptionRoleProvenanceError( + "window summary requires an exact tuple of at least three actors" + ) + actors = tuple( + _raw32(value, f"actor_commitments[{index}]") + for index, value in enumerate(self.actor_commitments) + ) + if len(set(actors)) != len(actors): + raise CaptionRoleProvenanceError("window summary repeats an actor commitment") + if ( + type(self.actor_caption_summaries) is not tuple + or len(self.actor_caption_summaries) != len(actors) + or any( + type(value) is not ActorCaptionContentSummary + for value in self.actor_caption_summaries + ) + ): + raise CaptionRoleProvenanceError( + "window summary actor-caption census is inconsistent" + ) + actor_caption_summaries = tuple( + sorted(self.actor_caption_summaries, key=lambda value: value.actor_commitment) + ) + if tuple(value.actor_commitment for value in actor_caption_summaries) != tuple( + sorted(actors) + ): + raise CaptionRoleProvenanceError( + "window summary actor-caption lineage is inconsistent" + ) + if self.holistic_context_summary is not None and type( + self.holistic_context_summary + ) is not HolisticContextContentSummary: + raise CaptionRoleProvenanceError( + "holistic_context_summary must be exact or absent" + ) + group = _raw32(self.group_commitment, "group_commitment") + if group_commitment(actors) != group: + raise CaptionRoleProvenanceError("window summary group commitment is inconsistent") + _lower_sha256(self.source_sha256, "source_sha256") + if type(self.source_start_frame) is not int or self.source_start_frame < 0: + raise CaptionRoleProvenanceError( + "source_start_frame must be an exact nonnegative int" + ) + object.__setattr__(self, "window_commitment", window) + object.__setattr__(self, "actor_commitments", actors) + object.__setattr__(self, "group_commitment", group) + object.__setattr__(self, "actor_caption_summaries", actor_caption_summaries) + + def _private_dict(self) -> dict[str, object]: + return { + "actor_commitments": [value.hex() for value in self.actor_commitments], + "actor_caption_summaries": [ + value._private_dict() for value in self.actor_caption_summaries + ], + "group_commitment": self.group_commitment.hex(), + "source_sha256": self.source_sha256, + "source_start_frame": self.source_start_frame, + "holistic_context_summary": ( + None + if self.holistic_context_summary is None + else self.holistic_context_summary._private_dict() + ), + "window_commitment": self.window_commitment.hex(), + "window_sha256": self.window_sha256, + } + + def __repr__(self) -> str: + return "WindowManifestRow()" + + +@dataclass(frozen=True, slots=True, repr=False) +class WindowManifestContentSummary: + """Content extracted from one verified private train-window manifest.""" + + manifest_sha256: str + rows: tuple[WindowManifestRow, ...] = field(repr=False) + split: str = "train" + schema: str = WINDOW_SUMMARY_SCHEMA + + def __post_init__(self) -> None: + _lower_sha256(self.manifest_sha256, "window manifest sha256") + if self.split != "train" or self.schema != WINDOW_SUMMARY_SCHEMA: + raise CaptionRoleProvenanceError("window summary schema or split is not registered") + if ( + type(self.rows) is not tuple + or not self.rows + or len(self.rows) > MAX_WINDOWS + or any(type(row) is not WindowManifestRow for row in self.rows) + ): + raise CaptionRoleProvenanceError("window summary rows are invalid or exceed the bound") + windows = tuple(row.window_sha256 for row in self.rows) + commitments = tuple(row.window_commitment for row in self.rows) + if len(set(windows)) != len(windows) or len(set(commitments)) != len(commitments): + raise CaptionRoleProvenanceError("window summary repeats a window identity") + + @property + def content_sha256(self) -> str: + return _sha256_bytes( + _canonical_json( + { + "manifest_sha256": self.manifest_sha256, + "rows": [row._private_dict() for row in self.rows], + "schema": self.schema, + "split": self.split, + } + ) + ) + + def __repr__(self) -> str: + return ( + f"WindowManifestContentSummary(windows={len(self.rows)}, " + ")" + ) + + +@dataclass(frozen=True, slots=True, repr=False) +class WindowFusionHandoff: + """Ephemeral fusion-to-CLIP handoff retained for one private validation.""" + + fusion_input: CaptionFusionInput = field(repr=False) + fused_caption: MachineFusedCaption = field(repr=False) + clip_caption_commitments: tuple[bytes, bytes] = field(repr=False) + + def __post_init__(self) -> None: + if type(self.fusion_input) is not CaptionFusionInput: + raise CaptionRoleProvenanceError("fusion_input must be exact CaptionFusionInput") + if type(self.fused_caption) is not MachineFusedCaption: + raise CaptionRoleProvenanceError("fused_caption must be exact MachineFusedCaption") + if ( + type(self.clip_caption_commitments) is not tuple + or len(self.clip_caption_commitments) != 2 + ): + raise CaptionRoleProvenanceError( + "clip_caption_commitments must contain exactly two rows" + ) + checked = tuple( + _raw32(value, f"clip_caption_commitments[{index}]") + for index, value in enumerate(self.clip_caption_commitments) + ) + if len(set(checked)) != 2: + raise CaptionRoleProvenanceError("fusion handoff repeats a caption commitment") + object.__setattr__(self, "clip_caption_commitments", checked) + + def __repr__(self) -> str: + return "WindowFusionHandoff()" + + +@dataclass(frozen=True, slots=True) +class CaptionRoleValidationReceipt: + """Canonical private digest receipt with no authority or policy selection.""" + + window_manifest_sha256: str + window_manifest_content_sha256: str + prepared_train_manifest_sha256: str + dataset_manifest_sha256: str + caption_manifest_sha256: str + human_text_rights_assertion_sha256: str + row_binding_manifest_sha256: str + fusion_receipt_manifest_sha256: str + clip_receipt_manifest_sha256: str + positive_family_manifest_sha256: str + window_count: int + caption_count: int + microbatch_count: int + inspection_seed: int = INSPECTION_SEED + candidate_policy_id: str = CANDIDATE_POLICY_ID + formal_supervision_policy_selected: bool = False + rights_authenticated_by_this_receipt: bool = False + semantic_equivalence_proved: bool = False + training_authorized: bool = False + scientific_result_claimed: bool = False + authority: int = 0 + status: str = STATUS + schema: str = SCHEMA + + def canonical_json_bytes(self) -> bytes: + for name in ( + "window_manifest_sha256", + "window_manifest_content_sha256", + "prepared_train_manifest_sha256", + "dataset_manifest_sha256", + "caption_manifest_sha256", + "human_text_rights_assertion_sha256", + "row_binding_manifest_sha256", + "fusion_receipt_manifest_sha256", + "clip_receipt_manifest_sha256", + "positive_family_manifest_sha256", + ): + _lower_sha256(getattr(self, name), name) + if ( + type(self.window_count) is not int + or not 1 <= self.window_count <= MAX_WINDOWS + or type(self.caption_count) is not int + or self.caption_count != 2 * self.window_count + or type(self.microbatch_count) is not int + or not 1 <= self.microbatch_count <= MAX_MICROBATCHES + or type(self.inspection_seed) is not int + or self.inspection_seed != INSPECTION_SEED + or self.formal_supervision_policy_selected is not False + or self.rights_authenticated_by_this_receipt is not False + or self.semantic_equivalence_proved is not False + or self.training_authorized is not False + or self.scientific_result_claimed is not False + or type(self.authority) is not int + or self.authority != 0 + ): + raise CaptionRoleProvenanceError( + "validation receipt census or frozen field type is invalid" + ) + frozen = { + "authority": 0, + "candidate_policy_id": CANDIDATE_POLICY_ID, + "formal_supervision_policy_selected": False, + "rights_authenticated_by_this_receipt": False, + "schema": SCHEMA, + "scientific_result_claimed": False, + "semantic_equivalence_proved": False, + "status": STATUS, + "training_authorized": False, + } + for name, expected in frozen.items(): + if getattr(self, name) != expected: + raise CaptionRoleProvenanceError(f"receipt frozen field {name} changed") + return _canonical_json( + { + **frozen, + "caption_count": self.caption_count, + "caption_manifest_sha256": self.caption_manifest_sha256, + "clip_receipt_manifest_sha256": self.clip_receipt_manifest_sha256, + "dataset_manifest_sha256": self.dataset_manifest_sha256, + "fusion_receipt_manifest_sha256": self.fusion_receipt_manifest_sha256, + "human_text_rights_assertion_sha256": ( + self.human_text_rights_assertion_sha256 + ), + "microbatch_count": self.microbatch_count, + "positive_family_manifest_sha256": self.positive_family_manifest_sha256, + "prepared_train_manifest_sha256": self.prepared_train_manifest_sha256, + "row_binding_manifest_sha256": self.row_binding_manifest_sha256, + "inspection_seed": self.inspection_seed, + "window_count": self.window_count, + "window_manifest_content_sha256": self.window_manifest_content_sha256, + "window_manifest_sha256": self.window_manifest_sha256, + } + ) + + @property + def sha256(self) -> str: + return _sha256_bytes(self.canonical_json_bytes()) + + +@dataclass(frozen=True, slots=True, repr=False) +class _BoundCaption: + role: str + commitment: bytes + text_sha256: str + embedding_sha256: str + embedding: torch.Tensor = field(repr=False) + + +@dataclass(frozen=True, slots=True, repr=False) +class _BoundWindow: + family: bytes + row: WindowManifestRow + captions: tuple[_BoundCaption, _BoundCaption] + fusion_receipt_sha256: str + clip_receipt_sha256: str + row_binding_sha256: str + + +def _validate_handoff( + *, + sample: PreparedGroupSample, + row: WindowManifestRow, + handoff: WindowFusionHandoff, + clip_commitments: tuple[bytes, bytes], + clip_text_digests: tuple[str, str], + clip_embedding_digests: tuple[str, str], + clip_embeddings: torch.Tensor, + clip_receipt_sha256: str, + expected_dataset_manifest_sha256: str, + expected_caption_manifest_sha256: str, + expected_human_text_rights_assertion_sha256: str, +) -> _BoundWindow: + # PreparedMotionTextBlock has already reconstructed exact PreparedGroupSample + # instances; avoid importing its implementation-only row helper. + if sample.window_sha256 != row.window_sha256: + raise CaptionRoleProvenanceError("prepared and summarized window identity differ") + if ( + sample.source_sha256 != row.source_sha256 + or sample.source_start_frame != row.source_start_frame + or sample.actor_commitments != row.actor_commitments + or sample.group_commitment != row.group_commitment + ): + raise CaptionRoleProvenanceError("prepared and summarized window lineage differ") + + fusion_input = handoff.fusion_input + fused = handoff.fused_caption + receipt = fused.receipt + if fusion_input.window_commitment != row.window_commitment: + raise CaptionRoleProvenanceError("fusion and summarized window commitments differ") + fusion_actors = tuple(sorted(item.actor_commitment for item in fusion_input.actor_captions)) + if fusion_actors != tuple(sorted(row.actor_commitments)): + raise CaptionRoleProvenanceError("fusion and summarized actor lineages differ") + fusion_actor_content = tuple( + sorted( + ( + item.actor_commitment, + item.source_record_sha256, + _sha256_bytes(item.text.encode("utf-8")), + ) + for item in fusion_input.actor_captions + ) + ) + summarized_actor_content = tuple( + ( + item.actor_commitment, + item.source_record_sha256, + item.text_utf8_sha256, + ) + for item in row.actor_caption_summaries + ) + if fusion_actor_content != summarized_actor_content: + raise CaptionRoleProvenanceError( + "fusion actor-caption content differs from verified window summary" + ) + holistic = fusion_input.holistic_context + holistic_summary = row.holistic_context_summary + if (holistic is None) != (holistic_summary is None): + raise CaptionRoleProvenanceError( + "fusion holistic-context presence differs from verified window summary" + ) + if holistic is not None and holistic_summary is not None and ( + holistic.window_commitment != row.window_commitment + or holistic.source_record_sha256 != holistic_summary.source_record_sha256 + or _sha256_bytes(holistic.text.encode("utf-8")) + != holistic_summary.text_utf8_sha256 + ): + raise CaptionRoleProvenanceError( + "fusion holistic content differs from verified window summary" + ) + provenance = fusion_input.provenance + if ( + provenance.split != "train" + or provenance.dataset_manifest_sha256 != expected_dataset_manifest_sha256 + or provenance.caption_manifest_sha256 != expected_caption_manifest_sha256 + or provenance.human_text_rights_assertion_sha256 + != expected_human_text_rights_assertion_sha256 + or provenance.sealed_test_manifest_sha256 is not None + or provenance.sealed_test_caption_sha256 is not None + ): + raise CaptionRoleProvenanceError("fusion provenance differs from admitted train inputs") + + try: + fusion_receipt_raw = canonical_caption_fusion_receipt_bytes(receipt) + except Exception: + raise CaptionRoleProvenanceError("caption-fusion receipt is invalid") from None + expected_admission = _sha256_bytes( + b"phaseset-caption-fusion-local-admission-v1\x00" + + provenance.digest().encode("ascii") + ) + if ( + type(receipt) is not CaptionFusionReceipt + or receipt.input_sha256 != _caption_input_digest(fusion_input) + or receipt.output_sha256 + != _caption_output_digest(fused.group_description, fused.semantic_paraphrase) + or receipt.provenance_sha256 != provenance.digest() + or receipt.admission_sha256 != expected_admission + or receipt.actor_count != len(row.actor_commitments) + or receipt.annotation_kind != ANNOTATION_KIND + or receipt.task_scope != TASK_SCOPE + or receipt.machine_fused is not True + or receipt.human_annotation is not False + or receipt.public_verification_performed is not False + or type(receipt.authority) is not int + or receipt.authority != 0 + or receipt.scientific_result_claimed is not False + or receipt.result_claimed is not False + ): + raise CaptionRoleProvenanceError("caption-fusion receipt does not bind this handoff") + + texts = (fused.group_description, fused.semantic_paraphrase) + text_digests = tuple(_sha256_bytes(value.encode("utf-8")) for value in texts) + if ( + handoff.clip_caption_commitments != clip_commitments + or text_digests != clip_text_digests + ): + raise CaptionRoleProvenanceError("fusion output differs from ordered frozen-CLIP rows") + caption_values = tuple( + _BoundCaption( + role=role, + commitment=commitment, + text_sha256=text_digest, + embedding_sha256=embedding_digest, + embedding=clip_embeddings[index : index + 1].detach().clone().contiguous(), + ) + for index, (role, commitment, text_digest, embedding_digest) in enumerate( + zip( + CAPTION_ROLES, + clip_commitments, + clip_text_digests, + clip_embedding_digests, + strict=True, + ) + ) + ) + family = bytes.fromhex(row.window_sha256) + fusion_receipt_sha256 = _sha256_bytes(fusion_receipt_raw) + row_binding = { + "actor_commitments": [value.hex() for value in row.actor_commitments], + "captions": [ + { + "clip_caption_commitment": caption.commitment.hex(), + "embedding_sha256": caption.embedding_sha256, + "role": caption.role, + "text_utf8_sha256": caption.text_sha256, + } + for caption in caption_values + ], + "clip_receipt_sha256": clip_receipt_sha256, + "fusion_input_sha256": receipt.input_sha256, + "fusion_output_sha256": receipt.output_sha256, + "fusion_provenance_sha256": receipt.provenance_sha256, + "fusion_receipt_sha256": fusion_receipt_sha256, + "group_commitment": row.group_commitment.hex(), + "positive_family": family.hex(), + "source_sha256": row.source_sha256, + "source_start_frame": row.source_start_frame, + "window_commitment": row.window_commitment.hex(), + "window_sha256": row.window_sha256, + } + return _BoundWindow( + family=family, + row=row, + captions=(caption_values[0], caption_values[1]), + fusion_receipt_sha256=fusion_receipt_sha256, + clip_receipt_sha256=clip_receipt_sha256, + row_binding_sha256=_sha256_bytes(_canonical_json(row_binding)), + ) + + +def validate_candidate_machine_fused_window_training( + *, + blocks: tuple[PreparedMotionTextBlock, ...], + handoffs: tuple[WindowFusionHandoff, ...], + window_summary: WindowManifestContentSummary, + train_source: PreparedTrainingDataSourceV2, + expected_window_manifest_sha256: str, + expected_prepared_train_manifest_sha256: str, + expected_dataset_manifest_sha256: str, + expected_caption_manifest_sha256: str, + expected_human_text_rights_assertion_sha256: str, +) -> CaptionRoleValidationReceipt: + """Validate one complete candidate auxiliary-window supervision census.""" + + expected_window = _lower_sha256( + expected_window_manifest_sha256, "expected window manifest sha256" + ) + expected_prepared = _lower_sha256( + expected_prepared_train_manifest_sha256, + "expected prepared train manifest sha256", + ) + expected_dataset = _lower_sha256( + expected_dataset_manifest_sha256, "expected dataset manifest sha256" + ) + expected_caption = _lower_sha256( + expected_caption_manifest_sha256, "expected caption manifest sha256" + ) + expected_rights = _lower_sha256( + expected_human_text_rights_assertion_sha256, + "expected human text rights assertion sha256", + ) + if type(window_summary) is not WindowManifestContentSummary: + raise CaptionRoleProvenanceError( + "window_summary must be exact WindowManifestContentSummary" + ) + if window_summary.manifest_sha256 != expected_window: + raise CaptionRoleProvenanceError("window summary differs from expected manifest digest") + if ( + type(blocks) is not tuple + or not blocks + or any(type(block) is not PreparedMotionTextBlock for block in blocks) + ): + raise CaptionRoleProvenanceError("blocks must be a nonempty exact prepared-v2 tuple") + if type(handoffs) is not tuple or any( + type(value) is not WindowFusionHandoff for value in handoffs + ): + raise CaptionRoleProvenanceError("handoffs must be an exact handoff tuple") + if type(train_source) is not PreparedTrainingDataSourceV2: + raise CaptionRoleProvenanceError("train_source must be exact prepared-data-v2 source") + if train_source.split != "train" or train_source.manifest_sha256 != expected_prepared: + raise CaptionRoleProvenanceError("prepared train source identity differs from expected") + + prepared_rows: list[tuple[PreparedGroupSample, int, int]] = [] + caption_offset = 0 + embedding_bytes = 0 + clip_receipt_digests: list[str] = [] + block_values: list[tuple[torch.Tensor, tuple[tuple[object, ...], ...], tuple[bytes, ...]]] = [] + for block_index, block in enumerate(blocks): + if len(block.samples) + len(prepared_rows) > MAX_WINDOWS: + raise CaptionRoleProvenanceError("prepared window census exceeds the bound") + if any(count != 2 for count in block.text_counts): + raise CaptionRoleProvenanceError( + "candidate policy requires exactly two text rows per motion window" + ) + embeddings = block.text_batch.embeddings + embedding_bytes += embeddings.numel() * embeddings.element_size() + if embedding_bytes > MAX_EMBEDDING_BYTES: + raise CaptionRoleProvenanceError("prepared frozen embeddings exceed the byte bound") + receipt = block.text_batch.receipt + try: + clip_receipt_sha256 = receipt.sha256 + except Exception: + raise CaptionRoleProvenanceError("frozen-CLIP receipt is invalid") from None + clip_receipt_digests.append(clip_receipt_sha256) + rows = receipt.caption_rows + commitments = block.text_batch.caption_commitments + local_offset = 0 + for sample, count in zip(block.samples, block.text_counts, strict=True): + prepared_rows.append((sample, block_index, local_offset)) + local_offset += count + caption_offset += count + if local_offset != len(commitments) or len(rows) != len(commitments): + raise CaptionRoleProvenanceError("prepared block text census is inconsistent") + block_values.append((embeddings, rows, commitments)) + if ( + not prepared_rows + or len(prepared_rows) != len(handoffs) + or len(prepared_rows) != len(window_summary.rows) + or caption_offset != 2 * len(prepared_rows) + or caption_offset > MAX_CAPTIONS + ): + raise CaptionRoleProvenanceError("prepared, fusion, and window censuses differ") + + bound: list[_BoundWindow] = [] + all_caption_commitments: set[bytes] = set() + all_families: set[bytes] = set() + all_fusion_receipts: set[str] = set() + for index, ((sample, block_index, local_offset), row, handoff) in enumerate( + zip(prepared_rows, window_summary.rows, handoffs, strict=True) + ): + embeddings, receipt_rows, block_commitments = block_values[block_index] + row_slice = receipt_rows[local_offset : local_offset + 2] + commitment_slice = block_commitments[local_offset : local_offset + 2] + if ( + len(row_slice) != 2 + or len(commitment_slice) != 2 + or tuple(item[0] for item in row_slice) != (local_offset, local_offset + 1) + or tuple(item[1] for item in row_slice) + != tuple(value.hex() for value in commitment_slice) + ): + raise CaptionRoleProvenanceError( + f"frozen-CLIP ordered rows differ at prepared window {index}" + ) + checked = _validate_handoff( + sample=sample, + row=row, + handoff=handoff, + clip_commitments=(commitment_slice[0], commitment_slice[1]), + clip_text_digests=(row_slice[0][2], row_slice[1][2]), + clip_embedding_digests=(row_slice[0][8], row_slice[1][8]), + clip_embeddings=embeddings[local_offset : local_offset + 2], + clip_receipt_sha256=clip_receipt_digests[block_index], + expected_dataset_manifest_sha256=expected_dataset, + expected_caption_manifest_sha256=expected_caption, + expected_human_text_rights_assertion_sha256=expected_rights, + ) + commitments = {caption.commitment for caption in checked.captions} + if ( + checked.family in all_families + or all_caption_commitments.intersection(commitments) + or checked.fusion_receipt_sha256 in all_fusion_receipts + ): + raise CaptionRoleProvenanceError("candidate binding repeats a row identity") + all_families.add(checked.family) + all_caption_commitments.update(commitments) + all_fusion_receipts.add(checked.fusion_receipt_sha256) + bound.append(checked) + + by_family = {value.family: value for value in bound} + by_caption = { + caption.commitment: (value.family, caption) + for value in bound + for caption in value.captions + } + seen_families: set[bytes] = set() + seen_captions: set[bytes] = set() + microbatch_count = 0 + observed_embedding_bytes = 0 + try: + iterator = train_source.iter_epoch(epoch=0, seed=INSPECTION_SEED) + for batch_index, batch in enumerate(iterator): + microbatch_count += 1 + if microbatch_count > MAX_MICROBATCHES: + raise CaptionRoleProvenanceError("prepared source exceeds the microbatch bound") + if type(batch) is not RetrievalTrainingBatch or batch.split != "train": + raise CaptionRoleProvenanceError("prepared source emitted a non-train batch") + observed_embedding_bytes += ( + batch.text_embeddings.numel() * batch.text_embeddings.element_size() + ) + if observed_embedding_bytes > MAX_EMBEDDING_BYTES: + raise CaptionRoleProvenanceError("materialized embeddings exceed the byte bound") + if len(seen_families) + batch.motion_count > MAX_WINDOWS: + raise CaptionRoleProvenanceError("materialized window census exceeds the bound") + if len(seen_captions) + len(batch.text_commitments) > MAX_CAPTIONS: + raise CaptionRoleProvenanceError("materialized caption census exceeds the bound") + for motion_index, family in enumerate(batch.motion_positive_ids): + expected = by_family.get(family) + if expected is None or family in seen_families: + raise CaptionRoleProvenanceError( + f"materialized motion family differs at batch {batch_index}" + ) + group_row = batch.groups.group_commitments[motion_index] + actor_row = tuple( + value + for value in batch.groups.actor_commitments[motion_index] + if value is not None + ) + if ( + group_row != expected.row.group_commitment + or actor_row != tuple(sorted(expected.row.actor_commitments)) + ): + raise CaptionRoleProvenanceError( + f"materialized motion lineage differs at batch {batch_index}" + ) + seen_families.add(family) + for text_index, (family, commitment) in enumerate( + zip(batch.text_positive_ids, batch.text_commitments, strict=True) + ): + expected_pair = by_caption.get(commitment) + if expected_pair is None or commitment in seen_captions: + raise CaptionRoleProvenanceError( + f"materialized caption lineage differs at batch {batch_index}" + ) + expected_family, expected_caption_row = expected_pair + if family != expected_family or not torch.equal( + batch.text_embeddings[text_index : text_index + 1], + expected_caption_row.embedding, + ): + raise CaptionRoleProvenanceError( + f"materialized caption value or family differs at batch {batch_index}" + ) + if ( + _sha256_bytes(_tensor_bytes(batch.text_embeddings[text_index : text_index + 1])) + != expected_caption_row.embedding_sha256 + ): + raise CaptionRoleProvenanceError( + f"materialized caption digest differs at batch {batch_index}" + ) + seen_captions.add(commitment) + except CaptionRoleProvenanceError: + raise + except Exception: + raise CaptionRoleProvenanceError("prepared train source iteration failed") from None + + if not microbatch_count: + raise CaptionRoleProvenanceError("prepared train source emitted no microbatches") + if seen_families != all_families or seen_captions != all_caption_commitments: + raise CaptionRoleProvenanceError("materialized prepared source has an incomplete census") + + result = CaptionRoleValidationReceipt( + window_manifest_sha256=expected_window, + window_manifest_content_sha256=window_summary.content_sha256, + prepared_train_manifest_sha256=expected_prepared, + dataset_manifest_sha256=expected_dataset, + caption_manifest_sha256=expected_caption, + human_text_rights_assertion_sha256=expected_rights, + row_binding_manifest_sha256=_manifest_digest( + b"phaseset-caption-role-row-binding-manifest-v1\x00", + tuple(value.row_binding_sha256 for value in bound), + ), + fusion_receipt_manifest_sha256=_manifest_digest( + b"phaseset-caption-role-fusion-receipt-manifest-v1\x00", + tuple(value.fusion_receipt_sha256 for value in bound), + ), + clip_receipt_manifest_sha256=_manifest_digest( + b"phaseset-caption-role-clip-receipt-manifest-v1\x00", + tuple(value.clip_receipt_sha256 for value in bound), + ), + positive_family_manifest_sha256=_manifest_digest( + b"phaseset-caption-role-positive-family-manifest-v1\x00", + tuple(value.family.hex() for value in bound), + ), + window_count=len(bound), + caption_count=len(all_caption_commitments), + microbatch_count=microbatch_count, + ) + result.canonical_json_bytes() + return result + + +__all__ = [ + "ActorCaptionContentSummary", + "CANDIDATE_POLICY_ID", + "CAPTION_ROLES", + "CaptionRoleProvenanceError", + "CaptionRoleValidationReceipt", + "HolisticContextContentSummary", + "INSPECTION_SEED", + "STATUS", + "WindowFusionHandoff", + "WindowManifestContentSummary", + "WindowManifestRow", + "validate_candidate_machine_fused_window_training", +] diff --git a/src/phaseset_core/capture_pooling.py b/src/phaseset_core/capture_pooling.py new file mode 100644 index 0000000..f732bd8 --- /dev/null +++ b/src/phaseset_core/capture_pooling.py @@ -0,0 +1,214 @@ +"""Deterministic capture-level pooling of already encoded motion windows. + +Window identity is independent of actor-set identity. This numeric seam does +not encode motion, score a gallery, establish participant identity, select +windows, or authorize access to sealed evaluation data. +""" + +from __future__ import annotations + +import hashlib +import json +from dataclasses import dataclass + +import numpy as np + +EMBEDDING_DIM = 512 +BAND_COUNT = 6 +SOURCE_WINDOW_FRAMES = 300 +DEFAULT_MAX_ENCODED_BYTES = 256 * 1024 * 1024 + + +class CapturePoolingError(ValueError): + """The supplied encoded windows differ from the complete capture plan.""" + + +class CapturePoolingResourceLimit(CapturePoolingError): + """An explicit memory bound was exceeded; no windows were sampled.""" + + +def _commitment(value: object) -> bytes: + if type(value) is not bytes or len(value) != 32: + raise CapturePoolingError("identity must be exact bytes[32]") + return value + + +@dataclass(frozen=True, slots=True, repr=False) +class CaptureWindowPlan: + """The complete accepted-window census, frozen before any model score. + + Rejected windows and trailing source frames belong to the upstream + preprocessing census. This plan may not be reconstructed by filtering + available model outputs. A capture with no accepted window is ineligible. + """ + + capture_commitment: bytes + window_commitments: tuple[bytes, ...] + source_start_frames: tuple[int, ...] + + def __post_init__(self) -> None: + _commitment(self.capture_commitment) + if type(self.window_commitments) is not tuple or not self.window_commitments: + raise CapturePoolingError("a capture must have its complete window census") + for value in self.window_commitments: + _commitment(value) + if len(set(self.window_commitments)) != len(self.window_commitments): + raise CapturePoolingError("window identities must be unique") + if ( + type(self.source_start_frames) is not tuple + or len(self.source_start_frames) != len(self.window_commitments) + or any( + type(value) is not int + or not 0 <= value < 2**64 + or value % SOURCE_WINDOW_FRAMES + for value in self.source_start_frames + ) + or tuple(sorted(set(self.source_start_frames))) != self.source_start_frames + ): + raise CapturePoolingError("window starts must be ordered nonoverlapping 10s grid positions") + + @property + def sha256(self) -> str: + raw = json.dumps( + { + "capture_commitment": self.capture_commitment.hex(), + "schema": "phaseset-capture-window-plan-v1", + "source_start_frames": self.source_start_frames, + "window_commitments": tuple(value.hex() for value in self.window_commitments), + }, + sort_keys=True, + separators=(",", ":"), + ).encode("ascii") + return hashlib.sha256(raw).hexdigest() + + def __repr__(self) -> str: + return f"CaptureWindowPlan(windows={len(self.window_commitments)},