diff --git a/.github/workflows/bench-regression.yml b/.github/workflows/bench-regression.yml index 3ad5a6cb7c..6299f24a85 100644 --- a/.github/workflows/bench-regression.yml +++ b/.github/workflows/bench-regression.yml @@ -42,6 +42,7 @@ on: - 'v2/Cargo.toml' - 'v2/Cargo.lock' - '.github/workflows/bench-regression.yml' + - 'scripts/ci-install-ubuntu-deps.sh' pull_request: paths: - 'v2/crates/**/benches/**' @@ -50,6 +51,7 @@ on: - 'v2/Cargo.toml' - 'v2/Cargo.lock' - '.github/workflows/bench-regression.yml' + - 'scripts/ci-install-ubuntu-deps.sh' workflow_dispatch: permissions: @@ -68,7 +70,7 @@ jobs: # ── HARD GATE: every bench must still compile + link ───────────────────── bench-compile: name: bench compile-verify (--no-run) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Checkout (recursive — wifi-densepose-rufield path-deps vendor/rufield) uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 @@ -84,8 +86,7 @@ jobs: # the whole graph, so these are required here too. - name: Install Tauri / GTK / serial system dev libraries run: | - sudo apt-get update - sudo apt-get install -y --no-install-recommends \ + bash scripts/ci-install-ubuntu-deps.sh \ libglib2.0-dev \ libgtk-3-dev \ libsoup-3.0-dev \ diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e1d9aa6668..b199d64d4c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -79,7 +79,7 @@ jobs: # Rust Workspace Tests rust-tests: name: Rust Workspace Tests - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Checkout code uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 @@ -96,8 +96,7 @@ jobs: # standard Tauri-on-Ubuntu set. - name: Install Tauri / GTK / serial system dev libraries run: | - sudo apt-get update - sudo apt-get install -y --no-install-recommends \ + bash scripts/ci-install-ubuntu-deps.sh \ libglib2.0-dev \ libgtk-3-dev \ libsoup-3.0-dev \ @@ -132,6 +131,15 @@ jobs: # device" once the engine/benchmark crates landed; the same tree's local # debug target measured 151 GB). Debuginfo is useless in CI — tests either # pass or print their failure — so build without it; target shrinks ~5-10x. + - name: Verify temporal Debug/Release test classification + env: + CARGO_PROFILE_DEV_DEBUG: "0" + CARGO_PROFILE_TEST_DEBUG: "0" + CARGO_BUILD_JOBS: "2" + run: | + python3 scripts/ci/test_temporal_test_profiles.py + python3 scripts/ci/temporal_test_profiles.py debug-check --evidence-dir "${{ runner.temp }}/temporal-test-profiles" + - name: Run Rust tests working-directory: v2 env: @@ -139,6 +147,25 @@ jobs: CARGO_PROFILE_TEST_DEBUG: "0" run: cargo test --workspace --no-default-features + - name: Execute native temporal persistence journeys in Release + env: + CARGO_BUILD_JOBS: "2" + run: python3 scripts/ci/temporal_test_profiles.py release-run --evidence-dir "${{ runner.temp }}/temporal-test-profiles" + + - name: Retain temporal profile registration and execution evidence + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: temporal-test-profiles + path: ${{ runner.temp }}/temporal-test-profiles + + - name: Verify secure companion over real sockets + env: + CARGO_PROFILE_DEV_DEBUG: "0" + run: | + cargo build --manifest-path v2/Cargo.toml --locked -p wifi-densepose-sensing-server --bin sensing-server + python3 scripts/tests/companion_socket_probe.py v2/target/debug/sensing-server + - name: Verify patched archive and cache compatibility working-directory: v2 env: @@ -405,6 +432,9 @@ jobs: # workflow, so it doesn't gate the rest of the pipeline. docker-build: name: Docker Build & Test + # PR/manual verification must not publish images. The existing main-push + # publisher behavior is separate from these verification-only events. + if: github.event_name == 'push' && github.ref == 'refs/heads/main' runs-on: ubuntu-latest needs: [code-quality, test, rust-tests] continue-on-error: true @@ -480,7 +510,7 @@ jobs: name: API Documentation runs-on: ubuntu-latest needs: [docker-build] - if: github.ref == 'refs/heads/main' + if: github.ref == 'refs/heads/main' && github.event_name != 'workflow_dispatch' permissions: contents: write # gh-pages deploy needs write (GITHUB_TOKEN is read-only by default -> 403) steps: @@ -525,7 +555,8 @@ jobs: name: Notify runs-on: ubuntu-latest needs: [code-quality, test, rust-tests, ui-tests, performance-test, docker-build, docs] - if: always() + # This job also creates releases and sends external notifications. + if: always() && github.event_name == 'push' && github.ref == 'refs/heads/main' permissions: contents: write # required by softprops/action-gh-release # GitHub Actions does not allow `secrets.X` directly in step-level `if:` diff --git a/.github/workflows/firmware-qemu.yml b/.github/workflows/firmware-qemu.yml index 1c295760c4..d792ed9ab6 100644 --- a/.github/workflows/firmware-qemu.yml +++ b/.github/workflows/firmware-qemu.yml @@ -22,49 +22,32 @@ on: - 'scripts/swarm_presets/**' - '.github/workflows/firmware-qemu.yml' +defaults: + run: + shell: bash + env: IDF_VERSION: "v5.4" - QEMU_REPO: "https://github.com/espressif/qemu.git" - QEMU_BRANCH: "esp-develop" jobs: build-qemu: name: Build Espressif QEMU - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 + # This SDK image installs Espressif's checksum-verified QEMU package and + # its runtime libraries. Match the image used by both consumers below. + # Avoid APT DNS and an unpinned floating QEMU source build on cold caches. + container: + image: espressif/idf:v5.4@sha256:f1e9f69dc052b9afc7801ca884e0ef40c17e014bb05ce73d9c09d29290bd17fb steps: - - name: Cache QEMU build - id: cache-qemu - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 - with: - path: /opt/qemu-esp32 - # Include date component so cache refreshes monthly when branch updates - key: qemu-esp32s3-${{ env.QEMU_BRANCH }}-v5 - restore-keys: | - qemu-esp32s3-${{ env.QEMU_BRANCH }}- - - - name: Install QEMU build dependencies - if: steps.cache-qemu.outputs.cache-hit != 'true' + - name: Prepare SDK-provided Espressif QEMU run: | - sudo apt-get update - sudo apt-get install -y \ - git build-essential ninja-build pkg-config \ - libglib2.0-dev libpixman-1-dev libslirp-dev \ - libgcrypt20-dev \ - python3 python3-venv - - - name: Clone and build Espressif QEMU - if: steps.cache-qemu.outputs.cache-hit != 'true' - run: | - git clone --depth 1 -b "$QEMU_BRANCH" "$QEMU_REPO" /tmp/qemu-esp - cd /tmp/qemu-esp - mkdir build && cd build - ../configure \ - --target-list=xtensa-softmmu \ - --prefix=/opt/qemu-esp32 \ - --enable-slirp \ - --disable-werror - ninja -j$(nproc) - ninja install + . "$IDF_PATH/export.sh" + QEMU_BIN="$(command -v qemu-system-xtensa)" + QEMU_ROOT="$(dirname "$(dirname "$QEMU_BIN")")" + "$QEMU_BIN" --version + "$QEMU_BIN" -machine help | grep -E '^esp32s3[[:space:]]' + mkdir -p /opt/qemu-esp32 + cp -a "$QEMU_ROOT/." /opt/qemu-esp32/ - name: Verify QEMU binary run: | @@ -84,7 +67,7 @@ jobs: needs: build-qemu runs-on: ubuntu-latest container: - image: espressif/idf:v5.4 + image: espressif/idf:v5.4@sha256:f1e9f69dc052b9afc7801ca884e0ef40c17e014bb05ce73d9c09d29290bd17fb strategy: fail-fast: false @@ -117,26 +100,26 @@ jobs: - name: Install Python dependencies run: | - . $IDF_PATH/export.sh + . "$IDF_PATH/export.sh" pip install esptool esp-idf-nvs-partition-gen - name: Set target ESP32-S3 working-directory: firmware/esp32-csi-node run: | - . $IDF_PATH/export.sh + . "$IDF_PATH/export.sh" idf.py set-target esp32s3 - name: Build firmware (mock CSI mode) working-directory: firmware/esp32-csi-node run: | - . $IDF_PATH/export.sh + . "$IDF_PATH/export.sh" idf.py \ -D SDKCONFIG_DEFAULTS="sdkconfig.defaults;sdkconfig.qemu" \ build - name: Generate NVS matrix run: | - . $IDF_PATH/export.sh + . "$IDF_PATH/export.sh" python3 scripts/generate_nvs_matrix.py \ --output-dir firmware/esp32-csi-node/build/nvs_matrix \ --only ${{ matrix.nvs_config }} @@ -144,12 +127,12 @@ jobs: - name: Create merged flash image working-directory: firmware/esp32-csi-node run: | - . $IDF_PATH/export.sh + . "$IDF_PATH/export.sh" # Determine merge_bin arguments - OTA_ARGS="" + OTA_ARGS=() if [ -f build/ota_data_initial.bin ]; then - OTA_ARGS="0xf000 build/ota_data_initial.bin" + OTA_ARGS=(0xf000 build/ota_data_initial.bin) fi python3 -m esptool --chip esp32s3 merge_bin \ @@ -158,7 +141,7 @@ jobs: --fill-flash-size 8MB \ 0x0 build/bootloader/bootloader.bin \ 0x8000 build/partition_table/partition-table.bin \ - $OTA_ARGS \ + "${OTA_ARGS[@]}" \ 0x20000 build/esp32-csi-node.bin file_size() { stat -c%s "$1" 2>/dev/null || stat -f%z "$1" 2>/dev/null || wc -c < "$1"; } @@ -213,32 +196,35 @@ jobs: fuzz-test: name: Fuzz Testing (ADR-061 Layer 6) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 + env: + CC: clang-18 steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 with: submodules: recursive - - name: Install clang + - name: Verify hosted Clang + # Ubuntu 24.04 hosted runners already contain Clang 18 and libFuzzer. + # Do not fetch the redundant distro meta-package before testing. run: | - sudo apt-get update - sudo apt-get install -y clang + "$CC" --version - name: Build fuzz targets working-directory: firmware/esp32-csi-node/test - run: make all CC=clang + run: make all CC="$CC" - name: Run serialize fuzzer (60s) working-directory: firmware/esp32-csi-node/test - run: make run_serialize FUZZ_DURATION=60 || echo "FUZZER_CRASH=serialize" >> "$GITHUB_ENV" + run: make run_serialize CC="$CC" FUZZ_DURATION=60 || echo "FUZZER_CRASH=serialize" >> "$GITHUB_ENV" - name: Run edge enqueue fuzzer (60s) working-directory: firmware/esp32-csi-node/test - run: make run_edge FUZZ_DURATION=60 || echo "FUZZER_CRASH=edge" >> "$GITHUB_ENV" + run: make run_edge CC="$CC" FUZZ_DURATION=60 || echo "FUZZER_CRASH=edge" >> "$GITHUB_ENV" - name: Run NVS config fuzzer (60s) working-directory: firmware/esp32-csi-node/test - run: make run_nvs FUZZ_DURATION=60 || echo "FUZZER_CRASH=nvs" >> "$GITHUB_ENV" + run: make run_nvs CC="$CC" FUZZ_DURATION=60 || echo "FUZZER_CRASH=nvs" >> "$GITHUB_ENV" - name: Check for crashes working-directory: firmware/esp32-csi-node/test @@ -299,7 +285,7 @@ jobs: echo "::error::$f has unexpected size $SIZE (expected 24576)" exit 1 fi - echo " OK: $(basename $f) ($SIZE bytes)" + echo " OK: $(basename "$f") ($SIZE bytes)" done # --------------------------------------------------------------------------- @@ -319,7 +305,7 @@ jobs: needs: [build-qemu] runs-on: ubuntu-latest container: - image: espressif/idf:v5.4 + image: espressif/idf:v5.4@sha256:f1e9f69dc052b9afc7801ca884e0ef40c17e014bb05ce73d9c09d29290bd17fb steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 @@ -337,13 +323,13 @@ jobs: - name: Install Python dependencies run: | - . $IDF_PATH/export.sh + . "$IDF_PATH/export.sh" pip install pyyaml esptool esp-idf-nvs-partition-gen - name: Build firmware for swarm working-directory: firmware/esp32-csi-node run: | - . $IDF_PATH/export.sh + . "$IDF_PATH/export.sh" idf.py set-target esp32s3 idf.py -D SDKCONFIG_DEFAULTS="sdkconfig.defaults;sdkconfig.qemu" build python3 -m esptool --chip esp32s3 merge_bin \ @@ -356,7 +342,7 @@ jobs: - name: Run swarm smoke test run: | - . $IDF_PATH/export.sh + . "$IDF_PATH/export.sh" EXIT_CODE=0 python3 scripts/qemu_swarm.py --preset ci_matrix \ --qemu-path /opt/qemu-esp32/bin/qemu-system-xtensa \ diff --git a/.github/workflows/mqtt-integration.yml b/.github/workflows/mqtt-integration.yml index 3a488dbe72..634ee21d73 100644 --- a/.github/workflows/mqtt-integration.yml +++ b/.github/workflows/mqtt-integration.yml @@ -33,11 +33,8 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 20 - # NB: we don't use a `services:` mosquitto container here because the - # eclipse-mosquitto:2.x image rejects anonymous connections by default - # and GH Actions `services` doesn't easily support mounting a custom - # config file. We start mosquitto manually in a step below with an - # inline `allow_anonymous true` config. + # Explicit vendor broker config; publish only the CI loopback port. + # No APT package install or production MQTT security policy is involved. env: RUVIEW_RUN_INTEGRATION: "1" @@ -50,29 +47,26 @@ jobs: with: submodules: recursive - - name: Install mosquitto + clients and start with allow_anonymous + - name: Start pinned Mosquitto broker with CI-only anonymous listener run: | - sudo apt-get update -qq - sudo apt-get install -y mosquitto mosquitto-clients - sudo systemctl stop mosquitto || true - # Inline config: anon listener on 11883 only — no TLS, no auth, - # OK for CI because we test the wire shape, not security. - # Production deployments enable mTLS per ADR-115 §3.9. - cat > /tmp/mosquitto-ci.conf <<'EOF' - listener 11883 + cat > "$RUNNER_TEMP/mosquitto-ci.conf" <<'CONFIG' + listener 1883 allow_anonymous true persistence false log_dest stdout - EOF - mosquitto -c /tmp/mosquitto-ci.conf -d + CONFIG + docker run --detach --user mosquitto --name ruview-mqtt-ci \ + --publish 127.0.0.1:11883:1883 \ + --mount "type=bind,source=$RUNNER_TEMP/mosquitto-ci.conf,target=/mosquitto/config/mosquitto.conf,readonly" \ + eclipse-mosquitto:2.0.18@sha256:d12c8f80dfc65b768bb9acecc7ef182b976f71fb681640b66358e5e0cf94e9e9 for i in {1..20}; do - if mosquitto_pub -h 127.0.0.1 -p 11883 -t healthcheck -m ok -q 0 2>/dev/null; then - echo "mosquitto reachable on 11883"; exit 0 + if docker exec ruview-mqtt-ci mosquitto_pub -h 127.0.0.1 -p 1883 -t healthcheck -m ok -q 0 \ + && python3 -c 'import socket; socket.create_connection(("127.0.0.1", 11883), timeout=3).close()'; then + exit 0 fi - sleep 2 + sleep 1 done - echo "mosquitto never became reachable" >&2 - tail -50 /var/log/mosquitto/*.log 2>/dev/null || true + docker logs ruview-mqtt-ci exit 1 - name: Install Rust toolchain @@ -90,16 +84,15 @@ jobs: python -m pip install --quiet pyyaml python scripts/validate-ha-blueprints.py - - name: Verify unit tests still pass under --features mqtt + - name: Verify MQTT-feature unit tests working-directory: v2 - # `cargo test` accepts a single TESTNAME filter, so we run the - # whole --lib suite here. That gives us the full 410-test green - # bar under --features mqtt (which is more reassuring than - # filtering anyway). + # Keep this broker workflow focused on the feature it changes. The full + # library/workspace and temporal Release suites run in ci.yml; repeating + # long learner training here exhausted this job before broker tests ran. run: >- - cargo test -p wifi-densepose-sensing-server + cargo test --locked -p wifi-densepose-sensing-server --features mqtt --no-default-features - --lib + --lib mqtt:: --no-fail-fast - name: Run integration tests against mosquitto @@ -114,5 +107,8 @@ jobs: - name: Dump broker logs on failure if: failure() run: | - docker ps -a - docker logs $(docker ps -aqf "ancestor=eclipse-mosquitto:2.0.18") || true + docker logs ruview-mqtt-ci || true + + - name: Stop the owned CI broker + if: always() + run: docker rm --force ruview-mqtt-ci diff --git a/.github/workflows/ruview-kernel-npm.yml b/.github/workflows/ruview-kernel-npm.yml index 8e850926ee..b7c32fdf90 100644 --- a/.github/workflows/ruview-kernel-npm.yml +++ b/.github/workflows/ruview-kernel-npm.yml @@ -77,6 +77,13 @@ jobs: with: node-version: '22' + # The ruv-FANN git dependency contains paths longer than Windows' + # 260-character limit. On a cold cargo cache, cargo's checkout fails + # with "path too long" unless git long-path support is on. + - name: Allow long paths for cargo git checkouts + if: runner.os == 'Windows' + run: git config --global core.longpaths true + - name: Rust ABI tests run: cargo test --manifest-path v2/Cargo.toml -p ruview-kernel --locked diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index 4b406ed775..bb47845012 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -38,7 +38,11 @@ jobs: working-directory: v2 run: | set -o pipefail - cargo audit --file Cargo.lock --json | tee cargo-audit.json + # Direct invocation avoids project-defined Cargo aliases. + cargo-audit audit --file Cargo.lock --json | tee cargo-audit.json + + - name: Exercise the active ZIP adapter security guards + run: cargo test --manifest-path v2/vendor/security/zip-candle-compat/Cargo.toml --locked - name: Upload Rust advisory report uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 @@ -450,7 +454,7 @@ jobs: # job-level env block (added below). - name: Notify security team on critical findings continue-on-error: true - if: ${{ env.SECURITY_SLACK_WEBHOOK_URL != '' && (needs.sast.result == 'failure' || needs.dependency-scan.result == 'failure' || needs.container-scan.result == 'failure') }} + if: ${{ github.event_name != 'workflow_dispatch' && env.SECURITY_SLACK_WEBHOOK_URL != '' && (needs.sast.result == 'failure' || needs.dependency-scan.result == 'failure' || needs.container-scan.result == 'failure') }} uses: 8398a7/action-slack@77eaa4f1c608a7d68b38af4e3f739dcd8cba273e # v3 with: status: failure @@ -466,7 +470,7 @@ jobs: - name: Create security issue on critical findings continue-on-error: true - if: needs.sast.result == 'failure' || needs.dependency-scan.result == 'failure' + if: github.event_name != 'workflow_dispatch' && (needs.sast.result == 'failure' || needs.dependency-scan.result == 'failure') uses: actions/github-script@00f12e3e20659f42342b1c0226afda7f7c042325 # v6 with: script: | diff --git a/docs/adr/ADR-001-wifi-mat-disaster-detection.md b/docs/adr/ADR-001-wifi-mat-disaster-detection.md index f03c55346b..cb997aee4f 100644 --- a/docs/adr/ADR-001-wifi-mat-disaster-detection.md +++ b/docs/adr/ADR-001-wifi-mat-disaster-detection.md @@ -165,6 +165,29 @@ The module integrates with existing crates through adapters: | False positives from animals | Size/pattern classification | | Power constraints in field | Low-power modes, solar charging | +## Hosted API and privacy boundary amendment (2026-10-22) + +PR #2249 composes the MAT API into the sensing-server host with one in-memory +MAT `AppState`. It remains inside host validation and authentication; optional +ML unavailability and empty incident state must be reported explicitly. This +composition does not implement a durable incident database or establish the +physical performance targets above. + +The host's `--privacy-mode` applies to MAT REST output and WebSocket broadcasts. +HTTP response middleware cannot inspect messages after an upgrade, so the host +installs the same recursive serialized-output filter in the MAT stream path. +Besides structured vital signs, this removes generated alert prose and domain +event Debug details that can contain literal heart/breathing rates. Permitted +status, identity and coarse metadata remain; ordinary error messages and +privacy-off serialization retain their existing behavior. Standalone MAT hosts +must install their own output policy; the extension hook alone is not a privacy +guarantee. + +Authenticated nonempty REST and real-socket stream regressions cover the output +boundary in both privacy modes. They use synthetic incidents and do not qualify +survivor detection, clinical accuracy, phone transport, or restart persistence. +Native companion stream support remains governed by ADR-419's allowlist. + ## References - [WiFi-based Vital Signs Monitoring](https://dl.acm.org/doi/10.1145/3130944) diff --git a/docs/adr/ADR-110-esp32-c6-firmware-extension.md b/docs/adr/ADR-110-esp32-c6-firmware-extension.md index 3905f167b2..03cd3f53f6 100644 --- a/docs/adr/ADR-110-esp32-c6-firmware-extension.md +++ b/docs/adr/ADR-110-esp32-c6-firmware-extension.md @@ -4,6 +4,7 @@ |-------|-------| | **Status** | Accepted — P1–P10 complete, firmware-side substrate closed at **v0.7.0-esp32** (2026-05-23) | | **Date** | 2026-05-22 (created) · 2026-05-23 (last revision — P10 + sprint summary) | +| **Updated** | 2026-11-04 — legacy calibration raw-grid compatibility boundary | | **Deciders** | ruv | | **Codename** | **C6-SOTA** | | **Relates to** | ADR-018 (CSI binary frame format), ADR-028 (ESP32 capability audit), ADR-029 (RuvSense multistatic), ADR-030 (RuvSense persistent field model), ADR-031 (RuView sensing-first), ADR-061 (QEMU CI), ADR-081 (adaptive CSI mesh kernel), ADR-097 (rvCSI adoption) | @@ -209,3 +210,55 @@ The firmware-side substrate for ADR-110 is now closed. Three categories remain, 3. **IDF upstream fixes**: - 802.15.4 RX path on C6 + IDF v5.4 — `c6_timesync` ships and initialises but never RXes a frame (D1, 5 hypotheses tested + rejected). ESP-NOW workaround (`c6_sync_espnow`) is the working primary mesh transport. The 802.15.4 source stays in for the day IDF fixes the driver. - Soft-AP HE/TWT-Responder advertise API — `c6_softap_he` ships as the in-place hook for when IDF v5.5+ exposes it. + + +## November 4, 2026: legacy calibration compatibility boundary + +The accepted firmware substrate above does not imply that HT-LTF and HE-LTF +raw vectors can be combined in one legacy FieldModel. That baseline has exactly +one author, one exact `(n_subcarriers, ppdu_type)` grid and one set of amplitude +offsets. Other placed nodes remain in the full room binding and must contribute +on that grid; they do not author or average their offsets into the model. +Equal bin counts with different PPDU types are not an interchangeable grid. + +Before any bootstrap/model/session/receipt mutation, legacy calibration start +checks every bound node's current raw-header evidence for the selected author's +exact grid. It searches **all** eligible grids per node, not only that node's +preferred or densest grid. Thus a C6 with a sufficiently sampled HT minority can +participate in an HT baseline authored by another node. Author selection itself +is unchanged; this does not automatically select a lower-density common grid. +A qualified nonmatching stream is refused before spending a room-capture window. +Missing, warming, weak or stale evidence is described separately and does not +become a hardware compatibility claim inferred from a user-set board label. + +The existing protected start/status responses gain optional bounded +`source_grid_readiness` diagnostics: the selected tuple and author, only the +bound node IDs, at most eight reported observed/eligible grids per node, +truncation flags and stable reasons. Reported eligible grids are a subset of +reported observations, and an observed selected tuple remains visible even +under truncation. The field is null without a valid active binding. No raw CSI, +RF values, device addresses, unbound inventory or new endpoint/grant is exposed. +Older clients may ignore this additive diagnostic. + +Current status recomputes these diagnostics without changing historical +contribution membership. For a **new unfinished** finalization only, a bound node +that now supplies a qualified different grid and lacks a current matching stream +blocks the new receipt while retaining the unfinished model and history. This +is not an additional blanket ten-second readiness gate on stop: unknown, +warming or stale diagnostics do not prove a hardware change, and existing +author freshness, contribution and sequence guards remain. Completed historical +receipts/metadata do not acquire this new finalization restriction. + +This rule applies only to the legacy single-author baseline. The separate +managed learner in Proposed ADR-420 preserves each source's own supported PHY/ +grid and projection; it must not inherit a global equality rule. Accepted +ADR-151's statistical specialist stages and Proposed ADR-008's distributed +coordination are not a claim that the legacy FieldModel is a joint heterogeneous +learner or that mixed-source sensing is physically qualified. + +The compatibility amendment is verified with synthetic raw-header fixtures +through the actual server start/status/stop handlers, including pre-mutation +preservation, PPDU mismatch, a shared minority grid, missing/warming/stale +sources, truncation/unbound exclusion and post-start drift. These software +results do not replace physical C5/C6/S3 or sensing-accuracy acceptance. No +firmware, production minimum duration, frame target or live node was changed. diff --git a/docs/adr/ADR-139-worldgraph-environmental-digital-twin.md b/docs/adr/ADR-139-worldgraph-environmental-digital-twin.md index d1a83624fa..c5277188cf 100644 --- a/docs/adr/ADR-139-worldgraph-environmental-digital-twin.md +++ b/docs/adr/ADR-139-worldgraph-environmental-digital-twin.md @@ -585,3 +585,37 @@ Add rows to `docs/WITNESS-LOG-028.md`: **Integration glue -- not yet on the live path:** feeding live fusion outputs and person tracks into nodes; the full `.rvf` bundle container (today it persists as JSON); and the live ADR-141 privacy-mode reducer. **Trust contribution:** the auditable map -- evidence and contradiction are first-class edges, and the privacy posture is *visible in the persisted graph* (an auditor can read what was suppressed). + + +## 2026-11-04 amendment: managed configured topology transport + +Status: implemented source slice; native delivery and physical geometry acceptance remain pending. + +The app-managed sensing server exposes authenticated `GET /api/v1/worldgraph/snapshot` +as protocol 2 when its authoritative companion room store is available. Protocol 1 +and genuine registered ENU WorldGraph snapshots remain a separate unchanged contract. +Protocol 2 is `ruview.worldgraph.managed-topology.v1`, with `managed_room_local` +coordinates and `x_right_y_down_metres` axes (room map origin at top left). Its only +geometry provenance is `operator_entered`; geographic registration and sensor +heights are explicitly null. Room dimensions may include the operator-entered +room height, but this never supplies sensor height or AR alignment. + +The projection takes one immutable published room envelope, preserving installation, +revision and binding digest. Canonical room/sensor and located-in vocabulary is +reused without injecting the engine's default room, person/semantic/vital records, +raw CSI or phone geometry. The configured AP remains part of the original room +binding but is not invented as a CSI receiver or RF link. This slice is not an ENU +registration, a spatial teacher, localization, occupancy model or model-selection +receipt. It requires no additional persistent graph store. + +An ephemeral random `wg-` epoch and safe-integer cursor belong to the loaded account/ +installation store. Equal canonical topology uses the same sequence; readiness, +binding or privacy changes advance it. Reads never publish geometry, renew a +baseline, train or select a model. Global server privacy mode conservatively returns +`privacy_denied` with no geometry. No published room yields `not_configured`; +missing managed support remains unsupported, not zero people. The same protected +owner and paired read paths apply, with `Cache-Control: private, no-store`. + +The managed client shares capability state between Zones, Settings and MAT, and +never converts this planar topology to a fabricated geographic or phone AR frame. +Cloud Cognitum Spaces/Zone projection remains separate under ADR-325. diff --git a/docs/adr/ADR-295-source-provenance-state-machine.md b/docs/adr/ADR-295-source-provenance-state-machine.md index c101d9e608..8bc4fddd6c 100644 --- a/docs/adr/ADR-295-source-provenance-state-machine.md +++ b/docs/adr/ADR-295-source-provenance-state-machine.md @@ -62,3 +62,38 @@ simulator status, sensing client source label). Broader UI adoption follows. simulator → never live without a verified frame; freshness expiry → `Stale`; watermark present on synthetic export. - `cargo test -p wifi-densepose-sensing-server`. + +## October 28 amendment: cached observations and stream keepalive + +A timer must not repeatedly publish the same sensing observation as transport +keepalive. Raw, edge and synthetic producers continue to publish their new +observations directly. The ESP32 timer now publishes a cached snapshot only on +an effective-source transition, including the first offline transition and a +later recovery. It preserves the observation's original tick and wall-clock +measurement timestamp rather than making old evidence fresh. + +A new raw or legacy-pose subscriber receives one local initial snapshot through +that connection's existing authorization and privacy filtering. This initial +snapshot is not sent to the shared broadcast bus. Both transports have an +independent protocol Ping keepalive, so quiet sensing requires no synthetic +measurement traffic. Late subscribers can inspect the historical timestamp and +offline source without amplifying traffic to every existing client. + +Cached snapshots refresh ESP32 receive-clock node age/stale metadata. Optional +expired vital, calibrated-presence, detector, room-inference and pose-proof blocks +are withheld; the historical classification is not rewritten as an observed +absence. Receive-rate estimates describe their original sample history: an +unchanging accepted CSI sequence alongside a receive rate can reflect feature-grid +rejection, and is not relabelled as packet loss or an invented accepted rate. + +Validation uses the real raw/legacy handlers and an isolated actual-helper +UDP-stop/late-subscribe/resume probe, including privacy-on behavior: +`node scripts/verify-sensing-stream-freshness.mjs /path/to/sensing-server`. +The probe uses synthetic loopback data and private temporary storage; it does +not establish real-hardware sensing accuracy or room-selective occupancy. + +The legacy pose conversion abstains before any model/signal reconstruction or +physics assessment for an offline or expired cached observation. It retains the +original timestamp/source but emits `pose_source`/`activity: unavailable`, null +confidence and null estimated-person count, with no produced person poses. The +empty pose array means no available pose candidates, not a measured empty room. diff --git a/docs/adr/ADR-355-local-empty-room-bootstrap-prior.md b/docs/adr/ADR-355-local-empty-room-bootstrap-prior.md index e03ef06d7e..2cf45f6fc1 100644 --- a/docs/adr/ADR-355-local-empty-room-bootstrap-prior.md +++ b/docs/adr/ADR-355-local-empty-room-bootstrap-prior.md @@ -72,6 +72,27 @@ host to provide a protected state directory. ## Consequences +### Explicit recovery without restarting, 2026-10-12 + +`POST /api/v1/calibration/bootstrap/recover` requires administrator authority, +the current server boot epoch and the stored payload digest. It validates the +installation-bound file and original expiry before changing live state, and +requires at least ten seconds of eligible raw CSI on its original source/grid. +An active unfinished capture must be cancelled first. + +Recovery clears old source sequence faults and runtime windows, withdraws any +full room session/receipt, and restores only `bootstrap_only`. New packets must +build a complete matching runtime window before negative suppression applies. +Duplicates and backward discontinuities remain rejected. No calibration frame +count, training timestamp, threshold or expiry is renewed, and the stored file +is not rewritten. Full calibrated evidence and numeric vitals remain disabled. + +The twelve-hour stale boundary still withdraws full calibrated authority. A +restored negative-only prior may match fresh runtime windows while stale until +its original twenty-four-hour hard expiry. Expiry stops matching and recovery. +This resolves the mismatch between the advertised twenty-four-hour prior and +the previous fresh-only runtime gate; it makes no room-selectivity claim. + Startup can use recent measured background structure instead of only generic heuristics. No accuracy lift is claimed until occupied and empty held out data from multiple installations is recorded. @@ -106,6 +127,45 @@ occupied sequences. Empty data is never relabeled as human training. performs the bounded one time held out refinement. It requires administrator authority when transport authentication is enabled. +## Publication and diagnostics clarification, 2026-10-27 + +The REST vital endpoint and both raw-CSI/edge sensing publishers share the same +room aggregate and bound-model publication gates. REST must not substitute the +single-link model's count for the room aggregate: a fresh second node can still +make occupant attribution ambiguous. A cached candidate is bound to its producing +node, packet tick/time and full immutable calibration receipt identity. A REST +read may withdraw a previously published candidate as gates expire or change; +it cannot promote a sample withheld at its original publication decision. A new +qualifying packet is required. Candidate freshness is bounded independently of +other nodes' arrival clocks. + +Raw CSI estimates carry the existing exact `ruview.vital-evidence.v1` envelope +with capability `esp32_csi_spectral_vitals_v1`. `source_node_ids` names only the +radio producing the estimate, not the active-node inventory or occupancy-model +source. The legacy five-field mirror may remain for dashboard compatibility, +but must match the exact envelope. Firmware edge estimates use the distinct +`esp32_firmware_edge_vitals_v1` capability; this does not qualify them for a +consumer that only accepts raw CSI spectral estimates. Calibrated presence tags +the radio whose history the model actually scored, rather than the radio whose +packet happened to trigger publication. Complementary vital/occupancy sensors +are not silently claimed as joint vital inference. + +`GET /api/v1/vital-signs` retains its existing managed owner/account/paired-read +authorization. Its bounded `ruview.vital-publication-diagnostics.v1` object +separates the last packet's publication decision from current reevaluation for +each node. It reports candidate kind/tick/time/age, occupancy source, gate +outcomes, actual per-node detector buffers, reset count, and distinct detector +and observed sample clocks. It contains no raw CSI, suppressed BPM, credentials, +room names or user identity. It is not sensing authority or clinical evidence. +No such diagnostic is added to public health. Privacy mode withholds node/buffer +details and both vital envelopes. Diagnostic snapshots explain only their own +observation time; they do not extend rate/model freshness. + +The single-link calibration algorithm and negative-only restart prior are +unchanged by this clarification. Multiple radios in a room receipt do not imply +separate trained baselines. A per-source model bank and fused presence require a +separately reviewed capture, coverage, persistence and holdout contract. + ## Measured installation evidence One physical empty home run on source Node 5 collected 1,762 accepted frames @@ -119,6 +179,19 @@ This is measured negative evidence for one installation. No occupied positive holdout was recorded, so it does not prove person recall, person counting, or cross home accuracy improvement. +## Full room receipt versus restart prior (2026-10-22) + +An explicit room calibration can name multiple contributing radios, but this +persisted field model remains bound to exactly one model-writing radio and its +CSI grid. On helper restart, the full explicit session, room binding and receipt +are not restored. Client history cannot recreate that live server authority. + +Verify all three radios' fresh reception independently of the restored prior's +single source. A one-radio `bootstrap_only` source set is neither proof that +other radios stopped receiving nor proof of a restored three-radio calibration. +Original expiry and negative-only authority remain unchanged. Retaining a room +or scan on an app is a separate lifecycle from restarting its paired helper. + ## Acceptance test With a physically empty room, complete calibration on stable nodes, promote @@ -132,3 +205,14 @@ the source set for the entire capture. After restart, a confirmed empty holdout may refine the restored boundary once. The stored snapshot must reject a second refinement and must continue to deny calibrated evidence and numeric vital authority. + + +## Proposed amendment: bounded full-room restoration (2026-10-31) + +The bootstrap-only contract above remains the fallback for existing or incomplete saved images. A separately integrity-checked `room-receipt-v1.json` companion may now preserve the completed full-room receipt, but only after the server's independent held-out promotion succeeds. It binds the exact bootstrap payload digest, installation identity, room digest, original model ID, sorted contributor set, original frame count and completion time. It contains no raw CSI, room names, geometry, credentials or live history; its size is capped at 64 KiB and writes are private/atomic. + +On restart, both files must validate together under the current account-owned data directory and installation ID. No original lifetime is renewed. The process generates a new boot/session receipt for the same saved model and original completion time. Every recorded contributor must deliver fresh current-boot CSI on the frozen grid, the scored source must rebuild a full runtime window, and sequence faults or missing/stale sources withhold calibrated publication. A client with its own previously saved and passed check may reconnect the unchanged model only after account/origin/room binding, every immutable model field, current contributor readiness and a fresh matching calibrated stream are verified. It retains the original check timestamp and original model lifetime; it does not claim a new empty observation. A different or previously unverified model still requires explicit adoption and a new independent check. Bootstrap-only images cannot gain full authority from client history. Numeric Vitals still require their independent existing qualification gates. + +Reset, replacement capture and explicit downgrade to a bootstrap prior retire the companion receipt. A crash between the two writes leaves a digest mismatch and falls back to bootstrap-only, never a mismatched full model. Account erasure removes both through the owned helper data directory. Older saved priors cannot be upgraded from client history and require one new complete capture after installation of this change. Expiry/staleness policy (24/12 hours) is unchanged. + +Existing upstream PR #1529 was assessed before this amendment: it proposes aggregate FieldModel persistence but does not carry this installation/room/session authority boundary. The implementation reuses the reviewed snapshot/atomic-write infrastructure already present in ADR-355 rather than introducing a competing field-model format. Physical logout/relaunch acceptance remains required before claiming delivered restoration. diff --git a/docs/adr/ADR-419-secure-local-companion-pairing.md b/docs/adr/ADR-419-secure-local-companion-pairing.md new file mode 100644 index 0000000000..45dcad5e31 --- /dev/null +++ b/docs/adr/ADR-419-secure-local-companion-pairing.md @@ -0,0 +1,182 @@ +# ADR-419: Secure local Mac–iPhone companion pairing + +- Status: Accepted decision (2026-10-12); implemented; software verified, physical qualification pending +- Decision owner: product owner, approved in the implementation session +- Related: ADR-271 (OAuth), ADR-272 (WebSocket tickets); ruview-mobile ADR-046 +- Updated: 2026-10-20 — bounded transient sensor preview; physical qualification pending. + +## Context + +The managed Mac helper uses an ephemeral, full-authority owner token. Earlier +phone testing connected to an unauthenticated LAN helper. Exporting the owner +token or disabling authentication would not establish a safe public companion +workflow. An IP address and Bonjour identity are routing hints, not trust anchors. + +## Decision + +Use a separate opt-in TLS companion listener (managed Mac port 3443), with a +persistent installation certificate. The owner app remains on authenticated +loopback HTTP. Its token never leaves that owner channel. Companion management +routes require both the actual loopback socket peer and the exact owner token; +OAuth administrator authority and forwarded headers do not substitute for either. + +The Mac displays a five-minute, one-use QR invitation containing version, +installation ID, HTTPS local endpoint, certificate DER SHA-256 fingerprint, +invitation ID, and a 256-bit secret. The phone pins the certificate before sending +any secret. It generates an independent 256-bit device secret, submits enrollment, +and displays a comparison code. The Mac operator approves the matching request. +Observing the invitation alone cannot retrieve an approved phone's credential. + +The phone stores its device secret and certificate pin in device-only Keychain. +The server persists only device-secret hashes. Pairing lasts at most 30 days; +expiry requires explicit pairing again. A paired phone obtains a 12-hour access +token, capped by pairing expiry; access tokens and calibration grants are not +persisted on the server. Restart refresh uses the durable device proof over pinned +TLS. Permission to calibrate is a separate owner-granted 15-minute capability. +Only one phone holds calibration authority at a time; granting another phone +explicitly transfers control and removes the previous grant, while read access +remains. +Read authority never authorizes model, recording, configuration or administrator +mutations. Existing OAuth scope semantics are unchanged. + +Companion WebSockets use native authenticated TLS upgrades, not legacy browser +WS tickets. The initial supported stream is `/ws/sensing`. Unsupported streams +fail closed. Revocation and access expiry close established streams and prevent +new requests. Corrupt private identity/grant state or failed grant persistence +fails closed. No authentication disablement, certificate trust fallback, redirect +following or plaintext fallback is allowed. + +Bonjour may rediscover a route, but the persisted certificate pin and installation +must still match. DHCP changes do not change pairing identity. There is no cloud +relay or automatic certificate rotation in this decision; lost identity requires +explicit re-pairing. This is distinct from Finder/Xcode device pairing and Cognitum +account login. + +## Shared calibration authority + +The Mac explicitly publishes one bounded current room geometry and RF placement +revision. Publication uses compare-and-swap; phone import is explicit. It does not +transfer raw CSI, photos, recordings, teacher samples, models, historical checks +or full-evidence receipts. Both clients derive the same installation/room/revision +binding from canonical integer millimetres and heading tenths, independently of +loopback, LAN origin and transport. Geometry changes require republishing. +Companion calibration must match the published digest and complete node set. +Existing server boot/session, accepted-grid and evidence checks remain mandatory. +A phone cannot label an old receipt current or shorten the required baseline. + +## Deployment, rollback and verification + +Land producer and consumer PRs separately and bundle the identified helper only +after both protocols are verified. An older helper must leave pairing unavailable +without weakening authentication. Roll back the app/helper together while retaining +private pairing files; no rollback silently reenables unauthenticated LAN access. + +Required verification: exact compiled helper over real sockets; owner peer/token +isolation; QR expiry/replay/duplicate enrollment; unknown and oversized input; +wrong certificate/installation and redirect rejection; read-versus-calibration +scope; revocation of live streams; durable restart and corrupt-state behavior; +shared-room digest parity, revision conflicts and no stale-evidence import; native +iOS and Mac builds; actual phone–Mac pairing and reconnect. Synthetic tests and +authenticated transport are not physical sensing or room-selectivity acceptance. + +## Implementation evidence (2026-10-12) + +Producer PR #2249 and consumer `cognitum-one/ruview-mobile#57` implement this +contract. The producer package passed 1,143 tests (three existing ignores); the +compiled debug and release helpers each passed the 26-check real-socket probe. +The consumer passed 1,547 Jest tests, iOS/Mac builds and production Swift TLS, +path, recovery and disconnect-race checks. Adversarial review has no remaining +evidence-backed blocker for PR review. These results do not establish physical +QR scanning, room controls or distribution acceptance. + +## Sensor preview amendment (2026-10-20) + +Sensor sharing is a separate transient owner-granted capability, lasting fifteen +minutes and never implied by read or calibration authority. Owner grant/revoke +routes are `/api/v1/companion/devices/:id/sensors`; snapshot readback is +`/api/v1/companion/sensors`. Existing actual-loopback plus exact-owner-token +protection applies to all three. No companion may call owner routes. + +The dedicated pinned TLS `/companion/v1/sensors` WebSocket authenticates the +existing pairing/access proof plus current sharing permission and immediately +sends `sensor_ready`. Each exact-schema geometry frame is bounded to 64 KiB, +1–512 integer-millimetre points, a rigid finite camera transform, explicit capture +lineage, monotonic sequence and bounded source age. Unknown fields, images, +empty/malformed geometry, replay, lineage substitution and excess rate fail +closed. Device identity comes from the authenticated grant, never packet data. + +Only the latest frame exists in transient memory. Pairing revocation, access or +sharing expiry, owner revocation, supersession and socket closure suppress it; +owner readback marks frames stale after three seconds. A superseded socket cannot +clear its replacement's preview. The receiver additionally checks both source +and receive age before drawing geometry. No storage, camera feed, legacy relay +admin token, calibration promotion or ARKit-to-room alignment is added. + +The consumer independently opts in to foreground sharing and renders remote +geometry separately from local sensing/calibration authority. Software protocol, +real socket and owner/companion middleware checks do not establish physical +phone-to-Mac capture accuracy or calibration acceptance. + +## Retained replica boundaries (2026-10-22) + +The helper's durable `companion-room.json` is one canonical geometry envelope, +not a replica of either app's scan, models or local evidence history. The app +containers own those records separately; Cognitum sign-in and local pairing do +not synchronize them. Expiry or revocation withdraws transport access and does +not establish deletion of already retained client data. + +Every accepted room PUT increments the server revision. A client may avoid an +unnecessary new binding by validating and reusing the exact unchanged published +configuration instead of issuing another PUT. Importing that identical envelope +must preserve legitimate local evidence; a deliberate changed binding withdraws +current authority while local records remain historical. The client retention +repair is proposed in ruview-mobile ADR-047; it is not present in TestFlight +build 2026102213 and requires a subsequent delivered-build restart check. + +The sensor preview remains transient and currently carries no room-binding +digest. A displayed phone mesh is not a saved canonical room, durable draft, +live AR alignment or RF calibration receipt. MAT HTTP composition likewise does +not expand native companion WebSocket paths; see ADR-001's privacy amendment. + +## App-owned helper lifetime (2026-10-29) + +An embedded Mac helper may opt into `--parent-pid`. The value must be greater +than1 and match the OS spawning parent before account state or listeners open. +The helper watches reparenting from a native thread, requests graceful HTTP +shutdown when its app ends, and exits within roughly2.1seconds even if async +shutdown or an open socket cannot finish. The guard stays armed through final +persistence. Reparenting is checked rather than merely testing whether an old +PID exists, so PID reuse cannot transfer lifetime authority. + +Standalone servers omit this flag and retain their independent lifetime. This +does not grant control over another process or authorize port-based PID killing. +The Mac launcher still refuses an occupied foreign endpoint, with a bounded +vacancy wait for a previous managed helper to finish. Existing durable room, +pairing and account records are not removed by parent loss. Physical capture and +TestFlight acceptance must name the newly embedded helper, not an older build. + +## Explicit saved spatial artifacts (mobile issue102, 2026-10-30) + +A separate owner-only receive action opens one15-minute artifact slot for an +already paired phone. The paired TLS route accepts a single bounded geometry +payload with a one-use nonce and SHA-256 digest. Publisher identity comes from +the authenticated grant. Upload and acceptance both match the current canonical +installation, room revision and binding; they serialize with room publication. +Pairing revocation and permission expiry withdraw pending review authority. + +Only an explicit owner acceptance retains the artifact. It is historical geometry, +not a calibration/model import: no points enter the detector, no room dimensions +are overwritten and no cold-restored scan establishes current AR alignment. +The payload has at most128 surfaces and64 anchors,48KiB encoded content, capture +and coordinate provenance, and an optional source calibration digest. Raw camera, +depth and CSI are excluded. Accepted history is capped at32 records without eviction. + +Pending permissions/artifacts are memory-only. Accepted history is atomically +written0600 in the account-owned private companion directory. Startup checks +account/installation ownership, geometry bounds and content digests; corrupt, +non-private or symlinked files fail closed. A persistence error freezes the inbox +until restart so an uncertain rename/fsync result cannot be reported as accepted +or followed by further writes. Pairing expiry does not delete already accepted +history. Consumers must implement review and distinguish this retained history +from current alignment. Rollout requires a rebuilt helper and mobile consumer; +source tests are not delivered-device or measured accuracy evidence. diff --git a/docs/adr/ADR-420-managed-source-aware-room-learner.md b/docs/adr/ADR-420-managed-source-aware-room-learner.md new file mode 100644 index 0000000000..162eae24f5 --- /dev/null +++ b/docs/adr/ADR-420-managed-source-aware-room-learner.md @@ -0,0 +1,430 @@ +# ADR-420: Managed source-aware room learning and durable candidate evidence + +**Status:** Proposed +**Date:** 2026-11-01 +**Deciders:** RuView maintainers and app delivery owners; privacy and reference-sensor owners for teacher admission + +## Context + +The managed application has an installation-bound selected-source background +detector. Its live feature path also receives multiple nodes. Receiving those +nodes does not establish that a room model was trained on each source. + +Existing implementations supply substantial building blocks: + +- [ADR-024](ADR-024-contrastive-csi-embedding-model.md) describes contrastive CSI + representations; AETHER supplies the Rust transformer and projection stack. +- [ADR-027](ADR-027-cross-environment-domain-generalization.md) covers domain + generalization. Geometry/domain adaptation requires measured inputs and a + composed training path. +- [ADR-151](ADR-151-room-calibration-specialist-training.md) defines room + specialists and statistical multi-node late fusion. It remains a useful + comparative baseline with its own enrollment contract. +- [ADR-008](ADR-008-distributed-consensus-multi-ap.md) concerns distributed + AP/server coordination. Several receivers observing one AP do not establish + that distributed consensus has occurred. +- `ruview-unified` provides native RF records, masks and provenance; + `ruview-groundtruth` and `ruview-fusion` provide distinct reference and claim + fusion mechanisms. Their presence does not establish managed-server callers. + +The missing seam is a managed source-preserving capture, actual learning, +account-owned persistence and evaluation lifecycle. The first target is +room-selective binary presence: empty, occupied-static, occupied-moving and +adjacent-only controls. Pose, person count and numeric Vitals require separate +acceptance. + +Implementation is reviewed in [#2423](https://github.com/ruvnet/RuView/pull/2423), +with the client in +[mobile #122](https://github.com/cognitum-one/ruview-mobile/pull/122). +[Tracker #2418](https://github.com/ruvnet/RuView/issues/2418) separates source, +artifact, device and physical acceptance. This ADR and those draft PRs do not +constitute architectural acceptance or measured physical accuracy. + +## Decision + +### Preserve the admitted RF view + +Use native `RfFrameV2` records with original IQ, validity masks and explicit +provenance. The initial learner consumes derived magnitude features only. +Preserve gain and DC information needed by the task; avoid per-window centering +that removes the evidence the model is expected to learn. + +Use an opt-in convex magnitude projection. The existing cubic control path can +undershoot below zero at valid spectral nulls. Keep that control path unchanged +for comparison; bind the new projection to `usable-drop2-linear-v2` so previous +candidate preprocessing cannot be silently relabelled. + +Key readiness by the reported receiver, antenna count, native bin grid, PPDU, +frequency and bandwidth. A stable arrival stream for one class cannot supply +cadence for another class later rejected by the adapter. Require both the +bounded stability window and the fresh packet window consumed by the learner. +Ignore other PHY classes from the same receiver without admitting them into the +selected view. Missing selected inputs continue to withdraw inference coverage. + +The first adapter supports its explicitly recognized narrow-band layouts. +Primary-channel metadata must not be silently treated as the centre of a bonded +wide channel. Receiver timing remains receiver timing; it does not establish a +measurement clock, coherent phase, authenticated transmitter or calibrated +signal quality. + +### Train a real source-conditioned candidate + +Reuse the AETHER transformer, with context-ordered source conditioning, +source-routed projection heads and a room-presence head. Optimize the actual +symmetric InfoNCE and supervised BCE objectives. Require intended backbone and +projection updates; a task-head improvement or a logged loss alone is +insufficient evidence of representation learning. + +Treat aligned views of the same observation as positives. Exclude sibling trials, +shared frame identities and overlapping windows from contrastive negatives. +Freeze train, validation and held-out roles before fitting. Fit normalization +profiles from Train only. Several receivers on the same AP are correlated views; +their count is not an independent-confidence multiplier. + +The initial bounded optimizer/model is a vertical integration candidate. Physical +comparison determines whether it has adequate capacity, robustness and latency. +This decision does not establish equivalence to a pretrained pose model or +justify scaling capacity without new evidence. + +### Keep lifecycle and authority distinct + +Expose explicit configure, prepare/start, complete/cancel, train, evaluate and +resume operations through the existing protected API. Feature support in a +verified helper manifest permits the runtime flag; it grants no model authority. + +Every mutation carries the observed runtime epoch, context, generation, workflow +revision and active phase identity. Compare and mutate under the same lock. +Return prediction identity and its score atomically. Client read polling and +mutation ownership must be separate so a queued poll cannot abort an action or +leave the UI busy. + +Preparation is excluded from labelled collection. Completion requires the +operator's conditions-held acknowledgement. A completed recording interval can +remain provisional while that acknowledgement is pending. By default, untrained +observations stay in bounded memory and can be lost if the helper or account +runtime closes. The proposed explicit-consent recovery extension below is a +narrow local-storage exception; it is not enabled by an app update or helper flag. + +Store trained candidates privately beneath the verified owner/installation +namespace, with immutable model versions, hashes and an exact-context pointer. +Preserve previous rooms/models and original provenance through cold load. +Logout retains owned records; another account cannot load them. Resume must use +the trained view rather than today's preferred, different source class. + +Candidate evaluation is diagnostic. Preserve aggregate confusion, abstention and +condition metrics with exact model/context/role/data hashes and original time. +Coverage describes evaluated cohorts, not wall-time availability. Distinct +declared trial IDs do not prove statistical independence. Missing or abstained +positives cannot disappear from unconditional sensitivity. + +Historical evaluation must remain historical after restart. Missing retained +Train identities cannot be converted into a newly verified lineage claim. +Corrupt evaluation evidence must be retained/quarantined without discarding a +valid candidate or manufacturing a pass. Evaluation receipts authorize neither +physical qualification nor adoption. + +### Proposed November 3 extension: local qualification, explicit selection and Room Save + +The current app's legacy Room Save requires a background-model receipt and its +empty check. A separately trained learner cannot satisfy that contract merely +by completing capture or evaluation. Do not manufacture a legacy receipt, or +make the user repeat both workflows without explaining the missing seam. + +Introduce a distinct, versioned **local operational qualification**. The server +computes it from the immutable candidate, exact retained Train lineage, +validated role-disjoint HeldOut data, managed sealed-phase history, and the +existing immutable evaluation artifact. The policy is fixed before evaluation; +clients cannot submit scores, thresholds or a claim that qualification passed. +Reuse the targets proposed in issue #2418: at least four Train and four HeldOut +cohorts for each of Empty, OccupiedStatic, OccupiedMoving and AdjacentOnly; +unconditional sensitivity at least 90% for each occupied condition; false +positive rate at most 5% for each negative condition; evaluated cohort coverage +at least 90% in every condition. Missing results, abstentions and unavailable +rates cannot be counted as correct. At the minimum cohort count these gates +require all four observations in every condition to be correctly scored. + +These are operator-labelled representative-cohort checks for one installation. +They do not measure wall-time coverage, independent trial statistics, p95 +decision latency, coherent source clocks or general physical accuracy. Preserve +the original diagnostic receipts' false `physically_validated`, +`independence_verified` and `adoption_authorized` fields. The separate receipt +names its narrower authority, policy hash, exact model/evaluation/data/context +hashes, label provenance, original observation bounds and evaluation time. +Older trusted managed sealed history may establish that the existing +conditions-held gate was passed; it cannot acquire invented historical +confirmation times or full-duration coverage. + +Qualification never selects a model automatically. Explicit guarded selection +binds the verified owner, installation, room revision, ordered source/PHY +context, immutable model and qualification hashes. Candidate discovery, +training, evaluation, a new latest-model pointer, or consent completion cannot +switch the selected model. Keep previous selection history for explicit +rollback. A newer completed check that fails for the selected model withdraws +current authority without rewriting its historical passing receipt or silently +falling back to another detector. + +Persist model and selection independently of optional capture recovery. The +24-hour reduced-sample recovery deadline does not expire a trained model or +renew its evidence. After restart/logout the original selection and evidence +times remain historical. Current use additionally requires the actual published +room binding, exact artifact/policy verification, complete fresh source windows, +and the model's existing out-of-distribution checks. Missing sources, changed +room/PHY context, corruption, failed checks or unknown model state produce an +explicit unavailable/abstaining result. They must not be rendered as an empty +room or trigger an unlabelled heuristic fallback. + +Publish the selected result through a versioned binary-only evidence surface. +It grants no person count, movement class, location, pose, numeric Vitals, +HomeCore or cloud authority. Keep legacy output visibly attributable as a +comparison where compatibility requires it; selected-model clients exclusively +use the selected proof and its withdrawal state. Withhold downstream projections +whose contracts cannot preserve this binary evidence boundary. + +The app stores a discriminated learner installation receipt and its own saved +timestamp/history. It does not reuse the legacy baseline's receipt or timestamp. +Room Save validates a fresh exact selection proof around durable persistence, +and reports saved progress separately from qualified/selected/current state. +Attaching a model must not republish geometry: the existing companion publish +operation increments its revision and would invalidate the candidate's binding. +Paired clients receive only the bounded read-scoped proof; qualification and +selection mutations remain with the managed owner. Any final native delivery +still requires composed tests and actual device qualification. + +### Proposed November 2 extension: opt-in provisional capture recovery + +The operator can explicitly choose encrypted local recovery when configuring a +run. This retains bounded **reduced magnitude training samples** in selected +representative/sealed cohorts, original source/context/role/time identities and +sequence watermarks. It excludes raw IQ/CSI packet files, camera/depth images, +teacher frames, live receiver windows and pending packet buffers. It is not a +blanket relaxation of the default no-observation-retention policy. API metadata +must disclose when derived samples persist rather than keeping a false raw-data +absence flag. + +Use the existing per-owner persistent Keychain signing seed only as input to a +domain-separated HKDF purpose key bound to owner and installation. AES-256-GCM +uses a fresh nonce and authenticates the descriptor. No key enters JavaScript, +HTTP responses or diagnostics; the existing native-to-helper secret mechanism +is unchanged. A nonsecret stability marker is true only for actual persistent +Keychain sources. Missing, ephemeral or unavailable keys cannot silently fall +back to plaintext or claim recoverability. Other platforms without a qualified +private-file/key path keep the RAM-only workflow. + +One owner-bound recovery slot is limited to64MiB of plaintext plus bounded +framing,0700 directories and0600 regular files, with authenticated atomic +write/fsync/rename/directory barriers. Symlinks, unsafe ownership/permissions, +wrong keys, identity/context mismatches and unknown/corrupt formats fail closed. +The original24-hour recovery-availability deadline never renews on save/read/ +resume. Expiry withdraws recovery; it does **not** promise that a stopped app +can automatically erase ciphertext. The UI states that an expired encrypted +copy may remain until explicitly discarded. Authenticated expired records can +be explicitly deleted by their guarded identity; unreadable unknown files remain +blocked rather than being overwritten or silently erased. + +After restart, recovery is explicit and requires the exact current room, +revision, source/PHY contract and verified owner/installation. Restore preserves +original evidence time and role isolation. A completed phase remains pending +operator conditions-held confirmation. An incomplete interrupted phase fails +and must restart; waiting through downtime never creates measured windows. +Restored live/pending buffers are empty, so new measured packets must establish +current freshness before inference. Interrupted fitting becomes an explicit +retry state, never an automatic job. A saved model is restored by its exact +immutable hash, not today's pointer. Neither recovery nor retention grants +physical qualification, statistical independence or adoption. + +Serialize checkpoint writes and disable/delete under the same runtime lock, +with generation/phase guards and bounded failed-write retry. Cancellation or +discard never acknowledges deletion after I/O failure. A late writer cannot +recreate a successfully discarded copy. Saved state is shown only after durable +success; unavailable/unverifiable retained contents report `null` (unknown), not +`false` (absence), in persistence metadata. Confirmed empty/deleted state may +report false. Full account erasure still drains/stops the helper before removing +its tree; narrower checkpoint discard preserves room scans, candidates and +control/evaluation artifacts. + +Software fixtures are not crash/power-loss or signed-device acceptance. The +extension remains Proposed pending privacy/maintainer review and actual Mac/ +iPhone interruption, owner-isolation, expiry and retention verification. + +### Admit optional teachers through a validated reference boundary + +The initial RF workflow does not activate radar, camera or LiDAR teaching. +Current receipt strings and historical geometry are insufficient admission +proofs. A server-owned registry must resolve authenticated source/session +identity, measured clock mapping, spatial registration, coverage, expiry and +revocation before constructing trusted derived teacher evidence. + +Visible-region absence and whole-room absence are different capabilities. +Occlusion, missing data and out-of-view observations are UNKNOWN. Adjacent-only +labels require target-room absence and separate adjacent activity evidence. +CSI/radar-fused outputs cannot serve as independent RF ground truth. + +References may teach a frozen RF input model. Phone departure ends future +teaching; RF-only inference must remain usable under its own acceptance gates. +Measured spatial benefit belongs to +[mobile #105](https://github.com/cognitum-one/ruview-mobile/issues/105), including +current anchors/transforms and aligned, whole-sequence comparisons. + +### Preserve the control and require explicit adoption + +Keep the selected-source detector and source-bank shadow as labelled controls. +The candidate does not replace their decisions automatically. Adopting a model +requires a separately reviewed policy and matched fresh evidence; saving a +candidate, reporting accuracy or restoring an old receipt cannot substitute. +Presence acceptance does not grant pose, count or Vitals authority. + +Full local account erasure must stop the helper before deleting its owned tree. +Selective learner erasure needs its own scope, generation/cancellation fence and +writer drain before removing artifacts or RAM. Legacy calibration reset and +phone-local spatial deletion must not imply they remove server learner history. + +## Options considered + +| Option | Complexity and reuse | Evidence and limitation | +|---|---|---| +| Independent background models | Reuses existing detector; useful shadow control | Does not train a joint representation or establish occupied/adjacent discrimination | +| Managed source-conditioned learner | Reuses native records and real AETHER computation; adds lifecycle/evidence seams | Requires physical qualification, reference admission and explicit adoption | +| Pretrained pose backbone first | Potential transfer benefit after input/task adaptation | Cannot repair source identity, preprocessing, ownership or reference-registration gaps by itself | + +Choose the managed composition as the next implementation, retaining the other +options as comparisons or later adapters. Keep new architecture review separate +from source CI, packaged native verification and end-user validation. + +## Consequences + +- Source admission and training attribution become explicit and testable. +- Candidate/model/evidence histories can survive sessions without renewing trust. +- Real-world preprocessing errors are visible before spending a room-capture window. +- Default RAM capture has an interruption limitation; explicit recovery adds encrypted sample-retention consent and cold-restore obligations. +- Additional sources or modalities require compatible view and reference contracts. +- Model capacity, generalization and uncertainty remain empirical questions. + +## Action items + +- [ ] Review this decision and the managed source/client PRs. +- [ ] Qualify the delivered native artifacts and independent physical conditions. +- [ ] Complete durable evaluation integration and historical UI attribution. +- [ ] Implement fenced selective learner erasure with absence/resurrection tests. +- [ ] Agree an adoption/rollback policy before enabling candidate authority. +- [ ] Validate reference clock/coverage/registration and optional derived teachers. +- [ ] Measure spatial, multi-source and reference benefit against frozen controls. + +## November 4 amendment: durable managed training and sealed-phase fences + +**Updated:** 2026-11-04. This extension is implemented in the source candidate; +the ADR remains Proposed. Software fixtures do not qualify a room, reference +sensor, delivered app or transmitter epoch. + +A v1 training manifest preserves the original Train SHA, total count and input +claims. It does not prove four managed, operator-sealed Train cohorts for every +condition. A stored v1 model without retained exact Train rows therefore remains +diagnostic: new HeldOut data alone cannot recover its missing training history. +Do not infer counts from the total, create acknowledgements from operator +recollections, or inspect expired encrypted samples to manufacture a proof. + +At successful managed fit, freeze a metadata-only seed while the actual native +Train rows and phase receipts are still available. Validate the exact original +Train SHA, four cohorts per required condition, unique sealed phase matches, +receiver-observed frame namespace, source ordinal/time ranges and original wire +counter watermarks. Bind this seed to the exact exported model SHA, complete +owner/installation/context contract, source/PHY contract, and domain-separated +trial/session role commitments. Store its immutable private proof and initial +sealed-phase fence, fsync and verify readback **before** publishing the candidate +pointer. Generic fitter/model-import persistence cannot issue this proof. +Retries reuse the first committed metadata, not later receive times. + +The proof contains no RF feature values, IQ, camera/depth frames or raw frame +lists. Metadata includes source counter/range bounds and hashed phase identities. +It uses the existing owner-private artifact directory, regular-file protections, +immutable hashes and durability primitives. This is integrity under the private +filesystem trust boundary, not a hardware signature or authenticated RF-source +attestation. A fully privileged local writer is outside that boundary. + +Fit-time fences alone are insufficient after later evaluations. Before a +post-fit phase becomes Sealed in RAM or its finish is acknowledged, commit a +bounded immutable metadata record and atomic verified ledger head. The record +extends the original model/proof identity, phase commitments and per-source +sequence/ordinal/receiver-time floors. Publish one newly sealed phase per link; +include no RF values. An I/O failure keeps the complete original phase retryable. +A retry can finish its exact pending publication but cannot replace its original +metadata with newer timestamps. A missing, torn, conflicting or unpublished +ledger blocks model resume/qualification; it cannot become an empty ledger. +This bounds previously sealed/labelled evidence, not every unused UDP packet. +Existing phase/cohort capacity limits also bound the ledger. + +Resume verifies the proof and complete ledger, restores the original capture +run/frame namespace, next phase index, frozen roles, original Train ranges and +maximum sealed source watermarks, and leaves all live/pending RF buffers empty. +It creates no dummy Train rows. Old sealed trial identities cannot be reused. +A current declared session may group multiple same-role conditions; a resumed +workflow needs a fresh group identity. Client IDs include the existing helper +runtime epoch and generation so a helper restart cannot collide with old groups. +Session declarations still do not establish statistical independence. + +Native ingress uses UTC-anchored monotonic receiver nanoseconds, not helper +uptime. The proof explicitly records that receiver clock domain. A lower new +helper wall anchor is incomparable and blocks capture; neither counter nor time +is silently rebased. Duplicate/reversed node counters remain rejected. Resetting +a node does not prove a new authenticated transmitter epoch. + +Qualification always recomputes the original Train SHA while rows remain. When +Train is empty, only a verified managed proof and a complete fresh disjoint +HeldOut set can provide that lineage basis. The per-condition minimums, coverage, +sensitivity, false-positive thresholds, OOD checks and explicit selection remain +unchanged. A failed latest assessment continues to withdraw positive authority; +restart cannot reuse its old labelled packets to clear the failure. + +Retained-row qualifications retain their v1 bytes/readers. Proof-backed receipts +use explicit `ruview.local-operational-qualification.v2`, bind the exact managed +training proof and sealed-phase fence SHA, and identify their lineage basis. +Their UTC nanoseconds are decimal strings, preserving precision beyond +JavaScript's safe integer range. Evaluation v1 still reports +`train_lineage_checked: false` at zero current Train rows. The bounded v2 public +summary carries only its version, lineage basis and opaque proof hash in addition +to the existing qualification summary; private phase identities/counters and +nanoseconds are not added to paired summaries. + +Owner status exposes a bounded qualification-readiness preflight before asking +for selection-oriented HeldOut captures: exact durable model, current room/source +binding, retained exact Train or verified managed proof, and comparable receiver +clock. Missing historical proof is a diagnostic limitation, not an invitation to +repeat unsaveable trials. Existing issued v1 qualification/selection readers are +preserved; no model is selected automatically and no general physical, +independence, count, pose or numeric Vitals authority is added. + +Managed lineage checkpoints use explicit capture-checkpoint v2 to retain these +metadata fences while v1 checkpoints remain readable. Optional reduced-sample +recovery keeps its original consent and expiry contract. Candidate weights and +bounded training/phase metadata survive that sample-recovery deadline in the +same owner-private model tree until explicitly erased under the model/account +lifecycle. They are not a retained sample set or a promise of automatic erasure. + +Verification covers synthetic packets through real managed fit, private writes, +cold resume, all-condition qualification, explicit selection, corrupted or pending +proof/ledger files, unchanged retained-row lineage, role/counter/time regression, +phase-ack ordering, and 2026-scale Rust-to-client wire fixtures. Physical +qualification and delivered-artifact acceptance remain separate work. + + +### Proposed November 5 extension: protect the current new RAM run in place + +An explicit local-owner-only `recovery/enable-current` command may checkpoint a +new RAM capture without configure/reset, training, selection or phase relabelling. +Its versioned request requires the current mutation guard, an opaque server-issued +run ticket and explicit reduced-sample consent. The ticket exists only for a new +training session configured in this process without recovery; restored captures +and resumed models cannot mint it. The fixed deadline is **24 hours from that +new run's creation**, not 24 hours from pressing Enable. Success consumes it; +discard, retry, delayed requests and clock regression cannot renew that deadline. + +The first version requires no active phase or training job. It saves and verifies +the exact existing snapshot through the existing encrypted atomic store before +publishing enabled status. RAM capture, counters, roles, model and selection remain +unchanged on failure. Persistence failures report unknown retained contents rather +than claiming no samples were saved. Only a partial write under the same opaque +ticket/context/original deadline can be retried. Any other retained slot, including +an expired copy, requires a separate explicit discard decision; it is never silently +replaced. Existing recovery descriptor/checkpoint versions and expiry semantics are +unchanged. This is proposed source work, not protection of a currently running +older helper: installing it would require stopping that helper, so live volatile +cohorts must not be sacrificed to obtain this capability. diff --git a/docs/adr/README.md b/docs/adr/README.md index d4b5167dff..24b3b97793 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -2,6 +2,8 @@ Latest proposed decisions: +- [ADR-420: Managed source-aware room learning and durable candidate evidence](ADR-420-managed-source-aware-room-learner.md) + - [ADR-379: An OTA'd image confirms itself with a bounded health check, or rolls back](ADR-379-ota-first-boot-health-check.md) - [ADR-380: Loopback UDP tee for second consumers of the CSI stream](ADR-380-loopback-udp-tee.md) - [ADR-382: `spatial.evidence.v1` export of RF Gaussians and link observations](ADR-382-spatial-evidence-rf-export.md) @@ -135,6 +137,7 @@ Statuses: **Proposed** (under discussion), **Accepted** (approved and/or impleme | [ADR-346](ADR-346-fail-closed-edge-occupancy-evidence.md) | Fail closed ESP32 occupancy evidence | Accepted (C6 occupancy integrity qualified) | | [ADR-347](ADR-347-rate-aware-esp32-temporal-sensing.md) | Rate aware ESP32 temporal sensing | Accepted (C6 timing and transport qualified) | | [ADR-363](ADR-363-native-macos-usb-node-onboarding.md) | Native macOS USB node onboarding with nonce bound discovery and fresh server verification | Accepted and physically qualified on one ESP32 S3; C6 and signed distribution pending | +| [ADR-419](ADR-419-secure-local-companion-pairing.md) | Secure local Mac–iPhone pairing, canonical room publication and transient sensor preview | Accepted decision; software verified, physical qualification pending | | [ADR-115](ADR-115-home-assistant-integration.md) | Home Assistant integration via MQTT auto-discovery + Matter bridge (HA-DISCO + HA-FABRIC + HA-MIND) | Accepted (MQTT track) / Proposed (Matter SDK P8b) | | [ADR-169](ADR-169-adam-mode-light-theme.md) | adam-mode — light theme toggle for the three.js realtime demo | Proposed | | [ADR-170](ADR-170-yoga-mode-pose-system.md) | yoga-mode — yoga pose detection, classification, and scoring for the three.js realtime demo | Proposed | diff --git a/docs/api/paired-calibration-progress-v1.md b/docs/api/paired-calibration-progress-v1.md new file mode 100644 index 0000000000..9b62f94298 --- /dev/null +++ b/docs/api/paired-calibration-progress-v1.md @@ -0,0 +1,70 @@ +# Paired calibration progress v1 + +Implemented as a proposed read-only companion surface. It reports the managed +Mac's RF learner, not a second phone learner or legacy baseline controller. + +`GET /api/v1/room-learner/progress` uses existing sensing-read authentication and +the companion's installation-pinned, owner/account-bound read grant. Neither a +read nor legacy calibration grant permits POST to this route or owner-only +learner controls. The existing selection envelope advertises +`progress_schema: "ruview.room-learner.progress.v1"`. Consumers discover through +that authorized read first, without first parsing an unsupported inner +selection version or requiring a historical selection to match today's room. +An absent discovery field means selection-only; +old TLS policy may reject an unknown progress route with403 before reaching404. +A404 can also degrade to the selection-only view, but privacy/authentication +failures are never a fallback signal. Do not probe owner status. + +Success schema: `ruview.room-learner.progress.v1`. The explicit DTO contains: + +- Public installation ID, published room binding/revision, context match, + `runtime_epoch`, `observed_at_unix_ms` and `expires_at_unix_ms`. +- `runtime_epoch` is the existing public calibration process identity. It is + **not** the learner's private mutation-guard runtime epoch. +- Recorded Train/HeldOut counts for four conditions; these are sealed records + in the current capture session, not proof of passing accuracy or all history + in a durable model. Zero temporary records do not mean a model was lost. +- Bounded phase role/label, preparation/countdown/elapsed/target and representative + counts, plus aggregate source readiness. No source IDs, packet counters, + RF values or physiological values are included. +- Separate verified qualification and selection summaries, a bounded machine + blocker and a workflow stage. These do not start a capture, train, qualify, + select or enable another capability. + Qualification names the current candidate where available; selection names + the separate durable selected model. Their model hashes may differ while a + replacement candidate is being prepared. No candidate means qualification is + not reported, rather than a claim that an older selected model lost its proof. +- `save_status` is always `not_reported`: model selection does not prove that + the Mac app's separate durable Room Save transaction completed. + +The snapshot expires at most10seconds after observation, shortened by the +existing paired/account grant's expiry. A revoked or expired grant is refused +before response. Consumers must also fence account, transport, installation, +publication, request generation and runtime changes; reject delayed/stale data. +Published binding is held stable through projection. A concurrent learner +context change cannot borrow readiness from the prior source/grid context. +A published-room mismatch withholds counts, phase, source readiness and all +current qualification/selection claims rather than relabelling another room. +A historical selection for another room/context is not reported as this room's +selection and does not alone block setup for the newly published room. + +An empty/unconfigured room has null progress fields. An optional durable-proof +readiness field is not implemented by this base producer. The blocker +`training_lineage_not_reported` means only that this view does not report that +lineage; it is not a statement that a stored model or future managed proof is +invalid. Review readiness on the Mac before requesting physical trials. + +Privacy mode returns403 with only schema/success/error_code. No owner scope, +mutation guard, trial/session IDs, model weights, private training-proof data, +raw source counters, media or physiology is projected. Unknown/corrupt artifact +readback cannot produce a qualified/current badge. + +"Read-only" means no logical capture, model, geometry, checkpoint or Save +mutation. Existing trusted artifact readers still enforce private-directory +security idempotently (0700); this is not a claim of zero filesystem metadata +side effects. The progress method deliberately does not call owner status, +which can write capture checkpoints. + +Synthetic producer fixtures are generated by the real HTTP handler test +`companion_integration::paired_progress_auth_tests::export_actual_paired_progress_producer_cases`. +They are software evidence, not physical sensing qualification. diff --git a/docs/security/wasmtime-audit-20261119.md b/docs/security/wasmtime-audit-20261119.md new file mode 100644 index 0000000000..77d09f9aa4 --- /dev/null +++ b/docs/security/wasmtime-audit-20261119.md @@ -0,0 +1,57 @@ +# Wasmtime 36.x patch and advisory metadata reconciliation + +## Problem and actual patch + +RustSec database7133866a reported five entries against core wasmtime36.0.20. +Primary upstream advisory HTML and repository advisory API distinguish the actual +package/version scopes from those raw matches: + +| RustSec ID | Upstream scope | Result for this workspace | +|---|---|---| +|0569|[Core below36.0.21](https://github.com/bytecodealliance/wasmtime/security/advisories/GHSA-hj27-mpw9-ff4c)|Version-applicable; patch to36.0.21| +|0567|[Core branches starting at39](https://github.com/bytecodealliance/wasmtime/security/advisories/GHSA-2674-w29h-wqvj)|36.x outside affected ranges| +|0573|[Core branches starting at47](https://github.com/bytecodealliance/wasmtime/security/advisories/GHSA-wvgq-g5hf-6gm6)|36.x outside affected ranges| +|0566|[wasmtime-wasi](https://github.com/bytecodealliance/wasmtime/security/advisories/GHSA-5m8g-xhq4-4h93)|Package absent; RustSec matched the wrong package| +|0571|[wasmtime-wasi-http](https://github.com/bytecodealliance/wasmtime/security/advisories/GHSA-f8mm-ggr3-89wh)|Package absent; RustSec matched the wrong package| + +The compatible36.0.21 patch updates32 Wasmtime/Cranelift family lock entries, +retains Rust1.89 compatibility and keeps the optional backend default-off. The +internal cache is re-vendored from its actual36.0.21 archive, preserving the +reviewed zstd0.14.3 manifest correction and byte-identical upstream executable +source/tests/licenses. Existing audit.toml ignores are unchanged. + +An independent controlled imported/re-exported host-function regression panics in +36.0.20's get_module_export implementation. The same test passes with36.0.21. +It is retained under the optional Wasmtime feature. This tests the dependency +patch; it is not a claim that the existing application invokes that faulty API or +that a deployed exploit was demonstrated. + +## Upstream correction and native audit — 2026-11-20 + +RustSec commit `074ea9a7ed619f511b5b7374c7a97e98d1e98be4` corrected both +range errors by recording `<39.0.0` and `<47.0.0` as unaffected for0567/0573. +With the patched lockfile, the CI-pinned cargo-audit0.22.2 now exits0 with zero +native registry findings under the unchanged audit policy. + +The temporary selector reconciliation, fixtures and live GitHub advisory API +calls have therefore been removed. CI again retains only the unmodified native +JSON report and uses its exit status, with pipefail preserving auditor failure. +No custom classification, extra token, offline database or advisory exception +is needed. Any future native advisory failure remains a job failure. + +Invoke `cargo-audit audit --file Cargo.lock --json` directly from `v2/`. +Direct binary invocation avoids executing a project-defined Cargo audit alias; +the installed tool remains pinned to0.22.2. An actual controlled Cargo-alias +negative probe established that the old dispatcher could execute a benign marker +and return artificial clean data; the direct auditor ignored that alias. Those +negative-control reports are not real security evidence. + +The dependency patch remains necessary:36.x before36.0.21 is still affected by +0569. The controlled API regression and archive provenance checks above remain. +Historical raw reports and reconciliation receipts describe the earlier database, +not the current native result. cargo-audit0.22.1 separately reports local path ZIP +packages as registry matches; it is not the pinned CI tool. Actual registry ZIP9 +and adapter tests remain separately relevant to the held server integration. + +No plugin is enabled, helper replaced, app installed or physical sensor qualified +by this change. Merge and release remain separate maintainer decisions. diff --git a/firmware/esp32-csi-node/README.md b/firmware/esp32-csi-node/README.md index d78fd4b5aa..81a359be56 100644 --- a/firmware/esp32-csi-node/README.md +++ b/firmware/esp32-csi-node/README.md @@ -304,6 +304,25 @@ The boot log must report the intended `internal ceramic` or `external U.FL` path before that measurement is labelled. The standard C6 build keeps the board default and emits no antenna-path claim. +#### Seeed XIAO ESP32-C6 MR60BHA2 carrier + +The assembled Seeed MR60BHA2 kit routes the radar UART to XIAO TX GPIO16 and +RX GPIO17. It also uses native USB Serial/JTAG for logs and onboarding, so the +radar UART does not overlap the console. Generic C6 boards keep the GPIO4 and +GPIO5 defaults. + +Build the dedicated carrier profile instead of flashing the generic C6 image: + +```bash +idf.py -D SDKCONFIG_DEFAULTS="sdkconfig.defaults;sdkconfig.defaults.esp32c6;sdkconfig.defaults.xiao-mr60bha2" \ + set-target esp32c6 +idf.py build +``` + +The boot log must identify `MR60BHA2` and the server must receive fresh +`edge_fused_vitals` packets before the radar is considered connected. Radar +reported heart and breathing values remain unvalidated, nonmedical evidence. + ### 3. Provision WiFi credentials (no reflash needed) ```bash diff --git a/firmware/esp32-csi-node/main/Kconfig.projbuild b/firmware/esp32-csi-node/main/Kconfig.projbuild index 6e40a659bc..a520633ebf 100644 --- a/firmware/esp32-csi-node/main/Kconfig.projbuild +++ b/firmware/esp32-csi-node/main/Kconfig.projbuild @@ -19,6 +19,16 @@ menu "CSI Node Configuration" external antenna. Leave this disabled for generic ESP32-C6 boards because GPIO3 and GPIO14 may have unrelated functions there. + config C6_XIAO_MR60BHA2 + bool "Use the Seeed XIAO ESP32-C6 MR60BHA2 carrier UART" + depends on IDF_TARGET_ESP32C6 + default n + help + Enable only for the assembled Seeed MR60BHA2 kit with a XIAO + ESP32-C6. The carrier routes the radar UART to XIAO TX GPIO16 and + RX GPIO17. The standard C6 profile keeps the generic GPIO4/GPIO5 + assignment so existing DevKit wiring is not changed. + choice C6_XIAO_ANTENNA_PATH prompt "Seeed XIAO ESP32-C6 antenna path" depends on C6_XIAO_ANTENNA_SELECT diff --git a/firmware/esp32-csi-node/main/mmwave_detect.h b/firmware/esp32-csi-node/main/mmwave_detect.h index 9fd9ade36d..f666497a97 100644 --- a/firmware/esp32-csi-node/main/mmwave_detect.h +++ b/firmware/esp32-csi-node/main/mmwave_detect.h @@ -25,6 +25,14 @@ static inline bool mmwave_mr60_identifying_type(uint16_t type) return type == 0x0A15 || type == 0x0A16 || type == 0x0F09; } +/** MR60BHA2 vital-sign firmware exposes one authoritative presence bit, not a + * dependable people-count frame. Map that bit to the single-target contract + * used by RuView's fused-vitals packet. */ +static inline uint8_t mmwave_mr60_target_count(bool person_present) +{ + return person_present ? 1u : 0u; +} + /** * True iff buf[i..] begins a *validated* LD2410 report frame within [0,len): * F4 F3 F2 F1 | len(LE,2) | data[len] | F8 F7 F6 F5 diff --git a/firmware/esp32-csi-node/main/mmwave_pins.h b/firmware/esp32-csi-node/main/mmwave_pins.h index 634265e92a..4605baf115 100644 --- a/firmware/esp32-csi-node/main/mmwave_pins.h +++ b/firmware/esp32-csi-node/main/mmwave_pins.h @@ -1,5 +1,21 @@ #pragma once #include +#include + +static inline void ruview_c6_mmwave_default_pins(bool xiao_mr60bha2, + int *tx, int *rx) +{ + if (tx != NULL) *tx = xiao_mr60bha2 ? 16 : 4; + if (rx != NULL) *rx = xiao_mr60bha2 ? 17 : 5; +} + +static inline bool ruview_mmwave_pins_overlap_console(int tx, int rx, + int console_tx, + int console_rx) +{ + return (console_tx >= 0 && (tx == console_tx || rx == console_tx)) + || (console_rx >= 0 && (tx == console_rx || rx == console_rx)); +} /* C5 SPI0/1 uses GPIO16..22. Never route a sensor onto the memory bus. * No generic C5 sensor pin assignment is safe without board wiring. */ diff --git a/firmware/esp32-csi-node/main/mmwave_sensor.c b/firmware/esp32-csi-node/main/mmwave_sensor.c index d63f3044ce..97713e8911 100644 --- a/firmware/esp32-csi-node/main/mmwave_sensor.c +++ b/firmware/esp32-csi-node/main/mmwave_sensor.c @@ -147,6 +147,16 @@ static void mr60_process_frame(uint16_t type, const uint8_t *data, uint16_t len) case MR60_TYPE_PRESENCE: if (len >= 1) { s_state.person_present = (data[0] != 0); + /* MR60BHA2 exposes a single-target vital-sign presence flag rather + * than a reliable people-count frame. Keep the public count in + * lockstep with that authoritative flag so downstream clients do + * not report zero targets while fresh radar vitals are arriving. */ + s_state.target_count = mmwave_mr60_target_count(s_state.person_present); + if (!s_state.person_present) { + s_state.heart_rate_bpm = 0.0f; + s_state.breathing_rate = 0.0f; + s_state.distance_cm = 0.0f; + } } break; @@ -517,13 +527,25 @@ esp_err_t mmwave_sensor_init(int uart_tx_pin, int uart_rx_pin) return ESP_ERR_NOT_SUPPORTED; } #elif defined(CONFIG_IDF_TARGET_ESP32C6) - if (uart_tx_pin < 0) uart_tx_pin = 4; - if (uart_rx_pin < 0) uart_rx_pin = 5; + int default_tx = 4, default_rx = 5; + ruview_c6_mmwave_default_pins( +#if defined(CONFIG_C6_XIAO_MR60BHA2) + true, +#else + false, +#endif + &default_tx, &default_rx); + if (uart_tx_pin < 0) uart_tx_pin = default_tx; + if (uart_rx_pin < 0) uart_rx_pin = default_rx; #else if (uart_tx_pin < 0) uart_tx_pin = 17; if (uart_rx_pin < 0) uart_rx_pin = 18; #endif -#if defined(CONFIG_ESP_CONSOLE_UART_CUSTOM) +#if defined(CONFIG_ESP_CONSOLE_USB_SERIAL_JTAG) || defined(CONFIG_ESP_CONSOLE_USB_CDC) + /* Native USB has no GPIO console pins. This is the standard C6 profile + * and leaves the XIAO carrier's GPIO16/GPIO17 hardware UART available. */ + const int console_tx = -1, console_rx = -1; +#elif defined(CONFIG_ESP_CONSOLE_UART_CUSTOM) const int console_tx = CONFIG_ESP_CONSOLE_UART_TX_GPIO; const int console_rx = CONFIG_ESP_CONSOLE_UART_RX_GPIO; #elif defined(CONFIG_IDF_TARGET_ESP32C6) @@ -542,8 +564,8 @@ esp_err_t mmwave_sensor_init(int uart_tx_pin, int uart_rx_pin) return ESP_ERR_INVALID_ARG; } #endif - if (uart_tx_pin == console_tx || uart_tx_pin == console_rx || - uart_rx_pin == console_tx || uart_rx_pin == console_rx) { + if (ruview_mmwave_pins_overlap_console(uart_tx_pin, uart_rx_pin, + console_tx, console_rx)) { ESP_LOGW(TAG, "mmWave probe pins (TX=%d, RX=%d) overlap the console UART (TX=%d, RX=%d); skipping probe", uart_tx_pin, uart_rx_pin, console_tx, console_rx); return ESP_ERR_INVALID_ARG; diff --git a/firmware/esp32-csi-node/sdkconfig.defaults.xiao-mr60bha2 b/firmware/esp32-csi-node/sdkconfig.defaults.xiao-mr60bha2 new file mode 100644 index 0000000000..5a96434b41 --- /dev/null +++ b/firmware/esp32-csi-node/sdkconfig.defaults.xiao-mr60bha2 @@ -0,0 +1,6 @@ +# Seeed XIAO ESP32-C6 on the assembled MR60BHA2 carrier. +# The carrier uses UART TX GPIO16 and RX GPIO17. Native USB Serial/JTAG remains +# the onboarding console. The photographed kit uses its external U.FL antenna. +CONFIG_C6_XIAO_MR60BHA2=y +CONFIG_C6_XIAO_ANTENNA_SELECT=y +CONFIG_C6_XIAO_ANTENNA_EXTERNAL=y diff --git a/firmware/esp32-csi-node/test/Makefile b/firmware/esp32-csi-node/test/Makefile index 06281b52c0..292fca5499 100644 --- a/firmware/esp32-csi-node/test/Makefile +++ b/firmware/esp32-csi-node/test/Makefile @@ -58,11 +58,12 @@ FUZZ_JOBS ?= 1 test_thermal run_thermal test_csi_sanitize run_csi_sanitize \ test_c6_antenna run_c6_antenna test_serial_onboarding run_serial_onboarding \ test_config_rules run_config_rules \ + test_mmwave_pins run_mmwave_pins \ run_delivery_contract test_ota_health run_ota_health run_onboarding_identity run_onboarding_console all: fuzz_serialize fuzz_edge fuzz_nvs test_adr110 test_vitals test_mmwave_detect \ test_thermal test_csi_sanitize test_c6_antenna test_serial_onboarding \ - test_ota_health test_config_rules + test_ota_health test_config_rules test_mmwave_pins # --- XIAO ESP32-C6 RF switch selection --- # Host-side truth table for the GPIO14 antenna selector. GPIO3 is always driven diff --git a/firmware/esp32-csi-node/test/test_mmwave_detect.c b/firmware/esp32-csi-node/test/test_mmwave_detect.c index 3646e530f9..bd3a993405 100644 --- a/firmware/esp32-csi-node/test/test_mmwave_detect.c +++ b/firmware/esp32-csi-node/test/test_mmwave_detect.c @@ -46,6 +46,8 @@ int main(void) CHECK(mmwave_mr60_identifying_type(0x0A15), "MR60BHA2 heart-rate type identifies sensor"); CHECK(mmwave_mr60_identifying_type(0x0A16), "MR60BHA2 distance type identifies sensor"); CHECK(mmwave_mr60_identifying_type(0x0F09), "MR60BHA2 presence type identifies sensor"); + CHECK(mmwave_mr60_target_count(false) == 0, "MR60BHA2 absence maps to zero targets"); + CHECK(mmwave_mr60_target_count(true) == 1, "MR60BHA2 presence maps to one target"); /* 1. A real basic-report frame (data len 13) validates. */ int n = make_frame(buf, 13); diff --git a/firmware/esp32-csi-node/test/test_mmwave_pins.c b/firmware/esp32-csi-node/test/test_mmwave_pins.c index 92c8ce4b49..87948845f7 100644 --- a/firmware/esp32-csi-node/test/test_mmwave_pins.c +++ b/firmware/esp32-csi-node/test/test_mmwave_pins.c @@ -3,6 +3,19 @@ int main(void) { + int tx = -1, rx = -1; + ruview_c6_mmwave_default_pins(false, &tx, &rx); + assert(tx == 4 && rx == 5); + ruview_c6_mmwave_default_pins(true, &tx, &rx); + assert(tx == 16 && rx == 17); + + /* Native USB has no GPIO console assignment, so the assembled XIAO + * carrier UART remains usable. A UART console on those pins still fails + * closed. */ + assert(!ruview_mmwave_pins_overlap_console(16, 17, -1, -1)); + assert(ruview_mmwave_pins_overlap_console(16, 17, 16, 17)); + assert(ruview_mmwave_pins_overlap_console(4, 5, 5, 6)); + assert(!ruview_c5_mmwave_pins_safe(-1, -1, 11, 12)); assert(!ruview_c5_mmwave_pins_safe(17, 18, 11, 12)); for (int pin = 16; pin <= 22; pin++) { diff --git a/scripts/ci-install-ubuntu-deps.sh b/scripts/ci-install-ubuntu-deps.sh new file mode 100755 index 0000000000..15324cb144 --- /dev/null +++ b/scripts/ci-install-ubuntu-deps.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# Ephemeral Ubuntu 24.04 CI dependency installation. Keep Ubuntu signature/TLS +# verification, but bypass the hosted image's failing mirror+file indirection. +set -euo pipefail + +if (( $# == 0 )); then + echo 'Pass one or more package names.' >&2 + exit 2 +fi +for package in "$@"; do + if [[ ! "$package" =~ ^[a-z0-9][a-z0-9+.-]*(:amd64)?$ ]]; then + echo 'Invalid package argument; APT options are not accepted.' >&2 + exit 2 + fi +done +if [[ $(uname -s) != Linux ]] || [[ ! -r /etc/os-release ]]; then + echo 'This helper requires an Ubuntu 24.04 amd64 CI runner.' >&2 + exit 2 +fi +os_id=$(. /etc/os-release; printf '%s' "$ID") +os_version=$(. /etc/os-release; printf '%s' "$VERSION_ID") +if [[ "$os_id" != ubuntu || "$os_version" != 24.04 || $(dpkg --print-architecture) != amd64 ]]; then + echo 'This helper requires an Ubuntu 24.04 amd64 CI runner.' >&2 + exit 2 +fi +keyring=/usr/share/keyrings/ubuntu-archive-keyring.gpg +[[ -r "$keyring" ]] || { echo 'Ubuntu archive keyring is unavailable.' >&2; exit 1; } +# Fail quickly on actual repository DNS loss, rather than spending hours trying +# each package through broken mirror fallbacks. +for host in archive.ubuntu.com security.ubuntu.com; do + timeout 20 getent ahostsv4 "$host" >/dev/null +done +apt_dir=$(mktemp -d -t ruview-ci-apt.XXXXXX) +source_list="$apt_dir/ubuntu.list" +parts_dir="$apt_dir/empty" +trap 'rm -f -- "$source_list"; rmdir -- "$parts_dir" "$apt_dir"' EXIT +mkdir "$parts_dir" +chmod 755 "$apt_dir" "$parts_dir" +cat > "$source_list" <= {minimum} registered tests, got {len(names)}') + if mode == 'release-run' and any(n in ignored for n in names): + raise ValueError('an opt-in ignored test would enter a Release batch: ' + selector) + batches.append({'component': component, 'selector': selector, 'exact': exact, 'names': names}) + return batches + +def list_tests(binary, ignored=False): + result = subprocess.run([str(binary), '--list'] + (['--ignored'] if ignored else []), check=True, capture_output=True, text=True) + return set(re.findall(r'^(.+): test$', result.stdout, re.M)) + +def validate_executable(path, target_directory, release): + raw = pathlib.Path(path) + target_raw = pathlib.Path(target_directory) + if not raw.is_absolute() or not target_raw.is_absolute(): + raise ValueError('Cargo artifact and target directory must be absolute') + target = target_raw.resolve(strict=True) + profile = 'release' if release else 'debug' + expected = target / profile / 'deps' + # The configured target root may be external/canonicalized by Cargo; below + # that root neither profile, deps nor executable may redirect through links. + for directory in [target_raw / profile, target_raw / profile / 'deps']: + if directory.is_symlink() or not directory.is_dir(): + raise ValueError('unsafe Cargo profile/deps directory') + if raw.parent not in (target_raw / profile / 'deps', expected): + raise ValueError('Cargo executable outside declared profile/deps') + info = raw.lstat() + if not stat.S_ISREG(info.st_mode) or raw.is_symlink() or not os.access(raw, os.X_OK): + raise ValueError('Cargo executable must be regular, executable and not a symlink') + if not re.fullmatch(r'wifi_densepose_sensing_server-[0-9a-f]{16}', raw.name): + raise ValueError('unexpected lib-test executable basename') + resolved = raw.resolve(strict=True) + if resolved.parent != expected.resolve(strict=True): + raise ValueError('Cargo executable escaped canonical target profile/deps') + return resolved + +def validate_cargo_artifacts(root, metadata, artifacts, release): + manifest = (root / 'v2/crates/wifi-densepose-sensing-server/Cargo.toml').resolve(strict=True) + source = (manifest.parent / 'src/lib.rs').resolve(strict=True) + packages = [p for p in metadata.get('packages', []) if p.get('name') == 'wifi-densepose-sensing-server' and pathlib.Path(p.get('manifest_path', '')).resolve() == manifest] + if len(packages) != 1: + raise ValueError('expected exactly one workspace sensing-server package') + package = packages[0] + libraries = [t for t in package.get('targets', []) if t.get('name') == 'wifi_densepose_sensing_server' and t.get('kind') == ['lib'] and pathlib.Path(t.get('src_path', '')).resolve() == source] + if len(libraries) != 1: + raise ValueError('expected exactly one actual workspace library source') + candidates = [a for a in artifacts if a.get('reason') == 'compiler-artifact' and a.get('executable') and a.get('package_id') == package['id'] and a.get('target', {}).get('name') == 'wifi_densepose_sensing_server' and a.get('profile', {}).get('test') is True] + if len(candidates) != 1: + raise ValueError('expected exactly one actual package/lib-test executable') + artifact = candidates[0] + target = artifact.get('target', {}) + profile = artifact.get('profile', {}) + if target.get('kind') != ['lib'] or pathlib.Path(target.get('src_path', '')).resolve() != source or profile.get('opt_level') != ('3' if release else '0') or profile.get('debug_assertions') is not (not release): + raise ValueError('Cargo executable library source or execution profile mismatch') + executable = validate_executable(artifact['executable'], metadata['target_directory'], release) + return {'path': executable, 'sha256': hashlib.sha256(executable.read_bytes()).hexdigest(), 'target_directory': pathlib.Path(metadata['target_directory']).resolve(strict=True), 'package_id': package['id'], 'lib_source': source, 'release': release} + +def verify_selected_executable(selected, expected_sha256): + binary = validate_executable(selected['path'], selected['target_directory'], selected['release']) + if hashlib.sha256(binary.read_bytes()).hexdigest() != expected_sha256: + raise ValueError('selected Cargo executable changed before invocation') + return binary + +def compile_binary(root, release): + metadata_result = subprocess.run(['cargo', 'metadata', '--locked', '--no-deps', '--format-version=1', '--no-default-features'], cwd=root/'v2', check=True, capture_output=True, text=True) + metadata = json.loads(metadata_result.stdout) + command = ['cargo', 'test', '--locked', '-p', 'wifi-densepose-sensing-server', '--lib', '--no-default-features', '--no-run', '--message-format=json'] + if release: + command.append('--release') + result = subprocess.run(command, cwd=root/'v2', capture_output=True, text=True) + sys.stderr.write(result.stderr) + artifacts = [] + for line in result.stdout.splitlines(): + try: item = json.loads(line) + except json.JSONDecodeError: continue + if item.get('reason') == 'compiler-message' and item.get('message', {}).get('rendered'): + sys.stderr.write(item['message']['rendered']) + if item.get('reason') == 'compiler-artifact': + artifacts.append(item) + if result.returncode: + raise ValueError('Cargo lib-test compilation failed with exit ' + str(result.returncode)) + return validate_cargo_artifacts(root, metadata, artifacts, release) + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('mode', choices=['debug-check', 'release-run']) + parser.add_argument('--evidence-dir', type=pathlib.Path, required=True) + args = parser.parse_args() + root = pathlib.Path(__file__).resolve().parents[2] + args.evidence_dir.mkdir(parents=True, exist_ok=True) + head = os.environ.get('GITHUB_SHA') or subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=root, text=True).strip() + dirty = bool(subprocess.check_output(['git', 'status', '--porcelain', '--untracked-files=normal', '--ignore-submodules=all'], cwd=root, text=True).strip()) + receipt = {'mode': args.mode, 'head': head, 'worktree_dirty': dirty, 'profile_runner_sha256': hashlib.sha256(pathlib.Path(__file__).read_bytes()).hexdigest(), 'batches': [], 'status': 'started'} + destination = args.evidence_dir / (args.mode + '.json') + try: + receipt['rustc'] = subprocess.check_output(['rustc', '-Vv'], cwd=root/'v2', text=True) + receipt['cargo'] = subprocess.check_output(['cargo', '-V'], cwd=root/'v2', text=True) + if args.mode == 'release-run': + selected = compile_binary(root, True) + else: + selected = compile_binary(root, False) + binary = selected['path'] + binary_sha256 = selected['sha256'] + verify_selected_executable(selected, binary_sha256) + registered = list_tests(binary) + verify_selected_executable(selected, binary_sha256) + ignored = list_tests(binary, True) + verify_selected_executable(selected, binary_sha256) + if args.mode == 'release-run': + batches = validate(policy(root), registered, ignored, 'release-run') + else: + batches = validate(policy(root), registered, ignored, 'debug-check') + receipt.update(binary=str(binary), binary_sha256=binary_sha256, cargo_artifact={'target_directory': str(selected['target_directory']), 'package_id': selected['package_id'], 'lib_source': str(selected['lib_source']), 'profile': 'release' if selected['release'] else 'debug'}, registered_count=len(registered), ignored_names=sorted(ignored), batches=batches) + print(json.dumps({'mode': args.mode, 'selected_batches': batches}, indent=2), flush=True) + if args.mode == 'release-run': + for index, batch in enumerate(batches): + verify_selected_executable(selected, binary_sha256) + command = [str(binary), batch['selector'], '--nocapture', '--test-threads=1'] + (['--exact'] if batch['exact'] else []) + result = subprocess.run(command, cwd=root/'v2', capture_output=True, text=True) + output = result.stdout + result.stderr + (args.evidence_dir / f'{index}-{batch["component"]}.log').write_text(output) + print(output, flush=True) + counts = re.findall(r'^test result: ok\. (\d+) passed; (\d+) failed; (\d+) ignored;', output, re.M) + if result.returncode or not counts or tuple(map(int, counts[-1])) != (len(batch['names']), 0, 0): + raise ValueError('Release batch failed or nonzero/exact-count verification failed: ' + batch['selector']) + batch['passed'] = len(batch['names']) + receipt['status'] = 'passed' + except Exception as error: + receipt.update(status='failed', error=str(error)) + raise + finally: + destination.write_text(json.dumps(receipt, indent=2) + '\n') + +if __name__ == '__main__': + main() diff --git a/scripts/ci/test_temporal_test_profiles.py b/scripts/ci/test_temporal_test_profiles.py new file mode 100644 index 0000000000..5290aba5ac --- /dev/null +++ b/scripts/ci/test_temporal_test_profiles.py @@ -0,0 +1,191 @@ +import importlib.util, pathlib, unittest +spec=importlib.util.spec_from_file_location('profiles',pathlib.Path(__file__).with_name('temporal_test_profiles.py'));profiles=importlib.util.module_from_spec(spec);spec.loader.exec_module(profiles) +class ClassificationTests(unittest.TestCase): + def component(self):return [('C2',None,[profiles.C2],[(profiles.C2,True,1)])] + def test_missing_registered_test_never_counts_as_coverage(self): + with self.assertRaises(ValueError):profiles.validate(self.component(),set(),set(),'release-run') + def test_debug_requires_transparent_ignored_classification(self): + with self.assertRaises(ValueError):profiles.validate(self.component(),{profiles.C2},set(),'debug-check') + def test_debug_classification_keeps_registered_name(self): + result=profiles.validate(self.component(),{profiles.C2},{profiles.C2},'debug-check');self.assertEqual(result[0]['names'],[profiles.C2]) + def test_release_case_is_present_active_and_nonzero(self): + result=profiles.validate(self.component(),{profiles.C2},set(),'release-run');self.assertEqual(len(result),1);self.assertTrue(result[0]['exact']) + def test_release_never_sweeps_ignored_optin_performance(self): + component=[('fixture',None,['module::positive'],[('module::',False,1)])] + with self.assertRaises(ValueError):profiles.validate(component,{'module::positive','module::manual_perf'},{'module::manual_perf'},'release-run') + def test_module_is_one_batch_to_preserve_fixture_cache(self): + component=[('fixture',None,['module::one'],[('module::',False,2)])] + result=profiles.validate(component,{'module::one','module::two'},set(),'release-run');self.assertEqual(len(result),1);self.assertEqual(result[0]['names'],['module::one','module::two']) + + +class C1ClassificationTests(unittest.TestCase): + def component(self): + return [next(item for item in profiles.policy(pathlib.Path(__file__).resolve().parents[2]) if item[0] == 'C1')] + def test_current_policy_requires_one_exact_release_reference(self): + result=profiles.validate(self.component(),{profiles.C1,profiles.C1+'_lookalike',profiles.C2},set(),'release-run') + self.assertEqual(result,[{'component':'C1','selector':profiles.C1,'exact':True,'names':[profiles.C1]}]) + def test_reference_stays_registered_but_transparently_ignored_in_debug(self): + result=profiles.validate(self.component(),{profiles.C1},{profiles.C1},'debug-check');self.assertEqual(result[0]['names'],[profiles.C1]) + def test_missing_reference_cannot_be_silently_omitted(self): + with self.assertRaises(ValueError):profiles.validate(self.component(),set(),set(),'release-run') + def test_reference_cannot_run_unclassified_in_debug(self): + with self.assertRaises(ValueError):profiles.validate(self.component(),{profiles.C1},set(),'debug-check') + def test_reference_cannot_be_ignored_in_required_release(self): + with self.assertRaises(ValueError):profiles.validate(self.component(),{profiles.C1},{profiles.C1},'release-run') + def test_zero_or_lookalike_only_is_not_the_exact_reference(self): + with self.assertRaises(ValueError):profiles.validate(self.component(),{profiles.C1+'_manual_perf'},set(),'release-run') + + +class C4BClassificationTests(unittest.TestCase): + def component(self):return [('C4B',None,profiles.C4B,[(profiles.C4B_PREFIX,False,11)])] + def test_exact_eleven_release_is_one_batch_with_fixture_cache(self): + batches=profiles.validate(self.component(),set(profiles.C4B_ALL),set(),'release-run');self.assertEqual(len(batches),1);self.assertEqual(len(batches[0]['names']),11);self.assertFalse(batches[0]['exact']) + def test_debug_keeps_seven_small_active_and_four_heavy_transparent(self): + batches=profiles.validate(self.component(),set(profiles.C4B_ALL),set(profiles.C4B),'debug-check');self.assertEqual(len(batches[0]['names']),11);self.assertEqual(len(set(profiles.C4B_ALL)-set(profiles.C4B)),7) + def test_missing_heavy_never_counts_as_coverage(self): + with self.assertRaises(ValueError):profiles.validate(self.component(),set(profiles.C4B_ALL)-{profiles.C4B[0]},set(),'release-run') + def test_extra_unclassified_test_requires_review(self): + with self.assertRaises(ValueError):profiles.validate(self.component(),set(profiles.C4B_ALL)|{profiles.C4B_PREFIX+'unexpected'},set(),'release-run') + def test_fourth_heavy_must_not_run_in_debug(self): + with self.assertRaises(ValueError):profiles.validate(self.component(),set(profiles.C4B_ALL),set(profiles.C4B[:-1]),'debug-check') + def test_small_controls_cannot_be_hidden_in_debug(self): + small=next(n for n in profiles.C4B_ALL if n not in profiles.C4B) + with self.assertRaises(ValueError):profiles.validate(self.component(),set(profiles.C4B_ALL),set(profiles.C4B)|{small},'debug-check') + def test_no_ignored_or_opt_in_release_selection(self): + with self.assertRaises(ValueError):profiles.validate(self.component(),set(profiles.C4B_ALL),{profiles.C4B[0]},'release-run') + def test_zero_tests_and_missing_source_marker_fail_closed(self): + import tempfile + with self.assertRaises(ValueError):profiles.validate(self.component(),set(),set(),'release-run') + with tempfile.TemporaryDirectory() as d: + root=pathlib.Path(d);p=root/'v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_tests.rs';p.parent.mkdir(parents=True);p.write_text('// synthetic marker only') + with self.assertRaises(ValueError):profiles.policy(root) + +class CargoExecutableCustodyTests(unittest.TestCase): + def setUp(self): + import tempfile, copy + self.temp=tempfile.TemporaryDirectory();self.addCleanup(self.temp.cleanup) + self.base=pathlib.Path(self.temp.name).resolve();self.root=self.base/'workspace';self.target=self.base/'external-warm-target' + self.manifest=self.root/'v2/crates/wifi-densepose-sensing-server/Cargo.toml';self.manifest.parent.mkdir(parents=True);self.manifest.write_text('[package]\nname="synthetic"\n') + self.source=self.manifest.parent/'src/lib.rs';self.source.parent.mkdir();self.source.write_text('// synthetic library source\n') + self.binary=self.target/'release/deps/wifi_densepose_sensing_server-0123456789abcdef';self.binary.parent.mkdir(parents=True);self.binary.write_text('#!/bin/sh\nexit 0\n');self.binary.chmod(0o700) + self.lib={'name':'wifi_densepose_sensing_server','kind':['lib'],'src_path':str(self.source)} + self.metadata={'target_directory':str(self.target),'packages':[{'name':'wifi-densepose-sensing-server','manifest_path':str(self.manifest),'id':'synthetic-workspace-package','targets':[self.lib]}]} + self.artifact={'reason':'compiler-artifact','package_id':'synthetic-workspace-package','target':copy.deepcopy(self.lib),'profile':{'test':True,'opt_level':'3','debug_assertions':False},'executable':str(self.binary)} + def valid(self,artifacts=None): + return profiles.validate_cargo_artifacts(self.root,self.metadata,artifacts if artifacts is not None else [self.artifact],True) + def test_actual_external_warm_target_is_preserved(self): + result=self.valid();self.assertEqual(result['path'],self.binary);self.assertEqual(result['target_directory'],self.target) + def test_executable_outside_reported_target_is_rejected(self): + outside=self.base/self.binary.name;outside.write_bytes(self.binary.read_bytes());outside.chmod(0o700);self.artifact['executable']=str(outside) + with self.assertRaises(ValueError):self.valid() + def test_symlink_executable_is_rejected(self): + original=self.binary.with_name('original');self.binary.rename(original);self.binary.symlink_to(original) + with self.assertRaises(ValueError):self.valid() + def test_wrong_profile_artifact_is_rejected(self): + self.artifact['profile']['opt_level']='0';self.artifact['profile']['debug_assertions']=True + with self.assertRaises(ValueError):self.valid() + def test_wrong_library_source_is_rejected(self): + wrong=self.source.with_name('wrong.rs');wrong.write_text('// wrong source\n');self.artifact['target']['src_path']=str(wrong) + with self.assertRaises(ValueError):self.valid() + def test_wrong_package_is_rejected(self): + self.artifact['package_id']='other-package' + with self.assertRaises(ValueError):self.valid() + def test_wrong_target_kind_is_rejected(self): + self.artifact['target']['kind']=['bin'] + with self.assertRaises(ValueError):self.valid() + def test_duplicate_test_artifacts_are_rejected(self): + with self.assertRaises(ValueError):self.valid([self.artifact,self.artifact]) + def test_nonexecutable_regular_file_is_rejected(self): + self.binary.chmod(0o600) + with self.assertRaises(ValueError):self.valid() + + + def test_swapped_bytes_after_admission_are_not_invoked(self): + import hashlib + selected=self.valid();digest=hashlib.sha256(self.binary.read_bytes()).hexdigest();self.binary.write_text('#!/bin/sh\necho changed\n') + with self.assertRaises(ValueError):profiles.verify_selected_executable(selected,digest) + def test_symlink_swap_after_admission_is_not_invoked(self): + import hashlib + selected=self.valid();digest=hashlib.sha256(self.binary.read_bytes()).hexdigest();other=self.base/'outside';other.write_bytes(self.binary.read_bytes());other.chmod(0o700);self.binary.unlink();self.binary.symlink_to(other) + with self.assertRaises(ValueError):profiles.verify_selected_executable(selected,digest) + + + def test_actual_main_rejects_replacement_before_any_registry_execution(self): + from unittest import mock + import sys + selected=self.valid();marker=self.base/'outside-executed';outside=self.base/'outside-libtest' + outside.write_text('#!/bin/sh\nprintf executed > '+str(marker)+'\n');outside.chmod(0o700) + def after_compile(root,release): + self.binary.unlink();self.binary.symlink_to(outside);return selected + with mock.patch.object(sys,'argv',['profiles','release-run','--evidence-dir',str(self.base/'receipt')]), mock.patch.object(profiles,'compile_binary',side_effect=after_compile), mock.patch.object(profiles.subprocess,'check_output',return_value='synthetic-provenance'): + with self.assertRaises(ValueError):profiles.main() + self.assertFalse(marker.exists(),'registry executed substituted out-of-target artifact') + +class C4CClassificationTests(unittest.TestCase): + def component(self): + return [('C4C', None, [profiles.C4C], [(profiles.C4C, True, 1)])] + def test_exact_default48_case_is_one_required_release_case(self): + result = profiles.validate(self.component(), {profiles.C4C, profiles.C4C + '_lookalike'}, set(), 'release-run') + self.assertEqual(result, [{'component': 'C4C', 'selector': profiles.C4C, 'exact': True, 'names': [profiles.C4C]}]) + def test_debug_requires_the_same_case_registered_and_transparently_ignored(self): + result = profiles.validate(self.component(), {profiles.C4C}, {profiles.C4C}, 'debug-check') + self.assertEqual(result[0]['names'], [profiles.C4C]) + def test_default48_cannot_be_left_unclassified_in_debug(self): + with self.assertRaises(ValueError): + profiles.validate(self.component(), {profiles.C4C}, set(), 'debug-check') + def test_default48_cannot_be_ignored_in_required_release(self): + with self.assertRaises(ValueError): + profiles.validate(self.component(), {profiles.C4C}, {profiles.C4C}, 'release-run') + def test_zero_or_missing_case_is_not_release_coverage(self): + with self.assertRaises(ValueError): + profiles.validate(self.component(), set(), set(), 'release-run') + def test_same_leaf_in_a_different_module_cannot_satisfy_exact_registration(self): + wrong = profiles.C4C.replace('public_workflow::acceptance::', 'public_workflow::wrong_module::') + with self.assertRaises(ValueError): + profiles.validate(self.component(), {wrong}, set(), 'release-run') + def test_exact_selector_does_not_sweep_opt_in_siblings(self): + perf = profiles.C4C + '_manual_performance' + result = profiles.validate(self.component(), {profiles.C4C, perf}, {perf}, 'release-run') + self.assertEqual(result[0]['names'], [profiles.C4C]) + def test_missing_c4c_source_marker_cannot_silently_drop_release_batch(self): + import tempfile + with tempfile.TemporaryDirectory() as d: + root = pathlib.Path(d) + source = root / 'v2/crates/wifi-densepose-sensing-server/src/room_learner' + source.mkdir(parents=True) + for name in ['lifecycle_temporal_tests.rs', 'lifecycle_temporal_held_out_tests.rs']: + (source / name).write_text('// synthetic source marker only') + with self.assertRaisesRegex(ValueError, 'C4C acceptance source marker'): + profiles.policy(root) + def test_current_policy_registers_exact_c4c_and_preserves_existing_components(self): + components = profiles.policy(pathlib.Path(__file__).resolve().parents[2]) + self.assertEqual([c[0] for c in components], ['C2', 'C1', 'C3', 'C4', 'budget343', 'C4B', 'C4C']) + current = components[-1] + self.assertEqual(current[1].name, 'lifecycle_temporal_public_acceptance_tests.rs') + self.assertEqual(current[2:], ([profiles.C4C], [(profiles.C4C, True, 1)])) + self.assertEqual(components[0][3], [(profiles.C2, True, 1)]) + self.assertEqual(components[1][3], [(profiles.C1, True, 1)]) + self.assertEqual(components[2][3], [(profiles.PREFIX + 'fit_control::tests::', False, 12)]) + self.assertEqual(components[3][3], [(profiles.PREFIX + 'full_run_tests::', False, 9)]) + self.assertEqual(components[4][3], [(name, True, 1) for name in profiles.BUDGET]) + self.assertEqual(components[5][3], [(profiles.C4B_PREFIX, False, 11)]) + + + +class OperatorEightClassificationTests(unittest.TestCase): + def component(self):return [('operator8',None,[profiles.OPERATOR8],[(profiles.OPERATOR8,True,1)])] + def test_exact_operator_case_is_mandatory_active_in_release(self): + result=profiles.validate(self.component(),{profiles.OPERATOR8,profiles.OPERATOR8+'_lookalike'},set(),'release-run') + self.assertEqual(result[0]['names'],[profiles.OPERATOR8]) + def test_debug_registration_is_explicitly_ignored_not_passed(self): + result=profiles.validate(self.component(),{profiles.OPERATOR8},{profiles.OPERATOR8},'debug-check') + self.assertEqual(result[0]['names'],[profiles.OPERATOR8]) + with self.assertRaises(ValueError):profiles.validate(self.component(),{profiles.OPERATOR8},set(),'debug-check') + def test_missing_or_ignored_release_case_fails(self): + with self.assertRaises(ValueError):profiles.validate(self.component(),set(),set(),'release-run') + with self.assertRaises(ValueError):profiles.validate(self.component(),{profiles.OPERATOR8},{profiles.OPERATOR8},'release-run') + def test_actual_policy_carries_operator_case_in_cached_c4_batch(self): + c4=next(c for c in profiles.policy(pathlib.Path(__file__).resolve().parents[2]) if c[0]=='C4') + self.assertIn(profiles.OPERATOR8,c4[2]);self.assertEqual(c4[3],[(profiles.PREFIX+'full_run_tests::',False,9)]) + +if __name__=='__main__':unittest.main() diff --git a/scripts/mat_http_smoke.py b/scripts/mat_http_smoke.py new file mode 100644 index 0000000000..cc9d00eef8 --- /dev/null +++ b/scripts/mat_http_smoke.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +"""Read-only smoke against the compiled server's actual HTTP listener.""" +import argparse +import json +import os +import urllib.error +import urllib.request + +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument('--base-url', required=True) +args = parser.parse_args() +token = os.environ.get('RUVIEW_MAT_SMOKE_TOKEN') +if not token: + raise SystemExit('Set RUVIEW_MAT_SMOKE_TOKEN for this isolated server; never pass credentials in arguments.') + +def request(path, authenticated): + headers = {'Authorization': 'Bearer ' + token} if authenticated else {} + req = urllib.request.Request(args.base_url.rstrip('/') + path, headers=headers) + try: + with urllib.request.urlopen(req, timeout=5) as response: + return response.status, json.load(response) + except urllib.error.HTTPError as error: + return error.code, None + +for path in ['/api/v1/mat/events?page=0&page_size=100', '/api/v1/mat/scan/status', '/ws/mat/stream']: + status, _ = request(path, False) + assert status in (401, 403), (path, 'unauthenticated access was not rejected', status) +status, events = request('/api/v1/mat/events?page=0&page_size=100', True) +assert status == 200 and isinstance(events['events'], list), ('MAT HTTP event router', status) +assert events['page'] == 0 and events['page_size'] == 100 +status, pipeline = request('/api/v1/mat/scan/status', True) +assert status == 200 and isinstance(pipeline['scanning'], bool), ('MAT HTTP pipeline router', status) +print(json.dumps({'result': 'PASS', 'evidence': 'EXECUTED_COMPILED_HTTP', + 'eventCount': len(events['events']), 'mlEnabled': pipeline['ml_enabled'], + 'mlReady': pipeline['ml_ready'], 'unauthenticatedReadsAndStream': 'denied'})) diff --git a/scripts/tests/companion_socket_probe.py b/scripts/tests/companion_socket_probe.py new file mode 100644 index 0000000000..5939845643 --- /dev/null +++ b/scripts/tests/companion_socket_probe.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +"""Real helper socket contract. Synthetic isolated state, never the live room.""" +import argparse, base64, hashlib, http.client, json, os, pathlib, secrets, socket, ssl, subprocess, tempfile, time + +def port(): + with socket.socket() as s: s.bind(('127.0.0.1',0)); return s.getsockname()[1] + +def main(): + parser=argparse.ArgumentParser(); parser.add_argument('binary'); args=parser.parse_args() + ports=[port() for _ in range(4)]; http_port,ws,udp,tls=ports + owner=secrets.token_urlsafe(32); device=secrets.token_urlsafe(32); proof=secrets.token_urlsafe(32) + context=ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT); context.check_hostname=False; context.verify_mode=ssl.CERT_NONE + count=0 + def check(condition,label): + nonlocal count + assert condition,label + count+=1; print('PASS',label,flush=True) + with tempfile.TemporaryDirectory(prefix='ruview-companion-probe-') as directory: + root=pathlib.Path(directory); proc=None; log=(root/'server.log').open('w') + def start(): + env=os.environ.copy();env['RUVIEW_API_TOKEN']=owner + p=subprocess.Popen([str(pathlib.Path(args.binary).resolve()),'--source','simulated','--bind-addr','127.0.0.1','--http-port',str(http_port),'--ws-port',str(ws),'--udp-port',str(udp),'--companion-port',str(tls),'--installation-id','companion-probe','--data-dir',directory,'--no-mdns','--no-edge-registry'],env=env,stdout=log,stderr=subprocess.STDOUT) + for _ in range(200): + if p.poll() is not None: raise RuntimeError('helper exited; '+(root/'server.log').read_text()[-1600:]) + try: + with socket.create_connection(('127.0.0.1',http_port),timeout=.2):return p + except OSError:time.sleep(.05) + raise RuntimeError('helper did not bind') + def request(path,method='GET',body=None,token=None,secure=False,pin=None): + conn=http.client.HTTPSConnection('127.0.0.1',tls,context=context,timeout=5) if secure else http.client.HTTPConnection('127.0.0.1',http_port,timeout=5) + conn.connect() + if secure and pin: assert hashlib.sha256(conn.sock.getpeercert(binary_form=True)).hexdigest()==pin,'TLS pin changed' + headers={'Content-Type':'application/json'} + if token:headers['Authorization']='Bearer '+token + if secure:headers['X-RuView-Installation-ID']='companion-probe' + conn.request(method,path,json.dumps(body) if body is not None else None,headers) + res=conn.getresponse(); status=res.status;raw=res.read();conn.close() + try:value=json.loads(raw) + except ValueError:value={} + return status,value + try: + proc=start() + check(request('/api/v1/companion')[0]==403,'owner endpoint rejects absent credential') + status,invite=request('/api/v1/companion/invites','POST',{},owner) + check(status==200 and invite['schema']=='ruview.companion.invite.v1','owner creates bounded invitation') + pin=invite['cert_sha256'] + payload={k:invite[k] for k in ['installation_id','invite_id','invite_secret']};payload.update(device_id=device,device_name='Contract probe',pairing_secret=proof) + status,enrolled=request('/companion/v1/enroll','POST',payload,secure=True,pin=pin) + check(status==200 and enrolled['status']=='pending','pinned enrollment pending owner') + session={'installation_id':'companion-probe','device_id':device,'pairing_secret':proof} + check(request('/companion/v1/session','POST',session,secure=True,pin=pin)[1]['status']=='pending','QR cannot autoapprove') + check(request('/companion/v1/enroll','POST',payload,secure=True,pin=pin)[0]>=400,'invitation cannot replay') + check(request('/api/v1/companion/requests/'+enrolled['request_id']+'/approve','POST',{'comparison_code':enrolled['comparison_code']},owner)[0]==200,'owner approves matched request') + status,access=request('/companion/v1/session','POST',session,secure=True,pin=pin);token=access['access_token'] + check(status==200,'device obtains read session') + check(request('/api/v1/status',token=token,secure=True,pin=pin)[0]==200,'read token works over pinned TLS') + check(request('/api/v1/status',token=token)[0]==401,'companion token rejected over plaintext') + check(request('/api/v1/companion',token=owner,secure=True,pin=pin)[0] in (403,404),'owner token cannot manage over TLS LAN listener') + check(request('/api/v1/ws-ticket','POST',{},token,True,pin)[0]==403,'legacy browser ticket denied') + check(request('/api/v1/calibration/start','POST',{},token,True,pin)[0]==403,'read grant cannot calibrate') + check(request('/api/v1/companion/devices/'+device+'/calibration','POST',{},owner)[0]==200,'owner grants calibration separately') + status,result=request('/api/v1/calibration/start','POST',{'binding_digest':'a'*64,'source_node_ids':[1]},token,True,pin) + check(status==200 and result.get('error_code')=='shared_room_binding_required','calibration rejects unshared room') + check(request('/api/v1/calibration/reset','POST',{},token,True,pin)[0]==403,'calibration grant is not administrator') + room={'schema':'ruview.companion.room.v1','id':'room-1','name':'Probe room','widthM':5,'lengthM':6,'heightM':2.6,'nodes':[{'nodeId':1,'slot':'north_west','xM':.45,'yM':.45,'role':'receiver','headingDeg':90,'orientationConfirmed':True,'confirmedAtUnixMs':1}],'accessPoint':{'slot':'north_east','xM':4.55,'yM':.45,'provenance':'operator_entered','confirmedAtUnixMs':1}} + status,published=request('/api/v1/companion/room','PUT',{'expected_revision':0,'room':room},owner) + check(status==200 and published['revision']==1,'owner publishes canonical room') + check(request('/companion/v1/room',token=token,secure=True,pin=pin)[1]==published,'paired phone reads exact shared room') + check(request('/api/v1/companion/room','PUT',{'expected_revision':0,'room':room},owner)[0]==409,'room publication rejects stale revision') + raw=socket.create_connection(('127.0.0.1',tls),timeout=5);stream=context.wrap_socket(raw,server_hostname='localhost') + check(hashlib.sha256(stream.getpeercert(binary_form=True)).hexdigest()==pin,'stream server pin matches') + key=base64.b64encode(secrets.token_bytes(16)).decode() + stream.sendall(('GET /ws/sensing HTTP/1.1\r\nHost: localhost\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Version: 13\r\nSec-WebSocket-Key: '+key+'\r\nAuthorization: Bearer '+token+'\r\nX-RuView-Installation-ID: companion-probe\r\n\r\n').encode()) + check(b'101 Switching Protocols' in stream.recv(4096),'authenticated live WebSocket upgrades') + proc.terminate();proc.wait(timeout=10);proc=start() + check(request('/api/v1/status',token=token,secure=True,pin=pin)[0]==401,'restart invalidates ephemeral access token') + status,access=request('/companion/v1/session','POST',session,secure=True,pin=pin);token=access['access_token'] + check(status==200 and request('/api/v1/status',token=token,secure=True,pin=pin)[0]==200,'durable device proof refreshes after restart') + stream.close() + raw=socket.create_connection(('127.0.0.1',tls),timeout=5);stream=context.wrap_socket(raw,server_hostname='localhost');stream.sendall(('GET /ws/sensing HTTP/1.1\r\nHost: localhost\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Version: 13\r\nSec-WebSocket-Key: '+key+'\r\nAuthorization: Bearer '+token+'\r\nX-RuView-Installation-ID: companion-probe\r\n\r\n').encode());stream.recv(4096) + check(request('/api/v1/companion/devices/'+device,'DELETE',token=owner)[0]==200,'owner revokes phone') + check(request('/api/v1/status',token=token,secure=True,pin=pin)[0]==401,'revocation blocks existing read token') + check(request('/companion/v1/session','POST',session,secure=True,pin=pin)[0]==401,'revocation blocks durable refresh') + stream.settimeout(3);closed=False + try: + for _ in range(100): + if not stream.recv(65536):closed=True;break + except (ssl.SSLEOFError,ConnectionResetError):closed=True + check(closed,'revocation closes already-open WebSocket');stream.close() + print('COMPANION SOCKET CONTRACT PASS',count) + finally: + if proc and proc.poll() is None:proc.terminate();proc.wait(timeout=10) + log.close() +if __name__=='__main__':main() diff --git a/scripts/verify-sensing-stream-freshness.mjs b/scripts/verify-sensing-stream-freshness.mjs new file mode 100644 index 0000000000..3ac62c3bd5 --- /dev/null +++ b/scripts/verify-sensing-stream-freshness.mjs @@ -0,0 +1,101 @@ +#!/usr/bin/env node +// SYNTHETIC loopback-only regression against an actual helper. Node22+. +// Usage: node scripts/verify-sensing-stream-freshness.mjs /path/to/sensing-server +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { mkdtempSync, readFileSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { spawn } from 'node:child_process'; +import net from 'node:net'; +import dgram from 'node:dgram'; +import { runInNewContext } from 'node:vm'; +const binary = resolve(process.argv[2] ?? 'v2/target/release/sensing-server'); +const binarySha256 = createHash('sha256').update(readFileSync(binary)).digest('hex'); +const evidence = mkdtempSync(join(tmpdir(), 'ruview-stream-freshness-')); +const delay = ms => new Promise(resolve => setTimeout(resolve, ms)); +const legacySource = readFileSync(new URL('../ui/services/pose.service.js', import.meta.url), 'utf8') + .replace(/^import[^\n]*\n/gm, '').replace('export class PoseService', 'class PoseService') + .replace('export const poseService', 'const poseService'); +const PoseService = runInNewContext(`${legacySource};PoseService;`, { + console: {log() {}, warn() {}, error() {}, debug() {}}, API_CONFIG: {}, +}); + +const freePort = () => new Promise(resolve => { + const socket = net.createServer(); + socket.listen(0, '127.0.0.1', () => { const port = socket.address().port; socket.close(() => resolve(port)); }); +}); +const connect = async url => { + const socket = new WebSocket(url), frames = []; + socket.onmessage = event => { if (typeof event.data === 'string') frames.push(JSON.parse(event.data)); }; + await new Promise((resolve, reject) => { socket.onopen = resolve; socket.onerror = reject; }); + return { socket, frames }; +}; +async function scenario(privacy) { + const root = join(evidence, privacy ? 'privacy' : 'normal'); + const [http, ws, udpPort] = [await freePort(), await freePort(), await freePort()]; + const args = ['--source', 'esp32', '--bind-addr', '127.0.0.1', '--udp-bind', '127.0.0.1', + '--http-port', String(http), '--ws-port', String(ws), '--udp-port', String(udpPort), + '--data-dir', root, '--no-mdns', ...(privacy ? ['--privacy-mode'] : [])]; + const child = spawn(binary, args, {env: {PATH: process.env.PATH, RUST_LOG: 'warn'}, stdio: ['ignore', 'pipe', 'pipe']}); + let logs = ''; child.stdout.on('data', b => logs += b); child.stderr.on('data', b => logs += b); + const sockets = [], udp = dgram.createSocket('udp4'); + try { + let ready = false; + for (let i = 0; i < 100; i++) { try { await fetch(`http://127.0.0.1:${http}/health`); ready = true; break; } catch { await delay(50); } } + assert(ready, 'isolated helper did not start'); + const first = await connect(`ws://127.0.0.1:${ws}/ws/sensing`); sockets.push(first.socket); + async function send(sequence) { + const tones = 56, packet = Buffer.alloc(20 + tones * 2); + packet.writeUInt32LE(0xC5110001, 0); packet[4] = 7; packet[5] = 1; + packet.writeUInt16LE(tones, 6); packet.writeUInt32LE(2437, 8); packet.writeUInt32LE(sequence, 12); + packet.writeInt8(-50, 16); packet.writeInt8(-90, 17); + for (let i = 0; i < tones; i++) { packet[20 + i * 2] = 5 + i % 40; packet[21 + i * 2] = i % 30; } + await new Promise((resolve, reject) => udp.send(packet, udpPort, '127.0.0.1', e => e ? reject(e) : resolve())); + await delay(33); + } + for (let sequence = 1; sequence <= 60; sequence++) await send(sequence); + await delay(100); const beforeQuiet = first.frames.length; + const last = first.frames.filter(f => f.type === 'sensing_update').at(-1); + assert(last, 'live producer did not publish'); + await delay(6200); + const quietFrames = first.frames.slice(beforeQuiet).filter(f => f.type === 'sensing_update'); + assert.equal(quietFrames.length, 1, 'quiet period must emit exactly one offline transition, not frozen snapshots'); + const offline = quietFrames[0]; + assert.equal(offline.source, 'esp32:offline'); assert.equal(offline.tick, last.tick); assert.equal(offline.timestamp, last.timestamp); + if (!privacy) { assert(offline.node_features[0].stale); assert(offline.node_features[0].last_seen_ms >= 5000); } + const count = first.frames.length; + const late = await connect(`ws://127.0.0.1:${ws}/ws/sensing`); sockets.push(late.socket); + const legacy = await connect(`ws://127.0.0.1:${http}/api/v1/stream/pose`); sockets.push(legacy.socket); + await delay(250); + assert.equal(first.frames.length, count, 'new subscriber must not broadcast its snapshot to existing clients'); + const initial = late.frames.filter(f => f.type === 'sensing_update'); + assert.equal(initial.length, 1); assert.equal(initial[0].tick, last.tick); assert.equal(initial[0].timestamp, last.timestamp); + assert.equal(initial[0].source, 'esp32:offline'); + const pose = legacy.frames.filter(f => f.type === 'pose_data'); assert.equal(pose.length, 1); + assert.equal(pose[0].timestamp, last.timestamp); assert.equal(pose[0].payload.metadata.source, 'esp32:offline'); + assert.equal(pose[0].payload.pose_source, 'unavailable'); assert.equal(pose[0].payload.confidence, null); + assert.equal(pose[0].payload.metadata.estimated_persons, null); assert.deepEqual(pose[0].payload.pose.persons, []); + assert.equal(pose[0].payload.metadata.evidence_state, 'unavailable'); + const consumer = new PoseService(); assert(consumer.validatePoseData(pose[0].payload).valid); + const converted = consumer.convertZoneDataToRestFormat(pose[0].payload, 'zone_1', pose[0]); + assert.equal(converted.pose_source, 'unavailable'); assert.equal(converted.metadata.confidence, null); + assert.equal(JSON.stringify(converted.zone_summary), '{}'); + if (privacy) for (const frame of [offline, initial[0], pose[0]]) { + const serialized = JSON.stringify(frame); assert(!serialized.includes('pose_keypoints')); assert(!serialized.includes('keypoints')); assert(!serialized.includes('vital_signs')); + } + for (let sequence = 61; sequence <= 70; sequence++) await send(sequence); + await delay(100); + assert(first.frames.some(f => f.type === 'sensing_update' && f.source === 'esp32' && f.tick > last.tick), 'fresh input must resume publication'); + return {privacy, quietFrames: quietFrames.length, lateInitialFrames: initial.length, legacyInitialFrames: pose.length, resumed: true}; + } finally { + sockets.forEach(socket => socket.close()); udp.close(); child.kill('SIGTERM'); + await Promise.race([new Promise(resolve => child.once('exit', resolve)), delay(3000)]); + if (child.exitCode === null) child.kill('SIGKILL'); + writeFileSync(join(evidence, privacy ? 'privacy.log' : 'normal.log'), logs); + } +} +const results = [await scenario(false), await scenario(true)]; +const receipt = {evidence: 'SYNTHETIC isolated actual-binary UDP/WS', binarySha256, results}; +writeFileSync(join(evidence, 'result.json'), JSON.stringify(receipt, null, 2)); +console.log(JSON.stringify({evidencePath: join(evidence, 'result.json'), ...receipt}, null, 2)); diff --git a/v2/Cargo.lock b/v2/Cargo.lock index 3ed1589569..739ee88d55 100644 --- a/v2/Cargo.lock +++ b/v2/Cargo.lock @@ -1825,36 +1825,36 @@ dependencies = [ [[package]] name = "cranelift-assembler-x64" -version = "0.123.20" +version = "0.123.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40d4018626c25bb153a72738b261bdfed97ad6b7f21b100d556c0b8457fa9b3b" +checksum = "142696a120a5dc298df74bcfb8c5ec43f0f8c53ff2f3607b3cffe67c1bc3fe0b" dependencies = [ "cranelift-assembler-x64-meta", ] [[package]] name = "cranelift-assembler-x64-meta" -version = "0.123.20" +version = "0.123.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c69b47b7ffbd42f8d162ee958886e685b374b102b3bd116b0470c911153428b4" +checksum = "24a12a121860c44ba6dc9d0bd2b4b9284f409c2f305aa04093184a7878148726" dependencies = [ "cranelift-srcgen", ] [[package]] name = "cranelift-bforest" -version = "0.123.20" +version = "0.123.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d1994514a2e498d42a7173804fb0618472c94176b72c54ef8d64acdada05d0f" +checksum = "26e0347ec8886cfe409616aeeeb3d1ee4ae0ec6ae0e053fc111e0214f67c830c" dependencies = [ "cranelift-entity", ] [[package]] name = "cranelift-bitset" -version = "0.123.20" +version = "0.123.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "019bfe57e970cf56fd9ad9ea45e4290457eb1ceb9bea954646abbe9d657a4a61" +checksum = "b8a8d504f1c49cf5b3c59166fa77c990d6618987baa16095a8adbc125218a710" dependencies = [ "serde", "serde_derive", @@ -1862,9 +1862,9 @@ dependencies = [ [[package]] name = "cranelift-codegen" -version = "0.123.20" +version = "0.123.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "397c0a3cc8a67f9f85dbe47fbba2c1e88430293b130f91f3c98411f6ae4e0e80" +checksum = "0826e7db0453c2e4def6cc16e661309b88acfe5c688727d2bd14f210c5979228" dependencies = [ "bumpalo", "cranelift-assembler-x64", @@ -1889,9 +1889,9 @@ dependencies = [ [[package]] name = "cranelift-codegen-meta" -version = "0.123.20" +version = "0.123.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5bc504c51980615d594a685367584d1fa95d36c32c14a15b0eb4b1d9cc0dc2e" +checksum = "5466899a46c1b457a2c40e62c0db595ead52ce070283349f24f1ea3e3ab95e59" dependencies = [ "cranelift-assembler-x64-meta", "cranelift-codegen-shared", @@ -1902,24 +1902,24 @@ dependencies = [ [[package]] name = "cranelift-codegen-shared" -version = "0.123.20" +version = "0.123.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc8ae2c52ca44f49719cd78e48480c076e115a490b4e51ae28a89535ec4d76df" +checksum = "4cb2af0ddf040bc688bb1c7c3f98026caf1f7be9735a90abe5210fb3ed61b31e" [[package]] name = "cranelift-control" -version = "0.123.20" +version = "0.123.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d94159e930a391bb589c8c4832769c3e2f9c81d8f1cd1266083a33a1787d41a9" +checksum = "23928359f722201147e559d1ad68b26e04f542df2da609615883373ca95488f9" dependencies = [ "arbitrary", ] [[package]] name = "cranelift-entity" -version = "0.123.20" +version = "0.123.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cc0257dc68c7d282dabc1b3c9effec3f5d00fff56a964a80e7c08af7a3aa4dc" +checksum = "544a5be17891f159c85d2a1200cbd1846d5ffdc9fbb26fa9140f4c0040cde8fe" dependencies = [ "cranelift-bitset", "serde", @@ -1928,9 +1928,9 @@ dependencies = [ [[package]] name = "cranelift-frontend" -version = "0.123.20" +version = "0.123.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a60aa092dbffc0b8e501845afc3f67a3c987d780d22dd302829aa9f0255738e" +checksum = "41a1775f555a016d16b212b38b7f4b4759f258ae90c7139ffc198bb3cb63829c" dependencies = [ "cranelift-codegen", "log", @@ -1940,15 +1940,15 @@ dependencies = [ [[package]] name = "cranelift-isle" -version = "0.123.20" +version = "0.123.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e62bea43dd7767887c6b7861021846b0f0a03d9cc72bafddd0827e79b5f5deb5" +checksum = "3cc4649f050a50eaf0ee29b47d797e76a7dbd6f4a1fa1247e21667fd50ccc197" [[package]] name = "cranelift-native" -version = "0.123.20" +version = "0.123.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f410a5e75d4758d0564106931a1809d50d17681385ea1686518d029593a06b2a" +checksum = "90b5fbff9fd5b0fef29865a393f77d1706f9ed50ff90ca40f2768ea2f53ccc7c" dependencies = [ "cranelift-codegen", "libc", @@ -1957,9 +1957,9 @@ dependencies = [ [[package]] name = "cranelift-srcgen" -version = "0.123.20" +version = "0.123.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5f98d0e96c17d2654a95820c77403050754561f2b2cd737dbf59462c64896f3" +checksum = "d2877cb00084918ff719184b88e247a2b46ebd0dca27fac3569a168590053ae1" [[package]] name = "crc" @@ -8102,9 +8102,9 @@ dependencies = [ [[package]] name = "pulley-interpreter" -version = "36.0.20" +version = "36.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f6a7699ba3c24456c4235f6578a5f182386a1a9230653e8783f165ea1085bbe" +checksum = "0781f03b600890175c17522913366e8543c96acd76d074b39a27835879f89702" dependencies = [ "cranelift-bitset", "log", @@ -8114,9 +8114,9 @@ dependencies = [ [[package]] name = "pulley-macros" -version = "36.0.20" +version = "36.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1efdd17eb6ed2d844c29350fffb7230c550ddb0cbe93e1fd2b9cfcc345d77450" +checksum = "dc7ad6f328adda4f2d169d9338402fad5f726630e9c9fdc26660771b13d1a788" dependencies = [ "proc-macro2", "quote", @@ -8213,7 +8213,7 @@ dependencies = [ "quinn-udp", "rustc-hash 2.1.1", "rustls", - "socket2 0.5.10", + "socket2 0.6.5", "thiserror 2.0.18", "tokio", "tracing", @@ -8253,9 +8253,9 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.5.10", + "socket2 0.6.5", "tracing", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -12643,9 +12643,9 @@ dependencies = [ [[package]] name = "wasmtime" -version = "36.0.20" +version = "36.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a32143e651d97f1b8d749713d242f70a72c9a0d7e16fd498a7e5a3f5574ed10b" +checksum = "c58d2cbba3817454af39ec9f0b39279870a9a98ef4c52bbbcdd9c1ffc1f5f17f" dependencies = [ "addr2line", "anyhow", @@ -12698,9 +12698,9 @@ dependencies = [ [[package]] name = "wasmtime-environ" -version = "36.0.20" +version = "36.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0501ad47df09ae57fbb76e3bdc6513b1f6d9dd0eadf3a115bb39360798890c7" +checksum = "2e64cbeb0aa73682360f848c931ec7e024a59bb15e855fc461c21c6ec665775e" dependencies = [ "anyhow", "cpp_demangle", @@ -12725,16 +12725,16 @@ dependencies = [ [[package]] name = "wasmtime-internal-asm-macros" -version = "36.0.20" +version = "36.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de4c70c3fb098b9b28b73f7fc2b0147c5624621fb017a5354cdedb660f273e07" +checksum = "ead251813500a15bb1fc31db43f8d2e612fcc0c723f3efff69c5d7bedd9edaf9" dependencies = [ "cfg-if", ] [[package]] name = "wasmtime-internal-cache" -version = "36.0.20" +version = "36.0.21" dependencies = [ "anyhow", "base64 0.22.1", @@ -12755,9 +12755,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-component-macro" -version = "36.0.20" +version = "36.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28729d33f5731506bcfe774a4b9eaf9e7e713449e1e3d848748f7de9978bea53" +checksum = "c6aba235e8bfa25cf76312a8b8295964968164cbd4e3fb6e2d9be167128ae3d7" dependencies = [ "anyhow", "proc-macro2", @@ -12770,15 +12770,15 @@ dependencies = [ [[package]] name = "wasmtime-internal-component-util" -version = "36.0.20" +version = "36.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91abf6b751d0f135cab1e78448f402af00a0f3e0638be858d18cc93170ab1009" +checksum = "0e1b1657d77558b86ace8e85e0f016655f7b68b8a8c5b8ad875515f4880aece3" [[package]] name = "wasmtime-internal-cranelift" -version = "36.0.20" +version = "36.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3840e7bb2d2593f3178b13a6717f215ad63e233ef54b61892f3cee80582f098" +checksum = "0dc2e39983f07c0e85ba5f83df2dfb8a4e8607a5cab01c761a3a5deb15abc5bd" dependencies = [ "anyhow", "cfg-if", @@ -12803,9 +12803,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-fiber" -version = "36.0.20" +version = "36.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "458d24ab006050e9b5fcf79f2d83ccc236f69846685fb27a7bf4c50a1583f1ea" +checksum = "3d8f744e4eea04b2b1070b941d32138c1c34e50887b27d9f11c47f9ce8187cd6" dependencies = [ "anyhow", "cc", @@ -12819,9 +12819,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-jit-debug" -version = "36.0.20" +version = "36.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eca1ce583227e82e6370939f48395ae902acdcd4c91ca32ea1ae7abbc86232a7" +checksum = "7e12bcf0d25fa4e1c4346d0ecea325d0ef850505ca1fabec5711b7a86679ae3e" dependencies = [ "cc", "object", @@ -12831,9 +12831,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-jit-icache-coherence" -version = "36.0.20" +version = "36.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c110f07b222a8eb29d957eabf90dce38f41461f883099ab28dc50861913ba17" +checksum = "147bdb5ff208cf503adb480a9bd9a3c9012b3289007e69d46117ca31cef51ed6" dependencies = [ "anyhow", "cfg-if", @@ -12843,24 +12843,24 @@ dependencies = [ [[package]] name = "wasmtime-internal-math" -version = "36.0.20" +version = "36.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a273dd4ec37457d435538c00b30f3ec9ae99e04a4f6ba89a737048e8de6095df" +checksum = "4e77a4cd77f1aa48a23ef9cc544cdc8732de512f2ad50f628a39092fb53cb34c" dependencies = [ "libm", ] [[package]] name = "wasmtime-internal-slab" -version = "36.0.20" +version = "36.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d12c7b3ae65639ab90d8d29135348dfd16de6d96eb36ab5b63861f3e12cf8e0" +checksum = "2652fc800965c03acb4e5a2a2bb0ce52650a1f1b3d23e91cd3f65ebafbd68ff0" [[package]] name = "wasmtime-internal-unwinder" -version = "36.0.20" +version = "36.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6da6131cb87700e06ffb7a83cc933658f521176f04d6d73511f1ee60db58c515" +checksum = "cb5796bfab805bdb0f13e8a8502c0ebd643f11bc209c9199da71c97c17234da3" dependencies = [ "anyhow", "cfg-if", @@ -12871,9 +12871,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-versioned-export-macros" -version = "36.0.20" +version = "36.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71efc44d34bb86408d7f620d308dad19457f34543e5c44ace680bdbe2d819917" +checksum = "f534370e1ccf59b3d903c6f4db72036c43b5d2b94a77ccd39c43239e90b07e61" dependencies = [ "proc-macro2", "quote", @@ -12882,9 +12882,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-winch" -version = "36.0.20" +version = "36.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af4da8c46ddb215374f023c0e3ec8917185b878b3d9874fbfaaefdc2d2df38e0" +checksum = "f286941452a9d79b9e681f7272354280129ef38c55497b757bfc9d5810277afd" dependencies = [ "anyhow", "cranelift-codegen", @@ -12899,9 +12899,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-wit-bindgen" -version = "36.0.20" +version = "36.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "356a1ae3cfc0534a84df35d9df214a241ca454983c0d311f8169b1674b4979a3" +checksum = "f0f67e6dc46acb465d17902b302f19bf29018d05b7cc00c8d6eeb2e8aa703399" dependencies = [ "anyhow", "bitflags 2.11.0", @@ -13618,20 +13618,26 @@ dependencies = [ "criterion", "futures-util", "hmac", + "hyper", + "hyper-util", "jsonwebtoken", "mdns-sd", "midstreamer-attractor", "midstreamer-temporal-compare", + "num-complex", "opentelemetry-appender-tracing", "opentelemetry-otlp", "opentelemetry_sdk", "proptest", "rand 0.8.5", + "rcgen", "ring", "rumqttc-v4-next", + "rustls", "ruvector-mincut", "ruview-auth", "ruview-fann-head", + "ruview-unified", "serde", "serde_json", "sha2 0.10.9", @@ -13639,6 +13645,7 @@ dependencies = [ "tempfile", "thiserror 1.0.69", "tokio", + "tokio-rustls", "tokio-tungstenite", "tower 0.4.13", "tower-http", @@ -13651,6 +13658,7 @@ dependencies = [ "wifi-densepose-engine", "wifi-densepose-geo", "wifi-densepose-hardware", + "wifi-densepose-mat", "wifi-densepose-physics", "wifi-densepose-rufield", "wifi-densepose-signal", @@ -13825,9 +13833,9 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "winch-codegen" -version = "36.0.20" +version = "36.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09aff830750e48ca9828e9df5ba4589688f49f2748e4a1d43fd543d6a673d272" +checksum = "48ff4619adfdac753592c13508b49067f857cb41a1e5c2579e12580ad7470ce6" dependencies = [ "anyhow", "cranelift-assembler-x64", diff --git a/v2/Cargo.toml b/v2/Cargo.toml index d546e97b36..2bf8679e3d 100644 --- a/v2/Cargo.toml +++ b/v2/Cargo.toml @@ -3,7 +3,7 @@ resolver = "2" members = [ # Checksum-verified dependency patches; keep their upstream tests in CI. "vendor/security/zip-0.6.6", - "vendor/security/wasmtime-internal-cache-36.0.20", + "vendor/security/wasmtime-internal-cache-36.0.21", "vendor/security/sqlx-sqlite-0.8.6", "crates/wifi-densepose-core", "crates/wifi-densepose-physics", # ADR-323 bounded pose assessment/correction @@ -148,7 +148,10 @@ members = [ # excluded for the same reason. Build separately: # cargo build --target wasm32-unknown-unknown --release -p homecore-plugin-example exclude = [ - "vendor/security/zip-candle-compat", # private ZIP adapter has its own guarded test workspace + # Retained ZIP 7 regression fixtures are not workspace members; current + # Candle IO delegates to the ZIP 9 adapter below. + "vendor/security/zip-7.2.0", + "vendor/security/zip-candle-compat", "crates/wifi-densepose-wasm-edge", "crates/homecore-plugin-example", # ADR-368: napi-rs transport for ruview-kernel. Own workspace so napi never @@ -346,16 +349,16 @@ strip = false inherits = "release" debug = true -# Narrow, checksum-verified parent manifest patches. Source/API code is -# unchanged. The Wasmtime cache and legacy ZIP parents use the fixed zstd +# Checksum-verified dependency patches. Candle delegates ZIP IO to the ZIP 9 +# adapter below; the Wasmtime cache and legacy ZIP manifests select zstd # 0.14.3 implementation; sqlx-sqlite selects libsqlite3-sys 0.39.1, which # bundles SQLite 3.54.1 (RUSTSEC-2026-0459). sqlx-sqlite 0.8.6 and 0.9.0 both # cap libsqlite3-sys below 0.39, so no registry release can reach it yet. -# See each directory's RUVIEW-PROVENANCE.md for archive digests and scope. +# See each directory's provenance/backport record for archive digests and scope. [patch.crates-io] zip-legacy = { package = "zip", path = "vendor/security/zip-0.6.6" } # Candle 0.9 expects ZIP 7's filename access. The private adapter validates # filename decoding and delegates ZIP operations to patched upstream ZIP 9. zip-candle = { package = "zip", path = "vendor/security/zip-candle-compat" } -wasmtime-internal-cache = { path = "vendor/security/wasmtime-internal-cache-36.0.20" } +wasmtime-internal-cache = { path = "vendor/security/wasmtime-internal-cache-36.0.21" } sqlx-sqlite = { path = "vendor/security/sqlx-sqlite-0.8.6" } diff --git a/v2/crates/homecore-plugins/Cargo.toml b/v2/crates/homecore-plugins/Cargo.toml index 5adb536849..f9815fde03 100644 --- a/v2/crates/homecore-plugins/Cargo.toml +++ b/v2/crates/homecore-plugins/Cargo.toml @@ -64,7 +64,7 @@ base64 = "0.22" # Upgraded from 25.0.3 to remediate the Cranelift/Winch sandbox advisories; # 40.0.4 was subsequently replaced because of RUSTSEC-2026-0114. # Patched 36.x retains HOMECORE's Rust 1.89 MSRV. -wasmtime = { version = "36.0.20", optional = true } +wasmtime = { version = "36.0.21", optional = true } [dev-dependencies] diff --git a/v2/crates/homecore-plugins/tests/wasmtime_advisory_regression.rs b/v2/crates/homecore-plugins/tests/wasmtime_advisory_regression.rs new file mode 100644 index 0000000000..b218bebf4b --- /dev/null +++ b/v2/crates/homecore-plugins/tests/wasmtime_advisory_regression.rs @@ -0,0 +1,23 @@ +//! RUSTSEC-2026-0569: a re-exported host function has no calling Wasm module. +#![cfg(feature = "wasmtime")] + +#[test] +fn reexported_host_function_has_no_calling_module() { + use wasmtime::{Caller, Engine, Linker, Module, Store}; + let engine = Engine::default(); + let module = Module::new(&engine, "(module (import \"host\" \"inspect\" (func)) (memory (export \"memory\") 1) (export \"run\" (func 0)))").unwrap(); + let memory = module.get_export_index("memory").unwrap(); + let mut linker = Linker::new(&engine); + linker + .func_wrap("host", "inspect", move |mut caller: Caller<'_, ()>| { + assert!(caller.get_module_export(&memory).is_none()); + }) + .unwrap(); + let mut store = Store::new(&engine, ()); + let instance = linker.instantiate(&mut store, &module).unwrap(); + instance + .get_typed_func::<(), ()>(&mut store, "run") + .unwrap() + .call(&mut store, ()) + .unwrap(); +} diff --git a/v2/crates/ruview-auth/src/lib.rs b/v2/crates/ruview-auth/src/lib.rs index 821abed8d1..50ce1acacd 100644 --- a/v2/crates/ruview-auth/src/lib.rs +++ b/v2/crates/ruview-auth/src/lib.rs @@ -86,4 +86,4 @@ pub use jwks::{JwksCache, JwksError, JwksFetcher}; #[cfg(feature = "ureq-transport")] pub use jwks::UreqFetcher; pub use principal::{scope, Principal}; -pub use verify::{extract_bearer, verify_access_token, VerifierConfig, VerifyError}; +pub use verify::{extract_bearer, verify_access_token, verify_account_identity, AccountIdentity, VerifierConfig, VerifyError}; diff --git a/v2/crates/ruview-auth/src/verify.rs b/v2/crates/ruview-auth/src/verify.rs index e5bd5c7b88..a512815c8c 100644 --- a/v2/crates/ruview-auth/src/verify.rs +++ b/v2/crates/ruview-auth/src/verify.rs @@ -95,6 +95,8 @@ pub enum VerifyError { LongLivedCredential, #[error("token carries no account_id and cannot be attributed")] MissingAccountId, + #[error("token does not contain a bounded interactive user ownership context")] + InvalidOwnership, #[error("token does not carry the required scope {required:?}")] MissingScope { required: String }, /// Minted for a different Cognitum product. `client_id` is the platform's @@ -126,6 +128,12 @@ struct AccessTokenClaims { #[serde(default)] jti: String, exp: i64, + #[serde(default)] + iat: i64, + #[serde(default)] + iss: Option, + #[serde(default)] + aud: Option, /// Long-lived, non-rotating setup credential. Absent on older tokens. #[serde(default)] setup: bool, @@ -166,11 +174,11 @@ pub struct VerifierConfig { /// /// Every rejection path returns a typed error; none of them return a partially /// trusted principal. -pub fn verify_access_token( +fn verified_interactive_claims( token: &str, jwks: &JwksCache, config: &VerifierConfig, -) -> Result { +) -> Result { let header = decode_header(token).map_err(|e| VerifyError::Malformed(e.to_string()))?; // Compared, never selected. `decode` below independently enforces the same @@ -217,6 +225,15 @@ pub fn verify_access_token( return Err(VerifyError::LongLivedCredential); } + Ok(claims) +} + +pub fn verify_access_token( + token: &str, + jwks: &JwksCache, + config: &VerifierConfig, +) -> Result { + let claims = verified_interactive_claims(token, jwks, config)?; let account_id = claims.account_id.filter(|a| !a.is_empty()); let Some(account_id) = account_id else { // meta-llm requires this so a token cannot bill an account it doesn't @@ -246,6 +263,60 @@ pub fn verify_access_token( Ok(principal) } +/// Local-data ownership is a user/tenant/workspace identity, not a billing +/// principal. This separate type cannot be used as a metering Principal. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct AccountIdentity { + pub issuer: String, + pub subject: String, + pub org_id: String, + pub workspace_id: String, + pub expires_at: i64, + _verified: (), +} + +fn canonical_uuid(value: &str) -> Option { + (value.len() == 36 && value.bytes().enumerate().all(|(index, byte)| { + if [8, 13, 18, 23].contains(&index) { byte == b'-' } else { byte.is_ascii_hexdigit() } + })).then(|| value.to_ascii_lowercase()) +} + +/// Verify account ownership with the existing fixed ES256/JWKS chain. Unlike +/// verify_access_token, this does not require the optional Firebase billing ID. +/// Existing billing/attribution acceptance is unchanged. +pub fn verify_account_identity( + token: &str, + jwks: &JwksCache, + config: &VerifierConfig, +) -> Result { + if token.len() > 16384 { return Err(VerifyError::InvalidOwnership); } + let claims = verified_interactive_claims(token, jwks, config)?; + let now = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH) + .map_err(|_| VerifyError::InvalidOwnership)?.as_secs() as i64; + let issuer = config.issuer.trim_end_matches('/'); + // Older Identity tokens omit iss/aud. The configured trusted JWKS binds + // their issuer, and exact client_id binds their audience. Present values + // must match; no token field can choose a key URL or broaden the client. + if claims.client_id != "ruview" + || claims.iss.as_deref().is_some_and(|value| value != issuer) + || claims.aud.as_deref().is_some_and(|value| value != "ruview") + || claims.iat > now + 30 || claims.exp <= now || claims.exp <= claims.iat + || claims.exp.checked_sub(claims.iat).is_none_or(|ttl| ttl > 3600) + { return Err(VerifyError::InvalidOwnership); } + let scopes: Vec<&str> = claims.scope.split_whitespace().collect(); + if !scopes.contains(&config.required_scope.as_str()) { + return Err(VerifyError::MissingScope { required: config.required_scope.clone() }); + } + Ok(AccountIdentity { + issuer: issuer.into(), + subject: canonical_uuid(&claims.sub).ok_or(VerifyError::InvalidOwnership)?, + org_id: canonical_uuid(&claims.org_id).ok_or(VerifyError::InvalidOwnership)?, + workspace_id: canonical_uuid(&claims.workspace_id).ok_or(VerifyError::InvalidOwnership)?, + expires_at: claims.exp, + _verified: (), + }) +} + /// Extract a bearer token from an `Authorization` header value. /// /// The scheme is matched **case-insensitively** per RFC 7235 §2.1, and leading diff --git a/v2/crates/ruview-auth/tests/verifier_matrix.rs b/v2/crates/ruview-auth/tests/verifier_matrix.rs index 2b2d799f8f..32ef2df4c7 100644 --- a/v2/crates/ruview-auth/tests/verifier_matrix.rs +++ b/v2/crates/ruview-auth/tests/verifier_matrix.rs @@ -22,7 +22,7 @@ use ring::{ }; use ruview_auth::{ jwks::{JwksError, JwksFetcher}, - scope, verify_access_token, JwksCache, VerifierConfig, VerifyError, + scope, verify_access_token, verify_account_identity, JwksCache, VerifierConfig, VerifyError, }; use serde_json::json; @@ -146,6 +146,50 @@ fn verify(token: &str, required_scope: &str) -> Result serde_json::Value { + let mut claims = valid_claims(); + claims["org_id"] = json!("20000000-0000-4000-8000-000000000001"); + claims["workspace_id"] = json!("30000000-0000-4000-8000-000000000001"); + claims["scope"] = json!("sensing:read spaces:read"); + claims.as_object_mut().unwrap().remove("account_id"); + claims +} + +#[test] +fn local_ownership_is_a_user_identity_without_requiring_firebase_billing() { + let token = sign(&ownership_claims()); + let identity = verify_account_identity(&token, &jwks_serving_test_key(), &config_for(scope::SENSING_READ)).unwrap(); + assert_eq!(identity.subject, "0f8fad5b-d9cb-469f-a165-70867728950e"); + assert_eq!(identity.org_id, "20000000-0000-4000-8000-000000000001"); + // This new verifier must not weaken the existing billing/attribution path. + assert!(matches!(verify(&token, scope::SENSING_READ), Err(VerifyError::MissingAccountId))); +} + +#[test] +fn local_ownership_rejects_wrong_or_unbounded_interactive_identity() { + for (field, value) in [ + ("sub", json!("")), ("sub", json!("../other-user")), + ("org_id", json!("org-1")), ("workspace_id", json!("")), + ("client_id", json!("other")), ("typ", json!("inference")), + ("setup", json!(true)), ("workload", json!(true)), + ("scope", json!("spaces:read")), ("iat", json!(now() + 300)), + ("exp", json!(now() + 7200)), ("exp", json!(now() - 1)), + ("iss", json!("https://attacker.example")), ("aud", json!("other")), + ] { + let mut claims = ownership_claims(); claims[field] = value; + assert!(verify_account_identity(&sign(&claims), &jwks_serving_test_key(), &config_for(scope::SENSING_READ)).is_err(), "accepted field {field}"); + } +} + +#[test] +fn local_ownership_verifies_signature_and_supports_current_issuer_claims() { + let mut claims = ownership_claims(); + claims["iss"] = json!(TEST_ISSUER); claims["aud"] = json!("ruview"); + assert!(verify_account_identity(&sign(&claims), &jwks_serving_test_key(), &config_for(scope::SENSING_READ)).is_ok()); + assert!(verify_account_identity(&sign_with(&claims, other_key()), &jwks_serving_test_key(), &config_for(scope::SENSING_READ)).is_err()); + assert!(verify_account_identity(ALG_NONE_TOKEN, &jwks_serving_test_key(), &config_for(scope::SENSING_READ)).is_err()); +} + // ─────────────────────────── accept ─────────────────────────── #[test] diff --git a/v2/crates/wifi-densepose-aether/src/graph_transformer.rs b/v2/crates/wifi-densepose-aether/src/graph_transformer.rs index 395752b8f0..f68a186d90 100644 --- a/v2/crates/wifi-densepose-aether/src/graph_transformer.rs +++ b/v2/crates/wifi-densepose-aether/src/graph_transformer.rs @@ -130,6 +130,18 @@ impl Linear { } out } + // Same scalar accumulation as forward, writing into an existing buffer. + fn forward_into(&self, input: &[f32], out: &mut [f32]) { + assert_eq!(input.len(), self.in_features); + assert_eq!(out.len(), self.out_features); + for (i, row) in self.weights.iter().enumerate() { + let mut s = self.bias[i]; + for (w, x) in row.iter().zip(input) { + s += w * x; + } + out[i] = s; + } + } pub fn weights(&self) -> &[Vec] { &self.weights } @@ -393,38 +405,59 @@ impl CrossAttention { return vec![vec![0.0; self.d_model]; n_q]; } - let q_proj: Vec> = query.iter().map(|q| self.w_q.forward(q)).collect(); - let k_proj: Vec> = key.iter().map(|k| self.w_k.forward(k)).collect(); - let v_proj: Vec> = value.iter().map(|v| self.w_v.forward(v)).collect(); + let mut q_proj = vec![0.0; n_q * self.d_model]; + let mut k_proj = vec![0.0; n_kv * self.d_model]; + let mut v_proj = vec![0.0; value.len() * self.d_model]; + for (i, row) in query.iter().enumerate() { + self.w_q + .forward_into(row, &mut q_proj[i * self.d_model..(i + 1) * self.d_model]); + } + for (i, row) in key.iter().enumerate() { + self.w_k + .forward_into(row, &mut k_proj[i * self.d_model..(i + 1) * self.d_model]); + } + for (i, row) in value.iter().enumerate() { + self.w_v + .forward_into(row, &mut v_proj[i * self.d_model..(i + 1) * self.d_model]); + } + // The former row-indexed V projection rejected missing value rows even + // for zero-width models. Flat empty slices alone cannot enforce that. + assert!(value.len() >= n_kv, "missing attention value rows"); let scale = (self.d_k as f32).sqrt(); let mut output = vec![vec![0.0f32; self.d_model]; n_q]; + // Scratch storage is reused across queries/heads. Keep the original + // scalar reduction order and concatenated width, including rejection of + // an inconsistent shape restored through unflatten_from. + let mut concat = vec![0.0; self.n_heads * self.d_k]; + let mut scores = vec![0.0; n_kv]; + let mut wts = vec![0.0; n_kv]; for qi in 0..n_q { - let mut concat = Vec::with_capacity(self.d_model); for h in 0..self.n_heads { let (start, end) = (h * self.d_k, (h + 1) * self.d_k); - let q_h = &q_proj[qi][start..end]; - let mut scores = vec![0.0f32; n_kv]; + let q_h = &q_proj[qi * self.d_model + start..qi * self.d_model + end]; for ki in 0..n_kv { let dot: f32 = q_h .iter() - .zip(&k_proj[ki][start..end]) + .zip(&k_proj[ki * self.d_model + start..ki * self.d_model + end]) .map(|(a, b)| a * b) .sum(); scores[ki] = dot / scale; } - let mut wts = vec![0.0f32; n_kv]; softmax(&scores, &mut wts); - let mut head_out = vec![0.0f32; self.d_k]; + let head_out = &mut concat[start..end]; + head_out.fill(0.0); for ki in 0..n_kv { - for (o, &v) in head_out.iter_mut().zip(&v_proj[ki][start..end]) { + for (o, &v) in head_out + .iter_mut() + .zip(&v_proj[ki * self.d_model + start..ki * self.d_model + end]) + { *o += wts[ki] * v; } } - concat.extend_from_slice(&head_out); } - output[qi] = self.w_o.forward(&concat); + self.w_o.forward_into(&concat, &mut output[qi]); } output } @@ -506,21 +539,26 @@ impl GraphMessagePassing { "expected 17 nodes, got {}", node_features.len() ); - let mut agg = vec![vec![0.0f32; self.in_features]; 17]; + // Rows are independent: reuse the aggregation buffer, clearing it + // before the same ascending-neighbor reduction used by the matrix path. + let mut agg = vec![0.0f32; self.in_features]; + let mut output = Vec::with_capacity(17); #[allow(clippy::needless_range_loop)] for i in 0..17 { + agg.fill(0.0); for j in 0..17 { let a = self.norm_adj[i][j]; if a.abs() > 1e-10 { - for (ag, &f) in agg[i].iter_mut().zip(&node_features[j]) { + for (ag, &f) in agg.iter_mut().zip(&node_features[j]) { *ag += a * f; } } } + let mut row = self.weight.forward(&agg); + row.iter_mut().for_each(|value| *value = relu(*value)); + output.push(row); } - agg.iter() - .map(|a| self.weight.forward(a).into_iter().map(relu).collect()) - .collect() + output } pub fn in_features(&self) -> usize { self.in_features @@ -1152,3 +1190,7 @@ mod tests { } } } + +#[cfg(test)] +#[path = "graph_transformer_equivalence_tests.rs"] +mod equivalence_tests; diff --git a/v2/crates/wifi-densepose-aether/src/graph_transformer_equivalence_tests.rs b/v2/crates/wifi-densepose-aether/src/graph_transformer_equivalence_tests.rs new file mode 100644 index 0000000000..6b168decb5 --- /dev/null +++ b/v2/crates/wifi-densepose-aether/src/graph_transformer_equivalence_tests.rs @@ -0,0 +1,186 @@ +//! Frozen pre-optimization scalar oracles. These deliberately retain the +//! allocation-heavy paths so stale scratch data, reduction-order changes and +//! finite-difference perturbations cannot silently change trained artifacts. +use super::*; +fn bits(rows: Vec>) -> Vec> { + rows.into_iter() + .map(|row| row.into_iter().map(f32::to_bits).collect()) + .collect() +} +fn matrix(rows: usize, width: usize, seed: usize) -> Vec> { + (0..rows) + .map(|i| { + (0..width) + .map(|j| (((i * 17 + j * 7 + seed) % 31) as f32 - 15.0) / 8.0) + .collect() + }) + .collect() +} +#[allow(clippy::needless_range_loop)] +fn reference_attention( + layer: &CrossAttention, + query: &[Vec], + key: &[Vec], + value: &[Vec], +) -> Vec> { + let (n_q, n_kv) = (query.len(), key.len()); + if n_q == 0 || n_kv == 0 { + return vec![vec![0.0; layer.d_model]; n_q]; + } + + let q_proj: Vec> = query.iter().map(|q| layer.w_q.forward(q)).collect(); + let k_proj: Vec> = key.iter().map(|k| layer.w_k.forward(k)).collect(); + let v_proj: Vec> = value.iter().map(|v| layer.w_v.forward(v)).collect(); + + let scale = (layer.d_k as f32).sqrt(); + let mut output = vec![vec![0.0f32; layer.d_model]; n_q]; + + for qi in 0..n_q { + let mut concat = Vec::with_capacity(layer.d_model); + for h in 0..layer.n_heads { + let (start, end) = (h * layer.d_k, (h + 1) * layer.d_k); + let q_h = &q_proj[qi][start..end]; + let mut scores = vec![0.0f32; n_kv]; + for ki in 0..n_kv { + let dot: f32 = q_h + .iter() + .zip(&k_proj[ki][start..end]) + .map(|(a, b)| a * b) + .sum(); + scores[ki] = dot / scale; + } + let mut wts = vec![0.0f32; n_kv]; + softmax(&scores, &mut wts); + let mut head_out = vec![0.0f32; layer.d_k]; + for ki in 0..n_kv { + for (o, &v) in head_out.iter_mut().zip(&v_proj[ki][start..end]) { + *o += wts[ki] * v; + } + } + concat.extend_from_slice(&head_out); + } + output[qi] = layer.w_o.forward(&concat); + } + output +} +#[allow(clippy::needless_range_loop)] +fn reference_graph(layer: &GraphMessagePassing, node_features: &[Vec]) -> Vec> { + assert_eq!( + node_features.len(), + 17, + "expected 17 nodes, got {}", + node_features.len() + ); + let mut agg = vec![vec![0.0f32; layer.in_features]; 17]; + #[allow(clippy::needless_range_loop)] + for i in 0..17 { + for j in 0..17 { + let a = layer.norm_adj[i][j]; + if a.abs() > 1e-10 { + for (ag, &f) in agg[i].iter_mut().zip(&node_features[j]) { + *ag += a * f; + } + } + } + } + agg.iter() + .map(|a| layer.weight.forward(a).into_iter().map(relu).collect()) + .collect() +} +#[test] +fn attention_preserves_bits_across_shapes_and_parameter_probes() { + for (width, heads) in [(0, 1), (2, 1), (3, 1), (4, 2), (8, 4), (16, 4)] { + let layer = CrossAttention::new(width, heads); + let mut parameters = Vec::new(); + layer.flatten_into(&mut parameters); + for (queries, keys, values) in [(0, 4, 4), (17, 0, 0), (1, 1, 1), (17, 4, 4), (3, 7, 9)] { + let q = matrix(queries, width, 3); + let k = matrix(keys, width, 7); + let v = matrix(values, width, 19); + assert_eq!( + bits(layer.forward(&q, &k, &v)), + bits(reference_attention(&layer, &q, &k, &v)) + ); + for index in 0..parameters.len() { + for epsilon in [-0.002, 0.002] { + let mut changed = parameters.clone(); + changed[index] += epsilon; + let (probe, _) = CrossAttention::unflatten_from(&changed, width, heads); + assert_eq!( + bits(probe.forward(&q, &k, &v)), + bits(reference_attention(&probe, &q, &k, &v)), + "width={width} heads={heads} parameter={index} epsilon={epsilon}" + ); + } + } + } + } +} +#[test] +fn graph_aggregation_preserves_bits_across_widths_and_parameter_probes() { + for (input, output) in [(0, 3), (3, 3), (4, 2), (8, 6)] { + let layer = GraphMessagePassing::new(input, output, &BodyGraph::new()); + let features = matrix(17, input, 11); + let mut parameters = Vec::new(); + layer.weight.flatten_into(&mut parameters); + assert_eq!( + bits(layer.forward(&features)), + bits(reference_graph(&layer, &features)) + ); + for index in 0..parameters.len() { + for epsilon in [-0.002, 0.002] { + let mut changed = parameters.clone(); + changed[index] += epsilon; + let mut probe = layer.clone(); + probe.weight = Linear::unflatten_from(&changed, input, output).0; + assert_eq!( + bits(probe.forward(&features)), + bits(reference_graph(&probe, &features)) + ); + } + } + } +} +#[test] +fn linear_reuse_overwrites_every_output_and_matches_allocating_path() { + for (input, output) in [(0, 3), (3, 3), (4, 2), (8, 6)] { + let layer = Linear::with_seed(input, output, 29); + let mut scratch = vec![f32::NAN; output]; + for row in matrix(17, input, 13) { + layer.forward_into(&row, &mut scratch); + assert_eq!( + scratch.iter().map(|x| x.to_bits()).collect::>(), + layer + .forward(&row) + .iter() + .map(|x| x.to_bits()) + .collect::>() + ); + } + } +} +#[test] +fn inconsistent_restored_attention_shape_still_rejects() { + let original = CrossAttention::new(3, 1); + let mut parameters = Vec::new(); + original.flatten_into(&mut parameters); + let (invalid, _) = CrossAttention::unflatten_from(¶meters, 3, 2); + let q = matrix(17, 3, 1); + let k = matrix(4, 3, 2); + assert!(std::panic::catch_unwind(|| reference_attention(&invalid, &q, &k, &k)).is_err()); + assert!(std::panic::catch_unwind(|| invalid.forward(&q, &k, &k)).is_err()); +} + +#[test] +fn missing_value_rows_still_reject_including_zero_width() { + for width in [0, 3] { + let layer = CrossAttention::new(width, 1); + let q = matrix(2, width, 1); + let k = matrix(4, width, 2); + for values in [0, 3] { + let v = matrix(values, width, 3); + assert!(std::panic::catch_unwind(|| reference_attention(&layer, &q, &k, &v)).is_err()); + assert!(std::panic::catch_unwind(|| layer.forward(&q, &k, &v)).is_err()); + } + } +} diff --git a/v2/crates/wifi-densepose-calibration/src/multistatic.rs b/v2/crates/wifi-densepose-calibration/src/multistatic.rs index 7fbcb422d7..46115bbe60 100644 --- a/v2/crates/wifi-densepose-calibration/src/multistatic.rs +++ b/v2/crates/wifi-densepose-calibration/src/multistatic.rs @@ -293,7 +293,45 @@ mod tests { m.add_node(1, bank("b1"), "b2"); // trained on b1, now observing b2 → stale let mut per = BTreeMap::new(); per.insert(1u8, live(12.0, 0.2, 0.3, 0.9)); - assert!(m.infer(&per).stale); + let s = m.infer(&per); + assert!(s.stale); + assert!(s.presence.is_none(), "stale is unknown, not empty"); + assert!(s.breathing.is_none()); + assert!(s.heartbeat.is_none()); + assert!(s.posture.is_none()); + assert!(s.anomaly.is_none()); + assert!(s.restlessness.is_none()); + } + + #[test] + fn stale_node_cannot_supply_vitals_or_veto_fresh_node() { + let mut m = MultiNodeMixture::new(); + m.add_node(1, bank("old"), "new"); + m.add_node(2, bank("fresh"), "fresh"); + let mut per = BTreeMap::new(); + per.insert(1u8, live(9000.0, 500.0, 0.2, 0.99)); + per.insert(2u8, live(12.0, 0.2, 0.3, 0.95)); + let s = m.infer(&per); + assert!(s.stale, "existing conservative room flag is preserved"); + assert!(!s.vetoed, "stale anomaly must abstain"); + assert_eq!(s.presence.unwrap().value, 1.0); + assert!((s.breathing.unwrap().value - 18.0).abs() < 0.3); + } + + #[test] + fn incomplete_node_bank_does_not_invent_absent_presence_or_vitals() { + let mut m = MultiNodeMixture::new(); + let mut incomplete = bank("b1"); + incomplete.presence = None; + m.add_node(1, incomplete, "b1"); + let mut per = BTreeMap::new(); + per.insert(1u8, live(12.0, 0.2, 0.3, 0.95)); + let s = m.infer(&per); + assert!(!s.stale); + assert!(s.presence.is_none()); + assert!(s.breathing.is_none()); + assert!(s.heartbeat.is_none()); + assert!(s.posture.is_none()); } #[test] diff --git a/v2/crates/wifi-densepose-calibration/src/runtime.rs b/v2/crates/wifi-densepose-calibration/src/runtime.rs index 6b4b25fec5..2bebb0fe3e 100644 --- a/v2/crates/wifi-densepose-calibration/src/runtime.rs +++ b/v2/crates/wifi-densepose-calibration/src/runtime.rs @@ -6,7 +6,7 @@ //! suppresses positive vitals/posture rather than propagating a hallucination; //! - **presence = absent** short-circuits breathing/heartbeat/posture to `None` //! (you cannot have a respiration rate in an empty room); -//! - a **STALE** bank (baseline drift) flags every reading. +//! - a **STALE** bank (baseline drift) abstains instead of emitting readings. use serde::{Deserialize, Serialize}; @@ -63,6 +63,9 @@ impl MixtureOfSpecialists { stale: self.bank.is_stale(current_baseline_id), ..Default::default() }; + if state.stale { + return state; + } // Anomaly first — it can veto everything else. state.anomaly = self.bank.anomaly.as_ref().and_then(|a| a.infer(f)); @@ -79,8 +82,8 @@ impl MixtureOfSpecialists { .presence .as_ref() .map(|r| r.value > 0.5) - // No presence specialist → assume present so vitals still run. - .unwrap_or(true); + // An incomplete bank has no authority to unlock dependent output. + .unwrap_or(false); // Restlessness is reported regardless of presence (movement implies presence). state.restlessness = self.bank.restlessness.as_ref().and_then(|r| r.infer(f)); @@ -174,5 +177,36 @@ mod tests { let mix = MixtureOfSpecialists::new(bank()); let s = mix.infer(&live(10.0, 0.2, 0.3, 0.9), "base-2"); assert!(s.stale); + assert!(s.presence.is_none(), "stale presence must abstain"); + assert!(s.breathing.is_none(), "stale vitals must abstain"); + assert!(s.posture.is_none(), "stale posture must abstain"); + assert!(s.heartbeat.is_none()); + assert!(s.restlessness.is_none()); + assert!(s.anomaly.is_none()); + assert!(!s.vetoed, "abstention is not an observed anomaly veto"); + let json = serde_json::to_value(&s).unwrap(); + assert!(json["presence"].is_null(), "unknown must not serialize as absent"); + assert_eq!(json["stale"], true); + } + + #[test] + fn missing_presence_specialist_abstains_and_suppresses_vitals() { + let mut incomplete = bank(); + incomplete.presence = None; + let mix = MixtureOfSpecialists::new(incomplete); + + let mut features = live(10.0, 0.2, 0.3, 0.9); + features.heart_score = 0.95; + features.heart_hz = 1.2; + let s = mix.infer(&features, "base-1"); + + assert!(!s.stale); + assert!(s.anomaly.is_some(), "unaffected specialist still executes"); + assert!(s.restlessness.is_some(), "independent movement evidence is retained"); + assert!(serde_json::to_value(&s).unwrap()["presence"].is_null()); + assert!(s.presence.is_none()); + assert!(s.breathing.is_none()); + assert!(s.heartbeat.is_none()); + assert!(s.posture.is_none()); } } diff --git a/v2/crates/wifi-densepose-mat/Cargo.toml b/v2/crates/wifi-densepose-mat/Cargo.toml index 848770f32c..d4377c2e76 100644 --- a/v2/crates/wifi-densepose-mat/Cargo.toml +++ b/v2/crates/wifi-densepose-mat/Cargo.toml @@ -29,8 +29,9 @@ ml = ["dep:wifi-densepose-nn"] # `cfg_attr(feature = "serde", ...)` derives on domain types are actually # active when the API is on (review finding 5: `api = ["dep:serde"]` enabled # the dependency but left every `feature = "serde"` cfg dead). -# The REST surface exposes ML status (`ml_ready`), so `api` implies `ml`. -api = ["ml", "serde", "dep:axum", "dep:futures-util"] +# API-only hosts expose an honest unavailable ML status without bundling ONNX. +# The default feature set still enables ML for existing full MAT deployments. +api = ["serde", "dep:axum", "dep:futures-util"] # Real ESP32 serial CSI ingest. Pulls the native `serialport` crate (libudev on # Linux) only when enabled, so the default/no-default appliance build stays free # of native serial deps. With the feature OFF, the ESP32 serial *parser* still diff --git a/v2/crates/wifi-densepose-mat/src/api/handlers.rs b/v2/crates/wifi-densepose-mat/src/api/handlers.rs index 39f790ee36..2d7c8e3e99 100644 --- a/v2/crates/wifi-densepose-mat/src/api/handlers.rs +++ b/v2/crates/wifi-densepose-mat/src/api/handlers.rs @@ -1048,7 +1048,7 @@ pub async fn pipeline_status( Ok(Json(PipelineStatusResponse { scanning: state.is_scanning(), buffer_duration_secs: 0.0, - ml_enabled: config.enable_ml, + ml_enabled: cfg!(feature = "ml") && config.enable_ml, ml_ready: pipeline.ml_ready(), sample_rate: config.sample_rate, heartbeat_enabled: config.enable_heartbeat, diff --git a/v2/crates/wifi-densepose-mat/src/api/websocket.rs b/v2/crates/wifi-densepose-mat/src/api/websocket.rs index 708488be25..0bdc1cee89 100644 --- a/v2/crates/wifi-densepose-mat/src/api/websocket.rs +++ b/v2/crates/wifi-densepose-mat/src/api/websocket.rs @@ -37,6 +37,7 @@ use axum::{ State, }, response::Response, + Extension, }; use futures_util::{SinkExt, StreamExt}; use parking_lot::Mutex; @@ -46,6 +47,14 @@ use uuid::Uuid; use super::dto::{WebSocketMessage, WebSocketRequest}; use super::state::AppState; +/// Optional host-owned output policy for serialized WebSocket frames. +/// +/// HTTP response middleware cannot filter frames after the upgrade. A host +/// applying a privacy policy must install this extension on the MAT router. +/// Returning `None` withholds a frame (fail closed). +#[derive(Clone, Copy)] +pub struct StreamOutputFilter(pub fn(&str) -> Option); + /// WebSocket connection handler. /// /// # OpenAPI Specification @@ -75,13 +84,17 @@ use super::state::AppState; /// 101: /// description: WebSocket connection established /// ``` -#[tracing::instrument(skip(state, ws))] -pub async fn ws_handler(State(state): State, ws: WebSocketUpgrade) -> Response { - ws.on_upgrade(move |socket| handle_socket(socket, state)) +#[tracing::instrument(skip(state, ws, filter))] +pub async fn ws_handler( + State(state): State, + filter: Option>, + ws: WebSocketUpgrade, +) -> Response { + ws.on_upgrade(move |socket| handle_socket(socket, state, filter.map(|Extension(f)| f))) } /// Handle an established WebSocket connection. -async fn handle_socket(socket: WebSocket, state: AppState) { +async fn handle_socket(socket: WebSocket, state: AppState, filter: Option) { let (mut sender, mut receiver) = socket.split(); // Subscription state for this connection @@ -102,7 +115,12 @@ async fn handle_socket(socket: WebSocket, state: AppState) { Ok(msg) => { // Check if this message matches subscription filter if subs_clone.lock().should_receive(&msg) { - if let Ok(json) = serde_json::to_string(&msg) { + if let Some(json) = serde_json::to_string(&msg).ok().and_then(|json| { + match filter { + Some(filter) => (filter.0)(&json), + None => Some(json), + } + }) { if sender.send(Message::Text(json)).await.is_err() { break; } diff --git a/v2/crates/wifi-densepose-mat/src/detection/pipeline.rs b/v2/crates/wifi-densepose-mat/src/detection/pipeline.rs index 0402a5b544..c7a07bd992 100644 --- a/v2/crates/wifi-densepose-mat/src/detection/pipeline.rs +++ b/v2/crates/wifi-densepose-mat/src/detection/pipeline.rs @@ -192,6 +192,12 @@ impl DetectionPipeline { self.ml_pipeline.as_ref().is_none_or(|ml| ml.is_ready()) } + /// API-only packages have no optional ML runtime to initialize. + #[cfg(not(feature = "ml"))] + pub fn ml_ready(&self) -> bool { + false + } + /// Process a scan zone and return detected vital signs. /// /// CSI data must be pushed into the pipeline via [`add_data`] before calling diff --git a/v2/crates/wifi-densepose-sensing-server/Cargo.toml b/v2/crates/wifi-densepose-sensing-server/Cargo.toml index 34da271ab3..96f5a1309e 100644 --- a/v2/crates/wifi-densepose-sensing-server/Cargo.toml +++ b/v2/crates/wifi-densepose-sensing-server/Cargo.toml @@ -27,6 +27,13 @@ tokio = { workspace = true, features = ["full", "process"] } futures-util = "0.3" ruvector-mincut = { workspace = true } mdns-sd = "0.11" +rcgen = "0.12.1" +rustls = { version = "0.23", default-features = false, features = ["ring", "std", "tls12"] } +# Opt-in encrypted capture recovery uses the same already-locked crypto package. +ring = "0.17" +tokio-rustls = { version = "0.26", default-features = false, features = ["ring", "tls12"] } +hyper = { workspace = true } +hyper-util = { version = "0.1", features = ["server-auto", "tokio", "service"] } # Serialization serde = { workspace = true } @@ -47,8 +54,11 @@ clap = { workspace = true } # from `lib.rs` so the Python `[aether]` wheel can bind it without this server's # Axum/tokio/worldgraph/ruvector tree. wifi-densepose-aether = { version = "0.3.0", path = "../wifi-densepose-aether" } +ruview-unified = { path = "../ruview-unified" } +num-complex = { workspace = true } wifi-densepose-core = { workspace = true, features = ["serde"] } wifi-densepose-physics = { workspace = true } +wifi-densepose-mat = { version = "0.3.2", path = "../wifi-densepose-mat", default-features = false, features = ["std", "api"] } # Multi-BSSID WiFi scanning pipeline (ADR-022 Phase 3) # ADR-426 D5: default-off ruv-FANN shadow scoring. Optional, so a default @@ -167,7 +177,6 @@ proptest = { version = "1.5", default-features = false, features = ["std"] } # end to end (router → middleware → verifier), not just mocked at the seam. # Keys are generated at test runtime; none are committed. jsonwebtoken = "9" -ring = "0.17" base64 = "0.21" [[bench]] diff --git a/v2/crates/wifi-densepose-sensing-server/README.md b/v2/crates/wifi-densepose-sensing-server/README.md index 0fae86a5fc..5749868bcf 100644 --- a/v2/crates/wifi-densepose-sensing-server/README.md +++ b/v2/crates/wifi-densepose-sensing-server/README.md @@ -76,6 +76,27 @@ cargo run -p wifi-densepose-sensing-server -- \ --ui-path ../ui ``` +### Experimental per-source shadow diagnostics + +`--source-bank-shadow` is off by default. When explicitly enabled, the next normal, +operator-confirmed room calibration also trains one independent, canonicalized +FieldModel per bound source (at least two). Each source must satisfy its own frame, +duration, grid and continuity requirements. Primary single-source calibration and +publication gates remain unchanged. This first adapter uses the existing frozen +raw grid for all sources; independent per-node grid selection is not implemented. +A radio missing that grid cannot complete its shadow calibration. + +After normal calibration completion, protected `GET /api/v1/calibration/status` +includes `source_bank_shadow`: per-source frame counts and empty-background +comparisons only after complete, fresh, post-training windows arrive. Privacy mode +omits the diagnostics. Cancel/reset clears the bank; normal `/calibration/stop` +completion freezes it for shadow comparison. Restart does not restore it. +No bank data is persisted or promoted into person count, presence or Vitals. +The existing UDP source/grid/sequence admission is not cryptographic sensor +authentication; diagnostics must retain that transport limitation. +This developer prototype needs independent physical empty/occupied/adjacent +qualification before any detection benefit can be claimed. + ### Empty-room startup baseline Provide a stable installation ID and an application-owned private state directory: diff --git a/v2/crates/wifi-densepose-sensing-server/src/account_profile.rs b/v2/crates/wifi-densepose-sensing-server/src/account_profile.rs new file mode 100644 index 0000000000..44b263bacd --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/account_profile.rs @@ -0,0 +1,152 @@ +//! Private helper working directory and durable account ownership witness. +//! Profile selection comes from the authenticated native app, never an HTTP +//! request. Unknown/nonempty legacy data cannot become a new account's data. +use std::{ + io, + path::{Path, PathBuf}, +}; + +const OWNER_FILE: &str = "account-owner-v1.json"; +const SCHEMA: &str = "ruview.account-owner.v1"; + +/// A desktop-owned helper cannot inherit a device-wide model mount. Manual +/// unowned server deployments retain their existing MODELS_DIR contract. +pub fn models_directory( + owned: bool, + data_dir: &Path, + override_dir: Option<&std::ffi::OsStr>, +) -> PathBuf { + if owned { + data_dir.join("models") + } else { + override_dir + .map(PathBuf::from) + .unwrap_or_else(|| data_dir.join("models")) + } +} + +pub fn prepare_directory(scope: &str, directory: &Path) -> io::Result { + if scope.len() != 64 + || !scope + .bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) + || !directory.is_absolute() + || directory.file_name().and_then(|v| v.to_str()) != Some("SensingServer") + || directory + .parent() + .and_then(Path::file_name) + .and_then(|v| v.to_str()) + != Some(scope) + { + return Err(io::Error::other("invalid owned helper directory")); + } + if let Some(parent) = directory.parent().and_then(|path| path.canonicalize().ok()) { + if parent.file_name().and_then(|v| v.to_str()) != Some(scope) { + return Err(io::Error::other("account directory alias rejected")); + } + } + crate::companion::private_dir(directory)?; + let canonical = directory.canonicalize()?; + if canonical + .parent() + .and_then(Path::file_name) + .and_then(|v| v.to_str()) + != Some(scope) + { + return Err(io::Error::other( + "helper directory belongs to another account namespace", + )); + } + let witness = canonical.join(OWNER_FILE); + match std::fs::symlink_metadata(&witness) { + Ok(_) => { + let value: serde_json::Value = + serde_json::from_slice(&crate::companion::private_read(&witness)?) + .map_err(io::Error::other)?; + if value.get("schema").and_then(|v| v.as_str()) != Some(SCHEMA) + || value.get("scope_key").and_then(|v| v.as_str()) != Some(scope) + { + return Err(io::Error::other( + "helper account ownership witness mismatch", + )); + } + } + Err(error) if error.kind() == io::ErrorKind::NotFound => { + if std::fs::read_dir(&canonical)?.next().is_some() { + return Err(io::Error::other( + "existing helper data needs explicit owner migration", + )); + } + let value = + serde_json::to_vec(&serde_json::json!({"schema": SCHEMA, "scope_key": scope})) + .map_err(io::Error::other)?; + crate::companion::atomic_private_write(&witness, &value)?; + if crate::companion::private_read(&witness)? != value { + return Err(io::Error::other("helper ownership readback failed")); + } + } + Err(error) => return Err(error), + } + Ok(canonical) +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn owned_models_ignore_a_device_wide_directory_override() { + let a = Path::new("/fixture/account-A/models"); + let b = Path::new("/fixture/account-B/SensingServer"); + assert_eq!( + models_directory(true, b, Some(a.as_os_str())), + b.join("models") + ); + assert_eq!(models_directory(false, b, Some(a.as_os_str())), a); + } + #[test] + fn independent_profiles_keep_their_own_witness_and_legacy_is_not_adopted() { + let root = tempfile::tempdir().unwrap(); + let a = "a".repeat(64); + let b = "b".repeat(64); + let a_dir = root.path().join(&a).join("SensingServer"); + let b_dir = root.path().join(&b).join("SensingServer"); + let a_path = prepare_directory(&a, &a_dir).unwrap(); + let a_before = std::fs::read(a_path.join(OWNER_FILE)).unwrap(); + assert_eq!(prepare_directory(&a, &a_dir).unwrap(), a_path); + prepare_directory(&b, &b_dir).unwrap(); + assert_eq!(std::fs::read(a_path.join(OWNER_FILE)).unwrap(), a_before); + assert!(prepare_directory(&b, &a_dir).is_err()); + let legacy = root.path().join("c".repeat(64)).join("SensingServer"); + std::fs::create_dir_all(&legacy).unwrap(); + std::fs::write(legacy.join("old-model.json"), "legacy bytes").unwrap(); + assert!(prepare_directory(&"c".repeat(64), &legacy).is_err()); + assert_eq!( + std::fs::read_to_string(legacy.join("old-model.json")).unwrap(), + "legacy bytes" + ); + assert!(!legacy.join(OWNER_FILE).exists()); + } + #[test] + fn mismatched_or_future_witness_is_preserved_and_rejected() { + let root = tempfile::tempdir().unwrap(); + let key = "a".repeat(64); + let dir = prepare_directory(&key, &root.path().join(&key).join("SensingServer")).unwrap(); + let future = br#"{"schema":"unknown.future","scope_key":"unreadable"}"#; + crate::companion::atomic_private_write(&dir.join(OWNER_FILE), future).unwrap(); + assert!(prepare_directory(&key, &dir).is_err()); + assert_eq!(std::fs::read(dir.join(OWNER_FILE)).unwrap(), future); + assert!(prepare_directory("../legacy", &dir).is_err()); + assert!(prepare_directory(&key, Path::new("relative/SensingServer")).is_err()); + } + #[cfg(unix)] + #[test] + fn symlink_to_another_account_profile_is_rejected() { + let root = tempfile::tempdir().unwrap(); + let a = "a".repeat(64); + let b = "b".repeat(64); + let a_dir = prepare_directory(&a, &root.path().join(&a).join("SensingServer")).unwrap(); + std::fs::create_dir(root.path().join(&b)).unwrap(); + std::os::unix::fs::symlink(&a_dir, root.path().join(&b).join("SensingServer")).unwrap(); + assert!(prepare_directory(&b, &root.path().join(&b).join("SensingServer")).is_err()); + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/bearer_auth.rs b/v2/crates/wifi-densepose-sensing-server/src/bearer_auth.rs index 88e4e0ce7c..71010d6d62 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/bearer_auth.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/bearer_auth.rs @@ -55,9 +55,18 @@ use axum::{ response::{IntoResponse, Response}, }; use ruview_auth::{scope, verify_access_token, JwksCache, UreqFetcher, VerifierConfig}; +use sha2::{Digest, Sha256}; /// Environment variable that gates the middleware. Unset / empty ⇒ auth off. pub const API_TOKEN_ENV: &str = "RUVIEW_API_TOKEN"; +pub const OWNER_ACCOUNT_SCOPE_ENV: &str = "RUVIEW_OWNER_ACCOUNT_SCOPE"; + +fn account_scope_key(identity: &ruview_auth::AccountIdentity) -> String { + let encoded = serde_json::to_vec(&["ruview.account-data.v1", identity.issuer.as_str(), + identity.subject.as_str(), identity.org_id.as_str(), identity.workspace_id.as_str()]) + .expect("string array serializes"); + format!("{:x}", Sha256::digest(encoded)) +} /// Issuer origin of the Cognitum authorization server. Setting this enables /// OAuth verification; unset ⇒ OAuth off and behaviour is unchanged. @@ -239,6 +248,8 @@ impl std::fmt::Debug for OAuthState { /// [`AuthState::from_env`]'s contract about failing closed. #[derive(Debug, thiserror::Error)] pub enum OAuthConfigError { + #[error("owned account scope requires a bounded namespace, owner credential, fixed Cognitum issuer/client and authenticated streams")] + InvalidOwnerContext, #[error("{OAUTH_ISSUER_ENV} is set but empty")] EmptyIssuer, #[error("JWKS at {url} is unreachable, so no token could ever be verified: {source}")] @@ -258,6 +269,7 @@ pub enum OAuthConfigError { /// should not be the one that does not. #[derive(Clone, Default)] pub struct AuthState { + owner_account_scope: Option, /// The expected static bearer token, if any. token: Option>, /// Cognitum OAuth verification, if enabled. @@ -288,6 +300,7 @@ impl AuthState { AuthState::default() } else { AuthState { + owner_account_scope: None, token: Some(Arc::new(s)), oauth: None, tickets: crate::ws_ticket::TicketStore::new(), @@ -362,7 +375,16 @@ impl AuthState { WS_PATHS.join(", ") ); } + let owner_account_scope = match std::env::var(OWNER_ACCOUNT_SCOPE_ENV) { + Err(std::env::VarError::NotPresent) => None, + Ok(value) if value.len() == 64 && value.bytes().all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) + && token.is_some() && !legacy_ws + && std::env::var(OAUTH_JWKS_URL_ENV).ok().as_deref().is_none_or(|url| url == "https://auth.cognitum.one/.well-known/jwks.json") + && oauth.as_ref().is_some_and(|o| o.issuer == COGNITUM_ISSUER && o.allowed_client_ids == [DEFAULT_CLIENT_ID]) => Some(value), + _ => return Err(OAuthConfigError::InvalidOwnerContext), + }; Ok(AuthState { + owner_account_scope, token, oauth, tickets: crate::ws_ticket::TicketStore::new(), @@ -394,6 +416,7 @@ impl AuthState { &self, token: &str, ) -> Result { + self.verify_account_ownership(token).map_err(|_| ruview_auth::VerifyError::InvalidOwnership)?; let oauth = self .oauth .as_ref() @@ -409,6 +432,22 @@ impl AuthState { ) } + pub fn owner_account_scope(&self) -> Option<&str> { self.owner_account_scope.as_deref() } + + /// The companion proof alone is not Cognitum account ownership. Require + /// the same verified user/tenant/workspace for every owned TLS request. + /// Unscoped manually configured servers retain their existing contract. + pub fn verify_account_ownership(&self, token: &str) -> Result, StatusCode> { + let Some(expected) = self.owner_account_scope.as_deref() else { return Ok(None); }; + let oauth = self.oauth.as_ref().ok_or(StatusCode::UNAUTHORIZED)?; + let identity = ruview_auth::verify_account_identity(token, &oauth.jwks, &VerifierConfig { + issuer: oauth.issuer.clone(), required_scope: scope::SENSING_READ.into(), + allowed_client_ids: vec![DEFAULT_CLIENT_ID.into()], + }).map_err(|_| StatusCode::UNAUTHORIZED)?; + if !ct_eq(expected.as_bytes(), account_scope_key(&identity).as_bytes()) { return Err(StatusCode::FORBIDDEN); } + Ok(Some(identity.expires_at as u64)) + } + /// Whether the legacy unauthenticated-WebSocket escape hatch is active. pub fn legacy_ws_enabled(&self) -> bool { self.legacy_ws @@ -424,6 +463,11 @@ impl AuthState { self.oauth.is_some() } + /// Exact owner capability check; caller must also verify a loopback socket peer. + pub fn is_owner_token(&self, supplied: &str) -> bool { + self.token.as_ref().is_some_and(|expected| ct_eq(supplied.as_bytes(), expected.as_bytes())) + } + /// Whether the legacy static `RUVIEW_API_TOKEN` is configured. pub fn static_token_enabled(&self) -> bool { self.token.is_some() @@ -521,6 +565,10 @@ pub async fn require_bearer( mut request: Request, next: Next, ) -> Response { + // Only the dedicated TLS router can insert this verified principal. + if request.extensions().get::().is_some() { + return next.run(request).await; + } if !auth.is_enabled() { return next.run(request).await; } @@ -603,9 +651,11 @@ pub async fn require_bearer( // as "the same mistake this codebase had to fix in jwks.rs". The hot // verification path had not been given the same treatment. let token = supplied.to_string(); + let owner_auth = auth.clone(); let oauth = Arc::clone(oauth); let required_owned = required.to_string(); let verified = tokio::task::spawn_blocking(move || { + owner_auth.verify_account_ownership(&token).map_err(|_| ruview_auth::VerifyError::InvalidOwnership)?; let config = VerifierConfig { issuer: oauth.issuer.clone(), required_scope: required_owned, @@ -661,6 +711,9 @@ pub async fn require_bearer( .and_then(|v| v.to_str().ok()) { if let Some(session) = crate::browser_session::from_cookie_header(cookie_header) { + if !session_matches_owner(&auth, &session) { + return unauthorized(&auth); + } let required = if is_ws_path(&path) { scope::SENSING_READ } else { @@ -709,6 +762,25 @@ pub async fn require_bearer( unauthorized(&auth) } +fn session_matches_owner(auth: &AuthState, session: &crate::browser_session::BrowserSession) -> bool { + auth.owner_account_scope().is_none() || auth.owner_account_scope() == session.owner_account_scope.as_deref() +} + +/// The public browser status must agree with the protected cookie boundary. +/// A legacy cookie may still verify against a migrated signing key without +/// belonging to this account-owned helper. +pub fn browser_status(auth: &AuthState, cookie: Option<&str>) -> serde_json::Value { + let session = cookie.and_then(crate::browser_session::from_cookie_header) + .filter(|session| session_matches_owner(auth, session)); + serde_json::json!({ + "auth_required": auth.is_enabled(), "oauth_enabled": auth.oauth_enabled(), + "browser_signin": auth.oauth_enabled() && crate::browser_session::is_configured(), + "signed_in": session.is_some(), + "account": session.as_ref().map(|value| value.account_id.clone()), + "scope": session.as_ref().map(|value| value.scope.clone()), + }) +} + /// The no-bearer path: try a browser session cookie, else 401. async fn session_or_unauthorized(auth: &AuthState, request: Request, next: Next) -> Response { let path = request.uri().path().to_string(); @@ -719,6 +791,7 @@ async fn session_or_unauthorized(auth: &AuthState, request: Request, next: Next) .and_then(|v| v.to_str().ok()) { if let Some(session) = crate::browser_session::from_cookie_header(h) { + if !session_matches_owner(auth, &session) { return unauthorized(auth); } let required = if is_ws_path(&path) { scope::SENSING_READ } else { @@ -1284,6 +1357,56 @@ mod oauth_tests { ) } + #[tokio::test] + async fn companion_owner_http_guard_matches_the_native_namespace_without_a_billing_id() { + let mut auth = oauth_only(); + Arc::get_mut(auth.oauth.as_mut().unwrap()).unwrap().issuer = "https://auth.cognitum.one".into(); + auth.token = Some(Arc::new("synthetic-local-owner-capability".into())); + // Same fixed user/tenant/workspace vector as the JS and Swift checks. + auth.owner_account_scope = Some("40b085eb7fdbb89941aa63cd909813c2b9a718e1b5b2b052f17a600d7bb372b4".into()); + let now = chrono::Utc::now().timestamp(); + let make_token = |subject: &str, workspace: &str| { + let claims = serde_json::json!({"typ":"access", "iss":"https://auth.cognitum.one", "aud":"ruview", "client_id":"ruview", + "sub":subject, "org_id":"20000000-0000-4000-8000-000000000001", "workspace_id":workspace, + "scope":"sensing:read spaces:read", "iat":now, "exp":now+300, "setup":false, "workload":false}); + let mut header = Header::new(jsonwebtoken::Algorithm::ES256); header.kid = Some(KID.into()); + encode(&header, &claims, &EncodingKey::from_ec_der(&key().der)).unwrap() + }; + let a = make_token("10000000-0000-4000-8000-000000000001", "30000000-0000-4000-8000-000000000001"); + let b = make_token("10000000-0000-4000-8000-000000000002", "30000000-0000-4000-8000-000000000001"); + let workspace = make_token("10000000-0000-4000-8000-000000000001", "30000000-0000-4000-8000-000000000002"); + for (credential, expected) in [(None, StatusCode::UNAUTHORIZED), (Some(b), StatusCode::FORBIDDEN), + (Some(workspace), StatusCode::FORBIDDEN), (Some(a.clone()), StatusCode::OK), (Some(format!("{a}x")), StatusCode::UNAUTHORIZED)] { + for (method,path) in [("POST","/companion/v1/enroll"),("GET","/api/v1/room-learner/progress")] { + let router = Router::new().route("/companion/v1/enroll", post(|| async { "accepted" })) + .route("/api/v1/room-learner/progress", get(|| async { "accepted" })) + .layer(axum::middleware::from_fn_with_state(auth.clone(), crate::companion::require_account_ownership)); + let mut request = Request::builder().method(method).uri(path); + if let Some(value) = credential.as_ref() { request = request.header("X-RuView-Account-Authorization", value); } + assert_eq!(router.oneshot(request.body(Body::empty()).unwrap()).await.unwrap().status(), expected); + } + } + } + + #[tokio::test] + async fn owned_helper_cookie_path_rejects_legacy_and_other_owner_but_accepts_its_owner() { + let owner = "a".repeat(64); + for candidate in [None, Some("b".repeat(64)), Some(owner.clone())] { + let mut auth = oauth_only(); auth.owner_account_scope = Some(owner.clone()); + let mut claims = serde_json::json!({"subject":"synthetic-user","account_id":"synthetic-billing", + "scope":scope::SENSING_READ,"exp":chrono::Utc::now().timestamp()+300}); + if let Some(key) = &candidate { claims["owner_account_scope"] = serde_json::json!(key); } + let raw = crate::browser_session::test_sign_for_tests(&serde_json::to_vec(&claims).unwrap()); + let cookie = format!("ruview_session={raw}"); + let status = browser_status(&auth, Some(&cookie)); + let actual = call_with_session(auth, "GET", "/api/v1/models", &cookie).await; + let expected = if candidate.as_deref() == Some(owner.as_str()) { StatusCode::OK } else { StatusCode::UNAUTHORIZED }; + assert_eq!(actual, expected, "cookie ownership mismatch on normal no-bearer request"); + assert_eq!(status["signed_in"], expected == StatusCode::OK, "public status disagrees with cookie authority"); + assert_eq!(status["account"].is_null(), expected != StatusCode::OK); + } + } + // ── browser session cookie as a credential ──────────────────────── // // The session cookie authorizes /api/v1/* and WebSocket upgrades, and had @@ -1536,6 +1659,7 @@ mod oauth_tests { fn oauth_only() -> AuthState { AuthState { + owner_account_scope: None, token: None, oauth: Some(oauth_state()), tickets: crate::ws_ticket::TicketStore::new(), @@ -1673,6 +1797,7 @@ mod oauth_tests { fn both() -> AuthState { AuthState { + owner_account_scope: None, token: Some(Arc::new("legacy-secret".to_string())), oauth: Some(oauth_state()), tickets: crate::ws_ticket::TicketStore::new(), @@ -1801,6 +1926,7 @@ mod ws_gate_tests { fn static_auth() -> AuthState { AuthState { + owner_account_scope: None, token: Some(Arc::new("secret".into())), oauth: None, tickets: crate::ws_ticket::TicketStore::new(), @@ -1986,6 +2112,7 @@ mod scope_gate_polarity_tests { (Method::DELETE, "/api/v1/recording/r1"), (Method::POST, "/api/v1/config/ground-truth"), (Method::POST, "/api/v1/calibration/reset"), + (Method::POST, "/api/v1/calibration/bootstrap/recover"), ] { assert_eq!( required_scope_for(&m, p), diff --git a/v2/crates/wifi-densepose-sensing-server/src/bootstrap_baseline.rs b/v2/crates/wifi-densepose-sensing-server/src/bootstrap_baseline.rs index 326e2f5fa0..b22b789e35 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/bootstrap_baseline.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/bootstrap_baseline.rs @@ -315,6 +315,37 @@ pub fn load( installation_id: &str, current_unix_ms: u64, ) -> Result<(FieldModel, BootstrapBaselineMetadata), BootstrapBaselineError> { + let image = read_image(path, installation_id, current_unix_ms)?; + let model = FieldModel::from_snapshot( + image.payload.field_model.clone(), + current_unix_ms.saturating_mul(1_000), + )?; + Ok((model, metadata(&image))) +} + +/// Read historical expiry diagnostics without restoring an expired model. +/// Identity, size, schema and exact stored-byte integrity checks still apply. +/// This returns metadata only and must never confer inference authority. +pub fn load_expired_metadata( + path: &Path, + installation_id: &str, + current_unix_ms: u64, +) -> Result { + // Zero skips only the deadline check; all other payload checks are shared + // with the model loader. A newly replaced, unexpired file is not accepted + // through this diagnostics-only path. + let image = read_image(path, installation_id, 0)?; + if current_unix_ms <= image.payload.expires_at_unix_ms { + return Err(BootstrapBaselineError::Malformed("reference is not expired".into())); + } + Ok(metadata(&image)) +} + +fn read_image( + path: &Path, + installation_id: &str, + current_unix_ms: u64, +) -> Result { let file_metadata = fs::symlink_metadata(path)?; if file_metadata.file_type().is_symlink() || !file_metadata.is_file() { return Err(BootstrapBaselineError::InvalidPath); @@ -340,15 +371,10 @@ pub fn load( } let payload: BootstrapPayloadV2 = serde_json::from_str(raw_image.payload.get())?; validate_payload(&payload, installation_id, current_unix_ms)?; - let image = BootstrapImageV2 { + Ok(BootstrapImageV2 { payload, content_sha256: raw_image.content_sha256, - }; - let model = FieldModel::from_snapshot( - image.payload.field_model.clone(), - current_unix_ms.saturating_mul(1_000), - )?; - Ok((model, metadata(&image))) + }) } pub fn remove(path: &Path) -> Result { @@ -409,12 +435,12 @@ fn metadata(image: &BootstrapImageV2) -> BootstrapBaselineMetadata { } } -fn hex_sha256(bytes: &[u8]) -> String { +pub(crate) fn hex_sha256(bytes: &[u8]) -> String { let digest = Sha256::digest(bytes); digest.iter().map(|byte| format!("{byte:02x}")).collect() } -fn atomic_write_private(path: &Path, bytes: &[u8]) -> Result<(), BootstrapBaselineError> { +pub(crate) fn atomic_write_private(path: &Path, bytes: &[u8]) -> Result<(), BootstrapBaselineError> { let parent = path.parent().ok_or(BootstrapBaselineError::InvalidPath)?; fs::create_dir_all(parent)?; let file_name = path @@ -501,6 +527,28 @@ mod tests { )); } + #[test] + fn expired_metadata_is_historical_only_and_retains_original_lifetime() { + let dir = tempfile::tempdir().unwrap(); + let path = path_in(dir.path()); + let now_ms = 1_000_000; + let stored = store(&path, "home-a", &[5], TEST_GRID, "model", now_ms, + &completed_model(now_ms * 1_000)).unwrap(); + let original = fs::read(&path).unwrap(); + let expired_at = stored.expires_at_unix_ms + 1; + assert!(matches!(load(&path, "home-a", expired_at), Err(BootstrapBaselineError::Expired))); + assert_eq!(load_expired_metadata(&path, "home-a", expired_at).unwrap(), stored); + assert_eq!(fs::read(&path).unwrap(), original, "diagnostics must not renew or rewrite the reference"); + assert!(load_expired_metadata(&path, "home-a", stored.expires_at_unix_ms).is_err()); + assert!(matches!(load_expired_metadata(&path, "home-b", expired_at), + Err(BootstrapBaselineError::InstallationMismatch))); + let mut image: serde_json::Value = serde_json::from_slice(&original).unwrap(); + image["payload"]["expires_at_unix_ms"] = serde_json::json!(expired_at + 99_999); + fs::write(&path, serde_json::to_vec(&image).unwrap()).unwrap(); + assert!(matches!(load_expired_metadata(&path, "home-a", expired_at), + Err(BootstrapBaselineError::DigestMismatch))); + } + #[test] fn tamper_and_expiry_fail_closed() { let dir = tempfile::tempdir().unwrap(); diff --git a/v2/crates/wifi-densepose-sensing-server/src/browser_session.rs b/v2/crates/wifi-densepose-sensing-server/src/browser_session.rs index 4a0080413b..e2531c7b18 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/browser_session.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/browser_session.rs @@ -163,6 +163,8 @@ struct Transaction { /// server already verified one — nothing replayable elsewhere. #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] pub struct BrowserSession { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub owner_account_scope: Option, pub subject: String, pub account_id: String, pub scope: String, @@ -476,14 +478,19 @@ pub fn verifier_for_callback(cookie_header: &str, state: &str) -> Result Result { + issue_for_account(principal, secure, None) +} + +pub fn issue_for_account(principal: &ruview_auth::Principal, secure: bool, owner_account_scope: Option<&str>) -> Result { let secret = secret()?; let session = BrowserSession { + owner_account_scope: owner_account_scope.map(str::to_owned), subject: principal.subject.clone(), account_id: principal.account_id.clone(), scope: principal.scopes().collect::>().join(" "), // Never outlive our own ceiling, and never inherit the access token's // 15 minutes either — this is a browser session, not the token. - exp: now() + SESSION_TTL_SECS, + exp: if owner_account_scope.is_some() { (now() + SESSION_TTL_SECS).min(principal.expires_at) } else { now() + SESSION_TTL_SECS }, // Stamped at issue, which is the moment a Cognitum token was actually // verified. Never refreshed by activity: it answers "when did you last // prove who you are", not "when were you last here". @@ -550,6 +557,7 @@ pub(crate) fn test_cookie_value_aged( init_secret_for_tests(); let secret = secret().expect("secret installed above"); let session = BrowserSession { + owner_account_scope: None, subject: subject.to_string(), account_id: account_id.to_string(), scope: scope.to_string(), @@ -720,6 +728,7 @@ mod tests { // so only the MAC stands between it and a valid session. let forged = sign( &serde_json::to_vec(&BrowserSession { + owner_account_scope: None, subject: "attacker".into(), account_id: "acct-attacker".into(), scope: "sensing:admin".into(), @@ -869,6 +878,7 @@ mod tests { fn session(exp: i64) -> BrowserSession { BrowserSession { + owner_account_scope: None, subject: "user-1".into(), account_id: "acct-1".into(), scope: "sensing:read".into(), diff --git a/v2/crates/wifi-densepose-sensing-server/src/companion.rs b/v2/crates/wifi-densepose-sensing-server/src/companion.rs new file mode 100644 index 0000000000..7a861f2cfb --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/companion.rs @@ -0,0 +1,986 @@ +//! Owner-approved, QR-pinned companion pairing; durable secrets are hashed. +use axum::{ + extract::{DefaultBodyLimit, Path as RoutePath, Request, State}, + http::StatusCode, + routing::{delete, get, post}, + middleware::Next, response::{IntoResponse, Response}, Json, Router, +}; +use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine}; +use rand::{rngs::OsRng, RngCore}; +use serde::{Deserialize, Serialize}; +use serde_json::{json, Value}; +use sha2::{Digest, Sha256}; +use std::{ + collections::HashMap, + io::{self, Write}, + path::{Path, PathBuf}, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, Mutex, + }, + time::{SystemTime, UNIX_EPOCH}, +}; +use tokio::sync::watch; +use crate::bearer_auth::AuthState; + +#[derive(Clone, Copy)] +pub struct AccountOwnershipExpiry(pub u64); + +pub async fn require_account_ownership(State(auth): State, mut request: Request, next: Next) -> Response { + if auth.owner_account_scope().is_none() { return next.run(request).await; } + let Some(token) = request.headers().get("x-ruview-account-authorization").and_then(|value| value.to_str().ok()) + .filter(|value| value.len() <= 16_384).map(str::to_owned) else { return StatusCode::UNAUTHORIZED.into_response(); }; + // A cold/rotated JWKS cache may fetch. Never block a Tokio worker on it. + let verified = tokio::task::spawn_blocking(move || auth.verify_account_ownership(&token)).await; + match verified { + Ok(Ok(Some(expiry))) => { request.extensions_mut().insert(AccountOwnershipExpiry(expiry)); }, + Ok(Err(status)) => return status.into_response(), + _ => return StatusCode::UNAUTHORIZED.into_response(), + } + next.run(request).await +} + + +pub fn secret() -> String { + let mut b = [0; 32]; + OsRng.fill_bytes(&mut b); + URL_SAFE_NO_PAD.encode(b) +} +fn hash(s: &str) -> String { + format!("{:x}", Sha256::digest(s.as_bytes())) +} +fn now() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs() +} +fn valid_secret(s: &str) -> bool { + s.len() == 43 && URL_SAFE_NO_PAD.decode(s).is_ok_and(|b| b.len() == 32) +} +fn valid_id(s: &str) -> bool { + !s.is_empty() + && s.len() <= 64 + && s.bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'-' || b == b'_') +} +pub(crate) fn private_dir(path: &Path) -> io::Result<()> { + std::fs::create_dir_all(path)?; + if std::fs::symlink_metadata(path)?.file_type().is_symlink() { + return Err(io::Error::other("symlink state directory")); + } + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o700))?; + } + Ok(()) +} +pub(crate) fn private_read(path: &Path) -> io::Result> { + let meta = std::fs::symlink_metadata(path)?; + if !meta.is_file() || meta.len() > 65536 { + return Err(io::Error::other("invalid private state file")); + } + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + if meta.permissions().mode() & 0o077 != 0 { + return Err(io::Error::other("insecure private state permissions")); + } + } + std::fs::read(path) +} +pub(crate) fn atomic_private_write(path:&Path,data:&[u8])->io::Result<()>{atomic_private_write_checked(path,data,&mut||Ok(()),&mut||Ok(()))} +/// One canonical writer body. Original callers use the no-op observer; C4C +/// instance-scoped test observers can exercise the actual post-rename boundary. +pub(crate) fn atomic_private_write_observed(path:&Path,data:&[u8],after_rename:&mut dyn FnMut()->io::Result<()>)->io::Result<()>{atomic_private_write_checked(path,data,&mut||Ok(()),after_rename)} +/// The optional admission callback runs after staging is fully synced and +/// immediately before the sole authoritative replacement. It cannot be delayed +/// by staging IO. A refusal removes only this invocation's own unique temp. +pub(crate) fn atomic_private_write_checked(path:&Path,data:&[u8],before_rename:&mut dyn FnMut()->io::Result<()>,after_rename:&mut dyn FnMut()->io::Result<()>)->io::Result<()>{ + let temp = path.with_extension(format!("tmp-{}", secret())); + let mut options = std::fs::OpenOptions::new(); + options.create_new(true).write(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + let mut f = options.open(&temp)?; + f.write_all(data)?; + f.sync_all()?; + if let Err(error)=before_rename(){ + drop(f); + std::fs::remove_file(&temp)?; + return Err(error); + } + std::fs::rename(&temp, path)?; + after_rename()?; + std::fs::File::open(path.parent().ok_or_else(|| io::Error::other("parent"))?)?.sync_all()?; + Ok(()) +} +#[derive(Clone, Serialize, Deserialize)] +struct Device { + device_id: String, + device_name: String, + pairing_hash: String, + expires_at: u64, +} +#[derive(Serialize, Deserialize)] +struct Disk { + version: u32, + installation_id: String, + devices: Vec, +} +struct Invite { + hash: String, + expires: u64, +} +struct Pending { + device: Device, + code: String, + expires: u64, +} +struct Access { + device_id: String, + expires: u64, +} +struct Inner { + devices: HashMap, + invites: HashMap, + pending: HashMap, + access: HashMap, + calibration: HashMap, + revocations: HashMap>, +} +pub struct Companion { + pub installation_id: String, + pub cert_sha256: String, + pub origin: String, + owner_account_scope: Option, + path: PathBuf, + fault: AtomicBool, + inner: Mutex, +} +pub type Shared = Arc; +pub type ApiResult = Result, StatusCode>; +#[derive(Clone)] +pub struct Grant { + pub device_id: String, + pub expires_at: u64, + pub revoked: watch::Receiver, +} +impl Companion { + pub fn load_for_account(path: &Path, installation_id: String, cert_sha256: String, origin: String, owner_account_scope: Option) -> io::Result { + if owner_account_scope.as_ref().is_some_and(|key| key.len() != 64 || !key.bytes().all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b))) { + return Err(io::Error::other("invalid owner account namespace")); + } + let mut service = Self::load(path, installation_id, cert_sha256, origin)?; + Arc::get_mut(&mut service).ok_or_else(|| io::Error::other("shared companion initialization"))?.owner_account_scope = owner_account_scope; + Ok(service) + } + pub fn load( + dir: &Path, + installation_id: String, + cert_sha256: String, + origin: String, + ) -> io::Result { + private_dir(dir)?; + if !origin.starts_with("https://") { + return Err(io::Error::other("https required")); + } + let path = dir.join("companion-grants.json"); + let devices = if path.exists() { + let disk: Disk = + serde_json::from_slice(&private_read(&path)?).map_err(io::Error::other)?; + if disk.version != 1 + || disk.installation_id != installation_id + || disk.devices.len() > 16 + { + return Err(io::Error::other("invalid grants")); + } + let mut m = HashMap::new(); + for d in disk.devices { + if !valid_id(&d.device_id) + || d.pairing_hash.len() != 64 + || m.insert(d.device_id.clone(), d).is_some() + { + return Err(io::Error::other("invalid device")); + } + } + m + } else { + HashMap::new() + }; + Ok(Arc::new(Self { + installation_id, + cert_sha256, + origin, + owner_account_scope: None, + path, + fault: AtomicBool::new(false), + inner: Mutex::new(Inner { + devices, + invites: HashMap::new(), + pending: HashMap::new(), + access: HashMap::new(), + calibration: HashMap::new(), + revocations: HashMap::new(), + }), + })) + } + fn persist(&self, devices: &HashMap, inner: &Inner) -> Result<(), StatusCode> { + let disk = Disk { + version: 1, + installation_id: self.installation_id.clone(), + devices: devices.values().cloned().collect(), + }; + atomic_private_write( + &self.path, + &serde_json::to_vec(&disk).map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?, + ) + .map_err(|_| { + self.fault.store(true, Ordering::SeqCst); + for tx in inner.revocations.values() { + let _ = tx.send(true); + } + StatusCode::SERVICE_UNAVAILABLE + }) + } + /// Metadata only; sharing never receives durable pairing credentials. + pub fn paired_device(&self, id: &str) -> Option<(String, u64)> { + if self.fault.load(Ordering::SeqCst) { return None; } + let inner = self.inner.lock().ok()?; + let device = inner.devices.get(id)?; + (device.expires_at > now()).then(|| (device.device_name.clone(), device.expires_at)) + } + pub fn device_revocation(&self, id: &str) -> Option> { + if self.fault.load(Ordering::SeqCst) { return None; } + let mut inner = self.inner.lock().ok()?; + if !inner.devices.get(id).is_some_and(|d| d.expires_at > now()) { return None; } + Some(inner.revocations.entry(id.into()).or_insert_with(|| watch::channel(false).0).subscribe()) + } + pub fn authorize( + &self, + installation: &str, + token: &str, + calibration: bool, + ) -> Result { + if self.fault.load(Ordering::SeqCst) { + return Err(StatusCode::SERVICE_UNAVAILABLE); + } + if installation != self.installation_id || !valid_secret(token) { + return Err(StatusCode::UNAUTHORIZED); + } + let mut i = self + .inner + .lock() + .map_err(|_| StatusCode::SERVICE_UNAVAILABLE)?; + let t = now(); + let a = i.access.get(&hash(token)).ok_or(StatusCode::UNAUTHORIZED)?; + let d = i + .devices + .get(&a.device_id) + .ok_or(StatusCode::UNAUTHORIZED)?; + if a.expires <= t || d.expires_at <= t { + return Err(StatusCode::UNAUTHORIZED); + } + if calibration + && i.calibration + .get(&d.device_id) + .copied() + .filter(|until| *until > now()) + .unwrap_or(0) + <= t + { + return Err(StatusCode::FORBIDDEN); + } + let device_id = d.device_id.clone(); + let expires_at = a.expires.min(d.expires_at); + let revoked = i + .revocations + .entry(device_id.clone()) + .or_insert_with(|| watch::channel(false).0) + .subscribe(); + Ok(Grant { + device_id, + expires_at, + revoked, + }) + } +} +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct Enroll { + installation_id: String, + invite_id: String, + invite_secret: String, + device_id: String, + device_name: String, + pairing_secret: String, +} +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct Session { + installation_id: String, + device_id: String, + pairing_secret: String, +} +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct Approve { + comparison_code: String, +} +pub fn public_router(state: Shared) -> Router { + Router::new() + .route("/companion/v1/enroll", post(enroll)) + .route("/companion/v1/session", post(session)) + .layer(DefaultBodyLimit::max(4096)) + .with_state(state) +} +/// Caller must wrap these endpoints in loopback-only owner authentication. +pub fn owner_router(state: Shared) -> Router { + Router::new() + .route("/api/v1/companion", get(snapshot)) + .route("/api/v1/companion/invites", post(invite)) + .route("/api/v1/companion/requests/:id/approve", post(approve)) + .route("/api/v1/companion/devices/:id", delete(revoke)) + .route( + "/api/v1/companion/devices/:id/calibration", + post(calibration), + ) + .layer(DefaultBodyLimit::max(4096)) + .with_state(state) +} +async fn invite(State(s): State) -> ApiResult { + let mut i = s + .inner + .lock() + .map_err(|_| StatusCode::SERVICE_UNAVAILABLE)?; + i.invites.retain(|_, v| v.expires > now()); + if i.invites.len() >= 4 { + return Err(StatusCode::TOO_MANY_REQUESTS); + } + let id = secret(); + let token = secret(); + let expires = now() + 300; + i.invites.insert( + id.clone(), + Invite { + hash: hash(&token), + expires, + }, + ); + Ok(Json( + json!({"schema":"ruview.companion.invite.v1","installation_id":s.installation_id,"cert_sha256":s.cert_sha256,"origin":s.origin,"invite_id":id,"invite_secret":token,"expires_at":expires,"owner_account_scope":s.owner_account_scope}), + )) +} +async fn enroll(State(s): State, Json(r): Json) -> ApiResult { + if r.installation_id != s.installation_id + || !valid_id(&r.device_id) + || !valid_secret(&r.pairing_secret) + || r.device_name.trim().is_empty() + || r.device_name.len() > 80 + || r.device_name.chars().any(char::is_control) + { + return Err(StatusCode::BAD_REQUEST); + } + let mut i = s + .inner + .lock() + .map_err(|_| StatusCode::SERVICE_UNAVAILABLE)?; + let t = now(); + i.pending.retain(|_, p| p.expires > t); + if i.pending.len() >= 8 || i.devices.len() >= 16 { + return Err(StatusCode::TOO_MANY_REQUESTS); + } + if i.devices.contains_key(&r.device_id) + || i.pending + .values() + .any(|p| p.device.device_id == r.device_id) + { + return Err(StatusCode::CONFLICT); + } + let inv = i + .invites + .get(&r.invite_id) + .ok_or(StatusCode::UNAUTHORIZED)?; + if inv.expires <= t || inv.hash != hash(&r.invite_secret) { + return Err(StatusCode::UNAUTHORIZED); + } + i.invites.remove(&r.invite_id); + let request_id = secret(); + let code = format!("{:06}", OsRng.next_u32() % 1_000_000); + i.pending.insert( + request_id.clone(), + Pending { + device: Device { + device_id: r.device_id, + device_name: r.device_name, + pairing_hash: hash(&r.pairing_secret), + expires_at: t + 30 * 86400, + }, + code: code.clone(), + expires: t + 300, + }, + ); + Ok(Json( + json!({"request_id":request_id,"comparison_code":code,"status":"pending"}), + )) +} +async fn approve( + State(s): State, + RoutePath(id): RoutePath, + Json(r): Json, +) -> ApiResult { + let mut i = s + .inner + .lock() + .map_err(|_| StatusCode::SERVICE_UNAVAILABLE)?; + let p = i.pending.get(&id).ok_or(StatusCode::NOT_FOUND)?; + if p.expires <= now() || p.code != r.comparison_code { + return Err(StatusCode::FORBIDDEN); + } + // Pending requests can queue while a slot is available. Recheck under the + // approval lock so another approval cannot exceed the persisted load limit. + if i.devices.len() >= 16 { + return Err(StatusCode::TOO_MANY_REQUESTS); + } + let mut devices = i.devices.clone(); + devices.insert(p.device.device_id.clone(), p.device.clone()); + s.persist(&devices, &i)?; + i.devices = devices; + i.pending.remove(&id); + Ok(Json(json!({"status":"approved"}))) +} +async fn session(State(s): State, Json(r): Json) -> ApiResult { + if s.fault.load(Ordering::SeqCst) { + return Err(StatusCode::SERVICE_UNAVAILABLE); + } + if r.installation_id != s.installation_id || !valid_secret(&r.pairing_secret) { + return Err(StatusCode::UNAUTHORIZED); + } + let mut i = s + .inner + .lock() + .map_err(|_| StatusCode::SERVICE_UNAVAILABLE)?; + let t = now(); + let h = hash(&r.pairing_secret); + if i.pending + .values() + .any(|p| p.device.device_id == r.device_id && p.device.pairing_hash == h && p.expires > t) + { + return Ok(Json(json!({"status":"pending"}))); + } + let d = i + .devices + .get(&r.device_id) + .ok_or(StatusCode::UNAUTHORIZED)?; + if d.pairing_hash != h || d.expires_at <= t { + return Err(StatusCode::UNAUTHORIZED); + } + let expires_at = (t + 12 * 3600).min(d.expires_at); + i.access + .retain(|_, a| a.expires > t && a.device_id != r.device_id); + let token = secret(); + i.access.insert( + hash(&token), + Access { + device_id: r.device_id.clone(), + expires: expires_at, + }, + ); + Ok(Json( + json!({"status":"approved","access_token":token,"expires_at":expires_at,"installation_id":s.installation_id,"device_id":r.device_id,"calibration_until":i.calibration.get(&r.device_id).copied().filter(|until| *until > now()).unwrap_or(0)}), + )) +} +async fn snapshot(State(s): State) -> ApiResult { + let i = s + .inner + .lock() + .map_err(|_| StatusCode::SERVICE_UNAVAILABLE)?; + let requests:Vec<_>=i.pending.iter().filter(|(_,p)|p.expires>now()).map(|(id,p)|json!({"request_id":id,"device_id":p.device.device_id,"device_name":p.device.device_name,"comparison_code":p.code,"expires_at":p.expires})).collect(); + let devices:Vec<_>=i.devices.values().map(|d|json!({"device_id":d.device_id,"device_name":d.device_name,"expires_at":d.expires_at,"calibration_until":i.calibration.get(&d.device_id).copied().filter(|until| *until > now()).unwrap_or(0)})).collect(); + Ok(Json( + json!({"installation_id":s.installation_id,"origin":s.origin,"cert_sha256":s.cert_sha256,"requests":requests,"devices":devices}), + )) +} +async fn revoke(State(s): State, RoutePath(id): RoutePath) -> ApiResult { + let mut i = s + .inner + .lock() + .map_err(|_| StatusCode::SERVICE_UNAVAILABLE)?; + let mut devices = i.devices.clone(); + if devices.remove(&id).is_none() { + return Err(StatusCode::NOT_FOUND); + } + s.persist(&devices, &i)?; + i.devices = devices; + i.access.retain(|_, a| a.device_id != id); + i.calibration.remove(&id); + if let Some(tx) = i.revocations.remove(&id) { + let _ = tx.send(true); + } + Ok(Json(json!({"status":"revoked"}))) +} +async fn calibration(State(s): State, RoutePath(id): RoutePath) -> ApiResult { + let mut i = s + .inner + .lock() + .map_err(|_| StatusCode::SERVICE_UNAVAILABLE)?; + if !i.devices.get(&id).is_some_and(|d| d.expires_at > now()) { + return Err(StatusCode::NOT_FOUND); + } + // An owner grant is an explicit controller handoff, never a second controller. + let until = now() + 900; + i.calibration.clear(); + i.calibration.insert(id, until); + Ok(Json(json!({"calibration_until":until}))) +} +#[cfg(test)] +mod tests { + use super::*; + fn setup() -> (tempfile::TempDir, Shared) { + let d = tempfile::tempdir().unwrap(); + let s = Companion::load( + d.path(), + "installation".into(), + "a".repeat(64), + "https://ruview.local:3443".into(), + ) + .unwrap(); + (d, s) + } + async fn pending(s: &Shared) -> (String, String, String) { + pending_device(s, "phone").await + } + async fn pending_device(s: &Shared, device_id: &str) -> (String, String, String) { + let v = invite(State(s.clone())).await.unwrap().0; + let proof = secret(); + let v = enroll( + State(s.clone()), + Json(Enroll { + installation_id: s.installation_id.clone(), + invite_id: v["invite_id"].as_str().unwrap().into(), + invite_secret: v["invite_secret"].as_str().unwrap().into(), + device_id: device_id.into(), + device_name: "Phone".into(), + pairing_secret: proof.clone(), + }), + ) + .await + .unwrap() + .0; + ( + v["request_id"].as_str().unwrap().into(), + v["comparison_code"].as_str().unwrap().into(), + proof, + ) + } + async fn access(s: &Shared, proof: &str) -> ApiResult { + session( + State(s.clone()), + Json(Session { + installation_id: s.installation_id.clone(), + device_id: "phone".into(), + pairing_secret: proof.into(), + }), + ) + .await + } + #[tokio::test] + async fn queued_approvals_cannot_exceed_durable_device_capacity() { + let (dir, s) = setup(); + for index in 0..15 { + let (id, code, _) = pending_device(&s, &format!("phone-{index}")).await; + let _ = approve( + State(s.clone()), + RoutePath(id), + Json(Approve { + comparison_code: code, + }), + ) + .await + .unwrap(); + } + let (first_id, first_code, _) = pending_device(&s, "last-slot").await; + let (waiting_id, waiting_code, _) = pending_device(&s, "waiting").await; + let _ = approve( + State(s.clone()), + RoutePath(first_id), + Json(Approve { + comparison_code: first_code, + }), + ) + .await + .unwrap(); + let persisted = std::fs::read(&s.path).unwrap(); + assert_eq!( + approve( + State(s.clone()), + RoutePath(waiting_id.clone()), + Json(Approve { + comparison_code: waiting_code.clone(), + }) + ) + .await + .unwrap_err(), + StatusCode::TOO_MANY_REQUESTS, + ); + assert_eq!(std::fs::read(&s.path).unwrap(), persisted); + { + let inner = s.inner.lock().unwrap(); + assert_eq!(inner.devices.len(), 16); + assert!(inner.pending.contains_key(&waiting_id)); + } + let reloaded = Companion::load( + dir.path(), + s.installation_id.clone(), + s.cert_sha256.clone(), + s.origin.clone(), + ) + .unwrap(); + assert_eq!(reloaded.inner.lock().unwrap().devices.len(), 16); + let _ = revoke(State(s.clone()), RoutePath("last-slot".into())) + .await + .unwrap(); + let _ = approve( + State(s.clone()), + RoutePath(waiting_id.clone()), + Json(Approve { + comparison_code: waiting_code, + }), + ) + .await + .unwrap(); + assert!(!s.inner.lock().unwrap().pending.contains_key(&waiting_id)); + let reloaded = Companion::load( + dir.path(), + s.installation_id.clone(), + s.cert_sha256.clone(), + s.origin.clone(), + ) + .unwrap(); + let inner = reloaded.inner.lock().unwrap(); + assert_eq!(inner.devices.len(), 16); + assert!(inner.devices.contains_key("waiting")); + } + + #[tokio::test] + async fn approval_read_scope_calibration_revocation() { + let (_d, s) = setup(); + let (id, code, proof) = pending(&s).await; + assert_eq!(access(&s, &proof).await.unwrap().0["status"], "pending"); + assert!(approve( + State(s.clone()), + RoutePath(id.clone()), + Json(Approve { + comparison_code: "wrong".into() + }) + ) + .await + .is_err()); + approve( + State(s.clone()), + RoutePath(id), + Json(Approve { + comparison_code: code, + }), + ) + .await + .unwrap(); + let v = access(&s, &proof).await.unwrap().0; + let token = v["access_token"].as_str().unwrap(); + let grant = s.authorize("installation", token, false).unwrap(); + assert!(s.authorize("other", token, false).is_err()); + assert!(s.authorize("installation", token, true).is_err()); + calibration(State(s.clone()), RoutePath("phone".into())) + .await + .unwrap(); + assert!(s.authorize("installation", token, true).is_ok()); + revoke(State(s.clone()), RoutePath("phone".into())) + .await + .unwrap(); + assert!(*grant.revoked.borrow()); + assert!(s.authorize("installation", token, false).is_err()); + assert!(access(&s, &proof).await.is_err()); + } + #[tokio::test] + async fn durable_pair_not_access_and_no_plaintext() { + let (d, s) = setup(); + let (id, code, proof) = pending(&s).await; + approve( + State(s.clone()), + RoutePath(id), + Json(Approve { + comparison_code: code, + }), + ) + .await + .unwrap(); + let v = access(&s, &proof).await.unwrap().0; + let token = v["access_token"].as_str().unwrap(); + let disk = std::fs::read_to_string(&s.path).unwrap(); + assert!(!disk.contains(&proof)); + assert!(!disk.contains(token)); + let loaded = Companion::load( + d.path(), + "installation".into(), + "a".repeat(64), + s.origin.clone(), + ) + .unwrap(); + assert!(loaded.authorize("installation", token, false).is_err()); + assert_eq!( + access(&loaded, &proof).await.unwrap().0["status"], + "approved" + ); + assert!( + Companion::load(d.path(), "wrong".into(), "a".repeat(64), s.origin.clone()).is_err() + ); + } + #[tokio::test] + async fn expires_and_cannot_replay_invite() { + let (_d, s) = setup(); + let v = invite(State(s.clone())).await.unwrap().0; + let make = || Enroll { + installation_id: s.installation_id.clone(), + invite_id: v["invite_id"].as_str().unwrap().into(), + invite_secret: v["invite_secret"].as_str().unwrap().into(), + device_id: secret(), + device_name: "Phone".into(), + pairing_secret: secret(), + }; + enroll(State(s.clone()), Json(make())).await.unwrap(); + assert!(enroll(State(s.clone()), Json(make())).await.is_err()); + let (id, code, proof) = pending(&s).await; + s.inner + .lock() + .unwrap() + .pending + .get_mut(&id) + .unwrap() + .expires = 0; + assert!(approve( + State(s.clone()), + RoutePath(id), + Json(Approve { + comparison_code: code + }) + ) + .await + .is_err()); + assert!(access(&s, &proof).await.is_err()); + } + #[tokio::test] + async fn persistence_failure_never_approves() { + let (_d, s) = setup(); + let (id, code, proof) = pending(&s).await; + std::fs::create_dir(&s.path).unwrap(); + assert_eq!( + approve( + State(s.clone()), + RoutePath(id), + Json(Approve { + comparison_code: code + }) + ) + .await + .unwrap_err(), + StatusCode::SERVICE_UNAVAILABLE + ); + assert!(s.inner.lock().unwrap().devices.is_empty()); + assert!(access(&s, &proof).await.is_err()); + } + #[tokio::test] + async fn renewal_invalidates_previous_and_pair_expiry() { + let (_d, s) = setup(); + let (id, code, proof) = pending(&s).await; + approve( + State(s.clone()), + RoutePath(id), + Json(Approve { + comparison_code: code, + }), + ) + .await + .unwrap(); + let first = access(&s, &proof).await.unwrap().0; + let second = access(&s, &proof).await.unwrap().0; + assert!(s + .authorize( + "installation", + first["access_token"].as_str().unwrap(), + false + ) + .is_err()); + assert!(s + .authorize( + "installation", + second["access_token"].as_str().unwrap(), + false + ) + .is_ok()); + s.inner + .lock() + .unwrap() + .devices + .get_mut("phone") + .unwrap() + .expires_at = 0; + assert!(access(&s, &proof).await.is_err()); + assert!(s + .authorize( + "installation", + second["access_token"].as_str().unwrap(), + false + ) + .is_err()); + } + #[tokio::test] + async fn router_rejects_unknown_fields_and_oversized_body() { + use tower::ServiceExt; + let (_d, s) = setup(); + let app = public_router(s); + let req=axum::http::Request::builder().method("POST").uri("/companion/v1/session").header("content-type","application/json").body(axum::body::Body::from(serde_json::to_vec(&json!({"installation_id":"installation","device_id":"phone","pairing_secret":secret(),"admin":true})).unwrap())).unwrap(); + assert_eq!( + app.clone().oneshot(req).await.unwrap().status(), + StatusCode::UNPROCESSABLE_ENTITY + ); + let req = axum::http::Request::builder() + .method("POST") + .uri("/companion/v1/session") + .header("content-type", "application/json") + .body(axum::body::Body::from("x".repeat(5000))) + .unwrap(); + assert_eq!( + app.oneshot(req).await.unwrap().status(), + StatusCode::PAYLOAD_TOO_LARGE + ); + } + #[tokio::test] + async fn invite_bound_and_duplicate_device_cannot_replace_proof() { + let (_d, s) = setup(); + let (id, code, proof) = pending(&s).await; + approve( + State(s.clone()), + RoutePath(id), + Json(Approve { + comparison_code: code, + }), + ) + .await + .unwrap(); + let v = invite(State(s.clone())).await.unwrap().0; + let attempted = enroll( + State(s.clone()), + Json(Enroll { + installation_id: s.installation_id.clone(), + invite_id: v["invite_id"].as_str().unwrap().into(), + invite_secret: v["invite_secret"].as_str().unwrap().into(), + device_id: "phone".into(), + device_name: "attacker".into(), + pairing_secret: secret(), + }), + ) + .await; + assert_eq!(attempted.unwrap_err(), StatusCode::CONFLICT); + assert_eq!(access(&s, &proof).await.unwrap().0["status"], "approved"); + for _ in 0..3 { + invite(State(s.clone())).await.unwrap(); + } + assert_eq!( + invite(State(s)).await.unwrap_err(), + StatusCode::TOO_MANY_REQUESTS + ); + } + #[tokio::test] + async fn calibration_handoff_is_exclusive_and_snapshot_expires() { + let (_dir, s) = setup(); + let (id, code, proof) = pending(&s).await; + approve( + State(s.clone()), + RoutePath(id), + Json(Approve { + comparison_code: code, + }), + ) + .await + .unwrap(); + let first = access(&s, &proof).await.unwrap().0; + let first_token = first["access_token"].as_str().unwrap(); + // A second approved device has its own credential and access token. + let second_proof = secret(); + { + let mut i = s.inner.lock().unwrap(); + i.devices.insert( + "phone-two".into(), + Device { + device_id: "phone-two".into(), + device_name: "Second phone".into(), + pairing_hash: hash(&second_proof), + expires_at: now() + 86400, + }, + ); + } + let second = session( + State(s.clone()), + Json(Session { + installation_id: s.installation_id.clone(), + device_id: "phone-two".into(), + pairing_secret: second_proof, + }), + ) + .await + .unwrap() + .0; + let second_token = second["access_token"].as_str().unwrap(); + calibration(State(s.clone()), RoutePath("phone".into())) + .await + .unwrap(); + assert!(s.authorize("installation", first_token, true).is_ok()); + calibration(State(s.clone()), RoutePath("phone-two".into())) + .await + .unwrap(); + assert!(matches!( + s.authorize("installation", first_token, true), + Err(StatusCode::FORBIDDEN) + )); + assert!(s.authorize("installation", second_token, true).is_ok()); + assert!(s.authorize("installation", first_token, false).is_ok()); + let view = snapshot(State(s.clone())).await.unwrap().0; + let devices = view["devices"].as_array().unwrap(); + assert_eq!( + devices.iter().find(|d| d["device_id"] == "phone").unwrap()["calibration_until"], + 0 + ); + assert!( + devices + .iter() + .find(|d| d["device_id"] == "phone-two") + .unwrap()["calibration_until"] + .as_u64() + .unwrap() + > now() + ); + s.inner + .lock() + .unwrap() + .calibration + .insert("phone-two".into(), now() - 1); + assert!(matches!( + s.authorize("installation", second_token, true), + Err(StatusCode::FORBIDDEN) + )); + let view = snapshot(State(s)).await.unwrap().0; + assert!(view["devices"] + .as_array() + .unwrap() + .iter() + .all(|d| d["calibration_until"] == 0)); + } +} + +#[cfg(test)] +#[path="companion_atomic_write_tests.rs"] +mod atomic_write_tests; diff --git a/v2/crates/wifi-densepose-sensing-server/src/companion_atomic_write_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/companion_atomic_write_tests.rs new file mode 100644 index 0000000000..97d3f12af0 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/companion_atomic_write_tests.rs @@ -0,0 +1,66 @@ +//! Actual canonical writer boundary controls; no model/selection permission is +//! manufactured by these generic byte fixtures. +use super::*; +use std::cell::Cell; +#[test] +fn preinstall_refusal_after_synced_staging_preserves_pointer_and_cleans_own_temp() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("pointer.json"); + atomic_private_write(&path, b"old-choice").unwrap(); + let observed = Cell::new(false); + let result = atomic_private_write_checked( + &path, + b"new-choice", + &mut || { + assert_eq!(std::fs::read(&path)?, b"old-choice"); + let staging = std::fs::read_dir(dir.path())? + .map(|e| e.unwrap().path()) + .filter(|p| p != &path) + .collect::>(); + assert_eq!(staging.len(), 1); + assert_eq!(std::fs::read(&staging[0])?, b"new-choice"); + observed.set(true); + Err(io::Error::other( + "synthetic-current-lease-expired-before-rename", + )) + }, + &mut || panic!("refused staging reached authoritative rename"), + ); + assert!(result.is_err()); + assert!(observed.get()); + assert_eq!(std::fs::read(&path).unwrap(), b"old-choice"); + assert_eq!(std::fs::read_dir(dir.path()).unwrap().count(), 1); +} +#[test] +fn admitted_rename_then_ambiguous_sync_retains_exact_choice_for_cold_readback() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("pointer.json"); + atomic_private_write(&path, b"old-choice").unwrap(); + let admitted = Cell::new(false); + let result = atomic_private_write_checked( + &path, + b"original-prepared-choice", + &mut || { + assert_eq!(std::fs::read(&path)?, b"old-choice"); + admitted.set(true); + Ok(()) + }, + &mut || { + assert!(admitted.get()); + assert_eq!(std::fs::read(&path)?, b"original-prepared-choice"); + Err(io::Error::other( + "synthetic-directory-sync-failure-after-valid-rename", + )) + }, + ); + assert!(result.is_err()); + assert_eq!(std::fs::read(&path).unwrap(), b"original-prepared-choice"); + assert_eq!(std::fs::read_dir(dir.path()).unwrap().count(), 1); +} +#[test] +fn original_noop_wrapper_keeps_private_exact_bytes() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("pointer.json"); + atomic_private_write(&path, b"unchanged-canonical-writer").unwrap(); + assert_eq!(private_read(&path).unwrap(), b"unchanged-canonical-writer"); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/companion_integration.rs b/v2/crates/wifi-densepose-sensing-server/src/companion_integration.rs new file mode 100644 index 0000000000..fa5183b5cd --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/companion_integration.rs @@ -0,0 +1,465 @@ +//! Dedicated TLS authority: never infer trust from headers or LAN reachability. +use super::*; +use axum::{ + extract::{ConnectInfo, Request}, + middleware::Next, + response::Response, +}; +use wifi_densepose_sensing_server::{ + bearer_auth::AuthState, + companion::{self, Companion, Grant}, + companion_tls, companion_sensors, +}; + +// Share the QR authority with Host validation; DNS-SD names retain their terminal dot. +fn companion_authority(identity: &companion_tls::Identity, port: u16) -> String { + let hostname = + wifi_densepose_sensing_server::discovery::discovery_hostname(&identity.installation_id); + format!("{}:{port}", hostname.trim_end_matches('.')) +} + +pub fn host_allowlist( + args: &Args, + companion_authority: Option<&str>, +) -> wifi_densepose_sensing_server::host_validation::HostAllowlist { + let discovery_label = args.installation_id.as_deref().unwrap_or("installation"); + let discovery_host = + wifi_densepose_sensing_server::discovery::discovery_hostname(discovery_label); + wifi_densepose_sensing_server::host_validation::HostAllowlist::from_cli_and_env( + args.allowed_hosts + .iter() + .map(String::as_str) + .chain(std::iter::once(discovery_host.as_str())) + .chain(companion_authority), + ) +} +pub struct Integration { + service: Arc, + authority: String, + auth: AuthState, + listener: tokio::net::TcpListener, + tls: Arc, + room: Arc, + control: Control, + sensors: companion_sensors::Shared, + spatial: Option>, + advertiser: Option, +} +impl Integration { + pub async fn prepare( + args: &Args, + state: &SharedState, + auth: AuthState, + ) -> std::io::Result> { + let Some(port) = args.companion_port else { + return Ok(None); + }; + if port == 0 + || port == args.http_port + || port == args.ws_port + || !auth.static_token_enabled() + { + return Err(std::io::Error::other( + "companion requires a dedicated port and owner credential", + )); + } + let dir = args.data_dir.join("companion"); + let identity = + companion_tls::Identity::load_or_create(&dir, args.installation_id.as_deref())?; + let hostname = + wifi_densepose_sensing_server::discovery::discovery_hostname(&identity.installation_id); + let authority = companion_authority(&identity, port); + let origin = format!("https://{authority}"); + let service = Companion::load_for_account( + &dir, + identity.installation_id.clone(), + identity.cert_sha256, + origin, + auth.owner_account_scope().map(str::to_owned), + )?; + state.write().await.installation_id = Some(identity.installation_id.clone()); + let ip: std::net::IpAddr = args.bind_addr.parse().map_err(std::io::Error::other)?; + let listener = tokio::net::TcpListener::bind(SocketAddr::from((ip, port))).await?; + let advertiser = if args.no_mdns || ip.is_loopback() { + None + } else { + Some( + wifi_densepose_sensing_server::discovery::start_tls_advertiser( + &format!("{} Companion", args.installation_name), + &identity.installation_id, + &hostname, + port, + ) + .map_err(std::io::Error::other)?, + ) + }; + let room = companion_room::SharedRoomStore::load(&dir, &identity.installation_id)?; + state.write().await.current_room_store = Some(room.clone()); + let control = Control { + state: state.clone(), + serial: Arc::new(tokio::sync::Mutex::new(())), + }; + let sensors = companion_sensors::SensorStore::new(service.clone()); + let spatial = auth.owner_account_scope().map(|scope| companion_spatial::SpatialStore::load(&dir, scope, &identity.installation_id, service.clone(), room.clone())).transpose()?; + Ok(Some(Self { + service, + authority, + auth, + listener, + tls: identity.config, + advertiser, + room, + control, + sensors, + spatial, + })) + } + pub fn authority(&self) -> &str { + &self.authority + } + pub fn owner_router(&self) -> Router { + companion::owner_router(self.service.clone()) + .merge(companion_sensors::owner_router(self.sensors.clone())) + .merge(companion_room::owner_router(self.room.clone())) + .merge(self.spatial.as_ref().map(|s| companion_spatial::owner_router(s.clone())).unwrap_or_default()) + .layer(axum::middleware::from_fn_with_state( + self.control.clone(), + serialize_control, + )) + .layer(axum::middleware::from_fn_with_state( + self.auth.clone(), + require_owner, + )) + } + pub fn protect_control(&self, app: Router) -> Router { + app.layer(Extension(self.room.clone())) + .layer(axum::middleware::from_fn_with_state( + self.control.clone(), + serialize_control, + )) + } + pub async fn start(self, app: Router) { + let app = attach_peer_artifacts( + app.merge(companion_sensors::companion_router(self.sensors.clone())) + .merge(companion_room::read_router(self.room.clone())), + self.spatial.as_ref().map(|s| companion_spatial::peer_router(s.clone())).unwrap_or_default(), + self.control.clone(), + ) + .layer(axum::middleware::from_fn_with_state( + self.service.clone(), + require_companion, + )) + .merge(companion::public_router(self.service)) + .layer(axum::middleware::from_fn_with_state(self.auth, companion::require_account_ownership)) + .layer(axum::extract::DefaultBodyLimit::max(64 * 1024)); + tokio::spawn(async move { + let _advertiser = self.advertiser; + if let Err(error) = companion_tls::serve(self.listener, self.tls, app).await { + error!("Companion listener stopped: {error}"); + } + }); + } +} +// Existing calibration routes already use protect_control. Layering their +// router a second time would reacquire the same non-reentrant mutex. +fn attach_peer_artifacts(app: Router, artifacts: Router, control: Control) -> Router { + app.merge(artifacts.layer(axum::middleware::from_fn_with_state(control, serialize_control))) +} + +#[derive(Clone)] +struct Control { + state: SharedState, + serial: Arc>, +} +async fn serialize_control( + State(control): State, + request: Request, + next: Next, +) -> Response { + let publish = request.method() == axum::http::Method::PUT + && request.uri().path() == "/api/v1/companion/room"; + let start = request.method() == axum::http::Method::POST + && matches!( + request.uri().path(), + "/api/v1/calibration/start" | "/api/v1/pose/calibrate" + ); + let artifact = request.method() == axum::http::Method::POST && matches!(request.uri().path(), "/api/v1/companion/spatial" | "/companion/v1/spatial"); + if !publish && !start && !artifact { + return next.run(request).await; + } + // Do not let a permitted phone hold the shared calibration lock while + // streaming an unfinished body. Bound and time-limit it before locking. + let request = if artifact { + let (parts, body) = request.into_parts(); + let bytes = match tokio::time::timeout(std::time::Duration::from_secs(5), axum::body::to_bytes(body, 64 * 1024)).await { + Ok(Ok(bytes)) => bytes, + Ok(Err(_)) => return StatusCode::PAYLOAD_TOO_LARGE.into_response(), + Err(_) => return StatusCode::REQUEST_TIMEOUT.into_response(), + }; + Request::from_parts(parts, axum::body::Body::from(bytes)) + } else { request }; + let _guard = control.serial.lock().await; + if publish { + let state = control.state.read().await; + let now = chrono::Utc::now().timestamp_millis().max(0) as u64; + if state.field_model_status_at(now).is_some_and(|status| { + matches!( + status, + CalibrationStatus::Collecting | CalibrationStatus::Uncalibrated + ) + }) { + return ( + StatusCode::CONFLICT, + "Calibration is active; stop it before publishing another room revision", + ) + .into_response(); + } + } + next.run(request).await +} + +fn bearer(request: &Request) -> Option<&str> { + let (scheme, token) = request + .headers() + .get(axum::http::header::AUTHORIZATION)? + .to_str() + .ok()? + .split_once(' ')?; + if !scheme.eq_ignore_ascii_case("Bearer") { + return None; + }; + Some(token) +} +pub(super) async fn require_owner(State(auth): State, request: Request, next: Next) -> Response { + let local = request + .extensions() + .get::>() + .is_some_and(|p| p.0.ip().is_loopback()); + if !local || !bearer(&request).is_some_and(|t| auth.is_owner_token(t)) { + return StatusCode::FORBIDDEN.into_response(); + } + next.run(request).await +} +fn policy(method: &axum::http::Method, path: &str) -> Option { + use axum::http::Method; + if *method == Method::GET || *method == Method::HEAD { + if matches!( + path, + "/health" + | "/api/v1/status" + | "/api/v1/info" + | "/api/v1/sensing/latest" + | "/api/v1/nodes" + | "/api/v1/stream/status" + | "/api/v1/vital-signs" + | "/api/v1/edge-vitals" + | "/api/v1/pose/current" + | "/api/v1/pose/stats" + | "/api/v1/pose/zones/summary" + | "/api/v1/pose/activities" + | "/api/v1/room-learner/selection" + | "/api/v1/room-learner/progress" + | "/api/v1/worldgraph/snapshot" + | "/api/v1/calibration/status" + | "/api/v1/pose/calibration/status" + | "/ws/sensing" + | "/companion/v1/room" + | "/companion/v1/sensors" + | "/companion/v1/spatial" + ) { + return Some(false); + } + } + if *method == Method::POST && path == "/companion/v1/spatial" { return Some(false); } + if *method == Method::POST + && matches!( + path, + "/api/v1/calibration/start" + | "/api/v1/calibration/stop" + | "/api/v1/calibration/cancel" + | "/api/v1/calibration/bootstrap/promote" + | "/api/v1/calibration/bootstrap/refine" + ) + { + return Some(true); + } + None +} +async fn require_companion( + State(service): State>, + mut request: Request, + next: Next, +) -> Response { + let Some(calibration) = policy(request.method(), request.uri().path()) else { + return StatusCode::FORBIDDEN.into_response(); + }; + // Ticket and credential query parameters are never accepted on this listener. + if request.uri().query().is_some_and(|q| { + q.split('&').any(|part| { + matches!( + part.split('=').next(), + Some("ticket" | "token" | "access_token") + ) + }) + }) { + return StatusCode::BAD_REQUEST.into_response(); + } + let installation = request + .headers() + .get("x-ruview-installation-id") + .and_then(|v| v.to_str().ok()) + .unwrap_or(""); + let mut grant = match service.authorize(installation, bearer(&request).unwrap_or(""), calibration) { + Ok(grant) => grant, + Err(status) => return status.into_response(), + }; + if let Some(companion::AccountOwnershipExpiry(expiry)) = request.extensions().get::() { + grant.expires_at = grant.expires_at.min(*expiry); + } + request.extensions_mut().insert(grant); + next.run(request).await +} +pub async fn guard_stream(grant: Grant, stream: impl std::future::Future) { + let mut revoked = grant.revoked; + if *revoked.borrow() { + return; + } + let now = chrono::Utc::now().timestamp().max(0) as u64; + if grant.expires_at <= now { + return; + } + tokio::select! { + _ = stream => {}, + _ = revoked.changed() => {}, + _ = tokio::time::sleep(Duration::from_secs(grant.expires_at-now)) => {}, + } +} +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn qr_authority_is_allowed_for_explicit_and_persisted_generated_identity() { + for preferred in [Some("standalone-installation"), None] { + let dir = tempfile::tempdir().unwrap(); + let identity = companion_tls::Identity::load_or_create(dir.path(), preferred).unwrap(); + let reloaded = companion_tls::Identity::load_or_create(dir.path(), preferred).unwrap(); + let mut argv = vec!["sensing-server"]; + if let Some(id) = preferred { + argv.extend(["--installation-id", id]); + } + let args = Args::try_parse_from(argv).unwrap(); + assert!(args.allowed_hosts.is_empty()); + for prepared in [&identity, &reloaded] { + let authority = companion_authority(prepared, 3443); + let origin = format!("https://{authority}"); + let parsed: axum::http::Uri = origin.parse().unwrap(); + let allowlist = host_allowlist(&args, Some(&authority)); + assert!(allowlist.is_enabled()); + assert!(allowlist.is_allowed(parsed.authority().unwrap().as_str())); + assert!(!allowlist.is_allowed("untrusted.example:3443")); + assert!(!allowlist.is_allowed(&format!("{authority}.untrusted.example"))); + // Only the exact QR authority is added, without allowing arbitrary ports. + assert!(!allowlist.is_allowed(&authority.replace(":3443", ":3444"))); + } + assert_eq!( + companion_authority(&identity, 3443), + companion_authority(&reloaded, 3443) + ); + } + } + + #[test] + fn device_policy_is_not_the_legacy_read_policy() { + use axum::http::Method; + assert_eq!(policy(&Method::GET, "/api/v1/sensing/latest"), Some(false)); + assert_eq!( + policy(&Method::POST, "/api/v1/calibration/start"), + Some(true) + ); + for path in [ + "/api/v1/ws-ticket", + "/api/v1/models/load", + "/api/v1/recording/start", + "/api/v1/calibration/reset", + "/api/v1/companion/invites", + ] { + assert_eq!(policy(&Method::POST, path), None, "{path}"); + } + assert_eq!(policy(&Method::GET, "/ws/train/progress"), None); + assert_eq!(policy(&Method::GET, "/api/v1/room-learner/selection"), Some(false)); + assert_eq!(policy(&Method::GET, "/api/v1/worldgraph/snapshot"), Some(false)); + assert_eq!(policy(&Method::POST, "/api/v1/worldgraph/snapshot"), None); + assert_eq!(policy(&Method::HEAD, "/api/v1/room-learner/selection"), Some(false)); + assert_eq!(policy(&Method::GET, "/api/v1/room-learner/status"), None); + assert_eq!(policy(&Method::GET, "/api/v1/room-learner/progress"), Some(false)); + assert_eq!(policy(&Method::HEAD, "/api/v1/room-learner/progress"), Some(false)); + assert_eq!(policy(&Method::POST, "/api/v1/room-learner/progress"), None); + for path in ["/api/v1/room-learner/recovery/enable-current", "/api/v1/room-learner/qualify", "/api/v1/room-learner/select", "/api/v1/room-learner/deselect", "/api/v1/room-learner/phase/cancel", "/api/v1/room-learner/train/cancel"] { + assert_eq!(policy(&Method::POST, path), None, "paired read/calibration grants cannot select: {path}"); + } + + } + #[tokio::test] + async fn peer_composition_never_takes_the_calibration_lock_twice() { + use tower::ServiceExt; + use axum::body::Body; + let control=Control{state:Arc::new(RwLock::new(AppStateInner::minimal())),serial:Arc::new(tokio::sync::Mutex::new(()))}; + let existing=Router::new().route("/api/v1/calibration/start",post(||async{StatusCode::OK})) + .layer(axum::middleware::from_fn_with_state(control.clone(),serialize_control)); + let artifacts=Router::new().route("/companion/v1/spatial",post(||async{StatusCode::OK})); + let app=attach_peer_artifacts(existing,artifacts,control); + for path in ["/api/v1/calibration/start","/companion/v1/spatial"] { + let request=Request::builder().method("POST").uri(path).body(Body::from("{}")).unwrap(); + let response=tokio::time::timeout(std::time::Duration::from_secs(1),app.clone().oneshot(request)).await.expect("nested calibration lock").unwrap(); + assert_eq!(response.status(),StatusCode::OK); + } + } + #[tokio::test] + async fn unfinished_spatial_upload_does_not_hold_calibration_lock() { + use tower::ServiceExt; + use axum::body::Body; + let serial=Arc::new(tokio::sync::Mutex::new(())); + let control=Control{state:Arc::new(RwLock::new(AppStateInner::minimal())),serial:serial.clone()}; + let router=Router::new().route("/companion/v1/spatial",post(||async{StatusCode::OK})) + .layer(axum::middleware::from_fn_with_state(control,serialize_control)); + let stream=futures_util::stream::pending::>(); + let request=Request::builder().method("POST").uri("/companion/v1/spatial").body(Body::from_stream(stream)).unwrap(); + let pending=tokio::spawn(router.clone().oneshot(request)); + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + assert!(serial.try_lock().is_ok()); + assert_eq!(pending.await.unwrap().unwrap().status(),StatusCode::REQUEST_TIMEOUT); + let huge=Request::builder().method("POST").uri("/companion/v1/spatial").body(Body::from(vec![b'x';65537])).unwrap(); + assert_eq!(router.oneshot(huge).await.unwrap().status(),StatusCode::PAYLOAD_TOO_LARGE); + } + #[tokio::test] + async fn sensor_routes_preserve_owner_peer_and_companion_isolation() { + use tower::ServiceExt; + use axum::{body::Body, http::{Request, Method}}; + let dir = tempfile::tempdir().unwrap(); + let service = Companion::load(dir.path(), "test".into(), "a".repeat(64), "https://test.local:3443".into()).unwrap(); + let sensors = companion_sensors::SensorStore::new(service.clone()); + let owner = companion_sensors::owner_router(sensors).layer(axum::middleware::from_fn_with_state(AuthState::from_token("owner"), require_owner)); + for (peer, credential, expected) in [("127.0.0.1:1", "owner", 200), ("127.0.0.1:1", "phone", 403), ("10.0.0.1:1", "owner", 403)] { + let mut req=Request::builder().uri("/api/v1/companion/sensors").header("authorization",format!("Bearer {credential}")).body(Body::empty()).unwrap(); + req.extensions_mut().insert(ConnectInfo(peer.parse::().unwrap())); + assert_eq!(owner.clone().oneshot(req).await.unwrap().status().as_u16(), expected); + } + assert_eq!(policy(&Method::GET,"/companion/v1/sensors"),Some(false)); + assert_eq!(policy(&Method::GET,"/companion/v1/spatial"),Some(false)); + assert_eq!(policy(&Method::POST,"/companion/v1/spatial"),Some(false)); + assert_eq!(policy(&Method::PUT,"/companion/v1/spatial"),None); + assert_eq!(policy(&Method::GET,"/api/v1/companion/spatial"),None); + assert_eq!(policy(&Method::POST,"/api/v1/companion/spatial"),None); + for path in ["/api/v1/companion/sensors", "/api/v1/companion/devices/phone/sensors"] { + assert_eq!(policy(&Method::GET,path),None); assert_eq!(policy(&Method::POST,path),None); assert_eq!(policy(&Method::DELETE,path),None); + } + let denied=Router::new().route("/companion/v1/sensors",get(||async {StatusCode::OK})).layer(axum::middleware::from_fn_with_state(service,require_companion)); + let response=denied.oneshot(Request::builder().uri("/companion/v1/sensors").body(Body::empty()).unwrap()).await.unwrap(); + assert_eq!(response.status(),StatusCode::UNAUTHORIZED); + } + +} + +#[cfg(test)] +#[path = "paired_progress_auth_tests.rs"] +mod paired_progress_auth_tests; diff --git a/v2/crates/wifi-densepose-sensing-server/src/companion_room.rs b/v2/crates/wifi-densepose-sensing-server/src/companion_room.rs new file mode 100644 index 0000000000..66f9d9d73a --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/companion_room.rs @@ -0,0 +1,433 @@ +//! Installation-owned room publication. Authentication is supplied by the caller. +use axum::{extract::State, http::StatusCode, routing::get, Json, Router}; +use serde::{Deserialize, Serialize}; +use serde_json::{json, Value}; +use sha2::{Digest, Sha256}; +#[cfg(unix)] +use std::os::unix::fs::{OpenOptionsExt, PermissionsExt}; +use std::{ + collections::HashSet, + fs::{self, OpenOptions}, + io::Write, + path::{Path, PathBuf}, + sync::{Arc, Mutex}, +}; +const MAX_SAFE: u64 = 9_007_199_254_740_991; +const SLOTS: &[&str] = &[ + "north_west", + "north_center", + "north_east", + "west_center", + "east_center", + "south_west", + "south_center", + "south_east", +]; +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Envelope { + pub installation_id: String, + pub revision: u64, + pub room: Option, + pub binding_digest: Option, +} +pub struct SharedRoomStore { + path: PathBuf, + current: Mutex, + worldgraph_cursor: Mutex, +} +fn invalid() -> std::io::Error { + std::io::Error::new(std::io::ErrorKind::InvalidData, "invalid shared room state") +} +fn keys(v: &Value, allowed: &[&str]) -> bool { + v.as_object() + .is_some_and(|o| o.keys().all(|k| allowed.contains(&k.as_str()))) +} +fn text(v: &Value, min: usize, max: usize) -> bool { + v.as_str().is_some_and(|s| { + (min..=max).contains(&s.encode_utf16().count()) + && s.chars().all(|c| c >= ' ' && c != '\u{7f}') + }) +} +fn number(v: &Value, min: f64, max: f64, scale: f64) -> bool { + v.as_f64().is_some_and(|n| { + n.is_finite() && n >= min && n <= max && (n * scale - (n * scale).round()).abs() < 1e-8 + }) +} +fn position(v: &Value, width: f64, length: f64, slots: &HashSet) -> bool { + v["slot"] + .as_str() + .is_some_and(|s| SLOTS.contains(&s) && !slots.contains(s)) + && number(&v["xM"], 0., width, 1000.) + && number(&v["yM"], 0., length, 1000.) + && v["confirmedAtUnixMs"] + .as_u64() + .is_some_and(|n| n > 0 && n <= MAX_SAFE) +} +fn validate(mut r: Value) -> Result { + if !keys( + &r, + &[ + "schema", + "id", + "name", + "widthM", + "lengthM", + "heightM", + "nodes", + "accessPoint", + ], + ) || r["schema"] != "ruview.companion.room.v1" + || !text(&r["id"], 4, 96) + || !text(&r["name"], 1, 48) + || !number(&r["widthM"], 1.5, 30., 1000.) + || !number(&r["lengthM"], 1.5, 30., 1000.) + || !number(&r["heightM"], 1.8, 6., 1000.) + || r.get("accessPoint").is_none() + { + return Err(invalid()); + } + let name = r["name"].as_str().unwrap(); + if name.split_whitespace().collect::>().join(" ") != name { + return Err(invalid()); + } + let width = r["widthM"].as_f64().unwrap(); + let length = r["lengthM"].as_f64().unwrap(); + let nodes = r["nodes"].as_array().ok_or_else(invalid)?; + if nodes.is_empty() || nodes.len() > 16 { + return Err(invalid()); + } + let mut ids = HashSet::new(); + let mut slots = HashSet::new(); + let mut base = false; + let mut receiver = false; + for n in nodes { + if !keys( + n, + &[ + "nodeId", + "slot", + "xM", + "yM", + "role", + "headingDeg", + "orientationConfirmed", + "confirmedAtUnixMs", + ], + ) || !number(&n["nodeId"], 0., 255., 1.) + || !position(n, width, length, &slots) + || n["orientationConfirmed"] != true + || !number(&n["headingDeg"], 0., 360., 10.) + { + return Err(invalid()); + } + let id = n["nodeId"].as_f64().unwrap() as u8; + if !ids.insert(id) { + return Err(invalid()); + } + slots.insert(n["slot"].as_str().unwrap().to_string()); + match n["role"].as_str() { + Some("receiver") => receiver = true, + Some("base_station") => base = true, + _ => return Err(invalid()), + } + } + let ap = &r["accessPoint"]; + if !ap.is_null() + && (!keys(ap, &["slot", "xM", "yM", "provenance", "confirmedAtUnixMs"]) + || !position(ap, width, length, &slots) + || ap["provenance"] != "operator_entered") + { + return Err(invalid()); + } + if (receiver && ap.is_null() && !base) || (base && !receiver) { + return Err(invalid()); + } + r["nodes"] + .as_array_mut() + .unwrap() + .sort_by_key(|n| n["nodeId"].as_f64().unwrap() as u8); + Ok(r) +} +fn scaled(v: &Value, scale: f64) -> i64 { + (v.as_f64().unwrap() * scale).round() as i64 +} +fn digest(room: &Value, installation: &str, revision: u64) -> String { + let nodes: Vec = room["nodes"] + .as_array() + .unwrap() + .iter() + .map(|n| { + json!([ + scaled(&n["nodeId"], 1.), + n["slot"], + scaled(&n["xM"], 1000.), + scaled(&n["yM"], 1000.), + n["role"], + scaled(&n["headingDeg"], 10.) + ]) + }) + .collect(); + let ap = &room["accessPoint"]; + let ap = if ap.is_null() { + Value::Null + } else { + json!([ + ap["slot"], + scaled(&ap["xM"], 1000.), + scaled(&ap["yM"], 1000.), + "operator_entered" + ]) + }; + let canonical = json!([ + "ruview.room-calibration-binding.v2", + installation, + revision, + room["id"], + [ + scaled(&room["widthM"], 1000.), + scaled(&room["lengthM"], 1000.), + scaled(&room["heightM"], 1000.) + ], + nodes, + ap + ]); + format!( + "{:x}", + Sha256::digest(serde_json::to_vec(&canonical).unwrap()) + ) +} +impl SharedRoomStore { + pub fn load(dir: &Path, installation_id: &str) -> std::io::Result> { + if !text(&json!(installation_id), 1, 128) { + return Err(invalid()); + } + fs::create_dir_all(dir)?; + let path = dir.join("companion-room.json"); + let current = match fs::symlink_metadata(&path) { + Ok(meta) => { + if !meta.file_type().is_file() || meta.len() > 65536 { + return Err(invalid()); + } + #[cfg(unix)] + if meta.permissions().mode() & 0o077 != 0 { + return Err(invalid()); + } + let e: Envelope = + serde_json::from_slice(&fs::read(&path)?).map_err(|_| invalid())?; + if e.installation_id != installation_id || e.revision == 0 || e.revision > MAX_SAFE + { + return Err(invalid()); + } + let room = validate(e.room.clone().ok_or_else(invalid)?)?; + if e.binding_digest.as_deref() + != Some(digest(&room, installation_id, e.revision).as_str()) + { + return Err(invalid()); + } + Envelope { + room: Some(room), + ..e + } + } + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Envelope { + installation_id: installation_id.into(), + revision: 0, + room: None, + binding_digest: None, + }, + Err(e) => return Err(e), + }; + Ok(Arc::new(Self { + path, + current: Mutex::new(current), + worldgraph_cursor: Mutex::new(crate::managed_worldgraph::Cursor::default()), + })) + } + /// Lock publication then cursor: no mixed binding/geometry response under CAS. + pub fn worldgraph_snapshot(&self, privacy_denied: bool) -> Result { + let room = self.current.lock().map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + self.worldgraph_cursor.lock().map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?.project(&room, privacy_denied) + } + + pub fn snapshot(&self) -> Option { self.current.lock().ok().map(|v| v.clone()) } + pub fn matches_binding(&self, binding: &str, node_ids: &[u8]) -> bool { + let Ok(e) = self.current.lock() else { + return false; + }; + let Some(room) = &e.room else { return false }; + let ids: HashSet = node_ids.iter().copied().collect(); + let expected: HashSet = room["nodes"] + .as_array() + .unwrap() + .iter() + .map(|n| scaled(&n["nodeId"], 1.) as u8) + .collect(); + e.binding_digest.as_deref() == Some(binding) + && ids.len() == node_ids.len() + && ids == expected + } + /// Read the current geometry binding without publishing or advancing its revision. + /// Model selection is separate metadata and must never mutate room geometry. + /// Hold the read binding stable across a guarded model-selection operation. + /// Lock order is app state -> shared room -> learner; no callback may publish + /// geometry or reacquire this store. + pub fn with_current_binding(&self, operation: impl FnOnce(Option<&str>) -> T) -> Result { + let room = self.current.lock().map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + Ok(operation(room.binding_digest.as_deref())) + } + + pub fn current_binding_digest(&self) -> Result, StatusCode> { + self.current.lock().map(|room| room.binding_digest.clone()) + .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR) + } + + fn publish(&self, expected: u64, room: Value) -> Result { + let room = validate(room).map_err(|_| StatusCode::BAD_REQUEST)?; + let mut current = self + .current + .lock() + .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + if current.revision != expected { + return Err(StatusCode::CONFLICT); + } + if expected >= MAX_SAFE { + return Err(StatusCode::CONFLICT); + } + let next = Envelope { + installation_id: current.installation_id.clone(), + revision: expected + 1, + binding_digest: Some(digest(&room, ¤t.installation_id, expected + 1)), + room: Some(room), + }; + let persist = || -> std::io::Result<()> { + let tmp = self + .path + .with_extension(format!("{}.tmp", rand::random::())); + let mut options = OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + options.mode(0o600); + let mut file = options.open(&tmp)?; + file.write_all(&serde_json::to_vec(&next)?)?; + file.sync_all()?; + fs::rename(&tmp, &self.path)?; + fs::File::open(self.path.parent().unwrap())?.sync_all()?; + Ok(()) + }; + persist().map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + *current = next.clone(); + Ok(next) + } +} +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct Publication { + expected_revision: u64, + room: Value, +} +async fn read(State(s): State>) -> Result, StatusCode> { + Ok(Json( + s.current + .lock() + .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)? + .clone(), + )) +} +async fn publish( + State(s): State>, + Json(p): Json, +) -> Result, StatusCode> { + s.publish(p.expected_revision, p.room).map(Json) +} +pub fn owner_router(store: Arc) -> Router { + Router::new() + .route("/api/v1/companion/room", get(read).put(publish)) + .with_state(store) +} +pub fn read_router(store: Arc) -> Router { + Router::new() + .route("/companion/v1/room", get(read)) + .with_state(store) +} +#[cfg(test)] +mod tests { + use super::*; + fn room() -> Value { + json!({"schema":"ruview.companion.room.v1","id":"room-1","name":"Living room","widthM":2.5,"lengthM":2.5,"heightM":2.4,"nodes":[{"nodeId":17,"slot":"north_west","xM":0,"yM":0,"role":"receiver","headingDeg":90,"orientationConfirmed":true,"confirmedAtUnixMs":1}],"accessPoint":{"slot":"south_east","xM":2.5,"yM":2.5,"provenance":"operator_entered","confirmedAtUnixMs":1}}) + } + #[test] + fn cross_language_fixture() { + let mut r = room(); + r["widthM"] = json!(5); + r["lengthM"] = json!(6); + r["heightM"] = json!(2.6); + r["nodes"][0]["nodeId"] = json!(1); + r["nodes"][0]["xM"] = json!(0.45); + r["nodes"][0]["yM"] = json!(0.45); + r["accessPoint"]["slot"] = json!("north_east"); + r["accessPoint"]["xM"] = json!(4.55); + r["accessPoint"]["yM"] = json!(0.45); + let r = validate(r).unwrap(); + assert_eq!( + digest(&r, "installation-1", 1), + "4b9276991f7d676f220ea053c3474e120f52698682c246cc92f2c66109a6be28" + ); + } + #[test] + fn persistence_cas_and_binding_are_installation_scoped() { + let dir = tempfile::tempdir().unwrap(); + let s = SharedRoomStore::load(dir.path(), "install-1").unwrap(); + assert_eq!(s.current.lock().unwrap().revision, 0); + let e = s.publish(0, room()).unwrap(); + assert!(s.matches_binding(e.binding_digest.as_ref().unwrap(), &[17])); + assert!(!s.matches_binding(e.binding_digest.as_ref().unwrap(), &[17, 17])); + assert!(!s.matches_binding(e.binding_digest.as_ref().unwrap(), &[18])); + assert!(matches!(s.publish(0, room()), Err(StatusCode::CONFLICT))); + let restored = SharedRoomStore::load(dir.path(), "install-1").unwrap(); + assert!(restored.matches_binding(e.binding_digest.as_ref().unwrap(), &[17])); + assert!(SharedRoomStore::load(dir.path(), "other").is_err()); + } + #[test] + fn reading_current_binding_never_publishes_a_new_geometry_revision() { + let dir = tempfile::tempdir().unwrap(); + let store = SharedRoomStore::load(dir.path(), "install-1").unwrap(); + assert_eq!(store.current_binding_digest().unwrap(), None); + let published = store.publish(0, room()).unwrap(); + let before = fs::read(&store.path).unwrap(); + for _ in 0..3 { + assert_eq!(store.current_binding_digest().unwrap(), published.binding_digest); + } + assert_eq!(store.current.lock().unwrap().revision, published.revision); + assert_eq!(fs::read(&store.path).unwrap(), before); + } + + #[test] + fn malformed_geometry_unknown_evidence_and_duplicate_slots_rejected() { + for (key, value) in [ + ("widthM", json!(2.5001)), + ("baseline", json!({})), + ("name", json!(" bad")), + ] { + let mut r = room(); + r[key] = value; + assert!(validate(r).is_err()); + } + let mut r = room(); + r["accessPoint"]["slot"] = json!("north_west"); + assert!(validate(r).is_err()); + let mut r = room(); + r["nodes"][0]["orientationConfirmed"] = json!(false); + assert!(validate(r).is_err()); + } + #[test] + fn disk_tampering_fails_closed() { + let dir = tempfile::tempdir().unwrap(); + let s = SharedRoomStore::load(dir.path(), "install-1").unwrap(); + s.publish(0, room()).unwrap(); + let mut e: Value = serde_json::from_slice(&fs::read(&s.path).unwrap()).unwrap(); + e["room"]["widthM"] = json!(3); + fs::write(&s.path, serde_json::to_vec(&e).unwrap()).unwrap(); + assert!(SharedRoomStore::load(dir.path(), "install-1").is_err()); + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/companion_sensors.rs b/v2/crates/wifi-densepose-sensing-server/src/companion_sensors.rs new file mode 100644 index 0000000000..147d0ad314 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/companion_sensors.rs @@ -0,0 +1,644 @@ +//! Transient geometry preview. A room digest never implies spatial registration. +use crate::companion::{Companion, Grant}; +use axum::{ + extract::{ + ws::{Message, WebSocket, WebSocketUpgrade}, + Path, State, + }, + http::StatusCode, + response::{IntoResponse, Response}, + routing::{get, post}, + Extension, Json, Router, +}; +use serde::{Deserialize, Serialize}; +use serde_json::{json, Value}; +use std::{ + collections::HashMap, + sync::{Arc, Mutex}, + time::{Duration, Instant}, +}; +use tokio::sync::watch; +const SAFE: u64 = 9_007_199_254_740_991; +const MAX_BYTES: usize = 65_536; +fn now_ms() -> u64 { + chrono::Utc::now().timestamp_millis().max(0) as u64 +} +fn identifier(s: &str) -> bool { + !s.is_empty() + && s.len() <= 128 + && s.bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'-' || b == b'_') +} +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub struct SensorFrame { + #[serde(rename = "type")] + pub kind: String, + pub schema: String, + pub share_session_id: String, + pub capture_session_id: String, + pub coordinate_frame_id: String, + pub sequence: u64, + pub captured_at_unix_ms: u64, + pub tracking_state: String, + pub points_mm: Vec, + pub confidences: Option>, + pub camera_transform: [f64; 16], + pub room_binding_digest: Option, +} +impl SensorFrame { + fn valid(&self, now: u64) -> bool { + let tracking = self.tracking_state.as_str(); + self.kind == "sensor_frame" + && self.schema == "ruview.companion.sensor-frame.v1" + && identifier(&self.share_session_id) + && identifier(&self.capture_session_id) + && identifier(&self.coordinate_frame_id) + && self.sequence <= SAFE + && self.captured_at_unix_ms > 0 + && self.captured_at_unix_ms <= SAFE + && self.captured_at_unix_ms <= now.saturating_add(1000) + && now <= self.captured_at_unix_ms.saturating_add(3000) + && matches!( + tracking, + "normal" + | "unavailable" + | "limited_initializing" + | "limited_excessiveMotion" + | "limited_insufficientFeatures" + | "limited_relocalizing" + ) + && !self.points_mm.is_empty() + && self.points_mm.len() <= 512 * 3 + && self.points_mm.len() % 3 == 0 + && self + .points_mm + .iter() + .all(|x| (-30_000..=30_000).contains(x)) + && self + .confidences + .as_ref() + .is_none_or(|c| c.len() == self.points_mm.len() / 3 && c.iter().all(|x| *x <= 2)) + && self.room_binding_digest.as_ref().is_none_or(|d| { + d.len() == 64 + && d.bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) + }) + && rigid(&self.camera_transform) + } + fn lineage(&self) -> (&str, &str, &str) { + ( + &self.share_session_id, + &self.capture_session_id, + &self.coordinate_frame_id, + ) + } +} +fn decode(text: &str) -> Result { + if text.len() > MAX_BYTES { + return Err(()); + } + let value: Value = serde_json::from_str(text).map_err(|_| ())?; + if !value.as_object().is_some_and(|m| { + m.len() == 12 && m.contains_key("confidences") && m.contains_key("roomBindingDigest") + }) { + return Err(()); + } + serde_json::from_str(text).map_err(|_| ()) +} +fn rigid(m: &[f64; 16]) -> bool { + if !m.iter().all(|v| v.is_finite()) + || [3, 7, 11].iter().any(|i| m[*i].abs() > 1e-5) + || (m[15] - 1.).abs() > 1e-5 + || [12, 13, 14].iter().any(|i| m[*i].abs() > 30.) + { + return false; + } + for a in 0..3 { + for b in 0..3 { + let dot: f64 = (0..3).map(|r| m[a * 4 + r] * m[b * 4 + r]).sum(); + if (dot - if a == b { 1. } else { 0. }).abs() > 1e-3 { + return false; + } + } + } + let det = m[0] * (m[5] * m[10] - m[9] * m[6]) - m[4] * (m[1] * m[10] - m[9] * m[2]) + + m[8] * (m[1] * m[6] - m[5] * m[2]); + (det - 1.).abs() < 1e-3 +} +struct Entry { + permission_revoked: watch::Receiver, + until: u64, + generation: u64, + stop: watch::Sender, + received: Option, + frame: Option, + access_until: u64, + access_revoked: Option>, + state: &'static str, +} +pub struct SensorStore { + companion: Arc, + entries: Mutex>, +} +pub type Shared = Arc; +impl SensorStore { + pub fn new(companion: Arc) -> Shared { + Arc::new(Self { + companion, + entries: Mutex::new(HashMap::new()), + }) + } + fn permit(&self, id: &str) -> Result { + let (_, expires) = self + .companion + .paired_device(id) + .ok_or(StatusCode::NOT_FOUND)?; + let permission_revoked = self + .companion + .device_revocation(id) + .ok_or(StatusCode::NOT_FOUND)?; + let mut entries = self + .entries + .lock() + .map_err(|_| StatusCode::SERVICE_UNAVAILABLE)?; + // Pairing membership is bounded. Forget departed devices rather than + // accumulating historical sensor grants across repeated enrollments. + entries.retain(|device_id, _| self.companion.paired_device(device_id).is_some()); + let entry = entries.entry(id.into()).or_insert_with(|| Entry { + permission_revoked: permission_revoked.clone(), + until: 0, + generation: 0, + stop: watch::channel(0).0, + received: None, + frame: None, + access_until: 0, + access_revoked: None, + state: "waiting", + }); + entry.permission_revoked = permission_revoked; + entry.generation = entry.generation.wrapping_add(1); + entry.stop.send_replace(entry.generation); + entry.until = (now_ms() / 1000 + 900).min(expires); + entry.frame = None; + entry.received = None; + entry.state = "waiting"; + entry.access_until = 0; + entry.access_revoked = None; + Ok(entry.until) + } + fn revoke(&self, id: &str) -> Result<(), StatusCode> { + let mut entries = self + .entries + .lock() + .map_err(|_| StatusCode::SERVICE_UNAVAILABLE)?; + let entry = entries.get_mut(id).ok_or(StatusCode::NOT_FOUND)?; + entry.until = 0; + entry.generation = entry.generation.wrapping_add(1); + entry.stop.send_replace(entry.generation); + entry.frame = None; + entry.received = None; + entry.state = "stopped"; + Ok(()) + } + fn begin(&self, grant: &Grant) -> Result<(u64, u64, watch::Receiver), StatusCode> { + if self.companion.paired_device(&grant.device_id).is_none() + || *grant.revoked.borrow() + || grant.expires_at <= now_ms() / 1000 + { + return Err(StatusCode::UNAUTHORIZED); + } + let mut entries = self + .entries + .lock() + .map_err(|_| StatusCode::SERVICE_UNAVAILABLE)?; + let e = entries + .get_mut(&grant.device_id) + .filter(|e| e.until > now_ms() / 1000 && !*e.permission_revoked.borrow()) + .ok_or(StatusCode::FORBIDDEN)?; + e.generation = e.generation.wrapping_add(1); + e.stop.send_replace(e.generation); + e.frame = None; + e.received = None; + e.state = "waiting"; + e.access_until = grant.expires_at; + e.access_revoked = Some(grant.revoked.clone()); + Ok(( + e.generation, + e.until.min(grant.expires_at), + e.stop.subscribe(), + )) + } + fn stop(&self, id: &str, generation: u64) { + if let Ok(mut entries) = self.entries.lock() { + if let Some(e) = entries.get_mut(id).filter(|e| e.generation == generation) { + e.frame = None; + e.received = None; + e.state = "stopped"; + } + } + } + fn accept(&self, id: &str, generation: u64, frame: SensorFrame) -> bool { + let Ok(mut entries) = self.entries.lock() else { + return false; + }; + let Some(e) = entries.get_mut(id).filter(|e| { + e.generation == generation + && !*e.permission_revoked.borrow() + && e.until > now_ms() / 1000 + && e.access_until > now_ms() / 1000 + && !e.access_revoked.as_ref().is_some_and(|r| *r.borrow()) + }) else { + return false; + }; + e.frame = Some(frame); + e.received = Some(now_ms()); + e.state = "live"; + true + } + fn snapshot(&self) -> Result { + let entries = self + .entries + .lock() + .map_err(|_| StatusCode::SERVICE_UNAVAILABLE)?; + let now = now_ms(); + let mut ids: Vec<_> = entries.keys().collect(); + ids.sort(); + let devices:Vec<_>=ids.into_iter().filter_map(|id| { + let (name,_)=self.companion.paired_device(id)?;let e=&entries[id]; + let authorized=!*e.permission_revoked.borrow() && e.until>now/1000 && (e.access_until==0 || e.access_until>now/1000) && !e.access_revoked.as_ref().is_some_and(|r| *r.borrow()); + let state=if !authorized {"stopped"} else if e.state=="live" && e.received.is_some_and(|t| now.saturating_sub(t)>3000) {"stale"} else {e.state}; + Some(json!({"deviceId":id,"deviceName":name,"sensorUntil":e.until,"state":state,"receivedAtUnixMs":if authorized {e.received} else {None},"frame":if state=="live" {e.frame.as_ref()} else {None}})) + }).collect(); + Ok(json!({"schema":"ruview.companion.sensor-snapshot.v1","devices":devices})) + } +} +async fn snapshot(State(s): State) -> Result, StatusCode> { + s.snapshot().map(Json) +} +async fn permit( + State(s): State, + Path(id): Path, +) -> Result, StatusCode> { + s.permit(&id) + .map(|until| Json(json!({"sensor_until":until}))) +} +async fn revoke( + State(s): State, + Path(id): Path, +) -> Result, StatusCode> { + s.revoke(&id)?; + Ok(Json(json!({"sensor_until":0}))) +} +/// Must be wrapped by the exact-owner-token and loopback-peer middleware. +pub fn owner_router(s: Shared) -> Router { + Router::new() + .route("/api/v1/companion/sensors", get(snapshot)) + .route( + "/api/v1/companion/devices/:id/sensors", + post(permit).delete(revoke), + ) + .with_state(s) +} +/// Must be wrapped by pinned companion authentication, which supplies Grant. +pub fn companion_router(s: Shared) -> Router { + Router::new() + .route("/companion/v1/sensors", get(upgrade)) + .with_state(s) +} +async fn upgrade( + State(s): State, + Extension(grant): Extension, + ws: WebSocketUpgrade, +) -> Response { + let (generation, until, stop) = match s.begin(&grant) { + Ok(v) => v, + Err(code) => return code.into_response(), + }; + ws.max_message_size(MAX_BYTES) + .max_frame_size(MAX_BYTES) + .on_upgrade(move |socket| run(s, grant, generation, until, stop, socket)) + .into_response() +} +async fn send(socket: &mut WebSocket, text: String) -> bool { + matches!( + tokio::time::timeout(Duration::from_millis(500), socket.send(Message::Text(text))).await, + Ok(Ok(())) + ) +} +async fn run( + s: Shared, + grant: Grant, + generation: u64, + until: u64, + mut stop: watch::Receiver, + mut socket: WebSocket, +) { + let id = grant.device_id; + let mut revoked = grant.revoked; + let ready = + json!({"type":"sensor_ready","schema":"ruview.companion.sensor-stream.v1"}).to_string(); + let mut previous: Option = None; + let mut last_at: Option = None; + let deadline = tokio::time::sleep(Duration::from_millis( + (until * 1000).saturating_sub(now_ms()), + )); + tokio::pin!(deadline); + if send(&mut socket, ready).await { + loop { + tokio::select! { + _=&mut deadline=>break, _=stop.changed()=>break, _=revoked.changed()=>break, + message=socket.recv()=>{ + if *revoked.borrow() || *stop.borrow()!=generation || until<=now_ms()/1000 {break;} + let text=match message {Some(Ok(Message::Text(t)))=>t,Some(Ok(Message::Ping(_)|Message::Pong(_)))=>continue,_=>break}; + if text.len()>MAX_BYTES {break;} + let frame=match decode(&text){Ok(f)=>f,Err(_)=>break}; + if !frame.valid(now_ms()) || last_at.is_some_and(|t|t.elapsed() (tempfile::TempDir, Shared, watch::Sender, Grant) { + let dir = tempfile::tempdir().unwrap(); + let proof = crate::companion::secret(); + let disk = json!({"version":1,"installation_id":"test","devices":[{"device_id":"phone","device_name":"Test phone","pairing_hash":format!("{:x}",Sha256::digest(proof.as_bytes())),"expires_at":now_ms()/1000+3600}]}); + crate::companion::private_dir(dir.path()).unwrap(); + crate::companion::atomic_private_write( + &dir.path().join("companion-grants.json"), + &serde_json::to_vec(&disk).unwrap(), + ) + .unwrap(); + let companion = Companion::load( + dir.path(), + "test".into(), + "a".repeat(64), + "https://test.local:3443".into(), + ) + .unwrap(); + let store = SensorStore::new(companion); + let (tx, rx) = watch::channel(false); + let grant = Grant { + device_id: "phone".into(), + expires_at: now_ms() / 1000 + 3600, + revoked: rx, + }; + (dir, store, tx, grant) + } + fn frame() -> SensorFrame { + serde_json::from_value(json!({"type":"sensor_frame","schema":"ruview.companion.sensor-frame.v1","shareSessionId":"share","captureSessionId":"capture","coordinateFrameId":"frame","sequence":1,"capturedAtUnixMs":now_ms(),"trackingState":"normal","pointsMm":[1,2,3],"confidences":[2],"cameraTransform":[1,0,0,0,0,1,0,0,0,0,1,0,0,0,0,1],"roomBindingDigest":null})).unwrap() + } + #[test] + fn geometry_validation_and_exact_nullable_keys() { + let f = frame(); + assert!(f.valid(now_ms())); + let mut value = serde_json::to_value(&f).unwrap(); + value.as_object_mut().unwrap().remove("confidences"); + assert!(decode(&value.to_string()).is_err()); + let mut empty = f.clone(); + empty.points_mm.clear(); + empty.confidences = Some(vec![]); + assert!(!empty.valid(now_ms())); + let mut uppercase = f.clone(); + uppercase.room_binding_digest = Some("A".repeat(64)); + assert!(!uppercase.valid(now_ms())); + let mut bad = f.clone(); + bad.points_mm = vec![1, 2]; + assert!(!bad.valid(now_ms())); + bad = f.clone(); + bad.camera_transform[0] = -1.; + assert!(!bad.valid(now_ms())); + bad = f.clone(); + bad.camera_transform[3] = 1.; + assert!(!bad.valid(now_ms())); + bad = f.clone(); + bad.captured_at_unix_ms = now_ms() - 3001; + assert!(!bad.valid(now_ms())); + bad = f.clone(); + bad.captured_at_unix_ms = now_ms() + 1001; + assert!(!bad.valid(now_ms())); + bad = f.clone(); + bad.confidences = Some(vec![]); + assert!(!bad.valid(now_ms())); + bad = f.clone(); + bad.points_mm = vec![1; 1539]; + assert!(!bad.valid(now_ms())); + bad = f.clone(); + bad.room_binding_digest = Some("x".repeat(64)); + assert!(!bad.valid(now_ms())); + value = serde_json::to_value(&f).unwrap(); + value["deviceId"] = json!("spoof"); + assert!(decode(&value.to_string()).is_err()); + } + #[test] + fn permission_expiry_revocation_supersession_and_stale() { + let (_dir, s, tx, g) = setup(); + assert!(matches!(s.begin(&g), Err(StatusCode::FORBIDDEN))); + s.permit("phone").unwrap(); + let (old, _, _) = s.begin(&g).unwrap(); + assert!(s.accept("phone", old, frame())); + let (current, _, _) = s.begin(&g).unwrap(); + s.stop("phone", old); + assert!(s.accept("phone", current, frame())); + assert_eq!(s.snapshot().unwrap()["devices"][0]["state"], "live"); + s.entries.lock().unwrap().get_mut("phone").unwrap().received = Some(now_ms() - 3001); + assert_eq!(s.snapshot().unwrap()["devices"][0]["state"], "stale"); + assert!(s.snapshot().unwrap()["devices"][0]["frame"].is_null()); + tx.send_replace(true); + assert!(s.snapshot().unwrap()["devices"][0]["frame"].is_null()); + assert!(!s.accept("phone", current, frame())); + s.permit("phone").unwrap(); + s.entries.lock().unwrap().get_mut("phone").unwrap().until = now_ms() / 1000; + assert_eq!(s.snapshot().unwrap()["devices"][0]["state"], "stopped"); + assert!(matches!(s.begin(&g), Err(StatusCode::UNAUTHORIZED))); + s.revoke("phone").unwrap(); + assert_eq!(s.snapshot().unwrap()["devices"][0]["sensorUntil"], 0); + assert!(matches!(s.permit("unknown"), Err(StatusCode::NOT_FOUND))); + } + async fn server(s: Shared, grant: Option) -> (String, tokio::task::JoinHandle<()>) { + let mut router = companion_router(s); + if let Some(grant) = grant { + router = router.layer(Extension(grant)); + } + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let task = tokio::spawn(async move { + axum::serve(listener, router).await.unwrap(); + }); + (format!("ws://{addr}/companion/v1/sensors"), task) + } + #[tokio::test] + async fn actual_upgrade_requires_permission_and_immediate_ready_then_revoke() { + let (_d, s, _tx, g) = setup(); + let (url, task) = server(s.clone(), Some(g)).await; + assert!(connect_async(&url).await.is_err()); + s.permit("phone").unwrap(); + let (mut socket, _) = connect_async(&url).await.unwrap(); + let ready = tokio::time::timeout(Duration::from_secs(1), socket.next()) + .await + .unwrap() + .unwrap() + .unwrap(); + assert!(ready.to_text().unwrap().contains("sensor_ready")); + socket + .send(ClientMessage::Text( + serde_json::to_string(&frame()).unwrap(), + )) + .await + .unwrap(); + assert!(socket + .next() + .await + .unwrap() + .unwrap() + .to_text() + .unwrap() + .contains("sensor_ack")); + assert_eq!(s.snapshot().unwrap()["devices"][0]["deviceId"], "phone"); + use tower::ServiceExt; + let response = owner_router(s.clone()) + .oneshot( + axum::http::Request::builder() + .method("DELETE") + .uri("/api/v1/companion/devices/phone/sensors") + .body(axum::body::Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let receipt: Value = serde_json::from_slice( + &axum::body::to_bytes(response.into_body(), 1024) + .await + .unwrap(), + ) + .unwrap(); + assert_eq!(receipt, json!({"sensor_until":0})); + assert!(s.snapshot().unwrap()["devices"][0]["frame"].is_null()); + let stopped = tokio::time::timeout(Duration::from_secs(1), socket.next()) + .await + .unwrap() + .unwrap() + .unwrap(); + assert!(stopped.to_text().unwrap().contains("sensor_stopped")); + task.abort(); + } + #[tokio::test] + async fn actual_socket_rejects_replay_lineage_rate_and_unknown_input() { + for mode in ["replay", "lineage", "rate", "unknown"] { + let (_d, s, _tx, g) = setup(); + s.permit("phone").unwrap(); + let (url, task) = server(s.clone(), Some(g)).await; + let (mut socket, _) = connect_async(&url).await.unwrap(); + socket.next().await.unwrap().unwrap(); + socket + .send(ClientMessage::Text( + serde_json::to_string(&frame()).unwrap(), + )) + .await + .unwrap(); + socket.next().await.unwrap().unwrap(); + if mode != "rate" { + tokio::time::sleep(Duration::from_millis(210)).await; + } + let mut next = frame(); + if mode != "replay" { + next.sequence = 2; + } + if mode == "lineage" { + next.coordinate_frame_id = "changed".into(); + } + let mut value = serde_json::to_value(next).unwrap(); + if mode == "unknown" { + value["rgb"] = json!("forbidden"); + } + socket + .send(ClientMessage::Text(value.to_string())) + .await + .unwrap(); + let stopped = tokio::time::timeout(Duration::from_secs(1), socket.next()) + .await + .unwrap() + .unwrap() + .unwrap(); + assert!( + stopped.to_text().unwrap().contains("sensor_stopped"), + "{mode}" + ); + assert!(s.snapshot().unwrap()["devices"][0]["frame"].is_null()); + task.abort(); + } + } + #[tokio::test] + async fn pairing_revocation_removes_preview_and_prevents_regrant() { + use tower::ServiceExt; + let (_d, s, _tx, g) = setup(); + s.permit("phone").unwrap(); + let (generation, _, _) = s.begin(&g).unwrap(); + assert!(s.accept("phone", generation, frame())); + let response = crate::companion::owner_router(s.companion.clone()) + .oneshot( + axum::http::Request::builder() + .method("DELETE") + .uri("/api/v1/companion/devices/phone") + .body(axum::body::Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + assert!(s.snapshot().unwrap()["devices"] + .as_array() + .unwrap() + .is_empty()); + assert!(matches!(s.permit("phone"), Err(StatusCode::NOT_FOUND))); + assert!(!s.accept("phone", generation, frame())); + } + #[tokio::test] + async fn established_socket_expires_sensor_and_access_permission() { + for sensor_expiry in [true, false] { + let (_d, s, _tx, mut g) = setup(); + s.permit("phone").unwrap(); + if sensor_expiry { + s.entries.lock().unwrap().get_mut("phone").unwrap().until = now_ms() / 1000 + 1; + } else { + g.expires_at = now_ms() / 1000 + 1; + } + let (url, task) = server(s.clone(), Some(g)).await; + let (mut socket, _) = connect_async(&url).await.unwrap(); + socket.next().await.unwrap().unwrap(); + socket + .send(ClientMessage::Text( + serde_json::to_string(&frame()).unwrap(), + )) + .await + .unwrap(); + socket.next().await.unwrap().unwrap(); + let stopped = tokio::time::timeout(Duration::from_secs(2), socket.next()) + .await + .unwrap() + .unwrap() + .unwrap(); + assert!(stopped.to_text().unwrap().contains("sensor_stopped")); + assert!(s.snapshot().unwrap()["devices"][0]["frame"].is_null()); + task.abort(); + } + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/companion_spatial.rs b/v2/crates/wifi-densepose-sensing-server/src/companion_spatial.rs new file mode 100644 index 0000000000..53a63bff59 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/companion_spatial.rs @@ -0,0 +1,772 @@ +//! Explicit, bounded artifact handoff. Imported geometry never grants RF/AR authority. +use super::companion_room::SharedRoomStore; +use axum::{extract::State, http::StatusCode, routing::get, Extension, Json, Router}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +#[cfg(unix)] +use std::os::unix::fs::OpenOptionsExt; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::{ + collections::HashMap, + fs, + io::Write, + path::{Path, PathBuf}, + sync::{Arc, Mutex}, +}; +use wifi_densepose_sensing_server::companion::{Companion, Grant}; +const MAX_PAYLOAD: usize = 48 * 1024; +fn now() -> u64 { + chrono::Utc::now().timestamp_millis().max(0) as u64 +} +fn id(s: &str) -> bool { + !s.is_empty() + && s.len() <= 128 + && s.bytes() + .all(|b| b.is_ascii_alphanumeric() || b"-_.:".contains(&b)) +} +fn hash(s: &str) -> String { + format!("{:x}", Sha256::digest(s.as_bytes())) +} +fn hex(s: &str) -> bool { + s.len() == 64 + && s.bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) +} +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +struct Surface { + id: String, + kind: String, + dimensions_m: [f64; 3], + transform: [f64; 16], +} +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +struct Node { + id: String, + role: String, + position_m: [f64; 3], +} +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +struct Geometry { + schema: String, + installation_id: String, + room_revision: u64, + room_binding_digest: String, + coordinate_frame_id: String, + source_scan_id: String, + captured_at_unix_ms: u64, + exported_at_unix_ms: u64, + provenance: String, + calibration_digest: Option, + surfaces: Vec, + nodes: Vec, +} +impl Geometry { + fn valid(&self, time: u64) -> bool { + self.schema == "ruview.companion.spatial-artifact.v1" + && id(&self.installation_id) + && self.room_revision > 0 + && self.room_revision <= 9_007_199_254_740_991 + && hex(&self.room_binding_digest) + && id(&self.coordinate_frame_id) + && id(&self.source_scan_id) + && self.captured_at_unix_ms > 0 + && self.captured_at_unix_ms <= self.exported_at_unix_ms + && self.exported_at_unix_ms <= time.saturating_add(5000) + && time <= self.exported_at_unix_ms.saturating_add(900_000) + && self.provenance == "historical_geometry_only" + && self.calibration_digest.as_deref().is_none_or(hex) + && !self.surfaces.is_empty() + && self.surfaces.len() <= 128 + && self.nodes.len() <= 64 + && self.surfaces.iter().all(|s| { + id(&s.id) + && ["wall", "door", "window", "opening", "floor"].contains(&s.kind.as_str()) + && s.dimensions_m + .iter() + .all(|v| v.is_finite() && *v >= 0. && *v <= 1000.) + && s.transform + .iter() + .all(|v| v.is_finite() && v.abs() <= 1000.) + && rigid(&s.transform) + }) + && self.nodes.iter().all(|n| { + id(&n.id) + && ["esp32_csi_receiver", "wifi_transmitter", "reference"] + .contains(&n.role.as_str()) + && n.position_m + .iter() + .all(|v| v.is_finite() && v.abs() <= 1000.) + }) + && self + .surfaces + .iter() + .map(|s| &s.id) + .collect::>() + .len() + == self.surfaces.len() + && self + .nodes + .iter() + .map(|s| &s.id) + .collect::>() + .len() + == self.nodes.len() + } +} +fn rigid(m: &[f64; 16]) -> bool { + if m[3].abs() > 1e-4 || m[7].abs() > 1e-4 || m[11].abs() > 1e-4 || (m[15] - 1.).abs() > 1e-4 { + return false; + } + for a in 0..3 { + for b in 0..3 { + let dot = (0..3).map(|r| m[a * 4 + r] * m[b * 4 + r]).sum::(); + if (dot - if a == b { 1. } else { 0. }).abs() > 0.015 { + return false; + } + } + } + if m[12..15].iter().any(|v| v.abs() > 30.) { + return false; + } + let det = m[0] * (m[5] * m[10] - m[9] * m[6]) - m[4] * (m[1] * m[10] - m[9] * m[2]) + + m[8] * (m[1] * m[6] - m[5] * m[2]); + (det - 1.).abs() < 0.02 +} +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +struct Artifact { + digest: String, + payload: String, + publisher_device_id: String, + received_at_unix_ms: u64, + accepted_at_unix_ms: Option, +} +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +struct Saved { + schema: String, + owner_scope: String, + installation_id: String, + accepted: Vec, +} +struct Receipt { + nonce: String, + until: u64, + pending: Option, + revoked: tokio::sync::watch::Receiver, +} +struct Inner { + saved: Saved, + receipts: HashMap, +} +pub struct SpatialStore { + fault: AtomicBool, + path: PathBuf, + service: Arc, + room: Arc, + inner: Mutex, +} +type Shared = Arc; +impl SpatialStore { + pub fn load( + dir: &Path, + scope: &str, + installation: &str, + service: Arc, + room: Arc, + ) -> std::io::Result { + if !hex(scope) { + return Err(std::io::Error::other( + "spatial handoff requires account ownership", + )); + } + let path = dir.join("accepted-spatial-artifacts.json"); + let saved = match (|| -> std::io::Result> { + let meta = fs::symlink_metadata(&path)?; + if !meta.is_file() || meta.file_type().is_symlink() || meta.len() > 2 * 1024 * 1024 { + return Err(std::io::Error::other("invalid spatial history file")); + } + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + if meta.permissions().mode() & 0o077 != 0 { + return Err(std::io::Error::other( + "spatial history permissions are not private", + )); + } + } + fs::read(&path) + })() { + Ok(bytes) => { + if bytes.len() > 2 * 1024 * 1024 { + return Err(std::io::Error::other("spatial history too large")); + } + let s: Saved = serde_json::from_slice(&bytes)?; + if s.schema != "ruview.companion.spatial-history.v1" + || s.owner_scope != scope + || s.installation_id != installation + || s.accepted.len() > 32 + || s.accepted.iter().any(|a| { + a.payload.len() > MAX_PAYLOAD + || hash(&a.payload) != a.digest + || a.accepted_at_unix_ms.is_none() + }) + { + return Err(std::io::Error::other( + "invalid spatial history ownership or digest", + )); + } + for a in &s.accepted { + let g: Geometry = serde_json::from_str(&a.payload)?; + if !g.valid(g.exported_at_unix_ms) || g.installation_id != installation { + return Err(std::io::Error::other("invalid spatial geometry")); + } + } + s + } + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Saved { + schema: "ruview.companion.spatial-history.v1".into(), + owner_scope: scope.into(), + installation_id: installation.into(), + accepted: vec![], + }, + Err(e) => return Err(e), + }; + Ok(Arc::new(Self { + fault: AtomicBool::new(false), + path, + service, + room, + inner: Mutex::new(Inner { + saved, + receipts: HashMap::new(), + }), + })) + } + fn persist(&self, s: &Saved) -> std::io::Result<()> { + let tmp = self + .path + .with_extension(format!("{}.tmp", rand::random::())); + let mut options = fs::OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + options.mode(0o600); + let mut file = options.open(&tmp)?; + file.write_all(&serde_json::to_vec(s)?)?; + file.sync_all()?; + fs::rename(tmp, &self.path)?; + fs::File::open(self.path.parent().unwrap())?.sync_all()?; + Ok(()) + } + fn matches(&self, g: &Geometry) -> bool { + let r = self.room.snapshot(); + r.is_some_and(|r| { + r.installation_id == g.installation_id + && r.revision == g.room_revision + && r.binding_digest.as_deref() == Some(&g.room_binding_digest) + }) + } +} +#[derive(Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +struct Upload { + nonce: String, + digest: String, + payload: String, +} +#[derive(Deserialize)] +#[serde(tag = "action", deny_unknown_fields, rename_all = "camelCase")] +enum Action { + Receive { + #[serde(rename = "deviceId")] + device_id: String, + }, + Accept { + digest: String, + }, + Reject { + digest: String, + }, +} +fn active(r: &Receipt, t: u64) -> bool { + r.until > t && !*r.revoked.borrow() +} +async fn snapshot(State(s): State) -> Result, StatusCode> { + if s.fault.load(Ordering::SeqCst) { + return Err(StatusCode::SERVICE_UNAVAILABLE); + } + let inner = s + .inner + .lock() + .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + let t = now(); + let pending: Vec<_> = inner + .receipts + .values() + .filter(|r| active(r, t)) + .filter_map(|r| r.pending.as_ref()) + .collect(); + Ok(Json( + serde_json::json!({"schema":"ruview.companion.spatial-inbox.v1","pending":pending,"accepted":inner.saved.accepted}), + )) +} +async fn ticket( + State(s): State, + Extension(g): Extension, +) -> Result, StatusCode> { + if s.fault.load(Ordering::SeqCst) { + return Err(StatusCode::SERVICE_UNAVAILABLE); + } + let inner = s + .inner + .lock() + .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + let t = now(); + if *g.revoked.borrow() || g.expires_at * 1000 <= t { + return Err(StatusCode::FORBIDDEN); + } + let r = inner + .receipts + .get(&g.device_id) + .filter(|r| active(r, t) && r.pending.is_none()) + .ok_or(StatusCode::FORBIDDEN)?; + Ok(Json(serde_json::json!({"nonce":r.nonce,"until":r.until}))) +} +async fn upload( + State(s): State, + Extension(g): Extension, + Json(u): Json, +) -> Result, StatusCode> { + if s.fault.load(Ordering::SeqCst) { + return Err(StatusCode::SERVICE_UNAVAILABLE); + } + let t = now(); + if *g.revoked.borrow() || g.expires_at * 1000 <= t { + return Err(StatusCode::FORBIDDEN); + } + if u.payload.len() > MAX_PAYLOAD || !hex(&u.digest) || hash(&u.payload) != u.digest { + return Err(StatusCode::BAD_REQUEST); + } + let raw: serde_json::Value = + serde_json::from_str(&u.payload).map_err(|_| StatusCode::BAD_REQUEST)?; + if raw.get("calibrationDigest").is_none() { + return Err(StatusCode::BAD_REQUEST); + } + let geometry: Geometry = + serde_json::from_str(&u.payload).map_err(|_| StatusCode::BAD_REQUEST)?; + if !geometry.valid(t) || !s.matches(&geometry) { + return Err(StatusCode::CONFLICT); + } + let mut i = s + .inner + .lock() + .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + if i.saved.accepted.iter().any(|a| a.digest == u.digest) { + return Err(StatusCode::CONFLICT); + } + let r = i + .receipts + .get_mut(&g.device_id) + .filter(|r| active(r, t) && r.nonce == u.nonce && r.pending.is_none()) + .ok_or(StatusCode::FORBIDDEN)?; + r.pending = Some(Artifact { + digest: u.digest.clone(), + payload: u.payload, + publisher_device_id: g.device_id, + received_at_unix_ms: t, + accepted_at_unix_ms: None, + }); + Ok(Json( + serde_json::json!({"digest":u.digest,"state":"pending_review"}), + )) +} +async fn action( + State(s): State, + Json(a): Json, +) -> Result, StatusCode> { + if s.fault.load(Ordering::SeqCst) { + return Err(StatusCode::SERVICE_UNAVAILABLE); + } + let t = now(); + let mut i = s + .inner + .lock() + .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + let accept = matches!(&a, Action::Accept { .. }); + match a { + Action::Receive { device_id } => { + let (_, expiry) = s + .service + .paired_device(&device_id) + .ok_or(StatusCode::FORBIDDEN)?; + let revoked = s + .service + .device_revocation(&device_id) + .ok_or(StatusCode::FORBIDDEN)?; + i.receipts.retain(|_, r| active(r, t)); + if i.receipts.len() >= 256 || i.saved.accepted.len() >= 32 { + return Err(StatusCode::CONFLICT); + } + let until = (t + 900_000).min(expiry * 1000); + if until <= t { + return Err(StatusCode::FORBIDDEN); + } + i.receipts.insert( + device_id, + Receipt { + nonce: format!("{:032x}", rand::random::()), + until, + pending: None, + revoked, + }, + ); + } + Action::Accept { digest } | Action::Reject { digest } => { + let (device, r) = i + .receipts + .iter() + .find(|(_, r)| { + active(r, t) && r.pending.as_ref().is_some_and(|p| p.digest == digest) + }) + .ok_or(StatusCode::CONFLICT)?; + let device = device.clone(); + let mut artifact = r.pending.clone().unwrap(); + if accept { + let geometry: Geometry = + serde_json::from_str(&artifact.payload).map_err(|_| StatusCode::BAD_REQUEST)?; + if !s.matches(&geometry) || i.saved.accepted.len() >= 32 { + return Err(StatusCode::CONFLICT); + } + artifact.accepted_at_unix_ms = Some(t); + i.saved.accepted.push(artifact); + if s.persist(&i.saved).is_err() { + s.fault.store(true, Ordering::SeqCst); + i.saved.accepted.pop(); + return Err(StatusCode::INTERNAL_SERVER_ERROR); + } + } + i.receipts.remove(&device); + } + } + Ok(Json(serde_json::json!({"state":"ok"}))) +} +pub fn owner_router(s: Shared) -> Router { + Router::new() + .route("/api/v1/companion/spatial", get(snapshot).post(action)) + .with_state(s) +} +pub fn peer_router(s: Shared) -> Router { + Router::new() + .route("/companion/v1/spatial", get(ticket).post(upload)) + .with_state(s) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + use tower::ServiceExt; + async fn setup() -> (tempfile::TempDir, Shared, Grant) { + let d = tempfile::tempdir().unwrap(); + fs::write(d.path().join("companion-grants.json"),serde_json::to_vec(&json!({"version":1,"installation_id":"test","devices":[{"device_id":"phone","device_name":"Phone","pairing_hash":"a".repeat(64),"expires_at":now()/1000+3600}]})).unwrap()).unwrap(); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions( + d.path().join("companion-grants.json"), + fs::Permissions::from_mode(0o600), + ) + .unwrap(); + } + let c = Companion::load( + d.path(), + "test".into(), + "a".repeat(64), + "https://test.local:3443".into(), + ) + .unwrap(); + let room = SharedRoomStore::load(d.path(), "test").unwrap(); + let body = json!({"expected_revision":0,"room":{"schema":"ruview.companion.room.v1","id":"room-1","name":"Room","widthM":2.5,"lengthM":2.5,"heightM":2.4,"nodes":[{"nodeId":15,"slot":"north_west","xM":0,"yM":0,"role":"receiver","headingDeg":90,"orientationConfirmed":true,"confirmedAtUnixMs":1}],"accessPoint":{"slot":"south_east","xM":2.5,"yM":2.5,"provenance":"operator_entered","confirmedAtUnixMs":1}}}); + let r = super::super::companion_room::owner_router(room.clone()) + .oneshot( + axum::http::Request::builder() + .method("PUT") + .uri("/api/v1/companion/room") + .header("content-type", "application/json") + .body(axum::body::Body::from(body.to_string())) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(r.status(), StatusCode::OK); + let grant = Grant { + device_id: "phone".into(), + expires_at: now() / 1000 + 3600, + revoked: c.device_revocation("phone").unwrap(), + }; + let s = SpatialStore::load(d.path(), &"a".repeat(64), "test", c, room).unwrap(); + (d, s, grant) + } + fn payload(s: &Shared) -> String { + json!({"schema":"ruview.companion.spatial-artifact.v1","installationId":"test","roomRevision":1,"roomBindingDigest":s.room.snapshot().unwrap().binding_digest,"coordinateFrameId":"frame","sourceScanId":"scan","capturedAtUnixMs":1,"exportedAtUnixMs":now(),"provenance":"historical_geometry_only","calibrationDigest":null,"surfaces":[{"id":"wall-1","kind":"wall","dimensionsM":[2.5,2.4,0.1],"transform":[1,0,0,0,0,1,0,0,0,0,1,0,0,0,0,1]}],"nodes":[]}).to_string() + } + async fn allow(s: &Shared, g: &Grant) -> String { + let _ = action( + State(s.clone()), + Json(Action::Receive { + device_id: g.device_id.clone(), + }), + ) + .await + .unwrap(); + ticket(State(s.clone()), Extension(g.clone())) + .await + .unwrap() + .0["nonce"] + .as_str() + .unwrap() + .into() + } + #[tokio::test] + async fn explicit_review_retains_across_restart_and_rejects_other_owner() { + let (d, s, g) = setup().await; + let nonce = allow(&s, &g).await; + let p = payload(&s); + let digest = hash(&p); + let _ = upload( + State(s.clone()), + Extension(g.clone()), + Json(Upload { + nonce: nonce.clone(), + digest: digest.clone(), + payload: p.clone(), + }), + ) + .await + .unwrap(); + assert!(!s.path.exists()); + assert_eq!( + snapshot(State(s.clone())).await.unwrap().0["pending"] + .as_array() + .unwrap() + .len(), + 1 + ); + assert!(upload( + State(s.clone()), + Extension(g), + Json(Upload { + nonce, + digest: digest.clone(), + payload: p + }) + ) + .await + .is_err()); + let _ = action( + State(s.clone()), + Json(Action::Accept { + digest: digest.clone(), + }), + ) + .await + .unwrap(); + let restored = SpatialStore::load( + d.path(), + &"a".repeat(64), + "test", + s.service.clone(), + s.room.clone(), + ) + .unwrap(); + assert_eq!( + snapshot(State(restored)).await.unwrap().0["accepted"][0]["digest"], + digest + ); + assert!(SpatialStore::load( + d.path(), + &"b".repeat(64), + "test", + s.service.clone(), + s.room.clone() + ) + .is_err()); + assert!(action(State(s), Json(Action::Accept { digest })) + .await + .is_err()); + } + #[tokio::test] + async fn rejects_expiry_wrong_device_nonce_binding_digest_and_authority() { + let (_d, s, g) = setup().await; + let nonce = allow(&s, &g).await; + let p = payload(&s); + for mode in [ + "wrong-device", + "nonce", + "binding", + "integrity", + "authority", + "reflection", + "unknown", + "expired", + "missing-null", + ] { + let mut g = g.clone(); + let mut v: serde_json::Value = serde_json::from_str(&p).unwrap(); + let mut n = nonce.clone(); + match mode { + "wrong-device" => g.device_id = "other".into(), + "nonce" => n = "wrong".into(), + "binding" => v["roomRevision"] = json!(2), + "authority" => v["provenance"] = json!("current_alignment"), + "reflection" => v["surfaces"][0]["transform"][0] = json!(-1), + "unknown" => v["cameraImage"] = json!("raw"), + "expired" => g.expires_at = 0, + "missing-null" => { + v.as_object_mut().unwrap().remove("calibrationDigest"); + } + _ => {} + } + let p = v.to_string(); + let digest = if mode == "integrity" { + "0".repeat(64) + } else { + hash(&p) + }; + assert!( + upload( + State(s.clone()), + Extension(g), + Json(Upload { + nonce: n, + digest, + payload: p + }) + ) + .await + .is_err(), + "{mode}" + ); + } + s.inner + .lock() + .unwrap() + .receipts + .get_mut("phone") + .unwrap() + .until = 0; + assert!(ticket(State(s), Extension(g)).await.is_err()); + } + #[tokio::test] + async fn revocation_during_review_removes_pending_authority() { + let (_d, s, g) = setup().await; + let nonce = allow(&s, &g).await; + let p = payload(&s); + let digest = hash(&p); + let _ = upload( + State(s.clone()), + Extension(g.clone()), + Json(Upload { + nonce, + digest: digest.clone(), + payload: p, + }), + ) + .await + .unwrap(); + let response = wifi_densepose_sensing_server::companion::owner_router(s.service.clone()) + .oneshot( + axum::http::Request::builder() + .method("DELETE") + .uri("/api/v1/companion/devices/phone") + .body(axum::body::Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + assert!(action(State(s.clone()), Json(Action::Accept { digest })) + .await + .is_err()); + assert_eq!(snapshot(State(s)).await.unwrap().0["pending"], json!([])); + } + #[tokio::test] + async fn uncertain_disk_failure_never_claims_acceptance_or_allows_more_writes() { + let (_d, s, g) = setup().await; + let nonce = allow(&s, &g).await; + let p = payload(&s); + let digest = hash(&p); + let _ = upload( + State(s.clone()), + Extension(g), + Json(Upload { + nonce, + digest: digest.clone(), + payload: p, + }), + ) + .await + .unwrap(); + fs::create_dir(&s.path).unwrap(); + assert!(action(State(s.clone()), Json(Action::Accept { digest })) + .await + .is_err()); + assert_eq!( + snapshot(State(s.clone())).await.unwrap_err(), + StatusCode::SERVICE_UNAVAILABLE + ); + assert!(s.inner.lock().unwrap().saved.accepted.is_empty()); + } + #[tokio::test] + async fn unknown_actions_are_rejected_by_real_router() { + let (_d, s, _g) = setup().await; + for body in [ + json!({"action":"receive","deviceId":"unknown"}).to_string(), + json!({"action":"accept","digest":"a".repeat(64),"promote":true}).to_string(), + ] { + let response = owner_router(s.clone()) + .oneshot( + axum::http::Request::builder() + .method("POST") + .uri("/api/v1/companion/spatial") + .header("content-type", "application/json") + .body(axum::body::Body::from(body)) + .unwrap(), + ) + .await + .unwrap(); + assert!(!response.status().is_success()); + } + } +} + +#[cfg(test)] +mod hostile_storage_tests { + use super::*; + #[test] + fn rejects_non_private_and_symlinked_history_before_reading() { + let d = tempfile::tempdir().unwrap(); + let c = Companion::load( + d.path(), + "test".into(), + "a".repeat(64), + "https://test.local:3443".into(), + ) + .unwrap(); + let room = SharedRoomStore::load(d.path(), "test").unwrap(); + let path = d.path().join("accepted-spatial-artifacts.json"); + fs::write(&path, b"{}").unwrap(); + assert!( + SpatialStore::load(d.path(), &"a".repeat(64), "test", c.clone(), room.clone()).is_err() + ); + #[cfg(unix)] + { + fs::remove_file(&path).unwrap(); + std::os::unix::fs::symlink("missing", &path).unwrap(); + assert!(SpatialStore::load(d.path(), &"a".repeat(64), "test", c, room).is_err()); + } + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/companion_tls.rs b/v2/crates/wifi-densepose-sensing-server/src/companion_tls.rs new file mode 100644 index 0000000000..371f7e4ae8 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/companion_tls.rs @@ -0,0 +1,141 @@ +//! Private, persistent TLS identity for the QR-pinned companion listener. +use rustls::pki_types::{CertificateDer, PrivatePkcs8KeyDer}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::{io, path::Path, sync::Arc}; +#[derive(Serialize, Deserialize)] +struct DiskIdentity { + installation_id: String, + certificate: Vec, + private_key: Vec, +} +// Publish a complete first identity without replacing another process's identity. +fn create_identity_exclusively(path: &Path, bytes: &[u8]) -> io::Result<()> { + let temporary = path.with_extension(format!("create-{}", super::companion::secret())); + super::companion::atomic_private_write(&temporary, bytes)?; + let result = std::fs::hard_link(&temporary, path); + let cleanup = std::fs::remove_file(&temporary); + result?; + cleanup?; + std::fs::File::open( + path.parent() + .ok_or_else(|| io::Error::other("identity parent"))?, + )? + .sync_all() +} +pub struct Identity { + pub installation_id: String, + pub cert_sha256: String, + pub config: Arc, +} +impl Identity { + pub fn load_or_create(dir: &Path, preferred_installation_id: Option<&str>) -> io::Result { + super::companion::private_dir(dir)?; + if preferred_installation_id.is_some_and(|id| { + id.is_empty() + || id.len() > 128 + || !id + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'-' || b == b'_') + }) { + return Err(io::Error::other("invalid configured installation")); + } + let path = dir.join("companion-identity.json"); + let data: DiskIdentity = if path.exists() { + serde_json::from_slice(&super::companion::private_read(&path)?) + .map_err(io::Error::other)? + } else { + let cert = rcgen::generate_simple_self_signed(vec!["ruview.local".into()]) + .map_err(io::Error::other)?; + let data = DiskIdentity { + installation_id: preferred_installation_id + .map(str::to_owned) + .unwrap_or_else(super::companion::secret), + certificate: cert.serialize_der().map_err(io::Error::other)?, + private_key: cert.serialize_private_key_der(), + }; + create_identity_exclusively( + &path, + &serde_json::to_vec(&data).map_err(io::Error::other)?, + )?; + data + }; + if data.installation_id.is_empty() + || data.installation_id.len() > 128 + || preferred_installation_id.is_some_and(|id| id != data.installation_id) + { + return Err(io::Error::other("invalid companion installation")); + } + let pin = format!("{:x}", Sha256::digest(&data.certificate)); + let config = rustls::ServerConfig::builder_with_provider(Arc::new( + rustls::crypto::ring::default_provider(), + )) + .with_safe_default_protocol_versions() + .map_err(io::Error::other)? + .with_no_client_auth() + .with_single_cert( + vec![CertificateDer::from(data.certificate)], + PrivatePkcs8KeyDer::from(data.private_key).into(), + ) + .map_err(io::Error::other)?; + Ok(Self { + installation_id: data.installation_id, + cert_sha256: pin, + config: Arc::new(config), + }) + } +} +pub async fn serve( + listener: tokio::net::TcpListener, + config: Arc, + app: axum::Router, +) -> io::Result<()> { + let acceptor = tokio_rustls::TlsAcceptor::from(config); + let slots = Arc::new(tokio::sync::Semaphore::new(64)); + loop { + let (stream, _) = listener.accept().await?; + let Ok(permit) = slots.clone().try_acquire_owned() else { + continue; + }; + let acceptor = acceptor.clone(); + let app = app.clone(); + tokio::spawn(async move { + let _permit = permit; + let Ok(Ok(tls)) = + tokio::time::timeout(std::time::Duration::from_secs(10), acceptor.accept(stream)) + .await + else { + return; + }; + let service = hyper_util::service::TowerToHyperService::new(app); + let builder = + hyper_util::server::conn::auto::Builder::new(hyper_util::rt::TokioExecutor::new()); + let _ = builder + .serve_connection_with_upgrades(hyper_util::rt::TokioIo::new(tls), service) + .await; + }); + } +} +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn stable_identity_and_configured_mismatch() { + let d = tempfile::tempdir().unwrap(); + let first = Identity::load_or_create(d.path(), Some("mac-installation")).unwrap(); + let second = Identity::load_or_create(d.path(), Some("mac-installation")).unwrap(); + assert_eq!(first.cert_sha256, second.cert_sha256); + assert_eq!(first.installation_id, "mac-installation"); + assert!(Identity::load_or_create(d.path(), Some("other")).is_err()); + std::fs::write(d.path().join("companion-identity.json"), b"corrupt").unwrap(); + assert!(Identity::load_or_create(d.path(), None).is_err()); + } + #[test] + fn first_identity_publication_never_replaces_existing() { + let d = tempfile::tempdir().unwrap(); + let p = d.path().join("identity"); + create_identity_exclusively(&p, b"first").unwrap(); + assert!(create_identity_exclusively(&p, b"second").is_err()); + assert_eq!(std::fs::read(&p).unwrap(), b"first"); + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/discovery.rs b/v2/crates/wifi-densepose-sensing-server/src/discovery.rs index 10eda76441..99617b6a4a 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/discovery.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/discovery.rs @@ -4,9 +4,16 @@ //! sensor values, credentials, or personal data. Mobile clients treat the //! installation id as a routing hint only and still health-check the service. -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; +use std::net::{IpAddr, Ipv4Addr}; +use std::sync::{ + atomic::{AtomicBool, Ordering}, + Arc, +}; +use std::thread::{self, JoinHandle}; +use std::time::{Duration, Instant}; -use mdns_sd::{ServiceDaemon, ServiceInfo}; +use mdns_sd::{DaemonEvent, ServiceDaemon, ServiceInfo}; pub const SERVICE_TYPE: &str = "_ruview._tcp.local."; pub const SCHEMA: &str = "ruview.installation.v1"; @@ -14,10 +21,14 @@ pub const SCHEMA: &str = "ruview.installation.v1"; pub struct DiscoveryAdvertiser { daemon: ServiceDaemon, fullname: String, + recovery: RecoveryWorker, } impl Drop for DiscoveryAdvertiser { fn drop(&mut self) { + // Join BEFORE unregister: any already-dispatched refresh precedes the + // goodbye, and no worker can enqueue a registration after shutdown. + self.recovery.stop(); let _ = self.daemon.unregister(&self.fullname); let _ = self.daemon.shutdown(); } @@ -80,12 +91,236 @@ pub fn start_advertiser( hostname: &str, http_port: u16, ) -> Result { + start_service(build_service( + instance_name, + installation_id, + hostname, + http_port, + false, + )?) +} + +/// Advertise the QR-pinned companion TLS endpoint independently of legacy HTTP. +pub fn start_tls_advertiser( + instance_name: &str, + installation_id: &str, + hostname: &str, + port: u16, +) -> Result { + start_service(build_service( + instance_name, + installation_id, + hostname, + port, + true, + )?) +} + +// IpAdd is only a wakeup, never an address/owner/health authority. Re-registering +// this pristine, empty-address template lets mdns-sd perform its EXISTING live +// interface enumeration; no event address is copied into a service record. +fn start_service(template: ServiceInfo) -> Result { let daemon = ServiceDaemon::new()?; - let service = build_service(instance_name, installation_id, hostname, http_port, false)?; - let fullname = service.get_fullname().to_owned(); - daemon.register(service)?; - Ok(DiscoveryAdvertiser { daemon, fullname }) + let fullname = template.get_fullname().to_owned(); + let result = (|| { + let events = daemon.monitor()?; + daemon.register(template.clone())?; + let recovery = RecoveryWorker::start(daemon.clone(), events, template)?; + Ok(DiscoveryAdvertiser { + daemon: daemon.clone(), + fullname: fullname.clone(), + recovery, + }) + })(); + if result.is_err() { + let _ = daemon.unregister(&fullname); + let _ = daemon.shutdown(); + } + result +} + +const COALESCE_MS: u64 = 250; +const MIN_REFRESH_MS: u64 = 1_000; +const MAX_RECOVERED_IPS: usize = 8; +const MAX_RETRIES: u8 = 3; + +/// Per-advertiser bounded event reducer. It does not retain packets, names, +/// credentials, or an additional service/address authority. +#[derive(Default)] +struct RecoveryState { + // Pending wakeups only, not a persistent interface cache. + recovered: HashSet, + due: Option, + last_attempt: Option, + retries: u8, + stopped: bool, +} +impl RecoveryState { + fn event(&mut self, event: DaemonEvent, now: u64) { + if self.stopped { + return; + } + match event { + DaemonEvent::IpAdd(IpAddr::V4(ip)) if ip.is_private() => { + if !self.recovered.contains(&ip) && self.recovered.len() >= MAX_RECOVERED_IPS { + return; + } + self.recovered.insert(ip); + // Duplicate notifications coalesce while pending. Once settled, + // a fresh IpAdd is allowed to wake us even if its IpDel was + // dropped by the bounded daemon monitor. Rate limiting remains. + if self.due.is_none() { + self.retries = 0; + self.due = Some( + now.saturating_add(COALESCE_MS).max( + self.last_attempt + .map_or(0, |at| at.saturating_add(MIN_REFRESH_MS)), + ), + ); + } + } + DaemonEvent::IpDel(IpAddr::V4(ip)) => { + self.recovered.remove(&ip); + if self.recovered.is_empty() { + self.due = None; + } + } + // Announce is not a successful wire-send receipt in mdns-sd0.11.5. + _ => {} + } + } + fn stop(&mut self) { + self.stopped = true; + self.due = None; + self.recovered.clear(); + } + fn poll(&mut self, now: u64, register: impl FnOnce() -> Result<(), mdns_sd::Error>) { + if self.stopped || self.due.is_none_or(|due| now < due) { + return; + } + self.due = None; + self.last_attempt = Some(now); + match register() { + Ok(()) => { + self.retries = 0; + self.recovered.clear(); + } + Err(mdns_sd::Error::Again) if self.retries < MAX_RETRIES => { + self.retries += 1; + self.due = Some(now.saturating_add(MIN_REFRESH_MS << (self.retries - 1))); + } + Err(error) => { + self.recovered.clear(); + // No busyloop, implicit publisher replacement or health claim. + tracing::warn!(%error, "Discovery interface refresh was not accepted"); + } + } + } +} + +enum EventRead { + Event(DaemonEvent), + Idle, + Closed, +} +trait RecoveryEvents: Send + 'static { + fn read(&self, timeout: Duration) -> EventRead; } +impl RecoveryEvents for mdns_sd::Receiver { + fn read(&self, timeout: Duration) -> EventRead { + match self.recv_timeout(timeout) { + Ok(event) => EventRead::Event(event), + Err(_) if self.is_disconnected() => EventRead::Closed, + Err(_) => EventRead::Idle, + } + } +} + +struct RecoveryWorker { + stop: Arc, + thread: Option>, +} +impl RecoveryWorker { + fn start( + daemon: ServiceDaemon, + events: mdns_sd::Receiver, + template: ServiceInfo, + ) -> Result { + Self::start_with(events, move || daemon.register(template.clone())) + } + fn start_with( + events: impl RecoveryEvents, + mut register: impl FnMut() -> Result<(), mdns_sd::Error> + Send + 'static, + ) -> Result { + let stop = Arc::new(AtomicBool::new(false)); + let cancelled = stop.clone(); + let thread = thread::Builder::new() + .name("ruview-mdns-recovery".into()) + .spawn(move || { + let origin = Instant::now(); + let now = || origin.elapsed().as_millis().min(u64::MAX as u128) as u64; + let mut state = RecoveryState::default(); + while !cancelled.load(Ordering::Acquire) { + match events.read(Duration::from_millis(100)) { + EventRead::Event(event) => state.event(event, now()), + EventRead::Closed => break, + EventRead::Idle => {} + } + let mut closed = false; + // Drain one bounded monitor batch before acting, including any + // queued IpDel. A continuous event producer cannot starve stop. + for _ in 0..100 { + if cancelled.load(Ordering::Acquire) { + break; + } + match events.read(Duration::ZERO) { + EventRead::Event(event) => state.event(event, now()), + EventRead::Closed => { + closed = true; + break; + } + EventRead::Idle => break, + } + } + if cancelled.load(Ordering::Acquire) || closed { + break; + } + state.poll(now(), || { + if cancelled.load(Ordering::Acquire) { + return Ok(()); + } + register() + }); + } + state.stop(); + }) + .map_err(|error| { + mdns_sd::Error::Msg(format!( + "Discovery recovery worker could not start: {error}" + )) + })?; + Ok(Self { + stop, + thread: Some(thread), + }) + } + fn stop(&mut self) { + self.stop.store(true, Ordering::Release); + if let Some(thread) = self.thread.take() { + let _ = thread.join(); + } + } +} + +impl Drop for RecoveryWorker { + fn drop(&mut self) { + self.stop(); + } +} + +#[cfg(test)] +#[path = "discovery_recovery_tests.rs"] +mod recovery_tests; #[cfg(test)] mod tests { diff --git a/v2/crates/wifi-densepose-sensing-server/src/discovery_recovery_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/discovery_recovery_tests.rs new file mode 100644 index 0000000000..43272fc592 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/discovery_recovery_tests.rs @@ -0,0 +1,223 @@ +//! No live daemon/socket/network: exercise the production reducer and worker +//! through synthetic event/registration ports. +use super::*; +use std::sync::{mpsc, Mutex}; + +fn add(ip: &str) -> DaemonEvent { + DaemonEvent::IpAdd(ip.parse().unwrap()) +} +fn del(ip: &str) -> DaemonEvent { + DaemonEvent::IpDel(ip.parse().unwrap()) +} + +#[test] +fn failed_bind_then_ip_add_refreshes_exact_pristine_service_without_event_address() { + let template = build_service( + "same companion", + "same-installation", + "ruview-same.local.", + 3443, + true, + ) + .unwrap(); + let original = format!("{template:?}"); + let mut state = RecoveryState::default(); + let mut sent = Vec::new(); + // A failed bind/Error itself is not an instruction to repeatedly announce. + state.event( + DaemonEvent::Error(mdns_sd::Error::Msg("synthetic failed bind".into())), + 0, + ); + state.poll(1000, || { + panic!("failure alone must not enqueue"); + }); + state.event(add("192.168.1.2"), 1000); + state.poll(1249, || { + panic!("coalescing boundary"); + }); + state.poll(1250, || { + sent.push(template.clone()); + Ok(()) + }); + assert_eq!(sent.len(), 1); + assert_eq!(format!("{:?}", sent[0]), original); + assert!( + sent[0].get_addresses().is_empty(), + "event address must never be injected" + ); + assert_eq!(sent[0].get_hostname(), "ruview-same.local."); + assert_eq!(sent[0].get_port(), 3443); + assert_eq!(sent[0].get_property_val_str("tls"), Some("1")); +} + +#[test] +fn duplicate_burst_and_irrelevant_events_are_bounded_without_postponing_refresh() { + let mut state = RecoveryState::default(); + for i in 1..=255 { + state.event(add(&format!("10.0.0.{i}")), i as u64); + } + for _ in 0..1000 { + state.event(add("10.0.0.1"), 249); + } + assert_eq!(state.recovered.len(), MAX_RECOVERED_IPS); + assert_eq!(state.due, Some(251)); + let mut calls = 0; + state.poll(251, || { + calls += 1; + Ok(()) + }); + state.event(add("127.0.0.1"), 1000); + state.event(add("8.8.8.8"), 1000); + state.event(add("169.254.1.2"), 1000); + state.event(add("::1"), 1000); + state.event( + DaemonEvent::Announce("foreign".into(), "not-health".into()), + 1000, + ); + state.poll(u64::MAX, || { + calls += 1; + Ok(()) + }); + assert_eq!(calls, 1); +} + +#[test] +fn same_ip_rebind_refreshes_with_or_without_a_del_notification() { + let mut state = RecoveryState::default(); + let mut calls = 0; + state.event(add("10.0.0.1"), 0); + state.poll(250, || { + calls += 1; + Ok(()) + }); + state.event(del("10.0.0.1"), 300); + state.event(add("10.0.0.1"), 301); + assert_eq!(state.due, Some(1250)); + state.poll(1250, || { + calls += 1; + Ok(()) + }); + // The monitor may drop IpDel when full: do not suppress this real rebind. + for _ in 0..1000 { + state.event(add("10.0.0.1"), 1251); + } + assert_eq!(state.due, Some(2250)); + state.poll(2249, || panic!("refresh storm")); + state.poll(2250, || { + calls += 1; + Ok(()) + }); + assert_eq!(calls, 3); +} + +#[test] +fn settled_wakeups_do_not_fill_a_permanent_interface_cache_when_ip_del_is_lost() { + let mut state = RecoveryState::default(); + let mut calls = 0; + for i in 1..=32 { + let now = i * 2000; + state.event(add(&format!("10.0.0.{i}")), now); + state.poll(now + COALESCE_MS, || { + calls += 1; + Ok(()) + }); + assert!(state.recovered.is_empty()); + } + assert_eq!(calls, 32); +} + +#[test] +fn queued_ip_del_and_stop_withdraw_pending_refresh() { + let mut state = RecoveryState::default(); + state.event(add("192.168.1.2"), 0); + state.event(del("192.168.1.2"), 249); + state.poll(250, || panic!("deleted wakeup must be withdrawn")); + state.event(add("192.168.1.2"), 251); + state.stop(); + state.event(add("10.0.0.1"), 1000); + state.poll(u64::MAX, || panic!("stopped generation must not register")); + assert!(state.recovered.is_empty()); +} + +#[test] +fn again_has_three_bounded_backoffs_then_stops_and_terminal_error_does_not_spin() { + let mut state = RecoveryState::default(); + state.event(add("10.0.0.1"), 0); + let mut calls = 0; + for (now, next) in [ + (250, Some(1250)), + (1250, Some(3250)), + (3250, Some(7250)), + (7250, None), + ] { + state.poll(now, || { + calls += 1; + Err(mdns_sd::Error::Again) + }); + assert_eq!(state.due, next); + state.poll(now, || panic!("no busyloop")); + } + assert_eq!(calls, 4); + state.event(add("10.0.0.2"), 8000); + state.poll(8250, || { + Err(mdns_sd::Error::Msg("synthetic closed daemon".into())) + }); + assert!(state.due.is_none()); +} + +struct TestEvents(mpsc::Receiver); +impl RecoveryEvents for TestEvents { + fn read(&self, timeout: Duration) -> EventRead { + match self.0.recv_timeout(timeout) { + Ok(event) => EventRead::Event(event), + Err(mpsc::RecvTimeoutError::Timeout) => EventRead::Idle, + Err(mpsc::RecvTimeoutError::Disconnected) => EventRead::Closed, + } + } +} + +#[test] +fn monitor_close_discards_pending_intent_and_worker_joins() { + let (tx, rx) = mpsc::channel(); + tx.send(add("10.0.0.1")).unwrap(); + drop(tx); + let mut worker = + RecoveryWorker::start_with(TestEvents(rx), || panic!("closed monitor cannot register")) + .unwrap(); + worker.thread.take().unwrap().join().unwrap(); +} + +#[test] +fn worker_stop_joins_inflight_registration_before_teardown_and_never_enqueues_again() { + let (tx, rx) = mpsc::channel(); + let (entered_tx, entered_rx) = mpsc::channel(); + let (release_tx, release_rx) = mpsc::channel(); + let (done_tx, done_rx) = mpsc::channel(); + let calls = Arc::new(Mutex::new(0)); + let seen = calls.clone(); + let worker = RecoveryWorker::start_with(TestEvents(rx), move || { + *seen.lock().unwrap() += 1; + entered_tx.send(()).unwrap(); + release_rx.recv_timeout(Duration::from_secs(3)).unwrap(); + Ok(()) + }) + .unwrap(); + tx.send(add("10.0.0.1")).unwrap(); + entered_rx.recv_timeout(Duration::from_secs(3)).unwrap(); + // Force stop publication before releasing the in-flight fake command. + worker.stop.store(true, Ordering::Release); + tx.send(del("10.0.0.1")).unwrap(); + tx.send(add("10.0.0.2")).unwrap(); + let stopper = thread::spawn(move || { + drop(worker); + done_tx.send(()).unwrap(); + }); + assert!( + done_rx.try_recv().is_err(), + "teardown must wait for dispatched registration" + ); + release_tx.send(()).unwrap(); + done_rx.recv_timeout(Duration::from_secs(3)).unwrap(); + stopper.join().unwrap(); + assert_eq!(*calls.lock().unwrap(), 1); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/field_bridge.rs b/v2/crates/wifi-densepose-sensing-server/src/field_bridge.rs index 6793cc26f3..7d6a3539b9 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/field_bridge.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/field_bridge.rs @@ -264,7 +264,7 @@ pub fn bootstrap_background_match( observed_at_us: u64, ) -> Option { if frame_history.len() < OCCUPANCY_WINDOW - || field.check_freshness(observed_at_us) != CalibrationStatus::Fresh + || !matches!(field.check_freshness(observed_at_us), CalibrationStatus::Fresh | CalibrationStatus::Stale) { return None; } @@ -923,6 +923,11 @@ mod tests { .is_some_and(|score| (0.5..=1.0).contains(&score))); let short_history: VecDeque> = empty_history.iter().take(49).cloned().collect(); + // Staleness withdraws full calibrated authority after half the lifetime, + // but the persisted negative-only reference remains useful until expiry. + assert!(bootstrap_background_match(&field, &empty_history, 32_000_000).is_some()); + assert!(calibrated_occupancy(&field, &empty_history, 32_000_000).is_none()); + assert!(bootstrap_background_match(&field, &empty_history, 62_000_000).is_none()); assert!(!bootstrap_empty_prior_applies( &field, &short_history, diff --git a/v2/crates/wifi-densepose-sensing-server/src/legacy_grid_readiness.rs b/v2/crates/wifi-densepose-sensing-server/src/legacy_grid_readiness.rs new file mode 100644 index 0000000000..5e55ebf155 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/legacy_grid_readiness.rs @@ -0,0 +1,179 @@ +//! Header-only admission for the legacy ONE-author exact-grid baseline. +//! This is not managed per-source learning or a hardware/accuracy claim. +use super::*; +use std::time::Instant; +const MAX_REPORTED_GRIDS: usize = 8; +#[derive(Debug, Clone, Copy, Serialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub(super) enum GridReadinessReason { + Compatible, + NodeMissing, + RawGridMissing, + GridEvidenceWarming, + GridEvidenceStale, + NonmatchingQualifiedGrid, + GridEvidenceUnqualified, +} +#[derive(Debug, Serialize)] +pub(super) struct BoundNodeGridReadiness { + pub node_id: u8, + pub ready: bool, + pub reason: GridReadinessReason, + pub observed_grids: Vec, + pub eligible_grids: Vec, + pub observed_grids_truncated: bool, + pub eligible_grids_truncated: bool, +} +#[derive(Debug, Serialize)] +pub(super) struct LegacyGridReadiness { + pub schema: &'static str, + pub author_source_node_id: u8, + pub selected_grid: CsiGridKey, + pub all_bound_nodes_ready: bool, + pub nodes: Vec, +} +/// Shared with the existing author selection: always inspect every candidate, +/// never infer compatibility from a node's preferred grid or board label. +pub(super) fn eligible_grid(grid: CsiGridKey, evidence: CalibrationGridEvidence) -> bool { + grid.n_subcarriers > 0 + && grid.ppdu_type <= 3 + && evidence.span_s >= CALIBRATION_GRID_MIN_SPAN_S + && evidence.rate_hz >= CALIBRATION_GRID_MIN_RATE_HZ + && evidence.max_gap_s < CALIBRATION_GRID_MAX_GAP_S + && evidence.latest_age_s < CALIBRATION_GRID_MAX_GAP_S +} +fn sorted_grids(mut values: Vec, selected: CsiGridKey) -> Vec { + values.retain(|g| g.n_subcarriers > 0); + values.sort_by_key(|g| { + ( + *g != selected, + std::cmp::Reverse(g.n_subcarriers), + g.ppdu_type, + ) + }); + values.dedup(); + values +} +fn node_readiness( + id: u8, + node: Option<&NodeState>, + selected: CsiGridKey, + now: Instant, +) -> BoundNodeGridReadiness { + let Some(node) = node else { + return BoundNodeGridReadiness { + node_id: id, + ready: false, + reason: GridReadinessReason::NodeMissing, + observed_grids: vec![], + eligible_grids: vec![], + observed_grids_truncated: false, + eligible_grids_truncated: false, + }; + }; + let candidates = node.calibration_grid_candidates(now); + let matching = candidates + .iter() + .find(|(g, _)| *g == selected) + .map(|(_, e)| *e); + let ready = matching.is_some_and(|e| eligible_grid(selected, e)); + let other_qualified = candidates + .iter() + .any(|(g, e)| *g != selected && eligible_grid(*g, *e)); + // A fresh but still warming/weak matching stream is not measured proof of a + // hardware change, even when a different grid is currently more frequent. + let reason = if ready { + GridReadinessReason::Compatible + } else if let Some(e) = matching.filter(|e| e.latest_age_s < CALIBRATION_GRID_MAX_GAP_S) { + if e.span_s < CALIBRATION_GRID_MIN_SPAN_S { + GridReadinessReason::GridEvidenceWarming + } else { + GridReadinessReason::GridEvidenceUnqualified + } + } else if other_qualified { + GridReadinessReason::NonmatchingQualifiedGrid + } else if node.raw_grid_observations.is_empty() { + GridReadinessReason::RawGridMissing + } else if candidates.is_empty() + || candidates + .iter() + .all(|(_, e)| e.latest_age_s >= CALIBRATION_GRID_MAX_GAP_S) + { + GridReadinessReason::GridEvidenceStale + } else if candidates.iter().any(|(g, e)| { + g.ppdu_type <= 3 && g.n_subcarriers > 0 && e.span_s < CALIBRATION_GRID_MIN_SPAN_S + }) { + GridReadinessReason::GridEvidenceWarming + } else { + GridReadinessReason::GridEvidenceUnqualified + }; + let all_observed = sorted_grids(candidates.iter().map(|(g, _)| *g).collect(), selected); + let all_eligible = sorted_grids( + candidates + .iter() + .filter(|(g, e)| eligible_grid(*g, *e)) + .map(|(g, _)| *g) + .collect(), + selected, + ); + // Joint projection preserves subset semantics under the cap. Reserve the + // selected observed-but-unqualified tuple rather than hiding why it blocks. + let selected_observed = all_observed.contains(&selected); + let reserve = usize::from(selected_observed && !all_eligible.contains(&selected)); + let eligible_grids: Vec<_> = all_eligible + .iter() + .copied() + .take(MAX_REPORTED_GRIDS - reserve) + .collect(); + let mut observed_grids = Vec::new(); + if selected_observed { + observed_grids.push(selected); + } + for g in eligible_grids.iter().chain(&all_observed) { + if observed_grids.len() == MAX_REPORTED_GRIDS { + break; + } + if !observed_grids.contains(g) { + observed_grids.push(*g); + } + } + let observed_grids_truncated = all_observed.len() > observed_grids.len(); + let eligible_grids_truncated = all_eligible.len() > eligible_grids.len(); + BoundNodeGridReadiness { + node_id: id, + ready, + reason, + observed_grids, + eligible_grids, + observed_grids_truncated, + eligible_grids_truncated, + } +} +impl LegacyGridReadiness { + pub(super) fn known_incompatibility(&self) -> bool { + self.nodes + .iter() + .any(|n| n.reason == GridReadinessReason::NonmatchingQualifiedGrid) + } +} +pub(super) fn inspect( + state: &AppStateInner, + ids: &[u8], + binding: CalibrationGridBinding, + now: Instant, +) -> Option { + if !valid_source_node_ids(ids) || ids.binary_search(&binding.source_node_id).is_err() { + return None; + } + let nodes: Vec<_> = ids + .iter() + .map(|id| node_readiness(*id, state.node_states.get(id), binding.grid, now)) + .collect(); + Some(LegacyGridReadiness { + schema: "ruview.legacy-calibration-grid-readiness.v1", + author_source_node_id: binding.source_node_id, + selected_grid: binding.grid, + all_bound_nodes_ready: nodes.iter().all(|n| n.ready), + nodes, + }) +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/legacy_grid_readiness_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/legacy_grid_readiness_tests.rs new file mode 100644 index 0000000000..a41ac32455 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/legacy_grid_readiness_tests.rs @@ -0,0 +1,543 @@ +//! SYNTHETIC header-grid fixtures through actual legacy calibration handlers. +use super::*; +use std::time::{Duration, Instant}; +fn grid(bins: u16, ppdu: u8) -> CsiGridKey { + CsiGridKey { + n_subcarriers: bins, + ppdu_type: ppdu, + } +} +fn ready_node(grids: &[CsiGridKey], now: Instant) -> NodeState { + let mut node = NodeState::new(); + for index in 0..=48 { + for grid in grids { + node.observe_raw_grid( + *grid, + now - Duration::from_secs(12) + Duration::from_millis(index * 250), + ); + } + } + node +} +fn request(ids: Vec) -> CalibrationBindingRequest { + CalibrationBindingRequest { + binding_digest: "a".repeat(64), + source_node_ids: ids, + } +} +#[tokio::test] +async fn mixed_he256_ht64_is_rejected_before_existing_baseline_or_receipt_mutation() { + let now = Instant::now(); + let mut inner = AppStateInner::minimal(); + inner + .node_states + .insert(3, ready_node(&[grid(256, 1)], now)); + inner.node_states.insert(5, ready_node(&[grid(64, 0)], now)); + inner + .node_states + .insert(99, ready_node(&[grid(192, 3)], now)); + inner.field_model = Some(FieldModel::new(field_bridge::single_link_config()).unwrap()); + inner.bootstrap_baseline_active = true; + inner.calibration_model_id = Some("old-model-must-stay".into()); + inner.calibration_session_id = Some("old-session-must-stay".into()); + inner.calibration_binding_digest = Some("b".repeat(64)); + inner.calibration_source_node_ids.extend([3, 5]); + inner.calibration_observed_source_node_ids.insert(3); + inner.calibration_model_receipt = Some(CalibrationModelReceipt { + schema: "ruview.room-calibration.model-receipt.v1".into(), + boot_epoch: inner.calibration_boot_epoch.clone(), + session_id: "old-session-must-stay".into(), + model_id: "old-model-must-stay".into(), + binding_digest: "b".repeat(64), + source_node_ids: vec![3, 5], + frame_count: 12345, + variance_explained: 0.95, + baseline_eigenvalue_count: 2, + completed_at_unix_ms: 1793800000000, + }); + let before = serde_json::to_value(inner.calibration_model_receipt.clone()).unwrap(); + let state = Arc::new(RwLock::new(inner)); + let Json(result) = calibration_start( + State(state.clone()), + Query(CalibrationStartQuery { + source_node_id: Some(3), + }), + None, + None, + Json(request(vec![3, 5])), + ) + .await; + assert_eq!( + result["success"], false, + "heterogeneous raw grids must fail before the ten-minute run: {result}" + ); + assert_eq!(result["error_code"], "calibration_bound_grid_unavailable"); + let preserved = state.read().await; + assert!(preserved.field_model.is_some()); + assert!(preserved.bootstrap_baseline_active); + assert_eq!( + preserved.calibration_model_id.as_deref(), + Some("old-model-must-stay") + ); + assert_eq!( + preserved.calibration_session_id.as_deref(), + Some("old-session-must-stay") + ); + assert_eq!( + preserved.calibration_binding_digest.as_deref(), + Some("b".repeat(64).as_str()) + ); + assert_eq!( + preserved.calibration_observed_source_node_ids, + [3].into_iter().collect() + ); + assert_eq!( + serde_json::to_value(&preserved.calibration_model_receipt).unwrap(), + before + ); +} +async fn start_nodes( + nodes: Vec<(u8, NodeState)>, + ids: Vec, + author: u8, +) -> (SharedState, serde_json::Value) { + let mut inner = AppStateInner::minimal(); + inner.node_states.extend(nodes); + let state = Arc::new(RwLock::new(inner)); + let Json(result) = calibration_start( + State(state.clone()), + Query(CalibrationStartQuery { + source_node_id: Some(author), + }), + None, + None, + Json(request(ids)), + ) + .await; + (state, result) +} +#[tokio::test] +async fn same_bin_count_different_ppdu_is_not_a_compatible_legacy_grid() { + let now = Instant::now(); + let (state, result) = start_nodes( + vec![ + (3, ready_node(&[grid(256, 1)], now)), + (5, ready_node(&[grid(256, 2)], now)), + ], + vec![3, 5], + 3, + ) + .await; + assert_eq!(result["success"], false); + assert_eq!( + result["source_grid_readiness"]["nodes"][1]["reason"], + "nonmatching_qualified_grid" + ); + assert_eq!( + result["source_grid_readiness"]["nodes"][1]["eligible_grids"][0], + serde_json::json!({"n_subcarriers":256,"ppdu_type":2}) + ); + assert!(state.read().await.field_model.is_none()); +} +#[tokio::test] +async fn every_bound_node_can_supply_the_exact_grid_and_unbound_nodes_do_not_leak() { + let now = Instant::now(); + let selected = grid(64, 0); + let (state, result) = start_nodes( + vec![ + (3, ready_node(&[selected], now)), + (5, ready_node(&[selected], now)), + (99, ready_node(&[grid(256, 3)], now)), + ], + vec![3, 5], + 3, + ) + .await; + assert_eq!(result["success"], true); + assert_eq!( + result["source_grid_readiness"]["all_bound_nodes_ready"], + true + ); + let Json(status) = calibration_status(State(state)).await; + assert_eq!( + status["source_grid_readiness"]["nodes"] + .as_array() + .unwrap() + .iter() + .map(|n| n["node_id"].as_u64().unwrap()) + .collect::>(), + vec![3, 5] + ); + assert!(!status["source_grid_readiness"].to_string().contains("99")); +} +#[tokio::test] +async fn nonpreferred_but_eligible_shared_grid_passes_without_blending_model_authors() { + let now = Instant::now(); + let selected = grid(64, 0); + let dense = grid(256, 1); + let c6 = ready_node(&[selected, dense], now); + assert_eq!(c6.select_calibration_grid(now).unwrap().0, dense); + let (state, result) = start_nodes( + vec![(3, c6), (5, ready_node(&[selected], now))], + vec![3, 5], + 5, + ) + .await; + assert_eq!(result["success"], true, "{result}"); + assert_eq!( + result["source_grid"], + serde_json::json!({"n_subcarriers":64,"ppdu_type":0}) + ); + assert_eq!( + result["source_grid_readiness"]["nodes"][0]["eligible_grids"][0], + result["source_grid"] + ); + let mut inner = state.write().await; + let values = vec![2.0; 64]; + let before = inner + .field_model + .as_ref() + .unwrap() + .calibration_frame_count(); + assert!(inner.maybe_feed_calibration_frame(3, selected, 1, &values, Instant::now())); + assert_eq!( + inner + .field_model + .as_ref() + .unwrap() + .calibration_frame_count(), + before + ); + assert!(inner.calibration_observed_source_node_ids.contains(&3)); + assert!(inner.maybe_feed_calibration_frame(5, selected, 1, &values, Instant::now())); + assert_eq!( + inner + .field_model + .as_ref() + .unwrap() + .calibration_frame_count(), + before + 1 + ); +} +#[tokio::test] +async fn missing_edge_only_warming_and_stale_nodes_are_not_claimed_incompatible_hardware() { + let now = Instant::now(); + let selected = grid(64, 0); + let cases = [ + "node_missing", + "raw_grid_missing", + "grid_evidence_warming", + "grid_evidence_stale", + "grid_evidence_unqualified", + ]; + for reason in cases { + let mut nodes = vec![(3, ready_node(&[selected], now))]; + if reason != "node_missing" { + let mut other = if reason == "grid_evidence_stale" { + ready_node(&[selected], now - Duration::from_secs(6)) + } else { + NodeState::new() + }; + other.last_frame_time = Some(now); // Edge/other liveness is not raw grid proof. + if reason == "grid_evidence_warming" { + other.observe_raw_grid(selected, now); + } + if reason == "grid_evidence_unqualified" { + for i in 0..=12 { + other.observe_raw_grid( + selected, + now - Duration::from_secs(12) + Duration::from_secs(i), + ); + } + } + nodes.push((5, other)); + } + let (_, result) = start_nodes(nodes, vec![3, 5], 3).await; + assert_eq!(result["success"], false, "{reason}: {result}"); + assert_eq!( + result["source_grid_readiness"]["nodes"][1]["reason"], reason, + "{result}" + ); + assert_eq!(result["source_grid_readiness"]["nodes"][1]["ready"], false); + } +} +#[tokio::test] +async fn truncated_diagnostics_keep_selected_and_all_reported_eligible_tuples() { + let now = Instant::now(); + let selected = grid(64, 0); + let mut other = ready_node( + &(0..9).map(|n| grid(80 + n * 16, 0)).collect::>(), + now, + ); + other.observe_raw_grid(selected, now); + let (_, result) = start_nodes( + vec![(3, ready_node(&[selected], now)), (5, other)], + vec![3, 5], + 3, + ) + .await; + let node = &result["source_grid_readiness"]["nodes"][1]; + assert_eq!(node["reason"], "grid_evidence_warming"); + assert_eq!(node["observed_grids_truncated"], true); + assert_eq!(node["eligible_grids_truncated"], true); + let observed = node["observed_grids"].as_array().unwrap(); + let eligible = node["eligible_grids"].as_array().unwrap(); + assert!(observed.len() <= 8 && eligible.len() <= 8); + assert_eq!(observed[0], result["source_grid"]); + assert!(eligible.iter().all(|g| observed.contains(g))); +} +#[tokio::test] +async fn current_status_reports_grid_drift_without_rewriting_historical_contributions() { + let now = Instant::now(); + let selected = grid(64, 0); + let (state, result) = start_nodes( + vec![ + (3, ready_node(&[selected], now)), + (5, ready_node(&[selected], now)), + ], + vec![3, 5], + 3, + ) + .await; + assert_eq!(result["success"], true); + { + let mut inner = state.write().await; + inner.calibration_observed_source_node_ids.extend([3, 5]); + inner + .node_states + .insert(5, ready_node(&[grid(256, 1)], Instant::now())); + } + let Json(status) = calibration_status(State(state.clone())).await; + assert_eq!(status["missing_source_node_ids"], serde_json::json!([])); + assert_eq!( + status["source_grid_readiness"]["nodes"][1]["reason"], + "nonmatching_qualified_grid" + ); + assert_eq!( + status["source_grid_readiness"]["all_bound_nodes_ready"], + false + ); + assert_eq!( + state.read().await.calibration_observed_source_node_ids, + [3, 5].into_iter().collect() + ); +} +#[tokio::test] +async fn idle_status_does_not_expose_leftover_or_unbound_grid_diagnostics() { + let mut inner = AppStateInner::minimal(); + let now = Instant::now(); + inner + .node_states + .insert(99, ready_node(&[grid(256, 1)], now)); + inner.calibration_grid_binding = Some(CalibrationGridBinding { + source_node_id: 99, + grid: grid(256, 1), + evidence: None, + }); + let Json(status) = calibration_status(State(Arc::new(RwLock::new(inner)))).await; + assert!(status["source_grid_readiness"].is_null()); + assert_eq!(status["source_node_ids"], serde_json::json!([])); +} +fn current_identity(state: &AppStateInner) -> CalibrationIdentityRequest { + CalibrationIdentityRequest { + boot_epoch: state.calibration_boot_epoch.clone(), + session_id: state.calibration_session_id.clone(), + binding_digest: state.calibration_binding_digest.clone(), + source_node_ids: Some(state.calibration_source_node_ids.iter().copied().collect()), + } +} +#[tokio::test] +async fn post_start_known_grid_drift_blocks_only_new_finalization_and_preserves_capture() { + let now = Instant::now(); + let selected = grid(64, 0); + let different = grid(256, 1); + let (state, started) = start_nodes( + vec![ + (3, ready_node(&[selected], now)), + (5, ready_node(&[selected], now)), + ], + vec![3, 5], + 3, + ) + .await; + assert_eq!(started["success"], true); + let identity; + { + let mut inner = state.write().await; + // Short SYNTHETIC model fixture, not a shortened production capture. + let mut config = field_bridge::single_link_config(); + config.min_calibration_frames = 4; + config.min_calibration_duration_s = 0.0; + inner.field_model = Some(FieldModel::new(config).unwrap()); + for seq in 1..=4 { + let values: Vec<_> = (0..64) + .map(|bin| { + 2.0 + f64::from(seq) * 0.04 + (bin as f64 * 0.3 + f64::from(seq)).sin() * 0.01 + }) + .collect(); + assert!(inner.maybe_feed_calibration_frame(3, selected, seq, &values, Instant::now())); + } + assert!(inner.maybe_feed_calibration_frame(5, selected, 1, &[2.0; 64], Instant::now())); + inner.node_states.get_mut(&5).unwrap().raw_grid_observations = + ready_node(&[different], Instant::now()).raw_grid_observations; + identity = current_identity(&inner); + } + let Json(blocked) = calibration_stop(State(state.clone()), Json(identity)).await; + assert_eq!( + blocked["error_code"], "calibration_bound_grid_unavailable", + "{blocked}" + ); + { + let inner = state.read().await; + assert_eq!( + inner.field_model.as_ref().unwrap().status(), + CalibrationStatus::Collecting + ); + assert_eq!( + inner + .field_model + .as_ref() + .unwrap() + .calibration_frame_count(), + 4 + ); + assert!(inner.calibration_model_receipt.is_none()); + assert_eq!( + inner.calibration_observed_source_node_ids, + [3, 5].into_iter().collect() + ); + assert_eq!(inner.node_states[&3].field_model_history.len(), 4); + assert_eq!(inner.node_states[&5].field_model_history.len(), 1); + } + // A qualified common minority grid is sufficient at finalization too. + let identity = { + let mut inner = state.write().await; + inner.node_states.get_mut(&5).unwrap().raw_grid_observations = + ready_node(&[selected, different], Instant::now()).raw_grid_observations; + current_identity(&inner) + }; + let Json(finished) = calibration_stop(State(state.clone()), Json(identity)).await; + assert_eq!(finished["success"], true, "{finished}"); + assert_eq!( + state.read().await.field_model.as_ref().unwrap().status(), + CalibrationStatus::Fresh + ); + // Completed receipt/metadata calls do not acquire a new all-node readiness gate. + let identity = { + let mut inner = state.write().await; + inner.node_states.get_mut(&5).unwrap().raw_grid_observations = + ready_node(&[different], Instant::now()).raw_grid_observations; + current_identity(&inner) + }; + let Json(historical) = calibration_stop(State(state.clone()), Json(identity)).await; + assert_ne!( + historical["error_code"], + "calibration_bound_grid_unavailable" + ); + assert!(state.read().await.calibration_model_receipt.is_some()); +} +#[tokio::test] +async fn warming_matching_grid_is_not_a_new_stop_incompatibility_gate() { + let now = Instant::now(); + let selected = grid(64, 0); + let (state, _) = start_nodes( + vec![ + (3, ready_node(&[selected], now)), + (5, ready_node(&[selected], now)), + ], + vec![3, 5], + 3, + ) + .await; + let identity = { + let mut inner = state.write().await; + assert!(inner.maybe_feed_calibration_frame(3, selected, 1, &[2.0; 64], Instant::now())); + assert!(inner.maybe_feed_calibration_frame(5, selected, 1, &[2.0; 64], Instant::now())); + let other = inner.node_states.get_mut(&5).unwrap(); + other.raw_grid_observations.clear(); + other.observe_raw_grid(selected, Instant::now()); + current_identity(&inner) + }; + let Json(stopped) = calibration_stop(State(state), Json(identity)).await; + assert_ne!(stopped["error_code"], "calibration_bound_grid_unavailable"); + assert!(stopped["error"].as_str().unwrap().contains("not complete")); +} +#[tokio::test] +async fn export_actual_legacy_grid_readiness_contract_cases() { + let now = Instant::now(); + let selected = grid(64, 0); + let dense = grid(256, 1); + let (_, mixed) = start_nodes( + vec![ + (3, ready_node(&[dense], now)), + (5, ready_node(&[selected], now)), + ], + vec![3, 5], + 3, + ) + .await; + let (healthy_state, started) = start_nodes( + vec![ + (3, ready_node(&[dense, selected], now)), + (5, ready_node(&[selected], now)), + ], + vec![3, 5], + 5, + ) + .await; + assert_eq!(started["success"], true); + let Json(healthy_status) = calibration_status(State(healthy_state.clone())).await; + { + let mut inner = healthy_state.write().await; + inner.node_states.get_mut(&3).unwrap().raw_grid_observations = + ready_node(&[dense], now).raw_grid_observations; + } + let Json(drift_status) = calibration_status(State(healthy_state)).await; + for status in [&healthy_status, &drift_status] { + assert_eq!( + status["source_grid_readiness"]["author_source_node_id"], + status["grid_binding"]["source_node_id"] + ); + assert_eq!( + status["source_grid_readiness"]["selected_grid"], + status["grid_binding"]["grid"] + ); + } + + let mut warming = NodeState::new(); + warming.observe_raw_grid(selected, now); + let (_, warming_refusal) = start_nodes( + vec![(3, ready_node(&[selected], now)), (5, warming)], + vec![3, 5], + 3, + ) + .await; + let (_, stale_refusal) = start_nodes( + vec![ + (3, ready_node(&[selected], now)), + (5, ready_node(&[selected], now - Duration::from_secs(6))), + ], + vec![3, 5], + 3, + ) + .await; + let (_, missing_refusal) = + start_nodes(vec![(3, ready_node(&[selected], now))], vec![3, 5], 3).await; + let Json(idle_status) = + calibration_status(State(Arc::new(RwLock::new(AppStateInner::minimal())))).await; + let mut many = ready_node( + &(0..9).map(|n| grid(80 + n * 16, 0)).collect::>(), + now, + ); + many.observe_raw_grid(selected, now); + let (_, bounded_refusal) = start_nodes( + vec![(3, ready_node(&[selected], now)), (5, many)], + vec![3, 5], + 3, + ) + .await; + let cases = serde_json::json!({"fixture_kind":"synthetic_legacy_grid_handlers","requested_source_node_ids":[3,5],"mixed_start_refusal":mixed,"healthy_start":started,"healthy_status":healthy_status,"drift_status":drift_status,"warming_start_refusal":warming_refusal,"stale_start_refusal":stale_refusal,"missing_start_refusal":missing_refusal,"idle_status":idle_status,"bounded_start_refusal":bounded_refusal}); + if let Ok(path) = std::env::var("RUVIEW_GRID_READINESS_FIXTURE_OUT") { + std::fs::write(path, serde_json::to_vec_pretty(&cases).unwrap()).unwrap(); + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/lib.rs b/v2/crates/wifi-densepose-sensing-server/src/lib.rs index bc0b730211..d92d579dfb 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/lib.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/lib.rs @@ -10,6 +10,7 @@ pub mod bearer_auth; pub mod bootstrap_baseline; +pub mod room_calibration; pub mod browser_session; pub mod ws_ticket; pub mod cli; @@ -24,9 +25,12 @@ pub mod http_bind; pub mod inference; pub mod introspection; pub mod matter; +/// Existing MAT incident API, hosted under the sensing server's security layers. +pub mod mat_surface; pub mod model_format; pub mod mqtt; pub mod path_safety; +pub mod parent_lifetime; /// ADR-323: fail-closed pose physics integration and raw/refined view selection. pub mod pose_physics; /// #2094: server-wide `--privacy-mode` filter for REST, WebSocket and recordings. @@ -69,3 +73,12 @@ pub mod vendor_rf; // `crate::sona`) and public API (`wifi_densepose_sensing_server::embedding`, …) // are unchanged. pub use wifi_densepose_aether::{embedding, graph_transformer, sona, sparse_inference}; + +pub mod companion; +pub mod account_profile; +pub mod companion_tls; + +pub mod companion_sensors; + +/// Account-bound multi-source room learning and managed lifecycle. +pub mod room_learner; diff --git a/v2/crates/wifi-densepose-sensing-server/src/main.rs b/v2/crates/wifi-densepose-sensing-server/src/main.rs index 3b313f2b75..2690668d98 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/main.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/main.rs @@ -10,11 +10,17 @@ #![allow(dead_code)] mod adaptive_classifier; +mod companion_integration; +mod companion_room; +mod managed_worldgraph; +mod companion_spatial; mod rssi_motion; pub mod cli; pub mod csi; mod engine_bridge; mod field_bridge; +mod source_bank_shadow; +mod room_learner_api; mod field_localize; mod model_format; mod multistatic_bridge; @@ -42,6 +48,7 @@ mod source_fusion; mod tracker_bridge; pub mod types; mod vital_signs; +mod vital_publication; // Training pipeline modules (exposed via lib.rs) use wifi_densepose_sensing_server::{ @@ -106,6 +113,10 @@ use wifi_densepose_signal::ruvsense::pose_tracker::PoseTracker; #[derive(Parser, Debug)] #[command(name = "sensing-server", about = "WiFi-DensePose sensing server")] struct Args { + /// App-managed helper only: exit when this verified spawning parent ends. + #[arg(long)] + parent_pid: Option, + /// HTTP port for UI and REST API #[arg(long, default_value = "8080")] http_port: u16, @@ -173,6 +184,10 @@ struct Args { #[arg(long, default_value = "127.0.0.1", env = "SENSING_BIND_ADDR")] bind_addr: String, + /// Enable QR-pinned HTTPS companion service on this separate port. + #[arg(long, env = "RUVIEW_COMPANION_PORT")] + companion_port: Option, + /// Disable local `_ruview._tcp` discovery. Discovery is automatically /// skipped for loopback-only binds and never carries sensor data. #[arg(long, env = "RUVIEW_NO_MDNS", value_parser = cli::parse_env_bool)] @@ -308,6 +323,14 @@ struct Args { #[arg(long)] calibrate: bool, + /// Collect a memory-only per-source shadow during explicit room calibration; never promote it. + #[arg(long)] + source_bank_shadow: bool, + + /// Enable the account-owned managed capture/train/candidate room learner. + #[arg(long)] + room_learner: bool, + // --------------------------------------------------------------- // ADR-102: Edge Module Registry — surface the canonical Cognitum // cog catalog via `GET /api/v1/edge/registry`. @@ -438,6 +461,29 @@ struct CalibrationGridEvidence { latest_age_s: f64, } +impl CalibrationGridEvidence { + fn from_arrivals(arrivals: &[std::time::Instant], now: std::time::Instant) -> Option { + let first = *arrivals.first()?; + let last = *arrivals.last()?; + let span_s = last.saturating_duration_since(first).as_secs_f64(); + let latest_age_s = now.saturating_duration_since(last).as_secs_f64(); + let mut max_gap_s = latest_age_s; + for pair in arrivals.windows(2) { + max_gap_s = max_gap_s.max( + pair[1] + .saturating_duration_since(pair[0]) + .as_secs_f64(), + ); + } + let rate_hz = if span_s > 0.0 { + arrivals.len().saturating_sub(1) as f64 / span_s + } else { + 0.0 + }; + Some(Self {sample_count:arrivals.len(),span_s,rate_hz,max_gap_s,latest_age_s}) + } +} + #[derive(Debug, Clone, Copy)] struct CalibrationGridBinding { source_node_id: u8, @@ -445,19 +491,7 @@ struct CalibrationGridBinding { evidence: Option, } -#[derive(Debug, Clone, Serialize)] -struct CalibrationModelReceipt { - schema: &'static str, - boot_epoch: String, - session_id: String, - model_id: String, - binding_digest: String, - source_node_ids: Vec, - frame_count: u64, - variance_explained: f64, - baseline_eigenvalue_count: usize, - completed_at_unix_ms: u64, -} +use wifi_densepose_sensing_server::room_calibration::{self, RoomReceipt as CalibrationModelReceipt}; /// Per-frame occupancy result bound to one immutable field-model calibration /// receipt. Absent whenever the model is stale, unavailable, or cannot score @@ -504,10 +538,18 @@ struct SensingUpdate { nodes: Vec, features: FeatureInfo, classification: ClassificationInfo, + /// Legacy classification is retained only as a labelled comparison while a + /// selected binary model owns presence. It does not acquire learner authority. + #[serde(default, skip_serializing_if = "Option::is_none")] + classification_source: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + selected_binary_evidence: Option, signal_field: SignalField, /// Vital sign estimates (breathing rate, heart rate, confidence). #[serde(skip_serializing_if = "Option::is_none")] vital_signs: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + vital_evidence: Option, /// Strict calibrated occupancy evidence for this frame, bound to the /// active model receipt. Omitted when no calibrated result is available. #[serde(default, skip_serializing_if = "Option::is_none")] @@ -1152,6 +1194,9 @@ struct NodeState { rssi_history: VecDeque, vital_detector: VitalSignDetector, latest_vitals: VitalSigns, + vital_candidate: Option, + last_vital_decision: Option, + vital_sample_rate_resets: u64, pub(crate) last_frame_time: Option, edge_vitals: Option, /// ADR-110 §A0.12: Latest sync packet received from this node. When a @@ -1176,6 +1221,13 @@ struct NodeState { csi_fps_ema: f64, /// Number of inter-frame deltas observed (need ≥5 before trusting EMA). csi_fps_samples: u32, + /// Start of the current cadence window for frames admitted to the active + /// feature history. This is separate from raw-arrival FPS because + /// grid-rejected packets must not affect DSP frequency bins. + accepted_csi_rate_window_started_at: Option, + accepted_csi_rate_window_frames: u32, + accepted_csi_rate_hz: f64, + accepted_csi_rate_windows: u32, /// Latest extracted features for cross-node fusion. latest_features: Option, // ── RuVector Phase 2: Temporal smoothing & coherence gating ── @@ -1376,7 +1428,158 @@ mod fps_ema_tests { } } +#[cfg(test)] +mod accepted_history_cadence_tests { + use super::{NodeState, GRID_VOTE_FRAMES, MAX_PHYSICAL_CSI_FPS}; + use std::time::{Duration, Instant}; + use wifi_densepose_hardware::PpduType; + + fn warm(node: &mut NodeState, start: Instant, period_ms: u64) -> Instant { + let count = 1000 / period_ms; + for i in 0..=count { + node.observe_accepted_csi_frame(i as u32, false, start + Duration::from_millis(i * period_ms)); + } + start + Duration::from_millis(count * period_ms) + } + + #[test] + fn rejected_grid_arrivals_do_not_clock_accepted_feature_history() { + let mut node = NodeState::new(); + let start = Instant::now(); + let active = (256, PpduType::HtLegacy); + let rejected = (64, PpduType::HtLegacy); + for _ in 0..GRID_VOTE_FRAMES { assert!(node.accept_grid(active)); } + for i in 0..=400 { + let now = start + Duration::from_millis(i * 25); + if i % 2 == 0 { + assert!(node.accept_grid(active)); + node.observe_accepted_csi_frame(i as u32, true, now); + } else { + assert!(!node.accept_grid(rejected)); + node.observe_csi_frame_arrival(now); + } + } + let now = start + Duration::from_secs(10); + assert!((node.measured_feature_sample_rate_hz(now).unwrap() - 20.0).abs() < 0.01); + assert!((node.measured_sample_rate_hz().unwrap() - 40.0).abs() < 2.0); + assert_eq!(node.latest_csi_sequence, Some(400)); + assert!(node.latest_csi_sync_valid); + assert_eq!(node.last_frame_time, Some(now)); + assert_eq!(node.latest_accepted_csi_at, Some(now)); + } + + #[test] + fn accepted_bursts_count_every_frame_and_keep_raw_clock() { + let mut node = NodeState::new(); + let start = Instant::now(); + for i in 0..=40 { + let at = start + Duration::from_millis(i * 50); + node.observe_accepted_csi_frame((i * 2) as u32, false, at); + node.observe_accepted_csi_frame((i * 2 + 1) as u32, false, at + Duration::from_micros(40)); + } + let now = start + Duration::from_secs(2) + Duration::from_micros(40); + assert!((node.measured_feature_sample_rate_hz(now).unwrap() - 40.0).abs() < 0.01); + assert!((node.measured_sample_rate_hz().unwrap() - 40.0).abs() < 3.0); + } + + #[test] + fn warmed_measurement_expires_then_restarts_without_stale_rate() { + let mut node = NodeState::new(); + let start = Instant::now(); + assert!(node.measured_feature_sample_rate_hz(start).is_none()); + let end = warm(&mut node, start, 25); + assert_eq!(node.measured_feature_sample_rate_hz(end), Some(40.0)); + let later = end + Duration::from_secs(4); + assert!(node.measured_feature_sample_rate_hz(later).is_none()); + assert_eq!(node.feature_sample_rate_hz(later), 20.0); + node.observe_accepted_csi_frame(99, false, later); + assert!(node.measured_feature_sample_rate_hz(later).is_none()); + assert_eq!(node.accepted_csi_rate_windows, 0); + let end = warm(&mut node, later + Duration::from_millis(50), 50); + assert_eq!(node.measured_feature_sample_rate_hz(end), Some(20.0)); + } + + #[test] + fn grid_switch_resets_only_accepted_clock() { + let mut node = NodeState::new(); + let active = (128, PpduType::HtLegacy); + for _ in 0..GRID_VOTE_FRAMES { assert!(node.accept_grid(active)); } + let end = warm(&mut node, Instant::now(), 25); + let raw_before = (node.csi_fps_ema, node.csi_fps_samples, node.last_frame_time); + assert!(!node.accept_grid((64, PpduType::HtLegacy))); + assert_eq!(node.measured_feature_sample_rate_hz(end), Some(40.0)); + for _ in 0..GRID_VOTE_FRAMES { node.accept_grid((64, PpduType::HtLegacy)); } + assert_eq!(node.active_grid, Some((64, PpduType::HtLegacy))); + assert!(node.measured_feature_sample_rate_hz(end).is_none()); + assert_eq!(raw_before, (node.csi_fps_ema, node.csi_fps_samples, node.last_frame_time)); + } + + #[test] + fn per_node_measurements_and_existing_consumption_bounds_are_preserved() { + let mut first = NodeState::new(); + let mut second = NodeState::new(); + let start = Instant::now(); + let end = warm(&mut first, start, 25); + warm(&mut second, start, 50); + assert_eq!(first.measured_feature_sample_rate_hz(end), Some(40.0)); + assert_eq!(second.measured_feature_sample_rate_hz(end), Some(20.0)); + first.accepted_csi_rate_hz = f64::NAN; + assert!(first.measured_feature_sample_rate_hz(end).is_none()); + first.accepted_csi_rate_hz = 9000.0; + assert_eq!(first.measured_feature_sample_rate_hz(end), Some(MAX_PHYSICAL_CSI_FPS)); + } +} + impl NodeState { + /// Track only frames admitted to this node's active-grid feature history. + /// One-to-three-second monotonic windows retain valid burst deliveries and + /// exclude rejected grids. Longer gaps restart warm-up instead of turning + /// a transport outage into a bogus low sample rate. + fn observe_accepted_feature_frame(&mut self, now: std::time::Instant) { + let Some(started_at) = self.accepted_csi_rate_window_started_at else { + self.accepted_csi_rate_window_started_at = Some(now); + self.accepted_csi_rate_window_frames = 1; + return; + }; + + let elapsed = now.saturating_duration_since(started_at); + if elapsed > std::time::Duration::from_secs(3) { + self.accepted_csi_rate_window_started_at = Some(now); + self.accepted_csi_rate_window_frames = 1; + self.accepted_csi_rate_hz = 20.0; + self.accepted_csi_rate_windows = 0; + return; + } + + self.accepted_csi_rate_window_frames = + self.accepted_csi_rate_window_frames.saturating_add(1); + if elapsed >= std::time::Duration::from_secs(1) { + let intervals = self.accepted_csi_rate_window_frames.saturating_sub(1); + let measured = intervals as f64 / elapsed.as_secs_f64(); + self.accepted_csi_rate_hz = measured; + self.accepted_csi_rate_windows = self.accepted_csi_rate_windows.saturating_add(1); + self.accepted_csi_rate_window_started_at = Some(now); + self.accepted_csi_rate_window_frames = 1; + } + } + + /// Accepted-history cadence is unknown before warm-up and after a stall. + /// Keep this optional: the DSP fallback is not an observed measurement. + fn measured_feature_sample_rate_hz(&self, now: std::time::Instant) -> Option { + let age = now.checked_duration_since(self.latest_accepted_csi_at?)?; + if age > std::time::Duration::from_secs(3) + || self.accepted_csi_rate_windows == 0 + || !self.accepted_csi_rate_hz.is_finite() + { + return None; + } + Some(self.accepted_csi_rate_hz.clamp(1.0, MAX_PHYSICAL_CSI_FPS)) + } + + fn feature_sample_rate_hz(&self, now: std::time::Instant) -> f64 { + self.measured_feature_sample_rate_hz(now).unwrap_or(20.0) + } + /// Measured sample rate only after enough inter-frame deltas have warmed /// the EMA. The physical ceiling is enforced again at the consumption /// boundary so restored or malformed state cannot overclock DSP math. @@ -1528,6 +1731,7 @@ impl NodeState { sync_valid: bool, now: std::time::Instant, ) -> bool { + self.observe_accepted_feature_frame(now); self.latest_accepted_csi_at = Some(now); self.latest_csi_sequence = Some(sequence); self.latest_csi_sync_valid = sync_valid; @@ -1555,6 +1759,9 @@ impl NodeState { rssi_history: VecDeque::new(), vital_detector: VitalSignDetector::new(20.0), latest_vitals: VitalSigns::default(), + vital_candidate: None, + last_vital_decision: None, + vital_sample_rate_resets: 0, last_frame_time: None, edge_vitals: None, latest_sync: None, @@ -1564,6 +1771,10 @@ impl NodeState { latest_csi_sync_valid: false, csi_fps_ema: 20.0, csi_fps_samples: 0, + accepted_csi_rate_window_started_at: None, + accepted_csi_rate_window_frames: 0, + accepted_csi_rate_hz: 20.0, + accepted_csi_rate_windows: 0, latest_features: None, prev_keypoints: None, motion_energy_history: VecDeque::with_capacity(COHERENCE_WINDOW), @@ -1627,35 +1838,7 @@ impl NodeState { } grouped .into_iter() - .filter_map(|(grid, arrivals)| { - let first = *arrivals.first()?; - let last = *arrivals.last()?; - let span_s = last.saturating_duration_since(first).as_secs_f64(); - let latest_age_s = now.saturating_duration_since(last).as_secs_f64(); - let mut max_gap_s = latest_age_s; - for pair in arrivals.windows(2) { - max_gap_s = max_gap_s.max( - pair[1] - .saturating_duration_since(pair[0]) - .as_secs_f64(), - ); - } - let rate_hz = if span_s > 0.0 { - arrivals.len().saturating_sub(1) as f64 / span_s - } else { - 0.0 - }; - Some(( - grid, - CalibrationGridEvidence { - sample_count: arrivals.len(), - span_s, - rate_hz, - max_gap_s, - latest_age_s, - }, - )) - }) + .filter_map(|(grid, arrivals)| CalibrationGridEvidence::from_arrivals(&arrivals, now).map(|evidence|(grid,evidence))) .collect() } @@ -1665,13 +1848,7 @@ impl NodeState { ) -> Option<(CsiGridKey, CalibrationGridEvidence)> { self.calibration_grid_candidates(now) .into_iter() - .filter(|(grid, evidence)| { - grid.ppdu_type <= 3 - && evidence.span_s >= CALIBRATION_GRID_MIN_SPAN_S - && evidence.rate_hz >= CALIBRATION_GRID_MIN_RATE_HZ - && evidence.max_gap_s < CALIBRATION_GRID_MAX_GAP_S - && evidence.latest_age_s < CALIBRATION_GRID_MAX_GAP_S - }) + .filter(|(grid, evidence)| legacy_grid_readiness::eligible_grid(*grid, *evidence)) // Density first. This was chosen to agree with `accept_grid` when // that gate locked each node onto the densest grid it had seen; // since #1894 the gate locks onto the most frequent grid instead, @@ -1749,6 +1926,10 @@ impl NodeState { if 2 * votes(grid) > 3 * votes(active) { self.active_grid = Some(grid); self.frame_history.clear(); + self.accepted_csi_rate_window_started_at = None; + self.accepted_csi_rate_window_frames = 0; + self.accepted_csi_rate_hz = 20.0; + self.accepted_csi_rate_windows = 0; self.baseline_motion = 0.0; self.baseline_frames = 0; return true; @@ -2201,6 +2382,11 @@ struct AppStateInner { engine_bridge: engine_bridge::EngineBridge, /// SVD-based room field model for eigenvalue person counting (None until calibration). field_model: Option, + source_bank_shadow_enabled: bool, + source_bank_shadow: Option, + room_learning: Option>, + room_learning_sources: HashMap, + current_room_store: Option>, /// Stable installation identity used only to bind local persisted state. installation_id: Option, /// Metadata for the privacy reduced empty room image, when available. @@ -2219,6 +2405,8 @@ struct AppStateInner { calibration_observed_source_node_ids: std::collections::BTreeSet, /// Immutable evidence created when the active model is finalized. calibration_model_receipt: Option, + /// Requires current-boot evidence from every saved contributor before publication. + calibration_restored: bool, /// Immutable source and CSI symbol grid selected from recent header-only /// evidence for the current field model. calibration_grid_binding: Option, @@ -2721,6 +2909,40 @@ fn calibration_sequence_order(previous: Option, sequence: u32) -> Calibrati } impl AppStateInner { + fn restore_saved_room(&mut self) -> Result { + // Historical expiry metadata is retained for diagnostics after restart; + // it is not a restored model and cannot restore a full-room receipt. + if self.field_model.is_none() { return Ok(false); } + let Some((installation, metadata)) = self.installation_id.as_deref().zip(self.bootstrap_baseline.as_ref()) else { return Ok(false); }; + match room_calibration::load(&room_calibration::path_in(&self.data_dir), installation, metadata) { + Ok(receipt) => { + let [source_node_id] = metadata.source_node_ids.as_slice() else { return Ok(false); }; + self.calibration_grid_binding = Some(CalibrationGridBinding { + source_node_id: *source_node_id, grid: CsiGridKey::from_bootstrap_grid(metadata.source_grid), evidence: None, + }); + self.install_restored_room(receipt); + Ok(true) + } + Err(bootstrap_baseline::BootstrapBaselineError::Io(error)) if error.kind() == std::io::ErrorKind::NotFound => Ok(false), + Err(error) => Err(error), + } + } + + fn install_restored_room(&mut self, mut receipt: CalibrationModelReceipt) { + receipt.boot_epoch = self.calibration_boot_epoch.clone(); + receipt.session_id = opaque_calibration_id("cal-session"); + self.calibration_model_id = Some(receipt.model_id.clone()); + self.calibration_session_id = Some(receipt.session_id.clone()); + self.calibration_binding_digest = Some(receipt.binding_digest.clone()); + self.calibration_source_node_ids = receipt.source_node_ids.iter().copied().collect(); + self.calibration_observed_source_node_ids.clear(); + self.clear_calibration_sequence_state(); + for node in self.node_states.values_mut() { node.clear_field_model_history(); } + self.calibration_model_receipt = Some(receipt); + self.calibration_restored = true; + self.bootstrap_baseline_active = false; + } + /// Advance the sensing tick and record the update for the measured /// stream rate (#2087). Returns the new tick. fn advance_tick(&mut self) -> u64 { @@ -2776,6 +2998,7 @@ impl AppStateInner { } CalibrationSequenceOrder::Discontinuity => { self.calibration_sequence_fault_node_ids.insert(node_id); + if let Some(shadow) = self.source_bank_shadow.as_mut() { shadow.invalidate(); } warn!( node_id, previous_sequence, @@ -2818,6 +3041,11 @@ impl AppStateInner { (chrono::Utc::now().timestamp_millis().max(0) as u64) .saturating_mul(1_000), ) == CalibrationStatus::Fresh + || (self.bootstrap_baseline_active + && field.check_freshness( + (chrono::Utc::now().timestamp_millis().max(0) as u64) + .saturating_mul(1_000), + ) == CalibrationStatus::Stale) } }); if accepted { @@ -2826,6 +3054,9 @@ impl AppStateInner { if let Some(node) = self.node_states.get_mut(&node_id) { node.push_field_model_frame(sequence, amplitudes, observed_at); } + if let Some(shadow) = self.source_bank_shadow.as_mut() { + shadow.observe(node_id, chrono::Utc::now().timestamp_micros().max(0) as u64, amplitudes); + } } accepted } @@ -2838,6 +3069,7 @@ impl AppStateInner { } fn clear_field_model_binding(&mut self) { + self.source_bank_shadow = None; self.calibration_grid_binding = None; for node in self.node_states.values_mut() { node.clear_field_model_history(); @@ -2898,7 +3130,16 @@ impl AppStateInner { } fn explicit_calibration_fresh_at(&self, observed_at_unix_ms: u64) -> bool { - !self.bootstrap_baseline_active + (!self.calibration_restored || (!self.calibration_source_node_ids.is_empty() + && self.calibration_source_node_ids.iter().all(|id| self.node_states.get(id).is_some_and(|node| + self.calibration_observed_source_node_ids.contains(id) + && !self.calibration_sequence_fault_node_ids.contains(id) + && node.field_model_latest_seen.is_some_and(|seen| seen.elapsed() < ESP32_OFFLINE_TIMEOUT))) + && self.calibration_grid_binding.is_some_and(|binding| self.field_model.as_ref() + .and_then(|model| model.modes()).and_then(|modes| modes.baseline_runtime_window_size) + .filter(|size| *size > 0 && *size <= FRAME_HISTORY_CAPACITY) + .is_some_and(|size| self.field_model_holdout_ready(binding, size, std::time::Instant::now()))))) + && !self.bootstrap_baseline_active && self.field_model_status_at(observed_at_unix_ms) == Some(CalibrationStatus::Fresh) } @@ -2953,7 +3194,7 @@ impl AppStateInner { /// heuristic, so a consumer may treat a present value as calibrated. fn calibrated_presence_evidence( &self, - inference_node_id: u8, + _inference_node_id: u8, source_tick: u64, observed_at_unix_ms: u64, ) -> Option { @@ -2974,7 +3215,7 @@ impl AppStateInner { binding_digest: receipt.binding_digest.clone(), source_node_ids: receipt.source_node_ids.clone(), model_completed_at_unix_ms: receipt.completed_at_unix_ms, - inference_node_id, + inference_node_id: self.bound_source_node_id()?, source_tick, observed_at_unix_ms, inference_method: occupancy.method.wire_name().to_string(), @@ -2998,7 +3239,7 @@ impl AppStateInner { // A persisted bootstrap model has negative-only authority. Its only // allowed occupancy effect is the explicit empty-background suppression // in `bootstrap_empty_prior_applies`; it must never infer positive count. - if self.bootstrap_baseline_active { + if self.bootstrap_baseline_active || (self.calibration_restored && !self.explicit_calibration_fresh_at(observed_at_unix_ms)) { return score_to_person_count(self.smoothed_person_score, self.prev_person_count); } match self.field_model.as_ref() { @@ -3248,6 +3489,11 @@ impl AppStateInner { None, ), field_model: None, + source_bank_shadow_enabled: false, + source_bank_shadow: None, + room_learning: None, + room_learning_sources: HashMap::new(), + current_room_store: None, installation_id: None, bootstrap_baseline: None, bootstrap_baseline_active: false, @@ -3258,6 +3504,7 @@ impl AppStateInner { calibration_source_node_ids: std::collections::BTreeSet::new(), calibration_observed_source_node_ids: std::collections::BTreeSet::new(), calibration_model_receipt: None, + calibration_restored: false, calibration_grid_binding: None, calibration_last_sequences: HashMap::new(), calibration_reordered_packets: HashMap::new(), @@ -3282,7 +3529,7 @@ mod calibration_expiry_tests { use super::*; use wifi_densepose_signal::ruvsense::field_model::FieldModelConfig; - fn finalized_model(calibrated_at_unix_ms: u64) -> FieldModel { + pub(super) fn finalized_model(calibrated_at_unix_ms: u64) -> FieldModel { let mut model = FieldModel::new(FieldModelConfig { n_links: 1, n_subcarriers: 56, @@ -3351,7 +3598,7 @@ mod calibration_expiry_tests { state.calibration_model_id = Some("cal-model-test".to_string()); state.calibration_binding_digest = Some("ab".repeat(32)); state.calibration_model_receipt = Some(CalibrationModelReceipt { - schema: field_bridge::CALIBRATION_MODEL_RECEIPT_SCHEMA, + schema: field_bridge::CALIBRATION_MODEL_RECEIPT_SCHEMA.to_string(), boot_epoch: state.calibration_boot_epoch.clone(), session_id: "cal-session-test".to_string(), model_id: "cal-model-test".to_string(), @@ -3425,6 +3672,58 @@ mod calibration_expiry_tests { } } + // Investigation fixture only: no production gate or publication change. + #[test] + fn investigation_vitals_candidate_source_is_not_checked_by_bound_model_gate() { + let mut state = state_with_runtime_window(); + let baseline = state.field_model.as_ref().unwrap().modes().unwrap().baseline[0].clone(); + state.node_states.get_mut(&5).unwrap().field_model_history = + (0..50).map(|_| baseline.iter().map(|value| value + 0.3).collect()).collect(); + assert_eq!(state.bound_source_node_id(), Some(5)); + assert_eq!(state.person_count_at(1_500), 1); + let mut foreign = NodeState::new(); + foreign.latest_vitals = VitalSigns { heart_rate_bpm: Some(101.0), ..qualified_vital_candidates() }; + state.node_states.insert(6, foreign); + let result = calibrated_vitals_for_publication( + &state, &state.node_states[&6].latest_vitals, 1, 1_500, + ).expect("baseline accepts a qualified candidate without checking its node identity"); + assert_eq!(result.heart_rate_bpm, Some(101.0)); + println!("INVESTIGATION: candidate node6 accepted using model bound to node5; publication API carries no candidate-node identity"); + } + + #[tokio::test] + async fn investigation_vitals_rest_preserves_websocket_multi_person_refusal() { + let observed = chrono::Utc::now().timestamp_millis().max(0) as u64; + let mut state = state_with_runtime_window(); + let mut snapshot = finalized_model(observed).export_snapshot().unwrap(); + snapshot.modes.baseline_runtime_window_size = Some(50); + state.field_model = Some(FieldModel::from_snapshot(snapshot, observed * 1_000).unwrap()); + state.calibration_model_receipt.as_mut().unwrap().completed_at_unix_ms = observed; + let baseline = state.field_model.as_ref().unwrap().modes().unwrap().baseline[0].clone(); + let node = state.node_states.get_mut(&5).unwrap(); + node.field_model_history = (0..50).map(|_| baseline.iter().map(|value| value + 0.3).collect()).collect(); + node.last_frame_time = Some(std::time::Instant::now()); + node.prev_person_count = 1; + let mut second = NodeState::new(); + second.last_frame_time = Some(std::time::Instant::now()); + second.prev_person_count = 2; + state.node_states.insert(6, second); + let room_count = update_room_person_count(&mut state, &ClassificationInfo { + motion_level: "present_still".into(), presence: true, confidence: 0.9, + }, std::time::Instant::now(), observed); + assert_eq!(room_count, 2); + assert_eq!(state.person_count_at(observed), 1); + let (ws_rates, _) = record_node_vital_decision(&mut state, 5, vital_publication::NodeVitalCandidate { + kind: vital_publication::CandidateKind::RawCsiSpectral, tick: 77, + observed_at_unix_ms: observed, received_at: std::time::Instant::now(), rates: qualified_vital_candidates(), calibration_identity: None, + }, room_count); + assert!(ws_rates.is_none()); + assert!(node_vital_decision(&state, 5, 1, observed).0.is_some(), "same candidate must qualify with a single room occupant"); + let Json(body) = vital_signs_endpoint(State(Arc::new(RwLock::new(state)))).await; + println!("INVESTIGATION: fused room count2 / bound count1: websocket withheld; REST authority={} heart={}", body["authority"], body["vital_signs"]["heart_rate_bpm"]); + assert_eq!(body["authority"], "abstained", "REST must not bypass the existing multi-person refusal used by WebSocket publication"); + } + fn state_with_runtime_window() -> AppStateInner { let mut state = state_with_receipt(); let model = state.field_model.as_mut().unwrap(); @@ -3434,6 +3733,366 @@ mod calibration_expiry_tests { state } + #[test] + fn saved_room_roundtrips_through_real_restart_loader_with_new_identity() { + let dir = tempfile::tempdir().unwrap(); + let now = chrono::Utc::now().timestamp_millis().max(0) as u64; + let source = qualified_vital_state(now); + let mut receipt = source.calibration_model_receipt.clone().unwrap(); + receipt.frame_count = 18942; + let metadata = bootstrap_baseline::store(&bootstrap_baseline::path_in(dir.path()), "account-A-installation", &[5], + source.calibration_grid_binding.unwrap().grid.as_bootstrap_grid(), &receipt.model_id, now, + source.field_model.as_ref().unwrap()).unwrap(); + room_calibration::store(&room_calibration::path_in(dir.path()), "account-A-installation", &metadata, &receipt).unwrap(); + let (model, metadata) = bootstrap_baseline::load(&bootstrap_baseline::path_in(dir.path()), "account-A-installation", now + 1).unwrap(); + let mut restarted = AppStateInner::minimal(); + restarted.data_dir = dir.path().to_path_buf(); + restarted.installation_id = Some("account-A-installation".into()); + restarted.field_model = Some(model); + restarted.bootstrap_baseline = Some(metadata.clone()); + restarted.bootstrap_baseline_active = true; + assert!(restarted.restore_saved_room().unwrap()); + let current = restarted.calibration_model_receipt.as_ref().unwrap(); + assert_eq!(current.frame_count, 18942); + assert_eq!(current.completed_at_unix_ms, receipt.completed_at_unix_ms); + assert_ne!(current.boot_epoch, receipt.boot_epoch); + assert_ne!(current.session_id, receipt.session_id); + assert!(!restarted.explicit_calibration_fresh_at(now)); + restarted.installation_id = Some("account-B-installation".into()); + assert!(restarted.restore_saved_room().is_err()); + room_calibration::remove(dir.path()).unwrap(); + assert!(!restarted.restore_saved_room().unwrap()); + } + + #[test] + fn expired_metadata_cannot_restore_a_full_room_without_a_model() { + let dir = tempfile::tempdir().unwrap(); + let old = (chrono::Utc::now().timestamp_millis().max(0) as u64) - 1_000_000_000; + let source = qualified_vital_state(old); + let receipt = source.calibration_model_receipt.as_ref().unwrap(); + let path = bootstrap_baseline::path_in(dir.path()); + let metadata = bootstrap_baseline::store(&path, "account-A-installation", &[5], + source.calibration_grid_binding.unwrap().grid.as_bootstrap_grid(), &receipt.model_id, old, + source.field_model.as_ref().unwrap()).unwrap(); + room_calibration::store(&room_calibration::path_in(dir.path()), "account-A-installation", &metadata, receipt).unwrap(); + let mut restarted = AppStateInner::minimal(); + restarted.data_dir = dir.path().to_path_buf(); + restarted.installation_id = Some("account-A-installation".into()); + restarted.bootstrap_baseline = Some(bootstrap_baseline::load_expired_metadata(&path, + "account-A-installation", metadata.expires_at_unix_ms + 1).unwrap()); + assert!(!restarted.restore_saved_room().unwrap()); + assert!(restarted.field_model.is_none()); + assert!(restarted.calibration_model_receipt.is_none()); + assert!(restarted.calibration_session_id.is_none()); + assert!(!restarted.bootstrap_baseline_active_at(metadata.expires_at_unix_ms + 1)); + } + + #[test] + fn restored_room_requires_new_boot_source_evidence_and_never_renews_completion() { + let mut state = state_with_runtime_window(); + let old = state.calibration_model_receipt.clone().unwrap(); + state.install_restored_room(old.clone()); + let current = state.calibration_model_receipt.as_ref().unwrap(); + assert_eq!(current.completed_at_unix_ms, old.completed_at_unix_ms); + assert_eq!(current.model_id, old.model_id); + assert_ne!(current.session_id, old.session_id); + assert!(!state.explicit_calibration_fresh_at(1_500)); + state.calibration_observed_source_node_ids.insert(5); + assert!(!state.explicit_calibration_fresh_at(1_500)); + let baseline = state.field_model.as_ref().unwrap().modes().unwrap().baseline[0].clone(); + let node = state.node_states.get_mut(&5).unwrap(); + node.field_model_history = (0..50).map(|_| baseline.clone()).collect(); + node.field_model_latest_seen = Some(std::time::Instant::now()); + node.field_model_latest_sequence = Some(60); + assert!(state.explicit_calibration_fresh_at(1_500)); + state.node_states.get_mut(&5).unwrap().field_model_history.pop_front(); + assert!(!state.explicit_calibration_fresh_at(1_500)); + state.calibration_source_node_ids.insert(17); + assert!(!state.explicit_calibration_fresh_at(1_500)); + } + + fn qualified_vital_state(observed: u64) -> AppStateInner { + let mut state = state_with_runtime_window(); + let mut snapshot = finalized_model(observed).export_snapshot().unwrap(); + snapshot.modes.baseline_runtime_window_size = Some(50); + state.field_model = Some(FieldModel::from_snapshot(snapshot, observed * 1_000).unwrap()); + state + .calibration_model_receipt + .as_mut() + .unwrap() + .completed_at_unix_ms = observed; + let baseline = state + .field_model + .as_ref() + .unwrap() + .modes() + .unwrap() + .baseline[0] + .clone(); + let node = state.node_states.get_mut(&5).unwrap(); + node.field_model_history = (0..50) + .map(|_| baseline.iter().map(|value| value + 0.3).collect()) + .collect(); + node.last_frame_time = Some(std::time::Instant::now()); + state.prev_person_count = 1; + state.tick = 77; + state + } + + fn record_qualified_vitals( + state: &mut AppStateInner, + observed: u64, + ) -> (Option, Option) { + record_node_vital_decision( + state, + 5, + vital_publication::NodeVitalCandidate { + kind: vital_publication::CandidateKind::RawCsiSpectral, + tick: 77, + observed_at_unix_ms: observed, + received_at: std::time::Instant::now(), + rates: qualified_vital_candidates(), + calibration_identity: None, + }, + 1, + ) + } + + #[tokio::test] + async fn vital_publication_rest_matches_recorded_single_person_decision() { + let observed = chrono::Utc::now().timestamp_millis().max(0) as u64; + let mut state = qualified_vital_state(observed); + let (ws, proof) = record_qualified_vitals(&mut state, observed); + assert_eq!(proof.unwrap().source_node_ids, vec![5]); + let Json(rest) = vital_signs_endpoint(State(Arc::new(RwLock::new(state)))).await; + assert_eq!( + rest["vital_signs"], + serde_json::to_value(ws.unwrap()).unwrap() + ); + assert_eq!( + rest["diagnostics"]["nodes"][0]["last_publication_decision"]["reason"], + "published" + ); + } + + #[test] + fn vital_publication_diagnostics_identify_gates_without_suppressed_rates() { + use vital_publication::Reason; + let mut state = qualified_vital_state(1_500); + record_qualified_vitals(&mut state, 1_500); + let inspect = |state: &AppStateInner| node_vital_decision(state, 5, 1, 1_500); + let node = state.node_states.get_mut(&5).unwrap(); + node.vital_candidate.as_mut().unwrap().rates.signal_quality = 0.1; + let (rates, diagnostic) = inspect(&state); + assert!(rates.is_none()); + assert_eq!(diagnostic.reason, Reason::LowSignalQuality); + assert!(!serde_json::to_string(&diagnostic) + .unwrap() + .contains("rate_bpm")); + state + .node_states + .get_mut(&5) + .unwrap() + .vital_candidate + .as_mut() + .unwrap() + .rates = VitalSigns { + heart_rate_bpm: Some(70.0), + heartbeat_confidence: 0.1, + signal_quality: 0.9, + ..VitalSigns::default() + }; + assert_eq!(inspect(&state).1.reason, Reason::LowRateConfidence); + state.node_states.get_mut(&5).unwrap().vital_candidate.as_mut().unwrap().rates = VitalSigns { + heart_rate_bpm: Some(f64::NAN), heartbeat_confidence: 0.9, signal_quality: 0.9, ..VitalSigns::default() + }; + assert_eq!(inspect(&state).1.reason, Reason::InvalidCandidateRate); + state + .node_states + .get_mut(&5) + .unwrap() + .vital_candidate + .as_mut() + .unwrap() + .rates = VitalSigns { + signal_quality: 0.9, + ..VitalSigns::default() + }; + assert_eq!(inspect(&state).1.reason, Reason::NoCandidateRate); + state.calibration_sequence_fault_node_ids.insert(5); + assert_eq!(inspect(&state).1.reason, Reason::SequenceFault); + } + + #[tokio::test] + async fn vital_publication_stale_candidate_cannot_be_revived_by_another_nodes_freshness() { + let observed = chrono::Utc::now().timestamp_millis().max(0) as u64; + let mut state = qualified_vital_state(observed); + record_qualified_vitals(&mut state, observed); + state + .node_states + .get_mut(&5) + .unwrap() + .vital_candidate + .as_mut() + .unwrap() + .received_at -= Duration::from_secs(4); + let Json(rest) = vital_signs_endpoint(State(Arc::new(RwLock::new(state)))).await; + assert_eq!(rest["authority"], "abstained"); + assert_eq!(rest["abstention_reason"], "candidate_stale"); + } + + #[test] + fn vital_publication_rest_cannot_promote_a_previously_withheld_packet() { + let mut state = qualified_vital_state(1_500); + let (rates, _) = record_node_vital_decision( + &mut state, + 5, + vital_publication::NodeVitalCandidate { + kind: vital_publication::CandidateKind::RawCsiSpectral, + tick: 77, + observed_at_unix_ms: 1_500, + received_at: std::time::Instant::now(), + rates: qualified_vital_candidates(), + calibration_identity: None, + }, + 2, + ); + assert!(rates.is_none()); + // Synthetic state transition without a new candidate: a read cannot + // grant publication authority to the packet withheld at capture time. + let (rates, decision) = reevaluate_node_vitals(&state, 5, 1, 1_500); + assert!(rates.is_none()); + assert_eq!( + decision.reason, + vital_publication::Reason::NotPublishedForSourceTick + ); + } + + #[test] + fn vital_publication_candidate_is_bound_to_full_calibration_identity() { + let mut state = qualified_vital_state(1_500); + record_qualified_vitals(&mut state, 1_500); + // Keep the same model ID, replacing only the session identity. + state.calibration_model_receipt.as_mut().unwrap().session_id = "other-session".into(); + let (rates, decision) = reevaluate_node_vitals(&state, 5, 1, 1_500); + assert!(rates.is_none()); + assert_eq!( + decision.reason, + vital_publication::Reason::CandidateCalibrationChanged + ); + } + + #[test] + fn vital_publication_rejected_feature_grid_cannot_promote_old_candidate() { + use wifi_densepose_hardware::PpduType; + let observed = chrono::Utc::now().timestamp_millis().max(0) as u64; + let mut state = qualified_vital_state(observed); + state + .node_states + .get_mut(&5) + .unwrap() + .field_model_history + .clear(); + let (rates, _) = record_qualified_vitals(&mut state, observed); + assert!( + rates.is_none(), + "candidate withheld while bound window is incomplete" + ); + let baseline = state + .field_model + .as_ref() + .unwrap() + .modes() + .unwrap() + .baseline[0] + .clone(); + let amplitudes: Vec<_> = (0..64) + .map(|i| { + let x = i as f64 * 55.0 / 63.0; + let lo = x.floor() as usize; + baseline[lo] * (1.0 - x.fract()) + baseline[(lo + 1).min(55)] * x.fract() + 0.3 + }) + .collect(); + for sequence in 100..150 { + // Mirror receive ordering: field model accepts its frozen grid + // before the independent feature grid refuses this same packet. + for _ in 0..5 { + state + .node_states + .get_mut(&5) + .unwrap() + .accept_grid((256, PpduType::HeSu)); + } + assert!(state.maybe_feed_calibration_frame( + 5, + CsiGridKey { + n_subcarriers: 64, + ppdu_type: 0 + }, + sequence, + &litudes, + std::time::Instant::now() + )); + assert!(!state + .node_states + .get_mut(&5) + .unwrap() + .accept_grid((64, PpduType::HtLegacy))); + } + assert!( + node_vital_decision(&state, 5, 1, observed).0.is_some(), + "updated model alone now permits the cached candidate" + ); + let (rates, decision) = reevaluate_node_vitals(&state, 5, 1, observed); + assert!(rates.is_none()); + assert_eq!( + decision.reason, + vital_publication::Reason::NotPublishedForSourceTick + ); + } + + #[test] + fn vital_publication_diagnostics_separate_observed_and_detector_clocks() { + let mut state = qualified_vital_state(1_500); + let node = state.node_states.get_mut(&5).unwrap(); + node.csi_fps_ema = 23.0; + node.csi_fps_samples = 10; + assert!(!node.vital_detector.reconfigure_sample_rate(23.0)); + let (_, diagnostic) = node_vital_decision(&state, 5, 1, 1_500); + assert_eq!(diagnostic.sample_rate_hz, 20.0); + assert_eq!(diagnostic.observed_sample_rate_hz, 23.0); + } + + #[tokio::test] + async fn vital_publication_privacy_hides_candidate_diagnostics_and_rates() { + let observed = chrono::Utc::now().timestamp_millis().max(0) as u64; + let mut state = qualified_vital_state(observed); + record_qualified_vitals(&mut state, observed); + state.privacy_mode = true; + let Json(rest) = vital_signs_endpoint(State(Arc::new(RwLock::new(state)))).await; + assert_eq!(rest["abstention_reason"], "privacy_redacted"); + assert_eq!(rest["diagnostics"]["nodes"], serde_json::json!([])); + assert!(rest["buffer_status"].is_null()); + assert!(rest["vital_signs"]["heart_rate_bpm"].is_null()); + let redacted = privacy_filter::redact_json_str( + &serde_json::json!({ + "vital_evidence": {"heart_rate_bpm": 70, "source_node_ids": [5]}, + "classification": {"presence": true} + }) + .to_string(), + ) + .unwrap(); + assert!(!redacted.contains("vital_evidence")); + } + + #[test] + fn vital_publication_presence_tags_scored_node_not_packet_arrival() { + let state = qualified_vital_state(1_500); + let proof = state.calibrated_presence_evidence(6, 77, 1_500).unwrap(); + assert_eq!(proof.inference_node_id, 5); + assert!(proof.source_node_ids.contains(&proof.inference_node_id)); + } + #[test] fn diagnostics_stay_outside_the_strict_calibration_authority_envelope() { let state = state_with_runtime_window(); @@ -3519,6 +4178,24 @@ mod calibration_expiry_tests { ); } + #[tokio::test] + async fn expired_metadata_after_restart_reports_original_deadline_without_authority() { + let mut state = state_with_model(true); + let expected_expiry = state.bootstrap_baseline.as_ref().unwrap().expires_at_unix_ms; + state.field_model = None; + state.bootstrap_baseline_active = false; + let state = Arc::new(RwLock::new(state)); + let Json(status) = calibration_status(State(state)).await; + assert_eq!(status["status"], "expired"); + assert_eq!(status["active"], false); + assert_eq!(status["binding_mode"], "none"); + assert_eq!(status["bootstrap_baseline"]["expires_at_unix_ms"], expected_expiry); + assert_eq!(status["bootstrap_baseline"]["active"], false); + assert_eq!(status["bootstrap_baseline"]["calibrated_evidence_authorized"], false); + assert_eq!(status["bootstrap_baseline"]["numeric_vitals_authorized"], false); + assert!(status["model_receipt"].is_null()); + } + #[tokio::test] async fn long_running_expiry_is_not_reported_active_for_runtime_or_bootstrap() { let runtime = Arc::new(RwLock::new(state_with_model(false))); @@ -3784,6 +4461,30 @@ fn parse_edge_fused_vitals(buf: &[u8]) -> Option { }) } +fn edge_fused_vitals_message(fused: &EdgeFusedVitalsPacket) -> serde_json::Value { + serde_json::json!({ + "type": "edge_fused_vitals", + "node_id": fused.node_id, + "breathing_rate_bpm": fused.breathing_rate_bpm, + "heartrate_bpm": fused.heartrate_bpm, + "n_persons": fused.n_persons, + "person_count_valid": fused.person_count_valid, + "fusion_confidence": fused.fusion_confidence, + "mmwave": { + "present": fused.flags & 0x08 != 0, + "hr_bpm": fused.mmwave_hr_bpm, + "br_bpm": fused.mmwave_br_bpm, + "distance_cm": fused.mmwave_distance_cm, + "targets": fused.mmwave_targets, + "confidence": fused.mmwave_confidence, + "type": fused.mmwave_type, + }, + "motion_energy": fused.motion_energy, + "presence_score": fused.presence_score, + "timestamp_ms": fused.timestamp_ms, + }) +} + #[cfg(test)] mod issue_928_magic_collision_tests { //! Issue #928 — `0xC511_0004` was being parsed as WASM output, eating the @@ -3803,7 +4504,7 @@ mod issue_928_magic_collision_tests { buf[8..12].copy_from_slice(&720_000u32.to_le_bytes()); // heartrate 72.0 BPM buf[12] = (-55i8) as u8; // rssi buf[13] = 1; // n_persons - buf[14] = 2; // mmwave_type + buf[14] = 1; // mmwave_type (MR60BHA2) buf[15] = 85; // fusion_confidence buf[16..20].copy_from_slice(&0.42f32.to_le_bytes()); // motion_energy buf[20..24].copy_from_slice(&0.95f32.to_le_bytes()); // presence_score @@ -3828,7 +4529,7 @@ mod issue_928_magic_collision_tests { assert_eq!(pkt.rssi, -55); assert_eq!(pkt.n_persons, 1); assert!(pkt.person_count_valid); - assert_eq!(pkt.mmwave_type, 2); + assert_eq!(pkt.mmwave_type, 1); assert_eq!(pkt.fusion_confidence, 85); assert!((pkt.motion_energy - 0.42).abs() < 1e-6); assert!((pkt.presence_score - 0.95).abs() < 1e-6); @@ -3838,6 +4539,11 @@ mod issue_928_magic_collision_tests { assert!((pkt.mmwave_distance_cm - 182.0).abs() < 1e-6); assert_eq!(pkt.mmwave_targets, 1); assert_eq!(pkt.mmwave_confidence, 90); + + let message = edge_fused_vitals_message(&pkt); + assert_eq!(message["mmwave"]["present"], true); + assert_eq!(message["mmwave"]["targets"], 1); + assert_eq!(message["mmwave"]["type"], 1); } #[test] @@ -5366,6 +6072,8 @@ async fn wifi_task(state: SharedState, tick_ms: u64) { }], features, classification, + classification_source: None, + selected_binary_evidence: None, signal_field: generate_signal_field( first_rssi, motion_score, @@ -5374,6 +6082,7 @@ async fn wifi_task(state: SharedState, tick_ms: u64) { &sub_variances, ), vital_signs: None, + vital_evidence: None, calibrated_presence_evidence: None, detector_diagnostics: None, enhanced_motion, @@ -5412,6 +6121,7 @@ async fn wifi_task(state: SharedState, tick_ms: u64) { attach_field_positions(&mut update); assess_legacy_image_pose(&mut s, &update); + room_learner_api::project_live_selection(&s, &mut update); if let Ok(json) = serde_json::to_string(&update) { let _ = s.tx.send(json); } @@ -5536,6 +6246,8 @@ async fn windows_wifi_fallback_tick(state: &SharedState, seq: u32) { }], features, classification, + classification_source: None, + selected_binary_evidence: None, signal_field: generate_signal_field( rssi_dbm, motion_score, @@ -5544,6 +6256,7 @@ async fn windows_wifi_fallback_tick(state: &SharedState, seq: u32) { &sub_variances, ), vital_signs: None, + vital_evidence: None, calibrated_presence_evidence: None, detector_diagnostics: None, enhanced_motion: None, @@ -5578,6 +6291,7 @@ async fn windows_wifi_fallback_tick(state: &SharedState, seq: u32) { attach_field_positions(&mut update); assess_legacy_image_pose(&mut s, &update); + room_learner_api::project_live_selection(&s, &mut update); if let Ok(json) = serde_json::to_string(&update) { let _ = s.tx.send(json); } @@ -6071,19 +6785,54 @@ fn generate_simulated_frame(tick: u64) -> Esp32Frame { } } +/// A cached observation is not a new measurement. Keep its original identity +/// and timestamp while refreshing receive-clock metadata for a new subscriber +/// or source-state transition. Never rebroadcast this snapshot per subscriber. +fn current_sensing_snapshot(state: &AppStateInner) -> Option { + let mut snapshot = state.latest_update.clone()?; + snapshot.source = state.effective_source(); + if snapshot.source.starts_with("esp32") { + let now = std::time::Instant::now(); + snapshot.node_features = build_node_features(&state.node_states, now); + for node in &mut snapshot.nodes { + node.node_inference = state.node_states.get(&node.node_id) + .map(|state| node_inference_for(state, now)); + } + } + let age_ms = chrono::Utc::now().timestamp_millis() as f64 - snapshot.timestamp * 1000.0; + if snapshot.source.ends_with(":offline") || age_ms > LATEST_UPDATE_STALE_AFTER.as_millis() as f64 { + // Preserve historical classification, not a fabricated absence. + snapshot.vital_signs = None; + snapshot.vital_evidence = None; + snapshot.calibrated_presence_evidence = None; + snapshot.detector_diagnostics = None; + snapshot.room_inference = None; + snapshot.persons = None; + snapshot.pose_keypoints = None; + } + room_learner_api::project_cached_selection(state, &mut snapshot); + Some(snapshot) +} + // ── WebSocket handler ──────────────────────────────────────────────────────── async fn ws_sensing_handler( ws: WebSocketUpgrade, State(state): State, + grant: Option>, ) -> impl IntoResponse { - ws.on_upgrade(|socket| handle_ws_client(socket, state)) + ws.on_upgrade(move |socket| async move { + if let Some(Extension(grant)) = grant { + companion_integration::guard_stream(grant, handle_ws_client(socket, state)).await; + } else { handle_ws_client(socket, state).await; } + }) } async fn handle_ws_client(mut socket: WebSocket, state: SharedState) { - let (mut rx, privacy_mode, _client) = { + let (mut rx, privacy_mode, _client, mut initial_snapshot) = { let s = state.read().await; - (s.tx.subscribe(), s.privacy_mode, s.ws_clients.enter()) + (s.tx.subscribe(), s.privacy_mode, s.ws_clients.enter(), + current_sensing_snapshot(&s).and_then(|update| serde_json::to_string(&update).ok())) }; info!("WebSocket client connected (sensing)"); @@ -6094,9 +6843,19 @@ async fn handle_ws_client(mut socket: WebSocket, state: SharedState) { loop { tokio::select! { - msg = rx.recv() => { + msg = async { + match initial_snapshot.take() { + Some(snapshot) => Ok(snapshot), + None => rx.recv().await, + } + } => { match msg { Ok(json) => { + let refreshed = { + let s = state.read().await; + room_learner_api::refresh_queued_update(&s, &json) + }; + let Some(json) = refreshed else { continue; }; let json = if privacy_mode { match privacy_filter::redact_json_str(&json) { Some(json) => json, @@ -6198,9 +6957,10 @@ async fn ws_pose_handler( } async fn handle_ws_pose_client(mut socket: WebSocket, state: SharedState) { - let (mut rx, privacy_mode, _client) = { + let (mut rx, privacy_mode, _client, mut initial_snapshot) = { let s = state.read().await; - (s.tx.subscribe(), s.privacy_mode, s.ws_clients.enter()) + (s.tx.subscribe(), s.privacy_mode, s.ws_clients.enter(), + current_sensing_snapshot(&s).and_then(|update| serde_json::to_string(&update).ok())) }; info!("WebSocket client connected (pose)"); @@ -6212,14 +6972,46 @@ async fn handle_ws_pose_client(mut socket: WebSocket, state: SharedState) { }); let _ = socket.send(Message::Text(conn_msg.to_string())).await; + let mut ping_interval = tokio::time::interval(Duration::from_secs(30)); + ping_interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + loop { tokio::select! { - msg = rx.recv() => { + msg = async { + match initial_snapshot.take() { + Some(snapshot) => Ok(snapshot), + None => rx.recv().await, + } + } => { match msg { Ok(json) => { + let refreshed = { + let s = state.read().await; + room_learner_api::refresh_queued_update(&s, &json) + }; + let Some(json) = refreshed else { continue; }; // Parse the sensing update and convert to pose format if let Ok(sensing) = serde_json::from_str::(&json) { if sensing.msg_type == "sensing_update" { + let age_ms = chrono::Utc::now().timestamp_millis() as f64 - sensing.timestamp * 1000.0; + if sensing.selected_binary_evidence.is_some() || sensing.source.ends_with(":offline") || age_ms > LATEST_UPDATE_STALE_AFTER.as_millis() as f64 { + // Unavailable is not measured absence. Never reconstruct + // a skeleton from a deliberately expired cached snapshot. + let mut unavailable = serde_json::json!({ + "type": "pose_data", "zone_id": "zone_1", + "timestamp": sensing.timestamp, "physics": null, + "payload": { + "pose": {"persons": []}, "confidence": null, + "activity": "unavailable", "pose_source": "unavailable", + "metadata": {"frame_id": format!("rust_frame_{}", sensing.tick), + "source": sensing.source, "tick": sensing.tick, + "estimated_persons": null, "evidence_state": "unavailable"} + } + }); + if privacy_mode { privacy_filter::redact_value(&mut unavailable); } + if socket.send(Message::Text(unavailable.to_string())).await.is_err() { break; } + continue; + } // Determine pose estimation mode for the UI indicator. // "model_inference" — a trained RVF model is loaded. // "signal_derived" — keypoints estimated from raw CSI features. @@ -6328,6 +7120,9 @@ async fn handle_ws_pose_client(mut socket: WebSocket, state: SharedState) { Err(_) => break, // channel closed } } + _ = ping_interval.tick() => { + if socket.send(Message::Ping(vec![])).await.is_err() { break; } + } msg = socket.recv() => { match msg { Some(Ok(Message::Text(text))) => { @@ -6428,11 +7223,14 @@ mod latest_update_staleness_tests { presence: false, confidence: 0.97, }, + classification_source: None, + selected_binary_evidence: None, signal_field: SignalField { grid_size: [0, 0, 0], values: vec![], }, vital_signs: None, + vital_evidence: None, classifier: None, activity: None, calibrated_presence_evidence: None, @@ -7593,6 +8391,10 @@ fn derive_single_person_pose( /// ever *lowers* a Derived cycle from P5 to P4, both of which are already held /// edge-local, so it cannot leak. fn emit_rufield_event(s: &AppStateInner, update: &SensingUpdate, node_id: u8) { + // The legacy field bridge cannot encode a selected model/qualification or + // its source-set/window identity. Never relabel packet features and signal + // field peaks as qualified binary-model motion/count/location evidence. + if update.selected_binary_evidence.is_some() { return; } // No-presence ⇒ no phantom event. if !update.classification.presence { return; @@ -7780,6 +8582,7 @@ fn assess_legacy_image_pose(state: &mut AppStateInner, update: &SensingUpdate) { } fn derive_pose_from_sensing(update: &SensingUpdate) -> Vec { + if update.selected_binary_evidence.is_some() { return vec![]; } let cls = &update.classification; if !cls.presence { return vec![]; @@ -8059,6 +8862,12 @@ async fn pose_current( Query(query): Query, ) -> impl IntoResponse { let s = state.read().await; + if room_learner_api::selection_owns_output(&s) { + return (StatusCode::CONFLICT, Json(serde_json::json!({ + "code":"selected_binary_model_has_no_pose_authority", "persons":[], "total_persons":null, + "detail":"The selected model qualifies room presence only; no pose or person count is available." + }))); + } let persons = match &s.latest_update { Some(update) => update .persons @@ -8144,6 +8953,10 @@ async fn pose_physics_metrics(State(state): State) -> impl IntoResp async fn pose_stats(State(state): State) -> Json { let s = state.read().await; + if room_learner_api::selection_owns_output(&s) { + return Json(serde_json::json!({"authority":"withheld","reason":"selected_binary_model_has_no_pose_authority", + "total_detections":null,"average_confidence":null,"frames_processed":s.tick,"source":s.effective_source()})); + } Json(serde_json::json!({ "total_detections": s.total_detections, "average_confidence": 0.87, @@ -8181,6 +8994,12 @@ async fn fann_shadow_status() -> Json { /// Legacy route. It used to return four invented zones; it now returns the /// fused zones, or none when no source is mapped to a zone. async fn pose_zones_summary(state: State) -> Json { + { + let s = state.0.read().await; + if room_learner_api::selection_owns_output(&s) { + return Json(serde_json::json!({"zones":{},"authority":"withheld","reason":"selected_binary_model_has_no_zone_count_authority"})); + } + } zones_endpoint(state).await } @@ -8484,9 +9303,9 @@ async fn activate_lora_profile(Json(body): Json) -> Json/models` (issue #2088). pub(crate) fn models_dir(data_dir: &std::path::Path) -> PathBuf { - std::env::var_os("MODELS_DIR") - .map(PathBuf::from) - .unwrap_or_else(|| data_dir.join("models")) + wifi_densepose_sensing_server::account_profile::models_directory( + std::env::var_os(wifi_densepose_sensing_server::bearer_auth::OWNER_ACCOUNT_SCOPE_ENV).is_some(), + data_dir, std::env::var_os("MODELS_DIR").as_deref()) } /// Return the directory CSI recordings are written to and read from: @@ -8889,26 +9708,50 @@ fn vitals_for_publication( if !explicit_calibration_fresh || person_count != 1 { return None; } + if !candidates.signal_quality.is_finite() { + return None; + } let signal_quality = candidates.signal_quality.clamp(0.0, 1.0); if signal_quality < VITAL_PUBLICATION_MIN_SIGNAL_QUALITY { return None; } - let breathing_rate_bpm = (candidates.breathing_confidence - >= VITAL_PUBLICATION_MIN_CONFIDENCE) - .then_some(candidates.breathing_rate_bpm) - .flatten(); - let heart_rate_bpm = - (candidates.heartbeat_confidence >= VITAL_PUBLICATION_MIN_CONFIDENCE) - .then_some(candidates.heart_rate_bpm) - .flatten(); + let qualified = |rate: Option, confidence: f64, minimum, maximum| { + rate.filter(|rate| { + rate.is_finite() + && *rate >= minimum + && *rate <= maximum + && confidence.is_finite() + && confidence >= VITAL_PUBLICATION_MIN_CONFIDENCE + }) + }; + let breathing_rate_bpm = qualified( + candidates.breathing_rate_bpm, + candidates.breathing_confidence, + 4.0, + 40.0, + ); + let heart_rate_bpm = qualified( + candidates.heart_rate_bpm, + candidates.heartbeat_confidence, + 30.0, + 240.0, + ); if breathing_rate_bpm.is_none() && heart_rate_bpm.is_none() { return None; } Some(VitalSigns { breathing_rate_bpm, heart_rate_bpm, - breathing_confidence: candidates.breathing_confidence.clamp(0.0, 1.0), - heartbeat_confidence: candidates.heartbeat_confidence.clamp(0.0, 1.0), + breathing_confidence: if breathing_rate_bpm.is_some() { + candidates.breathing_confidence.clamp(0.0, 1.0) + } else { + 0.0 + }, + heartbeat_confidence: if heart_rate_bpm.is_some() { + candidates.heartbeat_confidence.clamp(0.0, 1.0) + } else { + 0.0 + }, signal_quality, }) } @@ -8917,26 +9760,222 @@ fn vitals_for_publication( /// cannot authorize numeric rates when the bound field model says absent. /// Require both the existing room gate and a fresh, complete model window; /// never use a heuristic fallback as calibrated single-occupant evidence. +struct VitalGateAssessment { + published: Option, + reason: vital_publication::Reason, + bound_window_ready: bool, + calibrated_person_count: Option, +} + +fn assess_calibrated_vitals( + state: &AppStateInner, + candidates: &VitalSigns, + room_person_count: usize, + observed_at_unix_ms: u64, +) -> VitalGateAssessment { + use vital_publication::Reason; + let binding = state.calibration_grid_binding; + let window = state + .field_model + .as_ref() + .and_then(|m| m.modes()) + .and_then(|m| m.baseline_runtime_window_size); + let ready = binding.zip(window).is_some_and(|(binding, window)| { + state.field_model_holdout_ready(binding, window, std::time::Instant::now()) + }); + let evidence = binding.filter(|_| ready).and_then(|binding| { + state.calibrated_presence_evidence(binding.source_node_id, state.tick, observed_at_unix_ms) + }); + let count = evidence.as_ref().map(|e| e.person_count); + let reason = if state.privacy_mode { + Reason::PrivacyRedacted + } else if room_person_count != 1 { + Reason::RoomNotSingleOccupant + } else if !state.calibration_sequence_fault_node_ids.is_empty() { + Reason::SequenceFault + } else if !state.explicit_calibration_fresh_at(observed_at_unix_ms) { + Reason::ExplicitCalibrationRequired + } else if binding.is_none() || window.is_none() { + Reason::BoundModelUnavailable + } else if !ready { + Reason::BoundWindowNotReady + } else if count.is_none() { + Reason::CalibratedPresenceUnavailable + } else if count != Some(1) { + Reason::CalibratedNotSingleOccupant + } else if !candidates.signal_quality.is_finite() + || candidates.signal_quality < VITAL_PUBLICATION_MIN_SIGNAL_QUALITY + { + Reason::LowSignalQuality + } else if candidates.breathing_rate_bpm.is_none() && candidates.heart_rate_bpm.is_none() { + Reason::NoCandidateRate + } else if !candidates.breathing_rate_bpm.is_some_and(|rate| rate.is_finite() && (4.0..=40.0).contains(&rate)) + && !candidates.heart_rate_bpm.is_some_and(|rate| rate.is_finite() && (30.0..=240.0).contains(&rate)) { + Reason::InvalidCandidateRate + } else { + Reason::LowRateConfidence + }; + let published = if matches!(reason, Reason::LowRateConfidence) { + vitals_for_publication(candidates, true, 1) + } else { + None + }; + VitalGateAssessment { + reason: if published.is_some() { + Reason::Published + } else { + reason + }, + published, + bound_window_ready: ready, + calibrated_person_count: count, + } +} + fn calibrated_vitals_for_publication( state: &AppStateInner, candidates: &VitalSigns, room_person_count: usize, observed_at_unix_ms: u64, ) -> Option { - if room_person_count != 1 || !state.calibration_sequence_fault_node_ids.is_empty() { - return None; + assess_calibrated_vitals(state, candidates, room_person_count, observed_at_unix_ms).published +} + +/// REST reuses the last atomic room aggregate, never the single-link field +/// count in isolation. A stale candidate is independently refused below. +fn current_vital_room_count(state: &AppStateInner, now: std::time::Instant) -> usize { + if state.node_states.values().any(|n| node_is_fresh(n, now)) { + state.prev_person_count + } else { + 0 } - let binding = state.calibration_grid_binding?; - let window_size = state.field_model.as_ref()?.modes()?.baseline_runtime_window_size?; - if !state.field_model_holdout_ready(binding, window_size, std::time::Instant::now()) { - return None; +} + +fn vital_calibration_identity( + state: &AppStateInner, +) -> Option { + let receipt = state.calibration_model_receipt.as_ref()?; + Some(vital_publication::CalibrationIdentity { + boot_epoch: receipt.boot_epoch.clone(), + session_id: receipt.session_id.clone(), + model_id: receipt.model_id.clone(), + binding_digest: receipt.binding_digest.clone(), + source_node_ids: receipt.source_node_ids.clone(), + completed_at_unix_ms: receipt.completed_at_unix_ms, + }) +} + +fn node_vital_decision( + state: &AppStateInner, + node_id: u8, + room_count: usize, + observed_at_unix_ms: u64, +) -> (Option, vital_publication::Decision) { + use vital_publication::{Decision, Reason, CANDIDATE_MAX_AGE_MS}; + let node = &state.node_states[&node_id]; + let candidate = node.vital_candidate.as_ref(); + let rates = candidate.map(|c| c.rates.clone()).unwrap_or_default(); + let age = candidate.map(|c| c.received_at.elapsed().as_millis().min(u64::MAX as u128) as u64); + let mut gate = assess_calibrated_vitals(state, &rates, room_count, observed_at_unix_ms); + if !state.privacy_mode { + if candidate.is_none() { + gate.reason = Reason::NoCandidate; + gate.published = None; + } else if age.is_none_or(|age| age > CANDIDATE_MAX_AGE_MS) + || candidate.is_some_and(|c| { + observed_at_unix_ms.abs_diff(c.observed_at_unix_ms) > CANDIDATE_MAX_AGE_MS + }) + { + gate.reason = Reason::CandidateStale; + gate.published = None; + } + } + if gate.published.is_some() + && candidate.is_some_and(|c| { + c.calibration_identity.is_none() + || c.calibration_identity != vital_calibration_identity(state) + }) + { + gate.reason = Reason::CandidateCalibrationChanged; + gate.published = None; + } + let diagnostic = Decision { + node_id, + candidate_kind: candidate.map(|c| c.kind), + source_tick: candidate.map(|c| c.tick), + observed_at_unix_ms: candidate.map(|c| c.observed_at_unix_ms), + candidate_age_ms: age, + occupancy_source_node_id: state.bound_source_node_id(), + room_person_count: room_count, + calibrated_person_count: gate.calibrated_person_count, + explicit_calibration_fresh: state.explicit_calibration_fresh_at(observed_at_unix_ms), + sequence_continuity_ok: state.calibration_sequence_fault_node_ids.is_empty(), + bound_window_ready: gate.bound_window_ready, + breathing_rate_present: rates.breathing_rate_bpm.is_some(), + heart_rate_present: rates.heart_rate_bpm.is_some(), + signal_quality_passed: rates.signal_quality.is_finite() + && rates.signal_quality >= VITAL_PUBLICATION_MIN_SIGNAL_QUALITY, + breathing_confidence_passed: rates.breathing_confidence.is_finite() + && rates.breathing_confidence >= VITAL_PUBLICATION_MIN_CONFIDENCE, + heartbeat_confidence_passed: rates.heartbeat_confidence.is_finite() + && rates.heartbeat_confidence >= VITAL_PUBLICATION_MIN_CONFIDENCE, + buffers: node.vital_detector.buffer_status().into(), + sample_rate_hz: node.vital_detector.sample_rate_hz(), + observed_sample_rate_hz: node.effective_sample_rate_hz(), + sample_rate_resets: node.vital_sample_rate_resets, + reason: gate.reason, + }; + (gate.published, diagnostic) +} + +/// Called under the same write lock as the room aggregate and outgoing frame. +fn record_node_vital_decision( + state: &mut AppStateInner, + node_id: u8, + mut candidate: vital_publication::NodeVitalCandidate, + room_count: usize, +) -> (Option, Option) { + let observed = candidate.observed_at_unix_ms; + candidate.calibration_identity = vital_calibration_identity(state); + state + .node_states + .get_mut(&node_id) + .expect("active node") + .vital_candidate = Some(candidate); + let (published, diagnostic) = node_vital_decision(state, node_id, room_count, observed); + let node = state.node_states.get_mut(&node_id).expect("active node"); + let evidence = published.as_ref().and_then(|rates| { + vital_publication::evidence( + node_id, + node.vital_candidate.as_ref().expect("recorded candidate"), + rates, + ) + }); + node.last_vital_decision = Some(diagnostic); + (published, evidence) +} + +/// A read may withdraw a previously published candidate, never promote an old +/// withheld sample under a new room/privacy/calibration context. +fn reevaluate_node_vitals( + state: &AppStateInner, + node_id: u8, + room_count: usize, + observed: u64, +) -> (Option, vital_publication::Decision) { + let (mut rates, mut diagnostic) = node_vital_decision(state, node_id, room_count, observed); + let recorded = state.node_states[&node_id].last_vital_decision.as_ref(); + if rates.is_some() + && !recorded.is_some_and(|last| { + last.reason == vital_publication::Reason::Published + && last.source_tick == diagnostic.source_tick + && last.observed_at_unix_ms == diagnostic.observed_at_unix_ms + }) + { + rates = None; + diagnostic.reason = vital_publication::Reason::NotPublishedForSourceTick; } - let evidence = state.calibrated_presence_evidence( - binding.source_node_id, - state.tick, - observed_at_unix_ms, - )?; - vitals_for_publication(candidates, true, evidence.person_count) + (rates, diagnostic) } fn edge_vitals_message_for_publication( @@ -9155,6 +10194,8 @@ mod bootstrap_vital_publication_tests { Query(CalibrationStartQuery { source_node_id: Some(5), }), + None, + None, Json(test_binding_request(vec![5])), ) .await; @@ -9185,6 +10226,8 @@ mod bootstrap_vital_publication_tests { Query(CalibrationStartQuery { source_node_id: Some(5), }), + None, + None, Json(test_binding_request(vec![5])), ) .await; @@ -9793,12 +10836,243 @@ struct BootstrapRefineQuery { source_node_id: Option, } +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct BootstrapRecoverRequest { + boot_epoch: String, + content_sha256: String, +} + +/// Explicitly restore the verified negative-only prior, just as a clean boot +/// would. This never renews its lifetime or restores a full room receipt. +async fn calibration_recover_bootstrap( + State(state): State, + Json(request): Json, +) -> Json { + let mut s = state.write().await; + if request.boot_epoch != s.calibration_boot_epoch + || !valid_binding_digest(&request.content_sha256) + || s.bootstrap_baseline.as_ref().is_none_or(|metadata| { + metadata.content_sha256 != request.content_sha256 + }) + { + return Json(serde_json::json!({ + "success": false, "error_code": "bootstrap_recovery_identity_mismatch", + "error": "The saved reference or server boot changed. Refresh calibration status before restoring it.", + })); + } + if s.field_model.as_ref().is_some_and(|model| matches!( + model.status(), CalibrationStatus::Uncalibrated | CalibrationStatus::Collecting + )) && s.calibration_session_id.is_some() { + return Json(serde_json::json!({ + "success": false, "error_code": "calibration_capture_active", + "error": "Cancel the active capture before restoring the saved empty reference.", + })); + } + let Some(installation_id) = s.installation_id.as_deref() else { + return Json(serde_json::json!({ + "success": false, "error_code": "installation_id_required", + "error": "A stable installation identity is required to restore the saved reference.", + })); + }; + let path = bootstrap_baseline::path_in(&s.data_dir); + // Validate disk content before touching the live model, histories or faults. + let (model, metadata) = match bootstrap_baseline::load( + &path, installation_id, chrono::Utc::now().timestamp_millis().max(0) as u64, + ) { + Ok(value) => value, + Err(_) => return Json(serde_json::json!({ + "success": false, "error_code": "bootstrap_reference_unavailable", + "error": "The saved empty reference is expired or failed validation. Collect a new baseline.", + })), + }; + if metadata.content_sha256 != request.content_sha256 { + return Json(serde_json::json!({ + "success": false, "error_code": "bootstrap_recovery_identity_mismatch", + "error": "The saved reference changed. Refresh calibration status before restoring it.", + })); + } + let [source_node_id] = metadata.source_node_ids.as_slice() else { + return Json(serde_json::json!({ + "success": false, "error_code": "bootstrap_source_invalid", + "error": "The saved reference must have one source radio.", + })); + }; + let grid = CsiGridKey::from_bootstrap_grid(metadata.source_grid); + let now = std::time::Instant::now(); + let evidence = s.node_states.get(source_node_id).and_then(|node| { + node.calibration_grid_candidates(now).into_iter().find_map(|(candidate, evidence)| { + (candidate == grid + && evidence.span_s >= CALIBRATION_GRID_MIN_SPAN_S + && evidence.rate_hz >= CALIBRATION_GRID_MIN_RATE_HZ + && evidence.max_gap_s < CALIBRATION_GRID_MAX_GAP_S + && evidence.latest_age_s < CALIBRATION_GRID_MAX_GAP_S).then_some(evidence) + }) + }); + let Some(evidence) = evidence else { + return Json(serde_json::json!({ + "success": false, "error_code": "bootstrap_source_unavailable", + "error": "The saved source radio and CSI grid need at least ten seconds of fresh live data before recovery.", + "source_node_id": source_node_id, + })); + }; + let binding = CalibrationGridBinding { source_node_id: *source_node_id, grid, evidence: Some(evidence) }; + if let Err(error) = room_calibration::remove(&s.data_dir) { + return error_response::internal_error_json("room calibration downgrade", error); + } + s.clear_field_model_binding(); + s.clear_calibration_sequence_state(); + s.calibration_source_node_ids.clear(); + s.calibration_source_node_ids.insert(binding.source_node_id); + s.calibration_observed_source_node_ids.clear(); + s.calibration_model_id = None; + s.calibration_session_id = None; + s.calibration_binding_digest = None; + s.calibration_model_receipt = None; + s.calibration_restored = false; + s.field_model = Some(model); + s.calibration_grid_binding = Some(binding); + s.bootstrap_baseline = Some(metadata.clone()); + s.bootstrap_baseline_active = true; + Json(serde_json::json!({ + "success": true, "boot_epoch": s.calibration_boot_epoch, + "status": s.field_model_status_at(chrono::Utc::now().timestamp_millis().max(0) as u64).map(|value| format!("{value:?}").to_lowercase()), + "bootstrap_baseline": metadata, "restored_frames": 0, + "full_calibration_required": true, + "calibrated_evidence_authorized": false, "numeric_vitals_authorized": false, + })) +} + +#[cfg(test)] +mod bootstrap_recovery_tests { + use super::*; + + fn fixture() -> (tempfile::TempDir, SharedState, BootstrapRecoverRequest) { + let directory = tempfile::tempdir().unwrap(); + let now_ms = chrono::Utc::now().timestamp_millis() as u64; + let model = super::calibration_expiry_tests::finalized_model(now_ms); + let mut snapshot = model.export_snapshot().unwrap(); + snapshot.config.baseline_expiry_s = 86_400.0; + snapshot.modes.calibrated_at_us = (now_ms - 13 * 3_600_000) * 1_000; + let model = FieldModel::from_snapshot(snapshot, now_ms * 1_000).unwrap(); + let grid = CsiGridKey { n_subcarriers: 64, ppdu_type: 0 }; + let metadata = bootstrap_baseline::store( + &bootstrap_baseline::path_in(directory.path()), "recovery-test", &[5], + grid.as_bootstrap_grid(), "cal-model-saved", now_ms, &model, + ).unwrap(); + let mut state = AppStateInner::minimal(); + state.installation_id = Some("recovery-test".into()); + state.data_dir = directory.path().to_path_buf(); + state.bootstrap_baseline = Some(metadata.clone()); + state.field_model = Some(model); + state.calibration_session_id = Some("old-session".into()); + state.calibration_model_id = Some("old-model".into()); + state.calibration_binding_digest = Some("ab".repeat(32)); + state.calibration_sequence_fault_node_ids.insert(5); + state.calibration_last_sequences.insert(5, 50_000); + let mut node = NodeState::new(); + let now = std::time::Instant::now(); + for index in 0..=240 { + node.observe_raw_grid(grid, now - Duration::from_millis(12_000 - index * 50)); + } + node.field_model_history.push_back(vec![999.0; 56]); + node.field_model_latest_sequence = Some(50_000); + node.field_model_latest_seen = Some(now); + state.node_states.insert(5, node); + let request = BootstrapRecoverRequest { + boot_epoch: state.calibration_boot_epoch.clone(), content_sha256: metadata.content_sha256, + }; + (directory, Arc::new(RwLock::new(state)), request) + } + + #[tokio::test] + async fn stale_recovery_retains_expiry_withdraws_full_receipt_and_requires_new_frames() { + let (_directory, state, request) = fixture(); + let expiry = state.read().await.bootstrap_baseline.as_ref().unwrap().expires_at_unix_ms; + let Json(response) = calibration_recover_bootstrap(State(state.clone()), Json(request)).await; + assert_eq!(response["success"], true); + assert_eq!(response["bootstrap_baseline"]["expires_at_unix_ms"], expiry); + assert_eq!(response["restored_frames"], 0); + assert_eq!(response["calibrated_evidence_authorized"], false); + let mut s = state.write().await; + assert!(s.bootstrap_baseline_active); + assert!(s.calibration_model_receipt.is_none()); + assert!(s.calibration_session_id.is_none()); + assert!(s.calibration_sequence_fault_node_ids.is_empty()); + assert!(s.node_states[&5].field_model_history.is_empty()); + assert!(s.node_states[&5].field_model_latest_seen.is_none()); + assert!(!s.bootstrap_empty_prior_applies(chrono::Utc::now().timestamp_millis() as u64)); + let frame = s.field_model.as_ref().unwrap().modes().unwrap().baseline[0].clone(); + let grid = s.calibration_grid_binding.unwrap().grid; + assert!(s.maybe_feed_calibration_frame(5, grid, 1, &frame, std::time::Instant::now())); + assert!(!s.maybe_feed_calibration_frame(5, grid, 1, &frame, std::time::Instant::now())); + assert!(s.maybe_feed_calibration_frame(5, grid, 100, &frame, std::time::Instant::now())); + assert!(!s.maybe_feed_calibration_frame(5, grid, 2, &frame, std::time::Instant::now())); + assert!(s.calibration_sequence_fault_node_ids.contains(&5)); + assert!(!s.explicit_calibration_fresh_at(chrono::Utc::now().timestamp_millis() as u64)); + } + + #[tokio::test] + async fn rejected_recovery_preserves_live_state_and_saved_image() { + for invalid_case in ["boot", "digest", "tamper", "source", "grid", "capture", "installation", "expired"] { + let (directory, state, mut request) = fixture(); + let path = bootstrap_baseline::path_in(directory.path()); + match invalid_case { + "boot" => request.boot_epoch = "old-boot".into(), + "digest" => request.content_sha256 = "cd".repeat(32), + "tamper" => std::fs::write(&path, b"{}").unwrap(), + "source" => state.write().await.node_states.clear(), + "grid" => { + state.write().await.node_states.get_mut(&5).unwrap().raw_grid_observations.clear(); + } + "capture" => { + state.write().await.field_model = Some(FieldModel::new(field_bridge::single_link_config()).unwrap()); + } + "installation" => state.write().await.installation_id = Some("another-installation".into()), + "expired" => { + let mut s = state.write().await; + let now_ms = chrono::Utc::now().timestamp_millis() as u64; + let old = s.field_model.as_ref().unwrap().export_snapshot().unwrap(); + let mut old = old; + old.modes.calibrated_at_us = (now_ms - 25 * 3_600_000) * 1_000; + // Store a valid image when it was still young, then load now. + let model = FieldModel::from_snapshot(old, (now_ms - 24 * 3_600_000) * 1_000).unwrap(); + let metadata = bootstrap_baseline::store(&path, "recovery-test", &[5], + s.calibration_grid_binding.map_or(CsiGridKey { n_subcarriers:64, ppdu_type:0 }, |b| b.grid).as_bootstrap_grid(), + "cal-model-saved", now_ms - 24 * 3_600_000, &model).unwrap(); + request.content_sha256 = metadata.content_sha256.clone(); s.bootstrap_baseline = Some(metadata); + } + _ => unreachable!(), + } + let bytes = std::fs::read(&path).unwrap(); + let Json(response) = calibration_recover_bootstrap(State(state.clone()), Json(request)).await; + assert_eq!(response["success"], false, "{invalid_case}"); + let s = state.read().await; + assert_eq!(s.calibration_session_id.as_deref(), Some("old-session"), "{invalid_case}"); + assert!(s.calibration_sequence_fault_node_ids.contains(&5), "{invalid_case}"); + assert_eq!(std::fs::read(&path).unwrap(), bytes); + } + } +} + async fn calibration_start( State(state): State, Query(query): Query, + companion_grant: Option>, + shared_room: Option>>, Json(request): Json, ) -> Json { + if companion_grant.is_some() && !shared_room.as_ref().is_some_and(|Extension(room)| room.matches_binding(&request.binding_digest, &request.source_node_ids)) { + return Json(serde_json::json!({"success":false,"error_code":"shared_room_binding_required","error":"Import the current Mac room revision before starting calibration."})); + } let mut s = state.write().await; + if let Some(learner)=s.room_learning.as_ref() { + match learner.capture_active() { + Ok(false)=>{}, + Ok(true)=>return Json(serde_json::json!({"success":false,"error_code":"learner_capture_active","error":"Finish or cancel learner example capture before starting a legacy empty baseline."})), + Err(_)=>return Json(serde_json::json!({"success":false,"error_code":"learner_state_unavailable","error":"Learner capture state is unavailable; retry before changing calibration mode."})), + } + } if !valid_binding_digest(&request.binding_digest) || !valid_source_node_ids(&request.source_node_ids) { @@ -9868,6 +11142,18 @@ async fn calibration_start( }; *binding }; + let source_grid_readiness=legacy_grid_readiness::inspect(&s,&request.source_node_ids,selected_binding,now) + .expect("validated binding includes the selected author"); + if !source_grid_readiness.all_bound_nodes_ready { + return Json(serde_json::json!({ + "success":false, + "error_code":"calibration_bound_grid_unavailable", + "error":format!("Legacy baseline uses one author and exact {}-bin/PPDU{} data. Placed nodes {:?} are not grid-ready. No baseline changed. Review grid diagnostics; managed per-source learning is a separate workflow.",selected_binding.grid.n_subcarriers,selected_binding.grid.ppdu_type,source_grid_readiness.nodes.iter().filter(|n|!n.ready).map(|n|n.node_id).collect::>()), + "source_node_ids":request.source_node_ids, + "source_grid":selected_binding.grid, + "source_grid_readiness":source_grid_readiness, + })); + } if s.bootstrap_baseline_active { s.field_model = None; s.bootstrap_baseline_active = false; @@ -9893,6 +11179,9 @@ async fn calibration_start( _ => {} // Stale/Expired/Uncalibrated — ok to recalibrate } } + if let Err(error) = room_calibration::remove(&s.data_dir) { + return error_response::internal_error_json("room calibration replacement", error); + } match FieldModel::new(field_bridge::single_link_config()) { Ok(fm) => { s.field_model = Some(fm); @@ -9902,11 +11191,17 @@ async fn calibration_start( s.calibration_source_node_ids.clear(); s.calibration_observed_source_node_ids.clear(); s.calibration_model_receipt = None; + s.calibration_restored = false; s.clear_field_model_binding(); s.clear_calibration_sequence_state(); s.calibration_source_node_ids .extend(request.source_node_ids.iter().copied()); s.calibration_grid_binding = Some(selected_binding); + if s.source_bank_shadow_enabled { + s.source_bank_shadow = source_bank_shadow::ShadowCapture::new( + &request.source_node_ids, selected_binding.grid.n_subcarriers as usize, + format!("csi-{}-{}", selected_binding.grid.n_subcarriers, selected_binding.grid.ppdu_type)); + } Json(serde_json::json!({ "success": true, "message": "Calibration started — keep room empty while frames accumulate.", @@ -9919,6 +11214,7 @@ async fn calibration_start( "source_node_ids": s.calibration_source_node_ids, "source_grid": selected_binding.grid, "grid_evidence": selected_binding.evidence, + "source_grid_readiness":source_grid_readiness, })) } // ADR-080 #2: FieldModel init error chain stays server-side only. @@ -9994,6 +11290,19 @@ async fn calibration_stop( "latest_seen_ms": latest_seen_ms, })); } + let unfinished=s.field_model.as_ref().is_some_and(|fm|matches!(fm.status(),CalibrationStatus::Collecting|CalibrationStatus::Uncalibrated)); + if unfinished { + if let Some(readiness)=legacy_grid_readiness::inspect(&s,&source_node_ids,binding,std::time::Instant::now()) { + if readiness.known_incompatibility() { + return Json(serde_json::json!({ + "success":false,"error_code":"calibration_bound_grid_unavailable", + "error":"A placed node now has only a different qualified raw CSI grid from this legacy baseline. Restore the frozen grid before finalizing, or review the separate managed per-source learner. The unfinished baseline was preserved.", + "model_id":model_id,"source_node_ids":source_node_ids,"source_grid":source_grid, + "source_grid_readiness":readiness, + })); + } + } + } if let Some(ref mut fm) = s.field_model { // Guard: finalizing before enough empty-room frames have accumulated // is a client-side sequencing error, not a server fault. Return a @@ -10035,7 +11344,7 @@ async fn calibration_stop( let holdout_window_size = modes.baseline_runtime_window_size; let final_frame_count = fm.calibration_frame_count(); let receipt = CalibrationModelReceipt { - schema: field_bridge::CALIBRATION_MODEL_RECEIPT_SCHEMA, + schema: field_bridge::CALIBRATION_MODEL_RECEIPT_SCHEMA.to_string(), boot_epoch: s.calibration_boot_epoch.clone(), session_id: s.calibration_session_id.clone().expect("validated identity"), model_id: model_id.clone().expect("bound calibration model identity"), @@ -10046,7 +11355,9 @@ async fn calibration_stop( baseline_eigenvalue_count: baseline, completed_at_unix_ms: chrono::Utc::now().timestamp_millis().max(0) as u64, }; + s.calibration_restored = false; s.calibration_model_receipt = Some(receipt.clone()); + if let Some(shadow) = s.source_bank_shadow.as_mut() { shadow.freeze(ts); } s.begin_field_model_holdout(binding); info!("Field model calibrated: baseline_eigenvalues={baseline}, variance_explained={variance_explained:.2}"); Json(serde_json::json!({ @@ -10107,7 +11418,9 @@ async fn calibration_status(State(state): State) -> Json metadata.expires_at_unix_ms) + { "expired".to_string() } else { "none".to_string() }, |status| format!("{status:?}").to_lowercase(), ); // An expired calibration is inactive by definition, but "expired" is a @@ -10135,6 +11448,10 @@ async fn calibration_status(State(state): State) -> Json=s.calibration_source_node_ids.iter().copied().collect(); + legacy_grid_readiness::inspect(&s,&ids,binding,now) + })} else {None}; let mut response = serde_json::json!({ "active": active, "status": status, @@ -10186,15 +11503,26 @@ async fn calibration_status(State(state): State) -> Json>() } else { Vec::new() }); response["observed_source_node_ids"] = serde_json::json!(if bound { s.calibration_observed_source_node_ids.iter().copied().collect::>() } else { Vec::new() }); response["missing_source_node_ids"] = serde_json::json!(if bound { s.calibration_source_node_ids.difference(&s.calibration_observed_source_node_ids).copied().collect::>() } else { Vec::new() }); + if !s.privacy_mode { + if let Some(shadow) = s.source_bank_shadow.as_ref() { + response["source_bank_shadow"] = shadow.diagnostics(observed_at_unix_ms.saturating_mul(1_000)); + } + } Json(response) } @@ -10449,6 +11777,12 @@ async fn calibration_promote_bootstrap( return error_response::internal_error_json("bootstrap baseline store", error) } }; + let Some(receipt) = s.calibration_model_receipt.as_ref() else { + return error_response::internal_error_json("room calibration receipt", "missing receipt"); + }; + if let Err(error) = room_calibration::store(&room_calibration::path_in(&s.data_dir), &installation_id, &stored, receipt) { + return error_response::internal_error_json("room calibration store", error); + } s.bootstrap_baseline = Some(stored.clone()); info!( model_id = %stored.source_model_id, @@ -10739,6 +12073,9 @@ async fn calibration_cancel( })); } let cancelled_session_id = s.calibration_session_id.clone(); + if let Err(error) = room_calibration::remove(&s.data_dir) { + return error_response::internal_error_json("room calibration removal", error); + } s.field_model = None; s.calibration_model_id = None; s.calibration_session_id = None; @@ -10746,6 +12083,7 @@ async fn calibration_cancel( s.calibration_source_node_ids.clear(); s.calibration_observed_source_node_ids.clear(); s.calibration_model_receipt = None; + s.calibration_restored = false; s.clear_field_model_binding(); s.clear_calibration_sequence_state(); if let Some(installation_id) = s.installation_id.clone() { @@ -10796,6 +12134,9 @@ async fn calibration_reset( if !boot_only && !calibration_identity_matches(&s, &request) { return calibration_identity_error(); } + if let Err(error) = room_calibration::remove(&s.data_dir) { + return error_response::internal_error_json("room calibration removal", error); + } s.field_model = None; s.calibration_model_id = None; s.calibration_session_id = None; @@ -10803,6 +12144,7 @@ async fn calibration_reset( s.calibration_source_node_ids.clear(); s.calibration_observed_source_node_ids.clear(); s.calibration_model_receipt = None; + s.calibration_restored = false; s.clear_field_model_binding(); s.clear_calibration_sequence_state(); s.bootstrap_baseline_active = false; @@ -10847,33 +12189,80 @@ fn chrono_timestamp() -> u64 { async fn vital_signs_endpoint(State(state): State) -> Json { let s = state.read().await; - let observed_at_unix_ms = chrono::Utc::now().timestamp_millis().max(0) as u64; - let person_count = s.person_count_at(observed_at_unix_ms); - let published = calibrated_vitals_for_publication( - &s, - &s.latest_vitals, - person_count, - observed_at_unix_ms, - ); - let (br_len, br_cap, hb_len, hb_cap) = s.vital_detector.buffer_status(); + if room_learner_api::selection_owns_output(&s) { + return Json(serde_json::json!({"vital_signs":{ + "breathing_rate_bpm":null,"heart_rate_bpm":null,"breathing_confidence":null, + "heartbeat_confidence":null,"signal_quality":null},"authority":"abstained", + "abstention_reason":"selected_binary_model_has_no_vital_authority","buffer_status":null, + "buffer_source_node_id":null,"diagnostics":{"schema":vital_publication::DIAGNOSTICS_SCHEMA, + "privacy_redacted":s.privacy_mode,"nodes":[]},"source":s.effective_source(),"tick":s.tick})); + } + let observed = chrono::Utc::now().timestamp_millis().max(0) as u64; + let room_count = current_vital_room_count(&s, std::time::Instant::now()); + // Candidate, room aggregate, model and diagnostic snapshot share one lock. + let selected = s + .node_states + .iter() + .filter_map(|(&id, node)| node.vital_candidate.as_ref().map(|c| (id, c.tick))) + .max_by_key(|&(id, tick)| (tick, id)) + .map(|(id, _)| id); + let (published, reason) = selected + .map(|id| { + let (published, diagnostic) = reevaluate_node_vitals(&s, id, room_count, observed); + (published, diagnostic.reason) + }) + .unwrap_or(( + None, + if s.privacy_mode { + vital_publication::Reason::PrivacyRedacted + } else { + vital_publication::Reason::NoCandidate + }, + )); + let mut ids: Vec<_> = s.node_states.keys().copied().collect(); + ids.sort_unstable(); + // This existing endpoint retains its owner/paired-read/account middleware. + // Diagnostics are never added to public health or authority envelopes. + let diagnostics: Vec<_> = if s.privacy_mode { + vec![] + } else { + ids.into_iter() + .take(256) + .map(|id| { + let (_, current) = reevaluate_node_vitals(&s, id, room_count, observed); + serde_json::json!({"current": current, + "last_publication_decision": s.node_states[&id].last_vital_decision}) + }) + .collect() + }; + let buffer_source = if s.privacy_mode { + None + } else { + selected.or_else(|| { + s.node_states + .iter() + .max_by_key(|(&id, n)| (n.last_frame_time, id)) + .map(|(&id, _)| id) + }) + }; + let buffers = buffer_source.map(|id| { + vital_publication::BufferStatus::from(s.node_states[&id].vital_detector.buffer_status()) + }); Json(serde_json::json!({ "vital_signs": { - "breathing_rate_bpm": published.as_ref().and_then(|value| value.breathing_rate_bpm), - "heart_rate_bpm": published.as_ref().and_then(|value| value.heart_rate_bpm), - "breathing_confidence": published.as_ref().map(|value| value.breathing_confidence), - "heartbeat_confidence": published.as_ref().map(|value| value.heartbeat_confidence), - "signal_quality": published.as_ref().map(|value| value.signal_quality), + "breathing_rate_bpm": published.as_ref().and_then(|v| v.breathing_rate_bpm), + "heart_rate_bpm": published.as_ref().and_then(|v| v.heart_rate_bpm), + "breathing_confidence": published.as_ref().map(|v| v.breathing_confidence), + "heartbeat_confidence": published.as_ref().map(|v| v.heartbeat_confidence), + "signal_quality": published.as_ref().map(|v| v.signal_quality), }, "authority": if published.is_some() { "explicit_calibration" } else { "abstained" }, - "abstention_reason": if published.is_some() { serde_json::Value::Null } else { serde_json::json!("fresh explicit calibration with exactly one occupant and qualified evidence required") }, - "buffer_status": { - "breathing_samples": br_len, - "breathing_capacity": br_cap, - "heartbeat_samples": hb_len, - "heartbeat_capacity": hb_cap, - }, - "source": s.effective_source(), - "tick": s.tick, + "abstention_reason": if published.is_some() { serde_json::Value::Null } else { serde_json::json!(reason) }, + "buffer_status": buffers, + "buffer_source_node_id": buffer_source, + "diagnostics": {"schema": vital_publication::DIAGNOSTICS_SCHEMA, + "privacy_redacted": s.privacy_mode, "nodes": diagnostics}, + "source": s.effective_source(), "tick": s.tick, })) } @@ -10930,6 +12319,10 @@ async fn edge_registry_endpoint( /// GET /api/v1/edge-vitals — latest edge vitals from ESP32 (ADR-039). async fn edge_vitals_endpoint(State(state): State) -> Json { let s = state.read().await; + if room_learner_api::selection_owns_output(&s) { + return Json(serde_json::json!({"status":"abstained","edge_vitals":null, + "message":"The selected binary model does not grant vital-sign authority."})); + } let observed_at_unix_ms = chrono::Utc::now().timestamp_millis().max(0) as u64; let explicit_single_occupant = s.explicit_calibration_fresh_at(observed_at_unix_ms) && s.person_count_at(observed_at_unix_ms) == 1; @@ -12065,11 +13458,14 @@ fn mediatek_room_update( classification, // No spatial field: this heuristic has no per-subcarrier or // multi-antenna geometry to place a heat map from. + classification_source: None, + selected_binary_evidence: None, signal_field: SignalField { grid_size: [0, 0, 0], values: vec![], }, vital_signs: None, + vital_evidence: None, calibrated_presence_evidence: None, detector_diagnostics: None, enhanced_motion: None, @@ -12548,6 +13944,8 @@ async fn udp_receiver_task( loop { match socket.recv_from(&mut buf).await { Ok((len, src)) => { + // Host dequeue coordinate only, not kernel or RF measurement time. + let receiver_dequeued_at = std::time::Instant::now(); UDP_INGRESS .datagrams .fetch_add(1, std::sync::atomic::Ordering::Relaxed); @@ -12708,13 +14106,14 @@ async fn udp_receiver_task( .observe(now, mean_amp as f64, peak_amp as f64); s.tick += 1; let tick = s.tick; - let update = mediatek_room_update( + let mut update = mediatek_room_update( &mut s, now, ESP32_OFFLINE_TIMEOUT, tick, source_label, ); + room_learner_api::project_live_selection(&s, &mut update); if let Ok(json) = serde_json::to_string(&update) { let _ = s.tx.send(json); } @@ -12955,11 +14354,11 @@ async fn udp_receiver_task( }; let explicit_calibration_fresh = s.explicit_calibration_fresh_at(observed_at_unix_ms); - let published_vitals = calibrated_vitals_for_publication( - &s, - &vital_candidates, - total_persons, - observed_at_unix_ms, + let (published_vitals, vital_evidence) = record_node_vital_decision( + &mut s, node_id, vital_publication::NodeVitalCandidate { + kind: vital_publication::CandidateKind::FirmwareEdge, + tick, observed_at_unix_ms, received_at: now, rates: vital_candidates, calibration_identity: None, + }, total_persons, ); // Keep the legacy edge_vitals stream aligned with the @@ -12973,25 +14372,30 @@ async fn udp_receiver_task( explicit_calibration_fresh, total_persons, ); - if let Ok(json) = serde_json::to_string(&edge_vitals_message) { - let _ = s.tx.send(json); + if !room_learner_api::selection_owns_output(&s) { + if let Ok(json) = serde_json::to_string(&edge_vitals_message) { + let _ = s.tx.send(json); + } } let calibrated_presence_evidence = s.calibrated_presence_evidence( node_id, tick, - chrono::Utc::now().timestamp_millis().max(0) as u64, + observed_at_unix_ms, ); let mut update = SensingUpdate { msg_type: "sensing_update".to_string(), - timestamp: chrono::Utc::now().timestamp_millis() as f64 / 1000.0, + timestamp: observed_at_unix_ms as f64 / 1000.0, source: "esp32".to_string(), tick, nodes: active_nodes, features: fused_features.clone(), classification, + classification_source: None, + selected_binary_evidence: None, signal_field, vital_signs: published_vitals, + vital_evidence, detector_diagnostics: calibrated_presence_evidence.as_ref().and_then(|e| e.diagnostics.clone()), calibrated_presence_evidence, enhanced_motion: None, @@ -13034,6 +14438,7 @@ async fn udp_receiver_task( attach_field_positions(&mut update); assess_legacy_image_pose(&mut s, &update); + room_learner_api::project_live_selection(&s, &mut update); if let Ok(json) = serde_json::to_string(&update) { let _ = s.tx.send(json); } @@ -13088,26 +14493,7 @@ async fn udp_receiver_task( fused.mmwave_targets, fused.fusion_confidence, ); let s = state.write().await; - if let Ok(json) = serde_json::to_string(&serde_json::json!({ - "type": "edge_fused_vitals", - "node_id": fused.node_id, - "breathing_rate_bpm": fused.breathing_rate_bpm, - "heartrate_bpm": fused.heartrate_bpm, - "n_persons": fused.n_persons, - "person_count_valid": fused.person_count_valid, - "fusion_confidence": fused.fusion_confidence, - "mmwave": { - "hr_bpm": fused.mmwave_hr_bpm, - "br_bpm": fused.mmwave_br_bpm, - "distance_cm": fused.mmwave_distance_cm, - "targets": fused.mmwave_targets, - "confidence": fused.mmwave_confidence, - "type": fused.mmwave_type, - }, - "motion_energy": fused.motion_energy, - "presence_score": fused.presence_score, - "timestamp_ms": fused.timestamp_ms, - })) { + if let Ok(json) = serde_json::to_string(&edge_fused_vitals_message(&fused)) { let _ = s.tx.send(json); } continue; @@ -13135,13 +14521,18 @@ async fn udp_receiver_task( continue; } - if let Some(frame) = parse_esp32_frame(&buf[..len]) { + let receiver_parse_started_at = std::time::Instant::now(); + let csi_frame = parse_esp32_frame(&buf[..len]); + let receiver_parse_finished_at = std::time::Instant::now(); + if let Some(frame) = csi_frame { debug!( "ESP32 frame from {src}: node={}, subs={}, seq={}", frame.node_id, frame.n_subcarriers, frame.sequence ); + let receiver_lock_requested_at = std::time::Instant::now(); let mut s = state.write().await; + let receiver_lock_acquired_at = std::time::Instant::now(); s.source = "esp32".to_string(); let observed_at = std::time::Instant::now(); s.last_esp32_frame = Some(observed_at); @@ -13150,6 +14541,10 @@ async fn udp_receiver_task( .entry(frame.node_id) .or_insert_with(NodeState::new) .observe_raw_grid(grid_key, observed_at); + room_learner_api::observe_with_receiver_timing(&mut s, &frame, &buf[..len], observed_at, + Some(room_learner_api::ReceiverIngressTiming::new(receiver_dequeued_at, + receiver_parse_finished_at.checked_duration_since(receiver_parse_started_at), + receiver_lock_acquired_at.checked_duration_since(receiver_lock_requested_at)))); // The field model owns a separate, frozen grid path. Feed // it before the global inference grid gate so a stable @@ -13257,7 +14652,7 @@ async fn udp_receiver_task( ns.frame_history.pop_front(); } - let sample_rate_hz = ns.effective_sample_rate_hz(); + let sample_rate_hz = ns.feature_sample_rate_hz(std::time::Instant::now()); let ( features, mut classification, @@ -13303,6 +14698,7 @@ async fn udp_receiver_task( if ns.csi_fps_samples >= 5 && ns.vital_detector.reconfigure_sample_rate(sample_rate_hz) { + ns.vital_sample_rate_resets = ns.vital_sample_rate_resets.saturating_add(1); // Never smooth estimates computed against two clocks. ns.smoothed_hr = 0.0; ns.smoothed_br = 0.0; @@ -13456,21 +14852,21 @@ async fn udp_receiver_task( now, observed_at_unix_ms, ); - let published_vitals = calibrated_vitals_for_publication( - &s, - &vitals, - total_persons, - observed_at_unix_ms, + let (published_vitals, vital_evidence) = record_node_vital_decision( + &mut s, node_id, vital_publication::NodeVitalCandidate { + kind: vital_publication::CandidateKind::RawCsiSpectral, + tick, observed_at_unix_ms, received_at: now, rates: vitals, calibration_identity: None, + }, total_persons, ); let calibrated_presence_evidence = s.calibrated_presence_evidence( node_id, tick, - chrono::Utc::now().timestamp_millis().max(0) as u64, + observed_at_unix_ms, ); let mut update = SensingUpdate { msg_type: "sensing_update".to_string(), - timestamp: chrono::Utc::now().timestamp_millis() as f64 / 1000.0, + timestamp: observed_at_unix_ms as f64 / 1000.0, source: "esp32".to_string(), tick, nodes: active_nodes, @@ -13480,6 +14876,8 @@ async fn udp_receiver_task( // The count uses this same room presence gate; only // `motion_score` remains local to the arriving packet. classification: room_classification, + classification_source: None, + selected_binary_evidence: None, signal_field: generate_signal_field( fused_features.mean_rssi, motion_score, @@ -13488,6 +14886,7 @@ async fn udp_receiver_task( &sub_variances, ), vital_signs: published_vitals, + vital_evidence, detector_diagnostics: calibrated_presence_evidence.as_ref().and_then(|e| e.diagnostics.clone()), calibrated_presence_evidence, enhanced_motion: None, @@ -13530,6 +14929,7 @@ async fn udp_receiver_task( attach_field_positions(&mut update); assess_legacy_image_pose(&mut s, &update); + room_learner_api::project_live_selection(&s, &mut update); if let Ok(json) = serde_json::to_string(&update) { let _ = s.tx.send(json); } @@ -13720,6 +15120,8 @@ async fn simulated_data_task(state: SharedState, tick_ms: u64) { }], features: features.clone(), classification, + classification_source: None, + selected_binary_evidence: None, signal_field: generate_signal_field( features.mean_rssi, motion_score, @@ -13728,6 +15130,7 @@ async fn simulated_data_task(state: SharedState, tick_ms: u64) { &sub_variances, ), vital_signs: None, + vital_evidence: None, calibrated_presence_evidence: None, detector_diagnostics: None, enhanced_motion: None, @@ -13779,6 +15182,7 @@ async fn simulated_data_task(state: SharedState, tick_ms: u64) { if update.classification.presence { s.total_detections += 1; } + room_learner_api::project_live_selection(&s, &mut update); if let Ok(json) = serde_json::to_string(&update) { let _ = s.tx.send(json); } @@ -13791,26 +15195,19 @@ async fn simulated_data_task(state: SharedState, tick_ms: u64) { async fn broadcast_tick_task(state: SharedState, tick_ms: u64) { let mut interval = tokio::time::interval(Duration::from_millis(tick_ms)); - + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + let mut previous_source = None; loop { interval.tick().await; let s = state.read().await; - if let Some(ref update) = s.latest_update { - if s.tx.receiver_count() > 0 { - // Re-broadcast the latest sensing_update so pose WS clients - // always get data even when ESP32 pauses between frames. - // - // Issue #618: overwrite `source` with `effective_source()` - // before each broadcast so a stale latest_update (frozen - // payload from a now-offline ESP32) is emitted with - // `source: "esp32:offline"` instead of `source: "esp32"`. - // The REST `/health` endpoint already does this; before - // this fix the WS path was the only consumer that didn't, - // so the UI's "LIVE — ESP32 HARDWARE Connected" banner - // stayed green long after the hardware went away. - let mut tagged = update.clone(); - tagged.source = s.effective_source(); - if let Ok(json) = serde_json::to_string(&tagged) { + let source = s.effective_source(); + let changed = previous_source.as_ref().is_some_and(|previous| previous != &source); + previous_source = Some(source); + // Producers publish observations directly. Only announce source-state + // transitions here; WebSocket Ping provides keepalive without cached CSI. + if changed && s.tx.receiver_count() > 0 { + if let Some(snapshot) = current_sensing_snapshot(&s) { + if let Ok(json) = serde_json::to_string(&snapshot) { let _ = s.tx.send(json); } } @@ -14107,6 +15504,26 @@ async fn main() { let _telemetry = telemetry::init(); let mut args = Args::parse(); + let (_parent_lifetime_guard, parent_exit) = + match wifi_densepose_sensing_server::parent_lifetime::arm(args.parent_pid) { + Ok(lifetime) => lifetime, + Err(error) => { + eprintln!("Managed helper startup refused: {error}"); + std::process::exit(2); + } + }; + if let Ok(scope) = std::env::var(wifi_densepose_sensing_server::bearer_auth::OWNER_ACCOUNT_SCOPE_ENV) { + let directory = match wifi_densepose_sensing_server::account_profile::prepare_directory(&scope, &args.data_dir) { + Ok(directory) => directory, + Err(error) => { eprintln!("Owned helper startup refused: {error}"); std::process::exit(2); }, + }; + // Child-process cwd only: relative model, recording and cache paths + // must not reopen a different account's directory or the app's cwd. + if let Err(error) = std::env::set_current_dir(&directory) { + eprintln!("Owned helper working directory could not be selected: {error}"); std::process::exit(2); + } + args.data_dir = directory; + } args.ui_path = coalesce_ui_path(args.ui_path); if let Err(error) = validate_startup_calibration_request(args.calibrate) { @@ -14849,6 +16266,14 @@ async fn main() { { (None, None, false) } + Err(bootstrap_baseline::BootstrapBaselineError::Expired) => { + let metadata = bootstrap_baseline::load_expired_metadata( + &path, installation_id, chrono::Utc::now().timestamp_millis().max(0) as u64, + ).ok(); + // Keep original timestamps for honest status only. + // No FieldModel, active prior or authority is restored. + (None, metadata, false) + } Err(error) => { warn!(%error, "Ignored invalid empty room bootstrap prior"); (None, None, false) @@ -14898,65 +16323,6 @@ async fn main() { ); } - // #872: actually start the MQTT publisher when `--mqtt` is set. The publisher - // (mqtt::) consumes a typed VitalsSnapshot stream; we bridge the existing JSON - // sensing broadcast into it with a defensive serde_json::Value mapping (absent - // fields default — never publish wrong values). Gated on the `mqtt` feature - // (the Docker image is built `--features mqtt`); without it `--mqtt` WARNs and - // no-ops, matching the documented contract. - if args.mqtt_opts.mqtt { - #[cfg(feature = "mqtt")] - { - use wifi_densepose_sensing_server::mqtt; - let mut mcfg = mqtt::config::MqttConfig::from_args_with_data_dir( - &args.mqtt_opts, - &data_dir, - ); - mcfg.privacy_mode = args.privacy_mode; - let mcfg = std::sync::Arc::new(mcfg); - match mcfg.validate() { - Ok(()) => { - let node_id = mcfg.client_id.clone(); - let builder = mqtt::publisher::OwnedDiscoveryBuilder { - discovery_prefix: mcfg.discovery_prefix.clone(), - node_id: node_id.clone(), - node_friendly_name: Some("RuView".to_string()), - sw_version: env!("CARGO_PKG_VERSION").to_string(), - model: "RuView WiFi Sensing".to_string(), - via_device: None, - }; - let (vtx, vrx) = broadcast::channel::(64); - let (host, port) = (mcfg.host.clone(), mcfg.port); - mqtt::publisher::spawn(mcfg, builder, vrx); - let mut jrx = tx.subscribe(); - tokio::spawn(async move { - // #898/#872/#2085: one snapshot per physical node from - // each `sensing_update`; `edge_vitals` feeds the fall - // detector; other message types are ignored. - let mut bridge = mqtt::bridge::SensingBridge::new(node_id); - let started = std::time::Instant::now(); - while let Ok(json) = jrx.recv().await { - let Ok(v) = serde_json::from_str::(&json) else { - continue; - }; - for snap in bridge.ingest(&v, started.elapsed()) { - let _ = vtx.send(snap); - } - } - }); - tracing::info!("MQTT publisher started -> {host}:{port}"); - } - Err(e) => tracing::error!("MQTT config invalid: {e}; publisher not started"), - } - } - #[cfg(not(feature = "mqtt"))] - tracing::warn!( - "--mqtt set but this binary was built without the `mqtt` feature; the publisher is a \ - no-op. Use the official Docker image (built `--features mqtt`) or rebuild with \ - `cargo build -p wifi-densepose-sensing-server --features mqtt`." - ); - } - // ADR-262 P3: build the live RuField surface (dedicated ed25519 signer from // WDP_RUFIELD_SIGNING_SEED, else a logged dev default). The same Arc is // stored in AppStateInner (so the sensing loop can `emit()` per cycle) and @@ -14980,6 +16346,19 @@ async fn main() { .as_ref() .map(|p| mediatek_activity::load_activity_state(p)) .unwrap_or_default(); + let room_learning = if args.room_learner { + let owner = std::env::var(wifi_densepose_sensing_server::bearer_auth::OWNER_ACCOUNT_SCOPE_ENV).unwrap_or_default(); + let installation = args.installation_id.as_deref().unwrap_or(""); + let recovery = wifi_densepose_sensing_server::room_learner::recovery_store::RecoveryStore::from_managed_env(&args.data_dir, &owner, installation); + let runtime = match recovery { + Ok(store) => wifi_densepose_sensing_server::room_learner::lifecycle::RoomLearningRuntime::new_with_recovery(&args.data_dir, &owner, installation, store), + Err(reason) => wifi_densepose_sensing_server::room_learner::lifecycle::RoomLearningRuntime::new_with_recovery_unavailable(&args.data_dir, &owner, installation, reason), + }; + match runtime { + Ok(runtime) => Some(Arc::new(runtime)), + Err(error) => { error!("Managed room learner could not initialize: {error}"); std::process::exit(2); } + } + } else { None }; let state: SharedState = Arc::new(RwLock::new(AppStateInner { latest_update: None, source_fusion: source_fusion::FusionState::from_env(), @@ -15133,6 +16512,11 @@ async fn main() { ) .with_node_positions(engine_bridge_node_positions), field_model: bootstrap_field_model, + source_bank_shadow_enabled: args.source_bank_shadow, + source_bank_shadow: None, + room_learning, + room_learning_sources: HashMap::new(), + current_room_store: None, installation_id: args.installation_id.clone(), bootstrap_baseline: bootstrap_metadata.clone(), bootstrap_baseline_active, @@ -15146,6 +16530,7 @@ async fn main() { .unwrap_or_default(), calibration_observed_source_node_ids: std::collections::BTreeSet::new(), calibration_model_receipt: None, + calibration_restored: false, calibration_grid_binding: bootstrap_metadata.as_ref().and_then(|metadata| { let [source_node_id] = metadata.source_node_ids.as_slice() else { return None; @@ -15174,6 +16559,90 @@ async fn main() { .expect("default pose physics configuration is valid"), })); + // #872: actually start the MQTT publisher when `--mqtt` is set. The publisher + // (mqtt::) consumes a typed VitalsSnapshot stream; we bridge the existing JSON + // sensing broadcast into it with a defensive serde_json::Value mapping (absent + // fields default — never publish wrong values). Gated on the `mqtt` feature + // (the Docker image is built `--features mqtt`); without it `--mqtt` WARNs and + // no-ops, matching the documented contract. + // Selected models have no MQTT authority in this version. This admission + // guard does not retract a snapshot already queued inside the publisher or + // a broker-retained legacy value; known nodes retain the existing 10-second + // stale threshold / 30-second availability heartbeat. Managed native helpers do + // not enable MQTT. Network-time withdrawal needs a separate sink contract. + if args.mqtt_opts.mqtt { + #[cfg(feature = "mqtt")] + { + use wifi_densepose_sensing_server::mqtt; + let mut mcfg = mqtt::config::MqttConfig::from_args_with_data_dir( + &args.mqtt_opts, + &data_dir, + ); + mcfg.privacy_mode = args.privacy_mode; + let mcfg = std::sync::Arc::new(mcfg); + match mcfg.validate() { + Ok(()) => { + let node_id = mcfg.client_id.clone(); + let builder = mqtt::publisher::OwnedDiscoveryBuilder { + discovery_prefix: mcfg.discovery_prefix.clone(), + node_id: node_id.clone(), + node_friendly_name: Some("RuView".to_string()), + sw_version: env!("CARGO_PKG_VERSION").to_string(), + model: "RuView WiFi Sensing".to_string(), + via_device: None, + }; + let (vtx, vrx) = broadcast::channel::(64); + let (host, port) = (mcfg.host.clone(), mcfg.port); + mqtt::publisher::spawn(mcfg, builder, vrx); + let mut jrx = state.read().await.tx.subscribe(); + let mqtt_state = Arc::downgrade(&state); + tokio::spawn(async move { + // #898/#872/#2085: one snapshot per physical node from + // each `sensing_update`; `edge_vitals` feeds the fall + // detector; other message types are ignored. + let mut bridge = mqtt::bridge::SensingBridge::new(node_id.clone()); + let started = std::time::Instant::now(); + while let Ok(json) = jrx.recv().await { + let Ok(v) = serde_json::from_str::(&json) else { + continue; + }; + let Some(live_state) = mqtt_state.upgrade() else { break; }; + let current = live_state.read().await; + if !room_learner_api::legacy_sink_frame_allowed( + room_learner_api::selection_owns_output(¤t), &v, + ) { + // Do not replay pending legacy fall/motion state when + // the operator later explicitly returns to control. + bridge = mqtt::bridge::SensingBridge::new(node_id.clone()); + continue; + } + for snap in bridge.ingest(&v, started.elapsed()) { + let _ = vtx.send(snap); + } + } + }); + tracing::info!("MQTT publisher started -> {host}:{port}"); + } + Err(e) => tracing::error!("MQTT config invalid: {e}; publisher not started"), + } + } + #[cfg(not(feature = "mqtt"))] + tracing::warn!( + "--mqtt set but this binary was built without the `mqtt` feature; the publisher is a \ + no-op. Use the official Docker image (built `--features mqtt`) or rebuild with \ + `cargo build -p wifi-densepose-sensing-server --features mqtt`." + ); + } + + { + let mut s = state.write().await; + match s.restore_saved_room() { + Ok(true) => info!("Restored saved room model; waiting for fresh current-boot source windows"), + Ok(false) => {}, + Err(error) => warn!(%error, "Ignored invalid full-room receipt; retaining bootstrap-only authority"), + } + } + // Start background tasks from the resolved plan (issue #1004). // // In `auto` mode with no boot source, `bind_udp` AND `run_simulator` are @@ -15288,8 +16757,8 @@ async fn main() { Ok(s) => s, Err(e) => { error!( - "API auth: OAuth was requested but cannot be initialised: {e}. \ - Refusing to start — unset RUVIEW_OAUTH_ISSUER to run without it." + "API auth could not be initialised: {e}. Correct the configuration before restarting; \ + an owned helper profile must remain account-bound." ); std::process::exit(1); } @@ -15340,6 +16809,9 @@ async fn main() { } } + let companion = companion_integration::Integration::prepare(&args, &state, bearer_auth_state.clone()).await + .unwrap_or_else(|error| { error!("Companion startup refused: {error}"); std::process::exit(1) }); + // DNS-rebinding defense: validate the `Host` header against an allowlist // before any handler runs. Default is loopback-only (`localhost`, // `127.0.0.1`, `[::1]`, each with or without a port). Operators extend @@ -15353,16 +16825,10 @@ async fn main() { ); wifi_densepose_sensing_server::host_validation::HostAllowlist::disabled() } else { - let discovery_label = args.installation_id.as_deref().unwrap_or("installation"); - let discovery_host = - wifi_densepose_sensing_server::discovery::discovery_hostname(discovery_label); - let allowlist = - wifi_densepose_sensing_server::host_validation::HostAllowlist::from_cli_and_env( - args.allowed_hosts - .iter() - .cloned() - .chain(std::iter::once(discovery_host)), - ); + let allowlist = companion_integration::host_allowlist( + &args, + companion.as_ref().map(companion_integration::Integration::authority), + ); info!( "Host-header validation ON ({} entries; loopback names always included)", allowlist.entries_for_test().len() @@ -15413,7 +16879,7 @@ async fn main() { // HTTP server (serves UI + full DensePose-compatible REST API) let ui_path = args.ui_path.clone(); - let http_app = Router::new() + let mut http_app = Router::new() .route("/", get(info_page)) // Health endpoints (DensePose-compatible) .route("/health", get(health)) @@ -15454,6 +16920,7 @@ async fn main() { .route("/api/v1/rf/vendors/:vendor/events", post(ingest_vendor_events)) // Per-node health endpoint .route("/api/v1/nodes", get(nodes_endpoint)) + .route("/api/v1/worldgraph/snapshot", get(managed_worldgraph::snapshot)) // ADR-110 iter 29 — per-node mesh sync state for HTTP clients. .route("/api/v1/nodes/:id/sync", get(node_sync_endpoint)) .route("/api/v1/mesh", get(mesh_endpoint)) @@ -15549,7 +17016,23 @@ async fn main() { ) .route("/api/v1/calibration/cancel", post(calibration_cancel)) .route("/api/v1/calibration/status", get(calibration_status)) + // Mutations remain behind the existing default-admin bearer policy. + .route("/api/v1/room-learner/status", get(room_learner_api::status_dispatch)) + .route("/api/v1/room-learner/selection", get(room_learner_api::selection_dispatch)) + .route("/api/v1/room-learner/progress", get(room_learner_api::progress_dispatch)) + .merge(room_learner_api::owner_selection_routes(bearer_auth_state.clone())) + .route("/api/v1/room-learner/configure", post(room_learner_api::configure_dispatch)) + .route("/api/v1/room-learner/phase/start", post(room_learner_api::begin_dispatch)) + .route("/api/v1/room-learner/phase/finish", post(room_learner_api::finish_dispatch)) + .route("/api/v1/room-learner/phase/cancel", post(room_learner_api::pause_dispatch)) + .route("/api/v1/room-learner/train", post(room_learner_api::train_dispatch)) + .route("/api/v1/room-learner/train/cancel", post(room_learner_api::cancel_operation_dispatch)) + .route("/api/v1/room-learner/evaluate", post(room_learner_api::evaluate_dispatch)) + .route("/api/v1/room-learner/prediction", get(room_learner_api::prediction)) + .route("/api/v1/room-learner/recovery/restore", post(room_learner_api::restore_dispatch)) + .route("/api/v1/room-learner/recovery/discard", post(room_learner_api::discard_dispatch)) .route("/api/v1/calibration/reset", post(calibration_reset)) + .route("/api/v1/calibration/bootstrap/recover", post(calibration_recover_bootstrap)) // ADR-044 §5.3: runtime-configurable dedup factor .route( "/api/v1/config/dedup-factor", @@ -15576,6 +17059,9 @@ async fn main() { // Merged AFTER `.with_state` (so http_app is already `Router<()>` and // can absorb the field router's own `FieldState`). .merge(rufield_surface::router(field_surface.clone())) + // MAT REST and WebSocket share one incident state on the HTTP origin + // used by the app, inside privacy/auth/host and companion protection. + .merge(wifi_densepose_sensing_server::mat_surface::router(args.privacy_mode)) // #2094: strip biometrics from every JSON response in privacy mode // (no-op otherwise). Applied after the merge so it covers every route. .layer(axum::middleware::from_fn_with_state( @@ -15606,6 +17092,13 @@ async fn main() { wifi_densepose_sensing_server::host_validation::require_allowed_host, )); + if let Some(companion) = companion { + let owner_router = companion.owner_router(); + http_app = companion.protect_control(http_app); + companion.start(http_app.clone()).await; + http_app = http_app.merge(owner_router); + } + let http_addr = SocketAddr::from((bind_ip, args.http_port)); let http_listener = tokio::net::TcpListener::bind(http_addr) .await @@ -15644,10 +17137,13 @@ async fn main() { // Run the HTTP server with graceful shutdown support let shutdown_state = state.clone(); - let server = axum::serve(http_listener, http_app).with_graceful_shutdown(async { - tokio::signal::ctrl_c() - .await - .expect("failed to install CTRL+C handler"); + let server = axum::serve(http_listener, http_app.into_make_service_with_connect_info::()).with_graceful_shutdown(async move { + tokio::select! { + result = tokio::signal::ctrl_c() => result.expect("failed to install CTRL+C handler"), + () = wifi_densepose_sensing_server::parent_lifetime::wait_for_exit(parent_exit) => { + info!("Managed parent ended; closing helper listeners"); + } + } info!("Shutdown signal received"); }); @@ -16438,8 +17934,11 @@ mod observatory_persons_field_position_tests { presence, confidence: 0.8, }, + classification_source: None, + selected_binary_evidence: None, signal_field, vital_signs: None, + vital_evidence: None, calibrated_presence_evidence: None, detector_diagnostics: None, enhanced_motion: None, @@ -16750,7 +18249,7 @@ async fn oauth_callback( Err(e) => return bad(axum::http::StatusCode::UNAUTHORIZED, format!("{e}\n")), }; - let session_cookie = match bs::issue(&principal, secure) { + let session_cookie = match bs::issue_for_account(&principal, secure, auth.owner_account_scope()) { Ok(c) => c, Err(e) => return bad(axum::http::StatusCode::SERVICE_UNAVAILABLE, format!("{e}\n")), }; @@ -16808,19 +18307,10 @@ async fn oauth_status( axum::Extension(auth): axum::Extension, headers: axum::http::HeaderMap, ) -> axum::Json { - use wifi_densepose_sensing_server::browser_session as bs; let raw = headers .get(axum::http::header::COOKIE) .and_then(|v| v.to_str().ok()); - let session = raw.and_then(bs::from_cookie_header); - axum::Json(serde_json::json!({ - "auth_required": auth.is_enabled(), - "oauth_enabled": auth.oauth_enabled(), - "browser_signin": auth.oauth_enabled() && bs::is_configured(), - "signed_in": session.is_some(), - "account": session.as_ref().map(|s| s.account_id.clone()), - "scope": session.as_ref().map(|s| s.scope.clone()), - })) + axum::Json(wifi_densepose_sensing_server::bearer_auth::browser_status(&auth, raw)) } #[cfg(test)] mod adr186_http_tests { @@ -17270,3 +18760,195 @@ mod issue_2088_data_dir_tests { assert!(!models_dir.join("dd_model_2088.rvf").exists()); } } + + +#[cfg(test)] +mod investigation_vitals_endpoint_tests { + use super::*; + + #[tokio::test] + async fn investigation_vitals_rest_buffers_follow_active_esp32_detector() { + let mut state = AppStateInner::minimal(); + state.source = "esp32".to_string(); + state.last_esp32_frame = Some(std::time::Instant::now()); + let mut node = NodeState::new(); + for _ in 0..12 { + node.vital_detector.process_frame(&[10.0, 11.0, 12.0, 13.0], &[0.0, 0.1, 0.2, 0.3]); + } + let (expected, _, expected_heart, _) = node.vital_detector.buffer_status(); + assert_eq!(expected, 12); + state.node_states.insert(15, node); + let Json(body) = vital_signs_endpoint(State(Arc::new(RwLock::new(state)))).await; + println!("INVESTIGATION: ESP32 node15 has {} breathing/{} heartbeat samples; endpoint reports {} / {}", expected, expected_heart, body["buffer_status"]["breathing_samples"], body["buffer_status"]["heartbeat_samples"]); + assert_eq!(body["buffer_status"]["breathing_samples"], serde_json::json!(expected), "REST diagnostic must describe the detector populated by the active ESP32 path"); + } + + #[test] + fn investigation_vitals_regular_bursts_do_not_continuously_reset_buffers() { + let mut detector = VitalSignDetector::new(20.0); + let mut fps = 20.0; + let mut resets = 0; + for index in 0..300 { + let dt = [0.001, 0.001, 0.094][index % 3]; + fps = update_csi_fps_ema(fps, dt).unwrap().clamp(1.0, 50.0); + if index >= 5 && detector.reconfigure_sample_rate(fps) { resets += 1; } + detector.process_frame(&[10.0, 11.0, 12.0, 13.0], &[0.0, 0.1, 0.2, 0.3]); + } + let buffers = detector.buffer_status(); + println!("INVESTIGATION: synthetic31.25Hz3-packet bursts: resets={} buffers={:?} detector_rate={}", resets, buffers, detector.sample_rate_hz()); + assert!(resets < 6); + assert!(buffers.0 > 40 && buffers.2 > 30); + } +} + +#[cfg(test)] +mod cached_stream_freshness_tests { + use super::*; + + fn fixture() -> AppStateInner { + let mut state = AppStateInner::minimal(); + state.source = "esp32".into(); + state.last_esp32_frame = Some(std::time::Instant::now()); + state.latest_update = Some(serde_json::from_value(serde_json::json!({ + "type":"sensing_update", "tick":9, + "timestamp":chrono::Utc::now().timestamp_millis() as f64 / 1000.0, + "source":"esp32", "nodes":[], + "features":{"mean_rssi":-50.0,"variance":1.0,"motion_band_power":0.5, + "breathing_band_power":0.2,"dominant_freq_hz":0.25,"change_points":0,"spectral_power":1.0}, + "classification":{"motion_level":"present_still","presence":true,"confidence":0.9}, + "signal_field":{"grid_size":[1,1,1],"values":[0.0]}, + "vital_signs":{"breathing_rate_bpm":14.0,"heart_rate_bpm":62.0, + "breathing_confidence":0.8,"heartbeat_confidence":0.7,"signal_quality":0.9}, + "pose_keypoints":[[0.1,0.2,0.0,0.9]] + })).unwrap()); + state + } + + #[test] + fn cached_snapshot_retains_observation_identity_and_ages_node_metadata() { + let mut state = fixture(); + let old = std::time::Instant::now() - Duration::from_secs(6); + state.last_esp32_frame = Some(old); + let mut node = NodeState::new(); + node.last_frame_time = Some(old); + state.node_states.insert(7, node); + let previous = serde_json::to_string(state.latest_update.as_ref().unwrap()).unwrap(); + let snapshot = current_sensing_snapshot(&state).unwrap(); + let original = state.latest_update.as_ref().unwrap(); + assert_eq!(snapshot.tick, original.tick); + assert_eq!(snapshot.timestamp, original.timestamp); + assert_eq!(snapshot.source, "esp32:offline"); + assert!(snapshot.classification.presence, "history must not become a fabricated absence"); + assert!(snapshot.vital_signs.is_none()); + assert!(snapshot.pose_keypoints.is_none()); + let node = &snapshot.node_features.as_ref().unwrap()[0]; + assert!(node.stale); + assert!(node.last_seen_ms >= 6000); + assert_eq!(serde_json::to_string(original).unwrap(), previous, "snapshot must not rewrite stored observations"); + } + + #[tokio::test] + async fn cached_snapshots_use_each_clients_filter_and_protocol_keepalive() { + use futures_util::StreamExt; + for privacy in [false, true] { + let mut inner = fixture(); + inner.privacy_mode = privacy; + let state = Arc::new(RwLock::new(inner)); + let app = Router::new() + .route("/ws/sensing", get(ws_sensing_handler)) + .route("/api/v1/stream/pose", get(ws_pose_handler)) + .with_state(state); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + for (path, kind) in [("/ws/sensing", "sensing_update"), ("/api/v1/stream/pose", "pose_data")] { + let (mut socket, _) = tokio_tungstenite::connect_async(format!("ws://{address}{path}")).await.unwrap(); + let mut snapshot = None; + let mut ping = false; + for _ in 0..4 { + let message = tokio::time::timeout(Duration::from_secs(1), socket.next()).await.unwrap().unwrap().unwrap(); + match message { + tokio_tungstenite::tungstenite::Message::Ping(_) => ping = true, + tokio_tungstenite::tungstenite::Message::Text(text) => { + let value: serde_json::Value = serde_json::from_str(&text).unwrap(); + if value["type"] == kind { snapshot = Some(text); } + } + _ => {} + } + if snapshot.is_some() && ping { break; } + } + assert!(ping, "{path} needs keepalive independent of sensor snapshots"); + let snapshot = snapshot.expect("initial snapshot without timer or producer"); + if privacy { + assert!(!snapshot.contains("vital_signs")); + assert!(!snapshot.contains("keypoints")); + } else if kind == "sensing_update" { + assert!(snapshot.contains("heart_rate_bpm")); + } + socket.close(None).await.unwrap(); + } + server.abort(); let _ = server.await; + } + } + + #[tokio::test] + async fn offline_legacy_snapshot_abstains_instead_of_deriving_a_pose_or_absence() { + use futures_util::StreamExt; + for model_loaded in [false, true] { + let mut inner = fixture(); + inner.model_loaded = model_loaded; + inner.last_esp32_frame = Some(std::time::Instant::now() - Duration::from_secs(6)); + let original_timestamp = inner.latest_update.as_ref().unwrap().timestamp; + let app = Router::new().route("/pose", get(ws_pose_handler)) + .with_state(Arc::new(RwLock::new(inner))); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + let (mut socket, _) = tokio_tungstenite::connect_async(format!("ws://{address}/pose")).await.unwrap(); + let value = loop { + let message = tokio::time::timeout(Duration::from_secs(1), socket.next()).await.unwrap().unwrap().unwrap(); + if let tokio_tungstenite::tungstenite::Message::Text(text) = message { + let value: serde_json::Value = serde_json::from_str(&text).unwrap(); + if value["type"] == "pose_data" { break value; } + } + }; + assert_eq!(value["timestamp"], original_timestamp); + assert_eq!(value["payload"]["pose_source"], "unavailable"); + assert_eq!(value["payload"]["activity"], "unavailable"); + assert!(value["payload"]["confidence"].is_null()); + assert!(value["payload"]["metadata"]["estimated_persons"].is_null()); + assert_eq!(value["payload"]["metadata"]["source"], "esp32:offline"); + assert_eq!(value["payload"]["pose"]["persons"], serde_json::json!([])); + assert!(value["physics"].is_null()); + socket.close(None).await.unwrap(); + server.abort(); let _ = server.await; + } + } + + #[tokio::test] + async fn timer_emits_only_source_transitions_and_remains_quiet_between_them() { + let state = Arc::new(RwLock::new(fixture())); + let mut rx = state.read().await.tx.subscribe(); + let task = tokio::spawn(broadcast_tick_task(state.clone(), 2)); + tokio::time::sleep(Duration::from_millis(15)).await; + assert!(matches!(rx.try_recv(), Err(tokio::sync::broadcast::error::TryRecvError::Empty))); + state.write().await.last_esp32_frame = Some(std::time::Instant::now() - Duration::from_secs(6)); + let offline = tokio::time::timeout(Duration::from_secs(1), rx.recv()).await.unwrap().unwrap(); + let offline: SensingUpdate = serde_json::from_str(&offline).unwrap(); + assert_eq!(offline.source, "esp32:offline"); + assert_eq!(offline.tick, 9); + tokio::time::sleep(Duration::from_millis(15)).await; + assert!(matches!(rx.try_recv(), Err(tokio::sync::broadcast::error::TryRecvError::Empty))); + state.write().await.last_esp32_frame = Some(std::time::Instant::now()); + let resumed = tokio::time::timeout(Duration::from_secs(1), rx.recv()).await.unwrap().unwrap(); + assert_eq!(serde_json::from_str::(&resumed).unwrap().source, "esp32"); + task.abort(); + let _ = task.await; + } +} +#[cfg(test)] +#[path="legacy_grid_readiness_tests.rs"] +mod legacy_grid_readiness_tests; + +#[path="legacy_grid_readiness.rs"] +mod legacy_grid_readiness; diff --git a/v2/crates/wifi-densepose-sensing-server/src/managed_worldgraph.rs b/v2/crates/wifi-densepose-sensing-server/src/managed_worldgraph.rs new file mode 100644 index 0000000000..449d6827b8 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/managed_worldgraph.rs @@ -0,0 +1,154 @@ +//! Configured topology, not a geographic registration or perception result. +use axum::{extract::State, http::{header, StatusCode}, response::{IntoResponse, Response}, Json}; +use serde_json::{json, Value}; +use crate::{companion_room::Envelope, SharedState}; + +const MAX_SAFE: u64 = 9_007_199_254_740_991; + +/// Ephemeral per-authority cursor. Reads of identical content never freshen it. +pub struct Cursor { epoch: String, sequence: u64, content: Option } +impl Default for Cursor { + fn default() -> Self { + let random: [u8; 32] = rand::random(); + Self { epoch: format!("wg-{}", random.iter().map(|b| format!("{b:02x}")).collect::()), sequence: 0, content: None } + } +} +impl Cursor { + pub fn project(&mut self, envelope: &Envelope, privacy_denied: bool) -> Result { + let content = topology(envelope, privacy_denied)?; + if self.content.as_ref() != Some(&content) { + if self.sequence >= MAX_SAFE { return Err(StatusCode::SERVICE_UNAVAILABLE); } + self.sequence += 1; + self.content = Some(content.clone()); + } + Ok(json!({"protocol_version":2,"stream_epoch":self.epoch,"seq":self.sequence,"topology":content})) + } +} + +fn topology(envelope: &Envelope, privacy_denied: bool) -> Result { + let mut result = json!({"schema":"ruview.worldgraph.managed-topology.v1", + "coordinate_frame":"managed_room_local","axis_convention":"x_right_y_down_metres","provenance":"operator_entered", + "registration":null,"installation_id":envelope.installation_id, + "room_revision":envelope.revision,"binding_digest":null, + "status":if privacy_denied {"privacy_denied"} else {"not_configured"}, + "room":null,"access_point":null,"nodes":[],"edges":[]}); + if privacy_denied { return Ok(result); } + let Some(room) = &envelope.room else { return Ok(result); }; + let binding = envelope.binding_digest.as_deref().ok_or(StatusCode::INTERNAL_SERVER_ERROR)?; + let source_nodes = room["nodes"].as_array().ok_or(StatusCode::INTERNAL_SERVER_ERROR)?; + if envelope.revision == 0 || source_nodes.is_empty() || source_nodes.len() > 16 { + return Err(StatusCode::INTERNAL_SERVER_ERROR); + } + // The store validates/sorts this immutable publication before exposing it. + // Allowlist fields: never serialize live engine/default room, tracks, media, + // semantic prose, learned history, raw CSI, credentials or vital estimates. + let mut nodes: Vec = source_nodes.iter().map(|node| json!({ + "kind":"sensor","id":257 + node["nodeId"].as_u64().unwrap_or(256), + "node_id":node["nodeId"],"slot":node["slot"],"x_m":node["xM"],"y_m":node["yM"], + "height_m":null,"role":node["role"],"heading_deg":node["headingDeg"] + })).collect(); + nodes.sort_by_key(|node| node["id"].as_u64().unwrap_or(0)); + let edges: Vec = nodes.iter().map(|node| json!({"from":node["id"],"to":1,"rel":"located_in"})).collect(); + result["status"] = json!("configured"); + result["binding_digest"] = json!(binding); + result["room"] = json!({"kind":"room","id":1,"room_id":room["id"],"name":room["name"], + "width_m":room["widthM"],"length_m":room["lengthM"],"height_m":room["heightM"]}); + let ap = &room["accessPoint"]; + if !ap.is_null() { result["access_point"] = json!({"slot":ap["slot"],"x_m":ap["xM"],"y_m":ap["yM"],"provenance":"operator_entered"}); } + result["nodes"] = json!(nodes); + result["edges"] = json!(edges); + Ok(result) +} + +pub async fn snapshot(State(state): State) -> Result { + let state = state.read().await; + let store = state.current_room_store.as_ref().ok_or(StatusCode::NOT_IMPLEMENTED)?; + let value = store.worldgraph_snapshot(state.privacy_mode)?; + Ok(([(header::CACHE_CONTROL, "private, no-store")], Json(value)).into_response()) +} + +#[cfg(test)] +mod tests { + use super::*; + use axum::{body::{Body, to_bytes}, http::Request, routing::get, Router}; + use tower::ServiceExt; + fn envelope() -> Envelope { + Envelope { installation_id: "installation-1".into(), revision: 7, binding_digest: Some("a".repeat(64)), room: Some(json!({ + "id":"room-1","name":"Configured room","widthM":2.5,"lengthM":3.0,"heightM":2.4, + "nodes":[{"nodeId":15,"slot":"north_west","xM":1.25,"yM":2.0,"role":"receiver","headingDeg":150}], + "raw_csi":"forbidden","person_track":"forbidden","accessPoint":{"slot":"north_east","xM":0,"yM":0} + })) } + } + #[test] + fn configured_projection_is_planar_and_allowlisted() { + let value = topology(&envelope(), false).unwrap(); + assert_eq!(value["room"]["width_m"], 2.5); + assert_eq!(value["nodes"][0]["x_m"], 1.25); + assert_eq!(value["nodes"][0]["id"], 272); + assert!(value["nodes"][0]["height_m"].is_null()); + assert!(value["registration"].is_null()); + assert_eq!(value["edges"][0], json!({"from":272,"to":1,"rel":"located_in"})); + let text = value.to_string(); + for forbidden in ["raw_csi","person_track","accessPoint","confidence","latitude"] { assert!(!text.contains(forbidden)); } + } + #[test] + fn cursors_cover_privacy_binding_readiness_and_restart() { + let mut cursor = Cursor::default(); let original = envelope(); + let a = cursor.project(&original, false).unwrap(); + assert_eq!(a, cursor.project(&original, false).unwrap()); + let denied = cursor.project(&original, true).unwrap(); + assert_eq!(denied["seq"], 2); + assert!(denied["topology"]["room"].is_null()); assert_eq!(denied["topology"]["nodes"], json!([])); + let mut changed = original.clone(); changed.revision += 1; changed.binding_digest = Some("b".repeat(64)); + assert_eq!(cursor.project(&changed, false).unwrap()["seq"], 3); + let mut empty = changed; empty.room = None; empty.binding_digest = None; + let absent = cursor.project(&empty, false).unwrap(); assert_eq!(absent["topology"]["status"], "not_configured"); + assert_ne!(a["stream_epoch"], Cursor::default().project(&original, false).unwrap()["stream_epoch"]); + assert_eq!(original.revision, 7); + } + #[test] + fn refuses_missing_binding_and_cursor_overflow() { + let mut missing = envelope(); missing.binding_digest = None; assert!(topology(&missing, false).is_err()); + let mut cursor = Cursor::default(); cursor.sequence = MAX_SAFE; assert!(cursor.project(&envelope(), false).is_err()); + } + #[test] + fn actual_projection_matches_cross_language_fixture() { + let mut cursor = Cursor { epoch: format!("wg-{}", "1".repeat(64)), sequence: 0, content: None }; + let wire = cursor.project(&envelope(), false).unwrap(); + let fixture: Value = serde_json::from_str(include_str!("../tests/fixtures/managed-worldgraph-v1.json")).unwrap(); + assert_eq!(wire, fixture); + } + #[tokio::test] + async fn real_route_requires_read_authority_and_never_publishes_or_selects() { + use wifi_densepose_sensing_server::bearer_auth::{AuthState, require_bearer}; + let directory = tempfile::tempdir().unwrap(); + let store = crate::companion_room::SharedRoomStore::load(directory.path(), "installation-1").unwrap(); + let publication = json!({"expected_revision":0,"room":{ + "schema":"ruview.companion.room.v1","id":"room-1","name":"Configured room","widthM":2.5,"lengthM":3,"heightM":2.4, + "nodes":[{"nodeId":15,"slot":"north_west","xM":1.25,"yM":2,"role":"receiver","headingDeg":150,"orientationConfirmed":true,"confirmedAtUnixMs":1}], + "accessPoint":{"slot":"south_east","xM":2,"yM":2,"provenance":"operator_entered","confirmedAtUnixMs":1} + }}); + let published = crate::companion_room::owner_router(store.clone()).oneshot(Request::builder().method("PUT") + .uri("/api/v1/companion/room").header("content-type","application/json").body(Body::from(publication.to_string())).unwrap()).await.unwrap(); + assert_eq!(published.status(), StatusCode::OK); + let original = store.snapshot().unwrap(); + let mut inner = crate::AppStateInner::minimal(); inner.current_room_store = Some(store.clone()); + let state = std::sync::Arc::new(tokio::sync::RwLock::new(inner)); + let app = Router::new().route("/api/v1/worldgraph/snapshot", get(snapshot)).with_state(state.clone()) + .layer(axum::middleware::from_fn_with_state(AuthState::from_token("owner-test-token"), require_bearer)); + let request = || Request::builder().uri("/api/v1/worldgraph/snapshot").header("authorization","Bearer owner-test-token").body(Body::empty()).unwrap(); + let denied = app.clone().oneshot(Request::builder().uri("/api/v1/worldgraph/snapshot").body(Body::empty()).unwrap()).await.unwrap(); + assert_eq!(denied.status(), StatusCode::UNAUTHORIZED); + let response = app.clone().oneshot(request()).await.unwrap(); + assert_eq!(response.status(), StatusCode::OK); assert_eq!(response.headers()[header::CACHE_CONTROL], "private, no-store"); + let first: Value = serde_json::from_slice(&to_bytes(response.into_body(), 16_384).await.unwrap()).unwrap(); + assert_eq!(first["topology"]["status"],"configured"); + assert_eq!(store.snapshot().unwrap().revision, original.revision); + assert_eq!(store.snapshot().unwrap().binding_digest, original.binding_digest); + state.write().await.privacy_mode = true; + let response = app.oneshot(request()).await.unwrap(); + let privacy: Value = serde_json::from_slice(&to_bytes(response.into_body(), 16_384).await.unwrap()).unwrap(); + assert_eq!(privacy["topology"]["status"],"privacy_denied"); assert!(privacy["topology"]["room"].is_null()); + assert_eq!(privacy["seq"].as_u64().unwrap(), first["seq"].as_u64().unwrap()+1); + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/mat_surface.rs b/v2/crates/wifi-densepose-sensing-server/src/mat_surface.rs new file mode 100644 index 0000000000..8df3b81cca --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/mat_surface.rs @@ -0,0 +1,450 @@ +//! Host the existing MAT router without creating demo incidents or detections. +//! Incident state is in-memory; CSI ingest must be configured explicitly. + +/// Attach before host, bearer, companion and privacy middleware are applied. +pub fn router(privacy_mode: bool) -> axum::Router { + router_with_state(wifi_densepose_mat::api::AppState::new(), privacy_mode) +} + +fn router_with_state(state: wifi_densepose_mat::api::AppState, privacy_mode: bool) -> axum::Router { + let router = wifi_densepose_mat::api::create_router(state); + if privacy_mode { + router.layer(axum::Extension( + wifi_densepose_mat::api::websocket::StreamOutputFilter( + crate::privacy_filter::redact_json_str, + ), + )) + } else { + router + } +} + +#[cfg(test)] +mod tests { + use axum::{ + body::{to_bytes, Body}, + http::{Request, StatusCode}, + }; + use tower::ServiceExt; + + #[tokio::test] + async fn real_mat_router_starts_empty_and_reports_optional_ml_truthfully() { + let app = super::router(false); + for path in [ + "/api/v1/mat/events?page=0&page_size=100", + "/api/v1/mat/scan/status", + ] { + let response = app + .clone() + .oneshot(Request::builder().uri(path).body(Body::empty()).unwrap()) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK, "{path}"); + let body = to_bytes(response.into_body(), 128 * 1024).await.unwrap(); + let value: serde_json::Value = serde_json::from_slice(&body).unwrap(); + if path.contains("events") { + assert_eq!(value["events"].as_array().unwrap().len(), 0); + assert_eq!(value["total"], 0); + } else { + assert_eq!(value["scanning"], false); + assert_eq!(value["ml_enabled"], false); + assert_eq!(value["ml_ready"], false); + } + } + } + + #[tokio::test] + async fn mat_reads_mutations_and_stream_remain_behind_host_auth_layers() { + let app = super::router(false).layer(axum::middleware::from_fn_with_state( + crate::bearer_auth::AuthState::from_token("mat-test-fixture-token"), + crate::bearer_auth::require_bearer, + )); + for (method, path) in [ + ("GET", "/api/v1/mat/events"), + ("POST", "/api/v1/mat/scan/control"), + ("GET", "/ws/mat/stream"), + ] { + let response = app + .clone() + .oneshot( + Request::builder() + .method(method) + .uri(path) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!( + response.status(), + StatusCode::UNAUTHORIZED, + "{method} {path}" + ); + } + let response = app + .oneshot( + Request::builder() + .uri("/api/v1/mat/scan/status") + .header("authorization", "Bearer mat-test-fixture-token") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + } + + #[tokio::test] + async fn real_incident_create_and_read_use_only_explicit_test_state() { + let app = super::router(false); + let body = r#"{"event_type":"Earthquake","latitude":0,"longitude":0,"description":"EXPLICIT ISOLATED TEST FIXTURE"}"#; + let response = app + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri("/api/v1/mat/events") + .header("content-type", "application/json") + .body(Body::from(body)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::CREATED); + let response = app + .oneshot( + Request::builder() + .uri("/api/v1/mat/events?page=0&page_size=100") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + let bytes = to_bytes(response.into_body(), 128 * 1024).await.unwrap(); + let value: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(value["total"], 1); + assert_eq!(value["events"][0]["survivor_count"], 0); + assert_eq!( + value["events"][0]["description"], + "EXPLICIT ISOLATED TEST FIXTURE" + ); + } + + // SYNTHETIC values in isolated in-memory incident state. This exercises the + // real MAT handlers, upgrade, broadcaster and host output/auth policies; + // it makes no claim about sensor detection or the packaged app. + #[tokio::test] + async fn authenticated_nonempty_mat_stream_and_snapshot_obey_privacy() { + use futures_util::StreamExt; + use tokio_tungstenite::{connect_async, tungstenite::client::IntoClientRequest}; + use wifi_densepose_mat::{api::dto::*, domain::*}; + + for privacy in [false, true] { + let state = wifi_densepose_mat::api::AppState::new(); + let app = super::router_with_state(state.clone(), privacy) + .layer(axum::middleware::from_fn_with_state( + crate::privacy_filter::PrivacyFilter::new(privacy), + crate::privacy_filter::redact_json_responses, + )) + .layer(axum::middleware::from_fn_with_state( + crate::bearer_auth::AuthState::from_token("mat-test-fixture-token"), + crate::bearer_auth::require_bearer, + )); + let response = app.clone().oneshot(Request::builder() + .method("POST").uri("/api/v1/mat/events") + .header("authorization", "Bearer mat-test-fixture-token") + .header("content-type", "application/json") + .body(Body::from(r#"{"event_type":"Earthquake","latitude":0,"longitude":0,"description":"SYNTHETIC PRIVACY REGRESSION"}"#)).unwrap() + ).await.unwrap(); + assert_eq!(response.status(), StatusCode::CREATED); + let bytes = to_bytes(response.into_body(), 128 * 1024).await.unwrap(); + let event: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + let event_id = event["id"].as_str().unwrap().parse().unwrap(); + let zone = ScanZone::new("SYNTHETIC", ZoneBounds::rectangle(0., 0., 10., 10.)); + let zone_id = zone.id().clone(); + state + .update_event(event_id, |event| { + event.add_zone(zone); + event + .record_detection( + zone_id.clone(), + VitalSignsReading { + breathing: Some(BreathingPattern { + rate_bpm: 16., + amplitude: 0.8, + regularity: 0.9, + pattern_type: BreathingType::Normal, + }), + heartbeat: None, + movement: Default::default(), + timestamp: chrono::Utc::now(), + confidence: ConfidenceScore::new(0.9), + }, + None, + ) + .unwrap(); + }) + .unwrap(); + let domain_survivor = state.get_event(event_id).unwrap().survivors()[0].clone(); + state + .event_store() + .append(events::DomainEvent::Detection( + events::DetectionEvent::SurvivorDetected { + survivor_id: domain_survivor.id().clone(), + zone_id: zone_id.clone(), + vital_signs: domain_survivor.vital_signs().latest().unwrap().clone(), + location: None, + timestamp: chrono::Utc::now(), + }, + )) + .unwrap(); + let response = app + .clone() + .oneshot( + Request::builder() + .uri("/api/v1/mat/events/domain") + .header("authorization", "Bearer mat-test-fixture-token") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let bytes = to_bytes(response.into_body(), 128 * 1024).await.unwrap(); + let domain_events: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(domain_events["total"], 1); + assert_eq!(domain_events["events"][0]["event_type"], "SurvivorDetected"); + assert_eq!(domain_events["events"][0].get("details").is_none(), privacy); + if !privacy { + assert!(domain_events["events"][0]["details"] + .as_str() + .unwrap() + .contains("rate_bpm: 16.0")); + } + let alert = wifi_densepose_mat::alerting::AlertGenerator::new() + .generate(&domain_survivor) + .unwrap(); + assert!(alert.payload().message.contains("Breathing: 16.0 BPM")); + state.store_alert(alert.clone(), event_id); + let response = app + .clone() + .oneshot( + Request::builder() + .uri(format!("/api/v1/mat/events/{event_id}/alerts")) + .header("authorization", "Bearer mat-test-fixture-token") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let bytes = to_bytes(response.into_body(), 128 * 1024).await.unwrap(); + let alerts: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(alerts["total"], 1); + assert_eq!(alerts["alerts"][0].get("message").is_none(), privacy); + if !privacy { + assert!(alerts["alerts"][0]["message"] + .as_str() + .unwrap() + .contains("Breathing: 16.0 BPM")); + } + // Non-empty REST snapshot, including an array-nested vital_signs + // object: policy parity with the socket is the security invariant. + let response = app + .clone() + .oneshot( + Request::builder() + .uri(format!("/api/v1/mat/events/{event_id}/survivors")) + .header("authorization", "Bearer mat-test-fixture-token") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let bytes = to_bytes(response.into_body(), 128 * 1024).await.unwrap(); + let snapshot: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(snapshot["total"], 1); + assert_eq!( + snapshot["survivors"][0].get("vital_signs").is_none(), + privacy + ); + if !privacy { + assert_eq!( + snapshot["survivors"][0]["vital_signs"]["breathing_rate"], + 16. + ); + } + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let served_app = app.clone(); + let server = + tokio::spawn(async move { axum::serve(listener, served_app).await.unwrap() }); + let url = format!("ws://{addr}/ws/mat/stream"); + match connect_async(&url).await { + Err(tokio_tungstenite::tungstenite::Error::Http(response)) => { + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + } + other => panic!("unauthenticated MAT upgrade was not denied: {other:?}"), + } + let mut request = url.into_client_request().unwrap(); + request.headers_mut().insert( + "authorization", + "Bearer mat-test-fixture-token".parse().unwrap(), + ); + let (mut socket, response) = connect_async(request).await.unwrap(); + assert_eq!(response.status(), StatusCode::SWITCHING_PROTOCOLS); + tokio::time::timeout(std::time::Duration::from_secs(5), async { + while state.subscriber_count() == 0 { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + let survivor = SurvivorResponse { + id: *state.get_event(event_id).unwrap().survivors()[0] + .id() + .as_uuid(), + zone_id: *zone_id.as_uuid(), + status: SurvivorStatusDto::Active, + triage_status: TriageStatusDto::Minor, + location: None, + vital_signs: VitalSignsSummaryDto { + breathing_rate: Some(16.), + breathing_type: Some("Normal".into()), + heart_rate: Some(72.), + has_heartbeat: true, + has_movement: false, + movement_type: None, + timestamp: chrono::Utc::now(), + }, + confidence: 0.9, + first_detected: chrono::Utc::now(), + last_updated: chrono::Utc::now(), + is_deteriorating: false, + metadata: None, + }; + for (kind, message) in [ + ( + "survivor_detected", + WebSocketMessage::SurvivorDetected { + event_id, + survivor: survivor.clone(), + }, + ), + ( + "survivor_updated", + WebSocketMessage::SurvivorUpdated { event_id, survivor }, + ), + ( + "event_status_changed", + WebSocketMessage::EventStatusChanged { + event_id, + old_status: EventStatusDto::Initializing, + new_status: EventStatusDto::Active, + }, + ), + ] { + state.broadcast(message); + let frame = tokio::time::timeout(std::time::Duration::from_secs(5), socket.next()) + .await + .unwrap() + .unwrap() + .unwrap(); + let value: serde_json::Value = + serde_json::from_str(frame.to_text().unwrap()).unwrap(); + assert_eq!(value["type"], kind); + assert_eq!(value["event_id"], event_id.to_string()); + if kind.starts_with("survivor_") { + assert_eq!(value["survivor"]["confidence"], 0.9); + assert_eq!( + value["survivor"].get("vital_signs").is_none(), + privacy, + "{value}" + ); + if !privacy { + assert_eq!(value["survivor"]["vital_signs"]["heart_rate"], 72.); + assert_eq!(value["survivor"]["vital_signs"]["breathing_rate"], 16.); + } + } else { + assert_eq!(value["new_status"], "Active"); + } + } + let alert_response = AlertResponse { + id: *alert.id().as_uuid(), + survivor_id: *alert.survivor_id().as_uuid(), + priority: alert.priority().into(), + status: alert.status().clone().into(), + title: alert.payload().title.clone(), + message: alert.payload().message.clone(), + triage_status: alert.payload().triage_status.clone().into(), + location: None, + recommended_action: Some(alert.payload().recommended_action.clone()), + created_at: *alert.created_at(), + acknowledged_at: None, + acknowledged_by: None, + escalation_count: 0, + }; + for message in [ + WebSocketMessage::AlertCreated { + event_id, + alert: alert_response.clone(), + }, + WebSocketMessage::AlertUpdated { + event_id, + alert: alert_response, + }, + ] { + state.broadcast(message); + let frame = tokio::time::timeout(std::time::Duration::from_secs(5), socket.next()) + .await + .unwrap() + .unwrap() + .unwrap(); + let value: serde_json::Value = + serde_json::from_str(frame.to_text().unwrap()).unwrap(); + assert_eq!(value["alert"]["id"], alert.id().to_string()); + assert_eq!(value["alert"].get("message").is_none(), privacy, "{value}"); + assert!(value["alert"].get("recommended_action").is_some()); + if !privacy { + assert!(value["alert"]["message"] + .as_str() + .unwrap() + .contains("Breathing: 16.0 BPM")); + } + } + // Actual REST mutation -> generated AlertUpdated -> authenticated + // socket, not only a manually constructed broadcast fixture. + let response = app + .oneshot( + Request::builder() + .method("POST") + .uri(format!("/api/v1/mat/alerts/{}/acknowledge", alert.id())) + .header("authorization", "Bearer mat-test-fixture-token") + .header("content-type", "application/json") + .body(Body::from( + r#"{"acknowledged_by":"SYNTHETIC PRIVACY TEST"}"#, + )) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let bytes = to_bytes(response.into_body(), 128 * 1024).await.unwrap(); + let acknowledgement: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(acknowledgement["alert"].get("message").is_none(), privacy); + let frame = tokio::time::timeout(std::time::Duration::from_secs(5), socket.next()) + .await + .unwrap() + .unwrap() + .unwrap(); + let value: serde_json::Value = serde_json::from_str(frame.to_text().unwrap()).unwrap(); + assert_eq!(value["type"], "alert_updated"); + assert_eq!(value["alert"]["status"], "Acknowledged"); + assert_eq!(value["alert"].get("message").is_none(), privacy); + socket.close(None).await.unwrap(); + server.abort(); + } + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/model_manager.rs b/v2/crates/wifi-densepose-sensing-server/src/model_manager.rs index 6cbeebef9f..b982748611 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/model_manager.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/model_manager.rs @@ -39,6 +39,9 @@ pub const MODELS_DIR_DEFAULT: &str = "data/models"; /// Return the effective models directory, respecting `MODELS_DIR` env var. pub fn models_dir() -> PathBuf { + if std::env::var_os(crate::bearer_auth::OWNER_ACCOUNT_SCOPE_ENV).is_some() { + return std::env::current_dir().unwrap_or_else(|_| PathBuf::from(".")).join("models"); + } PathBuf::from( std::env::var("MODELS_DIR").unwrap_or_else(|_| MODELS_DIR_DEFAULT.to_string()), ) diff --git a/v2/crates/wifi-densepose-sensing-server/src/paired_progress_auth_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/paired_progress_auth_tests.rs new file mode 100644 index 0000000000..77efc18529 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/paired_progress_auth_tests.rs @@ -0,0 +1,126 @@ +//! Synthetic HTTP producer plus actual companion policy/credential boundaries. +use super::*; +use axum::{body::Body,http::{Method,Request},routing::get}; +use tower::ServiceExt; +use wifi_densepose_sensing_server::room_learner::{lifecycle::RoomLearningRuntime,observation::{CaptureContext,SourceSpec}}; +const PATH:&str="/api/v1/room-learner/progress"; +async fn json_call(router:Router,method:Method,path:&str,body:serde_json::Value)->(StatusCode,serde_json::Value){ + let r=router.oneshot(Request::builder().method(method).uri(path).header("content-type","application/json").body(Body::from(body.to_string())).unwrap()).await.unwrap(); + let status=r.status();let b=axum::body::to_bytes(r.into_body(),65536).await.unwrap();(status,serde_json::from_slice(&b).unwrap_or(serde_json::Value::Null)) +} +async fn pair(service:Arc,device:&str)->String { + let owner=companion::owner_router(service.clone());let public=companion::public_router(service); + let (s,invite)=json_call(owner.clone(),Method::POST,"/api/v1/companion/invites",serde_json::json!({})).await;assert_eq!(s,StatusCode::OK); + let proof=companion::secret(); + let (s,enrolled)=json_call(public.clone(),Method::POST,"/companion/v1/enroll",serde_json::json!({"installation_id":"synthetic-install","invite_id":invite["invite_id"],"invite_secret":invite["invite_secret"],"device_id":device,"device_name":"Synthetic phone","pairing_secret":proof})).await;assert_eq!(s,StatusCode::OK); + let (s,_)=json_call(owner,Method::POST,&format!("/api/v1/companion/requests/{}/approve",enrolled["request_id"].as_str().unwrap()),serde_json::json!({"comparison_code":enrolled["comparison_code"]})).await;assert_eq!(s,StatusCode::OK); + let (s,session)=json_call(public,Method::POST,"/companion/v1/session",serde_json::json!({"installation_id":"synthetic-install","device_id":device,"pairing_secret":proof})).await;assert_eq!(s,StatusCode::OK);session["access_token"].as_str().unwrap().into() +} +fn room()->serde_json::Value{serde_json::json!({"schema":"ruview.companion.room.v1","id":"synthetic-room","name":"Synthetic room","widthM":3,"lengthM":3,"heightM":2.8,"nodes":[{"nodeId":1,"slot":"north_west","xM":0,"yM":0,"role":"base_station","headingDeg":90,"orientationConfirmed":true,"confirmedAtUnixMs":1},{"nodeId":2,"slot":"south_east","xM":3,"yM":3,"role":"receiver","headingDeg":270,"orientationConfirmed":true,"confirmedAtUnixMs":1}],"accessPoint":null})} +async fn fixture()->(tempfile::TempDir,SharedState,Arc){ + let dir=tempfile::tempdir().unwrap();let owner="a".repeat(64); + let rt=Arc::new(RoomLearningRuntime::new(&dir.path().join(&owner).join("SensingServer"),&owner,"synthetic-install").unwrap()); + let publication=companion_room::SharedRoomStore::load(&dir.path().join("publication"),"synthetic-install").unwrap(); + let (status,envelope)=json_call(companion_room::owner_router(publication.clone()),Method::PUT,"/api/v1/companion/room",serde_json::json!({"expected_revision":0,"room":room()})).await;assert_eq!(status,StatusCode::OK); + let context=CaptureContext{owner_scope:owner,installation_id:"synthetic-install".into(),room_digest:envelope["binding_digest"].as_str().unwrap().into(),revision:77,sources:(1..=2).map(|id|SourceSpec{source_id:id,device_id:format!("esp32-node-{id}"),grid_id:"ant1-bins64-ppdu0-usable-drop2-linear-v2".into(),feature_count:56,signal_kind:ruview_unified::RfModality::WifiCsi,center_frequency_hz:2_437_000_000,bandwidth_hz:20_000_000,link_id:format!("synthetic-link-{id}"),link_index:0,geometry:None}).collect()}; + rt.configure(&rt.mutation_guard().unwrap(),context,"synthetic-actual-http-run".into(),true).unwrap(); + let mut inner=AppStateInner::minimal();inner.room_learning=Some(rt);inner.current_room_store=Some(publication.clone());inner.installation_id=Some("synthetic-install".into());inner.calibration_boot_epoch="cal-boot-synthetic".into(); + let now=std::time::Instant::now(); + for row in 0..120u32 {for node in 1..=2u8 { + let mut wire=Vec::new();wire.extend(0xC5110001u32.to_le_bytes());wire.extend([node,1]);wire.extend(64u16.to_le_bytes());wire.extend(2437u32.to_le_bytes());wire.extend((row+1).to_le_bytes());wire.extend([(-40i8)as u8,(-90i8)as u8,0,0x20]);for _ in 0..64 {wire.extend([3,4]);} + let frame=parse_esp32_frame(&wire).unwrap();room_learner_api::observe(&mut inner,&frame,&wire,now-std::time::Duration::from_millis(u64::from(120-row)*100)); + }} + (dir,Arc::new(RwLock::new(inner)),publication) +} +fn app(state:SharedState,service:Arc)->Router{ + Router::new().route(PATH,get(room_learner_api::progress)).route("/api/v1/room-learner/status",get(room_learner_api::status)) + .route("/api/v1/room-learner/phase/start",post(room_learner_api::begin_phase)) + .with_state(state).layer(axum::middleware::from_fn_with_state(service,require_companion)) +} +async fn fetch(router:Router,method:Method,installation:&str,token:&str,path:&str)->(StatusCode,serde_json::Value){ + let response=router.oneshot(Request::builder().method(method).uri(path).header("x-ruview-installation-id",installation).header("authorization",format!("Bearer {token}")).body(Body::empty()).unwrap()).await.unwrap(); + let status=response.status();let b=axum::body::to_bytes(response.into_body(),65536).await.unwrap();(status,serde_json::from_slice(&b).unwrap_or(serde_json::Value::Null)) +} +#[tokio::test] +async fn progress_http_read_scope_installation_revocation_and_no_mutation_grant_compose(){ + let (dir,state,_)=fixture().await;let service=Companion::load(&dir.path().join("pair"),"synthetic-install".into(),"f".repeat(64),"https://synthetic.local:3443".into()).unwrap();let token=pair(service.clone(),"phone").await;let router=app(state.clone(),service.clone()); + let rt=state.read().await.room_learning.clone().unwrap();let guard=rt.mutation_guard().unwrap(); + let (s,p)=fetch(router.clone(),Method::GET,"synthetic-install",&token,PATH).await;assert_eq!(s,StatusCode::OK);assert_eq!(p["schema"],"ruview.room-learner.progress.v1");assert_eq!(p["context_match"],true);assert_eq!(p["room"]["revision"],1);assert_eq!(p["sources"]["required"],2);assert_eq!(p["sources"]["ready"],2);assert_eq!(p["save_status"],"not_reported");assert_eq!(rt.mutation_guard().unwrap(),guard); + assert!(service.authorize("synthetic-install",&token,true).is_err(),"read grant must remain read-only"); + for (method,path,install,t) in [(Method::POST,PATH,"synthetic-install",token.as_str()),(Method::POST,"/api/v1/room-learner/phase/start","synthetic-install",token.as_str()),(Method::GET,"/api/v1/room-learner/status","synthetic-install",token.as_str()),(Method::GET,PATH,"foreign-install",token.as_str()),(Method::GET,PATH,"synthetic-install","invalid")] { + let status=fetch(router.clone(),method,install,t,path).await.0;assert!(matches!(status,StatusCode::UNAUTHORIZED|StatusCode::FORBIDDEN)); + } + assert_eq!(fetch(router.clone(),Method::GET,"synthetic-install",&token,&format!("{PATH}?token=ignored")).await.0,StatusCode::BAD_REQUEST); + assert_eq!(json_call(companion::owner_router(service.clone()),Method::POST,"/api/v1/companion/devices/phone/calibration",serde_json::json!({})).await.0,StatusCode::OK); + assert!(service.authorize("synthetic-install",&token,true).is_ok()); + assert_eq!(fetch(router.clone(),Method::POST,"synthetic-install",&token,"/api/v1/room-learner/phase/start").await.0,StatusCode::FORBIDDEN,"legacy calibration grant must not enable learner mutations"); + assert_eq!(fetch(router.clone(),Method::POST,"synthetic-install",&token,PATH).await.0,StatusCode::FORBIDDEN); + + // A separate account/installation service does not recognize this device proof. + let foreign=Companion::load(&dir.path().join("foreign-pair"),"synthetic-install".into(),"e".repeat(64),"https://foreign.local:3443".into()).unwrap(); + assert_eq!(fetch(app(state,foreign),Method::GET,"synthetic-install",&token,PATH).await.0,StatusCode::UNAUTHORIZED); + let (s,_)=json_call(companion::owner_router(service),Method::DELETE,"/api/v1/companion/devices/phone",serde_json::json!({})).await;assert_eq!(s,StatusCode::OK); + assert_eq!(fetch(router,Method::GET,"synthetic-install",&token,PATH).await.0,StatusCode::UNAUTHORIZED); +} +#[tokio::test] +async fn progress_http_private_projection_context_and_privacy_withhold_other_room_data(){ + let (dir,state,publication)=fixture().await;let service=Companion::load(&dir.path().join("pair"),"synthetic-install".into(),"f".repeat(64),"https://synthetic.local:3443".into()).unwrap();let token=pair(service.clone(),"phone").await;let router=app(state.clone(),service); + let (_,body)=fetch(router.clone(),Method::GET,"synthetic-install",&token,PATH).await; + let serialized=body.to_string();for secret in ["owner_scope","mutation_guard","trial_id","session_id","frame_ids","sequence_ids","last_accepted_sequence","training_data_sha256","encoder_weights","occupied_probability","heart_rate","amplitudes","geometry"]{assert!(!serialized.contains(secret),"leaked {secret}");} + let mut changed=room();changed["id"]=serde_json::json!("another-room");assert_eq!(json_call(companion_room::owner_router(publication.clone()),Method::PUT,"/api/v1/companion/room",serde_json::json!({"expected_revision":1,"room":changed})).await.0,StatusCode::OK); + let (_,mismatch)=fetch(router.clone(),Method::GET,"synthetic-install",&token,PATH).await;assert_eq!(mismatch["room"]["revision"],2);assert_eq!(mismatch["context_match"],false);assert!(mismatch["conditions"].is_null()&&mismatch["phase"].is_null()&&mismatch["sources"].is_null());assert_eq!(mismatch["blocker"],"context_mismatch"); + state.write().await.privacy_mode=true; + let (s,privacy)=fetch(router,Method::GET,"synthetic-install",&token,PATH).await;assert_eq!(s,StatusCode::FORBIDDEN);assert_eq!(privacy,serde_json::json!({"schema":"ruview.room-learner.progress.v1","success":false,"error_code":"privacy_redacted"})); +} +#[tokio::test] +async fn export_actual_paired_progress_producer_cases(){ + let (dir,state,publication)=fixture().await;let service=Companion::load(&dir.path().join("pair"),"synthetic-install".into(),"f".repeat(64),"https://synthetic.local:3443".into()).unwrap();let token=pair(service.clone(),"phone").await;let router=app(state.clone(),service); + let mut cases=vec![]; + let (s,p)=fetch(router.clone(),Method::GET,"synthetic-install",&token,PATH).await;assert_eq!(s,StatusCode::OK);cases.push(serde_json::json!({"name":"configured","payload":p})); + let rt=state.read().await.room_learning.clone().unwrap(); + rt.begin_phase(&rt.mutation_guard().unwrap(),wifi_densepose_sensing_server::room_learner::capture::PhaseRequest{trial_id:"private-operator-trial".into(),session_id:"private-operator-session".into(),role:wifi_densepose_sensing_server::room_learner::observation::DataRole::Train,label:wifi_densepose_sensing_server::room_learner::observation::RoomLabel::Empty,duration_ms:600_000,lead_in_ms:15_000,conditions_confirmed:true},rt.receiver_now_ns()).unwrap(); + let (_,p)=fetch(router.clone(),Method::GET,"synthetic-install",&token,PATH).await;assert_eq!(p["phase"]["stage"],"lead_in");cases.push(serde_json::json!({"name":"preparing_empty","payload":p}));rt.cancel_phase(&rt.mutation_guard().unwrap()).unwrap(); + let mut changed=room();changed["id"]=serde_json::json!("changed-room");assert_eq!(json_call(companion_room::owner_router(publication.clone()),Method::PUT,"/api/v1/companion/room",serde_json::json!({"expected_revision":1,"room":changed})).await.0,StatusCode::OK);let (_,p)=fetch(router.clone(),Method::GET,"synthetic-install",&token,PATH).await;cases.push(serde_json::json!({"name":"context_mismatch","payload":p})); + state.write().await.privacy_mode=true;let (_,p)=fetch(router,Method::GET,"synthetic-install",&token,PATH).await;cases.push(serde_json::json!({"name":"privacy_redacted","payload":p})); + if let Ok(path)=std::env::var("RUVIEW_TEST_PAIRED_PROGRESS_OUTPUT") { + use std::io::Write;use std::os::unix::fs::OpenOptionsExt; + let path=std::path::PathBuf::from(path);assert!(path.is_absolute()&&path.parent().unwrap().canonicalize().unwrap().starts_with(std::env::temp_dir().canonicalize().unwrap())); + let mut file=std::fs::OpenOptions::new().write(true).create_new(true).mode(0o600).open(path).unwrap(); + file.write_all(serde_json::to_string_pretty(&serde_json::json!({"schema":"ruview.test.paired-progress-producer.v1","evidence":"SYNTHETIC_ACTUAL_HTTP_PRODUCER_NOT_PHYSICAL_QUALIFICATION","cases":cases})).unwrap().as_bytes()).unwrap(); + } +} + +#[tokio::test] +async fn response_snapshot_cannot_outlive_or_resurrect_an_existing_peer_grant(){ + let (_dir,state,_)=fixture().await; + let (tx,rx)=tokio::sync::watch::channel(false); + let now=chrono::Utc::now().timestamp().max(0) as u64; + let grant=Grant{device_id:"private-phone-id".into(),expires_at:now+8,revoked:rx.clone()}; + let make=|grant:Grant|Router::new().route(PATH,get(room_learner_api::progress)).with_state(state.clone()).layer(axum::Extension(grant)); + let (s,body)=fetch(make(grant.clone()),Method::GET,"synthetic-install","unused",PATH).await; + assert_eq!(s,StatusCode::OK);assert!(body["expires_at_unix_ms"].as_u64().unwrap()<=grant.expires_at*1000);assert!(body["expires_at_unix_ms"].as_u64().unwrap()>,request:axum::extract::Request,next:Next)->Response { + if request.uri().path()==PATH {return StatusCode::FORBIDDEN.into_response();} + require_companion(State(service),request,next).await + } + let (dir,state,_)=fixture().await; + let service=Companion::load(&dir.path().join("pair"),"synthetic-install".into(),"f".repeat(64),"https://synthetic.local:3443".into()).unwrap();let token=pair(service.clone(),"phone").await; + let legacy=Router::new().route("/api/v1/room-learner/selection",get(|State(s):State|async move { + let Json(mut v)=room_learner_api::selection(State(s)).await; + v.as_object_mut().unwrap().remove("progress_schema");Json(v) + })).with_state(state.clone()).layer(axum::middleware::from_fn_with_state(service.clone(),old_policy)); + assert_eq!(fetch(legacy.clone(),Method::GET,"synthetic-install",&token,PATH).await.0,StatusCode::FORBIDDEN); + let (status,selection)=fetch(legacy,Method::GET,"synthetic-install",&token,"/api/v1/room-learner/selection").await; + assert_eq!(status,StatusCode::OK);assert!(selection.get("progress_schema").is_none());assert_eq!(selection["success"],true); + let new=Router::new().route("/api/v1/room-learner/selection",get(room_learner_api::selection)).with_state(state).layer(axum::middleware::from_fn_with_state(service,require_companion)); + assert_eq!(fetch(new,Method::GET,"synthetic-install",&token,"/api/v1/room-learner/selection").await.1["progress_schema"],"ruview.room-learner.progress.v1"); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/parent_lifetime.rs b/v2/crates/wifi-densepose-sensing-server/src/parent_lifetime.rs new file mode 100644 index 0000000000..1daff9fa00 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/parent_lifetime.rs @@ -0,0 +1,89 @@ +//! Opt-in lifetime boundary for an app-spawned helper, never port/PID discovery. +use std::{ + io, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }, + time::{Duration, Instant}, +}; +use tokio::sync::oneshot; + +/// Keep this guard until server shutdown and final persistence have completed. +pub struct ParentLifetimeGuard(Arc); + +impl Drop for ParentLifetimeGuard { + fn drop(&mut self) { + self.0.store(true, Ordering::Release); + } +} + +#[cfg(unix)] +fn current_parent() -> u32 { + extern "C" { + fn getppid() -> std::os::raw::c_int; + } + // getppid has no arguments, allocation, or borrowed memory. + unsafe { getppid() as u32 } +} + +/// Refuse a wrong/already-detached parent before opening listeners or records. +/// A native thread still observes parent death if the async runtime is blocked. +pub fn arm( + expected_parent: Option, +) -> io::Result<(ParentLifetimeGuard, Option>)> { + let stopped = Arc::new(AtomicBool::new(false)); + let guard = ParentLifetimeGuard(stopped.clone()); + let Some(expected_parent) = expected_parent else { + return Ok((guard, None)); + }; + #[cfg(not(unix))] + { + let _ = expected_parent; + Err(io::Error::new( + io::ErrorKind::Unsupported, + "managed parent lifetime is unsupported on this platform", + )) + } + #[cfg(unix)] + { + if expected_parent <= 1 || current_parent() != expected_parent { + return Err(io::Error::new( + io::ErrorKind::PermissionDenied, + "managed parent does not match the spawning process", + )); + } + let (sender, receiver) = oneshot::channel(); + std::thread::Builder::new() + .name("managed-parent".into()) + .spawn(move || { + while !stopped.load(Ordering::Acquire) { + // Reparenting changes getppid even if the old PID is reused. + if current_parent() != expected_parent { + let _ = sender.send(()); + let deadline = Instant::now() + Duration::from_secs(2); + while !stopped.load(Ordering::Acquire) && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(25)); + } + if !stopped.load(Ordering::Acquire) { + // Long-lived WebSockets must not keep former account + // authority listening after its owning app disappeared. + eprintln!("Managed parent ended; helper shutdown deadline reached."); + std::process::exit(0); + } + return; + } + std::thread::sleep(Duration::from_millis(100)); + } + })?; + Ok((guard, Some(receiver))) + } +} + +pub async fn wait_for_exit(receiver: Option>) { + if let Some(receiver) = receiver { + let _ = receiver.await; + } else { + std::future::pending::<()>().await; + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/privacy_filter.rs b/v2/crates/wifi-densepose-sensing-server/src/privacy_filter.rs index ad51370db9..7459860674 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/privacy_filter.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/privacy_filter.rs @@ -14,7 +14,10 @@ //! breathing rate and pose keypoints. Presence, motion, person count, zone //! and the coarse posture label are kept, as they are on MQTT. Keys are //! removed, not nulled, so a client can't tell a suppressed value from one -//! the server never had. +//! the server never had. MAT alert messages are also removed because their +//! generated prose embeds literal breathing and heart rates. Structured alert +//! identity, status, triage and recommended action remain available. MAT +//! domain-event Debug details are withheld for the same reason; type/time stay. //! //! The filter fails closed: a frame or JSON body that can't be parsed is //! dropped (WebSocket, recording) or replaced with a 500 (REST) rather than @@ -37,6 +40,7 @@ pub const BIOMETRIC_KEYS: &[&str] = &[ // confidences; the leaf keys cover edge vitals, fused vitals and the // mmWave block. "vital_signs", + "vital_evidence", "breathing_rate_bpm", "heart_rate_bpm", "heartrate_bpm", @@ -72,6 +76,20 @@ impl PrivacyFilter { pub fn redact_value(value: &mut serde_json::Value) { match value { serde_json::Value::Object(map) => { + // MAT's generated alert prose embeds literal HR/BR values. It is + // not safe merely to remove the structured vital_signs object. + // Match the alert DTO shape so ordinary error messages survive. + if map.contains_key("survivor_id") + && map.contains_key("priority") + && map.contains_key("triage_status") + { + map.remove("message"); + } + // MAT's domain-event DTO is a Debug snapshot, including the full + // vital reading as text. Keep its event type/time, withhold prose. + if map.contains_key("event_type") && map.contains_key("timestamp") { + map.remove("details"); + } map.retain(|key, _| !BIOMETRIC_KEYS.contains(&key.as_str())); for child in map.values_mut() { redact_value(child); @@ -197,6 +215,21 @@ mod tests { assert_eq!(v["mmwave"]["present"], true); } + #[test] + fn redact_mat_alert_prose_at_any_depth_without_hiding_errors() { + let mut value = json!({ + "alerts": [{"survivor_id": "fixture", "priority": "High", + "triage_status": "Delayed", "message": "Breathing: 16.0 BPM", + "title": "Survivor Detected", "status": "Pending"}], + "error": {"code": "fixture", "message": "ordinary error"} + }); + redact_value(&mut value); + assert!(value["alerts"][0].get("message").is_none()); + assert_eq!(value["alerts"][0]["status"], "Pending"); + assert_eq!(value["alerts"][0]["title"], "Survivor Detected"); + assert_eq!(value["error"]["message"], "ordinary error"); + } + #[test] fn redact_json_str_fails_closed_on_invalid_json() { assert!(redact_json_str("not json").is_none()); diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_calibration.rs b/v2/crates/wifi-densepose-sensing-server/src/room_calibration.rs new file mode 100644 index 0000000000..24e80d2937 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_calibration.rs @@ -0,0 +1,216 @@ +//! Full room receipt companion to a separately validated aggregate model. +//! No CSI/history is stored. A bootstrap image alone never gains this authority. +use crate::bootstrap_baseline::{self, BootstrapBaselineError, BootstrapBaselineMetadata}; +use serde::{Deserialize, Serialize}; +use serde_json::value::RawValue; +use std::{ + fs, + io::Read, + path::{Path, PathBuf}, +}; + +const MAX_BYTES: u64 = 65_536; +const SCHEMA: &str = "ruview.saved-room-calibration.v1"; + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RoomReceipt { + pub schema: String, + pub boot_epoch: String, + pub session_id: String, + pub model_id: String, + pub binding_digest: String, + pub source_node_ids: Vec, + pub frame_count: u64, + pub variance_explained: f64, + pub baseline_eigenvalue_count: usize, + pub completed_at_unix_ms: u64, +} +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Payload { + schema: String, + installation_binding_sha256: String, + bootstrap_content_sha256: String, + receipt: RoomReceipt, +} +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Envelope { + payload: Box, + content_sha256: String, +} +pub fn path_in(data_dir: &Path) -> PathBuf { + data_dir.join("calibration").join("room-receipt-v1.json") +} +fn validate( + payload: &Payload, + installation: &str, + model: &BootstrapBaselineMetadata, +) -> Result<(), BootstrapBaselineError> { + let r = &payload.receipt; + let valid_id = |id: &str, prefix: &str| { + id.starts_with(prefix) + && id.len() <= 80 + && id.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'-') + }; + let digest = |value: &str| { + value.len() == 64 + && value + .bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) + }; + if payload.schema != SCHEMA + || payload.installation_binding_sha256 + != bootstrap_baseline::installation_binding(installation)? + || payload.bootstrap_content_sha256 != model.content_sha256 + || r.schema != "ruview.calibration.field-model-receipt.v1" + || !valid_id(&r.boot_epoch, "cal-boot-") + || !valid_id(&r.session_id, "cal-session-") + || !valid_id(&r.model_id, "cal-model-") + || r.model_id != model.source_model_id + || !digest(&r.binding_digest) + || r.frame_count < 1000 + || !r.variance_explained.is_finite() + || !(0.0..=1.0).contains(&r.variance_explained) + || r.baseline_eigenvalue_count > 2048 + || r.source_node_ids.is_empty() + || r.source_node_ids.len() > 16 + || !r.source_node_ids.windows(2).all(|pair| pair[0] < pair[1]) + || model.source_node_ids.len() != 1 + || !r.source_node_ids.contains(&model.source_node_ids[0]) + || r.completed_at_unix_ms == 0 + || r.completed_at_unix_ms > model.created_at_unix_ms + || r.completed_at_unix_ms >= model.expires_at_unix_ms + { + return Err(BootstrapBaselineError::InvalidPath); + } + Ok(()) +} +pub fn store( + path: &Path, + installation: &str, + model: &BootstrapBaselineMetadata, + receipt: &RoomReceipt, +) -> Result<(), BootstrapBaselineError> { + let payload = Payload { + schema: SCHEMA.into(), + installation_binding_sha256: bootstrap_baseline::installation_binding(installation)?, + bootstrap_content_sha256: model.content_sha256.clone(), + receipt: receipt.clone(), + }; + validate(&payload, installation, model)?; + let raw = serde_json::value::to_raw_value(&payload)?; + let content_sha256 = bootstrap_baseline::hex_sha256(raw.get().as_bytes()); + let bytes = serde_json::to_vec(&Envelope { + payload: raw, + content_sha256, + })?; + if bytes.len() as u64 > MAX_BYTES { + return Err(BootstrapBaselineError::FileTooLarge); + } + bootstrap_baseline::atomic_write_private(path, &bytes) +} +pub fn load( + path: &Path, + installation: &str, + model: &BootstrapBaselineMetadata, +) -> Result { + let metadata = fs::symlink_metadata(path)?; + if metadata.file_type().is_symlink() || !metadata.is_file() { + return Err(BootstrapBaselineError::InvalidPath); + } + if metadata.len() > MAX_BYTES { + return Err(BootstrapBaselineError::FileTooLarge); + } + let mut bytes = Vec::new(); + fs::File::open(path)? + .take(MAX_BYTES + 1) + .read_to_end(&mut bytes)?; + if bytes.len() as u64 > MAX_BYTES { + return Err(BootstrapBaselineError::FileTooLarge); + } + let envelope: Envelope = serde_json::from_slice(&bytes)?; + if bootstrap_baseline::hex_sha256(envelope.payload.get().as_bytes()) != envelope.content_sha256 + { + return Err(BootstrapBaselineError::InvalidPath); + } + let payload: Payload = serde_json::from_str(envelope.payload.get())?; + validate(&payload, installation, model)?; + Ok(payload.receipt) +} +pub fn remove(data_dir: &Path) -> Result<(), BootstrapBaselineError> { + match fs::remove_file(path_in(data_dir)) { + Ok(()) => Ok(()), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(e) => Err(e.into()), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::bootstrap_baseline::BootstrapCsiGrid; + fn fixture() -> (BootstrapBaselineMetadata, RoomReceipt) { + ( + BootstrapBaselineMetadata { + authority: "bootstrap_only", + source_node_ids: vec![15], + source_grid: BootstrapCsiGrid { + n_subcarriers: 56, + ppdu_type: 1, + }, + source_model_id: "cal-model-abc".into(), + created_at_unix_ms: 2000, + expires_at_unix_ms: 9000, + content_sha256: "ab".repeat(32), + }, + RoomReceipt { + schema: "ruview.calibration.field-model-receipt.v1".into(), + boot_epoch: "cal-boot-abc".into(), + session_id: "cal-session-abc".into(), + model_id: "cal-model-abc".into(), + binding_digest: "cd".repeat(32), + source_node_ids: vec![15, 16, 17], + frame_count: 18942, + variance_explained: 0.9800000000000001, + baseline_eigenvalue_count: 5, + completed_at_unix_ms: 1900, + }, + ) + } + #[test] + fn exact_roundtrip_and_foreign_owner_or_replaced_model_refused() { + let dir = tempfile::tempdir().unwrap(); + let path = path_in(dir.path()); + let (metadata, receipt) = fixture(); + store(&path, "A", &metadata, &receipt).unwrap(); + let read = load(&path, "A", &metadata).unwrap(); + assert_eq!(read.source_node_ids, vec![15, 16, 17]); + assert_eq!(read.frame_count, 18942); + assert!(load(&path, "B", &metadata).is_err()); + let mut replaced = metadata.clone(); + replaced.content_sha256 = "ef".repeat(32); + assert!(load(&path, "A", &replaced).is_err()); + let mut changed = receipt; + changed.source_node_ids = vec![16, 17]; + assert!(store(&path, "A", &metadata, &changed).is_err()); + remove(dir.path()).unwrap(); + assert!(!path.exists()); + } + #[test] + fn corrupt_oversized_and_unfinished_receipts_refused() { + let dir = tempfile::tempdir().unwrap(); + let path = path_in(dir.path()); + let (metadata, mut receipt) = fixture(); + store(&path, "A", &metadata, &receipt).unwrap(); + let mut bytes = fs::read(&path).unwrap(); + bytes[20] ^= 1; + fs::write(&path, bytes).unwrap(); + assert!(load(&path, "A", &metadata).is_err()); + fs::write(&path, vec![b' '; MAX_BYTES as usize + 1]).unwrap(); + assert!(load(&path, "A", &metadata).is_err()); + receipt.frame_count = 999; + assert!(store(&path, "A", &metadata, &receipt).is_err()); + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/artifact.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/artifact.rs new file mode 100644 index 0000000000..da22a95e47 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/artifact.rs @@ -0,0 +1,331 @@ +//! Private, owner-bound candidate persistence. This store grants no qualification. +use super::{model::RoomModel, observation::CaptureContext}; +use crate::companion::{atomic_private_write, private_dir, private_read}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::{ + io, + path::{Path, PathBuf}, +}; + +const POINTER_SCHEMA: &str = "ruview.room-learner.candidate.v1"; +const MODEL_SCHEMA: &str = "ruview.room-learner.model.v1"; +const LIMIT: usize = 65_536; // Same bound as the actual private_read primitive. +const POINTER: &str = "candidate-v1.json"; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct CandidateReceipt { + pub schema: String, + pub owner_scope: String, + pub installation_id: String, + pub room_digest: String, + pub context_digest: String, + pub model_sha256: String, + pub model_version: u32, + pub parameter_count: usize, +} +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Envelope { + schema: String, + receipt: CandidateReceipt, + /// Exact RoomModel::export bytes, escaped as JSON without losing float bits. + learned_model_json: String, +} +#[derive(Debug)] +pub struct ArtifactStore { + data_dir: PathBuf, + root: PathBuf, + owner_scope: String, + installation_id: String, +} +fn digest(bytes: &[u8]) -> String { + format!("{:x}", Sha256::digest(bytes)) +} +fn hex_digest(value: &str) -> bool { + value.len() == 64 + && value + .bytes() + .all(|c| c.is_ascii_digit() || (b'a'..=b'f').contains(&c)) +} +fn bounded_id(value: &str) -> bool { + !value.is_empty() + && value.len() <= 256 + && value.is_ascii() + && value.trim() == value + && !value.bytes().any(|c| c.is_ascii_control()) +} +fn encode(value: &T) -> Result, String> { + let bytes = serde_json::to_vec(value).map_err(|e| e.to_string())?; + if bytes.len() > LIMIT { + return Err("candidate artifact exceeds private storage bound".into()); + } + Ok(bytes) +} +fn regular_or_missing(path: &Path) -> Result { + match std::fs::symlink_metadata(path) { + Ok(meta) if meta.is_file() && !meta.file_type().is_symlink() => Ok(true), + Ok(_) => Err("candidate path is not a private regular file".into()), + Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(false), + Err(e) => Err(e.to_string()), + } +} +impl ArtifactStore { + /// Load an immutable historical model by exact hash; never use today's pointer. + pub fn load_exact_candidate( + &self, + context: &CaptureContext, + sha: &str, + ) -> Result { + self.validate_context(context)?; + if !hex_digest(sha) { + return Err("invalid exact candidate digest".into()); + } + let dir = self + .context_dir(context, false)? + .ok_or("exact candidate missing")?; + let envelope: Envelope = serde_json::from_slice( + &private_read(&dir.join(format!("{sha}.json"))).map_err(|e| e.to_string())?, + ) + .map_err(|e| e.to_string())?; + if envelope.receipt.model_sha256 != sha { + return Err("exact candidate hash mismatch".into()); + } + self.read_model(&dir, &envelope.receipt, context) + } + pub fn new( + data_dir: &Path, + expected_owner_scope: &str, + expected_installation: &str, + ) -> Result { + if !bounded_id(expected_installation) { + return Err("invalid expected installation".into()); + } + let data_dir = crate::account_profile::prepare_directory(expected_owner_scope, data_dir) + .map_err(|e| e.to_string())?; + let root = data_dir.join("room-learner"); + private_dir(&root).map_err(|e| e.to_string())?; + Ok(Self { + data_dir, + root, + owner_scope: expected_owner_scope.into(), + installation_id: expected_installation.into(), + }) + } + fn verify_owner(&self) -> Result<(), String> { + let actual = crate::account_profile::prepare_directory(&self.owner_scope, &self.data_dir) + .map_err(|e| e.to_string())?; + if actual != self.data_dir { + return Err("owner directory changed".into()); + } + private_dir(&self.root).map_err(|e| e.to_string()) + } + fn validate_context(&self, context: &CaptureContext) -> Result<(), String> { + context.validate()?; + if context.owner_scope != self.owner_scope + || context.installation_id != self.installation_id + { + return Err("candidate owner or installation mismatch".into()); + } + Ok(()) + } + fn context_dir( + &self, + context: &CaptureContext, + create: bool, + ) -> Result, String> { + self.verify_owner()?; + self.validate_context(context)?; + let key = context.digest(); + if !hex_digest(&key) { + return Err("invalid context digest".into()); + } + let dir = self.root.join(key); + if !create { + match std::fs::symlink_metadata(&dir) { + Err(e) if e.kind() == io::ErrorKind::NotFound => return Ok(None), + Err(e) => return Err(e.to_string()), + Ok(meta) if !meta.is_dir() || meta.file_type().is_symlink() => { + return Err("invalid candidate context directory".into()) + } + _ => {} + } + } + private_dir(&dir).map_err(|e| e.to_string())?; + Ok(Some(dir)) + } + fn receipt_for(&self, model: &RoomModel, bytes: &[u8]) -> CandidateReceipt { + CandidateReceipt { + schema: POINTER_SCHEMA.into(), + owner_scope: self.owner_scope.clone(), + installation_id: self.installation_id.clone(), + room_digest: model.context().room_digest.clone(), + context_digest: model.context().digest(), + model_sha256: digest(bytes), + model_version: model.version, + parameter_count: model.parameter_count(), + } + } + fn validate_receipt( + &self, + receipt: &CandidateReceipt, + context: &CaptureContext, + ) -> Result<(), String> { + if receipt.schema != POINTER_SCHEMA + || receipt.owner_scope != self.owner_scope + || receipt.installation_id != self.installation_id + || receipt.room_digest != context.room_digest + || receipt.context_digest != context.digest() + || !hex_digest(&receipt.model_sha256) + || receipt.model_version != 1 + || !(1..=512).contains(&receipt.parameter_count) + { + return Err("candidate receipt/context mismatch".into()); + } + Ok(()) + } + fn read_model( + &self, + dir: &Path, + receipt: &CandidateReceipt, + context: &CaptureContext, + ) -> Result { + self.validate_receipt(receipt, context)?; + let bytes = private_read(&dir.join(format!("{}.json", receipt.model_sha256))) + .map_err(|e| e.to_string())?; + let envelope: Envelope = serde_json::from_slice(&bytes).map_err(|e| e.to_string())?; + if envelope.schema != MODEL_SCHEMA + || &envelope.receipt != receipt + || digest(envelope.learned_model_json.as_bytes()) != receipt.model_sha256 + { + return Err("candidate immutable artifact integrity mismatch".into()); + } + let model = RoomModel::import(envelope.learned_model_json.as_bytes())?; + if model.context() != context || self.receipt_for(&model, &model.export()?) != *receipt { + return Err("candidate learned model/context mismatch".into()); + } + Ok(model) + } + fn read_pointer( + &self, + dir: &Path, + context: &CaptureContext, + ) -> Result, String> { + let path = dir.join(POINTER); + if !regular_or_missing(&path)? { + return Ok(None); + } + let receipt: CandidateReceipt = + serde_json::from_slice(&private_read(&path).map_err(|e| e.to_string())?) + .map_err(|e| e.to_string())?; + self.validate_receipt(&receipt, context)?; + // A corrupt existing pointer/artifact is never silently overwritten. + self.read_model(dir, &receipt, context)?; + Ok(Some(receipt)) + } + pub fn save_candidate(&self, model: &RoomModel) -> Result { + self.validate_context(model.context())?; + let bytes = model.export()?; + let receipt = self.receipt_for(model, &bytes); + let envelope = Envelope { + schema: MODEL_SCHEMA.into(), + receipt: receipt.clone(), + learned_model_json: String::from_utf8(bytes).map_err(|e| e.to_string())?, + }; + let encoded = encode(&envelope)?; + let pointer = encode(&receipt)?; + let dir = self + .context_dir(model.context(), true)? + .ok_or("missing context directory")?; + self.read_pointer(&dir, model.context())?; + let immutable = dir.join(format!("{}.json", receipt.model_sha256)); + if regular_or_missing(&immutable)? { + if private_read(&immutable).map_err(|e| e.to_string())? != encoded { + return Err("existing immutable candidate differs".into()); + } + } else { + // Actual private writer supplies 0600 + fsync. An exclusive link + // installs the complete inode without replacing any previous model. + let staging = dir.join(format!("model-stage-{}", crate::companion::secret())); + atomic_private_write(&staging, &encoded).map_err(|e| e.to_string())?; + let installed = std::fs::hard_link(&staging, &immutable); + let cleanup = std::fs::remove_file(&staging); + match installed { + Ok(()) => {} + Err(e) if e.kind() == io::ErrorKind::AlreadyExists => { + if private_read(&immutable).map_err(|e| e.to_string())? != encoded { + return Err("immutable candidate collision".into()); + } + } + Err(e) => return Err(e.to_string()), + } + cleanup.map_err(|e| e.to_string())?; + std::fs::File::open(&dir) + .and_then(|f| f.sync_all()) + .map_err(|e| e.to_string())?; + } + self.read_model(&dir, &receipt, model.context())?; + atomic_private_write(&dir.join(POINTER), &pointer).map_err(|e| e.to_string())?; + if private_read(&dir.join(POINTER)).map_err(|e| e.to_string())? != pointer { + return Err("candidate pointer readback mismatch".into()); + } + Ok(receipt) + } + pub fn load_candidate( + &self, + expected_context: &CaptureContext, + ) -> Result, String> { + let Some(dir) = self.context_dir(expected_context, false)? else { + return Ok(None); + }; + let Some(receipt) = self.read_pointer(&dir, expected_context)? else { + return Ok(None); + }; + self.read_model(&dir, &receipt, expected_context).map(Some) + } +} + +#[cfg(test)] +#[path = "artifact_tests.rs"] +mod tests; + +#[path="artifact_catalog.rs"] +mod catalog; +pub use catalog::{CandidateCatalog,SavedCandidate}; + +#[path="evaluation_store.rs"] +mod evaluation_store; +pub use evaluation_store::EvaluationReceipt; + +#[path="selection_store.rs"] +mod selection_store; + +#[path="assessment_store.rs"] +mod assessment_store; + +#[path="managed_training_store.rs"] +mod managed_training_store; + +#[path="managed_capture_fence_store.rs"] +mod managed_capture_fence_store; + +#[path = "temporal_artifact_store.rs"] +mod temporal_artifacts; +pub use temporal_artifacts::TemporalCandidateReceipt; + +#[path="temporal_fit_summary.rs"] +mod temporal_fit_summary; +pub(crate) use temporal_fit_summary::{TemporalFitDiagnostics,TemporalFitSaveResult}; +pub(crate) use temporal_fit_summary::FitDiagnosticState; + +#[cfg(test)] +pub(crate) use temporal_fit_summary::rehash_fixture as rehash_temporal_fit_fixture; + +#[path="temporal_evaluation_store.rs"] +mod temporal_evaluation_store; +pub(crate) use temporal_evaluation_store::{StoredTemporalEvaluation,StoredTemporalAssessment}; + +#[path="temporal_selection_store.rs"] +mod temporal_selection_store; +pub(crate) use temporal_selection_store::StoredTemporalSelection; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/artifact_catalog.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/artifact_catalog.rs new file mode 100644 index 0000000000..385c725118 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/artifact_catalog.rs @@ -0,0 +1,117 @@ +//! Bounded owner-private metadata discovery. It does not select/adopt authority. +use super::*; +#[derive(Debug, Clone, Serialize)] +pub struct SavedCandidate { + pub receipt: CandidateReceipt, + pub context: CaptureContext, + /// Selection convenience only; never evidence freshness or qualification. + pub stored_at_unix_ms: u64, +} +#[derive(Debug, Clone, Serialize)] +pub struct CandidateCatalog { + pub candidates: Vec, + pub unreadable_contexts: usize, + pub truncated: bool, +} +impl ArtifactStore { + pub fn catalog(&self) -> Result { + self.catalog_filtered(None) + } + pub fn catalog_for_room(&self, room_digest: &str) -> Result { + if !bounded_id(room_digest) { + return Err("invalid catalog room".into()); + } + self.catalog_filtered(Some(room_digest)) + } + fn catalog_filtered(&self, room_digest: Option<&str>) -> Result { + self.verify_owner()?; + let mut contexts = Vec::new(); + let mut truncated = false; + let mut inspected = 0; + for entry in std::fs::read_dir(&self.root) + .map_err(|e| e.to_string())? + // One reserved non-v1 namespace cannot consume the v1 scan budget. + .take(258) + { + let entry = entry.map_err(|e| e.to_string())?; + if entry.file_name() == temporal_artifacts::DIRECTORY { continue; } + if inspected == 256 { + truncated = true; + break; + } + inspected += 1; + if contexts.len() >= 256 { + truncated = true; + break; + } + let Some(key) = entry.file_name().to_str().map(str::to_string) else { + continue; + }; + if hex_digest(&key) { + contexts.push((key, entry.path())); + } + } + let mut result = CandidateCatalog { + candidates: Vec::new(), + unreadable_contexts: 0, + truncated, + }; + contexts.sort_by(|a, b| a.0.cmp(&b.0)); + for (key, dir) in contexts { + let candidate = (|| { + let meta = std::fs::symlink_metadata(&dir).map_err(|e| e.to_string())?; + if !meta.is_dir() || meta.file_type().is_symlink() { + return Err("invalid catalog context directory".to_string()); + } + private_dir(&dir).map_err(|e| e.to_string())?; + let pointer = dir.join(POINTER); + let receipt: CandidateReceipt = + serde_json::from_slice(&private_read(&pointer).map_err(|e| e.to_string())?) + .map_err(|e| e.to_string())?; + if receipt.owner_scope != self.owner_scope + || receipt.installation_id != self.installation_id + || receipt.context_digest != key + || !hex_digest(&receipt.model_sha256) + { + return Err("catalog owner/context mismatch".into()); + } + let envelope: Envelope = serde_json::from_slice( + &private_read(&dir.join(format!("{}.json", receipt.model_sha256))) + .map_err(|e| e.to_string())?, + ) + .map_err(|e| e.to_string())?; + let model = RoomModel::import(envelope.learned_model_json.as_bytes())?; + self.validate_context(model.context())?; + self.read_model(&dir, &receipt, model.context())?; + let stored_at_unix_ms = std::fs::symlink_metadata(&pointer) + .ok() + .and_then(|m| m.modified().ok()) + .and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok()) + .map(|d| d.as_millis().min(u64::MAX as u128) as u64) + .unwrap_or(0); + Ok::<_, String>(SavedCandidate { + receipt, + context: model.context().clone(), + stored_at_unix_ms, + }) + })(); + match candidate { + Ok(c) if room_digest.is_none_or(|room| c.context.room_digest == room) => { + result.candidates.push(c) + } + Ok(_) => {} + Err(_) => result.unreadable_contexts += 1, + } + } + result.candidates.sort_by(|a, b| { + b.stored_at_unix_ms + .cmp(&a.stored_at_unix_ms) + .then(a.receipt.context_digest.cmp(&b.receipt.context_digest)) + }); + if result.candidates.len() > 32 { + result.candidates.truncate(32); + result.truncated = true; + } + Ok(result) + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/artifact_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/artifact_tests.rs new file mode 100644 index 0000000000..73d16a8c2e --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/artifact_tests.rs @@ -0,0 +1,452 @@ +//! Actual trained model + real private filesystem primitives; RF is SYNTHETIC. +use super::*; +use crate::room_learner::{observation::*, training::FitConfig}; +use ruview_unified::RfModality; +use std::sync::OnceLock; + +fn owner() -> String { + "a".repeat(64) +} +fn fitted(room: &str, revision: u64, epochs: usize) -> RoomModel { + let context = CaptureContext { + owner_scope: owner(), + installation_id: "synthetic-install".into(), + room_digest: room.into(), + revision, + sources: (1..=2) + .map(|id| SourceSpec { + source_id: id, + device_id: format!("synthetic-device-{id}"), + grid_id: "synthetic-4-bin".into(), + feature_count: 4, + signal_kind: RfModality::Synthetic, + link_id: format!("link-{id}"), + link_index: 0, + center_frequency_hz: 2_400_000_000, + bandwidth_hz: 20_000_000, + geometry: None, + }) + .collect(), + }; + let labels = [ + RoomLabel::Empty, + RoomLabel::AdjacentOnly, + RoomLabel::OccupiedStatic, + RoomLabel::OccupiedMoving, + ]; + let rows = (1..=8u64) + .map(|id| { + let label = labels[(id as usize - 1) % 4]; + let views = context + .sources + .iter() + .map(|s| { + let shift = match label { + RoomLabel::Empty => 0.0, + RoomLabel::AdjacentOnly => { + if s.source_id == 1 { + 1.5 + } else { + -1.5 + } + } + _ => 1.5, + }; + SourceView { + source_id: s.source_id, + device_id: s.device_id.clone(), + grid_id: s.grid_id.clone(), + signal_kind: s.signal_kind, + link_id: s.link_id.clone(), + link_index: 0, + center_frequency_hz: s.center_frequency_hz, + bandwidth_hz: s.bandwidth_hz, + geometry: None, + samples: (0..4) + .map(|t| { + (0..4) + .map(|b| { + 10.0 + shift + + t as f32 * 0.02 + + b as f32 * 0.01 + + id as f32 * 0.002 + }) + .collect() + }) + .collect(), + frame_ids: (0..4).map(|t| u128::from(id) * 10 + t + 1).collect(), + sequence_ids: vec![1, 2, 3, 4], + first_time_ns: id * 1000 + 1, + last_time_ns: id * 1000 + 100, + clock: ClockInfo { + kind: ClockKind::Unknown, + uncertainty_ns: u64::MAX, + }, + sample_age_ns: 0, + phase: PhaseAvailability::MagnitudeOnly, + provenance: CaptureProvenance::Synthetic, + } + }) + .collect(); + CaptureCohort { + id: format!("event-{id}"), + context_digest: context.digest(), + trial_id: format!("trial-{id}"), + session_id: format!("session-{id}"), + role: DataRole::Train, + label, + views, + teacher: None, + provenance: CaptureProvenance::Synthetic, + } + }) + .collect(); + RoomModel::fit( + &ValidatedDataset::new(context, rows).unwrap(), + FitConfig { + epochs, + ..FitConfig::default() + }, + ) + .unwrap() + .0 +} +fn model() -> RoomModel { + static MODEL: OnceLock = OnceLock::new(); + MODEL + .get_or_init(|| fitted("synthetic-room-a", 1, 2)) + .clone() +} +fn store(root: &Path) -> ArtifactStore { + ArtifactStore::new( + &root.join(owner()).join("SensingServer"), + &owner(), + "synthetic-install", + ) + .unwrap() +} +fn immutable(store: &ArtifactStore, receipt: &CandidateReceipt) -> PathBuf { + store + .root + .join(&receipt.context_digest) + .join(format!("{}.json", receipt.model_sha256)) +} +fn pointer(store: &ArtifactStore, model: &RoomModel) -> PathBuf { + store.root.join(model.context().digest()).join(POINTER) +} +#[test] +fn trained_model_private_save_cold_reload_inference_round_trip_has_no_raw_data() { + let temp = tempfile::tempdir().unwrap(); + let store = store(temp.path()); + let model = model(); + assert!(store.load_candidate(model.context()).unwrap().is_none()); + let receipt = store.save_candidate(&model).unwrap(); + let envelope = private_read(&immutable(&store, &receipt)).unwrap(); + let text = String::from_utf8(envelope).unwrap(); + for raw in ["samples", "frame_ids", "sequence_ids", "trial_id"] { + assert!(!text.contains(raw)); + } + let cold = ArtifactStore::new(&store.data_dir, &owner(), "synthetic-install").unwrap(); + let restored = cold.load_candidate(model.context()).unwrap().unwrap(); + assert_eq!(model.parameters(), restored.parameters()); + assert_eq!( + restored.training_manifest().training_provenance, + [CaptureProvenance::Synthetic] + ); + assert!(!restored.training_manifest().measurement_clock_available); + assert_eq!(model.export().unwrap(), restored.export().unwrap()); + let context = model.context(); + let live = CaptureCohort { + id: "live-input".into(), + context_digest: context.digest(), + trial_id: "live-trial".into(), + session_id: "live-session".into(), + role: DataRole::Inference, + label: RoomLabel::Unknown, + teacher: None, + provenance: CaptureProvenance::Synthetic, + views: context + .sources + .iter() + .map(|s| SourceView { + source_id: s.source_id, + device_id: s.device_id.clone(), + grid_id: s.grid_id.clone(), + signal_kind: s.signal_kind, + link_id: s.link_id.clone(), + link_index: 0, + center_frequency_hz: s.center_frequency_hz, + bandwidth_hz: s.bandwidth_hz, + geometry: None, + samples: vec![vec![11.5; 4]; 4], + frame_ids: vec![101, 102, 103, 104], + sequence_ids: vec![1, 2, 3, 4], + first_time_ns: 1000, + last_time_ns: 1100, + clock: ClockInfo { + kind: ClockKind::Unknown, + uncertainty_ns: u64::MAX, + }, + sample_age_ns: 0, + phase: PhaseAvailability::MagnitudeOnly, + provenance: CaptureProvenance::Synthetic, + }) + .collect(), + }; + assert_eq!( + model.infer(&live).occupied_probability, + restored.infer(&live).occupied_probability + ); + assert!(restored.infer(&live).occupied_probability.is_some()); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + assert_eq!( + std::fs::metadata(immutable(&store, &receipt)) + .unwrap() + .permissions() + .mode() + & 0o777, + 0o600 + ); + assert_eq!( + std::fs::metadata(pointer(&store, &model)) + .unwrap() + .permissions() + .mode() + & 0o777, + 0o600 + ); + assert_eq!( + std::fs::metadata(&store.root).unwrap().permissions().mode() & 0o777, + 0o700 + ); + } +} +#[test] +fn independent_rooms_revisions_and_replaced_candidates_retain_all_prior_artifacts() { + let temp = tempfile::tempdir().unwrap(); + let store = store(temp.path()); + let a = model(); + let b = fitted("synthetic-room-b", 1, 2); + let a2 = fitted("synthetic-room-a", 2, 2); + let ar = store.save_candidate(&a).unwrap(); + let before = private_read(&immutable(&store, &ar)).unwrap(); + store.save_candidate(&b).unwrap(); + store.save_candidate(&a2).unwrap(); + for expected in [&a, &b, &a2] { + assert_eq!( + store + .load_candidate(expected.context()) + .unwrap() + .unwrap() + .parameters(), + expected.parameters() + ); + } + let replacement = fitted("synthetic-room-a", 1, 3); + let next = store.save_candidate(&replacement).unwrap(); + assert_ne!(ar.model_sha256, next.model_sha256); + assert_eq!(private_read(&immutable(&store, &ar)).unwrap(), before); + assert_eq!( + store + .load_candidate(a.context()) + .unwrap() + .unwrap() + .parameters(), + replacement.parameters() + ); + assert!(immutable(&store, &next).is_file()); +} +#[test] +fn corrupt_pointer_or_immutable_bytes_fail_closed_and_preserve_existing_candidate() { + for pointer_corruption in [true, false] { + let temp = tempfile::tempdir().unwrap(); + let store = store(temp.path()); + let model = model(); + let receipt = store.save_candidate(&model).unwrap(); + let path = if pointer_corruption { + pointer(&store, &model) + } else { + immutable(&store, &receipt) + }; + atomic_private_write(&path, b"corrupt candidate bytes").unwrap(); + let before = private_read(&path).unwrap(); + assert!(store.load_candidate(model.context()).is_err()); + assert!(store.save_candidate(&model).is_err()); + assert_eq!(private_read(&path).unwrap(), before); + } +} +#[test] +fn model_hash_tampering_wrong_context_and_installation_cannot_be_adopted() { + let temp = tempfile::tempdir().unwrap(); + let store = store(temp.path()); + let model = model(); + let receipt = store.save_candidate(&model).unwrap(); + let mut envelope: serde_json::Value = + serde_json::from_slice(&private_read(&immutable(&store, &receipt)).unwrap()).unwrap(); + envelope["learned_model_json"] = serde_json::Value::String("{}".into()); + atomic_private_write( + &immutable(&store, &receipt), + &serde_json::to_vec(&envelope).unwrap(), + ) + .unwrap(); + assert!(store + .load_candidate(model.context()) + .unwrap_err() + .contains("integrity")); + let mut other = model.context().clone(); + other.installation_id = "other-install".into(); + assert!(store.load_candidate(&other).is_err()); + let wrong = ArtifactStore::new(&store.data_dir, &owner(), "other-install").unwrap(); + assert!(wrong.save_candidate(&model).is_err()); + other = model.context().clone(); + other.owner_scope = "b".repeat(64); + assert!(store.load_candidate(&other).is_err()); + assert!(ArtifactStore::new(&store.data_dir, &"b".repeat(64), "synthetic-install").is_err()); +} +#[test] +fn existing_account_witness_and_accepted_control_models_are_preserved() { + let temp = tempfile::tempdir().unwrap(); + let store = store(temp.path()); + let witness = store.data_dir.join("account-owner-v1.json"); + let before = private_read(&witness).unwrap(); + let accepted = store.data_dir.join("accepted-model.json"); + let control = store.data_dir.join("control-model.json"); + atomic_private_write(&accepted, b"accepted bytes").unwrap(); + atomic_private_write(&control, b"control bytes").unwrap(); + store.save_candidate(&model()).unwrap(); + assert_eq!(private_read(&witness).unwrap(), before); + assert_eq!(private_read(&accepted).unwrap(), b"accepted bytes"); + assert_eq!(private_read(&control).unwrap(), b"control bytes"); + atomic_private_write(&witness, b"changed owner witness").unwrap(); + assert!(store.save_candidate(&model()).is_err()); + assert_eq!(private_read(&witness).unwrap(), b"changed owner witness"); +} +#[cfg(unix)] +#[test] +fn symlink_candidate_pointer_context_and_root_are_rejected_without_touching_victims() { + use std::os::unix::fs::symlink; + for level in ["pointer", "immutable", "context", "root"] { + let temp = tempfile::tempdir().unwrap(); + let store = store(temp.path()); + let model = model(); + let receipt = store.save_candidate(&model).unwrap(); + let victim = temp.path().join("victim"); + atomic_private_write(&victim, b"victim private bytes").unwrap(); + let path = match level { + "pointer" => pointer(&store, &model), + "immutable" => immutable(&store, &receipt), + "context" => store.root.join(model.context().digest()), + _ => store.root.clone(), + }; + let preserved = path.with_extension("preserved"); + std::fs::rename(&path, &preserved).unwrap(); + symlink(&victim, &path).unwrap(); + assert!( + store.load_candidate(model.context()).is_err(), + "accepted symlink {level}" + ); + assert!( + store.save_candidate(&model).is_err(), + "overwrote symlink {level}" + ); + assert_eq!(private_read(&victim).unwrap(), b"victim private bytes"); + assert!(std::fs::symlink_metadata(path) + .unwrap() + .file_type() + .is_symlink()); + } +} +#[cfg(unix)] +#[test] +fn insecure_permissions_and_disk_path_failure_leave_previous_model_unchanged() { + use std::os::unix::fs::PermissionsExt; + let temp = tempfile::tempdir().unwrap(); + let store = store(temp.path()); + let model = model(); + let receipt = store.save_candidate(&model).unwrap(); + let path = pointer(&store, &model); + let before = private_read(&path).unwrap(); + let immutable_before = private_read(&immutable(&store, &receipt)).unwrap(); + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o644)).unwrap(); + assert!(store.load_candidate(model.context()).is_err()); + assert!(store.save_candidate(&model).is_err()); + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o600)).unwrap(); + assert_eq!(private_read(&path).unwrap(), before); + // A filesystem obstacle at the destination refuses publication and retains + // the old pointer (moved aside by the fixture) and immutable learned bytes. + let old = path.with_extension("old"); + std::fs::rename(&path, &old).unwrap(); + std::fs::create_dir(&path).unwrap(); + assert!(store.save_candidate(&model).is_err()); + assert_eq!(private_read(&old).unwrap(), before); + assert_eq!( + private_read(&immutable(&store, &receipt)).unwrap(), + immutable_before + ); +} +#[test] +fn claimed_qualification_and_unbounded_or_traversing_pointers_are_rejected() { + let temp = tempfile::tempdir().unwrap(); + let store = store(temp.path()); + let model = model(); + let receipt = store.save_candidate(&model).unwrap(); + let path = pointer(&store, &model); + let original = private_read(&path).unwrap(); + let artifact_before = private_read(&immutable(&store, &receipt)).unwrap(); + let mut forged: serde_json::Value = serde_json::from_slice(&original).unwrap(); + forged["physically_validated"] = serde_json::json!(true); + let mut traversing: serde_json::Value = serde_json::from_slice(&original).unwrap(); + traversing["model_sha256"] = serde_json::json!("../../accepted-model"); + for bytes in [ + serde_json::to_vec(&forged).unwrap(), + serde_json::to_vec(&traversing).unwrap(), + vec![b' '; LIMIT + 1], + ] { + atomic_private_write(&path, &bytes).unwrap(); + assert!(store.load_candidate(model.context()).is_err()); + assert!(store.save_candidate(&model).is_err()); + assert_eq!(std::fs::read(&path).unwrap(), bytes); + assert_eq!( + private_read(&immutable(&store, &receipt)).unwrap(), + artifact_before + ); + } +} +#[test] +fn unowned_legacy_data_and_blocked_context_paths_are_preserved() { + let temp = tempfile::tempdir().unwrap(); + let legacy = temp.path().join(owner()).join("SensingServer"); + std::fs::create_dir_all(&legacy).unwrap(); + std::fs::write(legacy.join("legacy-model"), b"unowned bytes").unwrap(); + assert!(ArtifactStore::new(&legacy, &owner(), "synthetic-install").is_err()); + assert_eq!( + std::fs::read(legacy.join("legacy-model")).unwrap(), + b"unowned bytes" + ); + assert!(!legacy.join("account-owner-v1.json").exists()); + let other = tempfile::tempdir().unwrap(); + let store = store(other.path()); + let model = model(); + let blocked = store.root.join(model.context().digest()); + atomic_private_write(&blocked, b"existing obstacle").unwrap(); + assert!(store.save_candidate(&model).is_err()); + assert_eq!(private_read(&blocked).unwrap(), b"existing obstacle"); +} + +#[test] +fn catalog_retains_exact_saved_source_contexts_and_quarantines_other_corrupt_entries() { + let temp=tempfile::tempdir().unwrap();let store=store(temp.path());let a=model(); + let receipt=store.save_candidate(&a).unwrap(); + let unknown=store.root.join("f".repeat(64));std::fs::create_dir(&unknown).unwrap(); + atomic_private_write(&unknown.join(POINTER),b"bad pointer").unwrap(); + let listed=store.catalog().unwrap(); + assert_eq!(listed.candidates.len(),1);assert_eq!(listed.unreadable_contexts,1); + assert_eq!(listed.candidates[0].context,*a.context()); + assert_eq!(listed.candidates[0].receipt,receipt); + assert!(store.load_candidate(a.context()).unwrap().is_some()); + assert_eq!(private_read(&unknown.join(POINTER)).unwrap(),b"bad pointer"); + let encoded=serde_json::to_string(&listed).unwrap(); + assert!(!encoded.contains("encoder_weights") && !encoded.contains("samples")); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/assessment.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/assessment.rs new file mode 100644 index 0000000000..348fb4bfc5 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/assessment.rs @@ -0,0 +1,122 @@ +//! Latest completed operator evaluation withdraws stale positive authority without erasing history. +use super::{ + artifact::EvaluationReceipt, + model::RoomModel, + observation::{CaptureProvenance, DataRole, MAX_COHORTS}, + qualification::{criteria, hash, ManagedEvidence, QualificationEnvelope, QualificationPolicy}, +}; +use serde::{Deserialize, Serialize}; +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct AssessmentReceipt { + pub schema: String, + pub owner_scope: String, + pub installation_id: String, + pub context_digest: String, + pub model_sha256: String, + pub evaluation_receipt_sha256: String, + pub policy_sha256: String, + pub evaluated_at_unix_ms: u64, + pub last_held_out_observed_ns: u64, + pub train_cohorts_per_condition: [usize; 4], + pub sealed_history_sha256: String, + pub qualification_sha256: Option, + pub failure_reason: Option, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct AssessmentEnvelope { + pub receipt_sha256: String, + pub receipt: AssessmentReceipt, +} +impl AssessmentEnvelope { + pub(crate) fn issue( + model: &RoomModel, + evaluation: &EvaluationReceipt, + evidence: &ManagedEvidence, + qualified: Option<&QualificationEnvelope>, + ) -> Result { + let outcome = criteria(&evaluation.report, &evidence.counts); + match (&outcome, qualified) { + (Ok(_), None) | (Err(_), Some(_)) => { + return Err("qualification outcome mismatch".into()) + } + _ => {} + } + let c = model.context(); + let receipt = AssessmentReceipt { + schema: "ruview.local-operational-assessment.v1".into(), + owner_scope: c.owner_scope.clone(), + installation_id: c.installation_id.clone(), + context_digest: c.digest(), + model_sha256: evaluation.model_sha256.clone(), + evaluation_receipt_sha256: evaluation.sha256()?, + policy_sha256: hash(&QualificationPolicy::default())?, + evaluated_at_unix_ms: evaluation.evaluated_at_unix_ms, + last_held_out_observed_ns: evidence.last_held_out_observed_ns, + train_cohorts_per_condition: evidence.counts, + sealed_history_sha256: evidence.sealed_history_sha256.clone(), + qualification_sha256: qualified.map(|q| q.receipt_sha256.clone()), + failure_reason: outcome.err(), + }; + let a = Self { + receipt_sha256: hash(&receipt)?, + receipt, + }; + a.validate(model, evaluation)?; + Ok(a) + } + pub(crate) fn validate( + &self, + model: &RoomModel, + eval: &EvaluationReceipt, + ) -> Result<(), String> { + let r = &self.receipt; + let c = model.context(); + let hex = |s: &str| { + s.len() == 64 + && s.bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) + }; + if self.receipt_sha256 != hash(r)? + || r.schema != "ruview.local-operational-assessment.v1" + || r.owner_scope != c.owner_scope + || r.installation_id != c.installation_id + || r.context_digest != c.digest() + || r.model_sha256 != eval.model_sha256 + || r.evaluation_receipt_sha256 != eval.sha256()? + || r.policy_sha256 != hash(&QualificationPolicy::default())? + || r.evaluated_at_unix_ms != eval.evaluated_at_unix_ms + || r.last_held_out_observed_ns == 0 + || !hex(&r.sealed_history_sha256) + || eval.role != DataRole::HeldOut + || eval.provenance != [CaptureProvenance::LiveReceiverObserved] + || model.training_manifest().training_provenance + != [CaptureProvenance::LiveReceiverObserved] + || r.train_cohorts_per_condition + .iter() + .any(|n| *n < 4 || *n > MAX_COHORTS) + || r.train_cohorts_per_condition.iter().sum::() + != model.training_manifest().training_cohorts + || eval.report.per_condition.iter().any(|v| v.samples < 4) + { + return Err("assessment identity/lineage mismatch".into()); + } + match criteria(&eval.report, &r.train_cohorts_per_condition) { + Ok(_) => { + if r.failure_reason.is_some() || !r.qualification_sha256.as_deref().is_some_and(hex) + { + return Err("assessment pass invalid".into()); + } + } + Err(e) => { + if r.failure_reason.as_deref() != Some(e.as_str()) + || r.qualification_sha256.is_some() + { + return Err("assessment failure invalid".into()); + } + } + } + Ok(()) + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/assessment_store.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/assessment_store.rs new file mode 100644 index 0000000000..3768136804 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/assessment_store.rs @@ -0,0 +1,104 @@ +use super::*; +use crate::room_learner::{assessment::AssessmentEnvelope, qualification::QualificationEnvelope}; +impl ArtifactStore { + fn assessment_dir(&self, model: &RoomModel, create: bool) -> Result, String> { + let Some(dir) = self.context_dir(model.context(), create)? else { + return Ok(None); + }; + let path = dir.join("assessments"); + match std::fs::symlink_metadata(&path) { + Err(e) if e.kind() == io::ErrorKind::NotFound && !create => return Ok(None), + Err(e) if e.kind() != io::ErrorKind::NotFound => return Err(e.to_string()), + _ => {} + } + private_dir(&path).map_err(|e| e.to_string())?; + Ok(Some(path)) + } + fn verify_assessment(&self, model: &RoomModel, a: &AssessmentEnvelope) -> Result<(), String> { + let e = self.load_exact_evaluation( + model, + crate::room_learner::observation::DataRole::HeldOut, + &a.receipt.evaluation_receipt_sha256, + )?; + a.validate(model, &e)?; + if let Some(sha) = &a.receipt.qualification_sha256 { + let q = self.load_qualification(model, sha)?; + if q.receipt.evaluation_receipt_sha256 != a.receipt.evaluation_receipt_sha256 + || q.receipt.sealed_history_sha256 != a.receipt.sealed_history_sha256 + { + return Err("assessment qualification differs".into()); + } + } + Ok(()) + } + pub(crate) fn load_latest_assessment( + &self, + model: &RoomModel, + ) -> Result, String> { + let Some(dir) = self.assessment_dir(model, false)? else { + return Ok(None); + }; + let path = dir.join(format!("{}.latest.json", digest(&model.export()?))); + if !regular_or_missing(&path)? { + return Ok(None); + } + let a: AssessmentEnvelope = + serde_json::from_slice(&private_read(&path).map_err(|e| e.to_string())?) + .map_err(|e| e.to_string())?; + self.verify_assessment(model, &a)?; + let immutable = dir.join(format!("{}.json", a.receipt_sha256)); + if private_read(&immutable).map_err(|e| e.to_string())? != encode(&a)? { + return Err("assessment pointer differs from immutable history".into()); + } + Ok(Some(a)) + } + pub(crate) fn save_assessment( + &self, + model: &RoomModel, + a: &AssessmentEnvelope, + ) -> Result<(), String> { + self.verify_assessment(model, a)?; + if let Some(previous) = self.load_latest_assessment(model)? { + if previous == *a { + return Ok(()); + } + if a.receipt.evaluated_at_unix_ms < previous.receipt.evaluated_at_unix_ms + || a.receipt.last_held_out_observed_ns <= previous.receipt.last_held_out_observed_ns + { + return Err( + "older qualification evidence cannot replace a newer assessment".into(), + ); + } + } + let dir = self + .assessment_dir(model, true)? + .ok_or("assessment directory unavailable")?; + let bytes = encode(a)?; + evaluation_store::write_immutable(&dir.join(format!("{}.json", a.receipt_sha256)), &bytes)?; + atomic_private_write( + &dir.join(format!("{}.latest.json", a.receipt.model_sha256)), + &bytes, + ) + .map_err(|e| e.to_string())?; + if self.load_latest_assessment(model)?.as_ref() != Some(a) { + return Err("assessment durable readback mismatch".into()); + } + Ok(()) + } + pub(crate) fn current_qualification_assessment( + &self, + model: &RoomModel, + q: &QualificationEnvelope, + ) -> Result { + let a = self + .load_latest_assessment(model)? + .ok_or("latest completed qualification assessment unavailable")?; + if a.receipt.failure_reason.is_some() { + return Err("newer_qualification_failed".into()); + } + if a.receipt.qualification_sha256.as_deref() != Some(q.receipt_sha256.as_str()) { + return Err("new qualification requires explicit selection".into()); + } + Ok(a) + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/c2_precision_pacer_test_support.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/c2_precision_pacer_test_support.rs new file mode 100644 index 0000000000..615c413d03 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/c2_precision_pacer_test_support.rs @@ -0,0 +1,96 @@ +//! PREPARED ONLY: std-only, no receiver/user-data/network access. +//! One baseline control + one absolute-deadline precision strategy; no tuning loop. +use std::{hint, time::{Duration, Instant}}; +const HZ: u64 = 33; +const NS: u64 = 1_000_000_000; +// Frozen r2 temporal_acquisition.rs217 and ingest_checked811–815; +// temporal_execution.rs484–489 also enforces this for whole-period input. +// This is ingestion compatibility, NOT an extra finish_checked predicate. +const MIN_INTERVAL: u64 = 1_000_000_000_000 / 36_000; +const MAX_GAP: u64 = 50_000_000; +const WINDOW_NS: u64 = 2_000_000_000; +const MAX_WINDOW_NS: u64 = 2_050_000_000; +const MIN_ROWS: usize = 61; +const LO: u128 = 30_000; +const HI: u128 = 36_000; +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Decision { Wait(u64), Emit(u64), MissedPeriod(u64) } +fn due_ns(index: usize) -> u64 { (index as u128 * u128::from(NS) / u128::from(HZ)) as u64 } +fn decide(index: usize, actual_ns: u64) -> Decision { + let due = due_ns(index); + if actual_ns < due { Decision::Wait(due - actual_ns) } + else if actual_ns >= due_ns(index + 1) { Decision::MissedPeriod(actual_ns - due) } + else { Decision::Emit(actual_ns - due) } +} +#[derive(Default, Clone, Copy)] +struct WaitCost { requested_sleep: u64, actual_sleep: u64, spin: u64 } +struct Wake { actual: u64, cost: WaitCost, missed: bool } +/// The emitted timestamp is ACTUAL monotonic wakeup, never due_ns(index). +/// Absolute deadlines never reset from actual wakeup. A missed whole period +/// stops the strategy rather than emitting catch-up/multiple batches. +fn wait_absolute(origin: Instant, index: usize, guard_ns: u64) -> Wake { + let due = origin.checked_add(Duration::from_nanos(due_ns(index))).expect("bounded study deadline"); + let mut cost = WaitCost::default(); + let now = Instant::now(); + if now < due { + let remaining = due.duration_since(now); + let guard = Duration::from_nanos(guard_ns); + if remaining > guard { + let sleep_for = remaining - guard; + cost.requested_sleep = sleep_for.as_nanos() as u64; + let slept = Instant::now(); std::thread::sleep(sleep_for); + cost.actual_sleep = slept.elapsed().as_nanos() as u64; + } + let spinning = Instant::now(); + while Instant::now() < due { hint::spin_loop(); } + cost.spin = spinning.elapsed().as_nanos() as u64; + } + let actual = origin.elapsed().as_nanos() as u64; + Wake { actual, cost, missed: matches!(decide(index, actual), Decision::MissedPeriod(_)) } +} +fn rate_ok(count: usize, span: u64) -> bool { + span > 0 && (count.saturating_sub(1) as u128) * 1_000_000_000_000 >= LO * u128::from(span) + && (count.saturating_sub(1) as u128) * 1_000_000_000_000 <= HI * u128::from(span) +} +// Mirrors retained-window readiness + finish span/mean cadence only. +fn first_window(times: &[u64]) -> Option<(usize, u64, bool)> { + let first = *times.first()?; + for count in MIN_ROWS..=times.len() { + let span = times[count - 1].checked_sub(first)?; + if span >= WINDOW_NS { return Some((count, span, span <= MAX_WINDOW_NS && rate_ok(count, span))); } + } + None +} +// Complete input compatibility combines REAL ingestion intervals with the +// separately checked retained-window mean/span/horizon. No invented gate. +fn qualifies(times: &[u64], requested: usize, missed: bool) -> bool { + if missed || times.len() != requested || times.len() < 2 { return false; } + if times.windows(2).any(|w| w[1] <= w[0] || w[1] - w[0] < MIN_INTERVAL || w[1] - w[0] > MAX_GAP) { return false; } + let span = times[times.len() - 1] - times[0]; + rate_ok(times.len(), span) && first_window(times).is_some_and(|(_,_,ok)| ok) +} +#[derive(Debug)] +pub(super) struct InputStudy { + pub qualified:bool, pub requested:usize, pub emitted:usize, + pub span_ns:u64, pub rate_millihz:u64, pub min_gap_ns:u64, pub max_gap_ns:u64, + pub missed_index:Option, pub spin_wall_ns:u128, pub sleep_overshoot_max_ns:u64, +} +pub(super) fn input_study(samples:usize,guard_ns:u64)->InputStudy { + assert!((70..=330).contains(&samples));assert!((500_000..=3_000_000).contains(&guard_ns)); + let origin=Instant::now();let mut times=Vec::with_capacity(samples);let mut missed=None;let mut spin=0u128;let mut overshoot=0; + for index in 0..samples { + let wake=wait_absolute(origin,index,guard_ns);spin+=u128::from(wake.cost.spin);overshoot=overshoot.max(wake.cost.actual_sleep.saturating_sub(wake.cost.requested_sleep)); + if wake.missed{missed=Some(index);break;}times.push(wake.actual); + } + let span=times.last().zip(times.first()).map_or(0,|(a,b)|a.saturating_sub(*b)); + let gaps=times.windows(2).map(|w|w[1].saturating_sub(w[0])).collect::>(); + InputStudy{qualified:qualifies(×,samples,missed.is_some()),requested:samples,emitted:times.len(),span_ns:span, + rate_millihz:if span==0{0}else{((times.len().saturating_sub(1)as u128)*1_000_000_000_000/u128::from(span))as u64}, + min_gap_ns:gaps.iter().min().copied().unwrap_or(0),max_gap_ns:gaps.iter().max().copied().unwrap_or(0),missed_index:missed,spin_wall_ns:spin,sleep_overshoot_max_ns:overshoot} +} +pub(super) fn next_actual_receipt(origin:Instant,index:usize,guard_ns:u64)->Result{ + let wake=wait_absolute(origin,index,guard_ns); + if wake.missed{return Err(format!("input_unqualified: whole period missed index={index} actual_ns={} planned_ns={}",wake.actual,due_ns(index)));} + Ok(wake.actual) +} +pub(super) fn actual_stream_qualifies(times:&[u64])->bool{qualifies(times,times.len(),false)} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/capture.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/capture.rs new file mode 100644 index 0000000000..7362933299 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/capture.rs @@ -0,0 +1,380 @@ +//! Bounded receiver-observed capture. Operator phase labels are experimental +//! claims, never independently verified empty-room or source-clock evidence. +use super::observation::*; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::collections::{HashMap, VecDeque}; + +pub const WINDOW_PACKETS: usize = 8; +pub const COHORTS_PER_PHASE: usize = 4; +pub const MIN_PHASE_MS: u64 = 12_000; +pub const MAX_PHASE_MS: u64 = 600_000; +pub const MAX_CAPTURE_BINS: usize = 128; +const MAX_PHASE_HISTORY: usize = 128; +// Protected diagnostics are consumed by JavaScript clients as exact integers. +const MAX_SEQUENCE_DIAGNOSTIC_COUNT: u64 = (1u64 << 53) - 1; + +/// Borrowed actual packet metadata. The caller must reject unknown bandwidth +/// and retain the known raw grid identity. Amplitudes may be length-only +/// interpolated to the declared feature_count; no gain/mean normalization. +/// This adapter never guesses PHY or original bin count from feature length. +pub struct RawSourceFrame<'a> { + pub source_id: u8, + pub grid_id: &'a str, + pub frequency_hz: u64, + pub bandwidth_hz: u64, + pub sequence: u32, + pub amplitudes: &'a [f64], +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PhaseRequest { + pub trial_id: String, + pub session_id: String, + pub role: DataRole, + pub label: RoomLabel, + pub duration_ms: u64, + #[serde(default)] + pub lead_in_ms: u64, + pub conditions_confirmed: bool, +} +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +pub enum PhaseStatus { Sealed, Cancelled, Failed } +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PhaseReceipt { + pub trial_id: String, + pub session_id: String, + pub role: DataRole, + pub label: RoomLabel, + pub status: PhaseStatus, + pub cohort_count: usize, + pub reason: Option, +} + +#[derive(Debug, Clone)] +struct Packet { values: Vec, frame_id: u128, ordinal: u64, received_ns: u64 } +#[derive(Debug, Default, Clone)] +struct SourceBuffer { + sequence: Option, + ordinal: u64, + last_time_ns: u64, + admitted_packets: u64, + // Transient diagnostics only. Never restore observations/rejections as + // fresh input, and never let them advance the admission watermark. + last_observed_sequence: Option, + sequence_rejection_count: u64, + sequence_rejection_reason: Option, + live: VecDeque, +} +#[derive(Debug,Clone)] +struct Phase { + request: PhaseRequest, + started_ns: u64, + bucket: usize, + pending: HashMap>, + representatives: Vec, + admitted: HashMap, +} + +/// A bounded diagnostic of the most recent sequence admission failure. +/// It does not establish a source reboot or authenticated source epoch. +#[derive(Debug, Clone, Copy, Serialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum SequenceRejectionReason { Duplicate, Reversed } + +/// Counts and receiver freshness only. No learned/qualified state is implied. +#[derive(Debug, Clone, Serialize)] +pub struct CaptureProgress { + pub phase: Option, + pub sources: Vec, + pub sealed_cohorts: usize, + pub last_failure: Option, +} +#[derive(Debug, Clone, Serialize)] +pub struct PhaseProgress { + pub trial_id: String, + pub session_id: String, + pub role: DataRole, + pub label: RoomLabel, + pub stage: &'static str, + pub lead_in_remaining_ms: u64, + pub capture_elapsed_ms: u64, + pub elapsed_ms: u64, + pub target_ms: u64, + pub representative_cohorts: usize, + pub required_cohorts: usize, +} +#[derive(Debug, Clone, Serialize)] +pub struct SourceProgress { + pub source_id: u8, + pub grid_id: String, + pub admitted_packets: u64, + /// Last admitted wire sequence, including an original recovery watermark. + pub last_accepted_sequence: Option, + /// Last sequence tested by this receiver after PHY/shape/time validation. + pub last_observed_sequence: Option, + /// Transient since this capture was created/restored; never persisted. + pub sequence_rejection_count: u64, + pub sequence_rejection_reason: Option, + pub phase_admitted_packets: u64, + pub live_buffer_frames: usize, + pub phase_buffer_frames: usize, + pub required_frames: usize, + pub latest_age_ms: Option, + pub window_eligible: bool, + pub reason: Option, +} + +#[derive(Debug,Clone)] +pub struct CaptureSession { + context: CaptureContext, + run_id: String, + prefix: u64, + last_seen_ns: u64, + sources: HashMap, + phase: Option, + cohorts: Vec, + history: Vec, + trials: HashMap, + sessions: HashMap, + metadata_lineage: Option, +} + +impl CaptureSession { + pub fn new(context: CaptureContext, run_id: String) -> Result { + context.validate()?; + identifier(&run_id)?; + if context.sources.iter().any(|s| s.feature_count > MAX_CAPTURE_BINS) { + return Err("capture supports at most 128 declared feature bins per source".into()); + } + let hash = Sha256::digest(format!("{}:{run_id}", context.digest()).as_bytes()); + let prefix = u64::from_be_bytes(hash[..8].try_into().map_err(|_| "run digest")?); + let sources = context.sources.iter().map(|s| (s.source_id, SourceBuffer::default())).collect(); + Ok(Self { context, run_id, prefix, last_seen_ns: 0, sources, phase: None, cohorts: vec![], history: vec![], + trials: HashMap::new(), sessions: HashMap::new(), metadata_lineage: None }) + } + + /// A rejected selected input withdraws old live windows, not sealed training rows. + pub(crate) fn clear_live_windows(&mut self) { for source in self.sources.values_mut() { source.live.clear(); } } + pub fn context(&self) -> &CaptureContext { &self.context } + pub fn cohorts(&self) -> &[CaptureCohort] { &self.cohorts } + pub fn history(&self) -> &[PhaseReceipt] { &self.history } + pub fn phase_active(&self) -> bool { self.phase.is_some() } + pub fn validated_dataset(&self) -> Result { + ValidatedDataset::new(self.context.clone(), self.cohorts.clone()) + } + + pub fn progress(&self, now_ns: u64) -> CaptureProgress { + let phase = self.phase.as_ref().map(|p| PhaseProgress { + trial_id: p.request.trial_id.clone(), session_id: p.request.session_id.clone(), + role: p.request.role, label: p.request.label, + stage: if now_ns < p.started_ns + p.request.lead_in_ms * 1_000_000 { "lead_in" } else { "collecting" }, + lead_in_remaining_ms: (p.started_ns + p.request.lead_in_ms * 1_000_000).saturating_sub(now_ns) / 1_000_000, + capture_elapsed_ms: now_ns.saturating_sub(p.started_ns + p.request.lead_in_ms * 1_000_000) / 1_000_000, + elapsed_ms: now_ns.saturating_sub(p.started_ns + p.request.lead_in_ms * 1_000_000) / 1_000_000, + target_ms: p.request.duration_ms, representative_cohorts: p.representatives.len(), + required_cohorts: COHORTS_PER_PHASE, + }); + let sources = self.context.sources.iter().map(|spec| { + let source = &self.sources[&spec.source_id]; + let latest_age = source.live.back().and_then(|p| now_ns.checked_sub(p.received_ns)); + let oldest_age = source.live.front().and_then(|p| now_ns.checked_sub(p.received_ns)); + let reason = if source.live.is_empty() { Some("missing_source") } + else if latest_age.is_none() || oldest_age.is_none() { Some("receiver_clock_regressed") } + else if oldest_age.is_some_and(|age| age > MAX_AGE_NS) { Some("stale_source") } + else if source.live.len() != WINDOW_PACKETS { Some("incomplete_window") } else { None }; + SourceProgress { source_id: spec.source_id, grid_id: spec.grid_id.clone(), + admitted_packets: source.admitted_packets, + last_accepted_sequence: source.sequence, + last_observed_sequence: source.last_observed_sequence, + sequence_rejection_count: source.sequence_rejection_count, + sequence_rejection_reason: source.sequence_rejection_reason, + phase_admitted_packets: self.phase.as_ref().and_then(|p| p.admitted.get(&spec.source_id)).copied().unwrap_or(0), + live_buffer_frames: source.live.len(), + phase_buffer_frames: self.phase.as_ref().and_then(|p| p.pending.get(&spec.source_id)).map_or(0, VecDeque::len), + required_frames: WINDOW_PACKETS, latest_age_ms: latest_age.map(|age| age / 1_000_000), + window_eligible: reason.is_none(), reason: reason.map(str::to_string) } + }).collect(); + CaptureProgress { phase, sources, sealed_cohorts: self.cohorts.len(), + last_failure: self.history.iter().rev().find(|r| r.status == PhaseStatus::Failed).and_then(|r| r.reason.clone()) } + } + + pub fn begin_phase(&mut self, request: PhaseRequest, now_ns: u64) -> Result<(), String> { + identifier(&request.trial_id)?; + identifier(&request.session_id)?; + if self.phase.is_some() { return Err("a phase is already active".into()); } + if !request.conditions_confirmed || request.role == DataRole::Inference || request.label == RoomLabel::Unknown + || !(MIN_PHASE_MS..=MAX_PHASE_MS).contains(&request.duration_ms) || request.lead_in_ms > 120_000 || now_ns == 0 + || now_ns < self.last_seen_ns || now_ns.checked_add((request.duration_ms + request.lead_in_ms) * 1_000_000).is_none() + || self.phase_index() >= MAX_PHASE_HISTORY || self.sealed_cohort_capacity_used() + COHORTS_PER_PHASE > MAX_COHORTS { + return Err("phase requires confirmed labelled conditions, bounded duration and capacity".into()); + } + if let Some(lineage)=&self.metadata_lineage { lineage.check_role(&request.trial_id,&request.session_id,request.role)?; } + for (roles, key) in [(&self.trials, &request.trial_id), (&self.sessions, &request.session_id)] { + if roles.get(key).is_some_and(|role| *role != request.role) { + return Err("trial/session role is frozen".into()); + } + } + if self.history.iter().any(|p|p.status==PhaseStatus::Sealed&&p.trial_id==request.trial_id) {return Err("phase identity was already sealed".into());} + self.trials.insert(request.trial_id.clone(), request.role); + self.sessions.insert(request.session_id.clone(), request.role); + self.phase = Some(Phase { request, started_ns: now_ns, bucket: 0, pending: HashMap::new(), representatives: vec![], admitted: HashMap::new() }); + Ok(()) + } + + pub fn ingest(&mut self, frame: RawSourceFrame<'_>, now_ns: u64) -> Result<(), String> { + let spec = match self.context.sources.iter().find(|s| s.source_id == frame.source_id) { + Some(s) => s, + None => return self.invalidate("unknown source changed capture context"), + }; + if frame.grid_id != spec.grid_id || frame.frequency_hz != spec.center_frequency_hz + || frame.bandwidth_hz != spec.bandwidth_hz || frame.amplitudes.len() != spec.feature_count + || frame.amplitudes.iter().any(|x| !x.is_finite() || *x < 0.0 || *x > f32::MAX as f64) { + return self.invalidate("source PHY/grid/shape or finite magnitude contract changed"); + } + let source = self.sources.get_mut(&frame.source_id).ok_or("source buffer unavailable")?; + if now_ns == 0 || now_ns < self.last_seen_ns || now_ns <= source.last_time_ns { return Err("nonmonotonic receiver time".into()); } + source.last_observed_sequence = Some(frame.sequence); + let delta = match source.sequence { + Some(last) => { + let delta = frame.sequence.wrapping_sub(last); + if delta == 0 || delta >= (1u32 << 31) { + source.sequence_rejection_count = source.sequence_rejection_count.saturating_add(1).min(MAX_SEQUENCE_DIAGNOSTIC_COUNT); + source.sequence_rejection_reason = Some(if delta == 0 { + SequenceRejectionReason::Duplicate + } else { SequenceRejectionReason::Reversed }); + return Err("replayed or reversed source sequence".into()); + } + u64::from(delta) + } + None => 1, + }; + let ordinal = source.ordinal.checked_add(delta).filter(|n| *n < (1u64 << 56)).ok_or("sequence ordinal exhausted")?; + let packet = Packet { values: frame.amplitudes.iter().map(|x| *x as f32).collect(), + frame_id: (u128::from(self.prefix) << 64) | (u128::from(frame.source_id) << 56) | u128::from(ordinal), + ordinal, received_ns: now_ns }; + source.sequence_rejection_reason = None; + source.sequence = Some(frame.sequence); + source.ordinal = ordinal; + source.last_time_ns = now_ns; + source.admitted_packets = source.admitted_packets.saturating_add(1); + self.last_seen_ns = now_ns; + push_bounded(&mut source.live, packet.clone()); + let phase_index=self.phase_index(); + let Some(phase) = self.phase.as_mut() else { return Ok(()); }; + if now_ns < phase.started_ns { return self.invalidate("receiver time precedes phase start"); } + let capture_start = phase.started_ns + phase.request.lead_in_ms * 1_000_000; + if now_ns < capture_start { return Ok(()); } + let elapsed = now_ns - capture_start; + let duration_ns = phase.request.duration_ms * 1_000_000; + if elapsed >= duration_ns { return Ok(()); } + *phase.admitted.entry(frame.source_id).or_default() += 1; + let bucket = ((elapsed * COHORTS_PER_PHASE as u64) / duration_ns) as usize; + if bucket != phase.bucket { phase.pending.clear(); phase.bucket = bucket; } + // Exactly one complete nonoverlapping representative per quarter. + if phase.representatives.iter().any(|c| c.id.ends_with(&format!("-bucket-{bucket}"))) { return Ok(()); } + push_bounded(phase.pending.entry(frame.source_id).or_default(), packet); + if phase.pending.len() != self.context.sources.len() + || phase.pending.values().any(|v| v.len() != WINDOW_PACKETS) { return Ok(()); } + let id = format!("{}-phase-{}-bucket-{bucket}", self.run_id, phase_index); + // Disjoint streams remain incomplete until a complete aligned window + // arrives. No pooling, padding, or fabricated source-clock alignment. + if let Ok(cohort) = assemble(&self.context, &phase.pending, now_ns, id, &phase.request) { + phase.representatives.push(cohort); phase.pending.clear(); + } + Ok(()) + } + + pub fn finish_phase(&mut self, now_ns: u64) -> Result { + let phase = self.phase.as_ref().ok_or("no active capture phase")?; + if now_ns < self.last_seen_ns || now_ns < phase.started_ns || now_ns - phase.started_ns < (phase.request.duration_ms + phase.request.lead_in_ms) * 1_000_000 { + return Err("phase duration has not elapsed".into()); + } + if phase.representatives.len() != COHORTS_PER_PHASE { + return self.invalidate("phase lacks four complete aligned representative windows"); + } + let phase = self.phase.take().ok_or("no active capture phase")?; + let receipt = receipt(&phase, PhaseStatus::Sealed, None); + let mut combined = self.cohorts.clone(); + combined.extend(phase.representatives); + if let Err(error) = ValidatedDataset::new(self.context.clone(), combined.clone()) { + let failed = PhaseReceipt { status: PhaseStatus::Failed, cohort_count: 0, reason: Some(error.clone()), ..receipt }; + self.history.push(failed); + return Err(error); + } + self.cohorts = combined; + self.history.push(receipt.clone()); + Ok(receipt) + } + + pub fn cancel_phase(&mut self) { + if let Some(phase) = self.phase.take() { + self.history.push(receipt(&phase, PhaseStatus::Cancelled, Some("operator cancelled phase".into()))); + } + } + + pub fn live_cohort(&self, now_ns: u64) -> Result { + let buffers = self.sources.iter().map(|(id,s)| (*id,s.live.clone())).collect(); + let request = PhaseRequest { trial_id: "live-unlabelled".into(), session_id: self.run_id.clone(), + role: DataRole::Inference, label: RoomLabel::Unknown, duration_ms: MIN_PHASE_MS, lead_in_ms: 0, conditions_confirmed: false }; + assemble(&self.context, &buffers, now_ns, format!("{}-live-{now_ns}", self.run_id), &request) + } + + fn invalidate(&mut self, reason: &str) -> Result { + if let Some(phase) = self.phase.take() { + self.history.push(receipt(&phase, PhaseStatus::Failed, Some(reason.into()))); + } + for source in self.sources.values_mut() { source.live.clear(); } + Err(reason.into()) + } +} + +fn assemble(context: &CaptureContext, buffers: &HashMap>, now_ns: u64, + id: String, request: &PhaseRequest) -> Result { + let mut views = Vec::with_capacity(context.sources.len()); + for spec in &context.sources { + let rows = buffers.get(&spec.source_id).filter(|v| v.len() == WINDOW_PACKETS).ok_or("incomplete source window")?; + let first = rows.front().ok_or("empty source window")?.received_ns; + let last = rows.back().ok_or("empty source window")?.received_ns; + let age = now_ns.checked_sub(first).filter(|age| *age <= MAX_AGE_NS).ok_or("stale or future source window")?; + views.push(SourceView { source_id: spec.source_id, device_id: spec.device_id.clone(), grid_id: spec.grid_id.clone(), + signal_kind: spec.signal_kind, center_frequency_hz: spec.center_frequency_hz, bandwidth_hz: spec.bandwidth_hz, + link_id: spec.link_id.clone(), link_index: spec.link_index, geometry: spec.geometry, + samples: rows.iter().map(|p| p.values.clone()).collect(), frame_ids: rows.iter().map(|p| p.frame_id).collect(), + sequence_ids: rows.iter().map(|p| p.ordinal).collect(), first_time_ns: first, last_time_ns: last, + clock: ClockInfo { kind: ClockKind::ReceiverOnly, uncertainty_ns: u64::MAX }, sample_age_ns: age, + phase: PhaseAvailability::MagnitudeOnly, provenance: CaptureProvenance::LiveReceiverObserved }); + } + let cohort = CaptureCohort { id, context_digest: context.digest(), trial_id: request.trial_id.clone(), + session_id: request.session_id.clone(), role: request.role, label: request.label, views, teacher: None, + provenance: CaptureProvenance::LiveReceiverObserved }; + validate_cohort(context, &cohort)?; + Ok(cohort) +} +fn push_bounded(buffer: &mut VecDeque, packet: Packet) { + if buffer.len() == WINDOW_PACKETS { buffer.pop_front(); } + buffer.push_back(packet); +} +fn receipt(phase: &Phase, status: PhaseStatus, reason: Option) -> PhaseReceipt { + PhaseReceipt { trial_id: phase.request.trial_id.clone(), session_id: phase.request.session_id.clone(), + role: phase.request.role, label: phase.request.label, status, + cohort_count: if status == PhaseStatus::Sealed { phase.representatives.len() } else { 0 }, reason } +} +fn identifier(s: &str) -> Result<(), String> { + if s.is_empty() || s.len() > 128 || !s.is_ascii() || s.trim() != s || s.chars().any(char::is_control) { + Err("invalid bounded capture identifier".into()) + } else { Ok(()) } +} + +#[cfg(test)] +#[path = "capture_tests.rs"] +mod tests; + +#[path = "capture_recovery.rs"] +mod recovery; +pub use recovery::{CaptureCheckpoint, CaptureCheckpointSummary, CaptureRestore, RecoveryPhaseDisposition, RecoveryPhaseSummary}; + +#[path = "managed_capture.rs"] +mod managed_capture; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/capture_recovery.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/capture_recovery.rs new file mode 100644 index 0000000000..5e8ae19b08 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/capture_recovery.rs @@ -0,0 +1,446 @@ +//! Typed reduced-sample recovery. Receiver buffers are deliberately never saved. +use super::*; +use std::collections::HashSet; +const SCHEMA: &str = "ruview.room-learner.capture-checkpoint.v1"; +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CaptureCheckpoint { + schema: String, + context: CaptureContext, + run_id: String, + saved_at_ns: u64, + last_seen_ns: u64, + sources: Vec, + cohorts: Vec, + history: Vec, + phase: Option, + #[serde(default, skip_serializing_if="Option::is_none")] + metadata_lineage: Option, +} +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct SourceWatermark { + source_id: u8, + sequence: Option, + ordinal: u64, + last_time_ns: u64, + admitted_packets: u64, +} +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct CheckpointPhase { + request: PhaseRequest, + started_ns: u64, + bucket: usize, + representatives: Vec, +} +#[derive(Debug, Clone, Copy, Serialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum RecoveryPhaseDisposition { + NoActivePhase, + CompletedNeedsConfirmation, + InterruptedNeedsRestart, +} +#[derive(Debug)] +pub struct CaptureRestore { + pub session: CaptureSession, + pub disposition: RecoveryPhaseDisposition, +} +#[derive(Debug, Clone, Serialize)] +pub struct CaptureCheckpointSummary { + pub run_id: String, + pub room_digest: String, + pub revision: u64, + pub source_ids: Vec, + pub sealed_cohorts: usize, + pub phase: Option, +} +#[derive(Debug, Clone, Serialize)] +pub struct RecoveryPhaseSummary { + pub role: DataRole, + pub label: RoomLabel, + pub complete: bool, + pub representative_cohorts: usize, + pub target_ms: u64, + pub capture_elapsed_ms: u64, +} +fn phase_end(p: &CheckpointPhase) -> Result { + if !p.request.conditions_confirmed + || p.request.role == DataRole::Inference + || p.request.label == RoomLabel::Unknown + || !(MIN_PHASE_MS..=MAX_PHASE_MS).contains(&p.request.duration_ms) + || p.request.lead_in_ms > 120_000 + || p.started_ns == 0 + || p.bucket >= COHORTS_PER_PHASE + { + return Err("invalid recovery phase request".into()); + } + identifier(&p.request.trial_id)?; + identifier(&p.request.session_id)?; + p.started_ns + .checked_add((p.request.duration_ms + p.request.lead_in_ms) * 1_000_000) + .ok_or_else(|| "recovery phase time overflow".into()) +} +impl CaptureCheckpoint { + pub fn context(&self) -> &CaptureContext { + &self.context + } + pub fn run_id(&self) -> &str { + &self.run_id + } + pub fn saved_at_ns(&self) -> u64 { + self.saved_at_ns + } + pub fn summary(&self) -> CaptureCheckpointSummary { + CaptureCheckpointSummary { + run_id: self.run_id.clone(), + room_digest: self.context.room_digest.clone(), + revision: self.context.revision, + source_ids: self.context.sources.iter().map(|s| s.source_id).collect(), + sealed_cohorts: self.cohorts.len(), + phase: self.phase.as_ref().map(|p| RecoveryPhaseSummary { + role: p.request.role, + label: p.request.label, + complete: phase_end(p).is_ok_and(|end| self.saved_at_ns >= end) + && p.representatives.len() == COHORTS_PER_PHASE, + representative_cohorts: p.representatives.len(), + target_ms: p.request.duration_ms, + capture_elapsed_ms: self.saved_at_ns.saturating_sub( + p.started_ns + .saturating_add(p.request.lead_in_ms.saturating_mul(1_000_000)), + ) / 1_000_000, + }), + } + } +} +impl CaptureSession { + pub fn checkpoint(&self, now_ns: u64) -> Result { + let snapshot = CaptureCheckpoint { + schema: if self.metadata_lineage.is_some() {"ruview.room-learner.capture-checkpoint.v2"} else {SCHEMA}.into(), + context: self.context.clone(), + run_id: self.run_id.clone(), + saved_at_ns: now_ns, + last_seen_ns: self.last_seen_ns, + sources: self + .context + .sources + .iter() + .map(|s| { + let b = &self.sources[&s.source_id]; + SourceWatermark { + source_id: s.source_id, + sequence: b.sequence, + ordinal: b.ordinal, + last_time_ns: b.last_time_ns, + admitted_packets: b.admitted_packets, + } + }) + .collect(), + cohorts: self.cohorts.clone(), + history: self.history.clone(), + metadata_lineage: self.metadata_lineage.clone(), + phase: self.phase.as_ref().map(|p| CheckpointPhase { + request: p.request.clone(), + started_ns: p.started_ns, + bucket: p.bucket, + representatives: p.representatives.clone(), + }), + }; + // Apply the same validation to exports and imports; do not roundtrip + // through serde_json::Value, which cannot retain arbitrary u128 IDs. + snapshot.validate(&self.context, now_ns)?; + Ok(snapshot) + } + pub fn restore_checkpoint( + snapshot: CaptureCheckpoint, + expected: &CaptureContext, + now_ns: u64, + ) -> Result { + snapshot.validate(expected, now_ns)?; + let mut session = Self::new(snapshot.context.clone(), snapshot.run_id.clone())?; + session.last_seen_ns = snapshot.last_seen_ns; + session.cohorts = snapshot.cohorts; + session.history = snapshot.history; + session.metadata_lineage = snapshot.metadata_lineage; + for receipt in &session.history { + session + .trials + .insert(receipt.trial_id.clone(), receipt.role); + session + .sessions + .insert(receipt.session_id.clone(), receipt.role); + } + for watermark in snapshot.sources { + session.sources.insert( + watermark.source_id, + SourceBuffer { + sequence: watermark.sequence, + ordinal: watermark.ordinal, + last_time_ns: watermark.last_time_ns, + admitted_packets: watermark.admitted_packets, + live: VecDeque::new(), + ..SourceBuffer::default() + }, + ); + } + let disposition = if let Some(p) = snapshot.phase { + session + .trials + .insert(p.request.trial_id.clone(), p.request.role); + session + .sessions + .insert(p.request.session_id.clone(), p.request.role); + let complete = snapshot.saved_at_ns >= phase_end(&p)? + && p.representatives.len() == COHORTS_PER_PHASE; + let phase = Phase { + request: p.request, + started_ns: p.started_ns, + bucket: p.bucket, + representatives: if complete { p.representatives } else { vec![] }, + pending: HashMap::new(), + admitted: HashMap::new(), + }; + if complete { + session.phase = Some(phase); + RecoveryPhaseDisposition::CompletedNeedsConfirmation + } else { + session.history.push(receipt( + &phase, + PhaseStatus::Failed, + Some( + "capture interrupted; restart required; downtime is not observation".into(), + ), + )); + RecoveryPhaseDisposition::InterruptedNeedsRestart + } + } else { + RecoveryPhaseDisposition::NoActivePhase + }; + Ok(CaptureRestore { + session, + disposition, + }) + } + /// Only semantic milestones trigger persistence; packet cadence and receiver + /// counters cannot turn this into a raw-packet write path. + pub fn recovery_marker(&self, now_ns: u64) -> String { + let phase = self.phase.as_ref().map(|p| { + let end = p + .started_ns + .saturating_add((p.request.duration_ms + p.request.lead_in_ms) * 1_000_000); + ( + &p.request, + p.started_ns, + p.representatives.len(), + now_ns >= end, + ) + }); + let bytes = serde_json::to_vec(&( + &self.context, + &self.run_id, + &self.history, + self.cohorts.len(), + phase, + )) + .unwrap_or_default(); + format!("{:x}", Sha256::digest(bytes)) + } +} +impl CaptureCheckpoint { + fn validate(&self, expected: &CaptureContext, now_ns: u64) -> Result<(), String> { + self.context.validate()?; + if let Some(seed)=&self.metadata_lineage {seed.validate()?;} + let lineage_offset=self.metadata_lineage.as_ref().map_or(0,|s|s.phase_claims.len()); + let expected_schema=if self.metadata_lineage.is_some() {"ruview.room-learner.capture-checkpoint.v2"} else {SCHEMA}; + if self.schema != expected_schema + || &self.context != expected + || self.saved_at_ns == 0 + || self.saved_at_ns > now_ns + || self.last_seen_ns > self.saved_at_ns + || self.sources.len() != self.context.sources.len() + || self.history.len()+lineage_offset > MAX_PHASE_HISTORY + || (self.phase.is_some() && self.history.len()+lineage_offset >= MAX_PHASE_HISTORY) + || self.cohorts.len()+self.metadata_lineage.as_ref().map_or(0,|s|s.phase_claims.iter().map(|p|p.cohort_count).sum::()) > MAX_COHORTS + { + return Err("invalid recovery context, time or capacity".into()); + } + let session = CaptureSession::new(self.context.clone(), self.run_id.clone())?; + if let Some(seed)=&self.metadata_lineage { + seed.validate()?; + if seed.context!=self.context || seed.run_id!=self.run_id || seed.last_seen_ns>self.last_seen_ns { + return Err("recovery managed lineage context/namespace mismatch".into()); + } + } + let mut latest = 0; + for (w, spec) in self.sources.iter().zip(&self.context.sources) { + if w.source_id != spec.source_id + || w.ordinal >= (1u64 << 56) + || w.last_time_ns > self.last_seen_ns + || w.admitted_packets > w.ordinal + || match w.sequence { + None => w.ordinal != 0 || w.last_time_ns != 0 || w.admitted_packets != 0, + Some(_) => w.ordinal == 0 || w.last_time_ns == 0 || w.admitted_packets == 0, + } + { + return Err("invalid source recovery watermark".into()); + } + if let Some(seed)=&self.metadata_lineage { + let original=seed.sources.iter().find(|s|s.source_id==w.source_id).ok_or("missing original recovery source")?; + if w.ordinal Result<(), String> { + identifier(trial)?; + identifier(capture_session)?; + if let Some(seed)=&self.metadata_lineage {seed.check_role(trial,capture_session,role)?;} + if role == DataRole::Inference + || trials + .insert(trial.clone(), role) + .is_some_and(|r| r != role) + || sessions + .insert(capture_session.clone(), role) + .is_some_and(|r| r != role) + { + return Err("recovery phase crosses frozen roles".into()); + } + Ok(()) + }; + for r in &self.history { + bind(&r.trial_id, &r.session_id, r.role)?; + if r.label == RoomLabel::Unknown + || r.cohort_count + != if r.status == PhaseStatus::Sealed { + COHORTS_PER_PHASE + } else { + 0 + } + || r.reason + .as_ref() + .is_some_and(|s| s.len() > 512 || s.chars().any(char::is_control)) + { + return Err("invalid recovery history".into()); + } + } + let mut all = self.cohorts.clone(); + if let Some(p) = &self.phase { + phase_end(p)?; + bind(&p.request.trial_id, &p.request.session_id, p.request.role)?; + if p.started_ns > self.saved_at_ns + || p.representatives.len() > COHORTS_PER_PHASE + || all.len() + p.representatives.len() > MAX_COHORTS + { + return Err("invalid provisional recovery phase".into()); + } + let mut previous_bucket = None; + let start = p.started_ns + p.request.lead_in_ms * 1_000_000; + let width = p.request.duration_ms * 1_000_000 / COHORTS_PER_PHASE as u64; + for c in &p.representatives { + let bucket = (0..COHORTS_PER_PHASE) + .find(|b| { + c.id == format!("{}-phase-{}-bucket-{b}", self.run_id, self.history.len()+lineage_offset) + }) + .ok_or("invalid provisional cohort identity")?; + if previous_bucket.is_some_and(|b| b >= bucket) + || bucket > p.bucket + || c.role != p.request.role + || c.label != p.request.label + || c.trial_id != p.request.trial_id + || c.session_id != p.request.session_id + || c.views.iter().any(|v| { + v.first_time_ns < start + bucket as u64 * width + || v.last_time_ns >= start + (bucket as u64 + 1) * width + }) + { + return Err("invalid provisional phase representative".into()); + } + previous_bucket = Some(bucket); + } + all.extend(p.representatives.clone()); + } + let mut expected_cohorts = HashSet::new(); + for (index, r) in self + .history + .iter() + .enumerate() + .filter(|(_, r)| r.status == PhaseStatus::Sealed) + { + for bucket in 0..COHORTS_PER_PHASE { + let index=index+lineage_offset; + let id = format!("{}-phase-{index}-bucket-{bucket}", self.run_id); + let c = self + .cohorts + .iter() + .find(|c| c.id == id) + .ok_or("missing sealed recovery cohort")?; + if c.role != r.role + || c.label != r.label + || c.trial_id != r.trial_id + || c.session_id != r.session_id + { + return Err("sealed recovery history mismatch".into()); + } + expected_cohorts.insert(id); + } + } + if expected_cohorts.len() != self.cohorts.len() { + return Err("unrecorded sealed recovery cohort".into()); + } + let mut frames = HashSet::new(); + for c in &all { + if c.provenance != CaptureProvenance::LiveReceiverObserved || c.teacher.is_some() + || (self.metadata_lineage.is_some() && c.role==DataRole::Train) { + return Err("recovery requires original managed capture provenance".into()); + } + for v in &c.views { + let w = self + .sources + .iter() + .find(|w| w.source_id == v.source_id) + .ok_or("unknown recovery source")?; + if v.samples.len() != WINDOW_PACKETS + || v.last_time_ns > w.last_time_ns + || v.clock.kind != ClockKind::ReceiverOnly + || v.clock.uncertainty_ns != u64::MAX + || v.phase != PhaseAvailability::MagnitudeOnly + { + return Err("invalid reduced recovery source window".into()); + } + if let Some(seed)=&self.metadata_lineage { + let old=seed.sources.iter().find(|s|s.source_id==v.source_id).ok_or("unknown original source")?; + if v.first_time_ns<=old.last_time_ns || v.sequence_ids.iter().any(|n|*n<=old.ordinal) { + return Err("recovered cohort overlaps original managed Train capture".into()); + } + } + for (frame, ordinal) in v.frame_ids.iter().zip(&v.sequence_ids) { + let expected = (u128::from(session.prefix) << 64) + | (u128::from(v.source_id) << 56) + | u128::from(*ordinal); + if *ordinal == 0 + || *ordinal > w.ordinal + || *frame != expected + || !frames.insert(*frame) + { + return Err("reused or invalid recovery frame/sequence watermark".into()); + } + } + } + } + if !all.is_empty() { + ValidatedDataset::new(self.context.clone(), all)?; + } + Ok(()) + } +} +#[cfg(test)] +#[path = "capture_recovery_tests.rs"] +mod tests; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/capture_recovery_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/capture_recovery_tests.rs new file mode 100644 index 0000000000..e819e40cca --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/capture_recovery_tests.rs @@ -0,0 +1,432 @@ +//! Synthetic recovery fixtures, never physical qualification. +use super::*; +use ruview_unified::RfModality; +const START: u64 = 1_000_000_000; +fn context() -> CaptureContext { + CaptureContext { + owner_scope: "synthetic-owner".into(), + installation_id: "synthetic-install".into(), + room_digest: "synthetic-room".into(), + revision: 1, + sources: (1..=2) + .map(|id| SourceSpec { + source_id: id, + device_id: format!("synthetic-{id}"), + grid_id: "ht20-64-raw".into(), + feature_count: 64, + signal_kind: RfModality::WifiCsi, + center_frequency_hz: 2_437_000_000, + bandwidth_hz: 20_000_000, + link_id: format!("tx-rx-{id}"), + link_index: 0, + geometry: None, + }) + .collect(), + } +} +fn session() -> CaptureSession { + CaptureSession::new(context(), "synthetic-run".into()).unwrap() +} +fn request(role: DataRole, label: RoomLabel, id: &str) -> PhaseRequest { + PhaseRequest { + trial_id: format!("synthetic-trial-{id}"), + session_id: format!("synthetic-session-{id}"), + role, + label, + duration_ms: MIN_PHASE_MS, + lead_in_ms: 0, + conditions_confirmed: true, + } +} +fn packet(source_id: u8, sequence: u32, values: &[f64]) -> RawSourceFrame<'_> { + RawSourceFrame { + source_id, + grid_id: "ht20-64-raw", + frequency_hz: 2_437_000_000, + bandwidth_hz: 20_000_000, + sequence, + amplitudes: values, + } +} +fn window(s: &mut CaptureSession, at: u64, start_sequence: u32, gain: f64) { + let values = vec![gain; 64]; + for t in 0..WINDOW_PACKETS { + for source in 1..=2 { + s.ingest( + packet(source, start_sequence.wrapping_add(t as u32), &values), + at + t as u64 * 10_000_000 + u64::from(source), + ) + .unwrap(); + } + } +} +fn phase( + s: &mut CaptureSession, + at: u64, + seq: u32, + role: DataRole, + label: RoomLabel, + id: &str, + gain: f64, +) { + s.begin_phase(request(role, label, id), at).unwrap(); + for quarter in 0..COHORTS_PER_PHASE { + window( + s, + at + quarter as u64 * 3_000_000_000 + 1_000_000, + seq + quarter as u32 * 8, + gain, + ); + } + s.finish_phase(at + MIN_PHASE_MS * 1_000_000).unwrap(); +} + +fn reopen(snapshot: &CaptureCheckpoint, now: u64) -> Result { + let bytes = serde_json::to_vec(snapshot).unwrap(); + let decoded: CaptureCheckpoint = serde_json::from_slice(&bytes).unwrap(); + CaptureSession::restore_checkpoint(decoded, &context(), now) +} +#[test] +fn completed_600s_requires_confirmation_after_typed_u128_roundtrip() { + let mut s = session(); + let mut r = request(DataRole::Train, RoomLabel::Empty, "600"); + r.duration_ms = 600_000; + s.begin_phase(r, START).unwrap(); + for bucket in 0..4 { + window( + &mut s, + START + bucket * 150_000_000_000 + 1_000_000, + 1 + bucket as u32 * 8, + 2.0, + ); + } + let end = START + 600_000_000_000; + let snapshot = s.checkpoint(end).unwrap(); + assert!(snapshot.summary().phase.unwrap().complete); + assert!( + snapshot.phase.as_ref().unwrap().representatives[0].views[0].frame_ids[0] + > u128::from(u64::MAX) + ); + let mut restored = reopen(&snapshot, end + 100_000_000_000).unwrap(); + assert_eq!( + restored.disposition, + RecoveryPhaseDisposition::CompletedNeedsConfirmation + ); + assert!(restored.session.cohorts().is_empty()); + assert!(restored.session.live_cohort(end + 100_000_000_000).is_err()); + assert!(restored + .session + .progress(end) + .sources + .iter() + .all(|s| s.live_buffer_frames == 0 && s.phase_buffer_frames == 0)); + let receipt = restored + .session + .finish_phase(end + 100_000_000_000) + .unwrap(); + assert_eq!(receipt.status, PhaseStatus::Sealed); + assert_eq!(restored.session.cohorts().len(), 4); + assert_eq!( + serde_json::to_vec(restored.session.cohorts()).unwrap(), + serde_json::to_vec(&snapshot.phase.unwrap().representatives).unwrap() + ); +} +#[test] +fn interrupted_phase_cannot_count_downtime_or_reuse_provisional_samples() { + let mut s = session(); + s.begin_phase(request(DataRole::Train, RoomLabel::Empty, "partial"), START) + .unwrap(); + window(&mut s, START + 1_000_000, 1, 2.0); + let snapshot = s.checkpoint(START + 2_000_000_000).unwrap(); + let mut recovered = reopen(&snapshot, START + 20_000_000_000).unwrap(); + assert_eq!( + recovered.disposition, + RecoveryPhaseDisposition::InterruptedNeedsRestart + ); + assert!(!recovered.session.phase_active()); + assert!(recovered.session.cohorts().is_empty()); + assert!(recovered + .session + .finish_phase(START + 20_000_000_000) + .is_err()); + assert_eq!(recovered.session.history()[0].status, PhaseStatus::Failed); + assert!(recovered + .session + .begin_phase( + request(DataRole::HeldOut, RoomLabel::Empty, "partial"), + START + 20_000_000_000 + ) + .is_err()); + recovered + .session + .begin_phase( + request(DataRole::Train, RoomLabel::Empty, "partial"), + START + 20_000_000_000, + ) + .unwrap(); + assert!(recovered + .session + .live_cohort(START + 20_000_000_000) + .is_err()); +} +#[test] +fn four_representatives_before_deadline_still_restore_as_interrupted() { + let mut s = session(); + s.begin_phase(request(DataRole::Train, RoomLabel::Empty, "early"), START) + .unwrap(); + for bucket in 0..4 { + window( + &mut s, + START + bucket * 3_000_000_000 + 1_000_000, + 1 + bucket as u32 * 8, + 2.0, + ); + } + let snapshot = s.checkpoint(START + 10_000_000_000).unwrap(); + assert!(!snapshot.summary().phase.unwrap().complete); + assert_eq!( + reopen(&snapshot, START + 20_000_000_000) + .unwrap() + .disposition, + RecoveryPhaseDisposition::InterruptedNeedsRestart + ); +} +#[test] +fn sealed_roles_watermarks_and_replay_guard_survive_without_live_buffers() { + let mut s = session(); + phase( + &mut s, + START, + 1, + DataRole::Train, + RoomLabel::Empty, + "sealed", + 2.0, + ); + let at = START + MIN_PHASE_MS * 1_000_000; + let snapshot = s.checkpoint(at).unwrap(); + let mut restored = reopen(&snapshot, at + 1).unwrap(); + assert_eq!( + restored.disposition, + RecoveryPhaseDisposition::NoActivePhase + ); + assert_eq!( + restored.session.validated_dataset().unwrap().train().len(), + 4 + ); + assert!(restored + .session + .begin_phase( + request(DataRole::Validation, RoomLabel::Empty, "sealed"), + at + 2 + ) + .is_err()); + let values = vec![2.0; 64]; + assert!(restored + .session + .ingest(packet(1, 32, &values), at + 3) + .is_err()); + assert!(restored + .session + .ingest(packet(1, 1, &values), at + 3) + .is_err()); + restored + .session + .ingest(packet(1, 33, &values), at + 3) + .unwrap(); + assert_eq!(restored.session.sources[&1].ordinal, 33); + assert_eq!(restored.session.sources[&1].live.len(), 1); + assert!(restored.session.live_cohort(at + 3).is_err()); +} +#[test] +fn malformed_context_times_sources_roles_and_frame_reuse_fail_closed() { + let mut s = session(); + phase( + &mut s, + START, + 1, + DataRole::Train, + RoomLabel::Empty, + "sealed", + 2.0, + ); + let now = START + MIN_PHASE_MS * 1_000_000; + let original = s.checkpoint(now).unwrap(); + let mut wrong = context(); + wrong.revision += 1; + assert!(CaptureSession::restore_checkpoint(original.clone(), &wrong, now).is_err()); + assert!(reopen(&original, now - 1).is_err()); + for mutant in 0..8 { + let mut c = original.clone(); + match mutant { + 0 => c.sources[0].source_id = 99, + 1 => c.sources[0].last_time_ns = now + 1, + 2 => c.sources[0].ordinal = 1, + 3 => c.history[0].role = DataRole::HeldOut, + 4 => c.cohorts[1].views[0].frame_ids[0] = c.cohorts[0].views[0].frame_ids[0], + 5 => c.cohorts[0].views[0].sequence_ids[0] = 0, + 6 => c.cohorts[0].views[0].samples[0][0] = f32::NAN, + _ => c.cohorts[0].views[0].last_time_ns = now + 1, + } + assert!( + CaptureSession::restore_checkpoint(c, &context(), now).is_err(), + "mutant {mutant}" + ); + } + let mut bytes = serde_json::to_vec(&original).unwrap(); + bytes.pop(); + bytes.extend_from_slice(br#","unknown":1}"#); + assert!(serde_json::from_slice::(&bytes).is_err()); +} +#[test] +fn marker_changes_only_at_capture_milestones_and_original_deadline() { + let mut s = session(); + let initial = s.recovery_marker(START); + let values = vec![2.0; 64]; + s.ingest(packet(1, 1, &values), START).unwrap(); + assert_eq!(initial, s.recovery_marker(START + 1)); + s.begin_phase( + request(DataRole::Train, RoomLabel::Empty, "marker"), + START + 1, + ) + .unwrap(); + let began = s.recovery_marker(START + 2); + assert_ne!(initial, began); + s.ingest(packet(1, 2, &values), START + 3).unwrap(); + assert_eq!(began, s.recovery_marker(START + 4)); + window(&mut s, START + 1_000_000, 3, 2.0); + let representative = s.recovery_marker(START + 100_000_000); + assert_ne!(began, representative); + assert_eq!(representative, s.recovery_marker(START + 1_000_000_000)); + assert_ne!( + representative, + s.recovery_marker(START + MIN_PHASE_MS * 1_000_000 + 1) + ); +} + +#[test] +fn unlabelled_live_windows_and_partial_packet_buffers_are_never_serialized() { + let mut s = session(); + window(&mut s, START, 1, 2.0); + assert!(s.live_cohort(START + 100_000_000).is_ok()); + let saved = s.checkpoint(START + 100_000_000).unwrap(); + let encoded = String::from_utf8(serde_json::to_vec(&saved).unwrap()).unwrap(); + assert!(!encoded.contains("\"samples\"")); + assert!(!encoded.contains("\"live\"")); + assert!(!encoded.contains("\"pending\"")); + let restored = reopen(&saved, START + 100_000_001).unwrap(); + assert!(restored.session.live_cohort(START + 100_000_001).is_err()); + s.begin_phase( + request(DataRole::Train, RoomLabel::Empty, "pending"), + START + 100_000_001, + ) + .unwrap(); + let values = vec![2.0; 64]; + s.ingest(packet(1, 9, &values), START + 100_000_002) + .unwrap(); + let saved = s.checkpoint(START + 100_000_003).unwrap(); + assert!(saved.phase.as_ref().unwrap().representatives.is_empty()); + assert!(!String::from_utf8(serde_json::to_vec(&saved).unwrap()) + .unwrap() + .contains("\"samples\"")); +} + +#[test] +fn recovered_sequence_diagnostics_explain_lower_input_without_rebasing_evidence() { + let mut s = session(); + let high = 1_000_000; + phase(&mut s, START, high, DataRole::Train, RoomLabel::Empty, "high", 2.0); + let at = START + MIN_PHASE_MS * 1_000_000; + let saved = s.checkpoint(at).unwrap(); + let original_rows = serde_json::to_vec(s.cohorts()).unwrap(); + let original_watermarks = serde_json::to_vec(&saved.sources).unwrap(); + let mut recovered = reopen(&saved, at + 1).unwrap().session; + let first = &recovered.progress(at + 1).sources[0]; + assert_eq!(first.last_accepted_sequence, Some(high + 31)); + assert_eq!(first.last_observed_sequence, None); + assert_eq!(first.sequence_rejection_count, 0); + assert_eq!(first.sequence_rejection_reason, None); + let values = vec![2.0; 64]; + for (index, seq) in [1, 2, high + 31].into_iter().enumerate() { + assert!(recovered.ingest(packet(1, seq, &values), at + 2 + index as u64).is_err()); + } + let progress = recovered.progress(at + 6); + let source = &progress.sources[0]; + assert_eq!(source.last_accepted_sequence, Some(high + 31)); + assert_eq!(source.last_observed_sequence, Some(high + 31)); + assert_eq!(source.sequence_rejection_count, 3); + assert_eq!(source.sequence_rejection_reason, Some(SequenceRejectionReason::Duplicate)); + assert_eq!(source.admitted_packets, 32); + assert_eq!(source.live_buffer_frames, 0); + assert!(!source.window_eligible); + assert_eq!(progress.sources[1].sequence_rejection_count, 0); + assert_eq!(serde_json::to_vec(&recovered.checkpoint(at + 6).unwrap().sources).unwrap(), original_watermarks); + assert_eq!(serde_json::to_vec(recovered.cohorts()).unwrap(), original_rows); + assert!(recovered.begin_phase(request(DataRole::HeldOut, RoomLabel::Empty, "high"), at + 7).is_err()); + + window(&mut recovered, at + 100_000_000, high + 32, 3.0); + let progress = recovered.progress(at + 180_000_000); + assert!(progress.sources.iter().all(|s| s.window_eligible && s.last_accepted_sequence == Some(high + 39))); + assert_eq!(progress.sources[0].last_observed_sequence, Some(high + 39)); + assert_eq!(progress.sources[0].sequence_rejection_count, 3); + assert_eq!(progress.sources[0].sequence_rejection_reason, None); + assert_eq!(progress.sources[0].admitted_packets, 40); + let live = recovered.live_cohort(at + 180_000_000).unwrap(); + assert!(live.views.iter().all(|v| v.sequence_ids == (33..=40).collect::>())); + assert_eq!(serde_json::to_vec(recovered.cohorts()).unwrap(), original_rows); + + let checkpoint = recovered.checkpoint(at + 200_000_000).unwrap(); + let serialized = serde_json::to_string(&checkpoint).unwrap(); + for transient in ["last_observed_sequence", "sequence_rejection_count", "sequence_rejection_reason"] { + assert!(!serialized.contains(transient), "diagnostics must not become retained evidence: {transient}"); + } + let restored = reopen(&checkpoint, at + 200_000_001).unwrap().session; + let progress = restored.progress(at + 200_000_001); + assert!(progress.sources.iter().all(|s| s.last_observed_sequence.is_none() && s.sequence_rejection_count == 0 && s.sequence_rejection_reason.is_none() && !s.window_eligible)); +} + +#[test] +fn reversed_sequence_reason_is_source_local_bounded_and_does_not_claim_reboot() { + let mut s = session(); + window(&mut s, START, 100, 2.0); + let values = vec![2.0; 64]; + s.sources.get_mut(&1).unwrap().sequence_rejection_count = MAX_SEQUENCE_DIAGNOSTIC_COUNT; + assert!(s.ingest(packet(1, 1, &values), START + 90_000_000).is_err()); + let progress = s.progress(START + 90_000_000); + let source = &progress.sources[0]; + assert_eq!(source.last_observed_sequence, Some(1)); + assert_eq!(source.last_accepted_sequence, Some(107)); + assert_eq!(source.sequence_rejection_count, MAX_SEQUENCE_DIAGNOSTIC_COUNT); + assert_eq!(source.sequence_rejection_reason, Some(SequenceRejectionReason::Reversed)); + assert_eq!(progress.sources[1].sequence_rejection_reason, None); + let diagnostic = serde_json::to_value(&progress).unwrap(); + assert_eq!(diagnostic["sources"][0]["sequence_rejection_reason"], "reversed"); + assert_eq!(diagnostic["sources"][0]["sequence_rejection_count"], 9_007_199_254_740_991_u64); + let json = serde_json::to_string(&progress).unwrap(); + for forbidden in ["amplitudes", "samples", "frame_ids", "ordinal", "receiver_epoch", "reboot"] { + assert!(!json.contains(forbidden), "unexpected diagnostic data/claim: {forbidden}"); + } +} + +#[test] +fn valid_sequence_wrap_remains_admitted_and_invalid_time_cannot_launder_diagnostics() { + let mut s = session(); + window(&mut s, START, u32::MAX - 3, 2.0); + let at = START + 80_000_000; + let checkpoint = s.checkpoint(at).unwrap(); + let mut recovered = reopen(&checkpoint, at + 1).unwrap().session; + window(&mut recovered, at + 100_000_000, 4, 3.0); + let source = &recovered.progress(at + 180_000_000).sources[0]; + assert_eq!(source.last_accepted_sequence, Some(11)); + assert_eq!(source.last_observed_sequence, Some(11)); + assert_eq!(source.sequence_rejection_count, 0); + assert_eq!(source.sequence_rejection_reason, None); + let values = vec![3.0; 64]; + assert!(recovered.ingest(packet(1, 2, &values), at + 190_000_000).is_err()); + assert!(recovered.ingest(packet(1, 999, &values), START).is_err()); + let source = &recovered.progress(at + 200_000_000).sources[0]; + assert_eq!(source.last_observed_sequence, Some(2)); + assert_eq!(source.last_accepted_sequence, Some(11)); + assert_eq!(source.sequence_rejection_count, 1); + assert_eq!(source.sequence_rejection_reason, Some(SequenceRejectionReason::Reversed)); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/capture_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/capture_tests.rs new file mode 100644 index 0000000000..6d6b31f93d --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/capture_tests.rs @@ -0,0 +1,195 @@ +//! SYNTHETIC packets exercising production public APIs; no hardware evidence. +use super::*; +use ruview_unified::RfModality; + +const START: u64 = 1_000_000_000; +fn context() -> CaptureContext { + CaptureContext { owner_scope:"synthetic-owner".into(),installation_id:"synthetic-install".into(), + room_digest:"synthetic-room".into(),revision:1,sources:(1..=2).map(|id| SourceSpec { + source_id:id,device_id:format!("synthetic-{id}"),grid_id:"ht20-64-raw".into(), + feature_count:64,signal_kind:RfModality::WifiCsi,center_frequency_hz:2_437_000_000, + bandwidth_hz:20_000_000,link_id:format!("tx-rx-{id}"),link_index:0,geometry:None }).collect() } +} +fn session() -> CaptureSession { CaptureSession::new(context(),"synthetic-run".into()).unwrap() } +fn request(role:DataRole,label:RoomLabel,id:&str) -> PhaseRequest { + PhaseRequest {trial_id:format!("synthetic-trial-{id}"),session_id:format!("synthetic-session-{id}"), + role,label,duration_ms:MIN_PHASE_MS, lead_in_ms: 0,conditions_confirmed:true} +} +fn packet(source_id:u8,sequence:u32,values:&[f64]) -> RawSourceFrame<'_> { + RawSourceFrame {source_id,grid_id:"ht20-64-raw",frequency_hz:2_437_000_000, + bandwidth_hz:20_000_000,sequence,amplitudes:values} +} +fn window(s:&mut CaptureSession,at:u64,start_sequence:u32,gain:f64) { + let values=vec![gain;64]; + for t in 0..WINDOW_PACKETS { + for source in 1..=2 { + s.ingest(packet(source,start_sequence.wrapping_add(t as u32),&values), + at+t as u64*10_000_000+u64::from(source)).unwrap(); + } + } +} +fn phase(s:&mut CaptureSession,at:u64,seq:u32,role:DataRole,label:RoomLabel,id:&str,gain:f64) { + s.begin_phase(request(role,label,id),at).unwrap(); + for quarter in 0..COHORTS_PER_PHASE { + window(s,at+quarter as u64*3_000_000_000+1_000_000,seq+quarter as u32*8,gain); + } + s.finish_phase(at+MIN_PHASE_MS*1_000_000).unwrap(); +} + +#[test] +fn four_confirmed_phase_quarters_seal_complete_source_cohorts_without_raw_receipt() { + let mut s=session(); + phase(&mut s,START,1,DataRole::Train,RoomLabel::Empty,"empty",2.0); + assert_eq!(s.cohorts().len(),4); + assert!(s.cohorts().iter().all(|c|c.views.len()==2 && c.views.iter().all(|v|v.samples.len()==8))); + assert_eq!(s.history()[0].status,PhaseStatus::Sealed); + let json=serde_json::to_string(&s.history()[0]).unwrap(); + assert!(!json.contains("samples") && !json.contains("amplitudes")); + assert!(!s.validated_dataset().unwrap().measurement_clock_available()); +} +#[test] +fn live_prediction_is_unknown_and_receiver_only_never_an_empty_reference() { + let mut s=session();window(&mut s,START,1,7.0); + let c=s.live_cohort(START+80_000_000).unwrap(); + assert_eq!((c.role,c.label),(DataRole::Inference,RoomLabel::Unknown)); + assert!(c.teacher.is_none()); + assert!(c.views.iter().all(|v|v.clock.kind==ClockKind::ReceiverOnly + && v.phase==PhaseAvailability::MagnitudeOnly && v.provenance==CaptureProvenance::LiveReceiverObserved)); + assert!(ValidatedDataset::new(context(),vec![c]).is_err()); +} +#[test] +fn gain_and_mean_shift_survive_distinct_operator_phase_labels() { + let mut s=session(); + phase(&mut s,START,1,DataRole::Train,RoomLabel::Empty,"empty",2.0); + phase(&mut s,START+13_000_000_000,33,DataRole::Train,RoomLabel::OccupiedStatic,"static",8.0); + assert!(s.cohorts()[0].views[0].samples.iter().flatten().all(|v|*v==2.0)); + assert!(s.cohorts()[4].views[0].samples.iter().flatten().all(|v|*v==8.0)); +} +#[test] +fn missing_source_dropout_never_pads_or_seals_partial_phase() { + let mut s=session();s.begin_phase(request(DataRole::HeldOut,RoomLabel::OccupiedMoving,"dropout"),START).unwrap(); + let values=vec![3.0;64]; + for t in 0..64 {s.ingest(packet(1,t,&values),START+1+t as u64*10_000_000).unwrap();} + assert!(s.finish_phase(START+MIN_PHASE_MS*1_000_000).is_err()); + assert!(s.cohorts().is_empty()); + assert_eq!(s.history()[0].status,PhaseStatus::Failed); + assert!(s.history()[0].reason.as_ref().unwrap().contains("four complete")); +} +#[test] +fn cancel_drops_only_inflight_phase_and_keeps_sealed_role_history() { + let mut s=session();phase(&mut s,START,1,DataRole::Train,RoomLabel::Empty,"sealed",2.0); + s.begin_phase(request(DataRole::Validation,RoomLabel::Empty,"cancel"),START+13_000_000_000).unwrap(); + window(&mut s,START+13_001_000_000,33,5.0);s.cancel_phase(); + assert_eq!(s.cohorts().len(),4);assert_eq!(s.history()[1].status,PhaseStatus::Cancelled); + assert!(!s.phase_active()); +} +#[test] +fn source_phy_grid_unknown_source_and_nonfinite_changes_fail_active_phase() { + for change in 0..5 { + let mut s=session();s.begin_phase(request(DataRole::Train,RoomLabel::Empty,"bad"),START).unwrap(); + let mut values=vec![2.0;64];if change==4{values[0]=f64::NAN;} + let mut f=packet(1,1,&values); + match change {0=>f.source_id=3,1=>f.grid_id="different-phy",2=>f.frequency_hz+=5_000_000, + 3=>f.bandwidth_hz*=2,_=>{}} + assert!(s.ingest(f,START+1).is_err());assert!(!s.phase_active()); + assert_eq!(s.history()[0].status,PhaseStatus::Failed); + } +} +#[test] +fn forward_sequence_wrap_extends_ordinals_and_replay_is_rejected() { + let mut s=session();window(&mut s,START,u32::MAX-3,2.0); + let c=s.live_cohort(START+80_000_000).unwrap(); + assert!(c.views.iter().all(|v|v.sequence_ids.windows(2).all(|w|w[1]>w[0]))); + let values=vec![2.0;64]; + assert!(s.ingest(packet(1,3,&values),START+90_000_000).unwrap_err().contains("replayed")); + assert!(s.ingest(packet(1,2,&values),START+91_000_000).is_err()); +} +#[test] +fn phase_boundary_never_reuses_live_window_or_previous_phase_frames() { + let mut s=session();window(&mut s,START,1,99.0); + let begin=START+100_000_000;s.begin_phase(request(DataRole::Train,RoomLabel::Empty,"first"),begin).unwrap(); + let values=vec![2.0;64]; + for t in 0..7 {for source in 1..=2 {s.ingest(packet(source,9+t,&values), + begin+1+t as u64*10_000_000+u64::from(source)).unwrap();}} + assert!(s.finish_phase(begin+MIN_PHASE_MS*1_000_000).is_err()); + assert!(s.cohorts().is_empty()); +} +#[test] +fn interleaved_train_validation_holdout_keep_distinct_frames_and_frozen_roles() { + let mut s=session(); + for (i,role) in [DataRole::Train,DataRole::HeldOut,DataRole::Validation].into_iter().enumerate() { + phase(&mut s,START+i as u64*13_000_000_000,1+i as u32*32,role,RoomLabel::Empty,&format!("{i}"),2.0); + } + let dataset=s.validated_dataset().unwrap(); + assert_eq!((dataset.train().len(),dataset.validation().len(),dataset.held_out().len()),(4,4,4)); + let mut wrong=request(DataRole::Validation,RoomLabel::Empty,"0");wrong.session_id="new-session".into(); + assert!(s.begin_phase(wrong,START+40_000_000_000).unwrap_err().contains("frozen")); +} +#[test] +fn preconditions_duration_and_unlabelled_training_requests_are_rejected() { + for change in 0..5 { + let mut s=session();let mut p=request(DataRole::Train,RoomLabel::Empty,"invalid"); + match change {0=>p.conditions_confirmed=false,1=>p.duration_ms=MIN_PHASE_MS-1, + 2=>p.duration_ms=MAX_PHASE_MS+1,3=>p.label=RoomLabel::Unknown,_=>p.role=DataRole::Inference} + assert!(s.begin_phase(p,START).is_err()); + } +} +#[test] +fn early_finish_leaves_phase_active_and_live_staleness_is_checked() { + let mut s=session();s.begin_phase(request(DataRole::Train,RoomLabel::Empty,"early"),START).unwrap(); + window(&mut s,START+1_000_000,1,2.0); + assert!(s.finish_phase(START+1_000_000_000).is_err());assert!(s.phase_active()); + assert!(s.live_cohort(START+MAX_AGE_NS+100_000_000).is_err()); +} +#[test] +fn quarter_windows_are_nonoverlapping_even_under_high_packet_rate() { + let mut s=session();s.begin_phase(request(DataRole::Train,RoomLabel::Empty,"rate"),START).unwrap(); + for w in 0..10 {window(&mut s,START+1_000_000+w as u64*100_000_000,1+w*8,2.0);} + for q in 1..4 {window(&mut s,START+q as u64*3_000_000_000+1_000_000,81+(q-1)*8,2.0);} + let receipt=s.finish_phase(START+MIN_PHASE_MS*1_000_000).unwrap(); + assert_eq!(receipt.cohort_count,4); + for pair in s.cohorts().windows(2) {assert!(pair[0].views[0].last_time_ns, + pub phase: Option, +} +impl RoomLearningRuntime { + pub(super) fn guard_for(&self, state: &Inner) -> MutationGuard { + MutationGuard { + runtime_epoch: self.runtime_epoch.clone(), + generation: state.generation, + workflow_revision: state.workflow_revision, + context_digest: state.capture_context().map(CaptureContext::digest), + phase: match state.capture.as_ref() { + Some(CaptureWorkflow::Legacy(c)) => c.progress(0).phase.map(|p| PhaseIdentity { trial_id: p.trial_id, session_id: p.session_id }), + Some(CaptureWorkflow::Temporal(c)) => c.phase_identity(), + None => None, + }, + } + } + pub fn mutation_guard(&self) -> Result { + let state = self.inner.lock().map_err(|_| "learner state unavailable")?; + Ok(self.guard_for(&state)) + } + /// Checks and consumes one expected workflow revision while holding the same + /// mutex used by the actual mutation. Raw packet arrival does not consume it. + pub(super) fn command_state( + &self, + expected: &MutationGuard, + ) -> Result, String> { + let mut state = self.inner.lock().map_err(|_| "learner state unavailable")?; + if *expected != self.guard_for(&state) { + return Err("learner_state_changed".into()); + } + state.workflow_revision = state + .workflow_revision + .checked_add(1) + .ok_or("learner command revision exhausted")?; + Ok(state) + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/evaluation_index.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/evaluation_index.rs new file mode 100644 index 0000000000..2e83c9208e --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/evaluation_index.rs @@ -0,0 +1,94 @@ +//! Direct immutable lookup by exact model/role/dataset evidence identity. +use super::*; +const DIRECTORY: &str = "by-dataset"; +const SCHEMA_INDEX: &str = "ruview.room-learner.evaluation-evidence-index.v1"; +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct EvidenceIndex { + schema: String, + owner_scope: String, + installation_id: String, + room_digest: String, + context_digest: String, + model_sha256: String, + model_version: u32, + role: DataRole, + dataset_sha256: String, + receipt_sha256: String, +} +impl ArtifactStore { + pub(super) fn read_evidence_index( + &self, + directory: &Path, + data_sha: &str, + model: &RoomModel, + role: DataRole, + ) -> Result, String> { + if !hex_digest(data_sha) { + return Err("invalid evaluation dataset digest".into()); + } + let Some(index_dir) = private_subdirectory(directory, DIRECTORY, false)? else { + return Ok(None); + }; + let path = index_dir.join(format!("{data_sha}.json")); + if !regular_or_missing(&path)? { + return Ok(None); + } + let index: EvidenceIndex = + serde_json::from_slice(&private_read(&path).map_err(|e| e.to_string())?) + .map_err(|e| e.to_string())?; + let context = model.context(); + if index.schema != SCHEMA_INDEX + || index.owner_scope != context.owner_scope + || index.installation_id != context.installation_id + || index.room_digest != context.room_digest + || index.context_digest != context.digest() + || index.model_sha256 != digest(&model.export()?) + || index.model_version != model.version + || index.role != role + || index.dataset_sha256 != data_sha + || !hex_digest(&index.receipt_sha256) + { + return Err("evaluation evidence index binding mismatch".into()); + } + let receipt = self.read_evaluation(directory, &index.receipt_sha256, model, role)?; + if receipt.dataset_sha256 != data_sha { + return Err("evaluation index references another dataset".into()); + } + Ok(Some(receipt)) + } + pub(super) fn save_evidence_index( + &self, + directory: &Path, + receipt: &EvaluationReceipt, + model: &RoomModel, + role: DataRole, + ) -> Result<(), String> { + receipt.validate(model, role)?; + let index_dir = private_subdirectory(directory, DIRECTORY, true)? + .ok_or("missing evaluation evidence index")?; + let index = EvidenceIndex { + schema: SCHEMA_INDEX.into(), + owner_scope: receipt.owner_scope.clone(), + installation_id: receipt.installation_id.clone(), + room_digest: receipt.room_digest.clone(), + context_digest: receipt.context_digest.clone(), + model_sha256: receipt.model_sha256.clone(), + model_version: receipt.model_version, + role, + dataset_sha256: receipt.dataset_sha256.clone(), + receipt_sha256: receipt.sha256()?, + }; + write_immutable( + &index_dir.join(format!("{}.json", receipt.dataset_sha256)), + &encode(&index)?, + )?; + let restored = self + .read_evidence_index(directory, &receipt.dataset_sha256, model, role)? + .ok_or("evaluation evidence index readback missing")?; + if restored.sha256()? != index.receipt_sha256 { + return Err("evaluation evidence index readback mismatch".into()); + } + Ok(()) + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/evaluation_index_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/evaluation_index_tests.rs new file mode 100644 index 0000000000..d6540ebd8e --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/evaluation_index_tests.rs @@ -0,0 +1,253 @@ +//! SYNTHETIC evidence identity across intervening evaluation datasets. +use super::*; +fn changed_evidence(ds: &ValidatedDataset) -> ValidatedDataset { + let mut rows = ds + .train() + .iter() + .chain(ds.validation()) + .chain(ds.held_out()) + .cloned() + .collect::>(); + rows.iter_mut() + .find(|r| r.role == DataRole::HeldOut) + .unwrap() + .views[0] + .samples[0][0] += 0.03; + ValidatedDataset::new(ds.context().clone(), rows).unwrap() +} +#[test] +fn alternating_datasets_reuse_original_evidence_identity_and_timestamp() { + let temp = tempfile::tempdir().unwrap(); + let (store, model) = saved(temp.path()); + let a = dataset(); + let b = changed_evidence(&a); + let first = store + .evaluate_candidate_at(&model, &a, DataRole::HeldOut, AT) + .unwrap(); + let path = receipt_path(&store, &model, &first); + let first_bytes = private_read(&path).unwrap(); + let middle = store + .evaluate_candidate_at(&model, &b, DataRole::HeldOut, AT + 1) + .unwrap(); + assert_ne!(first.dataset_sha256, middle.dataset_sha256); + // A direct evidence key must work beyond catalog-sized history and ignore + // unrelated unreadable entries; it must not scan or adopt another receipt. + let directory = store + .evaluation_directory(&model, DataRole::HeldOut, false) + .unwrap() + .unwrap(); + for n in 0..257 { + std::fs::write( + directory.join(format!("{n:064x}.json")), + b"unrelated unreadable history", + ) + .unwrap(); + } + let repeat = store + .evaluate_candidate_at(&model, &a, DataRole::HeldOut, AT + 2) + .unwrap(); + assert_eq!( + repeat.sha256().unwrap(), + first.sha256().unwrap(), + "A→B→A must not renew evidence A" + ); + assert_eq!(repeat.evaluated_at_unix_ms, AT); + assert_eq!(repeat.train_lineage_checked, first.train_lineage_checked); + assert_eq!(private_read(&path).unwrap(), first_bytes); + assert_eq!( + store + .load_latest_evaluation(&model, DataRole::HeldOut) + .unwrap(), + Some(first.clone()) + ); + let cold_a = ValidatedDataset::new(a.context().clone(), a.held_out().to_vec()).unwrap(); + let cold_repeat = store + .evaluate_candidate_at(&model, &cold_a, DataRole::HeldOut, AT + 3) + .unwrap(); + assert_eq!( + cold_repeat, first, + "historical lineage proof must not be rewritten on reuse" + ); +} + +fn index_path(store: &ArtifactStore, model: &RoomModel, receipt: &EvaluationReceipt) -> PathBuf { + store + .evaluation_directory(model, receipt.role, false) + .unwrap() + .unwrap() + .join("by-dataset") + .join(format!("{}.json", receipt.dataset_sha256)) +} +#[test] +fn corrupt_or_mismatched_index_never_overwrites_another_latest_receipt() { + for field in [ + "corrupt", + "owner_scope", + "installation_id", + "context_digest", + "model_sha256", + "role", + "dataset_sha256", + "receipt_sha256", + ] { + let temp = tempfile::tempdir().unwrap(); + let (store, model) = saved(temp.path()); + let a = dataset(); + let b = changed_evidence(&a); + let first = store + .evaluate_candidate_at(&model, &a, DataRole::HeldOut, AT) + .unwrap(); + let middle = store + .evaluate_candidate_at(&model, &b, DataRole::HeldOut, AT + 1) + .unwrap(); + let directory = store + .evaluation_directory(&model, DataRole::HeldOut, false) + .unwrap() + .unwrap(); + let pointer = directory.join(LATEST); + let before = private_read(&pointer).unwrap(); + let path = index_path(&store, &model, &first); + let mut bad: serde_json::Value = + serde_json::from_slice(&private_read(&path).unwrap()).unwrap(); + let changed = match field { + "owner_scope" => serde_json::json!("b".repeat(64)), + "installation_id" => serde_json::json!("wrong-install"), + "context_digest" | "model_sha256" => serde_json::json!("0".repeat(64)), + "role" => serde_json::json!("Validation"), + "dataset_sha256" => serde_json::json!(middle.dataset_sha256), + "receipt_sha256" => serde_json::json!(middle.sha256().unwrap()), + _ => serde_json::Value::Null, + }; + let bytes = if field == "corrupt" { + b"corrupt index".to_vec() + } else { + bad[field] = changed; + serde_json::to_vec(&bad).unwrap() + }; + atomic_private_write(&path, &bytes).unwrap(); + assert!( + store + .evaluate_candidate_at(&model, &a, DataRole::HeldOut, AT + 2) + .is_err(), + "index {field}" + ); + assert_eq!(private_read(&pointer).unwrap(), before); + assert_eq!(private_read(&path).unwrap(), bytes); + assert_eq!( + store + .load_latest_evaluation(&model, DataRole::HeldOut) + .unwrap(), + Some(middle) + ); + assert!(store.load_candidate(model.context()).unwrap().is_some()); + } +} +#[cfg(unix)] +#[test] +fn evidence_index_symlink_permission_and_size_errors_preserve_latest_and_victims() { + use std::os::unix::fs::{symlink, PermissionsExt}; + for failure in ["symlink", "permission", "oversized"] { + let temp = tempfile::tempdir().unwrap(); + let (store, model) = saved(temp.path()); + let a = dataset(); + let b = changed_evidence(&a); + let first = store + .evaluate_candidate_at(&model, &a, DataRole::HeldOut, AT) + .unwrap(); + store + .evaluate_candidate_at(&model, &b, DataRole::HeldOut, AT + 1) + .unwrap(); + let path = index_path(&store, &model, &first); + let directory = store + .evaluation_directory(&model, DataRole::HeldOut, false) + .unwrap() + .unwrap(); + let before = private_read(&directory.join(LATEST)).unwrap(); + let victim = temp.path().join("victim"); + atomic_private_write(&victim, b"preserved private victim").unwrap(); + match failure { + "symlink" => { + std::fs::rename(&path, path.with_extension("preserved")).unwrap(); + symlink(&victim, &path).unwrap(); + } + "permission" => { + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o644)).unwrap() + } + _ => atomic_private_write(&path, &vec![b' '; LIMIT + 1]).unwrap(), + } + assert!(store + .evaluate_candidate_at(&model, &a, DataRole::HeldOut, AT + 2) + .is_err()); + assert_eq!(private_read(&directory.join(LATEST)).unwrap(), before); + assert_eq!(private_read(&victim).unwrap(), b"preserved private victim"); + assert!(store.load_candidate(model.context()).unwrap().is_some()); + } +} +#[test] +fn blocked_new_dataset_index_fails_without_publishing_or_replacing_existing_evidence() { + let temp = tempfile::tempdir().unwrap(); + let (store, model) = saved(temp.path()); + let a = dataset(); + let b = changed_evidence(&a); + let first = store + .evaluate_candidate_at(&model, &a, DataRole::HeldOut, AT) + .unwrap(); + let directory = store + .evaluation_directory(&model, DataRole::HeldOut, false) + .unwrap() + .unwrap(); + let pointer = directory.join(LATEST); + let before = private_read(&pointer).unwrap(); + let old_receipt = private_read(&receipt_path(&store, &model, &first)).unwrap(); + let blocked = directory + .join("by-dataset") + .join(format!("{}.json", data_sha256(b.held_out()).unwrap())); + std::fs::create_dir(&blocked).unwrap(); + assert!(store + .evaluate_candidate_at(&model, &b, DataRole::HeldOut, AT + 1) + .is_err()); + assert_eq!(private_read(&pointer).unwrap(), before); + assert_eq!( + private_read(&receipt_path(&store, &model, &first)).unwrap(), + old_receipt + ); + assert!(blocked.is_dir()); + assert_eq!( + store + .load_latest_evaluation(&model, DataRole::HeldOut) + .unwrap(), + Some(first) + ); +} + +#[test] +fn persisted_evidence_index_recovers_original_receipt_after_missing_latest_pointer() { + let temp = tempfile::tempdir().unwrap(); + let (store, model) = saved(temp.path()); + let a = dataset(); + let original = store + .evaluate_candidate_at(&model, &a, DataRole::HeldOut, AT) + .unwrap(); + let directory = store + .evaluation_directory(&model, DataRole::HeldOut, false) + .unwrap() + .unwrap(); + let previous_pointer = directory.join("latest-before-interruption.json"); + std::fs::rename(directory.join(LATEST), &previous_pointer).unwrap(); + let bytes = private_read(&previous_pointer).unwrap(); + let cold = open_store(temp.path()); + let loaded = cold.load_candidate(model.context()).unwrap().unwrap(); + let retry = cold + .evaluate_candidate_at(&loaded, &a, DataRole::HeldOut, AT + 50) + .unwrap(); + assert_eq!( + retry, original, + "retry must not renew evidence after a partial publication" + ); + assert_eq!(private_read(&previous_pointer).unwrap(), bytes); + assert_eq!( + cold.load_latest_evaluation(&loaded, DataRole::HeldOut) + .unwrap(), + Some(original) + ); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/evaluation_receipt.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/evaluation_receipt.rs new file mode 100644 index 0000000000..04ac7d44f7 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/evaluation_receipt.rs @@ -0,0 +1,171 @@ +//! Hashed metadata only. Input provenance/declared trials do not grant adoption. +use super::super::{digest, encode, hex_digest}; +use crate::room_learner::{ + model::RoomModel, + observation::{CaptureCohort, CaptureProvenance, ClockKind, DataRole}, + training::EvaluationReport, +}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::io::{self, Write}; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct EvaluationReceipt { + pub schema: String, + pub owner_scope: String, + pub installation_id: String, + pub room_digest: String, + pub context_digest: String, + pub model_sha256: String, + pub model_version: u32, + pub role: DataRole, + pub dataset_sha256: String, + pub evaluated_at_unix_ms: u64, + pub provenance: Vec, + pub clock_claims: Vec, + pub measurement_clock_available: bool, + /// True only when the retained Train set matched this model's manifest and + /// the dataset validator checked role separation. Not statistical independence. + pub train_lineage_checked: bool, + pub training_data_sha256: String, + pub report: EvaluationReport, + pub independence_verified: bool, + pub physically_validated: bool, + pub adoption_authorized: bool, +} +struct HashWriter(Sha256); +impl Write for HashWriter { + fn write(&mut self, bytes: &[u8]) -> io::Result { + self.0.update(bytes); + Ok(bytes.len()) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} +pub(super) fn data_sha256(rows: &[CaptureCohort]) -> Result { + let mut writer = HashWriter(Sha256::new()); + serde_json::to_writer(&mut writer, rows).map_err(|e| e.to_string())?; + Ok(format!("{:x}", writer.0.finalize())) +} +fn provenance_rank(p: &CaptureProvenance) -> u8 { + match p { + CaptureProvenance::Synthetic => 0, + CaptureProvenance::CapturedReplay => 1, + CaptureProvenance::LiveReceiverObserved => 2, + } +} +fn clock_rank(p: &ClockKind) -> u8 { + match p { + ClockKind::Unknown => 0, + ClockKind::ReceiverOnly => 1, + ClockKind::SourceMeasurement => 2, + } +} +impl EvaluationReceipt { + pub fn sha256(&self) -> Result { + Ok(digest(&encode(self)?)) + } + pub(super) fn new( + model: &RoomModel, + rows: &[CaptureCohort], + role: DataRole, + at: u64, + train_checked: bool, + ) -> Result { + if rows.is_empty() + || rows.iter().any(|r| { + r.role != role + || r.views.iter().any(|v| { + v.provenance != r.provenance + || (v.signal_kind == ruview_unified::RfModality::Synthetic + && v.provenance != CaptureProvenance::Synthetic) + }) + }) + { + return Err("invalid evaluation role/provenance".into()); + } + let mut provenance: Vec<_> = rows.iter().map(|r| r.provenance).collect(); + provenance.sort_by_key(provenance_rank); + provenance.dedup(); + let mut clocks: Vec<_> = rows + .iter() + .flat_map(|r| r.views.iter().map(|v| v.clock.kind)) + .collect(); + clocks.sort_by_key(clock_rank); + clocks.dedup(); + let available = !provenance.contains(&CaptureProvenance::Synthetic) + && clocks == [ClockKind::SourceMeasurement]; + let c = model.context(); + let result = Self { + schema: "ruview.room-learner.evaluation.v1".into(), + owner_scope: c.owner_scope.clone(), + installation_id: c.installation_id.clone(), + room_digest: c.room_digest.clone(), + context_digest: c.digest(), + model_sha256: digest(&model.export()?), + model_version: model.version, + role, + dataset_sha256: data_sha256(rows)?, + evaluated_at_unix_ms: at, + provenance, + clock_claims: clocks, + measurement_clock_available: available, + train_lineage_checked: train_checked, + training_data_sha256: model.training_manifest().training_data_sha256.clone(), + report: model.evaluate(rows)?, + independence_verified: false, + physically_validated: false, + adoption_authorized: false, + }; + result.validate(model, role)?; + Ok(result) + } + pub(super) fn validate(&self, model: &RoomModel, role: DataRole) -> Result<(), String> { + let c = model.context(); + if self.schema != "ruview.room-learner.evaluation.v1" + || self.owner_scope != c.owner_scope + || self.installation_id != c.installation_id + || self.room_digest != c.room_digest + || self.context_digest != c.digest() + || self.model_sha256 != digest(&model.export()?) + || self.model_version != model.version + || self.role != role + || !matches!(role, DataRole::Validation | DataRole::HeldOut) + || !hex_digest(&self.dataset_sha256) + || self.evaluated_at_unix_ms == 0 + || self.evaluated_at_unix_ms > 253_402_300_799_999 + || self.training_data_sha256 != model.training_manifest().training_data_sha256 + || self.independence_verified + || self.physically_validated + || self.adoption_authorized + { + return Err("evaluation receipt/candidate binding mismatch".into()); + } + if self.provenance.is_empty() + || self.provenance.len() > 3 + || self + .provenance + .windows(2) + .any(|w| provenance_rank(&w[0]) >= provenance_rank(&w[1])) + || self.clock_claims.is_empty() + || self.clock_claims.len() > 3 + || self + .clock_claims + .windows(2) + .any(|w| clock_rank(&w[0]) >= clock_rank(&w[1])) + || self.measurement_clock_available + != (!self.provenance.contains(&CaptureProvenance::Synthetic) + && self.clock_claims == [ClockKind::SourceMeasurement]) + || (c + .sources + .iter() + .any(|s| s.signal_kind == ruview_unified::RfModality::Synthetic) + && self.provenance != [CaptureProvenance::Synthetic]) + { + return Err("invalid evaluation provenance/clock claims".into()); + } + self.report.validate() + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/evaluation_report.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/evaluation_report.rs new file mode 100644 index 0000000000..4c41bebecd --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/evaluation_report.rs @@ -0,0 +1,369 @@ +//! Candidate cohort diagnostics. Declared trials are not verified independence. +use super::super::{ + model::{PresenceDecision, RoomModel, RoomPrediction}, + observation::{CaptureCohort, DataRole, RoomLabel, MAX_COHORTS}, +}; +use serde::{Deserialize, Serialize}; +use std::collections::HashSet; + +pub(crate) const CONDITIONS: [RoomLabel; 4] = [ + RoomLabel::Empty, + RoomLabel::OccupiedStatic, + RoomLabel::OccupiedMoving, + RoomLabel::AdjacentOnly, +]; +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct BinaryConfusion { + pub true_empty: usize, + pub false_occupied: usize, + pub false_empty: usize, + pub true_occupied: usize, + pub abstained_empty: usize, + pub abstained_occupied: usize, + /// Missing probabilities are a subset of abstentions, never correct results. + pub missing_empty: usize, + pub missing_occupied: usize, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct EvaluationRates { + pub evaluated_cohort_coverage: Option, + pub unconditional_accuracy: Option, + pub scored_accuracy: Option, + /// Includes every positive cohort, including abstained/missing positives. + pub unconditional_sensitivity: Option, + pub scored_sensitivity: Option, + pub unconditional_specificity: Option, + pub scored_specificity: Option, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct ConditionEvaluation { + pub condition: RoomLabel, + pub samples: usize, + pub correct: usize, + pub abstained: usize, + pub scored: usize, + pub missing: usize, + pub binary_cross_entropy: Option, + pub binary_cross_entropy_samples: usize, + pub declared_trial_count: usize, + pub declared_session_count: usize, + pub confusion: BinaryConfusion, + pub rates: EvaluationRates, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct EvaluationReport { + pub samples: usize, + pub correct: usize, + pub abstained: usize, + /// Same numeric mean as before when probabilities exist; otherwise null. + pub binary_cross_entropy: Option, + pub binary_cross_entropy_samples: usize, + pub scored: usize, + pub missing: usize, + pub declared_trial_count: usize, + pub declared_session_count: usize, + pub confusion: BinaryConfusion, + pub rates: EvaluationRates, + pub per_condition: Vec, + pub independence_verified: bool, +} +fn ratio(n: usize, d: usize) -> Option { + if d == 0 { + None + } else { + Some(n as f32 / d as f32) + } +} +impl BinaryConfusion { + fn scored(&self) -> usize { + self.true_empty + self.false_occupied + self.false_empty + self.true_occupied + } + fn correct(&self) -> usize { + self.true_empty + self.true_occupied + } + fn abstained(&self) -> usize { + self.abstained_empty + self.abstained_occupied + } + fn missing(&self) -> usize { + self.missing_empty + self.missing_occupied + } + fn rates(&self) -> EvaluationRates { + let scored = self.scored(); + let samples = scored + self.abstained(); + EvaluationRates { + evaluated_cohort_coverage: ratio(scored, samples), + unconditional_accuracy: ratio(self.correct(), samples), + scored_accuracy: ratio(self.correct(), scored), + unconditional_sensitivity: ratio( + self.true_occupied, + self.true_occupied + self.false_empty + self.abstained_occupied, + ), + scored_sensitivity: ratio(self.true_occupied, self.true_occupied + self.false_empty), + unconditional_specificity: ratio( + self.true_empty, + self.true_empty + self.false_occupied + self.abstained_empty, + ), + scored_specificity: ratio(self.true_empty, self.true_empty + self.false_occupied), + } + } +} +#[derive(Default)] +struct Accumulator<'a> { + confusion: BinaryConfusion, + loss: f64, + probabilities: usize, + trials: HashSet<&'a str>, + sessions: HashSet<&'a str>, +} +impl<'a> Accumulator<'a> { + fn observe(&mut self, row: &'a CaptureCohort, p: &RoomPrediction) { + self.trials.insert(&row.trial_id); + self.sessions.insert(&row.session_id); + let occupied = matches!( + row.label, + RoomLabel::OccupiedStatic | RoomLabel::OccupiedMoving + ); + let probability = p + .occupied_probability + .filter(|v| v.is_finite() && (0.0..=1.0).contains(v)); + if let Some(v) = probability { + let v = v.clamp(1e-6, 1.0 - 1e-6) as f64; + self.loss -= if occupied { v.ln() } else { (1.0 - v).ln() }; + self.probabilities += 1; + } + let c = &mut self.confusion; + if p.decision == PresenceDecision::Abstain || probability.is_none() { + if occupied { + c.abstained_occupied += 1; + if probability.is_none() { + c.missing_occupied += 1; + } + } else { + c.abstained_empty += 1; + if probability.is_none() { + c.missing_empty += 1; + } + } + } else { + match (occupied, p.decision) { + (false, PresenceDecision::Empty) => c.true_empty += 1, + (false, PresenceDecision::Occupied) => c.false_occupied += 1, + (true, PresenceDecision::Empty) => c.false_empty += 1, + (true, PresenceDecision::Occupied) => c.true_occupied += 1, + _ => unreachable!(), + } + } + } + fn loss(&self) -> Option { + if self.probabilities == 0 { + None + } else { + Some((self.loss / self.probabilities as f64) as f32) + } + } + fn condition(&self, condition: RoomLabel) -> ConditionEvaluation { + ConditionEvaluation { + condition, + samples: self.confusion.scored() + self.confusion.abstained(), + correct: self.confusion.correct(), + abstained: self.confusion.abstained(), + scored: self.confusion.scored(), + missing: self.confusion.missing(), + binary_cross_entropy: self.loss(), + binary_cross_entropy_samples: self.probabilities, + declared_trial_count: self.trials.len(), + declared_session_count: self.sessions.len(), + confusion: self.confusion.clone(), + rates: self.confusion.rates(), + } + } +} +impl EvaluationReport { + pub(crate) fn compute(model: &RoomModel, rows: &[CaptureCohort]) -> Result { + model.validate()?; + if rows.is_empty() || rows.len() > MAX_COHORTS { + return Err("invalid bounded evaluation cohort count".into()); + } + let role = rows[0].role; + if !matches!(role, DataRole::Validation | DataRole::HeldOut) { + return Err("evaluation requires labeled validation or held-out cohorts".into()); + } + let mut ids = HashSet::new(); + let mut total = Accumulator::default(); + let mut conditions: [Accumulator; 4] = Default::default(); + let digest = model.context().digest(); + for row in rows { + if row.role != role || row.label == RoomLabel::Unknown { + return Err("evaluation requires one labeled non-inference role".into()); + } + if row.context_digest != digest { + return Err("evaluation context mismatch".into()); + } + if [&row.id, &row.trial_id, &row.session_id].iter().any(|id| { + id.is_empty() + || id.len() > 256 + || !id.is_ascii() + || id.trim() != id.as_str() + || id.chars().any(char::is_control) + }) || !ids.insert(&row.id) + { + return Err("invalid or duplicate evaluation cohort identity".into()); + } + let prediction = model.infer(row); + let condition = CONDITIONS + .iter() + .position(|label| *label == row.label) + .ok_or("unknown evaluation condition")?; + total.observe(row, &prediction); + conditions[condition].observe(row, &prediction); + } + let summary = total.condition(RoomLabel::Unknown); + let report = Self { + samples: summary.samples, + correct: summary.correct, + abstained: summary.abstained, + binary_cross_entropy: summary.binary_cross_entropy, + binary_cross_entropy_samples: summary.binary_cross_entropy_samples, + scored: summary.scored, + missing: summary.missing, + declared_trial_count: summary.declared_trial_count, + declared_session_count: summary.declared_session_count, + confusion: summary.confusion, + rates: summary.rates, + per_condition: conditions + .iter() + .zip(CONDITIONS) + .map(|(acc, label)| acc.condition(label)) + .collect(), + independence_verified: false, + }; + report.validate()?; + Ok(report) + } + pub(crate) fn validate(&self) -> Result<(), String> { + if !(1..=MAX_COHORTS).contains(&self.samples) + || self.independence_verified + || self.per_condition.len() != 4 + { + return Err("invalid evaluation report bounds".into()); + } + let mut sum = BinaryConfusion::default(); + let mut probability_count = 0usize; + let mut loss = 0.0; + for (condition, label) in self.per_condition.iter().zip(CONDITIONS) { + if condition.condition != label { + return Err("invalid evaluation conditions".into()); + } + validate_summary(condition)?; + let c = &condition.confusion; + if (matches!(label, RoomLabel::Empty | RoomLabel::AdjacentOnly) + && c.true_occupied + c.false_empty + c.abstained_occupied != 0) + || (matches!(label, RoomLabel::OccupiedStatic | RoomLabel::OccupiedMoving) + && c.true_empty + c.false_occupied + c.abstained_empty != 0) + { + return Err("condition confusion target mismatch".into()); + } + sum.true_empty += c.true_empty; + sum.false_occupied += c.false_occupied; + sum.false_empty += c.false_empty; + sum.true_occupied += c.true_occupied; + sum.abstained_empty += c.abstained_empty; + sum.abstained_occupied += c.abstained_occupied; + sum.missing_empty += c.missing_empty; + sum.missing_occupied += c.missing_occupied; + probability_count = probability_count + .checked_add(condition.binary_cross_entropy_samples) + .ok_or("evaluation probability count overflow")?; + loss += condition.binary_cross_entropy.unwrap_or(0.0) as f64 + * condition.binary_cross_entropy_samples as f64; + } + let summary = ConditionEvaluation { + condition: RoomLabel::Unknown, + samples: self.samples, + correct: self.correct, + abstained: self.abstained, + scored: self.scored, + missing: self.missing, + binary_cross_entropy: self.binary_cross_entropy, + binary_cross_entropy_samples: self.binary_cross_entropy_samples, + declared_trial_count: self.declared_trial_count, + declared_session_count: self.declared_session_count, + confusion: self.confusion.clone(), + rates: self.rates.clone(), + }; + validate_summary(&summary)?; + if sum != self.confusion + || probability_count != self.binary_cross_entropy_samples + || (probability_count > 0 + && (self.binary_cross_entropy.unwrap_or(0.0) as f64 + - loss / probability_count as f64) + .abs() + > 1e-4) + || !valid_declared_union( + self.declared_trial_count, + self.per_condition.iter().map(|c| c.declared_trial_count), + ) + || !valid_declared_union( + self.declared_session_count, + self.per_condition.iter().map(|c| c.declared_session_count), + ) + { + return Err("evaluation aggregate mismatch".into()); + } + Ok(()) + } +} +fn valid_declared_union(total: usize, mut counts: impl Iterator) -> bool { + let Some((maximum, sum)) = counts.try_fold((0usize, 0usize), |(maximum, sum), count| { + sum.checked_add(count).map(|sum| (maximum.max(count), sum)) + }) else { + return false; + }; + total >= maximum && total <= sum +} +fn validate_summary(s: &ConditionEvaluation) -> Result<(), String> { + let c = &s.confusion; + // Bound components before addition so malformed JSON cannot overflow usize. + if [ + c.true_empty, + c.false_occupied, + c.false_empty, + c.true_occupied, + c.abstained_empty, + c.abstained_occupied, + c.missing_empty, + c.missing_occupied, + ] + .iter() + .any(|n| *n > MAX_COHORTS) + || s.samples > MAX_COHORTS + || c.scored() + c.abstained() != s.samples + || c.correct() != s.correct + || c.scored() != s.scored + || c.abstained() != s.abstained + || c.missing() != s.missing + || c.missing_empty > c.abstained_empty + || c.missing_occupied > c.abstained_occupied + || s.samples.checked_sub(s.missing) != Some(s.binary_cross_entropy_samples) + || (s.samples > 0 && (s.declared_trial_count == 0 || s.declared_session_count == 0)) + || s.declared_trial_count > s.samples + || s.declared_session_count > s.samples + || c.rates() != s.rates + { + return Err("invalid evaluation cohort metrics".into()); + } + match (s.binary_cross_entropy_samples, s.binary_cross_entropy) { + (0, None) => {} + (n, Some(loss)) if n > 0 && loss.is_finite() && (0.0..=32.0).contains(&loss) => {} + _ => return Err("invalid evaluation probability loss".into()), + } + Ok(()) +} + +#[cfg(test)] +#[path = "evaluation_report_tests.rs"] +mod tests; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/evaluation_report_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/evaluation_report_tests.rs new file mode 100644 index 0000000000..f1f2a0bf3c --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/evaluation_report_tests.rs @@ -0,0 +1,234 @@ +//! SYNTHETIC metrics and denominator oracles; no sensing accuracy claim. +use super::*; +use crate::room_learner::training::{learning_tests::dataset, FitConfig}; +use std::sync::OnceLock; +fn model() -> RoomModel { + static MODEL: OnceLock = OnceLock::new(); + MODEL + .get_or_init(|| { + RoomModel::fit( + &dataset(8), + FitConfig { + epochs: 2, + ..FitConfig::default() + }, + ) + .unwrap() + .0 + }) + .clone() +} +#[test] +fn actual_evaluation_retains_legacy_fields_and_has_all_declared_conditions() { + let ds = dataset(8); + let report = model().evaluate(ds.held_out()).unwrap(); + assert_eq!(report.samples, 4); + assert_eq!(report.declared_trial_count, 4); + assert_eq!(report.per_condition.len(), 4); + assert_eq!(report.scored + report.abstained, report.samples); + assert_eq!(report.binary_cross_entropy_samples, 4); + assert!(report.binary_cross_entropy.unwrap().is_finite()); + assert!(!report.independence_verified); + report.validate().unwrap(); + let json = serde_json::to_value(&report).unwrap(); + for key in ["samples", "correct", "abstained", "binary_cross_entropy"] { + assert!(json.get(key).is_some()); + } + for condition in report.per_condition { + assert_eq!(condition.samples, 1); + assert_eq!(condition.declared_trial_count, 1); + } +} +#[test] +fn windows_are_not_distinct_declared_trials_and_missing_positives_reduce_unconditional_recall() { + let ds = dataset(8); + let mut windows = vec![ds.held_out()[2].clone(); 4]; + for (i, row) in windows.iter_mut().enumerate() { + row.id = format!("window-{i}"); + } + let report = model().evaluate(&windows).unwrap(); + assert_eq!(report.samples, 4); + assert_eq!(report.declared_trial_count, 1); + assert_eq!(report.declared_session_count, 1); + let mut missing = ds.held_out()[2].clone(); + missing.views.clear(); + let report = model().evaluate(&[missing]).unwrap(); + assert_eq!( + ( + report.samples, + report.correct, + report.abstained, + report.missing + ), + (1, 0, 1, 1) + ); + assert_eq!(report.rates.evaluated_cohort_coverage, Some(0.0)); + assert_eq!(report.rates.unconditional_sensitivity, Some(0.0)); + assert_eq!(report.rates.scored_sensitivity, None); + assert_eq!(report.binary_cross_entropy, None); + assert_eq!(report.binary_cross_entropy_samples, 0); + assert_eq!(report.rates.unconditional_specificity, None); +} +#[test] +fn abstentions_are_in_unconditional_sensitivity_and_absent_conditions_are_null() { + let ds = dataset(8); + let row = &ds.held_out()[2]; + let mut acc = Accumulator::default(); + acc.observe( + row, + &RoomPrediction { + decision: PresenceDecision::Occupied, + occupied_probability: Some(0.9), + reason: None, + physically_validated: false, + }, + ); + acc.observe( + row, + &RoomPrediction { + decision: PresenceDecision::Abstain, + occupied_probability: Some(0.5), + reason: None, + physically_validated: false, + }, + ); + acc.observe( + row, + &RoomPrediction { + decision: PresenceDecision::Abstain, + occupied_probability: None, + reason: None, + physically_validated: false, + }, + ); + let summary = acc.condition(RoomLabel::OccupiedStatic); + assert_eq!(summary.rates.unconditional_sensitivity, Some(1.0 / 3.0)); + assert_eq!(summary.rates.scored_sensitivity, Some(1.0)); + assert_eq!(summary.rates.evaluated_cohort_coverage, Some(1.0 / 3.0)); + assert_eq!(summary.binary_cross_entropy_samples, 2); + assert_eq!(summary.abstained, 2); + assert_eq!(summary.missing, 1); + let empty = Accumulator::default().condition(RoomLabel::Empty); + assert_eq!(empty.rates.evaluated_cohort_coverage, None); + assert_eq!(empty.rates.unconditional_accuracy, None); + assert_eq!(empty.binary_cross_entropy, None); +} +#[test] +fn unknown_train_inference_mixed_roles_wrong_context_and_duplicate_windows_are_rejected() { + let ds = dataset(8); + let model = model(); + assert!(model.evaluate(ds.train()).is_err()); + let mut row = ds.held_out()[0].clone(); + row.label = RoomLabel::Unknown; + assert!(model.evaluate(&[row.clone()]).is_err()); + row.role = DataRole::Inference; + assert!(model.evaluate(&[row]).is_err()); + assert!(model + .evaluate(&[ds.validation()[0].clone(), ds.held_out()[0].clone()]) + .is_err()); + assert!(model + .evaluate(&[ds.held_out()[0].clone(), ds.held_out()[0].clone()]) + .is_err()); + let mut wrong = ds.held_out()[0].clone(); + wrong.context_digest = "another-room".into(); + assert!(model.evaluate(&[wrong]).is_err()); +} +#[test] +fn tampered_metrics_nonfinite_losses_and_inflated_claims_fail_validation() { + let report = model().evaluate(dataset(8).held_out()).unwrap(); + let mut bad = report.clone(); + bad.correct = bad.samples + 1; + assert!(bad.validate().is_err()); + let mut bad = report.clone(); + bad.independence_verified = true; + assert!(bad.validate().is_err()); + let mut bad = report.clone(); + bad.binary_cross_entropy = Some(f32::NAN); + assert!(bad.validate().is_err()); + let mut bad = report.clone(); + bad.per_condition[0].rates.evaluated_cohort_coverage = Some(1.5); + assert!(bad.validate().is_err()); + let mut bad = report; + bad.confusion.true_empty = usize::MAX; + assert!(bad.validate().is_err()); +} + +fn sixteen_declared_windows() -> Vec { + let base = dataset(8); + let mut rows = Vec::new(); + for condition in base.held_out() { + for window in 0..4 { + let mut row = condition.clone(); + row.id = format!("{}-window-{window}", row.id); + rows.push(row); + } + } + rows +} +#[test] +fn impossible_declared_union_totals_are_rejected_without_rejecting_shared_trials() { + let rows = sixteen_declared_windows(); + let report = model().evaluate(&rows).unwrap(); + assert_eq!(report.samples, 16); + assert_eq!(report.declared_trial_count, 4); + assert_eq!( + report + .per_condition + .iter() + .map(|p| p.declared_trial_count) + .sum::(), + 4 + ); + let mut impossible = report.clone(); + impossible.declared_trial_count = 16; + assert!( + impossible.validate().is_err(), + "global trials cannot exceed per-condition union bound" + ); + let mut impossible = report; + impossible.declared_session_count = 16; + assert!( + impossible.validate().is_err(), + "global sessions cannot exceed per-condition union bound" + ); + let shared = rows + .into_iter() + .map(|mut row| { + row.trial_id = "shared-cross-condition-trial".into(); + row.session_id = "shared-cross-condition-session".into(); + row + }) + .collect::>(); + let shared = model().evaluate(&shared).unwrap(); + assert_eq!(shared.declared_trial_count, 1); + assert_eq!(shared.declared_session_count, 1); + shared.validate().unwrap(); +} +#[test] +fn nonmissing_abstention_probabilities_must_all_contribute_to_bce() { + let mut json = serde_json::to_value(model()).unwrap(); + for value in json["head_weights"].as_array_mut().unwrap() { + *value = serde_json::json!(0.0); + } + let uncertain = RoomModel::import(&serde_json::to_vec(&json).unwrap()).unwrap(); + let mut report = uncertain.evaluate(&sixteen_declared_windows()).unwrap(); + assert_eq!( + ( + report.samples, + report.scored, + report.missing, + report.binary_cross_entropy_samples + ), + (16, 0, 0, 16) + ); + report.binary_cross_entropy = None; + report.binary_cross_entropy_samples = 0; + for part in &mut report.per_condition { + part.binary_cross_entropy = None; + part.binary_cross_entropy_samples = 0; + } + assert!( + report.validate().is_err(), + "nonmissing uncertain probabilities cannot disappear from BCE" + ); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/evaluation_store.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/evaluation_store.rs new file mode 100644 index 0000000000..86983854b7 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/evaluation_store.rs @@ -0,0 +1,275 @@ +//! Per-model, per-role immutable evaluation history; no activation authority. +use super::*; +use crate::room_learner::observation::{DataRole, ValidatedDataset}; +#[path = "evaluation_receipt.rs"] +mod receipt; +use receipt::data_sha256; +#[path = "evaluation_index.rs"] +mod index; +pub use receipt::EvaluationReceipt; +const SCHEMA: &str = "ruview.room-learner.evaluation-artifact.v1"; +const POINTER_SCHEMA_EVAL: &str = "ruview.room-learner.evaluation-pointer.v1"; +const LATEST: &str = "latest-v1.json"; +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct EvaluationEnvelope { + schema: String, + receipt_sha256: String, + receipt: EvaluationReceipt, +} +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct EvaluationPointer { + schema: String, + receipt_sha256: String, + model_sha256: String, + context_digest: String, + role: DataRole, +} +fn role_directory(role: DataRole) -> Result<&'static str, String> { + match role { + DataRole::Validation => Ok("validation"), + DataRole::HeldOut => Ok("held-out"), + _ => Err("evaluation only accepts validation or held-out data".into()), + } +} +pub(super) fn private_subdirectory( + parent: &Path, + name: &str, + create: bool, +) -> Result, String> { + let path = parent.join(name); + match std::fs::symlink_metadata(&path) { + Ok(m) if !m.is_dir() || m.file_type().is_symlink() => { + return Err("invalid private evaluation directory".into()) + } + Err(e) if e.kind() == io::ErrorKind::NotFound && !create => return Ok(None), + Err(e) if e.kind() != io::ErrorKind::NotFound => return Err(e.to_string()), + _ => {} + } + private_dir(&path).map_err(|e| e.to_string())?; + Ok(Some(path)) +} +pub(super) fn write_immutable(path: &Path, bytes: &[u8]) -> Result<(), String> { + if regular_or_missing(path)? { + return if private_read(path).map_err(|e| e.to_string())? == bytes { + Ok(()) + } else { + Err("immutable evaluation collision/corruption".into()) + }; + } + let directory = path.parent().ok_or("missing evaluation parent")?; + let staging = directory.join(format!("evaluation-stage-{}", crate::companion::secret())); + atomic_private_write(&staging, bytes).map_err(|e| e.to_string())?; + let installed = std::fs::hard_link(&staging, path); + let cleanup = std::fs::remove_file(&staging); + match installed { + Ok(()) => {} + Err(e) if e.kind() == io::ErrorKind::AlreadyExists => { + if private_read(path).map_err(|e| e.to_string())? != bytes { + return Err("immutable evaluation collision/corruption".into()); + } + } + Err(e) => return Err(e.to_string()), + } + cleanup.map_err(|e| e.to_string())?; + std::fs::File::open(directory) + .and_then(|f| f.sync_all()) + .map_err(|e| e.to_string()) +} +impl ArtifactStore { + fn evaluation_directory( + &self, + model: &RoomModel, + role: DataRole, + create: bool, + ) -> Result, String> { + let role_name = role_directory(role)?; + self.validate_context(model.context())?; + let Some(context) = self.context_dir(model.context(), false)? else { + return Err("persist the frozen candidate before evaluation".into()); + }; + let bytes = model.export()?; + let model_receipt = self.receipt_for(model, &bytes); + self.read_model(&context, &model_receipt, model.context())?; + let Some(evaluations) = private_subdirectory(&context, "evaluations", create)? else { + return Ok(None); + }; + let Some(model_dir) = + private_subdirectory(&evaluations, &model_receipt.model_sha256, create)? + else { + return Ok(None); + }; + private_subdirectory(&model_dir, role_name, create) + } + fn read_evaluation( + &self, + directory: &Path, + key: &str, + model: &RoomModel, + role: DataRole, + ) -> Result { + if !hex_digest(key) { + return Err("invalid evaluation digest".into()); + } + let bytes = + private_read(&directory.join(format!("{key}.json"))).map_err(|e| e.to_string())?; + let envelope: EvaluationEnvelope = + serde_json::from_slice(&bytes).map_err(|e| e.to_string())?; + if envelope.schema != SCHEMA + || envelope.receipt_sha256 != key + || envelope.receipt.sha256()? != key + { + return Err("evaluation integrity mismatch".into()); + } + envelope.receipt.validate(model, role)?; + Ok(envelope.receipt) + } + fn read_latest_evaluation( + &self, + directory: &Path, + model: &RoomModel, + role: DataRole, + ) -> Result, String> { + let path = directory.join(LATEST); + if !regular_or_missing(&path)? { + return Ok(None); + } + let pointer: EvaluationPointer = + serde_json::from_slice(&private_read(&path).map_err(|e| e.to_string())?) + .map_err(|e| e.to_string())?; + if pointer.schema != POINTER_SCHEMA_EVAL + || pointer.context_digest != model.context().digest() + || pointer.role != role + || pointer.model_sha256 != digest(&model.export()?) + || !hex_digest(&pointer.receipt_sha256) + { + return Err("evaluation pointer/candidate mismatch".into()); + } + let receipt = self.read_evaluation(directory, &pointer.receipt_sha256, model, role)?; + let indexed = self + .read_evidence_index(directory, &receipt.dataset_sha256, model, role)? + .ok_or("evaluation evidence index missing")?; + if indexed.sha256()? != pointer.receipt_sha256 { + return Err("evaluation latest pointer differs from original evidence".into()); + } + Ok(Some(receipt)) + } + /// Compute metrics from a validated role split, then persist. No supplied + /// report, physical-qualification claim or caller-controlled filename enters. + pub fn evaluate_candidate( + &self, + model: &RoomModel, + dataset: &ValidatedDataset, + role: DataRole, + ) -> Result { + let at = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_err(|_| "evaluation clock unavailable")? + .as_millis() + .try_into() + .map_err(|_| "evaluation clock outside bounds")?; + self.evaluate_candidate_at(model, dataset, role, at) + } + fn evaluate_candidate_at( + &self, + model: &RoomModel, + dataset: &ValidatedDataset, + role: DataRole, + at: u64, + ) -> Result { + role_directory(role)?; + if dataset.context() != model.context() { + return Err("evaluation dataset/candidate context mismatch".into()); + } + let train_checked = !dataset.train().is_empty(); + if train_checked + && data_sha256(dataset.train())? != model.training_manifest().training_data_sha256 + { + return Err("retained Train set differs from the frozen candidate".into()); + } + let rows = if role == DataRole::HeldOut { + dataset.held_out() + } else { + dataset.validation() + }; + if rows.is_empty() { + return Err("no cohorts for the selected evaluation role".into()); + } + let directory = self + .evaluation_directory(model, role, true)? + .ok_or("missing evaluation directory")?; + let previous = self.read_latest_evaluation(&directory, model, role)?; + let data_sha = data_sha256(rows)?; + if let Some(original) = self.read_evidence_index(&directory, &data_sha, model, role)? { + // Selecting old evidence can move latest, but never changes its + // immutable receipt identity, original timestamp or lineage proof. + if previous + .as_ref() + .map(EvaluationReceipt::sha256) + .transpose()? + .as_deref() + != Some(original.sha256()?.as_str()) + { + self.publish_evaluation_pointer(&directory, &original)?; + } + return Ok(original); + } + let receipt = EvaluationReceipt::new(model, rows, role, at, train_checked)?; + let key = receipt.sha256()?; + let envelope = EvaluationEnvelope { + schema: SCHEMA.into(), + receipt_sha256: key.clone(), + receipt: receipt.clone(), + }; + let bytes = encode(&envelope)?; + write_immutable(&directory.join(format!("{key}.json")), &bytes)?; + self.read_evaluation(&directory, &key, model, role)?; + self.save_evidence_index(&directory, &receipt, model, role)?; + self.publish_evaluation_pointer(&directory, &receipt)?; + Ok(receipt) + } + fn publish_evaluation_pointer( + &self, + directory: &Path, + receipt: &EvaluationReceipt, + ) -> Result<(), String> { + let pointer = EvaluationPointer { + schema: POINTER_SCHEMA_EVAL.into(), + receipt_sha256: receipt.sha256()?, + model_sha256: receipt.model_sha256.clone(), + context_digest: receipt.context_digest.clone(), + role: receipt.role, + }; + let bytes = encode(&pointer)?; + atomic_private_write(&directory.join(LATEST), &bytes).map_err(|e| e.to_string())?; + if private_read(&directory.join(LATEST)).map_err(|e| e.to_string())? != bytes { + return Err("evaluation pointer readback failed".into()); + } + Ok(()) + } + pub fn load_latest_evaluation( + &self, + model: &RoomModel, + role: DataRole, + ) -> Result, String> { + let Some(directory) = self.evaluation_directory(model, role, false)? else { + return Ok(None); + }; + self.read_latest_evaluation(&directory, model, role) + } +} + +#[cfg(test)] +#[path = "evaluation_store_tests.rs"] +mod tests; + +impl ArtifactStore { + pub(crate) fn load_exact_evaluation(&self, model: &RoomModel, role: DataRole, key: &str) -> Result { + let directory=self.evaluation_directory(model,role,false)?.ok_or("evaluation unavailable")?; + let receipt=self.read_evaluation(&directory,key,model,role)?; + let original=self.read_evidence_index(&directory,&receipt.dataset_sha256,model,role)?.ok_or("original evaluation index missing")?; + if original.sha256()?!=key {return Err("evaluation differs from original dataset index".into())} + Ok(receipt) + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/evaluation_store_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/evaluation_store_tests.rs new file mode 100644 index 0000000000..af96f0f2b9 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/evaluation_store_tests.rs @@ -0,0 +1,475 @@ +//! SYNTHETIC RF through real fitting, private storage and cold-load boundaries. +use super::*; +use crate::room_learner::{ + observation::CaptureCohort, + training::{learning_tests, FitConfig}, +}; +use std::sync::OnceLock; +const AT: u64 = 1_800_000_000_000; +fn dataset() -> ValidatedDataset { + let base = learning_tests::dataset(8); + let mut context = base.context().clone(); + context.owner_scope = "a".repeat(64); + let mut rows = base + .train() + .iter() + .chain(base.validation()) + .chain(base.held_out()) + .cloned() + .collect::>(); + for row in &mut rows { + row.context_digest = context.digest(); + } + ValidatedDataset::new(context, rows).unwrap() +} +fn model() -> RoomModel { + static MODEL: OnceLock = OnceLock::new(); + MODEL + .get_or_init(|| { + RoomModel::fit( + &dataset(), + FitConfig { + epochs: 2, + ..FitConfig::default() + }, + ) + .unwrap() + .0 + }) + .clone() +} +fn open_store(root: &Path) -> ArtifactStore { + ArtifactStore::new( + &root.join("a".repeat(64)).join("SensingServer"), + &"a".repeat(64), + "synthetic-install", + ) + .unwrap() +} +fn saved(root: &Path) -> (ArtifactStore, RoomModel) { + let store = open_store(root); + let model = model(); + store.save_candidate(&model).unwrap(); + (store, model) +} +fn receipt_path(store: &ArtifactStore, model: &RoomModel, r: &EvaluationReceipt) -> PathBuf { + store + .evaluation_directory(model, r.role, false) + .unwrap() + .unwrap() + .join(format!("{}.json", r.sha256().unwrap())) +} +#[test] +fn real_fit_evaluate_private_store_cold_load_preserves_bound_metadata_without_raw_rows() { + let temp = tempfile::tempdir().unwrap(); + let (store, model) = saved(temp.path()); + let ds = dataset(); + let before = model.export().unwrap(); + assert!(store + .load_latest_evaluation(&model, DataRole::HeldOut) + .unwrap() + .is_none()); + let report = store + .evaluate_candidate_at(&model, &ds, DataRole::HeldOut, AT) + .unwrap(); + assert_eq!(model.export().unwrap(), before); + assert_eq!(report.dataset_sha256, data_sha256(ds.held_out()).unwrap()); + assert_eq!(report.model_sha256, digest(&before)); + assert!(report.train_lineage_checked); + assert!( + !report.adoption_authorized + && !report.physically_validated + && !report.independence_verified + && !report.measurement_clock_available + ); + let cold = open_store(temp.path()); + let restored = cold.load_candidate(model.context()).unwrap().unwrap(); + assert_eq!( + cold.load_latest_evaluation(&restored, DataRole::HeldOut) + .unwrap(), + Some(report.clone()) + ); + let text = + String::from_utf8(private_read(&receipt_path(&cold, &restored, &report)).unwrap()).unwrap(); + for raw in [ + "frame_ids", + "sequence_ids", + "trial_id\"", + "views\"", + "samples\":[", + "amplitudes", + ] { + assert!(!text.contains(raw), "raw field {raw}"); + } + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + assert_eq!( + std::fs::metadata(receipt_path(&cold, &restored, &report)) + .unwrap() + .permissions() + .mode() + & 0o777, + 0o600 + ); + } +} +#[test] +fn repeated_receipt_identity_is_idempotent_and_new_receipts_retain_history() { + let temp = tempfile::tempdir().unwrap(); + let (store, model) = saved(temp.path()); + let ds = dataset(); + let first = store + .evaluate_candidate_at(&model, &ds, DataRole::Validation, AT) + .unwrap(); + let path = receipt_path(&store, &model, &first); + let bytes = private_read(&path).unwrap(); + #[cfg(unix)] + let inode = { + use std::os::unix::fs::MetadataExt; + std::fs::metadata(&path).unwrap().ino() + }; + let repeat = store + .evaluate_candidate_at(&model, &ds, DataRole::Validation, AT) + .unwrap(); + assert_eq!(first, repeat); + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + assert_eq!(std::fs::metadata(&path).unwrap().ino(), inode); + } + let unchanged = store + .evaluate_candidate_at(&model, &ds, DataRole::Validation, AT + 1) + .unwrap(); + assert_eq!( + unchanged, first, + "identical evidence must not renew its timestamp" + ); + let mut changed_rows = ds + .train() + .iter() + .chain(ds.validation()) + .chain(ds.held_out()) + .cloned() + .collect::>(); + let changed = changed_rows + .iter_mut() + .find(|r| r.role == DataRole::Validation) + .unwrap(); + changed.views[0].samples[0][0] += 0.01; + let changed = ValidatedDataset::new(ds.context().clone(), changed_rows).unwrap(); + let next = store + .evaluate_candidate_at(&model, &changed, DataRole::Validation, AT + 2) + .unwrap(); + assert_ne!(first.sha256().unwrap(), next.sha256().unwrap()); + assert_eq!(private_read(&path).unwrap(), bytes); + assert_eq!( + store + .load_latest_evaluation(&model, DataRole::Validation) + .unwrap(), + Some(next) + ); +} +#[test] +fn frozen_model_role_and_context_isolation_and_missing_retained_train_are_explicit() { + let temp = tempfile::tempdir().unwrap(); + let (store, model) = saved(temp.path()); + let ds = dataset(); + for role in [DataRole::Train, DataRole::Inference] { + assert!(store.evaluate_candidate_at(&model, &ds, role, AT).is_err()); + } + let mut rows = ds.train().to_vec(); + rows[0].views[0].samples[0][0] += 0.01; + rows.extend_from_slice(ds.held_out()); + let changed = ValidatedDataset::new(ds.context().clone(), rows).unwrap(); + assert!(store + .evaluate_candidate_at(&model, &changed, DataRole::HeldOut, AT) + .unwrap_err() + .contains("Train set")); + let cold_rows = ValidatedDataset::new(ds.context().clone(), ds.held_out().to_vec()).unwrap(); + let result = store + .evaluate_candidate_at(&model, &cold_rows, DataRole::HeldOut, AT) + .unwrap(); + assert!(!result.train_lineage_checked); + let second = RoomModel::fit( + &ds, + FitConfig { + epochs: 3, + ..FitConfig::default() + }, + ) + .unwrap() + .0; + store.save_candidate(&second).unwrap(); + assert!(store + .load_latest_evaluation(&second, DataRole::HeldOut) + .unwrap() + .is_none()); + assert_eq!( + store + .load_latest_evaluation(&model, DataRole::HeldOut) + .unwrap(), + Some(result) + ); + let wrong = ArtifactStore::new( + &temp.path().join("b".repeat(64)).join("SensingServer"), + &"b".repeat(64), + "synthetic-install", + ) + .unwrap(); + assert!(wrong + .evaluate_candidate_at(&model, &ds, DataRole::HeldOut, AT) + .is_err()); + assert!(wrong + .load_latest_evaluation(&model, DataRole::HeldOut) + .is_err()); + let unsaved = tempfile::tempdir().unwrap(); + assert!(open_store(unsaved.path()) + .evaluate_candidate_at(&model, &ds, DataRole::HeldOut, AT) + .is_err()); +} +#[test] +fn role_leakage_and_repeat_cohort_ids_cannot_reach_receipt_generation() { + let ds = dataset(); + let mut rows = ds.train().to_vec(); + let mut overlap = rows[0].clone(); + overlap.role = DataRole::HeldOut; + overlap.id = "renamed-eval".into(); + overlap.trial_id = "renamed-trial".into(); + overlap.session_id = "renamed-session".into(); + rows.push(overlap); + assert!(ValidatedDataset::new(ds.context().clone(), rows) + .unwrap_err() + .contains("frame overlap")); + let duplicate: Vec = vec![ds.held_out()[0].clone(), ds.held_out()[0].clone()]; + assert!(ValidatedDataset::new(ds.context().clone(), duplicate).is_err()); +} +#[test] +fn corrupt_receipt_or_pointer_is_retained_without_discarding_the_model_or_overwriting_history() { + for pointer in [true, false] { + let temp = tempfile::tempdir().unwrap(); + let (store, model) = saved(temp.path()); + let ds = dataset(); + let receipt = store + .evaluate_candidate_at(&model, &ds, DataRole::HeldOut, AT) + .unwrap(); + let directory = store + .evaluation_directory(&model, DataRole::HeldOut, false) + .unwrap() + .unwrap(); + let path = if pointer { + directory.join(LATEST) + } else { + receipt_path(&store, &model, &receipt) + }; + atomic_private_write(&path, b"corrupt evaluation").unwrap(); + assert!(store + .load_latest_evaluation(&model, DataRole::HeldOut) + .is_err()); + assert!(store + .evaluate_candidate_at(&model, &ds, DataRole::HeldOut, AT + 1) + .is_err()); + assert_eq!(private_read(&path).unwrap(), b"corrupt evaluation"); + assert_eq!( + store + .load_candidate(model.context()) + .unwrap() + .unwrap() + .export() + .unwrap(), + model.export().unwrap() + ); + } +} +#[test] +fn self_consistent_false_authority_or_wrong_model_role_metrics_are_rejected() { + let temp = tempfile::tempdir().unwrap(); + let (store, model) = saved(temp.path()); + let ds = dataset(); + let original = store + .evaluate_candidate_at(&model, &ds, DataRole::HeldOut, AT) + .unwrap(); + let directory = store + .evaluation_directory(&model, DataRole::HeldOut, false) + .unwrap() + .unwrap(); + type Mutant = fn(&mut EvaluationReceipt); + let mutants: [Mutant; 8] = [ + |r: &mut EvaluationReceipt| r.adoption_authorized = true, + |r: &mut EvaluationReceipt| r.physically_validated = true, + |r: &mut EvaluationReceipt| r.independence_verified = true, + |r: &mut EvaluationReceipt| r.report.correct = 999, + |r: &mut EvaluationReceipt| r.model_sha256 = "0".repeat(64), + |r: &mut EvaluationReceipt| r.role = DataRole::Train, + |r: &mut EvaluationReceipt| { + r.provenance = + vec![crate::room_learner::observation::CaptureProvenance::LiveReceiverObserved] + }, + |r: &mut EvaluationReceipt| r.training_data_sha256 = "0".repeat(64), + ]; + for mutate in mutants { + let mut bad = original.clone(); + mutate(&mut bad); + let key = bad.sha256().unwrap(); + let envelope = EvaluationEnvelope { + schema: SCHEMA.into(), + receipt_sha256: key.clone(), + receipt: bad, + }; + write_immutable( + &directory.join(format!("{key}.json")), + &encode(&envelope).unwrap(), + ) + .unwrap(); + let pointer = EvaluationPointer { + schema: POINTER_SCHEMA_EVAL.into(), + receipt_sha256: key, + model_sha256: original.model_sha256.clone(), + context_digest: original.context_digest.clone(), + role: DataRole::HeldOut, + }; + atomic_private_write(&directory.join(LATEST), &encode(&pointer).unwrap()).unwrap(); + assert!(store + .load_latest_evaluation(&model, DataRole::HeldOut) + .is_err()); + } +} +#[cfg(unix)] +#[test] +fn symlinks_and_insecure_files_never_replace_private_evidence_or_touch_victims() { + use std::os::unix::fs::{symlink, PermissionsExt}; + for target in ["receipt", "pointer", "role", "model", "evaluations"] { + let temp = tempfile::tempdir().unwrap(); + let (store, model) = saved(temp.path()); + let ds = dataset(); + let receipt = store + .evaluate_candidate_at(&model, &ds, DataRole::HeldOut, AT) + .unwrap(); + let directory = store + .evaluation_directory(&model, DataRole::HeldOut, false) + .unwrap() + .unwrap(); + let path = match target { + "receipt" => receipt_path(&store, &model, &receipt), + "pointer" => directory.join(LATEST), + "role" => directory.clone(), + "model" => directory.parent().unwrap().to_path_buf(), + _ => directory.parent().unwrap().parent().unwrap().to_path_buf(), + }; + std::fs::rename(&path, path.with_extension("preserved")).unwrap(); + let victim = temp.path().join("victim"); + atomic_private_write(&victim, b"victim bytes").unwrap(); + symlink(&victim, &path).unwrap(); + assert!(store + .load_latest_evaluation(&model, DataRole::HeldOut) + .is_err()); + assert!(store + .evaluate_candidate_at(&model, &ds, DataRole::HeldOut, AT + 1) + .is_err()); + assert_eq!(private_read(&victim).unwrap(), b"victim bytes"); + } + let temp = tempfile::tempdir().unwrap(); + let (store, model) = saved(temp.path()); + let ds = dataset(); + let receipt = store + .evaluate_candidate_at(&model, &ds, DataRole::HeldOut, AT) + .unwrap(); + let path = receipt_path(&store, &model, &receipt); + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o644)).unwrap(); + assert!(store + .load_latest_evaluation(&model, DataRole::HeldOut) + .is_err()); +} +#[test] +fn disk_obstacles_and_oversized_pointers_leave_frozen_model_and_receipts_untouched() { + let temp = tempfile::tempdir().unwrap(); + let (store, model) = saved(temp.path()); + let ds = dataset(); + let context = store.context_dir(model.context(), false).unwrap().unwrap(); + let obstacle = context.join("evaluations"); + atomic_private_write(&obstacle, b"existing bytes").unwrap(); + assert!(store + .evaluate_candidate_at(&model, &ds, DataRole::HeldOut, AT) + .is_err()); + assert_eq!(private_read(&obstacle).unwrap(), b"existing bytes"); + assert!(store.load_candidate(model.context()).unwrap().is_some()); + let temp = tempfile::tempdir().unwrap(); + let (store, model) = saved(temp.path()); + let receipt = store + .evaluate_candidate_at(&model, &ds, DataRole::HeldOut, AT) + .unwrap(); + let path = receipt_path(&store, &model, &receipt); + let before = private_read(&path).unwrap(); + let pointer = store + .evaluation_directory(&model, DataRole::HeldOut, false) + .unwrap() + .unwrap() + .join(LATEST); + let oversized = vec![b' '; LIMIT + 1]; + atomic_private_write(&pointer, &oversized).unwrap(); + assert!(store + .load_latest_evaluation(&model, DataRole::HeldOut) + .is_err()); + assert!(store + .evaluate_candidate_at(&model, &ds, DataRole::HeldOut, AT + 1) + .is_err()); + assert_eq!(private_read(&path).unwrap(), before); + assert_eq!(std::fs::read(pointer).unwrap(), oversized); +} + +#[test] +fn preexisting_immutable_collision_preserves_the_previous_pointer_and_all_evidence() { + let temp = tempfile::tempdir().unwrap(); + let (store, model) = saved(temp.path()); + let ds = dataset(); + let original = store + .evaluate_candidate_at(&model, &ds, DataRole::HeldOut, AT) + .unwrap(); + let directory = store + .evaluation_directory(&model, DataRole::HeldOut, false) + .unwrap() + .unwrap(); + let pointer_before = private_read(&directory.join(LATEST)).unwrap(); + let original_before = private_read(&receipt_path(&store, &model, &original)).unwrap(); + let mut rows = ds + .train() + .iter() + .chain(ds.validation()) + .chain(ds.held_out()) + .cloned() + .collect::>(); + rows.iter_mut() + .find(|r| r.role == DataRole::HeldOut) + .unwrap() + .views[0] + .samples[0][0] += 0.02; + let changed = ValidatedDataset::new(ds.context().clone(), rows).unwrap(); + let next = EvaluationReceipt::new(&model, changed.held_out(), DataRole::HeldOut, AT + 1, true) + .unwrap(); + let collision = directory.join(format!("{}.json", next.sha256().unwrap())); + atomic_private_write(&collision, b"existing immutable evidence").unwrap(); + assert!(store + .evaluate_candidate_at(&model, &changed, DataRole::HeldOut, AT + 1) + .is_err()); + assert_eq!( + private_read(&collision).unwrap(), + b"existing immutable evidence" + ); + assert_eq!( + private_read(&directory.join(LATEST)).unwrap(), + pointer_before + ); + assert_eq!( + private_read(&receipt_path(&store, &model, &original)).unwrap(), + original_before + ); + assert_eq!( + store + .load_latest_evaluation(&model, DataRole::HeldOut) + .unwrap(), + Some(original) + ); +} + +#[path = "evaluation_index_tests.rs"] +mod index_tests; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/fit_config.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/fit_config.rs new file mode 100644 index 0000000000..8adec18220 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/fit_config.rs @@ -0,0 +1,101 @@ +//! Bounded fit settings and actual architecture preflight. +use super::*; +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct FitConfig { + pub epochs: usize, + pub learning_rate: f32, + pub contrastive_weight: f32, + pub supervised_weight: f32, + pub gradient_epsilon: f32, + pub max_parameters: usize, + pub max_duration_ms: u64, + pub max_objective_evaluations: usize, + pub hidden_dim: usize, + pub embedding_dim: usize, + pub temporal_tokens: usize, + pub temperature: f32, +} +impl Default for FitConfig { + fn default() -> Self { + Self { + epochs: 48, + learning_rate: 0.2, + contrastive_weight: 0.1, + supervised_weight: 1.0, + gradient_epsilon: 0.002, + max_parameters: 512, + max_duration_ms: 60_000, + max_objective_evaluations: 100_000, + hidden_dim: 3, + embedding_dim: 3, + temporal_tokens: 4, + temperature: 0.3, + } + } +} +impl FitConfig { + /// Keep every advertised source count within the bounded optimizer budget. + pub fn for_source_count(source_count: usize) -> Self { + let mut config = Self::default(); + if source_count >= 7 { + config.hidden_dim = 2; + config.embedding_dim = 2; + } + config + } + pub fn validate_context(&self, context: &CaptureContext) -> Result { + self.validate()?; + context.validate()?; + let config = EncoderConfig { + source_count: context.sources.len(), + bins: context + .sources + .iter() + .map(|s| s.feature_count) + .max() + .unwrap_or(0), + hidden_dim: self.hidden_dim, + embedding_dim: self.embedding_dim, + temporal_tokens: self.temporal_tokens, + temperature: self.temperature, + }; + let count = initial_encoder(&config)? + .transformer + .flatten_weights() + .len() + + initial_projection_weights(&config) + .iter() + .map(Vec::len) + .sum::() + + context.sources.len() * (self.embedding_dim + 2) + + 1; + if count > self.max_parameters { + return Err("model exceeds parameter budget".into()); + } + Ok(count) + } + pub fn validate(&self) -> Result<(), String> { + if !(1..=128).contains(&self.epochs) + || !(1..=512).contains(&self.max_parameters) + || !(1..=120_000).contains(&self.max_duration_ms) + || !(1..=200_000).contains(&self.max_objective_evaluations) + || !(2..=4).contains(&self.hidden_dim) + || !(2..=4).contains(&self.embedding_dim) + || !(2..=16).contains(&self.temporal_tokens) + || !self.learning_rate.is_finite() + || !(0.0001..=1.0).contains(&self.learning_rate) + || !self.contrastive_weight.is_finite() + || !(0.001..=10.0).contains(&self.contrastive_weight) + || !self.supervised_weight.is_finite() + || !(0.001..=10.0).contains(&self.supervised_weight) + || !self.gradient_epsilon.is_finite() + || !(0.0001..=0.05).contains(&self.gradient_epsilon) + || !self.temperature.is_finite() + || !(0.1..=2.0).contains(&self.temperature) + { + return Err("invalid bounded training configuration".into()); + } + Ok(()) + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/learning_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/learning_tests.rs new file mode 100644 index 0000000000..7bde77a21f --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/learning_tests.rs @@ -0,0 +1,545 @@ +//! SYNTHETIC leakage-free RF fixtures: learning mechanics, not room accuracy. +use crate::room_learner::model::*; +use crate::room_learner::observation::*; +use crate::room_learner::training::*; +use ruview_unified::RfModality; +use std::sync::atomic::AtomicBool; + +fn context(width: usize) -> CaptureContext { + CaptureContext { + owner_scope: "synthetic-owner".into(), + installation_id: "synthetic-install".into(), + room_digest: "synthetic-room".into(), + revision: 1, + sources: (1..=2) + .map(|id| SourceSpec { + source_id: id, + device_id: format!("synthetic-device-{id}"), + grid_id: format!("phy-{width}-bins"), + feature_count: width, + link_index: 0, + center_frequency_hz: 2_400_000_000, + bandwidth_hz: 20_000_000, + signal_kind: RfModality::Synthetic, + link_id: format!("link-{id}"), + geometry: None, + }) + .collect(), + } +} +fn cohort(c: &CaptureContext, id: u64, label: RoomLabel, role: DataRole) -> CaptureCohort { + let views = c + .sources + .iter() + .map(|source| { + let shift = match label { + RoomLabel::Empty | RoomLabel::Unknown => 0.0, + RoomLabel::AdjacentOnly => { + if source.source_id == 1 { + 1.7 + } else { + -1.7 + } + } + RoomLabel::OccupiedStatic | RoomLabel::OccupiedMoving => 1.7, + }; + let variation = (id % 5) as f32 * 0.015; + let samples = (0..4) + .map(|t| { + (0..source.feature_count) + .map(|b| { + let motion = if label == RoomLabel::OccupiedMoving { + (t as f32 * 1.3).sin() * 0.12 + } else { + 0.0 + }; + 10.0 + b as f32 * 0.003 + + (t as f32 - 1.5) * 0.03 + + shift + + variation + + motion + }) + .collect() + }) + .collect(); + SourceView { + source_id: source.source_id, + device_id: source.device_id.clone(), + grid_id: source.grid_id.clone(), + signal_kind: source.signal_kind, + link_index: 0, + center_frequency_hz: 2_400_000_000, + bandwidth_hz: 20_000_000, + link_id: source.link_id.clone(), + geometry: None, + samples, + frame_ids: (0..4).map(|t| u128::from(id) * 10 + t + 1).collect(), + sequence_ids: vec![1, 2, 3, 4], + first_time_ns: id * 1000 + 1, + last_time_ns: id * 1000 + 101, + clock: ClockInfo { + kind: ClockKind::Unknown, + uncertainty_ns: u64::MAX, + }, + sample_age_ns: 0, + phase: PhaseAvailability::MagnitudeOnly, + provenance: CaptureProvenance::Synthetic, + } + }) + .collect(); + CaptureCohort { + id: format!("synthetic-{id}"), + context_digest: c.digest(), + trial_id: format!("trial-{id}"), + session_id: format!("session-{id}"), + role, + label, + views, + teacher: None, + provenance: CaptureProvenance::Synthetic, + } +} +pub(crate) fn dataset(width: usize) -> ValidatedDataset { + let c = context(width); + let labels = [ + RoomLabel::Empty, + RoomLabel::AdjacentOnly, + RoomLabel::OccupiedStatic, + RoomLabel::OccupiedMoving, + ]; + let mut rows = Vec::new(); + for (i, label) in labels.iter().cycle().take(8).enumerate() { + rows.push(cohort(&c, i as u64 + 1, *label, DataRole::Train)); + } + for (i, label) in labels.iter().enumerate() { + rows.push(cohort(&c, 100 + i as u64, *label, DataRole::Validation)); + rows.push(cohort(&c, 200 + i as u64, *label, DataRole::HeldOut)); + } + ValidatedDataset::new(c, rows).unwrap() +} +fn config() -> FitConfig { + FitConfig { + epochs: 32, + ..FitConfig::default() + } +} + +#[test] +fn prepared_training_rows_match_uncached_representation_for_parameter_probes() { + use super::{represent_row, Row}; + for width in [8, 56, 128] { + let ds = dataset(width); + let (model, _) = RoomModel::fit( + &ds, + FitConfig { + epochs: 1, + hidden_dim: if width == 128 { 2 } else { 3 }, + embedding_dim: if width == 128 { 2 } else { 3 }, + ..config() + }, + ) + .unwrap(); + let parameters = model.parameters(); + for cohort in ds.train() { + let tokens = model.tokens(cohort, false).unwrap(); + let prepared = Row::prepare(&model, tokens.clone(), 0.0, true).unwrap(); + // Include the unchanged parameters and every positive/negative + // finite-difference probe, across backbone, projectors and head. + for probe in 0..=parameters.len() * 2 { + let mut params = parameters.clone(); + if probe > 0 { + let index = (probe - 1) / 2; + params[index] += if probe % 2 == 1 { 0.002 } else { -0.002 }; + } + let expected = model.represent(&tokens, ¶ms).unwrap(); + let mut encoder = extractor(&model.config); + encoder + .transformer + .unflatten_weights(¶ms[..model.encoder_weights.len()]) + .unwrap(); + let projectors = model.source_projectors(¶ms).unwrap(); + let actual = represent_row(&model, &prepared, &mut encoder, &projectors).unwrap(); + let bits = |values: &[f32]| values.iter().map(|x| x.to_bits()).collect::>(); + assert_eq!( + bits(&actual.0), + bits(&expected.0), + "width={width} probe={probe}" + ); + assert_eq!( + actual.1.iter().map(|x| bits(x)).collect::>(), + expected.1.iter().map(|x| bits(x)).collect::>(), + "width={width} probe={probe}" + ); + } + } + } +} + +#[test] +fn prepared_training_rows_reject_wrong_source_time_width_and_nonfinite_values() { + use super::Row; + let ds = dataset(8); + let (model, _) = RoomModel::fit( + &ds, + FitConfig { + epochs: 1, + ..config() + }, + ) + .unwrap(); + let valid = model.tokens(&ds.train()[0], false).unwrap(); + let mut wrong_source = valid.clone(); + wrong_source.pop(); + let mut wrong_time = valid.clone(); + wrong_time[0].pop(); + let mut wrong_width = valid.clone(); + wrong_width[0][0].pop(); + let mut nonfinite = valid.clone(); + nonfinite[0][0][0] = f32::NAN; + for invalid in [wrong_source, wrong_time, wrong_width, nonfinite] { + assert!(Row::prepare(&model, invalid, 0.0, true).is_err()); + } +} + +#[test] +fn actual_encoder_and_head_weights_reduce_declared_objective_and_learn_controls() { + let ds = dataset(8); + let (model, report) = RoomModel::fit(&ds, config()).unwrap(); + eprintln!("SYNTHETIC report: {report:?}"); + assert!(report.completed_epochs > 0); + assert!( + report.encoder_delta_l2 > 1.0e-5, + "zero-gradient/skipped-encoder mutant survived" + ); + assert!( + report.head_delta_l2 > 1.0e-3, + "skipped task update mutant survived" + ); + assert!(report.final_objective.total < report.initial.total * 0.8); + assert!(report.final_objective.supervised < report.initial.supervised * 0.5); + let held = model.evaluate(ds.held_out()).unwrap(); + assert_eq!( + held.correct, 4, + "SYNTHETIC disjoint occupancy/adjacent controls: {held:?}" + ); + assert_eq!(held.abstained, 0); + assert!(!model.infer(&ds.held_out()[0]).physically_validated); +} +#[test] +fn held_out_and_validation_changes_cannot_update_weights_or_training_profiles() { + let ds = dataset(8); + let mut rows = ds.train().to_vec(); + for c in ds.validation().iter().chain(ds.held_out()) { + let mut other = c.clone(); + other.label = RoomLabel::OccupiedMoving; + for v in &mut other.views { + for frame in &mut v.samples { + for value in frame { + *value += 0.05; + } + } + } + rows.push(other); + } + let changed = ValidatedDataset::new(ds.context().clone(), rows).unwrap(); + let cfg = FitConfig { + epochs: 8, + ..config() + }; + let (a, ar) = RoomModel::fit(&ds, cfg.clone()).unwrap(); + let (b, br) = RoomModel::fit(&changed, cfg).unwrap(); + assert_eq!(a.parameters(), b.parameters()); + assert_eq!(a.profiles(), b.profiles()); + assert_eq!(a.training_manifest(), b.training_manifest()); + assert_eq!(a.export().unwrap(), b.export().unwrap()); + assert_eq!(ar.final_objective.total, br.final_objective.total); +} +#[test] +fn source_order_dc_and_full_56_bin_grid_reach_real_aether_encoder() { + let ds = dataset(56); + let (model, report) = RoomModel::fit( + &ds, + FitConfig { + epochs: 2, + ..config() + }, + ) + .unwrap(); + assert_eq!(model.encoder_config().bins, 56); + assert!(report.parameter_count <= 512); + assert!(report.encoder_delta_l2 > 0.0); + let row = &ds.held_out()[1]; + let tokens = model.tokens(row, false).unwrap(); + assert_eq!(tokens[0][0].len(), 58); + assert_eq!(&tokens[0][0][56..], &[1.0, 0.0]); + assert_eq!(&tokens[1][0][56..], &[0.0, 1.0]); + // Individual native frequency bins survive; no fixed8-bin summary is used. + let mut changed = row.clone(); + changed.views[0].samples[0][55] += 0.05; + let changed_tokens = model.tokens(&changed, false).unwrap(); + assert_eq!(tokens[0][0][54], changed_tokens[0][0][54]); + assert_ne!(tokens[0][0][55], changed_tokens[0][0][55]); + let (_, a) = model.represent(&tokens, &model.parameters()).unwrap(); + let (_, b) = model + .represent(&changed_tokens, &model.parameters()) + .unwrap(); + let backbone = |input: &[Vec]| { + let mut encoder = extractor(model.encoder_config()); + encoder + .transformer + .unflatten_weights(&model.encoder_weights) + .unwrap(); + let mut input = input.to_vec(); + for row in &mut input { + for x in &mut row[..56] { + *x /= model.profiles()[0].max_normalized; + } + } + encoder.transformer.embed(&input) + }; + assert_ne!( + backbone(&tokens[0]), + backbone(&changed_tokens[0]), + "canonical bin must reach shared RF backbone" + ); + assert_eq!(a[1], b[1], "source0 cannot erase distinct source1 encoding"); + let empty = model.tokens(&ds.held_out()[0], false).unwrap(); + let occupied = model.tokens(&ds.held_out()[2], false).unwrap(); + assert!( + occupied[0][0][0] - empty[0][0][0] > 1.0, + "per-window normalization erased DC" + ); +} +#[test] +fn contrastive_objective_updates_encoder_and_is_not_a_logged_task_loss() { + let ds = dataset(8); + let (contrast, cr) = RoomModel::fit( + &ds, + FitConfig { + epochs: 16, + contrastive_weight: 5.0, + supervised_weight: 0.001, + ..config() + }, + ) + .unwrap(); + let (task, _) = RoomModel::fit( + &ds, + FitConfig { + epochs: 16, + contrastive_weight: 0.001, + ..config() + }, + ) + .unwrap(); + eprintln!( + "SYNTHETIC contrastive objective {} -> {}", + cr.initial.contrastive, cr.final_objective.contrastive + ); + assert!(cr.final_objective.contrastive < cr.initial.contrastive - 0.001); + assert!(cr.encoder_delta_l2 > 1.0e-4); + assert_ne!(contrast.encoder_weights, task.encoder_weights); +} +#[test] +fn model_round_trip_has_identical_predictions_and_no_observation_payload() { + let ds = dataset(8); + let (model, _) = RoomModel::fit( + &ds, + FitConfig { + epochs: 3, + ..config() + }, + ) + .unwrap(); + let bytes = model.export().unwrap(); + let json = String::from_utf8(bytes.clone()).unwrap(); + for raw in [ + "samples", + "frame_ids", + "sequence_ids", + "trial_id", + "synthetic-200", + ] { + assert!(!json.contains(raw)); + } + let imported = RoomModel::import(&bytes).unwrap(); + assert_eq!(model.parameters(), imported.parameters()); + assert_eq!( + model.infer(&ds.held_out()[2]).occupied_probability, + imported.infer(&ds.held_out()[2]).occupied_probability + ); + let original: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + for mutant in [ + "version", + "encoder_weights", + "head_weights", + "profiles", + "config", + "algorithm", + ] { + let mut value = original.clone(); + value[mutant] = serde_json::json!(null); + assert!( + RoomModel::import(&serde_json::to_vec(&value).unwrap()).is_err(), + "accepted malformed {mutant}" + ); + } + let mut invalid = model.clone(); + invalid.encoder_weights[0] = f32::NAN; + assert!(invalid.export().is_err()); + assert!(RoomModel::import(&vec![b' '; 1_048_577]).is_err()); +} +#[test] +fn missing_changed_stale_nonfinite_and_out_of_support_sources_abstain() { + let ds = dataset(8); + let (model, _) = RoomModel::fit( + &ds, + FitConfig { + epochs: 2, + ..config() + }, + ) + .unwrap(); + type Mutant = fn(&mut CaptureCohort); + let mutants: &[Mutant] = &[ + |c| { + c.views.pop(); + }, + |c| c.context_digest = "different-installation".into(), + |c| c.views[0].device_id = "different-device".into(), + |c| c.views[0].grid_id = "other-phy".into(), + |c| c.views[0].samples[0][0] = f32::NAN, + |c| c.views[0].samples[0][0] = 1.0e8, + |c| c.views[0].sample_age_ns = MAX_AGE_NS + 1, + ]; + for mutate in mutants { + let mut row = ds.held_out()[0].clone(); + mutate(&mut row); + let prediction = model.infer(&row); + assert_eq!(prediction.decision, PresenceDecision::Abstain); + assert!(prediction.occupied_probability.is_none()); + } +} +#[test] +fn parameter_time_evaluation_budgets_and_cancellation_fail_closed() { + let ds = dataset(8); + assert!(RoomModel::fit( + &ds, + FitConfig { + max_parameters: 1, + ..config() + } + ) + .unwrap_err() + .contains("parameter budget")); + assert!(RoomModel::fit( + &ds, + FitConfig { + max_objective_evaluations: 2, + ..config() + } + ) + .unwrap_err() + .contains("evaluation budget")); + assert!(RoomModel::fit( + &ds, + FitConfig { + max_duration_ms: 1, + ..config() + } + ) + .unwrap_err() + .contains("time budget")); + assert!( + RoomModel::fit_with_cancel(&ds, config(), &AtomicBool::new(true)) + .unwrap_err() + .contains("cancelled") + ); + for cfg in [ + FitConfig { + learning_rate: f32::NAN, + ..config() + }, + FitConfig { + epochs: 129, + ..config() + }, + FitConfig { + hidden_dim: 5000, + ..config() + }, + ] { + assert!(RoomModel::fit(&ds, cfg).is_err()); + } +} + +#[test] +fn sibling_trials_shared_frames_and_overlapping_windows_never_become_negatives() { + let ds = dataset(8); + let mut rows = ds.train().to_vec(); + rows[1].trial_id = rows[0].trial_id.clone(); + for i in 0..2 { + rows[2].views[i].first_time_ns = rows[0].views[i].first_time_ns; + rows[2].views[i].last_time_ns = rows[0].views[i].last_time_ns; + rows[3].views[i].frame_ids[0] = rows[0].views[i].frame_ids[0]; + } + let ds = ValidatedDataset::new(ds.context().clone(), rows).unwrap(); + let (_, report) = RoomModel::fit( + &ds, + FitConfig { + epochs: 1, + ..config() + }, + ) + .unwrap(); + assert_eq!(report.training_cohorts, 8); + assert_eq!(report.contrastive_trials, 5); +} +#[test] +fn running_fit_observes_cancellation_without_publishing_partial_model() { + use std::{ + sync::{atomic::Ordering, Arc}, + thread, + time::Duration, + }; + let ds = dataset(8); + let cancel = Arc::new(AtomicBool::new(false)); + let trigger = cancel.clone(); + let handle = thread::spawn(move || { + thread::sleep(Duration::from_millis(2)); + trigger.store(true, Ordering::SeqCst); + }); + let result = RoomModel::fit_with_cancel( + &ds, + FitConfig { + epochs: 128, + ..config() + }, + &cancel, + ); + handle.join().unwrap(); + assert!(result.unwrap_err().contains("cancelled")); +} + +#[test] +fn inference_does_not_invent_empty_ground_truth_or_enter_evaluation() { + let ds = dataset(8); + let (model, _) = RoomModel::fit( + &ds, + FitConfig { + epochs: 2, + ..config() + }, + ) + .unwrap(); + let mut live = ds.held_out()[2].clone(); + let expected = model.infer(&live).occupied_probability; + live.role = DataRole::Inference; + live.label = RoomLabel::Unknown; + assert_eq!(model.infer(&live).occupied_probability, expected); + assert!(model + .evaluate(&[live.clone()]) + .unwrap_err() + .contains("labeled")); + assert!(ValidatedDataset::new(ds.context().clone(), vec![live]).is_err()); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle.rs new file mode 100644 index 0000000000..2b47d522eb --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle.rs @@ -0,0 +1,587 @@ +//! Managed capture -> genuine learning -> private cold-loadable candidate. +//! Candidate predictions never grant physical qualification or replace control. +use super::{ + artifact::{ArtifactStore, CandidateReceipt}, + capture::{CaptureSession, PhaseReceipt, PhaseRequest, RawSourceFrame}, + model::{RoomModel, RoomPrediction}, + observation::{CaptureContext, DataRole, RoomLabel}, + training::{FitConfig, TrainingReport}, +}; +use super::temporal_model_input::managed::execution::DeferredReclamation; +use serde::{Deserialize, Serialize}; +use serde_json::{json, Value}; +use std::{ + path::Path, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, Mutex, + }, +}; + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum JobPhase { + Idle, + Running, + Cancelling, + Ready, + Failed, + Cancelled, +} +struct Job { + phase: JobPhase, + cancel: Arc, + report: Option, + error: Option, +} +impl Default for Job { + fn default() -> Self { + Self { + phase: JobPhase::Idle, + cancel: Arc::new(AtomicBool::new(false)), + report: None, + error: None, + } + } +} +/// Exactly one capture implementation. No public configure path constructs Temporal. +enum CaptureWorkflow { + Legacy(CaptureSession), + Temporal(Box), +} +struct Inner { + generation: u64, + workflow_revision: u64, + capture: Option, + model: Option>, + receipt: Option, + frozen_training: bool, + job: Job, + last_evaluation: Option, + evaluation_warning: Option, + input_error: Option, + recovery: RecoveryState, + recovery_enable: Option, + selection: selection_control::SelectionState, + qualification: Option, + qualification_error: Option, +} +impl Inner { + fn legacy_capture(&self) -> Option<&CaptureSession> { + match self.capture.as_ref() { Some(CaptureWorkflow::Legacy(c)) => Some(c), _ => None } + } + fn legacy_capture_mut(&mut self) -> Option<&mut CaptureSession> { + match self.capture.as_mut() { Some(CaptureWorkflow::Legacy(c)) => Some(c), _ => None } + } + fn capture_context(&self) -> Option<&CaptureContext> { + self.capture.as_ref().map(|capture| match capture { + CaptureWorkflow::Legacy(c) => c.context(), + CaptureWorkflow::Temporal(c) => c.context(), + }) + } + fn require_legacy_kind(&self) -> Result<(), String> { + if matches!(self.capture, Some(CaptureWorkflow::Temporal(_)))||self.selection.temporal.is_some() { + Err("learner_temporal_capture_private_unsupported_command".into()) + } else { Ok(()) } + } +} +pub struct RoomLearningRuntime { + runtime_epoch: String, + owner: String, + installation: String, + store: Arc, + recovery_store: Option>, + operator_recovery_store: Option>, + operator_profile_selected: AtomicBool, + operator_profile_initialized: AtomicBool, + status_reader: Arc, + inner: Mutex, + #[cfg(test)] + reclamation_probe:Mutex>>, + #[cfg(test)] + panic_after_reclamation_ack:AtomicBool, + #[cfg(test)] + temporal_fit_hook: Mutex>>, + #[cfg(test)] + temporal_status_profile_hook: Mutex>>, + #[cfg(test)] + temporal_evidence_io_hook: MutexResult<(),String> + Send + Sync>>>, + clock_origin: std::time::Instant, + clock_epoch_ns: u64, +} +impl RoomLearningRuntime { + fn deferred_reclamation(&self)->DeferredReclamation{ + let retired=DeferredReclamation::default(); + #[cfg(test)] let retired={let mut retired=retired;retired.probe=self.reclamation_probe.lock().unwrap_or_else(|e|e.into_inner()).clone();retired}; + retired + } +} +fn busy(job: &Job) -> bool { + matches!(job.phase, JobPhase::Running | JobPhase::Cancelling) +} +fn bounded(error: String) -> String { + error + .chars() + .filter(|c| !c.is_control()) + .take(256) + .collect() +} +impl RoomLearningRuntime { + pub fn new(directory: &Path, owner: &str, installation: &str) -> Result { + Self::build(directory, owner, installation, None) + } + pub fn new_with_recovery( + directory: &Path, + owner: &str, + installation: &str, + recovery: super::recovery_store::RecoveryStore, + ) -> Result { + Self::build(directory, owner, installation, Some(recovery)) + } + pub fn new_with_recovery_unavailable( + directory: &Path, + owner: &str, + installation: &str, + reason: String, + ) -> Result { + let runtime = Self::new(directory, owner, installation)?; + { + let mut state = runtime + .inner + .lock() + .map_err(|_| "learner state unavailable")?; + state.recovery.supported = true; + state.recovery.error = Some(bounded(reason)); + } + Ok(runtime) + } + /// Explicit time domain for deterministic fixtures; production uses the managed clock. + pub fn new_with_recovery_at( + directory: &Path, + owner: &str, + installation: &str, + recovery: super::recovery_store::RecoveryStore, + now_ns: u64, + ) -> Result { + if now_ns == 0 { + return Err("recovery_clock_invalid".into()); + } + let mut runtime = Self::build(directory, owner, installation, Some(recovery))?; + runtime.clock_epoch_ns = now_ns; + runtime.clock_origin = std::time::Instant::now(); + { + let mut state = runtime + .inner + .lock() + .map_err(|_| "learner state unavailable")?; + state.recovery = RecoveryState::default(); + runtime.inspect_recovery(&mut state, now_ns); + } + Ok(runtime) + } + fn build( + directory: &Path, + owner: &str, + installation: &str, + recovery_store: Option, + ) -> Result { + let clock_origin = std::time::Instant::now(); + let clock_epoch_ns = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_err(|_| "receiver clock unavailable")? + .as_nanos() + .try_into() + .map_err(|_| "receiver clock outside bounds")?; + let operator_recovery_store = recovery_store.as_ref().map(|store| store.operator_eight_scope()).transpose()?.map(Arc::new); + let runtime = Self { + clock_origin, + clock_epoch_ns, + runtime_epoch: crate::companion::secret(), + owner: owner.into(), + installation: installation.into(), + store: Arc::new(ArtifactStore::new(directory, owner, installation)?), + recovery_store: recovery_store.map(std::sync::Arc::new), + operator_recovery_store, + operator_profile_selected: AtomicBool::new(false), + operator_profile_initialized: AtomicBool::new(false), + status_reader: Arc::new(tokio::sync::Semaphore::new(1)), + #[cfg(test)] reclamation_probe:Mutex::new(None), + #[cfg(test)] panic_after_reclamation_ack:AtomicBool::new(false), + #[cfg(test)] temporal_fit_hook:Mutex::new(None), + #[cfg(test)] temporal_status_profile_hook:Mutex::new(None), + #[cfg(test)] temporal_evidence_io_hook:Mutex::new(None), + inner: Mutex::new(Inner { + generation: 0, + workflow_revision: 0, + capture: None, + model: None, + receipt: None, + frozen_training: false, + job: Job::default(), + last_evaluation: None, + evaluation_warning: None, + input_error: None, + recovery: RecoveryState::default(), + recovery_enable: None, + selection: selection_control::SelectionState::default(), + qualification: None, + qualification_error: None, + }), + }; + let temporal_selected=runtime.store.load_temporal_selection().and_then(|stored|stored.map(temporal_control::selected_control::TemporalSelectedRuntime::new).transpose()); + { + let mut state = runtime + .inner + .lock() + .map_err(|_| "learner state unavailable")?; + match temporal_selected{Ok(Some(selected))=>{state.selection.temporal=Some(Arc::new(selected));},Ok(None)=>runtime.restore_selection(&mut state),Err(error)=>state.selection.error=Some(bounded(error))} + runtime.inspect_recovery(&mut state, runtime.receiver_now_ns()); + } + Ok(runtime) + } + /// UTC-anchored monotonic receiver time. Wall-clock adjustments cannot + /// shorten an operator capture or turn a countdown into labelled RF data. + pub fn receiver_now_ns(&self) -> u64 { + self.clock_epoch_ns + .saturating_add(self.clock_origin.elapsed().as_nanos().min(u64::MAX as u128) as u64) + } + pub fn saved_candidates( + &self, + room_digest: &str, + ) -> Result { + self.inner.lock().map_err(|_| "learner state unavailable")?.require_legacy_kind()?; + self.store.catalog_for_room(room_digest) + } + /// Explicit new capture versus resuming an exact persisted candidate. Neither + /// path deletes old models, room receipts, pairing or artifact history. + pub fn configure( + &self, + expected: &MutationGuard, + context: CaptureContext, + run_id: String, + new_training: bool, + ) -> Result<(), String> { + self.configure_for_recovery( + expected, + context, + run_id, + new_training, + false, + self.receiver_now_ns(), + ) + } + pub fn configure_for_recovery( + &self, + expected: &MutationGuard, + context: CaptureContext, + run_id: String, + new_training: bool, + recovery_consent: bool, + now_ns: u64, + ) -> Result<(), String> { + context.validate()?; + if context.owner_scope != self.owner || context.installation_id != self.installation { + return Err("learner context does not belong to this managed profile".into()); + } + FitConfig::for_source_count(context.sources.len()).validate_context(&context)?; + let mut session = CaptureSession::new(context.clone(), run_id)?; + let mut state = self.command_state(expected)?; + state.require_legacy_kind()?; + if busy(&state.job) || state.legacy_capture().is_some_and(|c| c.phase_active()) { + return Err( + "finish/cancel the active capture or training job before changing context".into(), + ); + } + let restored = self.store.load_candidate(&context)?; + if !new_training && restored.is_none() { + return Err( + "no stored candidate for this exact context; start a new training capture".into(), + ); + } + if !new_training { + if let Some(model)=restored.as_ref() { + if let Some(proof)=self.store.load_managed_training_proof(model)? { + let fence=self.store.load_managed_capture_fence(model,&proof)?; + session=CaptureSession::resume_managed_fence(&proof,&fence)?; + } + } + } + let (restored_evaluation, evaluation_warning) = if !new_training { + restored + .as_ref() + .map(|model| self.restored_evaluation(model)) + .unwrap_or((None, None)) + } else { + (None, None) + }; + if recovery_consent { + self.prepare_recovery(&mut state, &context, now_ns)?; + } else { + state.recovery.enabled = false; + state.recovery.marker = None; + } + state.generation = state + .generation + .checked_add(1) + .ok_or("learner generation exhausted")?; + state.recovery_enable = if new_training && !recovery_consent { + recovery_enable::CurrentNewRun::new(&session, now_ns) + } else { None }; + state.capture = Some(CaptureWorkflow::Legacy(session)); + state.model = if new_training { + None + } else { + restored.map(Arc::new) + }; + state.frozen_training = !new_training; + state.receipt = None; + state.job = Job::default(); + if state.model.is_some() { + state.job.phase = JobPhase::Ready; + } + state.last_evaluation = restored_evaluation; + state.evaluation_warning = evaluation_warning; + state.input_error = None; + state.qualification = None; + state.qualification_error = None; + if let Some(model)=state.model.clone() { + match self.store.load_latest_qualification(&model) { + Ok(q)=>{ + state.qualification=q; + match self.store.load_latest_assessment(&model) { + Ok(Some(a))=>state.qualification_error=a.receipt.failure_reason, + Ok(None)=>{},Err(e)=>state.qualification_error=Some(bounded(e)), + } + }, + Err(e)=>state.qualification_error=Some(bounded(e)), + } + } + self.checkpoint_recovery(&mut state, now_ns, true)?; + Ok(()) + } + /// Called from an explicit protected action. Completion commits only a + /// candidate, with cancellation and context generation checked under lock. + pub fn start_training( + self: &Arc, + expected: &MutationGuard, + config: FitConfig, + ) -> Result { + config.validate()?; + let (generation, dataset, proof_seed, cancel) = { + let mut state = self.command_state(expected)?; + state.require_legacy_kind()?; + self.require_recovery_ready(&state)?; + if busy(&state.job) + || (state.frozen_training + && !matches!(state.job.phase, JobPhase::Failed | JobPhase::Cancelled)) + { + return Err("this run is already training or frozen".into()); + } + let capture = state.legacy_capture() + .ok_or("no learner capture configured")?; + config.validate_context(capture.context())?; + if capture.phase_active() { + return Err("finish or cancel the capture phase first".into()); + } + let dataset = capture.validated_dataset()?; + for label in [ + RoomLabel::Empty, + RoomLabel::OccupiedStatic, + RoomLabel::OccupiedMoving, + RoomLabel::AdjacentOnly, + ] { + if !dataset.train().iter().any(|c| c.label == label) { + return Err("capture Train empty, occupied-static, occupied-moving and adjacent-only trials before training".into()); + } + } + let proof_seed = capture.managed_training_seed()?; + let cancel = Arc::new(AtomicBool::new(false)); + state.job = Job { + phase: JobPhase::Running, + cancel: cancel.clone(), + report: None, + error: None, + }; + state.frozen_training = true; + let now = self.checkpoint_now(&state); + if let Err(error) = self.checkpoint_recovery(&mut state, now, true) { + state.job.phase = JobPhase::Failed; + state.job.error = Some(bounded(error.clone())); + return Err(error); + } + (state.generation, dataset, proof_seed, cancel) + }; + let runtime = self.clone(); + tokio::task::spawn_blocking(move || { + let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + RoomModel::fit_with_cancel(&dataset, config, &cancel) + })) + .unwrap_or_else(|_| Err("learner worker failed safely".into())); + let Ok(mut state) = runtime.inner.lock() else { + return; + }; + if state.generation != generation { + return; + } + state.workflow_revision = state.workflow_revision.saturating_add(1); + if cancel.load(Ordering::Acquire) { + state.job.phase = JobPhase::Cancelled; + state.job.error = + Some("training cancelled; previous stored candidates preserved".into()); + let now = runtime.checkpoint_now(&state); + let _ = runtime.checkpoint_recovery(&mut state, now, true); + return; + } + match outcome { + Ok((model, report)) => match runtime.store.save_managed_candidate(&model, proof_seed) { + Ok(receipt) => { + state.model = Some(Arc::new(model)); + state.receipt = Some(receipt); + state.job.report = Some(report); + state.job.phase = JobPhase::Ready; + } + Err(error) => { + state.job.phase = JobPhase::Failed; + state.job.error = Some(bounded(error)); + } + }, + Err(error) => { + state.job.phase = JobPhase::Failed; + state.job.error = Some(bounded(error)); + } + } + let now = runtime.checkpoint_now(&state); + let _ = runtime.checkpoint_recovery(&mut state, now, true); + }); + Ok(generation) + } + pub fn prediction(&self, now_ns: u64) -> Result<(MutationGuard, RoomPrediction), String> { + let state = self.inner.lock().map_err(|_| "learner state unavailable")?; + state.require_legacy_kind()?; + let model = state + .model + .as_ref() + .ok_or("no trained/restored candidate for this context")?; + let cohort = state.legacy_capture() + .ok_or("no current capture context")? + .live_cohort(now_ns)?; + Ok((self.guard_for(&state), model.infer(&cohort))) + } + pub fn status(&self,now_ns:u64)->Value{self.status_with_checkpoint(now_ns,true)} + pub(super) fn status_without_checkpoint(&self,now_ns:u64)->Value{self.status_with_checkpoint(now_ns,false)} + fn status_with_checkpoint(&self,now_ns:u64,checkpoint:bool)->Value{ + let Ok(mut state) = self.inner.lock() else { + return json!({"enabled":true,"state":"unavailable"}); + }; + if state.require_legacy_kind().is_err() { + return json!({"schema":"ruview.room-learner.status.v1","enabled":true,"success":false,"error_code":"learner_temporal_capture_private_unsupported_command"}); + } + if checkpoint{let _ = self.checkpoint_recovery(&mut state, now_ns, false);} + let required_ids: Vec<_> = state.legacy_capture() + .map(|c| c.context().sources.iter().map(|s| s.source_id).collect()) + .unwrap_or_default(); + let trained_ids: Vec<_> = state + .model + .as_ref() + .map(|m| m.context().sources.iter().map(|s| s.source_id).collect()) + .unwrap_or_default(); + let prediction = state.model.as_ref().and_then(|m| { + state.legacy_capture() + .and_then(|c| c.live_cohort(now_ns).ok()) + .map(|c| m.infer(&c)) + }); + let scored_ids = if prediction + .as_ref() + .is_some_and(|p| p.occupied_probability.is_some()) + { + trained_ids.clone() + } else { + Vec::new() + }; + json!({"schema":"ruview.room-learner.status.v1","enabled":true,"mutation_guard":self.guard_for(&state), + "required_source_ids":required_ids,"trained_source_ids":trained_ids,"scored_source_ids":scored_ids, + "candidate_prediction":prediction,"training_manifest":state.model.as_ref().map(|m|m.training_manifest()), + "phase_history":state.legacy_capture().map(|c|c.history()), + "phase_duration_defaults":{"Empty":600000,"OccupiedStatic":12000,"OccupiedMoving":12000,"AdjacentOnly":12000}, + "held_out_phase_duration_defaults":{"Empty":12000,"OccupiedStatic":12000,"OccupiedMoving":12000,"AdjacentOnly":12000}, + "phase_lead_in_defaults":{"Empty":15000,"OccupiedStatic":5000,"OccupiedMoving":5000,"AdjacentOnly":15000}, + "untrained_capture_progress_is_volatile":!state.recovery.enabled || state.recovery.error.is_some(),"recovery":self.recovery_status(&state, now_ns), + "generation":state.generation,"state":state.job.phase,"context_digest":state.legacy_capture().map(|c|c.context().digest()), + "room_digest":state.legacy_capture().map(|c| &c.context().room_digest), + "revision":state.legacy_capture().map(|c|c.context().revision), + "sources":state.legacy_capture().map(|c| &c.context().sources), + "capture":state.legacy_capture().map(|c|c.progress(now_ns)), + "training_frozen":state.frozen_training,"training_defaults":FitConfig::for_source_count(state.legacy_capture().map_or(0, |c| c.context().sources.len())),"training_report":state.job.report, + "error":state.job.error,"input_error":state.input_error, + "candidate":state.receipt,"candidate_loaded":state.model.is_some(),"candidate_identity":self.candidate_identity(&state), + "qualification_readiness":self.qualification_readiness_locked(&state,now_ns), + "model_selection":self.selection_status_locked(&state,now_ns,None), + "last_evaluation":state.last_evaluation,"evaluation_warning":state.evaluation_warning,"physically_validated":false, + "control_detector_changed":false,"raw_observations_persisted":state.recovery.derived_persisted(),"derived_samples_persisted":state.recovery.derived_persisted(),"raw_iq_persisted":false, + "limitations":["receiver-observed magnitude path; no coherent phase claim", + "legacy UDP transmitter identity is unverified","independent teacher registry not configured", + "candidate training is not physical qualification"]}) + } +} + +#[cfg(test)] +#[path = "lifecycle_tests.rs"] +mod tests; + +#[path = "lifecycle_capture.rs"] +mod capture_control; +pub use capture_control::ReceiverPipelineTiming; + +#[path = "command.rs"] +mod command; +pub use command::{MutationGuard, PhaseIdentity}; + +#[path = "lifecycle_recovery.rs"] +mod recovery_control; +use recovery_control::RecoveryState; + +#[cfg(all(test, unix))] +#[path = "lifecycle_recovery_tests.rs"] +mod recovery_tests; + +#[path = "lifecycle_evaluation.rs"] +mod evaluation_control; + +#[cfg(test)] +#[path = "lifecycle_evaluation_tests.rs"] +mod evaluation_tests; + +#[path = "lifecycle_selection.rs"] +mod selection_control; + +#[cfg(test)] +#[path="lifecycle_selection_tests.rs"] +mod selection_tests; + +#[cfg(test)] +#[path="lifecycle_assessment_tests.rs"] +mod assessment_tests; + +#[path="managed_training_readiness.rs"] +mod managed_training_readiness; + +#[cfg(test)] +#[path="managed_training_tests.rs"] +mod managed_training_tests; +#[path = "paired_progress.rs"] +mod paired_progress; +pub use paired_progress::{PairedCalibrationProgress, PublishedProgressRoom, PAIRED_PROGRESS_TTL_MS}; + +#[path = "lifecycle_recovery_enable.rs"] +mod recovery_enable; + +#[path = "lifecycle_temporal.rs"] +mod temporal_control; +pub(crate) use temporal_control::FullRunGenesisWitness; +pub(crate) use temporal_control::AcceptedTemporalFit; +pub(crate) use temporal_control::FrozenOwnerWitness; +pub(crate) use temporal_control::FullRunRecoveryWitness; + +#[cfg(test)] +#[path="lifecycle_legacy_byte_probe.rs"] +mod legacy_byte_probe; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_assessment_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_assessment_tests.rs new file mode 100644 index 0000000000..be27c749c2 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_assessment_tests.rs @@ -0,0 +1,395 @@ +//! Synthetic measured-path counterexamples; no hardware qualification claims. +use super::selection_tests::{feed, feed_sequence, fitted, phase_override}; +use super::tests::{context, finished, runtime, START}; +use super::*; +use crate::room_learner::selection::SelectedBinaryEvidence; + +#[tokio::test] +async fn newer_completed_failed_assessment_withdraws_persistent_selection_across_restart() { + let dir = tempfile::tempdir().unwrap(); + let rt = fitted(dir.path()).await; + let q = rt.qualify(&rt.mutation_guard().unwrap()).unwrap(); + let selected = rt + .select_model( + &rt.mutation_guard().unwrap(), + &q.receipt.model_sha256, + &q.receipt_sha256, + ) + .unwrap(); + let model = rt.inner.lock().unwrap().model.as_ref().unwrap().clone(); + let original_assessment = rt.store.load_latest_assessment(&model).unwrap().unwrap(); + // Record deliberately mislabelled held-out empty evidence through actual commands. + let now = phase_override(&rt, DataRole::HeldOut, RoomLabel::Empty, 8, 7.0); + let error = rt.qualify(&rt.mutation_guard().unwrap()).unwrap_err(); + assert!(error.contains("false presence"), "{error}"); + let snapshot = rt.selection_snapshot(now, Some("synthetic-room")); + assert_eq!( + snapshot.selected.unwrap().selection_sha256, + selected.receipt_sha256 + ); + assert!(snapshot.qualification.is_none()); + assert!( + matches!(snapshot.current,SelectedBinaryEvidence::Unavailable{reason,..} if reason=="newer_qualification_failed") + ); + assert!(rt + .select_model( + &rt.mutation_guard().unwrap(), + &q.receipt.model_sha256, + &q.receipt_sha256 + ) + .is_err()); + assert!( + rt.store + .save_assessment(&model, &original_assessment) + .is_err(), + "older pass must not erase a later completed failure" + ); + let cold = runtime(dir.path()); + let now = feed(&cold, cold.receiver_now_ns(), &[1, 2], 2.0); + let snapshot = cold.selection_snapshot(now, Some("synthetic-room")); + assert_eq!( + snapshot.selected.unwrap().selection_sha256, + selected.receipt_sha256 + ); + assert!( + matches!(snapshot.current,SelectedBinaryEvidence::Unavailable{reason,..} if reason=="newer_qualification_failed") + ); + assert!(cold + .store + .load_qualification( + cold.inner.lock().unwrap().model.as_ref().unwrap(), + &q.receipt_sha256 + ) + .is_ok()); + // Restoring the explicit legacy control is permitted; history/model stay intact. + cold.deselect_model(&cold.mutation_guard().unwrap()) + .unwrap(); + assert!(matches!( + cold.selected_binary_evidence(now, Some("synthetic-room")), + SelectedBinaryEvidence::None + )); +} + +#[tokio::test] +async fn rejected_selected_input_withdraws_old_window_until_all_sources_are_fresh_again() { + let dir = tempfile::tempdir().unwrap(); + let rt = fitted(dir.path()).await; + let q = rt.qualify(&rt.mutation_guard().unwrap()).unwrap(); + rt.select_model( + &rt.mutation_guard().unwrap(), + &q.receipt.model_sha256, + &q.receipt_sha256, + ) + .unwrap(); + let cold = runtime(dir.path()); + let at = cold.receiver_now_ns(); + let now = feed(&cold, at, &[1, 2], 2.0); + assert!(matches!( + cold.selected_binary_evidence(now, Some("synthetic-room")), + SelectedBinaryEvidence::Ready { .. } + )); + cold.record_native_rejection(99, "ht64-canonical8", "foreign source"); + cold.record_native_rejection(1, "other-grid", "unselected PHY"); + assert!(matches!( + cold.selected_binary_evidence(now, Some("synthetic-room")), + SelectedBinaryEvidence::Ready { .. } + )); + cold.record_native_rejection(1, "ht64-canonical8", "invalid native frame"); + assert!(matches!( + cold.selected_binary_evidence(now, Some("synthetic-room")), + SelectedBinaryEvidence::Unavailable { .. } + )); + let partial = feed_sequence(&cold, at + 200_000_000, &[1], 2.0, 20_000); + assert!(matches!( + cold.selected_binary_evidence(partial, Some("synthetic-room")), + SelectedBinaryEvidence::Unavailable { .. } + )); + let fresh = feed_sequence(&cold, at + 400_000_000, &[1, 2], 2.0, 30_000); + assert!(matches!( + cold.selected_binary_evidence(fresh, Some("synthetic-room")), + SelectedBinaryEvidence::Ready { .. } + )); + assert!(cold + .ingest( + RawSourceFrame { + source_id: 1, + grid_id: "ht64-canonical8", + frequency_hz: 2_437_000_000, + bandwidth_hz: 20_000_000, + sequence: 30_007, + amplitudes: &[2.3; 8] + }, + fresh + 1 + ) + .is_err()); + assert!(matches!( + cold.selected_binary_evidence(fresh + 1, Some("synthetic-room")), + SelectedBinaryEvidence::Unavailable { .. } + )); +} + +#[tokio::test] +async fn fresh_selection_snapshot_rechecks_durable_artifacts_not_only_cached_model() { + let dir = tempfile::tempdir().unwrap(); + let rt = fitted(dir.path()).await; + let q = rt.qualify(&rt.mutation_guard().unwrap()).unwrap(); + rt.select_model( + &rt.mutation_guard().unwrap(), + &q.receipt.model_sha256, + &q.receipt_sha256, + ) + .unwrap(); + let now = START + 103_000_000_000; + assert!(matches!( + rt.selection_snapshot(now, Some("synthetic-room")).current, + SelectedBinaryEvidence::Ready { .. } + )); + let path = dir + .path() + .join("a".repeat(64)) + .join("SensingServer/room-learner") + .join(context().digest()) + .join(format!("{}.json", q.receipt.model_sha256)); + let root = dir + .path() + .join("a".repeat(64)) + .join("SensingServer/room-learner"); + let qualification = root + .join(context().digest()) + .join("qualifications") + .join(format!("{}.json", q.receipt_sha256)); + let pointer = root.join("model-selection-v1.json"); + let assessment = root + .join(context().digest()) + .join("assessments") + .join(format!("{}.latest.json", q.receipt.model_sha256)); + for damaged in [path, qualification, pointer, assessment] { + let original = std::fs::read(&damaged).unwrap(); + std::fs::write(&damaged, b"corrupt durable evidence").unwrap(); + assert!( + matches!( + rt.selection_snapshot(now, Some("synthetic-room")).current, + SelectedBinaryEvidence::Unavailable { .. } + ), + "{}", + damaged.display() + ); + assert!(rt + .selection_snapshot(now, Some("synthetic-room")) + .integrity_error + .is_some()); + std::fs::write(&damaged, original).unwrap(); + assert!(matches!( + rt.selection_snapshot(now, Some("synthetic-room")).current, + SelectedBinaryEvidence::Ready { .. } + )); + } +} + +#[tokio::test] +async fn canonical_56_feature_context_qualifies_through_actual_managed_training_commands() { + let dir = tempfile::tempdir().unwrap(); + let rt = runtime(dir.path()); + let mut c = context(); + for s in &mut c.sources { + s.feature_count = 56; + s.grid_id = "ht64-canonical56".into(); + } + rt.configure( + &rt.mutation_guard().unwrap(), + c, + "canonical-width-fixture".into(), + true, + ) + .unwrap(); + let cases = [ + (RoomLabel::Empty, 2.0), + (RoomLabel::OccupiedStatic, 6.0), + (RoomLabel::OccupiedMoving, 7.0), + (RoomLabel::AdjacentOnly, 2.3), + ]; + for (i, (label, gain)) in cases.into_iter().enumerate() { + phase_override(&rt, DataRole::Train, label, i as u32, gain); + } + rt.start_training( + &rt.mutation_guard().unwrap(), + FitConfig { + epochs: 8, + learning_rate: 0.5, + hidden_dim: 2, + embedding_dim: 2, + temporal_tokens: 2, + ..FitConfig::default() + }, + ) + .unwrap(); + let training = finished(&rt).await; + assert_eq!(training["state"], "ready", "{training}"); + for (i, (label, gain)) in cases.into_iter().enumerate() { + phase_override(&rt, DataRole::HeldOut, label, i as u32 + 4, gain); + } + let q = rt.qualify(&rt.mutation_guard().unwrap()).unwrap(); + rt.select_model( + &rt.mutation_guard().unwrap(), + &q.receipt.model_sha256, + &q.receipt_sha256, + ) + .unwrap(); + assert!(matches!( + rt.selected_binary_evidence(START + 103_000_000_000, Some("synthetic-room")), + SelectedBinaryEvidence::Ready { .. } + )); +} + +#[tokio::test] +async fn missing_authoritative_pointer_with_history_is_unknown_not_legacy_after_cold_load() { + let dir = tempfile::tempdir().unwrap(); + let rt = fitted(dir.path()).await; + let q = rt.qualify(&rt.mutation_guard().unwrap()).unwrap(); + let selected = rt + .select_model( + &rt.mutation_guard().unwrap(), + &q.receipt.model_sha256, + &q.receipt_sha256, + ) + .unwrap(); + let root = dir + .path() + .join("a".repeat(64)) + .join("SensingServer/room-learner"); + let pointer = root.join("model-selection-v1.json"); + let original = std::fs::read(&pointer).unwrap(); + let history = root + .join("selections") + .join(format!("{}.json", selected.receipt_sha256)); + let history_bytes = std::fs::read(&history).unwrap(); + std::fs::rename(&pointer, root.join("selection-pointer-backup.json")).unwrap(); + let cold = runtime(dir.path()); + let status = cold.selection_snapshot(cold.receiver_now_ns(), Some("synthetic-room")); + assert!(status + .integrity_error + .as_deref() + .is_some_and(|s| s.contains("history"))); + assert!(matches!( + status.current, + SelectedBinaryEvidence::Unavailable { .. } + )); + assert!(!pointer.exists()); + assert_eq!(std::fs::read(&history).unwrap(), history_bytes); + assert_eq!( + std::fs::read(root.join("selection-pointer-backup.json")).unwrap(), + original + ); + // An actual explicit deselection pointer, unlike a missing pointer, proves legacy mode. + std::fs::rename(root.join("selection-pointer-backup.json"), &pointer).unwrap(); + let restored = runtime(dir.path()); + restored + .deselect_model(&restored.mutation_guard().unwrap()) + .unwrap(); + let deselected = runtime(dir.path()); + let status = + deselected.selection_snapshot(deselected.receiver_now_ns(), Some("synthetic-room")); + assert!(status.integrity_error.is_none()); + assert!(matches!(status.current, SelectedBinaryEvidence::None)); +} + +#[tokio::test] +async fn orphan_first_write_history_does_not_invent_a_selected_model_or_legacy_fallback() { + use crate::room_learner::{ + qualification::hash, + selection::{SelectionEnvelope, SelectionReceipt, SelectionTarget}, + }; + let dir = tempfile::tempdir().unwrap(); + let rt = fitted(dir.path()).await; + let q = rt.qualify(&rt.mutation_guard().unwrap()).unwrap(); + let root = dir + .path() + .join("a".repeat(64)) + .join("SensingServer/room-learner"); + let history = root.join("selections"); + crate::companion::private_dir(&history).unwrap(); + // Simulate the persisted boundary after immutable history but before the + // first authoritative pointer. No successful selection is asserted. + let receipt = SelectionReceipt { + schema: "ruview.room-learner.model-selection.v1".into(), + owner_scope: "a".repeat(64), + installation_id: "synthetic-install".into(), + revision: 1, + selected_at_unix_ms: rt.receiver_now_ns() / 1_000_000, + selected: Some(SelectionTarget { + context: context(), + model_sha256: q.receipt.model_sha256.clone(), + model_version: q.receipt.model_version, + qualification_sha256: q.receipt_sha256.clone(), + }), + previous: None, + }; + let envelope = SelectionEnvelope { + receipt_sha256: hash(&receipt).unwrap(), + receipt, + }; + let bytes = serde_json::to_vec(&envelope).unwrap(); + let record = history.join(format!("{}.json", envelope.receipt_sha256)); + crate::companion::atomic_private_write(&record, &bytes).unwrap(); + let pointer = root.join("model-selection-v1.json"); + std::fs::create_dir(&pointer).unwrap(); + assert!(crate::companion::atomic_private_write(&pointer, &bytes).is_err()); + std::fs::remove_dir(&pointer).unwrap(); + let cold = runtime(dir.path()); + let status = cold.selection_snapshot(cold.receiver_now_ns(), Some("synthetic-room")); + assert!(status.selected.is_none()); + assert!(status.integrity_error.is_some()); + assert!(matches!( + status.current, + SelectedBinaryEvidence::Unavailable { .. } + )); + assert!(!pointer.exists()); + assert_eq!(std::fs::read(record).unwrap(), bytes); +} + +#[cfg(unix)] +#[test] +fn never_selected_empty_history_is_legacy_but_unsafe_history_is_unknown() { + use std::os::unix::fs::{symlink, PermissionsExt}; + let dir = tempfile::tempdir().unwrap(); + let rt = runtime(dir.path()); + assert!(matches!( + rt.selection_snapshot(START, Some("synthetic-room")).current, + SelectedBinaryEvidence::None + )); + let root = dir + .path() + .join("a".repeat(64)) + .join("SensingServer/room-learner"); + let history = root.join("selections"); + crate::companion::private_dir(&history).unwrap(); + assert!(matches!( + runtime(dir.path()) + .selection_snapshot(START, Some("synthetic-room")) + .current, + SelectedBinaryEvidence::None + )); + std::fs::set_permissions(&history, std::fs::Permissions::from_mode(0o000)).unwrap(); + assert!(matches!( + runtime(dir.path()) + .selection_snapshot(START, Some("synthetic-room")) + .current, + SelectedBinaryEvidence::Unavailable { .. } + )); + std::fs::set_permissions(&history, std::fs::Permissions::from_mode(0o700)).unwrap(); + std::fs::remove_dir(&history).unwrap(); + let external = dir.path().join("empty-history-target"); + crate::companion::private_dir(&external).unwrap(); + symlink(&external, &history).unwrap(); + assert!(matches!( + runtime(dir.path()) + .selection_snapshot(START, Some("synthetic-room")) + .current, + SelectedBinaryEvidence::Unavailable { .. } + )); + assert!(std::fs::symlink_metadata(&history) + .unwrap() + .file_type() + .is_symlink()); + assert_eq!(std::fs::read_dir(external).unwrap().count(), 0); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_capture.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_capture.rs new file mode 100644 index 0000000000..c44f2d95d0 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_capture.rs @@ -0,0 +1,304 @@ +//! Managed capture controls and source/provenance admission. +use super::*; + +/// Transient metadata for one exact native packet. Deliberately not Serialize. +/// Relative receiver durations are not a source measurement clock. +/// +/// ```compile_fail +/// use wifi_densepose_sensing_server::room_learner::lifecycle::ReceiverPipelineTiming; +/// fn requires_serialize() {} +/// requires_serialize::>(); +/// ``` +#[derive(Clone, Copy)] +pub struct ReceiverPipelineTiming<'a> { + source_id: u8, + sequence: u32, + frame_id: u128, + grid_id: &'a str, + frequency_hz: u64, + bandwidth_hz: u64, + durations: [Option; 4], +} +impl<'a> ReceiverPipelineTiming<'a> { + pub fn for_packet(packet: &'a crate::room_learner::native::CanonicalPacket, + dequeue_interval_us: Option, parse_us: Option, state_lock_wait_us: Option, dequeue_to_stamp_us: Option) -> Self { + Self { source_id: packet.source_id, sequence: packet.sequence, frame_id: packet.native_frame.frame_id, + grid_id: &packet.grid_id, frequency_hz: packet.frequency_hz, bandwidth_hz: packet.bandwidth_hz, + durations: [dequeue_interval_us, parse_us, state_lock_wait_us, dequeue_to_stamp_us] } + } + fn failure_detail(&self, packet: &crate::room_learner::native::CanonicalPacket, primary: &str) -> Option { + if self.source_id != packet.source_id || self.sequence != packet.sequence || self.frame_id != packet.native_frame.frame_id + || self.grid_id != packet.grid_id || self.frequency_hz != packet.frequency_hz || self.bandwidth_hz != packet.bandwidth_hz { return None; } + crate::room_learner::temporal_model_input::managed::execution::receiver_pipeline_failure_detail(primary, self.durations) + } +} +impl RoomLearningRuntime { + pub fn bind_context( + &self, + room_digest: String, + revision: u64, + sources: Vec, + ) -> Result { + let context = CaptureContext { + owner_scope: self.owner.clone(), + installation_id: self.installation.clone(), + room_digest, + revision, + sources, + }; + context.validate()?; + Ok(context) + } + pub fn selects_grid(&self, source_id: u8, grid_id: &str) -> bool { + self.inner.lock().map(|state|state.capture_context().is_some_and(|c|c.sources.iter().any(|s|s.source_id==source_id&&s.grid_id==grid_id))||state.selection.temporal.as_ref().is_some_and(|selected|selected.stored.envelope().receipt.selected.context.sources.iter().any(|s|s.source_id==source_id&&s.grid_id==grid_id))).unwrap_or(false) + } + pub fn capture_active(&self) -> Result { + let state = self.inner.lock().map_err(|_| "learner state unavailable")?; + Ok(match state.capture.as_ref() { + Some(CaptureWorkflow::Legacy(c)) => c.phase_active(), + Some(CaptureWorkflow::Temporal(c)) => c.active(), + None => false, + }) + } + pub fn context(&self) -> Option { + self.inner.lock().ok()?.capture_context().cloned() + } + pub fn begin_phase( + &self, + expected: &MutationGuard, + request: PhaseRequest, + now_ns: u64, + ) -> Result<(), String> { + let mut state = self.command_state(expected)?; + state.require_legacy_kind()?; + self.expire_loaded_recovery(&mut state, now_ns)?; + self.require_recovery_ready(&state)?; + if busy(&state.job) { + return Err("training is active".into()); + } + if request.role == DataRole::Train && state.frozen_training { + return Err( + "training data is frozen; explicitly start a new capture run to retrain".into(), + ); + } + if request.role == DataRole::HeldOut && state.model.is_none() { + return Err( + "held-out capture starts only after a candidate is trained or restored".into(), + ); + } + if request.role==DataRole::HeldOut { + let readiness=self.qualification_readiness_locked(&state,now_ns); + if readiness["eligible_for_held_out"]!=true { + return Err(readiness["detail"].as_str().unwrap_or("managed training lineage unavailable before held-out capture").into()); + } + } + state.legacy_capture_mut() + .ok_or("configure the current room and source set first")? + .begin_phase(request, now_ns)?; + state.input_error = None; + self.checkpoint_recovery(&mut state, now_ns, true)?; + Ok(()) + } + pub fn ingest(&self, frame: RawSourceFrame<'_>, now_ns: u64) -> Result<(), String> { + let mut state = self.inner.lock().map_err(|_| "learner state unavailable")?; + state.require_legacy_kind()?; + let Some(capture) = state.legacy_capture_mut() else { + return Ok(()); + }; + let result = capture.ingest(frame, now_ns); + if let Err(error) = &result { + capture.clear_live_windows(); + state.input_error = Some(bounded(error.clone())); + } else if capture.live_cohort(now_ns).is_ok() { + state.input_error = None; + } + let saved = self.checkpoint_recovery(&mut state, now_ns, false); + result.and(saved) + } + pub fn ingest_native( + &self, + packet: &crate::room_learner::native::CanonicalPacket, + now_ns: u64, + ) -> Result<(), String> { + self.ingest_native_with_receiver_timing(packet, now_ns, None) + } + pub fn ingest_native_with_receiver_timing( + &self, + packet: &crate::room_learner::native::CanonicalPacket, + now_ns: u64, + receiver_timing: Option>, + ) -> Result<(), String> { + if packet.provenance + != crate::room_learner::observation::CaptureProvenance::LiveReceiverObserved + || packet.native_frame.provenance.class + != ruview_unified::frame::ProvenanceClass::Measured + { + self.record_native_rejection(packet.source_id,&packet.grid_id,"synthetic/replayed native provenance"); + return Err("live managed capture rejects synthetic/replayed native provenance".into()); + } + // Selected whole-model inference runs outside Runtime.inner. A stale + // Arc cannot publish evidence after selection is replaced; reads fence + // the exact current Arc again before returning a usable decision. + let selected={self.inner.lock().map_err(|_|"learner state unavailable")?.selection.temporal.clone()}; + let selected_present=selected.is_some(); + if let Some(selected)=selected{if selected.stored.envelope().receipt.selected.context.sources.iter().any(|s|s.source_id==packet.source_id&&s.grid_id==packet.grid_id&&s.center_frequency_hz==packet.frequency_hz&&s.bandwidth_hz==packet.bandwidth_hz){ + let mut driver=selected.driver.lock().map_err(|_|"selected_native_input_unavailable")?; + // Poll/inference may have advanced processing time while this + // packet waited for the driver. Measure the trusted processing + // clock HERE; preserve its original receive timestamp/identities. + let processing_now=self.receiver_now_ns(); + if processing_now Result { + let mut state = self.command_state(expected)?; + state.require_legacy_kind()?; + self.expire_loaded_recovery(&mut state, now_ns)?; + let recovered_complete = state.recovery.disposition + == Some( + crate::room_learner::capture::RecoveryPhaseDisposition::CompletedNeedsConfirmation, + ); + let capture = state.legacy_capture_mut() + .ok_or("no learner capture configured")?; + if !conditions_held { + capture.cancel_phase(); + let _ = self.checkpoint_recovery(&mut state, now_ns, true); + return Err( + "operator did not confirm conditions held; in-flight examples discarded".into(), + ); + } + if recovered_complete { + capture.live_cohort(now_ns)?; + } + // A phase is not published as sealed until its metadata-only replay + // fence is durable. Failure leaves the complete original phase retryable. + let mut completed=capture.clone(); + let receipt=completed.finish_phase(now_ns)?; + if let Some(model)=state.model.as_ref() { + if let Some(proof)=self.store.load_managed_training_proof(model)? { + completed.checkpoint(now_ns)?; + let seed=completed.managed_fence_seed(&proof)?; + self.store.save_managed_capture_fence(model,&proof,seed)?; + } + } + state.capture=Some(CaptureWorkflow::Legacy(completed)); + state.recovery.disposition = None; + self.checkpoint_recovery(&mut state, now_ns, true)?; + Ok(receipt) + } + pub fn cancel_phase(&self, expected: &MutationGuard) -> Result<(), String> { + let mut state = self.command_state(expected)?; + state.require_legacy_kind()?; + state.legacy_capture_mut() + .ok_or("no learner capture configured")? + .cancel_phase(); + let now = self.checkpoint_now(&state); + self.checkpoint_recovery(&mut state, now, true)?; + Ok(()) + } + pub fn cancel_training(&self, expected: &MutationGuard) -> Result { + let mut state = self.command_state(expected)?; + state.require_legacy_kind()?; + if busy(&state.job) { + state.job.cancel.store(true, Ordering::Release); + state.job.phase = JobPhase::Cancelling; + } + let now = self.checkpoint_now(&state); + self.checkpoint_recovery(&mut state, now, true)?; + Ok(state.job.phase) + } +} + +#[cfg(test)] +#[path = "receiver_pipeline_timing_tests.rs"] +mod receiver_timing_tests; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_evaluation.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_evaluation.rs new file mode 100644 index 0000000000..a1650d2670 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_evaluation.rs @@ -0,0 +1,81 @@ +//! Durable diagnostics only. Historical lineage checks are not current identity. +use super::*; +use crate::room_learner::artifact::EvaluationReceipt; +pub(super) fn presentation( + receipt: &EvaluationReceipt, + restored: bool, + current_train_checked: bool, +) -> Result { + Ok( + json!({"schema":"ruview.room-learner.evaluation.v1","role":receipt.role, + "context_digest":receipt.context_digest,"model_sha256":receipt.model_sha256,"report":receipt.report, + "measurement_clock_available":receipt.measurement_clock_available, + "receipt":receipt,"receipt_sha256":receipt.sha256()?,"durable":true,"restored":restored, + "train_lineage_checked":current_train_checked, + "historical_train_lineage_checked":receipt.train_lineage_checked, + "independence_verified":false,"physically_validated":false,"adoption_authorized":false}), + ) +} +impl RoomLearningRuntime { + pub(super) fn restored_evaluation(&self, model: &RoomModel) -> (Option, Option) { + let mut receipts = Vec::new(); + for role in [DataRole::Validation, DataRole::HeldOut] { + match self.store.load_latest_evaluation(model,role) { + Ok(Some(receipt))=>receipts.push(receipt),Ok(None)=>{}, + Err(_)=>return (None,Some("Saved evaluation could not be verified for this candidate; evidence files were retained.".into())), + } + } + let Some(receipt) = receipts.into_iter().max_by_key(|r| { + ( + r.evaluated_at_unix_ms, + usize::from(r.role == DataRole::HeldOut), + ) + }) else { + return (None, None); + }; + match presentation(&receipt, true, false) { + Ok(value) => (Some(value), None), + Err(_) => ( + None, + Some( + "Saved evaluation metadata could not be read; the candidate is still loaded." + .into(), + ), + ), + } + } + /// Only a frozen private candidate and a validated role split can produce a + /// durable diagnostic. This never adopts the candidate or changes control. + pub fn evaluate(&self, expected: &MutationGuard, role: DataRole) -> Result { + if !matches!(role, DataRole::Validation | DataRole::HeldOut) { + return Err("evaluate only validation or held-out data".into()); + } + let mut state = self.command_state(expected)?; + state.require_legacy_kind()?; + if busy(&state.job) || !state.frozen_training { + return Err("wait for the frozen candidate".into()); + } + let model = state.model.as_ref().ok_or("no frozen candidate")?.clone(); + let capture = state.legacy_capture().ok_or("no capture context")?; + if capture.phase_active() { + return Err("finish the current phase before evaluation".into()); + } + let dataset = capture.validated_dataset()?; + let result = self + .store + .evaluate_candidate(&model, &dataset, role) + .and_then(|receipt| presentation(&receipt, false, !dataset.train().is_empty())); + match result { + Ok(value) => { + state.last_evaluation = Some(value.clone()); + state.evaluation_warning = None; + Ok(value) + } + Err(error) => { + state.last_evaluation = None; + state.evaluation_warning=Some("Evaluation did not complete durably; previous evidence and the candidate were retained.".into()); + Err(bounded(error)) + } + } + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_evaluation_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_evaluation_tests.rs new file mode 100644 index 0000000000..45da3a35fc --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_evaluation_tests.rs @@ -0,0 +1,181 @@ +//! SYNTHETIC managed lifecycle, private persistence and cold diagnostics. +use super::tests::{context, finished, phase, runtime, train_phases, START}; +use super::*; +async fn fitted(root: &Path) -> Arc { + let rt = runtime(root); + rt.configure( + &rt.mutation_guard().unwrap(), + context(), + "evaluation-fixture".into(), + true, + ) + .unwrap(); + train_phases(&rt); + rt.start_training( + &rt.mutation_guard().unwrap(), + FitConfig { + epochs: 2, + ..FitConfig::default() + }, + ) + .unwrap(); + let status = finished(&rt).await; + assert_eq!(status["state"], "ready", "{status}"); + assert!( + status["training_report"]["backbone_delta_l2"] + .as_f64() + .unwrap() + > 0.0 + ); + rt +} +fn evaluation_directory(root: &Path, report: &Value) -> std::path::PathBuf { + root.join("a".repeat(64)) + .join("SensingServer/room-learner") + .join(report["context_digest"].as_str().unwrap()) + .join("evaluations") + .join(report["model_sha256"].as_str().unwrap()) + .join("held-out") +} +#[tokio::test] +async fn frozen_capture_evaluate_persist_cold_resume_restores_exact_diagnostic_without_current_train_claim( +) { + let dir = tempfile::tempdir().unwrap(); + let rt = fitted(dir.path()).await; + phase(&rt, DataRole::HeldOut, RoomLabel::Empty, 4); + phase(&rt, DataRole::HeldOut, RoomLabel::OccupiedStatic, 5); + let result = rt + .evaluate(&rt.mutation_guard().unwrap(), DataRole::HeldOut) + .unwrap(); + assert_eq!(result["durable"], true); + assert_eq!(result["train_lineage_checked"], true); + assert_eq!(result["report"]["samples"], 8); + assert_eq!(result["report"]["declared_trial_count"], 2); + assert_eq!(result["report"]["independence_verified"], false); + assert_eq!(result["adoption_authorized"], false); + let key = result["receipt_sha256"].as_str().unwrap(); + assert!(evaluation_directory(dir.path(), &result) + .join(format!("{key}.json")) + .is_file()); + let cold = runtime(dir.path()); + cold.configure( + &cold.mutation_guard().unwrap(), + context(), + "cold-evaluation".into(), + false, + ) + .unwrap(); + let status = cold.status(START); + let restored = &status["last_evaluation"]; + assert_eq!(status["candidate_loaded"], true); + assert!(status["evaluation_warning"].is_null()); + assert_eq!(restored["receipt_sha256"], result["receipt_sha256"]); + assert_eq!(restored["report"], result["report"]); + assert_eq!(restored["restored"], true); + assert_eq!(restored["train_lineage_checked"], false); + assert_eq!(restored["historical_train_lineage_checked"], true); + assert_eq!(status["physically_validated"], false); + assert_eq!(status["control_detector_changed"], false); + phase(&cold, DataRole::HeldOut, RoomLabel::Empty, 6); + let fresh = cold + .evaluate(&cold.mutation_guard().unwrap(), DataRole::HeldOut) + .unwrap(); + assert_eq!(fresh["train_lineage_checked"], false); + for raw in ["frame_ids", "sequence_ids", "\"views\"", "\"samples\":["] { + assert!(!serde_json::to_string(&cold.status(START)) + .unwrap() + .contains(raw)); + } +} +#[tokio::test] +async fn corrupted_report_is_retained_and_warned_while_the_valid_candidate_remains_loaded() { + let dir = tempfile::tempdir().unwrap(); + let rt = fitted(dir.path()).await; + phase(&rt, DataRole::HeldOut, RoomLabel::Empty, 4); + let result = rt + .evaluate(&rt.mutation_guard().unwrap(), DataRole::HeldOut) + .unwrap(); + let pointer = evaluation_directory(dir.path(), &result).join("latest-v1.json"); + crate::companion::atomic_private_write(&pointer, b"broken evaluation pointer").unwrap(); + let cold = runtime(dir.path()); + cold.configure( + &cold.mutation_guard().unwrap(), + context(), + "corrupt-eval".into(), + false, + ) + .unwrap(); + let status = cold.status(START); + assert_eq!(status["candidate_loaded"], true); + assert_eq!(status["state"], "ready"); + assert!(status["last_evaluation"].is_null()); + let warning = status["evaluation_warning"].as_str().unwrap(); + assert!(!warning.is_empty() && warning.len() <= 256); + assert_eq!( + std::fs::read(&pointer).unwrap(), + b"broken evaluation pointer" + ); + assert_eq!(status["physically_validated"], false); + phase(&cold, DataRole::HeldOut, RoomLabel::Empty, 6); + assert!(cold + .evaluate(&cold.mutation_guard().unwrap(), DataRole::HeldOut) + .is_err()); + let failed = cold.status(START); + assert!(failed["last_evaluation"].is_null()); + assert!(failed["evaluation_warning"].is_string()); + assert_eq!(failed["candidate_loaded"], true); +} +#[tokio::test] +async fn persistence_failure_never_claims_a_durable_evaluation_or_changes_control() { + let dir = tempfile::tempdir().unwrap(); + let rt = fitted(dir.path()).await; + phase(&rt, DataRole::HeldOut, RoomLabel::Empty, 4); + let root = dir + .path() + .join("a".repeat(64)) + .join("SensingServer/room-learner") + .join(context().digest()); + crate::companion::atomic_private_write(&root.join("evaluations"), b"filesystem obstacle") + .unwrap(); + assert!(rt + .evaluate(&rt.mutation_guard().unwrap(), DataRole::HeldOut) + .is_err()); + let status = rt.status(START); + assert!(status["last_evaluation"].is_null()); + assert!(status["evaluation_warning"].is_string()); + assert_eq!(status["candidate_loaded"], true); + assert_eq!(status["control_detector_changed"], false); + assert!(rt + .evaluate(&rt.mutation_guard().unwrap(), DataRole::Train) + .is_err()); +} + +pub(super) fn assert_no_raw_observation_arrays(value: &Value) { + match value { + Value::Object(object) => { + for (key, child) in object { + assert!( + !([ + "samples", + "views", + "frame_ids", + "sequence_ids", + "amplitudes", + "native_iq", + "valid_mask" + ] + .contains(&key.as_str()) + && child.is_array()), + "raw observation array: {key}" + ); + assert_no_raw_observation_arrays(child); + } + } + Value::Array(items) => { + for item in items { + assert_no_raw_observation_arrays(item); + } + } + _ => {} + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_legacy_byte_probe.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_legacy_byte_probe.rs new file mode 100644 index 0000000000..17bf6ecf9f --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_legacy_byte_probe.rs @@ -0,0 +1,90 @@ +//! SYNTHETIC fixed-clock public Legacy Runtime status-byte compatibility probe. +use super::*; +use sha2::{Digest, Sha256}; +#[test] +fn legacy_public_status_byte_probe() { + let dir = tempfile::tempdir().unwrap(); + let c = tests::context(); + let path = dir.path().join(&c.owner_scope).join("SensingServer"); + let recovery = crate::room_learner::recovery_store::RecoveryStore::new( + &path, + &c.owner_scope, + &c.installation_id, + Some(&[7; 32]), + ) + .unwrap(); + let mut rt = RoomLearningRuntime::new_with_recovery_at( + &path, + &c.owner_scope, + &c.installation_id, + recovery, + tests::START, + ) + .unwrap(); + rt.runtime_epoch = "synthetic-legacy-golden-runtime".into(); + rt.configure_for_recovery( + &rt.mutation_guard().unwrap(), + c.clone(), + "synthetic-legacy-golden-run".into(), + true, + false, + tests::START, + ) + .unwrap(); + fn stable(mut value: Value) -> Value { + if let Some(cap) = value.pointer_mut("/recovery/enable_current_run") { + if cap["run_id"].is_string() { + cap["run_id"] = json!("synthetic-generated-ticket"); + } + } + value + } + let initial = stable(rt.status(tests::START)); + let first = tests::START; + rt.begin_phase( + &rt.mutation_guard().unwrap(), + PhaseRequest { + trial_id: "golden-trial".into(), + session_id: "golden-session".into(), + role: DataRole::Train, + label: crate::room_learner::observation::RoomLabel::Empty, + duration_ms: 12000, + lead_in_ms: 0, + conditions_confirmed: true, + }, + first, + ) + .unwrap(); + for bucket in 0..4u64 { + for row in 0..8u32 { + let at = first + bucket * 3_000_000_000 + u64::from(row + 1) * 10_000_000; + for source in &c.sources { + let amplitudes = vec![2.; source.feature_count]; + rt.ingest( + RawSourceFrame { + source_id: source.source_id, + grid_id: &source.grid_id, + frequency_hz: source.center_frequency_hz, + bandwidth_hz: source.bandwidth_hz, + sequence: bucket as u32 * 8 + row + 1, + amplitudes: &litudes, + }, + at, + ) + .unwrap(); + } + } + } + rt.finish_phase(&rt.mutation_guard().unwrap(), first + 12_000_000_000, true) + .unwrap(); + let sealed = stable(rt.status(first + 12_000_000_000)); + let bytes = serde_json::to_vec(&(initial, sealed)).unwrap(); + let digest = format!("{:x}", Sha256::digest(&bytes)); + // The receiver clock and runtime ID are fixed; ONLY the random new-run + // receipt handle is normalized above. All other public bytes remain exact. + assert_eq!( + digest, + "8e35280c09309b4f039e1e7cc646364bbd7a92934cfe2960df1c4a0dbeb91b5a" + ); + println!("LEGACY_PUBLIC_STATUS_SHA256={digest} BYTES={}", bytes.len()); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_recovery.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_recovery.rs new file mode 100644 index 0000000000..0e49580865 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_recovery.rs @@ -0,0 +1,497 @@ +//! Explicit, encrypted recovery. No receiver buffers or automatic work resume. +use super::*; +use crate::room_learner::{ + capture::{CaptureCheckpoint, CaptureCheckpointSummary, RecoveryPhaseDisposition}, + recovery_store::{RecoveryDescriptor, RECOVERY_SCHEMA, RECOVERY_TTL_MS}, +}; +use sha2::{Digest, Sha256}; + +#[derive(Default)] +pub(super) struct RecoveryState { + pub(super) supported: bool, + pub(super) enabled: bool, + pub(super) descriptor: Option, + summary: Option, + pub(super) marker: Option, + pub(super) error: Option, + pub(super) disposition: Option, + last_now_ns: u64, + last_attempt_ns: u64, + volatile_after_expiry: bool, +} +impl RecoveryState { + pub(super) fn observe_clock(&mut self, now_ns: u64) -> u64 { + self.last_now_ns = self.last_now_ns.max(now_ns); + self.last_now_ns + } + pub(super) fn last_observed_ns(&self) -> u64 { self.last_now_ns } + pub(super) fn current_run_saved(descriptor: RecoveryDescriptor, summary: CaptureCheckpointSummary, now_ns: u64) -> Self { + Self { enabled:true, descriptor:Some(descriptor), summary:Some(summary), last_now_ns:now_ns, ..Default::default() } + } + pub(super) fn derived_persisted(&self) -> Option { + // Unreadable retained contents are unknown, never evidence of absence. + // An error also invalidates any older cached summary for this claim. + if self.error.is_some() || (self.descriptor.is_some() && self.summary.is_none()) { + return None; + } + Some(self.summary.as_ref().is_some_and(|s| { + s.sealed_cohorts > 0 + || s.phase + .as_ref() + .is_some_and(|p| p.representative_cohorts > 0) + })) + } +} +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct RuntimeCheckpoint { + pub(super) schema: String, + pub(super) capture: CaptureCheckpoint, + pub(super) frozen_training: bool, + pub(super) job_phase: JobPhase, + pub(super) model_sha256: Option, +} +pub(super) const SCHEMA: &str = "ruview.room-learner.runtime-recovery.v1"; +pub(super) fn model_sha(model: &RoomModel) -> Result { + Ok(format!("{:x}", Sha256::digest(model.export()?))) +} +pub(super) fn checkpoint_payload( + bytes: &[u8], + d: &RecoveryDescriptor, + owner: &str, + installation: &str, + now_ns: u64, +) -> Result { + let p: RuntimeCheckpoint = + serde_json::from_slice(bytes).map_err(|_| "recovery_payload_invalid")?; + if p.schema != SCHEMA + || p.capture.context().owner_scope != owner + || p.capture.context().installation_id != installation + || p.capture.context().digest() != d.context_digest + || p.capture.saved_at_ns() / 1_000_000 != d.saved_at_ms + || p.capture.saved_at_ns() < d.created_at_ms.saturating_mul(1_000_000) + || (p.model_sha256.is_some() && (!p.frozen_training || p.job_phase != JobPhase::Ready)) + || (p.job_phase == JobPhase::Ready && p.model_sha256.is_none()) + || (!p.frozen_training && p.job_phase != JobPhase::Idle) + || (p.frozen_training && p.job_phase == JobPhase::Idle) + { + return Err("recovery_payload_binding_mismatch".into()); + } + CaptureSession::restore_checkpoint(p.capture.clone(), p.capture.context(), now_ns)?; + Ok(p) +} +impl RoomLearningRuntime { + pub(super) fn require_recovery_ready(&self, state: &Inner) -> Result<(), String> { + if state.recovery.enabled && state.recovery.error.is_some() { + Err( + "recovery_save_failed; recover, discard, or explicitly configure RAM-only capture" + .into(), + ) + } else { + Ok(()) + } + } + /// Recovery availability is a disk deadline, not a lifetime for samples + /// already loaded into this active process. Authenticate the unchanged copy + /// before transitioning; other persistence failures never become RAM mode. + pub(super) fn expire_loaded_recovery( + &self, + state: &mut Inner, + now_ns: u64, + ) -> Result<(), String> { + state.require_legacy_kind()?; + state.recovery.last_now_ns = state.recovery.last_now_ns.max(now_ns); + let now_ns = state.recovery.last_now_ns; + if !state.recovery.enabled || state.capture.is_none() { + return Ok(()); + } + let Some(descriptor) = state.recovery.descriptor.as_ref() else { + return Ok(()); // Existing checkpoint validation reports missing state. + }; + if now_ns / 1_000_000 < descriptor.expires_at_ms { + return Ok(()); + } + if let Some(error) = &state.recovery.error { + return Err(error.clone()); + } + let verified = (|| -> Result<(), String> { + if state.legacy_capture().map(|capture| capture.context().digest()) + .as_deref() != Some(descriptor.context_digest.as_str()) + { + return Err("recovery_context_mismatch".into()); + } + let retained = self.recovery_store.as_ref().ok_or("recovery_unavailable")? + .inspect()?.ok_or("recovery_missing")?; + if retained != *descriptor { + return Err("recovery_descriptor_changed".into()); + } + Ok(()) + })(); + if let Err(error) = verified { + state.recovery.error = Some(bounded(error.clone())); + return Err(error); + } + state.recovery.enabled = false; + state.recovery.marker = None; + state.recovery.volatile_after_expiry = true; + Ok(()) + } + pub(super) fn checkpoint_now(&self, state: &Inner) -> u64 { + self.receiver_now_ns().max(state.recovery.last_now_ns) + } + pub(super) fn inspect_recovery(&self, state: &mut Inner, now_ns: u64) { + state.recovery.last_now_ns = state.recovery.last_now_ns.max(now_ns); + let now_ns = state.recovery.last_now_ns; + let Some(store) = &self.recovery_store else { + return; + }; + match store.inspect() { + Ok(None) => { + state.recovery.descriptor = None; + state.recovery.summary = None; + } + Ok(Some(d)) => { + if d.owner_scope != self.owner || d.installation_id != self.installation { + state.recovery.error = Some("recovery_owner_mismatch".into()); + return; + } + state.recovery.descriptor = Some(d.clone()); + match store + .load(now_ns / 1_000_000) + .and_then(|v| v.ok_or_else(|| "recovery_missing".into())) + .and_then(|loaded| { + checkpoint_payload( + &loaded.plaintext, + &d, + &self.owner, + &self.installation, + now_ns, + ) + }) { + Ok(p) => { + state.recovery.summary = Some(p.capture.summary()); + state.recovery.error = None; + } + Err(error) => state.recovery.error = Some(bounded(error)), + } + } + Err(error) => state.recovery.error = Some(bounded(error)), + } + } + pub(super) fn prepare_recovery( + &self, + state: &mut Inner, + context: &CaptureContext, + now_ns: u64, + ) -> Result<(), String> { + let store = self + .recovery_store + .as_ref() + .filter(|s| s.available()) + .ok_or("recovery_unavailable")?; + match store.inspect() { + Ok(Some(_)) => return Err("learner_recovery_pending".into()), + Ok(None) => {} + Err(error) => { + state.recovery.error = Some(bounded(error.clone())); + return Err(error); + } + } + let created = now_ns / 1_000_000; + let expires = created + .checked_add(RECOVERY_TTL_MS) + .ok_or("recovery_clock_invalid")?; + state.recovery = RecoveryState { + enabled: true, + descriptor: Some(RecoveryDescriptor { + schema: RECOVERY_SCHEMA.into(), + id: crate::companion::secret(), + owner_scope: self.owner.clone(), + installation_id: self.installation.clone(), + context_digest: context.digest(), + created_at_ms: created, + saved_at_ms: created, + expires_at_ms: expires, + }), + ..Default::default() + }; + Ok(()) + } + pub(super) fn checkpoint_recovery( + &self, + state: &mut Inner, + now_ns: u64, + force: bool, + ) -> Result<(), String> { + state.require_legacy_kind()?; + state.recovery.last_now_ns = state.recovery.last_now_ns.max(now_ns); + self.expire_loaded_recovery(state, now_ns)?; + if !state.recovery.enabled { + return Ok(()); + } + let result: Result = (|| { + // Borrow the capture field alone so recording an attempt still + // precedes checkpoint construction, exactly as on the legacy path. + let capture = match state.capture.as_ref() { + Some(CaptureWorkflow::Legacy(capture)) => capture, + _ => return Err("recovery_capture_missing".into()), + }; + let marker = format!( + "{}:{:?}:{}", + capture.recovery_marker(now_ns), + state.job.phase, + state.model.is_some() + ); + if !force + && state.recovery.marker.as_ref() == Some(&marker) + && (state.recovery.error.is_none() + || now_ns.saturating_sub(state.recovery.last_attempt_ns) < 5_000_000_000) + { + return Ok(false); + } + // Record attempts too, so one disk failure cannot cause packet-rate writes. + state.recovery.marker = Some(marker); + state.recovery.last_attempt_ns = now_ns; + let snapshot = capture.checkpoint(now_ns)?; + let summary = snapshot.summary(); + let payload = RuntimeCheckpoint { + schema: SCHEMA.into(), + capture: snapshot, + frozen_training: state.frozen_training, + job_phase: state.job.phase, + model_sha256: state.model.as_ref().map(|m| model_sha(m)).transpose()?, + }; + let mut descriptor = state + .recovery + .descriptor + .clone() + .ok_or("recovery_descriptor_missing")?; + descriptor.saved_at_ms = now_ns / 1_000_000; + let bytes = + serde_json::to_vec(&payload).map_err(|_| "recovery_payload_encode_failed")?; + let store = self.recovery_store.as_ref().ok_or("recovery_unavailable")?; + if let Err(error) = store.save(&descriptor, &bytes, now_ns / 1_000_000) { + // A rename may commit before directory fsync reports failure. + // Read back exactly this attempt; never replay a mutation or + // advertise durable success after the failed durability barrier. + if let Ok(Some(loaded)) = store.load(now_ns / 1_000_000) { + if loaded.descriptor == descriptor && loaded.plaintext == bytes { + state.recovery.descriptor = Some(descriptor); + state.recovery.summary = Some(summary); + } + } + return Err(error); + } + state.recovery.descriptor = Some(descriptor); + state.recovery.summary = Some(summary); + Ok(true) + })(); + match result { + Ok(true) => { + state.recovery.error = None; + Ok(()) + } + Ok(false) => Ok(()), + Err(error) => { + state.recovery.error = Some(bounded(error.clone())); + Err(error) + } + } + } + pub(super) fn recovery_status(&self, state: &Inner, now_ns: u64) -> Value { + let supported = self.recovery_store.is_some() || state.recovery.supported; + let available = self.recovery_store.as_ref().is_some_and(|s| s.available()); + // A request may have sampled receiver time before waiting for this + // mutex. Known expiry cannot be undone by that older observation. + let now_ns = now_ns.max(state.recovery.last_now_ns); + let expired = state.recovery.volatile_after_expiry || state + .recovery + .descriptor + .as_ref() + .is_some_and(|d| now_ns / 1_000_000 >= d.expires_at_ms); + let saved = state + .recovery + .descriptor + .as_ref() + .zip(state.recovery.summary.as_ref()) + .filter(|_| !expired) + .map(|(d, s)| { + #[derive(Serialize)] + struct Saved<'a> { + #[serde(flatten)] + descriptor: &'a RecoveryDescriptor, + #[serde(flatten)] + summary: &'a CaptureCheckpointSummary, + } + Saved { + descriptor: d, + summary: s, + } + }); + let status = if !supported { + "disabled" + } else if !available { + "unavailable" + } else if expired { + "expired" + } else if state.recovery.error.is_some() { + "error" + } else if state.recovery.summary.is_some() { + if state.recovery.enabled { + "saved" + } else { + "restore_required" + } + } else { + "disabled" + }; + let error = state.recovery.error.clone().or_else(|| { + state.recovery.volatile_after_expiry.then(|| { + "Recovery expired. This loaded run continues in memory only; new progress is not saved. Closing the helper loses these samples. The expired encrypted copy remains until explicitly discarded.".into() + }) + }).or_else(|| { + (supported && !available).then(|| { + "recovery_unavailable: stable managed encryption key is unavailable".to_string() + }) + }); + json!({"supported":supported,"enabled":state.recovery.enabled,"state":status,"error":error, + "saved":saved,"discard_id":state.recovery.descriptor.as_ref().map(|d| &d.id), + "restored_disposition":state.recovery.disposition,"raw_iq_persisted":false, + "derived_samples_persisted":state.recovery.derived_persisted(),"automatic_resume":false, + "enable_current_run":self.recovery_enable_status(state, now_ns)}) + } + pub fn saved_recovery_context( + &self, + recovery_id: &str, + now_ns: u64, + ) -> Result { + let mut state = self.inner.lock().map_err(|_| "learner state unavailable")?; + state.require_legacy_kind()?; + state.recovery.last_now_ns = state.recovery.last_now_ns.max(now_ns); + let now_ns = state.recovery.last_now_ns; + let loaded = self + .recovery_store + .as_ref() + .ok_or("recovery_unavailable")? + .load(now_ns / 1_000_000)? + .ok_or("recovery_missing")?; + if loaded.descriptor.id != recovery_id { + return Err("recovery_identity_mismatch".into()); + } + let payload = checkpoint_payload( + &loaded.plaintext, + &loaded.descriptor, + &self.owner, + &self.installation, + now_ns, + )?; + Ok(payload.capture.context().clone()) + } + pub fn restore_capture_recovery( + &self, + expected: &MutationGuard, + recovery_id: &str, + context: &CaptureContext, + now_ns: u64, + ) -> Result<(), String> { + let mut state = self.command_state(expected)?; + state.require_legacy_kind()?; + state.recovery.last_now_ns = state.recovery.last_now_ns.max(now_ns); + let now_ns = state.recovery.last_now_ns; + if busy(&state.job) || state.legacy_capture().is_some_and(|c| c.phase_active()) { + return Err("finish or cancel the active operation first".into()); + } + let loaded = self + .recovery_store + .as_ref() + .ok_or("recovery_unavailable")? + .load(now_ns / 1_000_000)? + .ok_or("recovery_missing")?; + if loaded.descriptor.id != recovery_id { + return Err("recovery_identity_mismatch".into()); + } + let payload = checkpoint_payload( + &loaded.plaintext, + &loaded.descriptor, + &self.owner, + &self.installation, + now_ns, + )?; + if payload.capture.context() != context { + return Err("recovery_context_mismatch".into()); + } + let restored = CaptureSession::restore_checkpoint(payload.capture, context, now_ns)?; + let model = payload + .model_sha256 + .as_ref() + .map(|sha| self.store.load_exact_candidate(context, sha)) + .transpose()?; + if let Some(seed)=restored.session.metadata_lineage() { + let candidate=model.as_ref().ok_or("managed recovery has no exact candidate")?; + let proof=self.store.load_managed_training_proof(candidate)?.ok_or("managed recovery training proof missing")?; + seed.ensure_extends(&proof.seed)?; + let fence=self.store.load_managed_capture_fence(candidate,&proof)?; + restored.session.managed_fence_seed(&proof)?.ensure_extends(&fence.seed)?; + } + state.generation = state + .generation + .checked_add(1) + .ok_or("learner generation exhausted")?; + state.recovery_enable = None; + state.capture = Some(CaptureWorkflow::Legacy(restored.session)); + state.model = model.map(Arc::new); + state.receipt = None; + state.frozen_training = payload.frozen_training; + state.job = Job::default(); + state.job.phase = if busy(&Job { + phase: payload.job_phase, + ..Job::default() + }) { + JobPhase::Failed + } else { + payload.job_phase + }; + if state.job.phase == JobPhase::Failed { + state.job.error = Some( + "Saved training was interrupted or failed; retry explicitly using the frozen data." + .into(), + ); + } + state.last_evaluation = None; + state.evaluation_warning = None; + state.input_error = None; + state.recovery = RecoveryState { + enabled: true, + descriptor: Some(loaded.descriptor), + disposition: Some(restored.disposition), + ..Default::default() + }; + self.checkpoint_recovery(&mut state, now_ns, true) + } + pub fn discard_capture_recovery( + &self, + expected: &MutationGuard, + recovery_id: &str, + ) -> Result<(), String> { + let mut state = self.command_state(expected)?; + state.require_legacy_kind()?; + // Fence every later ingest/status/worker save before touching the file. + state.recovery.enabled = false; + state.recovery.marker = None; + let result = self + .recovery_store + .as_ref() + .ok_or("recovery_unavailable")? + .discard(recovery_id); + match result { + Ok(()) => { + state.recovery = RecoveryState::default(); + Ok(()) + } + Err(error) => { + state.recovery.error = Some(bounded(error.clone())); + Err(error) + } + } + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_recovery_enable.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_recovery_enable.rs new file mode 100644 index 0000000000..91ed81d4d9 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_recovery_enable.rs @@ -0,0 +1,163 @@ +//! Explicitly protect an existing NEW RAM run without reconfiguring it. +//! Eligibility is process-local and consumed after success: restored/expired +//! captures cannot mint a fresh recovery deadline, even after explicit discard. +use super::recovery_control::{checkpoint_payload, model_sha, RuntimeCheckpoint, SCHEMA}; +use super::*; +use crate::room_learner::recovery_store::{RecoveryDescriptor, RECOVERY_SCHEMA, RECOVERY_TTL_MS}; + +#[derive(Clone)] +pub(super) struct CurrentNewRun { + id: String, + capture_run: String, + context_digest: String, + created_at_ms: u64, + expires_at_ms: u64, +} +impl CurrentNewRun { + pub(super) fn new(capture: &CaptureSession, now_ns: u64) -> Option { + let created_at_ms = now_ns / 1_000_000; + if created_at_ms == 0 { + return None; + } + Some(Self { + id: crate::companion::secret(), + capture_run: capture.checkpoint(now_ns).ok()?.run_id().into(), + context_digest: capture.context().digest(), + created_at_ms, + expires_at_ms: created_at_ms.checked_add(RECOVERY_TTL_MS)?, + }) + } +} +impl RoomLearningRuntime { + pub(super) fn recovery_enable_status(&self, state: &Inner, now_ns: u64) -> Value { + let Some(run) = &state.recovery_enable else { + return json!({"schema":"ruview.room-learner.enable-current-recovery.v1","eligible":false,"run_id":null,"reason":"not_a_new_ram_run"}); + }; + let now_ms = now_ns.max(state.recovery.last_observed_ns()) / 1_000_000; + let reason = if now_ms >= run.expires_at_ms { + Some("new_run_recovery_deadline_expired") + } else if state.recovery.enabled { + Some("recovery_already_enabled") + } else if busy(&state.job) || state.legacy_capture().is_some_and(|c| c.phase_active()) { + Some("finish_active_operation") + } else if self.recovery_store.as_ref().is_none_or(|s| !s.available()) { + Some("recovery_unavailable") + } else if state + .recovery + .descriptor + .as_ref() + .is_some_and(|d| d.id != run.id) + { + Some("learner_recovery_pending") + } else { + None + }; + json!({"schema":"ruview.room-learner.enable-current-recovery.v1","eligible":reason.is_none(),"run_id":run.id, + "created_at_ms":run.created_at_ms,"expires_at_ms":run.expires_at_ms,"reason":reason, + "consent_required":true,"deadline_basis":"new_run_created_at","reduced_samples_only":true}) + } + /// Saves the exact current capture under its original new-run deadline. + /// No reset, fit, role change, selection or renewal occurs. A failed write + /// may have installed this SAME ticket; retry never creates a second one. + pub fn enable_current_capture_recovery( + &self, + expected: &MutationGuard, + run_id: &str, + consent: bool, + now_ns: u64, + ) -> Result<(), String> { + if !consent { + return Err("recovery_consent_required".into()); + } + let mut state = self.command_state(expected)?; + state.require_legacy_kind()?; + let now_ns = state.recovery.observe_clock(now_ns); + let run = state.recovery_enable.clone().ok_or("not_a_new_ram_run")?; + if run.id != run_id { + return Err("recovery_run_identity_mismatch".into()); + } + if now_ns / 1_000_000 >= run.expires_at_ms { + return Err("new_run_recovery_deadline_expired".into()); + } + if state.recovery.enabled { + return Err("recovery_already_enabled".into()); + } + if busy(&state.job) || state.legacy_capture().is_some_and(|c| c.phase_active()) { + return Err("finish_active_operation".into()); + } + let capture = state.legacy_capture().ok_or("recovery_capture_missing")?; + if capture.context().digest() != run.context_digest { + return Err("recovery_context_mismatch".into()); + } + let snapshot = capture.checkpoint(now_ns)?; + if snapshot.run_id() != run.capture_run { + return Err("recovery_run_identity_mismatch".into()); + } + let summary = snapshot.summary(); + let payload = RuntimeCheckpoint { + schema: SCHEMA.into(), + capture: snapshot, + frozen_training: state.frozen_training, + job_phase: state.job.phase, + model_sha256: state.model.as_ref().map(|m| model_sha(m)).transpose()?, + }; + let bytes = serde_json::to_vec(&payload).map_err(|_| "recovery_payload_encode_failed")?; + let descriptor = RecoveryDescriptor { + schema: RECOVERY_SCHEMA.into(), + id: run.id, + owner_scope: self.owner.clone(), + installation_id: self.installation.clone(), + context_digest: run.context_digest, + created_at_ms: run.created_at_ms, + saved_at_ms: now_ns / 1_000_000, + expires_at_ms: run.expires_at_ms, + }; + let store = self + .recovery_store + .as_ref() + .filter(|s| s.available()) + .ok_or("recovery_unavailable")?; + let inspected = match store.inspect() { + Ok(value) => value, + Err(error) => { + state.recovery.error = Some(bounded(error.clone())); + return Err(error); + } + }; + if let Some(previous) = inspected { + // Only a partial durable write by this very action may be retried. + if previous.id != descriptor.id + || previous.owner_scope != descriptor.owner_scope + || previous.installation_id != descriptor.installation_id + || previous.context_digest != descriptor.context_digest + || previous.created_at_ms != descriptor.created_at_ms + || previous.expires_at_ms != descriptor.expires_at_ms + { + return Err("learner_recovery_pending".into()); + } + let loaded = store.load(now_ns / 1_000_000)?.ok_or("recovery_missing")?; + let prior = checkpoint_payload( + &loaded.plaintext, + &loaded.descriptor, + &self.owner, + &self.installation, + now_ns, + )?; + if prior.capture.run_id() != run.capture_run { + return Err("recovery_run_identity_mismatch".into()); + } + } + checkpoint_payload(&bytes, &descriptor, &self.owner, &self.installation, now_ns)?; + // save includes private atomic write, fsync and exact authenticated + // readback under the store lock; errors do not enable persistence. + if let Err(error) = store.save(&descriptor, &bytes, now_ns / 1_000_000) { + // A failed durability/readback barrier can still have installed + // ciphertext. Unknown retention must never be reported as absent. + state.recovery.error = Some(bounded(error.clone())); + return Err(error); + } + state.recovery = RecoveryState::current_run_saved(descriptor, summary, now_ns); + state.recovery_enable = None; + Ok(()) + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_recovery_enable_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_recovery_enable_tests.rs new file mode 100644 index 0000000000..40525b5c99 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_recovery_enable_tests.rs @@ -0,0 +1,405 @@ +//! SYNTHETIC guarded enable without a device, clock change or capture reset. +use super::*; +use crate::room_learner::recovery_store::RECOVERY_TTL_MS; +fn token(rt: &RoomLearningRuntime, now: u64) -> String { + rt.status(now)["recovery"]["enable_current_run"]["run_id"] + .as_str() + .unwrap() + .into() +} +fn ram_run(root: &Path) -> Arc { + let rt = runtime(root, BASE, true); + rt.configure_for_recovery( + &rt.mutation_guard().unwrap(), + context(), + "new-ram".into(), + true, + false, + BASE, + ) + .unwrap(); + rt +} +fn completed(rt: &RoomLearningRuntime) -> u64 { + rt.begin_phase(&rt.mutation_guard().unwrap(), request(12_000), BASE) + .unwrap(); + for q in 0..4 { + window( + rt, + BASE + u64::from(q) * 3_000_000_000 + 1_000_000, + q * 8 + 1, + ); + } + let at = BASE + 12_000_000_000; + rt.finish_phase(&rt.mutation_guard().unwrap(), at, true) + .unwrap(); + at +} +#[test] +fn current_new_run_enable_preserves_exact_rows_history_watermarks_and_cold_restores() { + let dir = tempfile::tempdir().unwrap(); + let rt = ram_run(dir.path()); + let at = completed(&rt); + let id = token(&rt, at); + let before = rt + .inner + .lock() + .unwrap() + .legacy_capture() + .unwrap() + .checkpoint(at) + .unwrap(); + rt.enable_current_capture_recovery(&rt.mutation_guard().unwrap(), &id, true, at) + .unwrap(); + let after = rt + .inner + .lock() + .unwrap() + .legacy_capture() + .unwrap() + .checkpoint(at) + .unwrap(); + assert_eq!( + serde_json::to_vec(&before).unwrap(), + serde_json::to_vec(&after).unwrap() + ); + let status = rt.status(at); + assert_eq!(status["capture"]["sealed_cohorts"], 4); + assert_eq!(status["recovery"]["enabled"], true); + assert!(status["candidate_loaded"] == false); + assert_eq!( + status["recovery"]["saved"]["created_at_ms"], + BASE / 1_000_000 + ); + assert_eq!( + status["recovery"]["saved"]["expires_at_ms"], + BASE / 1_000_000 + RECOVERY_TTL_MS + ); + let cold = runtime(dir.path(), at + 1, true); + cold.restore_capture_recovery(&cold.mutation_guard().unwrap(), &id, &context(), at + 1) + .unwrap(); + assert_eq!(cold.status(at + 1)["capture"]["sealed_cohorts"], 4); + assert!(cold + .enable_current_capture_recovery(&cold.mutation_guard().unwrap(), &id, true, at + 2) + .is_err()); +} +#[test] +fn wrong_ticket_consent_stale_guard_and_active_phase_do_not_enable_or_change_capture() { + let dir = tempfile::tempdir().unwrap(); + let rt = ram_run(dir.path()); + let id = token(&rt, BASE); + let stale = rt.mutation_guard().unwrap(); + assert!(rt + .enable_current_capture_recovery(&stale, &id, false, BASE) + .is_err()); + assert!(rt + .enable_current_capture_recovery(&stale, "wrong", true, BASE) + .is_err()); + assert!(rt + .enable_current_capture_recovery(&stale, &id, true, BASE) + .is_err()); + rt.begin_phase(&rt.mutation_guard().unwrap(), request(12_000), BASE) + .unwrap(); + let before = rt + .inner + .lock() + .unwrap() + .legacy_capture() + .unwrap() + .checkpoint(BASE) + .unwrap(); + assert!(rt + .enable_current_capture_recovery(&rt.mutation_guard().unwrap(), &id, true, BASE) + .is_err()); + assert_eq!( + serde_json::to_vec(&before).unwrap(), + serde_json::to_vec( + &rt.inner + .lock() + .unwrap() + .legacy_capture() + .unwrap() + .checkpoint(BASE) + .unwrap() + ) + .unwrap() + ); + assert!(rt + .recovery_store + .as_ref() + .unwrap() + .inspect() + .unwrap() + .is_none()); +} +#[test] +fn original_new_run_deadline_never_renews_even_after_regressed_time_or_discard() { + let dir = tempfile::tempdir().unwrap(); + let rt = ram_run(dir.path()); + let id = token(&rt, BASE); + let deadline = BASE + RECOVERY_TTL_MS * 1_000_000; + assert!(rt + .enable_current_capture_recovery(&rt.mutation_guard().unwrap(), &id, true, deadline) + .is_err()); + assert!(rt + .enable_current_capture_recovery(&rt.mutation_guard().unwrap(), &id, true, deadline - 1) + .is_err()); + assert_eq!( + rt.status(deadline - 1)["recovery"]["enable_current_run"]["eligible"], + false + ); + let second = tempfile::tempdir().unwrap(); + let rt = ram_run(second.path()); + let id = token(&rt, BASE); + rt.enable_current_capture_recovery(&rt.mutation_guard().unwrap(), &id, true, BASE) + .unwrap(); + rt.discard_capture_recovery(&rt.mutation_guard().unwrap(), &id) + .unwrap(); + assert!(rt + .enable_current_capture_recovery(&rt.mutation_guard().unwrap(), &id, true, BASE + 1) + .is_err()); +} +#[test] +fn foreign_and_expired_slots_are_never_overwritten_and_new_run_stays_intact() { + let dir = tempfile::tempdir().unwrap(); + let rt = runtime(dir.path(), BASE, true); + rt.configure_for_recovery( + &rt.mutation_guard().unwrap(), + context(), + "old".into(), + true, + true, + BASE, + ) + .unwrap(); + let retained = rt + .recovery_store + .as_ref() + .unwrap() + .load(BASE / 1_000_000) + .unwrap() + .unwrap(); + rt.configure_for_recovery( + &rt.mutation_guard().unwrap(), + context(), + "new".into(), + true, + false, + BASE + 1, + ) + .unwrap(); + let id = token(&rt, BASE + 1); + assert!(rt + .enable_current_capture_recovery(&rt.mutation_guard().unwrap(), &id, true, BASE + 2) + .is_err()); + let again = rt + .recovery_store + .as_ref() + .unwrap() + .load(BASE / 1_000_000) + .unwrap() + .unwrap(); + assert_eq!(again.descriptor, retained.descriptor); + assert_eq!(again.plaintext, retained.plaintext); + let later = BASE + (RECOVERY_TTL_MS + 1) * 1_000_000; + rt.configure_for_recovery( + &rt.mutation_guard().unwrap(), + context(), + "later-new".into(), + true, + false, + later, + ) + .unwrap(); + let id = token(&rt, later); + assert!(rt + .enable_current_capture_recovery(&rt.mutation_guard().unwrap(), &id, true, later) + .is_err()); + assert_eq!( + rt.recovery_store + .as_ref() + .unwrap() + .inspect() + .unwrap() + .unwrap(), + retained.descriptor + ); +} +#[test] +fn write_failure_keeps_ram_and_ticket_then_explicit_retry_can_succeed_without_reset() { + let dir = tempfile::tempdir().unwrap(); + let rt = ram_run(dir.path()); + let at = completed(&rt); + let id = token(&rt, at); + let path = dir + .path() + .join("a".repeat(64)) + .join("SensingServer") + .join("room-learner-recovery") + .join("capture-recovery-v1.bin"); + std::fs::create_dir(&path).unwrap(); + let before = serde_json::to_vec( + &rt.inner + .lock() + .unwrap() + .legacy_capture() + .unwrap() + .checkpoint(at) + .unwrap(), + ) + .unwrap(); + assert!(rt + .enable_current_capture_recovery(&rt.mutation_guard().unwrap(), &id, true, at) + .is_err()); + assert!(!rt.inner.lock().unwrap().recovery.enabled); + assert!(rt.status(at)["recovery"]["derived_samples_persisted"].is_null()); + assert_eq!( + serde_json::to_vec( + &rt.inner + .lock() + .unwrap() + .legacy_capture() + .unwrap() + .checkpoint(at) + .unwrap() + ) + .unwrap(), + before + ); + std::fs::remove_dir(&path).unwrap(); + rt.enable_current_capture_recovery(&rt.mutation_guard().unwrap(), &id, true, at + 1) + .unwrap(); + assert!(rt.inner.lock().unwrap().recovery.enabled); +} + +#[test] +fn concurrent_enable_consumes_one_guard_and_cannot_renew_the_ticket() { + let dir = tempfile::tempdir().unwrap(); + let rt = ram_run(dir.path()); + let id = token(&rt, BASE); + let guard = rt.mutation_guard().unwrap(); + let barrier = Arc::new(std::sync::Barrier::new(3)); + let handles: Vec<_> = (0..2) + .map(|_| { + let rt = rt.clone(); + let id = id.clone(); + let guard = guard.clone(); + let barrier = barrier.clone(); + std::thread::spawn(move || { + barrier.wait(); + rt.enable_current_capture_recovery(&guard, &id, true, BASE) + }) + }) + .collect(); + barrier.wait(); + let outcomes: Vec<_> = handles.into_iter().map(|h| h.join().unwrap()).collect(); + assert_eq!(outcomes.iter().filter(|r| r.is_ok()).count(), 1); + let saved = rt + .recovery_store + .as_ref() + .unwrap() + .load(BASE / 1_000_000) + .unwrap() + .unwrap(); + assert_eq!(saved.descriptor.id, id); + assert_eq!( + saved.descriptor.expires_at_ms, + BASE / 1_000_000 + RECOVERY_TTL_MS + ); + assert!(rt + .enable_current_capture_recovery(&rt.mutation_guard().unwrap(), &id, true, BASE + 1) + .is_err()); +} +#[test] +fn partial_committed_same_ticket_is_retried_with_unchanged_original_deadline() { + use super::super::recovery_control::{RuntimeCheckpoint, SCHEMA}; + use crate::room_learner::recovery_store::{RecoveryDescriptor, RECOVERY_SCHEMA}; + let dir = tempfile::tempdir().unwrap(); + let rt = ram_run(dir.path()); + let at = completed(&rt); + let id = token(&rt, at); + let payload = RuntimeCheckpoint { + schema: SCHEMA.into(), + capture: rt + .inner + .lock() + .unwrap() + .legacy_capture() + .unwrap() + .checkpoint(at) + .unwrap(), + frozen_training: false, + job_phase: JobPhase::Idle, + model_sha256: None, + }; + let descriptor = RecoveryDescriptor { + schema: RECOVERY_SCHEMA.into(), + id: id.clone(), + owner_scope: "a".repeat(64), + installation_id: "synthetic-install".into(), + context_digest: context().digest(), + created_at_ms: BASE / 1_000_000, + saved_at_ms: at / 1_000_000, + expires_at_ms: BASE / 1_000_000 + RECOVERY_TTL_MS, + }; + // Model a prior call whose rename committed but whose final barrier failed: + // durable bytes exist, yet the runtime never claimed persistence enabled. + rt.recovery_store + .as_ref() + .unwrap() + .save( + &descriptor, + &serde_json::to_vec(&payload).unwrap(), + at / 1_000_000, + ) + .unwrap(); + assert!(!rt.inner.lock().unwrap().recovery.enabled); + rt.enable_current_capture_recovery(&rt.mutation_guard().unwrap(), &id, true, at + 1_000_000) + .unwrap(); + let stored = rt + .recovery_store + .as_ref() + .unwrap() + .load((at + 1_000_000) / 1_000_000) + .unwrap() + .unwrap(); + assert_eq!(stored.descriptor.created_at_ms, descriptor.created_at_ms); + assert_eq!(stored.descriptor.expires_at_ms, descriptor.expires_at_ms); + assert_eq!(rt.status(at + 1_000_000)["capture"]["sealed_cohorts"], 4); +} +#[test] +fn unavailable_key_and_other_runtime_guard_cannot_enable() { + let dir = tempfile::tempdir().unwrap(); + let rt = runtime(dir.path(), BASE, false); + rt.configure_for_recovery( + &rt.mutation_guard().unwrap(), + context(), + "ram-no-key".into(), + true, + false, + BASE, + ) + .unwrap(); + let id = token(&rt, BASE); + assert!(rt + .enable_current_capture_recovery(&rt.mutation_guard().unwrap(), &id, true, BASE) + .is_err()); + assert!(!rt.inner.lock().unwrap().recovery.enabled); + let otherdir = tempfile::tempdir().unwrap(); + let other = ram_run(otherdir.path()); + assert!(other + .enable_current_capture_recovery( + &rt.mutation_guard().unwrap(), + &token(&other, BASE), + true, + BASE + ) + .is_err()); + assert!(other + .recovery_store + .as_ref() + .unwrap() + .inspect() + .unwrap() + .is_none()); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_recovery_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_recovery_tests.rs new file mode 100644 index 0000000000..deabc9cccb --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_recovery_tests.rs @@ -0,0 +1,849 @@ +//! SYNTHETIC process-loss and disk-failure integration; no real account or clock. +use super::*; +use crate::room_learner::{capture::RawSourceFrame, recovery_store::RecoveryStore}; +const BASE: u64 = 1_800_000_000_000_000_000; +fn runtime(root: &Path, now: u64, key: bool) -> Arc { + let owner = "a".repeat(64); + let path = root.join(&owner).join("SensingServer"); + let seed = [7u8; 32]; + let store = + RecoveryStore::new(&path, &owner, "synthetic-install", key.then_some(&seed)).unwrap(); + Arc::new( + RoomLearningRuntime::new_with_recovery_at(&path, &owner, "synthetic-install", store, now) + .unwrap(), + ) +} +fn context() -> CaptureContext { + super::tests::context() +} +fn request(duration: u64) -> PhaseRequest { + PhaseRequest { + trial_id: "original-empty".into(), + session_id: "original-train".into(), + role: DataRole::Train, + label: RoomLabel::Empty, + duration_ms: duration, + lead_in_ms: 0, + conditions_confirmed: true, + } +} +fn window(rt: &RoomLearningRuntime, at: u64, sequence: u32) { + for row in 0..8 { + for source in 1..=2 { + let values = vec![6.0 + f64::from(source) * 0.3; 8]; + rt.ingest( + RawSourceFrame { + source_id: source, + grid_id: "ht64-canonical8", + frequency_hz: 2_437_000_000, + bandwidth_hz: 20_000_000, + sequence: sequence + row, + amplitudes: &values, + }, + at + u64::from(row) * 10_000_000 + u64::from(source), + ) + .unwrap(); + } + } +} +#[test] +fn completed_six_hundred_second_capture_survives_process_loss_without_auto_confirmation() { + let temp = tempfile::tempdir().unwrap(); + let rt = runtime(temp.path(), BASE, true); + let ctx = context(); + rt.configure_for_recovery( + &rt.mutation_guard().unwrap(), + ctx.clone(), + "original-run".into(), + true, + true, + BASE, + ) + .unwrap(); + rt.begin_phase(&rt.mutation_guard().unwrap(), request(600_000), BASE) + .unwrap(); + for quarter in 0..4 { + window( + &rt, + BASE + u64::from(quarter) * 150_000_000_000 + 1_000_000, + quarter * 8 + 1, + ); + } + let complete = rt.status(BASE + 600_000_000_000); + assert_eq!(complete["recovery"]["saved"]["phase"]["complete"], true); + let id = complete["recovery"]["discard_id"] + .as_str() + .unwrap() + .to_owned(); + let original_created = complete["recovery"]["saved"]["created_at_ms"].clone(); + let original_expiry = complete["recovery"]["saved"]["expires_at_ms"].clone(); + drop(rt); // The only process state holding live/pending buffers is gone. + let cold = runtime(temp.path(), BASE + 601_000_000_000, true); + assert!(cold.context().is_none()); + cold.restore_capture_recovery( + &cold.mutation_guard().unwrap(), + &id, + &ctx, + BASE + 601_000_000_000, + ) + .unwrap(); + let restored = cold.status(BASE + 601_000_000_000); + assert_eq!( + restored["recovery"]["restored_disposition"], + "completed_needs_confirmation" + ); + assert_eq!( + restored["recovery"]["saved"]["created_at_ms"], + original_created + ); + assert_eq!( + restored["recovery"]["saved"]["expires_at_ms"], + original_expiry + ); + assert_eq!(restored["capture"]["sealed_cohorts"], 0); + assert!(restored["capture"]["sources"] + .as_array() + .unwrap() + .iter() + .all(|s| s["live_buffer_frames"] == 0)); + assert!(cold + .finish_phase( + &cold.mutation_guard().unwrap(), + BASE + 601_000_000_000, + true + ) + .is_err()); + window(&cold, BASE + 601_000_000_001, 33); + cold.finish_phase( + &cold.mutation_guard().unwrap(), + BASE + 601_100_000_000, + true, + ) + .unwrap(); + assert_eq!( + cold.status(BASE + 601_100_000_000)["capture"]["sealed_cohorts"], + 4 + ); + assert_eq!(restored["raw_iq_persisted"], false); + assert_eq!(restored["derived_samples_persisted"], true); + assert_eq!(restored["physically_validated"], false); +} +#[test] +fn unavailable_key_blocks_opt_in_without_starting_capture_and_ram_remains_explicit() { + let temp = tempfile::tempdir().unwrap(); + let rt = runtime(temp.path(), BASE, false); + assert!(rt + .configure_for_recovery( + &rt.mutation_guard().unwrap(), + context(), + "run".into(), + true, + true, + BASE + ) + .is_err()); + assert!(rt.context().is_none()); + assert_eq!(rt.status(BASE)["recovery"]["state"], "unavailable"); + rt.configure_for_recovery( + &rt.mutation_guard().unwrap(), + context(), + "ram".into(), + true, + false, + BASE, + ) + .unwrap(); + assert_eq!( + rt.status(BASE)["untrained_capture_progress_is_volatile"], + true + ); +} +#[test] +fn failed_initial_save_is_visible_and_blocks_long_capture_until_explicit_ram_configure() { + let temp = tempfile::tempdir().unwrap(); + let rt = runtime(temp.path(), BASE, true); + let dir = temp + .path() + .join("a".repeat(64)) + .join("SensingServer/room-learner-recovery"); + // A conflicting non-file destination is an owned fixture only. + let file = dir.join("capture-recovery-v1.bin"); + std::fs::create_dir(&file).unwrap(); + assert!(rt + .configure_for_recovery( + &rt.mutation_guard().unwrap(), + context(), + "blocked".into(), + true, + true, + BASE + ) + .is_err()); + assert!(rt + .begin_phase(&rt.mutation_guard().unwrap(), request(600_000), BASE) + .is_err()); + assert_ne!(rt.status(BASE)["recovery"]["state"], "saved"); + assert!(file.is_dir()); +} +#[test] +fn discard_fences_later_status_and_ingress_writes_and_does_not_remove_models() { + let temp = tempfile::tempdir().unwrap(); + let rt = runtime(temp.path(), BASE, true); + rt.configure_for_recovery( + &rt.mutation_guard().unwrap(), + context(), + "retained".into(), + true, + true, + BASE, + ) + .unwrap(); + let id = rt.status(BASE)["recovery"]["discard_id"] + .as_str() + .unwrap() + .to_owned(); + rt.discard_capture_recovery(&rt.mutation_guard().unwrap(), &id) + .unwrap(); + window(&rt, BASE + 100_000_000, 1); + let status = rt.status(BASE + 200_000_000); + assert_eq!(status["recovery"]["enabled"], false); + assert!(status["recovery"]["saved"].is_null()); + let owner = "a".repeat(64); + let path = temp.path().join(&owner).join("SensingServer"); + assert!( + RecoveryStore::new(&path, &owner, "synthetic-install", Some(&[7; 32])) + .unwrap() + .inspect() + .unwrap() + .is_none() + ); +} + +#[cfg(unix)] +#[test] +fn failed_completion_checkpoint_retries_after_bounded_backoff_with_original_expiry() { + use std::os::unix::fs::PermissionsExt; + let temp = tempfile::tempdir().unwrap(); + let rt = runtime(temp.path(), BASE, true); + rt.configure_for_recovery( + &rt.mutation_guard().unwrap(), + context(), + "retry-run".into(), + true, + true, + BASE, + ) + .unwrap(); + rt.begin_phase(&rt.mutation_guard().unwrap(), request(600_000), BASE) + .unwrap(); + for quarter in 0..4 { + window( + &rt, + BASE + u64::from(quarter) * 150_000_000_000 + 1_000_000, + quarter * 8 + 1, + ); + } + let expiry = rt.status(BASE + 450_100_000_000)["recovery"]["saved"]["expires_at_ms"].clone(); + let file = temp + .path() + .join("a".repeat(64)) + .join("SensingServer/room-learner-recovery/capture-recovery-v1.bin"); + std::fs::set_permissions(&file, std::fs::Permissions::from_mode(0o644)).unwrap(); + let failed = rt.status(BASE + 600_000_000_000); + assert_eq!(failed["recovery"]["state"], "error"); + assert_persisted_flags(&failed, Value::Null); + assert_eq!(failed["recovery"]["saved"]["phase"]["complete"], false); + std::fs::set_permissions(&file, std::fs::Permissions::from_mode(0o600)).unwrap(); + assert_eq!( + rt.status(BASE + 601_000_000_000)["recovery"]["state"], + "error" + ); + let retried = rt.status(BASE + 605_000_000_000); + assert_eq!(retried["recovery"]["state"], "saved"); + assert_eq!(retried["recovery"]["saved"]["phase"]["complete"], true); + assert_eq!(retried["recovery"]["saved"]["expires_at_ms"], expiry); + rt.finish_phase(&rt.mutation_guard().unwrap(), BASE + 605_000_000_000, true) + .unwrap(); + assert_eq!( + rt.status(BASE + 605_000_000_000)["capture"]["sealed_cohorts"], + 4 + ); +} +#[test] +fn expired_slot_is_summary_redacted_but_explicitly_discardable() { + let temp = tempfile::tempdir().unwrap(); + let rt = runtime(temp.path(), BASE, true); + rt.configure_for_recovery( + &rt.mutation_guard().unwrap(), + context(), + "expiry-run".into(), + true, + true, + BASE, + ) + .unwrap(); + drop(rt); + let cold = runtime(temp.path(), BASE + 86_400_000_000_000, true); + let status = cold.status(BASE + 86_400_000_000_000); + assert_eq!(status["recovery"]["state"], "expired"); + assert!(status["recovery"]["saved"].is_null()); + let id = status["recovery"]["discard_id"].as_str().unwrap(); + assert!(cold + .saved_recovery_context(id, BASE + 86_400_000_000_000) + .is_err()); + cold.discard_capture_recovery(&cold.mutation_guard().unwrap(), id) + .unwrap(); + assert!(cold.status(BASE + 86_400_000_000_000)["recovery"]["discard_id"].is_null()); +} + +#[tokio::test] +async fn trained_checkpoint_cold_restores_exact_candidate_without_live_buffers_or_auto_fit() { + let temp = tempfile::tempdir().unwrap(); + let now = super::tests::START; + let rt = runtime(temp.path(), now, true); + let ctx = context(); + rt.configure_for_recovery( + &rt.mutation_guard().unwrap(), + ctx.clone(), + "fit-run".into(), + true, + true, + now, + ) + .unwrap(); + super::tests::train_phases(&rt); + rt.start_training( + &rt.mutation_guard().unwrap(), + FitConfig { + epochs: 4, + ..FitConfig::default() + }, + ) + .unwrap(); + let ready = super::tests::finished(&rt).await; + assert_eq!(ready["state"], "ready", "{ready}"); + let id = ready["recovery"]["discard_id"].as_str().unwrap().to_owned(); + let expected = + super::recovery_control::model_sha(rt.inner.lock().unwrap().model.as_ref().unwrap()) + .unwrap(); + // Move today's candidate pointer after the recovery was saved. Recovery + // must still load the immutable candidate bound by its original hash. + let dataset = rt + .inner + .lock() + .unwrap() + .legacy_capture() + .unwrap() + .validated_dataset() + .unwrap(); + let (replacement, _) = RoomModel::fit_with_cancel( + &dataset, + FitConfig { + epochs: 1, + ..FitConfig::default() + }, + &AtomicBool::new(false), + ) + .unwrap(); + assert_ne!( + super::recovery_control::model_sha(&replacement).unwrap(), + expected + ); + rt.store.save_candidate(&replacement).unwrap(); + drop(rt); + let cold = runtime(temp.path(), now + 61_000_000_000, true); + cold.restore_capture_recovery( + &cold.mutation_guard().unwrap(), + &id, + &ctx, + now + 61_000_000_000, + ) + .unwrap(); + let status = cold.status(now + 61_000_000_000); + assert_eq!(status["state"], "ready"); + assert_eq!(status["training_frozen"], true); + assert_eq!(status["capture"]["sealed_cohorts"], 16); + assert_eq!( + super::recovery_control::model_sha(cold.inner.lock().unwrap().model.as_ref().unwrap()) + .unwrap(), + expected + ); + assert!(cold.prediction(now + 61_000_000_000).is_err()); + assert!(cold + .begin_phase( + &cold.mutation_guard().unwrap(), + request(12_000), + now + 61_000_000_000 + ) + .is_err()); + assert_eq!(status["physically_validated"], false); +} +#[test] +fn interrupted_fit_checkpoint_restores_failed_frozen_dataset_without_job() { + let temp = tempfile::tempdir().unwrap(); + let now = super::tests::START; + let rt = runtime(temp.path(), now, true); + let ctx = context(); + rt.configure_for_recovery( + &rt.mutation_guard().unwrap(), + ctx.clone(), + "interrupted-fit".into(), + true, + true, + now, + ) + .unwrap(); + super::tests::train_phases(&rt); + // Exact durable boundary immediately before handing work to the fitter. + { + let mut state = rt.inner.lock().unwrap(); + state.frozen_training = true; + state.job.phase = JobPhase::Running; + rt.checkpoint_recovery(&mut state, now + 60_000_000_000, true) + .unwrap(); + } + let id = rt.status(now + 60_000_000_000)["recovery"]["discard_id"] + .as_str() + .unwrap() + .to_owned(); + drop(rt); + let cold = runtime(temp.path(), now + 61_000_000_000, true); + cold.restore_capture_recovery( + &cold.mutation_guard().unwrap(), + &id, + &ctx, + now + 61_000_000_000, + ) + .unwrap(); + let status = cold.status(now + 61_000_000_000); + assert_eq!(status["state"], "failed"); + assert_eq!(status["training_frozen"], true); + assert_eq!(status["capture"]["sealed_cohorts"], 16); + assert!(cold.inner.lock().unwrap().model.is_none()); +} + +#[test] +fn partial_phase_restores_restart_required_and_ram_run_preserves_saved_slot() { + let temp = tempfile::tempdir().unwrap(); + let rt = runtime(temp.path(), BASE, true); + let ctx = context(); + rt.configure_for_recovery( + &rt.mutation_guard().unwrap(), + ctx.clone(), + "partial-run".into(), + true, + true, + BASE, + ) + .unwrap(); + rt.begin_phase(&rt.mutation_guard().unwrap(), request(600_000), BASE) + .unwrap(); + window(&rt, BASE + 1_000_000, 1); + let id = rt.status(BASE + 100_000_000)["recovery"]["discard_id"] + .as_str() + .unwrap() + .to_owned(); + drop(rt); + let cold = runtime(temp.path(), BASE + 1_000_000_000, true); + cold.configure_for_recovery( + &cold.mutation_guard().unwrap(), + ctx.clone(), + "explicit-ram".into(), + true, + false, + BASE + 1_000_000_000, + ) + .unwrap(); + assert_eq!( + cold.status(BASE + 1_000_000_000)["recovery"]["discard_id"], + id + ); + cold.restore_capture_recovery( + &cold.mutation_guard().unwrap(), + &id, + &ctx, + BASE + 1_000_000_000, + ) + .unwrap(); + let s = cold.status(BASE + 1_000_000_000); + assert_eq!( + s["recovery"]["restored_disposition"], + "interrupted_needs_restart" + ); + assert_eq!(s["capture"]["sealed_cohorts"], 0); + assert!(s["capture"]["phase"].is_null()); + assert_eq!(s["recovery"]["saved"]["phase"], Value::Null); +} +#[test] +fn failed_erase_is_error_and_disables_future_writer_without_claiming_data_removed() { + use std::os::unix::fs::PermissionsExt; + let temp = tempfile::tempdir().unwrap(); + let rt = runtime(temp.path(), BASE, true); + rt.configure_for_recovery( + &rt.mutation_guard().unwrap(), + context(), + "erase-fail".into(), + true, + true, + BASE, + ) + .unwrap(); + let id = rt.status(BASE)["recovery"]["discard_id"] + .as_str() + .unwrap() + .to_owned(); + let file = temp + .path() + .join("a".repeat(64)) + .join("SensingServer/room-learner-recovery/capture-recovery-v1.bin"); + std::fs::set_permissions(&file, std::fs::Permissions::from_mode(0o644)).unwrap(); + assert!(rt + .discard_capture_recovery(&rt.mutation_guard().unwrap(), &id) + .is_err()); + let s = rt.status(BASE + 10_000_000_000); + assert_eq!(s["recovery"]["enabled"], false); + assert_eq!(s["recovery"]["state"], "error"); + assert_eq!(s["recovery"]["discard_id"], id); + assert!(file.exists()); + std::fs::set_permissions(&file, std::fs::Permissions::from_mode(0o600)).unwrap(); + rt.discard_capture_recovery(&rt.mutation_guard().unwrap(), &id) + .unwrap(); + assert!(!file.exists()); +} + +fn assert_persisted_flags(status: &Value, expected: Value) { + assert_eq!(status["derived_samples_persisted"], expected, "{status}"); + assert_eq!(status["raw_observations_persisted"], expected); + assert_eq!(status["recovery"]["derived_samples_persisted"], expected); + assert_eq!(status["raw_iq_persisted"], false); + assert_eq!(status["recovery"]["raw_iq_persisted"], false); +} +fn assert_unverified_retained_contents(case: &str) { + let temp = tempfile::tempdir().unwrap(); + let rt = runtime(temp.path(), BASE, true); + rt.configure_for_recovery( + &rt.mutation_guard().unwrap(), + context(), + "retained-flags".into(), + true, + true, + BASE, + ) + .unwrap(); + rt.begin_phase(&rt.mutation_guard().unwrap(), request(600_000), BASE) + .unwrap(); + window(&rt, BASE + 1_000_000, 1); + assert_persisted_flags(&rt.status(BASE + 100_000_000), json!(true)); + drop(rt); + let owner = "a".repeat(64); + let path = temp.path().join(&owner).join("SensingServer"); + let file = path.join("room-learner-recovery/capture-recovery-v1.bin"); + if case == "corrupt" { + let mut bytes = std::fs::read(&file).unwrap(); + *bytes.last_mut().unwrap() ^= 1; + std::fs::write(&file, bytes).unwrap(); + } + let retained = std::fs::read(&file).unwrap(); + let now = if case == "expired" { + BASE + 86_400_000_000_000 + } else { + BASE + 1_000_000_000 + }; + let seed = if case == "wrong_key" { + [8; 32] + } else { + [7; 32] + }; + let store = RecoveryStore::new( + &path, + &owner, + "synthetic-install", + (case != "missing_key").then_some(&seed), + ) + .unwrap(); + let cold = + RoomLearningRuntime::new_with_recovery_at(&path, &owner, "synthetic-install", store, now) + .unwrap(); + let status = cold.status(now); + assert_persisted_flags(&status, Value::Null); + assert!(status["recovery"]["error"].is_string()); + assert!(status["recovery"]["saved"].is_null()); + assert_eq!( + std::fs::read(&file).unwrap(), + retained, + "{case} altered retained encrypted bytes" + ); +} +#[test] +fn persisted_flags_expired_contents_are_unknown_and_retained() { + assert_unverified_retained_contents("expired"); +} +#[test] +fn persisted_flags_corrupt_contents_are_unknown_and_retained() { + assert_unverified_retained_contents("corrupt"); +} +#[test] +fn persisted_flags_wrong_key_contents_are_unknown_and_retained() { + assert_unverified_retained_contents("wrong_key"); +} +#[test] +fn persisted_flags_missing_key_contents_are_unknown_and_retained() { + assert_unverified_retained_contents("missing_key"); +} +#[test] +fn persisted_flags_confirmed_empty_discard_and_default_ram_are_false() { + let temp = tempfile::tempdir().unwrap(); + let ram = super::tests::runtime(temp.path()); + assert_persisted_flags(&ram.status(BASE), json!(false)); + drop(ram); + let rt = runtime(temp.path(), BASE, true); + assert_persisted_flags(&rt.status(BASE), json!(false)); + rt.configure_for_recovery( + &rt.mutation_guard().unwrap(), + context(), + "verified-empty".into(), + true, + true, + BASE, + ) + .unwrap(); + let empty = rt.status(BASE); + assert_persisted_flags(&empty, json!(false)); + let id = empty["recovery"]["discard_id"].as_str().unwrap(); + rt.begin_phase(&rt.mutation_guard().unwrap(), request(600_000), BASE) + .unwrap(); + window(&rt, BASE + 1_000_000, 1); + assert_persisted_flags(&rt.status(BASE + 100_000_000), json!(true)); + rt.discard_capture_recovery(&rt.mutation_guard().unwrap(), id) + .unwrap(); + assert_persisted_flags(&rt.status(BASE), json!(false)); +} + +fn recovery_file(root: &Path) -> std::path::PathBuf { + root.join("a".repeat(64)).join("SensingServer/room-learner-recovery/capture-recovery-v1.bin") +} +fn expiry_request(role: DataRole, label: RoomLabel, index: u32) -> PhaseRequest { + PhaseRequest { + trial_id: format!("expiry-trial-{index}"), session_id: format!("expiry-session-{index}"), + role, label, duration_ms: 12_000, lead_in_ms: 0, conditions_confirmed: true, + } +} +fn expiry_phase(rt: &RoomLearningRuntime, at: u64, role: DataRole, label: RoomLabel, index: u32) { + rt.begin_phase(&rt.mutation_guard().unwrap(), expiry_request(role, label, index), at).unwrap(); + let gain = match label { RoomLabel::Empty => 2.0, RoomLabel::OccupiedStatic => 6.0, + RoomLabel::OccupiedMoving => 7.0, RoomLabel::AdjacentOnly => 2.3, RoomLabel::Unknown => panic!("label required") }; + for quarter in 0..4u32 { + for sample in 0..8u32 { + for source in 1..=2u8 { + let values: Vec<_> = (0..8).map(|bin| gain + f64::from(source)*0.3 + f64::from(bin)*0.04 + f64::from(sample)*0.003).collect(); + rt.ingest(RawSourceFrame {source_id: source, grid_id: "ht64-canonical8", + frequency_hz: 2_437_000_000, bandwidth_hz: 20_000_000, + sequence: index*32 + quarter*8 + sample + 1, amplitudes: &values}, + at + u64::from(quarter)*3_000_000_000 + 1_000_000 + u64::from(sample)*10_000_000 + u64::from(source)).unwrap(); + } + } + } + rt.finish_phase(&rt.mutation_guard().unwrap(), at + 12_000_000_000, true).unwrap(); +} +fn original_descriptor(root: &Path) -> crate::room_learner::recovery_store::RecoveryDescriptor { + let owner = "a".repeat(64); + RecoveryStore::new(&root.join(&owner).join("SensingServer"), &owner, "synthetic-install", Some(&[7;32])) + .unwrap().inspect().unwrap().unwrap() +} + +#[test] +fn loaded_recovery_expires_to_ram_at_exact_deadline_without_erasing_or_rebasing() { + let temp = tempfile::tempdir().unwrap(); + let rt = runtime(temp.path(), BASE, true); + rt.configure_for_recovery(&rt.mutation_guard().unwrap(), context(), "expiry-loaded".into(), true, true, BASE).unwrap(); + for index in 0..7 { expiry_phase(&rt, BASE + u64::from(index)*13_000_000_000, DataRole::Train, RoomLabel::Empty, index); } + let descriptor = original_descriptor(temp.path()); + drop(rt); + let recovered = runtime(temp.path(), BASE + 100_000_000_000, true); + recovered.restore_capture_recovery(&recovered.mutation_guard().unwrap(), &descriptor.id, &context(), BASE + 100_000_000_000).unwrap(); + let descriptor = original_descriptor(temp.path()); + let expires = descriptor.expires_at_ms*1_000_000; + let file = recovery_file(temp.path()); + let bytes = std::fs::read(&file).unwrap(); + let before = recovered.status(expires - 1); + assert_eq!(before["recovery"]["enabled"], true); + assert_eq!(before["capture"]["sealed_cohorts"], 28); + let (rows, history) = { + let state = recovered.inner.lock().unwrap();let capture = state.legacy_capture().unwrap(); + (serde_json::to_vec(capture.cohorts()).unwrap(), serde_json::to_vec(capture.history()).unwrap()) + }; + let at = recovered.status(expires); + assert_eq!(at["recovery"]["state"], "expired");assert_eq!(at["recovery"]["enabled"], false); + assert!(at["recovery"]["saved"].is_null());assert_eq!(at["capture"]["sealed_cohorts"], 28); + assert_eq!(at["capture"]["sources"], before["capture"]["sources"]); + assert!(at["recovery"]["error"].as_str().unwrap().contains("memory only")); + assert!(recovered.inner.lock().unwrap().recovery.error.is_none(), "informational expiry must not become a persistence error"); + assert_eq!(original_descriptor(temp.path()), descriptor);assert_eq!(std::fs::read(&file).unwrap(), bytes); + let values = vec![6.0;8]; + assert!(recovered.ingest(RawSourceFrame{source_id:1,grid_id:"ht64-canonical8",frequency_hz:2_437_000_000, + bandwidth_hz:20_000_000,sequence:224,amplitudes:&values},expires+1).unwrap_err().contains("replayed")); + assert!(recovered.begin_phase(&recovered.mutation_guard().unwrap(), expiry_request(DataRole::Validation,RoomLabel::Empty,0), expires+2).is_err(), "frozen phase roles must survive expiry"); + expiry_phase(&recovered,expires+1_000_000,DataRole::Train,RoomLabel::Empty,7); + let after = recovered.status(expires+13_000_000_000); + assert_eq!(after["capture"]["sealed_cohorts"],32);assert!(after["capture"]["phase"].is_null()); + assert_eq!(after["capture"]["sources"][0]["last_accepted_sequence"],256); + { + let state=recovered.inner.lock().unwrap();let capture=state.legacy_capture().unwrap(); + assert_eq!(serde_json::to_vec(&capture.cohorts()[..28]).unwrap(),rows); + assert_eq!(serde_json::to_vec(&capture.history()[..7]).unwrap(),history); + assert_eq!(capture.cohorts()[28].views[0].sequence_ids[0],225); + } + assert_eq!(original_descriptor(temp.path()),descriptor);assert_eq!(std::fs::read(&file).unwrap(),bytes); + drop(recovered); + let cold=runtime(temp.path(),expires+14_000_000_000,true); + assert!(cold.context().is_none()); + assert_eq!(cold.restore_capture_recovery(&cold.mutation_guard().unwrap(),&descriptor.id,&context(),expires+14_000_000_000).unwrap_err(),"recovery_expired"); + assert_eq!(std::fs::read(&file).unwrap(),bytes); +} + +#[test] +fn active_phase_can_cross_expiry_and_seal_without_a_false_failed_save() { + let temp=tempfile::tempdir().unwrap();let rt=runtime(temp.path(),BASE,true); + rt.configure_for_recovery(&rt.mutation_guard().unwrap(),context(),"crossing-phase".into(),true,true,BASE).unwrap(); + let descriptor=original_descriptor(temp.path());let expires=descriptor.expires_at_ms*1_000_000; + let start=expires-6_000_000_000; + rt.begin_phase(&rt.mutation_guard().unwrap(),request(12_000),start).unwrap(); + window(&rt,start+1_000_000,1);window(&rt,start+3_001_000_000,9); + let bytes=std::fs::read(recovery_file(temp.path())).unwrap();let final_saved=original_descriptor(temp.path()); + window(&rt,expires+1_000_000,17);window(&rt,expires+3_001_000_000,25); + assert_eq!(rt.status(expires+4_000_000_000)["capture"]["sealed_cohorts"],0,"expiry must not auto-confirm a phase"); + rt.finish_phase(&rt.mutation_guard().unwrap(),expires+6_000_000_000,true).unwrap(); + let status=rt.status(expires+7_000_000_000); + assert_eq!(status["capture"]["sealed_cohorts"],4);assert!(status["capture"]["phase"].is_null()); + assert_eq!(status["recovery"]["enabled"],false);assert_eq!(original_descriptor(temp.path()),final_saved); + assert_eq!(std::fs::read(recovery_file(temp.path())).unwrap(),bytes); +} + +#[test] +fn expiry_does_not_turn_io_tamper_missing_or_replacement_failures_into_ram_fallback() { + use std::os::unix::fs::PermissionsExt; + for case in ["latched_io", "tamper", "missing", "replacement"] { + let temp=tempfile::tempdir().unwrap();let rt=runtime(temp.path(),BASE,true); + rt.configure_for_recovery(&rt.mutation_guard().unwrap(),context(),format!("expiry-{case}"),true,true,BASE).unwrap(); + let original=original_descriptor(temp.path());let expires=original.expires_at_ms*1_000_000; + let file=recovery_file(temp.path()); + match case { + "latched_io"=>{ + std::fs::set_permissions(&file,std::fs::Permissions::from_mode(0o644)).unwrap(); + {let mut state=rt.inner.lock().unwrap();assert!(rt.checkpoint_recovery(&mut state,BASE+1_000_000,true).is_err());} + std::fs::set_permissions(&file,std::fs::Permissions::from_mode(0o600)).unwrap(); + }, + "tamper"=>{let mut bytes=std::fs::read(&file).unwrap();let last=bytes.len()-1;bytes[last]^=1;std::fs::write(&file,bytes).unwrap();}, + "missing"=>std::fs::remove_file(&file).unwrap(), + "replacement"=>{ + let owner="a".repeat(64);let store=RecoveryStore::new(&temp.path().join(&owner).join("SensingServer"),&owner,"synthetic-install",Some(&[7;32])).unwrap(); + store.discard(&original.id).unwrap();let mut other=original.clone();other.id="different-authenticated-copy".into();store.save(&other,b"SYNTHETIC REPLACEMENT",BASE/1_000_000).unwrap(); + },_=>unreachable!() + } + let bytes=std::fs::read(&file).ok(); + assert!(rt.begin_phase(&rt.mutation_guard().unwrap(),request(12_000),expires).is_err(),"{case}"); + let status=rt.status(expires+1); + assert!(status["capture"]["phase"].is_null(),"{case}: failed preflight must not start a phase"); + assert_eq!(status["recovery"]["enabled"],true,"{case}: generic error cannot disable recovery silently"); + assert!(!status["recovery"]["error"].as_str().unwrap().contains("memory only")); + assert_eq!(std::fs::read(&file).ok(),bytes,"{case}: expiry must not rewrite/delete a failed copy"); + } +} + +#[tokio::test] +async fn recovered_28_cohorts_can_finish_heldout_after_expiry_under_the_same_qualification_policy() { + let temp=tempfile::tempdir().unwrap();let rt=runtime(temp.path(),BASE,true); + rt.configure_for_recovery(&rt.mutation_guard().unwrap(),context(),"expiry-qualified".into(),true,true,BASE).unwrap(); + for (i,label) in [RoomLabel::Empty,RoomLabel::OccupiedStatic,RoomLabel::OccupiedMoving,RoomLabel::AdjacentOnly].into_iter().enumerate(){ + expiry_phase(&rt,BASE+i as u64*13_000_000_000,DataRole::Train,label,i as u32); + } + rt.start_training(&rt.mutation_guard().unwrap(),FitConfig{epochs:8,learning_rate:0.5,hidden_dim:2,embedding_dim:2,temporal_tokens:2,..FitConfig::default()}).unwrap(); + let wait_started=std::time::Instant::now(); + loop { + let ready=rt.status(BASE+51_000_000_000); + if ready["state"]=="ready" {break;} + assert!(ready["state"]=="running" && wait_started.elapsed(), + pub(super) temporal:Option>, + pub(super) withdrawal:Option, + model: Option>, + qualification: Option, + pub(super) error: Option, + assessment: Option, +} +impl SelectionState { + pub(super) fn known_revision(&self)->u64{self.temporal.as_ref().map(|s|s.stored.identity().selection_revision).or_else(||self.envelope.as_ref().map(|e|e.receipt.revision)).unwrap_or(0)} + pub(super) fn install_negative_temporal_tombstone(&mut self,envelope:SelectionEnvelope){self.envelope=Some(envelope);self.model=None;self.qualification=None;self.assessment=None;} + + pub(super) fn blocks_new_temporal_capture(&self)->bool{self.temporal.is_some()||self.envelope.as_ref().is_some_and(|e|e.receipt.selected.is_some())||self.error.is_some()} + pub(super) fn has_history(&self) -> bool { + self.temporal.is_some() || self.envelope.is_some() || self.model.is_some() || self.qualification.is_some() + || self.assessment.is_some() || self.error.is_some() + } +} +impl RoomLearningRuntime { + pub(super) fn restore_selection(&self, state: &mut Inner) { + let restored = (|| -> Result<(), String> { + let Some(envelope) = self.store.load_selection()? else { + return Ok(()); + }; + state.selection.envelope = Some(envelope.clone()); + if let Some(target) = &envelope.receipt.selected { + let (model, q) = self.store.resolve_selection_target(target)?; + let model = Arc::new(model); + state.capture = Some(CaptureWorkflow::Legacy(if let Some(proof)=self.store.load_managed_training_proof(&model)? { + let fence=self.store.load_managed_capture_fence(&model,&proof)?; + CaptureSession::resume_managed_fence(&proof,&fence)? + } else {CaptureSession::new(target.context.clone(),format!("selected-{}",envelope.receipt.revision))?})); + state.model = Some(model.clone()); + state.frozen_training = true; + state.job.phase = JobPhase::Ready; + state.qualification = self.store.load_latest_qualification(&model)?; + state.selection.assessment = self.store.load_latest_assessment(&model)?; + state.selection.model = Some(model); + state.selection.qualification = Some(q); + } + Ok(()) + })(); + if let Err(e) = restored { + state.selection.error = Some(bounded(e)); + } + } + pub fn qualify(&self, expected: &MutationGuard) -> Result { + let mut state = self.command_state(expected)?; + state.require_legacy_kind()?; + let result = (|| -> Result { + if busy(&state.job) || !state.frozen_training { + return Err("wait for the frozen candidate".into()); + } + let model = state.model.as_ref().ok_or("no candidate")?.clone(); + let capture = state.legacy_capture().ok_or("no capture context")?; + if capture.phase_active() { + return Err("finish the current capture phase first".into()); + } + let _policy = crate::room_learner::qualification::QualificationPolicy::default(); + let dataset = capture.validated_dataset()?; + let evaluation = self + .store + .evaluate_candidate(&model, &dataset, DataRole::HeldOut)?; + // Incomplete data or missing lineage is not a completed failed assessment. + let proof=self.store.load_managed_training_proof(&model)?; + let fence=if let Some(proof)=&proof { + let fence=self.store.load_managed_capture_fence(&model,proof)?; + let current=capture.managed_fence_seed(proof)?; + current.ensure_extends(&fence.seed)?; + if current.phase_claims!=fence.seed.phase_claims {return Err("managed sealed phases are not durably published".into());} + Some(fence) + } else {None}; + let evidence = crate::room_learner::qualification::managed_evidence_with_fence( + &model, + &dataset, + capture.history(), + &evaluation, + proof.as_ref(), + fence.as_ref().map(|f|f.fence_sha256.as_str()), + )?; + let outcome = + QualificationEnvelope::issue_with_proof(&model, &dataset, capture.history(), &evaluation,proof.as_ref(),fence.as_ref().map(|f|f.fence_sha256.as_str())); + let assessment = crate::room_learner::assessment::AssessmentEnvelope::issue( + &model, + &evaluation, + &evidence, + outcome.as_ref().ok(), + )?; + state.last_evaluation = Some(super::evaluation_control::presentation( + &evaluation, + false, + !dataset.train().is_empty(), + )?); + state.evaluation_warning = None; + if let Ok(q) = &outcome { + self.store.save_qualification(&model, q)?; + } + if let Err(e) = self.store.save_assessment(&model, &assessment) { + if state + .selection + .envelope + .as_ref() + .and_then(|e| e.receipt.selected.as_ref()) + .is_some_and(|t| t.model_sha256 == evaluation.model_sha256) + { + state.selection.error=Some("Latest qualification persistence could not be verified; restart and inspect retained evidence.".into()); + } + return Err(e); + } + if state + .selection + .envelope + .as_ref() + .and_then(|e| e.receipt.selected.as_ref()) + .is_some_and(|t| t.model_sha256 == evaluation.model_sha256) + { + state.selection.assessment = Some(assessment); + } + outcome + })(); + match result { + Ok(q) => { + state.qualification = Some(q.clone()); + state.qualification_error = None; + Ok(q) + } + Err(e) => { + state.qualification = None; + state.qualification_error = Some(bounded(e.clone())); + Err(e) + } + } + } + fn publish_selection( + &self, + state: &mut Inner, + target: Option, + previous: Option, + ) -> Result { + let resolved = target + .as_ref() + .map(|t| self.store.resolve_selection_target(t)) + .transpose()?; + let assessment = resolved + .as_ref() + .map(|(model, q)| self.store.current_qualification_assessment(model, q)) + .transpose()?; + let old_revision = state + .selection + .envelope + .as_ref() + .map_or(0, |s| s.receipt.revision); + let receipt = SelectionReceipt { + schema: "ruview.room-learner.model-selection.v1".into(), + owner_scope: self.owner.clone(), + installation_id: self.installation.clone(), + revision: old_revision + .checked_add(1) + .ok_or("selection revision exhausted")?, + selected_at_unix_ms: self.receiver_now_ns() / 1_000_000, + selected: target, + previous, + }; + let envelope = SelectionEnvelope { + receipt_sha256: hash(&receipt)?, + receipt, + }; + if let Err(error) = self.store.save_selection(&envelope, old_revision) { + state.selection.error=Some("Selection persistence outcome is unavailable; restart and verify the saved selection before continuing.".into()); + return Err(error); + } + // Only publish in-memory authority after durable pointer and readback succeed. + state.selection = SelectionState { + temporal:None, + withdrawal:None, + envelope: Some(envelope.clone()), + model: resolved.as_ref().map(|(m, _)| Arc::new(m.clone())), + qualification: resolved.map(|(_, q)| q), + error: None, + assessment, + }; + Ok(envelope) + } + pub fn select_model( + &self, + expected: &MutationGuard, + model_sha256: &str, + qualification_sha256: &str, + ) -> Result { + let mut state = self.command_state(expected)?; + state.require_legacy_kind()?; + if busy(&state.job) || state.legacy_capture().is_some_and(|c| c.phase_active()) { + return Err("finish active capture/training before selecting".into()); + } + let model = state + .model + .as_ref() + .ok_or("no current candidate to select")?; + let sha = format!("{:x}", Sha256::digest(model.export()?)); + if sha != model_sha256 { + return Err("candidate changed before selection".into()); + } + let qualification = self.store.load_qualification(model, qualification_sha256)?; + self.store + .current_qualification_assessment(model, &qualification)?; + let target = SelectionTarget { + context: model.context().clone(), + model_sha256: sha, + model_version: model.version, + qualification_sha256: qualification_sha256.into(), + }; + let previous = state + .selection + .envelope + .as_ref() + .and_then(|e| e.receipt.selected.clone()); + if previous.as_ref() == Some(&target) { + if state.selection.error.is_some() + || self.store.load_selection()?.as_ref() != state.selection.envelope.as_ref() + { + state.selection.error = Some( + "Saved selection differs from runtime; restart before selecting again.".into(), + ); + return Err("saved selection changed".into()); + } + return Ok(state + .selection + .envelope + .as_ref() + .ok_or("selection unavailable")? + .clone()); + } + self.publish_selection(&mut state, Some(target), previous) + } + pub fn deselect_model(&self, expected: &MutationGuard) -> Result { + let mut state = self.command_state(expected)?; + state.require_legacy_kind()?; + if busy(&state.job) || state.legacy_capture().is_some_and(|c| c.phase_active()) { + return Err("finish active capture/training before changing selection".into()); + } + if let Some(e) = &state.selection.envelope { + if e.receipt.selected.is_none() && state.selection.error.is_none() { + if self.store.load_selection()?.as_ref() != Some(e) { + return Err("saved selection changed".into()); + } + return Ok(e.clone()); + } + } + let previous = state + .selection + .envelope + .as_ref() + .and_then(|e| e.receipt.selected.clone()); + self.publish_selection(&mut state, None, previous) + } + pub fn rollback_model(&self, expected: &MutationGuard) -> Result { + let mut state = self.command_state(expected)?; + state.require_legacy_kind()?; + if busy(&state.job) || state.legacy_capture().is_some_and(|c| c.phase_active()) { + return Err("finish active capture/training before rollback".into()); + } + let previous = state + .selection + .envelope + .as_ref() + .and_then(|e| e.receipt.previous.clone()) + .ok_or("no prior selection")?; + let context = state.legacy_capture() + .map(|c| c.context()) + .ok_or("no current context")?; + if *context != previous.context { + return Err("rollback context does not match current installation".into()); + } + let current = state + .selection + .envelope + .as_ref() + .and_then(|e| e.receipt.selected.clone()); + self.publish_selection(&mut state, Some(previous), current) + } + fn selected_identity(state: &Inner) -> Option { + let e = state.selection.envelope.as_ref()?; + let t = e.receipt.selected.as_ref()?; + let q = state.selection.qualification.as_ref()?; + Some(SelectedModelIdentity { + selection_sha256: e.receipt_sha256.clone(), + selection_revision: e.receipt.revision, + model_sha256: t.model_sha256.clone(), + model_version: t.model_version, + qualification_sha256: t.qualification_sha256.clone(), + policy_sha256: q.receipt.policy_sha256.clone(), + context_digest: t.context.digest(), + room_digest: t.context.room_digest.clone(), + source_node_ids: t.context.sources.iter().map(|s| s.source_id).collect(), + selected_at_unix_ms: e.receipt.selected_at_unix_ms, + evaluated_at_unix_ms: q.receipt.evaluated_at_unix_ms, + authority_scope: "local_operator_binary_presence".into(), + }) + } + fn selected_evidence_locked( + &self, + state: &Inner, + now_ns: u64, + expected_room_binding: Option<&str>, + ) -> SelectedBinaryEvidence { + let identity = Self::selected_identity(state); + let unavailable = |reason: String| SelectedBinaryEvidence::Unavailable { + identity: identity.clone(), + reason: bounded(reason), + }; + if let Some(e) = &state.selection.error { + return unavailable(e.clone()); + } + let Some(target) = state + .selection + .envelope + .as_ref() + .and_then(|e| e.receipt.selected.as_ref()) + else { + return SelectedBinaryEvidence::None; + }; + let result = (|| -> Result { + if expected_room_binding != Some(target.context.room_digest.as_str()) { + return Err("selected model does not match the current published room".into()); + } + let identity = identity + .clone() + .ok_or("selected qualification unavailable")?; + let model = state + .selection + .model + .as_ref() + .ok_or("selected model unavailable")?; + let q = state + .selection + .qualification + .as_ref() + .ok_or("selected qualification unavailable")?; + if q.receipt.policy + != crate::room_learner::qualification::QualificationPolicy::default() + { + return Err("selected policy is not current".into()); + } + let assessment = state + .selection + .assessment + .as_ref() + .ok_or("latest completed qualification assessment unavailable")?; + if assessment.receipt.failure_reason.is_some() { + return Err("newer_qualification_failed".into()); + } + if assessment.receipt.qualification_sha256.as_deref() != Some(q.receipt_sha256.as_str()) + { + return Err("new qualification requires explicit selection".into()); + } + if let Some(error) = &state.input_error { + return Err(format!("selected input is unverified: {error}")); + } + let capture = state.legacy_capture() + .ok_or("awaiting current selected sources")?; + if capture.context() != &target.context { + return Err("selected context differs from current RF source contract".into()); + } + let cohort = capture.live_cohort(now_ns)?; + let prediction = model.infer(&cohort); + if prediction.decision == crate::room_learner::model::PresenceDecision::Abstain { + return Err(prediction + .reason + .unwrap_or_else(|| "selected model abstained".into())); + } + let probability = prediction + .occupied_probability + .filter(|x| x.is_finite() && (0.0..=1.0).contains(x)) + .ok_or("selected probability unavailable")?; + let observed = cohort + .views + .iter() + .map(|v| v.last_time_ns) + .min() + .ok_or("no selected source window")? + / 1_000_000; + // Source identities, not poll time/live-cohort id/age, define repeated observations. + let windows: Vec<_> = cohort + .views + .iter() + .map(|v| { + ( + v.source_id, + &v.frame_ids, + &v.sequence_ids, + v.first_time_ns, + v.last_time_ns, + ) + }) + .collect(); + let observation_digest = hash(&(target.context.digest(), windows))?; + Ok(SelectedBinaryEvidence::Ready { + identity, + presence: prediction.decision + == crate::room_learner::model::PresenceDecision::Occupied, + decision: prediction.decision, + occupied_probability: probability, + observed_at_unix_ms: observed, + observation_digest, + }) + })(); + result.unwrap_or_else(unavailable) + } + pub fn selected_binary_evidence( + &self, + now_ns: u64, + expected_room_binding: Option<&str>, + ) -> SelectedBinaryEvidence { + match self.inner.lock() { + Ok(state) if state.require_legacy_kind().is_err() => SelectedBinaryEvidence::Unavailable { identity: None, reason: "learner_temporal_capture_private_unsupported_command".into() }, + Ok(state) => self.selected_evidence_locked(&state, now_ns, expected_room_binding), + Err(_) => SelectedBinaryEvidence::Unavailable { + identity: None, + reason: "learner state unavailable".into(), + }, + } + } + pub(super) fn selection_status_locked( + &self, + state: &Inner, + now_ns: u64, + binding: Option<&str>, + ) -> ModelSelectionStatus { + ModelSelectionStatus { + schema: "ruview.room-learner.model-selection-status.v1", + revision: state + .selection + .envelope + .as_ref() + .map_or(0, |e| e.receipt.revision), + selected: Self::selected_identity(state), + previous_available: state + .selection + .envelope + .as_ref() + .is_some_and(|e| e.receipt.previous.is_some()), + qualification: state.qualification.as_ref().map(QualificationSummary::from), + qualification_error: state.qualification_error.clone(), + integrity_error: state.selection.error.clone(), + current: self.selected_evidence_locked(state, now_ns, binding), + } + } + pub fn selection_snapshot( + &self, + now_ns: u64, + expected_room_binding: Option<&str>, + ) -> ModelSelectionStatus { + match self.inner.lock() { + Ok(state) => { + let mut snapshot = + self.selection_status_locked(&state, now_ns, expected_room_binding); + if let Err(reason) = state.require_legacy_kind() { + snapshot.integrity_error = Some(reason.clone()); + snapshot.current = SelectedBinaryEvidence::Unavailable { identity: None, reason }; + return snapshot; + } + let verified = (|| -> Result<(), String> { + if self.store.load_selection()?.as_ref() != state.selection.envelope.as_ref() { + return Err("saved selection differs from current runtime".into()); + } + if let Some(target) = state + .selection + .envelope + .as_ref() + .and_then(|e| e.receipt.selected.as_ref()) + { + let (model, q) = self.store.resolve_selection_target(target)?; + self.store.current_qualification_assessment(&model, &q)?; + } + Ok(()) + })(); + if let Err(e) = verified { + snapshot.integrity_error = Some(bounded(e.clone())); + snapshot.current = SelectedBinaryEvidence::Unavailable { + identity: snapshot.selected.clone(), + reason: bounded(e), + }; + } + snapshot + } + Err(_) => ModelSelectionStatus { + schema: "ruview.room-learner.model-selection-status.v1", + revision: 0, + selected: None, + previous_available: false, + qualification: None, + qualification_error: None, + integrity_error: Some("learner state unavailable".into()), + current: SelectedBinaryEvidence::Unavailable { + identity: None, + reason: "learner state unavailable".into(), + }, + }, + } + } + pub(super) fn candidate_identity(&self, state: &Inner) -> Option { + let model = state.model.as_ref()?; + let sha = format!("{:x}", Sha256::digest(model.export().ok()?)); + Some( + json!({"model_sha256":sha,"model_version":model.version,"context_digest":model.context().digest(), + "room_digest":model.context().room_digest,"persisted":self.store.load_exact_candidate(model.context(),&sha).is_ok()}), + ) + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_selection_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_selection_tests.rs new file mode 100644 index 0000000000..f5f68ea985 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_selection_tests.rs @@ -0,0 +1,413 @@ +//! SYNTHETIC managed packets exercise the actual qualification/selection lifecycle. +//! No hardware, physical accuracy or operator-ground-truth validation is claimed. +use super::tests::{context, finished, phase, runtime, train_phases, START}; +use super::*; +use crate::room_learner::{ + qualification::QualificationEnvelope, selection::SelectedBinaryEvidence, +}; + +pub(super) async fn fitted(root: &Path) -> Arc { + let rt = runtime(root); + rt.configure( + &rt.mutation_guard().unwrap(), + context(), + "selection-fixture".into(), + true, + ) + .unwrap(); + train_phases(&rt); + rt.start_training( + &rt.mutation_guard().unwrap(), + FitConfig { + epochs: 8, + learning_rate: 0.5, + hidden_dim: 2, + embedding_dim: 2, + temporal_tokens: 2, + ..FitConfig::default() + }, + ) + .unwrap(); + let status = finished(&rt).await; + assert_eq!(status["state"], "ready", "{status}"); + for (i, label) in [ + RoomLabel::Empty, + RoomLabel::OccupiedStatic, + RoomLabel::OccupiedMoving, + RoomLabel::AdjacentOnly, + ] + .into_iter() + .enumerate() + { + phase(&rt, DataRole::HeldOut, label, i as u32 + 4); + } + rt +} +pub(super) fn feed(rt: &RoomLearningRuntime, at: u64, sources: &[u8], gain: f64) -> u64 { + feed_sequence(rt, at, sources, gain, 10_000) +} +pub(super) fn feed_sequence( + rt: &RoomLearningRuntime, + at: u64, + sources: &[u8], + gain: f64, + sequence_base: u32, +) -> u64 { + for i in 0..8u32 { + for source in sources { + let values: Vec = (0..8) + .map(|bin| { + gain + f64::from(*source) * 0.3 + bin as f64 * 0.04 + f64::from(i) * 0.003 + }) + .collect(); + rt.ingest( + RawSourceFrame { + source_id: *source, + grid_id: "ht64-canonical8", + frequency_hz: 2_437_000_000, + bandwidth_hz: 20_000_000, + sequence: sequence_base + i, + amplitudes: &values, + }, + at + u64::from(i) * 10_000_000 + u64::from(*source), + ) + .unwrap(); + } + } + at + 80_000_000 +} +fn ready(value: SelectedBinaryEvidence) -> crate::room_learner::selection::SelectedModelIdentity { + match value { + SelectedBinaryEvidence::Ready { identity, .. } => identity, + other => panic!("expected ready: {other:?}"), + } +} +#[tokio::test] +async fn qualification_selection_cold_restore_deselect_and_exact_rollback_compose() { + let dir = tempfile::tempdir().unwrap(); + let rt = fitted(dir.path()).await; + assert!(matches!( + rt.selected_binary_evidence(START + 104_000_000_000, Some("synthetic-room")), + SelectedBinaryEvidence::None + )); + let q = rt.qualify(&rt.mutation_guard().unwrap()).unwrap(); + assert_eq!(q.receipt.label_provenance, "managed-sealed-history-v1"); + assert!(!q.receipt.physically_validated && !q.receipt.independence_verified); + assert_eq!(q.receipt.criteria.len(), 4); + let evaltime = q.receipt.evaluated_at_unix_ms; + let oldguard = rt.mutation_guard().unwrap(); + let selected = rt + .select_model(&oldguard, &q.receipt.model_sha256, &q.receipt_sha256) + .unwrap(); + assert!(rt.deselect_model(&oldguard).is_err()); + let first = ready(rt.selected_binary_evidence(START + 103_000_000_000, Some("synthetic-room"))); + assert_eq!(first.selection_sha256, selected.receipt_sha256); + assert!(matches!( + rt.selected_binary_evidence(START + 103_000_000_000, Some("wrong-room")), + SelectedBinaryEvidence::Unavailable { .. } + )); + assert!(matches!( + rt.selected_binary_evidence(START + 200_000_000_000, Some("synthetic-room")), + SelectedBinaryEvidence::Unavailable { .. } + )); + + let cold = runtime(dir.path()); + let snapshot = cold.selection_snapshot(cold.receiver_now_ns(), Some("synthetic-room")); + assert_eq!( + snapshot.selected.unwrap().selection_sha256, + selected.receipt_sha256 + ); + assert_eq!( + snapshot.qualification.unwrap().evaluated_at_unix_ms, + evaltime + ); + assert!(matches!( + snapshot.current, + SelectedBinaryEvidence::Unavailable { .. } + )); + // Stored selection survives cold load, but historical capture buffers do not. + let at = cold.receiver_now_ns(); + let now = feed(&cold, at, &[1, 2], 2.0); + let a = cold.selected_binary_evidence(now, Some("synthetic-room")); + let b = cold.selected_binary_evidence(now + 10_000_000, Some("synthetic-room")); + match (&a, &b) { + ( + SelectedBinaryEvidence::Ready { + observation_digest: x, + observed_at_unix_ms: t, + .. + }, + SelectedBinaryEvidence::Ready { + observation_digest: y, + observed_at_unix_ms: u, + .. + }, + ) => { + assert_eq!(x, y); + assert_eq!(t, u); + assert!(*t < now / 1_000_000) + } + _ => panic!("fresh repeated proof unavailable: {a:?}, {b:?}"), + } + assert_eq!(ready(a).model_sha256, q.receipt.model_sha256); + let deselected = cold + .deselect_model(&cold.mutation_guard().unwrap()) + .unwrap(); + assert!(matches!( + cold.selected_binary_evidence(now, Some("synthetic-room")), + SelectedBinaryEvidence::None + )); + let repeated = cold + .deselect_model(&cold.mutation_guard().unwrap()) + .unwrap(); + assert_eq!(repeated, deselected); + let rolled = cold + .rollback_model(&cold.mutation_guard().unwrap()) + .unwrap(); + assert_eq!( + rolled.receipt.selected.unwrap().model_sha256, + q.receipt.model_sha256 + ); + assert_eq!( + ready(cold.selected_binary_evidence(now, Some("synthetic-room"))).evaluated_at_unix_ms, + evaltime + ); + // Selection/model are not coupled to the 24-hour provisional-capture TTL. + let tomorrow = feed_sequence( + &cold, + now + 25 * 60 * 60 * 1_000_000_000, + &[1, 2], + 2.0, + 20_000, + ); + assert_eq!( + ready(cold.selected_binary_evidence(tomorrow, Some("synthetic-room"))).evaluated_at_unix_ms, + evaltime + ); + // Starting another candidate must never automatically replace the selected model. + cold.configure( + &cold.mutation_guard().unwrap(), + context(), + "new-candidate".into(), + true, + ) + .unwrap(); + assert_eq!( + cold.selection_snapshot(now, Some("synthetic-room")) + .selected + .unwrap() + .model_sha256, + q.receipt.model_sha256 + ); + assert!(cold.status(now)["candidate_identity"].is_null()); + assert!(matches!( + cold.selected_binary_evidence(now, Some("synthetic-room")), + SelectedBinaryEvidence::Unavailable { .. } + )); + let fresh = feed(&cold, tomorrow + 1_000_000_000, &[1, 2], 2.0); + assert!(cold.qualify(&cold.mutation_guard().unwrap()).is_err()); + assert!( + matches!( + cold.selected_binary_evidence(fresh, Some("synthetic-room")), + SelectedBinaryEvidence::Ready { .. } + ), + "incomplete candidate must not revoke the existing selected model" + ); +} + +#[tokio::test] +async fn qualification_requires_real_lineage_all_conditions_and_matching_sealed_history() { + let dir = tempfile::tempdir().unwrap(); + let rt = fitted(dir.path()).await; + let state = rt.inner.lock().unwrap(); + let model = state.model.as_ref().unwrap(); + let capture = state.legacy_capture().unwrap(); + let dataset = capture.validated_dataset().unwrap(); + let evaluation = rt + .store + .evaluate_candidate(model, &dataset, DataRole::HeldOut) + .unwrap(); + let good = + QualificationEnvelope::issue(model, &dataset, capture.history(), &evaluation).unwrap(); + // Actual inference reports from deliberately wrong/abstaining model heads, + // not hand-edited success metrics, must fail the per-condition policy. + for (bias, reason) in [ + (-100.0, "occupied sensitivity"), + (100.0, "false presence"), + (0.0, "coverage"), + ] { + let mut wrong = (**model).clone(); + wrong.head_weights.fill(0.0); + *wrong.head_weights.last_mut().unwrap() = bias; + let report = wrong.evaluate(dataset.held_out()).unwrap(); + let error = crate::room_learner::qualification::criteria(&report, &[4; 4]).unwrap_err(); + assert!(error.contains(reason), "{error}"); + } + let incomplete: Vec<_> = dataset + .held_out() + .iter() + .filter(|c| c.label != RoomLabel::AdjacentOnly) + .cloned() + .collect(); + let report = model.evaluate(&incomplete).unwrap(); + assert!( + crate::room_learner::qualification::criteria(&report, &[4; 4]) + .unwrap_err() + .contains("AdjacentOnly") + ); + let mut missing = capture.history().to_vec(); + missing.retain(|h| h.label != RoomLabel::AdjacentOnly); + assert!(QualificationEnvelope::issue(model, &dataset, &missing, &evaluation).is_err()); + let no_train = crate::room_learner::observation::ValidatedDataset::new( + context(), + dataset.held_out().to_vec(), + ) + .unwrap(); + assert!( + QualificationEnvelope::issue(model, &no_train, capture.history(), &evaluation).is_err() + ); + let mut tampered = good.clone(); + tampered.receipt.policy.minimum_evaluated_cohort_coverage = 0.0; + tampered.receipt.policy_sha256 = + crate::room_learner::qualification::hash(&tampered.receipt.policy).unwrap(); + tampered.receipt_sha256 = crate::room_learner::qualification::hash(&tampered.receipt).unwrap(); + assert!(tampered.validate(model, &evaluation).is_err()); + let mut overflow = good.clone(); + overflow.receipt.criteria[0].train_cohorts = usize::MAX; + overflow.receipt_sha256 = crate::room_learner::qualification::hash(&overflow.receipt).unwrap(); + assert!(overflow.validate(model, &evaluation).is_err()); + let mut other = good; + other.receipt.model_sha256 = "b".repeat(64); + other.receipt_sha256 = crate::room_learner::qualification::hash(&other.receipt).unwrap(); + assert!(other.validate(model, &evaluation).is_err()); +} + +#[tokio::test] +async fn selection_corruption_owner_mismatch_missing_sources_and_ood_never_fall_back() { + let dir = tempfile::tempdir().unwrap(); + let rt = fitted(dir.path()).await; + let q = rt.qualify(&rt.mutation_guard().unwrap()).unwrap(); + rt.select_model( + &rt.mutation_guard().unwrap(), + &q.receipt.model_sha256, + &q.receipt_sha256, + ) + .unwrap(); + let cold = runtime(dir.path()); + let at = cold.receiver_now_ns(); + let now = feed(&cold, at, &[1], 2.0); + assert!(matches!( + cold.selected_binary_evidence(now, Some("synthetic-room")), + SelectedBinaryEvidence::Unavailable { .. } + )); + let ood = runtime(dir.path()); + let now = feed(&ood, ood.receiver_now_ns(), &[1, 2], 1_000_000.0); + assert!(matches!( + ood.selected_binary_evidence(now, Some("synthetic-room")), + SelectedBinaryEvidence::Unavailable { .. } + )); + let path = dir + .path() + .join("a".repeat(64)) + .join("SensingServer/room-learner/model-selection-v1.json"); + let original = std::fs::read(&path).unwrap(); + let foreign_dir = dir.path().join("b".repeat(64)).join("SensingServer"); + let foreign = + RoomLearningRuntime::new(&foreign_dir, &"b".repeat(64), "synthetic-install").unwrap(); + drop(foreign); + std::fs::write( + foreign_dir.join("room-learner/model-selection-v1.json"), + &original, + ) + .unwrap(); + let foreign = + RoomLearningRuntime::new(&foreign_dir, &"b".repeat(64), "synthetic-install").unwrap(); + assert!(matches!( + foreign.selected_binary_evidence(now, Some("synthetic-room")), + SelectedBinaryEvidence::Unavailable { .. } + )); + std::fs::write(&path, b"broken").unwrap(); + let broken = runtime(dir.path()); + assert!(matches!( + broken.selected_binary_evidence(now, Some("synthetic-room")), + SelectedBinaryEvidence::Unavailable { .. } + )); + assert_eq!(std::fs::read(&path).unwrap(), b"broken"); + assert!(rt + .store + .load_exact_candidate(&context(), &q.receipt.model_sha256) + .is_ok()); +} + +/// Genuine managed capture path with synthetic, deliberately selectable signal gain. +pub(super) fn phase_override( + rt: &RoomLearningRuntime, + role: DataRole, + label: RoomLabel, + index: u32, + gain: f64, +) -> u64 { + let at = START + u64::from(index) * 13_000_000_000; + rt.begin_phase( + &rt.mutation_guard().unwrap(), + PhaseRequest { + trial_id: format!("selection-trial-{index}"), + session_id: format!("selection-session-{index}"), + role, + label, + duration_ms: 12_000, + lead_in_ms: 0, + conditions_confirmed: true, + }, + at, + ) + .unwrap(); + let context = rt.context().unwrap(); + for quarter in 0..4u32 { + for sample in 0..8u32 { + for source in &context.sources { + let values: Vec = (0..source.feature_count) + .map(|bin| { + gain + f64::from(source.source_id) * 0.3 + + bin as f64 * 0.04 + + f64::from(sample) * 0.003 + }) + .collect(); + let t = at + + u64::from(quarter) * 3_000_000_000 + + 1_000_000 + + u64::from(sample) * 10_000_000 + + u64::from(source.source_id); + rt.ingest( + RawSourceFrame { + source_id: source.source_id, + grid_id: &source.grid_id, + frequency_hz: source.center_frequency_hz, + bandwidth_hz: source.bandwidth_hz, + sequence: index * 32 + quarter * 8 + sample + 1, + amplitudes: &values, + }, + t, + ) + .unwrap(); + } + } + } + rt.finish_phase(&rt.mutation_guard().unwrap(), at + 12_000_000_000, true) + .unwrap(); + at + 12_000_000_000 +} + +#[test] +fn held_out_duration_defaults_are_short_and_leave_training_empty_unchanged() { + let dir = tempfile::tempdir().unwrap(); + let rt = runtime(dir.path()); + let status = rt.status(START); + assert_eq!(status["phase_duration_defaults"]["Empty"], 600_000); + for label in ["Empty", "OccupiedStatic", "OccupiedMoving", "AdjacentOnly"] { + assert_eq!(status["held_out_phase_duration_defaults"][label], 12_000); + } + assert_eq!(status["phase_duration_defaults"]["OccupiedStatic"], 12_000); + assert_eq!(crate::room_learner::capture::COHORTS_PER_PHASE, 4); + assert_eq!(crate::room_learner::capture::MIN_PHASE_MS, 12_000); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal.rs new file mode 100644 index 0000000000..aa9fb2e95d --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal.rs @@ -0,0 +1,769 @@ +//! Default-inactive native Train capture bridge. No HTTP constructor or fit authority. +use super::*; +use crate::room_learner::{ + managed_schedule_contract::{LeadInPolicy, ManagedScheduleContract}, + native::CanonicalPacket, + temporal_model_input::managed::execution::{TemporalCaptureStage, TemporalTrainCapture}, +}; + +const TRAIN_LEAD_IN_NS: u64 = 5_000_000_000; +const PRIVATE_KIND: &str = "learner_temporal_capture_private_unsupported_command"; +/// Only the actual protected Runtime constructor can mint fresh full-run +/// custody. No serde/Clone or caller-supplied authority flag. +pub(crate) struct FullRunGenesisWitness { + runtime_epoch:String,generation:u64,context:CaptureContext,run:String, +} +impl FullRunGenesisWitness { + pub(crate) fn consume(self,context:&CaptureContext,run:&str)->Result<(String,u64),String>{ + if self.context!=*context||self.run!=run||self.runtime_epoch.is_empty()||self.generation==0{ + return Err("temporal_full_run_genesis_owner_changed".into()); + } + Ok((self.runtime_epoch,self.generation)) + } +} +pub(crate) struct FrozenOwnerWitness { + runtime_epoch:String,generation:u64,revision:u64,context:CaptureContext,run:String, +} +pub(crate) struct FullRunRecoveryWitness{runtime_epoch:String,generation:u64,context:CaptureContext} +impl FullRunRecoveryWitness{ + pub(crate) fn consume(self,context:&CaptureContext)->Result<(String,u64),String>{ + if self.context!=*context||self.runtime_epoch.is_empty()||self.generation==0{return Err("temporal_full_run_recovery_owner_changed".into());} + Ok((self.runtime_epoch,self.generation)) + } +} +impl FrozenOwnerWitness { + pub(crate) fn consume(self,context:&CaptureContext,run:&str)->Result<(String,u64,u64),String>{ + if self.context!=*context||self.run!=run||self.runtime_epoch.is_empty()||self.generation==0||self.revision==0{ + return Err("temporal_frozen_owner_operation_changed".into()); + } + Ok((self.runtime_epoch,self.generation,self.revision)) + } +} +#[path="lifecycle_temporal_frozen.rs"] +mod frozen_control; +#[path = "lifecycle_temporal_fit.rs"] +mod fit_control; +pub(crate) use fit_control::AcceptedTemporalFit; + +pub(super) struct RuntimeTemporalCapture { + context: CaptureContext, + owner_generation: u64, + core: Option, + lead_in_until: Option, + failure: Option, + last_now_ns: u64, + fit_job: Option, + evidence_job: Option, + training_proof: Option, + evaluation: Option>, + assessment: Option>, + diagnostic_fit: Option, + public_train_requested: bool, +} +impl RuntimeTemporalCapture { + pub(super) fn owner_generation(&self) -> u64 { + self.owner_generation + } + pub(super) fn invalidate_owner(&mut self) { + self.cancel_fit(); + self.lead_in_until = None; + if let Some(core) = self.core.as_mut() { + core.invalidate("temporal_owner_generation_changed"); + } + } + pub(super) fn context(&self) -> &CaptureContext { + &self.context + } + pub(super) fn phase_identity(&self) -> Option { + self.core + .as_ref()? + .phase_identity() + .map(|(trial_id, session_id)| PhaseIdentity { + trial_id, + session_id, + }) + } + pub(super) fn active(&self) -> bool { + self.lead_in_until.is_some() + || self.core.as_ref().is_some_and(|c| { + matches!( + c.stage(), + TemporalCaptureStage::Observing | TemporalCaptureStage::NeedsConfirmation + ) + }) + } + fn check_time(&mut self, now: u64) -> Result<(), String> { + if self.core.as_ref().is_some_and(|c|c.is_full_run()&&!c.current_context_matches(&self.context)){ + self.invalidate_owner();return Err("temporal_full_owner_context_changed".into()); + } + if now < self.last_now_ns { + self.cancel_fit(); + self.lead_in_until = None; + self.failure = Some("temporal_receiver_clock_reversed".into()); + if let Some(core) = self.core.as_mut() { + core.invalidate("temporal_receiver_clock_reversed"); + } + return Err("temporal_receiver_clock_reversed".into()); + } + let checked = self.core + .as_mut() + .ok_or("temporal_initial_barrier_unconfirmed")? + .check_time(now); + if let Err(error) = checked { + self.cancel_fit(); + return Err(error); + } + if let Some(job) = &self.fit_job { job.observe_now(now); } + if let Some(job) = &self.evidence_job { job.observe_now(now); } + self.last_now_ns = now; + Ok(()) + } + fn arm( + &mut self, + store: &super::super::recovery_store::RecoveryStore, + epoch: &str, + now: u64, + ) -> Result<(), String> { + if let Some(at) = self.lead_in_until { + if now >= at { + self.lead_in_until = None; + let core=self.core.as_mut().ok_or("temporal_initial_barrier_unconfirmed")?; + // Full-run preparation/lateness is never credited as observation. + // Start at this actual trusted receiver coordinate, not the old deadline. + let start=if core.is_full_run(){now}else{at}; + core.start_segment(epoch,start,store)?; + } + } + Ok(()) + } + pub(super) fn ingest( + &mut self, + store: &super::super::recovery_store::RecoveryStore, + epoch: &str, + packet: &CanonicalPacket, + now: u64, + retired: &mut DeferredReclamation, + ) -> Result<(), String> { + self.check_time(now)?; + let admission = self + .core + .as_mut() + .ok_or("temporal_initial_barrier_unconfirmed")? + .poll_async(now, retired)?; + let now = admission.map_or(now, |(_, at)| at); + self.last_now_ns = now; + if admission.is_some_and(|(start, _)| start) { + self.lead_in_until = Some( + now.checked_add(self.core.as_ref().filter(|c|c.is_full_run()).map(|c|c.current_lead_in_ns()).transpose()?.unwrap_or(TRAIN_LEAD_IN_NS)) + .ok_or("temporal_time_overflow")?, + ); + return self.core.as_mut().unwrap().prime_fence(packet, now); + } + self.arm(store, epoch, now)?; + if self.lead_in_until.is_some() { + return self + .core + .as_mut() + .ok_or("temporal_initial_barrier_unconfirmed")? + .prime_fence(packet, now); + } // Preparation never credits RF observation. + let core = self + .core + .as_mut() + .ok_or("temporal_initial_barrier_unconfirmed")?; + match core.stage() { + TemporalCaptureStage::Observing => core.ingest(packet, now, store), + TemporalCaptureStage::Failed => Err("temporal_capture_unavailable".into()), + TemporalCaptureStage::Blocked => core.prime_fence(packet, now), + TemporalCaptureStage::Ready + | TemporalCaptureStage::Paused + | TemporalCaptureStage::Preparing => core.prime_fence(packet, now), + TemporalCaptureStage::Complete|TemporalCaptureStage::Frozen|TemporalCaptureStage::FullComplete if core.is_full_run()=>core.prime_fence(packet,now), + _ => Ok(()), // No automatic start, confirmation or new phase from packet arrival. + } + } + /// Annotates only the already failed current capture; no failure transition is rerun. + pub(super) fn annotate_receiver_failure(&mut self, expected: &str, detail: String) -> bool { + self.core.as_mut().is_some_and(|core| core.annotate_receiver_failure(expected, detail)) + } + pub(super) fn reject(&mut self, source_id: u8, grid_id: &str, reason: &str) { + if self + .context + .sources + .iter() + .any(|s| s.source_id == source_id && s.grid_id == grid_id) + { + self.cancel_fit(); + self.lead_in_until = None; + self.failure = Some(bounded(reason.into())); + if let Some(core) = self.core.as_mut() { + core.invalidate(reason); + } + } + } + fn snapshot(&self, now: u64) -> Value { + let mut value = self.core.as_ref().map_or_else( + || json!({"stage":"blocked","error":self.failure}), + |c| c.snapshot(), + ); + value["lead_in_remaining_ms"] = json!(self + .lead_in_until + .map_or(0, |end| end.saturating_sub(now) / 1_000_000)); + value["runtime_state"] = json!(if self.lead_in_until.is_some() { + "lead_in" + } else { + "capture" + }); + value["runtime_error"] = json!(self.failure); + value["public_activation"] = json!(false); + value["fit_authority"] = json!(false); + value + } +} +impl RoomLearningRuntime { + pub(super) fn temporal_store( + &self, + ) -> Result<&super::super::recovery_store::RecoveryStore, String> { + if self.operator_profile_selected.load(Ordering::Acquire) { self.operator_recovery_store.as_deref() } else { self.recovery_store.as_deref() } + .ok_or("temporal_recovery_unavailable".into()) + } + fn temporal_store_arc(&self) -> Result, String> { + if self.operator_profile_selected.load(Ordering::Acquire) { self.operator_recovery_store.clone() } else { self.recovery_store.clone() }.ok_or("temporal_recovery_unavailable".into()) + } + /// Deliberately PRIVATE: current public configure always constructs Legacy. + /// Refuses to discard existing capture/candidate/selection/recovery state. + fn configure_temporal_capture( + &self, + expected: &MutationGuard, + context: CaptureContext, + run: String, + now: u64, + ) -> Result<(), String> { + self.configure_temporal_capture_with( + expected, + context, + run, + now, + |schedule, run, fit, now, _| { + // Existing-work guard precedes this off-lock producer. + let store=self.temporal_store_arc()?; + TemporalTrainCapture::prepare_async(schedule, run, fit, now, store) + }, + ) + } + /// Fresh genesis only; never upgrades a running old private capture. + fn configure_temporal_full_run(&self,expected:&MutationGuard,context:CaptureContext,run:String,now:u64)->Result<(),String>{ + self.configure_temporal_capture_with_custody(expected,context,run,now,|s,r,f,n,_,w|{ + let store=self.temporal_store_arc()?; + TemporalTrainCapture::prepare_full_run_async(s,r,f,n,store,w) + }) + } + /// Explicit trusted new-Train entry point. No public route or profile flag. + /// Active old generations must retire first; only authenticated retired + /// metadata can chain a new343 generation in the SAME store. + fn configure_fresh_train_343_capture(&self,expected:&MutationGuard,context:CaptureContext,run:String,now:u64)->Result<(),String>{ + self.configure_temporal_capture_with(expected,context,run,now,|schedule,run,fit,now,_|{ + let store=self.temporal_store_arc()?; + TemporalTrainCapture::prepare_async_fresh_train_343(schedule,run,fit,now,store) + }) + } + fn configure_temporal_capture_with( + &self, + expected: &MutationGuard, + context: CaptureContext, + run: String, + now: u64, + prepare: impl FnOnce( + ManagedScheduleContract, + String, + FitConfig, + u64, + &super::super::recovery_store::RecoveryStore, + ) -> Result<(TemporalTrainCapture, Option), String>, + ) -> Result<(), String> { + self.configure_temporal_capture_with_custody(expected,context,run,now,move|s,r,f,n,store,_|prepare(s,r,f,n,store)) + } + fn configure_temporal_capture_with_custody( + &self,expected:&MutationGuard,context:CaptureContext,run:String,now:u64, + prepare:impl FnOnce(ManagedScheduleContract,String,FitConfig,u64,&super::super::recovery_store::RecoveryStore,FullRunGenesisWitness)->Result<(TemporalTrainCapture,Option),String>, + )->Result<(),String>{self.configure_temporal_capture_with_custody_mode(expected,context,run,now,false,prepare)} + fn configure_temporal_capture_with_custody_mode( + &self,expected:&MutationGuard,context:CaptureContext,run:String,now:u64,replace_empty_legacy:bool, + prepare:impl FnOnce(ManagedScheduleContract,String,FitConfig,u64,&super::super::recovery_store::RecoveryStore,FullRunGenesisWitness)->Result<(TemporalTrainCapture,Option),String>, + )->Result<(),String>{ + context.validate()?; + if context.owner_scope != self.owner + || context.installation_id != self.installation + || now == 0 + { + return Err("temporal_context_owner_or_clock".into()); + } + let schedule = ManagedScheduleContract::build(context.clone(), if self.operator_profile_selected.load(Ordering::Acquire) { LeadInPolicy::OperatorEightStepV1 } else { LeadInPolicy::Train5HeldOut15EngineeringV1 })?; + let fit = FitConfig::for_source_count(context.sources.len()); + fit.validate_context(&context)?; + let mut state = self.command_state(expected)?; + let empty_legacy=replace_empty_legacy&&matches!(state.capture.as_ref(),Some(CaptureWorkflow::Legacy(c))if !c.phase_active()&&c.cohorts().is_empty()); + let public_fresh=replace_empty_legacy&&(state.capture.is_none()||empty_legacy)&&!busy(&state.job)&&!state.selection.blocks_new_temporal_capture(); + if (state.capture.is_some()&&!empty_legacy) + || (state.model.is_some()&&!public_fresh) + || (state.selection.has_history()&&!public_fresh) + || (state.job.phase != JobPhase::Idle&&!public_fresh) + || state.recovery.descriptor.is_some() + { + return Err("temporal_setup_would_replace_existing_work".into()); + } + let store = self.temporal_store()?; + let next_generation = state + .generation + .checked_add(1) + .ok_or("learner generation exhausted")?; + let witness=FullRunGenesisWitness{runtime_epoch:self.runtime_epoch.clone(),generation:next_generation,context:context.clone(),run:run.clone()}; + // Historical immutable models remain in ArtifactStore. Retire only + // their RAM presentation when explicitly starting a new consented run. + let retired_legacy=if public_fresh{let old=(state.model.take(),state.receipt.take(),state.last_evaluation.take(),state.qualification.take());state.job=Job::default();state.frozen_training=false;state.qualification_error=None;Some(old)}else{None}; + let retired_capture=state.capture.replace(CaptureWorkflow::Temporal(Box::new( + RuntimeTemporalCapture { + context: context.clone(), + owner_generation: next_generation, + core: None, + lead_in_until: None, + failure: None, + last_now_ns: now, + fit_job: None, + evidence_job: None, + training_proof: None, + evaluation: None, + assessment: None, + diagnostic_fit: None, + public_train_requested: false, + }, + ))); + state.generation = next_generation; + drop(state);drop(retired_capture);drop(retired_legacy); + // Initial crypto/filesystem execution never holds the receiver lock. + let result = prepare(schedule, run, fit, now, store,witness); + let (core, failure) = match result { + Ok((core, error)) => (Some(core), error), + Err(error) => (None, Some(error)), + }; + let mut state = self.inner.lock().map_err(|_| "learner state unavailable")?; + if state.generation != next_generation || state.capture_context() != Some(&context) { + // A discarded asynchronous constructor may already have a queued + // initial transaction. Drop/drain it only outside the ingress lock. + drop(state); + drop(core); + return Err("learner_state_changed".into()); + } + let Some(CaptureWorkflow::Temporal(capture)) = state.capture.as_mut() else { + return Err(PRIVATE_KIND.into()); + }; + capture.core = core; + capture.failure = failure.clone(); + if let Some(error) = failure { + state.input_error = Some(bounded(error.clone())); + return Err(error); + } + Ok(()) + } + fn start_temporal_phase(&self, expected: &MutationGuard, now: u64) -> Result<(), String> {self.start_temporal_phase_checked(expected,now,|_|Ok(()))} + fn start_temporal_phase_checked(&self,expected:&MutationGuard,now:u64,check:impl FnOnce(&RuntimeTemporalCapture)->Result<(),String>)->Result<(),String>{ + let mut state = self.command_state(expected)?; + if busy(&state.job) { return Err("temporal_fit_job_busy".into()); } + let owner_generation = state.generation; + let Some(CaptureWorkflow::Temporal(capture)) = state.capture.as_mut() else { + return Err(PRIVATE_KIND.into()); + }; + if capture.owner_generation != owner_generation { + capture + .core + .as_mut() + .map(|c| c.invalidate("temporal_owner_generation_changed")); + return Err("temporal_owner_generation_changed".into()); + } + check(capture)?; + capture.check_time(now)?; + let core = capture + .core + .as_mut() + .ok_or("temporal_initial_barrier_unconfirmed")?; + if capture.lead_in_until.is_some() + || !(matches!(core.stage(),TemporalCaptureStage::Ready|TemporalCaptureStage::Paused) + ||core.is_full_run()&&core.stage()==TemporalCaptureStage::Frozen) + { + return Err("temporal_phase_not_startable".into()); + } + if core.async_start(now)? { + return Ok(()); + } + // Synchronous reference path retains its original behavior. + capture.lead_in_until = Some( + now.checked_add(if core.is_full_run(){core.current_lead_in_ns()?}else{TRAIN_LEAD_IN_NS}) + .ok_or("temporal_time_overflow")?, + ); + Ok(()) + } + fn tick_temporal_capture(&self, expected: &MutationGuard, now: u64) -> Result<(), String> { + let store = self.temporal_store()?; + // Must precede MutexGuard for success, rejection and unwinding alike. + let mut retired = self.deferred_reclamation(); + let mut state = self.inner.lock().map_err(|_| "learner state unavailable")?; + if *expected != self.guard_for(&state) { + return Err("learner_state_changed".into()); + } + let owner_generation = state.generation; + let Some(CaptureWorkflow::Temporal(capture)) = state.capture.as_mut() else { + return Err(PRIVATE_KIND.into()); + }; + if capture.owner_generation != owner_generation { + capture + .core + .as_mut() + .map(|c| c.invalidate("temporal_owner_generation_changed")); + return Err("temporal_owner_generation_changed".into()); + } + capture.check_time(now)?; + let admission = capture + .core + .as_mut() + .ok_or("temporal_initial_barrier_unconfirmed")? + .poll_async(now, &mut retired)?; + #[cfg(test)] + if retired.has_ack() + && self + .panic_after_reclamation_ack + .swap(false, Ordering::AcqRel) + { + panic!("synthetic panic after actual ACK admission"); + } + let now = admission.map_or(now, |(_, at)| at); + capture.last_now_ns = now; + if admission.is_some_and(|(start, _)| start) { + capture.lead_in_until = Some( + now.checked_add(capture.core.as_ref().filter(|c|c.is_full_run()).map(|c|c.current_lead_in_ns()).transpose()?.unwrap_or(TRAIN_LEAD_IN_NS)) + .ok_or("temporal_time_overflow")?, + ); + return Ok(()); + } + capture.arm(store, &self.runtime_epoch, now)?; + if capture.lead_in_until.is_some() { + return Ok(()); + } + let core = capture + .core + .as_mut() + .ok_or("temporal_initial_barrier_unconfirmed")?; + if core.stage() == TemporalCaptureStage::Observing { + core.tick(now, store) + } else { + Ok(()) + } + } + fn temporal_confirmation_challenge(&self, expected: &MutationGuard) -> Result { + let state = self.inner.lock().map_err(|_| "learner state unavailable")?; + if *expected != self.guard_for(&state) { + return Err("learner_state_changed".into()); + } + let Some(CaptureWorkflow::Temporal(capture)) = state.capture.as_ref() else { + return Err(PRIVATE_KIND.into()); + }; + capture + .core + .as_ref() + .ok_or("temporal_initial_barrier_unconfirmed")? + .confirmation_challenge() + } + fn confirm_temporal_phase(&self,expected:&MutationGuard,challenge:&str,now:u64)->Result<(),String>{self.confirm_temporal_phase_checked(expected,challenge,now,|_|Ok(()))} + fn confirm_temporal_phase_checked(&self,expected:&MutationGuard,challenge:&str,now:u64,check:impl FnOnce(&RuntimeTemporalCapture)->Result<(),String>)->Result<(),String>{ + let store = self.temporal_store()?; + let mut state = self.command_state(expected)?; + if busy(&state.job) { return Err("temporal_fit_job_busy".into()); } + let owner_generation = state.generation; + let Some(CaptureWorkflow::Temporal(capture)) = state.capture.as_mut() else { + return Err(PRIVATE_KIND.into()); + }; + if capture.owner_generation != owner_generation { + capture + .core + .as_mut() + .map(|c| c.invalidate("temporal_owner_generation_changed")); + return Err("temporal_owner_generation_changed".into()); + } + check(capture)?; + capture.check_time(now)?; + if capture.lead_in_until.is_some() { + return Err("temporal_phase_incomplete".into()); + } + capture + .core + .as_mut() + .ok_or("temporal_initial_barrier_unconfirmed")? + .confirm(challenge, store, now) + } + fn pause_temporal_capture(&self, expected: &MutationGuard, now: u64) -> Result<(), String> { + let store = self.temporal_store()?; + let mut state = self.command_state(expected)?; + if busy(&state.job) { return Err("temporal_fit_job_busy".into()); } + let owner_generation = state.generation; + let Some(CaptureWorkflow::Temporal(capture)) = state.capture.as_mut() else { + return Err(PRIVATE_KIND.into()); + }; + if capture.owner_generation != owner_generation { + capture + .core + .as_mut() + .map(|c| c.invalidate("temporal_owner_generation_changed")); + return Err("temporal_owner_generation_changed".into()); + } + capture.check_time(now)?; + if capture.lead_in_until.is_some() { + return Err("temporal_preparation_has_no_observation_to_pause".into()); + } + capture + .core + .as_mut() + .ok_or("temporal_initial_barrier_unconfirmed")? + .pause(now, store) + } + fn restore_temporal_capture( + &self, + expected: &MutationGuard, + context: &CaptureContext, + now: u64, + ) -> Result<(), String> { + self.restore_temporal_capture_kind(expected,context,now,false) + } + fn restore_temporal_full_run(&self,expected:&MutationGuard,context:&CaptureContext,now:u64)->Result<(),String>{ + self.restore_temporal_capture_kind(expected,context,now,true) + } + fn restore_temporal_capture_kind(&self,expected:&MutationGuard,context:&CaptureContext,now:u64,full_run:bool)->Result<(),String>{ + context.validate()?; + if context.owner_scope != self.owner || context.installation_id != self.installation { + return Err("temporal_context_owner_or_clock".into()); + } + let store = self.temporal_store()?; + let mut state = self.command_state(expected)?; + if state.model.is_some() + || state.selection.has_history() + || state.job.phase != JobPhase::Idle + || state.recovery.descriptor.is_some() + { + return Err("temporal_restore_would_replace_existing_work".into()); + } + // A same-process reload must never lower known RAM replay/priming + // floors. Resume Paused directly; repair Blocked by its exact intent. + // A cold restore is admitted only into a fresh, unconfigured Runtime. + if state.capture.is_some() { + return Err("temporal_restore_requires_fresh_runtime".into()); + } + let next_generation = state + .generation + .checked_add(1) + .ok_or("learner generation exhausted")?; + let full_witness=FullRunRecoveryWitness{runtime_epoch:self.runtime_epoch.clone(),generation:next_generation,context:context.clone()}; + state.capture = Some(CaptureWorkflow::Temporal(Box::new( + RuntimeTemporalCapture { + context: context.clone(), + owner_generation: next_generation, + core: None, + lead_in_until: None, + failure: None, + last_now_ns: now, + fit_job: None, + evidence_job: None, + training_proof: None, + evaluation: None, + assessment: None, + diagnostic_fit: None, + public_train_requested: false, + }, + ))); + state.generation = next_generation; + drop(state); + let recovered=if full_run{ + TemporalTrainCapture::cold_restore_full_run(store,self.store.clone(),context,now,&self.runtime_epoch,now,full_witness) + }else{ + TemporalTrainCapture::cold_restore(store, context, now, &self.runtime_epoch, now) + }; + let mut core=match recovered{ + Ok(core)=>core, + Err(error)=>{ + if full_run{let mut state=self.inner.lock().map_err(|_|"learner state unavailable")?;if state.generation==next_generation{if let Some(CaptureWorkflow::Temporal(c))=state.capture.as_mut(){c.failure=Some(bounded(error.clone()));}}} + return Err(error); + } + }; + core.enable_async_restore( + self.temporal_store_arc()?, + now, + )?; + let mut state = self.inner.lock().map_err(|_| "learner state unavailable")?; + if state.generation != next_generation { + drop(state); + drop(core); + return Err("learner_state_changed".into()); + } + state.capture = Some(CaptureWorkflow::Temporal(Box::new( + RuntimeTemporalCapture { + context: context.clone(), + owner_generation: next_generation, + core: Some(core), + lead_in_until: None, + failure: None, + last_now_ns: now, + fit_job: None, + evidence_job: None, + training_proof: None, + evaluation: None, + assessment: None, + diagnostic_fit: None, + public_train_requested: false, + }, + ))); + state.generation = next_generation; + Ok(()) + } + fn repair_temporal_capture(&self, expected: &MutationGuard, now: u64) -> Result<(), String> { + let store = self.temporal_store()?; + let mut state = self.command_state(expected)?; + if busy(&state.job) { return Err("temporal_fit_job_busy".into()); } + let owner_generation = state.generation; + let Some(CaptureWorkflow::Temporal(capture)) = state.capture.as_mut() else { + return Err(PRIVATE_KIND.into()); + }; + if capture.owner_generation != owner_generation { + capture.invalidate_owner(); + return Err("temporal_owner_generation_changed".into()); + } + capture + .core + .as_mut() + .ok_or("temporal_initial_barrier_unconfirmed")? + .repair_pending(store, now)?; + capture.failure = None; + state.input_error = None; + Ok(()) + } + fn cancel_temporal_durability(&self, expected: &MutationGuard) -> Result<(), String> { + let (writer,fit,evidence,withdrawal,retired_selected)={ + let mut state=self.command_state(expected)?; + let withdrawal=state.selection.withdrawal.clone(); + let has_selected=state.selection.temporal.is_some(); + if !matches!(state.capture,Some(CaptureWorkflow::Temporal(_)))&&withdrawal.is_none()&&!has_selected{return Err(PRIVATE_KIND.into());} + if let Some(job)=&withdrawal{job.cancel();} + let retired_selected=state.selection.temporal.take(); + let(writer,fit,evidence)=match state.capture.as_mut(){Some(CaptureWorkflow::Temporal(capture))=>{ + capture.lead_in_until=None;capture.public_train_requested=false; + let fit=capture.fit_job.clone();let evidence=capture.evidence_job.clone();capture.cancel_fit(); + let writer=capture.core.as_mut().and_then(|core|{let writer=core.writer();core.invalidate("temporal_owner_write_cancelled");writer}); + if let Some(writer)=&writer{writer.cancel();}(writer,fit,evidence) + },_=>(None,None,None)}; + if busy(&state.job){state.job.cancel.store(true,Ordering::Release);state.job.phase=JobPhase::Cancelling;} + (writer,fit,evidence,withdrawal,retired_selected) + }; + let writer_result=writer.map_or(Ok(()),|writer|writer.drain()); + if let Some(fit)=fit{fit.drain();}if let Some(evidence)=evidence{evidence.drain();}if let Some(job)=withdrawal{job.drain();} + if let Some(selected)=retired_selected{selected.driver.lock().map_err(|_|"selected_driver_unavailable")?.deactivate();drop(selected);} + writer_result + } + fn temporal_capture_snapshot(&self, now: u64) -> Result { + let state = self.inner.lock().map_err(|_| "learner state unavailable")?; + let Some(CaptureWorkflow::Temporal(capture)) = state.capture.as_ref() else { + return Err(PRIVATE_KIND.into()); + }; + if capture.owner_generation != state.generation { + return Err("temporal_owner_generation_changed".into()); + } + Ok( + json!({"schema":"ruview.room-learner.private-temporal-capture.v1","mutation_guard":self.guard_for(&state),"capture":capture.snapshot(now),"fit_authority":false,"public_activation":false}), + ) + } +} + +#[cfg(test)] +#[path = "lifecycle_temporal_tests.rs"] +mod tests; +#[cfg(all(test,unix))] +#[path="lifecycle_temporal_full_run_tests.rs"] +mod full_run_tests; + +impl Drop for RoomLearningRuntime { + fn drop(&mut self) { + // The last Runtime owner has exclusive access; get_mut takes no ingress + // lock. Join before owner/store release so an old writer cannot outlive + // Runtime shutdown and recreate an erased profile later. + let state = match self.inner.get_mut() { + Ok(state) => state, + Err(poisoned) => poisoned.into_inner(), + }; + let fit = match state.capture.as_ref() { + Some(CaptureWorkflow::Temporal(capture)) => capture.fit_job.clone(), + _ => None, + }; + let evidence=match state.capture.as_ref(){Some(CaptureWorkflow::Temporal(capture))=>capture.evidence_job.clone(),_=>None}; + let withdrawal=state.selection.withdrawal.clone();if let Some(job)=&withdrawal{job.cancel();} + if let Some(fit) = &fit { fit.cancel(); } + if let Some(job)=&evidence{job.cancel();} + let writer = match state.capture.as_mut() { + Some(CaptureWorkflow::Temporal(capture)) => capture.core.as_mut().and_then(|core| { + let writer = core.writer(); + core.invalidate("temporal_runtime_dropped"); + writer + }), + _ => None, + }; + if let Some(writer) = writer { + let _ = writer.drain(); + } + if let Some(fit) = fit { fit.drain(); } + if let Some(job)=evidence{job.drain();} + if let Some(job)=withdrawal{job.drain();} + if let Some(selected)=state.selection.temporal.take(){if let Ok(mut driver)=selected.driver.lock(){driver.deactivate();}} + } +} + +#[cfg(test)] +#[path="lifecycle_temporal_held_out_tests.rs"] +mod held_out_tests; + +#[path="lifecycle_temporal_evaluation.rs"] +mod evidence_control; + +#[path="lifecycle_temporal_public.rs"] +mod public_workflow; + +#[path="lifecycle_temporal_selection.rs"] +pub(super) mod selected_control; + +impl RoomLearningRuntime { + fn retry_failed_temporal_capture(&self, expected:&MutationGuard, capture_generation:&str, now:u64) -> Result<(),String> { + let (snapshot,writer,context,post_guard,generation)={ + let mut state=self.command_state(expected)?; + if busy(&state.job) {return Err("temporal_retry_operation_busy".into());} + let generation=state.generation; + let Some(CaptureWorkflow::Temporal(c))=state.capture.as_mut()else{return Err(PRIVATE_KIND.into())}; + if c.owner_generation!=generation || c.fit_job.is_some() || c.evidence_job.is_some() {return Err("temporal_retry_operation_busy".into());} + let core=c.core.as_mut().ok_or("temporal_initial_barrier_unconfirmed")?; + if core.public_generation_id()!=capture_generation {return Err("learner_capture_identity_changed".into());} + let snapshot=core.failed_retry_state()?; + let writer=core.writer().ok_or("temporal_writer_required")?; + let context=c.context.clone(); + (snapshot,writer,context,self.guard_for(&state),generation) + }; + // Never join/repair/recertify while holding the ingress mutex. Terminal + // owner drain fences captured Writer Arcs before account erasure. + writer.reconcile_failed(now)?; + let at=self.receiver_now_ns().max(now); + let witness=FullRunRecoveryWitness{runtime_epoch:self.runtime_epoch.clone(),generation,context:context.clone()}; + let mut core=TemporalTrainCapture::cold_restore_full_run(self.temporal_store()?,self.store.clone(),&context,at,&self.runtime_epoch,at,witness)?; + core.apply_failed_retry_state(snapshot,at)?; + core.enable_async_restore(self.temporal_store_arc()?,at)?; + let mut state=self.inner.lock().map_err(|_|"learner state unavailable")?; + if self.guard_for(&state)!=post_guard || state.generation!=generation || state.capture_context()!=Some(&context) { + drop(state);drop(core);return Err("learner_state_changed".into()); + } + let Some(CaptureWorkflow::Temporal(c))=state.capture.as_mut()else{drop(state);drop(core);return Err("learner_state_changed".into())}; + let retired=c.core.replace(core);c.lead_in_until=None;c.failure=None;c.last_now_ns=at;c.public_train_requested=false; + state.input_error=None; + drop(state);drop(retired); + Ok(()) + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_evaluation.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_evaluation.rs new file mode 100644 index 0000000000..5e527362a0 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_evaluation.rs @@ -0,0 +1,530 @@ +//! C4C uses the SAME Runtime Job/executor/lifetime. No JSON authority issuer. +use super::fit_control::FitLifetime; +use super::*; +use crate::room_learner::artifact::StoredTemporalEvaluation; +use crate::room_learner::temporal_evaluation::TemporalTrainProofData; +use crate::room_learner::temporal_model_input::managed::execution::full_run::{ + PendingTemporalEvidenceSource, TemporalEvaluationBinding, +}; +use std::sync::{atomic::AtomicU64, Condvar, Weak}; +use std::time::Instant; +#[derive(Clone, Copy, PartialEq, Eq)] +pub(super) enum EvidencePurpose { + TrainProof, + HeldOutEvaluation, + Qualification, + Selection, + RestoreComplete, +} +#[derive(Clone, PartialEq)] +struct EvidenceIdentity { + epoch: String, + generation: u64, + attempt: u64, + purpose: EvidencePurpose, + binding: TemporalEvaluationBinding, +} +#[derive(Clone)] +pub(super) struct TemporalEvidenceJob { + identity: EvidenceIdentity, + lifetime: Arc, +} +impl TemporalEvidenceJob { + pub(super) fn uses_cancel(&self, cancel: &Arc) -> bool { + Arc::ptr_eq(cancel, &self.lifetime.cancel) + } + pub(super) fn observe_now(&self, now: u64) { + self.lifetime.observed.fetch_max(now, Ordering::AcqRel); + } + pub(super) fn cancel(&self) { + self.lifetime.cancel(); + } + pub(super) fn drain(&self) { + self.lifetime.drain(); + } + pub(super) fn done(&self) -> bool { + self.lifetime.done() + } + pub(super) fn operation_name(&self) -> &'static str { + match self.identity.purpose { + EvidencePurpose::TrainProof => "train_proof", + EvidencePurpose::HeldOutEvaluation => "evaluate", + EvidencePurpose::Qualification => "qualify", + EvidencePurpose::Selection => "select", + EvidencePurpose::RestoreComplete => "restore", + } + } +} +enum EvidenceOutcome { + Train(TemporalTrainProofData), + Evaluation(StoredTemporalEvaluation), + Assessment(crate::room_learner::artifact::StoredTemporalAssessment), + Selection(Arc), + Restored(TemporalTrainProofData, Option), +} +struct EvidenceCompletion { + runtime: Weak, + owner: Option>, + identity: EvidenceIdentity, + lifetime: Arc, +} +impl Drop for EvidenceCompletion { + fn drop(&mut self) { + let fallback = if self.owner.is_none() { + self.runtime.upgrade() + } else { + None + }; + if let Some(runtime) = self.owner.as_ref().or(fallback.as_ref()) { + let mut state = runtime.inner.lock().unwrap_or_else(|e| e.into_inner()); + let current = Arc::ptr_eq(&state.job.cancel, &self.lifetime.cancel) + && matches!(state.capture.as_ref(),Some(CaptureWorkflow::Temporal(c))if c.evidence_job.as_ref().is_some_and(|j|j.identity==self.identity&&Arc::ptr_eq(&j.lifetime,&self.lifetime))); + if current && busy(&state.job) { + state.job.phase = if self.lifetime.cancel.load(Ordering::Acquire) { + JobPhase::Cancelled + } else { + JobPhase::Failed + }; + state.job.error = Some("temporal_evidence_worker_unavailable".into()); + } + } + // Every result/raw snapshot and IO operation has completed before this + // signal. A last upgraded Runtime Arc may now safely enter its Drop. + *self + .lifetime + .completed + .lock() + .unwrap_or_else(|e| e.into_inner()) = true; + self.lifetime.completion.notify_all(); + drop(fallback); + } +} +impl RoomLearningRuntime { + pub(super) fn start_temporal_evidence( + self: &Arc, + expected: &MutationGuard, + now: u64, + purpose: EvidencePurpose, + ) -> Result { + let executor = tokio::runtime::Handle::try_current() + .map_err(|_| "temporal_evaluation_executor_unavailable")?; + let recovery = self.temporal_store_arc()?; + let store = self.store.clone(); + let (input, identity, lifetime) = self.reserve_temporal_evidence(expected, now, purpose)?; + let attempt = identity.attempt; + let completion = EvidenceCompletion { + runtime: Arc::downgrade(self), + owner: None, + identity: identity.clone(), + lifetime: lifetime.clone(), + }; + executor.spawn_blocking(move || { + let mut completion = completion; + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + Self::compute_temporal_evidence(&store, &recovery, input, &identity, &lifetime) + })) + .unwrap_or_else(|_| Err("temporal_evaluation_worker_failed_safely".into())); + completion.owner = completion.runtime.upgrade(); + if let Some(runtime) = &completion.owner { + runtime.admit_temporal_evidence(&identity, &lifetime, result); + } else { + drop(result); + } + }); + Ok(attempt) + } + fn reserve_temporal_evidence( + &self, + expected: &MutationGuard, + now: u64, + purpose: EvidencePurpose, + ) -> Result< + ( + PendingTemporalEvidenceSource, + EvidenceIdentity, + Arc, + ), + String, + > { + let mut retired_job = None; + let mut retired_evidence = None; + let (input, identity, lifetime) = { + let mut state = self.command_state(expected)?; + if busy(&state.job) { + return Err("learner_job_busy".into()); + } + let generation = state.generation; + let attempt = state.workflow_revision; + let Some(CaptureWorkflow::Temporal(c)) = state.capture.as_mut() else { + return Err(PRIVATE_KIND.into()); + }; + if c.owner_generation != generation + || c.failure.is_some() + || c.evidence_job.as_ref().is_some_and(|j| !j.done()) + || c.fit_job.as_ref().is_some_and(|j| !j.done()) + { + return Err("temporal_evaluation_owner_or_job_busy".into()); + } + c.check_time(now)?; + let input = c + .core + .as_ref() + .ok_or("temporal_initial_barrier_unconfirmed")? + .prepare_evaluation_source(now, purpose != EvidencePurpose::TrainProof)?; + if !input + .binding() + .owner_matches(&self.runtime_epoch, generation) + { + return Err("temporal_evaluation_owner_changed".into()); + } + let identity = EvidenceIdentity { + epoch: self.runtime_epoch.clone(), + generation, + attempt, + purpose, + binding: input.binding().clone(), + }; + let cancel = Arc::new(AtomicBool::new(false)); + let lifetime = Arc::new(FitLifetime { + cancel: cancel.clone(), + publication: Mutex::new(()), + completed: Mutex::new(false), + completion: Condvar::new(), + anchor: now, + origin: Instant::now(), + observed: AtomicU64::new(now), + expires_at_ms: input.expires_at_ms(), + #[cfg(test)] + hook: self + .temporal_fit_hook + .lock() + .unwrap_or_else(|e| e.into_inner()) + .clone(), + #[cfg(test)] + io_hook: self + .temporal_evidence_io_hook + .lock() + .unwrap_or_else(|e| e.into_inner()) + .clone(), + }); + retired_evidence = c.evidence_job.replace(TemporalEvidenceJob { + identity: identity.clone(), + lifetime: lifetime.clone(), + }); + retired_job = Some(std::mem::replace( + &mut state.job, + Job { + phase: JobPhase::Running, + cancel, + report: None, + error: None, + }, + )); + if purpose == EvidencePurpose::Selection { + state.selection.error=Some("Selection transition pending; current evidence is unavailable until durable reconciliation.".into()); + } + (input, identity, lifetime) + }; + drop(retired_evidence); + drop(retired_job); + Ok((input, identity, lifetime)) + } + pub(super) fn run_temporal_selection( + self: &Arc, + expected: &MutationGuard, + now: u64, + ) -> Result<(), String> { + let recovery = self.temporal_store_arc()?; + let store = self.store.clone(); + let (input, identity, lifetime) = + self.reserve_temporal_evidence(expected, now, EvidencePurpose::Selection)?; + let completion = EvidenceCompletion { + runtime: Arc::downgrade(self), + owner: Some(self.clone()), + identity: identity.clone(), + lifetime: lifetime.clone(), + }; + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + Self::compute_temporal_evidence(&store, &recovery, input, &identity, &lifetime) + })) + .unwrap_or_else(|_| Err("temporal_selection_worker_failed_safely".into())); + let failure = result.as_ref().err().cloned(); + self.admit_temporal_evidence(&identity, &lifetime, result); + // Finish/drain tracks the actual store operation, not an HTTP ACK. + drop(completion); + if let Some(error) = failure { + return Err(error); + } + let state = self.inner.lock().map_err(|_| "learner state unavailable")?; + if state.job.phase != JobPhase::Ready || !Arc::ptr_eq(&state.job.cancel, &lifetime.cancel) { + return Err("temporal_selection_not_adopted".into()); + } + Ok(()) + } + fn compute_temporal_evidence( + store: &crate::room_learner::artifact::ArtifactStore, + recovery: &crate::room_learner::recovery_store::RecoveryStore, + input: PendingTemporalEvidenceSource, + identity: &EvidenceIdentity, + lifetime: &FitLifetime, + ) -> Result { + lifetime.at("evidence_before_validate"); + let now = lifetime.allowed()?; + let outcome: Result = match identity.purpose { + EvidencePurpose::TrainProof => { + let source = input.recertify_train(recovery, now, &lifetime.cancel)?; + let _publication = lifetime + .publication + .lock() + .map_err(|_| "temporal_evaluation_publication_unavailable")?; + lifetime.at("evidence_publication_entered"); + let proof = store.persist_current_temporal_train_proof( + &source, + &lifetime.cancel, + &mut || lifetime.allowed(), + &mut |stage| { + lifetime.allowed()?; + lifetime.io(stage) + }, + )?; + lifetime.allowed()?; + Ok(EvidenceOutcome::Train(proof)) + } + EvidencePurpose::RestoreComplete => { + let source = input.recertify_evaluation(recovery, now, &lifetime.cancel)?; + let _publication = lifetime + .publication + .lock() + .map_err(|_| "temporal_restore_publication_unavailable")?; + let (proof, evaluation) = store.reconcile_temporal_snapshot( + source, + &lifetime.cancel, + &mut || lifetime.allowed(), + &mut |stage| lifetime.io(stage), + )?; + Ok(EvidenceOutcome::Restored(proof, evaluation)) + } + EvidencePurpose::Selection => { + let source = input.recertify_evaluation(recovery, now, &lifetime.cancel)?; + let _publication = lifetime + .publication + .lock() + .map_err(|_| "temporal_selection_publication_unavailable")?; + let revision = store.selection_revision_any()?; + let assessment = store.qualify_temporal_snapshot( + &source, + &lifetime.cancel, + &mut || lifetime.allowed(), + &mut |stage| lifetime.io(stage), + )?; + let stored = store.save_current_temporal_selection( + &source, + &assessment, + revision, + lifetime.allowed()?, + &lifetime.cancel, + &mut || lifetime.allowed(), + &mut || { + lifetime.io("selection_before_pointer_install")?; + identity + .binding + .recertify_current(recovery, lifetime.allowed()?)?; + lifetime.allowed()?; + Ok(()) + }, + &mut || lifetime.io("selection_after_pointer_rename"), + )?; + Ok(EvidenceOutcome::Selection(Arc::new( + super::selected_control::TemporalSelectedRuntime::new(stored)?, + ))) + } + EvidencePurpose::Qualification => { + let source = input.recertify_evaluation(recovery, now, &lifetime.cancel)?; + let _publication = lifetime + .publication + .lock() + .map_err(|_| "temporal_evaluation_publication_unavailable")?; + lifetime.at("evidence_publication_entered"); + let receipt = store.qualify_temporal_snapshot( + &source, + &lifetime.cancel, + &mut || lifetime.allowed(), + &mut |stage| { + lifetime.allowed()?; + lifetime.io(stage) + }, + )?; + lifetime.allowed()?; + Ok(EvidenceOutcome::Assessment(receipt)) + } + EvidencePurpose::HeldOutEvaluation => { + let source = input.recertify_evaluation(recovery, now, &lifetime.cancel)?; + let _publication = lifetime + .publication + .lock() + .map_err(|_| "temporal_evaluation_publication_unavailable")?; + lifetime.at("evidence_publication_entered"); + let receipt = store.evaluate_temporal_snapshot( + source, + &lifetime.cancel, + &mut || lifetime.allowed(), + &mut |stage| { + lifetime.allowed()?; + lifetime.io(stage) + }, + )?; + lifetime.allowed()?; + Ok(EvidenceOutcome::Evaluation(receipt)) + } + }; + let outcome = outcome?; + identity + .binding + .recertify_current(recovery, lifetime.allowed()?)?; + let model = store.load_exact_temporal_candidate( + identity.binding.context(), + identity.binding.model_sha256(), + )?; + if model.artifact_kind() != crate::room_learner::temporal_model_input::managed::SCHEMA_V3 { + return Err("temporal_evaluation_model_kind_changed".into()); + } + lifetime.allowed()?; + Ok(outcome) + } + fn admit_temporal_evidence( + &self, + identity: &EvidenceIdentity, + lifetime: &Arc, + outcome: Result, + ) { + // Outcome and replaced metadata are destroyed after inner is released. + let mut retired_proof = None; + let mut retired_evaluation = None; + let mut retired_assessment = None; + let mut retired_selection = None; + let mut state = match self.inner.lock() { + Ok(s) => s, + Err(_) => return, + }; + let same = identity.epoch == self.runtime_epoch + && identity.generation == state.generation + && Arc::ptr_eq(&state.job.cancel, &lifetime.cancel) + && matches!(state.capture.as_ref(),Some(CaptureWorkflow::Temporal(c))if c.owner_generation==identity.generation&&c.failure.is_none()&&c.evidence_job.as_ref().is_some_and(|j|j.identity==*identity&&Arc::ptr_eq(&j.lifetime,lifetime))&&c.core.as_ref().is_some_and(|core|core.evaluation_binding_matches(&identity.binding))); + if !same { + return; + } + if lifetime.allowed().is_err() { + state.job.phase = JobPhase::Cancelled; + state.job.error = Some("temporal_evaluation_cancelled_or_expired".into()); + return; + } + state.workflow_revision = state.workflow_revision.saturating_add(1); + match outcome { + Ok(EvidenceOutcome::Train(proof)) => { + if let Some(CaptureWorkflow::Temporal(c)) = state.capture.as_mut() { + retired_proof = c.training_proof.replace(proof); + } + state.job.phase = JobPhase::Ready; + state.job.error = None; + } + Ok(EvidenceOutcome::Evaluation(receipt)) => { + if receipt.binding() != &identity.binding { + state.job.phase = JobPhase::Failed; + state.job.error = Some("temporal_evaluation_result_identity_changed".into()); + return; + } + if let Some(CaptureWorkflow::Temporal(c)) = state.capture.as_mut() { + retired_proof = c.training_proof.replace(receipt.train().clone()); + retired_evaluation = c.evaluation.replace(Arc::new(receipt)); + } + state.job.phase = JobPhase::Ready; + state.job.error = None; + } + Ok(EvidenceOutcome::Assessment(receipt)) => { + if receipt.binding() != &identity.binding { + state.job.phase = JobPhase::Failed; + state.job.error = Some("temporal_assessment_result_identity_changed".into()); + return; + } + if let Some(CaptureWorkflow::Temporal(c)) = state.capture.as_mut() { + retired_assessment = c.assessment.replace(Arc::new(receipt)); + } + state.job.phase = JobPhase::Ready; + state.job.error = None; + } + Ok(EvidenceOutcome::Restored(proof, evaluation)) => { + if let Some(CaptureWorkflow::Temporal(c)) = state.capture.as_mut() { + retired_proof = c.training_proof.replace(proof); + if let Some(evaluation) = evaluation { + retired_evaluation = c.evaluation.replace(Arc::new(evaluation)); + } + } + state.job.phase = JobPhase::Ready; + state.job.error = None; + } + Ok(EvidenceOutcome::Selection(selected)) => { + let floors = match state.capture.as_ref() { + Some(CaptureWorkflow::Temporal(c)) => { + c.core.as_ref().map(|core| core.current_source_floors()) + } + _ => None, + }; + if let Some((floors, receiver)) = floors { + let seeded = selected + .driver + .lock() + .map_err(|_| "selected driver unavailable".to_string()) + .and_then(|mut d| d.advance_selected_floors(&floors, receiver)); + if let Err(error) = seeded { + state.job.phase = JobPhase::Failed; + state.job.error = Some(error); + return; + } + } + retired_selection = state.selection.temporal.replace(selected); + state.selection.error = None; + state.job.phase = JobPhase::Ready; + state.job.error = None; + } + Err(error) => { + state.job.phase = if lifetime.cancel.load(Ordering::Acquire) { + JobPhase::Cancelled + } else { + JobPhase::Failed + }; + state.job.error = Some(bounded(error)); + } + } + } +} +impl RoomLearningRuntime { + pub(super) fn cancel_temporal_evidence( + &self, + expected: &MutationGuard, + ) -> Result { + let job = { + let mut state = self.command_state(expected)?; + let Some(CaptureWorkflow::Temporal(c)) = state.capture.as_ref() else { + return Err(PRIVATE_KIND.into()); + }; + let job = c + .evidence_job + .clone() + .ok_or("temporal_evaluation_job_missing")?; + if state.job.phase == JobPhase::Ready + && Arc::ptr_eq(&state.job.cancel, &job.lifetime.cancel) + && c.failure.is_none() + { + return Ok(JobPhase::Ready); + } + job.cancel(); + if busy(&state.job) { + state.job.phase = JobPhase::Cancelling; + } + job + }; + job.drain(); + let state = self.inner.lock().map_err(|_| "learner state unavailable")?; + Ok(state.job.phase) + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_fit.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_fit.rs new file mode 100644 index 0000000000..a1a42b4319 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_fit.rs @@ -0,0 +1,502 @@ +//! Private fit-only adapter into the EXISTING Job, fitter and ArtifactStore. +//! No public route, fitted recovery, HeldOut, proof, selection or live authority. +use super::*; +use crate::room_learner::{ + artifact::{TemporalCandidateReceipt, TemporalFitDiagnostics, TemporalFitSaveResult}, + recovery_store::RecoveryStore, + temporal_model_input::{ + managed::execution::{PendingTemporalFitInput, TemporalFitCaptureIdentity,VerifiedTemporalFitSource}, + TemporalModel, + }, +}; +use std::sync::{atomic::AtomicU64, Condvar, Weak}; +use std::time::Instant; + +#[derive(Clone, PartialEq, Eq)] +struct TemporalFitIdentity { + runtime_epoch: String, + generation: u64, + // The consumed start revision identifies THIS job. Later denied commands + // may consume global revisions without replacing this active operation. + attempt: u64, + capture: TemporalFitCaptureIdentity, +} +pub(super) struct TemporalDiagnosticFit { + model: Arc, + saved: TemporalFitSaveResult, + accepted: Option, +} +/// Minted ONLY in the actual current C3 result-admission branch. Not serde, +/// Clone, a candidate-import constructor, or a report/claims permission. +pub(crate) struct AcceptedTemporalFit { + runtime_epoch:String,generation:u64,fit_attempt:u64, + model:Arc,source:VerifiedTemporalFitSource, +} +impl AcceptedTemporalFit { + pub(crate) fn consume_for_owner(self,epoch:&str,generation:u64,binding:&TemporalFitCaptureIdentity)->Result<(Arc,VerifiedTemporalFitSource),String>{ + if self.runtime_epoch!=epoch||self.generation!=generation||self.fit_attempt==0||self.source.binding()!=binding{ + return Err("temporal_accepted_fit_owner_changed".into()); + } + Ok((self.model,self.source)) + } +} +/// One lifetime barrier for the SAME existing Job/executor, not a new queue or +/// trainer. Cancellation is atomic; waiting/publication never holds inner. +pub(super) struct FitLifetime { + pub(super) cancel: Arc, + pub(super) publication: Mutex<()>, + pub(super) completed: Mutex, + pub(super) completion: Condvar, + pub(super) anchor: u64, + pub(super) origin: Instant, + pub(super) observed: AtomicU64, + pub(super) expires_at_ms: u64, + #[cfg(test)] + pub(super) hook: Option>, + #[cfg(test)] + pub(super) io_hook: OptionResult<(),String>+Send+Sync>>, +} +impl FitLifetime { + pub(super) fn io(&self,stage:&'static str)->Result<(),String>{ + #[cfg(test)] if let Some(hook)=&self.io_hook{return hook(stage);} + #[cfg(not(test))] let _=stage; + Ok(()) + } + pub(super) fn now(&self) -> Result { + let elapsed: u64 = self + .origin + .elapsed() + .as_nanos() + .try_into() + .map_err(|_| "temporal_fit_clock_overflow")?; + Ok(self.observed.load(Ordering::Acquire).max( + self.anchor + .checked_add(elapsed) + .ok_or("temporal_fit_clock_overflow")?, + )) + } + pub(super) fn allowed(&self) -> Result { + if self.cancel.load(Ordering::Acquire) { + return Err("temporal_fit_cancelled".into()); + } + let now = self.now()?; + if now / 1_000_000 >= self.expires_at_ms { + return Err("temporal_fit_original_deadline_expired".into()); + } + Ok(now) + } + pub(super) fn cancel(&self) { + self.cancel.store(true, Ordering::Release); + } + pub(super) fn done(&self) -> bool { + *self.completed.lock().unwrap_or_else(|e| e.into_inner()) + } + pub(super) fn drain(&self) { + self.cancel(); + let mut complete = self.completed.lock().unwrap_or_else(|e| e.into_inner()); + while !*complete { + complete = self + .completion + .wait(complete) + .unwrap_or_else(|e| e.into_inner()); + } + } + pub(super) fn at(&self, stage: &'static str) { + #[cfg(test)] + if let Some(hook) = &self.hook { + hook(stage); + } + #[cfg(not(test))] + let _ = stage; + } +} +#[derive(Clone)] +pub(super) struct TemporalFitJob { + identity: TemporalFitIdentity, + lifetime: Arc, +} +impl TemporalFitJob { + pub(super) fn uncancelled_completed(&self)->bool{self.done()&&!self.lifetime.cancel.load(Ordering::Acquire)} + pub(super) fn attempt_matches(&self,attempt:u64,cancel:&Arc)->bool{self.identity.attempt==attempt&&self.uses_cancel(cancel)&&!self.lifetime.cancel.load(Ordering::Acquire)} + pub(super) fn uses_cancel(&self,cancel:&Arc)->bool{Arc::ptr_eq(cancel,&self.lifetime.cancel)} + pub(super) fn observe_now(&self, now: u64) { + self.lifetime.observed.fetch_max(now, Ordering::AcqRel); + } + pub(super) fn cancel(&self) { + self.lifetime.cancel(); + } + pub(super) fn drain(&self) { + self.lifetime.drain(); + } + pub(super) fn done(&self) -> bool { self.lifetime.done() } +} +/// Created before scheduling. Dropping an unstarted/discarded closure also +/// releases waiters. Worker outcome/raw ownership is destroyed before this. +struct FitCompletion { + runtime: Weak, + // An upgraded owner is held only AFTER all fit/storage work. The done + // signal precedes its release, or the last Arc could self-drain here. + owner: Option>, + identity: TemporalFitIdentity, + lifetime: Arc, +} +impl Drop for FitCompletion { + fn drop(&mut self) { + let fallback = if self.owner.is_none() { + self.runtime.upgrade() + } else { + None + }; + if let Some(runtime) = self.owner.as_ref().or(fallback.as_ref()) { + let mut state = runtime.inner.lock().unwrap_or_else(|e| e.into_inner()); + let current = Arc::ptr_eq(&state.job.cancel, &self.lifetime.cancel) + && matches!(state.capture.as_ref(),Some(CaptureWorkflow::Temporal(c)) + if c.fit_job.as_ref().is_some_and(|j|j.identity==self.identity&&Arc::ptr_eq(&j.lifetime,&self.lifetime))); + if current && busy(&state.job) { + state.job.phase = if self.lifetime.cancel.load(Ordering::Acquire) { + JobPhase::Cancelled + } else { + JobPhase::Failed + }; + state.job.error = Some( + if state.job.phase == JobPhase::Cancelled { + "temporal_fit_cancelled" + } else { + "temporal_fit_worker_unavailable" + } + .into(), + ); + } + } + *self + .lifetime + .completed + .lock() + .unwrap_or_else(|e| e.into_inner()) = true; + self.lifetime.completion.notify_all(); + // No fit/store operation follows this signal. A final Runtime Arc may + // now run Drop safely because its exact fit lifetime is complete. + drop(fallback); + } +} +impl RuntimeTemporalCapture { + pub(super) fn cancel_fit(&self) { + if let Some(job) = &self.fit_job { job.cancel(); } + if let Some(job) = &self.evidence_job { job.cancel(); } + } + pub(super) fn take_current_accepted_fit(&mut self,epoch:&str,generation:u64,now:u64)->Result{ + if self.owner_generation!=generation||self.failure.is_some(){return Err("temporal_accepted_fit_owner_changed".into());} + let job=self.fit_job.as_ref().ok_or("temporal_actual_current_fit_required")?; + if job.identity.runtime_epoch!=epoch||job.lifetime.cancel.load(Ordering::Acquire)||!job.done(){return Err("temporal_actual_current_fit_required".into());} + let current=self.core.as_ref().ok_or("temporal_initial_barrier_unconfirmed")?.fit_capture_identity(now)?; + let accepted=self.diagnostic_fit.as_ref().and_then(|d|d.accepted.as_ref()).ok_or("temporal_full_run_actual_accepted_fit_required")?; + if accepted.source.binding()!=¤t{return Err("temporal_accepted_fit_capture_changed".into());} + Ok(self.diagnostic_fit.as_mut().unwrap().accepted.take().unwrap()) + } +} +impl RoomLearningRuntime { + /// Private cancellation of the SAME Job, preserving confirmed Train data. + /// Unlike an AtomicBool-only response, completion means publication drained. + pub(super) fn cancel_temporal_fit(&self, expected: &MutationGuard) -> Result { + let job = { + let mut state = self.command_state(expected)?; + let Some(CaptureWorkflow::Temporal(capture)) = state.capture.as_ref() else { + return Err(PRIVATE_KIND.into()); + }; + let job = capture.fit_job.clone().ok_or("temporal_fit_job_missing")?; + // Ready is published only after actual fit/store validation. A + // late compute-only cancel cannot revoke that accepted result or + // its separately owned Frozen transaction, even before the final + // completion notification. Capture/owner cancellation still uses + // cancel_fit + drain and revokes the entire owner operation. + if state.job.phase == JobPhase::Ready + && job.identity.runtime_epoch == self.runtime_epoch + && job.identity.generation == state.generation + && Arc::ptr_eq(&state.job.cancel, &job.lifetime.cancel) + && capture.owner_generation == state.generation + && capture.failure.is_none() + && capture.core.as_ref().is_some_and(|c| c.is_full_run()) + && capture.diagnostic_fit.is_some() + { + return Ok(JobPhase::Ready); + } + job.cancel(); + if busy(&state.job) { + state.job.phase = JobPhase::Cancelling; + } + job + }; + job.drain(); + let state = self.inner.lock().map_err(|_| "learner state unavailable")?; + Ok(state.job.phase) + } + /// Private entry only. The existing public start_training remains Legacy. + pub(super) fn start_temporal_fit( + self: &Arc, + expected: &MutationGuard, + now: u64, + ) -> Result { + let executor = tokio::runtime::Handle::try_current() + .map_err(|_| "temporal_fit_executor_unavailable")?; + let store = self.store.clone(); + let recovery = self.temporal_store_arc()?; + let retired_job; + let retired_fit; + let (input, identity, lifetime) = { + let mut state = self.command_state(expected)?; + if busy(&state.job) || state.job.phase == JobPhase::Ready { + return Err("temporal_fit_job_busy_or_frozen".into()); + } + let generation = state.generation; + let attempt = state.workflow_revision; + let Some(CaptureWorkflow::Temporal(capture)) = state.capture.as_mut() else { + return Err(PRIVATE_KIND.into()); + }; + if capture.owner_generation != generation { + capture.invalidate_owner(); + return Err("temporal_owner_generation_changed".into()); + } + if capture.lead_in_until.is_some() + || capture.failure.is_some() + || capture.fit_job.as_ref().is_some_and(|j| !j.done()) + { + return Err("temporal_fit_capture_not_ready".into()); + } + capture.check_time(now)?; + let input = capture + .core + .as_ref() + .ok_or("temporal_initial_barrier_unconfirmed")? + .prepare_fit_input(now)?; + let binding = input.identity().clone(); + if binding.owner() != self.owner + || binding.installation() != self.installation + || binding.context() != capture.context.digest() + { + return Err("temporal_fit_owner_binding_changed".into()); + } + let cancel = Arc::new(AtomicBool::new(false)); + let identity = TemporalFitIdentity { + runtime_epoch: self.runtime_epoch.clone(), + generation, + attempt, + capture: binding.clone(), + }; + let lifetime = Arc::new(FitLifetime { + cancel: cancel.clone(), + publication: Mutex::new(()), + completed: Mutex::new(false), + completion: Condvar::new(), + anchor: now, + origin: Instant::now(), + observed: AtomicU64::new(now), + expires_at_ms: binding.expires_at_ms(), + #[cfg(test)] + io_hook:None, + #[cfg(test)] + hook: self + .temporal_fit_hook + .lock() + .unwrap_or_else(|e| e.into_inner()) + .clone(), + }); + retired_fit = capture.fit_job.replace(TemporalFitJob { + identity: identity.clone(), + lifetime: lifetime.clone(), + }); + retired_job = Some(std::mem::replace( + &mut state.job, + Job { + phase: JobPhase::Running, + cancel, + report: None, + error: None, + }, + )); + state.frozen_training = true; + (input, identity, lifetime) + }; + drop(retired_fit); + drop(retired_job); + let attempt = identity.attempt; + let completion = FitCompletion { + runtime: Arc::downgrade(self), + owner: None, + identity: identity.clone(), + lifetime: lifetime.clone(), + }; + executor.spawn_blocking(move || { + // This guard outlives every outcome/session/prefix in the called + // body, including panic unwinding and rejected result admission. + let mut completion = completion; + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + Self::compute_temporal_fit(&store, &recovery, input, &identity, &lifetime) + })) + .unwrap_or_else(|_| Err("temporal_fit_worker_failed_safely".into())); + completion.owner = completion.runtime.upgrade(); + lifetime.at("completion_owner_upgraded"); + if let Some(runtime) = &completion.owner { + runtime.admit_temporal_fit(&identity, &lifetime, result); + #[cfg(test)] + lifetime.at("accepted_ready_published"); + } else { + drop(result); + } + }); + Ok(attempt) + } + fn compute_temporal_fit( + store: &ArtifactStore, + recovery: &RecoveryStore, + input: PendingTemporalFitInput, + identity: &TemporalFitIdentity, + lifetime: &FitLifetime, + ) -> Result<(Arc, TrainingReport, TemporalFitSaveResult,Option), String> { + lifetime.at("before_validate"); + let validated = input.validate(recovery, lifetime.allowed()?)?; + let (mut session, permit,mut source) = validated.into_session_admission_and_source()?; + lifetime.at("before_fit"); + lifetime.allowed()?; + let configured = session.bound_managed_fit_config()?; + let (model, report) = session.fit_managed_with_cancel(permit, &lifetime.cancel)?; + // Session owns raw refs; release it on the worker, never in admission. + drop(session); + lifetime.at("before_save"); + lifetime.allowed()?; + let saved = { + let _publication = lifetime + .publication + .lock() + .map_err(|_| "temporal_fit_publication_unavailable")?; + let now = lifetime.allowed()?; + identity.capture.recertify_current(recovery, now)?; + lifetime.at("publication_entered"); + lifetime.allowed()?; + #[cfg(not(test))] + let saved = + store.save_temporal_candidate_with_fit_summary(&model, &report, &configured)?; + #[cfg(test)] + let saved = store.save_temporal_candidate_with_fit_summary_probe( + &model, + &report, + &configured, + |_| lifetime.at("model_saved"), + )?; + lifetime.at("after_save"); + saved + }; + source.bind_saved_fit(&model,&saved)?; + identity + .capture + .recertify_current(recovery, lifetime.allowed()?)?; + lifetime.at("before_admission"); + lifetime.allowed()?; + Ok((Arc::new(model), report, saved,Some(source))) + } + fn admit_temporal_fit( + &self, + identity: &TemporalFitIdentity, + lifetime: &Arc, + mut outcome: Result<(Arc, TrainingReport, TemporalFitSaveResult,Option), String>, + ) { + // Outcome and retired owners precede the guard: even rejection/panic + // releases inner before destroying model/evidence/report allocations. + let mut retired_diagnostic = None; + let mut retired_report = None; + let mut state = match self.inner.lock() { + Ok(state) => state, + Err(_) => return, + }; + let same = identity.runtime_epoch == self.runtime_epoch + && identity.generation == state.generation + && Arc::ptr_eq(&state.job.cancel, &lifetime.cancel) + && matches!(state.capture.as_ref(),Some(CaptureWorkflow::Temporal(c)) if c.owner_generation==state.generation + && c.context.digest()==identity.capture.context() + && c.context.owner_scope==self.owner && c.context.installation_id==self.installation + && c.fit_job.as_ref().is_some_and(|j|j.identity==*identity&&Arc::ptr_eq(&j.lifetime,lifetime))); + if !same { + return; + } + let now = lifetime.now(); + let capture_valid=now.as_ref().is_ok_and(|now|{ + identity.capture.check_deadline(*now).is_ok()&&matches!(state.capture.as_ref(),Some(CaptureWorkflow::Temporal(c)) + if c.failure.is_none() && c.core.as_ref().and_then(|core|core.fit_capture_identity(*now).ok()).as_ref()==Some(&identity.capture)) + }); + if lifetime.cancel.load(Ordering::Acquire) || !capture_valid { + state.job.phase = JobPhase::Cancelled; + state.job.error = Some("temporal_fit_cancelled_or_capture_changed".into()); + return; + } + state.workflow_revision = state.workflow_revision.saturating_add(1); + match &mut outcome { + Ok((model, report, saved,source)) => { + if let Some(CaptureWorkflow::Temporal(capture)) = state.capture.as_mut() { + retired_diagnostic = capture.diagnostic_fit.take(); + capture.diagnostic_fit = Some(TemporalDiagnosticFit { + model: model.clone(), + saved: TemporalFitSaveResult { + candidate: saved.candidate.clone(), + fit_diagnostics: saved.fit_diagnostics.clone(), + }, + accepted:if source.as_ref().is_some_and(|s|s.is_full_run()){ + Some(AcceptedTemporalFit{runtime_epoch:identity.runtime_epoch.clone(),generation:identity.generation, + fit_attempt:identity.attempt,model:model.clone(),source:source.take().unwrap()}) + }else{None}, + }); + } + retired_report = state.job.report.replace(report.clone()); + state.job.phase = JobPhase::Ready; + state.job.error = None; + } + Err(error) => { + state.job.phase = JobPhase::Failed; + state.job.error = Some(bounded(error.clone())); + } + } + drop(state); + drop(retired_report); + drop(retired_diagnostic); + drop(outcome); + } + pub(super) fn temporal_diagnostic_fit( + &self, + ) -> Result< + Option<( + Arc, + TemporalCandidateReceipt, + TemporalFitDiagnostics, + )>, + String, + > { + let state = self.inner.lock().map_err(|_| "learner state unavailable")?; + let Some(CaptureWorkflow::Temporal(capture)) = state.capture.as_ref() else { + return Err(PRIVATE_KIND.into()); + }; + if capture.owner_generation != state.generation { + return Err("temporal_owner_generation_changed".into()); + } + if capture.diagnostic_fit.is_some() + && (capture.failure.is_some() + || state.job.phase != JobPhase::Ready + || capture.context.owner_scope != self.owner + || capture.context.installation_id != self.installation + || capture.fit_job.as_ref().is_none_or(|job| { + job.lifetime.cancel.load(Ordering::Acquire) + || job.identity.capture.context() != capture.context.digest() + })) + { + return Err("temporal_fit_owner_or_context_unavailable".into()); + } + Ok(capture.diagnostic_fit.as_ref().map(|fit| { + ( + fit.model.clone(), + fit.saved.candidate.clone(), + fit.saved.fit_diagnostics.clone(), + ) + })) + } +} + +#[cfg(all(test, unix))] +#[path = "lifecycle_temporal_fit_tests.rs"] +mod tests; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_fit_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_fit_tests.rs new file mode 100644 index 0000000000..569cccb21f --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_fit_tests.rs @@ -0,0 +1,738 @@ +//! SYNTHETIC packets/labels through the actual native/seal producer and fitter. +//! No test permit issuer, physical RF, teacher or qualification evidence. +use super::super::tests as capture_fixture; +use super::*; +use crate::room_learner::{ + native::{canonicalize_esp32, NativePacketInput}, + observation::CaptureProvenance, +}; +use std::sync::mpsc; +use std::time::Duration; +use wifi_densepose_hardware::{Adr018Flags, PpduType}; + +fn configured(root: &Path) -> (Arc, u64) { + let runtime = Arc::new(capture_fixture::runtime(root, capture_fixture::START)); + let store = runtime.recovery_store.clone().unwrap(); + runtime + .configure_temporal_capture_with( + &runtime.mutation_guard().unwrap(), + capture_fixture::context(), + "synthetic-c3-native-fit".into(), + capture_fixture::START, + move |schedule, run, mut config, now, _| { + // Bound when this capture is CREATED, never overridden by fit input. + config.epochs = 2; + TemporalTrainCapture::prepare_async(schedule, run, config, now, store) + }, + ) + .unwrap(); + let now = capture_fixture::settle_async_boundary(&runtime, capture_fixture::START); + (runtime, now) +} +fn packet(source: u8, sequence: u32, at: u64, label: RoomLabel) -> CanonicalPacket { + let gain = match label { + RoomLabel::Empty => 3, + RoomLabel::AdjacentOnly => 4, + RoomLabel::OccupiedStatic => 8, + RoomLabel::OccupiedMoving => 10 + (sequence % 2) as u8, + _ => unreachable!(), + }; + let iq = (0..64) + .flat_map(|bin| [gain + source + (bin % 2) as u8, 4]) + .collect::>(); + canonicalize_esp32(NativePacketInput { + node_id: source, + n_antennas: 1, + raw_bins: 64, + frequency_hz: 2_437_000_000, + ppdu_type: PpduType::HtLegacy, + sequence, + received_at_ns: at, + receiver_sample_rate_hz: 33., + iq_bytes: &iq, + flags: Adr018Flags::default(), + provenance: CaptureProvenance::LiveReceiverObserved, + }) + .unwrap() +} +fn record_train(root: &Path) -> (Arc, u64) { + assert!(!cfg!(debug_assertions), "complete native-persistence fixture requires Release; classify the caller"); + let (runtime, mut now) = configured(root); + let schedule = ManagedScheduleContract::build( + capture_fixture::context(), + LeadInPolicy::Train5HeldOut15EngineeringV1, + ) + .unwrap(); + let mut sequence = 0; + let mut starts = 0; + for episode in schedule + .episodes() + .iter() + .filter(|e| e.role == DataRole::Train) + { + loop { + let progress = capture_fixture::snapshot(&runtime, now); + match progress["stage"].as_str().unwrap() { + "ready" | "paused" => { + now = capture_fixture::async_lead_end(&runtime, now); + starts += 1; + runtime + .tick_temporal_capture(&runtime.mutation_guard().unwrap(), now) + .unwrap(); + } + "needs_confirmation" => break, + "failed" => panic!("synthetic C3 phase{} failed: {progress}", episode.index), + _ => {} + } + let pending = capture_fixture::snapshot(&runtime, now)["durability_pending"] == true; + sequence += 1; + for source in 1..=3 { + match runtime.ingest_native(&packet(source, sequence, now, episode.condition), now) + { + Ok(()) => {} + Err(error) if error == "temporal_storage_busy_pause" => break, + Err(error) => panic!("synthetic C3 ingest: {error}"), + } + } + now += capture_fixture::PERIOD; + if pending { + std::thread::sleep(Duration::from_nanos(capture_fixture::PERIOD)); + } + assert!(starts < 120, "synthetic C3 interruption bound exhausted"); + } + let guard = runtime.mutation_guard().unwrap(); + let challenge = runtime.temporal_confirmation_challenge(&guard).unwrap(); + runtime + .confirm_temporal_phase(&guard, &challenge, now) + .unwrap(); + assert_eq!( + capture_fixture::snapshot(&runtime, now)["durability_pending"], + true + ); + now = capture_fixture::settle_async_boundary(&runtime, now); + } + let complete = capture_fixture::snapshot(&runtime, now); + assert_eq!(complete["stage"], "complete"); + assert_eq!( + complete["sealed_train_episodes"], + schedule.counts().train_episodes + ); + assert_eq!(complete["durable_windows"], schedule.counts().train_windows); + println!( + "SYNTHETIC_C3_REAL_TRAIN_CAPTURE: rows={} seals={} starts={starts}", + schedule.counts().train_windows, + schedule.counts().train_episodes + ); + (runtime, now) +} +// This process-local cache is created by the REAL seven-phase native/seal +// producer once, then strictly cold-restored for independent fault cases. It +// contains only disposable SYNTHETIC private test files, never user samples. +fn complete_train(root: &Path) -> (Arc, u64) { + static RECORDED: std::sync::OnceLock<(tempfile::TempDir, u64)> = std::sync::OnceLock::new(); + let (recorded, at) = RECORDED.get_or_init(|| { + let directory = tempfile::tempdir().unwrap(); + let (runtime, at) = record_train(directory.path()); + drop(runtime); + (directory, at) + }); + fn copy_private(from: &Path, to: &Path) { + use std::os::unix::fs::PermissionsExt; + assert!(!std::fs::symlink_metadata(from) + .unwrap() + .file_type() + .is_symlink()); + if from.is_dir() { + std::fs::create_dir_all(to).unwrap(); + std::fs::set_permissions(to, std::fs::Permissions::from_mode(0o700)).unwrap(); + for entry in std::fs::read_dir(from).unwrap() { + let entry = entry.unwrap(); + copy_private(&entry.path(), &to.join(entry.file_name())); + } + } else { + assert!(from.is_file()); + std::fs::copy(from, to).unwrap(); + std::fs::set_permissions(to, std::fs::Permissions::from_mode(0o600)).unwrap(); + } + } + copy_private(recorded.path(), root); + let now = at + 1_000_000_000; + let runtime = Arc::new(capture_fixture::runtime(root, now)); + runtime + .restore_temporal_capture( + &runtime.mutation_guard().unwrap(), + &capture_fixture::context(), + now, + ) + .unwrap(); + assert_eq!( + capture_fixture::snapshot(&runtime, now)["stage"], + "complete" + ); + (runtime, now) +} +struct ReleaseGate(Option>); +impl ReleaseGate { + fn release(&mut self) { + if let Some(send) = self.0.take() { + let _ = send.try_send(()); + } + } +} +impl Drop for ReleaseGate { + fn drop(&mut self) { + self.release(); + } +} +fn hold(runtime: &RoomLearningRuntime, stage: &'static str) -> (mpsc::Receiver<()>, ReleaseGate) { + hold_outcome(runtime, stage, false) +} +fn hold_outcome( + runtime: &RoomLearningRuntime, + stage: &'static str, + unwind: bool, +) -> (mpsc::Receiver<()>, ReleaseGate) { + let (entered_tx, entered_rx) = mpsc::sync_channel(1); + let (release_tx, release_rx) = mpsc::sync_channel(1); + let release = Mutex::new(release_rx); + *runtime.temporal_fit_hook.lock().unwrap() = Some(Arc::new(move |at| { + if at == stage { + entered_tx.send(()).unwrap(); + release + .lock() + .unwrap() + .recv_timeout(Duration::from_secs(30)) + .expect("synthetic fit gate not released"); + assert!(!unwind, "synthetic last-completion-owner unwind"); + } + })); + (entered_rx, ReleaseGate(Some(release_tx))) +} +async fn wait_fit(runtime: &RoomLearningRuntime) -> JobPhase { + let start = Instant::now(); + loop { + let phase = runtime.inner.lock().unwrap().job.phase; + if !matches!(phase, JobPhase::Running | JobPhase::Cancelling) { + return phase; + } + assert!( + start.elapsed() < Duration::from_secs(120), + "bounded synthetic fit did not finish" + ); + tokio::time::sleep(Duration::from_millis(5)).await; + } +} +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn c3_incomplete_real_capture_cannot_issue_fit_or_weaken_public_legacy_guard() { + let directory = tempfile::tempdir().unwrap(); + let (runtime, now) = configured(directory.path()); + assert!(runtime + .start_temporal_fit(&runtime.mutation_guard().unwrap(), now) + .unwrap_err() + .contains("confirmed_durable_train")); + assert!(runtime + .start_training(&runtime.mutation_guard().unwrap(), FitConfig::default()) + .unwrap_err() + .contains("unsupported_command")); + assert_eq!(runtime.inner.lock().unwrap().job.phase, JobPhase::Idle); + assert!(runtime.temporal_diagnostic_fit().unwrap().is_none()); + runtime + .cancel_temporal_durability(&runtime.mutation_guard().unwrap()) + .unwrap(); +} +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[cfg_attr(debug_assertions, ignore = "native-persistence journey requires Release; CI explicitly executes and counts this test")] +async fn c3_actual_native_train_fit_and_same_store_cold_report_keep_whole_model_diagnostic() { + let directory = tempfile::tempdir().unwrap(); + let (runtime, now) = complete_train(directory.path()); + let before = capture_fixture::snapshot(&runtime, now); + let (entered, mut release) = hold(&runtime, "before_validate"); + runtime + .start_temporal_fit(&runtime.mutation_guard().unwrap(), now) + .unwrap(); + entered.recv_timeout(Duration::from_secs(10)).unwrap(); + // A denied unsupported command consumes a CAS revision, but does not + // replace the recorded fit operation or strand it in Running. + assert!(runtime + .start_training(&runtime.mutation_guard().unwrap(), FitConfig::default()) + .unwrap_err() + .contains("unsupported_command")); + assert_eq!(runtime.inner.lock().unwrap().job.phase, JobPhase::Running); + release.release(); + assert_eq!( + wait_fit(&runtime).await, + JobPhase::Ready, + "{:?}", + runtime.inner.lock().unwrap().job.error + ); + let (model, receipt, diagnostics) = runtime.temporal_diagnostic_fit().unwrap().unwrap(); + assert_eq!( + receipt.artifact_kind, + "ruview.room-learner.managed-temporal-input.v3" + ); + assert!(receipt.diagnostic_only); + assert!(!receipt.qualification_granted); + assert!(!receipt.physically_validated); + let report = runtime.inner.lock().unwrap().job.report.clone().unwrap(); + assert_eq!(report.training_cohorts, 64); + assert_eq!(report.completed_epochs, 2); + assert!(report.backbone_delta_l2 > 0.); + let summary = diagnostics.summary.unwrap(); + assert_eq!(summary.metrics.training_cohorts, report.training_cohorts); + assert_eq!( + summary.metrics.objective_evaluations, + report.objective_evaluations + ); + assert_eq!( + summary.metrics.parameter_delta_l2, + report.parameter_delta_l2 + ); + let cold = runtime + .store + .load_exact_temporal_candidate(&capture_fixture::context(), &receipt.artifact_sha256) + .unwrap(); + assert_eq!(cold.export().unwrap(), model.export().unwrap()); + let cold_report = runtime.store.temporal_fit_diagnostics(&cold); + assert_eq!( + serde_json::to_value(cold_report.summary).unwrap(), + serde_json::to_value(Some(summary)).unwrap() + ); + let state = runtime.inner.lock().unwrap(); + assert!(state.model.is_none()); + assert!(state.receipt.is_none()); + assert!(!state.selection.has_history()); + assert!(state.qualification.is_none()); + drop(state); + assert_eq!( + capture_fixture::snapshot(&runtime, now)["rows_sha256"], + before["rows_sha256"] + ); + assert!(runtime.qualify(&runtime.mutation_guard().unwrap()).is_err()); + runtime + .cancel_temporal_durability(&runtime.mutation_guard().unwrap()) + .unwrap(); + assert!( + runtime.temporal_diagnostic_fit().is_err(), + "cancelled owner retained a current diagnostic content read" + ); +} +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[cfg_attr(debug_assertions, ignore = "native-persistence journey requires Release; CI explicitly executes and counts this test")] +async fn c3_cancel_before_validation_drains_same_job_without_candidate_or_legacy_changes() { + let directory = tempfile::tempdir().unwrap(); + let (runtime, now) = complete_train(directory.path()); + let (entered, mut release) = hold(&runtime, "before_validate"); + runtime + .start_temporal_fit(&runtime.mutation_guard().unwrap(), now) + .unwrap(); + entered.recv_timeout(Duration::from_secs(10)).unwrap(); + let guard = runtime.mutation_guard().unwrap(); + let worker = runtime.clone(); + let (done_tx, done_rx) = mpsc::sync_channel(1); + let cancel = std::thread::spawn(move || { + done_tx.send(worker.cancel_temporal_fit(&guard)).unwrap(); + }); + assert!( + matches!( + done_rx.recv_timeout(Duration::from_millis(50)), + Err(mpsc::RecvTimeoutError::Timeout) + ), + "cancel falsely acknowledged an undrained worker" + ); + // Cancellation waits outside the actual ingress lock. + assert!(runtime.inner.try_lock().is_ok()); + release.release(); + assert_eq!( + done_rx + .recv_timeout(Duration::from_secs(10)) + .unwrap() + .unwrap(), + JobPhase::Cancelled + ); + cancel.join().unwrap(); + assert!(runtime.temporal_diagnostic_fit().unwrap().is_none()); + assert!(runtime + .store + .load_managed_temporal_candidate(&capture_fixture::context()) + .unwrap() + .is_none()); + assert_eq!( + capture_fixture::snapshot(&runtime, now)["stage"], + "complete" + ); + runtime + .cancel_temporal_durability(&runtime.mutation_guard().unwrap()) + .unwrap(); +} +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[cfg_attr(debug_assertions, ignore = "native-persistence journey requires Release; CI explicitly executes and counts this test")] +async fn c3_exact_store_recertification_rejects_tampered_head_before_any_fit_save() { + let directory = tempfile::tempdir().unwrap(); + let (runtime, now) = complete_train(directory.path()); + let (entered, mut release) = hold(&runtime, "before_validate"); + runtime + .start_temporal_fit(&runtime.mutation_guard().unwrap(), now) + .unwrap(); + entered.recv_timeout(Duration::from_secs(10)).unwrap(); + let head = directory + .path() + .join(capture_fixture::context().owner_scope) + .join( + "SensingServer/room-learner-recovery/temporal-prefix-commitment-v1/temporal-head.bin", + ); + let original = std::fs::read(&head).unwrap(); + std::fs::write(&head, b"synthetic-tampered-head").unwrap(); + release.release(); + assert_eq!(wait_fit(&runtime).await, JobPhase::Failed); + assert!(runtime.temporal_diagnostic_fit().unwrap().is_none()); + assert!(runtime + .store + .load_managed_temporal_candidate(&capture_fixture::context()) + .unwrap() + .is_none()); + std::fs::write(&head, &original).unwrap(); + runtime + .cancel_temporal_durability(&runtime.mutation_guard().unwrap()) + .unwrap(); +} +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[cfg_attr(debug_assertions, ignore = "native-persistence journey requires Release; CI explicitly executes and counts this test")] +async fn c3_real_model_save_is_drained_before_synthetic_owner_erase_ack_and_never_recreated() { + let directory = tempfile::tempdir().unwrap(); + let (runtime, now) = complete_train(directory.path()); + let (entered, mut release) = hold(&runtime, "model_saved"); + runtime + .start_temporal_fit(&runtime.mutation_guard().unwrap(), now) + .unwrap(); + entered.recv_timeout(Duration::from_secs(90)).unwrap(); + // The hook is INSIDE the shared real store body: an actual whole model is + // already saved, but optional report persistence has not executed yet. + assert!(runtime + .store + .load_managed_temporal_candidate(&capture_fixture::context()) + .unwrap() + .is_some()); + // Exercise the actual Runtime tick while genuine artifact persistence is + // held, not just an unrelated synthetic mutex. + runtime + .tick_temporal_capture(&runtime.mutation_guard().unwrap(), now + 1_000_000) + .unwrap(); + let worker = runtime.clone(); + let guard = runtime.mutation_guard().unwrap(); + let (done_tx, done_rx) = mpsc::sync_channel(1); + let cancel = std::thread::spawn(move || { + done_tx + .send(worker.cancel_temporal_durability(&guard)) + .unwrap(); + }); + assert!( + matches!( + done_rx.recv_timeout(Duration::from_millis(50)), + Err(mpsc::RecvTimeoutError::Timeout) + ), + "owner erase ACK escaped a real in-progress store body" + ); + assert!(runtime.inner.try_lock().is_ok()); + release.release(); + done_rx + .recv_timeout(Duration::from_secs(15)) + .unwrap() + .unwrap(); + cancel.join().unwrap(); + assert_eq!(runtime.inner.lock().unwrap().job.phase, JobPhase::Cancelled); + assert!(runtime.temporal_diagnostic_fit().unwrap().is_none()); + // Disposable, explicitly synthetic profile only. Never ask a store reader + // to recreate the erased owner namespace during this post-ACK oracle. + let owner = directory + .path() + .join(capture_fixture::context().owner_scope); + std::fs::remove_dir_all(&owner).unwrap(); + tokio::time::sleep(Duration::from_millis(100)).await; + assert!( + !owner.exists(), + "old fit recreated an erased profile after drain ACK" + ); + drop(runtime); + assert!(!owner.exists()); +} +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[cfg_attr(debug_assertions, ignore = "native-persistence journey requires Release; CI explicitly executes and counts this test")] +async fn c3_last_external_owner_drop_drains_real_store_before_return_and_synthetic_erase() { + let directory = tempfile::tempdir().unwrap(); + let (runtime, now) = complete_train(directory.path()); + let (entered, mut release) = hold(&runtime, "model_saved"); + runtime + .start_temporal_fit(&runtime.mutation_guard().unwrap(), now) + .unwrap(); + entered.recv_timeout(Duration::from_secs(90)).unwrap(); + assert_eq!( + Arc::strong_count(&runtime), + 1, + "worker retained the owner Runtime during storage" + ); + let weak = Arc::downgrade(&runtime); + let (done_tx, done_rx) = mpsc::sync_channel(1); + let dropper = std::thread::spawn(move || { + drop(runtime); + done_tx.send(()).unwrap(); + }); + assert!( + matches!( + done_rx.recv_timeout(Duration::from_millis(50)), + Err(mpsc::RecvTimeoutError::Timeout) + ), + "last owner Drop failed to drain actual storage" + ); + assert!( + weak.upgrade().is_none(), + "fit worker kept Runtime::Drop from starting" + ); + release.release(); + done_rx.recv_timeout(Duration::from_secs(15)).unwrap(); + dropper.join().unwrap(); + let owner = directory + .path() + .join(capture_fixture::context().owner_scope); + std::fs::remove_dir_all(&owner).unwrap(); + tokio::time::sleep(Duration::from_millis(100)).await; + assert!(!owner.exists()); +} +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[cfg_attr(debug_assertions, ignore = "native-persistence journey requires Release; CI explicitly executes and counts this test")] +async fn c3_last_completion_owner_releases_after_signal_on_normal_and_unwind_paths() { + for unwind in [false, true] { + let directory = tempfile::tempdir().unwrap(); + let (runtime, now) = complete_train(directory.path()); + let (entered, mut release) = hold_outcome(&runtime, "completion_owner_upgraded", unwind); + runtime + .start_temporal_fit(&runtime.mutation_guard().unwrap(), now) + .unwrap(); + entered.recv_timeout(Duration::from_secs(90)).unwrap(); + let lifetime = { + let state = runtime.inner.lock().unwrap(); + let Some(CaptureWorkflow::Temporal(capture)) = state.capture.as_ref() else { + panic!() + }; + capture.fit_job.as_ref().unwrap().lifetime.clone() + }; + assert_eq!( + Arc::strong_count(&runtime), + 2, + "only post-I/O completion may transiently own Runtime" + ); + let weak = Arc::downgrade(&runtime); + drop(runtime); + release.release(); + let started = Instant::now(); + while !lifetime.done() { + assert!( + started.elapsed() < Duration::from_secs(10), + "last completion Arc self-drained before signaling" + ); + tokio::time::sleep(Duration::from_millis(1)).await; + } + while weak.strong_count() != 0 { + assert!(started.elapsed() < Duration::from_secs(10)); + tokio::time::sleep(Duration::from_millis(1)).await; + } + let owner = directory + .path() + .join(capture_fixture::context().owner_scope); + std::fs::remove_dir_all(&owner).unwrap(); + tokio::time::sleep(Duration::from_millis(50)).await; + assert!(!owner.exists()); + } +} +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[cfg_attr(debug_assertions, ignore = "native-persistence journey requires Release; CI explicitly executes and counts this test")] +async fn c3_original_expiry_during_fitting_withdraws_publication_without_renewal() { + let directory = tempfile::tempdir().unwrap(); + let (runtime, now) = complete_train(directory.path()); + let original = capture_fixture::snapshot(&runtime, now)["descriptor"].clone(); + let (entered, mut release) = hold(&runtime, "before_save"); + runtime + .start_temporal_fit(&runtime.mutation_guard().unwrap(), now) + .unwrap(); + entered.recv_timeout(Duration::from_secs(90)).unwrap(); + let expiry = original["expires_at_ms"].as_u64().unwrap() * 1_000_000; + assert!(runtime + .tick_temporal_capture(&runtime.mutation_guard().unwrap(), expiry) + .is_err()); + release.release(); + assert_eq!(wait_fit(&runtime).await, JobPhase::Cancelled); + assert!(runtime.temporal_diagnostic_fit().unwrap().is_none()); + assert!(runtime + .store + .load_managed_temporal_candidate(&capture_fixture::context()) + .unwrap() + .is_none()); + assert_eq!( + capture_fixture::snapshot(&runtime, expiry)["descriptor"], + original + ); + runtime + .cancel_temporal_durability(&runtime.mutation_guard().unwrap()) + .unwrap(); +} +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[cfg_attr(debug_assertions, ignore = "native-persistence journey requires Release; CI explicitly executes and counts this test")] +async fn c3_stale_generation_after_real_save_cannot_adopt_or_strand_job() { + for change in ["generation", "room", "revision", "owner"] { + let directory = tempfile::tempdir().unwrap(); + let (runtime, now) = complete_train(directory.path()); + let (entered, mut release) = hold(&runtime, "before_admission"); + runtime + .start_temporal_fit(&runtime.mutation_guard().unwrap(), now) + .unwrap(); + entered.recv_timeout(Duration::from_secs(90)).unwrap(); + { + let mut state = runtime.inner.lock().unwrap(); + if change == "generation" { + state.generation += 1; + } else { + let Some(CaptureWorkflow::Temporal(capture)) = state.capture.as_mut() else { + panic!() + }; + match change { + "room" => capture.context.room_digest = "changed-room".into(), + "revision" => capture.context.revision += 1, + "owner" => capture.context.owner_scope = "c".repeat(64), + _ => unreachable!(), + } + } + } + release.release(); + assert_eq!(wait_fit(&runtime).await, JobPhase::Failed, "{change}"); + if change == "generation" { + assert!(runtime.temporal_diagnostic_fit().is_err()); + } else { + assert!(runtime.temporal_diagnostic_fit().unwrap().is_none()); + } + assert!(runtime.inner.lock().unwrap().model.is_none()); + assert!( + runtime + .store + .load_managed_temporal_candidate(&capture_fixture::context()) + .unwrap() + .is_some(), + "completed immutable history should not be deleted to fake cancellation" + ); + runtime + .cancel_temporal_durability(&runtime.mutation_guard().unwrap()) + .unwrap(); + } +} +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[cfg_attr(debug_assertions, ignore = "native-persistence journey requires Release; CI explicitly executes and counts this test")] +async fn c3_old_completion_cannot_fail_a_replacement_job_with_different_cancel_identity() { + let directory = tempfile::tempdir().unwrap(); + let (runtime, now) = complete_train(directory.path()); + let (entered, mut release) = hold(&runtime, "before_admission"); + runtime + .start_temporal_fit(&runtime.mutation_guard().unwrap(), now) + .unwrap(); + entered.recv_timeout(Duration::from_secs(90)).unwrap(); + let replacement = Arc::new(AtomicBool::new(false)); + let old_lifetime = { + let mut state = runtime.inner.lock().unwrap(); + let Some(CaptureWorkflow::Temporal(capture)) = state.capture.as_ref() else { + panic!() + }; + let lifetime = capture.fit_job.as_ref().unwrap().lifetime.clone(); + state.job = Job { + phase: JobPhase::Running, + cancel: replacement.clone(), + report: None, + error: None, + }; + lifetime + }; + release.release(); + let started = Instant::now(); + while !old_lifetime.done() { + assert!(started.elapsed() < Duration::from_secs(10)); + tokio::time::sleep(Duration::from_millis(1)).await; + } + { + let mut state = runtime.inner.lock().unwrap(); + assert_eq!(state.job.phase, JobPhase::Running); + assert!(Arc::ptr_eq(&state.job.cancel, &replacement)); + assert!(state.job.error.is_none()); + state.job.phase = JobPhase::Idle; + } + assert!(runtime.temporal_diagnostic_fit().unwrap().is_none()); + runtime + .cancel_temporal_durability(&runtime.mutation_guard().unwrap()) + .unwrap(); +} +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[cfg_attr(debug_assertions, ignore = "native-persistence journey requires Release; CI explicitly executes and counts this test")] +async fn c3_optional_actual_summary_failure_keeps_durable_diagnostic_model() { + let directory = tempfile::tempdir().unwrap(); + let (runtime, now) = complete_train(directory.path()); + let weak = Arc::downgrade(&runtime); + let root = directory.path().to_path_buf(); + *runtime.temporal_fit_hook.lock().unwrap() = Some(Arc::new(move |stage| { + if stage == "model_saved" { + let runtime = weak.upgrade().unwrap(); + let model = runtime + .store + .load_managed_temporal_candidate(&capture_fixture::context()) + .unwrap() + .unwrap(); + let binding = model.fit_diagnostic_binding().unwrap(); + let path = root + .join(capture_fixture::context().owner_scope) + .join("SensingServer/room-learner/temporal-models") + .join(binding.context_digest) + .join(format!("{}.fit-summary-v1.json", binding.artifact_sha256)); + std::fs::create_dir(path).unwrap(); + } + })); + runtime + .start_temporal_fit(&runtime.mutation_guard().unwrap(), now) + .unwrap(); + assert_eq!(wait_fit(&runtime).await, JobPhase::Ready); + let (model, receipt, diagnostics) = runtime.temporal_diagnostic_fit().unwrap().unwrap(); + assert_eq!( + serde_json::to_value(&diagnostics).unwrap()["state"], + "not_recorded" + ); + assert!(diagnostics.summary.is_none()); + assert_eq!( + runtime + .store + .load_exact_temporal_candidate(&capture_fixture::context(), &receipt.artifact_sha256) + .unwrap() + .export() + .unwrap(), + model.export().unwrap() + ); + runtime + .cancel_temporal_durability(&runtime.mutation_guard().unwrap()) + .unwrap(); +} +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[cfg_attr(debug_assertions, ignore = "native-persistence journey requires Release; CI explicitly executes and counts this test")] +async fn c3_worker_unwind_keeps_saved_history_but_never_adopts_current_candidate() { + let directory = tempfile::tempdir().unwrap(); + let (runtime, now) = complete_train(directory.path()); + *runtime.temporal_fit_hook.lock().unwrap() = Some(Arc::new(|stage| { + if stage == "before_admission" { + panic!("synthetic C3 worker failure") + } + })); + runtime + .start_temporal_fit(&runtime.mutation_guard().unwrap(), now) + .unwrap(); + assert_eq!(wait_fit(&runtime).await, JobPhase::Failed); + assert!(runtime.temporal_diagnostic_fit().unwrap().is_none()); + assert!(runtime + .store + .load_managed_temporal_candidate(&capture_fixture::context()) + .unwrap() + .is_some()); + runtime + .cancel_temporal_durability(&runtime.mutation_guard().unwrap()) + .unwrap(); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_frozen.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_frozen.rs new file mode 100644 index 0000000000..3148ba0f0f --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_frozen.rs @@ -0,0 +1,71 @@ +//! Private owner transition from actual C3 acceptance to a durable Frozen ACK. +//! No public fit/HeldOut/qualification/selection/Room Save activation. +use super::*; +impl RoomLearningRuntime { + pub(super) fn prepare_frozen_transition( + &self, + expected: &MutationGuard, + now: u64, + ) -> Result<(), String> { + let store = self.store.clone(); + let mut state = self.command_state(expected)?; + if state.job.phase != JobPhase::Ready { + return Err("temporal_actual_current_fit_required".into()); + } + let generation = state.generation; + let revision = state.workflow_revision; + let Some(CaptureWorkflow::Temporal(capture)) = state.capture.as_mut() else { + return Err(PRIVATE_KIND.into()); + }; + if capture.owner_generation != generation { + capture.invalidate_owner(); + return Err("temporal_owner_generation_changed".into()); + } + capture.check_time(now)?; + let core = capture + .core + .as_ref() + .ok_or("temporal_initial_barrier_unconfirmed")?; + if !core.is_full_run() { + return Err("temporal_full_run_no_upconversion".into()); + } + if core.stage() != TemporalCaptureStage::Complete { + return Err("temporal_full_run_train_complete_required".into()); + } + let accepted = capture.take_current_accepted_fit(&self.runtime_epoch, generation, now)?; + let run = capture + .core + .as_ref() + .unwrap() + .fit_capture_identity(now)? + .run_id() + .to_owned(); + let witness = FrozenOwnerWitness { + runtime_epoch: self.runtime_epoch.clone(), + generation, + revision, + context: capture.context.clone(), + run, + }; + capture + .core + .as_mut() + .unwrap() + .request_frozen(accepted, witness, store, now) + } + pub(super) fn current_verified_frozen_model( + &self, + ) -> Result>, String> { + let state = self.inner.lock().map_err(|_| "learner state unavailable")?; + let Some(CaptureWorkflow::Temporal(capture)) = state.capture.as_ref() else { + return Err(PRIVATE_KIND.into()); + }; + if capture.owner_generation != state.generation || capture.failure.is_some() { + return Err("temporal_frozen_owner_unavailable".into()); + } + Ok(capture + .core + .as_ref() + .and_then(|c| c.verified_frozen_model())) + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_full_run_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_full_run_tests.rs new file mode 100644 index 0000000000..7956565ee4 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_full_run_tests.rs @@ -0,0 +1,1308 @@ +//! SYNTHETIC private fresh-v3 genesis/Train-path checks; no physical evidence. +use super::tests as fixture; +use super::*; +use crate::room_learner::{ + native::{canonicalize_esp32, NativePacketInput}, + observation::CaptureProvenance, +}; +use std::path::Path; +use std::sync::Arc; +use std::time::{Duration, Instant}; +use wifi_densepose_hardware::{Adr018Flags, PpduType}; +#[test] +fn c4a_fresh_genesis_is_preparing_until_actual_ack_in_same_store_and_raw_read_cannot_write() { + let directory = tempfile::tempdir().unwrap(); + let rt = fixture::runtime(directory.path(), fixture::START); + rt.configure_temporal_full_run( + &rt.mutation_guard().unwrap(), + fixture::context(), + "synthetic-c4a-fresh".into(), + fixture::START, + ) + .unwrap(); + assert_eq!(fixture::snapshot(&rt, fixture::START)["stage"], "preparing"); + std::thread::sleep(Duration::from_millis(30)); + assert_eq!( + fixture::snapshot(&rt, fixture::START)["stage"], + "preparing", + "queued storage result is not receiver ACK admission" + ); + let now = fixture::settle_async_boundary(&rt, fixture::START); + assert_eq!(fixture::snapshot(&rt, now)["stage"], "ready"); + let store = rt.temporal_store().unwrap(); + let loaded = store.load_temporal(now).unwrap().unwrap(); + let prefix = loaded.prefix.unwrap(); + assert_eq!(prefix.metadata_kind(), "temporal-prefix-commitment-v2"); + assert_eq!(prefix.transition_limit(), 433); + assert_eq!(prefix.window_count(), 0); + let data: serde_json::Value = serde_json::from_slice(prefix.bytes()).unwrap(); + assert_eq!( + data["schema"], + "ruview.room-learner.temporal-runtime-prefix.v3" + ); + assert_eq!(data["payload"]["policy"]["body"]["max_total_windows"], 128); + assert_eq!(data["payload"]["stage"]["kind"], "train"); + assert!(store + .prepare_temporal(Some(&loaded.token), &prefix, now) + .err() + .unwrap() + .contains("producer_write_grant_required")); + assert_eq!( + store.load_temporal(now).unwrap().unwrap().token, + loaded.token + ); + let base = directory + .path() + .join(fixture::context().owner_scope) + .join("SensingServer/room-learner-recovery"); + assert!(base + .join("temporal-prefix-commitment-v1/temporal-head.bin") + .is_file()); + assert!(!base.join("temporal-prefix-commitment-v2").exists()); + if let Some(destination) = std::env::var_os("RUVIEW_C4A_EXPORT_PROFILE") { + use std::os::unix::fs::PermissionsExt; + let destination = std::path::PathBuf::from(destination); + assert!(destination.starts_with("/tmp/ruview-c4a-kind2-current-fixture-20261106")); + assert!(!destination.exists()); + fn copy(from: &std::path::Path, to: &std::path::Path) { + use std::os::unix::fs::PermissionsExt; + assert!(!std::fs::symlink_metadata(from) + .unwrap() + .file_type() + .is_symlink()); + if from.is_dir() { + std::fs::create_dir_all(to).unwrap(); + std::fs::set_permissions(to, std::fs::Permissions::from_mode(0o700)).unwrap(); + for e in std::fs::read_dir(from).unwrap() { + let e = e.unwrap(); + copy(&e.path(), &to.join(e.file_name())); + } + } else { + std::fs::copy(from, to).unwrap(); + std::fs::set_permissions(to, std::fs::Permissions::from_mode(0o600)).unwrap(); + } + } + copy(directory.path(), &destination); + let marker = destination.join("synthetic-c4a-fixture.json"); + std::fs::write(&marker,serde_json::to_vec(&serde_json::json!({"schema":"ruview.test-only.synthetic-c4a-kind2-fixture.v1","receiver_now_ns":now,"real_user_data":false})).unwrap()).unwrap(); + std::fs::set_permissions(marker, std::fs::Permissions::from_mode(0o600)).unwrap(); + } + rt.cancel_temporal_durability(&rt.mutation_guard().unwrap()) + .unwrap(); +} +#[test] +fn c4a_new_full_run_reuses_actual_native_train_writer_without_changing_model_input_identity() { + struct ReleaseOnDrop(std::sync::mpsc::Sender<()>); + impl Drop for ReleaseOnDrop { + fn drop(&mut self) { let _ = self.0.send(()); } + } + // Isolated synthetic stores: the old quiet tail must fail the unchanged + // source-custody guard; continued input must survive the same delayed ACK. + for quiet_tail in [true, false] { + let directory = tempfile::tempdir().unwrap(); + let rt = fixture::runtime(directory.path(), fixture::START); + rt.configure_temporal_full_run( + &rt.mutation_guard().unwrap(), + fixture::context(), + "synthetic-c4a-window".into(), + fixture::START, + ) + .unwrap(); + let mut now = fixture::settle_async_boundary(&rt, fixture::START); + let original = fixture::snapshot(&rt, now)["descriptor"].clone(); + now = fixture::async_lead_end(&rt, now); + let mut sequence = 0; + // Admit the actual empty segment before holding the first WINDOW write. + rt.tick_temporal_capture(&rt.mutation_guard().unwrap(), now).unwrap(); + now = fixture::settle_observing_with_sources(&rt, now, &mut sequence, Duration::from_secs(30)); + let writer = { + let state = rt.inner.lock().unwrap(); + let Some(CaptureWorkflow::Temporal(capture)) = &state.capture else { panic!() }; + capture.core.as_ref().unwrap().writer().unwrap() + }; + let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(1); + let (release_tx, release_rx) = std::sync::mpsc::channel(); + writer.hold_next(entered_tx, release_rx); + let release_on_unwind = ReleaseOnDrop(release_tx.clone()); + let end = now + 3_000_000_000; + while now <= end { + sequence += 1; + for source in 1..=3 { + rt.ingest_native(&fixture::packet(source, sequence, now), now) + .unwrap(); + } + now += fixture::PERIOD; + if fixture::snapshot(&rt, now)["durability_pending"] == true { + std::thread::sleep(Duration::from_nanos(fixture::PERIOD)); + } else { + std::thread::sleep(Duration::from_micros(500)); + } + } + entered_rx.recv_timeout(Duration::from_secs(2)).unwrap(); + assert_eq!(fixture::snapshot(&rt, now)["durability_pending"], true); + assert_eq!(fixture::snapshot(&rt, now)["retained_rows"], 1); + assert_eq!(fixture::snapshot(&rt, now)["durable_windows"], 0); + // Keep the actual WINDOW writer blocked for 150ms AFTER the old tail + // stopped feeding. A hold measured from row submission misses this seam. + if quiet_tail { + let wait = Instant::now(); + std::thread::sleep(Duration::from_millis(80)); + let error = rt.tick_temporal_capture( + &rt.mutation_guard().unwrap(), now + wait.elapsed().as_nanos() as u64, + ).unwrap_err(); + assert_eq!(error, "temporal_missing_segment_source"); + assert_eq!(fixture::snapshot(&rt, now)["durable_windows"], 0); + // Release only AFTER the negative observation; scheduler delay + // cannot let a valid ACK arrive before its zero-durable assertion. + drop(release_on_unwind); + rt.cancel_temporal_durability(&rt.mutation_guard().unwrap()).unwrap(); + continue; + } + let release = std::thread::spawn(move || { + std::thread::sleep(Duration::from_millis(150)); + let _ = release_tx.send(()); + }); + // Observing requires fresh canonical source packets even while storage + // is pending. The supervision bound is test-only, not a receiver budget. + now = fixture::settle_observing_with_sources(&rt, now, &mut sequence, Duration::from_secs(30)); + release.join().unwrap(); + drop(release_on_unwind); + assert_eq!(fixture::snapshot(&rt, now)["stage"], "observing"); + rt.pause_temporal_capture(&rt.mutation_guard().unwrap(), now).unwrap(); + now = fixture::settle_async_boundary(&rt, now); + assert!( + fixture::snapshot(&rt, now)["durable_windows"] + .as_u64() + .unwrap() + >= 1 + ); + let loaded = rt + .temporal_store() + .unwrap() + .load_temporal(now) + .unwrap() + .unwrap(); + let prefix = loaded.prefix.unwrap(); + assert_eq!(prefix.metadata_kind(), "temporal-prefix-commitment-v2"); + assert_eq!(prefix.transition_limit(), 433); + assert_eq!( + fixture::snapshot(&rt, now)["descriptor"]["id"], + original["id"] + ); + assert_eq!( + fixture::snapshot(&rt, now)["descriptor"]["expires_at_ms"], + original["expires_at_ms"] + ); + let data: serde_json::Value = serde_json::from_slice(prefix.bytes()).unwrap(); + let policy = &data["payload"]["policy"]["body"]; + assert_ne!( + policy["input_policy_sha256"], data["payload"]["policy"]["sha256"], + "continuation identity must not replace model input identity" + ); + rt.cancel_temporal_durability(&rt.mutation_guard().unwrap()) + .unwrap(); + } +} +#[test] +fn c4a_full_run_rejects_unsupported_source_profile_before_genesis_write() { + let directory = tempfile::tempdir().unwrap(); + let rt = fixture::runtime(directory.path(), fixture::START); + let mut context = fixture::context(); + context.sources.pop(); + assert!(rt + .configure_temporal_full_run( + &rt.mutation_guard().unwrap(), + context, + "synthetic-c4a-unsupported".into(), + fixture::START + ) + .is_err()); + let path = directory.path().join(fixture::context().owner_scope).join( + "SensingServer/room-learner-recovery/temporal-prefix-commitment-v1/temporal-head.bin", + ); + assert!(!path.exists()); +} + +fn configured(root: &Path) -> (Arc, u64) { + let runtime = Arc::new(fixture::runtime(root, fixture::START)); + let store = runtime.recovery_store.clone().unwrap(); + runtime + .configure_temporal_capture_with_custody( + &runtime.mutation_guard().unwrap(), + fixture::context(), + "synthetic-c4a-native-fit".into(), + fixture::START, + move |schedule, run, mut config, now, _, witness| { + // Bound when this capture is CREATED, never overridden by fit input. + config.epochs = 2; + TemporalTrainCapture::prepare_full_run_async( + schedule, run, config, now, store, witness, + ) + }, + ) + .unwrap(); + let now = fixture::settle_async_boundary(&runtime, fixture::START); + (runtime, now) +} +fn packet(source: u8, sequence: u32, at: u64, label: RoomLabel) -> CanonicalPacket { + let gain = match label { + RoomLabel::Empty => 3, + RoomLabel::AdjacentOnly => 4, + RoomLabel::OccupiedStatic => 8, + RoomLabel::OccupiedMoving => 10 + (sequence % 2) as u8, + _ => unreachable!(), + }; + let iq = (0..64) + .flat_map(|bin| [gain + source + (bin % 2) as u8, 4]) + .collect::>(); + canonicalize_esp32(NativePacketInput { + node_id: source, + n_antennas: 1, + raw_bins: 64, + frequency_hz: 2_437_000_000, + ppdu_type: PpduType::HtLegacy, + sequence, + received_at_ns: at, + receiver_sample_rate_hz: 33., + iq_bytes: &iq, + flags: Adr018Flags::default(), + provenance: CaptureProvenance::LiveReceiverObserved, + }) + .unwrap() +} +// This functional fixture supplies a synthetic receiver timeline. Preparation +// I/O must finish before that timeline starts producing packets; otherwise host +// serialization latency repeatedly consumes the first Check window's buffer. +// Real-paced and storage-backpressure acceptance helpers remain separate. +fn wait_for_segment_commit(rt: &RoomLearningRuntime, at: u64, phase_started: Instant) { + loop { + rt.tick_temporal_capture(&rt.mutation_guard().unwrap(), at).unwrap(); + let progress = fixture::snapshot(rt, at); + if progress["durability_pending"] == false { + assert_eq!(progress["stage"], "observing"); + assert_eq!(progress["retained_rows"], progress["durable_windows"]); + assert_eq!(progress["observed_ns"], progress["durable_observed_ns"], + "preparation cannot credit a synthetic observation"); + return; + } + assert!(phase_started.elapsed() < Duration::from_secs(240), + "segment preparation did not settle within the original phase guard: {progress}"); + std::thread::sleep(Duration::from_millis(1)); + } +} + +fn record_train_uncached(root: &Path) -> (Arc, u64) { + assert!(!cfg!(debug_assertions), "complete native-persistence fixture requires Release; classify the caller"); + let (runtime, mut now) = configured(root); + let schedule = ManagedScheduleContract::build( + fixture::context(), + LeadInPolicy::Train5HeldOut15EngineeringV1, + ) + .unwrap(); + let mut sequence = 0; + let mut starts = 0; + let mut forced_pause = false; + let mut forced_resume = false; + for episode in schedule + .episodes() + .iter() + .filter(|e| e.role == DataRole::Train) + { + loop { + let progress = fixture::snapshot(&runtime, now); + match progress["stage"].as_str().unwrap() { + "ready" | "paused" => { + let resume_revision = if progress["stage"] == "paused" + && forced_pause && !forced_resume + { + Some((runtime.temporal_store().unwrap().load_temporal(now) + .unwrap().unwrap().token.data().revision, + progress["durable_windows"].as_u64().unwrap())) + } else { None }; + now = fixture::async_lead_end(&runtime, now); + starts += 1; + runtime + .tick_temporal_capture(&runtime.mutation_guard().unwrap(), now) + .unwrap(); + if let Some((before, durable_rows)) = resume_revision { + // Lead readiness is only recertification. Admit the + // actual SEGMENT ACK while every source remains live. + let wait = Instant::now(); + loop { + sequence += 1; + for source in 1..=3 { + runtime.ingest_native( + &packet(source, sequence, now, episode.condition), now, + ).unwrap(); + } + let pending = fixture::snapshot(&runtime, now)["durability_pending"] == true; + now += fixture::PERIOD; + if !pending { break; } + assert!(wait.elapsed() < Duration::from_secs(30)); + std::thread::sleep(Duration::from_nanos(fixture::PERIOD)); + } + let after = runtime.temporal_store().unwrap().load_temporal(now) + .unwrap().unwrap().token.data().revision; + assert_eq!(after, before.checked_add(1).unwrap(), + "the explicit resumed segment is exactly one durable transition"); + let resumed = fixture::snapshot(&runtime, now); + assert_eq!(resumed["stage"], "observing"); + assert_eq!(resumed["durable_windows"], durable_rows); + assert_eq!(resumed["retained_rows"], durable_rows, + "the SEGMENT ACK must not be confused with a new WINDOW ACK"); + forced_resume = true; + } + } + "needs_confirmation" => break, + "failed" => panic!("synthetic C4A phase{} failed: {progress}", episode.index), + _ => {} + } + let pending = fixture::snapshot(&runtime, now)["durability_pending"] == true; + if !forced_pause && episode.index == 0 && !pending + && progress["stage"] == "observing" + && progress["durable_windows"].as_u64().is_some_and(|n| n >= 1) + && progress["retained_rows"] == progress["durable_windows"] + { + let before = runtime.temporal_store().unwrap().load_temporal(now) + .unwrap().unwrap().token.data().revision; + runtime.pause_temporal_capture(&runtime.mutation_guard().unwrap(), now).unwrap(); + now = fixture::settle_async_boundary(&runtime, now); + assert_eq!(fixture::snapshot(&runtime, now)["stage"], "paused"); + let after = runtime.temporal_store().unwrap().load_temporal(now) + .unwrap().unwrap().token.data().revision; + assert_eq!(after, before.checked_add(1).unwrap(), + "the explicit legal pause is exactly one durable transition"); + forced_pause = true; + continue; + } + sequence += 1; + for source in 1..=3 { + match runtime.ingest_native(&packet(source, sequence, now, episode.condition), now) + { + Ok(()) => {} + Err(error) if error == "temporal_storage_busy_pause" => break, + Err(error) => panic!("synthetic C4A ingest: {error}"), + } + } + now += fixture::PERIOD; + if pending { + std::thread::sleep(Duration::from_nanos(fixture::PERIOD)); + } + assert!(starts < 120, "synthetic C4A interruption bound exhausted"); + } + let guard = runtime.mutation_guard().unwrap(); + let challenge = runtime.temporal_confirmation_challenge(&guard).unwrap(); + runtime + .confirm_temporal_phase(&guard, &challenge, now) + .unwrap(); + assert_eq!(fixture::snapshot(&runtime, now)["durability_pending"], true); + now = fixture::settle_async_boundary(&runtime, now); + } + let complete = fixture::snapshot(&runtime, now); + assert!(forced_pause && forced_resume, + "the native producer must exercise an actual durable pause and resume"); + assert_eq!(complete["stage"], "complete"); + assert_eq!( + complete["sealed_train_episodes"], + schedule.counts().train_episodes + ); + assert_eq!(complete["durable_windows"], schedule.counts().train_windows); + println!( + "SYNTHETIC_C4A_REAL_TRAIN_CAPTURE: rows={} seals={} starts={starts}", + schedule.counts().train_windows, + schedule.counts().train_episodes + ); + (runtime, now) +} + +fn copy_private(from: &Path, to: &Path) { + use std::os::unix::fs::PermissionsExt; + assert!(!std::fs::symlink_metadata(from) + .unwrap() + .file_type() + .is_symlink()); + if from.is_dir() { + std::fs::create_dir_all(to).unwrap(); + std::fs::set_permissions(to, std::fs::Permissions::from_mode(0o700)).unwrap(); + for e in std::fs::read_dir(from).unwrap() { + let e = e.unwrap(); + copy_private(&e.path(), &to.join(e.file_name())); + } + } else { + std::fs::copy(from, to).unwrap(); + std::fs::set_permissions(to, std::fs::Permissions::from_mode(0o600)).unwrap(); + } +} +// Cache only the actual seven-phase native producer output. Each independent +// fault case gets its own files and strictly current recertification/owner. +fn record_train(root: &Path) -> (Arc, u64) { + static TRAIN: std::sync::OnceLock<(tempfile::TempDir, u64)> = std::sync::OnceLock::new(); + let (saved, at) = TRAIN.get_or_init(|| { + let d = tempfile::tempdir().unwrap(); + let (rt, at) = record_train_uncached(d.path()); + drop(rt); + (d, at) + }); + copy_private(saved.path(), root); + let now = at + 1_000_000_000; + let rt = Arc::new(fixture::runtime(root, now)); + rt.restore_temporal_full_run(&rt.mutation_guard().unwrap(), &fixture::context(), now) + .unwrap(); + assert_eq!(fixture::snapshot(&rt, now)["stage"], "complete"); + (rt, now) +} +fn assert_original_lifetime(value: &serde_json::Value, original: &serde_json::Value) { + for field in [ + "schema", + "id", + "created_at_ms", + "expires_at_ms", + "owner_scope", + "installation_id", + "context_digest", + ] { + assert_eq!(value[field], original[field], "{field}"); + } +} +struct ReleaseGate(Option>); +impl ReleaseGate { + fn release(&mut self) { + if let Some(tx) = self.0.take() { + let _ = tx.send(()); + } + } +} +impl Drop for ReleaseGate { + fn drop(&mut self) { + self.release(); + } +} +fn floors(rt: &RoomLearningRuntime) -> (Vec<(u8, u64, u64)>, Option) { + let s = rt.inner.lock().unwrap(); + let Some(CaptureWorkflow::Temporal(c)) = &s.capture else { + panic!() + }; + c.core.as_ref().unwrap().test_full_run_floors() +} +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[cfg_attr(debug_assertions, ignore = "native-persistence journey requires Release; CI explicitly executes and counts this test")] +async fn c4a_actual_seven_train_fit_frozen_ack_and_exact_cold_model() { + let directory = tempfile::tempdir().unwrap(); + let (runtime, now) = record_train(directory.path()); + let original = fixture::snapshot(&runtime, now)["descriptor"].clone(); + let train_rows_sha = fixture::snapshot(&runtime, now)["rows_sha256"].clone(); + let loaded = runtime + .temporal_store() + .unwrap() + .load_temporal(now) + .unwrap() + .unwrap(); + let train_revision = loaded.token.data().revision; + assert!(train_revision >= 80, + "all initial segments, 64 windows and 7 seals plus the forced pause/resume must be durable; extra legal pauses add transitions"); + let (compute_tx, compute_rx) = std::sync::mpsc::sync_channel(1); + let (compute_release_tx, compute_release_rx) = std::sync::mpsc::sync_channel(1); + let mut compute_gate = ReleaseGate(Some(compute_release_tx)); + let compute_release_rx = std::sync::Mutex::new(compute_release_rx); + let (ready_tx, ready_rx) = std::sync::mpsc::sync_channel(1); + let (done_tx, done_rx) = std::sync::mpsc::sync_channel(1); + let mut done_gate = ReleaseGate(Some(done_tx)); + let done_rx = std::sync::Mutex::new(done_rx); + *runtime.temporal_fit_hook.lock().unwrap() = Some(Arc::new(move |stage| { + if stage == "before_validate" { + compute_tx.send(()).unwrap(); + compute_release_rx + .lock() + .unwrap() + .recv_timeout(Duration::from_secs(30)) + .unwrap(); + } + if stage == "accepted_ready_published" { + ready_tx.send(()).unwrap(); + done_rx + .lock() + .unwrap() + .recv_timeout(Duration::from_secs(30)) + .unwrap(); + } + })); + let fit_started = Instant::now(); + runtime + .start_temporal_fit(&runtime.mutation_guard().unwrap(), now) + .unwrap(); + compute_rx.recv_timeout(Duration::from_secs(10)).unwrap(); + assert_eq!(runtime.inner.lock().unwrap().job.phase, JobPhase::Running); + let during_fit = now + fit_started.elapsed().as_nanos() as u64; + for source in 1..=3 { + runtime + .ingest_native( + &packet(source, 500_000, during_fit, RoomLabel::Empty), + during_fit, + ) + .unwrap(); + } + assert_eq!( + fixture::snapshot(&runtime, during_fit)["rows_sha256"], + train_rows_sha + ); + compute_gate.release(); + loop { + let phase = runtime.inner.lock().unwrap().job.phase; + if phase == JobPhase::Ready { + break; + } + assert_eq!( + phase, + JobPhase::Running, + "{:?}", + runtime.inner.lock().unwrap().job.error + ); + assert!(fit_started.elapsed() < Duration::from_secs(120)); + tokio::time::sleep(Duration::from_millis(5)).await; + } + ready_rx.recv_timeout(Duration::from_secs(10)).unwrap(); + let (model, receipt, _) = runtime.temporal_diagnostic_fit().unwrap().unwrap(); + assert_eq!( + runtime + .cancel_temporal_fit(&runtime.mutation_guard().unwrap()) + .unwrap(), + JobPhase::Ready, + "late compute cancellation cannot revoke actual Accepted Ready" + ); + assert_eq!( + runtime + .inner + .lock() + .unwrap() + .job + .report + .as_ref() + .unwrap() + .completed_epochs, + 2 + ); + assert!(runtime.current_verified_frozen_model().unwrap().is_none()); + let before_notify = now + fit_started.elapsed().as_nanos() as u64; + assert_eq!( + runtime + .prepare_frozen_transition(&runtime.mutation_guard().unwrap(), before_notify) + .unwrap_err(), + "temporal_actual_current_fit_required" + ); + done_gate.release(); + // This real native ingress updates the live receiver floor without adding + // Train rows or modifying the exact accepted Train commitment. + let prime_at = now + fit_started.elapsed().as_nanos() as u64; + for source in 1..=3 { + runtime + .ingest_native( + &packet(source, 1_000_000, prime_at, RoomLabel::Empty), + prime_at, + ) + .unwrap(); + } + let prepare_floors = floors(&runtime).0; + let orphan = tempfile::tempdir().unwrap(); + copy_private(directory.path(), orphan.path()); + let writer = { + let state = runtime.inner.lock().unwrap(); + let Some(CaptureWorkflow::Temporal(c)) = &state.capture else { + panic!() + }; + c.core.as_ref().unwrap().writer().unwrap() + }; + let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(1); + let (release_tx, release_rx) = std::sync::mpsc::sync_channel(1); + let mut writer_gate = ReleaseGate(Some(release_tx)); + writer.hold_next_after_prepare(entered_tx, release_rx); + // Completion publication and drain signal are separate. Only the latter + // permits transfer of the actual non-serde accepted-fit capability. + let mut freeze_now; + loop { + freeze_now = now + u64::try_from(fit_started.elapsed().as_nanos()).unwrap(); + match runtime.prepare_frozen_transition(&runtime.mutation_guard().unwrap(), freeze_now) { + Ok(()) => break, + Err(e) if e == "temporal_actual_current_fit_required" => { + tokio::time::sleep(Duration::from_millis(1)).await + } + Err(e) => panic!("actual Frozen preparation failed: {e}"), + } + } + entered_rx.recv_timeout(Duration::from_secs(10)).unwrap(); + assert!( + runtime + .start_training(&runtime.mutation_guard().unwrap(), FitConfig::default()) + .unwrap_err() + .contains("unsupported_command"), + "a rejected command must not replace the accepted Frozen operation" + ); + assert_eq!( + fixture::snapshot(&runtime, freeze_now)["stage"], + "preparing" + ); + assert!(runtime.current_verified_frozen_model().unwrap().is_none()); + let advanced = freeze_now + fixture::PERIOD; + for source in 1..=3 { + runtime + .ingest_native( + &packet(source, 2_000_000, advanced, RoomLabel::Empty), + advanced, + ) + .unwrap(); + } + let live_floors = floors(&runtime).0; + assert!(live_floors + .iter() + .all(|(_, seq, t)| *seq == 2_000_000 && *t == advanced)); + writer_gate.release(); + let frozen_now = fixture::settle_async_boundary(&runtime, advanced); + assert_eq!(fixture::snapshot(&runtime, frozen_now)["stage"], "frozen"); + assert_eq!( + runtime + .current_verified_frozen_model() + .unwrap() + .unwrap() + .export() + .unwrap(), + model.export().unwrap() + ); + let loaded = runtime + .temporal_store() + .unwrap() + .load_temporal(frozen_now) + .unwrap() + .unwrap(); + assert_eq!( + loaded.token.data().revision, + train_revision.checked_add(1).unwrap(), + "one actual durable Frozen transition" + ); + let data: serde_json::Value = serde_json::from_slice(loaded.prefix.unwrap().bytes()).unwrap(); + assert_eq!(data["payload"]["stage"]["kind"], "frozen"); + assert_eq!( + data["payload"]["stage"]["fit"]["body"]["artifact_sha256"], + receipt.artifact_sha256 + ); + assert_original_lifetime( + &fixture::snapshot(&runtime, frozen_now)["descriptor"], + &original, + ); + assert_eq!( + data["payload"]["stage"]["fit"]["body"]["freeze_prepare_receiver_ns"], + freeze_now + ); + assert_eq!( + fixture::snapshot(&runtime, frozen_now)["descriptor"]["saved_at_ms"], + freeze_now / 1_000_000 + ); + assert_eq!( + data["payload"]["stage"]["fit"]["body"]["source_floors"], + serde_json::to_value(&prepare_floors).unwrap() + ); + assert_eq!(floors(&runtime), (live_floors, Some(frozen_now))); + let loaded = runtime + .temporal_store() + .unwrap() + .load_temporal(frozen_now) + .unwrap() + .unwrap(); + assert!(runtime + .temporal_store() + .unwrap() + .prepare_temporal( + Some(&loaded.token), + loaded.prefix.as_ref().unwrap(), + frozen_now + ) + .err() + .unwrap() + .contains("producer_write_grant_required")); + + assert!(runtime.inner.lock().unwrap().model.is_none()); + assert!(runtime.inner.lock().unwrap().qualification.is_none()); + let post_ack = frozen_now + fixture::PERIOD; + for source in 1..=3 { + runtime + .ingest_native( + &packet(source, 3_000_000, post_ack, RoomLabel::Empty), + post_ack, + ) + .unwrap(); + } + assert!(floors(&runtime) + .0 + .iter() + .all(|(_, seq, t)| *seq == 3_000_000 && *t == post_ack)); + assert_eq!(floors(&runtime).1, Some(frozen_now)); + assert_eq!( + fixture::snapshot(&runtime, post_ack)["rows_sha256"], + train_rows_sha + ); + assert_eq!(fixture::snapshot(&runtime, post_ack)["retained_rows"], 64); + assert_eq!( + fixture::snapshot(&runtime, post_ack)["sealed_train_episodes"], + 7 + ); + assert_eq!( + runtime + .current_verified_frozen_model() + .unwrap() + .unwrap() + .export() + .unwrap(), + model.export().unwrap() + ); + assert_eq!( + runtime + .temporal_store() + .unwrap() + .load_temporal(post_ack) + .unwrap() + .unwrap() + .token + .data() + .revision, + train_revision.checked_add(1).unwrap() + ); + drop(runtime); + let cold_now = post_ack + 1_000_000_000; + let cold = Arc::new(fixture::runtime(directory.path(), cold_now)); + cold.restore_temporal_full_run( + &cold.mutation_guard().unwrap(), + &fixture::context(), + cold_now, + ) + .unwrap(); + assert_eq!(fixture::snapshot(&cold, cold_now)["stage"], "frozen"); + assert_eq!( + floors(&cold), + (prepare_floors, Some(cold_now)), + "cold recertification retains only fsynced floors and a fresh local floor" + ); + assert_eq!( + cold.current_verified_frozen_model() + .unwrap() + .unwrap() + .export() + .unwrap(), + model.export().unwrap() + ); + assert_original_lifetime(&fixture::snapshot(&cold, cold_now)["descriptor"], &original); + // C4B deliberately adds only the protected PRIVATE next-role producer. + // Public Legacy training stays unsupported, and no sample/observed time is + // credited by a request, recertification or preparation countdown. + assert!(cold + .start_training(&cold.mutation_guard().unwrap(), FitConfig::default()) + .unwrap_err() + .contains("unsupported_command")); + let before_start = fixture::snapshot(&cold, cold_now); + cold.start_temporal_phase(&cold.mutation_guard().unwrap(), cold_now) + .unwrap(); + let requested = fixture::snapshot(&cold, cold_now); + assert_eq!(requested["durability_pending"], true); + assert_eq!( + requested["durable_windows"], + before_start["durable_windows"] + ); + assert_eq!(requested["observed_ns"], "0"); + let acknowledged = fixture::settle_async_boundary(&cold, cold_now); + let prepared = fixture::snapshot(&cold, acknowledged); + assert_eq!(prepared["lead_in_remaining_ms"], 15_000); + assert_eq!(prepared["durable_windows"], 64); + assert_eq!(prepared["observed_ns"], "0"); + let before_lead_end = acknowledged + 14_999_999_999; + cold.tick_temporal_capture(&cold.mutation_guard().unwrap(), before_lead_end) + .unwrap(); + assert_eq!( + fixture::snapshot(&cold, before_lead_end)["durable_windows"], + 64 + ); + assert_eq!( + fixture::snapshot(&cold, before_lead_end)["observed_ns"], + "0" + ); + cold.cancel_temporal_durability(&cold.mutation_guard().unwrap()) + .unwrap(); + let orphan_rt = fixture::runtime(orphan.path(), cold_now); + orphan_rt + .restore_temporal_full_run( + &orphan_rt.mutation_guard().unwrap(), + &fixture::context(), + cold_now, + ) + .unwrap(); + assert_eq!(fixture::snapshot(&orphan_rt, cold_now)["stage"], "complete"); + assert!( + orphan_rt.current_verified_frozen_model().unwrap().is_none(), + "saved model before Frozen ACK is only a diagnostic orphan" + ); + assert!(orphan_rt + .store + .load_managed_temporal_candidate(&fixture::context()) + .unwrap() + .is_some()); + orphan_rt + .cancel_temporal_durability(&orphan_rt.mutation_guard().unwrap()) + .unwrap(); +} + +async fn fit_ready(rt: &Arc, now: u64) -> u64 { + let start = Instant::now(); + rt.start_temporal_fit(&rt.mutation_guard().unwrap(), now) + .unwrap(); + loop { + let phase = rt.inner.lock().unwrap().job.phase; + if phase == JobPhase::Ready { + break; + } + assert_eq!( + phase, + JobPhase::Running, + "{:?}", + rt.inner.lock().unwrap().job.error + ); + assert!(start.elapsed() < Duration::from_secs(120)); + tokio::time::sleep(Duration::from_millis(5)).await; + } + now + start.elapsed().as_nanos() as u64 +} +async fn prepare_frozen(rt: &Arc, at: u64) -> u64 { + let start = Instant::now(); + loop { + let now = at + start.elapsed().as_nanos() as u64; + match rt.prepare_frozen_transition(&rt.mutation_guard().unwrap(), now) { + Ok(()) => return now, + Err(e) if e == "temporal_actual_current_fit_required" => { + assert!(start.elapsed() < Duration::from_secs(10)); + tokio::time::sleep(Duration::from_millis(1)).await; + } + Err(e) => panic!("Frozen prepare {e}"), + } + } +} +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[cfg_attr(debug_assertions, ignore = "native-persistence journey requires Release; CI explicitly executes and counts this test")] +async fn c4a_actual_frozen_cold_missing_corrupt_wrong_kind_bytes_and_expiry_fail_closed() { + let directory = tempfile::tempdir().unwrap(); + let (rt, now) = record_train(directory.path()); + let at = fit_ready(&rt, now).await; + let (_, receipt, _) = rt.temporal_diagnostic_fit().unwrap().unwrap(); + let at = prepare_frozen(&rt, at).await; + let at = fixture::settle_async_boundary(&rt, at); + assert_eq!(fixture::snapshot(&rt, at)["stage"], "frozen"); + let descriptor = fixture::snapshot(&rt, at)["descriptor"].clone(); + drop(rt); + for corruption in ["missing", "corrupt", "wrong_kind_bytes", "head", "expired"] { + let copy = tempfile::tempdir().unwrap(); + copy_private(directory.path(), copy.path()); + let path = copy + .path() + .join(fixture::context().owner_scope) + .join("SensingServer/room-learner/temporal-models") + .join(fixture::context().digest()) + .join(format!("{}.json", receipt.artifact_sha256)); + match corruption{ + "missing"=>std::fs::remove_file(&path).unwrap(), + "corrupt"=>std::fs::write(&path,b"corrupt synthetic exact model").unwrap(), + "wrong_kind_bytes"=>std::fs::write(&path,b"{\"schema\":\"ruview.room-learner.temporal-input.v2\"}").unwrap(), + "head"=>std::fs::write(copy.path().join(fixture::context().owner_scope).join("SensingServer/room-learner-recovery/temporal-prefix-commitment-v1/temporal-head.bin"),b"corrupt synthetic current head").unwrap(), + _=>{}, + } + let now = if corruption == "expired" { + descriptor["expires_at_ms"].as_u64().unwrap() * 1_000_000 + } else { + at + 1_000_000_000 + }; + let cold = fixture::runtime(copy.path(), now); + assert!( + cold.restore_temporal_full_run( + &cold.mutation_guard().unwrap(), + &fixture::context(), + now + ) + .is_err(), + "{corruption}" + ); + assert!( + cold.current_verified_frozen_model().is_err(), + "{corruption}" + ); + assert!(cold.inner.lock().unwrap().model.is_none()); + assert!(!cold.inner.lock().unwrap().selection.has_history()); + // Failed recovery deliberately leaves a blocked placeholder with no + // core/writer; dropping it must not invent a cancellation target. + assert!(cold + .configure_temporal_capture( + &cold.mutation_guard().unwrap(), + fixture::context(), + "forbidden-fallback".into(), + now + ) + .is_err()); + drop(cold); + } +} +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[cfg_attr(debug_assertions, ignore = "native-persistence journey requires Release; CI explicitly executes and counts this test")] +async fn c4a_owner_cancel_drains_actual_prepared_frozen_writer_before_synthetic_erase() { + let directory = tempfile::tempdir().unwrap(); + let (rt, now) = record_train(directory.path()); + let at = fit_ready(&rt, now).await; + let writer = { + let s = rt.inner.lock().unwrap(); + let Some(CaptureWorkflow::Temporal(c)) = &s.capture else { + panic!() + }; + c.core.as_ref().unwrap().writer().unwrap() + }; + let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(1); + let (release_tx, release_rx) = std::sync::mpsc::sync_channel(1); + let mut gate = ReleaseGate(Some(release_tx)); + writer.hold_next_after_prepare(entered_tx, release_rx); + let at = prepare_frozen(&rt, at).await; + entered_rx.recv_timeout(Duration::from_secs(10)).unwrap(); + assert!(rt.current_verified_frozen_model().unwrap().is_none()); + let (worker_tx, worker_rx) = std::sync::mpsc::sync_channel(1); + let worker = rt.clone(); + let guard = rt.mutation_guard().unwrap(); + let cancel = std::thread::spawn(move || { + worker_tx + .send(worker.cancel_temporal_durability(&guard)) + .unwrap() + }); + assert!( + worker_rx.recv_timeout(Duration::from_millis(50)).is_err(), + "owner cancellation acknowledged an active actual prepared Frozen write" + ); + assert!(rt.inner.try_lock().is_ok()); + gate.release(); + worker_rx + .recv_timeout(Duration::from_secs(10)) + .unwrap() + .unwrap(); + cancel.join().unwrap(); + assert!(rt + .current_verified_frozen_model() + .map_or(true, |m| m.is_none())); + let owner = directory.path().join(fixture::context().owner_scope); + std::fs::remove_dir_all(&owner).unwrap(); + tokio::time::sleep(Duration::from_millis(100)).await; + assert!(!owner.exists()); + drop(rt); + drop(writer); + assert!(!owner.exists()); + println!("SYNTHETIC_C4A_OWNER_DRAIN_PREPARED_FROZEN_AT={at}"); +} +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[cfg_attr(debug_assertions, ignore = "native-persistence journey requires Release; CI explicitly executes and counts this test")] +async fn c4a_actual_frozen_lost_ack_same_intent_repair_installs_only_after_exact_ack() { + let directory = tempfile::tempdir().unwrap(); + let (rt, now) = record_train(directory.path()); + let train_revision = rt.temporal_store().unwrap().load_temporal(now) + .unwrap().unwrap().token.data().revision; + let at = fit_ready(&rt, now).await; + let writer = { + let s = rt.inner.lock().unwrap(); + let Some(CaptureWorkflow::Temporal(c)) = &s.capture else { + panic!() + }; + c.core.as_ref().unwrap().writer().unwrap() + }; + writer.lose_next_ack(); + let at = prepare_frozen(&rt, at).await; + let started = Instant::now(); + let mut now; + loop { + now = at + started.elapsed().as_nanos() as u64; + let _ = rt.tick_temporal_capture(&rt.mutation_guard().unwrap(), now); + if fixture::snapshot(&rt, now)["stage"] == "blocked" { + break; + } + assert!(started.elapsed() < Duration::from_secs(30)); + tokio::time::sleep(Duration::from_millis(5)).await; + } + assert!(rt.current_verified_frozen_model().unwrap().is_none()); + let token = rt + .temporal_store() + .unwrap() + .load_temporal(now) + .unwrap() + .unwrap() + .token; + assert_eq!(token.data().revision, train_revision.checked_add(1).unwrap(), + "Frozen adds exactly one transition to the actual completed Train prefix"); + rt.repair_temporal_capture(&rt.mutation_guard().unwrap(), now) + .unwrap(); + now = fixture::settle_async_boundary(&rt, now); + assert_eq!(fixture::snapshot(&rt, now)["stage"], "frozen"); + assert!(rt.current_verified_frozen_model().unwrap().is_some()); + assert_eq!( + rt.temporal_store() + .unwrap() + .load_temporal(now) + .unwrap() + .unwrap() + .token, + token, + "SAME-intent repair must not allocate another transition" + ); + rt.cancel_temporal_durability(&rt.mutation_guard().unwrap()) + .unwrap(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[cfg_attr(debug_assertions, ignore = "native-persistence journey requires Release; CI explicitly executes and counts this test")] +async fn c4a_running_compute_cancel_cannot_produce_accepted_fit_or_frozen() { + let directory = tempfile::tempdir().unwrap(); + let (rt, now) = record_train(directory.path()); + let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(1); + let (release_tx, release_rx) = std::sync::mpsc::sync_channel(1); + let mut gate = ReleaseGate(Some(release_tx)); + let release = std::sync::Mutex::new(release_rx); + *rt.temporal_fit_hook.lock().unwrap() = Some(Arc::new(move |stage| { + if stage == "before_validate" { + entered_tx.send(()).unwrap(); + release + .lock() + .unwrap() + .recv_timeout(Duration::from_secs(30)) + .unwrap(); + } + })); + rt.start_temporal_fit(&rt.mutation_guard().unwrap(), now) + .unwrap(); + entered_rx.recv_timeout(Duration::from_secs(10)).unwrap(); + let (done_tx, done_rx) = std::sync::mpsc::sync_channel(1); + let worker = rt.clone(); + let guard = rt.mutation_guard().unwrap(); + let cancel = + std::thread::spawn(move || done_tx.send(worker.cancel_temporal_fit(&guard)).unwrap()); + assert!(done_rx.recv_timeout(Duration::from_millis(50)).is_err()); + assert!(rt.inner.try_lock().is_ok()); + gate.release(); + assert_eq!( + done_rx + .recv_timeout(Duration::from_secs(10)) + .unwrap() + .unwrap(), + JobPhase::Cancelled + ); + cancel.join().unwrap(); + assert!(rt.temporal_diagnostic_fit().unwrap().is_none()); + assert!(rt + .prepare_frozen_transition(&rt.mutation_guard().unwrap(), now + 1_000_000_000) + .is_err()); + assert!(rt.current_verified_frozen_model().unwrap().is_none()); + rt.cancel_temporal_durability(&rt.mutation_guard().unwrap()) + .unwrap(); +} + +#[tokio::test(flavor="multi_thread",worker_threads=2)] +#[cfg_attr(debug_assertions,ignore="native-persistence operator journey requires Release; CI must explicitly execute this case")] +async fn operator_eight_actual_reserved_checks_fit_complete_cold_and_evaluate(){ + assert!(!cfg!(debug_assertions)); + let directory=tempfile::tempdir().unwrap(); + let mut runtime=Arc::new(fixture::runtime(directory.path(),fixture::START)); + runtime.temporal_select_profile_v2(Some("native-2s-three-source-56bin-operator-eight-step-v1")).unwrap(); + let store=runtime.temporal_store_arc().unwrap(); + runtime.configure_temporal_capture_with_custody(&runtime.mutation_guard().unwrap(),fixture::context(),"synthetic-eight-actual".into(),fixture::START, + move|schedule,run,mut config,now,_,witness|{config.epochs=2;TemporalTrainCapture::prepare_full_run_async(schedule,run,config,now,store,witness)}).unwrap(); + let mut now=fixture::settle_async_boundary(&runtime,fixture::START); + let original=fixture::snapshot(&runtime,now)["descriptor"].clone(); + let schedule=ManagedScheduleContract::build(fixture::context(),LeadInPolicy::OperatorEightStepV1).unwrap(); + let mut sequence=0u32; + // The same stopped-first-capture seam: segment write actually commits but + // its ACK is lost; a too-fast packet fails before any retained window. + now=fixture::async_lead_end(&runtime,now); + let (initial_enter_tx,initial_enter_rx)=std::sync::mpsc::sync_channel(1); + let (initial_release_tx,initial_release_rx)=std::sync::mpsc::sync_channel(1); + {let state=runtime.inner.lock().unwrap();let Some(CaptureWorkflow::Temporal(c))=&state.capture else{panic!()};let writer=c.core.as_ref().unwrap().writer().unwrap();writer.lose_next_ack();writer.hold_next_after_publish(initial_enter_tx,initial_release_rx);} + let mut initial_gate=ReleaseGate(Some(initial_release_tx)); + for source in 1..=3u8{let at=now+u64::from(source)*1000;runtime.ingest_native(&packet(source,1,at,RoomLabel::Empty),at).unwrap();} + initial_enter_rx.recv_timeout(Duration::from_secs(10)).unwrap(); + assert_eq!(fixture::snapshot(&runtime,now+3000)["durability_pending"],true); + let failed_at=now+1_001_000; + assert!(runtime.ingest_native(&packet(1,2,failed_at,RoomLabel::Empty),failed_at).unwrap_err().starts_with("temporal_observed_gap_or_rate:")); + initial_gate.release(); + let failed=runtime.temporal_status_v2(failed_at).unwrap(); + assert_eq!(failed["capture"]["stage"],"failed"); + let id=failed["capture"]["capture_generation_id"].as_str().unwrap().to_owned(); + let before=runtime.mutation_guard().unwrap();now=failed_at+1_000_000_000; + runtime.retry_failed_temporal_capture(&before,&id,now).unwrap(); + let retried=runtime.temporal_status_v2(now).unwrap(); + assert_eq!(retried["capture"]["capture_generation_id"],id); + assert_eq!(retried["capture"]["stage"],"paused"); + assert_eq!(retried["capture"]["durability_pending"],false); + assert_eq!(retried["capture"]["retained_windows"],0); + assert_eq!(retried["capture"]["durable_windows"],0); + assert!(retried["capture"]["failure"].is_null()); + assert_original_lifetime(&fixture::snapshot(&runtime,now)["descriptor"],&original); + assert_eq!(runtime.mutation_guard().unwrap().generation,before.generation); + assert!(runtime.mutation_guard().unwrap().workflow_revision>before.workflow_revision); + sequence=2; + for episode in schedule.episodes(){ + let supervised=Instant::now(); + loop{ + let progress=fixture::snapshot(&runtime,now); + match progress["stage"].as_str().unwrap(){ + "ready"|"paused"=>{ + now=fixture::async_lead_end(&runtime,now); + // Exercise preparation slower than the first Check slot's + // buffer horizon. No synthetic packet may precede its ACK. + let delayed = if episode.index == 7 { + let writer={let state=runtime.inner.lock().unwrap();let Some(CaptureWorkflow::Temporal(c))=&state.capture else{panic!()};c.core.as_ref().unwrap().writer().unwrap()}; + let(entered_tx,entered_rx)=std::sync::mpsc::sync_channel(1); + let(release_tx,release_rx)=std::sync::mpsc::sync_channel(1); + writer.hold_next_after_publish(entered_tx,release_rx); + let timer_release=release_tx.clone(); + let timer=std::thread::spawn(move||{ + entered_rx.recv_timeout(Duration::from_secs(30)).unwrap(); + std::thread::sleep(Duration::from_secs(5)); + let _=timer_release.send(()); + }); + Some((ReleaseGate(Some(release_tx)),timer)) + }else{None}; + wait_for_segment_commit(&runtime,now,supervised); + if let Some((mut gate,timer))=delayed{timer.join().unwrap();gate.release();} + } + "needs_confirmation"=>break, + "failed"=>panic!("operator episode{} failed: {progress}",episode.index), + _=>{} + } + sequence+=1; + for source in 1..=3{match runtime.ingest_native(&packet(source,sequence,now,episode.condition),now){Ok(())=>{},Err(e)if e=="temporal_storage_busy_pause"=>break,Err(e)=>panic!("operator ingest: {e}")}} + now+=fixture::PERIOD; + if fixture::snapshot(&runtime,now)["durability_pending"]==true{std::thread::sleep(Duration::from_nanos(fixture::PERIOD));} + assert!(supervised.elapsed()serde_json::Value{ + use sha2::{Digest,Sha256}; + let loaded=rt.temporal_store().unwrap().load_temporal(at).unwrap().unwrap(); + let data:serde_json::Value=serde_json::from_slice(loaded.prefix.unwrap().bytes()).unwrap(); + let payload=&data["payload"]; + for key in ["rows","evidence","identities"]{assert_eq!(payload["accepted"][key].as_array().unwrap().len(),20);} + assert_eq!(payload["common"]["ledger"]["spans"].as_array().unwrap().len(),20); + assert_eq!(payload["stage"]["bridge"]["seals"].as_array().unwrap().len(),2); + let digest=|v:&serde_json::Value|format!("{:x}",Sha256::digest(serde_json::to_vec(v).unwrap())); + serde_json::json!({"rows":digest(&payload["accepted"]["rows"]),"evidence":digest(&payload["accepted"]["evidence"]), + "identities":digest(&payload["accepted"]["identities"]),"spans":digest(&payload["common"]["ledger"]["spans"]), + "seals":digest(&payload["stage"]["bridge"]["seals"])}) + } + let known_prefix=prefix20(&runtime,now); + let known_rows=fixture::snapshot(&runtime,now)["rows_sha256"].clone();assert!(known_rows.is_string()); + for committed_lost_ack in [true,false]{ + let lead=fixture::async_lead_end(&runtime,now); + let writer={let state=runtime.inner.lock().unwrap();let Some(CaptureWorkflow::Temporal(c))=&state.capture else{panic!()};c.core.as_ref().unwrap().writer().unwrap()}; + let gate=if committed_lost_ack{ + let(entered_tx,entered_rx)=std::sync::mpsc::sync_channel(1);let(release_tx,release_rx)=std::sync::mpsc::sync_channel(1); + writer.lose_next_ack();writer.hold_next_after_publish(entered_tx,release_rx);Some((entered_rx,ReleaseGate(Some(release_tx)))) + }else{writer.fail_next_publication();None}; + sequence+=1; + // One packet arms the pending segment in both variants. + runtime.ingest_native(&packet(1,sequence,lead+1000,RoomLabel::OccupiedStatic),lead+1000).unwrap(); + let stop=lead+1_001_000; + if let Some((entered,mut release))=gate{ + for source in 2..=3u8{let at=lead+u64::from(source)*1000;runtime.ingest_native(&packet(source,sequence,at,RoomLabel::OccupiedStatic),at).unwrap();} + entered.recv_timeout(Duration::from_secs(10)).unwrap(); + assert!(writer.has_failed_repair(),"the HEAD/WITNESS publication is real before the ACK is lost"); + sequence+=1; + assert!(runtime.ingest_native(&packet(1,sequence,stop,RoomLabel::OccupiedStatic),stop).unwrap_err().starts_with("temporal_observed_gap_or_rate:")); + release.release(); + }else{ + let wait=Instant::now();while !writer.ack_is_queued(){assert!(wait.elapsed()=seq&&*t>=time),"Retry must retain live replay fences");} + assert!(runtime.retry_failed_temporal_capture(&guard,&capture_id,now).is_err(),"consumed command cannot retry again"); + sequence+=1; + println!("SYNTHETIC_OPERATOR20_RETRY committed_lost_ack={committed_lost_ack} rows=20 seals=2 prefix_preserved=true"); + } + // Probe a real previously admitted CanonicalPacket while paused. + // A fail-closed replay may stop the run; explicitly Retry once more + // before continuing, never count that packet as new progress. + let replay_floors=floors(&runtime).0; + let(source,seq,time)=replay_floors[0];let replay_now=now+1_000_000; + assert!(runtime.ingest_native(&packet(source,seq as u32,time,RoomLabel::OccupiedStatic),replay_now).is_err()); + assert_eq!(fixture::snapshot(&runtime,replay_now)["retained_rows"],20); + assert_eq!(fixture::snapshot(&runtime,replay_now)["durable_windows"],20); + assert_eq!(floors(&runtime).0,replay_floors,"rejected replay adds no source credit"); + if matches!(fixture::snapshot(&runtime,replay_now)["stage"].as_str(),Some("failed"|"blocked")){ + let status=runtime.temporal_status_v2(replay_now).unwrap();let id=status["capture"]["capture_generation_id"].as_str().unwrap(); + now=replay_now+1_000_000_000;runtime.retry_failed_temporal_capture(&runtime.mutation_guard().unwrap(),id,now).unwrap(); + }else{now=replay_now;} + assert_eq!(prefix20(&runtime,now),known_prefix); + } + } + assert_eq!(fixture::snapshot(&runtime,now)["durable_windows"],80); + assert!(runtime.current_verified_frozen_model().unwrap().is_none()); + runtime.start_temporal_fit(&runtime.mutation_guard().unwrap(),now).unwrap(); + let fit_clock=Instant::now(); + loop{ + let phase=runtime.inner.lock().unwrap().job.phase; + if phase==JobPhase::Ready{break;} + assert_eq!(phase,JobPhase::Running,"{:?}",runtime.inner.lock().unwrap().job.error); + assert!(fit_clock.elapsed() (Arc, u64) { + let runtime = Arc::new(fixture::runtime(root, fixture::START)); + let store = runtime.recovery_store.clone().unwrap(); + runtime + .configure_temporal_capture_with_custody( + &runtime.mutation_guard().unwrap(), + fixture::context(), + "synthetic-c4b-native-fit".into(), + fixture::START, + move |schedule, run, mut config, now, _, witness| { + // Bound when this capture is CREATED, never overridden by fit input. + config.epochs = 2; + TemporalTrainCapture::prepare_full_run_async( + schedule, run, config, now, store, witness, + ) + }, + ) + .unwrap(); + let now = fixture::settle_async_boundary(&runtime, fixture::START); + (runtime, now) +} +pub(super) fn packet(source: u8, sequence: u32, at: u64, label: RoomLabel) -> CanonicalPacket { + let gain = match label { + RoomLabel::Empty => 3, + RoomLabel::AdjacentOnly => 4, + RoomLabel::OccupiedStatic => 8, + RoomLabel::OccupiedMoving => 10 + (sequence % 2) as u8, + _ => unreachable!(), + }; + let iq = (0..64) + .flat_map(|bin| [gain + source + (bin % 2) as u8, 4]) + .collect::>(); + canonicalize_esp32(NativePacketInput { + node_id: source, + n_antennas: 1, + raw_bins: 64, + frequency_hz: 2_437_000_000, + ppdu_type: PpduType::HtLegacy, + sequence, + received_at_ns: at, + receiver_sample_rate_hz: 33., + iq_bytes: &iq, + flags: Adr018Flags::default(), + provenance: CaptureProvenance::LiveReceiverObserved, + }) + .unwrap() +} +fn record_train_uncached(root: &Path) -> (Arc, u64) { + let (runtime, mut now) = configured(root); + let schedule = ManagedScheduleContract::build( + fixture::context(), + LeadInPolicy::Train5HeldOut15EngineeringV1, + ) + .unwrap(); + let mut sequence = 0; + let mut starts = 0; + for episode in schedule + .episodes() + .iter() + .filter(|e| e.role == DataRole::Train) + { + loop { + // A durable terminal pause can be confirmable without resuming. + if runtime + .temporal_confirmation_challenge(&runtime.mutation_guard().unwrap()) + .is_ok() + { + break; + } + let progress = fixture::snapshot(&runtime, now); + match progress["stage"].as_str().unwrap() { + "ready" | "paused" => { + now = fixture::async_lead_end(&runtime, now); + starts += 1; + runtime + .tick_temporal_capture(&runtime.mutation_guard().unwrap(), now) + .unwrap(); + } + "needs_confirmation" => break, + "failed" => panic!("synthetic C4B phase{} failed: {progress}", episode.index), + _ => {} + } + let pending = fixture::snapshot(&runtime, now)["durability_pending"] == true; + sequence += 1; + for source in 1..=3 { + match runtime.ingest_native(&packet(source, sequence, now, episode.condition), now) + { + Ok(()) => {} + Err(error) if error == "temporal_storage_busy_pause" => break, + Err(error) => panic!("synthetic C4B ingest: {error}"), + } + } + now += fixture::PERIOD; + if pending { + std::thread::sleep(Duration::from_nanos(fixture::PERIOD)); + } + assert!(starts < 120, "synthetic C4B interruption bound exhausted"); + } + let guard = runtime.mutation_guard().unwrap(); + let challenge = runtime.temporal_confirmation_challenge(&guard).unwrap(); + runtime + .confirm_temporal_phase(&guard, &challenge, now) + .unwrap(); + assert_eq!(fixture::snapshot(&runtime, now)["durability_pending"], true); + now = fixture::settle_async_boundary(&runtime, now); + } + let complete = fixture::snapshot(&runtime, now); + assert_eq!(complete["stage"], "complete"); + assert_eq!( + complete["sealed_train_episodes"], + schedule.counts().train_episodes + ); + assert_eq!(complete["durable_windows"], schedule.counts().train_windows); + println!( + "SYNTHETIC_C4B_REAL_TRAIN_CAPTURE: rows={} seals={} starts={starts}", + schedule.counts().train_windows, + schedule.counts().train_episodes + ); + (runtime, now) +} + +pub(super) fn copy_private(from: &Path, to: &Path) { + use std::os::unix::fs::PermissionsExt; + assert!(!std::fs::symlink_metadata(from) + .unwrap() + .file_type() + .is_symlink()); + if from.is_dir() { + std::fs::create_dir_all(to).unwrap(); + std::fs::set_permissions(to, std::fs::Permissions::from_mode(0o700)).unwrap(); + for e in std::fs::read_dir(from).unwrap() { + let e = e.unwrap(); + copy_private(&e.path(), &to.join(e.file_name())); + } + } else { + std::fs::copy(from, to).unwrap(); + std::fs::set_permissions(to, std::fs::Permissions::from_mode(0o600)).unwrap(); + } +} +// Cache only the actual seven-phase native producer output. Each independent +// fault case gets its own files and strictly current recertification/owner. +fn record_train(root: &Path) -> (Arc, u64) { + static TRAIN: std::sync::OnceLock<(tempfile::TempDir, u64)> = std::sync::OnceLock::new(); + let (saved, at) = TRAIN.get_or_init(|| { + let d = tempfile::tempdir().unwrap(); + let (rt, at) = record_train_uncached(d.path()); + drop(rt); + (d, at) + }); + copy_private(saved.path(), root); + let now = at + 1_000_000_000; + let rt = Arc::new(fixture::runtime(root, now)); + rt.restore_temporal_full_run(&rt.mutation_guard().unwrap(), &fixture::context(), now) + .unwrap(); + assert_eq!(fixture::snapshot(&rt, now)["stage"], "complete"); + (rt, now) +} +pub(super) fn assert_original_lifetime(value: &serde_json::Value, original: &serde_json::Value) { + for field in [ + "schema", + "id", + "created_at_ms", + "expires_at_ms", + "owner_scope", + "installation_id", + "context_digest", + ] { + assert_eq!(value[field], original[field], "{field}"); + } +} + +async fn fit_ready(rt: &Arc, now: u64) -> u64 { + let start = Instant::now(); + rt.start_temporal_fit(&rt.mutation_guard().unwrap(), now) + .unwrap(); + loop { + let phase = rt.inner.lock().unwrap().job.phase; + if phase == JobPhase::Ready { + break; + } + assert_eq!( + phase, + JobPhase::Running, + "{:?}", + rt.inner.lock().unwrap().job.error + ); + assert!(start.elapsed() < Duration::from_secs(120)); + tokio::time::sleep(Duration::from_millis(5)).await; + } + now + start.elapsed().as_nanos() as u64 +} +async fn prepare_frozen(rt: &Arc, at: u64) -> u64 { + let start = Instant::now(); + loop { + let now = at + start.elapsed().as_nanos() as u64; + match rt.prepare_frozen_transition(&rt.mutation_guard().unwrap(), now) { + Ok(()) => return now, + Err(e) if e == "temporal_actual_current_fit_required" => { + assert!(start.elapsed() < Duration::from_secs(10)); + tokio::time::sleep(Duration::from_millis(1)).await; + } + Err(e) => panic!("Frozen prepare {e}"), + } + } +} + +pub(super) fn phase_target(rt: &RoomLearningRuntime) -> Option<(u64, u64)> { + let state = rt.inner.lock().unwrap(); + let Some(CaptureWorkflow::Temporal(c)) = &state.capture else { + panic!() + }; + c.core.as_ref().unwrap().c4b_test_target() +} +pub(super) fn receiver_floor(rt: &RoomLearningRuntime) -> u32 { + let state = rt.inner.lock().unwrap(); + let Some(CaptureWorkflow::Temporal(c)) = &state.capture else { + panic!() + }; + c.core + .as_ref() + .unwrap() + .test_full_run_floors() + .0 + .iter() + .map(|(_, q, _)| u32::try_from(*q).unwrap()) + .max() + .unwrap() +} +pub(super) fn prefix_value(rt: &RoomLearningRuntime, now: u64) -> serde_json::Value { + let loaded = rt + .temporal_store() + .unwrap() + .load_temporal(now) + .unwrap() + .unwrap(); + serde_json::from_slice(loaded.prefix.unwrap().bytes()).unwrap() +} +pub(super) fn same_train_and_frozen(before: &serde_json::Value, after: &serde_json::Value) { + for field in ["rows", "evidence", "identities"] { + assert_eq!( + &before["payload"]["accepted"][field].as_array().unwrap()[..64], + &after["payload"]["accepted"][field].as_array().unwrap()[..64], + "immutable Train {field}" + ); + } + assert_eq!( + before["payload"]["stage"]["fit"], + after["payload"]["stage"]["fit"] + ); + assert_eq!( + before["payload"]["stage"]["train_seals"], + after["payload"]["stage"]["train_seals"] + ); + assert_original_lifetime( + &after["payload"]["common"]["descriptor"], + &before["payload"]["common"]["descriptor"], + ); +} +async fn produce_complete_uncached(directory: &Path) -> u64 { + let (mut rt, mut now) = record_train(directory); + now = fit_ready(&rt, now).await; + let (model, receipt, _) = rt.temporal_diagnostic_fit().unwrap().unwrap(); + let model_bytes = model.export().unwrap(); + let report_before = serde_json::to_value(rt.store.temporal_fit_diagnostics(&model)).unwrap(); + now = prepare_frozen(&rt, now).await; + now = fixture::settle_async_boundary(&rt, now); + assert_eq!(fixture::snapshot(&rt, now)["stage"], "frozen"); + let frozen = prefix_value(&rt, now); + let original = frozen["payload"]["common"]["descriptor"].clone(); + let schedule = ManagedScheduleContract::build( + fixture::context(), + LeadInPolicy::Train5HeldOut15EngineeringV1, + ) + .unwrap(); + let mut sequence = receiver_floor(&rt); + let mut partial_cold = false; + let mut terminal_cold = false; + let mut starts = 0; + for episode in schedule + .episodes() + .iter() + .filter(|e| e.role == DataRole::HeldOut) + { + loop { + let challenge = rt.temporal_confirmation_challenge(&rt.mutation_guard().unwrap()); + if challenge.is_ok() { + if episode.index == 14 && !terminal_cold { + let before = prefix_value(&rt, now); + let rows = fixture::snapshot(&rt, now)["durable_windows"].clone(); + drop(rt); + now += 1_000_000_000; + rt = Arc::new(fixture::runtime(directory, now)); + rt.restore_temporal_full_run( + &rt.mutation_guard().unwrap(), + &fixture::context(), + now, + ) + .unwrap(); + assert_eq!(fixture::snapshot(&rt, now)["durable_windows"], rows); + same_train_and_frozen(&frozen, &prefix_value(&rt, now)); + assert_eq!( + before["payload"]["accepted"], + prefix_value(&rt, now)["payload"]["accepted"] + ); + terminal_cold = true; + continue; + } + break; + } + let progress = fixture::snapshot(&rt, now); + assert_eq!( + progress["label"], + serde_json::to_value(episode.condition).unwrap() + ); + assert_eq!(progress["role"], "HeldOut"); + assert_eq!(progress["target_ms"], 30_000); + match progress["stage"].as_str().unwrap() { + "frozen" | "ready" | "paused" => { + now = fixture::async_lead_end(&rt, now); + starts += 1; + if episode.index == 7 && starts == 1 { + now += 50_000_000; + } + rt.tick_temporal_capture(&rt.mutation_guard().unwrap(), now) + .unwrap(); + } + "failed" => panic!("HeldOut failed: {progress}"), + _ => {} + } + let pending = fixture::snapshot(&rt, now)["durability_pending"] == true; + sequence += 1; + for source in 1..=3 { + match rt.ingest_native(&packet(source, sequence, now, episode.condition), now) { + Ok(()) => {} + Err(e) if e == "temporal_storage_busy_pause" => break, + Err(e) => panic!("HeldOut ingress: {e}"), + } + } + now += fixture::PERIOD; + if pending { + std::thread::sleep(Duration::from_nanos(fixture::PERIOD)); + } + let progress = fixture::snapshot(&rt, now); + if !partial_cold + && episode.index == 7 + && progress["durable_windows"].as_u64().unwrap() >= 67 + && progress["durability_pending"] == false + && phase_target(&rt) + .is_some_and(|(start, end)| now >= start + 3 * fixture::PERIOD && now < end) + { + let durable = progress["durable_windows"].clone(); + let before = prefix_value(&rt, now); + drop(rt); + now += 1_000_000_000; + rt = Arc::new(fixture::runtime(directory, now)); + rt.restore_temporal_full_run( + &rt.mutation_guard().unwrap(), + &fixture::context(), + now, + ) + .unwrap(); + assert_eq!(fixture::snapshot(&rt, now)["durable_windows"], durable); + assert!(phase_target(&rt).is_none()); + assert_eq!( + before["payload"]["accepted"], + prefix_value(&rt, now)["payload"]["accepted"] + ); + same_train_and_frozen(&frozen, &prefix_value(&rt, now)); + partial_cold = true; + } + assert!( + starts < 300, + "bounded synthetic HeldOut interruption budget exhausted" + ); + } + let guard = rt.mutation_guard().unwrap(); + let challenge = rt.temporal_confirmation_challenge(&guard).unwrap(); + rt.confirm_temporal_phase(&guard, &challenge, now).unwrap(); + now = fixture::settle_async_boundary(&rt, now); + assert_eq!( + fixture::snapshot(&rt, now)["sealed_held_out_episodes"], + episode.index - 6 + ); + let sealed=prefix_value(&rt,now); + same_train_and_frozen(&frozen, &sealed); + println!("SYNTHETIC_C4B_HELDOUT_SEAL episode={} durable={} sealed={} starts={starts}",episode.index,fixture::snapshot(&rt,now)["durable_windows"],fixture::snapshot(&rt,now)["sealed_held_out_episodes"]); + if episode.index==13{ + let loaded=rt.temporal_store().unwrap().load_temporal(now).unwrap().unwrap();let prefix=loaded.prefix.unwrap(); + let oracle=crate::room_learner::temporal_model_input::managed::execution::checkpoint::c4b_reference_batch_and_stale_seal_probe(prefix.bytes(),prefix.descriptor(),now).unwrap(); + println!("SYNTHETIC_C4B_FRESH120_REFERENCE_STALE_SEAL={oracle}"); + } + } + assert!( + partial_cold && terminal_cold, + "two actual cold interruptions must execute" + ); + let status = fixture::snapshot(&rt, now); + assert_eq!(status["stage"], "full_complete"); + assert_eq!(status["durable_windows"], 128); + assert_eq!(status["sealed_train_episodes"], 7); + assert_eq!(status["sealed_held_out_episodes"], 8); + let complete = prefix_value(&rt, now); + same_train_and_frozen(&frozen, &complete); + assert_eq!(complete["payload"]["stage"]["kind"], "complete"); + assert_eq!( + rt.current_verified_frozen_model() + .unwrap() + .unwrap() + .export() + .unwrap(), + model_bytes + ); + assert_eq!( + serde_json::to_value(rt.store.temporal_fit_diagnostics(&model)).unwrap(), + report_before + ); + assert!(rt.inner.lock().unwrap().model.is_none()); + assert!(rt.inner.lock().unwrap().qualification.is_none()); + assert!(!rt.inner.lock().unwrap().selection.has_history()); + let loaded = rt + .temporal_store() + .unwrap() + .load_temporal(now) + .unwrap() + .unwrap(); + let final_prefix=loaded.prefix.unwrap(); + let bytes = final_prefix.bytes().len(); + let oracle=crate::room_learner::temporal_model_input::managed::execution::checkpoint::c4b_reference_batch_and_stale_seal_probe(final_prefix.bytes(),final_prefix.descriptor(),now).unwrap(); + println!("SYNTHETIC_C4B_FRESH128_REFERENCE_STALE_SEAL={oracle}"); + let raw_values=complete["payload"]["accepted"]["rows"].as_array().unwrap().iter().map(|row|row["views"].as_array().unwrap().iter().map(|view|view["samples"].as_array().unwrap().iter().map(|sample|sample.as_array().unwrap().len()).sum::()).sum::()).sum::(); + assert!(raw_values<=2_000_000); + fn slot_sizes(root:&Path,out:&mut Vec){for entry in std::fs::read_dir(root).unwrap(){let path=entry.unwrap().path();if path.is_dir(){slot_sizes(&path,out)}else if path.file_name().unwrap().to_string_lossy().starts_with("slot-")&&path.extension().is_some_and(|e|e=="bin"){out.push(std::fs::metadata(path).unwrap().len());}}} + let mut outer_slot_sizes=vec![];slot_sizes(directory,&mut outer_slot_sizes);outer_slot_sizes.sort_unstable(); + println!("SYNTHETIC_C4B_RAW_NUMERIC_VALUES={raw_values} AEAD_SLOT_FILE_BYTES={outer_slot_sizes:?}"); + assert!(bytes <= crate::room_learner::recovery_store::MAX_RECOVERY_BYTES); + println!( + "SYNTHETIC_C4B_FULL128_ACTUAL_JSON_BYTES={bytes} STARTS={starts} GLOBAL_GAPS={}", + complete["payload"]["common"]["ledger"]["gaps"] + .as_array() + .unwrap() + .len() + ); + drop(rt); + now += 1_000_000_000; + let cold = fixture::runtime(directory, now); + cold.restore_temporal_full_run(&cold.mutation_guard().unwrap(), &fixture::context(), now) + .unwrap(); + assert_eq!(fixture::snapshot(&cold, now)["stage"], "full_complete"); + assert_eq!( + cold.current_verified_frozen_model() + .unwrap() + .unwrap() + .export() + .unwrap(), + model_bytes + ); + assert_original_lifetime(&fixture::snapshot(&cold, now)["descriptor"], &original); + assert!(cold + .start_temporal_phase(&cold.mutation_guard().unwrap(), now) + .is_err()); + assert_eq!( + cold.store + .load_exact_temporal_candidate(&fixture::context(), &receipt.artifact_sha256) + .unwrap() + .export() + .unwrap(), + model_bytes + ); + cold.cancel_temporal_durability(&cold.mutation_guard().unwrap()) + .unwrap(); + now +} + +/// Isolated actual native/fence resource reproduction. This does NOT issue a +/// fitted/conditions/write permission or claim the full fifteen-phase path. +#[test] +fn c4b_native_partial_rollback_reproduces_retained_only_frame_fence_assumption() { + use crate::room_learner::temporal_acquisition::{ + AcquisitionPlan, ExperimentalPlanId, NativeWindowAssembler, ReceiverCadenceBand, + WindowAdmission, + }; + let budget = actual_full2_replay_budget(); + let context = fixture::context(); + let plan = AcquisitionPlan::build( + context, + ExperimentalPlanId::Native2000msSlots8, + &(1..=3) + .map(|id| (id, ReceiverCadenceBand::Hz30To36)) + .collect::>(), + ) + .unwrap(); + let maximum = plan + .source_policies() + .iter() + .map(|source| source.max_rows) + .sum::() + * 128; + let mut assembler = + NativeWindowAssembler::new(plan, "synthetic-b-fence-isolation".into()).unwrap(); + let mut sequence = 0u32; + let mut now = fixture::START; + let mut retained = 0usize; + let mut discarded = 0usize; + let mut reproduced = false; + 'attempts: for _ in 0..128 { + // One bounded partially consumed reservation, not a fabricated ID set. + assembler.begin_window(now, now + 6_000_000_000).unwrap(); + for index in 0..24 { + sequence += 1; + let at = now + index * fixture::PERIOD; + for source in 1..=3 { + assert_eq!( + assembler + .ingest(&packet(source, sequence, at, RoomLabel::Empty)) + .unwrap(), + WindowAdmission::Collecting + ); + } + } + discarded += 1; + let checkpoint = assembler + .fence_checkpoint() + .unwrap() + .for_prefix(retained) + .unwrap(); + let state = serde_json::to_value(&checkpoint).unwrap(); + if let Err(error) = assembler.restore_prefix_fences(checkpoint.clone(), retained) { + let next = state["next"].as_u64().unwrap() as usize; + let frames = state["frames"].as_array().unwrap().len(); + let watermarks = state["watermarks"].as_array().unwrap().len(); + println!("SYNTHETIC_FENCE_BOUND phase=partial next={next} expected={retained} next_gt128={} frames={frames} max={maximum} frames_gtmax={} watermarks={watermarks} sources=3 retained={retained} discarded={discarded}",next>128,frames>maximum); + assert_eq!(error, "temporal_checkpoint_fence_bounds"); + assert_eq!(next, retained); + assert!(next <= 128); + assert!(frames > maximum); + assert!(watermarks <= 3); + assembler + .restore_full_run_prefix_fences(checkpoint, retained, &budget) + .unwrap(); + assert_eq!( + assembler.c4b_test_fence_counts().1, + frames, + "typed budget must preserve EVERY actual ID" + ); + reproduced = true; + break 'attempts; + } + now += 6_000_000_000; + assembler.begin_window(now, now + 6_000_000_000).unwrap(); + let mut ready = false; + 'packets: for index in 0..77 { + sequence += 1; + let at = now + index * fixture::PERIOD; + for source in 1..=3 { + if assembler + .ingest(&packet(source, sequence, at, RoomLabel::Empty)) + .unwrap() + == WindowAdmission::Ready + { + assembler.finish_window(at).unwrap(); + ready = true; + break 'packets; + } + } + } + assert!(ready); + retained += 1; + let checkpoint = assembler.fence_checkpoint().unwrap(); + let state = serde_json::to_value(&checkpoint).unwrap(); + if let Err(error) = assembler.restore_prefix_fences(checkpoint.clone(), retained) { + let next = state["next"].as_u64().unwrap() as usize; + let frames = state["frames"].as_array().unwrap().len(); + let watermarks = state["watermarks"].as_array().unwrap().len(); + println!("SYNTHETIC_FENCE_BOUND phase=retained next={next} expected={retained} next_gt128={} frames={frames} max={maximum} frames_gtmax={} watermarks={watermarks} sources=3 retained={retained} discarded={discarded}",next>128,frames>maximum); + assert_eq!(error, "temporal_checkpoint_fence_bounds"); + assert_eq!(next, retained); + assert!(next <= 128); + assert!(frames > maximum); + assert!(watermarks <= 3); + assembler + .restore_full_run_prefix_fences(checkpoint, retained, &budget) + .unwrap(); + assert_eq!( + assembler.c4b_test_fence_counts().1, + frames, + "typed budget must preserve EVERY actual ID" + ); + reproduced = true; + break 'attempts; + } + now += 6_000_000_000; + } + assert!(reproduced,"bounded actual native/discard fixture did not reproduce the suspected retained-only resource assumption"); + assert!(retained <= 128 && discarded <= 128); +} + +#[test] +fn c4b_old_kind1_status_shape_has_no_full2_role_fields() { + for fresh in [false, true] { + let directory = tempfile::tempdir().unwrap(); + let rt = fixture::runtime(directory.path(), fixture::START); + if fresh { + rt.configure_fresh_train_343_capture( + &rt.mutation_guard().unwrap(), + fixture::context(), + "synthetic-old-status343".into(), + fixture::START, + ) + .unwrap(); + } else { + rt.configure_temporal_capture( + &rt.mutation_guard().unwrap(), + fixture::context(), + "synthetic-old-status329".into(), + fixture::START, + ) + .unwrap(); + } + let now = fixture::settle_async_boundary(&rt, fixture::START); + let value = fixture::snapshot(&rt, now); + assert!(value.get("sealed_held_out_episodes").is_none()); + assert_eq!(value["role"], "Train"); + assert_eq!( + value["schema"], + "ruview.room-learner.private-temporal-capture.v1" + ); + assert_eq!( + value["label"], + serde_json::to_value(RoomLabel::Empty).unwrap() + ); + rt.cancel_temporal_durability(&rt.mutation_guard().unwrap()) + .unwrap(); + } +} + +fn actual_full2_replay_budget( +) -> crate::room_learner::temporal_model_input::managed::execution::full_run::ReplayFenceBudget { + let directory = tempfile::tempdir().unwrap(); + let rt = fixture::runtime(directory.path(), fixture::START); + rt.configure_temporal_full_run( + &rt.mutation_guard().unwrap(), + fixture::context(), + "synthetic-full2-resource-budget".into(), + fixture::START, + ) + .unwrap(); + let now = fixture::settle_async_boundary(&rt, fixture::START); + let budget = { + let state = rt.inner.lock().unwrap(); + let Some(CaptureWorkflow::Temporal(c)) = &state.capture else { + panic!() + }; + c.core.as_ref().unwrap().c4b_test_replay_budget().unwrap() + }; + rt.cancel_temporal_durability(&rt.mutation_guard().unwrap()) + .unwrap(); + assert_eq!(budget.max_frames(), 59_136); + let _ = now; + budget +} +#[test] +fn c4b_full2_fence_budget_exact_boundary_and_other_guards_remain_strict() { + use crate::room_learner::temporal_acquisition::{ + AcquisitionPlan, AssemblerCheckpoint, ExperimentalPlanId, NativeWindowAssembler, + ReceiverCadenceBand, + }; + let budget = actual_full2_replay_budget(); + let make = |context| { + NativeWindowAssembler::new( + AcquisitionPlan::build( + context, + ExperimentalPlanId::Native2000msSlots8, + &(1..=3) + .map(|id| (id, ReceiverCadenceBand::Hz30To36)) + .collect::>(), + ) + .unwrap(), + "synthetic-fence-boundary".into(), + ) + .unwrap() + }; + let mut assembler = make(fixture::context()); + let per_source = 59_136 / 3; + let mut frames = Vec::new(); + let mut watermarks = Vec::new(); + for source in 1..=3 { + for sequence in 1..=per_source { + let at = fixture::START + sequence as u64 * fixture::PERIOD; + let p = packet(source, sequence as u32, at, RoomLabel::Empty); + frames.push(serde_json::json!([ + source, + p.native_frame.frame_id.to_string() + ])); + } + watermarks.push(serde_json::json!([ + source, + per_source, + fixture::START + per_source as u64 * fixture::PERIOD + ])); + } + let value = serde_json::json!({"next":128,"last_end_ns":fixture::START,"frames":frames,"watermarks":watermarks}); + let checkpoint: AssemblerCheckpoint = serde_json::from_value(value.clone()).unwrap(); + assert_eq!( + assembler + .restore_prefix_fences(checkpoint.clone(), 128) + .unwrap_err(), + "temporal_checkpoint_fence_bounds" + ); + assembler + .restore_full_run_prefix_fences(checkpoint, 128, &budget) + .unwrap(); + assert_eq!(assembler.c4b_test_fence_counts().1, 59_136); + assert!( + assembler + .begin_window( + fixture::START + 1_000_000_000_000, + fixture::START + 1_006_000_000_000 + ) + .is_err(), + "no129thwindow may createa newtail" + ); + assert_eq!(assembler.c4b_test_fence_counts().1, 59_136); + let mut beyond = value.clone(); + let sequence = per_source + 1; + let at = fixture::START + sequence as u64 * fixture::PERIOD; + let p = packet(1, sequence as u32, at, RoomLabel::Empty); + beyond["frames"] + .as_array_mut() + .unwrap() + .push(serde_json::json!([1, p.native_frame.frame_id.to_string()])); + beyond["watermarks"][0] = serde_json::json!([1, sequence, at]); + assert_eq!( + make(fixture::context()) + .restore_full_run_prefix_fences(serde_json::from_value(beyond).unwrap(), 128, &budget) + .unwrap_err(), + "temporal_checkpoint_fence_bounds" + ); + let mut wrong_next = value.clone(); + wrong_next["next"] = serde_json::json!(129); + assert_eq!( + make(fixture::context()) + .restore_full_run_prefix_fences( + serde_json::from_value(wrong_next).unwrap(), + 129, + &budget + ) + .unwrap_err(), + "temporal_checkpoint_fence_bounds" + ); + let mut extra_source = value.clone(); + extra_source["watermarks"] + .as_array_mut() + .unwrap() + .push(serde_json::json!([4, 1, fixture::START])); + assert_eq!( + make(fixture::context()) + .restore_full_run_prefix_fences( + serde_json::from_value(extra_source).unwrap(), + 128, + &budget + ) + .unwrap_err(), + "temporal_checkpoint_fence_bounds" + ); + let mut context = fixture::context(); + context.room_digest = "different-room".into(); + assert_eq!( + make(context) + .restore_full_run_prefix_fences(serde_json::from_value(value).unwrap(), 128, &budget) + .unwrap_err(), + "temporal_full2_fence_policy_changed" + ); +} +#[test] +fn c4b_rejected_native_packet_adds_no_fence_and_terminal_state_cannot_accumulate() { + use crate::room_learner::temporal_acquisition::{ + AcquisitionPlan, ExperimentalPlanId, NativeWindowAssembler, ReceiverCadenceBand, + }; + let plan = AcquisitionPlan::build( + fixture::context(), + ExperimentalPlanId::Native2000msSlots8, + &(1..=3) + .map(|id| (id, ReceiverCadenceBand::Hz30To36)) + .collect::>(), + ) + .unwrap(); + let mut assembler = + NativeWindowAssembler::new(plan, "synthetic-rejection-fence".into()).unwrap(); + assembler + .begin_window(fixture::START, fixture::START + 6_000_000_000) + .unwrap(); + let p = packet(1, 1, fixture::START, RoomLabel::Empty); + assembler.ingest(&p).unwrap(); + let before = assembler.c4b_test_fence_counts(); + assert_eq!( + assembler.ingest(&p).unwrap_err(), + "acquisition_replayed_or_reversed" + ); + let after = assembler.c4b_test_fence_counts(); + assert_eq!(after.1, before.1); + assert_eq!(after.2, before.2); + assert!(after.3); + for sequence in 2..10 { + assert_eq!( + assembler + .ingest(&packet( + 1, + sequence, + fixture::START + u64::from(sequence) * fixture::PERIOD, + RoomLabel::Empty + )) + .unwrap_err(), + "acquisition_window_state" + ); + assert_eq!(assembler.c4b_test_fence_counts().1, before.1); + } +} + +fn runtime_custody( + rt: &RoomLearningRuntime, +) -> (usize, usize, bool, (usize, usize, usize, bool, bool)) { + let state = rt.inner.lock().unwrap(); + let Some(CaptureWorkflow::Temporal(c)) = &state.capture else { + panic!() + }; + c.core.as_ref().unwrap().c4b_test_custody() +} +#[test] +fn c4b_full2_abort_contract_is_denied_before_any_parent_ticket_operation() { + let directory = tempfile::tempdir().unwrap(); + let rt = fixture::runtime(directory.path(), fixture::START); + rt.configure_temporal_full_run( + &rt.mutation_guard().unwrap(), + fixture::context(), + "synthetic-full2-abort-denial".into(), + fixture::START, + ) + .unwrap(); + let now = fixture::settle_async_boundary(&rt, fixture::START); + let before = rt + .temporal_store() + .unwrap() + .load_temporal(now) + .unwrap() + .unwrap() + .token; + let result = { + let state = rt.inner.lock().unwrap(); + let Some(CaptureWorkflow::Temporal(c)) = &state.capture else { + panic!() + }; + c.core.as_ref().unwrap().c4b_test_abort_contract() + }; + assert_eq!( + result.unwrap_err(), + "temporal_full2_abort_to_parent_unsupported" + ); + assert_eq!( + rt.temporal_store() + .unwrap() + .load_temporal(now) + .unwrap() + .unwrap() + .token, + before + ); + rt.cancel_temporal_durability(&rt.mutation_guard().unwrap()) + .unwrap(); +} +#[test] +fn c4b_full2_one_inflight_one_tail_no_third_and_same_repair_adopts_queued_window() { + let directory = tempfile::tempdir().unwrap(); + let rt = Arc::new(fixture::runtime(directory.path(), fixture::START)); + rt.configure_temporal_full_run( + &rt.mutation_guard().unwrap(), + fixture::context(), + "synthetic-full2-one-tail".into(), + fixture::START, + ) + .unwrap(); + let now = fixture::settle_async_boundary(&rt, fixture::START); + let mut now = fixture::async_lead_end(&rt, now); + rt.tick_temporal_capture(&rt.mutation_guard().unwrap(), now) + .unwrap(); + let mut sequence = 0; + now = fixture::settle_observing_with_sources(&rt, now, &mut sequence, Duration::from_secs(5)); + assert_eq!(runtime_custody(&rt).0, 0, "initial segment acknowledgement must precede the first retained window"); + let writer = { + let state = rt.inner.lock().unwrap(); + let Some(CaptureWorkflow::Temporal(c)) = &state.capture else { + panic!() + }; + c.core.as_ref().unwrap().writer().unwrap() + }; + let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(1); + let (release_tx, release_rx) = std::sync::mpsc::sync_channel(1); + let mut gate = ReleaseGate(Some(release_tx)); + writer.lose_next_ack(); + writer.hold_next(entered_tx, release_rx); + let mut blocked = false; + for _ in 0..6000 { + sequence += 1; + for source in 1..=3 { + match rt.ingest_native(&packet(source, sequence, now, RoomLabel::Empty), now) { + Ok(()) => {} + Err(e) if e == "temporal_storage_busy_pause" => { + blocked = true; + break; + } + Err(e) => panic!("one-tail ingress {e}"), + } + } + now += fixture::PERIOD; + if blocked { + break; + } + } + entered_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + assert!(blocked); + let custody = runtime_custody(&rt); + assert_eq!(custody.0, 2, "one queued plusone nextconsumed"); + assert_eq!(custody.1, 1); + assert!(!custody.2); + assert_eq!(custody.3 .0, 2); + let frames = custody.3 .1; + for _ in 0..10 { + sequence += 1; + for source in 1..=3 { + rt.ingest_native(&packet(source, sequence, now, RoomLabel::Empty), now) + .unwrap(); + } + now += fixture::PERIOD; + } + assert_eq!( + runtime_custody(&rt).3 .1, + frames, + "blockedprimingmustnotcreateathirdwindoworframeIDs" + ); + gate.release(); + let wait = Instant::now(); + let anchor = now; + loop { + now = anchor + wait.elapsed().as_nanos() as u64; + match rt.tick_temporal_capture(&rt.mutation_guard().unwrap(), now) { + Err(e) if e == "synthetic_lost_publish_ack" => break, + Err(e) => panic!("one-tail waiting {e}"), + Ok(()) => {} + } + assert!(wait.elapsed() < Duration::from_secs(30)); + std::thread::sleep(Duration::from_millis(1)); + } + let stored = rt + .temporal_store() + .unwrap() + .load_temporal(now) + .unwrap() + .unwrap(); + assert_eq!(stored.prefix.unwrap().window_count(), 1); + rt.repair_temporal_capture(&rt.mutation_guard().unwrap(), now) + .unwrap(); + now = fixture::settle_async_boundary(&rt, now); + assert_eq!(fixture::snapshot(&rt, now)["durable_windows"], 1); + let repaired = runtime_custody(&rt); + assert_eq!(repaired.0, 1); + assert_eq!(repaired.3 .0, 1); + assert_eq!( + repaired.3 .1, frames, + "SAMEadoptionmustpreservethetailreplayIDswithoutretainingtailraw" + ); + rt.cancel_temporal_durability(&rt.mutation_guard().unwrap()) + .unwrap(); +} + +struct ReleaseGate(Option>); +impl ReleaseGate { + fn release(&mut self) { + if let Some(tx) = self.0.take() { + let _ = tx.send(()); + } + } +} +impl Drop for ReleaseGate { + fn drop(&mut self) { + self.release(); + } +} + + +// Cache saved synthetic bytes ONLY. Every user of this fixture makes a new +// isolated copy and obtains actual current AEAD/model/owner recertification. +// No Runtime, token, producer/write capability or authority is cached. +fn complete_saved_copy(root:&Path)->u64{ + assert!(!cfg!(debug_assertions), "actual native full-run fixture requires Release profile; classify every cache consumer explicitly"); + static COMPLETE: std::sync::OnceLock<(tempfile::TempDir,u64)>=std::sync::OnceLock::new(); + let (saved,at)=COMPLETE.get_or_init(||std::thread::spawn(||{ + let directory=tempfile::tempdir().unwrap(); + let runtime=tokio::runtime::Builder::new_multi_thread().worker_threads(2).enable_all().build().unwrap(); + let now=runtime.block_on(produce_complete_uncached(directory.path())); + (directory,now) + }).join().unwrap()); + copy_private(saved.path(),root);at+1_000_000_000 +} +#[test] +#[cfg_attr(debug_assertions, ignore = "requires Release: actual native full-run fixture and current-store fault matrix")] +fn c4b_actual_native_train_frozen_eight_heldout_two_cold_interruptions_complete(){ + let d=tempfile::tempdir().unwrap();let now=complete_saved_copy(d.path()); + let rt=fixture::runtime(d.path(),now); + rt.restore_temporal_full_run(&rt.mutation_guard().unwrap(),&fixture::context(),now).unwrap(); + assert_eq!(fixture::snapshot(&rt,now)["stage"],"full_complete"); + assert_eq!(fixture::snapshot(&rt,now)["durable_windows"],128); + rt.cancel_temporal_durability(&rt.mutation_guard().unwrap()).unwrap(); +} +#[test] +#[cfg_attr(debug_assertions, ignore = "requires Release: actual native full-run fixture and current-store fault matrix")] +fn c4b_actual_complete_rehashed_role_phase_binding_floor_and_deadline_negatives(){ + let d=tempfile::tempdir().unwrap();let now=complete_saved_copy(d.path());let rt=fixture::runtime(d.path(),now); + let store=rt.temporal_store().unwrap();let loaded=store.load_temporal(now).unwrap().unwrap();let prefix=loaded.prefix.unwrap(); + let report=crate::room_learner::temporal_model_input::managed::execution::checkpoint::c4b_complete_fault_matrix(prefix.bytes(),prefix.descriptor(),now).unwrap(); + println!("SYNTHETIC_C4B_COMPLETE_STRUCTURAL_FAULTS={report}"); + assert!(store.prepare_temporal(Some(&loaded.token),&prefix,now).err().unwrap().contains("producer_write_grant_required")); + assert_eq!(store.load_temporal(now).unwrap().unwrap().token,loaded.token); + assert!(rt.inner.lock().unwrap().capture.is_none()); + assert!(rt.inner.lock().unwrap().qualification.is_none());assert!(!rt.inner.lock().unwrap().selection.has_history()); +} +#[test] +#[cfg_attr(debug_assertions, ignore = "requires Release: actual native full-run fixture and current-store fault matrix")] +fn c4b_actual_complete_owner_cancel_and_retirement_deny_reactivation(){ + let d=tempfile::tempdir().unwrap();let now=complete_saved_copy(d.path());let rt=fixture::runtime(d.path(),now); + rt.restore_temporal_full_run(&rt.mutation_guard().unwrap(),&fixture::context(),now).unwrap(); + let store=rt.recovery_store.clone().unwrap();let token=store.load_temporal(now).unwrap().unwrap().token; + rt.cancel_temporal_durability(&rt.mutation_guard().unwrap()).unwrap(); + assert!(rt.start_temporal_phase(&rt.mutation_guard().unwrap(),now).is_err()); + assert!(rt.current_verified_frozen_model().ok().flatten().is_none()); + assert_eq!(store.load_temporal(now).unwrap().unwrap().token,token);drop(rt); + let later=now+1_000_000_000;let intent=store.prepare_retire_temporal(&token,later,false).unwrap(); + let retired=store.publish_temporal(&intent,None,later).unwrap();assert_ne!(retired,token); + assert!(store.load_temporal(later).unwrap().unwrap().prefix.is_none()); + let cold=fixture::runtime(d.path(),later); + assert!(cold.restore_temporal_full_run(&cold.mutation_guard().unwrap(),&fixture::context(),later).is_err()); + assert!(cold.current_verified_frozen_model().ok().flatten().is_none()); + assert!(cold.inner.lock().unwrap().qualification.is_none());assert!(!cold.inner.lock().unwrap().selection.has_history()); +} +fn find_exact_model(root:&Path,filename:&str)->Option{ + for entry in std::fs::read_dir(root).unwrap(){let p=entry.unwrap().path();if p.is_dir(){if let Some(v)=find_exact_model(&p,filename){return Some(v)}}else if p.file_name().unwrap()==filename{return Some(p)}}None +} +#[test] +#[cfg_attr(debug_assertions, ignore = "requires Release: actual native full-run fixture and current-store fault matrix")] +fn c4b_actual_complete_cold_restore_requires_model_owner_context_and_original_deadline(){ + for case in ["missing_model","corrupt_model","wrong_owner","changed_context","expired","clock_regression"]{ + let d=tempfile::tempdir().unwrap();let mut now=complete_saved_copy(d.path());let rt=fixture::runtime(d.path(),now); + let store=rt.temporal_store().unwrap();let loaded=store.load_temporal(now).unwrap().unwrap();let prefix=loaded.prefix.unwrap();let original=prefix.descriptor().clone(); + let value:serde_json::Value=serde_json::from_slice(prefix.bytes()).unwrap();let sha=value["payload"]["stage"]["fit"]["body"]["artifact_sha256"].as_str().unwrap(); + if case=="missing_model"{std::fs::rename(find_exact_model(d.path(),&format!("{sha}.json")).unwrap(),d.path().join("SYNTHETIC-missing-model.json")).unwrap();} + if case=="corrupt_model"{std::fs::write(find_exact_model(d.path(),&format!("{sha}.json")).unwrap(),b"SYNTHETIC corrupt whole model").unwrap();} + let mut context=fixture::context();if case=="wrong_owner"{context.owner_scope="c".repeat(64);}if case=="changed_context"{context.room_digest="d".repeat(64);} + if case=="expired"{now=original.expires_at_ms*1_000_000;}if case=="clock_regression"{now=value["payload"]["common"]["last_now_ns"].as_u64().unwrap()-1;} + let error=rt.restore_temporal_full_run(&rt.mutation_guard().unwrap(),&context,now).err().expect(case); + println!("SYNTHETIC_C4B_COLD_DENIAL case={case} error={error}"); + assert!(rt.current_verified_frozen_model().ok().flatten().is_none());assert!(rt.inner.lock().unwrap().capture.as_ref().is_none_or(|c|matches!(c,CaptureWorkflow::Temporal(t) if t.core.is_none())));assert!(rt.inner.lock().unwrap().model.is_none());assert!(rt.inner.lock().unwrap().qualification.is_none());assert!(!rt.inner.lock().unwrap().selection.has_history()); + if case!="expired"&&case!="clock_regression"{assert_eq!(store.load_temporal(now).unwrap().unwrap().token,loaded.token);} + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_public.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_public.rs new file mode 100644 index 0000000000..1e1d779a23 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_public.rs @@ -0,0 +1,937 @@ +//! Versioned owner workflow over the existing tagged Runtime. No imported +//! JSON report/proof/Complete Boolean creates producer authority. +use super::evidence_control::EvidencePurpose; +use super::*; +use crate::room_learner::temporal_wire::*; +fn schema(actual: &str, expected: &str) -> Result<(), String> { + if actual != expected { + return Err("learner_unsupported_schema".into()); + } + Ok(()) +} +fn sha(s: &str) -> bool { + s.len() == 64 + && s.bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) +} +impl RoomLearningRuntime { + pub fn temporal_configure_v2( + &self, + expected: &MutationGuard, + context: CaptureContext, + run: String, + recovery_consent: bool, + now: u64, + ) -> Result<(), String> { + if !recovery_consent { + return Err("recovery_consent_required".into()); + } + if context.sources.len() != 3 { + return Err("learner_temporal_three_sources_required".into()); + } + let store = self.temporal_store_arc()?.clone(); + if !store.available(){return Err("recovery_unavailable".into());} + if store.inspect()?.is_some() { + return Err("learner_recovery_pending".into()); + } + self.configure_temporal_capture_with_custody_mode( + expected, + context, + run, + now, + true, + move |s, r, f, n, _, w| { + TemporalTrainCapture::prepare_full_run_async(s, r, f, n, store, w) + }, + ) + } + fn validate_profile_selection(&self,state:&Inner,profile:Option<&str>)->Result{ + let operator=match profile {Some(crate::room_learner::managed_schedule_contract::OPERATOR_EIGHT_PROFILE)=>true,Some("native-2s-three-source-56bin-option-a-engineering-v1")|None=>false,_=>return Err("learner_unsupported_profile".into())}; + if let Some(CaptureWorkflow::Temporal(c))=state.capture.as_ref(){ + if c.core.as_ref().is_none_or(|core|core.public_profile().profile_id.eq(crate::room_learner::managed_schedule_contract::OPERATOR_EIGHT_PROFILE)!=operator){return Err("temporal_previous_plan_loaded".into());} + } + Ok(operator) + } + pub fn temporal_select_profile_guarded_v2(&self,profile:Option<&str>,expected:&MutationGuard)->Result{ + let mut state=self.command_state(expected)?; + let operator=self.validate_profile_selection(&state,profile)?; + self.operator_profile_selected.store(operator,Ordering::Release); + self.operator_profile_initialized.store(true,Ordering::Release); + Ok(self.guard_for(&state)) + } + #[cfg(test)] + pub(super) fn temporal_select_profile_v2(&self,profile:Option<&str>)->Result<(),String>{ + let guard=self.mutation_guard()?;self.temporal_select_profile_guarded_v2(profile,&guard).map(|_|()) + } + pub fn temporal_default_operator_profile_v2(&self)->Result<(),String>{ + let mut state=self.inner.lock().map_err(|_|"learner state unavailable")?; + if state.capture.is_none()&&!self.operator_profile_initialized.load(Ordering::Acquire){ + state.workflow_revision=state.workflow_revision.checked_add(1).ok_or("learner command revision exhausted")?; + self.operator_profile_selected.store(true,Ordering::Release); + self.operator_profile_initialized.store(true,Ordering::Release); + } + Ok(()) + } + pub fn temporal_retry_v2(&self,request:TemporalRetryRequestV2,now:u64)->Result<(),String>{ + schema(&request.schema,RECOVERY_RETRY_REQUEST_SCHEMA)?; + if !request.retry_confirmed{return Err("temporal_retry_confirmation_required".into());} + if !sha(&request.capture_generation_id){return Err("learner_capture_identity_invalid".into());} + self.retry_failed_temporal_capture(&request.expected,&request.capture_generation_id,now) + } + pub fn temporal_begin_phase_v2( + &self, + request: TemporalPhaseStartRequestV2, + now: u64, + ) -> Result<(), String> { + schema(&request.schema, PHASE_START_REQUEST_SCHEMA)?; + if !request.conditions_confirmed { + return Err("learner_conditions_confirmation_required".into()); + } + self.start_temporal_phase_checked(&request.expected, now, |c| { + let core = c + .core + .as_ref() + .ok_or("temporal_initial_barrier_unconfirmed")?; + if !core.public_phase_matches( + &request.capture_generation_id, + &request.schedule_sha256, + request.episode_index as usize, + ) { + return Err("learner_phase_identity_changed".into()); + } + if core.stage() == TemporalCaptureStage::Frozen && c.training_proof.is_none() { + return Err("learner_training_proof_pending".into()); + } + Ok(()) + }) + } + pub fn temporal_finish_phase_v2( + &self, + request: TemporalPhaseFinishRequestV2, + now: u64, + ) -> Result<(), String> { + schema(&request.schema, PHASE_FINISH_REQUEST_SCHEMA)?; + if !request.conditions_confirmed { + return Err("learner_conditions_confirmation_required".into()); + } + self.confirm_temporal_phase_checked( + &request.expected, + &request.confirmation_challenge, + now, + |c| { + if c.core.as_ref().is_none_or(|core| { + !core.public_phase_matches( + &request.capture_generation_id, + &request.schedule_sha256, + request.episode_index as usize, + ) + }) { + return Err("learner_phase_identity_changed".into()); + } + Ok(()) + }, + ) + } + fn public_generation_matches(&self, id: &str) -> Result<(), String> { + if !sha(id) { + return Err("learner_capture_identity_invalid".into()); + } + let state = self.inner.lock().map_err(|_| "learner state unavailable")?; + if !matches!(state.capture.as_ref(),Some(CaptureWorkflow::Temporal(c))if c.core.as_ref().is_some_and(|core|core.is_full_run()&&core.public_generation_id()==id)) + { + return Err("learner_capture_identity_changed".into()); + } + Ok(()) + } + pub fn temporal_pause_phase_v2( + &self, + request: TemporalCurrentActionRequestV2, + now: u64, + ) -> Result<(), String> { + schema(&request.schema, PHASE_CANCEL_REQUEST_SCHEMA)?; + self.public_generation_matches(&request.capture_generation_id)?; + self.pause_temporal_capture(&request.expected, now) + } + pub fn temporal_cancel_operation_v2( + &self, + request: TemporalCurrentActionRequestV2, + _now: u64, + ) -> Result<(), String> { + schema(&request.schema, TRAIN_CANCEL_REQUEST_SCHEMA)?; + let (fit, evidence) = { + let mut state = self.command_state(&request.expected)?; + let phase = state.job.phase; + let cancel = state.job.cancel.clone(); + let Some(CaptureWorkflow::Temporal(c)) = state.capture.as_mut() else { + return Err(PRIVATE_KIND.into()); + }; + if c.core + .as_ref() + .is_none_or(|core| core.public_generation_id() != request.capture_generation_id) + { + return Err("learner_capture_identity_changed".into()); + } + if phase == JobPhase::Ready { + return Ok(()); + } + c.public_train_requested = false; + let fit = c + .fit_job + .as_ref() + .filter(|j| j.uses_cancel(&cancel)) + .cloned(); + let evidence = c + .evidence_job + .as_ref() + .filter(|j| j.uses_cancel(&cancel)) + .cloned(); + if let Some(j) = &fit { + j.cancel(); + } + if let Some(j) = &evidence { + j.cancel(); + } + if busy(&state.job) { + state.job.phase = JobPhase::Cancelling; + } + (fit, evidence) + }; + if let Some(j) = fit { + j.drain(); + } + if let Some(j) = evidence { + j.drain(); + } + Ok(()) + } + pub fn temporal_train_v2( + self: &Arc, + request: TemporalTrainRequestV2, + now: u64, + ) -> Result { + schema(&request.schema, TRAIN_REQUEST_SCHEMA)?; + self.public_generation_matches(&request.capture_generation_id)?; + let retry = { + let state = self.inner.lock().map_err(|_| "learner state unavailable")?; + let Some(CaptureWorkflow::Temporal(c)) = &state.capture else { + return Err(PRIVATE_KIND.into()); + }; + let core = c + .core + .as_ref() + .ok_or("temporal_initial_barrier_unconfirmed")?; + if core.public_candidate().is_some() { + if core.stage() == TemporalCaptureStage::FullComplete { + 4 + } else { + 3 + } + } else if c + .fit_job + .as_ref() + .is_some_and(|job| job.uncancelled_completed()) + && c.diagnostic_fit.is_some() + { + if core.stage() == TemporalCaptureStage::Blocked { + 2 + } else if core + .public_capture(&self.runtime_epoch, c.lead_in_until, now) + .durability_pending + { + 5 + } else { + 1 + } + } else { + 0 + } + }; + if retry >= 3 && retry <= 4 { + let attempt = self.start_temporal_evidence( + &request.expected, + now, + if retry == 4 { + EvidencePurpose::RestoreComplete + } else { + EvidencePurpose::TrainProof + }, + )?; + return Ok(attempt.to_string()); + } + if retry != 0 { + if retry == 2 { + self.repair_temporal_capture(&request.expected, now)?; + } else if retry == 5 { + self.tick_temporal_capture(&request.expected, now)?; + } else { + self.prepare_frozen_transition(&request.expected, now)?; + } + { + let mut state = self.inner.lock().map_err(|_| "learner state unavailable")?; + if let Some(CaptureWorkflow::Temporal(c)) = state.capture.as_mut() { + if c.failure.is_none() + && c.fit_job + .as_ref() + .is_some_and(|j| j.uncancelled_completed()) + && c.core.as_ref().is_some_and(|core| { + core.public_generation_id() == request.capture_generation_id + }) + { + c.public_train_requested = true; + } + } + } + self.spawn_temporal_training_progress(request.capture_generation_id); + return Ok(request + .expected + .workflow_revision + .saturating_add(1) + .to_string()); + } + let attempt = self.start_temporal_fit(&request.expected, now)?; + { + let mut state = self.inner.lock().map_err(|_| "learner state unavailable")?; + let cancel = state.job.cancel.clone(); + if let Some(CaptureWorkflow::Temporal(c)) = state.capture.as_mut() { + if c.core.as_ref().is_some_and(|core| { + core.public_generation_id() == request.capture_generation_id + }) && c + .fit_job + .as_ref() + .is_some_and(|j| j.attempt_matches(attempt, &cancel)) + { + c.public_train_requested = true; + } + } + } + self.spawn_temporal_training_progress(request.capture_generation_id); + Ok(attempt.to_string()) + } + fn spawn_temporal_training_progress(self: &Arc, id: String) { + let weak = Arc::downgrade(self); + tokio::spawn(async move { + loop { + let Some(runtime) = weak.upgrade() else { + return; + }; + let now = runtime.receiver_now_ns(); + let action = { + let state = match runtime.inner.lock() { + Ok(s) => s, + Err(_) => return, + }; + let Some(CaptureWorkflow::Temporal(c)) = state.capture.as_ref() else { + return; + }; + if !c.public_train_requested + || c.failure.is_some() + || c.core + .as_ref() + .is_none_or(|core| core.public_generation_id() != id) + || matches!(state.job.phase, JobPhase::Failed | JobPhase::Cancelled) + { + return; + } + if c.training_proof.is_some() { + return; + } + if c.core + .as_ref() + .unwrap() + .public_capture(&runtime.runtime_epoch, c.lead_in_until, now) + .durability_pending + { + 0 + } else { + match c.core.as_ref().unwrap().stage() { + TemporalCaptureStage::Complete + if state.job.phase == JobPhase::Ready + && c.fit_job.as_ref().is_some_and(|j| j.done()) => + { + 1 + } + TemporalCaptureStage::FullComplete if !busy(&state.job) => 3, + TemporalCaptureStage::Frozen + | TemporalCaptureStage::Paused + | TemporalCaptureStage::Ready + if !busy(&state.job) + && c.core.as_ref().unwrap().public_candidate().is_some() => + { + 2 + } + _ => 0, + } + } + }; + let result = (|| -> Result<(), String> { + if action == 1 { + runtime.prepare_frozen_transition(&runtime.mutation_guard()?, now)?; + } else if action == 2 { + runtime.start_temporal_evidence( + &runtime.mutation_guard()?, + now, + EvidencePurpose::TrainProof, + )?; + } else if action == 3 { + runtime.start_temporal_evidence( + &runtime.mutation_guard()?, + now, + EvidencePurpose::RestoreComplete, + )?; + } + // Only already-authorized post-Train ACK progress; no + // capture/configuration/ground-truth is started by GET. + let _ = runtime.tick_temporal_capture(&runtime.mutation_guard()?, now); + Ok(()) + })(); + if let Err(error) = result { + if error != "learner_state_changed" && error != "learner_job_busy" { + if let Ok(mut state) = runtime.inner.lock() { + state.input_error = Some(bounded(error)); + } + return; + } + } + drop(runtime); + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + }); + } + pub fn temporal_evaluate_v2( + self: &Arc, + request: TemporalEvaluateRequestV2, + now: u64, + ) -> Result { + schema(&request.schema, EVALUATE_REQUEST_SCHEMA)?; + self.public_generation_matches(&request.capture_generation_id)?; + { + let state = self.inner.lock().map_err(|_| "learner state unavailable")?; + let Some(CaptureWorkflow::Temporal(c)) = &state.capture else { + return Err(PRIVATE_KIND.into()); + }; + let proof = c + .training_proof + .as_ref() + .ok_or("learner_training_proof_pending")?; + let refs = c + .core + .as_ref() + .and_then(|core| core.public_complete_refs(&proof.sha256)) + .ok_or("temporal_actual_complete_required")?; + if refs != request.refs { + return Err("learner_evidence_identity_changed".into()); + } + } + self.start_temporal_evidence(&request.expected, now, EvidencePurpose::HeldOutEvaluation) + .map(|attempt| attempt.to_string()) + } + /// The permit must live inside the blocking read, not its cancellable HTTP + /// future: one stuck filesystem read cannot accumulate retry workers. + pub async fn acquire_temporal_status_reader(&self) -> Result { + self.status_reader.clone().acquire_owned().await.map_err(|_| "learner_status_reader_unavailable".into()) + } + pub fn temporal_status_v2(self: &Arc, now: u64) -> Result { + self.temporal_poll_status_v2(now)?; + self.temporal_status_snapshot_v2(now) + } + /// Caller serializes this clock-advancing poll with live ingress. No disk + /// discovery runs here; it belongs in the separately fenced snapshot. + pub fn temporal_poll_status_v2(self: &Arc, now: u64) -> Result<(), String> { + // ACKs may already be ready. This tick does not issue new consent, + // source samples, fit/proof/evaluation jobs or immutable metadata writes. + let temporal = { + let state = self.inner.lock().map_err(|_| "learner state unavailable")?; + matches!(state.capture, Some(CaptureWorkflow::Temporal(_))) + }; + if temporal { + let _ = self.tick_temporal_capture(&self.mutation_guard()?, now); + } + Ok(()) + } + /// Read-only recovery discovery may wait for a durable writer. Callers + /// must not hold the shared RF ingress lock across this operation. + pub fn temporal_status_snapshot_v2(self: &Arc, now: u64) -> Result { + let (generation,workflow_revision,operator_profile) = { + let state=self.inner.lock().map_err(|_| "learner state unavailable")?; + (state.generation,state.workflow_revision,self.operator_profile_selected.load(Ordering::Acquire)) + }; + let recovery_store=if operator_profile{self.operator_recovery_store.as_ref()}else{self.recovery_store.as_ref()}; + let recovery=recovery_store.and_then(|store|TemporalTrainCapture::recovery_discovery(store,now)); + #[cfg(test)] + if let Some(hook)=self.temporal_status_profile_hook.lock().unwrap().take(){hook();} + let mut response = { + let state = self.inner.lock().map_err(|_| "learner state unavailable")?; + if state.generation != generation || state.workflow_revision!=workflow_revision || self.operator_profile_selected.load(Ordering::Acquire)!=operator_profile { + return Err("learner_state_changed".into()); + } + let capture = match state.capture.as_ref() { + Some(CaptureWorkflow::Temporal(c)) => Some(c), + _ => None, + }; + let context = state.capture_context().cloned(); + let mut profile = if operator_profile{TemporalProfileWire::operator_eight_step()}else{TemporalProfileWire::engineering()}; + let mut wire_capture = None; + let mut candidate = None; + let mut train_proof = None; + let mut complete = None; + let mut evaluation = None; + let mut operation = None; + if let Some(c) = capture { + if let Some(core) = &c.core { + profile = core.public_profile(); + wire_capture = + Some(core.public_capture(&self.runtime_epoch, c.lead_in_until, now)); + candidate = core.public_candidate(); + train_proof = c.training_proof.as_ref().map(|p| p.sha256.clone()); + complete = train_proof + .as_ref() + .and_then(|sha| core.public_complete_refs(sha)); + } + if let Some(saved) = &c.evaluation { + evaluation = Some(evaluation_wire(saved)); + } + if let Some(job) = &c.evidence_job { + operation = Some(job.operation_name().into()); + } else if c.fit_job.is_some() { + operation = Some("train".into()); + } + } + TemporalLearnerStatusWire { + schema: STATUS_SCHEMA.into(), + enabled: true, + available_model_kinds: if self + .recovery_store + .as_ref() + .is_some_and(|s| s.available()) + { + vec![WholeTemporalKind::ManagedTemporalV3] + } else { + vec![] + }, + temporal_profile: profile, + workflow_kind: if capture.is_some() { + WorkflowKindWire::ManagedTemporalV3 + } else if state.capture.is_some() { + WorkflowKindWire::Legacy + } else { + WorkflowKindWire::Idle + }, + mutation_guard: self.guard_for(&state), + context_digest: context.as_ref().map(CaptureContext::digest), + room_digest: context.as_ref().map(|c| c.room_digest.clone()), + source_node_ids: context + .as_ref() + .map(|c| c.sources.iter().map(|s| s.source_id).collect()) + .unwrap_or_default(), + recovery, + capture: wire_capture, + candidate, + training_proof_sha256: train_proof, + complete, + evaluation, + assessment: capture.and_then(|c| { + c.assessment + .as_ref() + .map(|a| a.assessment.receipt.summary.clone()) + }), + model_selection: None, + job_state: if capture.is_some_and(|c| { + c.public_train_requested + && c.training_proof.is_none() + && c.failure.is_none() + && !matches!( + state.job.phase, + JobPhase::Failed | JobPhase::Cancelled | JobPhase::Cancelling + ) + }) { + "running".into() + } else { + serde_json::to_value(state.job.phase) + .unwrap() + .as_str() + .unwrap() + .into() + }, + job_operation: operation, + error_code: state + .job + .error + .as_ref() + .map(|_| "learner_operation_failed".into()), + error: state + .job + .error + .clone() + .or_else(|| state.input_error.clone()), + legacy_status: None, + } + }; + if response.workflow_kind != WorkflowKindWire::ManagedTemporalV3 { + response.legacy_status = Some(self.status_without_checkpoint(now)); + } + let selection = self.temporal_selection_v2(now, None, None)?; + response.model_selection = Some( + serde_json::from_value( + selection + .get("model_selection") + .cloned() + .ok_or("learner_selection_projection")?, + ) + .map_err(|_| "learner_selection_projection")?, + ); + serde_json::to_value(response).map_err(|_| "learner_status_encoding".into()) + } +} + +impl RoomLearningRuntime { + pub fn temporal_qualify_v2( + self: &Arc, + request: TemporalQualifyRequestV2, + now: u64, + ) -> Result { + schema(&request.schema, QUALIFY_REQUEST_SCHEMA)?; + self.public_generation_matches(&request.capture_generation_id)?; + { + let state = self.inner.lock().map_err(|_| "learner state unavailable")?; + let Some(CaptureWorkflow::Temporal(c)) = &state.capture else { + return Err(PRIVATE_KIND.into()); + }; + let saved = c + .evaluation + .as_ref() + .ok_or("temporal_evaluation_required")?; + if request.refs.evaluation_receipt_sha256 != saved.receipt().sha256 + || request.refs.held_out_evidence_sha256 != saved.held_out().sha256 + || request.refs.complete.training_proof_sha256 != saved.train().sha256 + || c.core + .as_ref() + .and_then(|core| core.public_complete_refs(&saved.train().sha256)) + .as_ref() + != Some(&request.refs.complete) + { + return Err("learner_evidence_identity_changed".into()); + } + } + self.start_temporal_evidence(&request.expected, now, EvidencePurpose::Qualification) + .map(|a| a.to_string()) + } + pub fn temporal_recovery_context_v2( + &self, + binding: &str, + revision: u64, + sources: &[u8], + now: u64, + ) -> Result { + let store = self.temporal_store()?; + let (context, _) = TemporalTrainCapture::saved_full_context(store, now)?; + let mut ids: Vec<_> = context.sources.iter().map(|s| s.source_id).collect(); + ids.sort(); + let mut supplied = sources.to_vec(); + supplied.sort(); + if context.owner_scope != self.owner + || context.installation_id != self.installation + || context.room_digest != binding + || context.revision != revision + || ids != supplied + { + return Err("recovery_context_mismatch".into()); + } + Ok(context) + } + pub fn temporal_restore_v2( + self: &Arc, + expected: &MutationGuard, + context: &CaptureContext, + capture_generation: &str, + now: u64, + ) -> Result<(), String> { + let (_, id) = TemporalTrainCapture::saved_full_context(self.temporal_store()?, now)?; + if id != capture_generation { + return Err("learner_capture_identity_changed".into()); + } + self.restore_temporal_full_run(expected, context, now)?; + let reconcile = { + let mut state = self.inner.lock().map_err(|_| "learner state unavailable")?; + if let Some(CaptureWorkflow::Temporal(c)) = state.capture.as_mut() { + let needed = c + .core + .as_ref() + .is_some_and(|core| core.public_candidate().is_some()); + c.public_train_requested = needed; + needed + } else { + false + } + }; + if reconcile { + self.spawn_temporal_training_progress(capture_generation.into()); + } + Ok(()) + } + pub fn temporal_discard_v2( + &self, + expected: &MutationGuard, + capture_generation: &str, + now: u64, + ) -> Result<(), String> { + let store = self.temporal_store()?; + if TemporalTrainCapture::retirement_generation(store, now)? != capture_generation { + return Err("learner_capture_identity_changed".into()); + } + // Stop/drain actual accepted work before the retirement commit. No + // late accepted write may recreate the retired recovery head. + let (writer, fit, evidence, generation) = { + let mut state = self.command_state(expected)?; + if let Some(CaptureWorkflow::Temporal(c)) = state.capture.as_mut() { + c.public_train_requested = false; + c.invalidate_owner(); + let writer = c.core.as_ref().and_then(|c| c.writer()); + if let Some(w) = &writer { + w.cancel(); + } + ( + writer, + c.fit_job.clone(), + c.evidence_job.clone(), + state.generation, + ) + } else { + (None, None, None, state.generation) + } + }; + if let Some(w) = writer { + w.drain()?; + } + if let Some(j) = fit { + j.drain(); + } + if let Some(j) = evidence { + j.drain(); + } + let inspection = store + .inspect_temporal_retirement(now)? + .ok_or("recovery_missing")?; + let intent = store.prepare_inspected_retirement(&inspection, now, false)?; + store.publish_temporal(&intent, None, now)?; + let retired = { + let mut state = self.inner.lock().map_err(|_| "learner state unavailable")?; + if state.generation != generation { + return Err("learner_state_changed".into()); + } + state.generation = state + .generation + .checked_add(1) + .ok_or("learner generation exhausted")?; + state.workflow_revision = state.workflow_revision.saturating_add(1); + state.job.phase = JobPhase::Idle; + state.input_error = None; + state.capture.take() + }; + drop(retired); + Ok(()) + } +} + +impl RoomLearningRuntime { + pub fn paired_temporal_progress_snapshot( + &self, + installation_id: &str, + boot_epoch: &str, + room: Option, + ready_ids: &[u8], + ready_context: Option<&str>, + now: u64, + ) -> Result { + const MAX_SAFE: u64 = 9_007_199_254_740_991; + let observed = now / 1_000_000; + if installation_id.is_empty() + || installation_id.len() > 128 + || installation_id.chars().any(char::is_control) + || boot_epoch.is_empty() + || boot_epoch.len() > 128 + || boot_epoch.chars().any(char::is_control) + || observed == 0 + || observed > MAX_SAFE - 10_000 + || room.as_ref().is_some_and(|r| { + !sha(&r.binding_digest) || r.revision == 0 || r.revision > MAX_SAFE + }) + { + return Err("progress_identity_or_time_unavailable".into()); + } + let mut result = TemporalPairedProgressWire { + schema: PAIRED_PROGRESS_SCHEMA.into(), + success: true, + installation_id: installation_id.into(), + runtime_epoch: boot_epoch.into(), + observed_at_unix_ms: observed, + expires_at_unix_ms: observed + 10_000, + room, + context_match: false, + stage: "setup".into(), + state: "idle".into(), + model_kind: None, + profile: None, + capture: None, + candidate: None, + evaluation: None, + qualification: None, + selection: None, + sources: None, + save_status: "not_reported".into(), + blocker: Some("room_not_published".into()), + }; + { + let state = self.inner.lock().map_err(|_| "learner state unavailable")?; + let context = state.capture_context().or_else(|| { + state + .selection + .temporal + .as_ref() + .map(|s| &s.stored.envelope().receipt.selected.context) + }); + if let (Some(published), Some(context)) = (result.room.as_ref(), context) { + // The full binding digest includes the monotonic publication revision. + // Context.revision is a separate learner structural fingerprint. + if installation_id != self.installation + || context.installation_id != installation_id + || context.room_digest != published.binding_digest + || state.selection.temporal.as_ref().is_some_and(|selected| { + let c = &selected.stored.envelope().receipt.selected.context; + c.room_digest != published.binding_digest + || c.digest() != context.digest() + }) + { + result.stage = "blocked".into(); + result.blocker = Some("context_mismatch".into()); + } else { + result.context_match = true; + result.model_kind = Some(WholeTemporalKind::ManagedTemporalV3); + result.state = serde_json::to_value(state.job.phase) + .unwrap() + .as_str() + .unwrap() + .into(); + if ready_context != Some(context.digest().as_str()) { + return Err("progress_context_changed".into()); + } + result.sources = Some(TemporalPairedSourcesWire { + required: context + .sources + .len() + .try_into() + .map_err(|_| "progress_source_count")?, + ready: context + .sources + .iter() + .filter(|s| ready_ids.contains(&s.source_id)) + .count() as u8, + }); + result.blocker = None; + if let Some(CaptureWorkflow::Temporal(c)) = &state.capture { + if let Some(core) = &c.core { + let capture = + core.public_capture(&self.runtime_epoch, c.lead_in_until, now); + result.stage = capture.stage.clone(); + result.profile = Some(core.public_profile()); + result.candidate = core.public_candidate(); + result.capture = Some(TemporalPairedCaptureProgressWire { + capture_generation_id: capture.capture_generation_id, + schedule_sha256: capture.schedule_sha256, + stage: capture.stage, + current_episode: capture.current_episode, + role: capture.role, + label: capture.label, + target_ms: capture.target_ms, + lead_in_remaining_ms: capture.lead_in_remaining_ms, + observed_ms: capture.observed_ms, + retained_windows: capture.retained_windows, + durable_windows: capture.durable_windows, + sealed_train_episodes: capture.sealed_train_episodes, + sealed_held_out_episodes: capture.sealed_held_out_episodes, + durability_pending: capture.durability_pending, + awaiting_ground_truth: capture.awaiting_ground_truth, + original_deadline: capture.original_deadline, + }); + } else { + result.stage = "preparing".into(); + } + if let Some(saved) = &c.evaluation { + result.evaluation = Some(evaluation_wire(saved)); + } + if let Some(a) = &c.assessment { + result.qualification = + a.qualification.as_ref().map(|q| q.receipt.summary.clone()); + } + } else if state.capture.is_some() { + result.stage = "unsupported_mixed_capture".into(); + result.blocker = Some("legacy_capture_with_temporal_selection".into()); + } else { + result.stage = "selected".into(); + } + } + } + } + if !result.context_match { + result.room = None; + return Ok(result); + } + let selection = self.temporal_selection_v2( + now, + result.room.as_ref().map(|r| r.binding_digest.as_str()), + Some(boot_epoch), + )?; + let summary: TemporalSelectionStatusWire = serde_json::from_value( + selection + .get("model_selection") + .cloned() + .ok_or("progress_selection_projection")?, + ) + .map_err(|_| "progress_selection_projection")?; + if summary.selected.is_some() { + result.model_kind = Some(WholeTemporalKind::ManagedTemporalV3); + if result.qualification.is_none() { + result.qualification = summary.qualification.clone(); + } + } + result.selection = Some(summary); + Ok(result) + } +} + +fn evaluation_wire( + saved: &crate::room_learner::artifact::StoredTemporalEvaluation, +) -> TemporalEvaluationWire { + let identity = &saved.receipt().body.identity; + let refs = TemporalEvaluationRefsWire { + complete: TemporalCompleteRefsWire { + model: TemporalModelIdentityWire { + model_kind: WholeTemporalKind::ManagedTemporalV3, + model_sha256: identity.artifact_sha256.clone(), + verified_fit_record_sha256: identity.verified_fit_record_sha256.clone(), + }, + training_proof_sha256: saved.train().sha256.clone(), + complete_record_sha256: saved.held_out().body.complete_record_sha256.clone(), + complete_commitment_sha256: saved + .held_out() + .body + .complete_commitment + .commitment_sha256 + .clone(), + }, + held_out_evidence_sha256: saved.held_out().sha256.clone(), + evaluation_receipt_sha256: saved.receipt().sha256.clone(), + }; + TemporalEvaluationWire { + refs, + window_unit: "native_receiver_2s_window".into(), + evaluated_at_unix_ms: saved.receipt().body.evaluated_at_unix_ms, + report: saved.receipt().body.report.clone(), + durable: true, + independence_verified: false, + physically_validated: false, + } +} + +#[cfg(all(test, unix))] +#[path = "lifecycle_temporal_public_acceptance_tests.rs"] +mod acceptance; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_public_acceptance_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_public_acceptance_tests.rs new file mode 100644 index 0000000000..2bb5924b52 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_public_acceptance_tests.rs @@ -0,0 +1,1233 @@ +//! SYNTHETIC production-default-profile software acceptance. No physical RF +//! or clock precision/generalization/cadence acceptance follows from this case. +use super::super::{held_out_tests as established, tests as fixture}; +use super::*; +use established::{ + assert_original_lifetime, copy_private, packet, phase_target, prefix_value, receiver_floor, + same_train_and_frozen, +}; +use std::{ + path::Path, + time::{Duration, Instant}, +}; +fn capture(rt: &RoomLearningRuntime, now: u64) -> TemporalCaptureWire { + let state = rt.inner.lock().unwrap(); + let Some(CaptureWorkflow::Temporal(c)) = &state.capture else { + panic!("actualTemporalcapturemissing") + }; + c.core + .as_ref() + .unwrap() + .public_capture(&rt.runtime_epoch, c.lead_in_until, now) +} +fn public_lead_end(rt: &Arc, at: u64) -> u64 { + let c = capture(rt, at); + rt.temporal_begin_phase_v2( + TemporalPhaseStartRequestV2 { + schema: PHASE_START_REQUEST_SCHEMA.into(), + expected: rt.mutation_guard().unwrap(), + capture_generation_id: c.capture_generation_id, + schedule_sha256: c.schedule_sha256, + episode_index: c.current_episode.unwrap(), + conditions_confirmed: true, + }, + at, + ) + .unwrap(); + let start = Instant::now(); + loop { + let now = at + start.elapsed().as_nanos() as u64; + rt.tick_temporal_capture(&rt.mutation_guard().unwrap(), now) + .unwrap(); + let state = rt.inner.lock().unwrap(); + let Some(CaptureWorkflow::Temporal(c)) = &state.capture else { + panic!() + }; + if let Some(end) = c.lead_in_until { + return end; + } + drop(state); + assert!(start.elapsed() < Duration::from_secs(30)); + std::thread::sleep(Duration::from_millis(1)); + } +} +fn public_finish(rt: &Arc, guard: MutationGuard, challenge: String, now: u64) { + let c = capture(rt, now); + rt.temporal_finish_phase_v2( + TemporalPhaseFinishRequestV2 { + schema: PHASE_FINISH_REQUEST_SCHEMA.into(), + expected: guard, + capture_generation_id: c.capture_generation_id, + schedule_sha256: c.schedule_sha256, + episode_index: c.current_episode.unwrap(), + confirmation_challenge: challenge, + conditions_confirmed: true, + }, + now, + ) + .unwrap(); +} +async fn metadata_ready(rt: &RoomLearningRuntime) { + let start = Instant::now(); + loop { + let state = rt.inner.lock().unwrap(); + let Some(CaptureWorkflow::Temporal(c)) = &state.capture else { + panic!() + }; + if state.job.phase == JobPhase::Ready + && c.training_proof.is_some() + && c.evidence_job.as_ref().is_some_and(|j| j.done()) + { + return; + } + assert!( + !matches!(state.job.phase, JobPhase::Failed | JobPhase::Cancelled), + "actualmetadatajobfailed {:?}", + state.job.error + ); + drop(state); + assert!( + start.elapsed() < Duration::from_secs(180), + "actualmetadatajobwatchdog" + ); + tokio::time::sleep(Duration::from_millis(5)).await; + } +} +async fn restore_public(rt: &Arc, now: u64) { + let status = rt.temporal_status_v2(now).unwrap(); + let id = status["recovery"]["capture_generation_id"] + .as_str() + .unwrap(); + rt.temporal_restore_v2(&rt.mutation_guard().unwrap(), &fixture::context(), id, now) + .unwrap(); + if rt.current_verified_frozen_model().unwrap().is_some() { + metadata_ready(rt).await; + } +} +fn record_train_default48(root: &Path) -> (Arc, u64) { + let runtime = Arc::new(fixture::runtime(root, fixture::START)); + runtime + .temporal_configure_v2( + &runtime.mutation_guard().unwrap(), + fixture::context(), + "synthetic-c4c-production-default48".into(), + true, + fixture::START, + ) + .unwrap(); + let mut now = fixture::settle_async_boundary(&runtime, fixture::START); + let schedule = ManagedScheduleContract::build( + fixture::context(), + LeadInPolicy::Train5HeldOut15EngineeringV1, + ) + .unwrap(); + let mut sequence = 0; + let mut starts = 0; + for episode in schedule + .episodes() + .iter() + .filter(|e| e.role == DataRole::Train) + { + loop { + // A durable terminal pause can be confirmable without resuming. + if runtime + .temporal_confirmation_challenge(&runtime.mutation_guard().unwrap()) + .is_ok() + { + break; + } + let progress = fixture::snapshot(&runtime, now); + match progress["stage"].as_str().unwrap() { + "ready" | "paused" => { + now = public_lead_end(&runtime, now); + starts += 1; + runtime + .tick_temporal_capture(&runtime.mutation_guard().unwrap(), now) + .unwrap(); + } + "needs_confirmation" => break, + "failed" => panic!("synthetic C4B phase{} failed: {progress}", episode.index), + _ => {} + } + let pending = fixture::snapshot(&runtime, now)["durability_pending"] == true; + sequence += 1; + for source in 1..=3 { + match runtime.ingest_native(&packet(source, sequence, now, episode.condition), now) + { + Ok(()) => {} + Err(error) if error == "temporal_storage_busy_pause" => break, + Err(error) => panic!("synthetic C4B ingest: {error}"), + } + } + now += fixture::PERIOD; + if pending { + std::thread::sleep(Duration::from_nanos(fixture::PERIOD)); + } + assert!(starts < 120, "synthetic C4B interruption bound exhausted"); + } + let guard = runtime.mutation_guard().unwrap(); + let challenge = runtime.temporal_confirmation_challenge(&guard).unwrap(); + public_finish(&runtime, guard, challenge, now); + assert_eq!(fixture::snapshot(&runtime, now)["durability_pending"], true); + now = fixture::settle_async_boundary(&runtime, now); + } + let complete = fixture::snapshot(&runtime, now); + assert_eq!(complete["stage"], "complete"); + assert_eq!( + complete["sealed_train_episodes"], + schedule.counts().train_episodes + ); + assert_eq!(complete["durable_windows"], schedule.counts().train_windows); + println!( + "SYNTHETIC_C4B_REAL_TRAIN_CAPTURE: rows={} seals={} starts={starts}", + schedule.counts().train_windows, + schedule.counts().train_episodes + ); + (runtime, now) +} + +async fn produce_default48_complete(directory: &Path) -> u64 { + let (rt, mut now) = record_train_default48(directory); + // Fresh actual cold Runtime uses its authenticated producer floor as this + // explicitly synthetic clock-domain anchor; no live clock clamp/rescue. + drop(rt); + now += 1_000_000_000; + let mut rt = Arc::new(fixture::runtime(directory, now)); + restore_public(&rt, now).await; + let generation = capture(&rt, rt.receiver_now_ns()).capture_generation_id; + rt.temporal_train_v2( + TemporalTrainRequestV2 { + schema: TRAIN_REQUEST_SCHEMA.into(), + expected: rt.mutation_guard().unwrap(), + capture_generation_id: generation, + }, + rt.receiver_now_ns(), + ) + .unwrap(); + metadata_ready(&rt).await; + now = rt.receiver_now_ns(); + let fit = rt.temporal_diagnostic_fit().unwrap().unwrap().2; + assert_eq!( + serde_json::to_value(&fit.summary.as_ref().unwrap().metrics.configured).unwrap(), + serde_json::to_value(FitConfig::for_source_count(3)).unwrap() + ); + assert_eq!( + fit.summary.as_ref().unwrap().metrics.completed_epochs, + 48, + "production default profile was not executed" + ); + println!( + "SYNTHETIC_C4C_ACTUAL_DEFAULT48_FIT={}", + serde_json::to_string(&fit.summary.as_ref().unwrap().metrics).unwrap() + ); + now = fixture::settle_async_boundary(&rt, now); + assert_eq!(fixture::snapshot(&rt, now)["stage"], "frozen"); + drive_remaining_held_out(directory, rt, now).await +} +async fn drive_remaining_held_out( + directory: &Path, + mut rt: Arc, + mut now: u64, +) -> u64 { + let model = rt.current_verified_frozen_model().unwrap().unwrap(); + let model_bytes = model.export().unwrap(); + let model_sha = { + use sha2::Digest; + format!("{:x}", sha2::Sha256::digest(&model_bytes)) + }; + let existing = rt.store.temporal_fit_diagnostics(&model); + assert_eq!( + existing.summary.as_ref().unwrap().metrics.completed_epochs, + 48 + ); + assert_eq!( + serde_json::to_value(&existing.summary.as_ref().unwrap().metrics.configured).unwrap(), + serde_json::to_value(FitConfig::for_source_count(3)).unwrap() + ); + let report_before = serde_json::to_value(rt.store.temporal_fit_diagnostics(&model)).unwrap(); + now = fixture::settle_async_boundary(&rt, now); + let initial = capture(&rt, now); + assert!(matches!(initial.stage.as_str(), "frozen" | "paused")); + assert!((64..128).contains(&initial.durable_windows)); + let frozen = prefix_value(&rt, now); + assert_eq!( + frozen["payload"]["stage"]["fit"]["body"]["configured_fit_sha256"], + crate::room_learner::qualification::hash(&FitConfig::for_source_count(3)).unwrap() + ); + let original = frozen["payload"]["common"]["descriptor"].clone(); + let schedule = ManagedScheduleContract::build( + fixture::context(), + LeadInPolicy::Train5HeldOut15EngineeringV1, + ) + .unwrap(); + let mut sequence = receiver_floor(&rt); + let mut partial_cold = initial.durable_windows > 64; + let mut terminal_cold = false; + let mut starts = 0; + let original_proof = { + let state = rt.inner.lock().unwrap(); + let Some(CaptureWorkflow::Temporal(c)) = &state.capture else { + panic!() + }; + c.training_proof.as_ref().unwrap().sha256.clone() + }; + let mut observing_refused = false; + let mut pending_refused = false; + let mut train_overlap_refused = false; + if initial.durable_windows > 64 { + let before_rows = initial.durable_windows; + let forged = rt.temporal_begin_phase_v2( + TemporalPhaseStartRequestV2 { + schema: PHASE_START_REQUEST_SCHEMA.into(), + expected: rt.mutation_guard().unwrap(), + capture_generation_id: initial.capture_generation_id.clone(), + schedule_sha256: initial.schedule_sha256.clone(), + episode_index: 0, + conditions_confirmed: true, + }, + now, + ); + assert_eq!(forged.unwrap_err(), "learner_phase_identity_changed"); + assert_eq!(capture(&rt, now).durable_windows, before_rows); + train_overlap_refused = true; + println!("SYNTHETIC_C4C_COLD_CONTINUATION_BASELINE rows={} episode={} train_seals=7 fit_epochs=48 same_model={} proof={}",initial.durable_windows,initial.current_episode.unwrap(),model_sha,original_proof); + } + + for episode in schedule.episodes().iter().filter(|e| { + e.role == DataRole::HeldOut && e.index >= initial.current_episode.unwrap_or(15) as usize + }) { + loop { + let challenge = rt.temporal_confirmation_challenge(&rt.mutation_guard().unwrap()); + if challenge.is_ok() { + if episode.index == 14 && !terminal_cold { + let before = prefix_value(&rt, now); + let rows = fixture::snapshot(&rt, now)["durable_windows"].clone(); + drop(rt); + now += 1_000_000_000; + rt = Arc::new(fixture::runtime(directory, now)); + restore_public(&rt, now).await; + now = rt.receiver_now_ns(); + assert_eq!(fixture::snapshot(&rt, now)["durable_windows"], rows); + same_accepted_prefix(&frozen, &prefix_value(&rt, now)); + same_train_and_frozen(&frozen, &prefix_value(&rt, now)); + assert_eq!( + before["payload"]["accepted"], + prefix_value(&rt, now)["payload"]["accepted"] + ); + terminal_cold = true; + continue; + } + break; + } + let progress = fixture::snapshot(&rt, now); + assert_eq!( + progress["label"], + serde_json::to_value(episode.condition).unwrap() + ); + assert_eq!(progress["role"], "HeldOut"); + assert_eq!(progress["target_ms"], 30_000); + match progress["stage"].as_str().unwrap() { + "frozen" | "ready" | "paused" => { + now = public_lead_end(&rt, now); + starts += 1; + if episode.index == 7 && starts == 1 { + now += 50_000_000; + } + rt.tick_temporal_capture(&rt.mutation_guard().unwrap(), now) + .unwrap(); + } + "failed" => panic!("HeldOut failed: {progress}"), + _ => {} + } + let pending = fixture::snapshot(&rt, now)["durability_pending"] == true; + if !observing_refused + && !pending + && fixture::snapshot(&rt, now)["stage"] == "observing" + && phase_target(&rt).is_some() + { + let state = rt.inner.lock().unwrap(); + let Some(CaptureWorkflow::Temporal(c)) = &state.capture else { + panic!() + }; + assert!( + c.core + .as_ref() + .unwrap() + .prepare_evaluation_source(now, false) + .is_err(), + "observingcannotmintstableTrainprojection" + ); + observing_refused = true; + } + sequence += 1; + for source in 1..=3 { + match rt.ingest_native(&packet(source, sequence, now, episode.condition), now) { + Ok(()) => {} + Err(e) if e == "temporal_storage_busy_pause" => break, + Err(e) => panic!("HeldOut ingress: {e}"), + } + } + now += fixture::PERIOD; + if pending { + std::thread::sleep(Duration::from_nanos(fixture::PERIOD)); + } + let progress = fixture::snapshot(&rt, now); + if !partial_cold + && episode.index == 7 + && progress["durable_windows"].as_u64().unwrap() >= 67 + && progress["durability_pending"] == false + && phase_target(&rt) + .is_some_and(|(start, end)| now >= start + 3 * fixture::PERIOD && now < end) + { + let durable = progress["durable_windows"].clone(); + let before = prefix_value(&rt, now); + drop(rt); + now += 1_000_000_000; + rt = Arc::new(fixture::runtime(directory, now)); + restore_public(&rt, now).await; + now = rt.receiver_now_ns(); + { + let state = rt.inner.lock().unwrap(); + let Some(CaptureWorkflow::Temporal(c)) = &state.capture else { + panic!() + }; + assert_eq!( + c.training_proof.as_ref().unwrap().sha256, + original_proof, + "partialHeldOutRestoremustreconcileSAMETrainproof" + ); + } + let current = capture(&rt, now); + let before_rejected_train = fixture::snapshot(&rt, now)["durable_windows"].clone(); + let forged = rt.temporal_begin_phase_v2( + TemporalPhaseStartRequestV2 { + schema: PHASE_START_REQUEST_SCHEMA.into(), + expected: rt.mutation_guard().unwrap(), + capture_generation_id: current.capture_generation_id, + schedule_sha256: current.schedule_sha256, + episode_index: 0, + conditions_confirmed: true, + }, + now, + ); + assert_eq!(forged.unwrap_err(), "learner_phase_identity_changed"); + assert_eq!( + fixture::snapshot(&rt, now)["durable_windows"], + before_rejected_train + ); + train_overlap_refused = true; + assert_eq!(fixture::snapshot(&rt, now)["durable_windows"], durable); + assert!(phase_target(&rt).is_none()); + assert_eq!( + before["payload"]["accepted"], + prefix_value(&rt, now)["payload"]["accepted"] + ); + same_accepted_prefix(&frozen, &prefix_value(&rt, now)); + same_train_and_frozen(&frozen, &prefix_value(&rt, now)); + partial_cold = true; + } + assert!( + starts < 300, + "bounded synthetic HeldOut interruption budget exhausted" + ); + } + let guard = rt.mutation_guard().unwrap(); + let challenge = rt.temporal_confirmation_challenge(&guard).unwrap(); + public_finish(&rt, guard, challenge, now); + { + let state = rt.inner.lock().unwrap(); + let Some(CaptureWorkflow::Temporal(c)) = &state.capture else { + panic!() + }; + assert!( + c.core + .as_ref() + .unwrap() + .public_capture(&rt.runtime_epoch, c.lead_in_until, now) + .durability_pending + ); + assert!( + c.core + .as_ref() + .unwrap() + .prepare_evaluation_source(now, false) + .is_err(), + "pendingdurabilitycannotmintstableTrainprojection" + ); + pending_refused = true; + } + now = fixture::settle_async_boundary(&rt, now); + assert_eq!( + fixture::snapshot(&rt, now)["sealed_held_out_episodes"], + episode.index - 6 + ); + let sealed = prefix_value(&rt, now); + same_accepted_prefix(&frozen, &sealed); + same_train_and_frozen(&frozen, &sealed); + println!( + "SYNTHETIC_C4B_HELDOUT_SEAL episode={} durable={} sealed={} starts={starts}", + episode.index, + fixture::snapshot(&rt, now)["durable_windows"], + fixture::snapshot(&rt, now)["sealed_held_out_episodes"] + ); + } + assert!( + partial_cold && terminal_cold, + "two actual cold interruptions must execute" + ); + assert!( + observing_refused && pending_refused && train_overlap_refused, + "actualprotectedsource/phaseadmissioncontrolsnotexecuted" + ); + println!("SYNTHETIC_C4C_ACTUAL_SOURCE_REFUSALS observing=true pending=true forged_train_episode=true same_stable_train_proof=true"); + + let status = fixture::snapshot(&rt, now); + assert_eq!(status["stage"], "full_complete"); + assert_eq!(status["durable_windows"], 128); + assert_eq!(status["sealed_train_episodes"], 7); + assert_eq!(status["sealed_held_out_episodes"], 8); + let complete = prefix_value(&rt, now); + same_accepted_prefix(&frozen, &complete); + same_train_and_frozen(&frozen, &complete); + assert_eq!(complete["payload"]["stage"]["kind"], "complete"); + assert_eq!( + rt.current_verified_frozen_model() + .unwrap() + .unwrap() + .export() + .unwrap(), + model_bytes + ); + assert_eq!( + serde_json::to_value(rt.store.temporal_fit_diagnostics(&model)).unwrap(), + report_before + ); + assert!(rt.inner.lock().unwrap().model.is_none()); + assert!(rt.inner.lock().unwrap().qualification.is_none()); + assert!(!rt.inner.lock().unwrap().selection.has_history()); + let loaded = rt + .temporal_store() + .unwrap() + .load_temporal(now) + .unwrap() + .unwrap(); + let final_prefix = loaded.prefix.unwrap(); + let bytes = final_prefix.bytes().len(); + let raw_values = complete["payload"]["accepted"]["rows"] + .as_array() + .unwrap() + .iter() + .map(|row| { + row["views"] + .as_array() + .unwrap() + .iter() + .map(|view| { + view["samples"] + .as_array() + .unwrap() + .iter() + .map(|sample| sample.as_array().unwrap().len()) + .sum::() + }) + .sum::() + }) + .sum::(); + assert!(raw_values <= 2_000_000); + fn slot_sizes(root: &Path, out: &mut Vec) { + for entry in std::fs::read_dir(root).unwrap() { + let path = entry.unwrap().path(); + if path.is_dir() { + slot_sizes(&path, out) + } else if path + .file_name() + .unwrap() + .to_string_lossy() + .starts_with("slot-") + && path.extension().is_some_and(|e| e == "bin") + { + out.push(std::fs::metadata(path).unwrap().len()); + } + } + } + let mut outer_slot_sizes = vec![]; + slot_sizes(directory, &mut outer_slot_sizes); + outer_slot_sizes.sort_unstable(); + println!( + "SYNTHETIC_C4B_RAW_NUMERIC_VALUES={raw_values} AEAD_SLOT_FILE_BYTES={outer_slot_sizes:?}" + ); + assert!(bytes <= crate::room_learner::recovery_store::MAX_RECOVERY_BYTES); + println!( + "SYNTHETIC_C4B_FULL128_ACTUAL_JSON_BYTES={bytes} STARTS={starts} GLOBAL_GAPS={}", + complete["payload"]["common"]["ledger"]["gaps"] + .as_array() + .unwrap() + .len() + ); + drop(rt); + now += 1_000_000_000; + let cold = Arc::new(fixture::runtime(directory, now)); + restore_public(&cold, now).await; + now = cold.receiver_now_ns(); + assert_eq!(fixture::snapshot(&cold, now)["stage"], "full_complete"); + assert_eq!( + cold.current_verified_frozen_model() + .unwrap() + .unwrap() + .export() + .unwrap(), + model_bytes + ); + assert_original_lifetime(&fixture::snapshot(&cold, now)["descriptor"], &original); + assert!(cold + .start_temporal_phase(&cold.mutation_guard().unwrap(), now) + .is_err()); + assert_eq!( + cold.store + .load_exact_temporal_candidate(&fixture::context(), &model_sha) + .unwrap() + .export() + .unwrap(), + model_bytes + ); + cold.cancel_temporal_durability(&cold.mutation_guard().unwrap()) + .unwrap(); + now +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[cfg_attr( + debug_assertions, + ignore = "actual native default-profile workflow runs in mandatory Temporal Release batch" +)] +async fn c4c_default48_public_operator_workflow_native128_and_cold_selection() { + let directory = tempfile::Builder::new() + .prefix("SYNTHETIC-c4c-default48-") + .tempdir() + .unwrap() + .keep(); + std::fs::write(directory.join("SYNTHETIC-OWNED-DEFAULT48-RUN.json"),br#"{"real_user_data":false,"physical_accuracy_claim":false,"profile":"production-default48","retention":"owned_test_evidence"}"#).unwrap(); + let configured = FitConfig::for_source_count(3); + assert_eq!(configured.epochs, 48); + println!( + "SYNTHETIC_C4C_ACTUAL_CONFIG={}", + serde_json::to_string(&configured).unwrap() + ); + let now = produce_default48_complete(&directory).await + 1_000_000_000; + finish_public_checks(&directory, now).await; +} +async fn finish_public_checks(directory: &Path, now: u64) { + let rt = Arc::new(fixture::runtime(directory, now)); + restore_public(&rt, now).await; + let status = rt.temporal_status_v2(rt.receiver_now_ns()).unwrap(); + let c: TemporalCaptureWire = serde_json::from_value(status["capture"].clone()).unwrap(); + let complete: TemporalCompleteRefsWire = + serde_json::from_value(status["complete"].clone()).unwrap(); + assert!(status["evaluation"].is_null()); + rt.temporal_evaluate_v2( + TemporalEvaluateRequestV2 { + schema: EVALUATE_REQUEST_SCHEMA.into(), + expected: rt.mutation_guard().unwrap(), + capture_generation_id: c.capture_generation_id.clone(), + refs: complete, + }, + rt.receiver_now_ns(), + ) + .unwrap(); + metadata_ready(&rt).await; + let status = rt.temporal_status_v2(rt.receiver_now_ns()).unwrap(); + let evaluation: TemporalEvaluationWire = + serde_json::from_value(status["evaluation"].clone()).unwrap(); + assert_eq!(evaluation.report.samples, 64); + assert_eq!(evaluation.report.declared_trial_count, 8); + assert!(evaluation + .report + .per_condition + .iter() + .all(|c| c.samples == 16 && c.declared_trial_count == 2)); + rt.temporal_qualify_v2( + TemporalQualifyRequestV2 { + schema: QUALIFY_REQUEST_SCHEMA.into(), + expected: rt.mutation_guard().unwrap(), + capture_generation_id: c.capture_generation_id.clone(), + refs: evaluation.refs.clone(), + }, + rt.receiver_now_ns(), + ) + .unwrap(); + metadata_ready(&rt).await; + let status = rt.temporal_status_v2(rt.receiver_now_ns()).unwrap(); + println!( + "SYNTHETIC_C4C_DEFAULT48_ACTUAL_EVALUATION_AND_ASSESSMENT={}", + serde_json::to_string( + &json!({"evaluation":status["evaluation"],"assessment":status["assessment"]}) + ) + .unwrap() + ); + let q = status["assessment"]["qualification_sha256"].as_str(); + if q.is_none() { + assert_eq!(status["assessment"]["outcome"], "not_qualified"); + assert!(!rt.inner.lock().unwrap().selection.has_history()); + let before = rt.store.load_selection().unwrap(); + let denied = rt.temporal_select_v2( + TemporalSelectRequestV2 { + schema: SELECT_REQUEST_SCHEMA.into(), + expected: rt.mutation_guard().unwrap(), + capture_generation_id: c.capture_generation_id.clone(), + refs: TemporalQualificationRefsWire { + evaluation: evaluation.refs.clone(), + qualification_sha256: "0".repeat(64), + }, + }, + rt.receiver_now_ns(), + ); + assert_eq!(denied.unwrap_err(), "temporal_candidate_not_qualified"); + assert_eq!( + rt.store.load_selection().unwrap(), + before, + "refusedSelectmustnotchangeprotectedpointerorhistory" + ); + assert!(!rt.inner.lock().unwrap().selection.has_history()); + panic!("production-default48 actual fixture refused qualification; positiveSelect remainsOPEN; preservedprofile={}",directory.display()); + } + let safety_preselect = tempfile::Builder::new() + .prefix("SYNTHETIC-c4c-safety-preselect-") + .tempdir() + .unwrap() + .keep(); + copy_private(directory, &safety_preselect); + let selection_request = TemporalSelectRequestV2 { + schema: SELECT_REQUEST_SCHEMA.into(), + expected: rt.mutation_guard().unwrap(), + capture_generation_id: c.capture_generation_id, + refs: TemporalQualificationRefsWire { + evaluation: evaluation.refs, + qualification_sha256: q.unwrap().into(), + }, + }; + let selected = rt + .temporal_select_v2(selection_request.clone(), rt.receiver_now_ns()) + .unwrap(); + assert_eq!(selected.as_object().unwrap().len(), 3); + assert_eq!( + selected["model_selection"]["schema"], + SELECTION_STATUS_SCHEMA + ); + let identity: TemporalSelectedIdentityWire = + serde_json::from_value(selected["model_selection"]["selected"].clone()).unwrap(); + assert_eq!(identity.model_kind, WholeTemporalKind::ManagedTemporalV3); + assert_ne!( + selected["model_selection"]["current"]["status"], "ready", + "historicalHeldOutisnotliveinput" + ); + assert!( + rt.deselect_model(&rt.mutation_guard().unwrap()).is_err(), + "cold/wholekindcannotuselegacywithdrawal" + ); + println!( + "SYNTHETIC_C4C_DEFAULT48_SELECTED={}", + serde_json::to_string(&identity).unwrap() + ); + exercise_live_and_cold_withdrawal(&rt, directory, &identity, &selection_request).await; + drop(rt); + safety::run(&safety_preselect, directory).await; +} + +#[derive(Debug, Default)] +struct LiveTiming { + packets: u64, + min_gap_ns: Option, + max_gap_ns: u64, + below_min: u64, + above_max: u64, + policy_min_ns: u64, + policy_max_ns: u64, +} +struct LiveFeed { + stop: Arc, + handle: Option>, + error: Arc>>, + timing: Arc>, +} +impl Drop for LiveFeed { + fn drop(&mut self) { + self.stop.store(true, Ordering::Release); + if let Some(handle) = self.handle.take() { + handle.join().unwrap(); + } + } +} +fn live_feed(rt: &Arc) -> LiveFeed { + let selected = rt.inner.lock().unwrap().selection.temporal.clone().unwrap(); + let mut sequence = selected + .driver + .lock() + .unwrap() + .c4c_replay_floor_probe() + .iter() + .map(|f| f.1) + .max() + .unwrap(); + let policy = selected.stored.model().window_input_policy().unwrap(); + let min_gap = policy + .source_policies() + .iter() + .map(|p| p.min_packet_interval_ns) + .max() + .unwrap(); + let max_gap = policy + .source_policies() + .iter() + .map(|p| p.max_gap_ns) + .min() + .unwrap(); + let previous = selected + .driver + .lock() + .unwrap() + .c4c_replay_floor_probe() + .iter() + .map(|f| f.2) + .max() + .unwrap(); + let timing = Arc::new(Mutex::new(LiveTiming { + policy_min_ns: min_gap, + policy_max_ns: max_gap, + ..LiveTiming::default() + })); + let stats = timing.clone(); + let stop = Arc::new(AtomicBool::new(false)); + let flag = stop.clone(); + let error = Arc::new(Mutex::new(None)); + let failure = error.clone(); + let weak = Arc::downgrade(rt); + let handle = std::thread::spawn(move || { + let origin = Instant::now(); + let mut ordinal = 0u32; + let mut previous = previous; + let mut next_minimum = origin + Duration::from_nanos(min_gap); + while !flag.load(Ordering::Acquire) { + // Schedule real emissions, never rewrite packet time. A missed + // nominal deadline cannot produce a catch-up burst below the + // unchanged source policy's minimum packet interval. + let target = (origin + Duration::from_nanos(u64::from(ordinal) * fixture::PERIOD)) + .max(next_minimum); + // This finite SOFTWARE stimulus owns one host thread. Sleeping + // timers may be coalesced beyond a30Hz receiver contract; wait + // against the actual monotonic deadline without forging time. + while Instant::now() < target && !flag.load(Ordering::Acquire) { + std::hint::spin_loop(); + } + if flag.load(Ordering::Acquire) { + break; + } + let Some(runtime) = weak.upgrade() else { break }; + sequence = sequence.checked_add(1).unwrap(); + let now = runtime.receiver_now_ns(); + next_minimum = Instant::now() + Duration::from_nanos(min_gap); + { + let gap = now - previous; + let mut st = stats.lock().unwrap(); + st.packets += 1; + st.min_gap_ns = Some(st.min_gap_ns.map_or(gap, |v| v.min(gap))); + st.max_gap_ns = st.max_gap_ns.max(gap); + st.below_min += u64::from(gap < min_gap); + st.above_max += u64::from(gap > max_gap); + } + previous = now; + for source in 1..=3 { + if !runtime.selects_grid( + source, + &fixture::context() + .sources + .iter() + .find(|s| s.source_id == source) + .unwrap() + .grid_id, + ) { + *failure.lock().unwrap() = Some("actualcoldselectedgridnotadmitted".into()); + return; + } + if let Err(e) = + runtime.ingest_native(&packet(source, sequence, now, RoomLabel::Empty), now) + { + *failure.lock().unwrap() = Some(e); + return; + } + } + drop(runtime); + ordinal = ordinal.checked_add(1).unwrap(); + } + }); + LiveFeed { + stop, + handle: Some(handle), + error, + timing, + } +} +async fn current_ready(rt: &RoomLearningRuntime, feed: &LiveFeed) -> Value { + let start = Instant::now(); + loop { + let error = feed.error.lock().unwrap().clone(); + assert!(error.is_none(), "actualsyntheticliveinputfailed {error:?}"); + let now = rt.receiver_now_ns(); + let frame = rt + .temporal_selected_binary_v2( + now, + Some(&fixture::context().room_digest), + Some("synthetic-c4c-public-boot"), + ) + .unwrap(); + if frame["status"] == "ready" { + println!( + "SYNTHETIC_C4C_ACTUAL_LIVE_EMISSION_TIMING={:?}", + feed.timing.lock().unwrap() + ); + return frame; + } + if start.elapsed() >= Duration::from_secs(12) { + let selected = rt.inner.lock().unwrap().selection.temporal.clone().unwrap(); + let error = selected + .driver + .lock() + .unwrap() + .c4c_last_input_failure() + .map(str::to_owned); + panic!("actualsyntheticselectedinputdidnotbecomeready input_error={error:?} timing={:?} {frame}",feed.timing.lock().unwrap()); + } + tokio::time::sleep(Duration::from_millis(10)).await; + } +} +async fn exercise_live_and_cold_withdrawal( + rt: &Arc, + directory: &Path, + identity: &TemporalSelectedIdentityWire, + selection_request: &TemporalSelectRequestV2, +) { + let feed = live_feed(rt); + let frame = current_ready(rt, &feed).await; + assert_eq!(frame["identity"], serde_json::to_value(identity).unwrap()); + assert_eq!(frame["schema"], SELECTED_BINARY_SCHEMA); + let first = frame["window_start_unix_ms"].as_u64().unwrap(); + let end = frame["window_end_unix_ms"].as_u64().unwrap(); + let expires = frame["valid_until_unix_ms"].as_u64().unwrap(); + assert!((2000..=2150).contains(&(end - first))); + assert_eq!(frame["observed_at_unix_ms"], end); + assert_eq!(expires, first + 5000); + let before_arc = rt.inner.lock().unwrap().selection.temporal.clone().unwrap(); + let before_floors = before_arc.driver.lock().unwrap().c4c_replay_floor_probe(); + let before_pointer = rt + .store + .load_temporal_selection() + .unwrap() + .unwrap() + .envelope() + .clone(); + let retried = rt + .temporal_select_v2( + TemporalSelectRequestV2 { + expected: rt.mutation_guard().unwrap(), + ..selection_request.clone() + }, + rt.receiver_now_ns(), + ) + .unwrap(); + assert_eq!( + retried["model_selection"]["selected"], + serde_json::to_value(identity).unwrap() + ); + let after_arc = rt.inner.lock().unwrap().selection.temporal.clone().unwrap(); + assert!( + Arc::ptr_eq(&before_arc, &after_arc), + "lostACKsamerateSelectresetdriver" + ); + assert_eq!( + *rt.store + .load_temporal_selection() + .unwrap() + .unwrap() + .envelope(), + before_pointer + ); + let after_floors = after_arc.driver.lock().unwrap().c4c_replay_floor_probe(); + for before in before_floors { + let after = after_floors.iter().find(|f| f.0 == before.0).unwrap(); + assert!(after.1 >= before.1 && after.2 >= before.2); + } + assert_eq!( + rt.temporal_selected_binary_v2( + rt.receiver_now_ns(), + Some(&fixture::context().room_digest), + Some("synthetic-c4c-public-boot") + ) + .unwrap()["status"], + "ready", + "idempotentSelectcaused2swarmup" + ); + let boot = frame["boot_epoch"].clone(); + let receiver = frame["receiver_epoch"].clone(); + for _ in 0..3 { + let proof = rt + .temporal_selection_v2( + rt.receiver_now_ns(), + Some(&fixture::context().room_digest), + Some("synthetic-c4c-public-boot"), + ) + .unwrap(); + assert_eq!(proof["model_selection"]["current"]["status"], "ready"); + assert_eq!( + proof["model_selection"]["current"]["identity"], + serde_json::to_value(identity).unwrap() + ); + assert_eq!(proof["model_selection"]["current"]["boot_epoch"], boot); + assert_eq!( + proof["model_selection"]["current"]["receiver_epoch"], + receiver + ); + } + let wrong = rt + .paired_temporal_progress_snapshot( + &fixture::context().installation_id, + "synthetic-c4c-public-boot", + Some(crate::room_learner::lifecycle::PublishedProgressRoom { + binding_digest: "c".repeat(64), + revision: fixture::context().revision, + }), + &[1, 2, 3], + Some(&fixture::context().digest()), + rt.receiver_now_ns(), + ) + .unwrap(); + let wrong = serde_json::to_value(wrong).unwrap(); + assert_eq!(wrong["context_match"], false); + for key in [ + "room", + "capture", + "candidate", + "profile", + "evaluation", + "qualification", + "selection", + "sources", + "model_kind", + ] { + assert!(wrong[key].is_null(), "pairedmismatchleaked{key}"); + } + println!( + "SYNTHETIC_C4C_ACTUAL_SELECTED_LIVE={}", + serde_json::to_string(&json!({"current":frame,"paired_mismatch":wrong})).unwrap() + ); + drop(feed); + let copy = tempfile::Builder::new() + .prefix("SYNTHETIC-c4c-selected-expired-") + .tempdir() + .unwrap() + .keep(); + copy_private(directory, ©); + // Already-selected cold operation intentionally has no capture. The + // protected admitted chain, not an expired raw reread, supplies its original + // retention deadline for this test's separate cold-copy clock domain. + let deadline = rt + .inner + .lock() + .unwrap() + .selection + .temporal + .as_ref() + .unwrap() + .stored + .held() + .body + .identity + .original_deadline + .expires_at_unix_ms; + fn deny_raw(root: &Path) { + use std::os::unix::fs::PermissionsExt; + for e in std::fs::read_dir(root).unwrap() { + let p = e.unwrap().path(); + if p.is_dir() { + deny_raw(&p) + } else if p + .file_name() + .unwrap() + .to_string_lossy() + .starts_with("slot-") + { + std::fs::set_permissions(p, std::fs::Permissions::from_mode(0o000)).unwrap(); + } + } + } + deny_raw(©); + let cold = Arc::new(fixture::runtime(©, deadline * 1_000_000 + 1)); + assert!(cold.inner.lock().unwrap().capture.is_none()); + assert!(cold.temporal_v2_state_present()); + let selection = cold + .temporal_selection_v2( + cold.receiver_now_ns(), + Some(&fixture::context().room_digest), + Some("synthetic-cold-boot"), + ) + .unwrap(); + assert_eq!( + selection["model_selection"]["selected"], + serde_json::to_value(identity).unwrap() + ); + assert_ne!(selection["model_selection"]["current"]["status"], "ready"); + assert_eq!(cold.temporal_progress_context(), Some(fixture::context())); + let cold_arc = cold + .inner + .lock() + .unwrap() + .selection + .temporal + .clone() + .unwrap(); + let cold_retry = cold + .temporal_select_v2( + TemporalSelectRequestV2 { + expected: cold.mutation_guard().unwrap(), + ..selection_request.clone() + }, + cold.receiver_now_ns(), + ) + .unwrap(); + assert_eq!( + cold_retry["model_selection"]["selected"], + serde_json::to_value(identity).unwrap() + ); + assert!(Arc::ptr_eq( + &cold_arc, + cold.inner + .lock() + .unwrap() + .selection + .temporal + .as_ref() + .unwrap() + )); + + // Actual fresh native input must reach the admitted model even when cold + // selected state has NO capture and the original raw slots are unreadable. + let cold_feed = live_feed(&cold); + let cold_frame = current_ready(&cold, &cold_feed).await; + assert_eq!( + cold_frame["identity"], + serde_json::to_value(identity).unwrap() + ); + drop(cold_feed); + tokio::time::sleep(Duration::from_millis(250)).await; + assert_ne!( + cold.temporal_selected_binary_v2( + cold.receiver_now_ns(), + Some(&fixture::context().room_digest), + Some("synthetic-cold-boot") + ) + .unwrap()["status"], + "ready" + ); + + assert!(cold + .deselect_model(&cold.mutation_guard().unwrap()) + .is_err()); + let off = cold + .temporal_deselect_v2( + TemporalDeselectRequestV2 { + schema: DESELECT_REQUEST_SCHEMA.into(), + model_kind: WholeTemporalKind::ManagedTemporalV3, + expected: cold.mutation_guard().unwrap(), + selection_sha256: identity.selection_sha256.clone(), + selection_revision: identity.selection_revision, + }, + cold.receiver_now_ns(), + ) + .unwrap(); + assert_eq!(off["model_selection"]["schema"], SELECTION_STATUS_SCHEMA); + assert_eq!( + off["model_selection"]["revision"], + identity.selection_revision + 1 + ); + assert!(off["model_selection"]["selected"].is_null()); + assert_eq!(off["model_selection"]["current"]["status"], "none"); + drop(cold); + let reread = fixture::runtime(©, deadline * 1_000_000 + 1_000_000_000); + assert!(!reread.temporal_v2_state_present()); + let pointer = reread.store.load_selection().unwrap().unwrap(); + assert!(pointer.receipt.selected.is_none()); + assert!(pointer.receipt.previous.is_none()); + assert_eq!(pointer.receipt.revision, identity.selection_revision + 1); + assert_eq!( + rt.store + .load_temporal_selection() + .unwrap() + .unwrap() + .identity(), + *identity, + "otherownedprofilewithdrawalmustnotchangeoriginalmodelselection" + ); +} + +fn same_accepted_prefix(before: &Value, after: &Value) { + let count = before["payload"]["accepted"]["rows"] + .as_array() + .unwrap() + .len(); + for field in ["rows", "evidence", "identities"] { + let a = &before["payload"]["accepted"][field].as_array().unwrap()[..count]; + let b = &after["payload"]["accepted"][field].as_array().unwrap()[..count]; + assert_eq!( + crate::room_learner::qualification::hash(&a).unwrap(), + crate::room_learner::qualification::hash(&b).unwrap(), + "originalacceptedprefixchanged{field}" + ); + } +} + +#[cfg(all(test, unix))] +#[path = "lifecycle_temporal_public_safety_tests.rs"] +mod safety; + +#[test] +fn explicit_stale_native_processing_coordinate_still_fails_closed() { + let temp = tempfile::Builder::new().prefix("SYNTHETIC-fullrun-primer-order-").tempdir().unwrap(); + let rt = Arc::new(fixture::runtime(temp.path(), fixture::START)); + let now = rt.receiver_now_ns(); + rt.temporal_configure_v2(&rt.mutation_guard().unwrap(), fixture::context(), "synthetic-primer-order".into(), true, now).unwrap(); + let wait = Instant::now(); + loop { + let status=rt.temporal_status_v2(rt.receiver_now_ns()).unwrap(); + if status["capture"]["stage"]=="ready" {break;} + assert!(wait.elapsed() std::path::PathBuf { + let dir = tempfile::Builder::new() + .prefix(&format!("SYNTHETIC-c4c-{name}-")) + .tempdir() + .unwrap() + .keep(); + copy_private(source, &dir); + dir +} +fn profile_root(root: &Path) -> std::path::PathBuf { + root.join(&fixture::context().owner_scope) + .join("SensingServer") +} +fn artifact_root(root: &Path) -> std::path::PathBuf { + profile_root(root).join("room-learner") +} +fn file_clock(root: &Path) -> u64 { + // Test clock candidate only. Actual owner/current AEAD cold restore and + // immutable metadata loaders must authenticate all data before admission. + fn visit(root: &Path, at: &mut u64) { + for e in std::fs::read_dir(root).unwrap() { + let p = e.unwrap().path(); + if p.is_dir() { + visit(&p, at) + } else { + let name = p.file_name().unwrap().to_string_lossy(); + if name.starts_with("slot-") && name.ends_with(".bin") { + let b = std::fs::read(&p).unwrap(); + let n = u32::from_be_bytes(b[8..12].try_into().unwrap()) as usize; + let v: Value = serde_json::from_slice(&b[12..12 + n]).unwrap(); + *at = (*at).max(v["receiver_ns"].as_u64().unwrap()); + } else if name.starts_with("evaluation-") { + let v: Value = serde_json::from_slice(&std::fs::read(&p).unwrap()).unwrap(); + *at = (*at).max( + v["body"]["evaluated_at_receiver_ns"] + .as_str() + .unwrap() + .parse() + .unwrap(), + ); + } else if name.starts_with("assessment-") { + let v: Value = serde_json::from_slice(&std::fs::read(&p).unwrap()).unwrap(); + *at = (*at).max( + v["receipt"]["summary"]["assessed_at_unix_ms"] + .as_u64() + .unwrap() + * 1_000_000, + ); + } else if name == "model-selection-v1.json" { + let v: Value = serde_json::from_slice(&std::fs::read(&p).unwrap()).unwrap(); + *at = (*at) + .max(v["receipt"]["selected_at_unix_ms"].as_u64().unwrap() * 1_000_000); + } + } + } + } + let mut at = 0; + visit(root, &mut at); + at + 1_000_000_000 +} +fn metadata_hashes(root: &Path) -> Vec<(String, String)> { + use sha2::Digest; + fn walk(base: &Path, path: &Path, out: &mut Vec<(String, String)>) { + for e in std::fs::read_dir(path).unwrap() { + let p = e.unwrap().path(); + if p.is_dir() { + walk(base, &p, out) + } else { + out.push(( + p.strip_prefix(base).unwrap().to_string_lossy().into(), + format!("{:x}", sha2::Sha256::digest(std::fs::read(p).unwrap())), + )); + } + } + } + let mut out = vec![]; + walk(root, root, &mut out); + out.sort(); + out +} +async fn terminal(rt: &RoomLearningRuntime) -> JobPhase { + let start = Instant::now(); + loop { + let state = rt.inner.lock().unwrap(); + let done = match state.capture.as_ref() { + Some(CaptureWorkflow::Temporal(c)) => c.evidence_job.as_ref().is_some_and(|j| j.done()), + _ => true, + }; + if done && !busy(&state.job) { + return state.job.phase; + } + drop(state); + assert!(start.elapsed() < Duration::from_secs(90)); + tokio::time::sleep(Duration::from_millis(2)).await; + } +} +async fn qualified(root: &Path) -> (Arc, TemporalSelectRequestV2) { + let now = file_clock(root); + let rt = Arc::new(fixture::runtime(root, now)); + restore_public(&rt, now).await; + let status = rt.temporal_status_v2(rt.receiver_now_ns()).unwrap(); + let c: TemporalCaptureWire = serde_json::from_value(status["capture"].clone()).unwrap(); + let e: TemporalEvaluationWire = serde_json::from_value(status["evaluation"].clone()).unwrap(); + rt.temporal_qualify_v2( + TemporalQualifyRequestV2 { + schema: QUALIFY_REQUEST_SCHEMA.into(), + expected: rt.mutation_guard().unwrap(), + capture_generation_id: c.capture_generation_id.clone(), + refs: e.refs.clone(), + }, + rt.receiver_now_ns(), + ) + .unwrap(); + assert_eq!(terminal(&rt).await, JobPhase::Ready); + let s = rt.temporal_status_v2(rt.receiver_now_ns()).unwrap(); + assert_eq!(s["assessment"]["outcome"], "qualified"); + let request = TemporalSelectRequestV2 { + schema: SELECT_REQUEST_SCHEMA.into(), + expected: rt.mutation_guard().unwrap(), + capture_generation_id: c.capture_generation_id, + refs: TemporalQualificationRefsWire { + evaluation: e.refs, + qualification_sha256: s["assessment"]["qualification_sha256"] + .as_str() + .unwrap() + .into(), + }, + }; + (rt, request) +} +fn selected(root: &Path) -> Arc { + let rt = Arc::new(fixture::runtime(root, file_clock(root))); + assert!(rt.inner.lock().unwrap().capture.is_none()); + assert!(rt.inner.lock().unwrap().selection.temporal.is_some()); + rt +} +fn deselect(rt: &RoomLearningRuntime) -> TemporalDeselectRequestV2 { + let identity = rt + .inner + .lock() + .unwrap() + .selection + .temporal + .as_ref() + .unwrap() + .stored + .identity(); + TemporalDeselectRequestV2 { + schema: DESELECT_REQUEST_SCHEMA.into(), + model_kind: WholeTemporalKind::ManagedTemporalV3, + expected: rt.mutation_guard().unwrap(), + selection_sha256: identity.selection_sha256, + selection_revision: identity.selection_revision, + } +} +struct Release { + state: Arc<(Mutex, Condvar)>, +} +impl Release { + fn new() -> Self { + Self { + state: Arc::new((Mutex::new(false), Condvar::new())), + } + } + fn open(&self) { + *self.state.0.lock().unwrap() = true; + self.state.1.notify_all(); + } +} +impl Drop for Release { + fn drop(&mut self) { + self.open(); + } +} + +pub(super) async fn run(preselect: &Path, selected_source: &Path) { + // Restore must never advertise future stored evaluation as Ready. The + // candidate clock is deliberately ONLY the older authenticated raw floor. + let root = branch(preselect, "future-evaluation"); + let mut raw = 0; + fn raw_clock(root: &Path, raw: &mut u64) { + for e in std::fs::read_dir(root).unwrap() { + let p = e.unwrap().path(); + if p.is_dir() { + raw_clock(&p, raw) + } else if p + .file_name() + .unwrap() + .to_string_lossy() + .starts_with("slot-") + { + let b = std::fs::read(p).unwrap(); + let n = u32::from_be_bytes(b[8..12].try_into().unwrap()) as usize; + let h: Value = serde_json::from_slice(&b[12..12 + n]).unwrap(); + *raw = (*raw).max(h["receiver_ns"].as_u64().unwrap()); + } + } + } + raw_clock(&root, &mut raw); + let rt = Arc::new(fixture::runtime(&root, raw + 1)); + let prior = metadata_hashes(&root); + let s = rt.temporal_status_v2(raw + 1).unwrap(); + let id = s["recovery"]["capture_generation_id"].as_str().unwrap(); + rt.temporal_restore_v2( + &rt.mutation_guard().unwrap(), + &fixture::context(), + id, + raw + 1, + ) + .unwrap(); + assert_eq!(terminal(&rt).await, JobPhase::Failed); + assert_eq!( + rt.inner.lock().unwrap().job.error.as_deref(), + Some("temporal_evaluation_stored_clock_reversed") + ); + assert_eq!(metadata_hashes(&root), prior); + drop(rt); + println!("C4C_ACTOR_SAFETY future_evaluation_refused=true original_bytes=true"); + + // The actual current source/job observes expiry AFTER staged file fsync, + // immediately before authoritative pointer rename. No pointer is installed. + for cancel in [false, true] { + let root = branch( + preselect, + if cancel { + "preinstall-cancel" + } else { + "preinstall-expiry" + }, + ); + let (rt, request) = qualified(&root).await; + let expiry = capture(&rt, rt.receiver_now_ns()) + .original_deadline + .expires_at_unix_ms + * 1_000_000; + let weak = Arc::downgrade(&rt); + let called = Arc::new(AtomicBool::new(false)); + let seen = called.clone(); + *rt.temporal_evidence_io_hook.lock().unwrap() = Some(Arc::new(move |stage| { + if stage == "selection_before_pointer_install" { + let owner = weak.upgrade().unwrap(); + let state = owner.inner.lock().unwrap(); + if cancel { + state.job.cancel.store(true, Ordering::Release) + } else { + let Some(CaptureWorkflow::Temporal(c)) = &state.capture else { + panic!() + }; + c.evidence_job.as_ref().unwrap().observe_now(expiry); + } + seen.store(true, Ordering::Release); + } + Ok(()) + })); + assert!(rt + .temporal_select_v2(request, rt.receiver_now_ns()) + .is_err()); + assert!(called.load(Ordering::Acquire)); + assert!(rt.inner.lock().unwrap().selection.temporal.is_none()); + assert!(!artifact_root(&root) + .join("model-selection-v1.json") + .exists()); + assert!(!std::fs::read_dir(artifact_root(&root)).unwrap().any(|e| e + .unwrap() + .file_name() + .to_string_lossy() + .starts_with("model-selection-v1.tmp-"))); + drop(rt); + } + println!("C4C_ACTOR_SAFETY staged_expiry_and_cancel_refused=true no_pointer_or_staging=true"); + + // A valid rename is the commit point even if the later directory barrier + // or ACK fails. RAM remains unavailable; cold trusted DAG reconciles it. + let root = branch(preselect, "select-after-rename"); + let (rt, request) = qualified(&root).await; + *rt.temporal_evidence_io_hook.lock().unwrap() = Some(Arc::new(|stage| { + if stage == "selection_after_pointer_rename" { + Err("synthetic_postrename_sync_failure".into()) + } else { + Ok(()) + } + })); + assert_eq!( + rt.temporal_select_v2(request, rt.receiver_now_ns()) + .unwrap_err(), + "temporal_selection_pointer_write_unknown" + ); + assert!(rt.inner.lock().unwrap().selection.temporal.is_none()); + assert!(rt.inner.lock().unwrap().selection.error.is_some()); + let committed = rt + .store + .load_temporal_selection() + .unwrap() + .unwrap() + .identity(); + drop(rt); + let cold = selected(&root); + assert_eq!( + cold.inner + .lock() + .unwrap() + .selection + .temporal + .as_ref() + .unwrap() + .stored + .identity(), + committed + ); + drop(cold); + println!("C4C_ACTOR_SAFETY select_postrename_unknown_cold_reconciled=true"); + + let root = branch(selected_source, "deselect-after-rename"); + let rt = selected(&root); + let identity = rt + .inner + .lock() + .unwrap() + .selection + .temporal + .as_ref() + .unwrap() + .stored + .identity(); + *rt.temporal_evidence_io_hook.lock().unwrap() = Some(Arc::new(|stage| { + if stage == "deselect_after_pointer_rename" { + Err("synthetic_postrename_sync_failure".into()) + } else { + Ok(()) + } + })); + assert_eq!( + rt.temporal_deselect_v2(deselect(&rt), rt.receiver_now_ns()) + .unwrap_err(), + "selection_pointer_write_unknown" + ); + assert!(rt.inner.lock().unwrap().selection.error.is_some()); + assert_ne!( + rt.temporal_selected_binary_v2( + rt.receiver_now_ns(), + Some(&fixture::context().room_digest), + Some("synthetic-safety-boot") + ) + .unwrap()["status"], + "ready" + ); + drop(rt); + let cold = Arc::new(fixture::runtime(&root, file_clock(&root))); + assert!(cold.inner.lock().unwrap().selection.temporal.is_none()); + assert!(cold + .store + .load_selection() + .unwrap() + .unwrap() + .receipt + .selected + .is_none()); + assert_eq!( + cold.store + .load_selection() + .unwrap() + .unwrap() + .receipt + .revision, + identity.selection_revision + 1 + ); + drop(cold); + println!("C4C_ACTOR_SAFETY deselect_postrename_unknown_cold_off=true"); + + // Real pending Select IO is drained before a simulated owner erasure ACK. + let root = branch(preselect, "owner-erase-drain"); + let (rt, request) = qualified(&root).await; + let release = Release::new(); + let held = release.state.clone(); + let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(1); + *rt.temporal_evidence_io_hook.lock().unwrap() = Some(Arc::new(move |stage| { + if stage == "selection_before_pointer_install" { + entered_tx.send(()).unwrap(); + let mut open = held.0.lock().unwrap(); + while !*open { + open = held.1.wait(open).unwrap(); + } + } + Ok(()) + })); + let worker = rt.clone(); + let operation = + std::thread::spawn(move || worker.temporal_select_v2(request, worker.receiver_now_ns())); + entered_rx.recv_timeout(Duration::from_secs(15)).unwrap(); + let owner = rt.clone(); + let (tx, rx) = std::sync::mpsc::sync_channel(1); + let cancellation = std::thread::spawn(move || { + let r = owner.cancel_temporal_durability(&owner.mutation_guard().unwrap()); + tx.send(r).unwrap(); + }); + assert!( + rx.recv_timeout(Duration::from_millis(100)).is_err(), + "eraseACKescapedpendingactualstore" + ); + release.open(); + assert!(operation.join().unwrap().is_err()); + rx.recv_timeout(Duration::from_secs(15)).unwrap().unwrap(); + cancellation.join().unwrap(); + drop(rt); + let protected = profile_root(&root); + std::fs::remove_dir_all(&protected).unwrap(); + tokio::time::sleep(Duration::from_millis(100)).await; + assert!(!protected.exists(), "latewriterrecreatederasedowner"); + println!("C4C_ACTOR_SAFETY owner_erase_waited_real_store_and_no_recreation=true"); + + // Learning of a corrupt current durable chain fences the SAME hot Arc; + // successful exact recertification restores only that committed choice. + let root = branch(selected_source, "hot-integrity"); + let rt = selected(&root); + let arc = rt.inner.lock().unwrap().selection.temporal.clone().unwrap(); + let identity = arc.stored.identity(); + let feed = live_feed(&rt); + current_ready(&rt, &feed).await; + fn model_file(root: &Path, sha: &str) -> Option { + for e in std::fs::read_dir(root).unwrap() { + let p = e.unwrap().path(); + if p.is_dir() { + if let Some(v) = model_file(&p, sha) { + return Some(v); + } + } else if p.file_name().unwrap().to_string_lossy() == format!("{sha}.json") { + return Some(p); + } + } + None + } + let file = model_file(&root, &identity.model_sha256).unwrap(); + let bytes = std::fs::read(&file).unwrap(); + std::fs::write(&file, b"corrupt-owned-model-control").unwrap(); + let snapshot = rt + .temporal_selection_v2( + rt.receiver_now_ns(), + Some(&fixture::context().room_digest), + Some("synthetic-safety-boot"), + ) + .unwrap(); + assert_eq!( + snapshot["model_selection"]["current"]["status"], + "unavailable" + ); + assert!(arc.invalid.load(Ordering::Acquire)); + assert_eq!( + rt.temporal_selected_binary_v2( + rt.receiver_now_ns(), + Some(&fixture::context().room_digest), + Some("synthetic-safety-boot") + ) + .unwrap()["status"], + "unavailable" + ); + std::fs::write(&file, bytes).unwrap(); + let snapshot = rt + .temporal_selection_v2( + rt.receiver_now_ns(), + Some(&fixture::context().room_digest), + Some("synthetic-safety-boot"), + ) + .unwrap(); + assert_eq!(snapshot["model_selection"]["current"]["status"], "ready"); + assert!(!arc.invalid.load(Ordering::Acquire)); + assert!(Arc::ptr_eq( + &arc, + rt.inner + .lock() + .unwrap() + .selection + .temporal + .as_ref() + .unwrap() + )); + drop(feed); + drop(rt); + println!("C4C_ACTOR_SAFETY deep_failure_fences_hot_arc_exact_recert_same_arc=true"); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_selection.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_selection.rs new file mode 100644 index 0000000000..6d8cadd949 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_selection.rs @@ -0,0 +1,544 @@ +//! Whole-kind selected state; same protected pointer/journal and live input. +use super::*; +use crate::room_learner::{ + artifact::StoredTemporalSelection, temporal_live_driver::ContinuousTemporalInput, + temporal_wire::*, +}; +pub(in crate::room_learner::lifecycle) struct TemporalSelectedRuntime { + pub(in crate::room_learner::lifecycle) stored: StoredTemporalSelection, + pub(in crate::room_learner::lifecycle) driver: Mutex, + pub(super) invalid: AtomicBool, +} +impl TemporalSelectedRuntime { + pub(in crate::room_learner::lifecycle) fn new( + stored: StoredTemporalSelection, + ) -> Result { + let driver = ContinuousTemporalInput::from_selected(&stored)?; + Ok(Self { + stored, + driver: Mutex::new(driver), + invalid: AtomicBool::new(false), + }) + } +} +impl RoomLearningRuntime { + pub fn temporal_progress_context(&self) -> Option { + let state = self.inner.lock().ok()?; + state.capture_context().cloned().or_else(|| { + state + .selection + .temporal + .as_ref() + .map(|s| s.stored.envelope().receipt.selected.context.clone()) + }) + } + pub fn temporal_v2_state_present(&self) -> bool { + self.inner.lock().map(|s|matches!(s.capture.as_ref(),Some(CaptureWorkflow::Temporal(c))if c.core.as_ref().is_some_and(|core|core.is_full_run()))||s.selection.temporal.is_some()).unwrap_or(true) + } + pub fn temporal_selection_v2( + &self, + now: u64, + expected_room_binding: Option<&str>, + boot_epoch: Option<&str>, + ) -> Result { + self.temporal_selection_snapshot(now, expected_room_binding, boot_epoch, true) + } + pub fn temporal_selected_binary_v2( + &self, + now: u64, + expected_room_binding: Option<&str>, + boot_epoch: Option<&str>, + ) -> Result { + let snapshot = + self.temporal_selection_snapshot(now, expected_room_binding, boot_epoch, false)?; + let mut current = snapshot + .pointer("/model_selection/current") + .cloned() + .ok_or("selected_projection_unavailable")?; + current + .as_object_mut() + .ok_or("selected_projection_unavailable")? + .insert( + "schema".into(), + Value::String(SELECTED_BINARY_SCHEMA.into()), + ); + Ok(current) + } + fn temporal_selection_snapshot( + &self, + now: u64, + expected_room_binding: Option<&str>, + boot_epoch: Option<&str>, + recertify: bool, + ) -> Result { + let (selected, assessment, integrity, revision) = { + let s = self.inner.lock().map_err(|_| "learner state unavailable")?; + ( + s.selection.temporal.clone(), + match s.capture.as_ref() { + Some(CaptureWorkflow::Temporal(c)) => c.assessment.clone(), + _ => None, + }, + s.selection.error.clone(), + s.selection.known_revision(), + ) + }; + let mut result = TemporalSelectionStatusWire { + schema: SELECTION_STATUS_SCHEMA.into(), + revision, + selected: None, + previous_available: false, + qualification: assessment + .as_ref() + .and_then(|a| a.qualification.as_ref().map(|q| q.receipt.summary.clone())), + qualification_error: assessment + .as_ref() + .filter(|a| a.qualification.is_none()) + .and_then(|a| a.assessment.receipt.summary.failure_reason.clone()), + integrity_error: integrity, + current: TemporalCurrentEvidenceWire::None, + }; + if let Some(selected) = selected { + let identity = selected.stored.identity(); + if expected_room_binding.is_some_and(|room| room != identity.room_digest) { + result.qualification = None; + result.qualification_error = Some("selected_room_context_mismatch".into()); + return Ok( + json!({"schema":SELECTION_SCHEMA,"success":true,"model_selection":result}), + ); + } + result.revision = identity.selection_revision; + result.selected = Some(identity.clone()); + result.qualification = Some(selected.stored.qualification().receipt.summary.clone()); + let unavailable = |reason: &str| TemporalCurrentEvidenceWire::Unavailable { + identity: Some(identity.clone()), + reason: bounded(reason.into()), + boot_epoch: boot_epoch.map(str::to_owned), + receiver_epoch: Some(self.runtime_epoch.clone()), + }; + let has_integrity_error = result.integrity_error.is_some(); + let check = (|| -> Result { + if has_integrity_error { + return Err("selected_integrity_unavailable".into()); + } + if expected_room_binding != Some(identity.room_digest.as_str()) { + return Err("selected_room_context_mismatch".into()); + } + let boot = boot_epoch + .filter(|s| !s.is_empty() && s.len() <= 128 && !s.chars().any(char::is_control)) + .ok_or("selected_boot_epoch_unavailable")?; + // Protected current pointer/journal and complete immutable DAG, + // not hashes returned by a status/JSON import. + if recertify { + let verified = self + .store + .load_temporal_selection() + .and_then(|current| { + current.ok_or("temporal_selection_pointer_missing".into()) + }) + .and_then(|current| { + if current.identity() == identity { + Ok(()) + } else { + Err("selected_pointer_changed".into()) + } + }); + match verified { + Ok(()) => { + let state = + self.inner.lock().map_err(|_| "learner state unavailable")?; + if state + .selection + .temporal + .as_ref() + .is_some_and(|s| Arc::ptr_eq(s, &selected)) + && state.selection.error.is_none() + && !assessment_blocks(&state, &identity) + { + selected.invalid.store(false, Ordering::Release); + } + } + Err(error) => { + let state = + self.inner.lock().map_err(|_| "learner state unavailable")?; + if state + .selection + .temporal + .as_ref() + .is_some_and(|s| Arc::ptr_eq(s, &selected)) + { + selected.invalid.store(true, Ordering::Release); + } + return Err(error); + } + } + } + if selected.invalid.load(Ordering::Acquire) { + return Err("selected_integrity_unavailable".into()); + } + if assessment.as_ref().is_some_and(|a| { + a.assessment.receipt.summary.model_sha256 == identity.model_sha256 + && (a.qualification.is_none() + || a.qualification + .as_ref() + .is_some_and(|q| q.receipt_sha256 != identity.qualification_sha256)) + }) { + return Err("newer_qualification_failed_or_requires_selection".into()); + } + let o = { + let mut driver = selected + .driver + .lock() + .map_err(|_| "selected_native_input_unavailable")?; + let current_now = self.receiver_now_ns(); + if current_now < now { + return Err("selected_receiver_clock_reversed".into()); + } + driver.selected_observation(current_now)? + }; + let probability = o + .prediction + .occupied_probability + .filter(|p| p.is_finite() && (0.0..=1.0).contains(p)) + .ok_or("selected_probability_unavailable")?; + let s = self.inner.lock().map_err(|_| "learner state unavailable")?; + if s.selection.error.is_some() + || selected.invalid.load(Ordering::Acquire) + || assessment_blocks(&s, &identity) + || !s + .selection + .temporal + .as_ref() + .is_some_and(|current| Arc::ptr_eq(current, &selected)) + { + return Err("selected_pointer_changed".into()); + } + if s.capture_context().is_some_and(|context| { + context != &selected.stored.envelope().receipt.selected.context + }) { + return Err("selected_source_context_changed".into()); + } + Ok(TemporalCurrentEvidenceWire::Ready { + identity: identity.clone(), + presence: o.prediction.decision + == crate::room_learner::model::PresenceDecision::Occupied, + decision: o.prediction.decision, + occupied_probability: probability, + observed_at_unix_ms: o.end_ns / 1_000_000, + observation_digest: o.digest, + boot_epoch: boot.into(), + receiver_epoch: self.runtime_epoch.clone(), + window_start_unix_ms: o.first_ns / 1_000_000, + window_end_unix_ms: o.end_ns / 1_000_000, + valid_until_unix_ms: o.expires_ns / 1_000_000, + source_observed_at_unix_ms: o.source_latest_ns / 1_000_000, + }) + })(); + result.current = check.unwrap_or_else(|error| unavailable(&error)); + if selected.invalid.load(Ordering::Acquire) && result.integrity_error.is_none() { + result.integrity_error = Some("selected_integrity_unavailable".into()); + } + } + Ok(json!({"schema":SELECTION_SCHEMA,"success":true,"model_selection":result})) + } + pub fn temporal_select_v2( + self: &Arc, + request: TemporalSelectRequestV2, + now: u64, + ) -> Result { + if request.schema != SELECT_REQUEST_SCHEMA { + return Err("learner_unsupported_schema".into()); + } + let prior = { + self.inner + .lock() + .map_err(|_| "learner state unavailable")? + .selection + .temporal + .clone() + }; + if let Some(selected) = prior { + let current = selected.stored.identity(); + let refs = &request.refs; + let complete = &refs.evaluation.complete; + let exact = current.model_kind == complete.model.model_kind + && current.model_sha256 == complete.model.model_sha256 + && current.verified_fit_record_sha256 == complete.model.verified_fit_record_sha256 + && current.training_proof_sha256 == complete.training_proof_sha256 + && current.complete_record_sha256 == complete.complete_record_sha256 + && selected + .stored + .held() + .body + .complete_commitment + .commitment_sha256 + == complete.complete_commitment_sha256 + && current.held_out_evidence_sha256 == refs.evaluation.held_out_evidence_sha256 + && current.evaluation_receipt_sha256 == refs.evaluation.evaluation_receipt_sha256 + && current.qualification_sha256 == refs.qualification_sha256; + if !exact { + return Err("selected_target_requires_explicit_deselect".into()); + } + if selected.stored.capture_generation_id()? != request.capture_generation_id { + return Err("learner_capture_identity_changed".into()); + } + { + let state = self.command_state(&request.expected)?; + if busy(&state.job) { + return Err("learner_job_busy".into()); + } + if !state + .selection + .temporal + .as_ref() + .is_some_and(|s| Arc::ptr_eq(s, &selected)) + { + return Err("selected_pointer_changed".into()); + } + if state.selection.error.is_some() + || selected.invalid.load(Ordering::Acquire) + || assessment_blocks(&state, ¤t) + { + return Err("selection_reconciliation_required".into()); + } + } + // Exact already-admitted choice, including a lost ACK or expired + // raw lease: no new adoption, IO, timestamp, revision or driver. + return self.temporal_selection_snapshot(self.receiver_now_ns(), None, None, false); + } + { + let s = self.inner.lock().map_err(|_| "learner state unavailable")?; + let Some(CaptureWorkflow::Temporal(c)) = &s.capture else { + return Err(PRIVATE_KIND.into()); + }; + if c.core + .as_ref() + .is_none_or(|core| core.public_generation_id() != request.capture_generation_id) + { + return Err("learner_capture_identity_changed".into()); + } + let a = c + .assessment + .as_ref() + .ok_or("temporal_qualification_required")?; + let q = a + .qualification + .as_ref() + .ok_or("temporal_candidate_not_qualified")?; + let e = &request.refs.evaluation; + if q.receipt_sha256 != request.refs.qualification_sha256 + || q.receipt.summary.evaluation_receipt_sha256 != e.evaluation_receipt_sha256 + || q.receipt.summary.held_out_evidence_sha256 != e.held_out_evidence_sha256 + || c.core + .as_ref() + .and_then(|core| { + core.public_complete_refs(&q.receipt.summary.training_proof_sha256) + }) + .as_ref() + != Some(&e.complete) + { + return Err("learner_qualification_identity_changed".into()); + } + } + self.run_temporal_selection(&request.expected, now)?; + // The actual operation already read back the durable DAG. Its drain + // signal has fired; ACK enrichment must not start another store read + // that could outlive an owner-erasure barrier. Poll/Save uses REST's + // separately deep-validated read path. + self.temporal_selection_snapshot(now, None, None, false) + } +} + +#[derive(Clone)] +pub(in crate::room_learner::lifecycle) struct TemporalWithdrawalJob { + lifetime: Arc, +} +impl TemporalWithdrawalJob { + pub(in crate::room_learner::lifecycle) fn cancel(&self) { + self.lifetime.cancel(); + } + pub(in crate::room_learner::lifecycle) fn drain(&self) { + self.lifetime.drain(); + } +} +struct WithdrawalCompletion { + runtime: std::sync::Weak, + job: TemporalWithdrawalJob, +} +impl Drop for WithdrawalCompletion { + fn drop(&mut self) { + let runtime = self.runtime.upgrade(); + if let Some(runtime) = &runtime { + let mut state = runtime.inner.lock().unwrap_or_else(|e| e.into_inner()); + if Arc::ptr_eq(&state.job.cancel, &self.job.lifetime.cancel) && busy(&state.job) { + state.job.phase = if self.job.lifetime.cancel.load(Ordering::Acquire) { + JobPhase::Cancelled + } else { + JobPhase::Failed + }; + state.job.error = Some("selection_withdrawal_unavailable".into()); + state.selection.error=Some("Selection withdrawal outcome unavailable; restart and verify the saved choice.".into()); + } + } + *self + .job + .lifetime + .completed + .lock() + .unwrap_or_else(|e| e.into_inner()) = true; + self.job.lifetime.completion.notify_all(); + drop(runtime); + } +} +impl RoomLearningRuntime { + pub fn temporal_deselect_v2( + self: &Arc, + request: TemporalDeselectRequestV2, + now: u64, + ) -> Result { + use std::sync::{atomic::AtomicU64, Condvar}; + use std::time::Instant; + if request.schema != DESELECT_REQUEST_SCHEMA { + return Err("learner_unsupported_schema".into()); + } + let retired_job; + let retired_withdrawal; + let (job, generation) = { + let mut state = self.command_state(&request.expected)?; + if busy(&state.job) { + return Err("learner_job_busy".into()); + } + let current = state + .selection + .temporal + .as_ref() + .ok_or("temporal_selection_missing")?; + let identity = current.stored.identity(); + if identity.selection_sha256 != request.selection_sha256 + || identity.selection_revision != request.selection_revision + { + return Err("selected_pointer_changed".into()); + } + let cancel = Arc::new(AtomicBool::new(false)); + let lifetime = Arc::new(super::fit_control::FitLifetime { + cancel: cancel.clone(), + publication: Mutex::new(()), + completed: Mutex::new(false), + completion: Condvar::new(), + anchor: now, + origin: Instant::now(), + observed: AtomicU64::new(now), + expires_at_ms: u64::MAX, + #[cfg(test)] + hook: self + .temporal_fit_hook + .lock() + .unwrap_or_else(|e| e.into_inner()) + .clone(), + #[cfg(test)] + io_hook: self + .temporal_evidence_io_hook + .lock() + .unwrap_or_else(|e| e.into_inner()) + .clone(), + }); + let job = TemporalWithdrawalJob { lifetime }; + state.selection.error=Some("Selection withdrawal pending; current evidence is unavailable until durable reconciliation.".into()); + retired_withdrawal = state.selection.withdrawal.replace(job.clone()); + retired_job = std::mem::replace( + &mut state.job, + Job { + phase: JobPhase::Running, + cancel, + report: None, + error: None, + }, + ); + (job, state.generation) + }; + drop(retired_job); + drop(retired_withdrawal); + let completion = WithdrawalCompletion { + runtime: Arc::downgrade(self), + job: job.clone(), + }; + let result = (|| -> Result { + let _publication = job + .lifetime + .publication + .lock() + .map_err(|_| "selection_publication_unavailable")?; + job.lifetime.allowed()?; + let envelope = self.store.withdraw_temporal_selection( + &request.selection_sha256, + request.selection_revision, + now, + &mut || { + job.lifetime.allowed()?; + Ok(()) + }, + &mut || { + job.lifetime.io("deselect_before_pointer_install")?; + job.lifetime.allowed()?; + Ok(()) + }, + &mut || job.lifetime.io("deselect_after_pointer_rename"), + )?; + job.lifetime.allowed()?; + Ok(envelope) + })(); + let mut retired_selected = None; + { + let mut state = self.inner.lock().map_err(|_| "learner state unavailable")?; + if state.generation != generation + || !Arc::ptr_eq(&state.job.cancel, &job.lifetime.cancel) + { + return Err("selection_withdrawal_not_adopted".into()); + } + match &result { + Ok(envelope) => { + retired_selected = state.selection.temporal.take(); + state + .selection + .install_negative_temporal_tombstone(envelope.clone()); + state.selection.error = None; + state.job.phase = JobPhase::Ready; + state.job.error = None; + } + Err(error) => { + state.selection.error=Some("Selection withdrawal outcome unavailable; reread the saved choice before continuing.".into()); + state.job.phase = if job.lifetime.cancel.load(Ordering::Acquire) { + JobPhase::Cancelled + } else { + JobPhase::Failed + }; + state.job.error = Some(bounded(error.clone())); + } + } + } + if let Some(selected) = retired_selected { + selected + .driver + .lock() + .map_err(|_| "selected_driver_unavailable")? + .deactivate(); + drop(selected); + } + drop(completion); + result?; + self.temporal_selection_v2(self.receiver_now_ns(), None, None) + } +} + +fn assessment_blocks(state: &Inner, identity: &TemporalSelectedIdentityWire) -> bool { + match state.capture.as_ref() { + Some(CaptureWorkflow::Temporal(c)) => c.assessment.as_ref().is_some_and(|a| { + a.assessment.receipt.summary.model_sha256 == identity.model_sha256 + && (a.qualification.is_none() + || a.qualification + .as_ref() + .is_some_and(|q| q.receipt_sha256 != identity.qualification_sha256)) + }), + _ => false, + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_tests.rs new file mode 100644 index 0000000000..f33bc3042c --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_tests.rs @@ -0,0 +1,1528 @@ +//! SYNTHETIC raw native inputs through the real Runtime, simulated receiver clocks. +//! No real RF, owner credentials, helper process, selected model or fit authority. +use super::*; +use crate::room_learner::{ + native::{canonicalize_esp32, NativePacketInput}, + observation::{CaptureProvenance, SourceSpec}, + recovery_store::RecoveryStore, +}; +use wifi_densepose_hardware::{Adr018Flags, PpduType}; +pub(super) const START: u64 = 1_000_000_000_000; +pub(super) const PERIOD: u64 = 1_000_000_000u64.div_ceil(33); +// Test-only supervision for asynchronous storage/authorization ACKs, including +// final-row waits while ingress continues; not observation or shipping budgets. +const QUIESCENT_ACK_TEST_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30); +pub(super) fn context() -> CaptureContext { + CaptureContext { + owner_scope: "a".repeat(64), + installation_id: "synthetic-runtime-temporal".into(), + room_digest: "b".repeat(64), + revision: 1, + sources: (1..=3) + .map(|source_id| SourceSpec { + source_id, + device_id: format!("esp32-node-{source_id}"), + grid_id: "ant1-bins64-ppdu0-usable-drop2-linear-v2".into(), + feature_count: 56, + signal_kind: ruview_unified::RfModality::WifiCsi, + center_frequency_hz: 2_437_000_000, + bandwidth_hz: 20_000_000, + link_id: format!("native-{source_id}"), + link_index: 0, + geometry: None, + }) + .collect(), + } +} +pub(super) fn runtime(root: &Path, now: u64) -> RoomLearningRuntime { + let c = context(); + let directory = root.join(&c.owner_scope).join("SensingServer"); + let store = RecoveryStore::new( + &directory, + &c.owner_scope, + &c.installation_id, + Some(&[7; 32]), + ) + .unwrap(); + RoomLearningRuntime::new_with_recovery_at( + &directory, + &c.owner_scope, + &c.installation_id, + store, + now, + ) + .unwrap() +} +fn configured(root: &Path) -> RoomLearningRuntime { + let rt = runtime(root, START); + // Original synchronous C1 reducer remains a reference/golden adapter. + rt.configure_temporal_capture_with( + &rt.mutation_guard().unwrap(), context(), "synthetic-runtime-train".into(), START, + TemporalTrainCapture::prepare_retained, + ) + .unwrap(); + rt +} +pub(super) fn packet(source: u8, seq: u32, at: u64) -> CanonicalPacket { + let iq = vec![8; 128]; + canonicalize_esp32(NativePacketInput { + node_id: source, + n_antennas: 1, + raw_bins: 64, + frequency_hz: 2_437_000_000, + ppdu_type: PpduType::HtLegacy, + sequence: seq, + received_at_ns: at, + receiver_sample_rate_hz: 33., + iq_bytes: &iq, + flags: Adr018Flags::default(), + provenance: CaptureProvenance::LiveReceiverObserved, + }) + .unwrap() +} +pub(super) fn snapshot(rt: &RoomLearningRuntime, now: u64) -> Value { + rt.temporal_capture_snapshot(now).unwrap()["capture"].clone() +} +fn start(rt: &RoomLearningRuntime, at: u64) -> u64 { + rt.start_temporal_phase(&rt.mutation_guard().unwrap(), at) + .unwrap(); + at + TRAIN_LEAD_IN_NS +} +fn feed(rt: &RoomLearningRuntime, mut at: u64, to: u64, seq: &mut u32) -> u64 { + while at <= to { + *seq += 1; + for source in 1..=3 { + rt.ingest_native(&packet(source, *seq, at), at).unwrap(); + } + at += PERIOD; + } + at +} +#[test] +fn private_temporal_setup_uses_real_native_runtime_and_public_configure_cannot_replace_it() { + let dir = tempfile::tempdir().unwrap(); + let rt = configured(dir.path()); + assert_eq!(snapshot(&rt, START)["stage"], "ready"); + assert_eq!(rt.context().unwrap(), context()); + assert!(rt.selects_grid(1, &context().sources[0].grid_id)); + let before = snapshot(&rt, START); + let guard = rt.mutation_guard().unwrap(); + assert!(rt + .configure(&guard, context(), "other".into(), true) + .unwrap_err() + .contains("unsupported_command")); + assert_eq!(snapshot(&rt, START)["rows_sha256"], before["rows_sha256"]); + assert_eq!( + rt.status(START)["error_code"], + "learner_temporal_capture_private_unsupported_command" + ); + assert!(rt.saved_candidates(&context().room_digest).is_err()); + assert!(rt.prediction(START).is_err()); + assert!(rt.qualify(&rt.mutation_guard().unwrap()).is_err()); + let first = start(&rt, START + 1); + let mut seq = 0; + feed(&rt, first, first + 3_000_000_000, &mut seq); + assert!( + snapshot(&rt, first + 3_000_000_000)["durable_windows"] + .as_u64() + .unwrap() + > 0 + ); + assert!(rt + .temporal_confirmation_challenge(&rt.mutation_guard().unwrap()) + .is_err()); +} +#[test] +#[cfg_attr(debug_assertions, ignore = "synchronous native-persistence reference requires Release; CI explicitly executes and counts this exact test")] +fn full_seven_train_phases_require_fresh_confirmation_and_never_issue_fit_authority() { + let dir = tempfile::tempdir().unwrap(); + let rt = configured(dir.path()); + let mut seq = 0; + let mut now = START + 1; + let original = snapshot(&rt, START)["descriptor"].clone(); + for episode in 0..7 { + let before = snapshot(&rt, now); + assert_eq!(before["episode_index"], episode); + let target = before["target_ms"].as_u64().unwrap() * 1_000_000; + let first = start(&rt, now); + now = feed(&rt, first, first + target + PERIOD, &mut seq); + assert_eq!(snapshot(&rt, now)["stage"], "needs_confirmation"); + assert!(rt + .confirm_temporal_phase( + &rt.mutation_guard().unwrap(), + "not-a-current-challenge", + now + ) + .is_err()); + assert_eq!(snapshot(&rt, now)["sealed_train_episodes"], episode); + let expected = rt.mutation_guard().unwrap(); + let challenge = rt.temporal_confirmation_challenge(&expected).unwrap(); + rt.confirm_temporal_phase(&expected, &challenge, now) + .unwrap(); + assert_eq!(snapshot(&rt, now)["sealed_train_episodes"], episode + 1); + assert!(rt + .confirm_temporal_phase(&expected, &challenge, now) + .is_err()); + now += 1; + } + let done = snapshot(&rt, now); + assert_eq!(done["stage"], "complete"); + assert_eq!(done["durable_windows"], 64); + assert_eq!(done["descriptor"]["id"], original["id"]); + assert_eq!( + done["descriptor"]["expires_at_ms"], + original["expires_at_ms"] + ); + assert_eq!(done["fit_admission_available"], false); + let rt = Arc::new(rt); + assert!(rt + .start_training( + &rt.mutation_guard().unwrap(), + FitConfig::for_source_count(3) + ) + .is_err()); +} +#[test] +fn a_missing_required_source_between_retained_strata_withdraws_runtime_quality_terminally() { + let dir = tempfile::tempdir().unwrap(); + let rt = configured(dir.path()); + let mut seq = 0; + let first = start(&rt, START + 1); + let mut now = feed(&rt, first, first + 3_000_000_000, &mut seq); + let prior = snapshot(&rt, now)["durable_windows"].clone(); + let mut failed = false; + for _ in 0..5 { + seq += 1; + for source in 1..=2 { + if rt.ingest_native(&packet(source, seq, now), now).is_err() { + failed = true; + } + } + now += PERIOD; + } + assert!(failed); + assert_eq!(snapshot(&rt, now)["stage"], "failed"); + assert_eq!(snapshot(&rt, now)["durable_windows"], prior); + assert!(rt + .temporal_confirmation_challenge(&rt.mutation_guard().unwrap()) + .is_err()); + assert!(rt + .start_temporal_phase(&rt.mutation_guard().unwrap(), now) + .is_err()); +} +#[test] +fn all_unsupported_legacy_commands_are_explicit_and_cannot_change_temporal_rows() { + let dir = tempfile::tempdir().unwrap(); + let rt = Arc::new(configured(dir.path())); + let before = snapshot(&rt, START); + let request = || PhaseRequest { + trial_id: "legacy-trial".into(), + session_id: "legacy-session".into(), + role: DataRole::Train, + label: crate::room_learner::observation::RoomLabel::Empty, + duration_ms: 600_000, + lead_in_ms: 5_000, + conditions_confirmed: true, + }; + macro_rules! denied { + ($call:expr) => { + assert!($call.unwrap_err().contains("unsupported_command")); + }; + } + denied!(rt.begin_phase(&rt.mutation_guard().unwrap(), request(), START)); + denied!(rt.finish_phase(&rt.mutation_guard().unwrap(), START, true)); + denied!(rt.cancel_phase(&rt.mutation_guard().unwrap())); + denied!(rt.cancel_training(&rt.mutation_guard().unwrap())); + denied!(rt.start_training( + &rt.mutation_guard().unwrap(), + FitConfig::for_source_count(3) + )); + denied!(rt.evaluate(&rt.mutation_guard().unwrap(), DataRole::HeldOut)); + denied!(rt.qualify(&rt.mutation_guard().unwrap())); + denied!(rt.select_model( + &rt.mutation_guard().unwrap(), + &"a".repeat(64), + &"b".repeat(64) + )); + denied!(rt.deselect_model(&rt.mutation_guard().unwrap())); + denied!(rt.rollback_model(&rt.mutation_guard().unwrap())); + denied!(rt.restore_capture_recovery( + &rt.mutation_guard().unwrap(), + "old-id", + &context(), + START + )); + denied!(rt.discard_capture_recovery(&rt.mutation_guard().unwrap(), "old-id")); + denied!(rt.saved_recovery_context("old-id", START)); + denied!(rt.enable_current_capture_recovery( + &rt.mutation_guard().unwrap(), + "old-id", + true, + START + )); + assert!(rt + .paired_progress_snapshot( + &context().installation_id, + "public-runtime", + None, + &[], + None, + START + ) + .unwrap_err() + .contains("unsupported_command")); + assert!(rt + .selection_snapshot(START, Some(&context().room_digest)) + .integrity_error + .unwrap() + .contains("unsupported_command")); + assert!(matches!( + rt.selected_binary_evidence(START, Some(&context().room_digest)), + crate::room_learner::selection::SelectedBinaryEvidence::Unavailable { .. } + )); + assert_eq!(snapshot(&rt, START)["rows_sha256"], before["rows_sha256"]); + assert_eq!(snapshot(&rt, START)["descriptor"], before["descriptor"]); + assert!(rt.inner.lock().unwrap().model.is_none()); +} +#[test] +fn preparation_replay_phy_and_receiver_clock_cannot_rescue_or_relabel_a_capture() { + for case in ["preparation_replay", "native_phy", "receiver_reverse"] { + let dir = tempfile::tempdir().unwrap(); + let rt = configured(dir.path()); + let first = start(&rt, START + 1); + if case == "preparation_replay" { + rt.ingest_native( + &packet(1, 100, START + 1_000_000_000), + START + 1_000_000_000, + ) + .unwrap(); + assert!(rt.ingest_native(&packet(1, 99, first), first).is_err()); + } else { + let mut seq = 0; + let now = feed(&rt, first, first + 3_000_000_000, &mut seq); + if case == "native_phy" { + let mut p = packet(1, seq + 1, now); + p.native_frame.centre_frequency_hz += 1.; + assert!(rt.ingest_native(&p, now).is_err()); + } else { + assert!(rt + .tick_temporal_capture(&rt.mutation_guard().unwrap(), first) + .is_err()); + } + } + assert_eq!( + snapshot(&rt, first + 4_000_000_000)["stage"], + "failed", + "{case}" + ); + assert!( + rt.restore_temporal_capture( + &rt.mutation_guard().unwrap(), + &context(), + first + 5_000_000_000 + ) + .is_err(), + "same process cannot rewind floors: {case}" + ); + } +} +#[test] +fn actual_runtime_two_cold_resumes_keep_original_rows_trial_and_expiry_without_downtime_credit() { + let dir = tempfile::tempdir().unwrap(); + let (mut rt,mut now) = configured_async(dir.path()); + let original = snapshot(&rt, now)["descriptor"].clone(); + let mut seq = 0; + let mut trial = None; + for _ in 0..2 { + // Cold restore installs the actual asynchronous writer. Await the + // real start authorization before choosing any observation times. + let first = async_lead_end(&rt, now); + rt.tick_temporal_capture(&rt.mutation_guard().unwrap(),first).unwrap(); + now = feed_async_paced(&rt, first, first + 100_000_000_000, &mut seq); + let phase = rt.mutation_guard().unwrap().phase.unwrap(); + if let Some(prior) = &trial { + assert_eq!(prior, &phase.trial_id); + } else { + trial = Some(phase.trial_id); + } + rt.pause_temporal_capture(&rt.mutation_guard().unwrap(), now) + .unwrap(); + now=settle_async_boundary(&rt,now); + let paused = snapshot(&rt, now); + let accepted = paused["durable_observed_ns"] + .as_str() + .unwrap() + .parse::() + .unwrap(); + assert!(accepted < 600_000_000_000); + let old_guard = rt.mutation_guard().unwrap(); + drop(rt); + now += 50_000_000_000; + rt = runtime(dir.path(), now); + assert!(rt + .restore_temporal_capture(&old_guard, &context(), now) + .is_err()); + rt.restore_temporal_capture(&rt.mutation_guard().unwrap(), &context(), now) + .unwrap(); + let restored = snapshot(&rt, now); + assert_eq!(restored["stage"], "paused"); + assert_eq!(restored["rows_sha256"], paused["rows_sha256"]); + assert_eq!( + restored["durable_observed_ns"], + paused["durable_observed_ns"] + ); + assert_eq!(restored["descriptor"]["id"], original["id"]); + assert_eq!( + restored["descriptor"]["expires_at_ms"], + original["expires_at_ms"] + ); + assert!(rt + .temporal_confirmation_challenge(&rt.mutation_guard().unwrap()) + .is_err()); + now += 1; + } + let accepted = snapshot(&rt, now)["durable_observed_ns"] + .as_str() + .unwrap() + .parse::() + .unwrap(); + let first = async_lead_end(&rt, now); + rt.tick_temporal_capture(&rt.mutation_guard().unwrap(),first).unwrap(); + now = feed_async_paced( + &rt, + first, + first + 600_000_000_000 - accepted + PERIOD, + &mut seq, + ); + // Keep the declared receiver stream alive while the final complete + // reservation awaits its actual ACK. Do not treat submission as confirmed. + let waiting=std::time::Instant::now(); + while snapshot(&rt,now)["stage"]!="needs_confirmation" { + now=feed_async_paced(&rt,now,now,&mut seq); + assert!(waiting.elapsed()u64{ + // Same accelerated receiver cadence as the existing synthetic journey; + // give actual asynchronous transactions time while pending. No source + // timestamp backdating or production quality limits are changed. + while at<=to { + let pending=snapshot(rt,at)["durability_pending"]==true; + *seq+=1; + for source in 1..=3{rt.ingest_native(&packet(source,*seq,at),at).unwrap();} + at+=PERIOD; + if pending{std::thread::sleep(std::time::Duration::from_nanos(PERIOD));} + } + at +} +pub(super) fn settle_async_boundary(rt: &RoomLearningRuntime, after: u64) -> u64 { + let started=std::time::Instant::now(); + loop { + let at=after+started.elapsed().as_nanos() as u64; + rt.tick_temporal_capture(&rt.mutation_guard().unwrap(),at).unwrap(); + if snapshot(rt,at)["durability_pending"]==false { + let admitted={let state=rt.inner.lock().unwrap();let Some(CaptureWorkflow::Temporal(c))=&state.capture else{panic!()};at.max(c.last_now_ns)}; + if started.elapsed()>std::time::Duration::from_secs(1){println!("SYNTHETIC_DURABLE_ACK_WAIT_NS={} ROWS={} SEALED={}",started.elapsed().as_nanos(),snapshot(rt,admitted)["durable_windows"],snapshot(rt,admitted)["sealed_train_episodes"]);} + return admitted; + } + assert!(started.elapsed()(RoomLearningRuntime,u64){ + let rt=runtime(root,START); + rt.configure_temporal_capture(&rt.mutation_guard().unwrap(),context(),"synthetic-c2-runtime".into(),START).unwrap(); + let at=settle_async_boundary(&rt,START); + assert_eq!(snapshot(&rt,at)["stage"],"ready"); + (rt,at) +} + +#[test] +fn c2_start_requires_exact_current_prefix_ack_before_any_lead_or_rows(){ + use std::{sync::mpsc,time::Duration}; + let dir=tempfile::tempdir().unwrap();let(rt,at)=configured_async(dir.path()); + let writer={let state=rt.inner.lock().unwrap();let Some(CaptureWorkflow::Temporal(capture))=&state.capture else{panic!()};capture.core.as_ref().unwrap().writer().unwrap()}; + let(entered_tx,entered_rx)=mpsc::sync_channel(1);let(release_tx,release_rx)=mpsc::sync_channel(1); + writer.hold_next(entered_tx,release_rx); + rt.start_temporal_phase(&rt.mutation_guard().unwrap(),at).unwrap(); + entered_rx.recv_timeout(Duration::from_secs(2)).unwrap(); + let mut now=at+1_000_000; + for source in 1..=3 {rt.ingest_native(&packet(source,1,now),now).unwrap();} + let pending=snapshot(&rt,now); + assert_eq!(pending["stage"],"preparing");assert_eq!(pending["retained_rows"],0); + assert_eq!(pending["durable_windows"],0);assert_eq!(pending["lead_in_remaining_ms"],0); + assert!(rt.temporal_confirmation_challenge(&rt.mutation_guard().unwrap()).is_err()); + release_tx.send(()).unwrap(); + // Exact recertification chooses a fresh start coordinate only at admission. + let started=std::time::Instant::now(); + loop { + now=now.max(at+started.elapsed().as_nanos() as u64); + rt.tick_temporal_capture(&rt.mutation_guard().unwrap(),now).unwrap(); + if snapshot(&rt,now)["lead_in_remaining_ms"].as_u64().unwrap()>0 {break;} + assert!(started.elapsed()now); + drop(state); + let durable=rt.temporal_store().unwrap().load_temporal(now).unwrap().unwrap(); + let bytes=durable.prefix.unwrap();let v:Value=serde_json::from_slice(bytes.bytes()).unwrap(); + assert_eq!(v["payload"]["rows"].as_array().unwrap().len(),0); + assert!(v["payload"]["mapping"].is_null()); + rt.cancel_temporal_durability(&rt.mutation_guard().unwrap()).unwrap(); +} + +#[test] +fn c2_real_native_ingress_and_source_floors_progress_while_segment_writer_is_blocked(){ + use std::{sync::{Arc,mpsc},time::Duration}; + let dir=tempfile::tempdir().unwrap();let(rt,at)=configured_async(dir.path());let rt=Arc::new(rt); + rt.start_temporal_phase(&rt.mutation_guard().unwrap(),at).unwrap(); + let started=std::time::Instant::now();let end; + loop { + let now=at+started.elapsed().as_nanos() as u64; + rt.tick_temporal_capture(&rt.mutation_guard().unwrap(),now).unwrap(); + let state=rt.inner.lock().unwrap();let Some(CaptureWorkflow::Temporal(c))=&state.capture else{panic!()}; + if let Some(value)=c.lead_in_until {end=value;break;} + drop(state);assert!(started.elapsed()u64{ + rt.start_temporal_phase(&rt.mutation_guard().unwrap(),at).unwrap(); + let started=std::time::Instant::now(); + loop{ + let now=at+started.elapsed().as_nanos() as u64; + rt.tick_temporal_capture(&rt.mutation_guard().unwrap(),now).unwrap(); + let state=rt.inner.lock().unwrap();let Some(CaptureWorkflow::Temporal(c))=&state.capture else{panic!()}; + if let Some(end)=c.lead_in_until{return end;} + drop(state);assert!(started.elapsed()break 'lost,Err(e)=>panic!("unexpected {e}"),Ok(())=>{}}} + assert!(began.elapsed()u64{ + let wait=std::time::Instant::now();let mut now=after; + loop { + *seq+=1; + for source in 1..=3 {rt.ingest_native(&packet(source,*seq,now),now).unwrap();} + if snapshot(rt,now)["durability_pending"]==false {return now+PERIOD;} + assert!(wait.elapsed(){},Err(error)if error=="temporal_storage_busy_pause"=>{busy=now;break 'observe;},Err(error)=>panic!("unexpected {error}"), + } + } + assert!(now0,"busy restore did not retire the actual unconfirmed raw tail"); + assert_eq!(reclamation.blocked.load(Ordering::Acquire),0,"busy raw-tail destructor ran under Runtime.inner"); + rt.cancel_temporal_durability(&rt.mutation_guard().unwrap()).unwrap(); +} + +#[test] +fn c2_three_sources_keep_declared_real_cadence_during_250ms_worker_stall(){ + use std::{sync::mpsc,time::{Duration,Instant}}; + let dir=tempfile::tempdir().unwrap();let rt=runtime(dir.path(),START);let mut c=context(); + let template=c.sources[0].clone(); + c.sources=(1..=3).map(|id|{let mut source=template.clone();source.source_id=id;source.device_id=format!("esp32-node-{id}");source.link_id=format!("native-{id}");source}).collect(); + rt.configure_temporal_capture(&rt.mutation_guard().unwrap(),c,"synthetic-c2-three".into(),START).unwrap(); + let at=settle_async_boundary(&rt,START);let end=async_lead_end(&rt,at); + let writer={let state=rt.inner.lock().unwrap();let Some(CaptureWorkflow::Temporal(c))=&state.capture else{panic!()};c.core.as_ref().unwrap().writer().unwrap()}; + let(entered_tx,entered_rx)=mpsc::sync_channel(1);let(release_tx,release_rx)=mpsc::sync_channel(1);writer.hold_next(entered_tx,release_rx); + rt.tick_temporal_capture(&rt.mutation_guard().unwrap(),end).unwrap();entered_rx.recv_timeout(Duration::from_secs(2)).unwrap(); + let clock=Instant::now();let mut max_ingest=Duration::ZERO;let mut last=end; + let result=std::panic::catch_unwind(std::panic::AssertUnwindSafe(||{ + let mut previous_batch=None; + for index in 0..9u32 { + if let Some(previous)=previous_batch { + let elapsed=Instant::now().duration_since(previous); + if let Some(wait)=Duration::from_nanos(PERIOD).checked_sub(elapsed){std::thread::sleep(wait);} + } + let arrival=Instant::now();previous_batch=Some(arrival); + // Three sources arrive together; record actual batch arrival once. + // Processing uses its own current clock. Never catch up missed + // deadlines by generating an artificially fast next batch. + let received=end+clock.elapsed().as_nanos() as u64; + for source in 1..=3 { + let packet=packet(source,index+1,received); + let now=end+clock.elapsed().as_nanos() as u64;let started=Instant::now(); + if let Err(error)=rt.ingest_native(&packet,now){panic!("paced index={index} source={source} max_ingest_ns={} error={error}",max_ingest.as_nanos());} + max_ingest=max_ingest.max(started.elapsed());last=now; + } + } + assert_eq!(snapshot(&rt,last)["stage"],"observing"); + assert_eq!(snapshot(&rt,last)["durable_windows"],0); + })); + release_tx.send(()).unwrap();rt.cancel_temporal_durability(&rt.mutation_guard().unwrap()).unwrap(); + if let Err(error)=result{std::panic::resume_unwind(error);} + println!("SYNTHETIC_3_SOURCE_PACED_WALL_NS={} MAX_INGEST_NS={}",clock.elapsed().as_nanos(),max_ingest.as_nanos()); + assert!(max_ingest{ + now=async_lead_end(&rt,now);fresh_starts+=1; + rt.tick_temporal_capture(&rt.mutation_guard().unwrap(),now).unwrap(); + } + "needs_confirmation"=>break, + "failed"=>panic!("phase{phase} failed: {state}"), + _=>{} + } + let pending=snapshot(&rt,now)["durability_pending"]==true; + seq+=1; + for source in 1..=3 { + match rt.ingest_native(&packet(source,seq,now),now){ + Ok(())=>{},Err(error) if error=="temporal_storage_busy_pause"=>break, + Err(error)=>panic!("phase{phase} input failed: {error}"), + } + } + now+=PERIOD; + if pending {std::thread::sleep(std::time::Duration::from_nanos(PERIOD));} + assert!(fresh_starts<120,"bounded interruption budget exhausted"); + } + let guard=rt.mutation_guard().unwrap();let challenge=rt.temporal_confirmation_challenge(&guard).unwrap(); + rt.confirm_temporal_phase(&guard,&challenge,now).unwrap(); + // Submission is not durable phase confirmation. + assert_eq!(snapshot(&rt,now)["sealed_train_episodes"],phase); + assert_eq!(snapshot(&rt,now)["durability_pending"],true); + now=settle_async_boundary(&rt,now); + assert_eq!(snapshot(&rt,now)["sealed_train_episodes"],phase+1); + println!("SYNTHETIC_ASYNC_PHASE_SEALED={} ROWS={} FRESH_STARTS={fresh_starts}",phase+1,snapshot(&rt,now)["durable_windows"]); + } + let complete=snapshot(&rt,now); + assert_eq!(complete["stage"],"complete");assert_eq!(complete["retained_rows"],64);assert_eq!(complete["durable_windows"],64); + assert_eq!(complete["fit_admission_available"],false); + assert_eq!(complete["descriptor"]["id"],original["id"]);assert_eq!(complete["descriptor"]["expires_at_ms"],original["expires_at_ms"]); + let inner=rt.inner.lock().unwrap();assert!(inner.model.is_none());assert!(!inner.selection.has_history());drop(inner); + println!("SYNTHETIC_ASYNC_TRAIN64_SEALS7_FRESH_STARTS={fresh_starts}"); + rt.cancel_temporal_durability(&rt.mutation_guard().unwrap()).unwrap(); +} + +#[derive(Default)] +struct ReclaimCounts { + unlocked:std::sync::atomic::AtomicUsize, + blocked:std::sync::atomic::AtomicUsize, + final_prefix_owner:std::sync::atomic::AtomicUsize, + raw_tail:std::sync::atomic::AtomicUsize, +} +fn install_reclaim_probe(rt:&std::sync::Arc)->std::sync::Arc{ + let counts=std::sync::Arc::new(ReclaimCounts::default());let observed=counts.clone();let weak=std::sync::Arc::downgrade(rt); + *rt.reclamation_probe.lock().unwrap()=Some(std::sync::Arc::new(move|stats|{ + if !stats.has_ack{return;} + // Never panic from a destructor, especially while testing unwinding. + let unlocked=weak.upgrade().is_some_and(|rt|match rt.inner.try_lock(){Ok(_)=>true,Err(std::sync::TryLockError::Poisoned(_))=>true,Err(std::sync::TryLockError::WouldBlock)=>false}); + if unlocked{observed.unlocked.fetch_add(1,Ordering::AcqRel);}else{observed.blocked.fetch_add(1,Ordering::AcqRel);} + if stats.old_prefix_bytes>0 && stats.old_prefix_last_owner{observed.final_prefix_owner.fetch_add(1,Ordering::AcqRel);} + if stats.raw_tail_rows>0{observed.raw_tail.fetch_add(1,Ordering::AcqRel);} + }));counts +} +#[test] +fn c2_actual_ack_retirement_unlocks_before_normal_rejection_and_unwind_destructors(){ + use std::{sync::{Arc,mpsc},time::{Duration,Instant}}; + for via_tick in [false,true]{for outcome in ["normal","rejected","unwind"]{ + let dir=tempfile::tempdir().unwrap();let(rt,at)=configured_async(dir.path());let rt=Arc::new(rt);let counts=install_reclaim_probe(&rt); + let writer={let state=rt.inner.lock().unwrap();let Some(CaptureWorkflow::Temporal(c))=&state.capture else{panic!()};c.core.as_ref().unwrap().writer().unwrap()}; + let(entered_tx,entered_rx)=mpsc::sync_channel(1);let(release_tx,release_rx)=mpsc::sync_channel(1);writer.hold_next(entered_tx,release_rx); + if outcome=="rejected"{writer.corrupt_next_ack_id();} + rt.start_temporal_phase(&rt.mutation_guard().unwrap(),at).unwrap();entered_rx.recv_timeout(Duration::from_secs(2)).unwrap();release_tx.send(()).unwrap(); + let wait=Instant::now();while !writer.ack_is_queued(){assert!(wait.elapsed()assert!(result.unwrap().is_ok()),"rejected"=>assert!(result.unwrap().is_err()),_=>assert!(result.is_err())} + assert!(counts.unlocked.load(Ordering::Acquire)+counts.blocked.load(Ordering::Acquire)>0,"destructor was never observed"); + assert_eq!(counts.blocked.load(Ordering::Acquire),0,"actual payload destructor ran while Runtime.inner was held"); + assert!(counts.unlocked.load(Ordering::Acquire)>0); + if outcome!="rejected"{assert!(counts.final_prefix_owner.load(Ordering::Acquire)>0,"did not retire last old-prefix Arc owner");} + writer.drain().unwrap(); + }} +} + +#[test] +#[ignore = "opt-in host/profile throughput acceptance; normal C2 tests retain growing-prefix custody, busy-pause and full-seven correctness"] +fn c2_real_growing_prefix_three_source_paced_seventeenth_row_throughput_acceptance(){ + use std::time::{Duration,Instant}; + let dir=tempfile::tempdir().unwrap();let(rt,mut now)=configured_async(dir.path());let mut seq=0; + // Prelude is synthetic receiver-clock collection, not a performance claim. + // Populate the SAME real Runtime/assembler/store with all16 Empty windows. + loop { + match snapshot(&rt,now)["stage"].as_str().unwrap(){ + "ready"|"paused"=>{now=async_lead_end(&rt,now);rt.tick_temporal_capture(&rt.mutation_guard().unwrap(),now).unwrap();}, + "needs_confirmation"=>break, + "failed"=>panic!("growing-prefix prelude failed"),_=>{} + } + let pending=snapshot(&rt,now)["durability_pending"]==true;seq+=1; + for source in 1..=3{match rt.ingest_native(&packet(source,seq,now),now){Ok(())=>{},Err(e)if e=="temporal_storage_busy_pause"=>break,Err(e)=>panic!("prelude: {e}")}} + now+=PERIOD;if pending{std::thread::sleep(Duration::from_nanos(PERIOD));} + } + let guard=rt.mutation_guard().unwrap();let challenge=rt.temporal_confirmation_challenge(&guard).unwrap(); + rt.confirm_temporal_phase(&guard,&challenge,now).unwrap();now=settle_async_boundary(&rt,now); + assert_eq!(snapshot(&rt,now)["durable_windows"],16); + let old_prefix=rt.temporal_store().unwrap().load_temporal(now).unwrap().unwrap().prefix.unwrap().bytes().len(); + let base=async_lead_end(&rt,now);rt.tick_temporal_capture(&rt.mutation_guard().unwrap(),base).unwrap(); + // Measure honest batch arrival and current processing clocks at real33Hz; + // no accelerated ticks, held-writer hook, replay queue, or deadline catch-up. + let clock=Instant::now();let mut previous_batch=None;let mut packet_wall=Vec::::new(); + let mut row_ready_at=None;let mut acked_at=None;let mut measured_now=base; + let mut batch_arrivals=Vec::::new();let mut sleep_samples=Vec::<(u64,u64)>::new(); + let mut source_arrivals:[Vec;3]=std::array::from_fn(|_|Vec::new()); + let mut snapshot_costs=Vec::::new();let mut failed_attempt:Option<(u8,u32,String)>=None; + let result:Result<(),String>=(||{ + loop { + if clock.elapsed()>Duration::from_secs(20){return Err("real growing-prefix ACK did not arrive within measurement window".into());} + if let Some(previous)=previous_batch{if let Some(wait)=Duration::from_nanos(PERIOD).checked_sub(Instant::now().duration_since(previous)){let slept=Instant::now();std::thread::sleep(wait);sleep_samples.push((wait.as_nanos()as u64,slept.elapsed().as_nanos()as u64));}} + previous_batch=Some(Instant::now());let received=base+clock.elapsed().as_nanos()as u64;seq+=1; + batch_arrivals.push(received-base); + for source in 1..=3{ + let before=Instant::now();let p=packet(source,seq,received);measured_now=base+clock.elapsed().as_nanos()as u64; + let outcome=rt.ingest_native(&p,measured_now); + let packet_elapsed=before.elapsed().as_nanos()as u64; + source_arrivals[usize::from(source)-1].push(p.native_frame.timestamp_ns-base); + packet_wall.push(packet_elapsed); + if let Err(error)=outcome{failed_attempt=Some((source,seq,error.clone()));return Err(error);} + } + let snapshot_started=Instant::now();let status=snapshot(&rt,measured_now);snapshot_costs.push(snapshot_started.elapsed().as_nanos()as u64); + if status["retained_rows"].as_u64().unwrap()>=17 && row_ready_at.is_none(){row_ready_at=Some(clock.elapsed());} + if status["durable_windows"].as_u64().unwrap()>=17 && acked_at.is_none(){acked_at=Some(clock.elapsed());} + if let Some(acked)=acked_at{if clock.elapsed()-acked>=Duration::from_millis(250){break;}} + } + if packet_wall.is_empty(){return Err("no measured packets".into());} + packet_wall.sort_unstable();let max=*packet_wall.last().unwrap();let p99=packet_wall[(packet_wall.len()-1)*99/100]; + if max>5_000_000 || p99>2_000_000{return Err(format!("packet wall budget exceeded: max_ns={max} p99_ns={p99}"));} + Ok(()) + })(); + rt.cancel_temporal_durability(&rt.mutation_guard().unwrap()).unwrap(); + let new_prefix=rt.temporal_store().unwrap().load_temporal(measured_now).unwrap().unwrap().prefix.unwrap().bytes().len(); + packet_wall.sort_unstable();let max=packet_wall.last().copied().unwrap_or(0);let p99=packet_wall.get(packet_wall.len().saturating_sub(1)*99/100).copied().unwrap_or(0); + println!("SYNTHETIC_REAL_3_SOURCE_GROWING_PREFIX status={} old_bytes={} new_bytes={} packets={} max_packet_wall_ns={} p99_packet_wall_ns={} row_ready_at_ns={:?} acked_at_ns={:?} row_ready_to_ack_ns={:?} debug_assertions={} error={:?}",if result.is_ok(){"PASS"}else{"UNQUALIFIED"},old_prefix,new_prefix,packet_wall.len(),max,p99,row_ready_at.map(|d|d.as_nanos()),acked_at.map(|d|d.as_nanos()),row_ready_at.zip(acked_at).map(|(q,a)|a.saturating_sub(q).as_nanos()),cfg!(debug_assertions),result.as_ref().err()); + let summaries=source_arrivals.iter().enumerate().map(|(index,times)|{ + let span=times.last().copied().unwrap_or(0).saturating_sub(times.first().copied().unwrap_or(0)); + let intervals=times.len().saturating_sub(1) as u128; + let gaps=times.windows(2).map(|w|w[1]-w[0]).collect::>(); + json!({"source":index+1,"attempts":times.len(),"first_relative_ns":times.first(),"last_relative_ns":times.last(),"span_ns":span, + "actual_rate_millihz":if span>0{Some(intervals*1_000_000_000_000u128/u128::from(span))}else{None}, + "min_gap_ns":gaps.iter().min(),"max_gap_ns":gaps.iter().max()}) + }).collect::>(); + let contract=ManagedScheduleContract::build(context(),LeadInPolicy::Train5HeldOut15EngineeringV1).unwrap();let policy=contract.native_window_policy();let slot=&contract.episodes()[1].slots[0]; + println!("SYNTHETIC_CADENCE_DIAGNOSTIC {}",json!({"batch_count":batch_arrivals.len(),"batch_first_relative_ns":batch_arrivals.first(),"batch_last_relative_ns":batch_arrivals.last(), + "sources":summaries,"failed_attempt":failed_attempt,"sleep_count":sleep_samples.len(),"requested_sleep_min_ns":sleep_samples.iter().map(|x|x.0).min(),"requested_sleep_max_ns":sleep_samples.iter().map(|x|x.0).max(), + "actual_sleep_min_ns":sleep_samples.iter().map(|x|x.1).min(),"actual_sleep_max_ns":sleep_samples.iter().map(|x|x.1).max(), + "sleep_overshoot_max_ns":sleep_samples.iter().map(|x|x.1.saturating_sub(x.0)).max(),"snapshot_max_ns":snapshot_costs.iter().max(), + "slot_target_ms":slot.target_offset_ms,"slot_deadline_ms":slot.deadline_offset_ms,"native_target_span_ms":policy.target_span_ms(), + "source_min_rows":policy.source_policies()[0].min_rows,"source_max_gap_ns":policy.source_policies()[0].max_gap_ns, + "rate_band_millihz":policy.source_policies()[0].receiver_cadence.rate_bounds_millihz()})); + assert!(result.is_ok(),"growing-prefix performance is UNQUALIFIED; no public activation: {:?}",result.err()); +} + +#[path = "c2_precision_pacer_test_support.rs"] +mod c2_precision_pacer_test_support; + +#[test] +#[ignore = "opt-in host/profile throughput acceptance; normal C2 tests retain growing-prefix custody, busy-pause and full-seven correctness"] +fn c2_real_growing_prefix_three_source_absolute_pacer_with_actual_input_precondition(){ + use std::time::{Duration,Instant}; + const SPIN_GUARD_NS:u64=3_000_000; + let precondition=c2_precision_pacer_test_support::input_study(100,SPIN_GUARD_NS); + println!("SYNTHETIC_INPUT_PRECONDITION {:?}",precondition); + assert!(precondition.qualified,"INPUT_UNQUALIFIED before backend exercise; no retry or public activation"); + let dir=tempfile::tempdir().unwrap();let(rt,mut now)=configured_async(dir.path());let mut seq=0; + // Prelude is synthetic receiver-clock collection, not a performance claim. + // Populate the SAME real Runtime/assembler/store with all16 Empty windows. + loop { + match snapshot(&rt,now)["stage"].as_str().unwrap(){ + "ready"|"paused"=>{now=async_lead_end(&rt,now);rt.tick_temporal_capture(&rt.mutation_guard().unwrap(),now).unwrap();}, + "needs_confirmation"=>break, + "failed"=>panic!("growing-prefix prelude failed"),_=>{} + } + let pending=snapshot(&rt,now)["durability_pending"]==true;seq+=1; + for source in 1..=3{match rt.ingest_native(&packet(source,seq,now),now){Ok(())=>{},Err(e)if e=="temporal_storage_busy_pause"=>break,Err(e)=>panic!("prelude: {e}")}} + now+=PERIOD;if pending{std::thread::sleep(Duration::from_nanos(PERIOD));} + } + let guard=rt.mutation_guard().unwrap();let challenge=rt.temporal_confirmation_challenge(&guard).unwrap(); + rt.confirm_temporal_phase(&guard,&challenge,now).unwrap();now=settle_async_boundary(&rt,now); + assert_eq!(snapshot(&rt,now)["durable_windows"],16); + let old_prefix=rt.temporal_store().unwrap().load_temporal(now).unwrap().unwrap().prefix.unwrap().bytes().len(); + let base=async_lead_end(&rt,now);rt.tick_temporal_capture(&rt.mutation_guard().unwrap(),base).unwrap(); + // Measure honest batch arrival and current processing clocks at real33Hz; + // no accelerated ticks, held-writer hook, replay queue, or deadline catch-up. + let clock=Instant::now();let mut absolute_index=0usize;let mut packet_wall=Vec::::new(); + let mut row_ready_at=None;let mut acked_at=None;let mut measured_now=base; + let mut batch_arrivals=Vec::::new(); + let mut source_arrivals:[Vec;3]=std::array::from_fn(|_|Vec::new()); + let mut snapshot_costs=Vec::::new();let mut failed_attempt:Option<(u8,u32,String)>=None; + let result:Result<(),String>=(||{ + loop { + if clock.elapsed()>Duration::from_secs(20){return Err("real growing-prefix ACK did not arrive within measurement window".into());} + let actual_relative=c2_precision_pacer_test_support::next_actual_receipt(clock,absolute_index,SPIN_GUARD_NS)?; + absolute_index+=1; + // ACTUAL wakeup, never nominal deadline. One batch only; an entire + // missed period returns UNQUALIFIED instead of catch-up emission. + let received=base+actual_relative;seq+=1; + batch_arrivals.push(received-base); + for source in 1..=3{ + let before=Instant::now();let p=packet(source,seq,received);measured_now=base+clock.elapsed().as_nanos()as u64; + let outcome=rt.ingest_native(&p,measured_now); + let packet_elapsed=before.elapsed().as_nanos()as u64; + source_arrivals[usize::from(source)-1].push(p.native_frame.timestamp_ns-base); + packet_wall.push(packet_elapsed); + if let Err(error)=outcome{failed_attempt=Some((source,seq,error.clone()));return Err(error);} + } + let snapshot_started=Instant::now();let status=snapshot(&rt,measured_now);snapshot_costs.push(snapshot_started.elapsed().as_nanos()as u64); + if status["retained_rows"].as_u64().unwrap()>=17 && row_ready_at.is_none(){row_ready_at=Some(clock.elapsed());} + if status["durable_windows"].as_u64().unwrap()>=17 && acked_at.is_none(){acked_at=Some(clock.elapsed());} + if let Some(acked)=acked_at{if clock.elapsed()-acked>=Duration::from_millis(250){break;}} + } + if !source_arrivals.iter().all(|times|c2_precision_pacer_test_support::actual_stream_qualifies(times)){return Err("INPUT_UNQUALIFIED during real backend measurement; planned deadlines are not evidence".into());} + if packet_wall.is_empty(){return Err("no measured packets".into());} + packet_wall.sort_unstable();let max=*packet_wall.last().unwrap();let p99=packet_wall[(packet_wall.len()-1)*99/100]; + if max>5_000_000 || p99>2_000_000{return Err(format!("packet wall budget exceeded: max_ns={max} p99_ns={p99}"));} + Ok(()) + })(); + rt.cancel_temporal_durability(&rt.mutation_guard().unwrap()).unwrap(); + let new_prefix=rt.temporal_store().unwrap().load_temporal(measured_now).unwrap().unwrap().prefix.unwrap().bytes().len(); + packet_wall.sort_unstable();let max=packet_wall.last().copied().unwrap_or(0);let p99=packet_wall.get(packet_wall.len().saturating_sub(1)*99/100).copied().unwrap_or(0); + println!("SYNTHETIC_REAL_3_SOURCE_GROWING_PREFIX status={} old_bytes={} new_bytes={} packets={} max_packet_wall_ns={} p99_packet_wall_ns={} row_ready_at_ns={:?} acked_at_ns={:?} row_ready_to_ack_ns={:?} debug_assertions={} error={:?}",if result.is_ok(){"PASS"}else{"UNQUALIFIED"},old_prefix,new_prefix,packet_wall.len(),max,p99,row_ready_at.map(|d|d.as_nanos()),acked_at.map(|d|d.as_nanos()),row_ready_at.zip(acked_at).map(|(q,a)|a.saturating_sub(q).as_nanos()),cfg!(debug_assertions),result.as_ref().err()); + let summaries=source_arrivals.iter().enumerate().map(|(index,times)|{ + let span=times.last().copied().unwrap_or(0).saturating_sub(times.first().copied().unwrap_or(0)); + let intervals=times.len().saturating_sub(1) as u128; + let gaps=times.windows(2).map(|w|w[1]-w[0]).collect::>(); + json!({"source":index+1,"attempts":times.len(),"first_relative_ns":times.first(),"last_relative_ns":times.last(),"span_ns":span, + "actual_rate_millihz":if span>0{Some(intervals*1_000_000_000_000u128/u128::from(span))}else{None}, + "min_gap_ns":gaps.iter().min(),"max_gap_ns":gaps.iter().max()}) + }).collect::>(); + let contract=ManagedScheduleContract::build(context(),LeadInPolicy::Train5HeldOut15EngineeringV1).unwrap();let policy=contract.native_window_policy();let slot=&contract.episodes()[1].slots[0]; + println!("SYNTHETIC_CADENCE_DIAGNOSTIC {}",json!({"batch_count":batch_arrivals.len(),"batch_first_relative_ns":batch_arrivals.first(),"batch_last_relative_ns":batch_arrivals.last(), + "sources":summaries,"failed_attempt":failed_attempt,"pacer_strategy":"absolute_actual_deadline_no_whole_period_catchup","spin_guard_ns":SPIN_GUARD_NS,"snapshot_max_ns":snapshot_costs.iter().max(), + "slot_target_ms":slot.target_offset_ms,"slot_deadline_ms":slot.deadline_offset_ms,"native_target_span_ms":policy.target_span_ms(), + "source_min_rows":policy.source_policies()[0].min_rows,"source_max_gap_ns":policy.source_policies()[0].max_gap_ns, + "rate_band_millihz":policy.source_policies()[0].receiver_cadence.rate_bounds_millihz()})); + assert!(result.is_ok(),"growing-prefix performance is UNQUALIFIED; no public activation: {:?}",result.err()); +} + +#[test] +fn c2_quiescent_start_waiter_handles_slow_recertification_without_observation_credit(){ + use std::{sync::mpsc,time::{Duration,Instant}}; + let dir=tempfile::tempdir().unwrap();let(rt,at)=configured_async(dir.path()); + let writer={let state=rt.inner.lock().unwrap();let Some(CaptureWorkflow::Temporal(c))=&state.capture else{panic!()};c.core.as_ref().unwrap().writer().unwrap()}; + let(entered_tx,entered_rx)=mpsc::sync_channel(1);let(release_tx,release_rx)=mpsc::sync_channel(1);writer.hold_next(entered_tx,release_rx); + let release=std::thread::spawn(move||{entered_rx.recv().unwrap();std::thread::sleep(Duration::from_secs(6));release_tx.send(()).unwrap();}); + let started=Instant::now();let lead_end=async_lead_end(&rt,at);release.join().unwrap(); + assert!(started.elapsed()>=Duration::from_secs(6)); + let admitted=lead_end-TRAIN_LEAD_IN_NS;let status=snapshot(&rt,admitted); + assert_eq!(status["retained_rows"],0);assert_eq!(status["durable_windows"],0);assert_eq!(status["observed_ns"],"0"); + assert_eq!(status["lead_in_remaining_ms"],5000,"lead-in must start only at actual authorization ACK"); + rt.cancel_temporal_durability(&rt.mutation_guard().unwrap()).unwrap(); +} + +// READ-ONLY production behavior probe: no budget/policy/clock/threshold changes. +// Private token Debug is parsed ONLY in this synthetic test to read the actual +// authenticated header revision without adding a production metadata accessor. +fn budget_repro_store_observation(rt:&RoomLearningRuntime,now:u64)->(usize,usize,usize,usize,String){ + let loaded=rt.temporal_store().unwrap().load_temporal(now).unwrap().unwrap(); + let token= format!("{:?}",loaded.token); + let revision=token.split("revision: ").nth(1).unwrap().split(',').next().unwrap().parse::().unwrap(); + let prefix=loaded.prefix.unwrap();assert_eq!(prefix.transition_limit(),329); + let value:Value=serde_json::from_slice(prefix.bytes()).unwrap(); + let rows=value["payload"]["rows"].as_array().unwrap().len(); + let gaps=value["payload"]["ledger"]["gaps"].as_array().unwrap().len(); + let bridge=&value["payload"]["bridge"]; + let parts=bridge["parts"].as_array().unwrap().len()+bridge["seals"].as_array().unwrap().iter().map(|s|s["parts"].as_array().unwrap().len()).sum::(); + (revision,rows,gaps,parts,format!("{:x}",::digest(prefix.bytes()))) +} +#[test] +#[cfg_attr(debug_assertions, ignore = "native-persistence journey requires Release; CI explicitly executes and counts this test")] +fn legacy_329_actual_runtime_saturation_retires_and_explicit_fresh_train_chains_343(){ + assert!(!cfg!(debug_assertions), "execute this native-persistence journey in the explicit Release batch"); + let dir=tempfile::tempdir().unwrap();let(rt,mut now)=configured_async(dir.path());let original=snapshot(&rt,now)["descriptor"].clone(); + assert_eq!(budget_repro_store_observation(&rt,now).0,0); + //127 zero-observation pauses followed by127 resumes: initialsegment0 is + //paused, segments1..126 paused, finalsegment127 will actually collectEmpty. + for cycle in 0..127usize{ + now=async_lead_end(&rt,now);rt.tick_temporal_capture(&rt.mutation_guard().unwrap(),now).unwrap(); + rt.pause_temporal_capture(&rt.mutation_guard().unwrap(),now).unwrap(); + now=settle_async_boundary(&rt,now); + assert_eq!(snapshot(&rt,now)["stage"],"paused");assert_eq!(snapshot(&rt,now)["durable_windows"],0); + if cycle==0 || cycle==126{ + let actual=budget_repro_store_observation(&rt,now); + assert_eq!(actual.0,2*(cycle+1));assert_eq!(actual.2,cycle); + println!("SYNTHETIC_BUDGET_ZERO_PREFIX cycle={} authenticated_revision={} gaps={} parts={}",cycle+1,actual.0,actual.2,actual.3); + } + } + let mut seq=0;let mut before_failure=None; + for phase in 0..7u64{ + if phase==6{ + //Newlast-episodeinitialsegment has no newgap; pause it once so the + //following realresume consumes the128thGLOBALgap. No laterstart. + now=async_lead_end(&rt,now);rt.tick_temporal_capture(&rt.mutation_guard().unwrap(),now).unwrap(); + rt.pause_temporal_capture(&rt.mutation_guard().unwrap(),now).unwrap();now=settle_async_boundary(&rt,now); + } + loop{ + let status=snapshot(&rt,now); + match status["stage"].as_str().unwrap(){ + "ready"|"paused"=>{now=async_lead_end(&rt,now);rt.tick_temporal_capture(&rt.mutation_guard().unwrap(),now).unwrap();}, + "needs_confirmation"=>break, + "failed"=>panic!("STATIC_TRACE_REFUTED earlierphase failure phase={phase} status={status}"),_=>{} + } + if status["durable_windows"]==59 && before_failure.is_none(){ + let observed=budget_repro_store_observation(&rt,now); + assert_eq!((observed.0,observed.1,observed.2,observed.3),(328,59,128,135)); + println!("SYNTHETIC_BUDGET_LAST_HEAD revision={} windows={} gaps={} parts={}",observed.0,observed.1,observed.2,observed.3); + before_failure=Some(observed); + } + let pending=snapshot(&rt,now)["durability_pending"]==true;seq+=1; + for source in 1..=3{ + match rt.ingest_native(&packet(source,seq,now),now){ + Ok(())=>{},Err(error)if error=="temporal_storage_busy_pause"=>break, + Err(error)if error=="temporal_transition_budget"=>{ + assert_eq!(phase,6,"wrong phase hit budget");let before=before_failure.as_ref().expect("no observed last valid head"); + let after=budget_repro_store_observation(&rt,now);assert_eq!(&after,before,"failed operation changed authenticated durable prefix"); + let state=snapshot(&rt,now);assert_eq!(state["durable_windows"],59);assert!(state["retained_rows"].as_u64().unwrap()>=60); + assert_eq!(state["stage"],"blocked");assert_eq!(state["descriptor"]["id"],original["id"]);assert_eq!(state["descriptor"]["expires_at_ms"],original["expires_at_ms"]); + println!("SYNTHETIC_ACTUAL_TRANSITION_BUDGET_CONFIRMED phase_index=6 failed_window_in_episode=4 proposed_revision=329 accepted_states=329 last_authenticated_revision={} last_durable_windows={} global_gaps={} quality_parts={} current_stage={} last_prefix_sha256_unchanged=true",after.0,after.1,after.2,after.3,state["stage"]); + rt.cancel_temporal_durability(&rt.mutation_guard().unwrap()).unwrap(); + let store=rt.temporal_store().unwrap(); + let token=store.load_temporal(now).unwrap().unwrap().token; + let original_prefix=store.load_temporal(now).unwrap().unwrap().prefix.unwrap(); + // Same Runtime generated authenticated revision328. No + // synthetic metadata/counter injection creates the head. + assert!(store.fresh_train_genesis_predecessor(now).is_err()); + // Independent current-time branch of the SAME real + // authenticated saturated prefix. Writer already drained; + // copying only this synthetic TempDir creates no live data. + let live=tempfile::tempdir().unwrap();let live_root=live.path().join("fixture"); + copy_synthetic_budget_fixture(dir.path(),&live_root); + {let live_rt=runtime(&live_root,now);let live_store=live_rt.temporal_store().unwrap(); + let current=live_store.load_temporal(now).unwrap().unwrap(); + let live_intent=live_store.prepare_retire_temporal(¤t.token,now,false).unwrap(); + let terminal=live_store.publish_temporal(&live_intent,None,now).unwrap(); + assert!(format!("{terminal:?}").contains("revision: 329,")); + assert!(live_store.load_temporal(now).unwrap().unwrap().prefix.is_none()); + assert_eq!(live_store.repair_same_temporal(&live_intent,None,now).unwrap(),terminal); + } + let expired=original_prefix.descriptor().expires_at_ms*1_000_000; + assert_eq!(store.load_temporal(expired).err().as_deref(),Some("recovery_expired")); + let inspection=store.inspect_temporal_retirement(expired).unwrap().unwrap(); + assert!(inspection.expired()); + let intent=store.prepare_inspected_retirement(&inspection,expired,true).unwrap(); + let retired=store.publish_temporal(&intent,None,expired).unwrap(); + assert!(format!("{retired:?}").contains("generation: 0, revision: 329,")); + // A lost ACK can only repair the exact same terminal, + // never consume revision330 or restore positive rows. + assert_eq!(store.repair_same_temporal(&intent,None,expired).unwrap(),retired); + let same=store.prepare_retire_temporal(&retired,expired,false).unwrap();assert_eq!(store.publish_temporal(&same,None,expired).unwrap(),retired); + assert!(store.load_temporal(expired).unwrap().unwrap().prefix.is_none()); + assert_eq!(store.fresh_train_genesis_predecessor(expired).unwrap(),Some(retired.clone())); + assert!(store.prepare_retire_temporal(&token,expired,false).is_err()); + drop(rt); + let fresh_at=expired+1_000_000; + let fresh=runtime(dir.path(),fresh_at); + fresh.configure_fresh_train_343_capture(&fresh.mutation_guard().unwrap(),context(),"synthetic-fresh-train-343".into(),fresh_at).unwrap(); + let fresh_now=settle_async_boundary(&fresh,fresh_at); + let loaded=fresh.temporal_store().unwrap().load_temporal(fresh_now).unwrap().unwrap(); + assert_eq!(loaded.prefix.as_ref().unwrap().transition_limit(),343); + assert!(format!("{:?}",loaded.token).contains("generation: 1, revision: 0,")); + assert_ne!(loaded.prefix.as_ref().unwrap().descriptor().id,original_prefix.descriptor().id); + assert_eq!(snapshot(&fresh,fresh_now)["durable_windows"],0); + assert_eq!(snapshot(&fresh,fresh_now)["stage"],"ready"); + fresh.cancel_temporal_durability(&fresh.mutation_guard().unwrap()).unwrap(); + println!("SYNTHETIC_LEGACY_SATURATED_RETIRE_AND_FRESH_343_PASS expired_samples_unavailable=true terminal_revision=329 same_intent_repair=true fresh_rows=0");return; + }, + Err(error)=>panic!("STATIC_TRACE_REFUTED earlierguard phase={phase} error={error}"), + } + } + now+=PERIOD;if pending{std::thread::sleep(std::time::Duration::from_nanos(PERIOD));} + } + let guard=rt.mutation_guard().unwrap();let challenge=rt.temporal_confirmation_challenge(&guard).unwrap();rt.confirm_temporal_phase(&guard,&challenge,now).unwrap();now=settle_async_boundary(&rt,now); + let actual=budget_repro_store_observation(&rt,now);println!("SYNTHETIC_BUDGET_PHASE_SEAL phase={} authenticated_revision={} windows={} gaps={} parts={}",phase+1,actual.0,actual.1,actual.2,actual.3); + } + panic!("STATIC_TRACE_REFUTED complete path did not hit predicted old329 budget"); +} + +// Fresh-profile complete upper-bound journey: original 128 gap / 135 part +// schedule plus a legitimate deferred pause before EACH of the seven seals. +// All counters below come from authenticated store readback, not a mock limit. +#[test] +#[cfg_attr(debug_assertions, ignore = "native-persistence journey requires Release; CI explicitly executes and counts this test")] +fn fresh_343_actual_runtime_all_128_gaps_seven_terminal_pauses_and_retirement_fit_budget(){ + assert!(!cfg!(debug_assertions), "execute this native-persistence journey in the explicit Release batch"); + let dir=tempfile::tempdir().unwrap();let rt=runtime(dir.path(),START); + rt.configure_fresh_train_343_capture(&rt.mutation_guard().unwrap(),context(),"synthetic-full-343".into(),START).unwrap(); + let mut now=settle_async_boundary(&rt,START);let original=snapshot(&rt,now)["descriptor"].clone(); + for _ in 0..127{ + now=async_lead_end(&rt,now);rt.tick_temporal_capture(&rt.mutation_guard().unwrap(),now).unwrap(); + rt.pause_temporal_capture(&rt.mutation_guard().unwrap(),now).unwrap();now=settle_async_boundary(&rt,now); + assert_eq!(snapshot(&rt,now)["durable_windows"],0); + } + let mut seq=0;let mut terminal_pauses=0; + for phase in 0..7u64{ + if phase==6{ + now=async_lead_end(&rt,now);rt.tick_temporal_capture(&rt.mutation_guard().unwrap(),now).unwrap(); + rt.pause_temporal_capture(&rt.mutation_guard().unwrap(),now).unwrap();now=settle_async_boundary(&rt,now); + } + let target=16+phase*8;let mut terminal_pause=false;let supervision=std::time::Instant::now(); + loop{ + assert!(supervision.elapsed(){now=async_lead_end(&rt,now);rt.tick_temporal_capture(&rt.mutation_guard().unwrap(),now).unwrap();}, + "needs_confirmation"=>break,"failed"|"blocked"=>panic!("fresh343 phase={phase} {status}"),_=>{} + } + let pending=snapshot(&rt,now)["durability_pending"]==true;seq+=1; + for source in 1..=3{ + match rt.ingest_native(&packet(source,seq,now),now){Ok(())=>{},Err(e)if e=="temporal_storage_busy_pause"=>break,Err(e)=>panic!("fresh343 phase={phase} ingress={e}")} + } + let after=snapshot(&rt,now); + if !terminal_pause && after["retained_rows"]==target && after["durability_pending"]==true{ + // Exact real pending final WINDOW. pause does not consume its + // ACK; no new raw data, target slide, or fake duration is used. + rt.pause_temporal_capture(&rt.mutation_guard().unwrap(),now).unwrap(); + terminal_pause=true;terminal_pauses+=1; + now=settle_async_boundary(&rt,now); + // Paused presentation precedes NeedsConfirmation; the real + // challenge below is the authority for complete-phase credit. + assert_eq!(snapshot(&rt,now)["durable_windows"],target);break; + }else{now+=PERIOD;if pending{std::thread::sleep(std::time::Duration::from_nanos(PERIOD));}} + } + assert!(terminal_pause,"did not exercise terminal deferred pause"); + let guard=rt.mutation_guard().unwrap();let challenge=rt.temporal_confirmation_challenge(&guard).unwrap(); + rt.confirm_temporal_phase(&guard,&challenge,now).unwrap();now=settle_async_boundary(&rt,now); + println!("SYNTHETIC_FRESH343_SEAL phase={} rows={} terminal_pauses={}",phase+1,target,terminal_pauses); + } + assert_eq!(snapshot(&rt,now)["sealed_train_episodes"],7);assert_eq!(snapshot(&rt,now)["durable_windows"],64); + rt.cancel_temporal_durability(&rt.mutation_guard().unwrap()).unwrap(); + let store=rt.temporal_store().unwrap();let loaded=store.load_temporal(now).unwrap().unwrap();let prefix=loaded.prefix.unwrap(); + assert_eq!(prefix.transition_limit(),343);assert_eq!(prefix.window_count(),64); + let value:Value=serde_json::from_slice(prefix.bytes()).unwrap(); + assert_eq!(value["payload"]["ledger"]["gaps"].as_array().unwrap().len(),128); + let bridge=&value["payload"]["bridge"]; + let parts=bridge["parts"].as_array().unwrap().len()+bridge["seals"].as_array().unwrap().iter().map(|seal|seal["parts"].as_array().unwrap().len()).sum::(); + assert_eq!(parts,135); + assert_eq!(prefix.descriptor().id,original["id"].as_str().unwrap()); + assert_eq!(prefix.descriptor().expires_at_ms,original["expires_at_ms"].as_u64().unwrap()); + let before=format!("{:?}",loaded.token);assert!(before.contains("revision: 341,"),"{before}"); + let intent=store.prepare_retire_temporal(&loaded.token,now,false).unwrap();let terminal=store.publish_temporal(&intent,None,now).unwrap(); + assert!(format!("{terminal:?}").contains("revision: 342,"));assert!(store.load_temporal(now).unwrap().unwrap().prefix.is_none()); + assert_eq!(store.repair_same_temporal(&intent,None,now).unwrap(),terminal); + println!("SYNTHETIC_FRESH343_COMPLETE authenticated_states=343 last_revision=342 rows=64 gaps=128 quality_parts=135 terminal_pauses=7 retired=true"); +} + +#[test] +fn fresh343_actual_genesis_fixture_for_frozen_old_runtime_consumer(){ + let dir=tempfile::tempdir().unwrap();let rt=runtime(dir.path(),START); + rt.configure_fresh_train_343_capture(&rt.mutation_guard().unwrap(),context(),"synthetic-old-reader-fixture".into(),START).unwrap(); + let now=settle_async_boundary(&rt,START);rt.cancel_temporal_durability(&rt.mutation_guard().unwrap()).unwrap(); + let loaded=rt.temporal_store().unwrap().load_temporal(now).unwrap().unwrap();assert_eq!(loaded.prefix.unwrap().transition_limit(),343); + if let Some(output)=std::env::var_os("RUVIEW_2473_SYNTHETIC_FIXTURE_OUTPUT"){ + fn copy_private(from:&Path,to:&Path){ + std::fs::create_dir(to).unwrap(); + #[cfg(unix)]{use std::os::unix::fs::PermissionsExt;std::fs::set_permissions(to,std::fs::Permissions::from_mode(0o700)).unwrap();} + for e in std::fs::read_dir(from).unwrap(){let e=e.unwrap();let m=std::fs::symlink_metadata(e.path()).unwrap();assert!(!m.file_type().is_symlink()); + if m.is_dir(){copy_private(&e.path(),&to.join(e.file_name()));}else{assert!(m.is_file());std::fs::copy(e.path(),to.join(e.file_name())).unwrap(); + #[cfg(unix)]{use std::os::unix::fs::PermissionsExt;std::fs::set_permissions(to.join(e.file_name()),std::fs::Permissions::from_mode(0o600)).unwrap();} + } + } + } + let output=std::path::PathBuf::from(output);std::fs::create_dir_all(&output).unwrap();copy_private(dir.path(),&output.join("fresh343")); + std::fs::write(output.join("fresh343.json"),serde_json::to_vec(&json!({"now_ns":now,"owner_scope":context().owner_scope,"installation_id":context().installation_id})).unwrap()).unwrap(); + } +} + +// Synthetic temp fixture only: no symbolic links, public state or user profile. +fn copy_synthetic_budget_fixture(from:&Path,to:&Path){ + std::fs::create_dir(to).unwrap(); + #[cfg(unix)]{use std::os::unix::fs::PermissionsExt;std::fs::set_permissions(to,std::fs::Permissions::from_mode(0o700)).unwrap();} + for entry in std::fs::read_dir(from).unwrap(){let entry=entry.unwrap();let m=std::fs::symlink_metadata(entry.path()).unwrap();assert!(!m.file_type().is_symlink()); + if m.is_dir(){copy_synthetic_budget_fixture(&entry.path(),&to.join(entry.file_name()));}else{assert!(m.is_file());std::fs::copy(entry.path(),to.join(entry.file_name())).unwrap(); + #[cfg(unix)]{use std::os::unix::fs::PermissionsExt;std::fs::set_permissions(to.join(entry.file_name()),std::fs::Permissions::from_mode(0o600)).unwrap();} + } + } +} + +#[test] +fn historical329_and_fresh343_priming_keep_the_original_cadence_band() { + for fresh in [false,true] { + let directory=tempfile::tempdir().unwrap();let rt=runtime(directory.path(),START); + if fresh {rt.configure_fresh_train_343_capture(&rt.mutation_guard().unwrap(),context(),"synthetic-old-profile-rate".into(),START).unwrap();} + else {rt.configure_temporal_capture(&rt.mutation_guard().unwrap(),context(),"synthetic-old-profile-rate".into(),START).unwrap();} + let now=settle_async_boundary(&rt,START); + let iq=vec![3u8;128]; + let packet=canonicalize_esp32(NativePacketInput{node_id:1,n_antennas:1,raw_bins:64,frequency_hz:2_437_000_000,ppdu_type:PpduType::HtLegacy,sequence:1,received_at_ns:now,receiver_sample_rate_hz:10.,iq_bytes:&iq,flags:Adr018Flags::default(),provenance:CaptureProvenance::LiveReceiverObserved}).unwrap(); + assert_eq!(rt.ingest_native(&packet,now),Err("acquisition_cadence_changed".into()),"fresh343={fresh}"); + assert_eq!(snapshot(&rt,now)["retained_rows"],0); + assert_eq!(snapshot(&rt,now)["durable_windows"],0); + assert_eq!(snapshot(&rt,now)["failure"],"acquisition_cadence_changed"); + } +} + +fn failed_full_capture_for_retry(root:&Path,lost_ack:bool)->(Arc,u64){ + let rt=Arc::new(runtime(root,START)); + rt.configure_temporal_full_run(&rt.mutation_guard().unwrap(),context(),"synthetic-stopped-retry".into(),START).unwrap(); + let ready=settle_async_boundary(&rt,START); + let start=async_lead_end(&rt,ready); + let publication_gate=if lost_ack { + let state=rt.inner.lock().unwrap();let Some(CaptureWorkflow::Temporal(c))=&state.capture else{panic!()}; + let writer=c.core.as_ref().unwrap().writer().unwrap();writer.lose_next_ack(); + let(entered_tx,entered_rx)=std::sync::mpsc::sync_channel(1);let(release_tx,release_rx)=std::sync::mpsc::sync_channel(1); + writer.hold_next_after_publish(entered_tx,release_rx);Some((entered_rx,release_tx)) + }else{None}; + for source in 1..=3u8 { + let at=start+u64::from(source)*1000; + rt.ingest_native(&packet(source,1,at),at).unwrap(); + } + if let Some((entered,_))=&publication_gate{entered.recv_timeout(std::time::Duration::from_secs(10)).unwrap();} + let at=start+1_001_000; + assert!(rt.ingest_native(&packet(1,2,at),at).unwrap_err().starts_with("temporal_observed_gap_or_rate:")); + if let Some((_,release))=publication_gate{release.send(()).unwrap();} + assert_eq!(snapshot(&rt,at)["stage"],"failed"); + (rt,at+1_000_000) +} + +#[test] +fn explicit_failed_capture_retry_reconciles_actual_head_without_changing_identity_or_deadline(){ + for lost_ack in [false,true] { + let directory=tempfile::tempdir().unwrap();let(rt,at)=failed_full_capture_for_retry(directory.path(),lost_ack); + let before=rt.temporal_status_v2(at).unwrap(); + let guard=rt.mutation_guard().unwrap(); + let id=before["capture"]["capture_generation_id"].as_str().unwrap().to_owned(); + rt.retry_failed_temporal_capture(&guard,&id,at).unwrap(); + let after=rt.temporal_status_v2(at+1).unwrap(); + assert_eq!(after["capture"]["stage"],"paused"); + assert_eq!(after["capture"]["failure"],Value::Null); + assert_eq!(after["capture"]["durability_pending"],false); + for field in ["capture_generation_id","schedule_sha256","continuation_policy_sha256","original_deadline","retained_windows","durable_windows","sealed_train_episodes","sealed_held_out_episodes"] { + assert_eq!(after["capture"][field],before["capture"][field],"{field}"); + } + assert_eq!(after["mutation_guard"]["generation"],before["mutation_guard"]["generation"]); + assert_eq!(after["mutation_guard"]["runtime_epoch"],before["mutation_guard"]["runtime_epoch"]); + assert!(rt.retry_failed_temporal_capture(&guard,&id,at+2).is_err(),"old command is consumed"); + assert_eq!(after["capture"]["observed_ms"],0); + } +} + +#[test] +fn retry_cannot_reconcile_a_writer_after_terminal_owner_drain(){ + let directory=tempfile::tempdir().unwrap();let(rt,at)=failed_full_capture_for_retry(directory.path(),true); + let writer={let state=rt.inner.lock().unwrap();let Some(CaptureWorkflow::Temporal(c))=&state.capture else{panic!()};c.core.as_ref().unwrap().writer().unwrap()}; + assert!(writer.has_failed_repair(),"actual publication happened and its ACK was withheld/lost"); + writer.drain().unwrap(); + assert!(!writer.has_failed_repair(),"terminal drain releases captured plaintext repair buffers"); + fn files(root:&Path)->Vec<(std::path::PathBuf,Vec)>{let mut result=vec![];for e in std::fs::read_dir(root).unwrap(){let e=e.unwrap();if e.file_type().unwrap().is_dir(){result.extend(files(&e.path()));}else{result.push((e.path(),std::fs::read(e.path()).unwrap()));}}result.sort_by(|a,b|a.0.cmp(&b.0));result} + let before=files(directory.path()); + assert_eq!(writer.reconcile_failed(at).unwrap_err(),"temporal_writer_terminally_drained"); + assert_eq!(files(directory.path()),before); +} + +#[test] +fn profile_namespace_selection_is_guarded_and_default_reads_cannot_undo_explicit_choice(){ + let directory=tempfile::tempdir().unwrap();let rt=runtime(directory.path(),START); + let old_guard=rt.mutation_guard().unwrap(); + rt.temporal_default_operator_profile_v2().unwrap(); + assert!(rt.operator_profile_selected.load(Ordering::Acquire)); + assert!(rt.temporal_select_profile_guarded_v2(None,&old_guard).is_err()); + assert!(rt.operator_profile_selected.load(Ordering::Acquire)); + let guard=rt.mutation_guard().unwrap(); + let after=rt.temporal_select_profile_guarded_v2(None,&guard).unwrap(); + assert!(!rt.operator_profile_selected.load(Ordering::Acquire)); + rt.temporal_default_operator_profile_v2().unwrap(); + assert!(!rt.operator_profile_selected.load(Ordering::Acquire),"read must not race an explicit old-plan restore"); + assert_eq!(rt.mutation_guard().unwrap(),after); +} + +#[test] +fn failed_capture_retry_rejects_wrong_identity_stale_owner_and_expiry_without_success(){ + let directory=tempfile::tempdir().unwrap();let(rt,at)=failed_full_capture_for_retry(directory.path(),false); + let before=rt.temporal_status_v2(at).unwrap(); + let id=before["capture"]["capture_generation_id"].as_str().unwrap(); + let mut stale=rt.mutation_guard().unwrap();stale.runtime_epoch="another-runtime".into(); + assert!(rt.retry_failed_temporal_capture(&stale,id,at).is_err()); + assert!(rt.retry_failed_temporal_capture(&rt.mutation_guard().unwrap(),&"f".repeat(64),at).is_err()); + let expiry=before["capture"]["original_deadline"]["expires_at_unix_ms"].as_u64().unwrap()*1_000_000; + assert!(rt.retry_failed_temporal_capture(&rt.mutation_guard().unwrap(),id,expiry).is_err()); + let state=rt.inner.lock().unwrap();let Some(CaptureWorkflow::Temporal(c))=&state.capture else{panic!()}; + let capture=c.core.as_ref().unwrap().public_capture(&rt.runtime_epoch,None,at); + assert_eq!(capture.stage,"failed");assert_eq!(capture.retained_windows,0);assert_eq!(capture.durable_windows,0); + assert_eq!(capture.original_deadline.expires_at_unix_ms,expiry/1_000_000); +} + +#[test] +fn retry_rejects_count_equal_but_wrong_known_accepted_prefix_digest(){ + let directory=tempfile::tempdir().unwrap();let(rt,at)=failed_full_capture_for_retry(directory.path(),false); + let id={let mut state=rt.inner.lock().unwrap();let Some(CaptureWorkflow::Temporal(c))=state.capture.as_mut()else{panic!()};let core=c.core.as_mut().unwrap();let id=core.public_generation_id().to_owned();core.test_corrupt_retry_prefix();id}; + assert_eq!(rt.retry_failed_temporal_capture(&rt.mutation_guard().unwrap(),&id,at).unwrap_err(),"temporal_retry_committed_prefix_changed"); + let state=rt.inner.lock().unwrap();let Some(CaptureWorkflow::Temporal(c))=&state.capture else{panic!()}; + assert_eq!(c.core.as_ref().unwrap().stage(),TemporalCaptureStage::Failed); +} + +#[test] +fn status_refuses_profile_and_recovery_snapshot_mixed_across_a_guarded_selection(){ + let directory=tempfile::tempdir().unwrap();let old=runtime(directory.path(),START); + old.configure_temporal_full_run(&old.mutation_guard().unwrap(),context(),"retained-old-plan".into(),START).unwrap(); + let settled=settle_async_boundary(&old,START);drop(old); + let rt=Arc::new(runtime(directory.path(),settled+1)); + rt.temporal_select_profile_v2(None).unwrap(); + let weak=Arc::downgrade(&rt); + *rt.temporal_status_profile_hook.lock().unwrap()=Some(Box::new(move||{let rt=weak.upgrade().unwrap();rt.temporal_select_profile_v2(Some(crate::room_learner::managed_schedule_contract::OPERATOR_EIGHT_PROFILE)).unwrap();})); + assert_eq!(rt.temporal_status_v2(settled+2).unwrap_err(),"learner_state_changed"); + let coherent=rt.temporal_status_v2(settled+3).unwrap(); + assert_eq!(coherent["temporal_profile"]["profile_id"],crate::room_learner::managed_schedule_contract::OPERATOR_EIGHT_PROFILE); + assert!(coherent["recovery"].is_null(),"new namespace cannot show the old plan recovery ID"); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_tests.rs new file mode 100644 index 0000000000..296dc34c61 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_tests.rs @@ -0,0 +1,563 @@ +//! @tier durable — SYNTHETIC packets through the actual managed public path. +use super::*; +use crate::room_learner::{ + capture::{MIN_PHASE_MS, WINDOW_PACKETS}, + observation::{RoomLabel, SourceSpec}, +}; +use ruview_unified::RfModality; +use std::time::{Duration, Instant}; +pub(super) const START: u64 = 1_000_000_000_000; +pub(super) fn context() -> CaptureContext { + CaptureContext { + owner_scope: "a".repeat(64), + installation_id: "synthetic-install".into(), + room_digest: "synthetic-room".into(), + revision: 1, + sources: (1..=2) + .map(|id| SourceSpec { + source_id: id, + device_id: format!("synthetic-esp32-{id}"), + grid_id: "ht64-canonical8".into(), + feature_count: 8, + signal_kind: RfModality::WifiCsi, + link_id: format!("synthetic-link-{id}"), + link_index: 0, + center_frequency_hz: 2_437_000_000, + bandwidth_hz: 20_000_000, + geometry: None, + }) + .collect(), + } +} +pub(super) fn runtime(root: &Path) -> Arc { + Arc::new( + RoomLearningRuntime::new( + &root.join("a".repeat(64)).join("SensingServer"), + &"a".repeat(64), + "synthetic-install", + ) + .unwrap(), + ) +} +pub(super) fn phase(rt: &RoomLearningRuntime, role: DataRole, label: RoomLabel, index: u32) -> u64 { + let at = START + u64::from(index) * 13_000_000_000; + rt.begin_phase( + &rt.mutation_guard().unwrap(), + PhaseRequest { + trial_id: format!("synthetic-trial-{index}"), + session_id: format!("synthetic-session-{index}"), + role, + label, + duration_ms: MIN_PHASE_MS, + lead_in_ms: 0, + conditions_confirmed: true, + }, + at, + ) + .unwrap(); + let gain = match label { + RoomLabel::Empty => 2.0, + RoomLabel::OccupiedStatic => 6.0, + RoomLabel::OccupiedMoving => 7.0, + RoomLabel::AdjacentOnly => 2.3, + RoomLabel::Unknown => panic!("not labelled"), + }; + for quarter in 0..4 { + for sample in 0..WINDOW_PACKETS { + for source in 1..=2 { + let t = at + + quarter * 3_000_000_000 + + 1_000_000 + + sample as u64 * 10_000_000 + + u64::from(source); + let values: Vec<_> = (0..8) + .map(|bin| { + gain + f64::from(source) * 0.3 + bin as f64 * 0.04 + sample as f64 * 0.003 + }) + .collect(); + rt.ingest( + RawSourceFrame { + source_id: source, + grid_id: "ht64-canonical8", + frequency_hz: 2_437_000_000, + bandwidth_hz: 20_000_000, + sequence: index * 32 + quarter as u32 * 8 + sample as u32 + 1, + amplitudes: &values, + }, + t, + ) + .unwrap(); + } + } + } + rt.finish_phase( + &rt.mutation_guard().unwrap(), + at + MIN_PHASE_MS * 1_000_000, + true, + ) + .unwrap(); + at + MIN_PHASE_MS * 1_000_000 +} +pub(super) fn train_phases(rt: &RoomLearningRuntime) { + for (index, label) in [ + RoomLabel::Empty, + RoomLabel::OccupiedStatic, + RoomLabel::OccupiedMoving, + RoomLabel::AdjacentOnly, + ] + .into_iter() + .enumerate() + { + phase(rt, DataRole::Train, label, index as u32); + } +} +pub(super) async fn finished(rt: &RoomLearningRuntime) -> Value { + let start = Instant::now(); + loop { + let s = rt.status(START + 60_000_000_000); + if !matches!(s["state"].as_str(), Some("running" | "cancelling")) { + return s; + } + assert!( + start.elapsed() < Duration::from_secs(65), + "bounded synthetic training did not finish: {s}" + ); + tokio::time::sleep(Duration::from_millis(5)).await; + } +} +#[tokio::test] +async fn real_capture_train_private_store_cold_load_and_unlabelled_inference_compose() { + let dir = tempfile::tempdir().unwrap(); + let rt = runtime(dir.path()); + let ctx = context(); + rt.configure( + &rt.mutation_guard().unwrap(), + ctx.clone(), + "synthetic-run".into(), + true, + ) + .unwrap(); + train_phases(&rt); + rt.start_training( + &rt.mutation_guard().unwrap(), + FitConfig { + epochs: 4, // Composition needs real updates, not accuracy convergence. + ..FitConfig::default() + }, + ) + .unwrap(); + let trained = finished(&rt).await; + assert_eq!(trained["state"], "ready", "{trained}"); + assert!( + trained["training_report"]["encoder_delta_l2"] + .as_f64() + .unwrap() + > 0.0 + ); + assert_eq!(trained["physically_validated"], false); + assert_eq!(trained["control_detector_changed"], false); + assert!(rt + .begin_phase( + &rt.mutation_guard().unwrap(), + PhaseRequest { + trial_id: "late-train".into(), + session_id: "late-session".into(), + role: DataRole::Train, + label: RoomLabel::Empty, + duration_ms: MIN_PHASE_MS, + lead_in_ms: 0, + conditions_confirmed: true + }, + START + 60_000_000_000 + ) + .is_err()); + phase(&rt, DataRole::HeldOut, RoomLabel::Empty, 4); + phase(&rt, DataRole::HeldOut, RoomLabel::OccupiedStatic, 5); + let report = rt + .evaluate(&rt.mutation_guard().unwrap(), DataRole::HeldOut) + .unwrap(); + assert_eq!(report["adoption_authorized"], false); + assert!(rt + .evaluate(&rt.mutation_guard().unwrap(), DataRole::Train) + .is_err()); + let cold = runtime(dir.path()); + cold.configure( + &cold.mutation_guard().unwrap(), + ctx, + "synthetic-cold".into(), + false, + ) + .unwrap(); + assert_eq!(cold.status(START)["candidate_loaded"], true); + for row in 0..8 { + for source in 1..=2 { + let values = vec![6.0 + f64::from(source) * 0.3; 8]; + cold.ingest( + RawSourceFrame { + source_id: source, + grid_id: "ht64-canonical8", + frequency_hz: 2_437_000_000, + bandwidth_hz: 20_000_000, + // Cold resume retains the original wire-sequence fence. + sequence: 1_000 + row, + amplitudes: &values, + }, + START + 100_000_000_000 + u64::from(row) * 10_000_000 + u64::from(source), + ) + .unwrap(); + } + } + let (_, prediction) = cold.prediction(START + 100_080_000_000).unwrap(); + assert!(prediction.occupied_probability.is_some()); + assert!(!prediction.physically_validated); + super::evaluation_tests::assert_no_raw_observation_arrays(&cold.status(START)); +} +#[tokio::test] +async fn cancellation_and_context_replacement_cannot_publish_a_late_model() { + let dir = tempfile::tempdir().unwrap(); + let rt = runtime(dir.path()); + let ctx = context(); + rt.configure( + &rt.mutation_guard().unwrap(), + ctx.clone(), + "synthetic-cancel".into(), + true, + ) + .unwrap(); + train_phases(&rt); + rt.start_training( + &rt.mutation_guard().unwrap(), + FitConfig { + epochs: 128, + ..FitConfig::default() + }, + ) + .unwrap(); + assert!(rt + .configure( + &rt.mutation_guard().unwrap(), + ctx.clone(), + "other-run".into(), + true + ) + .is_err()); + assert_eq!( + rt.cancel_training(&rt.mutation_guard().unwrap(),).unwrap(), + JobPhase::Cancelling + ); + assert_eq!(finished(&rt).await["state"], "cancelled"); + let cold = runtime(dir.path()); + assert!(cold + .configure( + &cold.mutation_guard().unwrap(), + ctx, + "no-late-model".into(), + false + ) + .is_err()); + assert_eq!(cold.status(START)["candidate_loaded"], false); +} +#[tokio::test] +async fn wrong_owner_and_incomplete_training_leave_records_and_model_store_untouched() { + let dir = tempfile::tempdir().unwrap(); + let rt = runtime(dir.path()); + let mut ctx = context(); + ctx.owner_scope = "b".repeat(64); + assert!(rt + .configure(&rt.mutation_guard().unwrap(), ctx, "wrong".into(), true) + .is_err()); + rt.configure( + &rt.mutation_guard().unwrap(), + context(), + "incomplete".into(), + true, + ) + .unwrap(); + phase(&rt, DataRole::Train, RoomLabel::Empty, 0); + assert!(rt + .start_training(&rt.mutation_guard().unwrap(), FitConfig::default()) + .is_err()); + assert_eq!(rt.status(START)["state"], "idle"); + assert_eq!(rt.status(START)["capture"]["sealed_cohorts"], 4); + assert!(rt.prediction(START).is_err()); +} + +#[tokio::test] +async fn operator_refusal_discards_only_unsealed_examples_and_bad_config_does_not_freeze_data() { + let dir = tempfile::tempdir().unwrap(); + let rt = runtime(dir.path()); + rt.configure( + &rt.mutation_guard().unwrap(), + context(), + "synthetic-refusal".into(), + true, + ) + .unwrap(); + phase(&rt, DataRole::Train, RoomLabel::Empty, 0); + rt.begin_phase( + &rt.mutation_guard().unwrap(), + PhaseRequest { + trial_id: "refused".into(), + session_id: "refused-session".into(), + role: DataRole::Train, + label: RoomLabel::OccupiedStatic, + duration_ms: MIN_PHASE_MS, + lead_in_ms: 0, + conditions_confirmed: true, + }, + START + 13_000_000_000, + ) + .unwrap(); + assert!(rt + .finish_phase(&rt.mutation_guard().unwrap(), START + 25_000_000_000, false) + .is_err()); + assert_eq!( + rt.status(START + 25_000_000_000)["capture"]["sealed_cohorts"], + 4 + ); + assert_eq!(rt.status(START + 25_000_000_000)["training_frozen"], false); + assert!(rt + .start_training( + &rt.mutation_guard().unwrap(), + FitConfig { + epochs: 0, + ..FitConfig::default() + } + ) + .is_err()); + assert_eq!(rt.status(START)["training_frozen"], false); +} +#[tokio::test] +async fn cancelled_training_can_retry_frozen_data_without_recapture_or_reopening_train() { + let dir = tempfile::tempdir().unwrap(); + let rt = runtime(dir.path()); + rt.configure( + &rt.mutation_guard().unwrap(), + context(), + "synthetic-retry".into(), + true, + ) + .unwrap(); + train_phases(&rt); + rt.start_training( + &rt.mutation_guard().unwrap(), + FitConfig { + epochs: 128, + ..FitConfig::default() + }, + ) + .unwrap(); + rt.cancel_training(&rt.mutation_guard().unwrap()).unwrap(); + assert_eq!(finished(&rt).await["state"], "cancelled"); + assert_eq!(rt.status(START)["capture"]["sealed_cohorts"], 16); + rt.start_training( + &rt.mutation_guard().unwrap(), + FitConfig { + epochs: 4, + ..FitConfig::default() + }, + ) + .unwrap(); + assert_eq!(finished(&rt).await["state"], "ready"); + assert_eq!(rt.status(START)["capture"]["sealed_cohorts"], 16); +} +#[test] +fn typed_synthetic_native_input_cannot_be_relabelled_live_by_runtime() { + use crate::room_learner::{ + native::{canonicalize_esp32, NativePacketInput}, + observation::CaptureProvenance, + }; + use wifi_densepose_hardware::{Adr018Flags, PpduType}; + let dir = tempfile::tempdir().unwrap(); + let rt = runtime(dir.path()); + let bytes: Vec<_> = (0..64).flat_map(|_| [3u8, 4]).collect(); + for provenance in [ + CaptureProvenance::Synthetic, + CaptureProvenance::CapturedReplay, + ] { + let packet = canonicalize_esp32(NativePacketInput { + node_id: 1, + n_antennas: 1, + raw_bins: 64, + frequency_hz: 2_437_000_000, + ppdu_type: PpduType::HtLegacy, + sequence: 1, + received_at_ns: START, + receiver_sample_rate_hz: 10.0, + iq_bytes: &bytes, + flags: Adr018Flags::default(), + provenance, + }) + .unwrap(); + assert!(rt.ingest_native(&packet, START).is_err()); + } +} + +#[test] +fn delayed_commands_cannot_cross_room_phase_or_helper_restart() { + let dir = tempfile::tempdir().unwrap(); + let rt = runtime(dir.path()); + let initial = rt.mutation_guard().unwrap(); + rt.configure(&initial, context(), "run-a".into(), true) + .unwrap(); + let room_a = rt.mutation_guard().unwrap(); + let mut b = context(); + b.room_digest = "other-room".into(); + rt.configure(&room_a, b.clone(), "run-b".into(), true) + .unwrap(); + let request = PhaseRequest { + trial_id: "delayed-a".into(), + session_id: "a".into(), + role: DataRole::Train, + label: RoomLabel::Empty, + duration_ms: MIN_PHASE_MS, + lead_in_ms: 0, + conditions_confirmed: true, + }; + assert_eq!( + rt.begin_phase(&room_a, request.clone(), START).unwrap_err(), + "learner_state_changed" + ); + assert_eq!( + rt.configure(&room_a, context(), "late-config".into(), true) + .unwrap_err(), + "learner_state_changed" + ); + let idle_b = rt.mutation_guard().unwrap(); + rt.begin_phase(&idle_b, request.clone(), START).unwrap(); + let phase_one = rt.mutation_guard().unwrap(); + rt.cancel_phase(&phase_one).unwrap(); + let mut next = request; + next.trial_id = "fresh-b".into(); + rt.begin_phase(&rt.mutation_guard().unwrap(), next, START) + .unwrap(); + assert_eq!( + rt.finish_phase(&phase_one, START + 13_000_000_000, true) + .unwrap_err(), + "learner_state_changed" + ); + assert_eq!( + rt.cancel_phase(&phase_one).unwrap_err(), + "learner_state_changed" + ); + assert_eq!( + rt.cancel_training(&phase_one).unwrap_err(), + "learner_state_changed" + ); + assert_eq!( + rt.start_training(&phase_one, FitConfig::default()) + .unwrap_err(), + "learner_state_changed" + ); + assert_eq!( + rt.evaluate(&phase_one, DataRole::HeldOut).unwrap_err(), + "learner_state_changed" + ); + assert_eq!( + rt.mutation_guard().unwrap().phase.unwrap().trial_id, + "fresh-b" + ); + let cold = runtime(dir.path()); + assert_ne!( + initial.runtime_epoch, + cold.mutation_guard().unwrap().runtime_epoch + ); + assert_eq!( + cold.configure(&initial, b, "stale-after-restart".into(), true) + .unwrap_err(), + "learner_state_changed" + ); + assert!(cold.context().is_none()); +} +#[test] +fn advertised_defaults_fit_all_native_source_counts_before_capture() { + for count in 2..=8 { + let dir = tempfile::tempdir().unwrap(); + let rt = runtime(dir.path()); + let mut ctx = context(); + let template = ctx.sources[0].clone(); + ctx.sources = (1..=count) + .map(|id| { + let mut spec = template.clone(); + spec.source_id = id as u8; + spec.device_id = format!("synthetic-device-{id}"); + spec.link_id = format!("synthetic-link-{id}"); + spec.feature_count = 56; + spec + }) + .collect(); + rt.configure( + &rt.mutation_guard().unwrap(), + ctx.clone(), + "budget".into(), + true, + ) + .unwrap(); + let defaults: FitConfig = + serde_json::from_value(rt.status(START)["training_defaults"].clone()).unwrap(); + let actual = defaults.validate_context(&ctx).unwrap(); + assert!(actual <= 512, "{count} sources require {actual} parameters"); + if count >= 7 { + assert!(FitConfig::default().validate_context(&ctx).is_err()); + } + } +} + +#[test] +fn native_rejection_diagnostics_are_context_bound_bounded_and_not_admission() { + let dir=tempfile::tempdir().unwrap();let rt=runtime(dir.path()); + rt.configure(&rt.mutation_guard().unwrap(),context(),"native-diagnostic".into(),true).unwrap(); + let observed=rt.mutation_guard().unwrap(); + rt.record_native_rejection(17,"ht64-canonical8","unselected"); + rt.record_native_rejection(1,"other-grid","unselected"); + assert!(rt.status(START)["input_error"].is_null()); + rt.record_native_rejection(1,"ht64-canonical8",&"x".repeat(1000)); + let status=rt.status(START); + assert!(status["input_error"].as_str().unwrap().starts_with("Node 1 native input rejected:")); + assert!(status["input_error"].as_str().unwrap().len()<=256); + assert_eq!(rt.mutation_guard().unwrap(),observed); + assert_eq!(status["capture"]["sources"][0]["admitted_packets"],0); + assert_eq!(status["physically_validated"],false); +} + +#[test] +fn protected_progress_explains_rejections_without_advancing_admission_or_live_authority() { + let dir = tempfile::tempdir().unwrap(); + let rt = runtime(dir.path()); + rt.configure(&rt.mutation_guard().unwrap(), context(), "synthetic-sequence-diagnostics".into(), true).unwrap(); + let values = vec![2.0; 8]; + let input = |source_id, sequence| RawSourceFrame { + source_id, grid_id: "ht64-canonical8", frequency_hz: 2_437_000_000, + bandwidth_hz: 20_000_000, sequence, amplitudes: &values, + }; + for sample in 0..8 { + for source in 1..=2 { + rt.ingest(input(source, 100 + sample), START + 1 + u64::from(sample) * 10_000_000 + u64::from(source)).unwrap(); + } + } + assert!(rt.status(START + 80_000_000)["capture"]["sources"].as_array().unwrap().iter().all(|s| s["window_eligible"] == true)); + assert!(rt.ingest(input(1, 1), START + 90_000_000).is_err()); + let status = rt.status(START + 90_000_000); + let source = &status["capture"]["sources"][0]; + assert_eq!(source["last_accepted_sequence"], 107); + assert_eq!(source["last_observed_sequence"], 1); + assert_eq!(source["sequence_rejection_count"], 1); + assert_eq!(source["sequence_rejection_reason"], "reversed"); + assert_eq!(source["admitted_packets"], 8); + assert!(status["capture"]["sources"].as_array().unwrap().iter().all(|s| s["live_buffer_frames"] == 0 && s["window_eligible"] == false)); + assert_eq!(status["input_error"], "replayed or reversed source sequence"); + let json = serde_json::to_string(&status["capture"]).unwrap(); + for raw in ["samples", "amplitudes", "frame_ids", "sequence_ids"] { + assert!(!json.contains(raw)); + } + for sample in 0..8 { + for source in 1..=2 { + rt.ingest(input(source, 108 + sample), START + 100_000_000 + u64::from(sample) * 10_000_000 + u64::from(source)).unwrap(); + } + } + let fresh = rt.status(START + 180_000_000); + assert!(fresh["input_error"].is_null()); + assert_eq!(fresh["capture"]["sources"][0]["sequence_rejection_count"], 1); + assert!(fresh["capture"]["sources"][0]["sequence_rejection_reason"].is_null()); + assert!(fresh["capture"]["sources"].as_array().unwrap().iter().all(|s| s["last_accepted_sequence"] == 115 && s["last_observed_sequence"] == 115 && s["window_eligible"] == true)); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/managed_capture.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/managed_capture.rs new file mode 100644 index 0000000000..4572cbed0a --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/managed_capture.rs @@ -0,0 +1,280 @@ +//! Private native-session provenance; no imported rows or invented phase acknowledgements. +use super::*; +use crate::room_learner::{ + managed_training::{ + commitment, ManagedTrainingProof, PhaseClaim, SourceWatermark, TrainingProofSeed, + TrainingSourceRange, + }, + qualification::{hash, CONDITIONS}, +}; +impl CaptureSession { + pub(super) fn sealed_cohort_capacity_used(&self) -> usize { + self.metadata_lineage.as_ref().map_or(0, |s| { + s.phase_claims.iter().map(|p| p.cohort_count).sum::() + }) + self.cohorts.len() + } + pub(crate) fn minimum_receiver_time_ns(&self) -> u64 { + self.last_seen_ns + } + pub(crate) fn metadata_lineage(&self) -> Option<&TrainingProofSeed> { + self.metadata_lineage.as_ref() + } + pub(super) fn phase_index(&self) -> usize { + self.metadata_lineage + .as_ref() + .map_or(0, |s| s.phase_claims.len()) + + self.history.len() + } + pub(crate) fn managed_training_seed(&self) -> Result { + if self.phase_active() || self.metadata_lineage.is_some() { + return Err( + "managed training proof requires original completed native Train capture".into(), + ); + } + // Validate the actual capture's frame namespace, source counters, phase + // identity and sealed-row correspondence, not a caller's Live claim. + self.checkpoint(self.last_seen_ns)?; + let dataset = self.validated_dataset()?; + let rows = dataset.train(); + let counts = CONDITIONS.map(|label| rows.iter().filter(|r| r.label == label).count()); + if counts.iter().any(|n| *n < COHORTS_PER_PHASE) { + return Err("managed fit requires four sealed Train cohorts per condition".into()); + } + let mut phases = Vec::new(); + for row in rows { + let matches: Vec<_> = self + .history + .iter() + .filter(|p| { + p.role == DataRole::Train + && p.status == PhaseStatus::Sealed + && p.trial_id == row.trial_id + && p.session_id == row.session_id + && p.label == row.label + }) + .collect(); + if matches.len() != 1 + || rows + .iter() + .filter(|r| { + r.trial_id == row.trial_id + && r.session_id == row.session_id + && r.label == row.label + }) + .count() + != matches[0].cohort_count + { + return Err("Train rows lack unique managed sealed acknowledgement".into()); + } + if !phases.iter().any(|p: &&PhaseReceipt| { + p.trial_id == row.trial_id && p.session_id == row.session_id + }) { + phases.push(matches[0]); + } + } + // Hash identities, times and native metadata; RF sample values never + // enter the metadata artifact (the existing Train SHA remains exact). + let metadata: Vec<_> = rows + .iter() + .map(|r| { + ( + &r.id, + &r.trial_id, + &r.session_id, + r.role, + r.label, + r.views + .iter() + .map(|v| { + ( + v.source_id, + &v.frame_ids, + &v.sequence_ids, + v.first_time_ns, + v.last_time_ns, + v.provenance, + v.clock, + ) + }) + .collect::>(), + ) + }) + .collect(); + let seed = TrainingProofSeed { + context: self.context.clone(), + run_id: self.run_id.clone(), + training_data_sha256: hash(&rows)?, + training_metadata_sha256: hash(&metadata)?, + sealed_train_history_sha256: hash(&phases)?, + train_counts: counts, + phase_claims: self + .history + .iter() + .map(|p| { + Ok(PhaseClaim { + trial_sha256: commitment("trial", &p.trial_id)?, + session_sha256: commitment("session", &p.session_id)?, + role: p.role, + label: p.label, + status: p.status, + cohort_count: p.cohort_count, + }) + }) + .collect::, String>>()?, + sources: self + .context + .sources + .iter() + .map(|s| { + let w = &self.sources[&s.source_id]; + Ok(SourceWatermark { + source_id: s.source_id, + sequence: w.sequence.ok_or("missing original wire sequence")?, + ordinal: w.ordinal, + last_time_ns: w.last_time_ns, + admitted_packets: w.admitted_packets, + train_range: TrainingSourceRange { + first_ordinal: rows + .iter() + .flat_map(|c| &c.views) + .filter(|v| v.source_id == s.source_id) + .flat_map(|v| &v.sequence_ids) + .copied() + .min() + .ok_or("missing Train ordinal")?, + last_ordinal: rows + .iter() + .flat_map(|c| &c.views) + .filter(|v| v.source_id == s.source_id) + .flat_map(|v| &v.sequence_ids) + .copied() + .max() + .ok_or("missing Train ordinal")?, + first_time_ns: rows + .iter() + .flat_map(|c| &c.views) + .filter(|v| v.source_id == s.source_id) + .map(|v| v.first_time_ns) + .min() + .ok_or("missing Train time")?, + last_time_ns: rows + .iter() + .flat_map(|c| &c.views) + .filter(|v| v.source_id == s.source_id) + .map(|v| v.last_time_ns) + .max() + .ok_or("missing Train time")?, + }, + }) + }) + .collect::, String>>()?, + first_observed_ns: rows + .iter() + .flat_map(|c| &c.views) + .map(|v| v.first_time_ns) + .min() + .ok_or("no managed Train time")?, + last_seen_ns: self.last_seen_ns, + source_epoch_verified: false, + receiver_clock_domain: "utc_anchored_monotonic_v1".into(), + }; + seed.validate()?; + Ok(seed) + } + pub(crate) fn resume_managed_proof(proof: &ManagedTrainingProof) -> Result { + proof.seed.validate()?; + let seed = &proof.seed; + let mut session = Self::new(seed.context.clone(), seed.run_id.clone())?; + session.last_seen_ns = seed.last_seen_ns; + for w in &seed.sources { + session.sources.insert( + w.source_id, + SourceBuffer { + sequence: Some(w.sequence), + ordinal: w.ordinal, + last_time_ns: w.last_time_ns, + admitted_packets: w.admitted_packets, + ..SourceBuffer::default() + }, + ); + } + session.metadata_lineage = Some(seed.clone()); + Ok(session) + } +} +impl CaptureSession { + pub(crate) fn managed_fence_seed( + &self, + proof: &ManagedTrainingProof, + ) -> Result { + if self.context != proof.seed.context || self.run_id != proof.seed.run_id { + return Err("capture has a different managed namespace".into()); + } + let mut seed = proof.seed.clone(); + seed.phase_claims = self + .metadata_lineage + .as_ref() + .map_or_else(Vec::new, |m| m.phase_claims.clone()); + for p in &self.history { + seed.phase_claims.push(PhaseClaim { + trial_sha256: commitment("trial", &p.trial_id)?, + session_sha256: commitment("session", &p.session_id)?, + role: p.role, + label: p.label, + status: p.status, + cohort_count: p.cohort_count, + }); + } + seed.sources = self + .context + .sources + .iter() + .map(|s| { + let w = &self.sources[&s.source_id]; + Ok(SourceWatermark { + source_id: s.source_id, + sequence: w.sequence.ok_or("missing original source sequence")?, + ordinal: w.ordinal, + last_time_ns: w.last_time_ns, + admitted_packets: w.admitted_packets, + train_range: proof + .seed + .sources + .iter() + .find(|old| old.source_id == s.source_id) + .ok_or("unknown original Train range")? + .train_range + .clone(), + }) + }) + .collect::, String>>()?; + seed.last_seen_ns = self.last_seen_ns; + seed.ensure_extends(&proof.seed)?; + if let Some(original) = &self.metadata_lineage { + seed.ensure_extends(original)?; + } + Ok(seed) + } + pub(crate) fn resume_managed_fence( + proof: &ManagedTrainingProof, + fence: &crate::room_learner::managed_capture_fences::ManagedCaptureFence, + ) -> Result { + fence.validate(proof)?; + let mut session = Self::resume_managed_proof(proof)?; + session.last_seen_ns = fence.seed.last_seen_ns; + for w in &fence.seed.sources { + session.sources.insert( + w.source_id, + SourceBuffer { + sequence: Some(w.sequence), + ordinal: w.ordinal, + last_time_ns: w.last_time_ns, + admitted_packets: w.admitted_packets, + ..SourceBuffer::default() + }, + ); + } + session.metadata_lineage = Some(fence.seed.clone()); + Ok(session) + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/managed_capture_fence_store.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/managed_capture_fence_store.rs new file mode 100644 index 0000000000..8c4a61a04f --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/managed_capture_fence_store.rs @@ -0,0 +1,267 @@ +//! Private immutable phase-fence chain and atomically published verified head. +use super::*; +use crate::room_learner::{ + managed_capture_fences::ManagedCaptureFence, + managed_training::{ManagedTrainingProof, TrainingProofSeed}, +}; +const HEAD: &str = "managed-sealed-phase-head-v1.json"; +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct FenceHead { + schema: String, + model_sha256: String, + proof_sha256: String, + fence_sha256: String, + revision: u64, +} +impl ArtifactStore { + fn fence_dir(&self, model: &RoomModel, create: bool) -> Result, String> { + self.validate_context(model.context())?; + let Some(dir) = self.context_dir(model.context(), create)? else { + return Ok(None); + }; + evaluation_store::private_subdirectory( + &dir, + &format!("managed-fences-{}", digest(&model.export()?)), + create, + ) + } + fn read_fence_chain( + &self, + model: &RoomModel, + proof: &ManagedTrainingProof, + allow_pending: bool, + ) -> Result, String> { + proof.validate(model)?; + let dir = self + .fence_dir(model, false)? + .ok_or("managed sealed phase ledger missing")?; + let path = dir.join(HEAD); + if !regular_or_missing(&path)? { + return Err("managed sealed phase ledger head missing".into()); + } + let head: FenceHead = + serde_json::from_slice(&private_read(&path).map_err(|e| e.to_string())?) + .map_err(|e| e.to_string())?; + if head.schema != "ruview.managed-sealed-phase-head.v1" + || head.model_sha256 != proof.model_sha256 + || head.proof_sha256 != proof.proof_sha256 + || head.revision >= 128 + || !hex_digest(&head.fence_sha256) + { + return Err("managed sealed phase ledger head mismatch".into()); + } + let mut result = Vec::new(); + let mut sha = head.fence_sha256; + let mut revision = head.revision; + loop { + if !hex_digest(&sha) { + return Err("invalid managed sealed phase ledger hash".into()); + } + let value: ManagedCaptureFence = serde_json::from_slice( + &private_read(&dir.join(format!("{sha}.json"))).map_err(|e| e.to_string())?, + ) + .map_err(|e| e.to_string())?; + value.validate(proof)?; + if value.fence_sha256 != sha || value.revision != revision { + return Err("managed sealed phase ledger chain changed".into()); + } + let previous = value.previous_sha256.clone(); + result.push(value); + if revision == 0 { + break; + } + sha = previous.ok_or("managed sealed phase ledger link missing")?; + revision -= 1; + } + result.reverse(); + for pair in result.windows(2) { + pair[1].seed.ensure_extends(&pair[0].seed)?; + let expected = ManagedCaptureFence::issue(proof, Some(&pair[0]), pair[1].seed.clone())?; + if expected != pair[1] { + return Err("managed phase ledger did not advance exact sealed history".into()); + } + } + let mut entries = 0; + for entry in std::fs::read_dir(&dir).map_err(|e| e.to_string())? { + entries += 1; + if entries > 130 { + return Err("managed sealed phase ledger exceeds capacity".into()); + } + let entry = entry.map_err(|e| e.to_string())?; + let name = entry + .file_name() + .into_string() + .map_err(|_| "invalid managed ledger path")?; + if name == HEAD { + continue; + } + let sha = name + .strip_suffix(".json") + .filter(|s| hex_digest(s)) + .ok_or("unknown pending managed phase artifact")?; + if !regular_or_missing(&entry.path())? { + return Err("managed ledger record disappeared".into()); + } + if !allow_pending && !result.iter().any(|f| f.fence_sha256 == sha) { + return Err("managed sealed phase publication is pending or conflicting".into()); + } + } + Ok(result) + } + pub(crate) fn load_managed_capture_fence( + &self, + model: &RoomModel, + proof: &ManagedTrainingProof, + ) -> Result { + self.read_fence_chain(model, proof, false)? + .pop() + .ok_or("managed capture fence unavailable".into()) + } + fn commit_fence( + &self, + model: &RoomModel, + proof: &ManagedTrainingProof, + fence: &ManagedCaptureFence, + ) -> Result<(), String> { + let dir = self + .fence_dir(model, true)? + .ok_or("managed fence directory unavailable")?; + evaluation_store::write_immutable( + &dir.join(format!("{}.json", fence.fence_sha256)), + &encode(fence)?, + )?; + let head = FenceHead { + schema: "ruview.managed-sealed-phase-head.v1".into(), + model_sha256: proof.model_sha256.clone(), + proof_sha256: proof.proof_sha256.clone(), + fence_sha256: fence.fence_sha256.clone(), + revision: fence.revision, + }; + let bytes = encode(&head)?; + atomic_private_write(&dir.join(HEAD), &bytes).map_err(|e| e.to_string())?; + if private_read(&dir.join(HEAD)).map_err(|e| e.to_string())? != bytes { + return Err("managed capture fence head readback mismatch".into()); + } + if self.load_managed_capture_fence(model, proof)? != *fence { + return Err("managed capture fence chain readback mismatch".into()); + } + Ok(()) + } + pub(crate) fn initialize_managed_capture_fence( + &self, + model: &RoomModel, + proof: &ManagedTrainingProof, + ) -> Result<(), String> { + if let Some(dir) = self.fence_dir(model, false)? { + if regular_or_missing(&dir.join(HEAD))? { + self.load_managed_capture_fence(model, proof)?; + return Ok(()); + } + // Retry only the exact initial inode before any model was installed. + // An existing candidate or another record means historical authority + // may have existed: never replace its missing head with an empty one. + let context = self + .context_dir(model.context(), false)? + .ok_or("managed context missing")?; + if regular_or_missing(&context.join(format!("{}.json", proof.model_sha256)))? { + return Err("existing candidate has an incomplete managed phase ledger".into()); + } + let initial = ManagedCaptureFence::issue(proof, None, proof.seed.clone())?; + let expected = format!("{}.json", initial.fence_sha256); + for entry in std::fs::read_dir(&dir).map_err(|e| e.to_string())? { + let entry = entry.map_err(|e| e.to_string())?; + if entry.file_name().to_str() != Some(expected.as_str()) + || private_read(&entry.path()).map_err(|e| e.to_string())? != encode(&initial)? + { + return Err("managed initial phase publication conflicts".into()); + } + } + } + let initial = ManagedCaptureFence::issue(proof, None, proof.seed.clone())?; + self.commit_fence(model, proof, &initial) + } + pub(crate) fn save_managed_capture_fence( + &self, + model: &RoomModel, + proof: &ManagedTrainingProof, + seed: TrainingProofSeed, + ) -> Result { + let chain = self.read_fence_chain(model, proof, true)?; + let previous = chain + .last() + .ok_or("managed capture fence unavailable")? + .clone(); + seed.ensure_extends(&previous.seed)?; + if seed.phase_claims == previous.seed.phase_claims { + // The same sealed phase may have committed before fsync reported + // failure. Retry its durability barrier, not its capture metadata. + self.load_managed_capture_fence(model, proof)?; + let dir = self + .fence_dir(model, false)? + .ok_or("managed fence directory unavailable")?; + std::fs::File::open(dir) + .and_then(|f| f.sync_all()) + .map_err(|e| e.to_string())?; + return Ok(previous); + } + let dir = self + .fence_dir(model, false)? + .ok_or("managed fence directory unavailable")?; + let mut pending = None; + for entry in std::fs::read_dir(&dir).map_err(|e| e.to_string())? { + let entry = entry.map_err(|e| e.to_string())?; + let name = entry + .file_name() + .into_string() + .map_err(|_| "invalid managed ledger path")?; + if name == HEAD { + continue; + } + let sha = name + .strip_suffix(".json") + .ok_or("invalid managed phase record")?; + if chain.iter().any(|f| f.fence_sha256 == sha) { + continue; + } + let record: ManagedCaptureFence = + serde_json::from_slice(&private_read(&entry.path()).map_err(|e| e.to_string())?) + .map_err(|e| e.to_string())?; + record.validate(proof)?; + if record.fence_sha256 != sha + || pending.is_some() + || record.previous_sha256.as_deref() != Some(previous.fence_sha256.as_str()) + || record.revision != previous.revision + 1 + || record.seed.phase_claims != seed.phase_claims + { + return Err( + "pending managed phase publication conflicts with this exact phase".into(), + ); + } + seed.ensure_extends(&record.seed)?; + pending = Some(record); + } + let next = match pending { + Some(original) => original, + None => ManagedCaptureFence::issue(proof, Some(&previous), seed)?, + }; + self.commit_fence(model, proof, &next)?; + Ok(next) + } + pub(crate) fn require_managed_capture_fence( + &self, + model: &RoomModel, + proof: &ManagedTrainingProof, + sha: &str, + ) -> Result<(), String> { + if self + .read_fence_chain(model, proof, false)? + .iter() + .any(|f| f.fence_sha256 == sha) + { + Ok(()) + } else { + Err("qualification sealed phase fence is not retained".into()) + } + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/managed_capture_fences.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/managed_capture_fences.rs new file mode 100644 index 0000000000..a1147eb70e --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/managed_capture_fences.rs @@ -0,0 +1,114 @@ +//! Bounded immutable metadata for sealed post-fit phases. No RF values. +use super::{ + managed_training::{ManagedTrainingProof, TrainingProofSeed}, + qualification::hash, +}; +use serde::{Deserialize, Serialize}; +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub(crate) struct ManagedCaptureFence { + pub schema: String, + pub revision: u64, + pub previous_sha256: Option, + pub model_sha256: String, + pub managed_training_proof_sha256: String, + pub seed: TrainingProofSeed, + pub fence_sha256: String, +} +impl TrainingProofSeed { + pub(crate) fn ensure_extends(&self, original: &Self) -> Result<(), String> { + self.validate()?; + original.validate()?; + if self.context != original.context + || self.run_id != original.run_id + || self.training_data_sha256 != original.training_data_sha256 + || self.training_metadata_sha256 != original.training_metadata_sha256 + || self.sealed_train_history_sha256 != original.sealed_train_history_sha256 + || self.train_counts != original.train_counts + || self.first_observed_ns != original.first_observed_ns + || self.receiver_clock_domain != original.receiver_clock_domain + || self.phase_claims.len() < original.phase_claims.len() + || self.phase_claims[..original.phase_claims.len()] != original.phase_claims + || self.last_seen_ns < original.last_seen_ns + { + return Err( + "managed sealed capture lineage regressed or changed original training".into(), + ); + } + for (old, new) in original.sources.iter().zip(&self.sources) { + if new.source_id != old.source_id + || new.train_range != old.train_range + || new.ordinal < old.ordinal + || new.last_time_ns < old.last_time_ns + || new.admitted_packets < old.admitted_packets + || new.sequence.wrapping_sub(old.sequence) != (new.ordinal - old.ordinal) as u32 + { + return Err("managed sealed capture source counter/clock regressed".into()); + } + } + Ok(()) + } +} +impl ManagedCaptureFence { + pub(crate) fn issue( + proof: &ManagedTrainingProof, + previous: Option<&Self>, + seed: TrainingProofSeed, + ) -> Result { + seed.ensure_extends(&proof.seed)?; + if let Some(old) = previous { + old.validate(proof)?; + seed.ensure_extends(&old.seed)?; + let added = &seed.phase_claims[old.seed.phase_claims.len()..]; + if added + .iter() + .filter(|p| p.status == super::capture::PhaseStatus::Sealed) + .count() + != 1 + || added + .iter() + .any(|p| p.role == super::observation::DataRole::Train) + { + return Err( + "managed ledger requires exactly one newly sealed post-fit phase".into(), + ); + } + } + let mut value = Self { + schema: "ruview.managed-sealed-phase-fence.v1".into(), + revision: previous.map_or(0, |p| p.revision + 1), + previous_sha256: previous.map(|p| p.fence_sha256.clone()), + model_sha256: proof.model_sha256.clone(), + managed_training_proof_sha256: proof.proof_sha256.clone(), + seed, + fence_sha256: String::new(), + }; + value.fence_sha256 = value.digest()?; + value.validate(proof)?; + Ok(value) + } + fn digest(&self) -> Result { + hash(&( + &self.schema, + self.revision, + &self.previous_sha256, + &self.model_sha256, + &self.managed_training_proof_sha256, + &self.seed, + )) + } + pub(crate) fn validate(&self, proof: &ManagedTrainingProof) -> Result<(), String> { + self.seed.ensure_extends(&proof.seed)?; + if self.schema != "ruview.managed-sealed-phase-fence.v1" + || self.revision >= 128 + || self.model_sha256 != proof.model_sha256 + || self.managed_training_proof_sha256 != proof.proof_sha256 + || self.fence_sha256 != self.digest()? + || (self.revision == 0) != self.previous_sha256.is_none() + || (self.revision == 0 && self.seed != proof.seed) + { + return Err("managed sealed phase fence identity/integrity mismatch".into()); + } + Ok(()) + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/managed_schedule_contract.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/managed_schedule_contract.rs new file mode 100644 index 0000000000..1392863653 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/managed_schedule_contract.rs @@ -0,0 +1,584 @@ +//! Inactive engineering schedule contract for the EXISTING managed learner. +//! No capture/runtime/model/storage caller and no quality/adequacy authority. +use super::{ + observation::{CaptureContext, DataRole, RoomLabel, MAX_COHORTS, MAX_ROWS, MAX_TOTAL_VALUES}, + temporal_acquisition::{ + AcquisitionPlan, ExperimentalPlanId, ReceiverCadenceBand, ResourcePreflight, + SourceWindowPolicy, + }, +}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +const SCHEMA: &str = "ruview.room-learner.trial-schedule.v2"; +const PROFILE: &str = "native-2s-three-source-56bin-option-a-engineering-v1"; +pub(crate) const OPERATOR_EIGHT_PROFILE: &str = "native-2s-three-source-56bin-operator-eight-step-v1"; +const MAX_BYTES: usize = 65_536; +const MAX_PROFILE_ROWS: usize = 77; +const MAX_TRAIN_WINDOWS: usize = 64; + +/// Explicit engineering choice, never a user-facing or scientific default. +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum LeadInPolicy { + Train5HeldOut15EngineeringV1, + OperatorEightStepV1, +} +impl LeadInPolicy { + fn milliseconds(self, role: DataRole, condition: RoomLabel) -> u64 { + if self == Self::OperatorEightStepV1 { + return if matches!(condition, RoomLabel::Empty | RoomLabel::AdjacentOnly) { 15_000 } else { 5_000 }; + } + match role { + DataRole::Train => 5_000, + DataRole::HeldOut => 15_000, + _ => unreachable!(), + } + } +} +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +pub struct WindowSlot { + pub index: usize, + pub target_offset_ms: u64, + pub deadline_offset_ms: u64, +} +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +pub struct Episode { + pub index: usize, + pub episode_key: String, + pub role: DataRole, + pub condition: RoomLabel, + pub target_active_recording_ms: u64, + pub lead_in_ms: u64, + pub requires_explicit_operator_start: bool, + pub requires_conditions_held_confirmation: bool, + pub separately_executed_episode_required: bool, + pub slots: Vec, + pub retained_nominal_window_ms: u64, + pub retained_maximum_window_ms: u64, +} +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +pub struct ScheduleCounts { + pub train_episodes: usize, + pub held_out_episodes: usize, + pub train_windows: usize, + pub held_out_windows: usize, + pub total_windows: usize, + pub train_active_recording_ms: u64, + pub held_out_active_recording_ms: u64, + pub active_recording_ms: u64, + pub train_lead_in_ms: u64, + pub held_out_lead_in_ms: u64, + pub total_lead_in_ms: u64, + pub nominal_recording_and_lead_in_ms: u64, + pub nominal_joint_window_ms: u64, + pub nominal_source_view_window_ms: u64, + pub verified_independent_trial_count: Option, +} +#[derive(Debug, Clone, Serialize)] +struct NativeInputPolicy { + id: &'static str, + original_frame_identity: &'static str, + projection: &'static str, + target_span_ms: u64, + maximum_span_ms: u64, + maximum_first_packet_delay_ms: u64, + maximum_pairwise_receiver_skew_ms: u64, + sources: Vec, +} +#[derive(Debug, Clone, Serialize)] +struct ContractClaims { + capture_ready: bool, + full_episode_quality_monitoring_implemented: bool, + independence_verified: bool, + scientific_adequacy_claimed: bool, + user_facing_default: bool, +} +#[derive(Debug, Clone, Serialize)] +struct Body { + profile_id: &'static str, + context: CaptureContext, + lead_in_policy: LeadInPolicy, + native_input: NativeInputPolicy, + episodes: Vec, + counts: ScheduleCounts, + resources: ResourcePreflight, + serialized_checkpoint_preflight_is_estimate_only: bool, + claims: ContractClaims, +} +#[derive(Serialize)] +struct Export<'a> { + schema: &'static str, + contract_sha256: &'a str, + body: &'a Body, +} +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct Import { + schema: String, + contract_sha256: String, + body: serde_json::Value, +} +/// Immutable metadata, not a preflight/checkpoint receipt or a capture controller. +#[derive(Debug, Clone)] +pub struct ManagedScheduleContract { + body: Body, + sha256: String, +} +fn digest(value: &[u8]) -> String { + format!("{:x}", Sha256::digest(value)) +} +fn hash(body: &Body) -> Result { + let bytes = serde_json::to_vec(body).map_err(|_| "managed_schedule_encoding")?; + let mut h = Sha256::new(); + h.update(SCHEMA); + h.update([0]); + h.update(bytes); + Ok(format!("{:x}", h.finalize())) +} +fn add(a: usize, b: usize) -> Result { + a.checked_add(b) + .ok_or("managed_schedule_capacity_overflow".into()) +} +fn mul(a: usize, b: usize) -> Result { + a.checked_mul(b) + .ok_or("managed_schedule_capacity_overflow".into()) +} +fn bounded_id(value: &str) -> bool { + !value.is_empty() + && value.len() <= 128 + && value.is_ascii() + && !value + .bytes() + .any(|b| b.is_ascii_control() || b.is_ascii_whitespace()) +} +fn sha(value: &str) -> bool { + value.len() == 64 + && value + .bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) +} +fn capacity( + context: &CaptureContext, + sources: &[SourceWindowPolicy], + train: usize, + total: usize, +) -> Result<(usize, usize), String> { + if train > MAX_TRAIN_WINDOWS + || total > MAX_COHORTS + || train == 0 + || total <= train + || sources.len() != context.sources.len() + { + return Err("managed_schedule_window_capacity".into()); + } + let mut rows = 0; + let mut values = 0; + for (spec, source) in context.sources.iter().zip(sources) { + if source.source_id != spec.source_id + || source.max_rows > MAX_ROWS + || source.max_rows > MAX_PROFILE_ROWS + || source.min_rows < 2 + || source.min_rows > source.max_rows + { + return Err("managed_schedule_row_capacity".into()); + } + rows = add(rows, source.max_rows)?; + values = add(values, mul(source.max_rows, spec.feature_count)?)?; + } + let rows = mul(rows, total)?; + let values = mul(values, total)?; + if values > MAX_TOTAL_VALUES { + return Err("managed_schedule_numeric_capacity".into()); + } + Ok((rows, values)) +} +fn slots(duration: u64, count: usize, reserved_ms: u64) -> Result, String> { + if count < 2 + || count > 16 + || reserved_ms == 0 + || duration < reserved_ms + || duration + < reserved_ms + .checked_mul(count as u64) + .ok_or("managed_schedule_slot_overflow")? + { + return Err("managed_schedule_slot_bounds".into()); + } + let stride_span = duration - reserved_ms; + let result = (0..count) + .map(|index| { + let start = stride_span + .checked_mul(index as u64) + .ok_or("managed_schedule_slot_overflow")? + / (count - 1) as u64; + Ok(WindowSlot { + index, + target_offset_ms: start, + deadline_offset_ms: start + reserved_ms, + }) + }) + .collect::, String>>()?; + if result + .windows(2) + .any(|pair| pair[0].deadline_offset_ms > pair[1].target_offset_ms) + { + return Err("managed_schedule_slot_overlap".into()); + } + Ok(result) +} +impl ManagedScheduleContract { + pub fn build(context: CaptureContext, lead_in_policy: LeadInPolicy) -> Result { + context.validate()?; + // Reuse the existing actual-native 2s validators/capacity calculation, + // but NEVER its different schedule, serialized plan or plan hash. + let bands = context + .sources + .iter() + .map(|source| (source.source_id, ReceiverCadenceBand::Hz30To36)) + .collect::>(); + let native = AcquisitionPlan::build( + context.clone(), + ExperimentalPlanId::Native2000msSlots8, + &bands, + )?; + if context.sources.len() != 3 + || context + .sources + .iter() + .any(|source| source.feature_count != 56) + { + return Err("managed_schedule_profile_supports_three_native56_sources".into()); + } + let reserved = native.max_observed_span_ms() + native.max_first_packet_delay_ms(); + let mut episodes = Vec::new(); + if lead_in_policy == LeadInPolicy::OperatorEightStepV1 { + for condition in [RoomLabel::Empty, RoomLabel::OccupiedStatic, RoomLabel::OccupiedMoving, RoomLabel::AdjacentOnly] { + for role in [DataRole::Train, DataRole::Validation] { + let duration = if role == DataRole::Validation { 15_000 } else if condition == RoomLabel::Empty { 600_000 } else { 60_000 }; + let count = if role == DataRole::Train { 16 } else { 4 }; + episodes.push(Episode { + index: episodes.len(), + episode_key: format!("operator-{role:?}-{condition:?}-episode-0"), + role, + condition, + target_active_recording_ms: duration, + lead_in_ms: lead_in_policy.milliseconds(role, condition), + requires_explicit_operator_start: true, + requires_conditions_held_confirmation: true, + separately_executed_episode_required: true, + slots: slots(duration, count, reserved)?, + retained_nominal_window_ms: count as u64 * native.target_span_ms(), + retained_maximum_window_ms: count as u64 * native.max_observed_span_ms(), + }); + } + } + } else { + for role in [DataRole::Train, DataRole::HeldOut] { + for condition in [ + RoomLabel::Empty, + RoomLabel::OccupiedStatic, + RoomLabel::OccupiedMoving, + RoomLabel::AdjacentOnly, + ] { + let empty_train = role == DataRole::Train && condition == RoomLabel::Empty; + for repeat in 0..if empty_train { 1 } else { 2 } { + let duration = if empty_train { 600_000 } else { 30_000 }; + let count = if empty_train { 16 } else { 8 }; + episodes.push(Episode { + index: episodes.len(), + episode_key: format!("{role:?}-{condition:?}-episode-{repeat}"), + role, + condition, + target_active_recording_ms: duration, + lead_in_ms: lead_in_policy.milliseconds(role, condition), + requires_explicit_operator_start: true, + requires_conditions_held_confirmation: true, + separately_executed_episode_required: true, + slots: slots(duration, count, reserved)?, + retained_nominal_window_ms: count as u64 * native.target_span_ms(), + retained_maximum_window_ms: count as u64 * native.max_observed_span_ms(), + }); + } + } + } + } + let role_counts = |role| episodes.iter().filter(|e| e.role == role).count(); + let role_windows = |role| { + episodes + .iter() + .filter(|e| e.role == role) + .map(|e| e.slots.len()) + .sum::() + }; + let role_time = |role| { + episodes + .iter() + .filter(|e| e.role == role) + .map(|e| e.target_active_recording_ms) + .sum::() + }; + let role_lead = |role| { + episodes + .iter() + .filter(|e| e.role == role) + .map(|e| e.lead_in_ms) + .sum::() + }; + let train = role_windows(DataRole::Train); + let check_role = if lead_in_policy == LeadInPolicy::OperatorEightStepV1 { DataRole::Validation } else { DataRole::HeldOut }; + let held = role_windows(check_role); + let total = add(train, held)?; + let (rows, values) = capacity(&context, native.source_policies(), train, total)?; + let resources = if lead_in_policy == LeadInPolicy::OperatorEightStepV1 { + let views = mul(total, context.sources.len())?; + let serialized = add(add(add(mul(values, 25)?, mul(rows, 86)?)?, mul(views, 4096)?)?, add(mul(total, 4096)?, 8 * 1024 * 1024)?)?; + ResourcePreflight { cohorts: total, source_views: views, max_retained_rows: rows, max_numeric_values: values, serialized_upper_bound_bytes: serialized } + } else { native.resources().clone() }; + if resources.cohorts != total + || resources.max_retained_rows != rows + || resources.max_numeric_values != values + { + return Err("managed_schedule_resource_contract_changed".into()); + } + let active = role_time(DataRole::Train) + role_time(check_role); + let lead = role_lead(DataRole::Train) + role_lead(check_role); + let counts = ScheduleCounts { + train_episodes: role_counts(DataRole::Train), + held_out_episodes: role_counts(check_role), + train_windows: train, + held_out_windows: held, + total_windows: total, + train_active_recording_ms: role_time(DataRole::Train), + held_out_active_recording_ms: role_time(check_role), + active_recording_ms: active, + train_lead_in_ms: role_lead(DataRole::Train), + held_out_lead_in_ms: role_lead(check_role), + total_lead_in_ms: lead, + nominal_recording_and_lead_in_ms: active + lead, + nominal_joint_window_ms: total as u64 * native.target_span_ms(), + nominal_source_view_window_ms: total as u64 + * native.target_span_ms() + * context.sources.len() as u64, + verified_independent_trial_count: None, + }; + let body = Body { + profile_id: if lead_in_policy == LeadInPolicy::OperatorEightStepV1 { OPERATOR_EIGHT_PROFILE } else { PROFILE }, + context, + lead_in_policy, + native_input: NativeInputPolicy { + id: "existing-native-2s-56bin-receiver-30to36-v2", + original_frame_identity: "existing-esp32-receiver-ns-node-u32-sequence-v1", + projection: "native-esp32-drop2-linear56-ordered-magnitude-v2-experimental", + target_span_ms: native.target_span_ms(), + maximum_span_ms: native.max_observed_span_ms(), + maximum_first_packet_delay_ms: native.max_first_packet_delay_ms(), + maximum_pairwise_receiver_skew_ms: native.max_pairwise_receiver_skew_ms(), + sources: native.source_policies().to_vec(), + }, + episodes, + counts, + resources, + serialized_checkpoint_preflight_is_estimate_only: true, + claims: ContractClaims { + capture_ready: false, + full_episode_quality_monitoring_implemented: false, + independence_verified: false, + scientific_adequacy_claimed: false, + user_facing_default: lead_in_policy == LeadInPolicy::OperatorEightStepV1, + }, + }; + let result = Self { + sha256: hash(&body)?, + body, + }; + result.export()?; + Ok(result) + } + pub fn export(&self) -> Result, String> { + let bytes = serde_json::to_vec(&Export { + schema: SCHEMA, + contract_sha256: &self.sha256, + body: &self.body, + }) + .map_err(|_| "managed_schedule_encoding")?; + if bytes.len() > MAX_BYTES { + return Err("managed_schedule_size_limit".into()); + } + Ok(bytes) + } + pub fn import(bytes: &[u8]) -> Result { + if bytes.len() > MAX_BYTES { + return Err("managed_schedule_size_limit".into()); + } + let value: Import = + serde_json::from_slice(bytes).map_err(|_| "managed_schedule_invalid_schema")?; + if value.schema != SCHEMA || (value.body["profile_id"] != PROFILE && value.body["profile_id"] != OPERATOR_EIGHT_PROFILE) { + return Err("managed_schedule_unsupported_profile".into()); + } + let context = serde_json::from_value(value.body["context"].clone()) + .map_err(|_| "managed_schedule_invalid_context")?; + let lead = serde_json::from_value(value.body["lead_in_policy"].clone()) + .map_err(|_| "managed_schedule_invalid_lead_in")?; + let rebuilt = Self::build(context, lead)?; + if value.contract_sha256 != rebuilt.sha256 + || value.body + != serde_json::to_value(&rebuilt.body).map_err(|_| "managed_schedule_encoding")? + || rebuilt.export() != Ok(bytes.to_vec()) + { + return Err("managed_schedule_contract_mismatch".into()); + } + Ok(rebuilt) + } + pub(crate) fn native_window_policy(&self)->super::temporal_acquisition::WindowInputPolicy { + let p=&self.body.native_input;super::temporal_acquisition::WindowInputPolicy::from_contract_fields(self.body.context.clone(),p.sources.clone(),p.target_span_ms,p.maximum_span_ms,p.maximum_first_packet_delay_ms,p.maximum_pairwise_receiver_skew_ms) + } + pub(crate) fn bound_matches_run(&self,bound:&BoundEpisode,run:&str)->bool { + let model=bound.frozen_artifact_sha256.as_ref().and_then(|sha|FrozenModelIdentity::from_runtime_verified_artifact(self,sha.clone()).ok()); + self.bind_episode_start(run,bound.episode.index,bound.attempt_ordinal,bound.segment_ordinal,model.as_ref()).ok().is_some_and(|expected|serde_json::to_vec(&expected).ok()==serde_json::to_vec(bound).ok()) + } + pub fn is_operator_eight_step(&self) -> bool { self.body.profile_id == OPERATOR_EIGHT_PROFILE } + pub fn profile_id(&self) -> &str { self.body.profile_id } + pub fn sha256(&self) -> &str { + &self.sha256 + } + pub fn context(&self) -> &CaptureContext { + &self.body.context + } + pub fn episodes(&self) -> &[Episode] { + &self.body.episodes + } + pub fn counts(&self) -> &ScheduleCounts { + &self.body.counts + } + pub fn resources(&self) -> &ResourcePreflight { + &self.body.resources + } + /// Identity binding only. Future Runtime MUST verify owner/checkpoint/current + /// step/conditions and the actual frozen managed artifact before calling. + pub fn bind_episode_start( + &self, + run_id: &str, + index: usize, + attempt: u16, + segment: u16, + frozen: Option<&FrozenModelIdentity>, + ) -> Result { + if !bounded_id(run_id) { + return Err("managed_schedule_invalid_run".into()); + } + let episode = self + .body + .episodes + .get(index) + .ok_or("managed_schedule_episode_missing")?; + match episode.role { + DataRole::Train if frozen.is_some() => { + return Err("managed_schedule_training_is_frozen".into()) + } + DataRole::HeldOut => { + let model = frozen.ok_or("managed_schedule_frozen_model_required")?; + if model.contract_sha256 != self.sha256 + || model.context_digest != self.body.context.digest() + || !sha(&model.artifact_sha256) + { + return Err("managed_schedule_frozen_model_mismatch".into()); + } + } + DataRole::Train => {} + DataRole::Validation if self.is_operator_eight_step() && frozen.is_none() => {} + _ => return Err("managed_schedule_invalid_role".into()), + } + let identity = |kind: &str, parts: &[serde_json::Value]| -> Result { + let bytes = serde_json::to_vec(&( + SCHEMA, + kind, + &self.sha256, + run_id, + &episode.episode_key, + parts, + )) + .map_err(|_| "managed_schedule_encoding")?; + Ok(format!("{kind}-{}", digest(&bytes))) + }; + let trial = identity("trial", &[])?; + Ok(BoundEpisode { + contract_sha256: self.sha256.clone(), + episode: episode.clone(), + episode_id: identity("episode", &[])?, + trial_id: trial, + session_id: identity("session", &[attempt.into(), segment.into()])?, + attempt_id: identity("attempt", &[attempt.into()])?, + attempt_ordinal: attempt, + segment_ordinal: segment, + frozen_artifact_sha256: frozen.map(|m| m.artifact_sha256.clone()), + }) + } +} +/// Opaque, non-Deserialize binding supplied only by the future managed Runtime. +/// Hash validity is not proof of fitting, frozen state or filesystem custody. +#[derive(Debug, Clone)] +pub struct FrozenModelIdentity { + artifact_sha256: String, + contract_sha256: String, + context_digest: String, +} +impl FrozenModelIdentity { + pub(crate) fn from_runtime_verified_artifact( + plan: &ManagedScheduleContract, + artifact_sha256: String, + ) -> Result { + if !sha(&artifact_sha256) { + return Err("managed_schedule_invalid_artifact_identity".into()); + } + Ok(Self { + artifact_sha256, + contract_sha256: plan.sha256.clone(), + context_digest: plan.body.context.digest(), + }) + } +} +#[derive(Debug, Clone, Serialize)] +pub struct BoundEpisode { + contract_sha256: String, + episode: Episode, + episode_id: String, + trial_id: String, + session_id: String, + attempt_id: String, + attempt_ordinal: u16, + segment_ordinal: u16, + frozen_artifact_sha256: Option, +} +impl BoundEpisode { + pub fn contract_sha256(&self) -> &str { + &self.contract_sha256 + } + pub fn episode(&self) -> &Episode { + &self.episode + } + pub fn episode_id(&self) -> &str { + &self.episode_id + } + pub fn trial_id(&self) -> &str { + &self.trial_id + } + pub fn session_id(&self) -> &str { + &self.session_id + } + pub fn attempt_id(&self) -> &str { + &self.attempt_id + } + pub fn attempt_ordinal(&self) -> u16 { + self.attempt_ordinal + } + pub fn segment_ordinal(&self) -> u16 { + self.segment_ordinal + } + pub fn frozen_artifact_sha256(&self) -> Option<&str> { + self.frozen_artifact_sha256.as_deref() + } +} + +#[cfg(test)] +#[path = "managed_schedule_contract_tests.rs"] +mod tests; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/managed_schedule_contract_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/managed_schedule_contract_tests.rs new file mode 100644 index 0000000000..20e63bb87f --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/managed_schedule_contract_tests.rs @@ -0,0 +1,391 @@ +//! SYNTHETIC metadata and resource oracles. No capture/fit/permission issued. +use super::*; +fn context(count: u8) -> CaptureContext { + CaptureContext { + owner_scope: "a".repeat(64), + installation_id: "synthetic-schedule".into(), + room_digest: "synthetic-room".into(), + revision: 1, + sources: (1..=count) + .map(|source_id| super::super::observation::SourceSpec { + source_id, + device_id: format!("esp32-node-{source_id}"), + grid_id: "ant1-bins64-ppdu0-usable-drop2-linear-v2".into(), + feature_count: 56, + signal_kind: ruview_unified::RfModality::WifiCsi, + center_frequency_hz: 2_437_000_000, + bandwidth_hz: 20_000_000, + link_id: format!("native-{source_id}"), + link_index: 0, + geometry: None, + }) + .collect(), + } +} +fn plan() -> ManagedScheduleContract { + ManagedScheduleContract::build(context(3), LeadInPolicy::Train5HeldOut15EngineeringV1).unwrap() +} +#[test] +fn option_a_is_fifteen_real_episode_descriptors_not_ids_as_independence() { + let p = plan(); + assert_eq!(p.episodes().len(), 15); + let empty = p + .episodes() + .iter() + .filter(|e| e.role == DataRole::Train && e.condition == RoomLabel::Empty) + .collect::>(); + assert_eq!(empty.len(), 1); + assert_eq!(empty[0].target_active_recording_ms, 600_000); + assert_eq!(empty[0].slots.len(), 16); + assert_eq!(empty[0].retained_nominal_window_ms, 32_000); + assert_eq!( + empty[0].retained_nominal_window_ms * 10000 / empty[0].target_active_recording_ms, + 533 + ); + for condition in [ + RoomLabel::OccupiedStatic, + RoomLabel::OccupiedMoving, + RoomLabel::AdjacentOnly, + ] { + let episodes = p + .episodes() + .iter() + .filter(|e| e.role == DataRole::Train && e.condition == condition) + .collect::>(); + assert_eq!(episodes.len(), 2); + assert!(episodes + .iter() + .all(|e| e.target_active_recording_ms == 30_000 && e.slots.len() == 8)); + } + for condition in [ + RoomLabel::Empty, + RoomLabel::OccupiedStatic, + RoomLabel::OccupiedMoving, + RoomLabel::AdjacentOnly, + ] { + assert_eq!( + p.episodes() + .iter() + .filter(|e| e.role == DataRole::HeldOut + && e.condition == condition + && e.target_active_recording_ms == 30_000 + && e.slots.len() == 8) + .count(), + 2 + ); + } + assert!(p.episodes()[..7].iter().all(|e| e.role == DataRole::Train)); + assert!(p.episodes()[7..] + .iter() + .all(|e| e.role == DataRole::HeldOut)); + assert!(p + .episodes() + .iter() + .all(|e| e.requires_explicit_operator_start + && e.requires_conditions_held_confirmation + && e.separately_executed_episode_required)); + assert_eq!(p.counts().verified_independent_trial_count, None); + let value: serde_json::Value = serde_json::from_slice(&p.export().unwrap()).unwrap(); + for field in [ + "capture_ready", + "full_episode_quality_monitoring_implemented", + "independence_verified", + "scientific_adequacy_claimed", + "user_facing_default", + ] { + assert_eq!(value["body"]["claims"][field], false); + } +} +#[test] +fn timing_retention_and_resource_totals_are_derived_and_honest() { + let p = plan(); + let c = p.counts(); + assert_eq!( + ( + c.train_episodes, + c.held_out_episodes, + c.train_windows, + c.held_out_windows, + c.total_windows + ), + (7, 8, 64, 64, 128) + ); + assert_eq!( + ( + c.train_active_recording_ms, + c.held_out_active_recording_ms, + c.active_recording_ms + ), + (780_000, 240_000, 1_020_000) + ); + assert_eq!( + ( + c.train_lead_in_ms, + c.held_out_lead_in_ms, + c.total_lead_in_ms + ), + (35_000, 120_000, 155_000) + ); + assert_eq!(c.nominal_recording_and_lead_in_ms, 1_175_000); + assert_eq!(c.nominal_joint_window_ms, 256_000); + assert_eq!(c.nominal_source_view_window_ms, 768_000); + let r = p.resources(); + assert_eq!( + ( + r.cohorts, + r.source_views, + r.max_retained_rows, + r.max_numeric_values + ), + (128, 384, 29_568, 1_655_808) + ); + assert!(r.serialized_upper_bound_bytes <= 64 * 1024 * 1024); + assert!(p.body.serialized_checkpoint_preflight_is_estimate_only); +} +#[test] +fn windows_span_whole_episodes_without_overlap_or_shortened_horizon() { + let p = plan(); + for e in p.episodes() { + assert_eq!(e.slots.first().unwrap().target_offset_ms, 0); + assert_eq!( + e.slots.last().unwrap().deadline_offset_ms, + e.target_active_recording_ms + ); + for s in &e.slots { + assert_eq!(s.deadline_offset_ms - s.target_offset_ms, 2150); + assert!(s.deadline_offset_ms <= e.target_active_recording_ms); + } + assert!(e + .slots + .windows(2) + .all(|w| w[0].deadline_offset_ms <= w[1].target_offset_ms)); + } + assert!(slots(30_000, 16, 2150).is_err()); + assert!(slots(30_000, 1, 2150).is_err()); +} +#[test] +fn held_out_binding_requires_future_runtime_frozen_identity_without_role_relabeling() { + let p = plan(); + for i in 7..15 { + assert!(p + .bind_episode_start("synthetic-run", i, 0, 0, None) + .is_err()); + } + let model = FrozenModelIdentity::from_runtime_verified_artifact(&p, "b".repeat(64)).unwrap(); + for i in 7..15 { + let b = p + .bind_episode_start("synthetic-run", i, 0, 0, Some(&model)) + .unwrap(); + assert_eq!(b.episode.role, DataRole::HeldOut); + assert_eq!(b.frozen_artifact_sha256, Some("b".repeat(64))); + } + assert!(p + .bind_episode_start("synthetic-run", 0, 0, 0, Some(&model)) + .is_err()); + assert!(FrozenModelIdentity::from_runtime_verified_artifact( + &p, + "inner-v1-not-a-whole-hash".into() + ) + .is_err()); + let mut changed = context(3); + changed.revision += 1; + let other = ManagedScheduleContract::build(changed, LeadInPolicy::Train5HeldOut15EngineeringV1) + .unwrap(); + let foreign = + FrozenModelIdentity::from_runtime_verified_artifact(&other, "b".repeat(64)).unwrap(); + assert!(p + .bind_episode_start("synthetic-run", 7, 0, 0, Some(&foreign)) + .is_err()); +} +#[test] +fn attempts_and_sessions_do_not_create_new_trial_groups_or_unbounded_ids() { + let p = plan(); + let a = p + .bind_episode_start(&"r".repeat(128), 0, 0, 0, None) + .unwrap(); + let b = p + .bind_episode_start(&"r".repeat(128), 0, 1, 2, None) + .unwrap(); + assert_eq!(a.episode_id, b.episode_id); + assert_eq!(a.trial_id, b.trial_id); + assert_ne!(a.session_id, b.session_id); + assert_ne!(a.attempt_id, b.attempt_id); + for id in [&a.episode_id, &a.trial_id, &a.session_id, &a.attempt_id] { + assert!(bounded_id(id)); + } + let next = p + .bind_episode_start("synthetic-run", 1, 0, 0, None) + .unwrap(); + let distinct = p + .bind_episode_start("synthetic-run", 2, 0, 0, None) + .unwrap(); + assert_ne!(next.trial_id, distinct.trial_id); + assert!(p.bind_episode_start("", 0, 0, 0, None).is_err()); + assert!(p.bind_episode_start("bad run", 0, 0, 0, None).is_err()); + assert!(p + .bind_episode_start("synthetic-run", 15, 0, 0, None) + .is_err()); + assert_eq!(p.counts().train_episodes, 7); +} +#[test] +fn canonical_versioned_hash_does_not_relabel_seed_or_accept_unknown_contracts() { + let p = plan(); + let bytes = p.export().unwrap(); + assert_eq!( + ManagedScheduleContract::import(&bytes) + .unwrap() + .export() + .unwrap(), + bytes + ); + let seed = AcquisitionPlan::build( + context(3), + ExperimentalPlanId::Native2000msSlots8, + &[ + (1, ReceiverCadenceBand::Hz30To36), + (2, ReceiverCadenceBand::Hz30To36), + (3, ReceiverCadenceBand::Hz30To36), + ], + ) + .unwrap(); + assert_ne!(p.sha256(), seed.sha256()); + // Even canonical, self-consistently rehashed metadata cannot redefine this + // finite profile's schedule, roles, counts or claims. + for change in [ + "duration", + "role", + "slots", + "quality", + "independence", + "resource", + ] { + let mut body = p.body.clone(); + match change { + "duration" => body.episodes[0].target_active_recording_ms = 300_000, + "role" => body.episodes[7].role = DataRole::Train, + "slots" => { + body.episodes[0].slots.pop(); + } + "quality" => body.claims.full_episode_quality_monitoring_implemented = true, + "independence" => body.claims.independence_verified = true, + "resource" => body.resources.max_numeric_values = 1, + _ => unreachable!(), + } + let forged_hash = hash(&body).unwrap(); + let forged = serde_json::to_vec(&Export { + schema: SCHEMA, + contract_sha256: &forged_hash, + body: &body, + }) + .unwrap(); + assert!( + ManagedScheduleContract::import(&forged).is_err(), + "{change}" + ); + } + let mut unknown: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + unknown["body"]["automatic_start"] = true.into(); + assert!(ManagedScheduleContract::import(&serde_json::to_vec(&unknown).unwrap()).is_err()); + let wrong_schema = serde_json::to_vec(&Export { + schema: "ruview.experimental.native-temporal-acquisition.v2", + contract_sha256: p.sha256(), + body: &p.body, + }) + .unwrap(); + assert!(ManagedScheduleContract::import(&wrong_schema).is_err()); + assert!(ManagedScheduleContract::import(&vec![b' '; MAX_BYTES + 1]).is_err()); +} +#[test] +fn native_capacity_and_shape_reject_before_capture_without_fallback() { + assert!( + ManagedScheduleContract::build(context(8), LeadInPolicy::Train5HeldOut15EngineeringV1) + .is_err() + ); + assert!( + ManagedScheduleContract::build(context(2), LeadInPolicy::Train5HeldOut15EngineeringV1) + .is_err() + ); + let mut wrong = context(3); + wrong.sources[0].feature_count = 55; + assert!( + ManagedScheduleContract::build(wrong, LeadInPolicy::Train5HeldOut15EngineeringV1).is_err() + ); + let p = plan(); + let mut sources = p.body.native_input.sources.clone(); + sources[0].max_rows = MAX_ROWS + 1; + assert!(capacity(&context(3), &sources, 64, 128).is_err()); + assert!(capacity(&context(3), &p.body.native_input.sources, 65, 128).is_err()); + assert!(capacity(&context(3), &p.body.native_input.sources, 64, 129).is_err()); + let many = context(8); + let policies = many + .sources + .iter() + .map(|s| SourceWindowPolicy { + source_id: s.source_id, + receiver_cadence: ReceiverCadenceBand::Hz30To36, + min_rows: 61, + max_rows: 77, + min_packet_interval_ns: 1_000_000_000_000 / 36_000, + max_gap_ns: 50_000_000, + }) + .collect::>(); + assert!(capacity(&many, &policies, 64, 128).is_err()); + assert_eq!(p.counts().total_windows, 128); +} + +#[test] +fn operator_eight_step_is_exact_interleaved_user_sequence_with_bounded_reserved_checks() { + let old = plan().export().unwrap(); + let p=ManagedScheduleContract::build(context(3),LeadInPolicy::OperatorEightStepV1).unwrap(); + assert_eq!(p.profile_id(),OPERATOR_EIGHT_PROFILE); + let expected=[ + (DataRole::Train,RoomLabel::Empty,15_000,600_000,16), + (DataRole::Validation,RoomLabel::Empty,15_000,15_000,4), + (DataRole::Train,RoomLabel::OccupiedStatic,5_000,60_000,16), + (DataRole::Validation,RoomLabel::OccupiedStatic,5_000,15_000,4), + (DataRole::Train,RoomLabel::OccupiedMoving,5_000,60_000,16), + (DataRole::Validation,RoomLabel::OccupiedMoving,5_000,15_000,4), + (DataRole::Train,RoomLabel::AdjacentOnly,15_000,60_000,16), + (DataRole::Validation,RoomLabel::AdjacentOnly,15_000,15_000,4), + ]; + assert_eq!(p.episodes().len(),8); + for (episode,(role,label,lead,duration,windows)) in p.episodes().iter().zip(expected) { + assert_eq!((episode.role,episode.condition,episode.lead_in_ms,episode.target_active_recording_ms,episode.slots.len()),(role,label,lead,duration,windows)); + assert!(episode.requires_explicit_operator_start&&episode.requires_conditions_held_confirmation); + assert!(episode.slots.windows(2).all(|pair|pair[0].deadline_offset_ms<=pair[1].target_offset_ms)); + assert!(episode.slots.last().unwrap().deadline_offset_ms<=duration); + } + let c=p.counts(); + assert_eq!((c.train_episodes,c.held_out_episodes,c.train_windows,c.held_out_windows,c.total_windows),(4,4,64,16,80)); + assert_eq!(c.nominal_recording_and_lead_in_ms,920_000); + assert_eq!(p.resources().max_numeric_values,1_034_880); + assert_eq!(p.resources().max_retained_rows,18_480); + assert_eq!(ManagedScheduleContract::import(&p.export().unwrap()).unwrap().export().unwrap(),p.export().unwrap()); + assert_eq!(ManagedScheduleContract::import(&old).unwrap().export().unwrap(),old); + assert_ne!(p.sha256(),plan().sha256()); +} + +#[test] +fn operator_reserved_checks_bind_without_a_model_but_cannot_be_relabelled_as_training() { + let p=ManagedScheduleContract::build(context(3),LeadInPolicy::OperatorEightStepV1).unwrap(); + let check=p.bind_episode_start("operator-run",1,0,0,None).unwrap(); + assert_eq!(check.episode().role,DataRole::Validation); + let frozen=FrozenModelIdentity::from_runtime_verified_artifact(&p,"f".repeat(64)).unwrap(); + assert!(p.bind_episode_start("operator-run",1,0,0,Some(&frozen)).is_err()); + assert!(plan().bind_episode_start("old-run",7,0,0,None).is_err()); + let mut value:serde_json::Value=serde_json::from_slice(&p.export().unwrap()).unwrap(); + value["body"]["episodes"][1]["role"]=serde_json::json!("Train"); + assert!(ManagedScheduleContract::import(&serde_json::to_vec(&value).unwrap()).is_err()); +} + +#[test] +fn operator_default_is_explicit_and_rehashed_authority_claims_stay_rejected(){ + let p=ManagedScheduleContract::build(context(3),LeadInPolicy::OperatorEightStepV1).unwrap(); + assert!(p.body.claims.user_facing_default); + assert!(!p.body.claims.capture_ready&&!p.body.claims.scientific_adequacy_claimed&&!p.body.claims.independence_verified); + assert!(!plan().body.claims.user_facing_default); + let mut forged=p.clone();forged.body.claims.capture_ready=true;forged.sha256=hash(&forged.body).unwrap(); + assert!(ManagedScheduleContract::import(&forged.export().unwrap()).is_err()); + let mut forged=plan();forged.body.claims.user_facing_default=true;forged.sha256=hash(&forged.body).unwrap(); + assert!(ManagedScheduleContract::import(&forged.export().unwrap()).is_err()); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/managed_training.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/managed_training.rs new file mode 100644 index 0000000000..0b94bfa293 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/managed_training.rs @@ -0,0 +1,265 @@ +//! Metadata-only provenance issued by the managed native capture path, not model imports. +use super::{ + capture::{PhaseStatus, COHORTS_PER_PHASE}, + model::RoomModel, + observation::{CaptureContext, CaptureProvenance, DataRole, RoomLabel, ValidatedDataset}, + qualification::{hash, CONDITIONS}, +}; +use serde::{Deserialize, Serialize}; + +pub(crate) const PROOF_SCHEMA: &str = "ruview.managed-training-proof.v1"; +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub(crate) struct TrainingSourceRange { + pub first_ordinal: u64, + pub last_ordinal: u64, + pub first_time_ns: u64, + pub last_time_ns: u64, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub(crate) struct SourceWatermark { + pub source_id: u8, + pub sequence: u32, + pub ordinal: u64, + pub last_time_ns: u64, + pub admitted_packets: u64, + pub train_range: TrainingSourceRange, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub(crate) struct PhaseClaim { + pub trial_sha256: String, + pub session_sha256: String, + pub role: DataRole, + pub label: RoomLabel, + pub status: PhaseStatus, + pub cohort_count: usize, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub(crate) struct TrainingProofSeed { + pub context: CaptureContext, + pub run_id: String, + pub training_data_sha256: String, + pub training_metadata_sha256: String, + pub sealed_train_history_sha256: String, + pub train_counts: [usize; 4], + pub phase_claims: Vec, + pub sources: Vec, + pub first_observed_ns: u64, + pub last_seen_ns: u64, + /// Receiver observations never establish an authenticated transmitter epoch. + pub source_epoch_verified: bool, + pub receiver_clock_domain: String, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub(crate) struct ManagedTrainingProof { + pub schema: String, + pub model_sha256: String, + pub seed: TrainingProofSeed, + pub proof_sha256: String, +} +pub(crate) fn commitment(domain: &str, value: &str) -> Result { + hash(&("ruview.managed-training-role.v1", domain, value)) +} +fn hex(s: &str) -> bool { + s.len() == 64 + && s.bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) +} +impl TrainingProofSeed { + pub fn validate(&self) -> Result<(), String> { + self.context.validate()?; + let _ = super::capture::CaptureSession::new(self.context.clone(), self.run_id.clone())?; + if !hex(&self.training_data_sha256) + || !hex(&self.training_metadata_sha256) + || !hex(&self.sealed_train_history_sha256) + || self.source_epoch_verified + || self.receiver_clock_domain != "utc_anchored_monotonic_v1" + || self + .train_counts + .iter() + .any(|n| *n < COHORTS_PER_PHASE || *n > 64) + || self.train_counts.iter().sum::() > 64 + || self.phase_claims.is_empty() + || self.phase_claims.len() > 128 + || self + .phase_claims + .iter() + .any(|p| p.cohort_count > COHORTS_PER_PHASE) + || self + .phase_claims + .iter() + .map(|p| p.cohort_count) + .sum::() + > super::observation::MAX_COHORTS + || self.sources.len() != self.context.sources.len() + || self.first_observed_ns == 0 + || self.last_seen_ns < self.first_observed_ns + { + return Err("invalid managed training metadata bounds".into()); + } + let mut trials = std::collections::HashMap::new(); + let mut sessions = std::collections::HashMap::new(); + for p in &self.phase_claims { + if !hex(&p.trial_sha256) + || !hex(&p.session_sha256) + || p.role == DataRole::Inference + || p.label == RoomLabel::Unknown + || p.cohort_count + != if p.status == PhaseStatus::Sealed { + COHORTS_PER_PHASE + } else { + 0 + } + || trials + .insert(&p.trial_sha256, p.role) + .is_some_and(|r| r != p.role) + || sessions + .insert(&p.session_sha256, p.role) + .is_some_and(|r| r != p.role) + { + return Err("managed training role/phase commitment mismatch".into()); + } + } + for (i, label) in CONDITIONS.iter().enumerate() { + let count: usize = self + .phase_claims + .iter() + .filter(|p| { + p.role == DataRole::Train + && p.label == *label + && p.status == PhaseStatus::Sealed + }) + .map(|p| p.cohort_count) + .sum(); + if count != self.train_counts[i] { + return Err("managed training condition counts lack sealed acknowledgement".into()); + } + } + for (s, w) in self.context.sources.iter().zip(&self.sources) { + if s.source_id != w.source_id + || w.ordinal == 0 + || w.ordinal >= 1u64 << 56 + || w.admitted_packets == 0 + || w.admitted_packets > w.ordinal + || w.last_time_ns == 0 + || w.last_time_ns > self.last_seen_ns + || w.train_range.first_ordinal == 0 + || w.train_range.last_ordinal < w.train_range.first_ordinal + || w.train_range.last_ordinal > w.ordinal + || w.train_range.first_time_ns == 0 + || w.train_range.last_time_ns <= w.train_range.first_time_ns + || w.train_range.last_time_ns > w.last_time_ns + { + return Err("invalid managed source watermark".into()); + } + } + if self.sources.iter().map(|s| s.last_time_ns).max() != Some(self.last_seen_ns) + || self + .sources + .iter() + .map(|s| s.train_range.first_time_ns) + .min() + != Some(self.first_observed_ns) + { + return Err("managed receiver watermark mismatch".into()); + } + Ok(()) + } + pub fn check_role(&self, trial: &str, session: &str, role: DataRole) -> Result<(), String> { + let trial = commitment("trial", trial)?; + let session = commitment("session", session)?; + if self.phase_claims.iter().any(|p| { + (p.trial_sha256 == trial || p.session_sha256 == session) + && (p.role != role || p.status == PhaseStatus::Sealed) + }) { + return Err("phase identity crosses original managed training roles".into()); + } + Ok(()) + } + pub fn prefix(&self) -> Result { + use sha2::{Digest, Sha256}; + let d = Sha256::digest(format!("{}:{}", self.context.digest(), self.run_id).as_bytes()); + Ok(u64::from_be_bytes( + d[..8].try_into().map_err(|_| "managed frame prefix")?, + )) + } +} +impl ManagedTrainingProof { + pub fn issue(model: &RoomModel, seed: TrainingProofSeed) -> Result { + let mut p = Self { + schema: PROOF_SCHEMA.into(), + model_sha256: String::new(), + seed, + proof_sha256: String::new(), + }; + // Model identity hashes its exact exported bytes, never a JSON Value roundtrip. + use sha2::{Digest, Sha256}; + p.model_sha256 = format!("{:x}", Sha256::digest(model.export()?)); + p.proof_sha256 = hash(&(&p.schema, &p.model_sha256, &p.seed))?; + p.validate(model)?; + Ok(p) + } + pub fn validate(&self, model: &RoomModel) -> Result<(), String> { + use sha2::{Digest, Sha256}; + self.seed.validate()?; + if self.schema != PROOF_SCHEMA + || self.proof_sha256 != hash(&(&self.schema, &self.model_sha256, &self.seed))? + || self.model_sha256 != format!("{:x}", Sha256::digest(model.export()?)) + || &self.seed.context != model.context() + || self.seed.training_data_sha256 != model.training_manifest().training_data_sha256 + || self.seed.train_counts.iter().sum::() + != model.training_manifest().training_cohorts + || model.training_manifest().training_provenance + != [CaptureProvenance::LiveReceiverObserved] + { + return Err("managed training proof model/context/Train identity mismatch".into()); + } + Ok(()) + } + pub fn check_held_out( + &self, + model: &RoomModel, + dataset: &ValidatedDataset, + ) -> Result<(), String> { + self.validate(model)?; + if dataset.context() != model.context() { + return Err("managed proof evaluation context mismatch".into()); + } + let prefix = self.seed.prefix()?; + for c in dataset.held_out() { + self.seed.check_role(&c.trial_id, &c.session_id, c.role)?; + if c.provenance != CaptureProvenance::LiveReceiverObserved || c.teacher.is_some() { + return Err("managed proof requires fresh native held-out capture".into()); + } + for v in &c.views { + let w = self + .seed + .sources + .iter() + .find(|w| w.source_id == v.source_id) + .ok_or("unknown managed held-out source")?; + if v.provenance != CaptureProvenance::LiveReceiverObserved + || v.first_time_ns <= w.last_time_ns + || v.sequence_ids + .iter() + .any(|n| *n <= w.ordinal || *n >= 1u64 << 56) + || v.frame_ids.iter().zip(&v.sequence_ids).any(|(f, n)| { + *f != ((u128::from(prefix) << 64) + | (u128::from(v.source_id) << 56) + | u128::from(*n)) + }) + { + return Err( + "held-out capture overlaps or replays original managed source lineage" + .into(), + ); + } + } + } + Ok(()) + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/managed_training_readiness.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/managed_training_readiness.rs new file mode 100644 index 0000000000..391912c85f --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/managed_training_readiness.rs @@ -0,0 +1,139 @@ +//! Qualification capture preflight, independent from a model's diagnostic availability. +use super::*; +use crate::room_learner::{ + capture::PhaseStatus, + qualification::{hash, CONDITIONS}, +}; +impl RoomLearningRuntime { + pub(super) fn qualification_readiness_locked(&self, state: &Inner, now_ns: u64) -> Value { + if state.require_legacy_kind().is_err() { + return json!({"schema":"ruview.qualification-readiness.v1","eligible_for_held_out":false,"training_lineage_basis":"unavailable","managed_training_proof_sha256":null,"raw_train_rows_retained":false,"train_cohorts_per_condition":[0,0,0,0],"reason":"unsupported_capture_kind","detail":"Private temporal capture has no fitting or qualification authority."}); + } + let retained = state.legacy_capture() + .is_some_and(|c| c.cohorts().iter().any(|r| r.role == DataRole::Train)); + let result = (|| -> Result<(&str, [usize; 4], Option), (&str, String)> { + let model = state.model.as_ref().ok_or(( + "missing_candidate", + "Train or restore a candidate first.".into(), + ))?; + use sha2::{Digest, Sha256}; + let model_bytes = model.export().map_err(|e| ("missing_candidate", e))?; + self.store + .load_exact_candidate( + model.context(), + &format!("{:x}", Sha256::digest(&model_bytes)), + ) + .map_err(|e| { + ( + "missing_candidate", + format!("Frozen candidate artifact could not be verified: {e}"), + ) + })?; + let capture = state.legacy_capture().ok_or(( + "context_mismatch", + "No current managed capture context.".into(), + ))?; + if capture.context() != model.context() { + return Err(( + "context_mismatch", + "Candidate does not match this room/source context.".into(), + )); + } + if now_ns < capture.minimum_receiver_time_ns() { + return Err(("receiver_clock_incomparable","The current receiver clock predates the original managed capture; no automatic clock/counter rebase is permitted.".into())); + } + let train: Vec<_> = capture + .cohorts() + .iter() + .filter(|c| c.role == DataRole::Train) + .collect(); + if !train.is_empty() { + capture + .validated_dataset() + .map_err(|e| ("invalid_managed_training_proof", e))?; + if hash(&train).map_err(|e| ("retained_train_mismatch", e))? + != model.training_manifest().training_data_sha256 + { + return Err(( + "retained_train_mismatch", + "Retained Train rows differ from this frozen model.".into(), + )); + } + let counts = + CONDITIONS.map(|label| train.iter().filter(|c| c.label == label).count()); + if counts.iter().any(|n| *n < 4) + || train.iter().any(|row| { + let phases: Vec<_> = capture + .history() + .iter() + .filter(|p| { + p.role == DataRole::Train + && p.status == PhaseStatus::Sealed + && p.trial_id == row.trial_id + && p.session_id == row.session_id + && p.label == row.label + }) + .collect(); + phases.len() != 1 + || phases[0].cohort_count + != train + .iter() + .filter(|r| { + r.trial_id == row.trial_id + && r.session_id == row.session_id + && r.label == row.label + }) + .count() + }) + { + return Err(( + "insufficient_managed_train", + "Four original managed sealed Train cohorts per condition are required." + .into(), + )); + } + if let Some(proof) = self + .store + .load_managed_training_proof(model) + .map_err(|e| ("invalid_managed_training_proof", e))? + { + let fence = self + .store + .load_managed_capture_fence(model, &proof) + .map_err(|e| ("invalid_managed_training_proof", e))?; + capture + .managed_fence_seed(&proof) + .and_then(|s| s.ensure_extends(&fence.seed)) + .map_err(|e| ("invalid_managed_training_proof", e))?; + } + return Ok(("retained_exact_train", counts, None)); + } + let proof=self.store.load_managed_training_proof(model).map_err(|e|("invalid_managed_training_proof",e))? + .ok_or(("missing_managed_training_proof","This diagnostic model has no retained Train rows or verified managed training proof; new held-out captures cannot qualify it.".into()))?; + let fence = self + .store + .load_managed_capture_fence(model, &proof) + .map_err(|e| ("invalid_managed_training_proof", e))?; + capture + .managed_fence_seed(&proof) + .and_then(|s| s.ensure_extends(&fence.seed)) + .map_err(|e| ("invalid_managed_training_proof", e))?; + proof + .validate(model) + .map_err(|e| ("invalid_managed_training_proof", e))?; + Ok(( + "managed_training_proof_v1", + proof.seed.train_counts, + Some(proof.proof_sha256), + )) + })(); + match result { + Ok((basis, counts, sha)) => { + json!({"schema":"ruview.qualification-readiness.v1","eligible_for_held_out":true,"training_lineage_basis":basis,"managed_training_proof_sha256":sha,"raw_train_rows_retained":retained,"train_cohorts_per_condition":counts,"reason":null,"detail":null}) + } + Err((reason, detail)) => { + json!({"schema":"ruview.qualification-readiness.v1","eligible_for_held_out":false,"training_lineage_basis":"unavailable","managed_training_proof_sha256":null,"raw_train_rows_retained":retained,"train_cohorts_per_condition":[0,0,0,0],"reason":reason,"detail":bounded(detail)}) + } + } + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/managed_training_store.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/managed_training_store.rs new file mode 100644 index 0000000000..b973a6b206 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/managed_training_store.rs @@ -0,0 +1,78 @@ +//! Immutable managed provenance is committed before the candidate pointer. +use super::*; +use crate::room_learner::managed_training::{ManagedTrainingProof, TrainingProofSeed}; +impl ArtifactStore { + pub(crate) fn load_managed_training_proof( + &self, + model: &RoomModel, + ) -> Result, String> { + self.validate_context(model.context())?; + let Some(dir) = self.context_dir(model.context(), false)? else { + return Ok(None); + }; + let sha = digest(&model.export()?); + let path = dir.join(format!("managed-training-{sha}.json")); + if !regular_or_missing(&path)? { + return Ok(None); + } + let proof: ManagedTrainingProof = + serde_json::from_slice(&private_read(&path).map_err(|e| e.to_string())?) + .map_err(|e| e.to_string())?; + proof.validate(model)?; + Ok(Some(proof)) + } + pub(crate) fn save_managed_candidate( + &self, + model: &RoomModel, + seed: TrainingProofSeed, + ) -> Result { + self.validate_context(model.context())?; + seed.validate()?; + let proposed = ManagedTrainingProof::issue(model, seed)?; + let proof = if let Some(original) = self.load_managed_training_proof(model)? { + // Retries use the first committed proof, never re-attest its original + // source watermark with observations received during the fit/retry. + if original.seed.context != proposed.seed.context + || original.seed.run_id != proposed.seed.run_id + || original.seed.training_data_sha256 != proposed.seed.training_data_sha256 + || original.seed.training_metadata_sha256 != proposed.seed.training_metadata_sha256 + || original.seed.sealed_train_history_sha256 + != proposed.seed.sealed_train_history_sha256 + || original.seed.train_counts != proposed.seed.train_counts + || original.seed.phase_claims != proposed.seed.phase_claims + || original + .seed + .sources + .iter() + .zip(&proposed.seed.sources) + .any(|(a, b)| { + a.source_id != b.source_id + || a.ordinal > b.ordinal + || a.last_time_ns > b.last_time_ns + }) + { + return Err( + "existing managed training proof differs from the original fit lineage".into(), + ); + } + proposed.seed.ensure_extends(&original.seed)?; + original + } else { + proposed + }; + let dir = self + .context_dir(model.context(), true)? + .ok_or("missing managed model context")?; + let path = dir.join(format!("managed-training-{}.json", proof.model_sha256)); + let bytes = encode(&proof)?; + evaluation_store::write_immutable(&path, &bytes)?; + if private_read(&path).map_err(|e| e.to_string())? != bytes { + return Err("managed training proof readback mismatch".into()); + } + self.load_managed_training_proof(model)? + .ok_or("managed training proof was not retained")?; + self.initialize_managed_capture_fence(model, &proof)?; + // Generic save_candidate remains diagnostic-only for all import callers. + self.save_candidate(model) + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/managed_training_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/managed_training_tests.rs new file mode 100644 index 0000000000..b4c92a88ef --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/managed_training_tests.rs @@ -0,0 +1,896 @@ +//! SYNTHETIC native packet fixtures; no hardware or general accuracy claim. +use super::tests::{context, phase, runtime, START}; +use super::*; +use crate::room_learner::{ + capture::CaptureSession, + managed_training::ManagedTrainingProof, + observation::ValidatedDataset, + qualification::{hash, managed_evidence_with_proof}, +}; +use std::path::PathBuf; +fn proof_path(root: &Path, model: &RoomModel) -> PathBuf { + use sha2::{Digest, Sha256}; + root.join("a".repeat(64)) + .join("SensingServer/room-learner") + .join(model.context().digest()) + .join(format!( + "managed-training-{:x}.json", + Sha256::digest(model.export().unwrap()) + )) +} +fn current(rt: &RoomLearningRuntime) -> (Arc, ManagedTrainingProof) { + let state = rt.inner.lock().unwrap(); + let model = state.model.as_ref().unwrap().clone(); + let proof = rt + .store + .load_managed_training_proof(&model) + .unwrap() + .unwrap(); + (model, proof) +} +fn restore(root: &Path) -> Arc { + let cold = runtime(root); + cold.configure( + &cold.mutation_guard().unwrap(), + context(), + "caller-new-run-must-not-replace-original".into(), + false, + ) + .unwrap(); + cold +} +#[tokio::test] +async fn metadata_proof_cold_resume_fresh_all_conditions_qualify_v2_without_raw_train() { + let dir = tempfile::tempdir().unwrap(); + let rt = super::selection_tests::fitted(dir.path()).await; + let (model, proof) = current(&rt); + let before = std::fs::read(proof_path(dir.path(), &model)).unwrap(); + let text = String::from_utf8(before.clone()).unwrap(); + for forbidden in [ + "\"samples\"", + "\"frame_ids\"", + "\"values\"", + "\"amplitudes\"", + ] { + assert!(!text.contains(forbidden)); + } + let legacy = rt.qualify(&rt.mutation_guard().unwrap()).unwrap(); + assert_eq!( + legacy.receipt.schema, + "ruview.local-operational-qualification.v1" + ); + assert!(!serde_json::to_value(&legacy.receipt) + .unwrap() + .as_object() + .unwrap() + .contains_key("training_lineage_basis")); + let cold = restore(dir.path()); + let ready = cold.status(START + 200_000_000_000)["qualification_readiness"].clone(); + assert_eq!(ready["eligible_for_held_out"], true, "{ready}"); + assert_eq!(ready["raw_train_rows_retained"], false); + assert_eq!(ready["training_lineage_basis"], "managed_training_proof_v1"); + assert_eq!(ready["train_cohorts_per_condition"], json!([4, 4, 4, 4])); + assert_eq!(ready["managed_training_proof_sha256"], proof.proof_sha256); + for (i, label) in crate::room_learner::qualification::CONDITIONS + .into_iter() + .enumerate() + { + phase(&cold, DataRole::HeldOut, label, i as u32 + 8); + } + let q = cold.qualify(&cold.mutation_guard().unwrap()).unwrap(); + assert_eq!( + q.receipt.schema, + "ruview.local-operational-qualification.v2" + ); + assert_eq!( + q.receipt.managed_training_proof_sha256.as_deref(), + Some(proof.proof_sha256.as_str()) + ); + assert_eq!( + q.receipt.training_lineage_basis.as_deref(), + Some("managed_training_proof_v1") + ); + assert!(!q.receipt.report.independence_verified && !q.receipt.physically_validated); + assert_eq!( + cold.status(START + 200_000_000_000)["last_evaluation"]["train_lineage_checked"], + false + ); + assert_eq!( + std::fs::read(proof_path(dir.path(), &model)).unwrap(), + before + ); + let summary = crate::room_learner::selection::QualificationSummary::from(&q); + let public = serde_json::to_value(&summary).unwrap(); + assert_eq!( + public["schema"], + "ruview.local-operational-qualification.v2" + ); + assert!(public.get("first_observed_at_ns").is_none()); + cold.select_model( + &cold.mutation_guard().unwrap(), + &q.receipt.model_sha256, + &q.receipt_sha256, + ) + .unwrap(); + let selected = runtime(dir.path()); + let s = selected.status(START + 200_000_000_000); + assert_eq!( + s["model_selection"]["selected"]["qualification_sha256"], + q.receipt_sha256 + ); + assert_eq!(s["qualification_readiness"]["eligible_for_held_out"], true); +} +#[tokio::test] +async fn old_diagnostic_model_without_proof_loads_but_stops_selection_capture_before_effort() { + let dir = tempfile::tempdir().unwrap(); + let rt = super::selection_tests::fitted(dir.path()).await; + let (model, _) = current(&rt); + std::fs::remove_file(proof_path(dir.path(), &model)).unwrap(); + let cold = restore(dir.path()); + let status = cold.status(START + 200_000_000_000); + assert_eq!(status["candidate_loaded"], true); + assert_eq!( + status["qualification_readiness"]["reason"], + "missing_managed_training_proof" + ); + let result = cold.begin_phase( + &cold.mutation_guard().unwrap(), + PhaseRequest { + trial_id: "fresh-empty".into(), + session_id: "fresh-empty-session".into(), + role: DataRole::HeldOut, + label: RoomLabel::Empty, + duration_ms: 12000, + lead_in_ms: 0, + conditions_confirmed: true, + }, + START + 200_000_000_000, + ); + assert!(result.unwrap_err().contains("cannot qualify")); + assert!(!cold.capture_active().unwrap()); + assert!(cold.qualify(&cold.mutation_guard().unwrap()).is_err()); +} +#[tokio::test] +async fn proof_resume_preserves_counter_namespace_roles_and_rejects_regressed_receiver_clock() { + let dir = tempfile::tempdir().unwrap(); + let rt = super::selection_tests::fitted(dir.path()).await; + let (_, proof) = current(&rt); + let mut capture = CaptureSession::resume_managed_proof(&proof).unwrap(); + let w = &proof.seed.sources[0]; + let values = vec![2.0; 8]; + for sequence in [w.sequence, w.sequence.wrapping_sub(1), 0] { + assert!(capture + .ingest( + RawSourceFrame { + source_id: w.source_id, + grid_id: "ht64-canonical8", + frequency_hz: 2_437_000_000, + bandwidth_hz: 20_000_000, + sequence, + amplitudes: &values + }, + proof.seed.last_seen_ns + 1 + ) + .is_err()); + } + assert!(capture + .ingest( + RawSourceFrame { + source_id: w.source_id, + grid_id: "ht64-canonical8", + frequency_hz: 2_437_000_000, + bandwidth_hz: 20_000_000, + sequence: w.sequence.wrapping_add(1), + amplitudes: &values + }, + proof.seed.last_seen_ns - 1 + ) + .is_err()); + capture + .ingest( + RawSourceFrame { + source_id: w.source_id, + grid_id: "ht64-canonical8", + frequency_hz: 2_437_000_000, + bandwidth_hz: 20_000_000, + sequence: w.sequence.wrapping_add(1), + amplitudes: &values, + }, + proof.seed.last_seen_ns + 1, + ) + .unwrap(); + assert_eq!( + capture.progress(proof.seed.last_seen_ns + 2).sources[0].last_accepted_sequence, + Some(w.sequence + 1) + ); + let crossed = PhaseRequest { + trial_id: "synthetic-trial-0".into(), + session_id: "unused-session".into(), + role: DataRole::HeldOut, + label: RoomLabel::Empty, + duration_ms: 12000, + lead_in_ms: 0, + conditions_confirmed: true, + }; + assert!(capture + .begin_phase(crossed, proof.seed.last_seen_ns + 3) + .unwrap_err() + .contains("original managed training roles")); + let cold = restore(dir.path()); + assert_eq!( + cold.status(proof.seed.last_seen_ns - 1)["qualification_readiness"]["reason"], + "receiver_clock_incomparable" + ); +} +#[tokio::test] +async fn v2_capture_checkpoint_retains_metadata_fences_and_no_dummy_training_rows() { + let dir = tempfile::tempdir().unwrap(); + let rt = super::selection_tests::fitted(dir.path()).await; + let (_, proof) = current(&rt); + let capture = CaptureSession::resume_managed_proof(&proof).unwrap(); + let at = proof.seed.last_seen_ns + 10; + let snapshot = capture.checkpoint(at).unwrap(); + let json = serde_json::to_value(&snapshot).unwrap(); + assert_eq!(json["schema"], "ruview.room-learner.capture-checkpoint.v2"); + assert!(json["cohorts"].as_array().unwrap().is_empty()); + let restored = + CaptureSession::restore_checkpoint(snapshot, &proof.seed.context, at + 1).unwrap(); + assert_eq!(restored.session.metadata_lineage(), Some(&proof.seed)); + assert_eq!( + restored.session.progress(at + 2).sources[0].last_accepted_sequence, + Some(proof.seed.sources[0].sequence) + ); + let mut missing = json; + missing.as_object_mut().unwrap().remove("metadata_lineage"); + let snapshot = serde_json::from_value(missing).unwrap(); + assert!(CaptureSession::restore_checkpoint(snapshot, &proof.seed.context, at + 1).is_err()); +} +#[tokio::test] +async fn corrupt_foreign_and_torn_metadata_never_replace_exact_train_checks() { + let dir = tempfile::tempdir().unwrap(); + let rt = super::selection_tests::fitted(dir.path()).await; + let (model, proof) = current(&rt); + for mutate in [ + "owner", + "installation", + "context", + "model", + "train", + "source", + "count", + "epoch", + "clock", + "floor", + "ack", + "hash", + ] { + let mut p = proof.clone(); + match mutate { + "owner" => p.seed.context.owner_scope = "b".repeat(64), + "installation" => p.seed.context.installation_id = "foreign-installation".into(), + "context" => p.seed.context.room_digest = "foreign-room".into(), + "model" => p.model_sha256 = "b".repeat(64), + "train" => p.seed.training_data_sha256 = "b".repeat(64), + "source" => { + p.seed.context.sources[1].source_id = 99; + p.seed.sources[1].source_id = 99; + } + "count" => p.seed.train_counts[1] = 0, + "epoch" => p.seed.source_epoch_verified = true, + "clock" => p.seed.receiver_clock_domain = "unknown".into(), + "floor" => { + p.seed.sources[0].ordinal = p.seed.sources[0].train_range.last_ordinal - 1; + p.seed.sources[0].admitted_packets = p.seed.sources[0].ordinal; + } + "ack" => p.seed.phase_claims[0].cohort_count = 0, + _ => p.proof_sha256 = "b".repeat(64), + } + if mutate != "hash" { + p.proof_sha256 = hash(&(&p.schema, &p.model_sha256, &p.seed)).unwrap(); + } + assert!(p.validate(&model).is_err(), "{mutate}"); + } + let evaluation = rt + .store + .evaluate_candidate( + &model, + &rt.inner + .lock() + .unwrap() + .legacy_capture() + .unwrap() + .validated_dataset() + .unwrap(), + DataRole::HeldOut, + ) + .unwrap(); + let state = rt.inner.lock().unwrap(); + let capture = state.legacy_capture().unwrap(); + let mut rows = capture.cohorts().to_vec(); + rows.iter_mut() + .find(|c| c.role == DataRole::Train) + .unwrap() + .views[0] + .samples[0][0] += 0.25; + let altered = ValidatedDataset::new(model.context().clone(), rows).unwrap(); + assert!(managed_evidence_with_proof( + &model, + &altered, + capture.history(), + &evaluation, + Some(&proof) + ) + .unwrap_err() + .contains("retained exact Train")); + drop(state); + crate::companion::atomic_private_write(&proof_path(dir.path(), &model), b"{").unwrap(); + assert!(rt.store.load_managed_training_proof(&model).is_err()); + let cold = runtime(dir.path()); + assert!(cold + .configure( + &cold.mutation_guard().unwrap(), + context(), + "torn".into(), + false + ) + .is_err()); +} +fn phase_at( + rt: &RoomLearningRuntime, + label: RoomLabel, + index: u32, + base: u64, + gain_override: Option, +) -> u64 { + phase_at_group(rt, label, index, base, gain_override, None) +} +fn phase_at_group( + rt: &RoomLearningRuntime, + label: RoomLabel, + index: u32, + base: u64, + gain_override: Option, + group: Option<&str>, +) -> u64 { + let end = prepare_phase_at(rt, label, index, base, gain_override, group); + rt.finish_phase(&rt.mutation_guard().unwrap(), end, true) + .unwrap(); + end +} +fn prepare_phase_at( + rt: &RoomLearningRuntime, + label: RoomLabel, + index: u32, + base: u64, + gain_override: Option, + group: Option<&str>, +) -> u64 { + let at = base + u64::from(index) * 13_000_000_000; + let role = if index < 4 { + DataRole::Train + } else { + DataRole::HeldOut + }; + rt.begin_phase( + &rt.mutation_guard().unwrap(), + PhaseRequest { + trial_id: format!("epoch-trial-{index}"), + session_id: group + .map(str::to_owned) + .unwrap_or_else(|| format!("epoch-session-{index}")), + role, + label, + duration_ms: 12000, + lead_in_ms: 0, + conditions_confirmed: true, + }, + at, + ) + .unwrap(); + let gain = gain_override.unwrap_or(match label { + RoomLabel::Empty => 2.0, + RoomLabel::OccupiedStatic => 6.0, + RoomLabel::OccupiedMoving => 7.0, + RoomLabel::AdjacentOnly => 2.3, + RoomLabel::Unknown => panic!("unlabelled"), + }); + let context = rt.context().unwrap(); + for quarter in 0..4u32 { + for sample in 0..8u32 { + for source in &context.sources { + let values: Vec<_> = (0..source.feature_count) + .map(|bin| { + gain + f64::from(source.source_id) * 0.3 + + bin as f64 * 0.04 + + f64::from(sample) * 0.003 + }) + .collect(); + rt.ingest( + RawSourceFrame { + source_id: source.source_id, + grid_id: &source.grid_id, + frequency_hz: source.center_frequency_hz, + bandwidth_hz: source.bandwidth_hz, + sequence: index * 32 + quarter * 8 + sample + 1, + amplitudes: &values, + }, + at + u64::from(quarter) * 3_000_000_000 + + 1_000_000 + + u64::from(sample) * 10_000_000 + + u64::from(source.source_id), + ) + .unwrap(); + } + } + } + at + 12_000_000_000 +} +#[tokio::test] +async fn failed_assessment_cold_resume_keeps_all_sealed_phase_fences_not_only_train() { + let dir = tempfile::tempdir().unwrap(); + let rt = super::selection_tests::fitted(dir.path()).await; + let (model, proof) = current(&rt); + rt.qualify(&rt.mutation_guard().unwrap()).unwrap(); + // Deliberately wrong synthetic Empty observations produce a failed policy. + phase_at(&rt, RoomLabel::Empty, 8, START, Some(7.0)); + assert!(rt.qualify(&rt.mutation_guard().unwrap()).is_err()); + let fence = rt.store.load_managed_capture_fence(&model, &proof).unwrap(); + assert_eq!(fence.revision, 5); + let cold = restore(dir.path()); + let status = cold.status(START + 200_000_000_000); + assert!(status["model_selection"]["qualification_error"] + .as_str() + .is_some()); + let state = cold.inner.lock().unwrap(); + let capture = state.legacy_capture().unwrap(); + assert_eq!(capture.metadata_lineage(), Some(&fence.seed)); + let last = &fence.seed.sources[0]; + drop(state); + let values = vec![2.0; 8]; + assert!(cold + .ingest( + RawSourceFrame { + source_id: last.source_id, + grid_id: "ht64-canonical8", + frequency_hz: 2_437_000_000, + bandwidth_hz: 20_000_000, + sequence: last.sequence, + amplitudes: &values + }, + START + 200_000_000_000 + ) + .is_err()); + assert!(cold + .begin_phase( + &cold.mutation_guard().unwrap(), + PhaseRequest { + trial_id: "epoch-trial-8".into(), + session_id: "new-session".into(), + role: DataRole::HeldOut, + label: RoomLabel::Empty, + duration_ms: 12000, + lead_in_ms: 0, + conditions_confirmed: true + }, + START + 200_000_000_000 + ) + .is_err()); + // New real conditions require new IDs AND advancing original source counters. + phase_at(&cold, RoomLabel::Empty, 9, START, None); + let next = cold + .store + .load_managed_capture_fence(&model, &proof) + .unwrap(); + assert_eq!(next.revision, 6); + assert!(next.seed.sources[0].ordinal > last.ordinal); +} +#[tokio::test] +async fn missing_torn_and_unpublished_sealed_ledger_never_reset_to_train_floor() { + let dir = tempfile::tempdir().unwrap(); + let rt = super::selection_tests::fitted(dir.path()).await; + let (model, proof) = current(&rt); + let directory = proof_path(dir.path(), &model) + .parent() + .unwrap() + .join(format!("managed-fences-{}", proof.model_sha256)); + let head = directory.join("managed-sealed-phase-head-v1.json"); + let retained = std::fs::read(&head).unwrap(); + std::fs::remove_file(&head).unwrap(); + assert!(rt.store.load_managed_capture_fence(&model, &proof).is_err()); + let cold = runtime(dir.path()); + assert!(cold + .configure( + &cold.mutation_guard().unwrap(), + context(), + "missing-ledger".into(), + false + ) + .is_err()); + crate::companion::atomic_private_write(&head, b"{").unwrap(); + assert!(rt.store.load_managed_capture_fence(&model, &proof).is_err()); + crate::companion::atomic_private_write(&head, &retained).unwrap(); + // A pending immutable child without published head blocks model resume. + let original = rt.store.load_managed_capture_fence(&model, &proof).unwrap(); + let mut next_seed = original.seed.clone(); + let mut claim = next_seed.phase_claims.last().unwrap().clone(); + claim.trial_sha256 = "c".repeat(64); + claim.session_sha256 = "d".repeat(64); + next_seed.phase_claims.push(claim); + let pending = crate::room_learner::managed_capture_fences::ManagedCaptureFence::issue( + &proof, + Some(&original), + next_seed.clone(), + ) + .unwrap(); + let bytes = serde_json::to_vec(&pending).unwrap(); + crate::companion::atomic_private_write( + &directory.join(format!("{}.json", pending.fence_sha256)), + &bytes, + ) + .unwrap(); + assert!(rt + .store + .load_managed_capture_fence(&model, &proof) + .unwrap_err() + .contains("pending")); + // The exact finishing caller can complete this pending publication, keeping + // its original identity instead of inventing a newer observation timestamp. + let saved = rt + .store + .save_managed_capture_fence(&model, &proof, next_seed) + .unwrap(); + assert_eq!(saved, pending); + assert_eq!( + rt.store.load_managed_capture_fence(&model, &proof).unwrap(), + pending + ); +} +#[tokio::test] +async fn unix_scale_v2_wire_fixture_uses_decimal_ns_and_bounded_summary_only() { + let dir = tempfile::tempdir().unwrap(); + let rt = runtime(dir.path()); + let mut context = context(); + context.room_digest = "c".repeat(64); + for source in &mut context.sources { + source.device_id = format!("esp32-node-{}", source.source_id); + source.link_id = format!("receiver-{}-transmitter-unverified", source.source_id); + source.grid_id = crate::room_learner::native::grid_id( + 1, + 64, + wifi_densepose_hardware::PpduType::HtLegacy, + ) + .unwrap(); + source.feature_count = 56; + } + rt.configure( + &rt.mutation_guard().unwrap(), + context.clone(), + "synthetic-unix-time-fixture".into(), + true, + ) + .unwrap(); + const BASE: u64 = 1_793_600_000_000_000_000; + for (i, label) in crate::room_learner::qualification::CONDITIONS + .into_iter() + .enumerate() + { + phase_at(&rt, label, i as u32, BASE, None); + } + rt.start_training( + &rt.mutation_guard().unwrap(), + FitConfig { + epochs: 8, + learning_rate: 0.5, + hidden_dim: 2, + embedding_dim: 2, + temporal_tokens: 2, + ..FitConfig::default() + }, + ) + .unwrap(); + assert_eq!(super::tests::finished(&rt).await["state"], "ready"); + for (i, label) in crate::room_learner::qualification::CONDITIONS + .into_iter() + .enumerate() + { + phase_at(&rt, label, i as u32 + 4, BASE, None); + } + let v1 = rt.qualify(&rt.mutation_guard().unwrap()).unwrap(); + let cold = runtime(dir.path()); + cold.configure( + &cold.mutation_guard().unwrap(), + context.clone(), + "synthetic-fresh-runtime".into(), + false, + ) + .unwrap(); + let owner_status = cold.status(BASE + 300_000_000_000); + let readiness = owner_status["qualification_readiness"].clone(); + let clock_ineligible_status = cold.status(BASE - 1); + let diagnostic_dir = tempfile::tempdir().unwrap(); + let diagnostic = runtime(diagnostic_dir.path()); + let (diagnostic_model, _) = current(&rt); + diagnostic.store.save_candidate(&diagnostic_model).unwrap(); + diagnostic + .configure( + &diagnostic.mutation_guard().unwrap(), + context.clone(), + "v1-diagnostic-import".into(), + false, + ) + .unwrap(); + let legacy_ineligible_status = diagnostic.status(BASE + 300_000_000_000); + for (i, label) in crate::room_learner::qualification::CONDITIONS + .into_iter() + .enumerate() + { + phase_at(&cold, label, i as u32 + 8, BASE, None); + } + let q = cold.qualify(&cold.mutation_guard().unwrap()).unwrap(); + let receipt = serde_json::to_value(&q).unwrap(); + assert_eq!( + receipt["receipt"]["first_observed_at_ns"], + BASE.saturating_add(1_000_001).to_string() + ); + assert!(receipt["receipt"]["last_observed_at_ns"].is_string()); + let summary = serde_json::to_value(crate::room_learner::selection::QualificationSummary::from( + &q, + )) + .unwrap(); + assert!(summary["evaluated_at_unix_ms"].as_u64().unwrap() < (1u64 << 53)); + for key in [ + "first_observed_at_ns", + "last_observed_at_ns", + "phase_claims", + "sources", + "frame_ids", + "sequence_ids", + ] { + assert!(summary.get(key).is_none()); + } + let fixture = json!({"fixture_kind":"synthetic_native_producer_2026_utc","readiness":readiness,"owner_status":owner_status,"legacy_ineligible_status":legacy_ineligible_status,"clock_ineligible_status":clock_ineligible_status,"v1_summary":crate::room_learner::selection::QualificationSummary::from(&v1),"v2_summary":summary,"v2_envelope":receipt,"raw_train_lineage_checked":cold.status(BASE+300_000_000_000)["last_evaluation"]["train_lineage_checked"]}); + if let Ok(path) = std::env::var("RUVIEW_MANAGED_PROOF_FIXTURE_OUT") { + std::fs::write(path, serde_json::to_vec_pretty(&fixture).unwrap()).unwrap(); + } +} +#[tokio::test] +async fn shared_current_role_session_is_allowed_but_sealed_resume_group_cannot_be_reused() { + let dir = tempfile::tempdir().unwrap(); + let rt = runtime(dir.path()); + rt.configure( + &rt.mutation_guard().unwrap(), + context(), + "shared-role-fixture".into(), + true, + ) + .unwrap(); + for (i, label) in crate::room_learner::qualification::CONDITIONS + .into_iter() + .enumerate() + { + phase_at_group( + &rt, + label, + i as u32, + START, + None, + Some("one-declared-train-session"), + ); + } + rt.start_training( + &rt.mutation_guard().unwrap(), + FitConfig { + epochs: 8, + learning_rate: 0.5, + hidden_dim: 2, + embedding_dim: 2, + temporal_tokens: 2, + ..FitConfig::default() + }, + ) + .unwrap(); + assert_eq!(super::tests::finished(&rt).await["state"], "ready"); + for (i, label) in crate::room_learner::qualification::CONDITIONS + .into_iter() + .enumerate() + { + phase_at_group( + &rt, + label, + i as u32 + 4, + START, + None, + Some("one-declared-heldout-session"), + ); + } + let q = rt.qualify(&rt.mutation_guard().unwrap()).unwrap(); + assert_eq!(q.receipt.report.declared_session_count, 1); + assert!(!q.receipt.independence_verified); + let cold = restore(dir.path()); + assert!(cold + .begin_phase( + &cold.mutation_guard().unwrap(), + PhaseRequest { + trial_id: "actually-new-trial".into(), + session_id: "one-declared-heldout-session".into(), + role: DataRole::HeldOut, + label: RoomLabel::Empty, + duration_ms: 12000, + lead_in_ms: 0, + conditions_confirmed: true + }, + START + 200_000_000_000 + ) + .is_err()); + assert!(!cold.capture_active().unwrap()); +} +#[tokio::test] +async fn phase_seal_waits_for_durable_fence_and_pending_retry_keeps_original_metadata() { + let dir = tempfile::tempdir().unwrap(); + let rt = super::selection_tests::fitted(dir.path()).await; + let (model, proof) = current(&rt); + let directory = proof_path(dir.path(), &model) + .parent() + .unwrap() + .join(format!("managed-fences-{}", proof.model_sha256)); + let head = directory.join("managed-sealed-phase-head-v1.json"); + let old_head = std::fs::read(&head).unwrap(); + let old = rt.store.load_managed_capture_fence(&model, &proof).unwrap(); + let end = prepare_phase_at(&rt, RoomLabel::Empty, 8, START, None, None); + crate::companion::atomic_private_write(&head, b"{").unwrap(); + assert!(rt + .finish_phase(&rt.mutation_guard().unwrap(), end, true) + .is_err()); + assert!(rt.capture_active().unwrap()); + assert_eq!( + rt.inner + .lock() + .unwrap() + .legacy_capture() + .unwrap() + .history() + .len(), + 8 + ); + crate::companion::atomic_private_write(&head, &old_head).unwrap(); + let mut completed = rt.inner.lock().unwrap().legacy_capture().unwrap().clone(); + completed.finish_phase(end).unwrap(); + let pending = crate::room_learner::managed_capture_fences::ManagedCaptureFence::issue( + &proof, + Some(&old), + completed.managed_fence_seed(&proof).unwrap(), + ) + .unwrap(); + crate::companion::atomic_private_write( + &directory.join(format!("{}.json", pending.fence_sha256)), + &serde_json::to_vec(&pending).unwrap(), + ) + .unwrap(); + // Unlabelled packets after a completed phase don't replace the first sealed + // phase record's timestamps when the finish acknowledgement is retried. + let values = vec![2.0; 8]; + let last = &pending.seed.sources[0]; + rt.ingest( + RawSourceFrame { + source_id: last.source_id, + grid_id: "ht64-canonical8", + frequency_hz: 2_437_000_000, + bandwidth_hz: 20_000_000, + sequence: last.sequence + 1, + amplitudes: &values, + }, + end + 1, + ) + .unwrap(); + rt.finish_phase(&rt.mutation_guard().unwrap(), end + 2, true) + .unwrap(); + assert!(!rt.capture_active().unwrap()); + assert_eq!( + rt.store.load_managed_capture_fence(&model, &proof).unwrap(), + pending + ); +} +#[tokio::test] +async fn managed_proof_and_initial_fence_must_commit_before_candidate_pointer_and_retry_is_immutable( +) { + let source_dir = tempfile::tempdir().unwrap(); + let source = super::selection_tests::fitted(source_dir.path()).await; + let (model, original) = current(&source); + let target_dir = tempfile::tempdir().unwrap(); + let target = runtime(target_dir.path()); + let proof_file = proof_path(target_dir.path(), &model); + crate::companion::private_dir(proof_file.parent().unwrap()).unwrap(); + let fence_directory = proof_file + .parent() + .unwrap() + .join(format!("managed-fences-{}", original.model_sha256)); + crate::companion::atomic_private_write(&fence_directory, b"not a directory").unwrap(); + assert!(target + .store + .save_managed_candidate(&model, original.seed.clone()) + .is_err()); + assert!(target + .store + .load_candidate(model.context()) + .unwrap() + .is_none()); + assert!(!proof_file + .parent() + .unwrap() + .join("candidate-v1.json") + .exists()); + let first = std::fs::read(&proof_file).unwrap(); + std::fs::remove_file(&fence_directory).unwrap(); + let mut later = original.seed.clone(); + for source in &mut later.sources { + source.ordinal += 1; + source.sequence = source.sequence.wrapping_add(1); + source.last_time_ns += 1; + source.admitted_packets += 1; + } + later.last_seen_ns += 1; + target.store.save_managed_candidate(&model, later).unwrap(); + assert_eq!(std::fs::read(&proof_file).unwrap(), first); + assert_eq!( + target + .store + .load_managed_training_proof(&model) + .unwrap() + .unwrap(), + original + ); + let fence = target + .store + .load_managed_capture_fence(&model, &original) + .unwrap(); + assert_eq!(fence.revision, 0); + assert_eq!(fence.seed, original.seed); + assert!(target + .store + .load_candidate(model.context()) + .unwrap() + .is_some()); +} +#[tokio::test] +async fn malformed_metadata_checkpoint_fails_closed_without_integer_overflow() { + let dir = tempfile::tempdir().unwrap(); + let rt = super::selection_tests::fitted(dir.path()).await; + let (_, proof) = current(&rt); + let capture = CaptureSession::resume_managed_proof(&proof).unwrap(); + let at = proof.seed.last_seen_ns + 10; + let mut value = serde_json::to_value(capture.checkpoint(at).unwrap()).unwrap(); + value["metadata_lineage"]["phase_claims"][0]["cohort_count"] = json!(u64::MAX); + let snapshot = serde_json::from_value(value).unwrap(); + let result = std::panic::catch_unwind(|| { + CaptureSession::restore_checkpoint(snapshot, &proof.seed.context, at + 1) + }); + assert!( + result.is_ok(), + "malformed metadata panicked before bounded validation" + ); + assert!(result.unwrap().is_err()); +} +#[tokio::test] +async fn cold_resume_preserves_total_cohort_budget_and_refuses_before_operator_capture() { + let dir = tempfile::tempdir().unwrap(); + let rt = super::selection_tests::fitted(dir.path()).await; + for index in 8..32 { + phase_at(&rt, RoomLabel::Empty, index, START, None); + } + let cold = restore(dir.path()); + let result = cold.begin_phase( + &cold.mutation_guard().unwrap(), + PhaseRequest { + trial_id: "fresh-but-over-budget".into(), + session_id: "new-group".into(), + role: DataRole::HeldOut, + label: RoomLabel::Empty, + duration_ms: 12000, + lead_in_ms: 0, + conditions_confirmed: true, + }, + START + 500_000_000_000, + ); + assert!(result.unwrap_err().contains("capacity")); + assert!(!cold.capture_active().unwrap()); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/manifest.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/manifest.rs new file mode 100644 index 0000000000..5587f18c73 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/manifest.rs @@ -0,0 +1,131 @@ +//! Immutable input claims and hash from Train rows only; no raw rows retained. +use crate::room_learner::observation::{ + CaptureCohort, CaptureProvenance, ClockKind, DataRole, RoomLabel, +}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::io::{self, Write}; + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum ConsumedChannel { + Magnitude, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct TrainingManifest { + pub schema: String, + pub training_provenance: Vec, + pub training_data_sha256: String, + pub consumed_channels: Vec, + /// Validated input clock declarations, never an independent hardware receipt. + pub clock_claims: Vec, + pub measurement_clock_available: bool, + pub training_cohorts: usize, +} +fn provenance_rank(value: &CaptureProvenance) -> u8 { + match value { + CaptureProvenance::Synthetic => 0, + CaptureProvenance::CapturedReplay => 1, + CaptureProvenance::LiveReceiverObserved => 2, + } +} +fn clock_rank(value: &ClockKind) -> u8 { + match value { + ClockKind::Unknown => 0, + ClockKind::ReceiverOnly => 1, + ClockKind::SourceMeasurement => 2, + } +} +struct HashWriter(Sha256); +impl Write for HashWriter { + fn write(&mut self, bytes: &[u8]) -> io::Result { + self.0.update(bytes); + Ok(bytes.len()) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} +impl TrainingManifest { + pub(crate) fn from_training(rows: &[CaptureCohort]) -> Result { + if !(4..=64).contains(&rows.len()) + || rows + .iter() + .any(|c| c.role != DataRole::Train || c.label == RoomLabel::Unknown) + { + return Err("training manifest requires bounded labeled Train rows".into()); + } + if rows.iter().any(|c| { + c.views.iter().any(|v| { + v.provenance != c.provenance + || (v.signal_kind == ruview_unified::RfModality::Synthetic + && v.provenance != CaptureProvenance::Synthetic) + }) + }) { + return Err( + "synthetic RF or view provenance cannot claim captured/live training".into(), + ); + } + let mut provenance: Vec<_> = rows.iter().map(|r| r.provenance).collect(); + provenance.sort_by_key(provenance_rank); + provenance.dedup(); + let mut clocks: Vec<_> = rows + .iter() + .flat_map(|r| r.views.iter().map(|v| v.clock.kind)) + .collect(); + clocks.sort_by_key(clock_rank); + clocks.dedup(); + let available = !provenance.contains(&CaptureProvenance::Synthetic) + && clocks == [ClockKind::SourceMeasurement]; + let mut hash = HashWriter(Sha256::new()); + serde_json::to_writer(&mut hash, rows).map_err(|e| e.to_string())?; + let manifest = Self { + schema: "ruview.room-learner.train-manifest.v1".into(), + training_provenance: provenance, + training_data_sha256: format!("{:x}", hash.0.finalize()), + consumed_channels: vec![ConsumedChannel::Magnitude], + clock_claims: clocks, + measurement_clock_available: available, + training_cohorts: rows.len(), + }; + manifest.validate()?; + Ok(manifest) + } + pub(crate) fn validate(&self) -> Result<(), String> { + let expected_clock = !self + .training_provenance + .contains(&CaptureProvenance::Synthetic) + && self.clock_claims == [ClockKind::SourceMeasurement]; + if self.schema != "ruview.room-learner.train-manifest.v1" + || !(4..=64).contains(&self.training_cohorts) + || self.training_data_sha256.len() != 64 + || !self + .training_data_sha256 + .bytes() + .all(|c| c.is_ascii_digit() || (b'a'..=b'f').contains(&c)) + || self.training_provenance.is_empty() + || self.training_provenance.len() > 3 + || self + .training_provenance + .windows(2) + .any(|w| provenance_rank(&w[0]) >= provenance_rank(&w[1])) + || self.clock_claims.is_empty() + || self.clock_claims.len() > 3 + || self + .clock_claims + .windows(2) + .any(|w| clock_rank(&w[0]) >= clock_rank(&w[1])) + || self.consumed_channels != [ConsumedChannel::Magnitude] + || self.measurement_clock_available != expected_clock + { + return Err("invalid immutable Train provenance manifest".into()); + } + Ok(()) + } +} + +#[cfg(test)] +#[path = "manifest_tests.rs"] +mod tests; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/manifest_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/manifest_tests.rs new file mode 100644 index 0000000000..c7b5585e27 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/manifest_tests.rs @@ -0,0 +1,142 @@ +//! SYNTHETIC manifest integrity; provenance remains an input claim, not a receipt. +use super::*; +use crate::room_learner::{ + model::RoomModel, + training::{learning_tests::dataset, FitConfig}, +}; +use std::sync::OnceLock; +fn model() -> RoomModel { + static MODEL: OnceLock = OnceLock::new(); + MODEL + .get_or_init(|| { + RoomModel::fit( + &dataset(8), + FitConfig { + epochs: 2, + ..FitConfig::default() + }, + ) + .unwrap() + .0 + }) + .clone() +} +#[test] +fn streamed_digest_covers_actual_train_bytes_and_consumed_channels_are_explicit() { + let ds = dataset(8); + let manifest = TrainingManifest::from_training(ds.train()).unwrap(); + assert_eq!( + manifest.training_data_sha256, + format!( + "{:x}", + Sha256::digest(serde_json::to_vec(ds.train()).unwrap()) + ) + ); + assert_eq!(manifest.training_provenance, [CaptureProvenance::Synthetic]); + assert_eq!(manifest.consumed_channels, [ConsumedChannel::Magnitude]); + assert_eq!(manifest.clock_claims, [ClockKind::Unknown]); + assert!(!manifest.measurement_clock_available); + assert_eq!(manifest.training_cohorts, 8); + let mut altered = ds.train().to_vec(); + altered[0].views[0].samples[0][0] += 0.01; + assert_ne!( + manifest.training_data_sha256, + TrainingManifest::from_training(&altered) + .unwrap() + .training_data_sha256 + ); + assert!(TrainingManifest::from_training(ds.held_out()).is_err()); + assert!(TrainingManifest::from_training(ds.validation()).is_err()); +} +#[test] +fn model_round_trip_retains_synthetic_origin_without_raw_rows_or_phone_channels() { + let trained = model(); + let bytes = trained.export().unwrap(); + let imported = RoomModel::import(&bytes).unwrap(); + assert_eq!(trained.training_manifest(), imported.training_manifest()); + assert_eq!( + imported.training_manifest().training_provenance, + [CaptureProvenance::Synthetic] + ); + assert!(!imported.training_manifest().measurement_clock_available); + let json = String::from_utf8(bytes).unwrap(); + for raw in [ + "samples", + "frame_ids", + "trial_id", + "phase", + "camera", + "phone", + ] { + assert!(!json.contains(raw)); + } +} +#[test] +fn missing_unknown_duplicated_forged_live_and_nonfinite_manifest_fields_are_rejected() { + let bytes = model().export().unwrap(); + let original: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + let mut missing = original.clone(); + missing.as_object_mut().unwrap().remove("training_manifest"); + assert!(RoomModel::import(&serde_json::to_vec(&missing).unwrap()).is_err()); + for (field, value) in [ + ("training_provenance", serde_json::json!([])), + ( + "training_provenance", + serde_json::json!(["UnknownPhysicalSource"]), + ), + ( + "training_provenance", + serde_json::json!(["Synthetic", "Synthetic"]), + ), + ( + "training_provenance", + serde_json::json!(["LiveReceiverObserved"]), + ), + ( + "training_data_sha256", + serde_json::json!("not-a-training-data-hash"), + ), + ("training_data_sha256", serde_json::json!("A".repeat(64))), + ("consumed_channels", serde_json::json!(["phase"])), + ("consumed_channels", serde_json::json!([])), + ("clock_claims", serde_json::json!(["Unknown", "Unknown"])), + ("measurement_clock_available", serde_json::json!(true)), + ("training_cohorts", serde_json::json!(1.5)), + ("training_cohorts", serde_json::Value::Null), + ] { + let mut candidate = original.clone(); + candidate["training_manifest"][field] = value; + assert!( + RoomModel::import(&serde_json::to_vec(&candidate).unwrap()).is_err(), + "accepted malformed {field}" + ); + } + let nonfinite = String::from_utf8(bytes).unwrap().replace( + "\"measurement_clock_available\":false", + "\"measurement_clock_available\":NaN", + ); + assert!(RoomModel::import(nonfinite.as_bytes()).is_err()); +} +#[test] +fn source_clock_claims_never_promote_synthetic_to_measured_or_live() { + let ds = dataset(8); + let mut rows = ds.train().to_vec(); + for c in &mut rows { + for v in &mut c.views { + v.clock.kind = ClockKind::SourceMeasurement; + v.clock.uncertainty_ns = 0; + } + } + let manifest = TrainingManifest::from_training(&rows).unwrap(); + assert_eq!(manifest.clock_claims, [ClockKind::SourceMeasurement]); + assert!(!manifest.measurement_clock_available); + for c in &mut rows { + c.provenance = CaptureProvenance::LiveReceiverObserved; + for v in &mut c.views { + v.provenance = CaptureProvenance::LiveReceiverObserved; + } + } + assert!(TrainingManifest::from_training(&rows) + .unwrap_err() + .contains("synthetic RF")); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/mod.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/mod.rs new file mode 100644 index 0000000000..d303f72bfb --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/mod.rs @@ -0,0 +1,38 @@ +//! Account-bound multi-source room calibration and learned inference. +pub mod observation; +pub mod capture; +pub mod model; +pub mod training; +pub mod artifact; +pub mod lifecycle; +pub mod native; +pub mod recovery_store; + +pub mod qualification; +pub mod selection; + +pub mod assessment; + +pub(crate) mod managed_training; + +pub(crate) mod managed_capture_fences; + +/// Experimental source-only temporal acquisition; not wired to v1 runtime. +pub mod temporal_acquisition; + +/// Experimental policy-bound adapter; no runtime or adoption wiring. +pub mod temporal_model_input; + +/// Default-inactive continuous temporal input diagnostic; no runtime wiring. +pub mod temporal_live_driver; + +/// Inactive engineering schedule contract; no managed capture authority. +pub mod managed_schedule_contract; + +/// Whole-model diagnostic proof/evaluation DATA; decoded bytes are not authority. +pub(crate) mod temporal_evaluation; + +/// Versioned public whole-kind DTOs; no decoder grants producer authority. +pub mod temporal_wire; + +pub(crate) mod temporal_qualification; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/model.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/model.rs new file mode 100644 index 0000000000..814ca2c995 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/model.rs @@ -0,0 +1,431 @@ +//! Portable RF-only room presence learner. No raw observations are persisted. +use super::observation::{CaptureCohort, CaptureContext, SourceSpec}; +use crate::embedding::{EmbeddingConfig, EmbeddingExtractor}; +use crate::graph_transformer::{CrossAttention, Linear, TransformerConfig}; +use serde::{Deserialize, Serialize}; + +#[path = "manifest.rs"] +mod manifest; +pub use manifest::{ConsumedChannel, TrainingManifest}; +#[path = "projection.rs"] +mod projection; +pub(crate) use projection::{center_empty_heads, initial_weights as initial_projection_weights}; + +const MAX_ARTIFACT_BYTES: usize = 1_048_576; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct SourceProfile { + pub source: SourceSpec, + pub mean: Vec, + pub scale: Vec, + pub max_normalized: f32, +} + +/// Versioned, bounded architecture. Values are checked before allocating a network. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct EncoderConfig { + pub bins: usize, + /// Stable ordered context sources; metadata tags are not measured RF bins. + pub source_count: usize, + pub temporal_tokens: usize, + pub hidden_dim: usize, + pub embedding_dim: usize, + pub temperature: f32, +} +impl Default for EncoderConfig { + fn default() -> Self { + Self { + bins: 56, + source_count: 2, + temporal_tokens: 4, + hidden_dim: 3, + embedding_dim: 3, + temperature: 0.3, + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RoomModel { + pub(crate) version: u32, + pub(crate) algorithm: String, + pub(crate) context: CaptureContext, + pub(crate) training_manifest: TrainingManifest, + pub(crate) config: EncoderConfig, + pub(crate) profiles: Vec, + pub(crate) encoder_weights: Vec, + pub(crate) source_projection_weights: Vec>, + pub(crate) head_weights: Vec, +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum PresenceDecision { + Empty, + Occupied, + Abstain, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RoomPrediction { + pub decision: PresenceDecision, + pub occupied_probability: Option, + pub reason: Option, + /// Always false: a trained candidate is not physical qualification evidence. + pub physically_validated: bool, +} +impl RoomPrediction { + fn abstain(reason: String) -> Self { + Self { + decision: PresenceDecision::Abstain, + occupied_probability: None, + reason: Some(reason), + physically_validated: false, + } + } +} + +pub(crate) fn extractor(config: &EncoderConfig) -> EmbeddingExtractor { + EmbeddingExtractor::new( + TransformerConfig { + n_subcarriers: config.bins + config.source_count, + n_keypoints: 17, + d_model: config.hidden_dim, + n_heads: 1, + n_gnn_layers: 1, + }, + EmbeddingConfig { + d_model: config.hidden_dim, + d_proj: config.embedding_dim, + temperature: config.temperature, + normalize: false, + }, + ) +} + +/// Keep the tiny reused ReLU graph active around the calibrated empty baseline. +/// Layout follows CsiToPoseTransformer::flatten_weights' documented ordering. +pub(crate) fn initial_encoder(config: &EncoderConfig) -> Result { + let mut encoder = extractor(config); + let d = config.hidden_dim; + let mut weights = encoder.transformer.flatten_weights(); + // A separate fan-in for source metadata keeps its scale independent of + // how many canonical RF bins are present. This assumes no response polarity. + let input_width = config.bins + config.source_count; + let tags = Linear::with_seed(config.source_count, d, 501); + for row in 0..d { + for source in 0..config.source_count { + weights[row * input_width + config.bins + source] = tags.weights()[row][source]; + } + } + let gnn = (config.bins + config.source_count + 1) * d + + 17 * d + + CrossAttention::new(d, 1).param_count(); + for r in 0..d { + for c in 0..d { + weights[gnn + r * d + c] = if r == c { 1.0 } else { 0.0 }; + } + } + weights[gnn + d * d..gnn + d * d + d].fill(1.0); + encoder.transformer.unflatten_weights(&weights)?; + Ok(encoder) +} + +impl RoomModel { + pub fn context(&self) -> &CaptureContext { + &self.context + } + pub fn training_manifest(&self) -> &TrainingManifest { + &self.training_manifest + } + pub fn profiles(&self) -> &[SourceProfile] { + &self.profiles + } + pub fn encoder_config(&self) -> &EncoderConfig { + &self.config + } + pub fn parameter_count(&self) -> usize { + self.representation_parameter_count() + self.head_weights.len() + } + pub fn representation_parameter_count(&self) -> usize { + self.encoder_weights.len() + + self + .source_projection_weights + .iter() + .map(Vec::len) + .sum::() + } + /// Read-only learned parameters for independent audits and regression oracles. + pub fn parameters(&self) -> Vec { + self.encoder_weights + .iter() + .chain(self.source_projection_weights.iter().flatten()) + .chain(&self.head_weights) + .copied() + .collect() + } + pub(crate) fn set_parameters(&mut self, values: &[f32]) -> Result<(), String> { + if values.len() != self.parameter_count() || !values.iter().all(|x| x.is_finite()) { + return Err("invalid model parameter vector".into()); + } + let mut offset = self.encoder_weights.len(); + self.encoder_weights.copy_from_slice(&values[..offset]); + for weights in &mut self.source_projection_weights { + let end = offset + weights.len(); + weights.copy_from_slice(&values[offset..end]); + offset = end; + } + self.head_weights.copy_from_slice(&values[offset..]); + Ok(()) + } + pub fn export(&self) -> Result, String> { + self.validate()?; + let bytes = serde_json::to_vec(self).map_err(|e| e.to_string())?; + if bytes.len() > MAX_ARTIFACT_BYTES { + return Err("model artifact too large".into()); + } + Ok(bytes) + } + pub fn import(bytes: &[u8]) -> Result { + if bytes.len() > MAX_ARTIFACT_BYTES { + return Err("model artifact too large".into()); + } + let model: Self = serde_json::from_slice(bytes).map_err(|e| e.to_string())?; + model.validate()?; + Ok(model) + } + pub(crate) fn validate(&self) -> Result<(), String> { + self.training_manifest.validate()?; + if self + .context + .sources + .iter() + .any(|s| s.signal_kind == ruview_unified::RfModality::Synthetic) + && self.training_manifest.training_provenance + != [super::observation::CaptureProvenance::Synthetic] + { + return Err("synthetic modality cannot have live training provenance".into()); + } + if self.version != 1 + || self.algorithm != "aether_multiview_infonce_room_bce_v1" + || !(1..=128).contains(&self.config.bins) + || !(2..=8).contains(&self.config.source_count) + || self.config.source_count != self.context.sources.len() + || !(2..=16).contains(&self.config.temporal_tokens) + || !(2..=4).contains(&self.config.hidden_dim) + || !(2..=4).contains(&self.config.embedding_dim) + || !self.config.temperature.is_finite() + || !(0.1..=2.0).contains(&self.config.temperature) + { + return Err("unsupported room model version or architecture".into()); + } + if !(2..=8).contains(&self.context.sources.len()) + || self.profiles.len() != self.context.sources.len() + { + return Err("invalid source profile count".into()); + } + // Validate the context through the same trusted contract as capture. + self.context.validate()?; + if self + .context + .sources + .iter() + .any(|s| s.feature_count != self.config.bins) + { + return Err("encoder width does not match source grids".into()); + } + for (profile, source) in self.profiles.iter().zip(&self.context.sources) { + if &profile.source != source + || profile.mean.len() != source.feature_count + || profile.scale.len() != source.feature_count + || !profile.mean.iter().all(|x| x.is_finite()) + || !profile.scale.iter().all(|x| x.is_finite() && *x >= 0.01) + || !profile.max_normalized.is_finite() + || !(1.0..=1.0e6).contains(&profile.max_normalized) + { + return Err("invalid normalization profile".into()); + } + } + if self.encoder_weights.len() != extractor(&self.config).transformer.param_count() + || self.source_projection_weights.len() != self.config.source_count + || self.source_projection_weights.iter().any(|w| { + w.len() + != crate::embedding::ProjectionHead::new(projection::config(&self.config)) + .param_count() + }) + || self.head_weights.len() != self.profiles.len() * (self.config.embedding_dim + 2) + 1 + || self.parameter_count() > 512 + || !self + .parameters() + .iter() + .all(|x| x.is_finite() && x.abs() <= 1.0e4) + { + return Err("invalid learned weights".into()); + } + Ok(()) + } + pub fn infer(&self, cohort: &CaptureCohort) -> RoomPrediction { + let result = (|| { + self.validate()?; + let tokens = self.tokens(cohort, true)?; + let (features, _) = self.represent(&tokens, &self.parameters())?; + let probability = sigmoid(self.logit(&features, &self.head_weights)); + if !probability.is_finite() { + return Err("non-finite prediction".into()); + } + Ok(probability) + })(); + match result { + Err(e) => RoomPrediction::abstain(e), + Ok(p) => RoomPrediction { + occupied_probability: Some(p), + decision: if (0.45..=0.55).contains(&p) { + PresenceDecision::Abstain + } else if p > 0.5 { + PresenceDecision::Occupied + } else { + PresenceDecision::Empty + }, + reason: if (0.45..=0.55).contains(&p) { + Some("uncertain task probability".into()) + } else { + None + }, + physically_validated: false, + }, + } + } + pub(crate) fn tokens( + &self, + cohort: &CaptureCohort, + check_range: bool, + ) -> Result>>, String> { + super::observation::validate_cohort(&self.context, cohort)?; + let mut result = Vec::with_capacity(self.profiles.len()); + for (source_index, profile) in self.profiles.iter().enumerate() { + let view = cohort + .views + .iter() + .find(|v| v.source_id == profile.source.source_id) + .ok_or("missing trained source")?; + if view.samples.len() < self.config.temporal_tokens { + return Err("insufficient measured temporal rows".into()); + } + let bins = self.config.bins; + let time = self.config.temporal_tokens; + let mut tokens = vec![vec![0.0; bins]; time]; + let mut counts = vec![vec![0usize; bins]; time]; + for (t, frame) in view.samples.iter().enumerate() { + let ti = t * time / view.samples.len(); + for (k, value) in frame.iter().enumerate() { + let value = (*value - profile.mean[k]) / profile.scale[k]; + if !value.is_finite() + || (check_range && value.abs() > profile.max_normalized * 2.0 + 4.0) + { + return Err("RF input outside trained support".into()); + } + let bi = k; + tokens[ti][bi] += value; + counts[ti][bi] += 1; + } + } + for t in 0..time { + for b in 0..bins { + if counts[t][b] > 0 { + tokens[t][b] /= counts[t][b] as f32; + } + } + } + if !tokens.iter().flatten().all(|x| x.is_finite()) { + return Err("invalid RF tokens".into()); + } + // Source identity is a conditioning channel, appended only after + // RF baseline normalization; metadata never enters RF statistics. + for token in &mut tokens { + token.extend((0..self.config.source_count).map(|i| { + if i == source_index { + 1.0 + } else { + 0.0 + } + })); + } + result.push(tokens); + } + Ok(result) + } + pub(crate) fn represent( + &self, + tokens: &[Vec>], + params: &[f32], + ) -> Result<(Vec, Vec>), String> { + let mut encoder = extractor(&self.config); + encoder + .transformer + .unflatten_weights(¶ms[..self.encoder_weights.len()])?; + let projectors = self.source_projectors(params)?; + self.represent_with_encoder(tokens, &mut encoder, &projectors) + } + pub(crate) fn represent_with_encoder( + &self, + tokens: &[Vec>], + encoder: &mut EmbeddingExtractor, + projectors: &[crate::embedding::ProjectionHead], + ) -> Result<(Vec, Vec>), String> { + let mut features = + Vec::with_capacity(self.profiles.len() * (self.config.embedding_dim + 2)); + let mut embeddings = Vec::with_capacity(tokens.len()); + for (index, source) in tokens.iter().enumerate() { + // A frozen Train-only support scale bounds encoder RF values while + // retaining their signed baseline-relative DC. The task skip below + // uses the original calibrated values; metadata is never rescaled. + let mut conditioned = source.clone(); + for row in &mut conditioned { + for x in &mut row[..self.config.bins] { + *x /= self.profiles[index].max_normalized; + } + } + encoder.projection = projectors[index].clone(); + let embedding = encoder.extract(&conditioned); + features.extend_from_slice(&embedding); + // Calibrated DC and temporal energy skip preserve static occupancy + // evidence that per-window centering or L2 projection would erase. + let mean = source + .iter() + .flat_map(|row| row[..self.config.bins].iter()) + .sum::() + / (self.config.temporal_tokens * self.config.bins) as f32; + let temporal = source + .windows(2) + .flat_map(|w| { + w[0][..self.config.bins] + .iter() + .zip(&w[1][..self.config.bins]) + }) + .map(|(a, b)| (a - b).abs()) + .sum::() + / ((self.config.temporal_tokens - 1) * self.config.bins) as f32; + features.extend_from_slice(&[mean, temporal]); + embeddings.push(embedding); + } + if !features.iter().all(|x| x.is_finite()) { + return Err("non-finite representation".into()); + } + Ok((features, embeddings)) + } + pub(crate) fn logit(&self, features: &[f32], head: &[f32]) -> f32 { + features.iter().zip(head).map(|(x, w)| x * w).sum::() + head[head.len() - 1] + } +} + +pub(crate) fn sigmoid(x: f32) -> f32 { + if x >= 0.0 { + 1.0 / (1.0 + (-x).exp()) + } else { + let e = x.exp(); + e / (1.0 + e) + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/native.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/native.rs new file mode 100644 index 0000000000..862f2d02c7 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/native.rs @@ -0,0 +1,167 @@ +//! Transient native ESP32 I/Q plus amplitude-only canonical features. +//! Receiver timestamps and compatibility metadata never establish a source +//! measurement clock, temporal sample rate, or physical qualification. +use super::observation::{CaptureProvenance, ClockInfo, ClockKind}; +use num_complex::Complex64; +use ruview_unified::frame::{CalibrationState, EvidenceLevel, FieldAxis, FrameProvenance, + PhaseState, ProvenanceClass, RfFrameV2, SignalQuality}; +use ruview_unified::RfModality; +use serde::Serialize; +use wifi_densepose_hardware::{Adr018Flags, PpduType}; +use wifi_densepose_signal::hardware_norm::HardwareNormalizer; + +pub const CANONICAL_PACKET_BINS: usize = 56; + +/// Parsed ADR-018 metadata and ORIGINAL payload bytes, excluding the header. +/// Integer-Hz input makes NaN/Inf unrepresentable at this boundary. +pub struct NativePacketInput<'a> { + pub node_id: u8, + pub n_antennas: usize, + pub raw_bins: usize, + pub frequency_hz: u64, + pub ppdu_type: PpduType, + pub sequence: u32, + pub received_at_ns: u64, + /// Actual measured receiver/grid cadence, never a source sample-rate claim. + pub receiver_sample_rate_hz: f64, + pub iq_bytes: &'a [u8], + pub flags: Adr018Flags, + pub provenance: CaptureProvenance, +} + +/// Contains raw native samples and deliberately has no Serialize implementation. +/// Only `summary()` is appropriate for public/status export. +/// +/// ```compile_fail +/// use wifi_densepose_sensing_server::room_learner::native::CanonicalPacket; +/// fn export(packet: &CanonicalPacket) { let _ = serde_json::to_string(packet); } +/// ``` +pub struct CanonicalPacket { + pub native_frame: RfFrameV2, + pub source_id: u8, + pub grid_id: String, + pub sequence: u32, + pub frequency_hz: u64, + pub bandwidth_hz: u64, + pub amplitudes: Vec, + pub clock: ClockInfo, + pub phase_consumed: bool, + pub receiver_sample_rate_hz: f64, + pub temporal_sample_rate_known: bool, + pub signal_quality_known: bool, + pub provenance: CaptureProvenance, +} + +/// Bounded counts/metadata, never native IQ or derived amplitude rows. +#[derive(Debug, Serialize)] +pub struct NativePacketSummary<'a> { + pub source_id: u8, + pub grid_id: &'a str, + pub raw_bins: usize, + pub canonical_bins: usize, + pub frequency_hz: u64, + pub bandwidth_hz: u64, + pub clock: ClockInfo, + pub phase_consumed: bool, + pub receiver_sample_rate_hz: f64, + pub temporal_sample_rate_known: bool, + pub signal_quality_known: bool, + pub provenance: CaptureProvenance, +} +impl CanonicalPacket { + pub fn summary(&self) -> NativePacketSummary<'_> { + NativePacketSummary { source_id: self.source_id, grid_id: &self.grid_id, + raw_bins: self.native_frame.native_shape[1], canonical_bins: self.amplitudes.len(), + frequency_hz: self.frequency_hz, bandwidth_hz: self.bandwidth_hz, clock: self.clock, + phase_consumed: self.phase_consumed, receiver_sample_rate_hz: self.receiver_sample_rate_hz, + temporal_sample_rate_known: self.temporal_sample_rate_known, + signal_quality_known: self.signal_quality_known, provenance: self.provenance } + } +} +impl std::fmt::Debug for CanonicalPacket { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.summary().fmt(f) + } +} + +/// Fixed projection identity binds original hardware grid plus excluded region. +pub fn grid_id(n_antennas: usize, raw_bins: usize, ppdu_type: PpduType) -> Result { + let supported = n_antennas == 1 && match ppdu_type { + PpduType::HtLegacy => raw_bins == 64, + PpduType::HeSu | PpduType::HeMu | PpduType::HeTb => raw_bins == 256, + PpduType::Unknown => false, + }; + if !supported { return Err("unsupported ESP32 antenna/grid/PPDU".into()); } + Ok(format!("ant{n_antennas}-bins{raw_bins}-ppdu{}-usable-drop2-linear-v2", ppdu_type.to_byte())) +} + +pub fn is_supported_frequency_hz(frequency_hz: u64) -> bool { + let mhz = frequency_hz / 1_000_000; + frequency_hz % 1_000_000 == 0 + && ((2412..=2484).contains(&mhz) || (4900..=5900).contains(&mhz) + || (5925..=7125).contains(&mhz)) +} + +/// Preserves original signed int8 complex samples; uses only the existing +/// length-only normalizer for the derived magnitude features. Never calls +/// `normalize()` or `RfFrameV2::to_canonical()` (both remove amplitude gain). +pub fn canonicalize_esp32(input: NativePacketInput<'_>) -> Result { + let grid_id = grid_id(input.n_antennas, input.raw_bins, input.ppdu_type)?; + if input.flags.bw40 { return Err("native capture does not support the reported 40 MHz bandwidth".into()); } + if input.received_at_ns == 0 { return Err("native receiver time is unavailable".into()); } + if !input.receiver_sample_rate_hz.is_finite() || input.receiver_sample_rate_hz <= 0.0 { + return Err("measured receiver cadence is unavailable".into()); + } + if !is_supported_frequency_hz(input.frequency_hz) { + return Err("unknown or unsupported ESP32 centre frequency".into()); + } + let expected = input.raw_bins * 2; + if input.iq_bytes.len() != expected { + return Err("ESP32 native IQ payload must exactly match bounded declared grid".into()); + } + // ADR-356 / firmware csi_collector.c: the invalid first word is FOUR + // bytes = TWO complex int8 I/Q samples. Keep those bytes unchanged and + // record their invalidity; every packet uses the same drop2 projection. + let mut valid_mask = vec![true; input.raw_bins]; + if input.flags.first_word_sanitized { valid_mask[..2].fill(false); } + let iq: Vec<_> = input.iq_bytes.chunks_exact(2).map(|pair| + Complex64::new(f64::from(pair[0] as i8), f64::from(pair[1] as i8))).collect(); + let frame_id = (u128::from(input.received_at_ns) << 40) + | (u128::from(input.node_id) << 32) | u128::from(input.sequence); + let (class, evidence) = match input.provenance { + CaptureProvenance::Synthetic => (ProvenanceClass::Synthetic, EvidenceLevel::L0Simulation), + CaptureProvenance::CapturedReplay | CaptureProvenance::LiveReceiverObserved => + (ProvenanceClass::Measured, EvidenceLevel::L1CapturedReplay), + }; + let native_frame = RfFrameV2::new(frame_id, input.received_at_ns, RfModality::WifiCsi, + vec![FieldAxis::Antenna, FieldAxis::Frequency, FieldAxis::Time], + input.frequency_hz as f64, 20_000_000.0, + // This complete frame is receiver-observed. The time axis and rate + // describe measured host reception, never a native/source clock. + input.receiver_sample_rate_hz, vec![1, input.raw_bins, 1], iq, valid_mask, None, None, vec![], 0, + CalibrationState { phase_state: PhaseState::Raw, gain_calibrated: false, + clock_ppm: f64::MAX, baseline_id: None, confidence: 0.0 }, + // No RSSI/noise/loss metadata in this narrow input: neutral constructor + // placeholders, explicitly unavailable via signal_quality_known=false. + SignalQuality { rssi_dbm: 0.0, noise_floor_dbm: 0.0, packet_loss: 0.0, interference: 0.0 }, + FrameProvenance { class, evidence, device_id: format!("esp32-node-{}", input.node_id), + firmware: "unreported".into(), receipt_id: frame_id }).map_err(|e| e.to_string())?; + if native_frame.valid_mask[2..].iter().any(|valid| !valid) { + return Err("retained native region contains an invalid sample".into()); + } + let magnitudes: Vec<_> = native_frame.native_iq[2..].iter().map(|z| z.norm()).collect(); + let amplitudes = HardwareNormalizer::new().resample_magnitude_to_canonical(&magnitudes).map_err(|e|e.to_string())?; + if amplitudes.len() != CANONICAL_PACKET_BINS || amplitudes.iter().any(|v| !v.is_finite() || *v < 0.0) { + return Err("length-only magnitude projection produced an invalid feature vector".into()); + } + Ok(CanonicalPacket { native_frame, source_id: input.node_id, + grid_id, + sequence: input.sequence, frequency_hz: input.frequency_hz, bandwidth_hz: 20_000_000, + amplitudes, clock: ClockInfo { kind: ClockKind::ReceiverOnly, uncertainty_ns: u64::MAX }, + phase_consumed: false, receiver_sample_rate_hz: input.receiver_sample_rate_hz, temporal_sample_rate_known: false, signal_quality_known: false, + provenance: input.provenance }) +} + +#[cfg(test)] +#[path = "native_tests.rs"] +mod tests; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/native_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/native_tests.rs new file mode 100644 index 0000000000..1a49f5a9c4 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/native_tests.rs @@ -0,0 +1,176 @@ +//! SYNTHETIC wire payloads through the real native and signal contracts. +//! These test contract behavior, never sensing accuracy or real hardware. +use super::*; + +fn bytes(bins:usize,i:i8,q:i8)->Vec { + (0..bins).flat_map(|_|[i as u8,q as u8]).collect() +} +fn input(payload:&[u8],bins:usize)->NativePacketInput<'_> { + NativePacketInput {node_id:7,n_antennas:1,raw_bins:bins,frequency_hz:2_437_000_000, + ppdu_type:if bins==64 {PpduType::HtLegacy} else {PpduType::HeSu},sequence:23, + received_at_ns:1_234_000_000,receiver_sample_rate_hz:42.0,iq_bytes:payload,flags:Adr018Flags::default(), + provenance:CaptureProvenance::Synthetic} +} + +#[test] +fn ht64_packet_retains_original_signed_iq_phase_mask_and_unknown_geometry() { + let payload=bytes(64,-3,4); + let packet=canonicalize_esp32(input(&payload,64)).unwrap(); + assert_eq!(packet.native_frame.native_shape,vec![1,64,1]); + assert_eq!(packet.native_frame.axes,vec![FieldAxis::Antenna,FieldAxis::Frequency,FieldAxis::Time]); + assert!(packet.native_frame.native_iq.iter().all(|z|z.re==-3.0 && z.im==4.0)); + assert_eq!(packet.native_frame.native_iq[0].arg(),4.0f64.atan2(-3.0)); + assert!(packet.native_frame.valid_mask.iter().all(|v|*v)); + assert!(packet.native_frame.transmitter_pose.is_none() && packet.native_frame.receiver_pose.is_none() + && packet.native_frame.antenna_geometry.is_empty()); + assert_eq!(packet.grid_id,"ant1-bins64-ppdu0-usable-drop2-linear-v2"); +} +#[test] +fn he256_packet_uses_actual_frame_contract_and_existing_56_bin_projection() { + let payload=bytes(256,6,8); + let packet=canonicalize_esp32(input(&payload,256)).unwrap(); + let direct=HardwareNormalizer::new().resample_magnitude_to_canonical(&vec![10.0;254]).unwrap(); + assert_eq!(packet.amplitudes,direct); + assert_eq!(packet.native_frame.native_shape,vec![1,256,1]); + assert_eq!(packet.grid_id,"ant1-bins256-ppdu1-usable-drop2-linear-v2"); + assert_eq!(packet.amplitudes.len(),56); +} +#[test] +fn gain_and_dc_levels_survive_canonicalization_without_median_or_zscore() { + let low=bytes(64,3,4);let high=bytes(64,9,12); + let a=canonicalize_esp32(input(&low,64)).unwrap(); + let b=canonicalize_esp32(input(&high,64)).unwrap(); + assert!(a.amplitudes.iter().all(|v|*v==5.0)); + assert!(b.amplitudes.iter().all(|v|*v==15.0)); + assert_eq!(a.native_frame.native_iq[0],Complex64::new(3.0,4.0)); +} +#[test] +fn varying_native_iq_is_not_reconstructed_from_magnitude_or_overwritten() { + let payload:Vec<_>=(0..64).flat_map(|n|[(n as i8-32) as u8,(-(n as i8)/2) as u8]).collect(); + let packet=canonicalize_esp32(input(&payload,64)).unwrap(); + for (sample,pair) in packet.native_frame.native_iq.iter().zip(payload.chunks_exact(2)) { + assert_eq!((sample.re,sample.im),(f64::from(pair[0] as i8),f64::from(pair[1] as i8))); + } +} +#[test] +fn raw_phase_is_retained_but_amplitude_learner_consumes_no_phase_or_source_clock() { + let payload=bytes(64,3,4);let mut input=input(&payload,64); + input.flags.ieee802154_sync_valid=true; + let packet=canonicalize_esp32(input).unwrap(); + assert_eq!(packet.native_frame.calibration.phase_state,PhaseState::Raw); + assert!(!packet.phase_consumed && !packet.temporal_sample_rate_known && !packet.signal_quality_known); + assert_eq!(packet.clock,ClockInfo {kind:ClockKind::ReceiverOnly,uncertainty_ns:u64::MAX}); + assert_eq!(packet.native_frame.timestamp_ns,1_234_000_000); + assert_eq!(packet.native_frame.calibration.confidence,0.0); + assert!(!packet.native_frame.calibration.gain_calibrated); +} +#[test] +fn provenance_never_grants_lab_or_physical_qualification() { + for provenance in [CaptureProvenance::Synthetic,CaptureProvenance::CapturedReplay,CaptureProvenance::LiveReceiverObserved] { + let payload=bytes(64,3,4);let mut input=input(&payload,64);input.provenance=provenance; + let packet=canonicalize_esp32(input).unwrap(); + let expected=if provenance==CaptureProvenance::Synthetic {(ProvenanceClass::Synthetic,EvidenceLevel::L0Simulation)} + else {(ProvenanceClass::Measured,EvidenceLevel::L1CapturedReplay)}; + assert_eq!((packet.native_frame.provenance.class,packet.native_frame.provenance.evidence),expected); + assert_eq!(packet.provenance,provenance); + assert_eq!(packet.clock.kind,ClockKind::ReceiverOnly); + } +} +#[test] +fn public_summary_and_debug_never_export_native_samples_amplitudes_or_frame_ids() { + let payload=bytes(64,-3,4);let packet=canonicalize_esp32(input(&payload,64)).unwrap(); + let public=serde_json::to_string(&packet.summary()).unwrap();let debug=format!("{packet:?}"); + for sensitive in ["native_iq","amplitudes","samples","frame_id","valid_mask"] { + assert!(!public.contains(sensitive));assert!(!debug.contains(sensitive)); + } + assert!(public.contains("ReceiverOnly") && public.contains("\"phase_consumed\":false")); +} +#[test] +fn malformed_grid_payload_bandwidth_frequency_and_validity_fail_closed() { + for change in 0..11 { + let payload=bytes(256,3,4);let mut i=input(&payload[..128],64); + match change {0=>i.n_antennas=0,1=>i.n_antennas=2,2=>i.raw_bins=usize::MAX, + 3=>i.raw_bins=128,4=>i.ppdu_type=PpduType::Unknown,5=>i.ppdu_type=PpduType::HeSu, + 6=>i.flags.bw40=true,7=>i.frequency_hz=0,8=>i.frequency_hz=2_437_000_001, + 9=>i.received_at_ns=0,_=>i.iq_bytes=&payload} + assert!(canonicalize_esp32(i).is_err(),"survived input mutant {change}"); + } +} +#[test] +fn iq_wire_integer_domain_cannot_produce_nan_or_inf_native_samples() { + let payload:Vec<_>=(0u8..=255).cycle().take(512).collect(); + let packet=canonicalize_esp32(input(&payload,256)).unwrap(); + assert!(packet.native_frame.native_iq.iter().all(|z|z.re.is_finite() && z.im.is_finite())); + assert!(packet.amplitudes.iter().all(|x|x.is_finite())); +} +#[test] +fn exact_declared_packet_size_is_required_without_padding_or_trailing_data() { + for size in [0,1,127,129,1024] { + let payload=vec![1;size]; + assert!(canonicalize_esp32(input(&payload,64)).is_err()); + } +} +#[test] +fn actual_raw_grid_identity_survives_both_supported_families() { + for ppdu in [PpduType::HtLegacy,PpduType::HeSu,PpduType::HeMu,PpduType::HeTb] { + let bins=if ppdu==PpduType::HtLegacy {64} else {256};let payload=bytes(bins,3,4); + let mut i=input(&payload,bins);i.ppdu_type=ppdu; + let packet=canonicalize_esp32(i).unwrap(); + assert_eq!(packet.grid_id,format!("ant1-bins{bins}-ppdu{}-usable-drop2-linear-v2",ppdu.to_byte())); + assert_eq!((packet.frequency_hz,packet.bandwidth_hz),(2_437_000_000,20_000_000)); + } +} + +#[test] +fn sanitized_word_is_retained_with_two_invalid_samples_and_fixed_projection() { + let original=bytes(256,3,4);let mut sanitized=original.clone();sanitized[..4].fill(0); + let normal=canonicalize_esp32(input(&original,256)).unwrap(); + let mut i=input(&sanitized,256);i.flags.first_word_sanitized=true; + let masked=canonicalize_esp32(i).unwrap(); + assert_eq!(normal.amplitudes,masked.amplitudes); + assert_eq!(normal.grid_id,masked.grid_id); + assert_eq!(&masked.native_frame.valid_mask[..2],&[false,false]); + assert!(masked.native_frame.valid_mask[2..].iter().all(|v|*v)); + assert_eq!(masked.native_frame.native_iq[0],Complex64::new(0.0,0.0)); + assert_eq!(normal.native_frame.native_iq[0],Complex64::new(3.0,4.0)); +} +#[test] +fn source_record_requires_actual_receiver_rate_and_never_invents_one_hz() { + let payload=bytes(64,3,4); + for rate in [0.0,-1.0,f64::NAN,f64::INFINITY,f64::NEG_INFINITY] { + let mut i=input(&payload,64);i.receiver_sample_rate_hz=rate; + assert!(canonicalize_esp32(i).is_err()); + } + let packet=canonicalize_esp32(input(&payload,64)).unwrap(); + assert_eq!(packet.native_frame.sample_rate_hz,42.0); + assert_eq!(packet.receiver_sample_rate_hz,42.0); + assert!(!packet.temporal_sample_rate_known); +} + +#[test] +fn valid_native_spectral_nulls_are_admitted_without_negative_projected_magnitudes() { + for bins in [64, 256] { + let payload: Vec = (0..bins) + .flat_map(|bin| [if bin % 8 == 0 { 100 } else { 0 }, 0]) + .collect(); + let result = canonicalize_esp32(input(&payload, bins)); + assert!( + result.is_ok(), + "valid {bins}-bin spectral nulls must not be rejected by interpolation: {result:?}" + ); + let packet = result.unwrap(); + assert!(packet + .amplitudes + .iter() + .all(|v| v.is_finite() && *v >= 0.0 && *v <= 100.0)); + for (actual, original) in packet + .native_frame + .native_iq + .iter() + .zip(payload.chunks_exact(2)) + { + assert_eq!(actual.re, f64::from(original[0] as i8)); + assert_eq!(actual.im, 0.0); + } + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/observation.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/observation.rs new file mode 100644 index 0000000000..2e77b75bdc --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/observation.rs @@ -0,0 +1,421 @@ +//! Source-preserving RF views and frozen, leakage-free capture cohorts. +//! All declarations are input claims: validation never grants physical authority. +use ruview_unified::{LinkGeometry, RfModality, RfTensor}; +use ruview_unified::frame::{FieldAxis, PhaseState, ProvenanceClass, RfFrameV2}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::collections::{HashMap, HashSet}; + +pub const MAX_SOURCES: usize = 16; +pub const MAX_FEATURES: usize = 512; +pub const MAX_ROWS: usize = 256; +pub const MAX_COHORTS: usize = 128; +pub const MAX_TOTAL_VALUES: usize = 2_000_000; +pub const MAX_AGE_NS: u64 = 5_000_000_000; +pub const MAX_WINDOW_NS: u64 = 60_000_000_000; +pub const MAX_CLOCK_UNCERTAINTY_NS: u64 = 100_000_000; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct SourceSpec { + pub source_id: u8, + pub device_id: String, + pub grid_id: String, + pub feature_count: usize, + pub signal_kind: RfModality, + pub center_frequency_hz: u64, + pub bandwidth_hz: u64, + pub link_id: String, + pub link_index: usize, + pub geometry: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct CaptureContext { + pub owner_scope: String, + pub installation_id: String, + pub room_digest: String, + pub revision: u64, + pub sources: Vec, +} + +impl CaptureContext { + /// SHA-256 of the full ordered source/configuration contract, not RF data. + pub fn digest(&self) -> String { + let bytes = serde_json::to_vec(self).unwrap_or_default(); + format!("{:x}", Sha256::digest(bytes)) + } + + pub fn validate(&self) -> Result<(), String> { + for s in [&self.owner_scope, &self.installation_id, &self.room_digest] { identifier(s)?; } + if self.revision == 0 || self.sources.is_empty() || self.sources.len() > MAX_SOURCES { + return Err("invalid context revision or source count".into()); + } + if self.sources.windows(2).any(|w| w[0].source_id >= w[1].source_id) { + return Err("source specs must be sorted uniquely by source id".into()); + } + let mut ids = HashSet::new(); + let mut links = HashSet::new(); + for s in &self.sources { + for id in [&s.device_id, &s.grid_id, &s.link_id] { identifier(id)?; } + if !ids.insert(s.source_id) || !links.insert((&s.device_id, &s.link_id)) + || s.feature_count == 0 || s.feature_count > MAX_FEATURES || s.link_index >= MAX_SOURCES + || s.center_frequency_hz == 0 || s.bandwidth_hz == 0 { + return Err("duplicate source/link or invalid feature count".into()); + } + if let Some(g) = s.geometry { geometry(g)?; } + } + Ok(()) + } +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +pub enum ClockKind { SourceMeasurement, ReceiverOnly, Unknown } +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct ClockInfo { pub kind: ClockKind, pub uncertainty_ns: u64 } +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +pub enum CaptureProvenance { Synthetic, CapturedReplay, LiveReceiverObserved } +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +pub enum PhaseAvailability { Raw, Sanitized, Calibrated, MagnitudeOnly, Unknown } + +/// Explicit per-row identities/times supplied by ingestion; tensor timestamp alone +/// does not establish a source clock (#1710). +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ViewMetadata { + pub frame_ids: Vec, + pub sequence_ids: Vec, + pub first_time_ns: u64, + pub last_time_ns: u64, + pub clock: ClockInfo, + pub phase: PhaseAvailability, + pub link_index: usize, + pub provenance: CaptureProvenance, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SourceView { + pub source_id: u8, + pub device_id: String, + pub grid_id: String, + pub signal_kind: RfModality, + pub center_frequency_hz: u64, + pub bandwidth_hz: u64, + pub link_id: String, + pub link_index: usize, + pub geometry: Option, + pub samples: Vec>, + pub frame_ids: Vec, + pub sequence_ids: Vec, + pub first_time_ns: u64, + pub last_time_ns: u64, + pub clock: ClockInfo, + pub sample_age_ns: u64, + pub phase: PhaseAvailability, + pub provenance: CaptureProvenance, +} + +impl SourceView { + /// Reads magnitudes without per-window centering, scaling or gain removal. + /// Unified adapter normalization averages away per-link gain: it is not + /// invertible. Such tensors must use the native-frame entry point instead. + pub fn from_tensor(source: &SourceSpec, tensor: &RfTensor, meta: ViewMetadata) -> Result { + let (links, bins, rows) = tensor.dims(); + if tensor.calibration.gain_offset_db != 0.0 || tensor.calibration.baseline_id.is_some() { + return Err("gain-normalized/baseline-subtracted tensor cannot preserve source amplitude".into()); + } + if links != 1 || meta.link_index != source.link_index || meta.link_index >= links || tensor.links.len() != links || bins != source.feature_count + || tensor.device_id != source.device_id || tensor.modality != source.signal_kind + || tensor.center_freq_hz != source.center_frequency_hz as f64 + || tensor.bandwidth_hz != source.bandwidth_hz as f64 + || !tensor.sample_age_s.is_finite() || tensor.sample_age_s < 0.0 + || tensor.sample_age_s > MAX_AGE_NS as f64 / 1e9 + || rows == 0 || rows > MAX_ROWS || bins > MAX_FEATURES + || tensor.timestamp_ns != meta.last_time_ns { + return Err("tensor/source/window contract mismatch".into()); + } + if meta.phase == PhaseAvailability::Raw || meta.phase == PhaseAvailability::Sanitized + || (meta.phase == PhaseAvailability::Calibrated && !tensor.calibration.phase_calibrated) { + return Err("tensor does not establish requested phase state".into()); + } + let actual_geometry = tensor.links[meta.link_index]; + if source.geometry.is_some_and(|g| g != actual_geometry) { + return Err("tensor link geometry changed".into()); + } + let samples = (0..rows).map(|t| (0..bins) + .map(|b| tensor.data[[meta.link_index, b, t]].norm() as f32).collect()).collect(); + let view = Self::build(source, samples, meta, (tensor.sample_age_s * 1e9) as u64, + tensor.modality == RfModality::Synthetic); + view.validate(source)?; + Ok(view) + } + + /// Native IQ avoids the existing canonical adapter's median normalization. + /// Only an explicit `(link, bin, time)` layout with complete valid samples + /// is accepted; missing bins cannot silently become measurements. + pub fn from_frame(source: &SourceSpec, frame: &RfFrameV2, meta: ViewMetadata) -> Result { + if frame.schema_version != 2 || frame.native_shape.len() != 3 + || frame.axes.len() != 3 || frame.axes[0] != FieldAxis::Antenna + || !matches!(frame.axes[1], FieldAxis::Frequency | FieldAxis::Delay | FieldAxis::Range) + || frame.axes[2] != FieldAxis::Time || frame.sample_age_ns > MAX_AGE_NS + || frame.modality != source.signal_kind || frame.provenance.device_id != source.device_id + || frame.centre_frequency_hz != source.center_frequency_hz as f64 + || frame.bandwidth_hz != source.bandwidth_hz as f64 + || frame.calibration.baseline_id.is_some() || frame.timestamp_ns != meta.last_time_ns { + return Err("native frame/source/layout contract mismatch".into()); + } + let (links, bins, rows) = (frame.native_shape[0], frame.native_shape[1], frame.native_shape[2]); + if links == 0 || links > MAX_SOURCES || bins != source.feature_count || bins > MAX_FEATURES + || rows == 0 || rows > MAX_ROWS || meta.link_index != source.link_index || meta.link_index >= links + || frame.native_iq.len() != links * bins * rows || frame.valid_mask.len() != links * bins * rows + || meta.frame_ids.iter().any(|id| *id != frame.frame_id) { + return Err("native shape/frame identity mismatch".into()); + } + let expected_phase = match frame.calibration.phase_state { + PhaseState::Raw => PhaseAvailability::Raw, + PhaseState::Sanitized => PhaseAvailability::Sanitized, + PhaseState::Calibrated => PhaseAvailability::Calibrated, + PhaseState::Unavailable => PhaseAvailability::MagnitudeOnly, + }; + if meta.phase != expected_phase { return Err("native phase declaration mismatch".into()); } + if let Some(g) = source.geometry { + if frame.transmitter_pose.map(|p| p.position_m) != Some(g.tx_pos) + || frame.receiver_pose.map(|p| p.position_m) != Some(g.rx_pos) { + return Err("native geometry changed".into()); + } + } + let mut samples = Vec::with_capacity(rows); + for t in 0..rows { + let mut row = Vec::with_capacity(bins); + for b in 0..bins { + let i = meta.link_index * bins * rows + b * rows + t; + if !frame.valid_mask[i] { return Err("native sample is invalid".into()); } + row.push(frame.native_iq[i].norm() as f32); + } + samples.push(row); + } + let view = Self::build(source, samples, meta, frame.sample_age_ns, + frame.provenance.class == ProvenanceClass::Synthetic); + view.validate(source)?; + Ok(view) + } + + fn build(s: &SourceSpec, samples: Vec>, m: ViewMetadata, sample_age_ns: u64, synthetic: bool) -> Self { + Self { source_id: s.source_id, device_id: s.device_id.clone(), grid_id: s.grid_id.clone(), + signal_kind: s.signal_kind, center_frequency_hz: s.center_frequency_hz, bandwidth_hz: s.bandwidth_hz, + link_id: s.link_id.clone(), link_index: s.link_index, geometry: s.geometry, samples, + frame_ids: m.frame_ids, sequence_ids: m.sequence_ids, first_time_ns: m.first_time_ns, + last_time_ns: m.last_time_ns, clock: m.clock, sample_age_ns, phase: m.phase, + provenance: if synthetic { CaptureProvenance::Synthetic } else { m.provenance } } + } + + pub fn validate(&self, s: &SourceSpec) -> Result<(), String> { + if self.source_id != s.source_id || self.grid_id != s.grid_id || self.device_id != s.device_id + || self.link_id != s.link_id || self.link_index != s.link_index || self.signal_kind != s.signal_kind || self.geometry != s.geometry + || self.center_frequency_hz != s.center_frequency_hz || self.bandwidth_hz != s.bandwidth_hz { + return Err("source identity, PHY grid, modality or geometry changed".into()); + } + if (self.signal_kind == RfModality::Synthetic && self.provenance != CaptureProvenance::Synthetic) + || self.samples.len() < 2 || self.samples.len() > MAX_ROWS + || self.frame_ids.len() != self.samples.len() || self.sequence_ids.len() != self.samples.len() + || self.first_time_ns == 0 || self.last_time_ns <= self.first_time_ns + || self.last_time_ns - self.first_time_ns > MAX_WINDOW_NS || self.sample_age_ns > MAX_AGE_NS + || self.frame_ids.contains(&0) || self.sequence_ids.windows(2).any(|w| w[1] <= w[0]) + || (self.clock.kind == ClockKind::SourceMeasurement + && self.clock.uncertainty_ns > MAX_CLOCK_UNCERTAINTY_NS) { + return Err("invalid bounded source window, identities, age or clock".into()); + } + if self.samples.iter().any(|row| row.len() != s.feature_count + || row.iter().any(|v| !v.is_finite() || *v < 0.0)) { + return Err("invalid magnitude sample/feature grid".into()); + } + Ok(()) + } +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Hash)] +pub enum DataRole { Train, Validation, HeldOut, Inference } +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Hash)] +pub enum RoomLabel { Empty, OccupiedStatic, OccupiedMoving, AdjacentOnly, Unknown } +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +pub enum TeacherKind { Camera, Radar } +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct TeacherEvidence { + pub kind: TeacherKind, + pub source_id: String, + /// Unverified receipt claim; only an external paired registry can resolve it. + pub validation_receipt: String, + pub context_digest: String, + pub room_digest: String, + pub trial_id: String, + pub label: RoomLabel, + pub frame_ids: Vec, + pub first_time_ns: u64, + pub last_time_ns: u64, + pub clock: ClockInfo, +} +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CaptureCohort { + pub id: String, + pub context_digest: String, + pub trial_id: String, + pub session_id: String, + pub role: DataRole, + pub label: RoomLabel, + pub views: Vec, + pub teacher: Option, + pub provenance: CaptureProvenance, +} + +/// Read-only after construction: held-out rows cannot accidentally be fitted. +#[derive(Debug, Clone)] +pub struct ValidatedDataset { + context: CaptureContext, + train: Vec, + validation: Vec, + held_out: Vec, + measurement_clock_available: bool, +} +type SourceRoleIntervals = HashMap<(u8, String), Vec<(u64, u64, DataRole)>>; +impl ValidatedDataset { + pub fn new(context: CaptureContext, cohorts: Vec) -> Result { + context.validate()?; + if cohorts.is_empty() || cohorts.len() > MAX_COHORTS { return Err("invalid cohort count".into()); } + let digest = context.digest(); + let mut result = Self { context, train: vec![], validation: vec![], held_out: vec![], measurement_clock_available: true }; + let mut total_values = 0usize; + let mut ids = HashSet::new(); + let mut trials = HashMap::new(); + let mut sessions = HashMap::new(); + let mut frames = HashMap::new(); + let mut sequences = HashMap::new(); + let mut intervals: SourceRoleIntervals = HashMap::new(); + for c in cohorts { + if c.role == DataRole::Inference || c.label == RoomLabel::Unknown { + return Err("inference/unlabelled input cannot enter a labelled dataset".into()); + } + validate_cohort(&result.context, &c)?; + for id in [&c.id, &c.trial_id, &c.session_id] { identifier(id)?; } + if !ids.insert(c.id.clone()) || c.context_digest != digest || c.views.len() != result.context.sources.len() { + return Err("duplicate cohort, wrong context or incomplete selected sources".into()); + } + for (map, key) in [(&mut trials, &c.trial_id), (&mut sessions, &c.session_id)] { + if map.insert(key.clone(), c.role).is_some_and(|role| role != c.role) { + return Err("trial/session crosses frozen roles".into()); + } + } + let mut selected = HashSet::new(); + let mut first = 0; + let mut last = u64::MAX; + for v in &c.views { + let spec = result.context.sources.iter().find(|s| s.source_id == v.source_id) + .ok_or("unknown source")?; + v.validate(spec)?; + total_values += v.samples.len() * spec.feature_count; + if total_values > MAX_TOTAL_VALUES { return Err("dataset value budget exceeded".into()); } + if !selected.insert(v.source_id) { return Err("duplicate source view".into()); } + result.measurement_clock_available &= v.provenance != CaptureProvenance::Synthetic + && v.clock.kind == ClockKind::SourceMeasurement; + first = first.max(v.first_time_ns); + last = last.min(v.last_time_ns); + for id in &v.frame_ids { + if frames.insert((v.source_id, *id), c.role).is_some_and(|r| r != c.role) { + return Err("frame overlap crosses roles".into()); + } + } + for seq in &v.sequence_ids { + if sequences.insert((v.source_id, c.session_id.clone(), *seq), c.id.clone()) + .is_some_and(|id| id != c.id) { return Err("reused source sequence".into()); } + } + let ranges = intervals.entry((v.source_id, c.session_id.clone())).or_default(); + if ranges.iter().any(|(a,b,r)| *r != c.role && v.first_time_ns <= *b && v.last_time_ns >= *a) { + return Err("source time window overlaps roles".into()); + } + ranges.push((v.first_time_ns, v.last_time_ns, c.role)); + } + if first >= last { return Err("selected source windows are not aligned".into()); } + if let Some(t) = &c.teacher { validate_teacher(t, &c, &result.context, first, last)?; } + // Labels without an independently validated teacher are still useful + // experimental/manual cohorts, never physical empty-room evidence. + + match c.role { DataRole::Train => result.train.push(c), DataRole::Validation => result.validation.push(c), DataRole::HeldOut => result.held_out.push(c), DataRole::Inference => return Err("inference dataset rejected".into()) } + } + Ok(result) + } + pub fn context(&self) -> &CaptureContext { &self.context } + pub fn train(&self) -> &[CaptureCohort] { &self.train } + pub fn validation(&self) -> &[CaptureCohort] { &self.validation } + pub fn held_out(&self) -> &[CaptureCohort] { &self.held_out } + /// Clock/provenance input claims only; never an activation or qualification receipt. + pub fn measurement_clock_available(&self) -> bool { self.measurement_clock_available } +} + +/// Validates one inference/capture boundary without assigning or changing splits. +pub fn validate_cohort(context: &CaptureContext, c: &CaptureCohort) -> Result<(), String> { + context.validate()?; + if (c.role == DataRole::Inference) != (c.label == RoomLabel::Unknown) + || (c.role == DataRole::Inference && c.teacher.is_some()) { + return Err("inference must be unlabelled and cannot carry teacher labels".into()); + } + for id in [&c.id, &c.trial_id, &c.session_id] { identifier(id)?; } + if c.context_digest != context.digest() || c.views.len() != context.sources.len() { + return Err("cohort context or selected source count mismatch".into()); + } + let mut ids = HashSet::new(); + let (mut first, mut last) = (0, u64::MAX); + for v in &c.views { + let spec = context.sources.iter().find(|s| s.source_id == v.source_id).ok_or("unknown source")?; + v.validate(spec)?; + if !ids.insert(v.source_id) || v.provenance != c.provenance { + return Err("duplicate view or inconsistent provenance".into()); + } + first = first.max(v.first_time_ns); + last = last.min(v.last_time_ns); + } + if first >= last { return Err("selected source windows are not aligned".into()); } + if let Some(t) = &c.teacher { validate_teacher(t, c, context, first, last)?; } + Ok(()) +} + +fn validate_teacher(t: &TeacherEvidence, c: &CaptureCohort, context: &CaptureContext, first: u64, last: u64) -> Result<(), String> { + // A client-supplied receipt string is never proof of a validated source. + // Until a paired-source resolver exists, only synthetic fixtures can attach + // teacher claims. Real capture uses explicit operator labels. + if c.provenance != CaptureProvenance::Synthetic { + return Err("teacher registry resolver unavailable for measured capture".into()); + } + identifier(&t.source_id)?; + identifier(&t.validation_receipt)?; + if t.context_digest != c.context_digest || t.room_digest != context.room_digest || t.trial_id != c.trial_id + || t.label != c.label || t.first_time_ns > first || t.last_time_ns < last + || c.views.iter().any(|v| t.first_time_ns > v.first_time_ns || t.last_time_ns < v.last_time_ns) + || t.clock.kind != ClockKind::SourceMeasurement || t.clock.uncertainty_ns > MAX_CLOCK_UNCERTAINTY_NS + || t.frame_ids.is_empty() || t.frame_ids.len() > MAX_ROWS || t.frame_ids.contains(&0) { + return Err("teacher is not validated and bound to this room/phase/time/frame".into()); + } + let rf_frames: HashSet<_> = c.views.iter().flat_map(|v| v.frame_ids.iter()).collect(); + if !t.frame_ids.iter().all(|id| rf_frames.contains(id)) + || !rf_frames.iter().all(|id| t.frame_ids.contains(id)) { + return Err("teacher references an unknown RF frame".into()); + } + Ok(()) +} +fn identifier(s: &str) -> Result<(), String> { + if s.is_empty() || s.len() > 256 || s.trim() != s || !s.is_ascii() || s.chars().any(char::is_control) { + Err("invalid bounded identifier".into()) + } else { Ok(()) } +} +fn geometry(g: LinkGeometry) -> Result<(), String> { + if g.tx_pos.iter().chain(g.rx_pos.iter()).any(|x| !x.is_finite()) { + Err("non-finite link geometry".into()) + } else { Ok(()) } +} + +#[cfg(test)] +#[path = "observation_tests.rs"] +mod tests; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/observation_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/observation_tests.rs new file mode 100644 index 0000000000..298ae12b96 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/observation_tests.rs @@ -0,0 +1,247 @@ +//! SYNTHETIC contract fixtures only; these tests make no sensing-accuracy claim. +use super::*; +use ruview_unified::frame::{CalibrationState, EvidenceLevel, FrameProvenance, SignalQuality}; + +fn context() -> CaptureContext { + CaptureContext { owner_scope: "synthetic-owner".into(), installation_id: "synthetic-install".into(), + room_digest: "synthetic-room".into(), revision: 1, sources: vec![SourceSpec { + source_id: 1, device_id: "synthetic-device".into(), grid_id: "phy-20mhz-2bins".into(), + feature_count: 2, signal_kind: RfModality::Synthetic, center_frequency_hz: 2_400_000_000, bandwidth_hz: 20_000_000, link_id: "tx-a-rx-b".into(), link_index: 0, geometry: None }] } +} +fn cohort(context: &CaptureContext, id: u64, role: DataRole) -> CaptureCohort { + CaptureCohort { id: format!("synthetic-{id}"), context_digest: context.digest(), + trial_id: format!("synthetic-trial-{id}"), session_id: format!("synthetic-session-{id}"), role, + label: RoomLabel::Empty, teacher: None, provenance: CaptureProvenance::Synthetic, + views: vec![SourceView { source_id: 1, device_id: "synthetic-device".into(), + grid_id: "phy-20mhz-2bins".into(), signal_kind: RfModality::Synthetic, center_frequency_hz: 2_400_000_000, bandwidth_hz: 20_000_000, + link_id: "tx-a-rx-b".into(), link_index: 0, geometry: None, samples: vec![vec![2.0,3.0], vec![4.0,5.0]], + frame_ids: vec![u128::from(id)*2+1,u128::from(id)*2+2], sequence_ids: vec![1,2], + first_time_ns: 100+id*10, last_time_ns: 102+id*10, + clock: ClockInfo {kind: ClockKind::Unknown, uncertainty_ns: u64::MAX}, sample_age_ns: 0, + phase: PhaseAvailability::MagnitudeOnly, provenance: CaptureProvenance::Synthetic }] } +} +fn native_frame() -> RfFrameV2 { + // Get the real complex element type through the actual native contract; + // no lookalike tensor type or direct ndarray dependency is introduced. + let iq = [2.0,4.0,3.0,5.0].into_iter().map(Into::into).collect(); + RfFrameV2::new(7, 102, RfModality::Synthetic, + vec![FieldAxis::Antenna,FieldAxis::Frequency,FieldAxis::Time], + 2.4e9,20e6,100.0,vec![1,2,2],iq,vec![true;4],None,None,vec![],0, + CalibrationState {phase_state: PhaseState::Unavailable,gain_calibrated:false,clock_ppm:20.0, + baseline_id:None,confidence:0.0}, + SignalQuality {rssi_dbm:-50.0,noise_floor_dbm:-90.0,packet_loss:0.0,interference:0.0}, + FrameProvenance {class:ProvenanceClass::Synthetic,evidence:EvidenceLevel::L0Simulation, + device_id:"synthetic-device".into(),firmware:"SYNTHETIC".into(),receipt_id:7}).unwrap() +} +fn metadata() -> ViewMetadata { + ViewMetadata {frame_ids:vec![7,7],sequence_ids:vec![1,2],first_time_ns:100,last_time_ns:102, + clock:ClockInfo {kind:ClockKind::Unknown,uncertainty_ns:u64::MAX}, + phase:PhaseAvailability::MagnitudeOnly,link_index:0,provenance:CaptureProvenance::Synthetic} +} + +#[test] +fn synthetic_dataset_exposes_frozen_roles_and_no_physical_clock_claim() { + let c = context(); + let ds = ValidatedDataset::new(c.clone(),vec![cohort(&c,1,DataRole::Train), + cohort(&c,2,DataRole::Validation),cohort(&c,3,DataRole::HeldOut)]).unwrap(); + assert_eq!((ds.train().len(),ds.validation().len(),ds.held_out().len()),(1,1,1)); + assert!(!ds.measurement_clock_available()); +} +#[test] +fn native_adapter_preserves_raw_gain_and_static_mean_shift() { + let c = context(); + let f = native_frame(); + let first = SourceView::from_frame(&c.sources[0],&f,metadata()).unwrap(); + let mut stronger = f.clone(); + for z in &mut stronger.native_iq { *z *= 3.0; } + let second = SourceView::from_frame(&c.sources[0],&stronger,metadata()).unwrap(); + assert_eq!(first.samples,vec![vec![2.0,3.0],vec![4.0,5.0]]); + assert_eq!(second.samples,vec![vec![6.0,9.0],vec![12.0,15.0]]); + assert_eq!(f.native_iq,native_frame().native_iq); +} +#[test] +fn actual_canonical_adapter_gain_normalization_is_refused() { + let c = context(); + let t = native_frame().to_canonical(vec![LinkGeometry{tx_pos:[0.0;3],rx_pos:[1.0,0.0,0.0]}]).unwrap(); + assert!(SourceView::from_tensor(&c.sources[0],&t,metadata()).unwrap_err().contains("gain-normalized")); +} +#[test] +fn tensor_adapter_reads_real_tensor_without_centering() { + let mut c = context(); + c.sources[0].feature_count = 56; + let mut t = native_frame().to_canonical(vec![LinkGeometry{tx_pos:[0.0;3],rx_pos:[1.0,0.0,0.0]}]).unwrap(); + // Explicitly synthetic raw-amplitude tensor: remove fixture adapter scale + // and overwrite every sample. No assertion is made about measured gain. + t.calibration.gain_offset_db = 0.0; + for z in &mut t.data { z.re = 7.0; z.im = 0.0; } + let mut m = metadata(); m.frame_ids = vec![7;8]; m.sequence_ids = (1..=8).collect(); + let view = SourceView::from_tensor(&c.sources[0],&t,m).unwrap(); + assert!(view.samples.iter().flatten().all(|x| *x == 7.0)); +} +#[test] +fn tensor_mutant_nonfinite_samples_are_rejected_despite_public_tensor_fields() { + let mut c = context(); c.sources[0].feature_count = 56; + let mut t = native_frame().to_canonical(vec![LinkGeometry{tx_pos:[0.0;3],rx_pos:[1.0,0.0,0.0]}]).unwrap(); + t.calibration.gain_offset_db = 0.0; + t.data[[0,0,0]].re = f64::NAN; + let mut m = metadata(); m.frame_ids = vec![7;8]; m.sequence_ids = (1..=8).collect(); + assert!(SourceView::from_tensor(&c.sources[0],&t,m).is_err()); +} +#[test] +fn native_invalid_mask_and_phase_are_not_invented_as_measurements() { + let c = context(); let mut f = native_frame(); f.valid_mask[0] = false; + assert!(SourceView::from_frame(&c.sources[0],&f,metadata()).is_err()); + f.valid_mask[0] = true; + let mut m = metadata(); m.phase = PhaseAvailability::Calibrated; + assert!(SourceView::from_frame(&c.sources[0],&f,m).is_err()); +} +#[test] +fn synthetic_boundary_mutants_are_rejected() { + let c = context(); + let original = cohort(&c,1,DataRole::Train); + type Mutant = fn(&mut CaptureCohort); + let mutants: &[(&str,Mutant)] = &[ + ("context",|x|x.context_digest="wrong-room-owner-revision".into()), + ("missing source",|x|x.views.clear()), + ("unknown source",|x|x.views[0].source_id=2), + ("duplicate source",|x|x.views.push(x.views[0].clone())), + ("changed PHY #1440",|x|x.views[0].grid_id="phy-40mhz".into()), + ("changed frequency #1440",|x|x.views[0].center_frequency_hz+=5_000_000), + ("changed bandwidth #1440",|x|x.views[0].bandwidth_hz*=2), + ("changed link",|x|x.views[0].link_id="other-link".into()), + ("changed device",|x|x.views[0].device_id="other-device".into()), + ("NaN",|x|x.views[0].samples[0][0]=f32::NAN), + ("oversized rows",|x|x.views[0].samples.resize(MAX_ROWS+1,vec![1.0,2.0])), + ("wrong feature count",|x|x.views[0].samples[0].push(1.0)), + ("stale",|x|x.views[0].sample_age_ns=MAX_AGE_NS+1), + ("sequence reversal",|x|x.views[0].sequence_ids=vec![2,1]), + ("source clock uncertainty #1710",|x|x.views[0].clock=ClockInfo{kind:ClockKind::SourceMeasurement,uncertainty_ns:u64::MAX}), + ("nonmonotonic time",|x|x.views[0].last_time_ns=1), + ("long window",|x|x.views[0].last_time_ns=MAX_WINDOW_NS+200), + ("forged provenance",|x|x.provenance=CaptureProvenance::LiveReceiverObserved), + ]; + for (name,mutate) in mutants { + let mut candidate = original.clone(); mutate(&mut candidate); + assert!(ValidatedDataset::new(c.clone(),vec![candidate]).is_err(),"survived mutant: {name}"); + } +} +#[test] +fn frame_reuse_cannot_cross_roles_even_when_trial_and_session_are_renamed() { + let c = context(); let a = cohort(&c,1,DataRole::Train); + let mut b = cohort(&c,2,DataRole::HeldOut); b.views[0].frame_ids = a.views[0].frame_ids.clone(); + assert!(ValidatedDataset::new(c,vec![a,b]).unwrap_err().contains("frame overlap")); +} +#[test] +fn trial_or_session_cannot_cross_roles() { + for use_trial in [true,false] { + let c = context(); let a = cohort(&c,1,DataRole::Train); let mut b = cohort(&c,2,DataRole::Validation); + if use_trial {b.trial_id=a.trial_id.clone();} else {b.session_id=a.session_id.clone();} + assert!(ValidatedDataset::new(c,vec![a,b]).unwrap_err().contains("frozen roles")); + } +} +#[test] +fn source_clock_is_not_inferred_from_receive_time_or_time_of_day() { + let mut c = context(); c.sources[0].signal_kind=RfModality::WifiCsi; + let mut a = cohort(&c,1,DataRole::Train); a.views[0].signal_kind=RfModality::WifiCsi; + a.provenance=CaptureProvenance::LiveReceiverObserved; + a.views[0].provenance=a.provenance; + a.views[0].clock.kind=ClockKind::ReceiverOnly; + let dataset=ValidatedDataset::new(c,vec![a]).unwrap(); + assert!(!dataset.measurement_clock_available()); +} +#[test] +fn context_digest_binds_owner_room_revision_and_phy_not_time_of_day() { + let c=context(); + for change in 0..4 { + let mut other=c.clone(); + match change {0=>other.owner_scope="other-owner".into(),1=>other.room_digest="other-room".into(), + 2=>other.revision+=1,_=>other.sources[0].grid_id="other-PHY".into()} + assert_ne!(other.digest(),c.digest()); + } +} +#[test] +fn missing_teacher_never_yields_a_camera_derived_empty_label() { + let c=context(); let a=cohort(&c,1,DataRole::Train); + let ds=ValidatedDataset::new(c,vec![a]).unwrap(); + assert!(ds.train()[0].teacher.is_none()); +} +fn teacher(c:&CaptureContext,a:&CaptureCohort)->TeacherEvidence { + TeacherEvidence {kind:TeacherKind::Radar,source_id:"synthetic-teacher".into(), + validation_receipt:"synthetic-receipt-NOT-qualification".into(),context_digest:c.digest(), + room_digest:c.room_digest.clone(),trial_id:a.trial_id.clone(),label:a.label, + frame_ids:a.views[0].frame_ids.clone(),first_time_ns:a.views[0].first_time_ns, + last_time_ns:a.views[0].last_time_ns,clock:ClockInfo{kind:ClockKind::SourceMeasurement,uncertainty_ns:1}} +} +#[test] +fn receive_time_only_radar_cannot_supply_empty_teacher_evidence() { + let c=context(); let mut a=cohort(&c,1,DataRole::Train); let mut t=teacher(&c,&a); + t.clock.kind=ClockKind::ReceiverOnly; a.teacher=Some(t); + assert!(ValidatedDataset::new(c,vec![a]).is_err()); +} +#[test] +fn teacher_room_phase_time_frame_and_validated_source_are_bound() { + let c=context(); let a=cohort(&c,1,DataRole::Train); let t=teacher(&c,&a); + let mutants: &[fn(&mut TeacherEvidence)] = &[ + |x|x.room_digest="elsewhere".into(),|x|x.label=RoomLabel::OccupiedMoving, + |x|x.first_time_ns=x.last_time_ns+1,|x|x.frame_ids=vec![999],|x|x.validation_receipt.clear(), + |x|x.source_id.clear(),|x|x.trial_id="different-trial".into()]; + for mutate in mutants { + let mut candidate=a.clone(); let mut evidence=t.clone(); mutate(&mut evidence); + candidate.teacher=Some(evidence); + assert!(ValidatedDataset::new(c.clone(),vec![candidate]).is_err()); + } +} +#[test] +fn context_rejects_unicode_unbounded_ids_and_unsorted_sources() { + let mut c=context(); c.owner_scope="untrusted-\u{202e}".into(); assert!(c.validate().is_err()); + c=context();c.owner_scope="x".repeat(257);assert!(c.validate().is_err()); + c=context();let mut second=c.sources[0].clone();second.source_id=0;second.link_id="other".into(); + c.sources.push(second);assert!(c.validate().is_err()); +} + +#[test] +fn nonempty_teacher_receipt_claim_never_replaces_validated_registry() { + let mut c=context(); c.sources[0].signal_kind=RfModality::WifiCsi; + let mut a=cohort(&c,1,DataRole::Train); a.views[0].signal_kind=RfModality::WifiCsi; + a.provenance=CaptureProvenance::CapturedReplay; a.views[0].provenance=a.provenance; + a.teacher=Some(teacher(&c,&a)); + assert!(ValidatedDataset::new(c,vec![a]).unwrap_err().contains("registry resolver unavailable")); +} + +#[test] +fn selected_sources_require_complete_aligned_windows_and_bound_link_indices() { + let mut c=context(); let mut second=c.sources[0].clone(); + second.source_id=2; second.link_id="second-link".into(); c.sources.push(second); + let mut a=cohort(&c,1,DataRole::Train); + let mut view=a.views[0].clone();view.source_id=2;view.link_id="second-link".into(); + a.views.push(view); + assert!(validate_cohort(&c,&a).is_ok()); + a.views[1].first_time_ns=1000;a.views[1].last_time_ns=1002; + assert!(validate_cohort(&c,&a).is_err()); + a.views[1].first_time_ns=a.views[0].first_time_ns;a.views[1].last_time_ns=a.views[0].last_time_ns; + a.views[1].link_index=1;assert!(validate_cohort(&c,&a).is_err()); +} +#[test] +fn unrecognized_fields_cannot_silently_claim_validated_sources() { + let c=context();let a=cohort(&c,1,DataRole::Train); + let mut json=serde_json::to_value(a).unwrap();json["validated"]=true.into(); + assert!(serde_json::from_value::(json).is_err()); +} +#[test] +fn tensor_cannot_promote_unknown_phase_to_calibrated() { + let mut c=context();c.sources[0].feature_count=56; + let mut t=native_frame().to_canonical(vec![LinkGeometry{tx_pos:[0.0;3],rx_pos:[1.0,0.0,0.0]}]).unwrap(); + t.calibration.gain_offset_db=0.0;t.calibration.phase_calibrated=false; + let mut m=metadata();m.phase=PhaseAvailability::Calibrated; + m.frame_ids=vec![7;8];m.sequence_ids=(1..=8).collect(); + assert!(SourceView::from_tensor(&c.sources[0],&t,m).unwrap_err().contains("phase state")); +} + +#[test] +fn inference_is_unlabelled_and_cannot_enter_training_or_evaluation_dataset() { + let c=context();let mut a=cohort(&c,1,DataRole::Inference); + assert!(validate_cohort(&c,&a).is_err()); + a.label=RoomLabel::Unknown;assert!(validate_cohort(&c,&a).is_ok()); + assert!(ValidatedDataset::new(c.clone(),vec![a.clone()]).is_err()); + a.role=DataRole::Train;assert!(validate_cohort(&c,&a).is_err()); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/optimization.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/optimization.rs new file mode 100644 index 0000000000..02416ba602 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/optimization.rs @@ -0,0 +1,142 @@ +//! Bounded RF descent followed by exact linear-task block descent. +//! Both blocks minimize the same declared joint objective; no validation tuning. +use super::{objective, represent_row, FitConfig, ObjectiveValues, Row}; +use crate::{ + room_learner::model::{extractor, sigmoid, RoomModel}, + trainer::{clip_gradients, estimate_gradient, SgdOptimizer}, +}; +use std::cell::Cell; +pub(super) fn head_step( + model: &RoomModel, + rows: &[Row], + params: &[f32], + config: &FitConfig, + current: ObjectiveValues, + check: &impl Fn() -> Result<(), String>, + evaluations: &Cell, +) -> Result, ObjectiveValues)>, String> { + let offset = model.representation_parameter_count(); + let mut gradient = vec![0.0; params.len()]; + let mut encoder = extractor(&model.config); + encoder + .transformer + .unflatten_weights(¶ms[..model.encoder_weights.len()])?; + let projectors = model.source_projectors(params)?; + for row in rows { + check()?; + let (features, _) = represent_row(model, row, &mut encoder, &projectors)?; + let residual = (sigmoid(model.logit(&features, ¶ms[offset..])) - row.occupied) + * config.supervised_weight + / rows.len() as f32; + for (g, x) in gradient[offset..].iter_mut().zip(features) { + *g += residual * x; + } + *gradient.last_mut().unwrap() += residual; + } + clip_gradients(&mut gradient, 5.0); + for backtrack in 0..8 { + check()?; + let mut next = params.to_vec(); + SgdOptimizer::new(config.learning_rate * 0.5f32.powi(backtrack), 0.0, 0.0) + .step(&mut next, &gradient); + evaluations.set(evaluations.get() + 1); + let loss = objective(model, rows, &next, config)?; + if loss.total < current.total { + return Ok(Some((next, loss))); + } + } + Ok(None) +} + +/// A second, smaller finite-difference radius handles ReLU boundaries without +/// changing the objective. Every probe remains inside the caller's hard budgets. +pub(super) fn joint_step( + model: &RoomModel, + rows: &[Row], + params: &[f32], + config: &FitConfig, + current: ObjectiveValues, + check: &impl Fn() -> Result<(), String>, + evaluations: &Cell, +) -> Result, ObjectiveValues)>, String> { + for attempt in 0..2 { + check()?; + let epsilon = (config.gradient_epsilon * 0.25f32.powi(attempt)).max(0.0001); + if attempt > 0 && epsilon == config.gradient_epsilon { + break; + } + let mut gradient = estimate_gradient( + |p| { + if check().is_err() { + return f32::NAN; + } + evaluations.set(evaluations.get() + 1); + objective(model, rows, p, config) + .map(|l| l.total) + .unwrap_or(f32::NAN) + }, + params, + epsilon, + ); + check()?; + if !gradient.iter().all(|g| g.is_finite()) { + return Err("non-finite training gradient".into()); + } + clip_gradients(&mut gradient, 5.0); + for backtrack in 0..16 { + check()?; + let mut next = params.to_vec(); + SgdOptimizer::new(config.learning_rate * 0.5f32.powi(backtrack), 0.0, 0.0) + .step(&mut next, &gradient); + evaluations.set(evaluations.get() + 1); + let loss = objective(model, rows, &next, config)?; + if loss.total < current.total { + return Ok(Some((next, loss))); + } + } + } + Ok(None) +} +pub(super) fn require_rf_update( + model: &RoomModel, + params: &[f32], + initial: &[f32], +) -> Result<(), String> { + let backbone = model.encoder_weights.len(); + let representation = model.representation_parameter_count(); + if params[..backbone] == initial[..backbone] + || params[backbone..representation] == initial[backbone..representation] + { + return Err( + "no improving backbone/projection update; refusing a head-only RF candidate".into(), + ); + } + Ok(()) +} + +/// Task gradients must not be suppressed by a clipped RF gradient or its small +/// accepted step. Optimize both blocks against the same Train-only objective. +pub(super) fn epoch_step( + model: &RoomModel, + rows: &[Row], + params: &[f32], + config: &FitConfig, + current: ObjectiveValues, + check: &impl Fn() -> Result<(), String>, + evaluations: &Cell, +) -> Result, ObjectiveValues)>, String> { + let mut next = params.to_vec(); + let mut loss = current; + let mut changed = false; + if let Some((p, l)) = joint_step(model, rows, params, config, current, check, evaluations)? { + next = p; + loss = l; + changed = true; + } + if let Some((p, l)) = head_step(model, rows, &next, config, loss, check, evaluations)? { + next = p; + loss = l; + changed = true; + } + Ok(if changed { Some((next, loss)) } else { None }) +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/paired_progress.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/paired_progress.rs new file mode 100644 index 0000000000..492518aeb3 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/paired_progress.rs @@ -0,0 +1,155 @@ +//! Read-only phone companion projection. Counts record capture, never accuracy. +use super::*; +use crate::room_learner::{capture::PhaseStatus, + selection::{SelectedBinaryEvidence, ModelSelectionStatus}}; + +const MAX_SAFE: u64 = 9_007_199_254_740_991; +pub const PAIRED_PROGRESS_TTL_MS: u64 = 10_000; +#[derive(Debug, Clone, Serialize)] +pub struct PublishedProgressRoom { pub binding_digest: String, pub revision: u64 } +#[derive(Debug, Clone, Serialize)] +pub struct RecordedCondition { pub label: RoomLabel, pub train_recorded: usize, pub held_out_recorded: usize } +#[derive(Debug, Clone, Serialize)] +pub struct ProgressPhase { + pub role: DataRole, pub label: RoomLabel, pub stage: &'static str, + pub target_ms: u64, pub lead_in_remaining_ms: u64, pub capture_elapsed_ms: u64, + pub representative_cohorts: usize, pub required_cohorts: usize, +} +#[derive(Debug, Clone, Serialize)] +pub struct ProgressSources { pub required: usize, pub ready: usize } +#[derive(Debug, Clone, Serialize)] +pub struct ProgressQualification { pub state: &'static str, pub receipt_sha256: Option, pub model_sha256: Option } +#[derive(Debug, Clone, Serialize)] +pub struct ProgressSelection { pub state: &'static str, pub selection_sha256: Option, pub model_sha256: Option, pub current_usable: bool } +#[derive(Debug, Clone, Serialize)] +pub struct PairedCalibrationProgress { + pub schema: &'static str, pub success: bool, pub installation_id: String, + /// Existing public calibration process identity, never the owner mutation guard. + pub runtime_epoch: String, pub observed_at_unix_ms: u64, pub expires_at_unix_ms: u64, + pub room: Option, pub context_match: bool, + pub stage: &'static str, pub state: &'static str, + pub conditions: Option>, pub phase: Option, + pub sources: Option, pub qualification: ProgressQualification, + pub selection: ProgressSelection, pub save_status: &'static str, pub blocker: Option<&'static str>, +} +fn bounded_text(value: &str) -> bool { !value.is_empty() && value.encode_utf16().count() <= 128 && !value.chars().any(char::is_control) } +fn digest(value: &str) -> bool { value.len()==64 && value.bytes().all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) } +impl RoomLearningRuntime { + /// Does not call owner status/checkpoint or advance workflow/model state. + /// Artifact readback reuses existing owner-private store enforcement. + /// Caller holds the published binding stable. + pub fn paired_progress_snapshot(&self, installation_id: &str, runtime_epoch: &str, + room: Option, externally_ready_sources: &[u8], ready_context_digest: Option<&str>, now_ns: u64, + ) -> Result { + let observed = now_ns / 1_000_000; + if !bounded_text(installation_id) || !bounded_text(runtime_epoch) || observed==0 || observed>MAX_SAFE-PAIRED_PROGRESS_TTL_MS + || room.as_ref().is_some_and(|r| !digest(&r.binding_digest) || r.revision==0 || r.revision>MAX_SAFE) + { return Err("progress identity/time unavailable".into()); } + let state=self.inner.lock().map_err(|_| "learner state unavailable")?; + state.require_legacy_kind()?; + let mut result=PairedCalibrationProgress { + schema:"ruview.room-learner.progress.v1",success:true,installation_id:installation_id.into(),runtime_epoch:runtime_epoch.into(), + observed_at_unix_ms:observed,expires_at_unix_ms:observed+PAIRED_PROGRESS_TTL_MS,room, + context_match:false,stage:"setup",state:"idle",conditions:None,phase:None,sources:None, + qualification:ProgressQualification{state:"not_reported",receipt_sha256:None,model_sha256:None}, + selection:ProgressSelection{state:"not_reported",selection_sha256:None,model_sha256:None,current_usable:false}, + save_status:"not_reported",blocker:Some("room_not_published"), + }; + let Some(published)=result.room.as_ref() else { return Ok(result); }; + let Some(capture)=state.legacy_capture() else { result.blocker=Some("learner_not_configured");return Ok(result); }; + let context=capture.context(); + if context.installation_id!=installation_id || context.room_digest!=published.binding_digest { + result.stage="blocked";result.blocker=Some("context_mismatch");return Ok(result); + } + if ready_context_digest!=Some(context.digest().as_str()) {return Err("progress context changed".into());} + if context.sources.is_empty() || context.sources.len()>8 {return Err("invalid progress source count".into());} + if state.model.as_ref().is_some_and(|m|m.context()!=context) {return Err("invalid progress candidate".into());} + let candidate_sha=state.model.as_ref().map(|m|m.export().map(|bytes|{ + use sha2::{Digest,Sha256};format!("{:x}",Sha256::digest(bytes)) + })).transpose().map_err(|_|"invalid progress candidate")?; + result.context_match=true; + result.state=match state.job.phase {JobPhase::Idle=>"idle",JobPhase::Running=>"running",JobPhase::Cancelling=>"cancelling",JobPhase::Ready=>"ready",JobPhase::Failed=>"failed",JobPhase::Cancelled=>"cancelled"}; + let labels=[RoomLabel::Empty,RoomLabel::OccupiedStatic,RoomLabel::OccupiedMoving,RoomLabel::AdjacentOnly]; + let conditions:Vec<_>=labels.into_iter().map(|label| { + let count=|role| capture.history().iter().filter(|p| p.status==PhaseStatus::Sealed && p.role==role && p.label==label) + .map(|p|p.cohort_count).sum::().min(128); + RecordedCondition{label,train_recorded:count(DataRole::Train),held_out_recorded:count(DataRole::HeldOut)} + }).collect(); + let train_complete=conditions.iter().all(|c|c.train_recorded>=4); + let heldout_complete=conditions.iter().all(|c|c.held_out_recorded>=4); + result.conditions=Some(conditions); + let progress=capture.progress(now_ns); + result.sources=Some(ProgressSources{required:context.sources.len(),ready:progress.sources.iter().filter(|s| s.window_eligible && externally_ready_sources.contains(&s.source_id)).count()}); + result.phase=progress.phase.map(|p| ProgressPhase { + role:p.role,label:p.label, + stage:if p.lead_in_remaining_ms>0 {"lead_in"} else if p.capture_elapsed_ms Result<(),String> { + let stored=self.store.load_selection()?; + if stored.as_ref().map_or(0,|e|e.receipt.revision)!=snapshot.revision {return Err("selection changed".into());} + match (stored.as_ref().and_then(|e|e.receipt.selected.as_ref()),snapshot.selected.as_ref()) { + (Some(target),Some(identity))=>{ + if stored.as_ref().map(|e|e.receipt_sha256.as_str())!=Some(identity.selection_sha256.as_str()) + || target.model_sha256!=identity.model_sha256 || target.qualification_sha256!=identity.qualification_sha256 {return Err("selection changed".into());} + let (model,q)=self.store.resolve_selection_target(target)?;self.store.current_qualification_assessment(&model,&q)?; + }, + (None,None)=>{},_=>return Err("selection changed".into()), + } + Ok(()) + })(); + result.qualification.state=if candidate_sha.is_some() {"not_qualified"} else {"not_reported"}; + result.qualification.model_sha256=candidate_sha; + result.selection.state="not_selected"; + if snapshot.qualification_error.is_some() { result.qualification.state="failed"; } + else if let (Some(q),Some(model))=(state.qualification.as_ref(),state.model.as_ref()) { + let loaded=self.store.load_qualification(model,&q.receipt_sha256).and_then(|loaded| { + if loaded.receipt.room_digest!=published.binding_digest {return Err("qualification context changed".into());} + self.store.current_qualification_assessment(model,&loaded)?;Ok(loaded) + }); + match loaded { + Ok(loaded)=>result.qualification=ProgressQualification{state:"qualified",receipt_sha256:Some(loaded.receipt_sha256),model_sha256:Some(loaded.receipt.model_sha256)}, + Err(_)=>result.qualification.state="failed", + } + } + project_selection(&mut result,&snapshot,verified.is_ok(),&context.digest()); + let source_unready=result.sources.as_ref().is_none_or(|s|s.ready!=s.required); + if source_unready {result.selection.current_usable=false;} + if let Some(phase)=&result.phase { + result.stage=if phase.role==DataRole::Train {"train"} else {"held_out"}; + result.blocker=if phase.stage=="incomplete" {Some("phase_incomplete")} else if source_unready {Some("sources_unready")} else {None}; + } else if busy(&state.job) {result.stage="training";result.blocker=None;} + else if state.job.phase==JobPhase::Failed {result.stage="blocked";result.blocker=Some("training_failed");} + else if result.qualification.state=="failed" {result.stage="blocked";result.blocker=Some("qualification_failed");} + else if result.selection.state=="unavailable" {result.stage="blocked";result.blocker=Some("selection_unavailable");} + else if result.qualification.state=="qualified" && result.qualification.model_sha256!=result.selection.model_sha256 {result.stage="selection";result.blocker=Some("selection_required");} + else if result.selection.state=="selected" {result.stage="selected";result.blocker=if result.selection.current_usable {None} else if source_unready {Some("sources_unready")} else {Some("selection_unavailable")};} + else if result.qualification.state=="qualified" {result.stage="selection";result.blocker=Some("selection_required");} + // Zero temporary rows do not invalidate a durable managed proof. This + // base projection does not report that proof's readiness or counts. + else if state.model.is_some() && !train_complete {result.stage=if heldout_complete {"qualification"} else {"held_out"};result.blocker=Some("training_lineage_not_reported");} + else if state.model.is_some() {result.stage=if heldout_complete {"qualification"} else {"held_out"};result.blocker=if source_unready {Some("sources_unready")} else if heldout_complete {Some("qualification_required")} else {None};} + else {result.stage="train";result.blocker=if source_unready {Some("sources_unready")} else {None};} + Ok(result) + } +} +fn project_selection(result:&mut PairedCalibrationProgress,snapshot:&ModelSelectionStatus,verified:bool,context_digest:&str) { + if let Some(selected)=&snapshot.selected { + if Some(selected.room_digest.as_str())!=result.room.as_ref().map(|r|r.binding_digest.as_str()) || selected.context_digest!=context_digest { + result.selection.state="not_reported";return; + } + } + if !verified || snapshot.integrity_error.is_some() {result.selection.state="unavailable";return;} + if let Some(selected)=&snapshot.selected { + result.selection=ProgressSelection {state:"selected",selection_sha256:Some(selected.selection_sha256.clone()),model_sha256:Some(selected.model_sha256.clone()), + current_usable:matches!(snapshot.current,SelectedBinaryEvidence::Ready{..})}; + } +} + +#[cfg(test)] +#[path = "paired_progress_tests.rs"] +mod tests; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/paired_progress_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/paired_progress_tests.rs new file mode 100644 index 0000000000..28aabd8ebc --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/paired_progress_tests.rs @@ -0,0 +1,131 @@ +//! SYNTHETIC current/read-only progress, not physical qualification. +use super::*; +use super::super::tests::{context,runtime,phase,train_phases,finished,START}; +use crate::room_learner::recovery_store::RecoveryStore; +fn binding()->String {"a".repeat(64)} +fn publication()->Option{Some(PublishedProgressRoom{binding_digest:binding(),revision:1})} +fn configured(root:&Path)->Arc{ + let rt=runtime(root);let mut ctx=context();ctx.room_digest=binding(); + rt.configure(&rt.mutation_guard().unwrap(),ctx,"paired-synthetic-run".into(),true).unwrap();rt +} +fn snapshot(rt:&RoomLearningRuntime,now:u64)->PairedCalibrationProgress { + rt.paired_progress_snapshot("synthetic-install","cal-boot-synthetic",publication(),&[1,2],rt.context().as_ref().map(|c|c.digest()).as_deref(),now).unwrap() +} +fn assert_private_free(value:&PairedCalibrationProgress) { + let s=serde_json::to_string(value).unwrap(); + for forbidden in ["owner_scope","mutation_guard","trial_id","session_id","frame_ids","sequence_ids","last_accepted_sequence","source_id","encoder_weights","head_weights","occupied_probability","heart_rate","amplitudes","\"samples\"","training_data_sha256","\"geometry\""] { + assert!(!s.contains(forbidden),"paired progress leaked {forbidden}"); + } + assert!(!s.contains("NaN")&&!s.contains("Infinity")); +} +#[test] +fn setup_and_foreign_publication_withhold_current_counts_and_private_identity() { + let dir=tempfile::tempdir().unwrap();let rt=configured(dir.path()); + let missing=rt.paired_progress_snapshot("synthetic-install","cal-boot-synthetic",None,&[1,2],None,START).unwrap(); + assert_eq!(missing.blocker,Some("room_not_published"));assert!(missing.conditions.is_none()); + let mut other=publication().unwrap();other.binding_digest="b".repeat(64); + let mismatch=rt.paired_progress_snapshot("synthetic-install","cal-boot-synthetic",Some(other),&[1,2],None,START).unwrap(); + assert_eq!(mismatch.blocker,Some("context_mismatch"));assert!(!mismatch.context_match); + assert!(mismatch.conditions.is_none()&&mismatch.phase.is_none()&&mismatch.sources.is_none()); + assert_eq!(mismatch.qualification.state,"not_reported");assert_eq!(mismatch.selection.state,"not_reported"); + let foreign=rt.paired_progress_snapshot("other-install","cal-boot-synthetic",publication(),&[1,2],None,START).unwrap(); + assert_eq!(foreign.blocker,Some("context_mismatch"));assert_private_free(&foreign); +} +#[test] +fn phases_are_bounded_and_recorded_counts_are_separate_from_qualification_and_save() { + let dir=tempfile::tempdir().unwrap();let rt=configured(dir.path()); + phase(&rt,DataRole::Train,RoomLabel::Empty,0); + let before=snapshot(&rt,START+12_000_000_000); + assert_eq!(before.conditions.as_ref().unwrap()[0].train_recorded,4); + assert_eq!(before.qualification.state,"not_reported");assert_eq!(before.save_status,"not_reported"); + rt.begin_phase(&rt.mutation_guard().unwrap(),PhaseRequest {trial_id:"private-trial-name".into(),session_id:"private-session-name".into(),role:DataRole::Train,label:RoomLabel::OccupiedStatic,duration_ms:12_000,lead_in_ms:5000,conditions_confirmed:true},START+13_000_000_000).unwrap(); + let guard=rt.mutation_guard().unwrap();let preparing=snapshot(&rt,START+14_000_000_000); + assert_eq!(preparing.phase.as_ref().unwrap().stage,"lead_in");assert_eq!(preparing.phase.as_ref().unwrap().lead_in_remaining_ms,4000); + assert_eq!(preparing.observed_at_unix_ms+10_000,preparing.expires_at_unix_ms); + let incomplete=snapshot(&rt,START+100_000_000_000); + let p=incomplete.phase.as_ref().unwrap();assert_eq!(p.stage,"incomplete");assert_eq!(p.capture_elapsed_ms,p.target_ms);assert_eq!(p.representative_cohorts,0); + assert_eq!(guard,rt.mutation_guard().unwrap(),"GET projection must not consume workflow revisions"); + let s=serde_json::to_string(&incomplete).unwrap();assert!(!s.contains("private-trial-name")&&!s.contains("private-session-name"));assert_private_free(&incomplete); +} +#[test] +fn progress_does_not_checkpoint_or_advance_expired_recovery() { + let dir=tempfile::tempdir().unwrap();let owner="a".repeat(64);let data=dir.path().join(&owner).join("SensingServer"); + let store=RecoveryStore::new(&data,&owner,"synthetic-install",Some(&[7;32])).unwrap(); + let rt=RoomLearningRuntime::new_with_recovery_at(&data,&owner,"synthetic-install",store,START).unwrap(); + let mut ctx=context();ctx.room_digest=binding();rt.configure_for_recovery(&rt.mutation_guard().unwrap(),ctx,"private-recovery-run".into(),true,true,START).unwrap(); + let file=data.join("room-learner-recovery/capture-recovery-v1.bin");let bytes=std::fs::read(&file).unwrap(); + let guard=rt.mutation_guard().unwrap(); + let view=snapshot(&rt,START+86_400_000_000_000); + assert_eq!(view.save_status,"not_reported");assert_eq!(rt.mutation_guard().unwrap(),guard); + assert!(rt.inner.lock().unwrap().recovery.enabled); + assert_eq!(std::fs::read(file).unwrap(),bytes); +} +#[test] +fn invalid_bounds_times_and_nonfinite_candidate_are_unavailable_without_raw_values() { + let dir=tempfile::tempdir().unwrap();let rt=configured(dir.path()); + assert!(rt.paired_progress_snapshot("synthetic-install","public\ninvalid",publication(),&[],None,START).is_err()); + assert!(rt.paired_progress_snapshot("synthetic-install","cal-boot-synthetic",publication(),&[],None,0).is_err()); + assert!(rt.paired_progress_snapshot("synthetic-install","cal-boot-synthetic",Some(PublishedProgressRoom{binding_digest:binding(),revision:MAX_SAFE+1}),&[],None,START).is_err()); + assert!(rt.paired_progress_snapshot("synthetic-install","cal-boot-synthetic",Some(PublishedProgressRoom{binding_digest:binding(),revision:0}),&[],None,START).is_err()); + assert!(rt.paired_progress_snapshot("synthetic-install","cal-boot-synthetic",publication(),&[1,2],Some(&"b".repeat(64)),START).is_err(),"ready-source class from a prior context must not supply current readiness"); + assert_private_free(&snapshot(&rt,START)); +} +async fn fitted(root:&Path)->Arc{ + let rt=configured(root);train_phases(&rt); + rt.start_training(&rt.mutation_guard().unwrap(),FitConfig{epochs:8,learning_rate:0.5,hidden_dim:2,embedding_dim:2,temporal_tokens:2,..FitConfig::default()}).unwrap(); + assert_eq!(finished(&rt).await["state"],"ready"); + for (i,l) in [RoomLabel::Empty,RoomLabel::OccupiedStatic,RoomLabel::OccupiedMoving,RoomLabel::AdjacentOnly].into_iter().enumerate(){phase(&rt,DataRole::HeldOut,l,i as u32+4);} + rt +} +#[tokio::test] +async fn verified_selection_is_not_mac_save_and_corrupt_artifacts_withhold_claims() { + let dir=tempfile::tempdir().unwrap();let rt=fitted(dir.path()).await; + let recorded=snapshot(&rt,START+104_000_000_000);assert!(recorded.conditions.as_ref().unwrap().iter().all(|c|c.train_recorded==4&&c.held_out_recorded==4));assert_eq!(recorded.qualification.state,"not_qualified"); + let q=rt.qualify(&rt.mutation_guard().unwrap()).unwrap();let qualified=snapshot(&rt,START+104_000_000_000); + assert_eq!(qualified.qualification.state,"qualified");assert_eq!(qualified.stage,"selection"); + rt.select_model(&rt.mutation_guard().unwrap(),&q.receipt.model_sha256,&q.receipt_sha256).unwrap(); + let now=super::super::selection_tests::feed(&rt,START+105_000_000_000,&[1,2],6.0); + let selected=snapshot(&rt,now);assert_eq!(selected.selection.state,"selected");assert!(selected.selection.current_usable);assert_eq!(selected.save_status,"not_reported");assert_private_free(&selected); + let context=rt.context().unwrap().digest();let unready=rt.paired_progress_snapshot("synthetic-install","cal-boot-synthetic",publication(),&[],Some(&context),now).unwrap(); + assert_eq!(unready.selection.state,"selected");assert!(!unready.selection.current_usable);assert_eq!(unready.blocker,Some("sources_unready")); + let path=dir.path().join("a".repeat(64)).join("SensingServer/room-learner").join(rt.context().unwrap().digest()).join("qualifications").join(format!("{}.json",q.receipt_sha256)); + let original=std::fs::read(&path).unwrap(); + std::fs::write(&path,b"{\"synthetic_corruption\":true}").unwrap(); + let corrupt=snapshot(&rt,now);assert_ne!(corrupt.qualification.state,"qualified");assert_eq!(corrupt.selection.state,"unavailable");assert!(!corrupt.selection.current_usable);assert_private_free(&corrupt); + #[cfg(unix)] { + let outside=dir.path().join("synthetic-other-file.json");std::fs::write(&outside,original).unwrap();std::fs::remove_file(&path).unwrap();std::os::unix::fs::symlink(outside,&path).unwrap(); + let alias=snapshot(&rt,now);assert_ne!(alias.qualification.state,"qualified");assert_eq!(alias.selection.state,"unavailable");assert!(!alias.selection.current_usable); + } +} +#[tokio::test] +async fn nonfinite_model_and_foreign_owner_witness_do_not_grant_current_progress() { + let dir=tempfile::tempdir().unwrap();let rt=fitted(dir.path()).await; + {let mut state=rt.inner.lock().unwrap();let model=Arc::make_mut(state.model.as_mut().unwrap());model.encoder_weights[0]=f32::NAN;} + assert!(rt.paired_progress_snapshot("synthetic-install","cal-boot-synthetic",publication(),&[1,2],rt.context().as_ref().map(|c|c.digest()).as_deref(),START+104_000_000_000).is_err()); + let dir2=tempfile::tempdir().unwrap();let good=fitted(dir2.path()).await;let q=good.qualify(&good.mutation_guard().unwrap()).unwrap();good.select_model(&good.mutation_guard().unwrap(),&q.receipt.model_sha256,&q.receipt_sha256).unwrap(); + std::fs::write(dir2.path().join("a".repeat(64)).join("SensingServer/account-owner-v1.json"),br#"{"schema":"ruview.account-owner.v1","scope_key":"foreign"}"#).unwrap(); + let denied=snapshot(&good,START+104_000_000_000);assert_eq!(denied.selection.state,"unavailable");assert!(!denied.selection.current_usable); +} + +#[tokio::test] +async fn restored_model_with_no_temporary_rows_is_not_reported_as_lost_or_invalid() { + let dir=tempfile::tempdir().unwrap();let rt=fitted(dir.path()).await;let ctx=rt.context().unwrap(); + rt.configure(&rt.mutation_guard().unwrap(),ctx,"new-progress-session".into(),false).unwrap(); + let progress=snapshot(&rt,START+104_000_000_000); + assert_eq!(progress.state,"ready");assert_eq!(progress.stage,"held_out"); + assert_eq!(progress.blocker,Some("training_lineage_not_reported")); + assert!(progress.conditions.as_ref().unwrap().iter().all(|c|c.train_recorded==0&&c.held_out_recorded==0)); + assert_ne!(progress.stage,"blocked");assert_eq!(progress.save_status,"not_reported"); +} + +#[tokio::test] +async fn another_rooms_selected_model_does_not_block_or_label_current_room_setup() { + let dir=tempfile::tempdir().unwrap();let rt=fitted(dir.path()).await; + let q=rt.qualify(&rt.mutation_guard().unwrap()).unwrap();rt.select_model(&rt.mutation_guard().unwrap(),&q.receipt.model_sha256,&q.receipt_sha256).unwrap(); + let mut next=rt.context().unwrap();next.room_digest="b".repeat(64);rt.configure(&rt.mutation_guard().unwrap(),next,"current-room-b".into(),true).unwrap(); + let context=rt.context().unwrap().digest(); + let view=rt.paired_progress_snapshot("synthetic-install","cal-boot-synthetic",Some(PublishedProgressRoom{binding_digest:"b".repeat(64),revision:2}),&[],Some(&context),START+104_000_000_000).unwrap(); + assert!(view.context_match);assert_eq!(view.stage,"train");assert_eq!(view.selection.state,"not_reported");assert!(view.selection.model_sha256.is_none()); + assert_eq!(view.qualification.state,"not_reported");assert!(view.qualification.model_sha256.is_none()); + assert!(view.conditions.as_ref().unwrap().iter().all(|c|c.train_recorded==0&&c.held_out_recorded==0));assert_private_free(&view); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/projection.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/projection.rs new file mode 100644 index 0000000000..01452f115c --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/projection.rs @@ -0,0 +1,110 @@ +//! Existing AETHER projection heads routed by frozen source identity. +//! Opposite RF responses can swap across room states. Source-routed coordinates +//! preserve receiver identity during alignment while the transformer is shared. +use super::{EncoderConfig, RoomModel}; +use crate::{ + embedding::{EmbeddingConfig, ProjectionHead}, + graph_transformer::Linear, +}; +pub(crate) fn config(c: &EncoderConfig) -> EmbeddingConfig { + EmbeddingConfig { + d_model: c.hidden_dim, + d_proj: c.embedding_dim, + temperature: c.temperature, + normalize: false, + } +} +pub(crate) fn initial_weights(c: &EncoderConfig) -> Vec> { + (0..c.source_count) + .map(|source| { + let mut head = ProjectionHead::new(config(c)); + // Source-specific deterministic Xavier seeds break routing symmetry; + // no label, amplitude polarity or fixture value enters initialization. + head.proj_1 = + Linear::with_seed(c.hidden_dim, c.embedding_dim, 20_240 + source as u64 * 2); + head.proj_1.set_bias(vec![0.1; c.embedding_dim]); + head.proj_2 = + Linear::with_seed(c.embedding_dim, c.embedding_dim, 20_241 + source as u64 * 2); + let mut weights = Vec::new(); + head.flatten_into(&mut weights); + weights + }) + .collect() +} +impl RoomModel { + pub(crate) fn source_projectors(&self, params: &[f32]) -> Result, String> { + if params.len() != self.parameter_count() { + return Err("invalid conditional parameter count".into()); + } + let mut offset = self.encoder_weights.len(); + let mut heads = Vec::new(); + for weights in &self.source_projection_weights { + let (head, n) = ProjectionHead::unflatten_from( + ¶ms[offset..offset + weights.len()], + &config(&self.config), + ); + offset += n; + heads.push(head); + } + Ok(heads) + } +} + +/// Center each initial first-layer activation on Train Empty only. This keeps +/// every small ReLU head trainable without assuming any source's signal polarity. +pub(crate) fn center_empty_heads( + model: &mut RoomModel, + rows: &[crate::room_learner::observation::CaptureCohort], +) -> Result<(), String> { + use crate::room_learner::observation::RoomLabel; + if rows + .iter() + .any(|r| r.role != crate::room_learner::observation::DataRole::Train) + { + return Err("projection centering only accepts Train".into()); + } + let mut encoder = super::extractor(&model.config); + encoder + .transformer + .unflatten_weights(&model.encoder_weights)?; + let mut baseline = vec![vec![0.0; model.config.hidden_dim]; model.config.source_count]; + let mut count = 0usize; + for row in rows.iter().filter(|r| r.label == RoomLabel::Empty) { + count += 1; + for (source, mut tokens) in model.tokens(row, false)?.into_iter().enumerate() { + for frame in &mut tokens { + for x in &mut frame[..model.config.bins] { + *x /= model.profiles[source].max_normalized; + } + } + let features = encoder.transformer.embed(&tokens); + for vector in &features { + for (mean, x) in baseline[source].iter_mut().zip(vector) { + *mean += x / features.len() as f32; + } + } + } + } + if count == 0 { + return Err("projection centering requires Train Empty".into()); + } + for (source, mean) in baseline.iter_mut().enumerate() { + mean.iter_mut().for_each(|x| *x /= count as f32); + let (mut head, _) = ProjectionHead::unflatten_from( + &model.source_projection_weights[source], + &config(&model.config), + ); + // Initial bias is 0.1; retain a 0.1 positive activation at the baseline. + let bias = head + .proj_1 + .forward(mean) + .iter() + .map(|value| 0.2 - value) + .collect(); + head.proj_1.set_bias(bias); + let mut weights = Vec::new(); + head.flatten_into(&mut weights); + model.source_projection_weights[source] = weights; + } + Ok(()) +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/qualification.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/qualification.rs new file mode 100644 index 0000000000..393c7b4787 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/qualification.rs @@ -0,0 +1,449 @@ +//! Narrow operator-labelled local qualification; never statistical independence. +use super::{ + capture::{PhaseReceipt, PhaseStatus}, + model::RoomModel, + observation::{CaptureProvenance, DataRole, RoomLabel, ValidatedDataset}, + training::EvaluationReport, +}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +const PERCENT_SCALE: u128 = 100; +const MIN_SENSITIVITY_PERCENT: u128 = 90; +const MAX_FALSE_POSITIVE_PERCENT: u128 = 5; +const MIN_COVERAGE_PERCENT: u128 = 90; +pub const POLICY_ID: &str = "ruview.local-operational-policy.v1"; +pub(crate) fn hash(value: &T) -> Result { + Ok(format!( + "{:x}", + Sha256::digest(serde_json::to_vec(value).map_err(|e| e.to_string())?) + )) +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct QualificationPolicy { + pub id: String, + pub minimum_cohorts_per_condition: usize, + pub minimum_unconditional_sensitivity: f64, + pub maximum_false_positive_rate: f64, + pub minimum_evaluated_cohort_coverage: f64, + pub coverage_scope: String, +} +impl Default for QualificationPolicy { + fn default() -> Self { + Self { + id: POLICY_ID.into(), + minimum_cohorts_per_condition: 4, + minimum_unconditional_sensitivity: MIN_SENSITIVITY_PERCENT as f64 + / PERCENT_SCALE as f64, + maximum_false_positive_rate: MAX_FALSE_POSITIVE_PERCENT as f64 / PERCENT_SCALE as f64, + minimum_evaluated_cohort_coverage: MIN_COVERAGE_PERCENT as f64 / PERCENT_SCALE as f64, + coverage_scope: "representative_cohorts_not_wall_time".into(), + } + } +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct ConditionCriterion { + pub condition: RoomLabel, + pub train_cohorts: usize, + pub held_out_cohorts: usize, + pub evaluated_cohort_coverage: f64, + pub unconditional_sensitivity: Option, + pub false_positive_rate: Option, +} +/// v1 keeps historical numeric bytes; v2 uses exact decimal strings for UTC ns +/// beyond JavaScript's safe-integer range. Neither form asserts source time. +#[derive(Debug,Clone,Serialize,Deserialize,PartialEq)] +#[serde(untagged)] +pub enum ReceiptTimestamp { Legacy(u64), Decimal(String) } +impl ReceiptTimestamp { + fn new(value:u64,proof:bool)->Self {if proof {Self::Decimal(value.to_string())} else {Self::Legacy(value)}} + fn value(&self)->Result {match self { + Self::Legacy(n)=>Ok(*n), + Self::Decimal(s) if !s.is_empty()&&s.len()<=20&&s.bytes().all(|b|b.is_ascii_digit())&&(!s.starts_with('0')||s=="0")=>s.parse().map_err(|_|"invalid qualification timestamp".into()), + _=>Err("invalid qualification timestamp".into()), + }} +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct QualificationReceipt { + pub schema: String, + pub owner_scope: String, + pub installation_id: String, + pub room_digest: String, + pub context_digest: String, + pub model_sha256: String, + pub model_version: u32, + pub training_data_sha256: String, + pub held_out_dataset_sha256: String, + pub evaluation_receipt_sha256: String, + pub evaluated_at_unix_ms: u64, + pub policy: QualificationPolicy, + pub policy_sha256: String, + pub label_provenance: String, + pub sealed_history_sha256: String, + pub first_observed_at_ns: ReceiptTimestamp, + pub last_observed_at_ns: ReceiptTimestamp, + pub criteria: Vec, + pub report: EvaluationReport, + pub independence_verified: bool, + pub physically_validated: bool, + pub authority_scope: String, + #[serde(default, skip_serializing_if="Option::is_none")] + pub training_lineage_basis: Option, + #[serde(default, skip_serializing_if="Option::is_none")] + pub managed_training_proof_sha256: Option, + #[serde(default, skip_serializing_if="Option::is_none")] + pub managed_sealed_phase_fence_sha256: Option, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct QualificationEnvelope { + pub receipt_sha256: String, + pub receipt: QualificationReceipt, +} + +pub(crate) const CONDITIONS: [RoomLabel; 4] = [ + RoomLabel::Empty, + RoomLabel::OccupiedStatic, + RoomLabel::OccupiedMoving, + RoomLabel::AdjacentOnly, +]; + +pub(crate) fn criteria( + report: &EvaluationReport, + train_counts: &[usize; 4], +) -> Result, String> { + report.validate()?; + let policy = QualificationPolicy::default(); + CONDITIONS + .iter() + .enumerate() + .map(|(i, label)| { + let c = report + .per_condition + .iter() + .find(|c| c.condition == *label) + .ok_or("missing required held-out condition")?; + if train_counts[i] < policy.minimum_cohorts_per_condition + || c.samples < policy.minimum_cohorts_per_condition + { + return Err(format!( + "qualification requires four Train and HeldOut cohorts for {label:?}" + )); + } + // The diagnostic report remains f32. Operational public rates are + // computed from validated integer counts so JSON Value/JS widening + // cannot turn exact 90% or 5% boundaries into rejection. + c.rates + .evaluated_cohort_coverage + .ok_or("held-out coverage unavailable")?; + let coverage = c.scored as f64 / c.samples as f64; + if (c.scored as u128) * PERCENT_SCALE < (c.samples as u128) * MIN_COVERAGE_PERCENT { + return Err(format!("held-out coverage failed for {label:?}")); + } + let occupied = matches!(label, RoomLabel::OccupiedStatic | RoomLabel::OccupiedMoving); + let (sensitivity, false_positive) = if occupied { + c.rates + .unconditional_sensitivity + .ok_or("occupied sensitivity unavailable")?; + let s = c.confusion.true_occupied as f64 / c.samples as f64; + if (c.confusion.true_occupied as u128) * PERCENT_SCALE + < (c.samples as u128) * MIN_SENSITIVITY_PERCENT + { + return Err(format!("occupied sensitivity failed for {label:?}")); + } + (Some(s), None) + } else { + let f = c.confusion.false_occupied as f64 / c.samples as f64; + if (c.confusion.false_occupied as u128) * PERCENT_SCALE + > (c.samples as u128) * MAX_FALSE_POSITIVE_PERCENT + { + return Err(format!("false presence failed for {label:?}")); + } + (None, Some(f)) + }; + Ok(ConditionCriterion { + condition: *label, + train_cohorts: train_counts[i], + held_out_cohorts: c.samples, + evaluated_cohort_coverage: coverage, + unconditional_sensitivity: sensitivity, + false_positive_rate: false_positive, + }) + }) + .collect() +} + +#[derive(Debug, Clone)] +pub(crate) struct ManagedEvidence { + pub counts: [usize; 4], + pub sealed_history_sha256: String, + pub first_observed_at_ns: u64, + pub last_observed_at_ns: u64, + pub last_held_out_observed_ns: u64, + pub training_proof_sha256: Option, + pub sealed_phase_fence_sha256: Option, +} +pub(crate) fn managed_evidence( + model: &RoomModel, + dataset: &ValidatedDataset, + history: &[PhaseReceipt], + evaluation: &super::artifact::EvaluationReceipt, +) -> Result { + managed_evidence_with_proof(model,dataset,history,evaluation,None) +} +pub(crate) fn managed_evidence_with_proof( + model:&RoomModel, dataset:&ValidatedDataset, history:&[PhaseReceipt], + evaluation:&super::artifact::EvaluationReceipt, + proof:Option<&super::managed_training::ManagedTrainingProof>, +)->Result { + if dataset.context()!=model.context() {return Err("qualification context mismatch".into());} + let persisted_proof=if dataset.train().is_empty() { + let p=proof.ok_or("qualification requires retained exact Train rows or a verified managed training proof")?; + p.check_held_out(model,dataset)?; Some(p) + } else { + // Never hide changed retained rows behind a formerly valid proof. + if hash(&dataset.train())? != model.training_manifest().training_data_sha256 { + return Err("qualification requires retained exact Train manifest lineage".into()); + } + None + }; + let rows: Vec<_> = dataset.train().iter().chain(dataset.held_out()).collect(); + if rows.iter().any(|c| { + c.provenance != CaptureProvenance::LiveReceiverObserved + || c.views + .iter() + .any(|v| v.provenance != CaptureProvenance::LiveReceiverObserved) + }) { + return Err( + "qualification requires live receiver-observed Train and HeldOut sources".into(), + ); + } + if model.training_manifest().training_provenance != [CaptureProvenance::LiveReceiverObserved] { + return Err("candidate training provenance is not live receiver-observed".into()); + } + let mut relevant = Vec::new(); + for row in &rows { + let matching: Vec<_> = history + .iter() + .filter(|p| { + p.trial_id == row.trial_id + && p.session_id == row.session_id + && p.role == row.role + && p.label == row.label + && p.status == PhaseStatus::Sealed + }) + .collect(); + if matching.len() != 1 { + return Err("required managed sealed phase history is missing or ambiguous".into()); + } + let phase = matching[0]; + let count = rows + .iter() + .filter(|r| { + r.trial_id == row.trial_id + && r.session_id == row.session_id + && r.role == row.role + && r.label == row.label + }) + .count(); + if phase.cohort_count != count { + return Err("sealed phase history does not match exact cohorts".into()); + } + if !relevant.iter().any(|p: &&PhaseReceipt| { + p.trial_id == phase.trial_id && p.session_id == phase.session_id + }) { + relevant.push(phase); + } + } + let counts = persisted_proof.map_or_else( + || CONDITIONS.map(|label|dataset.train().iter().filter(|c|c.label==label).count()), + |p|p.seed.train_counts); + evaluation.report.validate()?; + for (i, label) in CONDITIONS.iter().enumerate() { + if counts[i] < 4 + || dataset + .held_out() + .iter() + .filter(|c| c.label == *label) + .count() + < 4 + { + return Err(format!( + "qualification requires four Train and HeldOut cohorts for {label:?}" + )); + } + } + let c = model.context(); + if evaluation.role != DataRole::HeldOut + || evaluation.context_digest != c.digest() + || evaluation.training_data_sha256 != model.training_manifest().training_data_sha256 + || evaluation.dataset_sha256 != hash(&dataset.held_out())? + { + return Err("qualification evaluation lineage mismatch".into()); + } + let first = rows + .iter() + .flat_map(|c| &c.views) + .map(|v| v.first_time_ns) + .min() + .ok_or("no source time")?; + let last = rows + .iter() + .flat_map(|c| &c.views) + .map(|v| v.last_time_ns) + .max() + .ok_or("no source time")?; + + Ok(ManagedEvidence { + counts, + sealed_history_sha256: if let Some(p)=persisted_proof {hash(&("managed-training-proof-v1+sealed-heldout-v1",&p.proof_sha256,&p.seed.sealed_train_history_sha256,&relevant))?} else {hash(&relevant)?}, + first_observed_at_ns: persisted_proof.map_or(first,|p|p.seed.first_observed_ns.min(first)), + last_observed_at_ns: last, + training_proof_sha256: persisted_proof.map(|p|p.proof_sha256.clone()), + sealed_phase_fence_sha256: None, + last_held_out_observed_ns: dataset + .held_out() + .iter() + .flat_map(|c| &c.views) + .map(|v| v.last_time_ns) + .max() + .ok_or("no held-out source time")?, + }) +} + +pub(crate) fn managed_evidence_with_fence( + model:&RoomModel,dataset:&ValidatedDataset,history:&[PhaseReceipt],evaluation:&super::artifact::EvaluationReceipt, + proof:Option<&super::managed_training::ManagedTrainingProof>,fence_sha:Option<&str>, +)->Result { + let mut evidence=managed_evidence_with_proof(model,dataset,history,evaluation,proof)?; + if evidence.training_proof_sha256.is_some() { + let sha=fence_sha.filter(|s|s.len()==64&&s.bytes().all(|b|b.is_ascii_digit()||(b'a'..=b'f').contains(&b))).ok_or("managed qualification requires its durable sealed phase fence")?; + evidence.sealed_history_sha256=hash(&("managed-phase-fence-v1",&evidence.sealed_history_sha256,sha))?; + evidence.sealed_phase_fence_sha256=Some(sha.into()); + } + Ok(evidence) +} +impl QualificationEnvelope { + pub(crate) fn issue( + model: &RoomModel, + dataset: &ValidatedDataset, + history: &[PhaseReceipt], + evaluation: &super::artifact::EvaluationReceipt, + ) -> Result { + Self::issue_with_proof(model,dataset,history,evaluation,None,None) + } + pub(crate) fn issue_with_proof( + model:&RoomModel,dataset:&ValidatedDataset,history:&[PhaseReceipt], + evaluation:&super::artifact::EvaluationReceipt, + proof:Option<&super::managed_training::ManagedTrainingProof>, + fence_sha:Option<&str>, + )->Result { + let evidence = managed_evidence_with_fence(model,dataset,history,evaluation,proof,fence_sha)?; + let checks = criteria(&evaluation.report, &evidence.counts)?; + let c = model.context(); + let policy = QualificationPolicy::default(); + let receipt = QualificationReceipt { + schema: if evidence.training_proof_sha256.is_some() {"ruview.local-operational-qualification.v2"} else {"ruview.local-operational-qualification.v1"}.into(), + owner_scope: c.owner_scope.clone(), + installation_id: c.installation_id.clone(), + room_digest: c.room_digest.clone(), + context_digest: c.digest(), + model_sha256: evaluation.model_sha256.clone(), + model_version: evaluation.model_version, + training_data_sha256: evaluation.training_data_sha256.clone(), + held_out_dataset_sha256: evaluation.dataset_sha256.clone(), + evaluation_receipt_sha256: evaluation.sha256()?, + evaluated_at_unix_ms: evaluation.evaluated_at_unix_ms, + policy_sha256: hash(&policy)?, + policy, + label_provenance: if evidence.training_proof_sha256.is_some() {"managed-training-proof-v1+sealed-heldout-v1"} else {"managed-sealed-history-v1"}.into(), + sealed_history_sha256: evidence.sealed_history_sha256, + first_observed_at_ns: ReceiptTimestamp::new(evidence.first_observed_at_ns,evidence.training_proof_sha256.is_some()), + last_observed_at_ns: ReceiptTimestamp::new(evidence.last_observed_at_ns,evidence.training_proof_sha256.is_some()), + criteria: checks, + report: evaluation.report.clone(), + independence_verified: false, + physically_validated: false, + authority_scope: "local_operator_binary_presence".into(), + training_lineage_basis: evidence.training_proof_sha256.as_ref().map(|_|"managed_training_proof_v1".into()), + managed_training_proof_sha256: evidence.training_proof_sha256, + managed_sealed_phase_fence_sha256: evidence.sealed_phase_fence_sha256, + }; + let envelope = Self { + receipt_sha256: hash(&receipt)?, + receipt, + }; + envelope.validate(model, evaluation)?; + Ok(envelope) + } + pub(crate) fn validate( + &self, + model: &RoomModel, + eval: &super::artifact::EvaluationReceipt, + ) -> Result<(), String> { + let r = &self.receipt; + let c = model.context(); + let hex = |x: &str| { + x.len() == 64 + && x.bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) + }; + let v1=r.schema=="ruview.local-operational-qualification.v1" && r.training_lineage_basis.is_none() &&r.managed_training_proof_sha256.is_none()&&r.managed_sealed_phase_fence_sha256.is_none() &&r.label_provenance=="managed-sealed-history-v1" &&matches!(r.first_observed_at_ns,ReceiptTimestamp::Legacy(_))&&matches!(r.last_observed_at_ns,ReceiptTimestamp::Legacy(_)); + let v2=r.schema=="ruview.local-operational-qualification.v2" &&r.training_lineage_basis.as_deref()==Some("managed_training_proof_v1") &&r.managed_training_proof_sha256.as_deref().is_some_and(hex)&&r.managed_sealed_phase_fence_sha256.as_deref().is_some_and(hex) &&r.label_provenance=="managed-training-proof-v1+sealed-heldout-v1" && !eval.train_lineage_checked &&matches!(r.first_observed_at_ns,ReceiptTimestamp::Decimal(_))&&matches!(r.last_observed_at_ns,ReceiptTimestamp::Decimal(_)); + if !(v1||v2) + || self.receipt_sha256 != hash(r)? + || r.owner_scope != c.owner_scope + || r.installation_id != c.installation_id + || r.room_digest != c.room_digest + || r.context_digest != c.digest() + || r.model_sha256 != eval.model_sha256 + || r.model_version != eval.model_version + || r.training_data_sha256 != model.training_manifest().training_data_sha256 + || r.held_out_dataset_sha256 != eval.dataset_sha256 + || r.evaluation_receipt_sha256 != eval.sha256()? + || r.evaluated_at_unix_ms != eval.evaluated_at_unix_ms + || r.report != eval.report + || eval.role != DataRole::HeldOut + || r.policy != QualificationPolicy::default() + || r.policy_sha256 != hash(&r.policy)? + || !hex(&r.sealed_history_sha256) + || r.first_observed_at_ns.value()? == 0 + || r.last_observed_at_ns.value()? < r.first_observed_at_ns.value()? + || r.independence_verified + || r.physically_validated + || r.authority_scope != "local_operator_binary_presence" + || model.training_manifest().training_provenance + != [CaptureProvenance::LiveReceiverObserved] + || eval.provenance != [CaptureProvenance::LiveReceiverObserved] + || r.criteria.len() != 4 + { + return Err("local qualification identity, provenance or policy mismatch".into()); + } + let counts = std::array::from_fn(|i| r.criteria[i].train_cohorts); + if counts.iter().any(|n| *n > super::observation::MAX_COHORTS) + || counts.iter().sum::() != model.training_manifest().training_cohorts + || criteria(&r.report, &counts)? != r.criteria + { + return Err("local qualification criterion mismatch".into()); + } + Ok(()) + } +} + +#[cfg(test)] +#[path = "qualification_boundary_tests.rs"] +mod boundary_tests; + +/// Descriptive native-window failure flags use the SAME integer thresholds and +/// denominators as criteria. Legacy policy/behavior/serialization is unchanged. +pub(crate) fn native_condition_failure_codes(c:&super::training::ConditionEvaluation)->Vec{ + let mut codes=Vec::new(); + if (c.scored as u128)*PERCENT_SCALE<(c.samples as u128)*MIN_COVERAGE_PERCENT{codes.push("coverage_below_minimum".into());} + if matches!(c.condition,RoomLabel::OccupiedStatic|RoomLabel::OccupiedMoving){ + if(c.confusion.true_occupied as u128)*PERCENT_SCALE<(c.samples as u128)*MIN_SENSITIVITY_PERCENT{codes.push("sensitivity_below_minimum".into());} + }else if(c.confusion.false_occupied as u128)*PERCENT_SCALE>(c.samples as u128)*MAX_FALSE_POSITIVE_PERCENT{codes.push("false_positive_rate_above_maximum".into());} + codes +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/qualification_boundary_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/qualification_boundary_tests.rs new file mode 100644 index 0000000000..ab71b8fe14 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/qualification_boundary_tests.rs @@ -0,0 +1,167 @@ +//! Exact count boundaries and Rust Value/JSON wire contracts; synthetic diagnostics only. +use super::*; +use crate::room_learner::training::{BinaryConfusion, ConditionEvaluation, EvaluationRates}; +fn rates(c: &BinaryConfusion) -> EvaluationRates { + let scored = c.true_empty + c.false_occupied + c.false_empty + c.true_occupied; + let n = scored + c.abstained_empty + c.abstained_occupied; + let ratio = |a: usize, b: usize| { + if b == 0 { + None + } else { + Some(a as f32 / b as f32) + } + }; + EvaluationRates { + evaluated_cohort_coverage: ratio(scored, n), + unconditional_accuracy: ratio(c.true_empty + c.true_occupied, n), + scored_accuracy: ratio(c.true_empty + c.true_occupied, scored), + unconditional_sensitivity: ratio( + c.true_occupied, + c.true_occupied + c.false_empty + c.abstained_occupied, + ), + scored_sensitivity: ratio(c.true_occupied, c.true_occupied + c.false_empty), + unconditional_specificity: ratio( + c.true_empty, + c.true_empty + c.false_occupied + c.abstained_empty, + ), + scored_specificity: ratio(c.true_empty, c.true_empty + c.false_occupied), + } +} +fn condition(label: RoomLabel, c: BinaryConfusion) -> ConditionEvaluation { + let scored = c.true_empty + c.false_occupied + c.false_empty + c.true_occupied; + let abstained = c.abstained_empty + c.abstained_occupied; + ConditionEvaluation { + condition: label, + samples: scored + abstained, + correct: c.true_empty + c.true_occupied, + abstained, + scored, + missing: c.missing_empty + c.missing_occupied, + binary_cross_entropy: Some(0.0), + binary_cross_entropy_samples: scored + abstained, + declared_trial_count: 1, + declared_session_count: 1, + rates: rates(&c), + confusion: c, + } +} +fn report( + false_positive: usize, + occupied_correct: usize, + occupied_abstained: usize, +) -> EvaluationReport { + let conditions = vec![ + condition( + RoomLabel::Empty, + BinaryConfusion { + true_empty: 40 - false_positive, + false_occupied: false_positive, + ..Default::default() + }, + ), + condition( + RoomLabel::OccupiedStatic, + BinaryConfusion { + true_occupied: occupied_correct, + false_empty: 10 - occupied_correct - occupied_abstained, + abstained_occupied: occupied_abstained, + ..Default::default() + }, + ), + condition( + RoomLabel::OccupiedMoving, + BinaryConfusion { + true_occupied: 4, + ..Default::default() + }, + ), + condition( + RoomLabel::AdjacentOnly, + BinaryConfusion { + true_empty: 4, + ..Default::default() + }, + ), + ]; + let mut sum = BinaryConfusion::default(); + for c in &conditions { + sum.true_empty += c.confusion.true_empty; + sum.false_occupied += c.confusion.false_occupied; + sum.true_occupied += c.confusion.true_occupied; + sum.false_empty += c.confusion.false_empty; + sum.abstained_occupied += c.confusion.abstained_occupied; + } + let report = EvaluationReport { + samples: conditions.iter().map(|c| c.samples).sum(), + correct: conditions.iter().map(|c| c.correct).sum(), + abstained: conditions.iter().map(|c| c.abstained).sum(), + scored: conditions.iter().map(|c| c.scored).sum(), + missing: 0, + binary_cross_entropy: Some(0.0), + binary_cross_entropy_samples: conditions + .iter() + .map(|c| c.binary_cross_entropy_samples) + .sum(), + declared_trial_count: 4, + declared_session_count: 4, + rates: rates(&sum), + confusion: sum, + per_condition: conditions, + independence_verified: false, + }; + report.validate().unwrap(); + report +} +#[test] +fn exact_ninety_percent_coverage_and_sensitivity_and_five_percent_false_positives_pass() { + let checks = criteria(&report(2, 9, 1), &[4; 4]).unwrap(); + assert_eq!(checks[0].false_positive_rate, Some(0.05)); + assert_eq!(checks[1].evaluated_cohort_coverage, 0.9); + assert_eq!(checks[1].unconditional_sensitivity, Some(0.9)); + let wire = serde_json::to_value(&checks).unwrap(); + assert_eq!(wire[0]["false_positive_rate"], serde_json::json!(0.05)); + assert_eq!(wire[1]["evaluated_cohort_coverage"], serde_json::json!(0.9)); + assert_eq!(wire[1]["unconditional_sensitivity"], serde_json::json!(0.9)); + let json = serde_json::to_string(&wire).unwrap(); + assert!(!json.contains("0.899999") && !json.contains("0.050000000")); +} +#[test] +fn below_threshold_coverage_sensitivity_and_excess_false_positives_fail_separately() { + assert!(criteria(&report(2, 8, 2), &[4; 4]) + .unwrap_err() + .contains("coverage")); + assert!(criteria(&report(2, 8, 0), &[4; 4]) + .unwrap_err() + .contains("sensitivity")); + assert!(criteria(&report(3, 9, 1), &[4; 4]) + .unwrap_err() + .contains("false presence")); +} +#[test] +fn policy_is_canonical_through_json_value_and_direct_hash_encoding() { + let p = QualificationPolicy::default(); + let value = serde_json::to_value(&p).unwrap(); + assert_eq!( + value["minimum_unconditional_sensitivity"], + serde_json::json!(0.9) + ); + assert_eq!( + value["minimum_evaluated_cohort_coverage"], + serde_json::json!(0.9) + ); + assert_eq!( + value["maximum_false_positive_rate"], + serde_json::json!(0.05) + ); + let encoded = serde_json::to_string(&p).unwrap(); + let expected = r#"{"id":"ruview.local-operational-policy.v1","minimum_cohorts_per_condition":4,"minimum_unconditional_sensitivity":0.9,"maximum_false_positive_rate":0.05,"minimum_evaluated_cohort_coverage":0.9,"coverage_scope":"representative_cohorts_not_wall_time"}"#; + assert_eq!(encoded, expected); + assert_eq!( + hash(&p).unwrap(), + format!("{:x}", Sha256::digest(expected.as_bytes())) + ); + assert_eq!( + serde_json::from_value::(value).unwrap(), + p + ); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/receiver_failure_annotation_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/receiver_failure_annotation_tests.rs new file mode 100644 index 0000000000..9731eec3ca --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/receiver_failure_annotation_tests.rs @@ -0,0 +1,36 @@ +//! SYNTHETIC private-core metadata update; no actual capture or durability claim. +use super::*; +use super::super::tests::bridge_setup; +const PRIMARY: &str = "temporal_observed_gap_or_rate: source=1 direction=below_min receiver_interval_ns=1000000 min_ns=27777777 max_ns=50000000"; +#[test] +fn failure_annotation_only_replaces_exact_failed_metadata_and_preserves_all_snapshot_facts() { + let (_dir, _store, mut c) = bridge_setup(); + let initial = c.snapshot(); + assert!(!c.annotate_receiver_failure(PRIMARY, format!("{PRIMARY} deq_us=30303"))); + assert_eq!(c.snapshot(), initial, "nonfailed capture must be untouched"); + c.invalidate(PRIMARY); + let before = c.snapshot(); + let floor = c.workflow.source_high.clone(); + let receiver_clock = c.workflow.last_now_ns; + let durable = c.workflow.durable.clone(); + let detail = format!("{PRIMARY} deq_us=30303 parse_us=10 lock_us=50000 stamp_age_us=50015"); + assert!(c.annotate_receiver_failure(PRIMARY, detail.clone())); + let mut after = c.snapshot(); + assert_eq!(after["failure"], detail); + after["failure"] = before["failure"].clone(); + assert_eq!(after, before, "no stage, time, pending ACK, retained rows, or counters may change"); + assert_eq!(c.workflow.source_high, floor); + assert_eq!(c.workflow.last_now_ns, receiver_clock); + assert_eq!(c.workflow.durable, durable); + let annotated = c.snapshot(); + assert!(!c.annotate_receiver_failure(PRIMARY, format!("{PRIMARY} stale_expected=1"))); + assert_eq!(c.snapshot(), annotated); +} +#[test] +fn failure_annotation_refuses_oversized_or_unrelated_metadata_without_cutting_primary() { + let (_dir, _store, mut c) = bridge_setup(); c.invalidate(PRIMARY); + let before = c.snapshot(); + assert!(!c.annotate_receiver_failure(PRIMARY, format!("{PRIMARY}{}", "x".repeat(256)))); + assert!(!c.annotate_receiver_failure(PRIMARY, "different failure".into())); + assert_eq!(c.snapshot(), before); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/receiver_pipeline_timing_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/receiver_pipeline_timing_tests.rs new file mode 100644 index 0000000000..cff186c38e --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/receiver_pipeline_timing_tests.rs @@ -0,0 +1,55 @@ +//! SYNTHETIC relative receiver timing; no captured RF or physical clock claim. +use super::*; +use crate::room_learner::{native::{canonicalize_esp32, NativePacketInput, CanonicalPacket}, observation::CaptureProvenance}; +use wifi_densepose_hardware::{Adr018Flags, PpduType}; + +fn packet(source: u8, sequence: u32, stamp: u64, ppdu: PpduType, bins: usize) -> CanonicalPacket { + canonicalize_esp32(NativePacketInput { node_id: source, n_antennas: 1, raw_bins: bins, + frequency_hz: 2_437_000_000, ppdu_type: ppdu, sequence, received_at_ns: stamp, + receiver_sample_rate_hz: 33.0, iq_bytes: &vec![4u8; bins * 2], + flags: Adr018Flags::from_byte(0), provenance: CaptureProvenance::Synthetic }).unwrap() +} +const PRIMARY: &str = "temporal_observed_gap_or_rate: source=15 direction=below_min receiver_interval_ns=1000000 min_ns=27777777 max_ns=50000000"; + +#[test] +fn pipeline_failure_reports_relative_durations_without_changing_packet_identity_or_values() { + let p = packet(15, 12, 2_000_000_000, PpduType::HtLegacy, 64); + let before = serde_json::to_vec(&p.summary()).unwrap(); + let iq = p.native_frame.native_iq.clone(); + let magnitudes = p.amplitudes.clone(); + let id = p.native_frame.frame_id; + // Distinct synthetic dequeue spacing and processing interval are NOT clocks. + let timing = ReceiverPipelineTiming::for_packet(&p, Some(30_303), Some(42), Some(60_000), Some(60_050)); + let detail = timing.failure_detail(&p, PRIMARY).unwrap(); + assert_eq!(detail, format!("{PRIMARY} deq_us=30303 parse_us=42 lock_us=60000 stamp_age_us=60050")); + assert_eq!(serde_json::to_vec(&p.summary()).unwrap(), before); + assert_eq!(p.native_frame.native_iq, iq); + assert_eq!(p.amplitudes, magnitudes); + assert_eq!(p.native_frame.frame_id, id); + assert!(!detail.contains("owner") && !detail.contains("token") && !detail.contains("timestamp")); +} + +#[test] +fn receiver_timing_never_attaches_another_packet_or_phy_class() { + let p = packet(15, 12, 2_000_000_000, PpduType::HtLegacy, 64); + let timing = ReceiverPipelineTiming::for_packet(&p, Some(30_303), None, Some(12), None); + for other in [packet(16, 12, 2_000_000_000, PpduType::HtLegacy, 64), + packet(15, 13, 2_000_000_000, PpduType::HtLegacy, 64), + packet(15, 12, 2_000_000_001, PpduType::HtLegacy, 64), + packet(15, 12, 2_000_000_000, PpduType::HeSu, 256)] { + assert!(timing.failure_detail(&other, PRIMARY).is_none()); + } + assert!(timing.failure_detail(&p, "temporal_missing_segment_source").is_none()); +} + +#[test] +fn failure_primary_is_preserved_and_unknown_durations_are_not_fabricated() { + let p = packet(15, 12, 2_000_000_000, PpduType::HtLegacy, 64); + let timing = ReceiverPipelineTiming::for_packet(&p, None, Some(u32::MAX), None, Some(0)); + let detail = timing.failure_detail(&p, PRIMARY).unwrap(); + assert!(detail.starts_with(PRIMARY)); + assert!(detail.ends_with("deq_us=na parse_us=4294967295 lock_us=na stamp_age_us=0")); + assert!(detail.len() <= 256); + let long_primary = format!("{PRIMARY}{}", "x".repeat(256 - PRIMARY.len())); + assert!(timing.failure_detail(&p, &long_primary).is_none(), "never truncate the primary to make metadata fit"); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/recovery_store.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/recovery_store.rs new file mode 100644 index 0000000000..7661bcfbb9 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/recovery_store.rs @@ -0,0 +1,481 @@ +//! Explicitly consented encrypted reduced-sample recovery; no plaintext fallback. +use ring::{ + aead, hkdf, + rand::{SecureRandom, SystemRandom}, +}; +use serde::{Deserialize, Serialize}; +use std::{ + fs::{self, File, OpenOptions}, + io::{Read, Write}, + path::{Path, PathBuf}, + sync::Mutex, +}; + +pub const RECOVERY_SCHEMA: &str = "ruview.room-learner.capture-recovery.v1"; +pub const MAX_RECOVERY_BYTES: usize = 64 * 1024 * 1024; +pub const RECOVERY_TTL_MS: u64 = 24 * 60 * 60 * 1000; +const MAX_DESCRIPTOR: usize = 4096; +const MAGIC: &[u8; 8] = b"RVCREC01"; +const FILE: &str = "capture-recovery-v1.bin"; +const MAX_FILE: usize = MAX_RECOVERY_BYTES + MAX_DESCRIPTOR + 40; +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct RecoveryDescriptor { + pub schema: String, + pub id: String, + pub owner_scope: String, + pub installation_id: String, + pub context_digest: String, + pub created_at_ms: u64, + pub saved_at_ms: u64, + pub expires_at_ms: u64, +} +pub struct LoadedRecovery { + pub descriptor: RecoveryDescriptor, + pub plaintext: Vec, +} +pub struct RecoveryStore { + directory: PathBuf, + owner_directory: PathBuf, + owner: String, + installation: String, + key: Option>, + operator_key: Option>, + serial: Mutex<()>, +} +fn id(value: &str) -> bool { + !value.is_empty() + && value.len() <= 256 + && value.is_ascii() + && value.trim() == value + && !value.bytes().any(|b| b.is_ascii_control()) +} +fn hex(value: &str) -> bool { + value.len() == 64 + && value + .bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) +} +#[cfg(unix)] +fn private_metadata(meta: &fs::Metadata, directory: bool) -> Result<(), String> { + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + extern "C" { + fn geteuid() -> u32; + } + // SAFETY: geteuid has no arguments or memory side effects. + let uid = unsafe { geteuid() }; + if meta.uid() != uid + || meta.permissions().mode() & 0o7777 != if directory { 0o700 } else { 0o600 } + { + return Err("recovery_private_permissions".into()); + } + Ok(()) +} +#[cfg(not(unix))] +fn private_metadata(_: &fs::Metadata, _: bool) -> Result<(), String> { + Err("recovery_private_permissions_unsupported".into()) +} +fn private_directory(path: &Path) -> Result<(), String> { + match fs::symlink_metadata(path) { + Ok(m) if m.file_type().is_symlink() || !m.is_dir() => { + return Err("recovery_unsafe_directory".into()) + } + Ok(m) => return private_metadata(&m, true), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + Err(_) => return Err("recovery_directory_unavailable".into()), + } + #[cfg(unix)] + { + use std::os::unix::fs::DirBuilderExt; + let mut b = fs::DirBuilder::new(); + b.mode(0o700); + match b.create(path) { + Ok(()) => {} + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {} + Err(_) => return Err("recovery_directory_unavailable".into()), + } + } + let m = fs::symlink_metadata(path).map_err(|_| "recovery_directory_unavailable")?; + if m.file_type().is_symlink() || !m.is_dir() { + return Err("recovery_unsafe_directory".into()); + } + private_metadata(&m, true) +} +fn managed_seed(stable: Option<&str>, value: Option<&str>) -> Option<[u8; 32]> { + if stable != Some("1") { + return None; + } + let value = value?; + if value.len() != 64 || !value.bytes().all(|b| b.is_ascii_hexdigit()) { + return None; + } + let mut bytes = [0u8; 32]; + for (i, slot) in bytes.iter_mut().enumerate() { + *slot = u8::from_str_radix(&value[i * 2..i * 2 + 2], 16).ok()?; + } + Some(bytes) +} +impl RecoveryStore { + /// Same account-owned recovery root, isolated ciphertext/key/commitment + /// namespace. Creating its empty private directory grants no capture consent. + pub(crate) fn operator_eight_scope(&self) -> Result { + self.verify_directory()?; + if self.directory!=self.owner_directory.join("room-learner-recovery"){return Err("recovery_profile_scope_invalid".into());} + let directory = self.directory.join("operator-eight-step-v1"); + private_directory(&directory)?; + Ok(Self { directory, owner_directory:self.owner_directory.clone(), owner:self.owner.clone(), installation:self.installation.clone(), key:self.operator_key.clone(), operator_key:None, serial:Mutex::new(()) }) + } + pub fn new( + data_dir: &Path, + owner: &str, + installation: &str, + seed: Option<&[u8; 32]>, + ) -> Result { + if !hex(owner) || !id(installation) { + return Err("recovery_invalid_identity".into()); + } + // Check existing permissions before the legacy owner preparer can repair them. + for path in [ + data_dir.to_path_buf(), + data_dir.join("account-owner-v1.json"), + ] { + if let Ok(m) = fs::symlink_metadata(&path) { + if m.file_type().is_symlink() { + return Err("recovery_unsafe_owner_path".into()); + } + private_metadata(&m, path == data_dir)?; + } + } + if data_dir + .parent() + .is_some_and(|p| fs::symlink_metadata(p).is_ok_and(|m| m.file_type().is_symlink())) + { + return Err("recovery_unsafe_owner_path".into()); + } + let owned = crate::account_profile::prepare_directory(owner, data_dir) + .map_err(|_| "recovery_owner_mismatch")?; + let directory = owned.join("room-learner-recovery"); + private_directory(&directory)?; + let key = seed + .map(|seed| { + let mut info = Vec::new(); + for item in [ + b"ruview.capture-recovery.aes256gcm.v1".as_slice(), + owner.as_bytes(), + installation.as_bytes(), + ] { + info.extend_from_slice(&(item.len() as u32).to_be_bytes()); + info.extend_from_slice(item); + } + let salt = hkdf::Salt::new(hkdf::HKDF_SHA256, b"ruview.capture-recovery.hkdf.v1"); + let prk = salt.extract(seed); + let infos = [info.as_slice()]; + let okm = prk + .expand(&infos, &aead::AES_256_GCM) + .map_err(|_| "recovery_key_unavailable")?; + let mut material = [0u8; 32]; + okm.fill(&mut material) + .map_err(|_| "recovery_key_unavailable")?; + let result = aead::UnboundKey::new(&aead::AES_256_GCM, &material) + .map(aead::LessSafeKey::new) + .map_err(|_| "recovery_key_unavailable"); + material.fill(0); + result + }) + .transpose()?.map(std::sync::Arc::new); + // Purpose-separated key for the explicitly versioned operator plan; + // old ciphertext and its derivation remain unchanged/readable. + let operator_key = seed.map(|seed| { + let mut info = Vec::new(); + for item in [b"ruview.capture-recovery.operator-eight-step.aes256gcm.v1".as_slice(), owner.as_bytes(), installation.as_bytes()] { + info.extend_from_slice(&(item.len() as u32).to_be_bytes()); info.extend_from_slice(item); + } + let salt = hkdf::Salt::new(hkdf::HKDF_SHA256, b"ruview.capture-recovery.hkdf.v1"); + let prk = salt.extract(seed); let infos = [info.as_slice()]; + let okm = prk.expand(&infos, &aead::AES_256_GCM).map_err(|_| "recovery_key_unavailable")?; + let mut material = [0u8; 32]; okm.fill(&mut material).map_err(|_| "recovery_key_unavailable")?; + let result = aead::UnboundKey::new(&aead::AES_256_GCM, &material).map(aead::LessSafeKey::new).map(std::sync::Arc::new).map_err(|_| "recovery_key_unavailable"); + material.fill(0); result + }).transpose()?; + Ok(Self { + directory, + owner_directory:owned, + owner: owner.into(), + installation: installation.into(), + key, + operator_key, + serial: Mutex::new(()), + }) + } + pub fn from_managed_env( + data_dir: &Path, + owner: &str, + installation: &str, + ) -> Result { + let stable = std::env::var("RUVIEW_CAPTURE_RECOVERY_KEY_STABLE").ok(); + let value = if stable.as_deref() == Some("1") { + std::env::var("WDP_RUFIELD_SIGNING_SEED").ok() + } else { + None + }; + let mut seed = managed_seed(stable.as_deref(), value.as_deref()); + let result = Self::new(data_dir, owner, installation, seed.as_ref()); + if let Some(ref mut bytes) = seed { + bytes.fill(0); + } + result + } + pub fn available(&self) -> bool { + self.key.is_some() + } + fn verify_directory(&self) -> Result<(), String> { + let owned = self.owner_directory.as_path(); + for (path, is_directory) in [ + (owned.to_path_buf(), true), + (owned.join("account-owner-v1.json"), false), + ] { + let m = fs::symlink_metadata(path).map_err(|_| "recovery_owner_mismatch")?; + if m.file_type().is_symlink() + || (is_directory && !m.is_dir()) + || (!is_directory && !m.is_file()) + { + return Err("recovery_unsafe_owner_path".into()); + } + private_metadata(&m, is_directory)?; + } + if owned + .parent() + .is_some_and(|p| fs::symlink_metadata(p).is_ok_and(|m| m.file_type().is_symlink())) + { + return Err("recovery_unsafe_owner_path".into()); + } + let verified = crate::account_profile::prepare_directory(&self.owner, owned) + .map_err(|_| "recovery_owner_mismatch")?; + if verified != owned { + return Err("recovery_owner_mismatch".into()); + } + private_directory(&owned.join("room-learner-recovery"))?; + if self.directory!=owned.join("room-learner-recovery")&&self.directory!=owned.join("room-learner-recovery/operator-eight-step-v1"){return Err("recovery_profile_scope_invalid".into());} + private_directory(&self.directory) + } + /// Non-repairing owner validation for read-scoped status discovery. + /// Unlike prepare_directory this never creates directories or chmods them. + fn verify_directory_read_only(&self)->Result<(),String>{ + let owned=self.owner_directory.as_path(); + if !hex(&self.owner)||owned.file_name().and_then(|v|v.to_str())!=Some("SensingServer")||owned.parent().and_then(Path::file_name).and_then(|v|v.to_str())!=Some(self.owner.as_str())||owned.canonicalize().map_err(|_|"recovery_owner_mismatch")?!=owned{return Err("recovery_owner_mismatch".into());} + for(path,directory)in[(owned.to_path_buf(),true),(owned.join("account-owner-v1.json"),false),(owned.join("room-learner-recovery"),true),(self.directory.clone(),true)]{ + let m=fs::symlink_metadata(&path).map_err(|_|"recovery_owner_mismatch")?; + if m.file_type().is_symlink()||(directory&&!m.is_dir())||(!directory&&!m.is_file()){return Err("recovery_unsafe_owner_path".into());}private_metadata(&m,directory)?; + } + let witness=crate::companion::private_read(&owned.join("account-owner-v1.json")).map_err(|_|"recovery_owner_mismatch")?; + let value:serde_json::Value=serde_json::from_slice(&witness).map_err(|_|"recovery_owner_mismatch")?; + if value.get("schema").and_then(|v|v.as_str())!=Some("ruview.account-owner.v1")||value.get("scope_key").and_then(|v|v.as_str())!=Some(self.owner.as_str()){return Err("recovery_owner_mismatch".into());}Ok(()) + } + fn descriptor(&self, d: &RecoveryDescriptor, now: Option) -> Result<(), String> { + if d.schema != RECOVERY_SCHEMA + || !id(&d.id) + || d.owner_scope != self.owner + || d.installation_id != self.installation + || !hex(&d.context_digest) + || d.created_at_ms == 0 + || d.saved_at_ms < d.created_at_ms + || d.expires_at_ms <= d.created_at_ms + || d.expires_at_ms - d.created_at_ms > RECOVERY_TTL_MS + || d.saved_at_ms >= d.expires_at_ms + { + return Err("recovery_descriptor_invalid".into()); + } + if let Some(now) = now { + if now < d.created_at_ms || now < d.saved_at_ms { + return Err("recovery_clock_regression".into()); + } + if now >= d.expires_at_ms { + return Err("recovery_expired".into()); + } + } + Ok(()) + } + fn read(&self, now: Option) -> Result, String> { + let key = self.key.as_ref().ok_or("recovery_unavailable")?; + self.verify_directory()?; + let path = self.directory.join(FILE); + let m = match fs::symlink_metadata(&path) { + Ok(m) => m, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(_) => return Err("recovery_read_failed".into()), + }; + if !m.is_file() || m.file_type().is_symlink() || m.len() > MAX_FILE as u64 { + return Err("recovery_unsafe_file".into()); + } + private_metadata(&m, false)?; + let mut options = OpenOptions::new(); + options.read(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + #[cfg(target_os = "macos")] + options.custom_flags(0x100); // O_NOFOLLOW + #[cfg(target_os = "linux")] + options.custom_flags(0x20000); // O_NOFOLLOW + } + let file = options.open(&path).map_err(|_| "recovery_read_failed")?; + let opened = file.metadata().map_err(|_| "recovery_read_failed")?; + private_metadata(&opened, false)?; + if !opened.is_file() || opened.len() > MAX_FILE as u64 { + return Err("recovery_unsafe_file".into()); + } + let mut bytes = Vec::new(); + file.take((MAX_FILE + 1) as u64) + .read_to_end(&mut bytes) + .map_err(|_| "recovery_read_failed")?; + if bytes.len() > MAX_FILE || bytes.len() < 40 || &bytes[..8] != MAGIC { + return Err("recovery_corrupt".into()); + } + let n = + u32::from_be_bytes(bytes[8..12].try_into().map_err(|_| "recovery_corrupt")?) as usize; + if n == 0 || n > MAX_DESCRIPTOR || bytes.len() < 12 + n + 12 + 16 { + return Err("recovery_corrupt".into()); + } + let aad = bytes[..12 + n].to_vec(); + let d: RecoveryDescriptor = serde_json::from_slice(&bytes[12..12 + n]) + .map_err(|_| "recovery_descriptor_invalid")?; + let nonce = aead::Nonce::try_assume_unique_for_key(&bytes[12 + n..24 + n]) + .map_err(|_| "recovery_corrupt")?; + let mut payload = bytes[24 + n..].to_vec(); + let plain = key + .open_in_place(nonce, aead::Aad::from(&aad), &mut payload) + .map_err(|_| "recovery_authentication_failed")?; + if plain.len() > MAX_RECOVERY_BYTES { + return Err("recovery_budget_exceeded".into()); + } + let plaintext = plain.to_vec(); + self.descriptor(&d, now)?; + Ok(Some(LoadedRecovery { + descriptor: d, + plaintext, + })) + } + /// Authenticate retained metadata for an explicit restore/discard decision. + /// Expiry and current-clock checks belong to load; inspection never renews + /// the original deadline and never exposes decrypted sample bytes. + pub fn inspect(&self) -> Result, String> { + let _lock = self + .serial + .lock() + .map_err(|_| "recovery_store_unavailable")?; + Ok(self.read(None)?.map(|loaded| loaded.descriptor)) + } + pub fn load(&self, now_ms: u64) -> Result, String> { + let _lock = self + .serial + .lock() + .map_err(|_| "recovery_store_unavailable")?; + self.read(Some(now_ms)) + } + pub fn save( + &self, + d: &RecoveryDescriptor, + plaintext: &[u8], + now_ms: u64, + ) -> Result<(), String> { + let _lock = self + .serial + .lock() + .map_err(|_| "recovery_store_unavailable")?; + let key = self.key.as_ref().ok_or("recovery_unavailable")?; + self.descriptor(d, Some(now_ms))?; + if plaintext.len() > MAX_RECOVERY_BYTES { + return Err("recovery_budget_exceeded".into()); + } + if let Some(previous) = self.read(Some(now_ms))? { + let p = previous.descriptor; + if p.id != d.id + || p.context_digest != d.context_digest + || p.created_at_ms != d.created_at_ms + || p.expires_at_ms != d.expires_at_ms + || d.saved_at_ms < p.saved_at_ms + { + return Err("recovery_discard_required".into()); + } + } + let descriptor = serde_json::to_vec(d).map_err(|_| "recovery_descriptor_invalid")?; + if descriptor.len() > MAX_DESCRIPTOR { + return Err("recovery_descriptor_invalid".into()); + } + let mut output = MAGIC.to_vec(); + output.extend_from_slice(&(descriptor.len() as u32).to_be_bytes()); + output.extend_from_slice(&descriptor); + let mut nonce = [0u8; 12]; + SystemRandom::new() + .fill(&mut nonce) + .map_err(|_| "recovery_random_unavailable")?; + let mut ciphertext = plaintext.to_vec(); + key.seal_in_place_append_tag( + aead::Nonce::assume_unique_for_key(nonce), + aead::Aad::from(&output), + &mut ciphertext, + ) + .map_err(|_| "recovery_encrypt_failed")?; + output.extend_from_slice(&nonce); + output.extend_from_slice(&ciphertext); + let mut suffix = [0u8; 16]; + SystemRandom::new() + .fill(&mut suffix) + .map_err(|_| "recovery_random_unavailable")?; + let temp = self + .directory + .join(format!("recovery-stage-{:x}", u128::from_be_bytes(suffix))); + let result = (|| { + let mut options = OpenOptions::new(); + options.create_new(true).write(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + let mut file = options.open(&temp).map_err(|_| "recovery_write_failed")?; + file.write_all(&output) + .and_then(|_| file.sync_all()) + .map_err(|_| "recovery_write_failed")?; + fs::rename(&temp, self.directory.join(FILE)).map_err(|_| "recovery_write_failed")?; + File::open(&self.directory) + .and_then(|f| f.sync_all()) + .map_err(|_| "recovery_write_failed")?; + let restored = self.read(Some(now_ms))?.ok_or("recovery_readback_failed")?; + if restored.descriptor != *d || restored.plaintext != plaintext { + return Err("recovery_readback_failed".into()); + } + Ok(()) + })(); + if temp.exists() { + let _ = fs::remove_file(&temp); + } + result + } + pub fn discard(&self, expected_id: &str) -> Result<(), String> { + let _lock = self + .serial + .lock() + .map_err(|_| "recovery_store_unavailable")?; + if !id(expected_id) { + return Err("recovery_invalid_identity".into()); + } + if let Some(existing) = self.read(None)? { + if existing.descriptor.id != expected_id { + return Err("recovery_identity_mismatch".into()); + } + fs::remove_file(self.directory.join(FILE)).map_err(|_| "recovery_discard_failed")?; + File::open(&self.directory) + .and_then(|f| f.sync_all()) + .map_err(|_| "recovery_discard_failed")?; + } + Ok(()) + } +} +#[cfg(test)] +#[path = "recovery_store_tests.rs"] +mod tests; + +// Typed temporal consistency uses the SAME profile directory/key, not V1 files. +#[path = "temporal_recovery_commitment.rs"] +pub(crate) mod temporal_commitment; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/recovery_store_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/recovery_store_tests.rs new file mode 100644 index 0000000000..f96e72822a --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/recovery_store_tests.rs @@ -0,0 +1,381 @@ +//! Synthetic keys and temporary owned directories only. +use super::*; +const NOW: u64 = 1_800_000_000_000; +fn fixture(seed: Option<&[u8; 32]>) -> (tempfile::TempDir, RecoveryStore) { + let t = tempfile::tempdir().unwrap(); + let s = RecoveryStore::new( + &t.path().join("a".repeat(64)).join("SensingServer"), + &"a".repeat(64), + "install", + seed, + ) + .unwrap(); + (t, s) +} +fn descriptor() -> RecoveryDescriptor { + RecoveryDescriptor { + schema: RECOVERY_SCHEMA.into(), + id: "original-run".into(), + owner_scope: "a".repeat(64), + installation_id: "install".into(), + context_digest: "b".repeat(64), + created_at_ms: NOW, + saved_at_ms: NOW, + expires_at_ms: NOW + RECOVERY_TTL_MS, + } +} +#[test] +fn binary_roundtrip_fresh_nonce_and_original_deadline() { + let (_t, s) = fixture(Some(&[7; 32])); + let mut d = descriptor(); + let payload = b"opaque u128=340282366920938463463374607431768211455\0\xff"; + s.save(&d, payload, NOW).unwrap(); + let before = fs::read(s.directory.join(FILE)).unwrap(); + let r = s.load(NOW + 1).unwrap().unwrap(); + assert_eq!(r.descriptor, d); + assert_eq!(r.plaintext, payload); + assert!(!before.windows(payload.len()).any(|w| w == payload)); + d.saved_at_ms += 1; + s.save(&d, payload, NOW + 1).unwrap(); + let after = fs::read(s.directory.join(FILE)).unwrap(); + assert_ne!(before, after); + assert_eq!( + s.load(NOW + 2).unwrap().unwrap().descriptor.expires_at_ms, + NOW + RECOVERY_TTL_MS + ); +} +#[test] +fn unavailable_never_writes_plaintext() { + let (_t, s) = fixture(None); + assert!(!s.available()); + assert_eq!( + s.save(&descriptor(), b"secret", NOW).unwrap_err(), + "recovery_unavailable" + ); + assert!(s.load(NOW).is_err()); + assert!(!s.directory.join(FILE).exists()); +} +#[test] +fn identity_context_deadline_and_clock_cannot_be_replaced_or_renewed() { + let (_t, s) = fixture(Some(&[7; 32])); + let d = descriptor(); + s.save(&d, b"old", NOW).unwrap(); + let path = s.directory.join(FILE); + let before = fs::read(&path).unwrap(); + for field in [ + "id", "context", "created", "expires", "saved", "owner", "install", "schema", + ] { + let mut bad = d.clone(); + match field { + "id" => bad.id = "new".into(), + "context" => bad.context_digest = "c".repeat(64), + "created" => bad.created_at_ms += 1, + "expires" => bad.expires_at_ms -= 1, + "saved" => bad.saved_at_ms -= 1, + "owner" => bad.owner_scope = "d".repeat(64), + "install" => bad.installation_id = "other".into(), + _ => bad.schema = "future".into(), + }; + assert!(s.save(&bad, b"new", NOW + 5).is_err(), "{field}"); + assert_eq!(fs::read(&path).unwrap(), before); + } + assert!(s.load(NOW - 1).unwrap_err_string().contains("clock")); + assert!(s.discard("wrong-id").is_err()); + assert_eq!(fs::read(&path).unwrap(), before); +} +// Avoid requiring Debug on a value containing plaintext. +trait ErrorText { + fn unwrap_err_string(self) -> String; +} +impl ErrorText for Result { + fn unwrap_err_string(self) -> String { + match self { + Err(e) => e, + Ok(_) => panic!("unexpected success"), + } + } +} +#[test] +fn expiry_retained_and_explicit_authenticated_discard() { + let (_t, s) = fixture(Some(&[7; 32])); + s.save(&descriptor(), b"bytes", NOW).unwrap(); + let path = s.directory.join(FILE); + let before = fs::read(&path).unwrap(); + assert_eq!( + s.load(NOW + RECOVERY_TTL_MS).unwrap_err_string(), + "recovery_expired" + ); + let mut next = descriptor(); + next.saved_at_ms = NOW + 1; + assert_eq!( + s.save(&next, b"new", NOW + RECOVERY_TTL_MS).unwrap_err(), + "recovery_expired" + ); + assert_eq!(fs::read(&path).unwrap(), before); + s.discard("original-run").unwrap(); + assert!(!path.exists()); + s.save(&descriptor(), b"new", NOW).unwrap(); +} +#[test] +fn corruption_descriptor_nonce_tag_and_ciphertext_fail_closed() { + for position in [0, 8, 12, 50, 350, 400] { + let (_t, s) = fixture(Some(&[7; 32])); + s.save(&descriptor(), &[42; 512], NOW).unwrap(); + let path = s.directory.join(FILE); + let mut bytes = fs::read(&path).unwrap(); + let n = bytes.len(); + bytes[position % n] ^= 1; + fs::write(&path, &bytes).unwrap(); + assert!(s.load(NOW).is_err(), "position {position}"); + assert!(s.save(&descriptor(), b"new", NOW).is_err()); + assert!(s.discard("original-run").is_err()); + assert_eq!(fs::read(&path).unwrap(), bytes); + } +} +#[test] +fn wrong_key_and_installation_cannot_read_or_overwrite() { + let (_t, s) = fixture(Some(&[7; 32])); + s.save(&descriptor(), b"opaque", NOW).unwrap(); + let path = s.directory.join(FILE); + let before = fs::read(&path).unwrap(); + let data = s.directory.parent().unwrap(); + for (seed, installation) in [([8; 32], "install"), ([7; 32], "other-install")] { + let other = RecoveryStore::new(data, &"a".repeat(64), installation, Some(&seed)).unwrap(); + assert!(other.load(NOW).is_err()); + let mut d = descriptor(); + d.installation_id = installation.into(); + assert!(other.save(&d, b"bad", NOW).is_err()); + assert_eq!(fs::read(&path).unwrap(), before); + } + assert!(RecoveryStore::new(data, &"c".repeat(64), "install", Some(&[7; 32])).is_err()); +} +#[test] +fn bounded_large_payload_and_oversize_file_fail_closed() { + let (_t, s) = fixture(Some(&[7; 32])); + assert_eq!( + s.save(&descriptor(), &vec![1; MAX_RECOVERY_BYTES + 1], NOW) + .unwrap_err(), + "recovery_budget_exceeded" + ); + assert!(!s.directory.join(FILE).exists()); + s.save(&descriptor(), b"small", NOW).unwrap(); + let f = OpenOptions::new() + .write(true) + .open(s.directory.join(FILE)) + .unwrap(); + f.set_len((MAX_FILE + 1) as u64).unwrap(); + assert!(s.load(NOW).is_err()); + assert!(s.save(&descriptor(), b"new", NOW).is_err()); +} +#[cfg(unix)] +#[test] +fn private_files_symlinks_and_owner_witness_revalidated() { + use std::os::unix::fs::{symlink, PermissionsExt}; + for target in ["file", "directory", "witness"] { + let (t, s) = fixture(Some(&[7; 32])); + s.save(&descriptor(), b"small", NOW).unwrap(); + let path = match target { + "file" => s.directory.join(FILE), + "directory" => s.directory.clone(), + _ => s.directory.parent().unwrap().join("account-owner-v1.json"), + }; + fs::rename(&path, path.with_extension("preserved")).unwrap(); + let victim = t.path().join("victim"); + fs::write(&victim, b"untouched").unwrap(); + fs::set_permissions(&victim, fs::Permissions::from_mode(0o600)).unwrap(); + symlink(&victim, &path).unwrap(); + assert!(s.load(NOW).is_err()); + assert!(s.save(&descriptor(), b"bad", NOW).is_err()); + assert_eq!(fs::read(victim).unwrap(), b"untouched"); + } + let (_t, s) = fixture(Some(&[7; 32])); + s.save(&descriptor(), b"small", NOW).unwrap(); + let p = s.directory.join(FILE); + fs::set_permissions(&p, fs::Permissions::from_mode(0o644)).unwrap(); + assert!(s.load(NOW).is_err()); + assert!(s.save(&descriptor(), b"bad", NOW).is_err()); +} +#[test] +fn invalid_paths_and_descriptor_bounds_do_not_disclose_key_or_payload() { + let t = tempfile::tempdir().unwrap(); + assert!(RecoveryStore::new( + Path::new("relative"), + &"a".repeat(64), + "install", + Some(&[7; 32]) + ) + .is_err()); + let (_t, s) = fixture(Some(&[7; 32])); + let mut d = descriptor(); + d.id = "../escape".into(); // IDs are opaque authenticated metadata, never paths. + s.save(&d, b"sensitive", NOW).unwrap(); + assert!(!t.path().join("escape").exists()); + d.id = "x".repeat(257); + let e = s.save(&d, b"sensitive", NOW).unwrap_err(); + assert!(!e.contains("sensitive")); + assert!(!e.contains("070707")); +} + +#[test] +fn managed_key_policy_has_no_implicit_dev_or_ephemeral_fallback() { + let value = "07".repeat(32); + for stable in [None, Some(""), Some("true"), Some("0"), Some("1 ")] { + assert!(managed_seed(stable, Some(&value)).is_none()); + } + for invalid in [ + None, + Some("short"), + Some("adr262-ruview-rufield-dev-seed!!"), + ] { + assert!(managed_seed(Some("1"), invalid).is_none()); + } + assert_eq!(managed_seed(Some("1"), Some(&value)), Some([7; 32])); + assert!(managed_seed(Some("1"), Some(&"z".repeat(64))).is_none()); +} +#[test] +fn copied_other_owner_ciphertext_does_not_authenticate() { + let (_a, a) = fixture(Some(&[7; 32])); + a.save(&descriptor(), b"private-owner-a", NOW).unwrap(); + let b_root = tempfile::tempdir().unwrap(); + let b = RecoveryStore::new( + &b_root.path().join("b".repeat(64)).join("SensingServer"), + &"b".repeat(64), + "install", + Some(&[7; 32]), + ) + .unwrap(); + fs::copy(a.directory.join(FILE), b.directory.join(FILE)).unwrap(); + assert!(b.load(NOW).is_err()); + let mut d = descriptor(); + d.owner_scope = "b".repeat(64); + assert!(b.save(&d, b"bad", NOW).is_err()); +} +#[cfg(unix)] +#[test] +fn owner_permissions_drift_is_not_silently_repaired() { + use std::os::unix::fs::PermissionsExt; + for witness in [true, false] { + let (_t, s) = fixture(Some(&[7; 32])); + s.save(&descriptor(), b"original", NOW).unwrap(); + let path = if witness { + s.directory.parent().unwrap().join("account-owner-v1.json") + } else { + s.directory.parent().unwrap().to_path_buf() + }; + fs::set_permissions( + &path, + fs::Permissions::from_mode(if witness { 0o644 } else { 0o755 }), + ) + .unwrap(); + assert!(s.load(NOW).is_err()); + assert!(s.save(&descriptor(), b"bad", NOW).is_err()); + assert_eq!( + fs::metadata(path).unwrap().permissions().mode() & 0o777, + if witness { 0o644 } else { 0o755 } + ); + } +} + +#[test] +fn descriptor_nonce_and_tag_are_all_authenticated_and_truncation_is_retained() { + for part in ["descriptor", "nonce", "ciphertext", "tag", "truncated"] { + let (_t, s) = fixture(Some(&[7; 32])); + s.save(&descriptor(), b"bounded opaque payload", NOW) + .unwrap(); + let path = s.directory.join(FILE); + let mut bytes = fs::read(&path).unwrap(); + let descriptor_len = u32::from_be_bytes(bytes[8..12].try_into().unwrap()) as usize; + match part { + "descriptor" => { + let at = bytes + .windows(12) + .position(|w| w == b"original-run") + .unwrap(); + bytes[at] = b'O'; + } + "nonce" => bytes[12 + descriptor_len] ^= 1, + "ciphertext" => bytes[24 + descriptor_len] ^= 1, + "tag" => { + let end = bytes.len() - 1; + bytes[end] ^= 1; + } + _ => { + bytes.truncate(bytes.len() - 1); + } + } + fs::write(&path, &bytes).unwrap(); + assert!(s.load(NOW).is_err(), "{part}"); + assert!(s.save(&descriptor(), b"replacement", NOW).is_err()); + assert_eq!(fs::read(path).unwrap(), bytes); + } +} +#[test] +fn exact_plaintext_budget_roundtrips() { + let (_t, s) = fixture(Some(&[7; 32])); + let bytes = vec![0x65; MAX_RECOVERY_BYTES]; + s.save(&descriptor(), &bytes, NOW).unwrap(); + assert_eq!(s.load(NOW).unwrap().unwrap().plaintext, bytes); +} + +#[test] +fn expired_authenticated_inspection_preserves_deadline_then_explicit_discard() { + let (_t, s) = fixture(Some(&[7; 32])); + let d = descriptor(); + s.save(&d, b"private reduced samples", NOW).unwrap(); + let path = s.directory.join(FILE); + let before = fs::read(&path).unwrap(); + assert_eq!( + s.load(d.expires_at_ms).unwrap_err_string(), + "recovery_expired" + ); + assert_eq!(s.inspect().unwrap(), Some(d.clone())); + assert!(s.load(NOW - 1).is_err()); + assert_eq!(s.inspect().unwrap(), Some(d)); + assert_eq!(fs::read(&path).unwrap(), before); + s.discard("original-run").unwrap(); + assert_eq!(s.inspect().unwrap(), None); +} +#[test] +fn wrong_key_or_tampered_inspection_fails_without_altering_retained_evidence() { + let (_t, s) = fixture(Some(&[7; 32])); + s.save(&descriptor(), b"private reduced samples", NOW) + .unwrap(); + let path = s.directory.join(FILE); + let before = fs::read(&path).unwrap(); + let other = RecoveryStore::new( + s.directory.parent().unwrap(), + &"a".repeat(64), + "install", + Some(&[8; 32]), + ) + .unwrap(); + assert!(other.inspect().is_err()); + assert_eq!(fs::read(&path).unwrap(), before); + let mut bad = before; + let last = bad.len() - 1; + bad[last] ^= 1; + fs::write(&path, &bad).unwrap(); + assert!(s.inspect().is_err()); + assert_eq!(fs::read(&path).unwrap(), bad); +} + +#[test] +fn operator_profile_scope_preserves_old_bytes_and_uses_a_separate_key_without_recording_consent() { + let (_t,old)=fixture(Some(&[7;32])); + old.save(&descriptor(),b"synthetic previous-plan evidence",NOW).unwrap(); + let bytes=fs::read(old.directory.join(FILE)).unwrap(); + let new=old.operator_eight_scope().unwrap(); + assert!(new.available()); + assert!(fs::read_dir(&new.directory).unwrap().next().is_none()); + assert!(new.inspect().unwrap().is_none()); + new.save(&descriptor(),b"synthetic eight-step evidence",NOW).unwrap(); + assert_eq!(new.load(NOW+1).unwrap().unwrap().plaintext,b"synthetic eight-step evidence"); + assert_eq!(fs::read(old.directory.join(FILE)).unwrap(),bytes); + assert_eq!(old.load(NOW+1).unwrap().unwrap().plaintext,b"synthetic previous-plan evidence"); + // The same owner's old ciphertext cannot authenticate in the new key domain. + let valid_new=fs::read(new.directory.join(FILE)).unwrap(); + fs::write(new.directory.join(FILE),&bytes).unwrap(); + assert!(new.load(NOW+1).is_err()); + fs::write(new.directory.join(FILE),valid_new).unwrap(); + assert!(new.operator_eight_scope().is_err()); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/selection.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/selection.rs new file mode 100644 index 0000000000..0837f383b1 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/selection.rs @@ -0,0 +1,115 @@ +//! Explicit durable selection is distinct from candidate discovery and current usability. +use super::{ + model::PresenceDecision, observation::CaptureContext, qualification::QualificationEnvelope, +}; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct SelectionTarget { + pub context: CaptureContext, + pub model_sha256: String, + pub model_version: u32, + pub qualification_sha256: String, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct SelectionReceipt { + pub schema: String, + pub owner_scope: String, + pub installation_id: String, + pub revision: u64, + pub selected_at_unix_ms: u64, + pub selected: Option, + pub previous: Option, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct SelectionEnvelope { + pub receipt_sha256: String, + pub receipt: SelectionReceipt, +} + +/// Bounded identity suitable for read-scoped paired phones; no owner identifier/raw context. +#[derive(Debug, Clone, Serialize, PartialEq)] +pub struct SelectedModelIdentity { + pub selection_sha256: String, + pub selection_revision: u64, + pub model_sha256: String, + pub model_version: u32, + pub qualification_sha256: String, + pub policy_sha256: String, + pub context_digest: String, + pub room_digest: String, + pub source_node_ids: Vec, + pub selected_at_unix_ms: u64, + pub evaluated_at_unix_ms: u64, + pub authority_scope: String, +} +#[derive(Debug, Clone, Serialize)] +#[serde(tag = "status", rename_all = "snake_case")] +pub enum SelectedBinaryEvidence { + None, + Unavailable { + identity: Option, + reason: String, + }, + Ready { + identity: SelectedModelIdentity, + decision: PresenceDecision, + presence: bool, + occupied_probability: f32, + observed_at_unix_ms: u64, + observation_digest: String, + }, +} +#[derive(Debug, Clone, Serialize)] +pub struct QualificationSummary { + #[serde(skip_serializing_if="Option::is_none")] + pub schema: Option, + #[serde(skip_serializing_if="Option::is_none")] + pub training_lineage_basis: Option, + #[serde(skip_serializing_if="Option::is_none")] + pub managed_training_proof_sha256: Option, + pub receipt_sha256: String, + pub model_sha256: String, + pub context_digest: String, + pub room_digest: String, + pub evaluated_at_unix_ms: u64, + pub policy: super::qualification::QualificationPolicy, + pub policy_sha256: String, + pub label_provenance: String, + pub criteria: Vec, + pub authority_scope: String, +} +impl From<&QualificationEnvelope> for QualificationSummary { + fn from(q: &QualificationEnvelope) -> Self { + let r = &q.receipt; + Self { + schema: (r.schema=="ruview.local-operational-qualification.v2").then(||r.schema.clone()), + training_lineage_basis:r.training_lineage_basis.clone(), + managed_training_proof_sha256:r.managed_training_proof_sha256.clone(), + receipt_sha256: q.receipt_sha256.clone(), + model_sha256: r.model_sha256.clone(), + context_digest: r.context_digest.clone(), + room_digest: r.room_digest.clone(), + evaluated_at_unix_ms: r.evaluated_at_unix_ms, + policy: r.policy.clone(), + policy_sha256: r.policy_sha256.clone(), + label_provenance: r.label_provenance.clone(), + criteria: r.criteria.clone(), + authority_scope: r.authority_scope.clone(), + } + } +} +#[derive(Debug, Clone, Serialize)] +pub struct ModelSelectionStatus { + pub schema: &'static str, + pub revision: u64, + pub selected: Option, + pub previous_available: bool, + pub qualification: Option, + pub qualification_error: Option, + pub integrity_error: Option, + pub current: SelectedBinaryEvidence, +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/selection_store.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/selection_store.rs new file mode 100644 index 0000000000..4983db4c38 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/selection_store.rs @@ -0,0 +1,246 @@ +//! Owner-private immutable qualification and guarded durable selected-model pointer. +use super::*; +use crate::room_learner::{ + qualification::{hash, QualificationEnvelope}, + selection::{SelectionEnvelope, SelectionTarget}, +}; +const SELECTION: &str = "model-selection-v1.json"; +impl ArtifactStore { + fn qualification_dir( + &self, + context: &CaptureContext, + create: bool, + ) -> Result, String> { + let Some(dir) = self.context_dir(context, create)? else { + return Ok(None); + }; + let path = dir.join("qualifications"); + match std::fs::symlink_metadata(&path) { + Err(e) if e.kind() == io::ErrorKind::NotFound && !create => return Ok(None), + Err(e) if e.kind() != io::ErrorKind::NotFound => return Err(e.to_string()), + _ => {} + } + private_dir(&path).map_err(|e| e.to_string())?; + Ok(Some(path)) + } + pub(crate) fn save_qualification( + &self, + model: &RoomModel, + q: &QualificationEnvelope, + ) -> Result<(), String> { + self.load_exact_candidate(model.context(), &q.receipt.model_sha256)?; + let eval = self.load_exact_evaluation( + model, + crate::room_learner::observation::DataRole::HeldOut, + &q.receipt.evaluation_receipt_sha256, + )?; + q.validate(model, &eval)?; + if let Some(proof)=self.load_managed_training_proof(model)? {self.load_managed_capture_fence(model,&proof)?;} + + if let Some(expected)=&q.receipt.managed_training_proof_sha256 { + let proof=self.load_managed_training_proof(model)?.ok_or("qualification managed training proof missing")?; + if &proof.proof_sha256!=expected {return Err("qualification managed training proof changed".into());} + self.require_managed_capture_fence(model,&proof,q.receipt.managed_sealed_phase_fence_sha256.as_deref().ok_or("qualification sealed phase fence missing")?)?; + } + let dir = self + .qualification_dir(model.context(), true)? + .ok_or("qualification directory unavailable")?; + evaluation_store::write_immutable( + &dir.join(format!("{}.json", q.receipt_sha256)), + &encode(q)?, + )?; + self.load_qualification(model, &q.receipt_sha256)?; + let pointer = serde_json::json!({"receipt_sha256":q.receipt_sha256,"model_sha256":q.receipt.model_sha256,"context_digest":model.context().digest()}); + atomic_private_write( + &dir.join(format!("{}.latest.json", q.receipt.model_sha256)), + &encode(&pointer)?, + ) + .map_err(|e| e.to_string())?; + Ok(()) + } + pub(crate) fn load_qualification( + &self, + model: &RoomModel, + sha: &str, + ) -> Result { + if !hex_digest(sha) { + return Err("invalid qualification digest".into()); + } + let dir = self + .qualification_dir(model.context(), false)? + .ok_or("qualification unavailable")?; + let q: QualificationEnvelope = serde_json::from_slice( + &private_read(&dir.join(format!("{sha}.json"))).map_err(|e| e.to_string())?, + ) + .map_err(|e| e.to_string())?; + if q.receipt_sha256 != sha { + return Err("qualification hash mismatch".into()); + } + let eval = self.load_exact_evaluation( + model, + crate::room_learner::observation::DataRole::HeldOut, + &q.receipt.evaluation_receipt_sha256, + )?; + q.validate(model, &eval)?; + if let Some(proof)=self.load_managed_training_proof(model)? {self.load_managed_capture_fence(model,&proof)?;} + + if let Some(expected)=&q.receipt.managed_training_proof_sha256 { + let proof=self.load_managed_training_proof(model)?.ok_or("qualification managed training proof missing")?; + if &proof.proof_sha256!=expected {return Err("qualification managed training proof changed".into());} + self.require_managed_capture_fence(model,&proof,q.receipt.managed_sealed_phase_fence_sha256.as_deref().ok_or("qualification sealed phase fence missing")?)?; + } + Ok(q) + } + pub(crate) fn load_latest_qualification( + &self, + model: &RoomModel, + ) -> Result, String> { + let Some(dir) = self.qualification_dir(model.context(), false)? else { + return Ok(None); + }; + let sha = digest(&model.export()?); + let path = dir.join(format!("{sha}.latest.json")); + if !regular_or_missing(&path)? { + return Ok(None); + } + #[derive(Deserialize)] + #[serde(deny_unknown_fields)] + struct Pointer { + receipt_sha256: String, + model_sha256: String, + context_digest: String, + } + let p: Pointer = serde_json::from_slice(&private_read(&path).map_err(|e| e.to_string())?) + .map_err(|e| e.to_string())?; + if p.model_sha256 != sha || p.context_digest != model.context().digest() { + return Err("qualification pointer identity mismatch".into()); + } + let q = self.load_qualification(model, &p.receipt_sha256)?; + let Some(a) = self.load_latest_assessment(model)? else { + return Ok(None); + }; + if a.receipt.failure_reason.is_some() { + return Ok(None); + } + if a.receipt.qualification_sha256.as_deref() != Some(q.receipt_sha256.as_str()) { + return Err("latest qualification pointer differs from completed assessment".into()); + } + Ok(Some(q)) + } + pub(crate) fn resolve_selection_target( + &self, + t: &SelectionTarget, + ) -> Result<(RoomModel, QualificationEnvelope), String> { + self.validate_context(&t.context)?; + let model = self.load_exact_candidate(&t.context, &t.model_sha256)?; + let q = self.load_qualification(&model, &t.qualification_sha256)?; + if model.version != t.model_version || q.receipt.model_sha256 != t.model_sha256 { + return Err("selection model identity mismatch".into()); + } + Ok((model, q)) + } + fn verify_no_selection_history(&self) -> Result<(), String> { + let history = self.root.join("selections"); + let metadata = match std::fs::symlink_metadata(&history) { + Err(e) if e.kind() == io::ErrorKind::NotFound => return Ok(()), + Err(_) => { + return Err( + "selection history cannot be inspected while its pointer is missing".into(), + ) + } + Ok(m) => m, + }; + if !metadata.is_dir() || metadata.file_type().is_symlink() { + return Err("selection history is unsafe while its pointer is missing".into()); + } + #[cfg(unix)] + { + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + unsafe extern "C" { + fn geteuid() -> u32; + } + // SAFETY: POSIX geteuid has no arguments or memory side effects. + let uid = unsafe { geteuid() }; + if metadata.permissions().mode() & 0o777 != 0o700 || metadata.uid() != uid { + return Err( + "selection history is not privately readable while its pointer is missing" + .into(), + ); + } + } + let mut entries = std::fs::read_dir(&history) + .map_err(|_| "selection history cannot be read while its pointer is missing")?; + match entries.next() { + None=>Ok(()), + Some(Ok(_))=>Err("selection pointer is missing but retained selection history exists; explicit recovery is required".into()), + Some(Err(_))=>Err("selection history cannot be read while its pointer is missing".into()), + } + } + pub(crate) fn load_selection(&self) -> Result, String> { + self.verify_owner()?; + let path = self.root.join(SELECTION); + if !regular_or_missing(&path)? { + // Absence is only known when no selection commit was attempted. + // Immutable history may outlive a lost pointer or a failed first + // pointer write. Never guess a replacement from that history. + self.verify_no_selection_history()?; + return Ok(None); + } + let envelope: SelectionEnvelope = + serde_json::from_slice(&private_read(&path).map_err(|e| e.to_string())?) + .map_err(|e| e.to_string())?; + let r = &envelope.receipt; + if r.schema != "ruview.room-learner.model-selection.v1" + || envelope.receipt_sha256 != hash(r)? + || r.owner_scope != self.owner_scope + || r.installation_id != self.installation_id + || r.revision == 0 + || r.selected_at_unix_ms == 0 + || r.selected_at_unix_ms > 253_402_300_799_999 + { + return Err("selected model pointer identity mismatch".into()); + } + for t in [&r.selected, &r.previous].into_iter().flatten() { + self.validate_context(&t.context)?; + if !hex_digest(&t.model_sha256) || !hex_digest(&t.qualification_sha256) { + return Err("invalid selection target digest".into()); + } + } + Ok(Some(envelope)) + } + pub(crate) fn save_selection( + &self, + envelope: &SelectionEnvelope, + expected_revision: u64, + ) -> Result<(), String> { + let current = self.load_selection()?; + if current.as_ref().map_or(0, |e| e.receipt.revision) != expected_revision + || envelope.receipt.revision + != expected_revision + .checked_add(1) + .ok_or("selection revision exhausted")? + || envelope.receipt.owner_scope != self.owner_scope + || envelope.receipt.installation_id != self.installation_id + || envelope.receipt.schema != "ruview.room-learner.model-selection.v1" + || envelope.receipt_sha256 != hash(&envelope.receipt)? + { + return Err("selection pointer changed or invalid".into()); + } + if let Some(t) = &envelope.receipt.selected { + self.resolve_selection_target(t)?; + } + let bytes = encode(envelope)?; + // Save history before atomically replacing the sole authoritative pointer. + let history = self.root.join("selections"); + private_dir(&history).map_err(|e| e.to_string())?; + evaluation_store::write_immutable( + &history.join(format!("{}.json", envelope.receipt_sha256)), + &bytes, + )?; + atomic_private_write(&self.root.join(SELECTION), &bytes).map_err(|e| e.to_string())?; + if self.load_selection()?.as_ref() != Some(envelope) { + return Err("selection durable readback mismatch".into()); + } + Ok(()) + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/source_conditioning_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/source_conditioning_tests.rs new file mode 100644 index 0000000000..2641665a1c --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/source_conditioning_tests.rs @@ -0,0 +1,230 @@ +//! SYNTHETIC exact sign-inversion oracle: pooled views cannot identify the room. +use crate::room_learner::{ + model::*, + observation::*, + training::{learning_tests::dataset, FitConfig}, +}; + +fn inverted_dataset(reverse: bool) -> ValidatedDataset { + let base = dataset(8); + let mut rows = base + .train() + .iter() + .chain(base.validation()) + .chain(base.held_out()) + .cloned() + .collect::>(); + for c in &mut rows { + for v in &mut c.views { + for (t, frame) in v.samples.iter_mut().enumerate() { + for (b, x) in frame.iter_mut().enumerate() { + let empty = 10.0 + t as f32 * 0.02 + b as f32 * 0.001; + let state = match c.label { + RoomLabel::Empty => 0.0, + RoomLabel::AdjacentOnly => -1.0, + _ => 1.0, + }; + let source = if (v.source_id == 1) != reverse { + 1.0 + } else { + -1.0 + }; + *x = empty + source * state * 0.25; + } + } + } + } + ValidatedDataset::new(base.context().clone(), rows).unwrap() +} +fn cosine(a: &[f32], b: &[f32]) -> f32 { + a.iter().zip(b).map(|(x, y)| x * y).sum::() + / (a.iter().map(|x| x * x).sum::().sqrt() + * b.iter().map(|x| x * x).sum::().sqrt()) + .max(1e-10) +} +fn margin( + model: &RoomModel, + occupied: &CaptureCohort, + adjacent: &CaptureCohort, + condition: bool, +) -> f32 { + let mut stripped = model.clone(); + if !condition { + for i in 1..stripped.source_projection_weights.len() { + stripped.source_projection_weights[i] = stripped.source_projection_weights[0].clone(); + } + } + let model = if condition { model } else { &stripped }; + let mut a = model.tokens(occupied, false).unwrap(); + let mut b = model.tokens(adjacent, false).unwrap(); + if !condition { + for tokens in a.iter_mut().chain(b.iter_mut()) { + for row in tokens { + row[model.encoder_config().bins..].fill(0.0); + } + } + } + let (_, a) = model.represent(&a, &model.parameters()).unwrap(); + let (_, b) = model.represent(&b, &model.parameters()).unwrap(); + (cosine(&a[0], &a[1]) + cosine(&b[0], &b[1]) - cosine(&a[0], &b[1]) - cosine(&b[0], &a[1])) + * 0.5 +} +fn check_inverted_case(reverse: bool) { + let ds = inverted_dataset(reverse); + let occupied = &ds.held_out()[2]; + let adjacent = &ds.held_out()[1]; + let pooled = |c: &CaptureCohort| { + c.views + .iter() + .flat_map(|v| v.samples.iter().flatten()) + .sum::() + }; + assert!((pooled(occupied) - pooled(adjacent)).abs() < 0.001); + assert_eq!(occupied.views[0].samples, adjacent.views[1].samples); + assert_eq!(occupied.views[1].samples, adjacent.views[0].samples); + let (model, report) = RoomModel::fit( + &ds, + FitConfig { + epochs: 48, + max_duration_ms: 120_000, + learning_rate: 0.5, + hidden_dim: 4, + embedding_dim: 4, + contrastive_weight: 5.0, + supervised_weight: 0.5, + ..FitConfig::default() + }, + ) + .unwrap(); + let conditional = margin(&model, occupied, adjacent, true); + let unconditioned = margin(&model, occupied, adjacent, false); + eprintln!("SYNTHETIC inverted RF InfoNCE {}->{}; positive-vs-opposite margin conditioned={conditional}, removed-tags={unconditioned}; report={report:?}",report.initial.contrastive,report.final_objective.contrastive); + assert!(report.backbone_delta_l2 > 1e-4); + assert!(report.projection_delta_l2 > 1e-4); + assert!(report.final_objective.contrastive < report.initial.contrastive - 0.02); + assert!( + conditional > 0.1, + "shared encoder must align same-event opposite-sign views" + ); + assert!( + unconditioned <= 1e-5, + "removed conditioning must recover the impossible symmetric-view case" + ); + let evaluation = model.evaluate(ds.held_out()).unwrap(); + assert_eq!(evaluation.correct, 4, "{evaluation:?}"); + assert_eq!(evaluation.abstained, 0); + assert!(!model.infer(occupied).physically_validated); +} +#[test] +fn opposite_source_responses_learn_real_alignment_and_disjoint_room_controls() { + check_inverted_case(false); +} +#[test] +fn reversed_response_polarities_learn_with_the_same_generic_initialization() { + check_inverted_case(true); +} +#[test] +fn source_count_and_metadata_cannot_alias_canonical_rf_or_evade_import_bounds() { + let ds = dataset(56); + let (model, report) = RoomModel::fit( + &ds, + FitConfig { + epochs: 1, + ..FitConfig::default() + }, + ) + .unwrap(); + assert_eq!(model.encoder_config().bins, 56); + assert_eq!(model.encoder_config().source_count, 2); + assert!(report.parameter_count <= 512); + let original: serde_json::Value = serde_json::from_slice(&model.export().unwrap()).unwrap(); + for count in [0, 1, 3, 5000] { + let mut bad = original.clone(); + bad["config"]["source_count"] = serde_json::json!(count); + assert!(RoomModel::import(&serde_json::to_vec(&bad).unwrap()).is_err()); + } + let mut missing = original; + missing["config"] + .as_object_mut() + .unwrap() + .remove("source_count"); + assert!(RoomModel::import(&serde_json::to_vec(&missing).unwrap()).is_err()); + for malformed in [ + serde_json::Value::Null, + serde_json::json!([]), + serde_json::json!([[], []]), + ] { + let mut invalid: serde_json::Value = + serde_json::from_slice(&model.export().unwrap()).unwrap(); + invalid["source_projection_weights"] = malformed; + assert!(RoomModel::import(&serde_json::to_vec(&invalid).unwrap()).is_err()); + } + let tokens = model.tokens(&ds.held_out()[0], false).unwrap(); + let (features, _) = model.represent(&tokens, &model.parameters()).unwrap(); + let mut zeroed = tokens; + for source in &mut zeroed { + for row in source { + row[56..].fill(0.0); + } + } + let (without_tags, _) = model.represent(&zeroed, &model.parameters()).unwrap(); + let width = model.encoder_config().embedding_dim + 2; + for source in 0..2 { + assert_eq!( + &features[source * width + width - 2..source * width + width], + &without_tags[source * width + width - 2..source * width + width] + ); + } +} + +#[test] +fn three_native_56_bin_sources_train_every_projection_within_actual_budget() { + let base = dataset(56); + let mut context = base.context().clone(); + let mut source = context.sources[0].clone(); + source.source_id = 3; + source.device_id = "synthetic-device-3".into(); + source.link_id = "link-3".into(); + context.sources.push(source.clone()); + let mut rows = base + .train() + .iter() + .chain(base.validation()) + .chain(base.held_out()) + .cloned() + .collect::>(); + for row in &mut rows { + let mut view = row.views[0].clone(); + view.source_id = 3; + view.device_id = source.device_id.clone(); + view.link_id = source.link_id.clone(); + row.views.push(view); + row.context_digest = context.digest(); + } + let ds = ValidatedDataset::new(context, rows).unwrap(); + let (model, report) = RoomModel::fit( + &ds, + FitConfig { + epochs: 1, + ..FitConfig::default() + }, + ) + .unwrap(); + assert_eq!(model.encoder_config().source_count, 3); + assert_eq!(report.parameter_count, 395); + assert!(report.backbone_delta_l2 > 0.0); + let mut initial_model = model.clone(); + initial_model.encoder_weights = initial_encoder(model.encoder_config()) + .unwrap() + .transformer + .flatten_weights(); + initial_model.source_projection_weights = initial_projection_weights(model.encoder_config()); + center_empty_heads(&mut initial_model, ds.train()).unwrap(); + let initial = initial_model.source_projection_weights; + for (before, after) in initial.iter().zip(&model.source_projection_weights) { + assert!(before.iter().zip(after).any(|(a, b)| (a - b).abs() > 1e-6)); + } + let tokens = model.tokens(&ds.held_out()[0], false).unwrap(); + assert_eq!(tokens[2][0].len(), 59); + assert_eq!(&tokens[2][0][56..], &[0.0, 0.0, 1.0]); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_acquisition.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_acquisition.rs new file mode 100644 index 0000000000..e8344de80c --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_acquisition.rs @@ -0,0 +1,1164 @@ +//! Source-only v2 acquisition seed. Not wired to capture, inference, artifacts or +//! qualification. All policy arms are experimental engineering choices. +use super::managed_schedule_contract::{BoundEpisode, ManagedScheduleContract}; +use super::{ + native::{self, CanonicalPacket}, + observation::*, +}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::collections::{HashMap, HashSet}; + +const NS_PER_MS: u64 = 1_000_000; +const NS_PER_MILLI_HZ: u64 = 1_000_000_000_000; +const MAX_NATIVE_SPAN_NS: u64 = 4_000_000_000; +const SERIALIZED_LIMIT: usize = 64 * 1024 * 1024; +const SERIALIZED_METADATA_RESERVE: usize = 8 * 1024 * 1024; +const PREPROCESSING: &str = "native-esp32-drop2-linear56-ordered-magnitude-v2-experimental"; + +/// Finite, visible alternatives. Selecting a smaller variant is explicit; there +/// is no automatic truncation or silent fallback to fewer slots/shorter input. +#[derive(Debug, Clone, Copy, Serialize, PartialEq, Eq)] +pub enum ExperimentalPlanId { + Native1000msSlots8, + Native1000msSlots4, + Native1000msSlots2, + Native2000msSlots8, + Native2000msSlots4, + Native2000msSlots2, + Native3900msMax4000Slots8, + Native3900msMax4000Slots4, + Native3900msMax4000Slots2, +} +impl ExperimentalPlanId { + fn dimensions(self) -> (u64, usize) { + use ExperimentalPlanId::*; + match self { + Native1000msSlots8 => (1000, 8), + Native1000msSlots4 => (1000, 4), + Native1000msSlots2 => (1000, 2), + Native2000msSlots8 => (2000, 8), + Native2000msSlots4 => (2000, 4), + Native2000msSlots2 => (2000, 2), + Native3900msMax4000Slots8 => (3900, 8), + Native3900msMax4000Slots4 => (3900, 4), + Native3900msMax4000Slots2 => (3900, 2), + } + } +} +/// A frozen receiver-cadence study band, NOT a source measurement clock or +/// scientific adequacy claim. Actual packet intervals/rate must also fit it. +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +pub enum ReceiverCadenceBand { + Hz30To36, + Hz45To55, +} +impl ReceiverCadenceBand { + /// Read-only frozen-rate bounds for versioned input validation. + pub fn rate_bounds_millihz(self) -> (u64, u64) { + let (lo, hi, _) = self.range(); + (lo, hi) + } + fn range(self) -> (u64, u64, u64) { + match self { + Self::Hz30To36 => (30_000, 36_000, 50), + Self::Hz45To55 => (45_000, 55_000, 35), + } + } +} +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +pub struct ResourcePreflight { + pub cohorts: usize, + pub source_views: usize, + pub max_retained_rows: usize, + pub max_numeric_values: usize, + pub serialized_upper_bound_bytes: usize, +} +#[derive(Debug, Clone, Serialize)] +pub struct TrialStep { + pub step_id: String, + pub role: DataRole, + pub label: RoomLabel, + pub duration_ms: u64, + pub lead_in_ms: u64, +} +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SourceWindowPolicy { + pub source_id: u8, + pub receiver_cadence: ReceiverCadenceBand, + pub min_rows: usize, + pub max_rows: usize, + pub min_packet_interval_ns: u64, + pub max_gap_ns: u64, +} +#[derive(Debug, Clone, Serialize)] +struct HardLimits { + rows_per_source: usize, + cohorts: usize, + numeric_values: usize, + serialized_bytes: usize, + metadata_reserve_bytes: usize, + max_native_span_ns: u64, + max_first_packet_age_ns: u64, +} +#[derive(Debug, Clone, Serialize)] +struct PlanData { + schema: &'static str, + id: ExperimentalPlanId, + context_digest: String, + preprocessing_id: &'static str, + quality_policy_id: &'static str, + limits: HardLimits, + encoder_id: &'static str, + encoder: super::model::EncoderConfig, + target_span_ms: u64, + min_observed_span_ms: u64, + max_observed_span_ms: u64, + max_pairwise_receiver_skew_ms: u64, + max_first_packet_delay_ms: u64, + slots_per_trial: usize, + schedule: Vec, + sources: Vec, + resources: ResourcePreflight, +} +/// Private fields and no Deserialize: network input cannot supply thresholds or +/// issue a new policy by changing serialized bytes. Context/spec validators are +/// the existing learner's; this is not another model/dataset implementation. +#[derive(Debug, Clone)] +pub struct AcquisitionPlan { + context: CaptureContext, + data: PlanData, + sha256: String, +} +fn add(a: usize, b: usize) -> Result { + a.checked_add(b) + .ok_or_else(|| "acquisition_resource_overflow".into()) +} +fn mul(a: usize, b: usize) -> Result { + a.checked_mul(b) + .ok_or_else(|| "acquisition_resource_overflow".into()) +} +fn ceil_div(a: u64, b: u64) -> Result { + a.checked_add(b - 1) + .map(|v| v / b) + .ok_or_else(|| "acquisition_resource_overflow".into()) +} +impl AcquisitionPlan { + pub fn build( + context: CaptureContext, + id: ExperimentalPlanId, + cadences: &[(u8, ReceiverCadenceBand)], + ) -> Result { + context.validate()?; + if cadences.len() != context.sources.len() + || cadences + .iter() + .map(|s| s.0) + .ne(context.sources.iter().map(|s| s.source_id)) + { + return Err("acquisition_cadence_source_binding".into()); + } + let (target_ms, slots) = id.dimensions(); + let max_ms = if target_ms == 3900 { + 4000 + } else { + target_ms + 50 + }; + let target_ns = target_ms * NS_PER_MS; + let max_ns = max_ms * NS_PER_MS; + if max_ns > MAX_NATIVE_SPAN_NS { + return Err("acquisition_native_horizon_limit".into()); + } + let mut sources = vec![]; + for (spec, (_, band)) in context.sources.iter().zip(cadences) { + if spec.feature_count != native::CANONICAL_PACKET_BINS + || spec.signal_kind != ruview_unified::RfModality::WifiCsi + || spec.device_id != format!("esp32-node-{}", spec.source_id) + || spec.link_index != 0 + || !native::is_supported_frequency_hz(spec.center_frequency_hz) + || spec.bandwidth_hz != 20_000_000 + || !(0..=3).any(|ppdu| { + spec.grid_id + == format!( + "ant1-bins{}-ppdu{ppdu}-usable-drop2-linear-v2", + if ppdu == 0 { 64 } else { 256 } + ) + }) + { + return Err("acquisition_requires_existing_native56_source".into()); + } + let (lo, hi, gap_ms) = band.range(); + let max_rows = usize::try_from( + ceil_div( + max_ns + .checked_mul(hi) + .ok_or("acquisition_resource_overflow")?, + NS_PER_MILLI_HZ, + )? + 3, + ) + .map_err(|_| "acquisition_resource_overflow")?; + let min_rows = usize::try_from( + target_ns + .checked_mul(lo) + .ok_or("acquisition_resource_overflow")? + / NS_PER_MILLI_HZ + + 1, + ) + .map_err(|_| "acquisition_resource_overflow")?; + if max_rows > MAX_ROWS || min_rows < 2 || min_rows > max_rows { + return Err("acquisition_row_limit".into()); + } + sources.push(SourceWindowPolicy { + source_id: spec.source_id, + receiver_cadence: *band, + min_rows, + max_rows, + min_packet_interval_ns: NS_PER_MILLI_HZ / hi, + max_gap_ns: gap_ms * NS_PER_MS, + }); + } + let mut schedule = vec![]; + for (role_index, role) in [DataRole::Train, DataRole::HeldOut].into_iter().enumerate() { + for (condition, label) in [ + RoomLabel::Empty, + RoomLabel::OccupiedStatic, + RoomLabel::OccupiedMoving, + RoomLabel::AdjacentOnly, + ] + .into_iter() + .enumerate() + { + for trial in 0..2 { + // Whole-trial duration is distinct from the retained model + // horizon. Space each sparse slot so its window can finish. + let duration_ms = if label == RoomLabel::Empty { + 600_000 + } else { + (slots as u64 * 2 * (max_ms + 100)).div_ceil(1000).max(12) * 1000 + }; + schedule.push(TrialStep { + step_id: format!("v2-r{role_index}-c{condition}-t{trial}"), + role, + label, + duration_ms, + lead_in_ms: if matches!(label, RoomLabel::Empty | RoomLabel::AdjacentOnly) { + 15_000 + } else { + 5_000 + }, + }); + } + } + } + let cohorts = mul(schedule.len(), slots)?; + if cohorts > MAX_COHORTS { + return Err("acquisition_cohort_limit".into()); + } + let rows_per_cohort = sources.iter().try_fold(0, |sum, s| add(sum, s.max_rows))?; + let values_per_cohort = sources + .iter() + .zip(&context.sources) + .try_fold(0, |sum, (s, spec)| { + add(sum, mul(s.max_rows, spec.feature_count)?) + })?; + let rows = mul(cohorts, rows_per_cohort)?; + let values = mul(cohorts, values_per_cohort)?; + if values > MAX_TOTAL_VALUES { + return Err("acquisition_numeric_capacity".into()); + } + let views = mul(cohorts, sources.len())?; + // Conservative JSON upper bound: <=24-byte finite f32 text + comma; + // original u128 frame/u64 sequence/u64 row time + array delimiters; + // escaped bounded SourceView/identities, cohort wrappers, and reserved + // future policy/quality/checkpoint metadata. Uses MAX rows, never mean. + let serialized = add( + add(add(mul(values, 25)?, mul(rows, 86)?)?, mul(views, 4096)?)?, + add(mul(cohorts, 4096)?, SERIALIZED_METADATA_RESERVE)?, + )?; + if serialized > SERIALIZED_LIMIT { + return Err("acquisition_serialized_capacity".into()); + } + let resources = ResourcePreflight { + cohorts, + source_views: views, + max_retained_rows: rows, + max_numeric_values: values, + serialized_upper_bound_bytes: serialized, + }; + let data = PlanData { + schema: "ruview.experimental.native-temporal-acquisition.v2", + id, + context_digest: context.digest(), + preprocessing_id: PREPROCESSING, + quality_policy_id: "receiver-cadence-gap-skew-first-age-v1-experimental", + limits: HardLimits { + rows_per_source: MAX_ROWS, + cohorts: MAX_COHORTS, + numeric_values: MAX_TOTAL_VALUES, + serialized_bytes: SERIALIZED_LIMIT, + metadata_reserve_bytes: SERIALIZED_METADATA_RESERVE, + max_native_span_ns: MAX_NATIVE_SPAN_NS, + max_first_packet_age_ns: MAX_AGE_NS, + }, + encoder_id: "existing-aether-ordered-block-means-v1", + encoder: super::model::EncoderConfig { + source_count: context.sources.len(), + ..Default::default() + }, + target_span_ms: target_ms, + min_observed_span_ms: target_ms, + max_observed_span_ms: max_ms, + max_pairwise_receiver_skew_ms: 50, + max_first_packet_delay_ms: 100, + slots_per_trial: slots, + schedule, + sources, + resources, + }; + let sha256 = format!( + "{:x}", + Sha256::digest(serde_json::to_vec(&data).map_err(|_| "acquisition_plan_encoding")?) + ); + Ok(Self { + context, + data, + sha256, + }) + } + /// Read-only adapter seam; does not change serialized policy or its hash. + pub fn context(&self) -> &CaptureContext { + &self.context + } + pub fn id(&self) -> ExperimentalPlanId { + self.data.id + } + pub fn slots_per_trial(&self) -> usize { + self.data.slots_per_trial + } + pub fn max_pairwise_receiver_skew_ms(&self) -> u64 { + self.data.max_pairwise_receiver_skew_ms + } + pub fn max_first_packet_delay_ms(&self) -> u64 { + self.data.max_first_packet_delay_ms + } + pub fn sha256(&self) -> &str { + &self.sha256 + } + pub fn resources(&self) -> &ResourcePreflight { + &self.data.resources + } + pub fn schedule(&self) -> &[TrialStep] { + &self.data.schedule + } + pub fn source_policies(&self) -> &[SourceWindowPolicy] { + &self.data.sources + } + pub fn encoder_config(&self) -> &super::model::EncoderConfig { + &self.data.encoder + } + pub fn target_span_ms(&self) -> u64 { + self.data.target_span_ms + } + pub fn max_observed_span_ms(&self) -> u64 { + self.data.max_observed_span_ms + } +} + +struct PacketRow { + received_ns: u64, + frame_id: u128, + sequence: u64, + values: Vec, +} +struct ActiveWindow { + target_ns: u64, + end_ns: u64, + rows: HashMap>, + provenance: Option, + failed: bool, + ready: bool, +} +/// Read-only native window rules shared by seed-v2, managed-v3 and live input. +/// No experimental seed schedule/hash is returned for a managed contract. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct WindowInputPolicy { + context: CaptureContext, + sources: Vec, + target_span_ms: u64, + max_observed_span_ms: u64, + max_first_packet_delay_ms: u64, + max_pairwise_receiver_skew_ms: u64, +} +impl WindowInputPolicy { + pub(crate) fn from_seed(plan: &AcquisitionPlan) -> Self { + Self { + context: plan.context.clone(), + sources: plan.data.sources.clone(), + target_span_ms: plan.data.target_span_ms, + max_observed_span_ms: plan.data.max_observed_span_ms, + max_first_packet_delay_ms: plan.data.max_first_packet_delay_ms, + max_pairwise_receiver_skew_ms: plan.data.max_pairwise_receiver_skew_ms, + } + } + pub(crate) fn from_managed(contract: &ManagedScheduleContract) -> Self { + contract.native_window_policy() + } + pub(crate) fn from_contract_fields( + context: CaptureContext, + sources: Vec, + target: u64, + max: u64, + delay: u64, + skew: u64, + ) -> Self { + Self { + context, + sources, + target_span_ms: target, + max_observed_span_ms: max, + max_first_packet_delay_ms: delay, + max_pairwise_receiver_skew_ms: skew, + } + } + /// Same stateless native-source checks for retained windows and whole-period + /// observation. This does not admit a window, establish replay freshness or + /// create a measured clock/teacher claim. + pub(crate) fn validate_packet_source( + &self, + p: &CanonicalPacket, + ) -> Result<(&SourceSpec, &SourceWindowPolicy), String> { + let (spec, policy) = self.validate_packet_source_identity(p)?; + let (lo, hi, _) = policy.receiver_cadence.range(); + if p.receiver_sample_rate_hz < lo as f64 / 1000. + || p.receiver_sample_rate_hz > hi as f64 / 1000. + { + return Err("acquisition_cadence_changed".into()); + } + Ok((spec, policy)) + } + /// Identity-only check for closed Full2 priming, never a window admission. + /// The owning capture derives and checks its non-observing stage first. + pub(super) fn validate_priming_packet_source_identity( + &self, + p: &CanonicalPacket, + ) -> Result<(&SourceSpec, &SourceWindowPolicy), String> { + self.validate_packet_source_identity(p) + } + fn validate_packet_source_identity( + &self, + p: &CanonicalPacket, + ) -> Result<(&SourceSpec, &SourceWindowPolicy), String> { + let spec = self + .context + .sources + .iter() + .find(|s| s.source_id == p.source_id) + .ok_or("acquisition_unbound_source")?; + let policy = self + .sources + .iter() + .find(|s| s.source_id == p.source_id) + .ok_or("acquisition_unbound_source")?; + if p.grid_id != spec.grid_id + || p.frequency_hz != spec.center_frequency_hz + || p.bandwidth_hz != spec.bandwidth_hz + || p.native_frame.modality != spec.signal_kind + || p.native_frame.provenance.device_id != spec.device_id + || p.native_frame.centre_frequency_hz != spec.center_frequency_hz as f64 + || p.native_frame.bandwidth_hz != spec.bandwidth_hz as f64 + || p.amplitudes.len() != spec.feature_count + || p.amplitudes + .iter() + .any(|x| !x.is_finite() || *x < 0. || *x > f32::MAX as f64) + || p.native_frame.frame_id == 0 + || p.clock.kind != ClockKind::ReceiverOnly + || p.clock.uncertainty_ns != u64::MAX + || p.phase_consumed + { + return Err("acquisition_native_source_changed".into()); + } + if let Some(g) = spec.geometry { + if p.native_frame.transmitter_pose.map(|p| p.position_m) != Some(g.tx_pos) + || p.native_frame.receiver_pose.map(|p| p.position_m) != Some(g.rx_pos) + { + return Err("acquisition_native_geometry_changed".into()); + } + } + if !p.receiver_sample_rate_hz.is_finite() || p.receiver_sample_rate_hz <= 0.0 { + return Err("acquisition_cadence_changed".into()); + } + Ok((spec, policy)) + } + pub fn context(&self) -> &CaptureContext { + &self.context + } + pub fn source_policies(&self) -> &[SourceWindowPolicy] { + &self.sources + } + pub fn target_span_ms(&self) -> u64 { + self.target_span_ms + } + pub fn max_observed_span_ms(&self) -> u64 { + self.max_observed_span_ms + } + pub fn max_first_packet_delay_ms(&self) -> u64 { + self.max_first_packet_delay_ms + } + pub fn max_pairwise_receiver_skew_ms(&self) -> u64 { + self.max_pairwise_receiver_skew_ms + } +} +/// Historical mapping, not operator confirmation, quality or managed proof. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct ManagedWindowIdentity { + pub contract_sha256: String, + pub episode_index: usize, + pub slot_index: usize, + pub episode_id: String, + pub trial_id: String, + pub session_id: String, + pub attempt_id: String, + pub attempt_ordinal: u16, + pub segment_ordinal: u16, + pub role: DataRole, + pub label: RoomLabel, + pub collection_started_ns: String, + pub target_ns: String, + pub end_ns: String, + pub frozen_artifact_sha256: Option, + pub mapping: crate::room_learner::temporal_model_input::managed::execution::ObservedSegmentMap, +} +/// Only a future Runtime mapping (or explicit SYNTHETIC unit fixture) can issue +/// this non-Deserialize target. No client timestamps/roles are accepted here. +#[derive(Debug, Clone)] +pub struct ManagedWindowTarget { + run_id: String, + identity: ManagedWindowIdentity, +} +impl ManagedWindowTarget { + #[cfg(test)] + pub(crate) fn from_runtime_episode( + contract: &ManagedScheduleContract, + bound: &BoundEpisode, + run: &str, + slot: usize, + collection_started_ns: u64, + ) -> Result { + let mapping = crate::room_learner::temporal_model_input::managed::execution::ObservedSegmentMap::synthetic_uninterrupted(bound, collection_started_ns); + Self::from_observed_episode(contract, bound, run, slot, mapping) + } + pub(crate) fn from_observed_episode( + contract: &ManagedScheduleContract, + bound: &BoundEpisode, + run: &str, + slot: usize, + mapping: crate::room_learner::temporal_model_input::managed::execution::ObservedSegmentMap, + ) -> Result { + if !contract.bound_matches_run(bound, run) { + return Err("managed_window_episode_binding".into()); + } + let spec = bound + .episode() + .slots + .get(slot) + .ok_or("managed_window_slot_missing")?; + let (target, end) = mapping.slot_bounds( + bound, + spec.target_offset_ms * NS_PER_MS, + spec.deadline_offset_ms * NS_PER_MS, + )?; + Ok(Self { + run_id: run.into(), + identity: ManagedWindowIdentity { + contract_sha256: contract.sha256().into(), + episode_index: bound.episode().index, + slot_index: slot, + episode_id: bound.episode_id().into(), + trial_id: bound.trial_id().into(), + session_id: bound.session_id().into(), + attempt_id: bound.attempt_id().into(), + attempt_ordinal: bound.attempt_ordinal(), + segment_ordinal: bound.segment_ordinal(), + role: bound.episode().role, + label: bound.episode().condition, + collection_started_ns: mapping.receiver_start_ns.to_string(), + target_ns: target.to_string(), + end_ns: end.to_string(), + frozen_artifact_sha256: bound.frozen_artifact_sha256().map(str::to_owned), + mapping, + }, + }) + } + pub(crate) fn identity(&self) -> &ManagedWindowIdentity { + &self.identity + } + pub fn target_ns(&self) -> u64 { + self.identity + .target_ns + .parse() + .expect("internally bounded target") + } + pub fn end_ns(&self) -> u64 { + self.identity + .end_ns + .parse() + .expect("internally bounded target") + } +} +#[derive(Clone)] +enum WindowIdentityMode { + Seed(AcquisitionPlan), + Managed { + contract: ManagedScheduleContract, + binding_sha256: String, + pending: Option, + }, + Inference { + binding_sha256: String, + }, +} +/// Internal exact original row times are not a serialized/public progress DTO. +pub struct TemporalWindow { + pub policy_sha256: String, + pub cohort: CaptureCohort, + pub row_times_ns: Vec<(u8, Vec)>, + pub(crate) managed_identity: Option, +} +/// A bounded single-run seed. Runtime authorization, whole-trial quality/seal, +/// recovery and artifact version integration are deliberately not implemented. +/// Source watermarks survive successful windows/role transitions within this +/// object. A failure is terminal for this seed run: it cannot relabel/retry it. +pub struct NativeWindowAssembler { + policy: WindowInputPolicy, + identity: WindowIdentityMode, + run_id: String, + next: usize, + active: Option, + watermarks: HashMap, + frames: HashSet<(u8, u128)>, + last_end_ns: u64, + failed: bool, +} +#[derive(Debug, PartialEq, Eq)] +pub enum WindowAdmission { + Collecting, + Ready, +} +impl NativeWindowAssembler { + pub fn new(plan: AcquisitionPlan, run_id: String) -> Result { + if run_id.is_empty() + || run_id.len() > 128 + || !run_id.is_ascii() + || run_id + .bytes() + .any(|b| b.is_ascii_control() || b.is_ascii_whitespace()) + { + return Err("acquisition_run_id".into()); + } + Ok(Self { + policy: WindowInputPolicy::from_seed(&plan), + identity: WindowIdentityMode::Seed(plan), + run_id, + next: 0, + active: None, + watermarks: HashMap::new(), + frames: HashSet::new(), + last_end_ns: 0, + failed: false, + }) + } + pub(crate) fn managed( + contract: ManagedScheduleContract, + binding_sha256: String, + run_id: String, + ) -> Result { + let seed = AcquisitionPlan::build( + contract.context().clone(), + ExperimentalPlanId::Native2000msSlots8, + &contract + .context() + .sources + .iter() + .map(|s| (s.source_id, ReceiverCadenceBand::Hz30To36)) + .collect::>(), + )?; + let mut result = Self::new(seed, run_id)?; + result.policy = WindowInputPolicy::from_managed(&contract); + result.identity = WindowIdentityMode::Managed { + contract, + binding_sha256, + pending: None, + }; + Ok(result) + } + pub(crate) fn inference( + policy: WindowInputPolicy, + binding_sha256: String, + run_id: String, + ) -> Result { + // Run-id validation and native rules are identical; the seed is only an + // internal rules constructor and never supplies inference role/hash. + let bands = policy + .sources + .iter() + .map(|p| (p.source_id, p.receiver_cadence)) + .collect::>(); + let seed = AcquisitionPlan::build( + policy.context.clone(), + ExperimentalPlanId::Native2000msSlots8, + &bands, + )?; + let mut result = Self::new(seed, run_id)?; + result.policy = policy; + result.identity = WindowIdentityMode::Inference { binding_sha256 }; + Ok(result) + } + pub fn begin_managed_window(&mut self, target: &ManagedWindowTarget) -> Result<(), String> { + let WindowIdentityMode::Managed { + contract, pending, .. + } = &mut self.identity + else { + return Err("managed_window_mode_required".into()); + }; + if pending.is_some() + || target.run_id != self.run_id + || target.identity.contract_sha256 != contract.sha256() + { + return Err("managed_window_contract_changed".into()); + } + let expected = contract + .episodes() + .iter() + .take(target.identity.episode_index) + .map(|e| e.slots.len()) + .sum::() + + target.identity.slot_index; + if expected != self.next || expected >= contract.counts().total_windows { + return Err("managed_window_schedule_order".into()); + } + *pending = Some(target.identity.clone()); + if let Err(e) = self.begin_checked(target.target_ns(), target.end_ns()) { + if let WindowIdentityMode::Managed { pending, .. } = &mut self.identity { + *pending = None; + } + return Err(e); + } + Ok(()) + } + /// Target and end come from future server slot scheduling, not model scores. + /// First packets must begin near target, and the COMPLETE horizon must fit. + pub fn begin_window(&mut self, target_ns: u64, end_ns: u64) -> Result<(), String> { + if matches!(self.identity, WindowIdentityMode::Managed { .. }) { + return Err("managed_window_target_required".into()); + } + self.begin_checked(target_ns, end_ns) + } + fn begin_checked(&mut self, target_ns: u64, end_ns: u64) -> Result<(), String> { + let capacity = match &self.identity { + WindowIdentityMode::Seed(p) => p.resources().cohorts, + WindowIdentityMode::Managed { contract, .. } => contract.counts().total_windows, + WindowIdentityMode::Inference { .. } => 1, + }; + if self.failed || self.active.is_some() || self.next >= capacity { + return Err("acquisition_window_state".into()); + } + if target_ns == 0 + || target_ns <= self.last_end_ns + || end_ns <= target_ns + || end_ns - target_ns + < self.policy.max_observed_span_ms * NS_PER_MS + + self.policy.max_first_packet_delay_ms * NS_PER_MS + { + return Err("acquisition_slot_bounds".into()); + } + self.active = Some(ActiveWindow { + target_ns, + end_ns, + rows: HashMap::new(), + provenance: None, + failed: false, + ready: false, + }); + Ok(()) + } + pub fn ingest(&mut self, packet: &CanonicalPacket) -> Result { + let result = self.ingest_checked(packet); + if result.is_err() { + self.failed = true; + if let Some(a) = &mut self.active { + a.failed = true; + } + } + result + } + fn ingest_checked(&mut self, p: &CanonicalPacket) -> Result { + if self.failed { + return Err("acquisition_window_state".into()); + } + let a = self + .active + .as_mut() + .filter(|a| !a.failed && !a.ready) + .ok_or("acquisition_window_not_collecting")?; + let (_spec, policy) = self.policy.validate_packet_source(p)?; + let t = p.native_frame.timestamp_ns; + if t < a.target_ns || t >= a.end_ns { + return Err("acquisition_packet_outside_slot".into()); + } + if self + .watermarks + .get(&p.source_id) + .is_some_and(|(seq, last)| u64::from(p.sequence) <= *seq || t <= *last) + || self + .frames + .contains(&(p.source_id, p.native_frame.frame_id)) + { + return Err("acquisition_replayed_or_reversed".into()); + } + if a.provenance.is_some_and(|v| v != p.provenance) { + return Err("acquisition_provenance_changed".into()); + } + let rows = a.rows.entry(p.source_id).or_default(); + if rows.len() >= policy.max_rows { + return Err("acquisition_row_capacity".into()); + } + if let Some(last) = rows.last() { + let gap = t - last.received_ns; + if gap > policy.max_gap_ns || gap < policy.min_packet_interval_ns { + return Err("acquisition_packet_gap_or_rate".into()); + } + } else if t - a.target_ns > self.policy.max_first_packet_delay_ms * NS_PER_MS { + return Err("acquisition_late_first_packet".into()); + } + if let Some(first) = rows.first() { + if t - first.received_ns > self.policy.max_observed_span_ms * NS_PER_MS { + return Err("acquisition_window_span_limit".into()); + } + } + a.provenance = Some(p.provenance); + rows.push(PacketRow { + received_ns: t, + frame_id: p.native_frame.frame_id, + sequence: u64::from(p.sequence), + values: p.amplitudes.iter().map(|v| *v as f32).collect(), + }); + self.watermarks + .insert(p.source_id, (u64::from(p.sequence), t)); + self.frames.insert((p.source_id, p.native_frame.frame_id)); + a.ready = self.policy.sources.iter().all(|s| { + a.rows.get(&s.source_id).is_some_and(|r| { + r.len() >= s.min_rows + && r.last().unwrap().received_ns - r[0].received_ns + >= self.policy.target_span_ms * NS_PER_MS + }) + }); + if a.ready { + Ok(WindowAdmission::Ready) + } else { + Ok(WindowAdmission::Collecting) + } + } + pub fn finish_window(&mut self, now_ns: u64) -> Result { + let result = self.finish_checked(now_ns); + if result.is_err() { + self.failed = true; + } + result + } + fn finish_checked(&mut self, now_ns: u64) -> Result { + if self.failed { + return Err("acquisition_window_state".into()); + } + let a = self + .active + .as_ref() + .filter(|a| a.ready && !a.failed) + .ok_or("acquisition_horizon_incomplete")?; + if now_ns >= a.end_ns { + return Err("acquisition_slot_expired".into()); + } + + let mut views = vec![]; + let mut times = vec![]; + let mut first_min = u64::MAX; + let mut first_max = 0; + let mut last_min = u64::MAX; + let mut last_max = 0; + for (spec, policy) in self.policy.context.sources.iter().zip(&self.policy.sources) { + let rows = a + .rows + .get(&spec.source_id) + .ok_or("acquisition_missing_source")?; + let first = rows + .first() + .ok_or("acquisition_missing_source")? + .received_ns; + let last = rows.last().unwrap().received_ns; + let age = now_ns + .checked_sub(first) + .filter(|age| *age <= MAX_AGE_NS) + .ok_or("acquisition_first_packet_age")?; + if now_ns < last { + return Err("acquisition_future_packet".into()); + } + let span = last - first; + if span < self.policy.target_span_ms * NS_PER_MS + || span > self.policy.max_observed_span_ms * NS_PER_MS + || span > MAX_NATIVE_SPAN_NS + { + return Err("acquisition_observed_span".into()); + } + let (lo, hi, _) = policy.receiver_cadence.range(); + let scaled = (rows.len() - 1) as u128 * NS_PER_MILLI_HZ as u128; + if scaled < (lo as u128 * span as u128) || scaled > (hi as u128 * span as u128) { + return Err("acquisition_observed_cadence".into()); + } + first_min = first_min.min(first); + first_max = first_max.max(first); + last_min = last_min.min(last); + last_max = last_max.max(last); + let view = SourceView { + source_id: spec.source_id, + device_id: spec.device_id.clone(), + grid_id: spec.grid_id.clone(), + signal_kind: spec.signal_kind, + center_frequency_hz: spec.center_frequency_hz, + bandwidth_hz: spec.bandwidth_hz, + link_id: spec.link_id.clone(), + link_index: spec.link_index, + geometry: spec.geometry, + samples: rows.iter().map(|r| r.values.clone()).collect(), + frame_ids: rows.iter().map(|r| r.frame_id).collect(), + sequence_ids: rows.iter().map(|r| r.sequence).collect(), + first_time_ns: first, + last_time_ns: last, + clock: ClockInfo { + kind: ClockKind::ReceiverOnly, + uncertainty_ns: u64::MAX, + }, + sample_age_ns: age, + phase: PhaseAvailability::MagnitudeOnly, + provenance: a.provenance.ok_or("acquisition_provenance_missing")?, + }; + view.validate(spec)?; + views.push(view); + times.push((spec.source_id, rows.iter().map(|r| r.received_ns).collect())); + } + let skew = self.policy.max_pairwise_receiver_skew_ms * NS_PER_MS; + if first_max - first_min > skew || last_max - last_min > skew || first_max >= last_min { + return Err("acquisition_source_skew".into()); + } + let (id, trial_id, session_id, role, label, policy_sha256, managed_identity) = match &self + .identity + { + WindowIdentityMode::Seed(plan) => { + let step = &plan.data.schedule[self.next / plan.data.slots_per_trial]; + let slot = self.next % plan.data.slots_per_trial; + ( + format!("{}-{}-{slot}", self.run_id, step.step_id), + format!("{}-{}", self.run_id, step.step_id), + format!("{}-{}", self.run_id, step.step_id), + step.role, + step.label, + plan.sha256.clone(), + None, + ) + } + WindowIdentityMode::Managed { + binding_sha256, + pending, + .. + } => { + let p = pending.as_ref().ok_or("managed_window_target_missing")?; + let id = format!( + "window-{:x}", + Sha256::digest( + serde_json::to_vec(p).map_err(|_| "managed_window_identity_encoding")? + ) + ); + ( + id, + p.trial_id.clone(), + p.session_id.clone(), + p.role, + p.label, + binding_sha256.clone(), + Some(p.clone()), + ) + } + WindowIdentityMode::Inference { binding_sha256 } => ( + format!("{}-inference-0", self.run_id), + format!("{}-inference", self.run_id), + format!("{}-inference", self.run_id), + DataRole::Inference, + RoomLabel::Unknown, + binding_sha256.clone(), + None, + ), + }; + let cohort = CaptureCohort { + id, + context_digest: self.policy.context.digest(), + trial_id, + session_id, + role, + label, + views, + teacher: None, + provenance: a.provenance.unwrap(), + }; + validate_cohort(&self.policy.context, &cohort)?; + let result = TemporalWindow { + policy_sha256, + cohort, + row_times_ns: times, + managed_identity, + }; + if let WindowIdentityMode::Managed { pending, .. } = &mut self.identity { + *pending = None; + } + self.last_end_ns = last_max; + self.next += 1; + self.active = None; + Ok(result) + } +} +#[cfg(test)] +#[path = "temporal_acquisition_tests.rs"] +mod tests; + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct AssemblerCheckpoint { + next: usize, + last_end_ns: u64, + watermarks: Vec<(u8, u64, u64)>, + frames: Vec<(u8, String)>, +} +impl AssemblerCheckpoint { + pub(crate) fn for_prefix(mut self, n: usize) -> Result { + if n > self.next { + return Err("temporal_prefix_ahead".into()); + } + self.next = n; + Ok(self) + } +} +impl NativeWindowAssembler { + #[cfg(test)] + pub(crate) fn c4b_test_fence_counts(&self)->(usize,usize,usize,bool,bool){(self.next,self.frames.len(),self.watermarks.len(),self.failed,self.active.is_some())} + pub(crate) fn fence_checkpoint(&self) -> Result { + if self.failed { + return Err("temporal_failed_fences".into()); + } + let mut watermarks = self + .watermarks + .iter() + .map(|(id, (seq, t))| (*id, *seq, *t)) + .collect::>(); + watermarks.sort(); + let mut frames = self + .frames + .iter() + .map(|(id, f)| (*id, f.to_string())) + .collect::>(); + frames.sort(); + Ok(AssemblerCheckpoint { + next: self.next, + last_end_ns: self.last_end_ns, + watermarks, + frames, + }) + } + pub(crate) fn restore_prefix_fences( + &mut self, + s: AssemblerCheckpoint, + expected_next: usize, + ) -> Result<(), String> { + let max = self.policy.sources.iter().map(|s|s.max_rows).sum::()*128; + self.restore_prefix_fences_with_limit(s,expected_next,max) + } + /// Only the separately validated canonical Full2 contract may select its + /// derived replay bookkeeping bound. Historical callers keep the old API. + pub(crate) fn restore_full_run_prefix_fences( + &mut self,s:AssemblerCheckpoint,expected_next:usize, + budget:&crate::room_learner::temporal_model_input::managed::execution::full_run::ReplayFenceBudget, + )->Result<(),String>{ + budget.matches_native_policy(&self.policy)?; + self.restore_prefix_fences_with_limit(s,expected_next,budget.max_frames()) + } + fn restore_prefix_fences_with_limit(&mut self,s:AssemblerCheckpoint,expected_next:usize,max:usize)->Result<(),String>{ + if s.next != expected_next + || s.next > 128 + || s.frames.len() > max + || s.watermarks.len() > self.policy.sources.len() + { + return Err("temporal_checkpoint_fence_bounds".into()); + } + let mut watermarks = HashMap::new(); + let mut frames = HashSet::new(); + for (id, seq, t) in s.watermarks { + if !self + .policy + .context + .sources + .iter() + .any(|x| x.source_id == id) + || seq == 0 + || t == 0 + || watermarks.insert(id, (seq, t)).is_some() + { + return Err("temporal_checkpoint_fence_source".into()); + } + } + for (id, f) in s.frames { + let f = f.parse::().map_err(|_| "temporal_checkpoint_frame")?; + let (sequence, time) = watermarks + .get(&id) + .copied() + .ok_or("temporal_checkpoint_frame")?; + if ((f >> 32) & 255) != u128::from(id) + || (f >> 40) > u128::from(time) + || (f & u128::from(u32::MAX)) > u128::from(sequence) + || !frames.insert((id, f)) + { + return Err("temporal_checkpoint_frame".into()); + } + } + self.next = s.next; + self.last_end_ns = s.last_end_ns; + self.watermarks = watermarks; + self.frames = frames; + self.active = None; + self.failed = false; + if let WindowIdentityMode::Managed { pending, .. } = &mut self.identity { + *pending = None; + } + Ok(()) + } + pub(crate) fn fences_cover(&self, other: &AssemblerCheckpoint) -> bool { + other.watermarks.iter().all(|(id, seq, t)| { + self.watermarks + .get(id) + .is_some_and(|(s, n)| s >= seq && n >= t) + }) && other.frames.iter().all(|(id, f)| { + f.parse() + .ok() + .is_some_and(|f| self.frames.contains(&(*id, f))) + }) + } +} + +impl NativeWindowAssembler { + pub(crate) fn retained_fences_cover(&self, rows: &[(u8, u64, u64, u128)]) -> bool { + rows.iter().all(|(id, seq, t, f)| { + self.watermarks + .get(id) + .is_some_and(|(s, n)| s >= seq && n >= t) + && self.frames.contains(&(*id, *f)) + }) + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_acquisition_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_acquisition_tests.rs new file mode 100644 index 0000000000..87111e709c --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_acquisition_tests.rs @@ -0,0 +1,648 @@ +//! SYNTHETIC original native IQ through the existing native adapter and SourceView +//! validators. No physical timing/accuracy or integrated v2 runtime claim. +use super::super::native::{canonicalize_esp32, NativePacketInput}; +use super::*; +use wifi_densepose_hardware::{Adr018Flags, PpduType}; + +fn context(count: u8) -> CaptureContext { + CaptureContext { + owner_scope: "a".repeat(64), + installation_id: "synthetic-install".into(), + room_digest: "synthetic-room".into(), + revision: 1, + sources: (1..=count) + .map(|source_id| SourceSpec { + source_id, + device_id: format!("esp32-node-{source_id}"), + grid_id: "ant1-bins64-ppdu0-usable-drop2-linear-v2".into(), + feature_count: 56, + signal_kind: ruview_unified::RfModality::WifiCsi, + center_frequency_hz: 2_437_000_000, + bandwidth_hz: 20_000_000, + link_id: format!("native-{source_id}"), + link_index: 0, + geometry: None, + }) + .collect(), + } +} +fn plan( + count: u8, + id: ExperimentalPlanId, + band: ReceiverCadenceBand, +) -> Result { + AcquisitionPlan::build( + context(count), + id, + &(1..=count).map(|n| (n, band)).collect::>(), + ) +} +fn packet(source: u8, sequence: u32, at: u64, hz: f64) -> CanonicalPacket { + let bytes = (0..64).flat_map(|_| [3u8, 4]).collect::>(); + canonicalize_esp32(NativePacketInput { + node_id: source, + n_antennas: 1, + raw_bins: 64, + frequency_hz: 2_437_000_000, + ppdu_type: PpduType::HtLegacy, + sequence, + received_at_ns: at, + receiver_sample_rate_hz: hz, + iq_bytes: &bytes, + flags: Adr018Flags::default(), + provenance: CaptureProvenance::Synthetic, + }) + .unwrap() +} +fn collect( + assembler: &mut NativeWindowAssembler, + target: u64, + hz: u64, + sequence: &mut u32, +) -> TemporalWindow { + let period = 1_000_000_000u64.div_ceil(hz); + let end = target + 6_000_000_000; + assembler.begin_window(target, end).unwrap(); + for index in 0..MAX_ROWS { + *sequence += 1; + for source in 1..=assembler.policy.context.sources.len() as u8 { + let last = target + index as u64 * period + u64::from(source) * 1000; + let result = assembler + .ingest(&packet(source, *sequence, last, hz as f64)) + .unwrap(); + if result == WindowAdmission::Ready { + return assembler.finish_window(last).unwrap(); + } + } + } + panic!("window did not reach its actual horizon"); +} +#[test] +fn immutable_plan_hash_and_actual_max_capacity_are_context_cadence_and_variant_bound() { + use ExperimentalPlanId::*; + use ReceiverCadenceBand::*; + let a = plan(3, Native2000msSlots8, Hz30To36).unwrap(); + let b = plan(3, Native2000msSlots8, Hz30To36).unwrap(); + assert_eq!(a.sha256(), b.sha256()); + assert_eq!(a.resources().cohorts, 128); + assert_eq!(a.source_policies()[0].max_rows, 77); + assert_eq!(a.resources().max_numeric_values, 128 * 3 * 56 * 77); + assert!(a.resources().serialized_upper_bound_bytes <= SERIALIZED_LIMIT); + assert_eq!( + a.schedule() + .iter() + .filter(|s| s.role == DataRole::Train) + .count(), + 8 + ); + assert_eq!( + a.schedule() + .iter() + .filter(|s| s.role == DataRole::HeldOut) + .count(), + 8 + ); + for label in [ + RoomLabel::Empty, + RoomLabel::OccupiedStatic, + RoomLabel::OccupiedMoving, + RoomLabel::AdjacentOnly, + ] { + for role in [DataRole::Train, DataRole::HeldOut] { + assert_eq!( + a.schedule() + .iter() + .filter(|s| s.role == role && s.label == label) + .count(), + 2 + ); + } + } + assert!(plan(2, Native2000msSlots8, Hz45To55).is_ok()); + assert_eq!( + plan(3, Native2000msSlots8, Hz45To55).unwrap_err(), + "acquisition_numeric_capacity" + ); + assert!(plan(3, Native2000msSlots4, Hz45To55).is_ok()); + assert!(plan(8, Native2000msSlots4, Hz30To36).is_err()); + let smaller = plan(8, Native2000msSlots2, Hz30To36).unwrap(); + assert_eq!(smaller.resources().cohorts, 32); + assert_ne!( + a.sha256(), + plan(2, Native2000msSlots8, Hz30To36).unwrap().sha256() + ); + assert_ne!( + a.sha256(), + plan(3, Native2000msSlots4, Hz30To36).unwrap().sha256() + ); + let mut ctx = context(2); + ctx.sources[1].source_id = 1; + assert!( + AcquisitionPlan::build(ctx, Native1000msSlots2, &[(1, Hz30To36), (1, Hz30To36)]).is_err() + ); + assert!(AcquisitionPlan::build( + context(2), + Native1000msSlots2, + &[(2, Hz30To36), (1, Hz30To36)] + ) + .is_err()); + let mut ctx = context(2); + ctx.sources[0].feature_count = 64; + assert!( + AcquisitionPlan::build(ctx, Native1000msSlots2, &[(1, Hz30To36), (2, Hz30To36)]).is_err() + ); +} +#[test] +fn actual_two_seconds_at_33_and_50_hz_preserve_all_original_rows_phy_and_frame_identity() { + for (hz, band, rows) in [ + (33, ReceiverCadenceBand::Hz30To36, 67), + (50, ReceiverCadenceBand::Hz45To55, 101), + ] { + let p = plan(2, ExperimentalPlanId::Native2000msSlots2, band).unwrap(); + let hash = p.sha256().to_string(); + let mut a = NativeWindowAssembler::new(p, "synthetic-run".into()).unwrap(); + let window = collect(&mut a, 10_000_000_000, hz, &mut 0); + assert_eq!(window.policy_sha256, hash); + assert_eq!(window.cohort.role, DataRole::Train); + assert_eq!(window.cohort.provenance, CaptureProvenance::Synthetic); + assert!(window.cohort.teacher.is_none()); + for (view, (source, times)) in window.cohort.views.iter().zip(&window.row_times_ns) { + assert_eq!(view.source_id, *source); + assert_eq!(view.samples.len(), rows); + assert_eq!( + view.last_time_ns - view.first_time_ns, + times.last().unwrap() - times[0] + ); + assert!(view.last_time_ns - view.first_time_ns >= 2_000_000_000); + assert_eq!(view.center_frequency_hz, 2_437_000_000); + assert_eq!(view.bandwidth_hz, 20_000_000); + assert_eq!(view.clock.kind, ClockKind::ReceiverOnly); + assert_eq!(view.clock.uncertainty_ns, u64::MAX); + for (index, ((sequence, frame), row)) in view + .sequence_ids + .iter() + .zip(&view.frame_ids) + .zip(&view.samples) + .enumerate() + { + let original = packet(*source, *sequence as u32, times[index], hz as f64); + assert_eq!(*frame, original.native_frame.frame_id); + assert_eq!( + row, + &original + .amplitudes + .iter() + .map(|n| *n as f32) + .collect::>() + ); + assert!(row.iter().all(|v| *v == 5.0)); + } + } + } +} +#[test] +fn fixed_64_rows_cannot_claim_two_seconds_and_legacy_eight_rows_are_not_upgraded() { + for count in [8, 64] { + for hz in [33, 50] { + let band = if hz == 33 { + ReceiverCadenceBand::Hz30To36 + } else { + ReceiverCadenceBand::Hz45To55 + }; + let p = plan(2, ExperimentalPlanId::Native2000msSlots2, band).unwrap(); + let mut a = NativeWindowAssembler::new(p, "short-control".into()).unwrap(); + let target = 10_000_000_000; + a.begin_window(target, target + 3_000_000_000).unwrap(); + let period = 1_000_000_000u64.div_ceil(hz); + for index in 0..count { + for source in 1..=2 { + assert_eq!( + a.ingest(&packet( + source, + index + 1, + target + u64::from(index) * period + u64::from(source) * 1000, + hz as f64 + )) + .unwrap(), + WindowAdmission::Collecting + ); + } + } + assert_eq!( + a.finish_window(target + u64::from(count) * period) + .err() + .as_deref(), + Some("acquisition_horizon_incomplete") + ); + } + } +} +#[test] +fn gaps_replay_wrong_source_grid_phy_age_and_skew_fail_without_downsampling() { + for failure in [ + "gap", + "sequence", + "frame", + "source", + "grid", + "phy", + "nonfinite", + "cadence", + "fast_actual", + "provenance", + ] { + let p = plan( + 2, + ExperimentalPlanId::Native1000msSlots2, + ReceiverCadenceBand::Hz30To36, + ) + .unwrap(); + let mut a = NativeWindowAssembler::new(p, "negative".into()).unwrap(); + let t = 10_000_000_000; + a.begin_window(t, t + 2_000_000_000).unwrap(); + let first = packet(1, 1, t + 1000, 33.); + let original = first.native_frame.frame_id; + a.ingest(&first).unwrap(); + let mut bad = packet(1, 2, t + 30_304_000, 33.); + match failure { + "gap" => bad = packet(1, 2, t + 100_000_000, 33.), + "sequence" => bad.sequence = 1, + "frame" => bad.native_frame.frame_id = original, + "source" => bad.source_id = 99, + "grid" => bad.grid_id = "fake".into(), + "phy" => bad.frequency_hz = 5_180_000_000, + "nonfinite" => bad.amplitudes[0] = f64::NAN, + "cadence" => bad.receiver_sample_rate_hz = 50., + "fast_actual" => bad = packet(1, 2, t + 20_000_000, 33.), + "provenance" => bad.provenance = CaptureProvenance::LiveReceiverObserved, + _ => unreachable!(), + } + assert!(a.ingest(&bad).is_err(), "{failure}"); + assert_eq!( + a.active.as_ref().unwrap().rows[&1].len(), + 1, + "{failure} replaced a row" + ); + assert!(a + .begin_window(t + 3_000_000_000, t + 5_000_000_000) + .is_err()); + } + let mut a = NativeWindowAssembler::new( + plan( + 2, + ExperimentalPlanId::Native3900msMax4000Slots2, + ReceiverCadenceBand::Hz30To36, + ) + .unwrap(), + "age".into(), + ) + .unwrap(); + let w = collect(&mut a, 10_000_000_000, 33, &mut 0); + assert!(w + .cohort + .views + .iter() + .all(|v| v.last_time_ns - v.first_time_ns >= 3_900_000_000 + && v.last_time_ns - v.first_time_ns <= 4_000_000_000)); + let mut a = NativeWindowAssembler::new( + plan( + 2, + ExperimentalPlanId::Native3900msMax4000Slots2, + ReceiverCadenceBand::Hz30To36, + ) + .unwrap(), + "stale".into(), + ) + .unwrap(); + let t = 10_000_000_000; + a.begin_window(t, t + 7_000_000_000).unwrap(); + let period = 1_000_000_000u64.div_ceil(33); + let mut last = t; + 'ready: for index in 0..MAX_ROWS { + for source in 1..=2 { + last = t + index as u64 * period + u64::from(source) * 1000; + if a.ingest(&packet(source, index as u32 + 1, last, 33.)) + .unwrap() + == WindowAdmission::Ready + { + break 'ready; + } + } + } + assert!(last - t <= 4_000_000_000); + assert_eq!( + a.finish_window(t + 5_000_000_001 + 1000).err().as_deref(), + Some("acquisition_first_packet_age") + ); +} +#[test] +fn schedule_roles_and_original_sequence_fences_survive_multiple_windows() { + let ctx = context(2); + let p = plan( + 2, + ExperimentalPlanId::Native1000msSlots2, + ReceiverCadenceBand::Hz30To36, + ) + .unwrap(); + let mut a = NativeWindowAssembler::new(p, "role-run".into()).unwrap(); + let mut rows = vec![]; + let mut sequence = 0; + for index in 0..32 { + let w = collect( + &mut a, + 10_000_000_000 + index * 3_000_000_000, + 33, + &mut sequence, + ); + assert_eq!( + w.cohort.role, + if index < 16 { + DataRole::Train + } else { + DataRole::HeldOut + } + ); + rows.push(w.cohort); + } + let valid = ValidatedDataset::new(ctx.clone(), rows.clone()).unwrap(); + assert_eq!(valid.train().len(), 16); + assert_eq!(valid.held_out().len(), 16); + assert!(!valid.measurement_clock_available()); + rows[16].views[0].frame_ids[0] = rows[0].views[0].frame_ids[0]; + assert!(ValidatedDataset::new(ctx, rows).is_err()); + assert!(a.begin_window(200_000_000_000, 202_000_000_000).is_err()); + let p = plan( + 2, + ExperimentalPlanId::Native1000msSlots2, + ReceiverCadenceBand::Hz30To36, + ) + .unwrap(); + let mut a = NativeWindowAssembler::new(p, "replay".into()).unwrap(); + collect(&mut a, 10_000_000_000, 33, &mut 0); + a.begin_window(20_000_000_000, 22_000_000_000).unwrap(); + assert_eq!( + a.ingest(&packet(1, 1, 20_000_001_000, 33.)) + .err() + .as_deref(), + Some("acquisition_replayed_or_reversed") + ); +} +#[test] +fn resource_upper_bound_includes_max_rows_and_actual_json_not_just_float_memory() { + let p = plan( + 3, + ExperimentalPlanId::Native2000msSlots8, + ReceiverCadenceBand::Hz30To36, + ) + .unwrap(); + let resources = p.resources(); + assert!( + resources.serialized_upper_bound_bytes + > resources.max_numeric_values * std::mem::size_of::() + ); + let mut a = NativeWindowAssembler::new(p.clone(), "serialization".into()).unwrap(); + let mut w = collect(&mut a, 10_000_000_000, 33, &mut 0); + for view in &mut w.cohort.views { + let max = p + .source_policies() + .iter() + .find(|s| s.source_id == view.source_id) + .unwrap() + .max_rows; + view.samples = vec![vec![1.2345678e-30f32; 56]; max]; + view.frame_ids = vec![u128::MAX; max]; + view.sequence_ids = vec![u64::MAX; max]; + } + let one = serde_json::to_vec(&w.cohort).unwrap().len(); + assert!( + one * resources.cohorts + SERIALIZED_METADATA_RESERVE + < resources.serialized_upper_bound_bytes + ); +} + +#[test] +fn individually_long_enough_sources_still_need_bounded_pairwise_receiver_skew() { + let p = plan( + 2, + ExperimentalPlanId::Native1000msSlots2, + ReceiverCadenceBand::Hz30To36, + ) + .unwrap(); + let mut a = NativeWindowAssembler::new(p, "skew".into()).unwrap(); + let t = 10_000_000_000; + a.begin_window(t, t + 2_000_000_000).unwrap(); + let period = 1_000_000_000u64.div_ceil(33); + let mut now = t; + 'ready: for index in 0..MAX_ROWS { + for source in 1..=2 { + now = t + index as u64 * period + if source == 1 { 1000 } else { 80_001_000 }; + if a.ingest(&packet(source, index as u32 + 1, now, 33.)) + .unwrap() + == WindowAdmission::Ready + { + break 'ready; + } + } + } + assert_eq!( + a.finish_window(now).err().as_deref(), + Some("acquisition_source_skew") + ); +} +#[test] +fn capacity_failure_is_terminal_and_does_not_evict_or_shorten_the_horizon() { + let p = plan( + 2, + ExperimentalPlanId::Native2000msSlots2, + ReceiverCadenceBand::Hz30To36, + ) + .unwrap(); + let capacity = p.source_policies()[0].max_rows; + let mut a = NativeWindowAssembler::new(p, "capacity".into()).unwrap(); + let t = 10_000_000_000; + a.begin_window(t, t + 3_000_000_000).unwrap(); + a.ingest(&packet(1, 1, t + 1000, 33.)).unwrap(); + // Boundary fault injection of a full buffer: the public admission operation + // must fail rather than pop old rows, regardless of any future caller bug. + let original = a.active.as_ref().unwrap().rows[&1][0].frame_id; + let buffer = a.active.as_mut().unwrap().rows.get_mut(&1).unwrap(); + while buffer.len() < capacity { + buffer.push(PacketRow { + received_ns: t + 1000, + frame_id: original, + sequence: 1, + values: vec![5.; 56], + }); + } + assert_eq!( + a.ingest(&packet(1, 2, t + 30_304_000, 33.)) + .err() + .as_deref(), + Some("acquisition_row_capacity") + ); + assert_eq!(a.active.as_ref().unwrap().rows[&1].len(), capacity); + assert_eq!(a.active.as_ref().unwrap().rows[&1][0].frame_id, original); + assert!(a.finish_window(t + 2_100_000_000).is_err()); +} + +#[test] +fn four_second_native_ceiling_is_independent_of_the_existing_five_second_age_guard() { + let p = plan( + 2, + ExperimentalPlanId::Native3900msMax4000Slots2, + ReceiverCadenceBand::Hz30To36, + ) + .unwrap(); + assert_eq!(p.max_observed_span_ms(), 4000); + let mut a = NativeWindowAssembler::new(p, "four-second-limit".into()).unwrap(); + let t = 10_000_000_000; + a.begin_window(t, t + 6_000_000_000).unwrap(); + let period = 1_000_000_000u64.div_ceil(33); + 'ready: for index in 0..MAX_ROWS { + for source in 1..=2 { + if a.ingest(&packet( + source, + index as u32 + 1, + t + index as u64 * period + u64::from(source) * 1000, + 33., + )) + .unwrap() + == WindowAdmission::Ready + { + break 'ready; + } + } + } + // Corrupted/overlong buffer negative control: the inherited SourceView + // validator permits up to60s, so v2's own <=4s guard must independently win. + let rows = a.active.as_mut().unwrap().rows.get_mut(&1).unwrap(); + let first = rows[0].received_ns; + rows.last_mut().unwrap().received_ns = first + 4_000_000_001; + assert_eq!( + a.finish_window(t + 4_100_000_000).err().as_deref(), + Some("acquisition_observed_span") + ); +} + +#[test] +fn approved_variants_preflight_every_supported_source_count_without_silent_fallback() { + use ExperimentalPlanId::*; + for count in 2..=16 { + for id in [ + Native1000msSlots8, + Native1000msSlots4, + Native1000msSlots2, + Native2000msSlots8, + Native2000msSlots4, + Native2000msSlots2, + Native3900msMax4000Slots8, + Native3900msMax4000Slots4, + Native3900msMax4000Slots2, + ] { + for band in [ReceiverCadenceBand::Hz30To36, ReceiverCadenceBand::Hz45To55] { + if let Ok(plan) = plan(count, id, band) { + assert_eq!(plan.data.id, id); + assert!(plan.resources().cohorts <= MAX_COHORTS); + assert!(plan.resources().max_numeric_values <= MAX_TOTAL_VALUES); + assert!(plan.resources().serialized_upper_bound_bytes <= SERIALIZED_LIMIT); + assert!(plan + .source_policies() + .iter() + .all(|s| s.max_rows <= MAX_ROWS)); + assert_eq!(plan.encoder_config().source_count, usize::from(count)); + assert_eq!(plan.encoder_config().temporal_tokens, 4); + } + } + } + } +} +#[test] +fn mixed_original_ht_and_he_phy_grids_remain_distinct_in_the_same_temporal_window() { + let mut ctx = context(2); + ctx.sources[1].grid_id = "ant1-bins256-ppdu1-usable-drop2-linear-v2".into(); + let p = AcquisitionPlan::build( + ctx, + ExperimentalPlanId::Native1000msSlots2, + &[ + (1, ReceiverCadenceBand::Hz30To36), + (2, ReceiverCadenceBand::Hz30To36), + ], + ) + .unwrap(); + let mut a = NativeWindowAssembler::new(p, "mixed-native".into()).unwrap(); + let t = 10_000_000_000; + a.begin_window(t, t + 2_000_000_000).unwrap(); + let period = 1_000_000_000u64.div_ceil(33); + for index in 0..MAX_ROWS { + for source in 1..=2 { + let bins = if source == 1 { 64 } else { 256 }; + let bytes = (0..bins).flat_map(|_| [3u8, 4]).collect::>(); + let packet = canonicalize_esp32(NativePacketInput { + node_id: source, + n_antennas: 1, + raw_bins: bins, + frequency_hz: 2_437_000_000, + ppdu_type: if source == 1 { + PpduType::HtLegacy + } else { + PpduType::HeSu + }, + sequence: index as u32 + 1, + received_at_ns: t + index as u64 * period + u64::from(source) * 1000, + receiver_sample_rate_hz: 33., + iq_bytes: &bytes, + flags: Adr018Flags::default(), + provenance: CaptureProvenance::Synthetic, + }) + .unwrap(); + let now = packet.native_frame.timestamp_ns; + if a.ingest(&packet).unwrap() == WindowAdmission::Ready { + let w = a.finish_window(now).unwrap(); + assert!(w.cohort.views[0].grid_id.contains("bins64-ppdu0")); + assert!(w.cohort.views[1].grid_id.contains("bins256-ppdu1")); + assert!(w + .cohort + .views + .iter() + .all(|v| v.samples.len() == 34 && v.samples.iter().all(|r| r.len() == 56))); + return; + } + } + } + panic!("mixed source window incomplete"); +} + +#[test] +fn a_failed_finish_cannot_be_retried_with_an_earlier_clock_or_more_packets() { + let p = plan( + 2, + ExperimentalPlanId::Native1000msSlots2, + ReceiverCadenceBand::Hz30To36, + ) + .unwrap(); + let mut a = NativeWindowAssembler::new(p, "failed-finish".into()).unwrap(); + let t = 10_000_000_000; + a.begin_window(t, t + 7_000_000_000).unwrap(); + let period = 1_000_000_000u64.div_ceil(33); + let mut last = t; + 'ready: for index in 0..MAX_ROWS { + for source in 1..=2 { + last = t + index as u64 * period + u64::from(source) * 1000; + if a.ingest(&packet(source, index as u32 + 1, last, 33.)) + .unwrap() + == WindowAdmission::Ready + { + break 'ready; + } + } + } + assert_eq!( + a.finish_window(t + 6_000_000_000).err().as_deref(), + Some("acquisition_first_packet_age") + ); + assert_eq!( + a.finish_window(last).err().as_deref(), + Some("acquisition_window_state") + ); + assert!(a.ingest(&packet(1, 200, last + period, 33.)).is_err()); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_artifact_store.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_artifact_store.rs new file mode 100644 index 0000000000..92374b5d68 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_artifact_store.rs @@ -0,0 +1,385 @@ +//! Whole temporal-input diagnostics in the EXISTING owner-bound ArtifactStore. +//! Private diagnostic Runtime callers only; no managed proof, qualification, +//! selection or inner-v1 access. +use super::*; +use crate::room_learner::temporal_model_input::TemporalModel; +use std::{fs, io::Read}; + +// Stable kind namespace; codec kind/pointer are versioned inside it. +// This reader supports only the existing d4a v2 codec, with no upconversion. +pub(super) const DIRECTORY: &str = "temporal-models"; +const SCHEMA: &str = "ruview.room-learner.temporal-candidate.v1"; +const KIND: &str = "ruview.experimental.temporal-model-input.v2"; +const MANAGED_KIND: &str = "ruview.room-learner.managed-temporal-input.v3"; +const MANAGED_POINTER: &str = "candidate-managed-temporal-v3.json"; +const BLOB_LIMIT: usize = 4 * 1024 * 1024; +const POINTER: &str = "candidate-temporal-v2.json"; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct TemporalCandidateReceipt { + pub schema: String, + pub artifact_kind: String, + pub owner_scope: String, + pub installation_id: String, + pub context_digest: String, + pub room_digest: String, + pub artifact_sha256: String, + pub input_policy_sha256: String, + pub parameter_count: usize, + pub diagnostic_only: bool, + pub physically_validated: bool, + pub qualification_granted: bool, +} +fn metadata(meta: &fs::Metadata, directory: bool) -> Result<(), String> { + if meta.file_type().is_symlink() + || (directory && !meta.is_dir()) + || (!directory && !meta.is_file()) + { + return Err("temporal_artifact_unsafe_path".into()); + } + #[cfg(unix)] + { + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + extern "C" { + fn geteuid() -> u32; + } + // SAFETY: geteuid has no arguments or memory side effects. + if meta.uid() != unsafe { geteuid() } + || meta.permissions().mode() & 0o7777 != if directory { 0o700 } else { 0o600 } + { + return Err("temporal_artifact_private_permissions".into()); + } + } + #[cfg(not(unix))] + { + return Err("temporal_artifact_private_permissions_unsupported".into()); + } + #[allow(unreachable_code)] + Ok(()) +} +pub(super) fn directory(path: &Path, create: bool) -> Result { + match fs::symlink_metadata(path) { + Ok(meta) => { + metadata(&meta, true)?; + return Ok(true); + } + Err(error) if error.kind() == io::ErrorKind::NotFound && !create => return Ok(false), + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(_) => return Err("temporal_artifact_directory_unavailable".into()), + } + let mut builder = fs::DirBuilder::new(); + #[cfg(unix)] + { + use std::os::unix::fs::DirBuilderExt; + builder.mode(0o700); + } + match builder.create(path) { + Ok(()) => {} + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => {} + Err(_) => return Err("temporal_artifact_directory_unavailable".into()), + } + metadata( + &fs::symlink_metadata(path).map_err(|_| "temporal_artifact_directory_unavailable")?, + true, + )?; + Ok(true) +} +// Only fixed callers select 64KiB pointer or 4MiB temporal-codec limits. This is +// not a replacement/global enlargement of companion::private_read. +pub(super) fn bounded_read(path: &Path, limit: usize) -> Result, String> { + let check = |meta: &fs::Metadata| -> Result<(), String> { + metadata(meta, false)?; + if meta.len() > limit as u64 { + return Err("temporal_artifact_size_limit".into()); + } + Ok(()) + }; + check(&fs::symlink_metadata(path).map_err(|_| "temporal_artifact_unavailable")?)?; + let mut options = fs::OpenOptions::new(); + options.read(true); + #[cfg(target_os = "macos")] + { + use std::os::unix::fs::OpenOptionsExt; + options.custom_flags(0x100); + } + #[cfg(target_os = "linux")] + { + use std::os::unix::fs::OpenOptionsExt; + options.custom_flags(0x20000); + } + #[cfg(not(any(target_os = "macos", target_os = "linux")))] + { + return Err("temporal_artifact_nofollow_unsupported".into()); + } + let file = options + .open(path) + .map_err(|_| "temporal_artifact_unavailable")?; + check( + &file + .metadata() + .map_err(|_| "temporal_artifact_unavailable")?, + )?; + let mut bytes = Vec::new(); + file.take((limit + 1) as u64) + .read_to_end(&mut bytes) + .map_err(|_| "temporal_artifact_read_failed")?; + if bytes.len() > limit { + return Err("temporal_artifact_size_limit".into()); + } + Ok(bytes) +} +pub(super) fn immutable(path: &Path, bytes: &[u8]) -> Result<(), String> { + immutable_with_io(path,bytes,&mut |_|Ok(())) +} +// Same body, with a bounded IO-stage seam used by the C4C current job. A +// callback cannot create source/result authority or choose paths/content. +pub(super) fn immutable_with_io(path:&Path,bytes:&[u8],io:&mut dyn FnMut(&'static str)->Result<(),String>)->Result<(),String>{ + if bytes.len() > BLOB_LIMIT { + return Err("temporal_artifact_size_limit".into()); + } + if regular_or_missing(path)? { + return if bounded_read(path, BLOB_LIMIT)? == bytes { + Ok(()) + } else { + Err("temporal_artifact_immutable_mismatch".into()) + }; + } + let parent = path + .parent() + .ok_or("temporal_artifact_directory_unavailable")?; + let stage = parent.join(format!("temporal-stage-{}", crate::companion::secret())); + atomic_private_write(&stage, bytes).map_err(|_| "temporal_artifact_stage_failed")?; + let result = fs::hard_link(&stage, path); + let cleanup = fs::remove_file(&stage); + match result { + Ok(()) => {} + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => { + if bounded_read(path, BLOB_LIMIT)? != bytes { + return Err("temporal_artifact_immutable_mismatch".into()); + } + } + Err(_) => return Err("temporal_artifact_install_failed".into()), + } + cleanup.map_err(|_| "temporal_artifact_stage_cleanup_failed")?; + io("before:directory_fsync")?; + fs::File::open(parent) + .and_then(|file| file.sync_all()) + .map_err(|_| "temporal_artifact_sync_failed")?; + Ok(()) +} +/// Re-establish a failed/ambiguous immutable metadata durability barrier. Equal +/// existing bytes alone do not prove an earlier file/directory sync succeeded. +pub(super) fn recertify_immutable(path:&Path,expected:&[u8],limit:usize,io:&mut dyn FnMut(&'static str)->Result<(),String>)->Result<(),String>{ + if expected.len()>limit||bounded_read(path,limit)?!=expected{return Err("temporal_artifact_immutable_mismatch".into());} + let mut options=fs::OpenOptions::new();options.read(true); + #[cfg(target_os="macos")] {use std::os::unix::fs::OpenOptionsExt;options.custom_flags(0x100);} + #[cfg(target_os="linux")] {use std::os::unix::fs::OpenOptionsExt;options.custom_flags(0x20000);} + #[cfg(not(any(target_os="macos",target_os="linux")))] {return Err("temporal_artifact_nofollow_unsupported".into());} + let file=options.open(path).map_err(|_|"temporal_artifact_unavailable")?;metadata(&file.metadata().map_err(|_|"temporal_artifact_unavailable")?,false)?; + io("before:file_fsync")?;file.sync_all().map_err(|_|"temporal_artifact_sync_failed")?; + let parent=path.parent().ok_or("temporal_artifact_directory_unavailable")?;directory(parent,false)?; + io("before:directory_fsync")?;fs::File::open(parent).and_then(|f|f.sync_all()).map_err(|_|"temporal_artifact_sync_failed")?; + if bounded_read(path,limit)?!=expected{return Err("temporal_artifact_immutable_mismatch".into());}Ok(()) +} + +impl ArtifactStore { + pub(super) fn temporal_directory( + &self, + context: &CaptureContext, + create: bool, + ) -> Result, String> { + self.verify_owner()?; + self.validate_context(context)?; + let kind = self.root.join(DIRECTORY); + if !directory(&kind, create)? { + return Ok(None); + } + let path = kind.join(context.digest()); + if !directory(&path, create)? { + return Ok(None); + } + Ok(Some(path)) + } + fn temporal_receipt(&self, model: &TemporalModel, bytes: &[u8]) -> TemporalCandidateReceipt { + TemporalCandidateReceipt { + schema: SCHEMA.into(), + artifact_kind: model.artifact_kind().into(), + owner_scope: self.owner_scope.clone(), + installation_id: self.installation_id.clone(), + context_digest: model.context().digest(), + room_digest: model.context().room_digest.clone(), + artifact_sha256: digest(bytes), + input_policy_sha256: model.plan_sha256().into(), + parameter_count: model.parameter_count(), + diagnostic_only: true, + physically_validated: false, + qualification_granted: false, + } + } + fn validate_temporal_receipt( + &self, + receipt: &TemporalCandidateReceipt, + context: &CaptureContext, + ) -> Result<(), String> { + self.validate_context(context)?; + if receipt.schema != SCHEMA + || !matches!(receipt.artifact_kind.as_str(), KIND | MANAGED_KIND) + || receipt.owner_scope != self.owner_scope + || receipt.installation_id != self.installation_id + || receipt.context_digest != context.digest() + || receipt.room_digest != context.room_digest + || !hex_digest(&receipt.artifact_sha256) + || !hex_digest(&receipt.input_policy_sha256) + || !(1..=512).contains(&receipt.parameter_count) + || !receipt.diagnostic_only + || receipt.physically_validated + || receipt.qualification_granted + { + return Err("temporal_artifact_receipt_mismatch".into()); + } + Ok(()) + } + fn temporal_read_model( + &self, + path: &Path, + receipt: &TemporalCandidateReceipt, + context: &CaptureContext, + ) -> Result { + self.validate_temporal_receipt(receipt, context)?; + let bytes = bounded_read( + &path.join(format!("{}.json", receipt.artifact_sha256)), + BLOB_LIMIT, + )?; + if digest(&bytes) != receipt.artifact_sha256 { + return Err("temporal_artifact_hash_mismatch".into()); + } + let model = TemporalModel::import(&bytes)?; + if model.context() != context || self.temporal_receipt(&model, &model.export()?) != *receipt + { + return Err("temporal_artifact_context_or_policy_mismatch".into()); + } + Ok(model) + } + fn temporal_read_pointer( + &self, + path: &Path, + context: &CaptureContext, + ) -> Result, String> { + self.temporal_read_pointer_kind(path, context, KIND) + } + fn temporal_read_pointer_kind( + &self, + path: &Path, + context: &CaptureContext, + kind: &str, + ) -> Result, String> { + let pointer = path.join(match kind { + KIND => POINTER, + MANAGED_KIND => MANAGED_POINTER, + _ => return Err("temporal_artifact_kind_unsupported".into()), + }); + if !regular_or_missing(&pointer)? { + return Ok(None); + } + let receipt: TemporalCandidateReceipt = + serde_json::from_slice(&bounded_read(&pointer, LIMIT)?) + .map_err(|_| "temporal_artifact_pointer_invalid")?; + if receipt.artifact_kind != kind { + return Err("temporal_artifact_pointer_kind_changed".into()); + } + self.temporal_read_model(path, &receipt, context)?; + Ok(Some(receipt)) + } + /// Diagnostic persistence only; it cannot create managed proof or selection. + pub fn save_temporal_candidate( + &self, + model: &TemporalModel, + ) -> Result { + self.validate_context(model.context())?; + let bytes = model.export()?; + let receipt = self.temporal_receipt(model, &bytes); + self.validate_temporal_receipt(&receipt, model.context())?; + let pointer = encode(&receipt)?; + let path = self + .temporal_directory(model.context(), true)? + .ok_or("temporal_artifact_directory_missing")?; + self.temporal_read_pointer_kind(&path, model.context(), model.artifact_kind())?; + immutable( + &path.join(format!("{}.json", receipt.artifact_sha256)), + &bytes, + )?; + self.temporal_read_model(&path, &receipt, model.context())?; + let pointer_name = if model.artifact_kind() == KIND { + POINTER + } else if model.artifact_kind() == MANAGED_KIND { + MANAGED_POINTER + } else { + return Err("temporal_artifact_kind_unsupported".into()); + }; + atomic_private_write(&path.join(pointer_name), &pointer) + .map_err(|_| "temporal_artifact_pointer_write_failed")?; + if bounded_read(&path.join(pointer_name), LIMIT)? != pointer { + return Err("temporal_artifact_pointer_readback_failed".into()); + } + self.temporal_read_pointer_kind(&path, model.context(), model.artifact_kind())?; + Ok(receipt) + } + pub fn temporal_candidate_receipt( + &self, + context: &CaptureContext, + ) -> Result, String> { + let Some(path) = self.temporal_directory(context, false)? else { + return Ok(None); + }; + self.temporal_read_pointer(&path, context) + } + pub fn load_temporal_candidate( + &self, + context: &CaptureContext, + ) -> Result, String> { + let Some(path) = self.temporal_directory(context, false)? else { + return Ok(None); + }; + let Some(receipt) = self.temporal_read_pointer(&path, context)? else { + return Ok(None); + }; + self.temporal_read_model(&path, &receipt, context).map(Some) + } + pub fn load_managed_temporal_candidate( + &self, + context: &CaptureContext, + ) -> Result, String> { + let Some(path) = self.temporal_directory(context, false)? else { + return Ok(None); + }; + let Some(receipt) = self.temporal_read_pointer_kind(&path, context, MANAGED_KIND)? else { + return Ok(None); + }; + self.temporal_read_model(&path, &receipt, context).map(Some) + } + pub fn load_exact_temporal_candidate( + &self, + context: &CaptureContext, + artifact_sha256: &str, + ) -> Result { + if !hex_digest(artifact_sha256) { + return Err("temporal_artifact_invalid_hash".into()); + } + let path = self + .temporal_directory(context, false)? + .ok_or("temporal_artifact_directory_missing")?; + let bytes = bounded_read(&path.join(format!("{artifact_sha256}.json")), BLOB_LIMIT)?; + if digest(&bytes) != artifact_sha256 { + return Err("temporal_artifact_hash_mismatch".into()); + } + let model = TemporalModel::import(&bytes)?; + if model.context() != context { + return Err("temporal_artifact_context_mismatch".into()); + } + Ok(model) + } +} +#[cfg(all(test, unix))] +#[path = "temporal_artifact_store_tests.rs"] +mod tests; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_artifact_store_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_artifact_store_tests.rs new file mode 100644 index 0000000000..3f1e9bbabc --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_artifact_store_tests.rs @@ -0,0 +1,544 @@ +//! SYNTHETIC native inputs + actual fitter/filesystem; no runtime or user data. +use super::*; +use crate::room_learner::{ + capture::{CaptureSession, PhaseRequest, RawSourceFrame}, + native::{canonicalize_esp32, CanonicalPacket, NativePacketInput}, + observation::*, + selection::{SelectionEnvelope, SelectionReceipt}, + temporal_acquisition::{ExperimentalPlanId, ReceiverCadenceBand, WindowAdmission}, + temporal_model_input::TemporalTrainingSession, + training::FitConfig, +}; +use std::{collections::BTreeMap, sync::OnceLock}; +use wifi_densepose_hardware::{Adr018Flags, PpduType}; +fn context() -> CaptureContext { + CaptureContext { + owner_scope: "a".repeat(64), + installation_id: "synthetic-temporal-store".into(), + room_digest: "synthetic-room".into(), + revision: 1, + sources: (1..=2) + .map(|source_id| SourceSpec { + source_id, + device_id: format!("esp32-node-{source_id}"), + grid_id: "ant1-bins64-ppdu0-usable-drop2-linear-v2".into(), + feature_count: 56, + signal_kind: ruview_unified::RfModality::WifiCsi, + center_frequency_hz: 2_437_000_000, + bandwidth_hz: 20_000_000, + link_id: format!("native-{source_id}"), + link_index: 0, + geometry: None, + }) + .collect(), + } +} +fn packet(source: u8, sequence: u32, time: u64, label: RoomLabel, row: usize) -> CanonicalPacket { + let gain = match label { + RoomLabel::Empty => 3, + RoomLabel::AdjacentOnly => 4, + RoomLabel::OccupiedStatic => 8, + RoomLabel::OccupiedMoving => 10 + (row % 2) as u8, + RoomLabel::Unknown => 8, + }; + let bytes = (0..64) + .flat_map(|bin| [gain + source + (bin % 2) as u8, 4]) + .collect::>(); + canonicalize_esp32(NativePacketInput { + node_id: source, + n_antennas: 1, + raw_bins: 64, + frequency_hz: 2_437_000_000, + ppdu_type: PpduType::HtLegacy, + sequence, + received_at_ns: time, + receiver_sample_rate_hz: 33., + iq_bytes: &bytes, + flags: Adr018Flags::default(), + provenance: CaptureProvenance::Synthetic, + }) + .unwrap() +} +fn fitted() -> TemporalModel { + static BYTES: OnceLock> = OnceLock::new(); + TemporalModel::import(BYTES.get_or_init(|| { + let mut session = TemporalTrainingSession::new( + context(), + ExperimentalPlanId::Native2000msSlots2, + &[ + (1, ReceiverCadenceBand::Hz30To36), + (2, ReceiverCadenceBand::Hz30To36), + ], + "synthetic-store-fit".into(), + FitConfig { + epochs: 2, + ..Default::default() + }, + ) + .unwrap(); + let labels = [ + RoomLabel::Empty, + RoomLabel::OccupiedStatic, + RoomLabel::OccupiedMoving, + RoomLabel::AdjacentOnly, + ]; + let mut sequence = 0; + for window in 0..16usize { + let target = 10_000_000_000 + window as u64 * 3_000_000_000; + session + .begin_window(target, target + 3_000_000_000) + .unwrap(); + let label = labels[window / 4]; + let mut complete = false; + 'rows: for row in 0..256usize { + sequence += 1; + for source in 1..=2 { + let at = target + + row as u64 * 1_000_000_000u64.div_ceil(33) + + u64::from(source) * 1000; + if session + .ingest(&packet(source, sequence, at, label, row)) + .unwrap() + == WindowAdmission::Ready + { + session.finish_window(at).unwrap(); + complete = true; + break 'rows; + } + } + } + assert!(complete); + } + let (model, report) = session.fit().unwrap(); + assert!(report.backbone_delta_l2 > 0.); + model.export().unwrap() + })) + .unwrap() +} +fn store(root: &Path) -> ArtifactStore { + let c = context(); + ArtifactStore::new( + &root.join(&c.owner_scope).join("SensingServer"), + &c.owner_scope, + &c.installation_id, + ) + .unwrap() +} +fn live(model: &TemporalModel) -> serde_json::Value { + let mut input = model.live_input("synthetic-fresh-live".into()).unwrap(); + let target = 1_000_000_000_000; + input.begin_window(target, target + 3_000_000_000).unwrap(); + for row in 0..256usize { + for source in 1..=2 { + let at = target + row as u64 * 1_000_000_000u64.div_ceil(33) + u64::from(source) * 1000; + if input + .ingest(&packet( + source, + 100_000 + row as u32, + at, + RoomLabel::Unknown, + row, + )) + .unwrap() + == WindowAdmission::Ready + { + return serde_json::to_value(model.predict(input.finish(at).unwrap(), at).unwrap()) + .unwrap(); + } + } + } + panic!("missing complete native horizon") +} +fn files(root: &Path) -> BTreeMap> { + fn visit(root: &Path, at: &Path, out: &mut BTreeMap>) { + for entry in fs::read_dir(at).unwrap() { + let path = entry.unwrap().path(); + if path.is_dir() { + visit(root, &path, out); + } else { + out.insert( + path.strip_prefix(root).unwrap().to_path_buf(), + fs::read(path).unwrap(), + ); + } + } + } + let mut result = BTreeMap::new(); + visit(root, root, &mut result); + result +} +fn blob_path(store: &ArtifactStore, receipt: &TemporalCandidateReceipt) -> PathBuf { + store + .root + .join(DIRECTORY) + .join(&receipt.context_digest) + .join(format!("{}.json", receipt.artifact_sha256)) +} +#[test] +fn actual_native_fit_whole_envelope_private_persist_coldload_same_inference() { + let root = tempfile::tempdir().unwrap(); + let storage = store(root.path()); + let model = fitted(); + let bytes = model.export().unwrap(); + assert!(bytes.len() > LIMIT); + assert!(bytes.len() < BLOB_LIMIT); + let before = serde_json::to_vec(&storage.catalog().unwrap()).unwrap(); + let receipt = storage.save_temporal_candidate(&model).unwrap(); + assert_eq!(receipt.artifact_sha256, digest(&bytes)); + assert_eq!(receipt.input_policy_sha256, model.plan_sha256()); + assert!(receipt.diagnostic_only); + assert!(!receipt.qualification_granted && !receipt.physically_validated); + let path = blob_path(&storage, &receipt); + assert_eq!(fs::read(&path).unwrap(), bytes); + assert!( + private_read(&path).is_err(), + "global64KiB reader was broadened" + ); + assert_eq!( + serde_json::to_vec(&storage.catalog().unwrap()).unwrap(), + before + ); + assert!(storage.load_candidate(&context()).unwrap().is_none()); + assert!(storage.load_selection().unwrap().is_none()); + let cold = store(root.path()); + let loaded = cold.load_temporal_candidate(&context()).unwrap().unwrap(); + assert_eq!(loaded.export().unwrap(), bytes); + assert_eq!(live(&model), live(&loaded)); + assert_eq!( + cold.temporal_candidate_receipt(&context()).unwrap(), + Some(receipt.clone()) + ); + assert_eq!( + cold.load_exact_temporal_candidate(&context(), &receipt.artifact_sha256) + .unwrap() + .export() + .unwrap(), + bytes + ); + // Whole codec includes private weights/policy AND per-row receive/frame/sequence + // lineage. It is NOT aggregate-only; raw IQ/CSI sample arrays are absent. + let value: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + assert!(value["payload"]["model_json"].is_string()); + assert!(value["payload"]["evidence"][0]["sources"][0]["times_ns"].is_array()); + assert!(value["payload"]["evidence"][0]["sources"][0]["frame_ids"].is_array()); + assert!(value["payload"]["evidence"][0]["sources"][0]["sequence_ids"].is_array()); + assert!(crate::room_learner::model::RoomModel::import(&bytes).is_err()); +} +fn altered(bytes: &[u8], case: &str) -> Vec { + let value: serde_json::Value = serde_json::from_slice(bytes).unwrap(); + let mut text = String::from_utf8(bytes.to_vec()).unwrap(); + match case { + "policy" => { + text = text.replacen("\"target_span_ms\":2000", "\"target_span_ms\":1000", 1); + } + "source" => { + text = text.replacen("\"bandwidth_hz\":20000000", "\"bandwidth_hz\":40000000", 1); + } + "frame" => { + let old = value["payload"]["evidence"][0]["sources"][0]["frame_ids"][0] + .as_str() + .unwrap(); + text = text.replacen( + &format!("\"frame_ids\":[\"{old}\""), + "\"frame_ids\":[\"0\"", + 1, + ); + let a = text.find("\"evidence\":").unwrap() + "\"evidence\":".len(); + let b = text.find(",\"model_sha256\"").unwrap(); + let hash = digest(text[a..b].as_bytes()); + text = text.replacen( + value["payload"]["input_evidence_sha256"].as_str().unwrap(), + &hash, + 1, + ); + } + _ => unreachable!(), + } + let start = text.find("\"payload\":").unwrap() + "\"payload\":".len(); + let hash = digest(text[start..text.len() - 1].as_bytes()); + text = text.replacen(value["payload_sha256"].as_str().unwrap(), &hash, 1); + text.into_bytes() +} +#[test] +fn policy_source_frame_and_expected_context_tampering_cannot_load() { + let root = tempfile::tempdir().unwrap(); + let storage = store(root.path()); + let model = fitted(); + let receipt = storage.save_temporal_candidate(&model).unwrap(); + let bytes = model.export().unwrap(); + let path = blob_path(&storage, &receipt); + let parent = path.parent().unwrap(); + for case in ["policy", "source", "frame"] { + let bad = altered(&bytes, case); + let hash = digest(&bad); + let target = parent.join(format!("{hash}.json")); + atomic_private_write(&target, &bad).unwrap(); + let error = storage + .load_exact_temporal_candidate(&context(), &hash) + .err() + .expect("tampered policy/source/frame accepted"); + assert!( + !error.contains("temporal_input_envelope_integrity"), + "negative was only noncanonical JSON: {error}" + ); + assert!( + !error.contains("temporal_artifact_hash_mismatch"), + "negative did not reach codec: {error}" + ); + } + let wrong_hash = "2".repeat(64); + atomic_private_write(&parent.join(format!("{wrong_hash}.json")), &bytes).unwrap(); + assert_eq!( + storage + .load_exact_temporal_candidate(&context(), &wrong_hash) + .err() + .unwrap(), + "temporal_artifact_hash_mismatch" + ); + let mut wrong = context(); + wrong.revision += 1; + assert!(storage + .load_exact_temporal_candidate(&wrong, &receipt.artifact_sha256) + .is_err()); + assert!(storage.load_temporal_candidate(&wrong).unwrap().is_none()); + assert_eq!( + storage.temporal_candidate_receipt(&context()).unwrap(), + Some(receipt) + ); + assert_eq!(fs::read(path).unwrap(), bytes); +} +#[test] +fn immutable_corruption_and_pointer_privilege_changes_are_preserved_and_rejected() { + let root = tempfile::tempdir().unwrap(); + let storage = store(root.path()); + let model = fitted(); + let receipt = storage.save_temporal_candidate(&model).unwrap(); + let path = blob_path(&storage, &receipt); + let original = fs::read(&path).unwrap(); + atomic_private_write(&path, b"synthetic corrupted immutable record").unwrap(); + let corrupt = fs::read(&path).unwrap(); + assert!(storage.save_temporal_candidate(&model).is_err()); + assert_eq!(fs::read(&path).unwrap(), corrupt); + assert!(storage.load_temporal_candidate(&context()).is_err()); + atomic_private_write(&path, &original).unwrap(); + let pointer = path.parent().unwrap().join(POINTER); + let mut forged = receipt.clone(); + forged.qualification_granted = true; + atomic_private_write(&pointer, &encode(&forged).unwrap()).unwrap(); + let pointer_bytes = fs::read(&pointer).unwrap(); + assert!(storage.load_temporal_candidate(&context()).is_err()); + assert!(storage.save_temporal_candidate(&model).is_err()); + assert_eq!(fs::read(&pointer).unwrap(), pointer_bytes); +} +#[test] +fn owner_installation_and_private_size_permissions_are_fail_closed() { + use std::os::unix::fs::PermissionsExt; + let root = tempfile::tempdir().unwrap(); + let storage = store(root.path()); + let model = fitted(); + let receipt = storage.save_temporal_candidate(&model).unwrap(); + let wrong = ArtifactStore::new( + &root.path().join("b".repeat(64)).join("SensingServer"), + &"b".repeat(64), + &context().installation_id, + ) + .unwrap(); + assert!(wrong.save_temporal_candidate(&model).is_err()); + assert!(wrong + .load_exact_temporal_candidate(&context(), &receipt.artifact_sha256) + .is_err()); + let install = ArtifactStore::new( + &storage.data_dir, + &context().owner_scope, + "foreign-installation", + ) + .unwrap(); + assert!(install.save_temporal_candidate(&model).is_err()); + let path = blob_path(&storage, &receipt); + fs::set_permissions(&path, fs::Permissions::from_mode(0o644)).unwrap(); + assert!(storage.load_temporal_candidate(&context()).is_err()); + assert_eq!( + fs::metadata(&path).unwrap().permissions().mode() & 0o777, + 0o644 + ); + fs::set_permissions(&path, fs::Permissions::from_mode(0o600)).unwrap(); + let oversized = path + .parent() + .unwrap() + .join(format!("{}.json", "1".repeat(64))); + let file = fs::File::create(&oversized).unwrap(); + fs::set_permissions(&oversized, fs::Permissions::from_mode(0o600)).unwrap(); + file.set_len((BLOB_LIMIT + 1) as u64).unwrap(); + assert_eq!( + storage + .load_exact_temporal_candidate(&context(), &"1".repeat(64)) + .err() + .unwrap(), + "temporal_artifact_size_limit" + ); + let pointer = path.parent().unwrap().join(POINTER); + let file = fs::File::create(&pointer).unwrap(); + fs::set_permissions(&pointer, fs::Permissions::from_mode(0o600)).unwrap(); + file.set_len((LIMIT + 1) as u64).unwrap(); + assert_eq!( + storage.load_temporal_candidate(&context()).err().unwrap(), + "temporal_artifact_size_limit" + ); +} +#[test] +fn blob_and_namespace_symlinks_never_follow_outside_owned_kind() { + use std::os::unix::fs::symlink; + let root = tempfile::tempdir().unwrap(); + let storage = store(root.path()); + let model = fitted(); + let receipt = storage.save_temporal_candidate(&model).unwrap(); + let path = blob_path(&storage, &receipt); + let outside = root.path().join("external-synthetic-blob"); + fs::rename(&path, &outside).unwrap(); + symlink(&outside, &path).unwrap(); + assert!(storage.load_temporal_candidate(&context()).is_err()); + assert!(storage.save_temporal_candidate(&model).is_err()); + assert_eq!(fs::read(&outside).unwrap(), model.export().unwrap()); + fs::remove_file(&path).unwrap(); + fs::rename(&outside, &path).unwrap(); + let namespace = storage.root.join(DIRECTORY); + let moved = storage.root.join("outside-kind-test"); + fs::rename(&namespace, &moved).unwrap(); + symlink(&moved, &namespace).unwrap(); + assert!(storage.load_temporal_candidate(&context()).is_err()); + assert!(storage.save_temporal_candidate(&model).is_err()); +} +fn v1_model_and_seed() -> ( + crate::room_learner::model::RoomModel, + crate::room_learner::managed_training::TrainingProofSeed, +) { + let mut capture = CaptureSession::new(context(), "synthetic-v1-preserved".into()).unwrap(); + for (index, label) in [ + RoomLabel::Empty, + RoomLabel::OccupiedStatic, + RoomLabel::OccupiedMoving, + RoomLabel::AdjacentOnly, + ] + .into_iter() + .enumerate() + { + let start = 1_000_000_000 + index as u64 * 13_000_000_000; + capture + .begin_phase( + PhaseRequest { + trial_id: format!("v1-trial-{index}"), + session_id: format!("v1-session-{index}"), + role: DataRole::Train, + label, + duration_ms: 12000, + lead_in_ms: 0, + conditions_confirmed: true, + }, + start, + ) + .unwrap(); + for quarter in 0..4u64 { + for sample in 0..8u32 { + for source in 1..=2u8 { + let values = vec![ + match label { + RoomLabel::Empty => 2., + RoomLabel::AdjacentOnly => 2.3, + _ => 6., + }; + 56 + ]; + capture + .ingest( + RawSourceFrame { + source_id: source, + grid_id: "ant1-bins64-ppdu0-usable-drop2-linear-v2", + frequency_hz: 2_437_000_000, + bandwidth_hz: 20_000_000, + sequence: index as u32 * 100 + quarter as u32 * 8 + sample + 1, + amplitudes: &values, + }, + start + + quarter * 3_000_000_000 + + 1_000_000 + + u64::from(sample) * 10_000_000 + + u64::from(source), + ) + .unwrap(); + } + } + } + capture.finish_phase(start + 12_000_000_000).unwrap(); + } + let seed = capture.managed_training_seed().unwrap(); + let (model, _) = crate::room_learner::model::RoomModel::fit( + &capture.validated_dataset().unwrap(), + FitConfig { + epochs: 2, + ..Default::default() + }, + ) + .unwrap(); + (model, seed) +} +#[test] +fn existing_v1_model_proof_fence_pointer_selection_and_catalog_bytes_do_not_change() { + let root = tempfile::tempdir().unwrap(); + let storage = store(root.path()); + let (v1, seed) = v1_model_and_seed(); + storage.save_managed_candidate(&v1, seed).unwrap(); + let receipt = SelectionReceipt { + schema: "ruview.room-learner.model-selection.v1".into(), + owner_scope: context().owner_scope, + installation_id: context().installation_id, + revision: 1, + selected_at_unix_ms: 1, + selected: None, + previous: None, + }; + let envelope = SelectionEnvelope { + receipt_sha256: crate::room_learner::qualification::hash(&receipt).unwrap(), + receipt, + }; + storage.save_selection(&envelope, 0).unwrap(); + let before = files(&storage.data_dir); + assert!(before + .keys() + .any(|p| p.to_string_lossy().contains("managed-training"))); + assert!(before.keys().any(|p| p.to_string_lossy().contains("fence"))); + let catalog = serde_json::to_vec(&storage.catalog().unwrap()).unwrap(); + let v1bytes = v1.export().unwrap(); + storage.save_temporal_candidate(&fitted()).unwrap(); + for (path, bytes) in before { + assert_eq!(fs::read(storage.data_dir.join(path)).unwrap(), bytes); + } + assert_eq!( + storage + .load_candidate(&context()) + .unwrap() + .unwrap() + .export() + .unwrap(), + v1bytes + ); + assert_eq!(storage.load_selection().unwrap(), Some(envelope)); + assert_eq!( + serde_json::to_vec(&storage.catalog().unwrap()).unwrap(), + catalog + ); +} +#[test] +fn reserved_temporal_kind_does_not_consume_v1_catalog_context_scan_budget() { + let root = tempfile::tempdir().unwrap(); + let storage = store(root.path()); + for index in 0..256u32 { + let path = storage.root.join(format!("{index:064x}")); + private_dir(&path).unwrap(); + } + let before = serde_json::to_vec(&storage.catalog().unwrap()).unwrap(); + directory(&storage.root.join(DIRECTORY), true).unwrap(); + assert_eq!( + serde_json::to_vec(&storage.catalog().unwrap()).unwrap(), + before + ); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_budget_profile_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_budget_profile_tests.rs new file mode 100644 index 0000000000..12e6565a2f --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_budget_profile_tests.rs @@ -0,0 +1,41 @@ +//! Synthetic profile/hash compatibility; the fresh constructor remains private. +use super::*; +use super::super::checkpoint; +const OLD_LITERAL:&str="native-receiver-whole-train-period-v1;all-selected-packets;existing-cadence-gap-arrival-skew;confirmed-durable-seals;no-fit-authority"; +#[test] +fn historical_constructor_and_seal_hash_keep_exact_bytes(){ + let old=BridgeState::new();assert_eq!(old.policy,OLD_LITERAL);assert_eq!(BRIDGE_POLICY,OLD_LITERAL); + let expected=format!("{{\"policy\":\"{OLD_LITERAL}\",\"disposition\":\"ready\",\"seals\":[],\"parts\":[]}}"); + assert_eq!(serde_json::to_vec(&old).unwrap(),expected.as_bytes()); + let seal=TrainSeal{episode:0,trial_id:"synthetic-trial".into(),context_sha256:"a".repeat(64),schedule_sha256:"b".repeat(64),row_prefix_sha256:"c".repeat(64),quality_sha256:"d".repeat(64),parts:vec![],confirmed_at_ns:1,previous_sha256:None,sha256:String::new()}; + // Independent frozen tuple golden, preserving the historical domain. + assert_eq!(seal.digest(TrainBudgetProfile::Historical329).unwrap(),"467e2740fd4a20ed6e09af36071a5b816522c20888936be1e976b3b8f4b4f4c1"); + assert_ne!(seal.digest(TrainBudgetProfile::FreshTrain343).unwrap(),seal.digest(TrainBudgetProfile::Historical329).unwrap()); +} +#[test] +fn generic_and_stored_historical_prefixes_remain329_and_byte_roundtrip(){ + let(_dir,_store,mut w)=super::super::tests::setup(); + for bridge in [None,Some(BridgeState::new())]{ + w.bridge=bridge; + let bytes=checkpoint::encode(&w).unwrap();let prefix=checkpoint::validated_prefix(&bytes,&w.descriptor,w.last_now_ns).unwrap(); + assert_eq!(prefix.transition_limit(),329); + let restored=checkpoint::decode(&bytes,&w.descriptor,w.last_now_ns).unwrap();assert_eq!(checkpoint::encode(&restored).unwrap(),bytes); + if w.bridge.is_some(){assert_eq!(prefix.policy_sha256(),hash(&(w.session.plan_sha256(),OLD_LITERAL)).unwrap());} + } +} +#[test] +fn fresh_train_profile343_is_explicit_and_cannot_upgrade_existing_generation(){ + let(_dir,_store,mut w)=super::super::tests::setup();w.bridge=Some(BridgeState::new()); + let old_bytes=checkpoint::encode(&w).unwrap();let old=checkpoint::validated_prefix(&old_bytes,&w.descriptor,w.last_now_ns).unwrap(); + w.bridge=Some(BridgeState::fresh_train_343()); + let new_bytes=checkpoint::encode(&w).unwrap();let fresh=checkpoint::validated_prefix(&new_bytes,&w.descriptor,w.last_now_ns).unwrap(); + assert_eq!(old.transition_limit(),329);assert_eq!(fresh.transition_limit(),343);assert_ne!(old.policy_sha256(),fresh.policy_sha256()); + assert!(checkpoint::validate_extension(&old,&fresh).is_err()); + assert_eq!(BridgeState::new().policy,OLD_LITERAL,"generic/C4 constructor must not flip"); +} +#[test] +fn unknown_stored_profile_and_injected_budget_never_choose_larger_capacity(){ + let(_dir,_store,mut w)=super::super::tests::setup();w.bridge=Some(BridgeState::new()); + w.bridge.as_mut().unwrap().policy="unknown-budget-profile".into();let bytes=checkpoint::encode(&w).unwrap();assert!(checkpoint::validated_prefix(&bytes,&w.descriptor,w.last_now_ns).is_err()); + w.bridge=Some(BridgeState::fresh_train_343());let bytes=checkpoint::encode(&w).unwrap();let mut v:serde_json::Value=serde_json::from_slice(&bytes).unwrap();v["payload"]["transition_limit"]=serde_json::json!(512);assert!(checkpoint::decode(&serde_json::to_vec(&v).unwrap(),&w.descriptor,w.last_now_ns).is_err()); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_capture_quality.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_capture_quality.rs new file mode 100644 index 0000000000..db05dcddcd --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_capture_quality.rs @@ -0,0 +1,1272 @@ +//! Private Runtime capture facade. Whole-period receiver quality is engineering +//! admission only: never source-clock precision, teacher truth or qualification. +//! Seals alone grant no fit authority; the private owner must separately consume +//! the complete durable producer prefix into its opaque fit-only admission. +use super::*; +use crate::room_learner::temporal_acquisition::WindowInputPolicy; +use serde_json::{json, Value}; + +pub(super) const BRIDGE_POLICY: &str = "native-receiver-whole-train-period-v1;all-selected-packets;existing-cadence-gap-arrival-skew;confirmed-durable-seals;no-fit-authority"; +// Explicit NEW Train genesis only. The generic constructor and old literal +// remain historical329; restore never upgrades a stored generation. +pub(super) const FRESH_TRAIN_343_POLICY:&str="native-receiver-whole-train-period-v2;all-selected-packets;existing-cadence-gap-arrival-skew;confirmed-durable-seals;train-budget343;no-fit-authority"; +pub(super) const OPERATOR_EIGHT_POLICY:&str="native-receiver-whole-operator-period-v1;reserved-validation-not-training;confirmed-durable-seals;no-fit-authority"; +#[derive(Clone,Copy,Debug,PartialEq,Eq)] +pub(super) enum TrainBudgetProfile { Historical329, FreshTrain343, OperatorEightStep } +impl TrainBudgetProfile { + pub(super) fn from_stored(policy:&str)->Result{match policy{ + BRIDGE_POLICY=>Ok(Self::Historical329),FRESH_TRAIN_343_POLICY=>Ok(Self::FreshTrain343),OPERATOR_EIGHT_POLICY=>Ok(Self::OperatorEightStep), + _=>Err("temporal_train_profile_unknown".into()), + }} + pub(super) fn policy(self)->&'static str{match self{Self::Historical329=>BRIDGE_POLICY,Self::FreshTrain343=>FRESH_TRAIN_343_POLICY,Self::OperatorEightStep=>OPERATOR_EIGHT_POLICY}} + pub(super) fn transition_limit(self,schedule:&ManagedScheduleContract)->Result{ + let episodes=schedule.counts().train_episodes; + let old=schedule.counts().train_windows.checked_add(2*128).and_then(|n|n.checked_add(episodes)).and_then(|n|n.checked_add(2)).ok_or("temporal_transition_capacity")?; + match self{Self::Historical329=>Ok(old),Self::FreshTrain343=>old.checked_add(2*episodes).ok_or("temporal_transition_capacity".into()),Self::OperatorEightStep=>Ok(schedule.counts().total_windows+2*(128+schedule.episodes().len())+schedule.episodes().len()+4)} + } +} + +#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub(super) enum Disposition { + Ready, + Observing, + Paused, + NeedsConfirmation, + Complete, +} +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub(super) struct SourceQuality { + pub(super) source_id: u8, + pub(super) first_ns: u64, + pub(super) last_ns: u64, + pub(super) first_sequence: u64, + pub(super) last_sequence: u64, + pub(super) packets: u64, + pub(super) maximum_gap_ns: u64, + pub(super) minimum_interval_ns: u64, +} +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub(super) struct QualityPart { + pub(super) mapping: ObservedSegmentMap, + pub(super) observed_end_ns: u64, + pub(super) sources: Vec, +} +impl QualityPart { + fn observe(&mut self, p: &CanonicalPacket) -> Result<(), String> { + let t = p.native_frame.timestamp_ns; + if let Some(s) = self.sources.iter_mut().find(|s| s.source_id == p.source_id) { + let gap = t.checked_sub(s.last_ns).ok_or("temporal_quality_clock")?; + s.maximum_gap_ns = s.maximum_gap_ns.max(gap); + s.minimum_interval_ns = s.minimum_interval_ns.min(gap); + s.last_ns = t; + s.last_sequence = u64::from(p.sequence); + s.packets = s.packets.checked_add(1).ok_or("temporal_quality_count")?; + } else { + self.sources.push(SourceQuality { + source_id: p.source_id, + first_ns: t, + last_ns: t, + first_sequence: u64::from(p.sequence), + last_sequence: u64::from(p.sequence), + packets: 1, + maximum_gap_ns: 0, + minimum_interval_ns: u64::MAX, + }); + self.sources.sort_by_key(|s| s.source_id); + } + Ok(()) + } + fn validate( + &self, + native: &WindowInputPolicy, + bound: &BoundEpisode, + now: u64, + ) -> Result<(), String> { + let start = self.mapping.observed_start_ns; + if self.observed_end_ns < start + || self.observed_end_ns > bound.episode().target_active_recording_ms * 1_000_000 + || self.mapping.receiver_start_ns > now + || self.sources.len() > native.source_policies().len() + { + return Err("temporal_quality_part_bounds".into()); + } + if self.observed_end_ns == start && !self.sources.is_empty() { + return Err("temporal_quality_uncommitted_rows".into()); + } + let mut prior = None; + for s in &self.sources { + let policy = native + .source_policies() + .iter() + .find(|p| p.source_id == s.source_id) + .ok_or("temporal_quality_source")?; + if prior.is_some_and(|p| p >= s.source_id) + || s.packets == 0 + || s.packets > 1_000_000 + || s.first_ns < self.mapping.receiver_start_ns + || s.last_ns < s.first_ns + || s.last_ns > now + || s.first_ns - self.mapping.receiver_start_ns + > native.max_first_packet_delay_ms() * 1_000_000 + || s.last_sequence < s.first_sequence + || s.last_sequence - s.first_sequence + 1 < s.packets + || (s.packets == 1 + && (s.first_ns != s.last_ns + || s.maximum_gap_ns != 0 + || s.minimum_interval_ns != u64::MAX)) + || (s.packets > 1 + && (s.last_ns == s.first_ns + || s.minimum_interval_ns < policy.min_packet_interval_ns + || s.maximum_gap_ns > policy.max_gap_ns + || s.minimum_interval_ns > s.maximum_gap_ns)) + { + return Err("temporal_quality_source_bounds".into()); + } + if self.observed_end_ns > start { + let end = self + .mapping + .receiver_start_ns + .checked_add(self.observed_end_ns - start) + .ok_or("temporal_quality_time")?; + let (lo, hi) = policy.receiver_cadence.rate_bounds_millihz(); + let intervals = s.packets.saturating_sub(1) as u128; + let span = (s.last_ns - s.first_ns) as u128; + // The exact integer packet/rate bounds already declared by the + // native policy, including its integer interval representation. + if s.packets < 2 + || s.last_ns < end + || intervals < span * u128::from(lo) / 1_000_000_000_000u128 + || span < intervals * u128::from(1_000_000_000_000u64 / hi) + { + return Err("temporal_whole_period_cadence".into()); + } + } + prior = Some(s.source_id); + } + if self.observed_end_ns > start + && self + .sources + .iter() + .map(|s| s.source_id) + .ne(native.source_policies().iter().map(|s| s.source_id)) + { + return Err("temporal_quality_required_sources".into()); + } + if let (Some(lo), Some(hi)) = ( + self.sources.iter().map(|s| s.first_ns).min(), + self.sources.iter().map(|s| s.first_ns).max(), + ) { + if hi - lo > native.max_pairwise_receiver_skew_ms() * 1_000_000 { + return Err("temporal_quality_source_skew".into()); + } + } + Ok(()) + } +} +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub(super) struct TrainSeal { + pub(super) episode: usize, + pub(super) trial_id: String, + pub(super) context_sha256: String, + pub(super) schedule_sha256: String, + pub(super) row_prefix_sha256: String, + pub(super) quality_sha256: String, + pub(super) parts: Vec, + pub(super) confirmed_at_ns: u64, + pub(super) previous_sha256: Option, + pub(super) sha256: String, +} +impl TrainSeal { + fn digest(&self,profile:TrainBudgetProfile) -> Result { + hash(&( + profile.policy(), + self.episode, + &self.trial_id, + &self.context_sha256, + &self.schedule_sha256, + &self.row_prefix_sha256, + &self.quality_sha256, + &self.parts, + self.confirmed_at_ns, + &self.previous_sha256, + )) + } +} +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub(super) struct BridgeState { + pub(super) policy: String, + pub(super) disposition: Disposition, + pub(super) seals: Vec, + pub(super) parts: Vec, +} +impl BridgeState { + pub(super) fn new() -> Self { + Self { + policy: BRIDGE_POLICY.into(), + disposition: Disposition::Ready, + seals: vec![], + parts: vec![], + } + } + pub(super) fn operator_eight_step()->Self{Self{policy:OPERATOR_EIGHT_POLICY.into(),..Self::new()}} + pub(super) fn fresh_train_343()->Self{ + Self{policy:FRESH_TRAIN_343_POLICY.into(),..Self::new()} + } + pub(super) fn profile(&self)->Result{TrainBudgetProfile::from_stored(&self.policy)} + pub(super) fn validate( + &self, + schedule: &ManagedScheduleContract, + run: &str, + rows: &[CaptureCohort], + evidence: &[WindowEvidence], + ledger: &Ledger, + episode: usize, + mapping: Option<&ObservedSegmentMap>, + now: u64, + ) -> Result<(), String> { + self.validate_with_prefixes(schedule,run,rows,evidence,ledger,episode,mapping,now,None) + } + pub(super) fn validate_full_run_with_prefixes( + &self,schedule:&ManagedScheduleContract,run:&str,rows:&[CaptureCohort],evidence:&[WindowEvidence],ledger:&Ledger,episode:usize,mapping:Option<&ObservedSegmentMap>,now:u64,digests:&prefix_digests::CanonicalPrefixDigests, + )->Result<(),String>{ + if self.policy!=if schedule.is_operator_eight_step(){OPERATOR_EIGHT_POLICY}else{BRIDGE_POLICY}{return Err("temporal_full_run_train_profile_changed".into());} + self.validate_with_prefixes(schedule,run,rows,evidence,ledger,episode,mapping,now,Some(digests)) + } + fn validate_with_prefixes( + &self,schedule:&ManagedScheduleContract,run:&str,rows:&[CaptureCohort],evidence:&[WindowEvidence],ledger:&Ledger,episode:usize,mapping:Option<&ObservedSegmentMap>,now:u64,digests:Option<&prefix_digests::CanonicalPrefixDigests>, + )->Result<(),String>{ + let train = if schedule.is_operator_eight_step() { schedule.episodes().len() } else { + schedule.episodes().iter().take_while(|e| e.role == DataRole::Train).count() + }; + let profile=self.profile()?; + if self.seals.len() > train + || episode != self.seals.len() + || episode > train + || self.parts.len() > 128 + || self.seals.iter().map(|s| s.parts.len()).sum::() + self.parts.len() + > 128 + train + { + return Err("temporal_bridge_phase_order".into()); + } + let mut count = 0; + let mut previous = None; + let mut last_confirm = 0; + for (i, s) in self.seals.iter().enumerate() { + count += schedule.episodes()[i].slots.len(); + let bound = schedule.bind_episode_start(run, i, 0, 0, None)?; + if s.episode != i + || s.trial_id != bound.trial_id() + || s.context_sha256 != schedule.context().digest() + || s.schedule_sha256 != schedule.sha256() + || count > rows.len() + || s.confirmed_at_ns < last_confirm + || s.confirmed_at_ns > now + || s.previous_sha256 != previous + || s.sha256 != s.digest(profile)? + || s.quality_sha256 != hash(&s.parts)? + || s.row_prefix_sha256 != match digests{ + Some(digests)=>digests.at(count)?.to_owned(), + None=>hash(&(&rows[..count], &evidence[..count], &ledger.spans[..count]))?, + } + { + return Err("temporal_train_seal_binding".into()); + } + validate_parts( + &s.parts, + schedule, + run, + i, + Some(schedule.episodes()[i].target_active_recording_ms * 1_000_000), + s.confirmed_at_ns, + )?; + validate_part_lineage(&s.parts, i, ledger, rows, None)?; + previous = Some(s.sha256.clone()); + last_confirm = s.confirmed_at_ns; + } + if episode == train { + if self.disposition != Disposition::Complete + || !self.parts.is_empty() + || mapping.is_some() + || rows.len() != if schedule.is_operator_eight_step(){schedule.counts().total_windows}else{schedule.counts().train_windows} + { + return Err("temporal_bridge_complete_state".into()); + } + } else { + if rows.len() < count || rows.len() > count + schedule.episodes()[episode].slots.len() { + return Err("temporal_bridge_row_phase".into()); + } + validate_parts(&self.parts, schedule, run, episode, None, now)?; + validate_part_lineage(&self.parts, episode, ledger, rows, mapping)?; + if let Some(m) = mapping { + if self.parts.last().map(|p| &p.mapping) != Some(m) { + return Err("temporal_quality_current_mapping".into()); + } + } else if !self.parts.is_empty() || rows.len() != count { + return Err("temporal_bridge_mapping_missing".into()); + } + let accepted = ledger + .spans + .last() + .filter(|s| s.episode == episode) + .map_or(0, |s| s.observed_end_ns); + if self.parts.last().map_or(0, |p| p.observed_end_ns) != accepted { + return Err("temporal_quality_durable_frontier".into()); + } + if self.disposition == Disposition::Ready + && (mapping.is_some() || !self.parts.is_empty() || rows.len() != count) + { + return Err("temporal_bridge_ready_with_active_data".into()); + } + if matches!( + self.disposition, + Disposition::Observing | Disposition::NeedsConfirmation | Disposition::Paused + ) && mapping.is_none() + { + return Err("temporal_bridge_phase_without_mapping".into()); + } + if self.disposition == Disposition::Complete { + return Err("temporal_bridge_early_complete".into()); + } + if self.disposition == Disposition::NeedsConfirmation + && accepted != schedule.episodes()[episode].target_active_recording_ms * 1_000_000 + { + return Err("temporal_bridge_confirmation_before_end".into()); + } + } + Ok(()) + } +} +pub(super) fn validate_part_lineage( + parts: &[QualityPart], + episode: usize, + ledger: &Ledger, + rows: &[CaptureCohort], + current: Option<&ObservedSegmentMap>, +) -> Result<(), String> { + for part in parts { + let spans = ledger + .spans + .iter() + .enumerate() + .filter(|(_, s)| s.episode == episode && s.mapping == part.mapping) + .collect::>(); + let frontier = spans + .last() + .map_or(part.mapping.observed_start_ns, |(_, s)| s.observed_end_ns); + let known = !spans.is_empty() + || current == Some(&part.mapping) + || ledger.gaps.iter().any(|g| { + g.episode == episode + && (g.new_mapping == part.mapping + || g.genesis + .as_ref() + .is_some_and(|v| v.previous_mapping == part.mapping)) + }); + if !known || part.observed_end_ns != frontier { + return Err("temporal_quality_lineage_changed".into()); + } + for source in &part.sources { + let mut retained = 0u64; + for (index, _) in &spans { + let row = rows.get(*index).ok_or("temporal_quality_row_missing")?; + let view = row + .views + .iter() + .find(|v| v.source_id == source.source_id) + .ok_or("temporal_quality_row_source")?; + retained = retained + .checked_add(view.samples.len() as u64) + .ok_or("temporal_quality_count")?; + if source.first_ns > view.first_time_ns + || source.last_ns < view.last_time_ns + || view + .sequence_ids + .first() + .is_none_or(|v| *v < source.first_sequence) + || view + .sequence_ids + .last() + .is_none_or(|v| *v > source.last_sequence) + { + return Err("temporal_quality_retained_identity".into()); + } + } + if source.packets < retained { + return Err("temporal_quality_retained_count".into()); + } + } + } + Ok(()) +} +fn validate_parts( + parts: &[QualityPart], + schedule: &ManagedScheduleContract, + run: &str, + episode: usize, + complete: Option, + now: u64, +) -> Result<(), String> { + validate_parts_with_frozen(parts,schedule,run,episode,complete,now,None) +} +pub(super) fn validate_parts_with_frozen( + parts:&[QualityPart],schedule:&ManagedScheduleContract,run:&str,episode:usize, + complete:Option,now:u64, + frozen:Option<&crate::room_learner::managed_schedule_contract::FrozenModelIdentity>, +)->Result<(),String>{ + let native = schedule.native_window_policy(); + let mut frontier = 0; + let mut ordinal = None; + let mut receiver_end = 0; + for p in parts { + let bound = + schedule.bind_episode_start(run, episode, 0, p.mapping.segment_ordinal, frozen)?; + if p.mapping.session_id != bound.session_id() + || (ordinal.is_none() && p.mapping.segment_ordinal != 0) + || p.mapping.observed_start_ns != frontier + || ordinal.is_some_and(|n| p.mapping.segment_ordinal != n + 1) + || p.mapping.receiver_start_ns < receiver_end + { + return Err("temporal_quality_segment_order".into()); + } + p.validate(&native, &bound, now)?; + frontier = p.observed_end_ns; + ordinal = Some(p.mapping.segment_ordinal); + receiver_end = p + .mapping + .receiver_start_ns + .checked_add(p.observed_end_ns - p.mapping.observed_start_ns) + .ok_or("temporal_quality_time")?; + } + if complete.is_some_and(|v| v != frontier) || complete.is_some() && parts.is_empty() { + return Err("temporal_quality_period_incomplete".into()); + } + Ok(()) +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum TemporalCaptureStage { + Preparing, + Ready, + Observing, + NeedsConfirmation, + Complete, + Frozen, + FullComplete, + Paused, + Blocked, + Failed, +} +impl TemporalCaptureStage { + pub(crate) fn as_str(self) -> &'static str { + match self { + Self::Preparing => "preparing", + Self::Ready => "ready", + Self::Observing => "observing", + Self::NeedsConfirmation => "needs_confirmation", + Self::Complete => "complete", + Self::Frozen => "frozen", + Self::FullComplete => "full_complete", + Self::Paused => "paused", + Self::Blocked => "blocked", + Self::Failed => "failed", + } + } +} +pub(crate) struct TemporalTrainCapture { + workflow: TemporalWorkflow, + public_generation_id: String, + // The exact immutable contract decoded from this workflow's validated + // binding. Packet handling must not reparse its full JSON on every tick. + schedule: ManagedScheduleContract, + failure: Option, + durable_seals: usize, +} +impl TemporalTrainCapture { + #[cfg(test)] + pub(crate) fn c4b_test_abort_contract(&self)->Result<(),String>{self.workflow.require_abortable_contract()} + #[cfg(test)] + pub(crate) fn c4b_test_custody(&self)->(usize,usize,bool,(usize,usize,usize,bool,bool)){(self.workflow.session.rows.len(),self.workflow.ledger.spans.len(),self.workflow.session.active,self.workflow.session.assembler.c4b_test_fence_counts())} + #[cfg(test)] + pub(crate) fn c4b_test_replay_budget(&self)->Result{self.workflow.full_run.as_ref().ok_or("temporal_full_run_required")?.policy.replay_fence_budget(&self.workflow.session.managed.as_ref().unwrap().binding)} + #[cfg(test)] + pub(crate) fn c4b_test_target(&self)->Option<(u64,u64)>{self.workflow.pending_window.as_ref().map(|w|(w.target_ns(),w.end_ns()))} + #[cfg(test)] + pub(crate) fn test_full_run_floors(&self)->(Vec<(u8,u64,u64)>,Option){ + (self.workflow.source_high.clone(),self.workflow.full_run.as_ref().and_then(|m|m.local_start_not_before_ns)) + } + pub(crate) fn is_full_run(&self)->bool{self.workflow.full_run.is_some()} + pub(crate) fn current_context_matches(&self,context:&CaptureContext)->bool{self.workflow.context()==context} + pub(crate) fn verified_frozen_model(&self)->Option>{if self.failure.is_some(){return None;}self.workflow.full_run.as_ref().and_then(|m|m.verified_model())} + pub(crate) fn request_frozen(&mut self,accepted:crate::room_learner::lifecycle::AcceptedTemporalFit,witness:crate::room_learner::lifecycle::FrozenOwnerWitness,store:std::sync::Arc,now:u64)->Result<(),String>{ + self.ensure_live()?; + if self.stage()!=TemporalCaptureStage::Complete{return Err("temporal_full_run_train_complete_required".into());} + self.workflow.request_frozen(accepted,witness,store,now) + } + pub(crate) fn prepare_full_run_async( + schedule:ManagedScheduleContract,run:String,config:FitConfig,now:u64, + store:std::sync::Arc,witness:crate::room_learner::lifecycle::FullRunGenesisWitness, + )->Result<(Self,Option),String>{ + // Fail unsupported full-run profiles before creating a writer/genesis. + full_run::ContinuationPolicy::build(&ManagedBinding::build(&schedule,config.clone())?)?; + let owned_store=store.clone(); + // Reuse authenticated retired-head chaining. An active predecessor is + // refused; this never upgrades/replaces an existing capture in place. + let predecessor=store.fresh_train_genesis_predecessor(now)?; + Self::prepare_with(schedule,run,config,now,&store,move|w,store|{ + let binding=&w.session.managed.as_ref().ok_or("managed_session_required")?.binding; + w.full_run=Some(full_run::FullRunState::fresh(binding,&w.session.run_id,witness)?); + w.commit_token=predecessor; + let writer=std::sync::Arc::new(durability::Writer::new(owned_store,now)?); + w.async_state=Some(durability::AsyncState::new(writer)); + w.persist_initial(store) + }) + } + pub(crate) fn prepare_fit_input(&self, now: u64) -> Result { + self.ensure_live()?; + if self.stage() != TemporalCaptureStage::Complete { + return Err("temporal_fit_confirmed_durable_train_required".into()); + } + fit_admission::PendingTemporalFitInput::capture(&self.workflow, &self.schedule, now) + } + pub(crate) fn fit_capture_identity(&self, now: u64) -> Result { + self.ensure_live()?; + if self.stage() != TemporalCaptureStage::Complete { + return Err("temporal_fit_confirmed_durable_train_required".into()); + } + fit_admission::current_identity(&self.workflow, &self.schedule, now) + } + pub(crate) fn prepare( + schedule: ManagedScheduleContract, + run: String, + config: FitConfig, + now: u64, + store: &RecoveryStore, + ) -> Result { + let (capture, error) = Self::prepare_retained(schedule, run, config, now, store)?; + if let Some(error) = error { + Err(error) + } else { + Ok(capture) + } + } + pub(crate) fn prepare_async( + schedule: ManagedScheduleContract, run:String, config:FitConfig, now:u64, + store:std::sync::Arc, + ) -> Result<(Self,Option),String> { + let owned_store=store.clone(); + Self::prepare_with(schedule,run,config,now,&store,move |w,store|{ + let writer=std::sync::Arc::new(durability::Writer::new(owned_store,now)?); + w.async_state=Some(durability::AsyncState::new(writer)); + w.persist_initial(store) + }) + } + /// Explicit NEW Train genesis only. Existing generic/old constructors and + /// cold restore retain their stored profile; C4 never inherits this choice. + pub(crate) fn prepare_async_fresh_train_343( + schedule:ManagedScheduleContract,run:String,config:FitConfig,now:u64, + store:std::sync::Arc, + )->Result<(Self,Option),String>{ + let predecessor=store.fresh_train_genesis_predecessor(now)?; + let mut workflow=TemporalWorkflow::prepare(schedule,run,config,now)?; + let schedule=workflow.schedule()?; + workflow.bridge=Some(BridgeState::fresh_train_343()); + workflow.commit_token=predecessor; + let writer=std::sync::Arc::new(durability::Writer::new(store.clone(),now)?); + workflow.async_state=Some(durability::AsyncState::new(writer)); + let error=workflow.persist_initial(&store).err(); + if error.is_some(){workflow.stage=Stage::BlockedDurability;} + Ok((Self{public_generation_id:public_generation_id(&workflow)?,workflow,schedule,failure:None,durable_seals:0},error)) + } + pub(crate) fn async_start(&mut self,now:u64)->Result{ + self.ensure_live()?; + if self.workflow.async_state.is_none(){return Ok(false);} + self.workflow.request_async_start(now)?;Ok(true) + } + pub(crate) fn poll_async(&mut self,now:u64,retired:&mut DeferredReclamation)->Result,String>{ + if self.failure.is_some(){return Err("temporal_capture_failed".into());} + let start=match self.workflow.poll_async(now,retired){ + Ok(start)=>start, + Err(error)=>{ + if matches!(error.as_str(),"temporal_writer_ack_clock_or_expiry"|"temporal_writer_ack_identity_changed"|"temporal_writer_ack_row_lineage_changed"|"temporal_writer_ack_binding_changed") {self.invalidate(&error);} + return Err(error); + } + }; + if let Some(state)=&self.workflow.async_state{self.durable_seals=state.accepted_seals;} + if start.is_some(){self.workflow.prepare_complete_from_actual_seal_ack(now)?;} + Ok(start) + } + pub(crate) fn enable_async_restore(&mut self,store:std::sync::Arc,now:u64)->Result<(),String>{ + if self.workflow.async_state.is_some(){return Err("temporal_writer_already_present".into());} + let mut state=durability::AsyncState::new(std::sync::Arc::new(durability::Writer::new(store,now)?)); + state.seed_restored(&self.workflow)?; + self.workflow.async_state=Some(state); + self.workflow.prepare_complete_from_actual_seal_ack(now)?;Ok(()) + } + pub(crate) fn current_lead_in_ns(&self)->Result{self.schedule.episodes().get(self.workflow.episode).ok_or("temporal_capture_complete")?.lead_in_ms.checked_mul(1_000_000).ok_or("temporal_time_overflow".into())} + pub(crate) fn writer(&self)->Option>{ + self.workflow.async_state.as_ref().map(|a|a.writer.clone()) + } + pub(crate) fn prepare_retained( + schedule: ManagedScheduleContract, + run: String, + config: FitConfig, + now: u64, + store: &RecoveryStore, + ) -> Result<(Self, Option), String> { + Self::prepare_with(schedule, run, config, now, store, |w, store| { + w.persist_initial(store) + }) + } + fn prepare_with( + schedule: ManagedScheduleContract, + run: String, + config: FitConfig, + now: u64, + store: &RecoveryStore, + initial: impl FnOnce(&mut TemporalWorkflow, &RecoveryStore) -> Result<(), String>, + ) -> Result<(Self, Option), String> { + let mut workflow = TemporalWorkflow::prepare(schedule, run, config, now)?; + let schedule = workflow.schedule()?; + workflow.bridge = Some(if schedule.is_operator_eight_step(){BridgeState::operator_eight_step()}else{BridgeState::new()}); + let error = initial(&mut workflow, store).err(); + if error.is_some() { + workflow.stage = Stage::BlockedDurability; + } + Ok(( + Self { + public_generation_id:public_generation_id(&workflow)?, + workflow, + schedule, + failure: None, + durable_seals: 0, + }, + error, + )) + } + #[cfg(test)] + pub(crate) fn synthetic_prepare_lost_initial_ack( + schedule: ManagedScheduleContract, + run: String, + config: FitConfig, + now: u64, + store: &RecoveryStore, + ) -> Result<(Self, Option), String> { + Self::prepare_with(schedule, run, config, now, store, |w, store| { + w.persist_with(store, false, |s, i, p, n| { + s.publish_temporal(i, Some(p), n)?; + Err("synthetic-lost-initial-ack-after-real-publish".into()) + }) + }) + } + pub(crate) fn context(&self) -> &CaptureContext { + self.workflow.context() + } + pub(crate) fn check_time(&mut self, now: u64) -> Result<(), String> { + if self.workflow.async_state.is_none(){self.ensure_live()?;}else if self.failure.is_some(){return Err("temporal_capture_failed".into());} + if now < self.workflow.last_now_ns + || now / 1_000_000 >= self.workflow.descriptor.expires_at_ms + { + self.invalidate("temporal_receiver_clock_or_expiry"); + return Err("temporal_receiver_clock_or_expiry".into()); + } + Ok(()) + } + pub(crate) fn prime_fence(&mut self, p: &CanonicalPacket, now: u64) -> Result<(), String> { + if self.workflow.async_state.is_none(){self.ensure_live()?;}else if self.failure.is_some(){return Err("temporal_capture_failed".into());} + // A full-run owner keeps listening between Train completion and the + // future HeldOut stage. This admits only the existing replay/clock + // fence update below: never another row, seal, model or role credit. + let full_run_continuation_fence = self.is_full_run() + && matches!(self.stage(), TemporalCaptureStage::Complete | TemporalCaptureStage::Frozen | TemporalCaptureStage::FullComplete); + if !matches!( + self.stage(), + TemporalCaptureStage::Ready | TemporalCaptureStage::Paused | TemporalCaptureStage::Preparing | TemporalCaptureStage::Blocked + ) && !(self.workflow.async_state.is_some() && self.logical_phase_complete()) + && !full_run_continuation_fence { + return Err("temporal_priming_state".into()); + } + // No recorded observation exists in these closed Full2 stages. A + // changing host-arrival estimate may prime identity/replay fences, + // while the frozen recording band remains mandatory in ingest(). + let full_run_idle = self.is_full_run() && matches!(self.stage(), + TemporalCaptureStage::Ready | TemporalCaptureStage::Paused | + TemporalCaptureStage::Preparing | TemporalCaptureStage::Blocked | + TemporalCaptureStage::Complete | TemporalCaptureStage::Frozen | + TemporalCaptureStage::FullComplete); + let result: Result<(), String> = (|| { + let w = &mut self.workflow; + w.check_clock(now)?; + let native = &w.session.managed.as_ref() + .ok_or("managed_session_required")?.binding.native; + if full_run_idle { + native.validate_priming_packet_source_identity(p)?; + } else { + native.validate_packet_source(p)?; + } + let t = p.native_frame.timestamp_ns; + if p.provenance != CaptureProvenance::LiveReceiverObserved + || p.native_frame.provenance.class + != ruview_unified::frame::ProvenanceClass::Measured + || t > now + || w.source_high.iter().any(|(id, seq, last)| { + *id == p.source_id && (u64::from(p.sequence) <= *seq || t <= *last) + }) + { + return Err("temporal_priming_source_or_fence".into()); + } + w.source_high.retain(|(id, _, _)| *id != p.source_id); + w.source_high.push((p.source_id, u64::from(p.sequence), t)); + w.source_high.sort(); + Ok(()) + })(); + if let Err(e) = &result { + self.invalidate(e); + } + result + } + + pub(crate) fn stage(&self) -> TemporalCaptureStage { + if self.failure.is_some() { + return TemporalCaptureStage::Failed; + } + let w = &self.workflow; + if w.async_state.as_ref().is_some_and(|a| a.pending.as_ref().is_some_and(|p|p.start_ticket || !matches!(w.stage,Stage::Observing|Stage::BlockedDurability)) || w.durable.is_none()) { + return TemporalCaptureStage::Preparing; + } + if w.stage == Stage::BlockedDurability { + return TemporalCaptureStage::Blocked; + } + if w.full_run.as_ref().is_some_and(|m|m.is_complete()){return TemporalCaptureStage::FullComplete;} + if w.full_run.as_ref().is_some_and(|m|m.is_frozen()&&m.held_out.is_none()){return TemporalCaptureStage::Frozen;} + if w.full_run.as_ref().is_none_or(|m|m.held_out.is_none()) && w.bridge + .as_ref() + .is_some_and(|b| b.disposition == Disposition::Complete) + { + return TemporalCaptureStage::Complete; + } + if w.stage == Stage::PausedNeedsReconfirmation { + return TemporalCaptureStage::Paused; + } + if self.phase_ready() { + return TemporalCaptureStage::NeedsConfirmation; + } + if w.stage == Stage::Observing { + TemporalCaptureStage::Observing + } else { + TemporalCaptureStage::Ready + } + } + pub(crate) fn phase_identity(&self) -> Option<(String, String)> { + let w = &self.workflow; + let schedule = &self.schedule; + let e = schedule.episodes().get(w.episode)?; + if e.role != DataRole::Train && !(schedule.is_operator_eight_step()&&e.role==DataRole::Validation) && !w.full_run.as_ref().is_some_and(|m|m.is_frozen()) {return None;} + let frozen=w.frozen_identity().ok()?; + let bound = schedule + .bind_episode_start( + &w.session.run_id,w.episode,0,w.mapping.as_ref().map_or(0, |m| m.segment_ordinal),frozen.as_ref(), + ) + .ok()?; + Some((bound.trial_id().into(), bound.session_id().into())) + } + fn logical_phase_complete(&self)->bool { + let w=&self.workflow; + self.schedule.episodes().get(w.episode).is_some_and(|e|(e.role==DataRole::Train || self.schedule.is_operator_eight_step()&&e.role==DataRole::Validation || w.full_run.as_ref().is_some_and(|m|m.held_out.is_some())) + && w.pending_window.is_none() + && w.session.rows.len()==self.schedule.episodes().iter().take(w.episode+1).map(|e|e.slots.len()).sum::()) + } + fn phase_ready(&self) -> bool { + let w = &self.workflow; + let s = &self.schedule; + let Some(e) = s.episodes().get(w.episode) else { + return false; + }; + (e.role == DataRole::Train || s.is_operator_eight_step()&&e.role==DataRole::Validation || w.full_run.as_ref().is_some_and(|m|m.held_out.is_some())) + && w.accepted() == e.target_active_recording_ms * 1_000_000 + && w.pending_window.is_none() + && w.session.rows.len() + == s.episodes() + .iter() + .take(w.episode + 1) + .map(|e| e.slots.len()) + .sum::() + } + pub(crate) fn start_segment( + &mut self, + epoch: &str, + now: u64, + store: &RecoveryStore, + ) -> Result<(), String> { + self.ensure_live()?; + if self.phase_ready() { + return Err("temporal_confirmation_required".into()); + } + let w = &mut self.workflow; + let schedule = w.schedule()?; + let role=schedule.episodes().get(w.episode).ok_or("temporal_capture_complete")?.role; + if role==DataRole::HeldOut{w.begin_held_out_role(now)?;} + else if role!=DataRole::Train&&!(schedule.is_operator_eight_step()&&role==DataRole::Validation){return Err("temporal_train_complete".into());} + else if w.full_run.is_some()&&w.mapping.is_none(){ + let b=w.bridge.as_ref().ok_or("temporal_bridge_required")?; + let expected=schedule.episodes().iter().take(w.episode).map(|e|e.slots.len()).sum::(); + if b.seals.len()!=w.episode||!b.parts.is_empty()||w.session.rows.len()!=expected||w.async_state.as_ref().is_none_or(|a|a.pending.is_some()||a.accepted_seals!=b.seals.len()){ + return Err("temporal_initial_episode_prior_seal_ack_required".into()); + } + } + let permit = ConditionsPermit { + context: w.context().digest(), + run: w.session.run_id.clone(), + durable_sha256: w.durable.as_ref().map(|b| bytes_hash(b)), + }; + w.start_segment(&permit, epoch, now, now)?; + let mapping = w.mapping.clone().ok_or("temporal_segment_missing")?; + w.set_current_disposition(Disposition::Observing)?; + w.current_parts_mut()?.push(QualityPart { + observed_end_ns: mapping.observed_start_ns, + mapping, + sources: vec![], + }); + w.persist_segment(store)?; + w.begin_slot()?; + Ok(()) + } + pub(crate) fn ingest( + &mut self, + p: &CanonicalPacket, + now: u64, + store: &RecoveryStore, + ) -> Result<(), String> { + self.ensure_live()?; + if self.phase_ready() || (self.workflow.async_state.is_some() && self.logical_phase_complete()) { + if self.workflow.async_state.is_some(){return self.prime_fence(p,now);} + return Ok(()); + } + let result = (|| { + let w = &mut self.workflow; + let native = &w + .session + .managed + .as_ref() + .ok_or("managed_session_required")? + .binding + .native; + native.validate_packet_source(p)?; + if p.provenance != CaptureProvenance::LiveReceiverObserved + || p.native_frame.provenance.class + != ruview_unified::frame::ProvenanceClass::Measured + { + return Err("temporal_measured_receiver_required".into()); + } + w.observe(p, now)?; + w.current_parts_mut()?.last_mut() + .ok_or("temporal_quality_segment_missing")? + .observe(p)?; + self.commit_if_ready(store) + })(); + if let Err(e) = &result { + if self.workflow.stage != Stage::BlockedDurability { + self.invalidate(e); + } + } + result + } + pub(crate) fn tick(&mut self, now: u64, store: &RecoveryStore) -> Result<(), String> { + self.ensure_live()?; + if self.phase_ready() { + return self.workflow.check_clock(now); + } + let result = self + .workflow + .observe_tick(now) + .and_then(|_| self.commit_if_ready(store)); + if let Err(e) = &result { + if self.workflow.stage != Stage::BlockedDurability { + self.invalidate(e); + } + } + result + } + fn commit_if_ready(&mut self, store: &RecoveryStore) -> Result<(), String> { + let w = &mut self.workflow; + let Some(target) = w.pending_window.as_ref() else { + return Ok(()); + }; + if w.session.active + || w.source_high.len() != w.context().sources.len() + || w.source_high.iter().any(|(_, _, t)| *t < target.end_ns()) + { + return Ok(()); + } + let observed = target.identity().slot_index; + let schedule = w.schedule()?; + let deadline = + schedule.episodes()[w.episode].slots[observed].deadline_offset_ms * 1_000_000; + if w.async_state.as_ref().is_some_and(|a|a.pending.is_some()) { + let next=schedule.episodes()[w.episode].slots.get(observed+1); + if let Some(next)=next { + let mapping=w.mapping.as_ref().ok_or("temporal_segment_missing")?; + let deadline=mapping.receiver_start_ns.checked_add(next.target_offset_ms*1_000_000-mapping.observed_start_ns).ok_or("temporal_time_overflow")?; + if w.last_now_ns>=deadline { + let state=w.async_state.as_mut().unwrap();state.busy_pause=true;state.error=Some("temporal_storage_busy_pause".into()); + w.stage=Stage::BlockedDurability;return Err("temporal_storage_busy_pause".into()); + } + } + return Ok(()); + } + w.current_parts_mut()?.last_mut() + .ok_or("temporal_quality_segment_missing")? + .observed_end_ns = deadline; + if deadline == schedule.episodes()[w.episode].target_active_recording_ms * 1_000_000 { + w.set_current_disposition(Disposition::NeedsConfirmation)?; + } + w.commit_slot(store, w.last_now_ns)?; + if !self.logical_phase_complete() { + self.workflow.begin_slot()?; + } + Ok(()) + } + pub(crate) fn confirmation_challenge(&self) -> Result { + self.ensure_live()?; + if !self.phase_ready() || self.workflow.async_state.as_ref().is_some_and(|a|a.pending.is_some()) { + return Err("temporal_phase_not_complete".into()); + } + let w = &self.workflow; + if let Some(mode)=w.full_run.as_ref().filter(|m|m.held_out.is_some()){ + let held=mode.held_out.as_ref().unwrap();let fit=mode.frozen.as_ref().ok_or("temporal_verified_frozen_required")?; + return hash(&(held_out::HELD_OUT_CONDITIONS_DOMAIN,&w.descriptor.id,&w.session.run_id,w.episode,w.durable_digest(),&fit.sha256,&held.parts,&held.seals)); + } + let b = w.bridge.as_ref().ok_or("temporal_bridge_required")?; + hash(&( + b.profile()?.policy(), + &w.descriptor.id, + &w.session.run_id, + w.episode, + w.durable_digest(), + &b.parts, + &b.seals, + )) + } + pub(crate) fn confirm( + &mut self, + challenge: &str, + store: &RecoveryStore, + now: u64, + ) -> Result<(), String> { + self.ensure_live()?; + if challenge != self.confirmation_challenge()? { + return Err("temporal_stale_conditions_challenge".into()); + } + let w = &mut self.workflow; + w.check_clock(now)?; + let schedule = w.schedule()?; + if schedule.episodes().get(w.episode).is_some_and(|e|e.role==DataRole::HeldOut){return self.confirm_held_out(store,now);} + let context_sha256 = w.context().digest(); + let row_prefix_sha256 = match &w.async_state { + Some(state)=>state.row_prefix_sha256.clone().ok_or("temporal_row_prefix_unconfirmed")?, + None=>hash(&(&w.session.rows, &w.session.evidence, &w.ledger.spans))?, + }; + let b = w.bridge.as_mut().ok_or("temporal_bridge_required")?; + validate_parts( + &b.parts, + &schedule, + &w.session.run_id, + w.episode, + Some(schedule.episodes()[w.episode].target_active_recording_ms * 1_000_000), + now, + )?; + let bound = schedule.bind_episode_start(&w.session.run_id, w.episode, 0, 0, None)?; + let mut seal = TrainSeal { + episode: w.episode, + trial_id: bound.trial_id().into(), + context_sha256, + schedule_sha256: schedule.sha256().into(), + row_prefix_sha256, + quality_sha256: hash(&b.parts)?, + parts: b.parts.clone(), + confirmed_at_ns: now, + previous_sha256: b.seals.last().map(|s| s.sha256.clone()), + sha256: String::new(), + }; + seal.sha256 = seal.digest(b.profile()?)?; + b.seals.push(seal); + b.parts.clear(); + w.episode += 1; + w.mapping = None; + w.custody = SegmentCustody::default(); + w.live_observed_ns = 0; + b.disposition = if w.episode == if schedule.is_operator_eight_step(){schedule.episodes().len()}else{schedule.counts().train_episodes} { + Disposition::Complete + } else { + Disposition::Ready + }; + w.stage = Stage::DurableReady; + w.persist_with(store, false, |s, i, p, n| s.publish_temporal(i, Some(p), n))?; + if w.async_state.is_none(){self.durable_seals = w.bridge.as_ref().unwrap().seals.len();} + Ok(()) + } + fn confirm_held_out(&mut self,store:&RecoveryStore,now:u64)->Result<(),String>{ + let w=&mut self.workflow;let schedule=w.schedule()?; + let frozen=w.frozen_identity()?.ok_or("temporal_verified_frozen_required")?; + let mode=w.full_run.as_ref().unwrap();let fit=mode.frozen.as_ref().unwrap();let held=mode.held_out.as_ref().ok_or("temporal_heldout_bridge_required")?; + if w.episode!=7+held.seals.len()||held.seals.len()>=8{return Err("temporal_heldout_seal_order".into());} + validate_parts_with_frozen(&held.parts,&schedule,&w.session.run_id,w.episode,Some(30_000_000_000),now,Some(&frozen))?; + let state=w.async_state.as_ref().ok_or("temporal_async_required")?; + if state.pending.is_some()||state.accepted_held_out_seals!=held.seals.len(){return Err("temporal_heldout_prior_seal_ack_required".into());} + let row_prefix_sha256=state.row_prefix_sha256.clone().ok_or("temporal_row_prefix_unconfirmed")?; + let bound=schedule.bind_episode_start(&w.session.run_id,w.episode,0,0,Some(&frozen))?; + let seal=held_out::HeldOutSealData::from_body(held_out::HeldOutSealBody{role:DataRole::HeldOut,episode:w.episode,trial_id:bound.trial_id().into(),context_sha256:w.context().digest(),schedule_sha256:schedule.sha256().into(),continuation_policy_sha256:mode.policy.sha256().into(),verified_fit_sha256:fit.sha256.clone(),artifact_kind:fit.body.artifact_kind.clone(),artifact_sha256:fit.body.artifact_sha256.clone(),row_prefix_sha256,quality_sha256:hash(&held.parts)?,parts:held.parts.clone(),confirmed_at_ns:now,previous_sha256:held.seals.last().map(|s|s.sha256.clone())})?; + seal.validate_binding_data(&schedule,&w.session.run_id,fit,now)?; + let held=w.full_run.as_mut().unwrap().held_out.as_mut().unwrap();held.seals.push(seal);held.parts.clear(); + w.episode+=1;w.mapping=None;w.custody=SegmentCustody::default();w.live_observed_ns=0; + held.disposition=if w.episode==15{Disposition::Complete}else{Disposition::Ready};w.stage=Stage::DurableReady;w.descriptor.saved_at_ms=now/1_000_000; + w.persist_with(store,false,|s,i,p,n|s.publish_temporal(i,Some(p),n)) + } + pub(crate) fn pause(&mut self, now: u64, store: &RecoveryStore) -> Result<(), String> { + self.ensure_live()?; + if self.workflow.async_state.is_some(){return self.workflow.request_async_pause(now);} + self.workflow.check_clock(now)?; + self.workflow.pause()?; + self.workflow.set_current_disposition(Disposition::Paused)?; + self.workflow + .persist_with(store, false, |s, i, p, n| s.publish_temporal(i, Some(p), n)) + } + pub(crate) fn cold_restore( + store: &RecoveryStore, + context: &CaptureContext, + now: u64, + epoch: &str, + anchor: u64, + ) -> Result { + let w = TemporalWorkflow::cold_restore(store, context, now, epoch, anchor)?; + if w.full_run.is_some(){return Err("temporal_full_run_restore_requires_exact_model_consumer".into());} + let b = w + .bridge + .as_ref() + .ok_or("temporal_bridge_checkpoint_required")?; + let n = b.seals.len(); + let schedule = w.schedule()?; + Ok(Self { + public_generation_id:public_generation_id(&w)?, + workflow: w, + schedule, + failure: None, + durable_seals: n, + }) + } + pub(crate) fn cold_restore_full_run( + store:&RecoveryStore,artifact:std::sync::Arc,context:&CaptureContext, + now:u64,epoch:&str,anchor:u64,witness:crate::room_learner::lifecycle::FullRunRecoveryWitness, + )->Result{ + let w=full_run::recover_current(store,artifact,context,now,epoch,anchor,witness)?; + let n=w.bridge.as_ref().ok_or("temporal_bridge_checkpoint_required")?.seals.len();let schedule=w.schedule()?; + Ok(Self{public_generation_id:public_generation_id(&w)?,workflow:w,schedule,failure:None,durable_seals:n}) + } + pub(crate) fn repair_pending(&mut self, store: &RecoveryStore, now: u64) -> Result<(), String> { + if self.workflow.async_state.is_some(){return self.workflow.request_async_repair(now);} + self.workflow.repair_pending(store, now)?; + if self.workflow.schedule()?.sha256() != self.schedule.sha256() { + self.invalidate("temporal_repaired_schedule_changed"); + return Err("temporal_repaired_schedule_changed".into()); + } + self.durable_seals = self + .workflow + .bridge + .as_ref() + .ok_or("temporal_bridge_required")? + .seals + .len(); + Ok(()) + } + /// Metadata-only update after the exact packet has already failed admission. + pub(crate) fn annotate_receiver_failure(&mut self, expected: &str, detail: String) -> bool { + if self.stage() != TemporalCaptureStage::Failed || self.failure.as_deref() != Some(expected) + || detail.len() > 256 || !detail.starts_with(expected) { return false; } + self.failure = Some(detail); + true + } + pub(crate) fn invalidate(&mut self, reason: &str) { + if let Some(state)=&self.workflow.async_state{state.writer.cancel();} + self.failure = Some( + reason + .chars() + .filter(|c| !c.is_control()) + .take(256) + .collect(), + ); + } + fn ensure_live(&self) -> Result<(), String> { + if self.failure.is_some() { + Err("temporal_capture_failed".into()) + } else if self.workflow.stage == Stage::BlockedDurability { + Err("temporal_capture_durability_blocked".into()) + } else { + Ok(()) + } + } + pub(crate) fn snapshot(&self) -> Value { + let w = &self.workflow; + let e = self.schedule.episodes().get(w.episode); + let full=w.full_run.is_some(); + let mut value=json!({"schema":"ruview.room-learner.private-temporal-capture.v1","stage":self.stage().as_str(),"episode_index":w.episode,"label":e.filter(|e|e.role==DataRole::Train||full).map(|e|e.condition),"role":if self.schedule.is_operator_eight_step(){e.map(|e|match e.role{DataRole::Train=>"Train",DataRole::Validation=>"Validation",DataRole::HeldOut=>"HeldOut",DataRole::Inference=>"Inference"}).unwrap_or("Train")}else if full&&e.is_some_and(|e|e.role==DataRole::HeldOut){"HeldOut"}else{"Train"},"sealed_train_episodes":if self.schedule.is_operator_eight_step(){self.schedule.episodes().iter().take(self.durable_seals).filter(|e|e.role==DataRole::Train).count()}else{self.durable_seals},"retained_rows":w.session.rows.len(),"durable_windows":w.durable_windows,"rows_sha256":w.async_state.as_ref().map(|a|if w.session.rows.len()==w.durable_windows{a.rows_sha256.clone()}else{None}).unwrap_or_else(||hash(&w.session.rows).ok()),"durability_pending":w.async_state.as_ref().is_some_and(|a|a.pending.is_some()),"durability_error":w.async_state.as_ref().and_then(|a|a.error.clone()),"observed_ns":w.live_observed_ns.to_string(),"durable_observed_ns":w.accepted().to_string(),"target_ms":e.filter(|e|e.role==DataRole::Train||full).map(|e|e.target_active_recording_ms),"descriptor":w.descriptor,"failure":self.failure,"fit_admission_available":false,"quality_is_engineering_only":true,"independence_verified":false,"source_clock_measured":false}); + if full{value["sealed_held_out_episodes"]=json!(if self.schedule.is_operator_eight_step(){self.schedule.episodes().iter().take(self.durable_seals).filter(|e|e.role==DataRole::Validation).count()}else{w.async_state.as_ref().map_or(0,|s|s.accepted_held_out_seals)});} + value + } +} + +impl Drop for TemporalTrainCapture { + fn drop(&mut self) { + // Runtime drains installed captures before dropping its inner state. + // Constructor failures/obsolete unpublished cores must also retain the + // owner store until their one writer has actually stopped. + if let Some(state)=&self.workflow.async_state {let _=state.writer.drain();} + } +} + +#[cfg(test)] +#[path="temporal_budget_profile_tests.rs"] +mod budget_profile_tests; + +impl TemporalTrainCapture { + pub(crate) fn prepare_evaluation_source(&self,now:u64,held_out:bool)->Result{ + self.ensure_live()?;self.workflow.prepare_temporal_evidence_source(now,held_out,&self.schedule) + } + pub(crate) fn evaluation_binding_matches(&self,binding:&full_run::TemporalEvaluationBinding)->bool{ + self.failure.is_none()&&self.workflow.current_temporal_evidence_matches(binding) + } +} + +// Informational public generation binding only, never an owner/source permit. +// Computed once by off-lock construction/recovery, not on packet ingress. +fn public_generation_id(w:&TemporalWorkflow)->Result{hash(&("ruview.room-learner.capture-generation.v2",&w.descriptor.id,&w.session.run_id,w.context().digest()))} +impl TemporalTrainCapture { + pub(crate) fn public_generation_id(&self)->&str{&self.public_generation_id} + pub(crate) fn public_phase_matches(&self,generation:&str,schedule:&str,episode:usize)->bool{self.public_generation_id==generation&&self.schedule.sha256()==schedule&&self.workflow.episode==episode} + pub(crate) fn public_profile(&self)->crate::room_learner::temporal_wire::TemporalProfileWire{ + use crate::room_learner::temporal_wire::*; + TemporalProfileWire{profile_id:self.schedule.profile_id().into(),window_unit:"native_receiver_2s_window".into(),native_window_ms:2000,train_windows:self.schedule.counts().train_windows as u16,held_out_windows:self.schedule.counts().held_out_windows as u16,train_episodes:self.schedule.counts().train_episodes as u8,held_out_episodes:self.schedule.counts().held_out_episodes as u8,windows_per_condition:[16;4],train_episodes_per_condition:if self.schedule.is_operator_eight_step(){[1;4]}else{[1,2,2,2]},held_out_episodes_per_condition:if self.schedule.is_operator_eight_step(){[1;4]}else{[2;4]},episodes:self.schedule.episodes().iter().map(|e|TemporalEpisodeWire{index:e.index as u8,role:e.role,label:e.condition,target_ms:e.target_active_recording_ms,lead_in_ms:e.lead_in_ms,native_windows:e.slots.len()as u16}).collect()} + } + pub(crate) fn public_capture(&self,epoch:&str,lead_until:Option,now:u64)->crate::room_learner::temporal_wire::TemporalCaptureWire{ + use crate::room_learner::temporal_wire::*; + let w=&self.workflow;let episode=self.schedule.episodes().get(w.episode);let lead=lead_until.map_or(0,|end|end.saturating_sub(now)/1_000_000); + let challenge=if matches!(self.stage(),TemporalCaptureStage::NeedsConfirmation|TemporalCaptureStage::Paused){self.confirmation_challenge().ok()}else{None}; + TemporalCaptureWire{capture_generation_id:self.public_generation_id.clone(),schedule_sha256:self.schedule.sha256().into(),continuation_policy_sha256:w.full_run.as_ref().map(|m|m.policy.sha256().into()).unwrap_or_default(),durable_commitment_sha256:w.commit_token.as_ref().map(|t|t.data().commitment),original_deadline:TemporalOriginalDeadlineWire{created_at_unix_ms:w.descriptor.created_at_ms,expires_at_unix_ms:w.descriptor.expires_at_ms},receiver_epoch:epoch.into(),stage:if lead>0{"preparing".into()}else{self.stage().as_str().into()},current_episode:episode.map(|e|e.index as u8),role:episode.map(|e|e.role),label:episode.map(|e|e.condition),target_ms:episode.map(|e|e.target_active_recording_ms),lead_in_remaining_ms:lead,observed_ms:w.live_observed_ns/1_000_000,retained_windows:w.session.rows.len()as u16,durable_windows:w.durable_windows as u16,sealed_train_episodes:if self.schedule.is_operator_eight_step(){self.schedule.episodes().iter().take(self.durable_seals).filter(|e|e.role==DataRole::Train).count()as u8}else{self.durable_seals as u8},sealed_held_out_episodes:if self.schedule.is_operator_eight_step(){self.schedule.episodes().iter().take(self.durable_seals).filter(|e|e.role==DataRole::Validation).count()as u8}else{w.async_state.as_ref().map_or(0,|a|a.accepted_held_out_seals as u8)},durability_pending:w.async_state.as_ref().is_some_and(|a|a.pending.is_some()),awaiting_ground_truth:challenge.is_some(),confirmation_challenge:challenge,failure:self.failure.clone()} + } + pub(crate) fn public_candidate(&self)->Option{ + use crate::room_learner::temporal_wire::*; + let mode=self.workflow.full_run.as_ref()?;let record=mode.public_verified_fit()?; + Some(TemporalCandidateIdentityWire{model_kind:WholeTemporalKind::ManagedTemporalV3,model_sha256:record.body.artifact_sha256.clone(),verified_fit_record_sha256:record.sha256.clone(),context_digest:record.body.context_digest.clone(),room_digest:self.context().room_digest.clone(),persisted:true}) + } + pub(crate) fn public_complete_refs(&self,proof_sha:&str)->Option{ + use crate::room_learner::temporal_wire::*; + let mode=self.workflow.full_run.as_ref()?;let record=mode.public_complete(self.workflow.commit_token.as_ref())?;let fit=mode.public_verified_fit()?; + Some(TemporalCompleteRefsWire{model:TemporalModelIdentityWire{model_kind:WholeTemporalKind::ManagedTemporalV3,model_sha256:fit.body.artifact_sha256.clone(),verified_fit_record_sha256:fit.sha256.clone()},training_proof_sha256:proof_sha.into(),complete_record_sha256:record.sha256.clone(),complete_commitment_sha256:self.workflow.commit_token.as_ref()?.data().commitment}) + } +} + +impl TemporalTrainCapture { + /// Authenticated current recovery is decoded only to discover its context. + /// This DATA does not mint a continued/Frozen witness; restore does that. + pub(crate) fn saved_full_context(store:&RecoveryStore,now:u64)->Result<(CaptureContext,String),String>{ + let loaded=store.load_temporal(now)?.ok_or("recovery_missing")?; + let prefix=loaded.prefix.ok_or("temporal_retired")?; + if prefix.metadata_kind()!=full_run::META_KIND{return Err("temporal_full_run_no_upconversion".into());} + let w=checkpoint::decode(prefix.bytes(),prefix.descriptor(),now)?; + Ok((w.context().clone(),public_generation_id(&w)?)) + } + pub(crate) fn retirement_generation(store:&RecoveryStore,now:u64)->Result{ + let inspection=store.inspect_temporal_retirement(now)?.ok_or("recovery_missing")?; + if inspection.metadata_kind()!=full_run::META_KIND{return Err("temporal_full_run_no_upconversion".into());} + hash(&("ruview.room-learner.capture-generation.v2",&inspection.descriptor().id,inspection.run(),&inspection.descriptor().context_digest)) + } +} + +impl TemporalTrainCapture {pub(crate) fn current_source_floors(&self)->(Vec<(u8,u64,u64)>,u64){(self.workflow.source_high.clone(),self.workflow.last_now_ns)}} + +impl TemporalTrainCapture { + pub(crate) fn recovery_discovery(store:&RecoveryStore,now:u64)->Option{ + use crate::room_learner::temporal_wire::*; + let result=(||->Result,String>{ + let Some(inspection)=store.inspect_temporal_read_only(now)?else{return Ok(None)}; + if inspection.metadata_kind()!=full_run::META_KIND||inspection.retired(){return Ok(None)} + let descriptor=inspection.descriptor();let generation=hash(&("ruview.room-learner.capture-generation.v2",&descriptor.id,inspection.run(),&descriptor.context_digest))?; + Ok(Some(TemporalRecoveryStatusWire{schema:RECOVERY_STATUS_SCHEMA.into(),state:if inspection.expired(){"expired"}else{"available"}.into(),capture_generation_id:Some(generation),context_digest:Some(descriptor.context_digest.clone()),original_deadline:Some(TemporalOriginalDeadlineWire{created_at_unix_ms:descriptor.created_at_ms,expires_at_unix_ms:descriptor.expires_at_ms})})) + })(); + result.unwrap_or_else(|_|Some(TemporalRecoveryStatusWire{schema:RECOVERY_STATUS_SCHEMA.into(),state:"unavailable".into(),capture_generation_id:None,context_digest:None,original_deadline:None})) + } +} + +/// Opaque same-process Retry fence snapshot. It cannot be deserialized and +/// never grants recording or conditions authority. +pub(crate) struct FailedRetryState { + generation: String, + schedule: String, + created_at_ms: u64, + expires_at_ms: u64, + durable_windows: usize, + seals: usize, + accepted_prefix_sha256: Option, + accepted_seal_digests: Vec, + fences: AssemblerCheckpoint, + source_high: Vec<(u8, u64, u64)>, +} +impl TemporalTrainCapture { + pub(crate) fn failed_retry_state(&mut self) -> Result { + if !matches!(self.stage(), TemporalCaptureStage::Failed | TemporalCaptureStage::Blocked) + || !self.is_full_run() + || self.workflow.full_run.as_ref().is_some_and(|m| m.is_frozen() || m.is_complete()) + { return Err("temporal_retry_requires_stopped_capture".into()); } + let state = FailedRetryState { + generation:self.public_generation_id.clone(), schedule:self.schedule.sha256().into(), + created_at_ms:self.workflow.descriptor.created_at_ms, expires_at_ms:self.workflow.descriptor.expires_at_ms, + durable_windows:self.workflow.durable_windows, seals:self.durable_seals, + accepted_prefix_sha256:self.workflow.async_state.as_ref().and_then(|state|state.row_prefix_sha256.clone()), + accepted_seal_digests:self.workflow.bridge.as_ref().map_or_else(Vec::new,|b|b.seals.iter().take(self.durable_seals).map(|seal|seal.sha256.clone()).collect()), + fences:self.workflow.session.assembler.fence_checkpoint()?, source_high:self.workflow.source_high.clone(), + }; + if self.failure.is_none() { self.invalidate("temporal_retry_stopped_capture"); } + Ok(state) + } + pub(crate) fn apply_failed_retry_state(&mut self, old: FailedRetryState, now:u64) -> Result<(), String> { + if self.public_generation_id != old.generation || self.schedule.sha256()!=old.schedule + || self.workflow.descriptor.created_at_ms!=old.created_at_ms || self.workflow.descriptor.expires_at_ms!=old.expires_at_ms + || now/1_000_000>=old.expires_at_ms || self.workflow.durable_windows>()!=old.accepted_seal_digests){ + return Err("temporal_retry_confirmed_seals_changed".into()); + } + let n=self.workflow.session.rows.len(); + let budget=self.workflow.full_run.as_ref().ok_or("temporal_full_run_required")?.policy.replay_fence_budget(&self.workflow.session.managed.as_ref().unwrap().binding)?; + self.workflow.session.assembler.restore_full_run_prefix_fences(old.fences.for_prefix(n)?,n,&budget)?; + for (id,seq,time) in old.source_high { + if time>now {return Err("temporal_retry_receiver_clock_changed".into());} + if let Some((_,old_seq,old_time))=self.workflow.source_high.iter_mut().find(|(source,_,_)|*source==id) { + *old_seq=(*old_seq).max(seq); *old_time=(*old_time).max(time); + } else {self.workflow.source_high.push((id,seq,time));} + } + self.workflow.source_high.sort(); + self.workflow.stage=Stage::PausedNeedsReconfirmation; + self.workflow.last_now_ns=now; + self.failure=None; + Ok(()) + } +} + +#[cfg(test)] +impl TemporalTrainCapture { + pub(crate) fn test_corrupt_retry_prefix(&mut self){self.workflow.async_state.as_mut().unwrap().row_prefix_sha256=Some("f".repeat(64));} +} + +#[cfg(test)] +#[path = "receiver_failure_annotation_tests.rs"] +mod receiver_failure_annotation_tests; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_checkpoint.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_checkpoint.rs new file mode 100644 index 0000000000..675573dc27 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_checkpoint.rs @@ -0,0 +1,930 @@ +//! Typed complete native-prefix payload INSIDE existing RecoveryStore AEAD. +//! This never issues operator confirmation, scientific quality, or fit authority. +use super::*; +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct Prefix { + schema: String, + binding: ManagedBinding, + run: String, + descriptor: RecoveryDescriptor, + last_now_ns: u64, + rows: Vec, + evidence: Vec, + identities: Vec, + fences: AssemblerCheckpoint, + source_high: Vec<(u8, u64, u64)>, + ledger: Ledger, + mapping: Option, + episode: usize, + frozen_model: Option, + quality_issued: bool, + confirmation_issued: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + bridge: Option, +} +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Envelope

{ + schema: String, + payload_sha256: String, + payload: P, +} +const SCHEMA: &str = "ruview.room-learner.temporal-runtime-prefix.v1"; +const BRIDGE_SCHEMA: &str = "ruview.room-learner.temporal-runtime-prefix.v2"; +pub(super) type OldSnapshot = Prefix, SharedRecord>; +#[path="temporal_full_run_checkpoint.rs"] +mod full_checkpoint; +pub(super) enum Snapshot { + Old(OldSnapshot), + Full(full_checkpoint::FullSnapshot), +} +#[derive(Clone)] +pub(super) struct SharedRecords { + rows: Vec>, + evidence: Vec>, +} +pub(super) fn snapshot_records(snapshot:&Snapshot)->SharedRecords { + match snapshot { + Snapshot::Old(snapshot)=>SharedRecords{rows:snapshot.rows.clone(),evidence:snapshot.evidence.clone()}, + Snapshot::Full(snapshot)=>snapshot.records(), + } +} +pub(super) fn snapshot_full_permit(snapshot:&mut Snapshot)->Option{ + match snapshot{Snapshot::Old(_)=>None,Snapshot::Full(snapshot)=>snapshot.permit.take()} +} +pub(super) fn workflow_records(w:&TemporalWorkflow)->SharedRecords { + SharedRecords{rows:w.session.rows.clone(),evidence:w.session.evidence.clone()} +} +pub(super) fn workflow_record_prefix(w:&TemporalWorkflow,count:usize)->Result{ + if count>w.session.rows.len()||count>w.session.evidence.len(){return Err("temporal_record_prefix_missing".into());} + Ok(SharedRecords{rows:w.session.rows[..count].to_vec(),evidence:w.session.evidence[..count].to_vec()}) +} +fn reuse_snapshot_records(w:&mut TemporalWorkflow,records:&SharedRecords)->Result<(),String>{ + if w.session.rows.len()!=records.rows.len() || records.rows.len()!=records.evidence.len() + || hash(&w.session.evidence)?!=hash(&records.evidence)? { + return Err("temporal_writer_shared_records_changed".into()); + } + // decode() already validates every decoded row's exact canonical SHA + // against its WindowEvidence. Validate the SAME expected immutable rows + // against those evidence hashes once, then reuse their cached canonical + // hashes on later prefixes. This is not a new hash formula or authority. + for (row,evidence) in records.rows.iter().zip(&records.evidence) { + if row.canonical_sha256()?!=evidence.cohort_sha256 {return Err("temporal_writer_shared_records_changed".into());} + } + // This call executes on the writer. Free validation's decoded raw copies + // here; the receiver completion contains only refs to its SAME immutable + // authoritative rows/evidence plus bounded reducer metadata. + w.session.rows=records.rows.clone();w.session.evidence=records.evidence.clone();Ok(()) +} +pub(super) fn decode_shared( + bytes:&[u8], descriptor:&RecoveryDescriptor, now:u64, records:&SharedRecords, +)->Result{ + // Keep validation and immutable-ref reuse one private admission boundary. + // No caller-provided/mutated workflow can substitute for this decode. + let mut validated=decode(bytes,descriptor,now)?; + reuse_snapshot_records(&mut validated,records)?; + Ok(validated) +} +pub(super) fn snapshot(v: &TemporalWorkflow) -> Result { + if v.full_run.is_some(){return Ok(Snapshot::Full(full_checkpoint::FullSnapshot::from_workflow(v)?));} + Ok(Snapshot::Old(snapshot_old(v)?)) +} +fn snapshot_old(v: &TemporalWorkflow) -> Result { + // The Full operator codec reuses this structural carrier, with explicit + // reserved Validation rows. Old Train-only and old15 roles stay unchanged. + let operator_checks=v.full_run.is_some()&&v.schedule()?.is_operator_eight_step(); + if v.session.fitted + || v.session.failed + || v.session.active + || v.session.rows.iter().any(|r| r.role != DataRole::Train && !(operator_checks&&r.role==DataRole::Validation)) + || v.session.rows.len() != v.ledger.spans.len() + { + return Err("temporal_checkpoint_complete_train_prefix_only".into()); + } + let m = v + .session + .managed + .as_ref() + .ok_or("managed_session_required")?; + let schema = if v.bridge.is_some() { + BRIDGE_SCHEMA + } else { + SCHEMA + }; + let payload = Prefix { + schema: schema.into(), + binding: m.binding.clone(), + run: v.session.run_id.clone(), + descriptor: v.descriptor.clone(), + last_now_ns: v.last_now_ns, + rows: v.session.rows.clone(), + evidence: v.session.evidence.clone(), + identities: m.windows.clone(), + fences: v.session.assembler.fence_checkpoint()?, + source_high: v.source_high.clone(), + ledger: v.ledger.clone(), + mapping: v.mapping.clone(), + episode: v.episode, + frozen_model: None, + quality_issued: false, + confirmation_issued: false, + bridge: v.bridge.clone(), + }; + Ok(payload) +} +pub(super) fn encode_snapshot(payload: Snapshot) -> Result, String> { + match payload { + Snapshot::Old(payload)=>encode_old_snapshot(payload), + Snapshot::Full(payload)=>payload.encode(), + } +} +fn encode_old_snapshot(payload: OldSnapshot) -> Result, String> { + let envelope = Envelope { + schema: payload.schema.clone(), + payload_sha256: hash(&payload)?, + payload, + }; + let bytes = serde_json::to_vec(&envelope).map_err(|_| "temporal_checkpoint_encoding")?; + if bytes.len() > crate::room_learner::recovery_store::MAX_RECOVERY_BYTES { + return Err("temporal_checkpoint_budget".into()); + } + Ok(bytes) +} +pub(super) fn encode(v: &TemporalWorkflow) -> Result, String> { + encode_snapshot(snapshot(v)?) +} +pub(super) fn decode( + bytes: &[u8], + descriptor: &RecoveryDescriptor, + now: u64, +) -> Result { + if full_checkpoint::is_full(bytes)? {return full_checkpoint::decode_structural(bytes,descriptor,now);} + decode_old(bytes,descriptor,now) +} +fn decode_old(bytes:&[u8],descriptor:&RecoveryDescriptor,now:u64)->Result{ + decode_old_contract(bytes,descriptor,now,StructuralDecode::OldTrain) +} +pub(super) fn decode_full_train_projection(bytes:&[u8],descriptor:&RecoveryDescriptor,now:u64,budget:&full_run::ReplayFenceBudget)->Result{ + decode_old_contract(bytes,descriptor,now,StructuralDecode::FullTrain{budget}) +} +fn decode_old_contract(bytes:&[u8],descriptor:&RecoveryDescriptor,now:u64,contract:StructuralDecode<'_>)->Result{ + if bytes.len() > crate::room_learner::recovery_store::MAX_RECOVERY_BYTES { + return Err("temporal_checkpoint_budget".into()); + } + let e: Envelope = serde_json::from_slice(bytes).map_err(|_| "temporal_checkpoint_schema")?; + if !matches!(e.schema.as_str(), SCHEMA | BRIDGE_SCHEMA) + || e.payload.schema != e.schema + || (e.schema == SCHEMA) != e.payload.bridge.is_none() + || e.payload_sha256 != hash(&e.payload)? + || serde_json::to_vec(&e).map_err(|_| "temporal_checkpoint_encoding")? != bytes + { + return Err("temporal_checkpoint_integrity".into()); + } + decode_prefix(e.payload,descriptor,now,contract) +} +/// DATA validation contract only; this never installs fitted/owner/write state. +enum StructuralDecode<'a> { + OldTrain, + FullTrain { budget:&'a full_run::ReplayFenceBudget }, + FullPostFrozen { fit:&'a full_run::VerifiedFitRecordData, held_out_seals:usize, budget:&'a full_run::ReplayFenceBudget }, +} +pub(super) fn decode_full_post_frozen( + prefix:Prefix,descriptor:&RecoveryDescriptor,now:u64, + fit:&full_run::VerifiedFitRecordData,held_out_seals:usize,budget:&full_run::ReplayFenceBudget, +)->Result{ + decode_prefix(prefix,descriptor,now,StructuralDecode::FullPostFrozen{fit,held_out_seals,budget}) +} +fn decode_prefix(p:Prefix,descriptor:&RecoveryDescriptor,now:u64,contract:StructuralDecode<'_>)->Result{ + let schedule = p.binding.contract()?; + let full=matches!(&contract,StructuralDecode::FullPostFrozen{..})||schedule.is_operator_eight_step(); + let frozen=match &contract{ + StructuralDecode::OldTrain|StructuralDecode::FullTrain{..}=>None, + StructuralDecode::FullPostFrozen{fit,..}=>Some(crate::room_learner::managed_schedule_contract::FrozenModelIdentity::from_runtime_verified_artifact(&schedule,fit.body.artifact_sha256.clone())?), + }; + let frozen_for=|episode:usize| -> Option<&crate::room_learner::managed_schedule_contract::FrozenModelIdentity>{ + if schedule.episodes().get(episode).is_some_and(|e|e.role==DataRole::HeldOut){frozen.as_ref()}else{None} + }; + let context = schedule.context(); + if &p.descriptor != descriptor + || p.descriptor.owner_scope != context.owner_scope + || p.descriptor.installation_id != context.installation_id + || p.descriptor.context_digest != context.digest() + || p.last_now_ns / 1_000_000 != descriptor.saved_at_ms + || p.last_now_ns > now + || now / 1_000_000 >= descriptor.expires_at_ms + || descriptor + .expires_at_ms + .checked_sub(descriptor.created_at_ms) + != Some(RECOVERY_TTL_MS) + || p.frozen_model.is_some() + || p.quality_issued + || p.confirmation_issued + || if full {p.episode>schedule.episodes().len()}else{p.episode>=schedule.episodes().len()} + || p.rows.len() > if full {schedule.counts().total_windows}else{schedule.counts().train_windows} + || p.rows.len() != p.evidence.len() + || p.rows.len() != p.identities.len() + || p.rows.len() != p.ledger.spans.len() + || p.ledger.gaps.len() > 128 + || p.source_high.len() > context.sources.len() + || (!p.rows.is_empty() && p.source_high.len() != context.sources.len()) + { + return Err("temporal_checkpoint_binding_or_unsupported_state".into()); + } + if let Some(mapping) = &p.mapping { + if mapping.segment_ordinal >= 128 || mapping.receiver_start_ns > p.last_now_ns { + return Err("temporal_checkpoint_mapping".into()); + } + let bound = + schedule.bind_episode_start(&p.run, p.episode, 0, mapping.segment_ordinal, frozen_for(p.episode))?; + let slot = schedule.episodes()[p.episode] + .slots + .iter() + .find(|s| s.target_offset_ms * 1_000_000 >= mapping.observed_start_ns) + .ok_or("temporal_checkpoint_mapping")?; + mapping.slot_bounds( + &bound, + slot.target_offset_ms * 1_000_000, + slot.deadline_offset_ms * 1_000_000, + )?; + let expected_start = p + .ledger + .gaps + .iter() + .rev() + .find(|g| g.episode == p.episode && g.new_mapping == *mapping) + .map_or(0, |g| g.accepted_observed_ns); + if mapping.observed_start_ns != expected_start { + return Err("temporal_checkpoint_mapping_start".into()); + } + } else if !p.rows.is_empty() { + let valid=match &contract{ + StructuralDecode::OldTrain|StructuralDecode::FullTrain{..}=>p.bridge.as_ref().is_some_and(|b|b.seals.len()==p.episode), + StructuralDecode::FullPostFrozen{held_out_seals,..}=>*held_out_seals<=8 && p.episode==7+held_out_seals && p.rows.len()==64+8*held_out_seals, + }; + if !valid{return Err("temporal_checkpoint_mapping_missing".into());} + } + validate_records( + &schedule, + &p.run, + &p.evidence, + &p.identities, + &p.binding.validation_binding()?, + !full, + )?; + let mut retained = Vec::new(); + let mut prior: Option = None; + for (i, ((row, evidence), span)) in p + .rows + .iter() + .zip(&p.evidence) + .zip(&p.ledger.spans) + .enumerate() + { + validate_cohort(context, row)?; + if row.views.len() != evidence.sources.len() { + return Err("temporal_checkpoint_row_evidence_shape".into()); + } + for (view, source) in row.views.iter().zip(&evidence.sources) { + if view.source_id != source.source_id + || view.samples.len() != source.times_ns.len() + || view + .frame_ids + .iter() + .map(u128::to_string) + .collect::>() + != source.frame_ids + || view.sequence_ids != source.sequence_ids + || source.times_ns.first() != Some(&view.first_time_ns.to_string()) + || source.times_ns.last() != Some(&view.last_time_ns.to_string()) + { + return Err("temporal_checkpoint_original_row_identity".into()); + } + } + let record = &p.identities[i]; + let episode = &schedule.episodes()[record.episode_index]; + let slot = &episode.slots[record.slot_index]; + if hash(row)? != evidence.cohort_sha256 + || row.id != evidence.cohort_id + || row.trial_id != evidence.trial_id + || row.session_id != evidence.session_id + || row.role != if full {episode.role}else{DataRole::Train} + || if let StructuralDecode::FullPostFrozen{fit,..}=&contract { + if i<64 {record.role!=DataRole::Train || record.frozen_artifact_sha256.is_some()} + else {record.role!=DataRole::HeldOut || record.frozen_artifact_sha256.as_ref()!=Some(&fit.body.artifact_sha256)} + }else{false} + || row.label != evidence.label + || span.previous_sha256 != prior + || span.window_sha256 != hash(row)? + || span.mapping != record.mapping + || span.episode != record.episode_index + || span.slot != record.slot_index + || span.observed_end_ns != slot.deadline_offset_ms * 1_000_000 + || span.receiver_end_ns != integer::(&record.end_ns)? + || span.receiver_end_ns > p.last_now_ns + { + return Err("temporal_checkpoint_row_or_span_changed".into()); + } + let expected_start = if i > 0 && p.ledger.spans[i - 1].episode == span.episode { + p.ledger.spans[i - 1].observed_end_ns + } else { + 0 + }; + if span.observed_start_ns != expected_start + || span.observed_end_ns <= span.observed_start_ns + { + return Err("temporal_committed_span_overlap".into()); + } + for source in &evidence.sources { + for ((t, f), seq) in source + .times_ns + .iter() + .zip(&source.frame_ids) + .zip(&source.sequence_ids) + { + retained.push(( + source.source_id, + *seq, + integer::(t)?, + integer::(f)?, + )); + } + } + prior = Some(hash(span)?); + } + for (gap_index, gap) in p.ledger.gaps.iter().enumerate() { + if gap.episode >= schedule.episodes().len() { + return Err("temporal_gap_episode".into()); + } + let bound = schedule.bind_episode_start( + &p.run, + gap.episode, + 0, + gap.new_mapping.segment_ordinal, + frozen_for(gap.episode), + )?; + let next_slot = schedule.episodes()[gap.episode] + .slots + .iter() + .find(|s| s.target_offset_ms * 1_000_000 >= gap.accepted_observed_ns) + .ok_or("temporal_gap_mapping_changed")?; + gap.new_mapping.slot_bounds( + &bound, + next_slot.target_offset_ms * 1_000_000, + next_slot.deadline_offset_ms * 1_000_000, + )?; + if let Some(parent) = &gap.after_prefix_sha256 { + if gap.genesis.is_some() { + return Err("temporal_gap_conflicting_parent".into()); + } + let span = p + .ledger + .spans + .iter() + .find(|s| hash(s).ok().as_ref() == Some(parent)) + .ok_or("temporal_gap_prefix_missing")?; + if gap.episode != span.episode + || gap.accepted_observed_ns != span.observed_end_ns + || gap.previous_receiver_ns != span.receiver_end_ns + { + return Err("temporal_gap_mapping_changed".into()); + } + } else { + let genesis = gap.genesis.as_ref().ok_or("temporal_gap_genesis_missing")?; + let previous = &genesis.previous_mapping; + if gap.accepted_observed_ns != 0 + || previous.observed_start_ns != 0 + || genesis.sha256 != genesis.digest(descriptor, &p.run, gap.episode)? + || genesis.paused_receiver_ns < previous.receiver_start_ns + || gap.previous_receiver_ns != genesis.paused_receiver_ns + || gap.new_mapping.segment_ordinal + != previous + .segment_ordinal + .checked_add(1) + .ok_or("temporal_gap_segment_overflow")? + || p.ledger.spans.iter().any(|s| { + s.episode == gap.episode + && s.mapping.segment_ordinal <= previous.segment_ordinal + }) + { + return Err("temporal_gap_invalid_genesis".into()); + } + let prior_bound = schedule.bind_episode_start( + &p.run, + gap.episode, + 0, + previous.segment_ordinal, + frozen_for(gap.episode), + )?; + let first = &schedule.episodes()[gap.episode].slots[0]; + previous.slot_bounds( + &prior_bound, + first.target_offset_ms * 1_000_000, + first.deadline_offset_ms * 1_000_000, + )?; + if let Some(prior) = p.ledger.gaps[..gap_index] + .iter() + .rev() + .find(|g| g.episode == gap.episode) + { + if &prior.new_mapping != previous { + return Err("temporal_gap_genesis_history_changed".into()); + } + } else if previous.segment_ordinal != 0 { + return Err("temporal_gap_genesis_history_missing".into()); + } + if genesis.source_floors.len() > context.sources.len() { + return Err("temporal_gap_genesis_floors".into()); + } + let mut seen = HashSet::new(); + for (id, seq, t) in &genesis.source_floors { + if !seen.insert(*id) + || !context.sources.iter().any(|s| s.source_id == *id) + || *seq == 0 + || *t > genesis.paused_receiver_ns + || !p + .source_high + .iter() + .any(|(s, q, n)| s == id && q >= seq && n >= t) + { + return Err("temporal_gap_genesis_floors".into()); + } + } + } + if gap.resume_receiver_ns < gap.previous_receiver_ns + || gap.new_mapping.observed_start_ns != gap.accepted_observed_ns + || gap.new_mapping.receiver_start_ns != gap.resume_receiver_ns + || gap.new_mapping.clock_anchor_ns < gap.previous_receiver_ns + { + return Err("temporal_gap_mapping_changed".into()); + } + } + if let Some(bridge) = &p.bridge { + if !full || schedule.is_operator_eight_step() { + if matches!(contract,StructuralDecode::FullTrain{..}){ + let max_seals=if schedule.is_operator_eight_step(){schedule.episodes().len()}else{schedule.counts().train_episodes}; + if bridge.seals.len()>max_seals{return Err("temporal_bridge_phase_order".into());} + let mut count=0;let cuts=bridge.seals.iter().enumerate().map(|(i,_)|{count+=schedule.episodes()[i].slots.len();count}).collect::>(); + let digests=prefix_digests::canonical_tuple_prefixes(&p.rows,&p.evidence,&p.ledger.spans,&cuts)?; + bridge.validate_full_run_with_prefixes(&schedule,&p.run,&p.rows,&p.evidence,&p.ledger,p.episode,p.mapping.as_ref(),p.last_now_ns,&digests)?; + }else{bridge.validate(&schedule,&p.run,&p.rows,&p.evidence,&p.ledger,p.episode,p.mapping.as_ref(),p.last_now_ns)?;} + } + // Whole-period quality also observes the reservation tail and packets + // between retained strata. Those durably acknowledged observations + // must not disappear from the replay floors merely because they are + // not samples in a retained native window. + for part in bridge + .seals + .iter() + .flat_map(|seal| &seal.parts) + .chain(&bridge.parts) + { + for observed in &part.sources { + if !p.source_high.iter().any(|(id, sequence, time)| { + *id == observed.source_id + && *sequence >= observed.last_sequence + && *time >= observed.last_ns + }) { + return Err("temporal_quality_floors_regressed".into()); + } + } + } + } + let mut session = + TemporalTrainingSession::managed(schedule.clone(), p.run, p.binding.fit.clone())?; + session.rows = p.rows.into_iter().map(SharedRecord::new).collect(); + session.evidence = p.evidence.into_iter().map(SharedRecord::new).collect(); + session.managed.as_mut().unwrap().windows = p.identities; + match &contract{ + StructuralDecode::OldTrain=>session.assembler.restore_prefix_fences(p.fences,session.rows.len())?, + StructuralDecode::FullTrain{budget}|StructuralDecode::FullPostFrozen{budget,..}=>session.assembler.restore_full_run_prefix_fences(p.fences,session.rows.len(),budget)?, + } + if !session.assembler.retained_fences_cover(&retained) { + return Err("temporal_checkpoint_fences_regressed".into()); + } + let mut seen = HashSet::new(); + for (id, seq, t) in &p.source_high { + if !seen.insert(*id) + || !context.sources.iter().any(|s| s.source_id == *id) + || *seq == 0 + || *t > p.last_now_ns + || retained + .iter() + .any(|(s, q, n, _)| s == id && (*q > *seq || *n > *t)) + { + return Err("temporal_checkpoint_source_floor".into()); + } + } + let accepted = p + .ledger + .spans + .last() + .filter(|s| s.episode == p.episode) + .map_or(0, |s| s.observed_end_ns); + let durable_windows = session.rows.len(); + Ok(TemporalWorkflow { + full_run: None, + session, + descriptor: p.descriptor, + ledger: p.ledger, + stage: Stage::PausedNeedsReconfirmation, + mapping: p.mapping, + episode: p.episode, + last_now_ns: p.last_now_ns, + live_observed_ns: accepted, + source_high: p.source_high, + pending_window: None, + durable: None, + durable_windows, + custody: SegmentCustody::default(), + commit_token: None, + pending_intent: None, + pending_prefix: None, + bridge: p.bridge, + async_state: None, + }) +} +#[cfg(test)] +#[path = "temporal_checkpoint_tests.rs"] +mod tests; + +/// Opaque, crate-local storage input; generic JSON cannot claim validity/proof. +#[derive(Clone)] +pub(super) enum ValidatedFullStage { + Train, + Frozen(crate::room_learner::recovery_store::temporal_commitment::TemporalCommitmentData), + HeldOut, + Complete(crate::room_learner::recovery_store::temporal_commitment::TemporalCommitmentData), +} +impl ValidatedFullStage{ + fn predecessor(&self)->Option<&crate::room_learner::recovery_store::temporal_commitment::TemporalCommitmentData>{match self{Self::Frozen(p)|Self::Complete(p)=>Some(p),_=>None}} + fn requires_model(&self)->bool{!matches!(self,Self::Train)} +} +#[derive(Clone)] +pub(crate) struct ValidatedTemporalPrefix { + bytes: Vec, + descriptor: RecoveryDescriptor, + run: String, + policy_sha256: String, + window_count: usize, + transition_limit: usize, + saved_receiver_ns: u64, + metadata_kind: &'static str, + full_write_grant: Option, + full_stage: Option, + full_fit_record_sha256:Option, +} +pub(crate) fn validated_prefix( + bytes: &[u8], + descriptor: &RecoveryDescriptor, + now_ns: u64, +) -> Result { + if full_checkpoint::is_full(bytes)?{return full_checkpoint::validated_structural(bytes,descriptor,now_ns);} + let v = decode(bytes, descriptor, now_ns)?; + validated_old_from_workflow(bytes,descriptor,v) +} +pub(super) fn validated_full_train_projection(bytes:&[u8],descriptor:&RecoveryDescriptor,now:u64,budget:&full_run::ReplayFenceBudget)->Result{ + let v=decode_full_train_projection(bytes,descriptor,now,budget)?; + validated_old_from_workflow(bytes,descriptor,v) +} +fn validated_old_from_workflow(bytes:&[u8],descriptor:&RecoveryDescriptor,v:TemporalWorkflow)->Result{ + let schedule = v.schedule()?; + // Historical profiles retain their ORIGINAL limit/hash; only an explicit + // fresh-Train genesis can select the recognized343 budget profile. + let profile=match &v.bridge{Some(bridge)=>bridge.profile()?,None=>bridge::TrainBudgetProfile::Historical329}; + let transition_limit=profile.transition_limit(&schedule)?; + if transition_limit > 512 { + return Err("temporal_transition_capacity".into()); + } + Ok(ValidatedTemporalPrefix { + bytes: bytes.to_vec(), + descriptor: descriptor.clone(), + run: v.session.run_id.clone(), + policy_sha256: if v.bridge.is_some() { + hash(&(v.session.plan_sha256(), profile.policy()))? + } else { + v.session.plan_sha256().into() + }, + window_count: v.session.rows.len(), + transition_limit, + saved_receiver_ns: v.last_now_ns, + metadata_kind:"temporal-prefix-commitment-v1", + full_write_grant:None, + full_stage:None,full_fit_record_sha256:None, + }) +} +pub(crate) fn validate_extension( + old: &ValidatedTemporalPrefix, + new: &ValidatedTemporalPrefix, +) -> Result<(), String> { + if old.metadata_kind!=new.metadata_kind{return Err("temporal_prefix_generation_kind_changed".into());} + if old.metadata_kind==full_run::META_KIND{return full_checkpoint::validate_full_extension(old,new);} + let a: Envelope = + serde_json::from_slice(&old.bytes).map_err(|_| "temporal_checkpoint_schema")?; + let b: Envelope = + serde_json::from_slice(&new.bytes).map_err(|_| "temporal_checkpoint_schema")?; + let a = a.payload; + let b = b.payload; + if old.run != new.run + || old.policy_sha256 != new.policy_sha256 + || old.transition_limit != new.transition_limit + || old.descriptor.id != new.descriptor.id + || old.descriptor.owner_scope != new.descriptor.owner_scope + || old.descriptor.installation_id != new.descriptor.installation_id + || old.descriptor.context_digest != new.descriptor.context_digest + || old.descriptor.created_at_ms != new.descriptor.created_at_ms + || old.descriptor.expires_at_ms != new.descriptor.expires_at_ms + || new.descriptor.saved_at_ms < old.descriptor.saved_at_ms + || b.last_now_ns < a.last_now_ns + { + return Err("temporal_commit_identity_or_time_changed".into()); + } + macro_rules! prefix { + ($old:expr,$new:expr) => { + if $new.len() < $old.len() + || serde_json::to_vec(&$new[..$old.len()]).map_err(|_| "temporal_encoding")? + != serde_json::to_vec(&$old).map_err(|_| "temporal_encoding")? + { + return Err("temporal_committed_prefix_changed".into()); + } + }; + } + prefix!(a.rows, b.rows); + prefix!(a.evidence, b.evidence); + prefix!(a.identities, b.identities); + prefix!(a.ledger.spans, b.ledger.spans); + prefix!(a.ledger.gaps, b.ledger.gaps); + for (id, seq, t) in &a.source_high { + if !b + .source_high + .iter() + .any(|(s, q, n)| s == id && q >= seq && n >= t) + { + return Err("temporal_commit_floor_regressed".into()); + } + } + if old.bytes == new.bytes { + return Ok(()); + } + let window = b.rows.len() == a.rows.len() + 1 + && b.ledger.gaps.len() == a.ledger.gaps.len() + && b.mapping == a.mapping; + let segment = b.rows.len() == a.rows.len() + && ((a.mapping.is_none() + && b.mapping.is_some() + && b.ledger.gaps.len() == a.ledger.gaps.len()) + || (b.ledger.gaps.len() == a.ledger.gaps.len() + 1)); + match (&a.bridge, &b.bridge) { + (None, None) => { + if !window && !segment { + return Err("temporal_transition_without_window_or_segment".into()); + } + } + (Some(old), Some(new)) => { + prefix!(old.seals, new.seals); + let seal = b.rows.len() == a.rows.len() + && b.episode == a.episode + 1 + && b.ledger.gaps == a.ledger.gaps + && b.mapping.is_none() + && new.seals.len() == old.seals.len() + 1 + && new.parts.is_empty() + && new + .seals + .last() + .is_some_and(|s| s.parts == old.parts && s.confirmed_at_ns >= a.last_now_ns); + let pause = b.rows.len() == a.rows.len() + && b.episode == a.episode + && b.mapping == a.mapping + && b.ledger.gaps == a.ledger.gaps + && new.seals == old.seals + && new.parts == old.parts + && new.disposition == bridge::Disposition::Paused; + if seal { + return Ok(()); + } + if new.seals != old.seals || b.episode != a.episode { + return Err("temporal_bridge_seal_transition".into()); + } + if window { + if new.parts.len() != old.parts.len() || old.parts.is_empty() { + return Err("temporal_quality_part_transition".into()); + } + let n = old.parts.len() - 1; + if old.parts[..n] != new.parts[..n] + || old.parts[n].mapping != new.parts[n].mapping + || new.parts[n].observed_end_ns < old.parts[n].observed_end_ns + { + return Err("temporal_quality_prefix_changed".into()); + } + for source in &old.parts[n].sources { + if !new.parts[n].sources.iter().any(|s| { + s.source_id == source.source_id + && s.first_ns == source.first_ns + && s.first_sequence == source.first_sequence + && s.packets >= source.packets + && s.last_ns >= source.last_ns + && s.last_sequence >= source.last_sequence + && s.maximum_gap_ns >= source.maximum_gap_ns + && s.minimum_interval_ns <= source.minimum_interval_ns + }) { + return Err("temporal_quality_counts_regressed".into()); + } + } + } else if segment { + if new.parts.len() != old.parts.len() + 1 + || new.parts[..old.parts.len()] != old.parts + || new.parts.last().is_none_or(|p| { + !p.sources.is_empty() || p.observed_end_ns != p.mapping.observed_start_ns + }) + { + return Err("temporal_quality_segment_changed".into()); + } + } else if !pause { + return Err("temporal_bridge_transition_invalid".into()); + } + } + _ => return Err("temporal_bridge_kind_changed".into()), + } + Ok(()) +} + +impl ValidatedTemporalPrefix { + pub(crate) fn metadata_kind(&self)->&str{self.metadata_kind} + pub(crate) fn authorizes_write(&self,expected:Option<&TemporalToken>)->bool{ + self.metadata_kind!=full_run::META_KIND||(self.full_write_grant.as_ref().is_some_and(|g|g.permits(&self.bytes,expected)) + &&self.full_stage.as_ref().and_then(|s|s.predecessor()).is_none_or(|p|expected.is_some_and(|e|e.data()==*p))) + } + pub(super) fn verify_full_artifact(&self)->Result<(),String>{ + if self.metadata_kind==full_run::META_KIND&&self.full_stage.as_ref().is_some_and(|s|s.requires_model()){ + return self.full_write_grant.as_ref().ok_or("temporal_frozen_producer_model_required")?.verify_frozen_model(); + } + Ok(()) + } + pub(super) fn continued_witness(&self,w:&TemporalWorkflow,token:&TemporalToken)->Result,String>{ + if !matches!(self.full_stage,Some(ValidatedFullStage::HeldOut|ValidatedFullStage::Complete(_))){return Ok(None);} + self.full_write_grant.as_ref().ok_or("temporal_continued_producer_required")?.continued_witness(w,token) + } + pub(super) fn attach_full_grant(&mut self,grant:full_run::FullRunWriteGrant)->Result<(),String>{ + if self.metadata_kind!=full_run::META_KIND{return Err("temporal_full_run_grant_kind".into());} + if !grant.matches_fit_record(self.full_fit_record_sha256.as_deref()){return Err("temporal_full_run_grant_model_record_changed".into());} + self.full_write_grant=Some(grant);Ok(()) + } + pub(crate) fn saved_receiver_ns(&self) -> u64 { + self.saved_receiver_ns + } + pub(crate) fn bytes(&self) -> &[u8] { + &self.bytes + } + pub(crate) fn descriptor(&self) -> &RecoveryDescriptor { + &self.descriptor + } + pub(crate) fn run(&self) -> &str { + &self.run + } + pub(crate) fn policy_sha256(&self) -> &str { + &self.policy_sha256 + } + pub(crate) fn window_count(&self) -> usize { + self.window_count + } + pub(crate) fn transition_limit(&self) -> usize { + self.transition_limit + } +} + +pub(crate) fn is_new_empty_run(p: &ValidatedTemporalPrefix) -> Result { + if p.metadata_kind==full_run::META_KIND{return full_checkpoint::is_new_full_run(p);} + let e: Envelope = + serde_json::from_slice(p.bytes()).map_err(|_| "temporal_checkpoint_schema")?; + let q = e.payload; + let schedule = q.binding.contract()?; + let fresh = TemporalTrainingSession::managed(schedule, q.run.clone(), q.binding.fit.clone())?; + Ok(q.rows.is_empty() + && q.evidence.is_empty() + && q.identities.is_empty() + && q.ledger.spans.is_empty() + && q.ledger.gaps.is_empty() + && q.source_high.is_empty() + && q.mapping.is_none() + && q.episode == 0 + && q.bridge.as_ref().is_none_or(|b| { + b.seals.is_empty() && b.parts.is_empty() && b.disposition == bridge::Disposition::Ready + }) + && serde_json::to_vec(&q.fences).map_err(|_| "temporal_encoding")? + == serde_json::to_vec(&fresh.assembler.fence_checkpoint()?) + .map_err(|_| "temporal_encoding")?) +} + +#[cfg(test)] +pub(crate) fn synthetic_storage_prefixes() -> Vec { + let (_dir, store, mut w) = super::tests::setup(); + let mut out = Vec::new(); + let add = |w: &TemporalWorkflow| { + let bytes = encode(w).unwrap(); + validated_prefix(&bytes, &w.descriptor, w.last_now_ns).unwrap() + }; + out.push(add(&w)); + w.persist_initial(&store).unwrap(); + super::tests::start(&mut w, &store, 2_000_000_000, "one"); + out.push(add(&w)); + let mut seq = 0; + let next = super::tests::commit_next(&mut w, &store, 2_000_000_000, &mut seq); + out.push(add(&w)); + super::tests::commit_next(&mut w, &store, next, &mut seq); + out.push(add(&w)); + out +} +#[cfg(test)] +pub(crate) fn synthetic_segment_fork(initial: &ValidatedTemporalPrefix) -> ValidatedTemporalPrefix { + let mut w = decode( + initial.bytes(), + initial.descriptor(), + initial.descriptor().saved_at_ms * 1_000_000, + ) + .unwrap(); + w.stage = Stage::DurableReady; + w.durable = Some(std::sync::Arc::new(initial.bytes().to_vec())); + let p = w.synthetic_conditions(); + w.start_segment(&p, "different-segment", 2_000_000_000, 2_000_000_000) + .unwrap(); + w.descriptor.saved_at_ms = w.last_now_ns / 1_000_000; + let b = encode(&w).unwrap(); + validated_prefix(&b, &w.descriptor, w.last_now_ns).unwrap() +} +#[cfg(test)] +pub(crate) fn synthetic_changed_old_row( + input: &ValidatedTemporalPrefix, +) -> ValidatedTemporalPrefix { + let mut e: Envelope = serde_json::from_slice(input.bytes()).unwrap(); + e.payload.rows[0].views[0].samples[0][0] += 1.; + e.payload.evidence[0].cohort_sha256 = hash(&e.payload.rows[0]).unwrap(); + e.payload.ledger.spans[0].window_sha256 = hash(&e.payload.rows[0]).unwrap(); + for i in 1..e.payload.ledger.spans.len() { + e.payload.ledger.spans[i].previous_sha256 = + Some(hash(&e.payload.ledger.spans[i - 1]).unwrap()); + } + e.payload_sha256 = hash(&e.payload).unwrap(); + let b = serde_json::to_vec(&e).unwrap(); + validated_prefix(&b, &e.payload.descriptor, e.payload.last_now_ns).unwrap() +} +#[cfg(test)] +pub(crate) fn synthetic_new_empty_run( + input: &ValidatedTemporalPrefix, + created: u64, +) -> ValidatedTemporalPrefix { + let mut e: Envelope = serde_json::from_slice(input.bytes()).unwrap(); + assert!(e.payload.rows.is_empty()); + e.payload.run = "synthetic-next-run".into(); + e.payload.descriptor.id = "synthetic-new-recovery-id".into(); + e.payload.descriptor.created_at_ms = created; + e.payload.descriptor.saved_at_ms = created; + e.payload.descriptor.expires_at_ms = created + RECOVERY_TTL_MS; + e.payload.last_now_ns = created * 1_000_000; + e.payload_sha256 = hash(&e.payload).unwrap(); + let b = serde_json::to_vec(&e).unwrap(); + validated_prefix(&b, &e.payload.descriptor, e.payload.last_now_ns).unwrap() +} +#[cfg(test)] +pub(crate) fn synthetic_floor_regression( + input: &ValidatedTemporalPrefix, +) -> ValidatedTemporalPrefix { + let mut w = decode(input.bytes(), input.descriptor(), input.saved_receiver_ns()).unwrap(); + w.durable = Some(std::sync::Arc::new(input.bytes().to_vec())); + let permit = w.synthetic_conditions(); + let start = input.saved_receiver_ns() + 10_000_000_000; + w.start_segment(&permit, "floor-regression", start, start) + .unwrap(); + w.descriptor.saved_at_ms = w.last_now_ns / 1_000_000; + let b = encode(&w).unwrap(); + let mut e: Envelope = serde_json::from_slice(&b).unwrap(); + let source = &e.payload.evidence.last().unwrap().sources[0]; + let minimum = *source.sequence_ids.last().unwrap(); + let old = e.payload.source_high[0].1; + assert!(old > minimum); + e.payload.source_high[0].1 = minimum; + e.payload_sha256 = hash(&e.payload).unwrap(); + let b = serde_json::to_vec(&e).unwrap(); + validated_prefix(&b, &e.payload.descriptor, e.payload.last_now_ns).unwrap() +} + +// Test-only typed fixture: create a NEW empty run and encode its explicit +// profile through the same strict codec; never upgrades an existing prefix. +#[cfg(test)] +pub(crate) fn synthetic_fresh_train343_empty_run(input:&ValidatedTemporalPrefix,created:u64)->ValidatedTemporalPrefix{ + let empty=synthetic_new_empty_run(input,created); + let mut workflow=decode(empty.bytes(),empty.descriptor(),empty.saved_receiver_ns()).unwrap(); + workflow.bridge=Some(super::bridge::BridgeState::fresh_train_343()); + let bytes=encode(&workflow).unwrap();validated_prefix(&bytes,&workflow.descriptor,workflow.last_now_ns).unwrap() +} + +#[cfg(test)] +pub(crate) fn c4b_reference_batch_and_stale_seal_probe(bytes:&[u8],descriptor:&RecoveryDescriptor,now:u64)->Result{ + full_checkpoint::c4b_reference_batch_and_stale_seal_probe(bytes,descriptor,now) +} + +#[cfg(test)] +pub(crate) fn c4b_complete_fault_matrix(bytes:&[u8],descriptor:&RecoveryDescriptor,now:u64)->Result{full_checkpoint::c4b_complete_fault_matrix(bytes,descriptor,now)} + +// Reuse the existing whole-prefix Train validator for C4C; this DATA check +// cannot construct a snapshot or owner/write/evaluation capability. +pub(super) fn validate_temporal_evaluation_train_prefix(w:&TemporalWorkflow,record:&full_run::VerifiedFitRecordData)->Result<(),String>{full_checkpoint::validate_immutable_train(w,record)} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_checkpoint_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_checkpoint_tests.rs new file mode 100644 index 0000000000..5ca5f4d414 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_checkpoint_tests.rs @@ -0,0 +1,333 @@ +use super::super::tests::{commit_next, setup, start}; +use super::*; +#[test] +fn real_aead_initial_barrier_and_ambiguous_save_fail_closed_without_new_authority() { + let (_dir, store, mut w) = setup(); + assert!(w + .persist_with(&store, false, |_, _, _, _| Err( + "injected-before-save".into() + )) + .is_err()); + assert_eq!(w.stage, Stage::BlockedDurability); + assert!(store.load_temporal(w.last_now_ns).unwrap().is_none()); + assert!(w.begin_slot().is_err()); + let (_dir, store, mut w) = setup(); + let result = w.persist_with(&store, false, |s, d, b, n| { + s.publish_temporal(d, Some(b), n)?; + Err("injected-after-real-save-before-ack".into()) + }); + assert!(result.is_err()); + assert_eq!(w.stage, Stage::BlockedDurability); + assert!(w.begin_slot().is_err()); + let loaded = store + .load_temporal(w.last_now_ns) + .unwrap() + .unwrap() + .prefix + .unwrap(); + assert!(decode(loaded.bytes(), loaded.descriptor(), w.last_now_ns).is_ok()); + let c = crate::room_learner::temporal_model_input::managed::tests::context(); + let encrypted = std::fs::read( + _dir.path() + .join(&c.owner_scope) + .join("SensingServer/room-learner-recovery/temporal-prefix-commitment-v1/slot-0.bin"), + ) + .unwrap(); + assert!(encrypted.starts_with(b"RVTPAY01")); + assert!(serde_json::from_slice::(&encrypted).is_err()); + assert!(!encrypted + .windows(SCHEMA.len()) + .any(|w| w == SCHEMA.as_bytes())); +} +#[test] +fn rehashed_checkpoint_role_policy_epoch_fence_expiry_and_fake_receipt_tampering_fail() { + let (_dir, store, mut w) = setup(); + w.persist_initial(&store).unwrap(); + start(&mut w, &store, 2_000_000_000, "one"); + let mut seq = 0; + commit_next(&mut w, &store, 2_000_000_000, &mut seq); + let bytes = w.durable.as_ref().unwrap(); + for case in 0..9 { + let mut e: Envelope = serde_json::from_slice(bytes).unwrap(); + match case { + 0 => e.payload.rows[0].role = DataRole::HeldOut, + 1 => e.payload.binding.target_mapping = "old-uninterrupted".into(), + 2 => e.payload.identities[0].mapping.receiver_epoch = "".into(), + 3 => e.payload.source_high[0].1 = 1, + 4 => e.payload.descriptor.expires_at_ms += 1, + 5 => e.payload.frozen_model = Some("a".repeat(64)), + 6 => e.payload.quality_issued = true, + 7 => e.payload.ledger.spans[0].observed_start_ns = 1, + _ => { + e.payload.rows[0].views[0].frame_ids[0] += 1; + e.payload.evidence[0].cohort_sha256 = hash(&e.payload.rows[0]).unwrap(); + e.payload.ledger.spans[0].window_sha256 = hash(&e.payload.rows[0]).unwrap(); + } + }; + e.payload_sha256 = hash(&e.payload).unwrap(); + assert!( + decode( + &serde_json::to_vec(&e).unwrap(), + &w.descriptor, + w.last_now_ns + ) + .is_err(), + "case{case}" + ); + } + let expired = w.descriptor.expires_at_ms * 1_000_000; + assert!(decode(bytes, &w.descriptor, expired).is_err()); +} + +#[test] +fn actual_encrypted_file_corruption_during_readback_blocks_collection() { + let (dir, store, mut w) = setup(); + let c = crate::room_learner::temporal_model_input::managed::tests::context(); + let file = dir + .path() + .join(&c.owner_scope) + .join("SensingServer/room-learner-recovery/temporal-prefix-commitment-v1/slot-0.bin"); + assert!(w + .persist_with(&store, false, |s, d, b, n| { + let token = s.publish_temporal(d, Some(b), n)?; + let mut encrypted = std::fs::read(&file).unwrap(); + let i = encrypted.len() - 1; + encrypted[i] ^= 1; + std::fs::write(&file, encrypted).unwrap(); + Ok(token) + }) + .is_err()); + assert_eq!(w.stage, Stage::BlockedDurability); + assert!(w.begin_slot().is_err()); + assert!(store.load_temporal(w.last_now_ns).is_err()); +} + +#[test] +fn failed_completed_window_barrier_never_advances_promised_durable_prefix() { + let (_dir, store, mut w) = setup(); + w.persist_initial(&store).unwrap(); + start(&mut w, &store, 2_000_000_000, "one"); + let mut seq = 0; + let target = w.begin_slot().unwrap(); + super::super::tests::drive(&mut w, 2_000_000_000, target.end_ns(), &mut seq); + assert!(w + .persist_with(&store, true, |_, _, _, _| Err( + "injected-window-save-failure".into() + )) + .is_err()); + assert_eq!(w.accepted(), 0); + assert_eq!(w.stage, Stage::BlockedDurability); +} + +#[test] +fn credited_prefix_cannot_be_replaced_by_a_rehashed_genesis_parent() { + let (_dir, store, mut w) = setup(); + w.persist_initial(&store).unwrap(); + start(&mut w, &store, 2_000_000_000, "initial"); + let mut seq = 0; + let next = commit_next(&mut w, &store, 2_000_000_000, &mut seq); + w.pause().unwrap(); + start(&mut w, &store, next + 10_000_000_000, "resumed"); + let mut e: Envelope = serde_json::from_slice(w.durable.as_ref().unwrap()).unwrap(); + let old_mapping = e.payload.ledger.spans[0].mapping.clone(); + let paused = e.payload.last_now_ns; + let anchor = GenesisGap::new( + &e.payload.descriptor, + &e.payload.run, + 0, + old_mapping, + paused, + e.payload.source_high.clone(), + ) + .unwrap(); + let gap = &mut e.payload.ledger.gaps[0]; + gap.after_prefix_sha256 = None; + gap.genesis = Some(anchor); + gap.accepted_observed_ns = 0; + gap.previous_receiver_ns = paused; + gap.new_mapping.observed_start_ns = 0; + e.payload.mapping = Some(gap.new_mapping.clone()); + e.payload_sha256 = hash(&e.payload).unwrap(); + let err = decode( + &serde_json::to_vec(&e).unwrap(), + &w.descriptor, + w.last_now_ns, + ) + .err() + .expect("missing parent after credited prefix must reject"); + assert_eq!(err, "temporal_gap_invalid_genesis"); +} + +#[test] +fn zero_prefix_anchor_binds_original_run_clock_and_source_floor_history() { + let (_dir, store, mut w) = setup(); + w.persist_initial(&store).unwrap(); + start(&mut w, &store, 2_000_000_000, "initial"); + let target = w.begin_slot().unwrap(); + let mut seq = 0; + let next = super::super::tests::drive( + &mut w, + 2_000_000_000, + target.target_ns() + 1_000_000_000, + &mut seq, + ); + w.pause().unwrap(); + start(&mut w, &store, next + 10_000_000_000, "resumed"); + let bytes = w.durable.as_ref().unwrap(); + for case in 0..4 { + let mut e: Envelope = serde_json::from_slice(bytes).unwrap(); + match case { + 0 => e.payload.run = "foreign-run".into(), + 1 => { + e.payload.ledger.gaps[0] + .genesis + .as_mut() + .unwrap() + .paused_receiver_ns = 0 + } + 2 => { + e.payload.ledger.gaps[0] + .genesis + .as_mut() + .unwrap() + .source_floors[0] + .1 = 1 + } + _ => e.payload.ledger.gaps[0].genesis = None, + }; + e.payload_sha256 = hash(&e.payload).unwrap(); + assert!( + decode( + &serde_json::to_vec(&e).unwrap(), + &w.descriptor, + w.last_now_ns + ) + .is_err(), + "case{case}" + ); + } +} + +#[test] +fn bridge_prefix_version_is_strict_and_rehashed_quality_tampering_is_rejected() { + let (_dir, store, mut capture) = super::super::tests::bridge_setup(); + capture + .start_segment("bridge-checkpoint", 2_000_000_000, &store) + .unwrap(); + let mut sequence = 0; + let now = super::super::tests::drive_bridge( + &mut capture, + &store, + 2_000_000_000, + 5_000_000_000, + &mut sequence, + ); + let loaded = store.load_temporal(now).unwrap().unwrap(); + let prefix = loaded.prefix.unwrap(); + let mut envelope: Envelope = serde_json::from_slice(prefix.bytes()).unwrap(); + assert_eq!(envelope.schema, BRIDGE_SCHEMA); + assert!(!envelope.payload.quality_issued && !envelope.payload.confirmation_issued); + envelope.payload.bridge.as_mut().unwrap().parts[0].sources[0].packets = 1; + envelope.payload_sha256 = hash(&envelope.payload).unwrap(); + let bytes = serde_json::to_vec(&envelope).unwrap(); + assert!(validated_prefix(&bytes, prefix.descriptor(), now).is_err()); +} +#[test] +fn a_rehashed_prefix_cannot_claim_an_unearned_next_train_phase() { + let (_dir, store, capture) = super::super::tests::bridge_setup(); + let now = 1_000_000_000; + let loaded = store.load_temporal(now).unwrap().unwrap(); + let prefix = loaded.prefix.unwrap(); + let mut envelope: Envelope = serde_json::from_slice(prefix.bytes()).unwrap(); + envelope.payload.episode = 1; + envelope.payload_sha256 = hash(&envelope.payload).unwrap(); + assert!(validated_prefix( + &serde_json::to_vec(&envelope).unwrap(), + prefix.descriptor(), + now + ) + .is_err()); + assert_eq!(capture.snapshot()["sealed_train_episodes"], 0); +} + +#[test] +fn bridge_quality_observed_tail_must_remain_covered_by_durable_replay_floors() { + let (_dir, store, mut capture) = super::super::tests::bridge_setup(); + capture + .start_segment("quality-floor", 2_000_000_000, &store) + .unwrap(); + let mut sequence = 0; + let now = super::super::tests::drive_bridge( + &mut capture, + &store, + 2_000_000_000, + 5_000_000_000, + &mut sequence, + ); + let prefix = store.load_temporal(now).unwrap().unwrap().prefix.unwrap(); + let mut envelope: Envelope = serde_json::from_slice(prefix.bytes()).unwrap(); + let view = &envelope.payload.rows.last().unwrap().views[0]; + let observed = envelope + .payload + .bridge + .as_ref() + .unwrap() + .parts + .last() + .unwrap() + .sources + .iter() + .find(|s| s.source_id == view.source_id) + .unwrap(); + assert!( + observed.last_sequence > *view.sequence_ids.last().unwrap(), + "the actual reservation tail must include packets after the retained native horizon" + ); + let floor = envelope + .payload + .source_high + .iter_mut() + .find(|s| s.0 == view.source_id) + .unwrap(); + floor.1 = *view.sequence_ids.last().unwrap(); + floor.2 = view.last_time_ns; + envelope.payload_sha256 = hash(&envelope.payload).unwrap(); + let bytes = serde_json::to_vec(&envelope).unwrap(); + assert!( + validated_prefix(&bytes, prefix.descriptor(), now).is_err(), + "rehashed valid rows cannot hide replay-floor rollback below the observed quality tail" + ); +} + +#[test] +fn writer_readback_reuses_exact_authoritative_arc_records_and_rejects_changed_records(){ + let(_dir,store,mut workflow)=setup();workflow.persist_initial(&store).unwrap(); + start(&mut workflow,&store,2_000_000_000,"synthetic-shared-records");let mut seq=0; + commit_next(&mut workflow,&store,2_000_000_000,&mut seq); + let frozen=snapshot(&workflow).unwrap();let records=snapshot_records(&frozen); + let bytes=encode_snapshot(frozen).unwrap();let mut restored=decode(&bytes,&workflow.descriptor,workflow.last_now_ns).unwrap(); + assert!(!restored.session.rows[0].same_record(&workflow.session.rows[0])); + reuse_snapshot_records(&mut restored,&records).unwrap(); + assert!(restored.session.rows[0].same_record(&workflow.session.rows[0])); + assert!(restored.session.evidence[0].same_record(&workflow.session.evidence[0])); + assert_eq!(encode(&restored).unwrap(),bytes); + let mut changed=records.clone();let mut row=(*changed.rows[0]).clone();row.label=RoomLabel::Unknown; + changed.rows[0]=SharedRecord::new(row); + assert_eq!(reuse_snapshot_records(&mut restored,&changed).unwrap_err(),"temporal_writer_shared_records_changed"); + assert!(restored.session.rows[0].same_record(&workflow.session.rows[0])); +} + +#[test] +fn old_train_snapshot_still_refuses_validation_heldout_and_inference_rows(){ + let(_dir,store,mut workflow)=setup();workflow.persist_initial(&store).unwrap(); + start(&mut workflow,&store,2_000_000_000,"synthetic-old-role-boundary");let mut sequence=0; + commit_next(&mut workflow,&store,2_000_000_000,&mut sequence); + let original=workflow.session.rows[0].clone(); + let canonical=encode(&workflow).unwrap(); + for role in [DataRole::Validation,DataRole::HeldOut,DataRole::Inference]{ + let mut row=(*original).clone();row.role=role;workflow.session.rows[0]=SharedRecord::new(row); + assert!(matches!(snapshot_old(&workflow),Err(error)if error=="temporal_checkpoint_complete_train_prefix_only")); + } + workflow.session.rows[0]=original; + assert_eq!(encode(&workflow).unwrap(),canonical); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_durability.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_durability.rs new file mode 100644 index 0000000000..ee172d333c --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_durability.rs @@ -0,0 +1,980 @@ +//! Private, bounded serial persistence executor. No packet queue or authority issuer. +use super::*; +use std::sync::{ + atomic::{AtomicBool, AtomicU64, Ordering}, + mpsc, Arc, Mutex, +}; +use std::time::Instant; + +pub(super) struct CommitRequest { + pub id: u64, + pub saved_receiver_ns: u64, + pub snapshot: checkpoint::Snapshot, + pub descriptor: RecoveryDescriptor, + pub expected: Option, + pub slot: bool, + pub rows: usize, + pub seals: usize, +} +pub(super) struct CommitAck { + pub id: u64, + pub slot: bool, + pub rows: usize, + pub seals: usize, + pub start_ticket: bool, + pub completed_at_ns: u64, + pub outcome: Result, +} +pub(super) struct Committed { + pub token: TemporalToken, + pub bytes: Arc>, + pub durable_sha256: String, + pub rows_sha256: String, + pub row_prefix_sha256: String, + pub restored: Option>, + pub continued:Option, + pub full_hashes:Option, +} +#[cfg(test)] +#[derive(Clone, Copy, Default)] +pub(crate) struct ReclamationStats { + pub has_ack: bool, + pub old_prefix_bytes: usize, + pub old_prefix_last_owner: bool, + pub raw_tail_rows: usize, +} +#[derive(Default)] +pub(crate) struct DeferredReclamation { + ack: Option, + pending: Option, + previous_bytes: Option>>, + previous_workflow: Option>, + fences: Option, + #[cfg(test)] + pub probe: Option>, +} +impl DeferredReclamation { + pub(crate) fn has_ack(&self) -> bool { + self.ack.is_some() + } +} +impl Drop for DeferredReclamation { + fn drop(&mut self) { + // Callers declare this value BEFORE Runtime.inner's MutexGuard, so + // return/error/unwind unlocks before these owning payloads are freed. + #[cfg(test)] + if self.has_ack() { + if let Some(probe) = &self.probe { + probe(ReclamationStats { + has_ack: true, + old_prefix_bytes: self.previous_bytes.as_ref().map_or(0, |b| b.len()), + old_prefix_last_owner: self + .previous_bytes + .as_ref() + .is_some_and(|b| Arc::strong_count(b) == 1), + raw_tail_rows: self.previous_workflow.as_ref().map_or(0, |w| { + w.session.rows.len().saturating_sub(w.durable_windows) + }), + }); + } + } + drop(self.ack.take()); + drop(self.previous_workflow.take()); + drop(self.pending.take()); + drop(self.previous_bytes.take()); + drop(self.fences.take()); + } +} +enum Work { + Commit(CommitRequest), + Recertify { + id: u64, + expected: TemporalToken, + descriptor: RecoveryDescriptor, + rows: usize, + seals: usize, + start_ticket: bool, + records: checkpoint::SharedRecords, + model:Option, + }, + Repair { + id: u64, + }, + Stop, +} +struct Pending { + intent: TemporalIntent, + prefix: Arc, + id: u64, + slot: bool, + rows: usize, + seals: usize, + records: checkpoint::SharedRecords, +} +struct Shared { + failed_repair: Mutex)>>, + terminal_drained: AtomicBool, + cancelled: AtomicBool, + busy: AtomicBool, + now_ns: AtomicU64, + anchor_ns: u64, + origin: Instant, + #[cfg(test)] + lose_next_ack: AtomicBool, + #[cfg(test)] + fail_next_publication: AtomicBool, + #[cfg(test)] + ack_queued: AtomicBool, + #[cfg(test)] + corrupt_ack_id: AtomicBool, + #[cfg(test)] + hold: Mutex, mpsc::Receiver<()>)>>, + #[cfg(test)] + hold_after_prepare: Mutex, mpsc::Receiver<()>)>>, + #[cfg(test)] + hold_after_publish: Mutex, mpsc::Receiver<()>)>>, +} +impl Shared { + fn now(&self) -> Result { + let elapsed: u64 = self + .origin + .elapsed() + .as_nanos() + .try_into() + .map_err(|_| "temporal_writer_clock_overflow")?; + Ok(self.now_ns.load(Ordering::Acquire).max( + self.anchor_ns + .checked_add(elapsed) + .ok_or("temporal_writer_clock_overflow")?, + )) + } + fn allowed(&self, descriptor: &RecoveryDescriptor) -> Result { + if self.cancelled.load(Ordering::Acquire) { + return Err("temporal_writer_cancelled".into()); + } + let now = self.now()?; + if now / 1_000_000 >= descriptor.expires_at_ms { + return Err("temporal_writer_original_deadline_expired".into()); + } + Ok(now) + } +} +pub(crate) struct Writer { + store: Arc, + send: mpsc::SyncSender, + receive: Mutex>, + shared: Arc, + thread: Mutex>>, +} +impl Writer { + pub fn new(store: Arc, now: u64) -> Result { + let (send, requests) = mpsc::sync_channel(1); + let (replies, receive) = mpsc::sync_channel(1); + let shared = Arc::new(Shared { + failed_repair: Mutex::new(None), + terminal_drained: AtomicBool::new(false), + cancelled: AtomicBool::new(false), + busy: AtomicBool::new(false), + now_ns: AtomicU64::new(now), + anchor_ns: now, + origin: Instant::now(), + #[cfg(test)] + lose_next_ack: AtomicBool::new(false), + #[cfg(test)] + fail_next_publication: AtomicBool::new(false), + #[cfg(test)] + ack_queued: AtomicBool::new(false), + #[cfg(test)] + corrupt_ack_id: AtomicBool::new(false), + #[cfg(test)] + hold: Mutex::new(None), + #[cfg(test)] + hold_after_prepare: Mutex::new(None), + #[cfg(test)] + hold_after_publish: Mutex::new(None), + }); + let state = shared.clone(); + let worker_store = store.clone(); + let thread = std::thread::Builder::new() + .name("ruview-temporal-durable".into()) + .spawn(move || { + let store = worker_store; + let mut pending: Option = None; + while let Ok(work) = requests.recv() { + let (id, slot, rows, seals, start_ticket, result) = match work { + Work::Stop => break, + Work::Commit(mut request) => { + let meta = (request.id, request.slot, request.rows, request.seals); + let result = (|| { + state.allowed(&request.descriptor)?; + #[cfg(test)] + if let Some((entered, release)) = state + .hold + .lock() + .map_err(|_| "temporal_writer_test_lock")? + .take() + { + let _ = entered.send(()); + let _ = release.recv(); + } + state.allowed(&request.descriptor)?; + if pending.is_some() { + return Err("temporal_writer_exact_repair_required".into()); + } + let records = checkpoint::snapshot_records(&request.snapshot); + let full_permit=checkpoint::snapshot_full_permit(&mut request.snapshot); + let bytes = checkpoint::encode_snapshot(request.snapshot)?; + let at = request.saved_receiver_ns; + let mut prefix = + checkpoint::validated_prefix(&bytes, &request.descriptor, at)?; + if let Some(permit)=full_permit{prefix.attach_full_grant(permit.encoded(&bytes))?;} + prefix.verify_full_artifact()?; + let intent = store.prepare_temporal( + request.expected.as_ref(), + &prefix, + state.allowed(&request.descriptor)?, + )?; + let prefix = Arc::new(prefix); + *state.failed_repair.lock().map_err(|_| "temporal_writer_repair_lock")? = Some((intent.clone(), prefix.clone())); + pending = Some(Pending { + intent, + prefix, + id: meta.0, + slot: meta.1, + rows: meta.2, + seals: meta.3, + records, + }); + #[cfg(test)] + if let Some((entered, release)) = state.hold_after_prepare.lock().map_err(|_|"temporal_writer_test_lock")?.take() { + let _ = entered.send(()); + let _ = release.recv(); + } + let p = pending.as_ref().unwrap(); + #[cfg(not(test))] + let token = store.publish_temporal(&p.intent, Some(&p.prefix), state.allowed(p.prefix.descriptor())?)?; + #[cfg(test)] + let token = if state.fail_next_publication.swap(false,Ordering::AcqRel){ + store.synthetic_fail_before_head_publication(&p.intent,&p.prefix,state.allowed(p.prefix.descriptor())?) + }else{store.publish_temporal(&p.intent,Some(&p.prefix),state.allowed(p.prefix.descriptor())?)}?; + #[cfg(test)] + if let Some((entered,release))=state.hold_after_publish.lock().map_err(|_|"temporal_writer_test_lock")?.take(){let _=entered.send(());let _=release.recv();} + #[cfg(test)] + if state.lose_next_ack.swap(false, Ordering::AcqRel) { + return Err("synthetic_lost_publish_ack".into()); + } + readback(&store, &state, p, token) + })(); + if result.is_ok() { + pending = None; + if let Ok(mut ticket) = state.failed_repair.lock() { *ticket = None; } + } + (meta.0, meta.1, meta.2, meta.3, false, result) + } + Work::Recertify { + id, + expected, + descriptor, + rows, + seals, + start_ticket, + records,model, + } => { + let result = (|| { + state.allowed(&descriptor)?; + #[cfg(test)] + if let Some((entered, release)) = state + .hold + .lock() + .map_err(|_| "temporal_writer_test_lock")? + .take() + { + let _ = entered.send(()); + let _ = release.recv(); + } + if pending.is_some() { + return Err("temporal_writer_exact_repair_required".into()); + } + if let Some(model)=&model{model.verify()?;} + let recertified = store + .recertify_temporal(&expected, state.allowed(&descriptor)?)?; + let restored = checkpoint::decode_shared( + recertified.prefix.bytes(), + recertified.prefix.descriptor(), + state.allowed(&descriptor)?, + &records, + )?; + if restored.full_run.as_ref().is_some_and(|m|m.frozen.is_some())&&model.is_none(){return Err("temporal_current_model_recertification_required".into());} + let continued=match &model{Some(m)=>{m.check_workflow_data(&restored)?;m.verify()?;m.continued_witness(&restored,&recertified.token)?},None=>None}; + let full_hashes=if restored.full_run.is_some(){Some(full_run::FullPrefixHashes::from_workflow(&restored)?)}else{None}; + let rows_sha256 = hash(&restored.session.rows)?; + let row_prefix_sha256 = hash(&( + &restored.session.rows, + &restored.session.evidence, + &restored.ledger.spans, + ))?; + state.allowed(&descriptor)?; + Ok(Committed { + token: recertified.token, + durable_sha256: bytes_hash(recertified.prefix.bytes()), + bytes: Arc::new(recertified.prefix.bytes().to_vec()), + rows_sha256, + row_prefix_sha256,continued,full_hashes, + restored: Some(Box::new(restored)), + }) + })(); + (id, false, rows, seals, start_ticket, result) + } + Work::Repair { id } => { + let Some(p) = pending.as_ref() else { + let _ = replies.send(CommitAck { + id, + slot: false, + rows: 0, + seals: 0, + start_ticket: false, + completed_at_ns: state.now().unwrap_or(0), + outcome: Err("temporal_writer_pending_intent_missing".into()), + }); + continue; + }; + let meta = (p.id, p.slot, p.rows, p.seals); + let result = (|| { + if id != p.id { + return Err("temporal_writer_repair_ticket_changed".into()); + } + let token = store.repair_same_temporal( + &p.intent, + Some(&p.prefix), + state.allowed(p.prefix.descriptor())?, + )?; + readback(&store, &state, p, token) + })(); + if result.is_ok() { + pending = None; + if let Ok(mut ticket) = state.failed_repair.lock() { *ticket = None; } + } + (meta.0, meta.1, meta.2, meta.3, false, result) + } + }; + // Capacity one and only one accepted transaction. The receiver + // must consume this result before another request is admitted. + #[cfg(test)] + let id = if state.corrupt_ack_id.swap(false, Ordering::AcqRel) { + id.wrapping_add(1) + } else { + id + }; + if replies + .send(CommitAck { + id, + slot, + rows, + seals, + start_ticket, + completed_at_ns: state.now().unwrap_or(0), + outcome: result, + }) + .is_err() + { + break; + } + #[cfg(test)] + state.ack_queued.store(true, Ordering::Release); + if state.cancelled.load(Ordering::Acquire) { + break; + } + } + }) + .map_err(|_| "temporal_writer_spawn_failed")?; + Ok(Self { + store, + send, + receive: Mutex::new(receive), + shared, + thread: Mutex::new(Some(thread)), + }) + } + pub fn observe_now(&self, now: u64) { + self.shared.now_ns.fetch_max(now, Ordering::AcqRel); + } + pub fn submit(&self, request: CommitRequest) -> Result<(), String> { + if self.shared.cancelled.load(Ordering::Acquire) { + return Err("temporal_writer_cancelled".into()); + } + if self + .shared + .busy + .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) + .is_err() + { + return Err("temporal_storage_busy".into()); + } + #[cfg(test)] + self.shared.ack_queued.store(false, Ordering::Release); + if self.send.try_send(Work::Commit(request)).is_err() { + self.shared.busy.store(false, Ordering::Release); + return Err("temporal_writer_unavailable".into()); + } + Ok(()) + } + fn recertify( + &self, + id: u64, + expected: TemporalToken, + descriptor: RecoveryDescriptor, + rows: usize, + seals: usize, + start_ticket: bool, + records: checkpoint::SharedRecords, + model:Option, + ) -> Result<(), String> { + if self.shared.cancelled.load(Ordering::Acquire) + || self + .shared + .busy + .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) + .is_err() + { + return Err("temporal_storage_busy".into()); + } + #[cfg(test)] + self.shared.ack_queued.store(false, Ordering::Release); + if self + .send + .try_send(Work::Recertify { + id, + expected, + descriptor, + rows, + seals, + start_ticket, + records,model, + }) + .is_err() + { + self.shared.busy.store(false, Ordering::Release); + return Err("temporal_writer_unavailable".into()); + } + Ok(()) + } + pub fn poll(&self) -> Result, String> { + match self + .receive + .lock() + .map_err(|_| "temporal_writer_reply_lock")? + .try_recv() + { + Ok(ack) => { + self.shared.busy.store(false, Ordering::Release); + Ok(Some(ack)) + } + Err(mpsc::TryRecvError::Empty) => Ok(None), + Err(_) => Err("temporal_writer_disconnected".into()), + } + } + pub fn repair(&self, id: u64) -> Result<(), String> { + if self + .shared + .busy + .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) + .is_err() + { + return Err("temporal_storage_busy".into()); + } + #[cfg(test)] + self.shared.ack_queued.store(false, Ordering::Release); + if self.send.try_send(Work::Repair { id }).is_err() { + self.shared.busy.store(false, Ordering::Release); + return Err("temporal_writer_unavailable".into()); + } + Ok(()) + } + /// Explicit owner Retry only, outside Runtime.inner. The thread lock also + /// serializes erasure's drain, so no exact-intent repair can publish after + /// that drain has returned. Never infer a failed write from a lost ACK. + pub(crate) fn reconcile_failed(&self, now: u64) -> Result<(), String> { + self.observe_now(now); + self.cancel(); + let mut thread = self.thread.lock().map_err(|_| "temporal_writer_join_lock")?; + if self.shared.terminal_drained.load(Ordering::Acquire) { return Err("temporal_writer_terminally_drained".into()); } + if let Some(worker) = thread.take() { + worker.join().map_err(|_| "temporal_writer_panicked")?; + } + let mut repair = self.shared.failed_repair.lock().map_err(|_| "temporal_writer_repair_lock")?; + if let Some((intent, prefix)) = repair.as_ref() { + self.store.repair_same_temporal(intent, Some(prefix.as_ref()), self.shared.now()?)?; + *repair = None; + } + Ok(()) + } + pub fn cancel(&self) { + self.shared.cancelled.store(true, Ordering::Release); + let _ = self.send.try_send(Work::Stop); + } + /// Caller MUST be outside Runtime.inner. Finishing a publication cannot be + /// undone; this barrier prevents an old worker from publishing after erase. + pub fn drain(&self) -> Result<(), String> { + self.cancel(); + let mut handle = self.thread.lock().map_err(|_| "temporal_writer_join_lock")?; + // Terminal owner invalidation/erase barrier. Unlike Retry's own join, + // no captured Writer Arc may reconcile or publish after this returns. + self.shared.terminal_drained.store(true, Ordering::Release); + let joined = if let Some(thread) = handle.take() { + thread.join().map_err(|_| "temporal_writer_panicked".to_string()) + } else { Ok(()) }; + let repair = self.shared.failed_repair.lock().map(|mut value|value.take()).unwrap_or_else(|error|error.into_inner().take()); + drop(repair); + drop(handle); + // A revoked/failed capture may never poll its final ACK. Release its + // large serialized/candidate buffers here, not when Core drops inner. + if let Ok(receiver) = self.receive.lock() { + while let Ok(ack) = receiver.try_recv() { + drop(ack); + } + } + joined + } + #[cfg(test)] + pub(crate) fn fail_next_publication(&self){self.shared.fail_next_publication.store(true,Ordering::Release);} + #[cfg(test)] + pub(crate) fn has_failed_repair(&self)->bool{self.shared.failed_repair.lock().unwrap().is_some()} + #[cfg(test)] + pub(crate) fn hold_next_after_publish(&self,entered:mpsc::SyncSender<()>,release:mpsc::Receiver<()>){*self.shared.hold_after_publish.lock().unwrap()=Some((entered,release));} + #[cfg(test)] + pub fn ack_is_queued(&self) -> bool { + self.shared.ack_queued.load(Ordering::Acquire) + } + #[cfg(test)] + pub fn corrupt_next_ack_id(&self) { + self.shared.corrupt_ack_id.store(true, Ordering::Release); + } + #[cfg(test)] + pub fn lose_next_ack(&self) { + self.shared.lose_next_ack.store(true, Ordering::Release); + } + #[cfg(test)] + pub fn hold_next_after_prepare(&self, entered:mpsc::SyncSender<()>, release:mpsc::Receiver<()>) { + *self.shared.hold_after_prepare.lock().unwrap()=Some((entered,release)); + } + #[cfg(test)] + pub fn hold_next(&self, entered: mpsc::SyncSender<()>, release: mpsc::Receiver<()>) { + *self.shared.hold.lock().unwrap() = Some((entered, release)); + } +} +impl Drop for Writer { + fn drop(&mut self) { + self.shared.cancelled.store(true, Ordering::Release); + let _ = self.send.try_send(Work::Stop); + } +} +fn readback( + store: &RecoveryStore, + state: &Shared, + p: &Pending, + token: TemporalToken, +) -> Result { + let now = state.allowed(p.prefix.descriptor())?; + let loaded = store + .load_temporal(now)? + .ok_or("temporal_readback_missing")?; + if loaded.token != token || loaded.prefix.as_ref().map(|x| x.bytes()) != Some(p.prefix.bytes()) + { + return Err("temporal_readback_changed".into()); + } + let restored = + checkpoint::decode_shared(p.prefix.bytes(), p.prefix.descriptor(), now, &p.records)?; + p.prefix.verify_full_artifact()?; + let continued=p.prefix.continued_witness(&restored,&token)?; + let full_hashes=if restored.full_run.is_some(){Some(full_run::FullPrefixHashes::from_workflow(&restored)?)}else{None}; + let rows_sha256 = hash(&restored.session.rows)?; + let row_prefix_sha256 = hash(&( + &restored.session.rows, + &restored.session.evidence, + &restored.ledger.spans, + ))?; + state.allowed(p.prefix.descriptor())?; + Ok(Committed { + token, + durable_sha256: bytes_hash(p.prefix.bytes()), + bytes: Arc::new(p.prefix.bytes().to_vec()), + rows_sha256, + row_prefix_sha256,continued,full_hashes, + restored: Some(Box::new(restored)), + }) +} + +#[derive(Clone)] +pub(super) struct PendingMeta { + id: u64, + slot: bool, + rows: usize, + seals: usize, + pub start_ticket: bool, + predecessor: Option, + row_lineage: Vec>, + evidence_lineage: Vec>, +} +pub(super) struct AsyncState { + pub writer: Arc, + pub pending: Option, + next_id: u64, + pub busy_pause: bool, + repairing: bool, + durable_sha256: Option, + pub rows_sha256: Option, + pub row_prefix_sha256: Option, + pub error: Option, + pub accepted_seals: usize, + pub accepted_held_out_seals:usize, + pub full_hashes:Option, + // At most the same ONE completion, awaiting an actually fresh receiver + // timestamp for new full-run authority. Never a packet or work queue. + deferred_ack:Option, +} +impl AsyncState { + pub fn new(writer: Arc) -> Self { + Self { + writer, + pending: None, + next_id: 0, + busy_pause: false, + repairing: false, + durable_sha256: None, + rows_sha256: None, + row_prefix_sha256: None, + error: None, + accepted_seals: 0,accepted_held_out_seals:0,full_hashes:None, + deferred_ack:None, + } + } + pub fn seed_restored(&mut self, w: &TemporalWorkflow) -> Result<(), String> { + self.durable_sha256 = w.durable.as_ref().map(|b| bytes_hash(b)); + self.rows_sha256 = Some(hash(&w.session.rows)?); + self.row_prefix_sha256 = Some(hash(&( + &w.session.rows, + &w.session.evidence, + &w.ledger.spans, + ))?); + self.accepted_seals = w.bridge.as_ref().map_or(0, |b| b.seals.len()); + self.accepted_held_out_seals=w.full_run.as_ref().map_or(0,|m|m.held_out_seals()); + self.full_hashes=if w.full_run.is_some(){Some(full_run::FullPrefixHashes::from_workflow(w)?)}else{None}; + Ok(()) + } + fn id(&mut self) -> Result { + self.next_id = self + .next_id + .checked_add(1) + .ok_or("temporal_writer_revision_exhausted")?; + Ok(self.next_id) + } +} +impl TemporalWorkflow { + pub(super) fn durable_digest(&self) -> Option { + self.async_state + .as_ref() + .map(|a| a.durable_sha256.clone()) + .unwrap_or_else(|| self.durable.as_ref().map(|b| bytes_hash(b))) + } + pub(super) fn submit_async_commit(&mut self, slot: bool) -> Result<(), String> { + let snapshot = checkpoint::snapshot(self)?; + let state = self.async_state.as_mut().ok_or("temporal_async_required")?; + let id = state.id()?; + let seals = self.bridge.as_ref().map_or(0, |b| b.seals.len()); + let rows = self.session.rows.len(); + state.writer.observe_now(self.last_now_ns); + state.writer.submit(CommitRequest { + id, + saved_receiver_ns: self.last_now_ns, + snapshot, + descriptor: self.descriptor.clone(), + expected: self.commit_token.clone(), + slot, + rows, + seals, + })?; + state.pending = Some(PendingMeta { + id, + slot, + rows, + seals, + start_ticket: false, + predecessor: self.commit_token.clone(), + row_lineage: self.session.rows.clone(), + evidence_lineage: self.session.evidence.clone(), + }); + state.error = None; + if slot { + self.pending_window = None; + } + Ok(()) + } + pub(super) fn request_async_start(&mut self, now: u64) -> Result<(), String> { + self.check_clock(now)?; + if !matches!( + self.stage, + Stage::DurableReady | Stage::PausedNeedsReconfirmation + ) { + return Err("temporal_start_barrier_state".into()); + } + if self.session.rows.len() != self.durable_windows { + return Err("temporal_unconfirmed_rows".into()); + } + let records = checkpoint::workflow_records(self); + let model=self.full_run.as_ref().and_then(|m|m.model_verifier()); + let state = self.async_state.as_mut().ok_or("temporal_async_required")?; + if state.pending.is_some() || state.busy_pause { + return Err("temporal_storage_busy".into()); + } + let token = self + .commit_token + .clone() + .ok_or("temporal_initial_barrier_unconfirmed")?; + let id = state.id()?; + let rows = self.durable_windows; + let seals = self.bridge.as_ref().map_or(0, |b| b.seals.len()); + state.writer.observe_now(now); + state.writer.recertify( + id, + token, + self.descriptor.clone(), + rows, + seals, + true, + records,model, + )?; + state.pending = Some(PendingMeta { + id, + slot: false, + rows, + seals, + start_ticket: true, + predecessor: self.commit_token.clone(), + row_lineage: self.session.rows.clone(), + evidence_lineage: self.session.evidence.clone(), + }); + state.error = None; + Ok(()) + } + pub(super) fn request_async_pause(&mut self, now: u64) -> Result<(), String> { + self.check_clock(now)?; + if self.stage != Stage::Observing { + return Err("temporal_pause_state".into()); + } + let records = checkpoint::workflow_record_prefix(self, self.durable_windows)?; + let model=self.full_run.as_ref().and_then(|m|m.model_verifier()); + let state = self.async_state.as_mut().ok_or("temporal_async_required")?; + state.busy_pause = true; + state.error = Some("temporal_pause_pending_durability".into()); + if state.pending.is_none() { + let id = state.id()?; + let rows = self.durable_windows; + let seals = state.accepted_seals; + let token = self + .commit_token + .clone() + .ok_or("temporal_initial_barrier_unconfirmed")?; + state.writer.observe_now(now); + state.writer.recertify( + id, + token, + self.descriptor.clone(), + rows, + seals, + false, + records,model, + )?; + state.pending = Some(PendingMeta { + id, + slot: false, + rows, + seals, + start_ticket: false, + predecessor: self.commit_token.clone(), + row_lineage: self.session.rows.clone(), + evidence_lineage: self.session.evidence.clone(), + }); + } + self.stage = Stage::BlockedDurability; + Ok(()) + } + /// Only the exact current operation can admit a durable result. Live source + /// floors, current custody and uncheckpointed rows are never replaced by an + /// older normal ACK. Returns true only for a consumed one-use StartTicket. + pub(super) fn poll_async( + &mut self, + now: u64, + retired: &mut DeferredReclamation, + ) -> Result, String> { + let full_run=self.full_run.is_some(); + let Some(state) = self.async_state.as_mut() else { + return Ok(None); + }; + if retired.has_ack() { + return Err("temporal_reclamation_already_occupied".into()); + } + state.writer.observe_now(now); + let next=if state.deferred_ack.is_some(){state.deferred_ack.take()}else{state.writer.poll()?}; + let Some(ack) = next else { + return Ok(None); + }; + if full_run && now= self.descriptor.expires_at_ms { + return Err("temporal_writer_ack_clock_or_expiry".into()); + } + let committed = match &mut ack.outcome { + Ok(result) => result, + Err(error) => { + state.error = Some(error.clone()); + if pending.start_ticket { + state.pending = None; + self.stage = Stage::PausedNeedsReconfirmation; + } else { + self.stage = Stage::BlockedDurability; + } + return Err(error.clone()); + } + }; + let candidate = committed + .restored + .as_ref() + .ok_or("temporal_writer_restore_missing")?; + if candidate.context() != self.context() + || candidate.session.run_id != self.session.run_id + || candidate.descriptor.id != self.descriptor.id + || candidate.descriptor.expires_at_ms != self.descriptor.expires_at_ms + || candidate.session.rows.len() != pending.rows + { + return Err("temporal_writer_ack_binding_changed".into()); + } + if match (&self.full_run,&candidate.full_run) { + (None,None)=>false, + (Some(a),Some(b))=>a.policy.sha256()!=b.policy.sha256(), + _=>true, + } {return Err("temporal_writer_ack_continuation_changed".into());} + let frozen_transition=self.prepare_frozen_ack(candidate,now)?; + if let Some(continued)=&committed.continued{continued.check_owner(self,&committed.token)?;} + let complete_transition=self.prepare_complete_ack(candidate,&committed.token)?; + let candidate_held_out_seals=candidate.full_run.as_ref().map_or(0,|m|m.held_out_seals()); + let must_restore = self + .async_state + .as_ref() + .is_some_and(|s| s.busy_pause || s.repairing) && frozen_transition.is_none() && complete_transition.is_none(); + if must_restore { + retired.fences = Some(self.session.assembler.fence_checkpoint()?); + let restored = committed.restored.as_mut().unwrap(); + let fences=retired.fences.as_ref().unwrap().clone().for_prefix(pending.rows)?; + if let Some(mode)=&self.full_run{ + let budget=mode.policy.replay_fence_budget(&self.session.managed.as_ref().ok_or("managed_session_required")?.binding)?; + restored.session.assembler.restore_full_run_prefix_fences(fences,pending.rows,&budget)?; + }else{restored.session.assembler.restore_prefix_fences(fences,pending.rows)?;} + restored.source_high = self.source_high.clone(); + restored.mapping = self.mapping.clone(); + restored.live_observed_ns = self.live_observed_ns; + restored.last_now_ns = now; + if let Some(continued)=&committed.continued{continued.install_raw(restored)?;} + restored.stage = Stage::PausedNeedsReconfirmation; + restored.set_current_disposition(bridge::Disposition::Paused)?; + } + // All fallible candidate checks finished while its ownership remained + // in retired. From here transfer old owners, never destroy them here. + let mut state = self.async_state.take().unwrap(); + state.pending = None; + state.error = None; + state.durable_sha256 = Some(committed.durable_sha256.clone()); + state.accepted_seals = pending.seals; + state.accepted_held_out_seals=candidate_held_out_seals; + state.full_hashes=committed.full_hashes.clone(); + state.rows_sha256 = Some(committed.rows_sha256.clone()); + state.row_prefix_sha256 = Some(committed.row_prefix_sha256.clone()); + if frozen_transition.is_some()||complete_transition.is_some(){state.busy_pause=false;state.repairing=false;} + retired.previous_bytes = self.durable.take(); + self.commit_token = Some(committed.token.clone()); + self.durable = Some(committed.bytes.clone()); + self.durable_windows = pending.rows; + if must_restore { + let restored = committed.restored.as_mut().unwrap(); + restored.commit_token = self.commit_token.clone(); + restored.durable = self.durable.clone(); + let persist_pause = state.busy_pause; + state.busy_pause = false; + state.repairing = false; + restored.async_state = Some(state); + if let Some(old)=self.full_run.as_mut(){ + // Exact continuation presence was already validated above; + // all fallible checks finished before moving live custody. + let recovered=restored.full_run.as_mut().unwrap(); + recovered.owner=old.owner.take(); + recovered.local_start_not_before_ns=old.local_start_not_before_ns; + } + let replacement = *committed.restored.take().unwrap(); + retired.previous_workflow = Some(Box::new(std::mem::replace(self, replacement))); + if persist_pause { + self.descriptor.saved_at_ms = now / 1_000_000; + self.submit_async_commit(false)?; + } + } else { + self.async_state = Some(state); + } + if let Some(transition)=frozen_transition{self.apply_frozen_ack(transition);} + if let Some(transition)=complete_transition{self.apply_complete_ack(transition);} + if pending.start_ticket{if let Some(mode)=self.full_run.as_mut(){mode.local_start_not_before_ns=Some(now);}} + self.last_now_ns = self.last_now_ns.max(now); + Ok(Some((pending.start_ticket, now))) + } + pub(super) fn request_async_repair(&mut self, now: u64) -> Result<(), String> { + self.check_clock(now)?; + if self.stage != Stage::BlockedDurability { + return Err("temporal_pending_repair_state".into()); + } + let state = self.async_state.as_mut().ok_or("temporal_async_required")?; + let pending = state + .pending + .as_ref() + .ok_or("temporal_pending_intent_missing")?; + state.writer.observe_now(now); + state.writer.repair(pending.id)?; + state.repairing = true; + Ok(()) + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_evaluation.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_evaluation.rs new file mode 100644 index 0000000000..16c5c7661e --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_evaluation.rs @@ -0,0 +1,584 @@ +//! Whole Temporal evaluation DATA. No decoded producer or runtime/store permit, +//! model accessor, evaluator invocation or qualification/adoption authority. +//! Successful decoding validates only bounded metadata and its integrity links. +use super::{ + observation::{CaptureProvenance, ClockKind, DataRole}, + training::EvaluationReport, +}; +use serde::{de::DeserializeOwned, Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +pub(super) const TRAIN_SCHEMA: &str = "ruview.room-learner.temporal-training-proof.v1"; +pub(super) const HELD_OUT_SCHEMA: &str = "ruview.room-learner.temporal-heldout-evidence.v1"; +pub(super) const EVALUATION_SCHEMA: &str = "ruview.room-learner.temporal-evaluation.v1"; +pub(super) const OPERATOR_TRAIN_SCHEMA: &str = "ruview.room-learner.temporal-training-proof.operator-eight-step.v1"; +pub(super) const OPERATOR_CHECK_SCHEMA: &str = "ruview.room-learner.temporal-validation-evidence.operator-eight-step.v1"; +pub(super) const OPERATOR_EVALUATION_SCHEMA: &str = "ruview.room-learner.temporal-evaluation.operator-eight-step.v1"; +const MODEL_KIND: &str = "ruview.room-learner.managed-temporal-input.v3"; +const PREFIX_SCHEMA: &str = "ruview.room-learner.temporal-runtime-prefix.v3"; +const METADATA_KIND: &str = "temporal-prefix-commitment-v2"; +pub(super) const MAX_METADATA_BYTES: usize = 64 * 1024; +const TTL_MS: u64 = 86_400_000; +fn invalid() -> Result { + Err("temporal_evaluation_data_invalid".into()) +} +fn sha(value: &str) -> bool { + value.len() == 64 + && value + .bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) +} +fn identifier(value: &str) -> bool { + !value.is_empty() + && value.len() <= 256 + && value.is_ascii() + && value.trim() == value + && !value.bytes().any(|b| b.is_ascii_control()) +} +fn decimal(value: &str) -> Result { + if value.is_empty() + || value.len() > 20 + || !value.bytes().all(|b| b.is_ascii_digit()) + || (value.starts_with('0') && value != "0") + { + return invalid(); + } + value + .parse() + .map_err(|_| "temporal_evaluation_data_integer".into()) +} +fn decimal_frame(value: &str) -> Result { + if value.is_empty() + || value.len() > 39 + || !value.bytes().all(|b| b.is_ascii_digit()) + || (value.starts_with('0') && value != "0") + { + return invalid(); + } + value + .parse() + .map_err(|_| "temporal_evaluation_data_frame_integer".into()) +} +fn data_sha(schema: &str, body: &T) -> Result { + let bytes = + serde_json::to_vec(&(schema, body)).map_err(|_| "temporal_evaluation_data_encoding")?; + Ok(format!("{:x}", Sha256::digest(bytes))) +} +fn decode_data(bytes: &[u8]) -> Result { + if bytes.is_empty() || bytes.len() > MAX_METADATA_BYTES { + return invalid(); + } + let value: T = serde_json::from_slice(bytes).map_err(|_| "temporal_evaluation_data_schema")?; + if serde_json::to_vec(&value).map_err(|_| "temporal_evaluation_data_encoding")? != bytes { + return invalid(); + } + Ok(value) +} + +/// Stable original identity ONLY; intentionally excludes mutable saved_at_ms. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub(super) struct OriginalDeadlineData { + pub id: String, + pub created_at_unix_ms: u64, + pub expires_at_unix_ms: u64, +} +impl OriginalDeadlineData { + fn validate_data(&self) -> Result<(), String> { + if !identifier(&self.id) + || self.created_at_unix_ms == 0 + || self.expires_at_unix_ms > 253_402_300_799_999 + || self.created_at_unix_ms.checked_add(TTL_MS) != Some(self.expires_at_unix_ms) + { + return invalid(); + } + Ok(()) + } + fn contains_receiver_ns(&self, ns: u64) -> bool { + ns > 0 + && ns / 1_000_000 >= self.created_at_unix_ms + && ns / 1_000_000 < self.expires_at_unix_ms + } +} +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub(super) struct TemporalEvaluationIdentityData { + pub owner_scope: String, + pub installation_id: String, + pub context_digest: String, + pub room_digest: String, + pub run: String, + pub original_deadline: OriginalDeadlineData, + pub artifact_kind: String, + pub artifact_sha256: String, + pub verified_fit_record_sha256: String, + pub prefix_schema: String, + pub metadata_kind: String, + pub continuation_policy_sha256: String, + pub input_policy_sha256: String, + pub execution_policy_sha256: String, + pub schedule_sha256: String, + pub configured_fit_sha256: String, + pub source_ids: [u8; 3], +} +impl TemporalEvaluationIdentityData { + fn validate_data(&self) -> Result<(), String> { + self.original_deadline.validate_data()?; + if [&self.owner_scope, &self.installation_id, &self.run] + .iter() + .any(|s| !identifier(s)) + || [ + &self.context_digest, + &self.room_digest, + &self.artifact_sha256, + &self.verified_fit_record_sha256, + &self.continuation_policy_sha256, + &self.input_policy_sha256, + &self.execution_policy_sha256, + &self.schedule_sha256, + &self.configured_fit_sha256, + ] + .iter() + .any(|s| !sha(s)) + || self.artifact_kind != MODEL_KIND + || self.prefix_schema != PREFIX_SCHEMA + || self.metadata_kind != METADATA_KIND + || self.source_ids.windows(2).any(|w| w[0] >= w[1]) + { + return invalid(); + } + Ok(()) + } +} +/// Projection, never a deserializable TemporalToken/storage permission. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub(super) struct CommitmentData { + pub generation: u64, + pub revision: u16, + pub run: String, + pub id: String, + pub commitment_sha256: String, +} +impl CommitmentData { + fn validate_data(&self, identity: &TemporalEvaluationIdentityData) -> Result<(), String> { + if self.revision >= 433 + || self.run != identity.run + || self.id != identity.original_deadline.id + || !sha(&self.commitment_sha256) + { + return invalid(); + } + Ok(()) + } +} +/// Negative descriptive claims only; no field can grant authority. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub(super) struct DiagnosticClaimsData { + pub independence_verified: bool, + pub physically_validated: bool, + pub qualification_granted: bool, + pub adoption_authorized: bool, + pub measurement_clock_available: bool, +} +impl DiagnosticClaimsData { + fn validate_data(&self) -> Result<(), String> { + if self.independence_verified + || self.physically_validated + || self.qualification_granted + || self.adoption_authorized + || self.measurement_clock_available + { + return invalid(); + } + Ok(()) + } +} +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub(super) struct FrozenSourceFloorData { + pub source_id: u8, + pub sequence: u64, + pub receiver_ns: String, +} +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub(super) struct TemporalTrainProofBodyData { + pub identity: TemporalEvaluationIdentityData, + pub training_data_sha256: String, + pub training_evidence_sha256: String, + pub training_identities_sha256: String, + pub train_seal_sha256s: Vec, + pub train_seal_chain_sha256: String, + pub train_windows: u16, + pub windows_per_condition: [u16; 4], + pub declared_episodes_per_condition: [u16; 4], + pub accepted_train_commitment: CommitmentData, + pub freeze_prepare_receiver_ns: String, + pub source_floors: [FrozenSourceFloorData; 3], + pub claims: DiagnosticClaimsData, +} +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub(super) struct TemporalTrainProofData { + pub schema: String, + pub body: TemporalTrainProofBodyData, + pub sha256: String, +} +impl TemporalTrainProofData { + pub(super) fn is_operator_eight_step(&self) -> bool { self.schema == OPERATOR_TRAIN_SCHEMA } + pub(super) fn decode_data(bytes: &[u8]) -> Result { + let value: Self = decode_data(bytes)?; + value.validate_data()?; + Ok(value) + } + pub(super) fn validate_data(&self) -> Result<(), String> { + let b = &self.body; + b.identity.validate_data()?; + b.claims.validate_data()?; + b.accepted_train_commitment.validate_data(&b.identity)?; + let cut = decimal(&b.freeze_prepare_receiver_ns)?; + if ![TRAIN_SCHEMA, OPERATOR_TRAIN_SCHEMA].contains(&self.schema.as_str()) + || !sha(&self.sha256) + || self.sha256 != data_sha(&self.schema, b)? + || [ + &b.training_data_sha256, + &b.training_evidence_sha256, + &b.training_identities_sha256, + &b.train_seal_chain_sha256, + ] + .iter() + .any(|s| !sha(s)) + || b.train_seal_sha256s.iter().any(|s| !sha(s)) + || b.train_seal_sha256s + .iter() + .collect::>() + .len() + != if self.is_operator_eight_step() { 4 } else { 7 } + || b.train_seal_sha256s.len() != if self.is_operator_eight_step() { 4 } else { 7 } + || b.train_windows != 64 + || b.windows_per_condition != [16; 4] + || b.declared_episodes_per_condition != if self.is_operator_eight_step() { [1; 4] } else { [1, 2, 2, 2] } + || !b.identity.original_deadline.contains_receiver_ns(cut) + { + return invalid(); + } + for (floor, id) in b.source_floors.iter().zip(b.identity.source_ids) { + let ns = decimal(&floor.receiver_ns)?; + if floor.source_id != id || floor.sequence > u32::MAX as u64 || ns == 0 || ns > cut { + return invalid(); + } + } + Ok(()) + } +} +/// DATA from the acknowledged Complete prefix's whole-period source_high. +/// This is not a future ACK/RAM floor, hardware epoch or live continuation permit. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub(super) struct FinalSourceFenceData { + pub source_id: u8, + pub sequence: u64, + pub receiver_ns: String, + pub native_frame_id: String, +} +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub(super) struct TemporalHeldOutBodyData { + pub identity: TemporalEvaluationIdentityData, + pub training_proof_sha256: String, + pub held_out_data_sha256: String, + pub held_out_evidence_sha256: String, + pub held_out_identities_sha256: String, + pub complete_rows_sha256: String, + pub complete_evidence_sha256: String, + pub complete_identities_sha256: String, + pub complete_ledger_sha256: String, + pub held_out_seal_sha256s: Vec, + pub held_out_seal_chain_sha256: String, + pub complete_record_sha256: String, + pub complete_commitment: CommitmentData, + pub complete_prepare_receiver_ns: String, + pub complete_persisted_receiver_ns: String, + pub final_source_fences: [FinalSourceFenceData; 3], + pub held_out_windows: u16, + pub windows_per_condition: [u16; 4], + pub declared_episodes_per_condition: [u16; 4], + pub claims: DiagnosticClaimsData, +} +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub(super) struct TemporalHeldOutEvidenceData { + pub schema: String, + pub body: TemporalHeldOutBodyData, + pub sha256: String, +} +impl TemporalHeldOutEvidenceData { + pub(super) fn is_operator_eight_step(&self) -> bool { self.schema == OPERATOR_CHECK_SCHEMA } + pub(super) fn decode_data(bytes: &[u8]) -> Result { + let value: Self = decode_data(bytes)?; + value.validate_data()?; + Ok(value) + } + pub(super) fn validate_data(&self) -> Result<(), String> { + let b = &self.body; + b.identity.validate_data()?; + b.claims.validate_data()?; + b.complete_commitment.validate_data(&b.identity)?; + if ![HELD_OUT_SCHEMA, OPERATOR_CHECK_SCHEMA].contains(&self.schema.as_str()) + || !sha(&self.sha256) + || self.sha256 != data_sha(&self.schema, b)? + || [ + &b.training_proof_sha256, + &b.held_out_data_sha256, + &b.held_out_evidence_sha256, + &b.held_out_identities_sha256, + &b.complete_rows_sha256, + &b.complete_evidence_sha256, + &b.complete_identities_sha256, + &b.complete_ledger_sha256, + &b.held_out_seal_chain_sha256, + &b.complete_record_sha256, + ] + .iter() + .any(|s| !sha(s)) + || b.held_out_seal_sha256s.iter().any(|s| !sha(s)) + || b.held_out_seal_sha256s + .iter() + .collect::>() + .len() + != if self.is_operator_eight_step() { 4 } else { 8 } + || b.held_out_seal_sha256s.len() != if self.is_operator_eight_step() { 4 } else { 8 } + || b.held_out_windows != if self.is_operator_eight_step() { 16 } else { 64 } + || b.windows_per_condition != if self.is_operator_eight_step() { [4; 4] } else { [16; 4] } + || b.declared_episodes_per_condition != if self.is_operator_eight_step() { [1; 4] } else { [2; 4] } + || !b + .identity + .original_deadline + .contains_receiver_ns(decimal(&b.complete_prepare_receiver_ns)?) + { + return invalid(); + } + let persisted = decimal(&b.complete_persisted_receiver_ns)?; + if !b.identity.original_deadline.contains_receiver_ns(persisted) + || persisted != decimal(&b.complete_prepare_receiver_ns)? + { + return invalid(); + } + for (floor, id) in b.final_source_fences.iter().zip(b.identity.source_ids) { + let ns = decimal(&floor.receiver_ns)?; + let frame = decimal_frame(&floor.native_frame_id)?; + let expected = + (u128::from(ns) << 40) | (u128::from(id) << 32) | u128::from(floor.sequence); + if floor.source_id != id + || floor.sequence > u32::MAX as u64 + || ns == 0 + || ns > persisted + || frame != expected + { + return invalid(); + } + } + Ok(()) + } + /// Metadata equality/integrity only, NOT a complete capture or evaluation permit. + pub(super) fn validate_data_links(&self, train: &TemporalTrainProofData) -> Result<(), String> { + self.validate_data()?; + train.validate_data()?; + if self.is_operator_eight_step() != train.is_operator_eight_step() + || self.body.identity != train.body.identity + || self.body.training_proof_sha256 != train.sha256 + || decimal(&self.body.complete_prepare_receiver_ns)? + < decimal(&train.body.freeze_prepare_receiver_ns)? + || self.body.complete_commitment.generation + != train.body.accepted_train_commitment.generation + || self.body.complete_commitment.revision + <= train.body.accepted_train_commitment.revision + { + return invalid(); + } + for (final_floor, train_floor) in self + .body + .final_source_fences + .iter() + .zip(&train.body.source_floors) + { + if final_floor.source_id != train_floor.source_id + || final_floor.sequence < train_floor.sequence + || decimal(&final_floor.receiver_ns)? < decimal(&train_floor.receiver_ns)? + { + return invalid(); + } + } + Ok(()) + } +} +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub(super) struct TemporalEvaluationBodyData { + pub identity: TemporalEvaluationIdentityData, + pub training_proof_sha256: String, + pub held_out_evidence_sha256: String, + pub complete_record_sha256: String, + pub training_data_sha256: String, + pub held_out_data_sha256: String, + pub role: DataRole, + pub window_unit: String, + pub evaluated_at_receiver_ns: String, + pub evaluated_at_unix_ms: u64, + pub provenance: Vec, + pub clock_claims: Vec, + pub train_lineage_checked: bool, + pub report: EvaluationReport, + pub claims: DiagnosticClaimsData, +} +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub(super) struct TemporalEvaluationReceiptData { + pub schema: String, + pub body: TemporalEvaluationBodyData, + pub sha256: String, +} +impl TemporalEvaluationReceiptData { + pub(super) fn is_operator_eight_step(&self) -> bool { self.schema == OPERATOR_EVALUATION_SCHEMA } + pub(super) fn decode_data(bytes: &[u8]) -> Result { + let value: Self = decode_data(bytes)?; + value.validate_data()?; + Ok(value) + } + pub(super) fn validate_data(&self) -> Result<(), String> { + let b = &self.body; + b.identity.validate_data()?; + b.claims.validate_data()?; + b.report.validate()?; + let ns = decimal(&b.evaluated_at_receiver_ns)?; + if ![EVALUATION_SCHEMA, OPERATOR_EVALUATION_SCHEMA].contains(&self.schema.as_str()) + || !sha(&self.sha256) + || self.sha256 != data_sha(&self.schema, b)? + || [ + &b.training_proof_sha256, + &b.held_out_evidence_sha256, + &b.complete_record_sha256, + &b.training_data_sha256, + &b.held_out_data_sha256, + ] + .iter() + .any(|s| !sha(s)) + || b.role != if self.is_operator_eight_step() { DataRole::Validation } else { DataRole::HeldOut } + || b.window_unit != "native_receiver_2s_window" + || !b.train_lineage_checked + || ns / 1_000_000 != b.evaluated_at_unix_ms + || !b.identity.original_deadline.contains_receiver_ns(ns) + || b.provenance != [CaptureProvenance::LiveReceiverObserved] + || b.clock_claims != [ClockKind::ReceiverOnly] + || b.report.samples != if self.is_operator_eight_step() { 16 } else { 64 } + || b.report.declared_trial_count != if self.is_operator_eight_step() { 4 } else { 8 } + || b.report + .per_condition + .iter() + .any(|c| c.samples != if self.is_operator_eight_step() { 4 } else { 16 } || c.declared_trial_count != if self.is_operator_eight_step() { 1 } else { 2 }) + { + return invalid(); + } + Ok(()) + } + pub(super) fn validate_data_links( + &self, + train: &TemporalTrainProofData, + held: &TemporalHeldOutEvidenceData, + ) -> Result<(), String> { + self.validate_data()?; + held.validate_data_links(train)?; + let b = &self.body; + if self.is_operator_eight_step() != train.is_operator_eight_step() + || b.identity != held.body.identity + || b.training_proof_sha256 != train.sha256 + || b.held_out_evidence_sha256 != held.sha256 + || b.complete_record_sha256 != held.body.complete_record_sha256 + || b.training_data_sha256 != train.body.training_data_sha256 + || b.held_out_data_sha256 != held.body.held_out_data_sha256 + || decimal(&b.evaluated_at_receiver_ns)? + < decimal(&held.body.complete_persisted_receiver_ns)? + { + return invalid(); + } + Ok(()) + } +} + +// These constructors compute bounded DATA integrity only. They do not issue a +// producer, current-store, evaluation, qualification or selection permission. +impl TemporalTrainProofData { + pub(super) fn from_body_data(body: TemporalTrainProofBodyData) -> Result { + let schema = if body.train_seal_sha256s.len() == 4 && body.declared_episodes_per_condition == [1; 4] { OPERATOR_TRAIN_SCHEMA } else { TRAIN_SCHEMA }; + let value = Self { + schema: schema.into(), + sha256: data_sha(schema, &body)?, + body, + }; + value.validate_data()?; + value.encode_data()?; + Ok(value) + } + pub(super) fn encode_data(&self) -> Result, String> { + self.validate_data()?; + bounded_encoding(self) + } +} +impl TemporalHeldOutEvidenceData { + pub(super) fn from_body_data(body: TemporalHeldOutBodyData) -> Result { + let schema = if body.held_out_windows == 16 && body.held_out_seal_sha256s.len() == 4 { OPERATOR_CHECK_SCHEMA } else { HELD_OUT_SCHEMA }; + let value = Self { + schema: schema.into(), + sha256: data_sha(schema, &body)?, + body, + }; + value.validate_data()?; + value.encode_data()?; + Ok(value) + } + pub(super) fn encode_data(&self) -> Result, String> { + self.validate_data()?; + bounded_encoding(self) + } +} +impl TemporalEvaluationReceiptData { + pub(super) fn from_body_data(body: TemporalEvaluationBodyData) -> Result { + let schema = if body.role == DataRole::Validation { OPERATOR_EVALUATION_SCHEMA } else { EVALUATION_SCHEMA }; + let value = Self { + schema: schema.into(), + sha256: data_sha(schema, &body)?, + body, + }; + value.validate_data()?; + value.encode_data()?; + Ok(value) + } + pub(super) fn encode_data(&self) -> Result, String> { + self.validate_data()?; + bounded_encoding(self) + } +} +impl DiagnosticClaimsData { + pub(super) fn diagnostic_only() -> Self { + Self { + independence_verified: false, + physically_validated: false, + qualification_granted: false, + adoption_authorized: false, + measurement_clock_available: false, + } + } +} +fn bounded_encoding(value: &T) -> Result, String> { + let bytes = serde_json::to_vec(value).map_err(|_| "temporal_evaluation_data_encoding")?; + if bytes.is_empty() || bytes.len() > MAX_METADATA_BYTES { + return invalid(); + } + Ok(bytes) +} + +#[cfg(test)] +#[path = "temporal_evaluation_data_tests.rs"] +mod tests; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_evaluation_data_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_evaluation_data_tests.rs new file mode 100644 index 0000000000..5a2cce8a0e --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_evaluation_data_tests.rs @@ -0,0 +1,499 @@ +//! Synthetic metadata fixtures only. None creates a runtime/fit/Complete permit. +use super::*; +use serde_json::{json, Value}; +fn fixture_sha(n: u64) -> String { + format!("{n:064x}") +} +fn claims() -> DiagnosticClaimsData { + DiagnosticClaimsData { + independence_verified: false, + physically_validated: false, + qualification_granted: false, + adoption_authorized: false, + measurement_clock_available: false, + } +} +fn identity() -> TemporalEvaluationIdentityData { + TemporalEvaluationIdentityData { + owner_scope: "owner-a".into(), + installation_id: "installation-a".into(), + context_digest: fixture_sha(1), + room_digest: fixture_sha(2), + run: "run-a".into(), + original_deadline: OriginalDeadlineData { + id: "recovery-a".into(), + created_at_unix_ms: 1_800_000_000_000, + expires_at_unix_ms: 1_800_086_400_000, + }, + artifact_kind: MODEL_KIND.into(), + artifact_sha256: fixture_sha(3), + verified_fit_record_sha256: fixture_sha(4), + prefix_schema: PREFIX_SCHEMA.into(), + metadata_kind: METADATA_KIND.into(), + continuation_policy_sha256: fixture_sha(5), + input_policy_sha256: fixture_sha(6), + execution_policy_sha256: fixture_sha(7), + schedule_sha256: fixture_sha(8), + configured_fit_sha256: fixture_sha(9), + source_ids: [15, 16, 17], + } +} +fn token(id: &TemporalEvaluationIdentityData, revision: u16) -> CommitmentData { + CommitmentData { + generation: 0, + revision, + run: id.run.clone(), + id: id.original_deadline.id.clone(), + commitment_sha256: fixture_sha(u64::from(revision) + 100), + } +} +fn train() -> TemporalTrainProofData { + let id = identity(); + let cut = "1800000900000000000".to_owned(); + let body = TemporalTrainProofBodyData { + accepted_train_commitment: token(&id, 78), + source_floors: std::array::from_fn(|i| FrozenSourceFloorData { + source_id: id.source_ids[i], + sequence: 10_000, + receiver_ns: "1800000899999000000".into(), + }), + identity: id, + training_data_sha256: fixture_sha(20), + training_evidence_sha256: fixture_sha(21), + training_identities_sha256: fixture_sha(22), + train_seal_sha256s: (0..7).map(|i| fixture_sha(200 + i)).collect(), + train_seal_chain_sha256: fixture_sha(23), + train_windows: 64, + windows_per_condition: [16; 4], + declared_episodes_per_condition: [1, 2, 2, 2], + freeze_prepare_receiver_ns: cut, + claims: claims(), + }; + TemporalTrainProofData { + schema: TRAIN_SCHEMA.into(), + sha256: data_sha(TRAIN_SCHEMA, &body).unwrap(), + body, + } +} +fn held(t: &TemporalTrainProofData) -> TemporalHeldOutEvidenceData { + let body = TemporalHeldOutBodyData { + identity: t.body.identity.clone(), + training_proof_sha256: t.sha256.clone(), + held_out_data_sha256: fixture_sha(30), + held_out_evidence_sha256: fixture_sha(31), + held_out_identities_sha256: fixture_sha(32), + complete_rows_sha256: fixture_sha(33), + complete_evidence_sha256: fixture_sha(34), + complete_identities_sha256: fixture_sha(35), + complete_ledger_sha256: fixture_sha(36), + held_out_seal_sha256s: (0..8).map(|i| fixture_sha(300 + i)).collect(), + held_out_seal_chain_sha256: fixture_sha(37), + complete_record_sha256: fixture_sha(38), + complete_commitment: token(&t.body.identity, 160), + complete_prepare_receiver_ns: "1800001300000000000".into(), + complete_persisted_receiver_ns: "1800001300000000000".into(), + final_source_fences: std::array::from_fn(|i| { + let id = t.body.identity.source_ids[i]; + let ns = 1_800_001_299_999_000_000u64; + FinalSourceFenceData { + source_id: id, + sequence: 20_000, + receiver_ns: ns.to_string(), + native_frame_id: ((u128::from(ns) << 40) | (u128::from(id) << 32) | 20_000) + .to_string(), + } + }), + held_out_windows: 64, + windows_per_condition: [16; 4], + declared_episodes_per_condition: [2; 4], + claims: claims(), + }; + TemporalHeldOutEvidenceData { + schema: HELD_OUT_SCHEMA.into(), + sha256: data_sha(HELD_OUT_SCHEMA, &body).unwrap(), + body, + } +} +fn rates(sensitivity: Option, specificity: Option) -> Value { + json!({"evaluated_cohort_coverage":1.0,"unconditional_accuracy":1.0,"scored_accuracy":1.0,"unconditional_sensitivity":sensitivity,"scored_sensitivity":sensitivity,"unconditional_specificity":specificity,"scored_specificity":specificity}) +} +fn confusion(empty: usize, occupied: usize) -> Value { + json!({"true_empty":empty,"false_occupied":0,"false_empty":0,"true_occupied":occupied,"abstained_empty":0,"abstained_occupied":0,"missing_empty":0,"missing_occupied":0}) +} +fn report() -> EvaluationReport { + let labels = ["Empty", "OccupiedStatic", "OccupiedMoving", "AdjacentOnly"]; + let per:Vec=labels.into_iter().map(|label|{let occupied=label.starts_with("Occupied");json!({"condition":label,"samples":16,"correct":16,"abstained":0,"scored":16,"missing":0,"binary_cross_entropy":0.1,"binary_cross_entropy_samples":16,"declared_trial_count":2,"declared_session_count":2,"confusion":confusion(if occupied{0}else{16},if occupied{16}else{0}),"rates":rates(occupied.then_some(1.0),(!occupied).then_some(1.0))})}).collect(); + serde_json::from_value(json!({"samples":64,"correct":64,"abstained":0,"binary_cross_entropy":0.1,"binary_cross_entropy_samples":64,"scored":64,"missing":0,"declared_trial_count":8,"declared_session_count":8,"confusion":confusion(32,32),"rates":rates(Some(1.0),Some(1.0)),"per_condition":per,"independence_verified":false})).unwrap() +} +fn evaluation( + t: &TemporalTrainProofData, + h: &TemporalHeldOutEvidenceData, +) -> TemporalEvaluationReceiptData { + let body = TemporalEvaluationBodyData { + identity: t.body.identity.clone(), + training_proof_sha256: t.sha256.clone(), + held_out_evidence_sha256: h.sha256.clone(), + complete_record_sha256: h.body.complete_record_sha256.clone(), + training_data_sha256: t.body.training_data_sha256.clone(), + held_out_data_sha256: h.body.held_out_data_sha256.clone(), + role: DataRole::HeldOut, + window_unit: "native_receiver_2s_window".into(), + evaluated_at_receiver_ns: "1800001301000000000".into(), + evaluated_at_unix_ms: 1_800_001_301_000, + provenance: vec![CaptureProvenance::LiveReceiverObserved], + clock_claims: vec![ClockKind::ReceiverOnly], + train_lineage_checked: true, + report: report(), + claims: claims(), + }; + TemporalEvaluationReceiptData { + schema: EVALUATION_SCHEMA.into(), + sha256: data_sha(EVALUATION_SCHEMA, &body).unwrap(), + body, + } +} +fn rehash_train(t: &mut TemporalTrainProofData) { + t.sha256 = data_sha(TRAIN_SCHEMA, &t.body).unwrap(); +} +fn rehash_held(h: &mut TemporalHeldOutEvidenceData) { + h.sha256 = data_sha(HELD_OUT_SCHEMA, &h.body).unwrap(); +} +fn rehash_eval(e: &mut TemporalEvaluationReceiptData) { + e.sha256 = data_sha(EVALUATION_SCHEMA, &e.body).unwrap(); +} +#[test] +fn three_canonical_data_records_roundtrip_without_authority() { + let t = train(); + let h = held(&t); + let e = evaluation(&t, &h); + let tb = serde_json::to_vec(&t).unwrap(); + let hb = serde_json::to_vec(&h).unwrap(); + let eb = serde_json::to_vec(&e).unwrap(); + assert_eq!(TemporalTrainProofData::decode_data(&tb).unwrap(), t); + assert_eq!(TemporalHeldOutEvidenceData::decode_data(&hb).unwrap(), h); + assert_eq!(TemporalEvaluationReceiptData::decode_data(&eb).unwrap(), e); + e.validate_data_links(&t, &h).unwrap(); + assert!(!e.body.claims.adoption_authorized); + assert!(!e.body.claims.qualification_granted); +} +#[test] +fn unknown_inner_identity_and_mutable_descriptor_fields_are_rejected() { + let t = train(); + let h = held(&t); + let e = evaluation(&t, &h); + for key in ["model_sha256", "model_version", "adoption_authorized"] { + let mut value = serde_json::to_value(&e).unwrap(); + value["body"]["report"][key] = json!("untrusted-inner-identity"); + assert!( + TemporalEvaluationReceiptData::decode_data(&serde_json::to_vec(&value).unwrap()) + .is_err() + ); + } + let mut value = serde_json::to_value(&t).unwrap(); + value["body"]["identity"]["original_deadline"]["saved_at_ms"] = json!(1_800_000_500_000u64); + assert!(TemporalTrainProofData::decode_data(&serde_json::to_vec(&value).unwrap()).is_err()); +} +#[test] +fn canonical_encoding_size_digest_and_domain_near_misses_fail() { + let t = train(); + let bytes = serde_json::to_vec(&t).unwrap(); + assert!(TemporalTrainProofData::decode_data(&[bytes.clone(), vec![b' ']].concat()).is_err()); + assert!(TemporalTrainProofData::decode_data(&vec![b' '; MAX_METADATA_BYTES + 1]).is_err()); + let mut bad = t.clone(); + bad.sha256 = fixture_sha(999); + assert!(bad.validate_data().is_err()); + bad = t.clone(); + bad.schema = HELD_OUT_SCHEMA.into(); + bad.sha256 = data_sha(&bad.schema, &bad.body).unwrap(); + assert!(bad.validate_data().is_err()); +} +#[test] +fn rehashed_positive_claims_roles_fake_cohort_counts_and_wrong_kind_fail() { + let t = train(); + let h = held(&t); + let e = evaluation(&t, &h); + for i in 0..5 { + let mut bad = t.clone(); + match i { + 0 => bad.body.claims.adoption_authorized = true, + 1 => bad.body.claims.qualification_granted = true, + 2 => bad.body.claims.independence_verified = true, + 3 => bad.body.claims.physically_validated = true, + _ => bad.body.claims.measurement_clock_available = true, + }; + rehash_train(&mut bad); + assert!(bad.validate_data().is_err()); + } + let mut bad = t.clone(); + bad.body.windows_per_condition = [4; 4]; + rehash_train(&mut bad); + assert!(bad.validate_data().is_err()); + bad = t.clone(); + bad.body.identity.artifact_kind = "ruview.room-model.v1".into(); + rehash_train(&mut bad); + assert!(bad.validate_data().is_err()); + let mut badh = h.clone(); + badh.body.declared_episodes_per_condition = [16; 4]; + rehash_held(&mut badh); + assert!(badh.validate_data().is_err()); + let mut bade = e.clone(); + bade.body.role = DataRole::Inference; + rehash_eval(&mut bade); + assert!(bade.validate_data().is_err()); + bade = e.clone(); + bade.body.clock_claims = vec![ClockKind::SourceMeasurement]; + rehash_eval(&mut bade); + assert!(bade.validate_data().is_err()); + bade = e.clone(); + bade.body.report.declared_trial_count = 64; + rehash_eval(&mut bade); + assert!(bade.validate_data().is_err()); +} +#[test] +fn deadline_clock_floor_and_canonical_integer_checks_do_not_rebase() { + let t = train(); + let h = held(&t); + let e = evaluation(&t, &h); + for cut in [ + "01800000900000000000", + "+1800000900000000000", + "18446744073709551616", + "1800086400000000000", + "1799999999999999999", + ] { + let mut bad = t.clone(); + bad.body.freeze_prepare_receiver_ns = cut.into(); + rehash_train(&mut bad); + assert!(bad.validate_data().is_err(), "{cut}"); + } + let mut bad = t.clone(); + bad.body.identity.original_deadline.expires_at_unix_ms += 1; + rehash_train(&mut bad); + assert!(bad.validate_data().is_err()); + bad = t.clone(); + bad.body.source_floors[0].receiver_ns = "1800000900000000001".into(); + rehash_train(&mut bad); + assert!(bad.validate_data().is_err()); + let mut bade = e.clone(); + bade.body.evaluated_at_unix_ms += 1; + rehash_eval(&mut bade); + assert!(bade.validate_data().is_err()); + let mut badh = h.clone(); + badh.body.complete_prepare_receiver_ns = "1800000800000000000".into(); + rehash_held(&mut badh); + assert!(badh.validate_data_links(&t).is_err()); +} +#[test] +fn rehashed_cross_record_owner_model_policy_dataset_and_revision_mismatches_fail() { + let t = train(); + let h = held(&t); + let e = evaluation(&t, &h); + for i in 0..5 { + let mut bad = h.clone(); + match i { + 0 => bad.body.identity.owner_scope = "owner-b".into(), + 1 => bad.body.identity.artifact_sha256 = fixture_sha(999), + 2 => bad.body.identity.continuation_policy_sha256 = fixture_sha(998), + 3 => bad.body.training_proof_sha256 = fixture_sha(997), + _ => bad.body.complete_commitment.revision = 1, + }; + rehash_held(&mut bad); + assert!(bad.validate_data_links(&t).is_err()); + } + let mut bad = e.clone(); + bad.body.held_out_data_sha256 = fixture_sha(999); + rehash_eval(&mut bad); + assert!(bad.validate_data_links(&t, &h).is_err()); + bad = e.clone(); + bad.body.complete_record_sha256 = fixture_sha(998); + rehash_eval(&mut bad); + assert!(bad.validate_data_links(&t, &h).is_err()); +} +#[test] +fn train_data_identity_does_not_follow_heldout_progress_or_evaluation_time() { + let t = train(); + let original = serde_json::to_vec(&t).unwrap(); + let mut h = held(&t); + let mut e = evaluation(&t, &h); + h.body.complete_commitment.revision += 1; + h.body.complete_record_sha256 = fixture_sha(777); + rehash_held(&mut h); + e.body.held_out_evidence_sha256 = h.sha256.clone(); + e.body.complete_record_sha256 = h.body.complete_record_sha256.clone(); + e.body.evaluated_at_unix_ms += 1; + e.body.evaluated_at_receiver_ns = "1800001301001000000".into(); + rehash_eval(&mut e); + e.validate_data_links(&t, &h).unwrap(); + assert_eq!(original, serde_json::to_vec(&t).unwrap()); +} +#[test] +fn bounded_max_width_metadata_fixture_fits_without_relaxing_reader_limit() { + let mut t = train(); + let id = &mut t.body.identity; + let escaped = "\"".repeat(256); + id.owner_scope = escaped.clone(); + id.installation_id = escaped.clone(); + id.run = escaped.clone(); + id.original_deadline.id = escaped.clone(); + id.source_ids = [253, 254, 255]; + id.original_deadline.expires_at_unix_ms = u64::MAX / 1_000_000 + 1; + id.original_deadline.created_at_unix_ms = id.original_deadline.expires_at_unix_ms - TTL_MS; + t.body.freeze_prepare_receiver_ns = (u64::MAX - 1_000_000_000_000).to_string(); + t.body.accepted_train_commitment.run = escaped.clone(); + t.body.accepted_train_commitment.id = escaped; + t.body.accepted_train_commitment.generation = u64::MAX; + t.body.accepted_train_commitment.revision = 430; + for (floor, id) in t + .body + .source_floors + .iter_mut() + .zip(t.body.identity.source_ids) + { + floor.source_id = id; + floor.sequence = u32::MAX as u64 - 100_000; + floor.receiver_ns = (u64::MAX - 1_001_000_000_000).to_string(); + } + rehash_train(&mut t); + let mut h = held(&t); + h.body.complete_commitment.generation = u64::MAX; + h.body.complete_commitment.revision = 432; + h.body.complete_prepare_receiver_ns = (u64::MAX - 1_000_000_000).to_string(); + h.body.complete_persisted_receiver_ns = h.body.complete_prepare_receiver_ns.clone(); + for floor in &mut h.body.final_source_fences { + let ns = u64::MAX - 1_000_000_001; + floor.sequence = u32::MAX as u64; + floor.receiver_ns = ns.to_string(); + floor.native_frame_id = ((u128::from(ns) << 40) + | (u128::from(floor.source_id) << 32) + | u128::from(floor.sequence)) + .to_string(); + } + rehash_held(&mut h); + let mut e = evaluation(&t, &h); + e.body.evaluated_at_receiver_ns = (u64::MAX - 1).to_string(); + e.body.evaluated_at_unix_ms = (u64::MAX - 1) / 1_000_000; + for condition in &mut e.body.report.per_condition { + condition.binary_cross_entropy = Some(1.234_567_8e-30); + condition.declared_session_count = 16; + } + e.body.report.binary_cross_entropy = Some(1.234_567_8e-30); + e.body.report.declared_session_count = 64; + rehash_eval(&mut e); + e.validate_data_links(&t, &h).unwrap(); + let sizes = [ + serde_json::to_vec(&t).unwrap().len(), + serde_json::to_vec(&h).unwrap().len(), + serde_json::to_vec(&e).unwrap().len(), + ]; + assert!(sizes.iter().all(|n| *n < MAX_METADATA_BYTES)); + eprintln!("SYNTHETIC bounded max-width DATA bytes={sizes:?}; not a raw128 recovery-size or performance proof"); +} + +#[test] +fn final_complete_source_fences_are_exact_bounded_native_metadata_not_a_live_permit() { + let t = train(); + let h = held(&t); + h.validate_data_links(&t).unwrap(); + for case in 0..9 { + let mut bad = h.clone(); + match case { + 0 => bad.body.final_source_fences[0].source_id = 99, + 1 => bad.body.final_source_fences.swap(0, 1), + 2 => bad.body.final_source_fences[0].sequence = u32::MAX as u64 + 1, + 3 => bad.body.final_source_fences[0].receiver_ns = "01800001299999000000".into(), + 4 => bad.body.final_source_fences[0].native_frame_id = "01".into(), + 5 => { + bad.body.final_source_fences[0].native_frame_id = + "340282366920938463463374607431768211456".into() + } + 6 => bad.body.final_source_fences[0].native_frame_id = "999".into(), + 7 => bad.body.complete_persisted_receiver_ns = "1800001299000000000".into(), + _ => bad.body.complete_persisted_receiver_ns = "1800001300000000001".into(), + } + rehash_held(&mut bad); + assert!(bad.validate_data().is_err(), "case{case}"); + } + let mut lower = h.clone(); + let floor = &mut lower.body.final_source_fences[0]; + floor.sequence = t.body.source_floors[0].sequence - 1; + let ns = decimal(&floor.receiver_ns).unwrap(); + floor.native_frame_id = + ((u128::from(ns) << 40) | (u128::from(floor.source_id) << 32) | u128::from(floor.sequence)) + .to_string(); + rehash_held(&mut lower); + assert!(lower.validate_data_links(&t).is_err()); + let mut value = serde_json::to_value(&h).unwrap(); + value["body"]["final_source_fences"] + .as_array_mut() + .unwrap() + .pop(); + assert!( + TemporalHeldOutEvidenceData::decode_data(&serde_json::to_vec(&value).unwrap()).is_err() + ); + let mut value = serde_json::to_value(&h).unwrap(); + value["body"]["final_source_fences"][0]["sequence"] = json!("20000"); + assert!( + TemporalHeldOutEvidenceData::decode_data(&serde_json::to_vec(&value).unwrap()).is_err() + ); + assert!(!h.body.claims.adoption_authorized); +} + + +fn operator_records() -> (TemporalTrainProofData, TemporalHeldOutEvidenceData, TemporalEvaluationReceiptData) { + let mut tb = train().body; + tb.train_seal_sha256s.truncate(4); + tb.declared_episodes_per_condition = [1; 4]; + let t = TemporalTrainProofData::from_body_data(tb).unwrap(); + let mut hb = held(&t).body; + hb.held_out_seal_sha256s.truncate(4); + hb.held_out_windows = 16; + hb.windows_per_condition = [4; 4]; + hb.declared_episodes_per_condition = [1; 4]; + let h = TemporalHeldOutEvidenceData::from_body_data(hb).unwrap(); + let mut eb = evaluation(&t, &h).body; + eb.role = DataRole::Validation; + let mut report = serde_json::to_value(&eb.report).unwrap(); + for key in ["samples", "correct", "scored", "binary_cross_entropy_samples"] { report[key] = json!(16); } + for key in ["declared_trial_count", "declared_session_count"] { report[key] = json!(4); } + report["confusion"] = confusion(8, 8); + for c in report["per_condition"].as_array_mut().unwrap() { + for key in ["samples", "correct", "scored", "binary_cross_entropy_samples"] { c[key] = json!(4); } + for key in ["declared_trial_count", "declared_session_count"] { c[key] = json!(1); } + let occupied = c["condition"].as_str().unwrap().starts_with("Occupied"); + c["confusion"] = confusion(if occupied { 0 } else { 4 }, if occupied { 4 } else { 0 }); + } + eb.report = serde_json::from_value(report).unwrap(); + let e = TemporalEvaluationReceiptData::from_body_data(eb).unwrap(); + (t, h, e) +} + +#[test] +fn operator_eight_step_validation_metadata_roundtrips_without_heldout_claims() { + let (t, h, e) = operator_records(); + assert_eq!(t.schema, OPERATOR_TRAIN_SCHEMA); + assert_eq!(h.schema, OPERATOR_CHECK_SCHEMA); + assert_eq!(e.schema, OPERATOR_EVALUATION_SCHEMA); + assert_eq!(e.body.role, DataRole::Validation); + assert_eq!(e.body.report.samples, 16); + assert!(!e.body.report.independence_verified); + e.validate_data_links(&t, &h).unwrap(); + assert_eq!(TemporalTrainProofData::decode_data(&t.encode_data().unwrap()).unwrap(), t); + assert_eq!(TemporalHeldOutEvidenceData::decode_data(&h.encode_data().unwrap()).unwrap(), h); + assert_eq!(TemporalEvaluationReceiptData::decode_data(&e.encode_data().unwrap()).unwrap(), e); +} + +#[test] +fn operator_and_legacy_metadata_cannot_be_relabelled_or_cross_linked_after_rehash() { + let (t, h, e) = operator_records(); + let mut old_name = t.clone(); old_name.schema = TRAIN_SCHEMA.into(); old_name.sha256 = data_sha(&old_name.schema, &old_name.body).unwrap(); + assert!(old_name.validate_data().is_err()); + let mut invented = train(); invented.schema = OPERATOR_TRAIN_SCHEMA.into(); invented.sha256 = data_sha(&invented.schema, &invented.body).unwrap(); + assert!(invented.validate_data().is_err()); + let mut wrong_role = e.clone(); wrong_role.body.role = DataRole::HeldOut; wrong_role.sha256 = data_sha(&wrong_role.schema, &wrong_role.body).unwrap(); + assert!(wrong_role.validate_data().is_err()); + let mut wrong_count = h.clone(); wrong_count.body.held_out_windows = 64; wrong_count.sha256 = data_sha(&wrong_count.schema, &wrong_count.body).unwrap(); + assert!(wrong_count.validate_data().is_err()); + let legacy = train(); + let mut crossed = h.clone(); crossed.body.training_proof_sha256 = legacy.sha256.clone(); crossed.sha256 = data_sha(&crossed.schema, &crossed.body).unwrap(); + assert!(crossed.validate_data_links(&legacy).is_err()); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_evaluation_projection.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_evaluation_projection.rs new file mode 100644 index 0000000000..4c3ff453f2 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_evaluation_projection.rs @@ -0,0 +1,582 @@ +//! Protected C4C projections from the SAME current FullRun producer. +//! Raw prefix/DATA decoding cannot create either projection or this pending +//! one-use source. All current-store IO, validation and hashing run off-lock. +use super::*; +use crate::room_learner::temporal_evaluation as data; +use std::sync::atomic::{AtomicBool, Ordering}; + +#[derive(Clone, PartialEq)] +pub(crate) struct TemporalEvaluationBinding { + epoch: String, + generation: u64, + context: CaptureContext, + context_sha256: String, + run: String, + token: TemporalToken, + fit: String, + model: String, + complete: Option, +} +impl TemporalEvaluationBinding { + pub(crate) fn owner_matches(&self, epoch: &str, generation: u64) -> bool { + self.epoch == epoch && self.generation == generation + } + pub(crate) fn context_digest(&self) -> &str { + &self.context_sha256 + } + pub(crate) fn context(&self) -> &CaptureContext { + &self.context + } + pub(crate) fn recertify_current(&self, store: &RecoveryStore, now: u64) -> Result<(), String> { + store.recertify_temporal(&self.token, now).map(|_| ()) + } + pub(crate) fn model_sha256(&self) -> &str { + &self.model + } + pub(crate) fn complete_sha256(&self) -> Option<&str> { + self.complete.as_deref() + } +} +/// No Clone/serde/import constructor. Minted below only from current owned +/// verified Frozen/Complete facts; consuming it authenticates the current store. +pub(crate) struct PendingTemporalEvidenceSource { + binding: TemporalEvaluationBinding, + model: Arc, + artifact: Arc, + context: CaptureContext, + start_floor: u64, + deadline: u64, + held_out: bool, +} +pub(crate) struct VerifiedTemporalTrainSnapshot { + model: Arc, + artifact: Arc, + proof: data::TemporalTrainProofData, +} +pub(crate) struct VerifiedTemporalEvaluationSnapshot { + train: VerifiedTemporalTrainSnapshot, + binding: TemporalEvaluationBinding, + held_out: data::TemporalHeldOutEvidenceData, + rows: Vec>, +} +impl VerifiedTemporalTrainSnapshot { + pub(crate) fn model(&self) -> &Arc { + &self.model + } + pub(crate) fn require_same_store(&self, store: &ArtifactStore) -> Result<(), String> { + if !std::ptr::eq(store, self.artifact.as_ref()) { + return Err("temporal_evaluation_same_store_required".into()); + } + Ok(()) + } + pub(crate) fn proof(&self) -> &data::TemporalTrainProofData { + &self.proof + } +} +impl VerifiedTemporalEvaluationSnapshot { + pub(crate) fn train_snapshot(&self) -> &VerifiedTemporalTrainSnapshot { + &self.train + } + pub(crate) fn proof(&self) -> &data::TemporalTrainProofData { + self.train.proof() + } + pub(crate) fn held_out(&self) -> &data::TemporalHeldOutEvidenceData { + &self.held_out + } + pub(crate) fn binding(&self) -> &TemporalEvaluationBinding { + &self.binding + } + pub(crate) fn model(&self) -> &Arc { + &self.train.model + } + pub(crate) fn require_same_store(&self, store: &ArtifactStore) -> Result<(), String> { + if !std::ptr::eq(store, self.train.artifact.as_ref()) { + return Err("temporal_evaluation_same_store_required".into()); + } + Ok(()) + } + pub(crate) fn evaluation_rows(&self) -> Vec { + self.rows.iter().map(|r| (**r).clone()).collect() + } + pub(crate) fn require_same_model(&self, model: &TemporalModel) -> Result<(), String> { + if !std::ptr::eq(model, self.train.model.as_ref()) { + return Err("temporal_evaluation_actual_model_required".into()); + } + Ok(()) + } +} +impl TemporalWorkflow { + pub(crate) fn prepare_temporal_evidence_source( + &self, + now: u64, + held_out: bool, + schedule: &ManagedScheduleContract, + ) -> Result { + let operator = schedule.is_operator_eight_step(); + let mode = self.full_run.as_ref().ok_or("temporal_full_run_required")?; + let owner = mode + .owner + .as_ref() + .ok_or("temporal_full_run_owner_custody_required")?; + let verified = mode + .verified + .as_ref() + .ok_or("temporal_verified_frozen_required")?; + let fit = mode + .frozen + .as_ref() + .ok_or("temporal_verified_fit_record_required")?; + let token = self + .commit_token + .as_ref() + .ok_or("temporal_current_token_required")?; + // A paused suffix mapping cannot change the already fitted/confirmed + // first64 Train rows. It is permitted ONLY for stable TrainProof, never + // for a HeldOut evaluation/qualification/selection capability. + let suffix_mapping = !held_out + && self.stage == Stage::PausedNeedsReconfirmation + && schedule.sha256() == fit.body.schedule_sha256 + && schedule + .episodes() + .get(self.episode) + .is_some_and(|e| e.role == DataRole::HeldOut) + && self.bridge.as_ref().is_some_and(|b| b.seals.len() == 7) + && self.session.rows.len() >= 64; + if self.mapping.is_some() && !suffix_mapping { + return Err("temporal_train_projection_mapping_overlaps_active_or_train".into()); + } + if self.session.failed + || self.session.active + || self.stage == Stage::BlockedDurability + || self.pending_intent.is_some() + || self.pending_prefix.is_some() + || self + .async_state + .as_ref() + .is_none_or(|a| a.pending.is_some() || a.busy_pause || a.error.is_some()) + || self.durable_windows != self.session.rows.len() + || fit != &verified.record + || now < self.last_now_ns + || now / 1_000_000 >= self.descriptor.expires_at_ms + || mode.pending.is_some() + || mode.pending_complete.is_some() + { + return Err("temporal_evidence_current_durable_owner_required".into()); + } + let complete = if held_out { + let fact = mode + .completed + .as_ref() + .ok_or("temporal_actual_complete_required")?; + let record = mode + .complete + .as_ref() + .ok_or("temporal_complete_record_required")?; + if fact.token != *token + || fact.record_sha256 != record.sha256 + || (if operator { + self.bridge.as_ref().map_or(0, |b| b.seals.iter().filter(|s| + schedule.episodes().get(s.episode).is_some_and(|e| e.role == DataRole::Validation) + ).count()) + } else { mode.held_out_seals() }) != if operator { 4 } else { 8 } + || self.session.rows.len() != if operator { 80 } else { 128 } + { + return Err("temporal_current_complete_changed".into()); + } + Some(record.sha256.clone()) + } else { + if (operator && self.session.rows.len() != 80) + || (!operator && !(64..=128).contains(&self.session.rows.len())) { + return Err("temporal_stable_train_projection_required".into()); + } + None + }; + // Arc refs and bounded identity only. Native tensors/records, SHA work, + // full model validation and disk access are deliberately not done here. + Ok(PendingTemporalEvidenceSource { + binding: TemporalEvaluationBinding { + epoch: owner.runtime_epoch.clone(), + generation: owner.generation, + context: self.context().clone(), + context_sha256: fit.body.context_digest.clone(), + run: self.session.run_id.clone(), + token: token.clone(), + fit: fit.sha256.clone(), + model: fit.body.artifact_sha256.clone(), + complete, + }, + model: verified.model.clone(), + artifact: verified.store.clone(), + context: self.context().clone(), + start_floor: now, + deadline: self.descriptor.expires_at_ms, + held_out, + }) + } + pub(crate) fn current_temporal_evidence_matches(&self, b: &TemporalEvaluationBinding) -> bool { + self.full_run.as_ref().is_some_and(|m| { + m.owner + .as_ref() + .is_some_and(|o| o.runtime_epoch == b.epoch && o.generation == b.generation) + && m.frozen + .as_ref() + .is_some_and(|f| f.sha256 == b.fit && f.body.artifact_sha256 == b.model) + && b.complete.as_ref().is_none_or(|sha| { + m.completed + .as_ref() + .is_some_and(|c| c.record_sha256 == *sha && c.token == b.token) + }) + }) && self.commit_token.as_ref() == Some(&b.token) + && self.session.run_id == b.run + && self.context() == &b.context + && !self.session.failed + } +} +fn commitment( + value: &crate::room_learner::recovery_store::temporal_commitment::TemporalCommitmentData, +) -> Result { + Ok(data::CommitmentData { + generation: value.generation, + revision: value + .revision + .try_into() + .map_err(|_| "temporal_commitment_revision")?, + run: value.run.clone(), + id: value.id.clone(), + commitment_sha256: value.commitment.clone(), + }) +} +impl PendingTemporalEvidenceSource { + pub(crate) fn binding(&self) -> &TemporalEvaluationBinding { + &self.binding + } + pub(crate) fn expires_at_ms(&self) -> u64 { + self.deadline + } + fn check(&self, now: u64, cancel: &AtomicBool) -> Result<(), String> { + if cancel.load(Ordering::Acquire) { + return Err("temporal_evaluation_cancelled".into()); + } + if now < self.start_floor || now / 1_000_000 >= self.deadline { + return Err("temporal_evaluation_clock_or_original_deadline".into()); + } + Ok(()) + } + /// One-use actual-current-store recertification; structural decoders have + /// no constructor for this source and cannot reach the authority return. + pub(crate) fn recertify_evaluation( + self, + store: &RecoveryStore, + now: u64, + cancel: &AtomicBool, + ) -> Result { + if !self.held_out { + return Err("temporal_complete_projection_required".into()); + } + let (binding, train, held, rows) = self.recertify(store, now, cancel)?; + Ok(VerifiedTemporalEvaluationSnapshot { + train, + binding, + held_out: held.ok_or("temporal_complete_evidence_required")?, + rows, + }) + } + pub(crate) fn recertify_train( + self, + store: &RecoveryStore, + now: u64, + cancel: &AtomicBool, + ) -> Result { + let (_, train, _, _) = self.recertify(store, now, cancel)?; + Ok(train) + } + fn recertify( + self, + store: &RecoveryStore, + now: u64, + cancel: &AtomicBool, + ) -> Result< + ( + TemporalEvaluationBinding, + VerifiedTemporalTrainSnapshot, + Option, + Vec>, + ), + String, + > { + self.check(now, cancel)?; + let current = store.recertify_temporal(&self.binding.token, now)?; + self.check(now, cancel)?; + let w = checkpoint::decode(current.prefix.bytes(), current.prefix.descriptor(), now)?; + let mode = w + .full_run + .as_ref() + .ok_or("temporal_full_run_no_upconversion")?; + let record = mode + .frozen + .as_ref() + .ok_or("temporal_verified_fit_record_required")?; + if w.context() != &self.context + || w.session.run_id != self.binding.run + || record.sha256 != self.binding.fit + || record.body.artifact_sha256 != self.binding.model + { + return Err("temporal_evaluation_current_source_changed".into()); + } + let loaded = self + .artifact + .load_exact_temporal_candidate(&self.context, &self.binding.model)?; + verify_model_binding(record, &loaded)?; + verify_model_binding(record, &self.model)?; + checkpoint::validate_temporal_evaluation_train_prefix(&w, record)?; + self.check(now, cancel)?; + let schedule = w.schedule()?; + let operator = schedule.is_operator_eight_step(); + if loaded.managed_is_operator_eight_step()? != operator + || self.model.managed_is_operator_eight_step()? != operator { + return Err("temporal_evaluation_profile_changed".into()); + } + let check_role = if operator { DataRole::Validation } else { DataRole::HeldOut }; + let check_rows: Vec<_> = w.session.rows.iter().filter(|r| r.role == check_role).cloned().collect(); + let check_evidence: Vec<_> = w.session.evidence.iter().zip(&w.session.rows) + .filter(|(_, r)| r.role == check_role).map(|(e, _)| e.clone()).collect(); + let identity = data::TemporalEvaluationIdentityData { + owner_scope: w.descriptor.owner_scope.clone(), + installation_id: w.descriptor.installation_id.clone(), + context_digest: w.context().digest(), + room_digest: w.context().room_digest.clone(), + run: w.session.run_id.clone(), + original_deadline: data::OriginalDeadlineData { + id: w.descriptor.id.clone(), + created_at_unix_ms: w.descriptor.created_at_ms, + expires_at_unix_ms: w.descriptor.expires_at_ms, + }, + artifact_kind: MODEL_KIND.into(), + artifact_sha256: self.binding.model.clone(), + verified_fit_record_sha256: record.sha256.clone(), + prefix_schema: PREFIX_SCHEMA.into(), + metadata_kind: META_KIND.into(), + continuation_policy_sha256: record.body.continuation_policy_sha256.clone(), + input_policy_sha256: record.body.input_policy_sha256.clone(), + execution_policy_sha256: record.body.execution_policy_sha256.clone(), + schedule_sha256: record.body.schedule_sha256.clone(), + configured_fit_sha256: record.body.configured_fit_sha256.clone(), + source_ids: w + .context() + .sources + .iter() + .map(|s| s.source_id) + .collect::>() + .try_into() + .map_err(|_| "temporal_evaluation_three_sources_required")?, + }; + let seals = &w + .bridge + .as_ref() + .ok_or("temporal_train_seals_required")? + .seals; + let proof = + data::TemporalTrainProofData::from_body_data(data::TemporalTrainProofBodyData { + identity: identity.clone(), + training_data_sha256: record.body.training_data_sha256.clone(), + training_evidence_sha256: record.body.input_evidence_sha256.clone(), + training_identities_sha256: record.body.identity_evidence_sha256.clone(), + train_seal_sha256s: seals + .iter() + .filter(|s| schedule.episodes().get(s.episode).is_some_and(|e| e.role == DataRole::Train)) + .map(|s| s.sha256.clone()) + .collect(), + train_seal_chain_sha256: record.body.train_seal_chain_sha256.clone(), + train_windows: 64, + windows_per_condition: [16; 4], + declared_episodes_per_condition: if operator { [1; 4] } else { [1, 2, 2, 2] }, + accepted_train_commitment: commitment(&record.body.accepted_train_commitment)?, + freeze_prepare_receiver_ns: record.body.freeze_prepare_receiver_ns.to_string(), + source_floors: record + .body + .source_floors + .iter() + .map(|(id, seq, ns)| data::FrozenSourceFloorData { + source_id: *id, + sequence: *seq, + receiver_ns: ns.to_string(), + }) + .collect::>() + .try_into() + .map_err(|_| "temporal_evaluation_three_sources_required")?, + claims: data::DiagnosticClaimsData::diagnostic_only(), + })?; + let held = if self.held_out { + let complete = mode + .complete + .as_ref() + .ok_or("temporal_actual_complete_required")?; + if Some(&complete.sha256) != self.binding.complete.as_ref() + || w.session.rows.len() != if operator { 80 } else { 128 } + || current.token != self.binding.token + { + return Err("temporal_current_complete_changed".into()); + } + let managed = w + .session + .managed + .as_ref() + .ok_or("managed_session_required")?; + let check_windows: Vec<_> = managed.windows.iter().filter(|w| w.role == check_role).cloned().collect(); + if check_rows.len() != if operator { 16 } else { 64 } + || check_evidence.len() != check_rows.len() + || check_windows.len() != check_rows.len() { + return Err("temporal_evaluation_reserved_check_shape".into()); + } + let check_seals: Vec = if operator { + seals.iter().filter(|s| schedule.episodes().get(s.episode).is_some_and(|e| e.role == DataRole::Validation)) + .map(|s| s.sha256.clone()).collect() + } else { + mode.held_out.as_ref().ok_or("temporal_heldout_seals_required")?.seals.iter().map(|s| s.sha256.clone()).collect() + }; + let evidence = + data::TemporalHeldOutEvidenceData::from_body_data(data::TemporalHeldOutBodyData { + identity, + training_proof_sha256: proof.sha256.clone(), + held_out_data_sha256: hash(&check_rows)?, + held_out_evidence_sha256: hash(&check_evidence)?, + held_out_identities_sha256: hash(&check_windows)?, + complete_rows_sha256: complete.body.raw_sha256.clone(), + complete_evidence_sha256: complete.body.evidence_sha256.clone(), + complete_identities_sha256: complete.body.identity_sha256.clone(), + complete_ledger_sha256: complete.body.ledger_sha256.clone(), + held_out_seal_sha256s: check_seals, + held_out_seal_chain_sha256: complete.body.held_out_chain_sha256.clone(), + complete_record_sha256: complete.sha256.clone(), + complete_commitment: commitment(¤t.token.data())?, + complete_prepare_receiver_ns: complete + .body + .complete_prepare_receiver_ns + .to_string(), + complete_persisted_receiver_ns: w.last_now_ns.to_string(), + final_source_fences: w + .source_high + .iter() + .map(|(id, seq, ns)| data::FinalSourceFenceData { + source_id: *id, + sequence: *seq, + receiver_ns: ns.to_string(), + native_frame_id: ((u128::from(*ns) << 40) + | (u128::from(*id) << 32) + | u128::from(*seq)) + .to_string(), + }) + .collect::>() + .try_into() + .map_err(|_| "temporal_evaluation_three_sources_required")?, + held_out_windows: if operator { 16 } else { 64 }, + windows_per_condition: if operator { [4; 4] } else { [16; 4] }, + declared_episodes_per_condition: if operator { [1; 4] } else { [2; 4] }, + claims: data::DiagnosticClaimsData::diagnostic_only(), + })?; + evidence.validate_data_links(&proof)?; + Some(evidence) + } else { + None + }; + let rows = if self.held_out { + check_rows + } else { + vec![] + }; + self.check(now, cancel)?; + Ok(( + self.binding, + VerifiedTemporalTrainSnapshot { + model: self.model, + artifact: self.artifact, + proof, + }, + held, + rows, + )) + } +} + +/// No serde/Clone/from-DATA constructor: computed only from the protected +/// snapshot and the SAME whole model's existing evaluator. +pub(crate) struct VerifiedTemporalEvaluationResult { + artifact: Arc, + binding: TemporalEvaluationBinding, + train: data::TemporalTrainProofData, + held: data::TemporalHeldOutEvidenceData, + receipt: data::TemporalEvaluationReceiptData, +} +impl VerifiedTemporalEvaluationResult { + pub(crate) fn require_same_store(&self, store: &ArtifactStore) -> Result<(), String> { + if !std::ptr::eq(store, self.artifact.as_ref()) { + return Err("temporal_evaluation_same_store_required".into()); + } + Ok(()) + } + pub(crate) fn binding(&self) -> &TemporalEvaluationBinding { + &self.binding + } + pub(crate) fn train(&self) -> &data::TemporalTrainProofData { + &self.train + } + pub(crate) fn held_out(&self) -> &data::TemporalHeldOutEvidenceData { + &self.held + } + pub(crate) fn receipt(&self) -> &data::TemporalEvaluationReceiptData { + &self.receipt + } +} +impl VerifiedTemporalEvaluationSnapshot { + pub(crate) fn compute_verified( + &self, + cancel: &AtomicBool, + clock: &mut dyn FnMut() -> Result, + ) -> Result { + let before = clock()?; + if cancel.load(Ordering::Acquire) { + return Err("temporal_evaluation_cancelled".into()); + } + let deadline = &self.proof().body.identity.original_deadline; + if before / 1_000_000 >= deadline.expires_at_unix_ms { + return Err("temporal_evaluation_original_deadline_expired".into()); + } + let report = self.model().evaluate_managed_held_out(self)?; + // Existing compute is bounded64 rows. Cancellation withdraws admission + // and drains the entire call; no per-row interruption is claimed. + let after = clock()?; + if cancel.load(Ordering::Acquire) + || after < before + || after / 1_000_000 >= deadline.expires_at_unix_ms + { + return Err("temporal_evaluation_cancelled_or_clock".into()); + } + let receipt = data::TemporalEvaluationReceiptData::from_body_data( + data::TemporalEvaluationBodyData { + identity: self.proof().body.identity.clone(), + training_proof_sha256: self.proof().sha256.clone(), + held_out_evidence_sha256: self.held_out().sha256.clone(), + complete_record_sha256: self.held_out().body.complete_record_sha256.clone(), + training_data_sha256: self.proof().body.training_data_sha256.clone(), + held_out_data_sha256: self.held_out().body.held_out_data_sha256.clone(), + role: if self.proof().is_operator_eight_step() { DataRole::Validation } else { DataRole::HeldOut }, + window_unit: "native_receiver_2s_window".into(), + evaluated_at_receiver_ns: after.to_string(), + evaluated_at_unix_ms: after / 1_000_000, + provenance: vec![CaptureProvenance::LiveReceiverObserved], + clock_claims: vec![ClockKind::ReceiverOnly], + train_lineage_checked: true, + report, + claims: data::DiagnosticClaimsData::diagnostic_only(), + }, + )?; + receipt.validate_data_links(self.proof(), self.held_out())?; + Ok(VerifiedTemporalEvaluationResult { + artifact: self.train.artifact.clone(), + binding: self.binding.clone(), + train: self.proof().clone(), + held: self.held_out().clone(), + receipt, + }) + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_evaluation_store.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_evaluation_store.rs new file mode 100644 index 0000000000..483f360fce --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_evaluation_store.rs @@ -0,0 +1,546 @@ +//! Immutable whole-Temporal metadata in the SAME ArtifactStore owner root. +//! Raw decoder returns DATA only. Protected source/result is required to write +//! or claim a current evaluation; no parallel DB/key/latest pointer is added. +use super::*; +use crate::room_learner::{ + observation::DataRole, + temporal_evaluation as data, + temporal_model_input::managed::execution::full_run::{ + TemporalEvaluationBinding, VerifiedTemporalEvaluationResult, + VerifiedTemporalEvaluationSnapshot, VerifiedTemporalTrainSnapshot, + }, +}; +use std::sync::atomic::{AtomicBool, Ordering}; +const CHILD: &str = "managed-evaluation"; +pub(crate) struct StoredTemporalEvaluation { + binding: TemporalEvaluationBinding, + train: data::TemporalTrainProofData, + held: data::TemporalHeldOutEvidenceData, + receipt: data::TemporalEvaluationReceiptData, + lookup_sha256: String, +} +impl StoredTemporalEvaluation { + pub(crate) fn binding(&self) -> &TemporalEvaluationBinding { + &self.binding + } + pub(crate) fn train(&self) -> &data::TemporalTrainProofData { + &self.train + } + pub(crate) fn held_out(&self) -> &data::TemporalHeldOutEvidenceData { + &self.held + } + pub(crate) fn receipt(&self) -> &data::TemporalEvaluationReceiptData { + &self.receipt + } + pub(crate) fn lookup_sha256(&self) -> &str { + &self.lookup_sha256 + } +} +fn lookup(source: &VerifiedTemporalEvaluationSnapshot) -> Result { + let identity = &source.proof().body.identity; + crate::room_learner::qualification::hash(&( + if source.proof().is_operator_eight_step() { data::OPERATOR_EVALUATION_SCHEMA } else { data::EVALUATION_SCHEMA }, + "lookup", + &identity.artifact_kind, + &identity.artifact_sha256, + &source.proof().sha256, + &source.held_out().sha256, + if source.proof().is_operator_eight_step() { DataRole::Validation } else { DataRole::HeldOut }, + )) +} +fn allowed( + cancel: &AtomicBool, + clock: &mut dyn FnMut() -> Result, +) -> Result { + if cancel.load(Ordering::Acquire) { + return Err("temporal_evaluation_cancelled".into()); + } + clock() +} +impl ArtifactStore { + pub(crate) fn persist_current_temporal_train_proof( + &self, + source: &VerifiedTemporalTrainSnapshot, + cancel: &AtomicBool, + clock: &mut dyn FnMut() -> Result, + io: &mut dyn FnMut(&'static str) -> Result<(), String>, + ) -> Result { + source.require_same_store(self)?; + allowed(cancel, clock)?; + let directory = self + .temporal_evaluation_directory(source.model().context(), true)? + .ok_or("temporal_evaluation_directory_unavailable")?; + let bytes = source.proof().encode_data()?; + let path = directory.join(format!("train-proof-{}.json", source.proof().sha256)); + temporal_artifacts::immutable_with_io(&path, &bytes, &mut |_| { + io("train_proof_install_before_directory_fsync") + })?; + allowed(cancel, clock)?; + let read = data::TemporalTrainProofData::decode_data(&temporal_artifacts::bounded_read( + &path, + data::MAX_METADATA_BYTES, + )?)?; + if read != *source.proof() { + return Err("temporal_train_proof_readback_changed".into()); + } + temporal_artifacts::recertify_immutable( + &path, + &bytes, + data::MAX_METADATA_BYTES, + &mut |stage| { + allowed(cancel, clock)?; + io(if stage == "before:file_fsync" { + "train_proof_recertify_before_file_fsync" + } else { + "train_proof_recertify_before_directory_fsync" + }) + }, + )?; + self.recertify_temporal_metadata_directories(&directory, cancel, clock, io)?; + Ok(read) + } + pub(super) fn temporal_evaluation_directory( + &self, + context: &CaptureContext, + create: bool, + ) -> Result, String> { + let Some(root) = self.temporal_directory(context, create)? else { + return Ok(None); + }; + let directory = root.join(CHILD); + if !temporal_artifacts::directory(&directory, create)? { + return Ok(None); + } + Ok(Some(directory)) + } + /// The current protected Complete snapshot supplies exact paths/identities. + /// A generic imported receipt/proof never reaches this return constructor. + pub(super) fn load_current_temporal_evaluation( + &self, + source: &VerifiedTemporalEvaluationSnapshot, + ) -> Result, String> { + source.require_same_store(self)?; + let Some(directory) = + self.temporal_evaluation_directory(source.model().context(), false)? + else { + return Ok(None); + }; + let key = lookup(source)?; + let path = directory.join(format!("evaluation-{key}.json")); + match std::fs::symlink_metadata(&path) { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(_) => return Err("temporal_evaluation_file_unavailable".into()), + Ok(_) => {} + } + let receipt = data::TemporalEvaluationReceiptData::decode_data( + &temporal_artifacts::bounded_read(&path, data::MAX_METADATA_BYTES)?, + )?; + let train = data::TemporalTrainProofData::decode_data(&temporal_artifacts::bounded_read( + &directory.join(format!("train-proof-{}.json", source.proof().sha256)), + data::MAX_METADATA_BYTES, + )?)?; + let held = + data::TemporalHeldOutEvidenceData::decode_data(&temporal_artifacts::bounded_read( + &directory.join(format!( + "heldout-evidence-{}.json", + source.held_out().sha256 + )), + data::MAX_METADATA_BYTES, + )?)?; + receipt.validate_data_links(&train, &held)?; + if train != *source.proof() || held != *source.held_out() { + return Err("temporal_evaluation_current_source_changed".into()); + } + Ok(Some(StoredTemporalEvaluation { + binding: source.binding().clone(), + train, + held, + receipt, + lookup_sha256: key, + })) + } + /// Existing compute and immutable IO; exact retry loads the previous record + /// BEFORE inference, preserving the ORIGINAL evaluation timestamp/report. + pub(crate) fn evaluate_temporal_snapshot( + &self, + source: VerifiedTemporalEvaluationSnapshot, + cancel: &AtomicBool, + clock: &mut dyn FnMut() -> Result, + io: &mut dyn FnMut(&'static str) -> Result<(), String>, + ) -> Result { + allowed(cancel, clock)?; + source.require_same_store(self)?; + if let Some(existing) = self.load_current_temporal_evaluation(&source)? { + let now = allowed(cancel, clock)?; + let evaluated = existing + .receipt() + .body + .evaluated_at_receiver_ns + .parse::() + .map_err(|_| "temporal_evaluation_data_integer")?; + if now < evaluated { + return Err("temporal_evaluation_stored_clock_reversed".into()); + } + return self.recertify_current_temporal_evaluation(&source, existing, cancel, clock, io); + } + let key = lookup(&source)?; + let computed = source.compute_verified(cancel, clock)?; + self.persist_temporal_evaluation_result(&source, &computed, &key, cancel, clock, io) + } + fn persist_temporal_evaluation_result( + &self, + source: &VerifiedTemporalEvaluationSnapshot, + result: &VerifiedTemporalEvaluationResult, + key: &str, + cancel: &AtomicBool, + clock: &mut dyn FnMut() -> Result, + io: &mut dyn FnMut(&'static str) -> Result<(), String>, + ) -> Result { + source.require_same_store(self)?; + result.require_same_store(self)?; + if result.binding() != source.binding() { + return Err("temporal_evaluation_operation_changed".into()); + } + allowed(cancel, clock)?; + let directory = self + .temporal_evaluation_directory(source.model().context(), true)? + .ok_or("temporal_evaluation_directory_unavailable")?; + let writes = [ + ( + format!("train-proof-{}.json", result.train().sha256), + result.train().encode_data()?, + ), + ( + format!("heldout-evidence-{}.json", result.held_out().sha256), + result.held_out().encode_data()?, + ), + ( + format!("evaluation-{key}.json"), + result.receipt().encode_data()?, + ), + ]; + for (name, bytes) in writes { + allowed(cancel, clock)?; + // Same immutable private stage/link/fsync primitive as whole models. + let stage = if name.starts_with("evaluation-") { + "evaluation_receipt_install_before_directory_fsync" + } else if name.starts_with("train-proof-") { + "train_proof_install_before_directory_fsync" + } else { + "heldout_evidence_install_before_directory_fsync" + }; + temporal_artifacts::immutable_with_io(&directory.join(&name), &bytes, &mut |_| { + io(stage) + })?; + allowed(cancel, clock)?; + if temporal_artifacts::bounded_read(&directory.join(name), data::MAX_METADATA_BYTES)? + != bytes + { + return Err("temporal_evaluation_immutable_readback_changed".into()); + } + } + allowed(cancel, clock)?; + let stored = self + .load_current_temporal_evaluation(source)? + .ok_or("temporal_evaluation_readback_missing")?; + self.recertify_current_temporal_evaluation(source, stored, cancel, clock, io) + } + fn recertify_current_temporal_evaluation( + &self, + source: &VerifiedTemporalEvaluationSnapshot, + stored: StoredTemporalEvaluation, + cancel: &AtomicBool, + clock: &mut dyn FnMut() -> Result, + io: &mut dyn FnMut(&'static str) -> Result<(), String>, + ) -> Result { + source.require_same_store(self)?; + allowed(cancel, clock)?; + let directory = self + .temporal_evaluation_directory(source.model().context(), false)? + .ok_or("temporal_evaluation_directory_unavailable")?; + let records = [ + ( + format!("train-proof-{}.json", stored.train.sha256), + stored.train.encode_data()?, + "train", + ), + ( + format!("heldout-evidence-{}.json", stored.held.sha256), + stored.held.encode_data()?, + "held", + ), + ( + format!("evaluation-{}.json", stored.lookup_sha256), + stored.receipt.encode_data()?, + "evaluation", + ), + ]; + for (name, bytes, kind) in records { + temporal_artifacts::recertify_immutable( + &directory.join(name), + &bytes, + data::MAX_METADATA_BYTES, + &mut |stage| { + allowed(cancel, clock)?; + io(match (kind, stage) { + ("evaluation", "before:file_fsync") => { + "evaluation_receipt_recertify_before_file_fsync" + } + ("evaluation", _) => "evaluation_receipt_recertify_before_directory_fsync", + ("train", "before:file_fsync") => "train_proof_recertify_before_file_fsync", + ("train", _) => "train_proof_recertify_before_directory_fsync", + (_, "before:file_fsync") => "heldout_evidence_recertify_before_file_fsync", + _ => "heldout_evidence_recertify_before_directory_fsync", + }) + }, + )?; + } + self.recertify_temporal_metadata_directories(&directory, cancel, clock, io)?; + allowed(cancel, clock)?; + let read = self + .load_current_temporal_evaluation(source)? + .ok_or("temporal_evaluation_readback_missing")?; + if read.train != stored.train || read.held != stored.held || read.receipt != stored.receipt + { + return Err("temporal_evaluation_recertify_changed".into()); + } + Ok(read) + } + fn recertify_temporal_metadata_directories( + &self, + directory: &Path, + cancel: &AtomicBool, + clock: &mut dyn FnMut() -> Result, + io: &mut dyn FnMut(&'static str) -> Result<(), String>, + ) -> Result<(), String> { + let mut path = directory; + loop { + allowed(cancel, clock)?; + temporal_artifacts::directory(path, false)?; + io("metadata_directory_chain_before_fsync")?; + std::fs::File::open(path) + .and_then(|f| f.sync_all()) + .map_err(|_| "temporal_evaluation_directory_sync_failed")?; + if path == self.root { + break; + } + path = path + .parent() + .ok_or("temporal_evaluation_directory_outside_owner")?; + if !path.starts_with(&self.root) { + return Err("temporal_evaluation_directory_outside_owner".into()); + } + } + Ok(()) + } + /// Explicit inventory for privacy/reporting; contents include labels, + /// run/trial and receiver/frame/sequence lineage metadata, not aggregates only. + pub(crate) fn temporal_evaluation_inventory( + &self, + context: &CaptureContext, + ) -> Result, String> { + let Some(directory) = self.temporal_evaluation_directory(context, false)? else { + return Ok(vec![]); + }; + let mut files = vec![]; + for entry in + std::fs::read_dir(directory).map_err(|_| "temporal_evaluation_inventory_unavailable")? + { + if files.len() >= 512 { + return Err("temporal_evaluation_inventory_too_many_records".into()); + } + let entry = entry.map_err(|_| "temporal_evaluation_inventory_unavailable")?; + let name = entry.file_name().to_string_lossy().into_owned(); + if !(name.starts_with("train-proof-") + || name.starts_with("heldout-evidence-") + || name.starts_with("evaluation-") + || name.starts_with("latest-assessment-") + || name.starts_with("assessment-") + || name.starts_with("qualification-") + || name.starts_with("temporal-stage-")) + { + return Err("temporal_evaluation_inventory_unknown_file".into()); + } + let bytes = temporal_artifacts::bounded_read(&entry.path(), data::MAX_METADATA_BYTES)?; + files.push((name, bytes.len() as u64)); + } + files.sort(); + Ok(files) + } +} + +pub(crate) struct StoredTemporalAssessment { + binding: TemporalEvaluationBinding, + pub(crate) evaluation: StoredTemporalEvaluation, + pub(crate) qualification: + Option, + pub(crate) assessment: crate::room_learner::temporal_qualification::TemporalAssessmentEnvelope, +} +impl StoredTemporalAssessment { + pub(crate) fn binding(&self) -> &TemporalEvaluationBinding { + &self.binding + } +} +impl ArtifactStore { + pub(crate) fn qualify_temporal_snapshot( + &self, + source: &VerifiedTemporalEvaluationSnapshot, + cancel: &AtomicBool, + clock: &mut dyn FnMut() -> Result, + io: &mut dyn FnMut(&'static str) -> Result<(), String>, + ) -> Result { + use crate::room_learner::temporal_qualification::*; + source.require_same_store(self)?; + allowed(cancel, clock)?; + // A qualification operation consumes only an already durable evaluation; + // it does not invent or silently recompute an evaluation report. + let existing = self + .load_current_temporal_evaluation(&source)? + .ok_or("temporal_evaluation_required")?; + let evaluation = + self.recertify_current_temporal_evaluation(&source, existing, cancel, clock, io)?; + let directory = self + .temporal_evaluation_directory(source.model().context(), false)? + .ok_or("temporal_evaluation_directory_unavailable")?; + let key = crate::room_learner::qualification::hash(&( + crate::room_learner::temporal_wire::ASSESSMENT_SCHEMA, + &evaluation.receipt().sha256, + crate::room_learner::temporal_wire::TemporalQualificationPolicyWire::for_operator_eight_step(source.proof().is_operator_eight_step()), + ))?; + let path = directory.join(format!("assessment-{key}.json")); + let (assessment, qualification) = match std::fs::symlink_metadata(&path) { + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + let computed = ComputedTemporalAssessment::issue_current( + &source, + &evaluation, + allowed(cancel, clock)?, + )?; + if let Some(q) = &computed.qualification { + let bytes = + serde_json::to_vec(q).map_err(|_| "temporal_qualification_encoding")?; + temporal_artifacts::immutable_with_io( + &directory.join(format!("qualification-{}.json", q.receipt_sha256)), + &bytes, + &mut |_| { + allowed(cancel, clock)?; + io("qualification_install_before_directory_fsync") + }, + )?; + } + let bytes = serde_json::to_vec(&computed.assessment) + .map_err(|_| "temporal_assessment_encoding")?; + temporal_artifacts::immutable_with_io(&path, &bytes, &mut |_| { + allowed(cancel, clock)?; + io("assessment_install_before_directory_fsync") + })?; + (computed.assessment, computed.qualification) + } + Err(_) => return Err("temporal_assessment_unavailable".into()), + Ok(_) => { + let assessment: TemporalAssessmentEnvelope = serde_json::from_slice( + &temporal_artifacts::bounded_read(&path, data::MAX_METADATA_BYTES)?, + ) + .map_err(|_| "temporal_assessment_encoding")?; + let qualification = match &assessment.receipt.summary.qualification_sha256 { + Some(sha) => { + if !hex_digest(sha) { + return Err("temporal_qualification_digest".into()); + } + Some( + serde_json::from_slice(&temporal_artifacts::bounded_read( + &directory.join(format!("qualification-{sha}.json")), + data::MAX_METADATA_BYTES, + )?) + .map_err(|_| "temporal_qualification_encoding")?, + ) + } + None => None, + }; + (assessment, qualification) + } + }; + assessment.validate(&evaluation, qualification.as_ref())?; + if allowed(cancel, clock)? / 1_000_000 < assessment.receipt.summary.assessed_at_unix_ms { + return Err("temporal_assessment_clock_reversed".into()); + } + let mut records = vec![( + path, + serde_json::to_vec(&assessment).map_err(|_| "temporal_assessment_encoding")?, + )]; + if let Some(q) = &qualification { + records.push(( + directory.join(format!("qualification-{}.json", q.receipt_sha256)), + serde_json::to_vec(q).map_err(|_| "temporal_qualification_encoding")?, + )); + } + for (path, bytes) in records { + temporal_artifacts::recertify_immutable( + &path, + &bytes, + data::MAX_METADATA_BYTES, + &mut |_| { + allowed(cancel, clock)?; + io("assessment_recertify_before_sync") + }, + )?; + } + self.recertify_temporal_metadata_directories(&directory, cancel, clock, io)?; + allowed(cancel, clock)?; + let latest = serde_json::json!({"schema":"ruview.room-learner.temporal-current-assessment.v1","model_sha256":source.binding().model_sha256(),"context_digest":source.binding().context_digest(),"assessment_sha256":assessment.receipt_sha256,"assessment_lookup_sha256":key}); + allowed(cancel, clock)?; + atomic_private_write( + &directory.join(format!( + "latest-assessment-{}.json", + source.binding().model_sha256() + )), + &serde_json::to_vec(&latest).map_err(|_| "temporal_assessment_encoding")?, + ) + .map_err(|_| "temporal_assessment_pointer_write")?; + allowed(cancel, clock)?; + Ok(StoredTemporalAssessment { + binding: source.binding().clone(), + evaluation, + qualification, + assessment, + }) + } +} + +impl ArtifactStore { + pub(crate) fn reconcile_temporal_snapshot( + &self, + source: VerifiedTemporalEvaluationSnapshot, + cancel: &AtomicBool, + clock: &mut dyn FnMut() -> Result, + io: &mut dyn FnMut(&'static str) -> Result<(), String>, + ) -> Result< + ( + data::TemporalTrainProofData, + Option, + ), + String, + > { + source.require_same_store(self)?; + let proof = + self.persist_current_temporal_train_proof(source.train_snapshot(), cancel, clock, io)?; + let existing = self.load_current_temporal_evaluation(&source)?; + if let Some(existing) = &existing { + let evaluated = existing + .receipt() + .body + .evaluated_at_receiver_ns + .parse::() + .map_err(|_| "temporal_evaluation_data_integer")?; + if allowed(cancel, clock)? < evaluated { + return Err("temporal_evaluation_stored_clock_reversed".into()); + } + } + let stored = existing + .map(|existing| { + self.recertify_current_temporal_evaluation(&source, existing, cancel, clock, io) + }) + .transpose()?; + allowed(cancel, clock)?; + Ok((proof, stored)) + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_execution.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_execution.rs new file mode 100644 index 0000000000..a71522eedc --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_execution.rs @@ -0,0 +1,861 @@ +//! Private future ONE Runtime Temporal workflow; no public constructor/API issuer. +//! Durability is not scientific qualification or an assertion of whole-trial quality. +use super::*; +use crate::room_learner::recovery_store::temporal_commitment::{ + TemporalAbortTicket, TemporalIntent, TemporalToken, +}; +use crate::room_learner::{ + managed_schedule_contract::BoundEpisode, + recovery_store::{RecoveryDescriptor, RecoveryStore, RECOVERY_TTL_MS}, + temporal_acquisition::{AssemblerCheckpoint, ManagedWindowTarget}, +}; +use serde::{Deserialize, Serialize}; +#[path = "temporal_durability.rs"] +mod durability; +use durability::AsyncState; +pub(crate) use durability::DeferredReclamation; +#[cfg(test)] pub(crate) use durability::ReclamationStats; +#[path = "temporal_capture_quality.rs"] +mod bridge; +use bridge::BridgeState; +pub(crate) use bridge::{TemporalCaptureStage, TemporalTrainCapture}; +#[path = "temporal_fit_admission.rs"] +mod fit_admission; +pub(crate) use fit_admission::{PendingTemporalFitInput, TemporalFitCaptureIdentity,VerifiedTemporalFitSource}; +#[path = "temporal_full_run.rs"] +pub(crate) mod full_run; +#[path="temporal_held_out_capture.rs"] +mod held_out; +#[path="temporal_prefix_digests.rs"] +mod prefix_digests; + +pub(crate) const EXECUTION_POLICY:&str="observed-complete-reservation-prefix-v1;paused-time;immutable-spans;partial-tail-drop;max128segments;persegment-native-firstarrival-skew-gaps;explicit-genesis-gap-anchor;quality-unimplemented;independence-false"; +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub(crate) struct ObservedSegmentMap { + pub(crate) segment_ordinal: u16, + pub(crate) session_id: String, + pub(crate) receiver_epoch: String, + pub(crate) clock_anchor_ns: u64, + pub(crate) receiver_start_ns: u64, + pub(crate) observed_start_ns: u64, +} +impl ObservedSegmentMap { + pub(crate) fn slot_bounds( + &self, + b: &BoundEpisode, + a: u64, + z: u64, + ) -> Result<(u64, u64), String> { + if self.segment_ordinal != b.segment_ordinal() + || self.session_id != b.session_id() + || self.receiver_epoch.is_empty() + || self.receiver_epoch.len() > 128 + || self.receiver_start_ns == 0 + || self.clock_anchor_ns == 0 + || self.receiver_start_ns < self.clock_anchor_ns + || a < self.observed_start_ns + || z <= a + || z > b.episode().target_active_recording_ms * 1_000_000 + { + return Err("managed_observed_segment_mapping".into()); + } + Ok(( + self.receiver_start_ns + .checked_add(a - self.observed_start_ns) + .ok_or("managed_mapping_overflow")?, + self.receiver_start_ns + .checked_add(z - self.observed_start_ns) + .ok_or("managed_mapping_overflow")?, + )) + } + #[cfg(test)] + pub(crate) fn synthetic_uninterrupted(b: &BoundEpisode, start: u64) -> Self { + Self { + segment_ordinal: b.segment_ordinal(), + session_id: b.session_id().into(), + receiver_epoch: "synthetic-uninterrupted".into(), + clock_anchor_ns: start, + receiver_start_ns: start, + observed_start_ns: 0, + } + } +} +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +struct CommittedSpan { + episode: usize, + slot: usize, + mapping: ObservedSegmentMap, + observed_start_ns: u64, + observed_end_ns: u64, + receiver_end_ns: u64, + previous_sha256: Option, + window_sha256: String, +} +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +struct GenesisGap { + previous_mapping: ObservedSegmentMap, + paused_receiver_ns: u64, + source_floors: Vec<(u8, u64, u64)>, + sha256: String, +} +impl GenesisGap { + fn digest( + &self, + descriptor: &RecoveryDescriptor, + run: &str, + episode: usize, + ) -> Result { + hash(&( + "temporal-zero-prefix-genesis-v1", + EXECUTION_POLICY, + &descriptor.id, + &descriptor.owner_scope, + &descriptor.installation_id, + &descriptor.context_digest, + descriptor.created_at_ms, + descriptor.expires_at_ms, + run, + episode, + &self.previous_mapping, + self.paused_receiver_ns, + &self.source_floors, + )) + } + fn new( + descriptor: &RecoveryDescriptor, + run: &str, + episode: usize, + previous_mapping: ObservedSegmentMap, + paused_receiver_ns: u64, + source_floors: Vec<(u8, u64, u64)>, + ) -> Result { + let mut v = Self { + previous_mapping, + paused_receiver_ns, + source_floors, + sha256: String::new(), + }; + v.sha256 = v.digest(descriptor, run, episode)?; + Ok(v) + } +} +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +struct Gap { + episode: usize, + after_prefix_sha256: Option, + genesis: Option, + accepted_observed_ns: u64, + previous_receiver_ns: u64, + resume_receiver_ns: u64, + discarded_live_ns: u64, + new_mapping: ObservedSegmentMap, +} +#[derive(Clone, Default, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +struct Ledger { + spans: Vec, + gaps: Vec, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Stage { + Prepared, + DurableReady, + Observing, + PausedNeedsReconfirmation, + PreparedSegment, + BlockedDurability, +} +/// Neither JSON nor a generic saved checkpoint can construct an operator permit. +struct ConditionsPermit { + context: String, + run: String, + durable_sha256: Option, +} +/// Transient per-segment observation custody, distinct from persistent replay +/// high-watermarks. It is NEVER restored as active observation from JSON. +#[derive(Default)] +struct SegmentCustody { + first_last: Vec<(u8, u64, u64)>, + failed: bool, +} +/// Future value owned by existing RoomLearningRuntime::Inner's ONE tagged workflow. +/// No production constructor/permit issuer is exposed by this source slice. +struct TemporalWorkflow { + full_run: Option, + session: TemporalTrainingSession, + descriptor: RecoveryDescriptor, + ledger: Ledger, + stage: Stage, + mapping: Option, + episode: usize, + last_now_ns: u64, + live_observed_ns: u64, + source_high: Vec<(u8, u64, u64)>, + pending_window: Option, + durable: Option>>, + durable_windows: usize, + custody: SegmentCustody, + commit_token: Option, + pending_intent: Option, + pending_prefix: Option, + bridge: Option, + async_state: Option, +} +impl TemporalWorkflow { + fn context(&self) -> &CaptureContext { + &self.session.binding.context + } + fn schedule(&self) -> Result { + self.session + .managed + .as_ref() + .ok_or("managed_session_required")? + .binding + .contract() + } + fn prefix_hash(&self) -> Result, String> { + self.ledger.spans.last().map(hash).transpose() + } + fn accepted(&self) -> u64 { + self.ledger + .spans + .iter() + .take(self.durable_windows) + .last() + .filter(|s| s.episode == self.episode) + .map_or(0, |s| s.observed_end_ns) + } + fn check_clock(&mut self, now: u64) -> Result<(), String> { + if now < self.last_now_ns || now / 1_000_000 >= self.descriptor.expires_at_ms { + return Err("temporal_receiver_clock_or_expiry".into()); + } + self.last_now_ns = now; + Ok(()) + } + fn prepare( + schedule: ManagedScheduleContract, + run: String, + config: FitConfig, + now: u64, + ) -> Result { + if now == 0 { + return Err("temporal_receiver_clock".into()); + } + let c = schedule.context().clone(); + let digest = c.digest(); + let session = TemporalTrainingSession::managed(schedule, run, config)?; + Ok(Self { + full_run: None, + session, + descriptor: RecoveryDescriptor { + schema: crate::room_learner::recovery_store::RECOVERY_SCHEMA.into(), + id: crate::companion::secret(), + owner_scope: c.owner_scope, + installation_id: c.installation_id, + context_digest: digest, + created_at_ms: now / 1_000_000, + saved_at_ms: now / 1_000_000, + expires_at_ms: now / 1_000_000 + RECOVERY_TTL_MS, + }, + ledger: Ledger::default(), + stage: Stage::Prepared, + mapping: None, + episode: 0, + last_now_ns: now, + live_observed_ns: 0, + source_high: vec![], + pending_window: None, + durable: None, + durable_windows: 0, + custody: SegmentCustody::default(), + commit_token: None, + pending_intent: None, + pending_prefix: None, + bridge: None, + async_state: None, + }) + } + #[cfg(test)] + fn synthetic_conditions(&self) -> ConditionsPermit { + ConditionsPermit { + context: self.context().digest(), + run: self.session.run_id.clone(), + durable_sha256: self.durable_digest(), + } + } + fn start_segment( + &mut self, + permit: &ConditionsPermit, + epoch: &str, + anchor: u64, + start: u64, + ) -> Result<(), String> { + if !matches!( + self.stage, + Stage::DurableReady | Stage::PausedNeedsReconfirmation + ) || permit.context != self.context().digest() + || permit.run != self.session.run_id + || permit.durable_sha256 != self.durable_digest() + || epoch.is_empty() + || epoch.len() > 128 + || anchor < self.last_now_ns + || start < anchor + || (self.ledger.gaps.len() >= 128 && (self.full_run.is_none() || self.mapping.is_some())) + { + return Err("temporal_reconfirmation_or_clock_required".into()); + } + let paused_receiver_ns = self.last_now_ns; + self.check_clock(start)?; + let schedule = self.schedule()?; + let segment = self.mapping.as_ref().map_or(0, |m| m.segment_ordinal + 1); + let frozen=self.frozen_identity()?; + let bound = schedule.bind_episode_start(&self.session.run_id, self.episode, 0, segment, frozen.as_ref())?; + let mapping = ObservedSegmentMap { + segment_ordinal: segment, + session_id: bound.session_id().into(), + receiver_epoch: epoch.into(), + clock_anchor_ns: anchor, + receiver_start_ns: start, + observed_start_ns: self.accepted(), + }; + if let Some(old) = &self.mapping { + let parent = self + .ledger + .spans + .last() + .filter(|s| s.episode == self.episode) + .map(hash) + .transpose()?; + let genesis = if parent.is_none() { + Some(GenesisGap::new( + &self.descriptor, + &self.session.run_id, + self.episode, + old.clone(), + paused_receiver_ns, + self.source_high.clone(), + )?) + } else { + None + }; + let prev = self + .ledger + .spans + .last() + .filter(|s| s.episode == self.episode) + .map_or(paused_receiver_ns, |s| s.receiver_end_ns); + self.ledger.gaps.push(Gap { + episode: self.episode, + after_prefix_sha256: parent, + genesis, + accepted_observed_ns: self.accepted(), + previous_receiver_ns: prev, + resume_receiver_ns: start, + discarded_live_ns: self.live_observed_ns.saturating_sub(self.accepted()), + new_mapping: mapping.clone(), + }); + } + self.custody = SegmentCustody::default(); + self.mapping = Some(mapping); + self.live_observed_ns = self.accepted(); + self.stage = Stage::PreparedSegment; + Ok(()) + } + + fn persist_segment(&mut self, store: &RecoveryStore) -> Result<(), String> { + if self.stage != Stage::PreparedSegment { + return Err("temporal_segment_not_prepared".into()); + } + self.persist_with(store, false, |s, intent, prefix, now_ns| { + s.publish_temporal(intent, Some(prefix), now_ns) + })?; + self.stage = Stage::Observing; + Ok(()) + } + fn begin_slot(&mut self) -> Result { + if self.stage != Stage::Observing || self.pending_window.is_some() { + return Err("temporal_slot_state".into()); + } + let schedule = self.schedule()?; + let mapping = self.mapping.clone().ok_or("temporal_segment_missing")?; + let frozen=self.frozen_identity()?; + let bound = schedule.bind_episode_start( + &self.session.run_id,self.episode,0,mapping.segment_ordinal,frozen.as_ref(), + )?; + let slot = self.session.rows.len() + - schedule + .episodes() + .iter() + .take(self.episode) + .map(|e| e.slots.len()) + .sum::(); + let target = ManagedWindowTarget::from_observed_episode( + &schedule, + &bound, + &self.session.run_id, + slot, + mapping, + )?; + self.session.begin_managed_window(&target)?; + self.pending_window = Some(target.clone()); + Ok(target) + } + /// Busy packets and trusted receiver ticks both withdraw custody when a + /// required source is absent/stale; neither can backfill missing intervals. + fn custody_at(&mut self, now: u64) -> Result<(), String> { + if self.stage != Stage::Observing || self.custody.failed { + return Err("temporal_segment_custody_unavailable".into()); + } + let mapping = self.mapping.as_ref().ok_or("temporal_segment_missing")?; + let native = &self + .session + .managed + .as_ref() + .ok_or("managed_session_required")? + .binding + .native; + let first_limit = native + .max_first_packet_delay_ms() + .checked_mul(1_000_000) + .ok_or("temporal_time_overflow")?; + if now < mapping.receiver_start_ns { + return Err("temporal_segment_clock".into()); + } + for source in native.source_policies() { + let invalid = match self + .custody + .first_last + .iter() + .find(|(id, _, _)| *id == source.source_id) + { + None => now - mapping.receiver_start_ns > first_limit, + Some((_, _, last)) => now < *last || now - *last > source.max_gap_ns, + }; + if invalid { + self.custody.failed = true; + return Err("temporal_missing_segment_source".into()); + } + } + Ok(()) + } + fn observe_tick(&mut self, now: u64) -> Result<(), String> { + self.check_clock(now)?; + self.custody_at(now) + } + fn observe(&mut self, p: &CanonicalPacket, now: u64) -> Result<(), String> { + if self.stage != Stage::Observing { + return Err("temporal_not_observing".into()); + } + self.check_clock(now)?; + self.custody_at(now)?; + let source_count = self.context().sources.len(); + let c = self.context(); + let spec = c + .sources + .iter() + .find(|s| s.source_id == p.source_id) + .ok_or("temporal_unbound_source")?; + let t = p.native_frame.timestamp_ns; + if t > now + || p.grid_id != spec.grid_id + || p.frequency_hz != spec.center_frequency_hz + || p.bandwidth_hz != spec.bandwidth_hz + || p.native_frame.provenance.device_id != spec.device_id + || p.amplitudes.len() != spec.feature_count + || p.amplitudes.iter().any(|v| !v.is_finite() || *v < 0.) + || self.source_high.iter().any(|(id, seq, last)| { + *id == p.source_id && (u64::from(p.sequence) <= *seq || t <= *last) + }) + { + return Err("temporal_observation_source_or_fence".into()); + } + let mapping = self.mapping.as_ref().ok_or("temporal_segment_missing")?; + if t < mapping.receiver_start_ns { + return Err("temporal_segment_replay".into()); + } + let native = &self + .session + .managed + .as_ref() + .ok_or("managed_session_required")? + .binding + .native; + let policy = native + .source_policies() + .iter() + .find(|s| s.source_id == p.source_id) + .ok_or("temporal_unbound_source")?; + if let Some((_, _, last)) = self.source_high.iter().find(|(id, _, last)| { + *id == p.source_id + && *last >= mapping.receiver_start_ns + && (t - *last > policy.max_gap_ns || t - *last < policy.min_packet_interval_ns) + }) { + let interval_ns = t - *last; + let direction = if interval_ns < policy.min_packet_interval_ns { + "below_min" + } else { + "above_max" + }; + // Receiver-processing interval only: no RF values, absolute time or + // source-clock claim. Keep the existing failure prefix and guards. + return Err(format!( + "temporal_observed_gap_or_rate: source={} direction={} receiver_interval_ns={} min_ns={} max_ns={}", + p.source_id, direction, interval_ns, policy.min_packet_interval_ns, policy.max_gap_ns + )); + } + let first_limit = native.max_first_packet_delay_ms() * 1_000_000; + if let Some((_, _, last)) = self + .custody + .first_last + .iter_mut() + .find(|(id, _, _)| *id == p.source_id) + { + *last = t; + } else { + if t - mapping.receiver_start_ns > first_limit { + self.custody.failed = true; + return Err("temporal_late_segment_source".into()); + } + self.custody.first_last.push((p.source_id, t, t)); + } + let first_min = self + .custody + .first_last + .iter() + .map(|(_, first, _)| *first) + .min() + .unwrap(); + let first_max = self + .custody + .first_last + .iter() + .map(|(_, first, _)| *first) + .max() + .unwrap(); + if first_max - first_min > native.max_pairwise_receiver_skew_ms() * 1_000_000 { + self.custody.failed = true; + return Err("temporal_segment_first_source_skew".into()); + } + self.source_high.retain(|(id, _, _)| *id != p.source_id); + self.source_high + .push((p.source_id, u64::from(p.sequence), t)); + self.source_high.sort(); + if self.custody.first_last.len() == source_count { + let frontier = self + .custody + .first_last + .iter() + .map(|(_, _, t)| *t) + .min() + .unwrap(); + if frontier >= mapping.receiver_start_ns { + self.live_observed_ns = + mapping.observed_start_ns + frontier - mapping.receiver_start_ns; + } + } + if self.session.active { + let target = self + .pending_window + .as_ref() + .ok_or("temporal_target_missing")?; + if t >= target.target_ns() { + if self.session.ingest(p)? == WindowAdmission::Ready { + self.session.finish_window(t)?; + } + } + } + Ok(()) + } + fn pause(&mut self) -> Result<(), String> { + if self.stage != Stage::Observing { + return Err("temporal_pause_state".into()); + } + let bytes = self + .durable + .clone() + .ok_or("temporal_durable_prefix_missing")?; + let ram = self.session.assembler.fence_checkpoint()?; + let high = self.source_high.clone(); + let mapping = self.mapping.clone(); + let live = self.live_observed_ns; + let last = self.last_now_ns; + let token = self.commit_token.clone(); + let mut restored = checkpoint::decode(&bytes, &self.descriptor, last)?; + restored.commit_token = token; + restored.session.assembler.restore_prefix_fences( + ram.for_prefix(restored.session.rows.len())?, + restored.session.rows.len(), + )?; + restored.mapping = mapping; + restored.live_observed_ns = live; + restored.source_high = high; + restored.last_now_ns = last; + restored.stage = Stage::PausedNeedsReconfirmation; + restored.durable = Some(bytes); + *self = restored; + Ok(()) + } + fn persist_initial(&mut self, store: &RecoveryStore) -> Result<(), String> { + if self.stage != Stage::Prepared { + return Err("temporal_prepare_state".into()); + } + self.persist_with(store, false, |store, intent, prefix, now_ns| { + store.publish_temporal(intent, Some(prefix), now_ns) + })?; + self.stage = Stage::DurableReady; + Ok(()) + } + fn commit_slot(&mut self, store: &RecoveryStore, now: u64) -> Result<(), String> { + self.check_clock(now)?; + self.persist_with(store, true, |store, intent, prefix, now_ns| { + store.publish_temporal(intent, Some(prefix), now_ns) + })?; + self.last_now_ns = now.max(self.last_now_ns); + Ok(()) + } + fn persist_with( + &mut self, + store: &RecoveryStore, + slot: bool, + save: impl FnOnce( + &RecoveryStore, + &TemporalIntent, + &checkpoint::ValidatedTemporalPrefix, + u64, + ) -> Result, + ) -> Result<(), String> { + if self.async_state.as_ref().is_some_and(|a| a.pending.is_some()) { + self.async_state.as_mut().unwrap().busy_pause = true; + self.stage = Stage::BlockedDurability; + return Err("temporal_storage_busy_pause".into()); + } + if slot { + self.custody_at(self.last_now_ns)?; + let t = self + .pending_window + .as_ref() + .ok_or("temporal_no_completed_window")?; + let schedule = self.schedule()?; + let e = &schedule.episodes()[self.episode]; + let s = &e.slots[t.identity().slot_index]; + if self.stage != Stage::Observing + || self.session.active + || self.session.rows.len() != self.ledger.spans.len() + 1 + || self.last_now_ns < t.end_ns() + || self.source_high.len() != self.context().sources.len() + || self.source_high.iter().any(|(_, _, n)| *n < t.end_ns()) + { + return Err("temporal_full_reservation_not_observed".into()); + } + let mapping = t.identity().mapping.clone(); + let accepted = self.accepted(); + let span = CommittedSpan { + episode: self.episode, + slot: s.index, + mapping, + observed_start_ns: accepted, + observed_end_ns: s.deadline_offset_ms * 1_000_000, + receiver_end_ns: t.end_ns(), + previous_sha256: self.prefix_hash()?, + window_sha256: hash(self.session.rows.last().unwrap())?, + }; + self.ledger.spans.push(span); + } + self.descriptor.saved_at_ms = self.last_now_ns / 1_000_000; + if self.async_state.is_some() { + let result=self.submit_async_commit(slot); + if result.is_err(){self.stage=Stage::BlockedDurability;} + return result; + } + let bytes = checkpoint::encode(self)?; + let prefix = checkpoint::validated_prefix(&bytes, &self.descriptor, self.last_now_ns)?; + let intent = + match store.prepare_temporal(self.commit_token.as_ref(), &prefix, self.last_now_ns) { + Ok(i) => i, + Err(e) => { + self.stage = Stage::BlockedDurability; + return Err(e); + } + }; + self.pending_intent = Some(intent.clone()); + self.pending_prefix = Some(prefix.clone()); + let token = match save(store, &intent, &prefix, self.last_now_ns).and_then(|token| { + let loaded = store + .load_temporal(self.last_now_ns)? + .ok_or("temporal_readback_missing")?; + if loaded.token != token + || loaded.prefix.as_ref().map(|p| p.bytes()) != Some(prefix.bytes()) + { + return Err("temporal_readback_changed".into()); + } + Ok(token) + }) { + Ok(token) => token, + Err(e) => { + self.stage = Stage::BlockedDurability; + return Err(e); + } + }; + self.commit_token = Some(token); + self.pending_intent = None; + self.pending_prefix = None; + self.durable_windows = self.session.rows.len(); + self.durable = Some(std::sync::Arc::new(bytes)); + if slot { + self.pending_window = None; + } + Ok(()) + } + fn abort_pending_artifact( + &mut self, + store: &RecoveryStore, + ticket: &TemporalAbortTicket, + now_ns: u64, + ) -> Result<(), String> { + // The Full2 frame-budget proof requires adopt/SAME then at most one + // next tail. Aborting to an older parent could discard TWO consumed + // windows per resume; this generic synchronous route is not Full2. + self.require_abortable_contract()?; + if self.stage != Stage::BlockedDurability + || Some(ticket.expected_token()) != self.commit_token.as_ref() + || now_ns < self.last_now_ns + { + return Err("temporal_abort_workflow_changed".into()); + } + let ram = self.session.assembler.fence_checkpoint()?; + let high = self.source_high.clone(); + let mapping = self.mapping.clone(); + let live = self.live_observed_ns; + let loaded = store.abort_temporal_pending(ticket, now_ns)?; + let prefix = loaded.prefix.ok_or("temporal_recovery_retired")?; + let mut restored = checkpoint::decode(prefix.bytes(), prefix.descriptor(), now_ns)?; + restored.session.assembler.restore_prefix_fences( + ram.for_prefix(restored.session.rows.len())?, + restored.session.rows.len(), + )?; + restored.source_high = high; + restored.mapping = mapping; + restored.live_observed_ns = live; + restored.commit_token = Some(loaded.token); + restored.durable = Some(std::sync::Arc::new(prefix.bytes().to_vec())); + restored.stage = Stage::PausedNeedsReconfirmation; + restored.last_now_ns = now_ns; + *self = restored; + Ok(()) + } + fn repair_pending(&mut self, store: &RecoveryStore, now_ns: u64) -> Result<(), String> { + if self.stage != Stage::BlockedDurability || now_ns < self.last_now_ns { + return Err("temporal_pending_repair_state".into()); + } + let intent = self + .pending_intent + .as_ref() + .ok_or("temporal_pending_intent_missing")?; + let prefix = self + .pending_prefix + .as_ref() + .ok_or("temporal_pending_prefix_missing")?; + let token = store.repair_same_temporal(intent, Some(prefix), now_ns)?; + let mut restored = checkpoint::decode(prefix.bytes(), prefix.descriptor(), now_ns)?; + restored.commit_token = Some(token); + restored.durable = Some(std::sync::Arc::new(prefix.bytes().to_vec())); + restored.stage = Stage::PausedNeedsReconfirmation; + restored.last_now_ns = now_ns; + *self = restored; + Ok(()) + } + fn cold_restore( + store: &RecoveryStore, + expected: &CaptureContext, + now: u64, + epoch: &str, + anchor: u64, + ) -> Result { + Self::cold_restore_with(store, expected, now, epoch, anchor, |s, t, n| { + s.recertify_temporal(t, n) + }) + } + fn cold_restore_with( + store: &RecoveryStore, + expected: &CaptureContext, + now: u64, + epoch: &str, + anchor: u64, + recertify: impl FnOnce( + &RecoveryStore, + &TemporalToken, + u64, + ) -> Result< + crate::room_learner::recovery_store::temporal_commitment::TemporalRestored, + String, + >, + ) -> Result { + let loaded = store + .load_temporal(now)? + .ok_or("temporal_recovery_missing")?; + if loaded.prefix.is_none() { + return Err("temporal_recovery_retired".into()); + } + // A read is only an authenticated observation. Cold restoration must + // re-establish actual barriers before promising a durable saved prefix. + let certified = recertify(store, &loaded.token, now)?; + let prefix = certified.prefix; + let mut v = checkpoint::decode(prefix.bytes(), prefix.descriptor(), now)?; + v.commit_token = Some(certified.token); + if v.context() != expected || epoch.is_empty() || anchor < v.last_now_ns || now < anchor { + return Err("temporal_restore_context_or_clock".into()); + } + v.stage = Stage::PausedNeedsReconfirmation; + v.last_now_ns = now; + v.durable = Some(std::sync::Arc::new(prefix.bytes().to_vec())); + Ok(v) + } +} +#[path = "temporal_checkpoint.rs"] +pub(crate) mod checkpoint; +#[cfg(test)] +#[path = "temporal_execution_tests.rs"] +mod tests; + +impl TemporalWorkflow { + fn require_abortable_contract(&self)->Result<(),String>{if self.full_run.is_some(){Err("temporal_full2_abort_to_parent_unsupported".into())}else{Ok(())}} + fn current_parts(&self)->Result<&Vec,String>{ + if let Some(held)=self.full_run.as_ref().and_then(|m|m.held_out.as_ref()){return Ok(&held.parts);} + Ok(&self.bridge.as_ref().ok_or("temporal_bridge_required")?.parts) + } + fn current_parts_mut(&mut self)->Result<&mut Vec,String>{ + if self.full_run.as_ref().is_some_and(|m|m.held_out.is_some()){return Ok(&mut self.full_run.as_mut().unwrap().held_out.as_mut().unwrap().parts);} + Ok(&mut self.bridge.as_mut().ok_or("temporal_bridge_required")?.parts) + } + fn set_current_disposition(&mut self,disposition:bridge::Disposition)->Result<(),String>{ + if let Some(held)=self.full_run.as_mut().and_then(|m|m.held_out.as_mut()){held.disposition=disposition;} + else{self.bridge.as_mut().ok_or("temporal_bridge_required")?.disposition=disposition;} + Ok(()) + } +} + + +/// Failure-only receiver pipeline metadata. No clock value or RF row is exported. +/// Unknown/oversized durations stay unknown; the original failure is never cut. +pub(crate) fn receiver_pipeline_failure_detail( + primary: &str, + durations: [Option; 4], +) -> Option { + if primary.len() > 256 || !primary.starts_with("temporal_observed_gap_or_rate:") { return None; } + const KEYS: [&str; 4] = ["deq_us", "parse_us", "lock_us", "stamp_age_us"]; + let digits = |value: Option| -> usize { value.map_or(2, |n| if n == 0 { 1 } else { n.ilog10() as usize + 1 }) }; + let length = primary.len() + KEYS.iter().zip(durations).map(|(key, value)| key.len() + 2 + digits(value)).sum::(); + if length > 256 { return None; } + let mut detail = String::with_capacity(length); + detail.push_str(primary); + for (key, value) in KEYS.iter().zip(durations) { + detail.push(' '); detail.push_str(key); detail.push('='); + if let Some(value) = value { use std::fmt::Write; write!(&mut detail, "{value}").ok()?; } + else { detail.push_str("na"); } + } + Some(detail) +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_execution_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_execution_tests.rs new file mode 100644 index 0000000000..cbe9a3a5e9 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_execution_tests.rs @@ -0,0 +1,1026 @@ +//! SYNTHETIC simulated receiver clocks; no sleeps or physical sensing claims. +use super::super::tests::{context, packet}; +use super::*; +use crate::room_learner::managed_schedule_contract::LeadInPolicy; +pub(super) fn setup() -> (tempfile::TempDir, RecoveryStore, TemporalWorkflow) { + let dir = tempfile::tempdir().unwrap(); + let c = context(); + let data = dir.path().join(&c.owner_scope).join("SensingServer"); + let store = + RecoveryStore::new(&data, &c.owner_scope, &c.installation_id, Some(&[7; 32])).unwrap(); + let plan = + ManagedScheduleContract::build(c, LeadInPolicy::Train5HeldOut15EngineeringV1).unwrap(); + let w = TemporalWorkflow::prepare( + plan, + "synthetic-prefix".into(), + FitConfig { + epochs: 2, + ..Default::default() + }, + 1_000_000_000, + ) + .unwrap(); + (dir, store, w) +} +pub(super) fn start(w: &mut TemporalWorkflow, store: &RecoveryStore, time: u64, epoch: &str) { + let permit = w.synthetic_conditions(); + w.start_segment(&permit, epoch, time, time).unwrap(); + assert!( + w.begin_slot().is_err(), + "segment must be durable before collection" + ); + w.persist_segment(store).unwrap(); +} +pub(super) fn drive(w: &mut TemporalWorkflow, from: u64, to: u64, seq: &mut u32) -> u64 { + let period = 1_000_000_000u64.div_ceil(33); + let mut at = from; + while at <= to + period { + *seq += 1; + for source in 1..=3u8 { + let t = at + u64::from(source) * 1000; + w.observe(&packet(source, *seq, t, RoomLabel::Empty, 0), t) + .unwrap(); + } + at += period; + } + at +} +pub(super) fn commit_next( + w: &mut TemporalWorkflow, + store: &RecoveryStore, + from: u64, + seq: &mut u32, +) -> u64 { + let target = w.begin_slot().unwrap(); + let next = drive(w, from, target.end_ns(), seq); + w.commit_slot(store, w.last_now_ns).unwrap(); + next +} +#[test] +fn actual_600second_episode_interruption_keeps_only_durable_prefix_and_original_rows() { + let (_dir, store, mut w) = setup(); + assert!(w.begin_slot().is_err()); + w.persist_initial(&store).unwrap(); + start(&mut w, &store, 2_000_000_000, "epoch-one"); + let mut seq = 0; + let mut next = 2_000_000_000; + for _ in 0..3 { + next = commit_next(&mut w, &store, next, &mut seq); + } + let saved_rows = serde_json::to_vec(&w.session.rows).unwrap(); + let old_ledger = w.ledger.clone(); + let deadline = w.descriptor.expires_at_ms; + let accepted = w.accepted(); + let original_trial = w.session.rows[0].trial_id.clone(); + let old_session = w.session.rows[0].session_id.clone(); + let partial = w.begin_slot().unwrap(); + let next = drive(&mut w, next, partial.target_ns() + 1_000_000_000, &mut seq); + assert!(w.session.active); + assert!(w.live_observed_ns > accepted); + let before_high = w.source_high.clone(); + w.pause().unwrap(); + assert_eq!(w.accepted(), accepted); + assert_eq!(w.source_high, before_high); + assert_eq!(serde_json::to_vec(&w.session.rows).unwrap(), saved_rows); + let resumed = next + 300_000_000_000; + start(&mut w, &store, resumed, "epoch-two"); + assert_eq!(w.ledger.gaps.len(), 1); + assert!(w.ledger.gaps[0].discarded_live_ns > 0); + assert_eq!(w.ledger.gaps[0].accepted_observed_ns, accepted); + assert_eq!(w.ledger.spans, old_ledger.spans); + assert_eq!(w.mapping.as_ref().unwrap().observed_start_ns, accepted); + let newtarget = w.begin_slot().unwrap(); + assert_eq!( + newtarget.target_ns(), + resumed + w.schedule().unwrap().episodes()[0].slots[3].target_offset_ms * 1_000_000 + - accepted + ); + assert_eq!(newtarget.identity().trial_id, original_trial); + assert_ne!(newtarget.identity().session_id, old_session); + let mut next = drive(&mut w, resumed, newtarget.end_ns(), &mut seq); + w.commit_slot(&store, w.last_now_ns).unwrap(); + while w.session.rows.len() < 16 { + next = commit_next(&mut w, &store, next, &mut seq); + } + assert_eq!(w.accepted(), 600_000_000_000); + assert_eq!(w.descriptor.expires_at_ms, deadline); + assert_eq!( + serde_json::to_vec(&w.session.rows[..3]).unwrap(), + saved_rows + ); + assert!(w.last_now_ns - resumed < 600_000_000_000); + assert!(w.session.fit().is_err()); + let mut cold = TemporalWorkflow::cold_restore( + &store, + &context(), + w.last_now_ns + 1_000_000, + "epoch-three", + w.last_now_ns + 1_000_000, + ) + .unwrap(); + assert_eq!(cold.accepted(), 600_000_000_000); + assert_eq!( + serde_json::to_vec(&cold.session.rows).unwrap(), + serde_json::to_vec(&w.session.rows).unwrap() + ); + assert_eq!(cold.descriptor.expires_at_ms, deadline); + assert_eq!(cold.stage, Stage::PausedNeedsReconfirmation); + assert!(cold.session.fit().is_err()); +} +#[test] +fn complete_native_ready_does_not_commit_unobserved_reservation_or_downtime() { + let (_dir, store, mut w) = setup(); + w.persist_initial(&store).unwrap(); + start(&mut w, &store, 2_000_000_000, "one"); + let target = w.begin_slot().unwrap(); + let mut seq = 0; + drive( + &mut w, + 2_000_000_000, + target.target_ns() + 2_100_000_000, + &mut seq, + ); + assert!(!w.session.active); + assert!(w.commit_slot(&store, w.last_now_ns).is_err()); + assert!(w.ledger.spans.is_empty()); + assert_eq!(w.accepted(), 0); +} +#[test] +fn explicit_pause_and_cold_restore_preserve_fences_reject_counter_epoch_and_clock_rescue() { + let (_dir, store, mut w) = setup(); + w.persist_initial(&store).unwrap(); + start(&mut w, &store, 2_000_000_000, "one"); + let mut seq = 0; + let next = commit_next(&mut w, &store, 2_000_000_000, &mut seq); + let last = w.last_now_ns; + assert!(TemporalWorkflow::cold_restore( + &store, + &context(), + last + 1_000_000, + "bad-clock", + last - 1 + ) + .is_err()); + w.pause().unwrap(); + let permit = w.synthetic_conditions(); + assert!(w.start_segment(&permit, "two", last - 1, last + 1).is_err()); + start(&mut w, &store, next + 1_000_000_000, "two"); + let t = w.begin_slot().unwrap(); + assert!(w + .observe( + &packet(1, 1, t.target_ns(), RoomLabel::Empty, 0), + t.target_ns() + ) + .is_err()); + let mut foreign = context(); + foreign.room_digest = "foreign-room".into(); + assert!(TemporalWorkflow::cold_restore( + &store, + &foreign, + w.last_now_ns + 1_000_000, + "three", + w.last_now_ns + 1_000_000 + ) + .is_err()); +} + +#[test] +fn missing_full_period_packets_and_stale_confirmation_cannot_turn_time_into_observation() { + let (_dir, store, mut w) = setup(); + w.persist_initial(&store).unwrap(); + let old = w.synthetic_conditions(); + start(&mut w, &store, 2_000_000_000, "one"); + let mut seq = 0; + let next = commit_next(&mut w, &store, 2_000_000_000, &mut seq); + let accepted = w.accepted(); + let bad_at = next + 10_000_000_000; + assert!(w + .observe(&packet(1, seq + 1, bad_at, RoomLabel::Empty, 0), bad_at) + .is_err()); + assert_eq!(w.accepted(), accepted); + w.pause().unwrap(); + assert!(w + .start_segment(&old, "two", bad_at + 1, bad_at + 1) + .is_err()); +} + +#[test] +fn two_successive_aead_cold_resumes_extend_prefix_without_renewing_descriptor() { + let (_dir, store, mut w) = setup(); + w.persist_initial(&store).unwrap(); + start(&mut w, &store, 2_000_000_000, "one"); + let mut seq = 0; + commit_next(&mut w, &store, 2_000_000_000, &mut seq); + let identity = ( + w.descriptor.id.clone(), + w.descriptor.created_at_ms, + w.descriptor.expires_at_ms, + ); + let original = serde_json::to_vec(&w.session.rows).unwrap(); + for epoch in ["two", "three"] { + let at = w.last_now_ns + 10_000_000_000; + w = TemporalWorkflow::cold_restore(&store, &context(), at, epoch, at).unwrap(); + start(&mut w, &store, at + 1_000_000_000, epoch); + commit_next(&mut w, &store, at + 1_000_000_000, &mut seq); + assert_eq!( + ( + w.descriptor.id.clone(), + w.descriptor.created_at_ms, + w.descriptor.expires_at_ms + ), + identity + ); + } + assert_eq!(w.session.rows.len(), 3); + assert_eq!(w.ledger.gaps.len(), 2); + assert_eq!(serde_json::to_vec(&w.session.rows[..1]).unwrap(), original); +} + +/// EXECUTED LIMITATION, not a security success: AEAD authenticates bytes but +/// published593 RecoveryStore has no independent latest-revision anchor. +#[test] +fn legacy_v1_authenticated_slot_remains_unchanged_and_has_no_new_freshness_claim() { + let (dir, store, mut w) = setup(); + w.persist_initial(&store).unwrap(); + start(&mut w, &store, 2_000_000_000, "one"); + let mut seq = 0; + let next = commit_next(&mut w, &store, 2_000_000_000, &mut seq); + let c = context(); + let file = dir + .path() + .join(&c.owner_scope) + .join("SensingServer/room-learner-recovery/capture-recovery-v1.bin"); + store + .save( + &w.descriptor, + w.durable.as_ref().unwrap(), + w.last_now_ns / 1_000_000, + ) + .unwrap(); + let old = std::fs::read(&file).unwrap(); + commit_next(&mut w, &store, next, &mut seq); + assert_eq!(w.session.rows.len(), 2); + store + .save( + &w.descriptor, + w.durable.as_ref().unwrap(), + w.last_now_ns / 1_000_000, + ) + .unwrap(); + std::fs::write(&file, &old).unwrap(); + let now = w.last_now_ns + 1_000_000; + let legacy = store.load(now / 1_000_000).unwrap().unwrap(); + let mut restored = checkpoint::decode(&legacy.plaintext, &legacy.descriptor, now).unwrap(); + assert_eq!(restored.session.rows.len(),1,"V1 legacy API bytes/meaning remain unchanged; typed temporal path uses independent commitments"); + assert!(restored.session.fit().is_err()); +} + +#[test] +fn independent_resumed_segment_cannot_credit_period_before_all_sources_arrive() { + let (_dir, store, mut w) = setup(); + w.persist_initial(&store).unwrap(); + start(&mut w, &store, 2_000_000_000, "initial"); + let mut seq = 0; + let next = commit_next(&mut w, &store, 2_000_000_000, &mut seq); + let original = w.accepted(); + w.pause().unwrap(); + let resume = next + 10_000_000_000; + start(&mut w, &store, resume, "resumed"); + let target = w.begin_slot().unwrap(); + assert!(target.target_ns() - resume > 1_000_000_000); + let period = 1_000_000_000u64.div_ceil(33); + let mut at = resume; + let mut node3_started = false; + while at <= target.end_ns() + period { + seq += 1; + for source in 1..=3u8 { + if source == 3 && at < target.target_ns() { + continue; + } + if source == 3 { + node3_started = true; + } + let t = at + u64::from(source) * 1000; + let result = w.observe(&packet(source, seq, t, RoomLabel::Empty, 0), t); + if result.is_err() { + assert_eq!(w.accepted(), original); + return; + } + } + at += period; + } + assert!(node3_started); + assert!(!w.session.active); + let result = w.commit_slot(&store, w.last_now_ns); + assert!( + result.is_err(), + "missing source before sparse window was credited as a fully observed reservation" + ); + assert_eq!(w.accepted(), original); +} + +#[test] +fn initial_segment_missing_source_is_rejected_at_native_first_arrival_limit() { + let (_dir, store, mut w) = setup(); + w.persist_initial(&store).unwrap(); + let start_ns = 2_000_000_000; + start(&mut w, &store, start_ns, "initial"); + let limit = w + .session + .managed + .as_ref() + .unwrap() + .binding + .native + .max_first_packet_delay_ms() + * 1_000_000; + assert!(w.observe_tick(start_ns + limit).is_ok()); + assert_eq!(w.live_observed_ns, 0); + assert_eq!(w.accepted(), 0); + assert!(w.observe_tick(start_ns + limit + 1).is_err()); + assert!(w.custody.failed); + assert_eq!(w.live_observed_ns, 0); + assert_eq!(w.accepted(), 0); + assert!(w + .observe( + &packet(3, 1, start_ns + limit + 2, RoomLabel::Empty, 0), + start_ns + limit + 2 + ) + .is_err()); +} + +#[test] +fn busy_sources_detect_dropout_between_sparse_windows_and_preserve_prefix_floors() { + let (_dir, store, mut w) = setup(); + w.persist_initial(&store).unwrap(); + start(&mut w, &store, 2_000_000_000, "initial"); + let mut seq = 0; + let next = commit_next(&mut w, &store, 2_000_000_000, &mut seq); + let accepted = w.accepted(); + let raw = w.live_observed_ns; + let third_floor = w + .source_high + .iter() + .find(|(id, _, _)| *id == 3) + .copied() + .unwrap(); + let saved = serde_json::to_vec(&w.session.rows).unwrap(); + let period = 1_000_000_000u64.div_ceil(33); + let mut rejected = false; + for i in 0..5 { + seq += 1; + for source in 1..=2u8 { + let at = next + i * period + u64::from(source) * 1000; + if w.observe(&packet(source, seq, at, RoomLabel::Empty, 0), at) + .is_err() + { + rejected = true; + break; + } + } + if rejected { + break; + } + } + assert!( + rejected, + "busy nodes must detect absent third node between retained windows" + ); + assert!(w.custody.failed); + assert_eq!(w.accepted(), accepted); + let known_frontier = w.mapping.as_ref().unwrap().observed_start_ns + third_floor.2 + - w.mapping.as_ref().unwrap().receiver_start_ns; + assert!( + w.live_observed_ns >= raw && w.live_observed_ns <= known_frontier, + "busy sources cannot advance beyond the absent source's last actual receiver timestamp" + ); + let failed_live = w.live_observed_ns; + assert_eq!( + w.source_high.iter().find(|(id, _, _)| *id == 3).copied(), + Some(third_floor) + ); + let at = w.last_now_ns + period; + assert!(w + .observe(&packet(3, seq + 1, at, RoomLabel::Empty, 0), at) + .is_err()); + assert_eq!(w.live_observed_ns, failed_live); + w.pause().unwrap(); + assert_eq!(w.accepted(), accepted); + assert_eq!(serde_json::to_vec(&w.session.rows).unwrap(), saved); + assert!(w + .source_high + .iter() + .find(|(id, _, _)| *id == 3) + .is_some_and(|(_, s, t)| *s >= third_floor.1 && *t >= third_floor.2)); +} + +#[test] +fn receiver_tick_withdraws_stale_segment_without_advancing_observed_time() { + let (_dir, store, mut w) = setup(); + w.persist_initial(&store).unwrap(); + start(&mut w, &store, 2_000_000_000, "initial"); + let mut seq = 0; + drive(&mut w, 2_000_000_000, 2_000_000_000, &mut seq); + let raw = w.live_observed_ns; + let gap = w + .session + .managed + .as_ref() + .unwrap() + .binding + .native + .source_policies()[0] + .max_gap_ns; + assert!(w.observe_tick(w.last_now_ns + gap + 1).is_err()); + assert!(w.custody.failed); + assert_eq!(w.live_observed_ns, raw); + assert_eq!(w.accepted(), 0); +} + +#[test] +fn receiver_interval_diagnostics_preserve_exact_frozen_admission_boundaries() { + for boundary in ["below", "minimum", "maximum", "above"] { + let (_dir, store, mut w) = setup(); + w.persist_initial(&store).unwrap(); + let base = 2_000_000_000; + start(&mut w, &store, base, "interval-boundaries"); + for source in 1..=3u8 { + let at = base + u64::from(source) * 1000; + w.observe(&packet(source, 1, at, RoomLabel::Empty, 0), at) + .unwrap(); + } + let policy = &w.session.managed.as_ref().unwrap().binding.native.source_policies()[0]; + let (minimum, maximum) = (policy.min_packet_interval_ns, policy.max_gap_ns); + assert_eq!((minimum, maximum), (27_777_777, 50_000_000)); + let interval = match boundary { + "below" => minimum - 1, + "minimum" => minimum, + "maximum" => maximum, + _ => maximum + 1, + }; + let before = (w.live_observed_ns, w.source_high.clone(), w.accepted()); + let at = base + 1000 + interval; + let result = w.observe(&packet(1, 2, at, RoomLabel::Empty, 0), at); + match boundary { + "minimum" | "maximum" => result.unwrap(), + "below" => { + let error = result.unwrap_err(); + assert_eq!(error, format!( + "temporal_observed_gap_or_rate: source=1 direction=below_min receiver_interval_ns={} min_ns={} max_ns={}", + interval, minimum, maximum + )); + assert!(error.len() <= 256); + assert_eq!((w.live_observed_ns, w.source_high.clone(), w.accepted()), before); + } + _ => { + // Custody still withdraws a too-late source before the interval + // diagnostic. Do not replace the earlier failure or admit it. + assert_eq!(result.unwrap_err(), "temporal_missing_segment_source"); + assert!(w.custody.failed); + assert_eq!((w.live_observed_ns, w.source_high.clone(), w.accepted()), before); + } + } + assert!(w.session.rows.is_empty()); + assert_eq!(w.durable_windows, 0); + } +} + +#[test] +fn synthetic_receiver_burst_reports_interval_without_advancing_observation() { + let (_dir, store, mut w) = setup(); + w.persist_initial(&store).unwrap(); + // A synthetic delay before the actual receiver start earns no observation. + let base = 12_000_000_000; + start(&mut w, &store, base, "synthetic-queued-drain"); + for source in 1..=3u8 { + let at = base + u64::from(source) * 1000; + w.observe(&packet(source, 1, at, RoomLabel::Empty, 0), at) + .unwrap(); + } + let before = (w.live_observed_ns, w.source_high.clone()); + let at = base + 1_001_000; + let error = w.observe(&packet(1, 2, at, RoomLabel::Empty, 0), at).unwrap_err(); + assert_eq!(error, "temporal_observed_gap_or_rate: source=1 direction=below_min receiver_interval_ns=1000000 min_ns=27777777 max_ns=50000000"); + assert_eq!((w.live_observed_ns, w.source_high.clone()), before); + assert_eq!(w.accepted(), 0); + assert!(w.session.rows.is_empty()); +} + +#[test] +fn initial_source_skew_uses_native_limit_even_within_first_arrival_deadline() { + let (_dir, store, mut w) = setup(); + w.persist_initial(&store).unwrap(); + let start_ns = 2_000_000_000; + start(&mut w, &store, start_ns, "initial"); + for row in 0..3u32 { + for source in 1..=2u8 { + let at = start_ns + u64::from(row) * 30_000_000 + u64::from(source) * 1000; + w.observe(&packet(source, row + 1, at, RoomLabel::Empty, 0), at) + .unwrap(); + } + } + let at = start_ns + 75_000_000; + assert!(w + .observe(&packet(3, 3, at, RoomLabel::Empty, 0), at) + .is_err()); + assert!(w.custody.failed); + assert_eq!(w.live_observed_ns, 0); + assert_eq!(w.accepted(), 0); + assert!(!w.source_high.iter().any(|(id, _, _)| *id == 3)); +} + +#[test] +fn independent_zero_prefix_partial_pause_then_saved_segment_cold_restores() { + let (_dir, store, mut w) = setup(); + w.persist_initial(&store).unwrap(); + let id = w.descriptor.id.clone(); + let expiry = w.descriptor.expires_at_ms; + start(&mut w, &store, 2_000_000_000, "zero-initial"); + let target = w.begin_slot().unwrap(); + let mut seq = 0; + let next = drive( + &mut w, + 2_000_000_000, + target.target_ns() + 1_000_000_000, + &mut seq, + ); + assert!(w.session.active); + assert!(w.session.rows.is_empty()); + let floors = w.source_high.clone(); + w.pause().unwrap(); + assert_eq!(w.accepted(), 0); + assert!(w.session.rows.is_empty()); + let resume = next + 10_000_000_000; + start(&mut w, &store, resume, "zero-resumed"); + assert_eq!(w.ledger.gaps.len(), 1); + assert!(w.ledger.gaps[0].after_prefix_sha256.is_none()); + let at = w.last_now_ns + 1_000_000; + let restored = TemporalWorkflow::cold_restore(&store, &context(), at, "zero-cold", at) + .expect("legitimate zero-prefix pause gap must cold-restore without inventing rows"); + assert!(restored.session.rows.is_empty()); + assert_eq!(restored.accepted(), 0); + assert_eq!(restored.descriptor.id, id); + assert_eq!(restored.descriptor.expires_at_ms, expiry); + for floor in floors { + assert!(restored + .source_high + .iter() + .any(|s| s.0 == floor.0 && s.1 >= floor.1 && s.2 >= floor.2)); + } + assert_eq!(restored.stage, Stage::PausedNeedsReconfirmation); +} + +#[test] +fn genesis_history_cold_restores_before_and_after_the_first_committed_window() { + let (_dir, store, mut w) = setup(); + w.persist_initial(&store).unwrap(); + let original = ( + w.descriptor.id.clone(), + w.descriptor.created_at_ms, + w.descriptor.expires_at_ms, + ); + start(&mut w, &store, 2_000_000_000, "initial"); + let mut seq = 0; + let target = w.begin_slot().unwrap(); + let next = drive( + &mut w, + 2_000_000_000, + target.target_ns() + 1_000_000_000, + &mut seq, + ); + let floors = w.source_high.clone(); + w.pause().unwrap(); + start(&mut w, &store, next + 10_000_000_000, "resume-one"); + let now = w.last_now_ns + 1_000_000; + w = TemporalWorkflow::cold_restore(&store, &context(), now, "cold-one", now).unwrap(); + assert_eq!(w.accepted(), 0); + assert_eq!(w.session.rows.len(), 0); + let resume = now + 1_000_000_000; + start(&mut w, &store, resume, "resume-two"); + assert_eq!(w.ledger.gaps.len(), 2); + assert!(w + .ledger + .gaps + .iter() + .all(|g| g.genesis.is_some() && g.after_prefix_sha256.is_none())); + commit_next(&mut w, &store, resume, &mut seq); + let committed = serde_json::to_vec(&w.session.rows).unwrap(); + let now = w.last_now_ns + 1_000_000; + let cold = TemporalWorkflow::cold_restore(&store, &context(), now, "cold-two", now).unwrap(); + assert_eq!(cold.session.rows.len(), 1); + assert_eq!(serde_json::to_vec(&cold.session.rows).unwrap(), committed); + assert_eq!(cold.ledger.gaps.len(), 2); + assert_eq!( + ( + cold.descriptor.id.clone(), + cold.descriptor.created_at_ms, + cold.descriptor.expires_at_ms + ), + original + ); + for (id, seq, t) in floors { + assert!(cold + .source_high + .iter() + .any(|(s, q, n)| *s == id && *q >= seq && *n >= t)); + } +} + +#[test] +fn reducer_acknowledges_only_explicit_same_intent_repair_then_requires_reconfirmation() { + let (_dir, store, mut w) = setup(); + w.persist_initial(&store).unwrap(); + let original_expiry = w.descriptor.expires_at_ms; + start(&mut w, &store, 2_000_000_000, "one"); + let mut seq = 0; + let target = w.begin_slot().unwrap(); + drive(&mut w, 2_000_000_000, target.end_ns(), &mut seq); + assert!(w + .persist_with(&store, true, |s, i, p, n| { + s.publish_temporal(i, Some(p), n)?; + Err("injected-ack-ambiguity".into()) + }) + .is_err()); + assert_eq!(w.accepted(), 0); + assert_eq!(w.stage, Stage::BlockedDurability); + assert!(w.begin_slot().is_err()); + let now = w.last_now_ns + 1_000_000; + w.repair_pending(&store, now).unwrap(); + assert_eq!(w.session.rows.len(), 1); + assert!(w.accepted() > 0); + assert_eq!(w.stage, Stage::PausedNeedsReconfirmation); + assert_eq!(w.descriptor.expires_at_ms, original_expiry); + assert!(w.begin_slot().is_err()); +} + +#[test] +fn crashed_truncated_stage_can_be_explicitly_aborted_then_resume_saved_run() { + let (_dir, store, mut w) = setup(); + w.persist_initial(&store).unwrap(); + let original = (w.descriptor.id.clone(), w.descriptor.expires_at_ms); + start(&mut w, &store, 2_000_000_000, "one"); + let mut seq = 0; + let next = commit_next(&mut w, &store, 2_000_000_000, &mut seq); + let saved = serde_json::to_vec(&w.session.rows).unwrap(); + let floors = w.source_high.clone(); + let target = w.begin_slot().unwrap(); + drive(&mut w, next, target.end_ns(), &mut seq); + assert!(w + .persist_with(&store, true, |s, i, p, n| s + .synthetic_partial_write(i, p, n)) + .is_err()); + let now = w.last_now_ns + 1_000_000; + drop(w); + let loaded = store.load_temporal(now).unwrap().unwrap(); + let token = loaded.token; + let tickets = store.inspect_abortable_temporal(&token, now).unwrap(); + assert!(!tickets.is_empty()); + store.abort_temporal_pending(&tickets[0], now).unwrap(); + let mut restored = + TemporalWorkflow::cold_restore(&store, &context(), now, "after-abort", now).unwrap(); + assert_eq!(restored.stage, Stage::PausedNeedsReconfirmation); + assert_eq!(serde_json::to_vec(&restored.session.rows).unwrap(), saved); + assert_eq!( + ( + restored.descriptor.id.clone(), + restored.descriptor.expires_at_ms + ), + original + ); + assert_eq!(restored.source_high, floors); + let start_ns = now + 1_000_000_000; + start(&mut restored, &store, start_ns, "resume"); + commit_next(&mut restored, &store, start_ns, &mut seq); + assert_eq!(restored.session.rows.len(), 2); + assert_eq!( + serde_json::to_vec(&restored.session.rows[..1]).unwrap(), + saved + ); +} + +#[test] +fn live_abort_retains_known_ram_highwater_while_dropping_only_unsaved_rows() { + let (_dir, store, mut w) = setup(); + w.persist_initial(&store).unwrap(); + start(&mut w, &store, 2_000_000_000, "one"); + let mut seq = 0; + let next = commit_next(&mut w, &store, 2_000_000_000, &mut seq); + let saved = serde_json::to_vec(&w.session.rows).unwrap(); + let target = w.begin_slot().unwrap(); + drive(&mut w, next, target.end_ns(), &mut seq); + let high = w.source_high.clone(); + assert!(w + .persist_with(&store, true, |s, i, p, n| s + .synthetic_partial_write(i, p, n)) + .is_err()); + let now = w.last_now_ns + 1_000_000; + let tickets = store + .inspect_abortable_temporal(w.commit_token.as_ref().unwrap(), now) + .unwrap(); + w.abort_pending_artifact(&store, &tickets[0], now).unwrap(); + assert_eq!(w.source_high, high); + assert_eq!(serde_json::to_vec(&w.session.rows).unwrap(), saved); + assert_eq!(w.stage, Stage::PausedNeedsReconfirmation); + let start_ns = now + 1_000_000_000; + start(&mut w, &store, start_ns, "resumed"); + commit_next(&mut w, &store, start_ns, &mut seq); + assert_eq!(w.session.rows.len(), 2); + assert_eq!(serde_json::to_vec(&w.session.rows[..1]).unwrap(), saved); +} + +#[test] +fn cold_restore_cannot_ack_visible_saved_rows_when_recertification_fails() { + let (_dir, store, mut w) = setup(); + w.persist_initial(&store).unwrap(); + let original = (w.descriptor.id.clone(), w.descriptor.expires_at_ms); + start(&mut w, &store, 2_000_000_000, "one"); + let mut seq = 0; + let target = w.begin_slot().unwrap(); + drive(&mut w, 2_000_000_000, target.end_ns(), &mut seq); + assert!(w + .persist_with(&store, true, |s, i, p, n| s + .synthetic_lost_witness_ack(i, p, n)) + .is_err()); + assert_eq!(w.accepted(), 0); + let expected_rows = serde_json::to_vec(&w.session.rows).unwrap(); + let now = w.last_now_ns + 1_000_000; + drop(w); + assert!(store.load_temporal(now).unwrap().unwrap().prefix.is_some()); + let mut called = false; + assert!(TemporalWorkflow::cold_restore_with( + &store, + &context(), + now, + "cold", + now, + |_, _, _| { + called = true; + Err("injected-restoration-durability-denied".into()) + } + ) + .is_err()); + assert!(called); + let restored = TemporalWorkflow::cold_restore(&store, &context(), now, "cold", now).unwrap(); + assert_eq!(restored.stage, Stage::PausedNeedsReconfirmation); + assert_eq!( + serde_json::to_vec(&restored.session.rows).unwrap(), + expected_rows + ); + assert_eq!( + ( + restored.descriptor.id.clone(), + restored.descriptor.expires_at_ms + ), + original + ); +} + +pub(super) fn bridge_packet(source: u8, sequence: u32, time: u64) -> CanonicalPacket { + // Synthetic bytes go through the actual receiver canonicalizer. The live + // provenance arm is exercised as it would be by the real UDP dispatcher; + // this is not a physical observation or user-owned capture. + use crate::room_learner::native::{canonicalize_esp32, NativePacketInput}; + use wifi_densepose_hardware::{Adr018Flags, PpduType}; + canonicalize_esp32(NativePacketInput { + node_id: source, + n_antennas: 1, + raw_bins: 64, + frequency_hz: 2_437_000_000, + ppdu_type: PpduType::HtLegacy, + sequence, + received_at_ns: time, + receiver_sample_rate_hz: 33., + iq_bytes: &[7; 128], + flags: Adr018Flags::default(), + provenance: CaptureProvenance::LiveReceiverObserved, + }) + .unwrap() +} +pub(super) fn bridge_setup() -> (tempfile::TempDir, RecoveryStore, TemporalTrainCapture) { + let d = tempfile::tempdir().unwrap(); + let c = context(); + let store = RecoveryStore::new( + &d.path().join(&c.owner_scope).join("SensingServer"), + &c.owner_scope, + &c.installation_id, + Some(&[7; 32]), + ) + .unwrap(); + let schedule = + ManagedScheduleContract::build(c, LeadInPolicy::Train5HeldOut15EngineeringV1).unwrap(); + let capture = TemporalTrainCapture::prepare( + schedule, + "synthetic-runtime-bridge".into(), + FitConfig { + epochs: 2, + ..Default::default() + }, + 1_000_000_000, + &store, + ) + .unwrap(); + (d, store, capture) +} +pub(super) fn drive_bridge( + c: &mut TemporalTrainCapture, + store: &RecoveryStore, + from: u64, + to: u64, + sequence: &mut u32, +) -> u64 { + let period = 1_000_000_000u64.div_ceil(33); + let mut at = from; + while at <= to + period && c.stage() == TemporalCaptureStage::Observing { + *sequence += 1; + for source in 1..=3 { + let t = at + u64::from(source) * 1000; + c.ingest(&bridge_packet(source, *sequence, t), t, store) + .unwrap(); + } + at += period; + } + at +} +#[test] +fn private_bridge_priming_preserves_replay_floors_without_observed_credit() { + let (_d, store, mut capture) = bridge_setup(); + capture + .prime_fence(&bridge_packet(1, 100, 2_000_000_000), 2_000_000_000) + .unwrap(); + assert_eq!(capture.snapshot()["durable_windows"], 0); + assert_eq!(capture.snapshot()["durable_observed_ns"], "0"); + capture + .start_segment("receiver-one", 7_000_000_000, &store) + .unwrap(); + let error = capture + .ingest(&bridge_packet(1, 99, 7_000_001_000), 7_000_001_000, &store) + .unwrap_err(); + assert!(error.contains("fence")); + assert_eq!(capture.stage(), TemporalCaptureStage::Failed); + assert!(capture.confirmation_challenge().is_err()); +} +#[test] +fn private_bridge_clock_validation_does_not_retime_exact_lead_in_start() { + let (_d, store, mut capture) = bridge_setup(); + capture.check_time(7_000_001_000).unwrap(); + capture + .start_segment("receiver-one", 7_000_000_000, &store) + .unwrap(); + assert!(capture.check_time(6_999_999_999).is_err()); + assert_eq!(capture.stage(), TemporalCaptureStage::Failed); +} +#[test] +fn private_bridge_detects_missing_source_between_sparse_slots_terminally() { + let (_d, store, mut capture) = bridge_setup(); + capture + .start_segment("receiver-one", 2_000_000_000, &store) + .unwrap(); + let mut sequence = 0; + let next = drive_bridge( + &mut capture, + &store, + 2_000_000_000, + 5_000_000_000, + &mut sequence, + ); + assert_eq!(capture.snapshot()["durable_windows"], 1); + assert!(capture.tick(next + 100_000_000, &store).is_err()); + assert_eq!(capture.stage(), TemporalCaptureStage::Failed); + assert!(capture.confirmation_challenge().is_err()); +} +#[test] +fn private_bridge_observation_reuses_native_validation_outside_retained_slots() { + let (_d, store, mut capture) = bridge_setup(); + capture + .start_segment("receiver-one", 2_000_000_000, &store) + .unwrap(); + let mut sequence = 0; + let next = drive_bridge( + &mut capture, + &store, + 2_000_000_000, + 5_000_000_000, + &mut sequence, + ); + let mut changed = bridge_packet(1, sequence + 1, next + 1000); + changed.native_frame.centre_frequency_hz += 1.; + assert!(capture.ingest(&changed, next + 1000, &store).is_err()); + assert_eq!(capture.stage(), TemporalCaptureStage::Failed); +} +#[test] +fn private_bridge_full_train_schedule_requires_fresh_seal_each_episode() { + let (_d, store, mut capture) = bridge_setup(); + let mut at = 2_000_000_000; + let mut sequence = 0; + let created = capture.snapshot()["descriptor"]["created_at_ms"].clone(); + let expires = capture.snapshot()["descriptor"]["expires_at_ms"].clone(); + let mut previous_challenge: Option = None; + for episode in 0..7 { + assert!(matches!( + capture.stage(), + TemporalCaptureStage::Ready | TemporalCaptureStage::Paused + )); + assert!(capture.confirmation_challenge().is_err()); + capture + .start_segment(&format!("receiver-{episode}"), at, &store) + .unwrap(); + let duration = if episode == 0 { + 600_000_000_000 + } else { + 30_000_000_000 + }; + at = drive_bridge(&mut capture, &store, at, at + duration, &mut sequence); + assert_eq!(capture.stage(), TemporalCaptureStage::NeedsConfirmation); + if episode == 0 { + let before = capture.snapshot(); + drop(capture); + capture = + TemporalTrainCapture::cold_restore(&store, &context(), at, "fresh-runtime", at) + .unwrap(); + assert_eq!(capture.stage(), TemporalCaptureStage::Paused); + assert_eq!(capture.snapshot()["rows_sha256"], before["rows_sha256"]); + assert_eq!(capture.snapshot()["sealed_train_episodes"], 0); + assert!(capture.start_segment("not-a-new-slot", at, &store).is_err()); + assert!(capture.confirmation_challenge().is_ok(), "full saved but unconfirmed phase needs a fresh owner confirmation, not recollection"); + } + if let Some(old) = previous_challenge.as_ref() { + assert!(capture.confirm(old, &store, at).is_err()); + } + let challenge = capture.confirmation_challenge().unwrap(); + capture.confirm(&challenge, &store, at).unwrap(); + previous_challenge = Some(challenge); + assert_eq!(capture.snapshot()["sealed_train_episodes"], episode + 1); + assert_eq!(capture.snapshot()["descriptor"]["created_at_ms"], created); + assert_eq!(capture.snapshot()["descriptor"]["expires_at_ms"], expires); + at += 5_000_000_000; + } + assert_eq!(capture.stage(), TemporalCaptureStage::Complete); + assert_eq!(capture.snapshot()["durable_windows"], 64); + assert_eq!(capture.snapshot()["fit_admission_available"], false); + assert!(capture.start_segment("no-held-out", at, &store).is_err()); +} + +#[test] +fn private_bridge_retains_initial_intent_until_exact_repair_after_real_lost_ack() { + let dir = tempfile::tempdir().unwrap(); + let c = context(); + let store = RecoveryStore::new( + &dir.path().join(&c.owner_scope).join("SensingServer"), + &c.owner_scope, + &c.installation_id, + Some(&[7; 32]), + ) + .unwrap(); + let schedule = + ManagedScheduleContract::build(c, LeadInPolicy::Train5HeldOut15EngineeringV1).unwrap(); + let (mut capture, error) = TemporalTrainCapture::synthetic_prepare_lost_initial_ack( + schedule, + "synthetic-retained-initial".into(), + FitConfig { + epochs: 2, + ..Default::default() + }, + 1_000_000_000, + &store, + ) + .unwrap(); + assert!(error.is_some()); + assert_eq!(capture.stage(), TemporalCaptureStage::Blocked); + assert!(capture.confirmation_challenge().is_err()); + assert!(capture + .start_segment("never", 2_000_000_000, &store) + .is_err()); + capture.repair_pending(&store, 2_000_000_000).unwrap(); + assert_eq!(capture.stage(), TemporalCaptureStage::Paused); + assert_eq!(capture.snapshot()["durable_windows"], 0); + assert_eq!(capture.snapshot()["sealed_train_episodes"], 0); + capture + .start_segment("explicit-after-repair", 3_000_000_000, &store) + .unwrap(); + assert_eq!(capture.stage(), TemporalCaptureStage::Observing); +} + +#[test] +fn private_bridge_preserves_native_first_sequence_zero_semantics() { + let (_dir, store, mut capture) = bridge_setup(); + let start = 2_000_000_000; + capture + .start_segment("zero-sequence", start, &store) + .unwrap(); + for source in 1..=3 { + let at = start + u64::from(source) * 1000; + capture + .ingest(&bridge_packet(source, 0, at), at, &store) + .unwrap(); + } + let mut sequence = 0; + drive_bridge( + &mut capture, + &store, + start + 1_000_000_000u64.div_ceil(33), + start + 3_000_000_000, + &mut sequence, + ); + assert_eq!(capture.snapshot()["durable_windows"], 1); + assert_eq!(capture.stage(), TemporalCaptureStage::Observing); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_fit_admission.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_fit_admission.rs new file mode 100644 index 0000000000..a496bcb5fc --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_fit_admission.rs @@ -0,0 +1,317 @@ +//! Private producer-owned permission to COMPUTE one diagnostic candidate. +//! Existing durable Train seals are not physical/scientific qualification. +use super::*; +use std::sync::Arc; + +/// Fixed binding to the capture's accepted prefix, not a wire proof or a +/// caller-supplied permission. Only this producer module can construct it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct TemporalFitCaptureIdentity { + owner: String, + installation: String, + context: String, + run: String, + schedule: String, + policy: String, + configuration: String, + descriptor: RecoveryDescriptor, + token: TemporalToken, + prefix: String, + prefix_schema: &'static str, + metadata_kind: &'static str, + continuation_policy: Option, + operator_profile: Option, +} +impl TemporalFitCaptureIdentity { + pub(crate) fn owner(&self) -> &str { + &self.owner + } + pub(crate) fn installation(&self) -> &str { + &self.installation + } + pub(crate) fn context(&self) -> &str { + &self.context + } + pub(crate) fn run_id(&self)->&str{&self.run} + pub(crate) fn expires_at_ms(&self) -> u64 { + self.descriptor.expires_at_ms + } + pub(crate) fn check_deadline(&self, now: u64) -> Result<(), String> { + if now == 0 || now / 1_000_000 >= self.descriptor.expires_at_ms { + Err("temporal_fit_original_deadline_expired".into()) + } else { + Ok(()) + } + } + /// Off-lock only. Same-store observation; never publishes or renews TTL. + pub(crate) fn recertify_current(&self, store: &RecoveryStore, now: u64) -> Result<(), String> { + self.check_deadline(now)?; + let current = store.recertify_temporal(&self.token, now)?; + if current.token != self.token + || current.prefix.descriptor() != &self.descriptor + || bytes_hash(current.prefix.bytes()) != self.prefix + { + return Err("temporal_fit_accepted_prefix_changed".into()); + } + Ok(()) + } +} + +/// A bounded ref snapshot of ACTUAL accepted Runtime data. No Clone or serde: +/// imported models, JSON claims and generic sessions cannot create this value. +pub(crate) struct PendingTemporalFitInput { + identity: TemporalFitCaptureIdentity, + accepted_bytes: Arc>, + records: checkpoint::SharedRecords, + not_before_ns: u64, +} +/// Constructed only after exact same-store recertification and strict decoding, +/// off the receiver lock. It owns a fit session, never the live capture owner. +pub(crate) struct ValidatedTemporalFitInput { + identity: TemporalFitCaptureIdentity, + session: TemporalTrainingSession, + train_seals_sha256:String, +} +/// Off-lock strict producer validation, later bound to an ACTUAL saved fit. +/// This is not serializable/importable, and has no public constructor. +pub(crate) struct VerifiedTemporalFitSource { + identity:TemporalFitCaptureIdentity, + train_data:String,evidence:String,identities:String,seals:String, + actual_candidate:Option, + actual_summary:full_run::FitSummaryReference, +} +impl VerifiedTemporalFitSource { + pub(crate) fn is_full_run(&self)->bool{self.identity.continuation_policy.is_some()} + pub(crate) fn binding(&self)->&TemporalFitCaptureIdentity{&self.identity} + pub(crate) fn bind_saved_fit(&mut self,model:&TemporalModel,saved:&crate::room_learner::artifact::TemporalFitSaveResult)->Result<(),String>{ + let b=model.fit_diagnostic_binding()?; + if b.artifact_kind!=full_run::MODEL_KIND||b.context_digest!=self.identity.context + ||b.training_data_sha256!=self.train_data||b.input_evidence_sha256!=self.evidence + ||b.identity_evidence_sha256.as_ref()!=Some(&self.identities)||b.schedule_sha256.as_ref()!=Some(&self.identity.schedule) + ||b.input_policy_sha256!=self.identity.policy + ||b.bound_fit_config.as_ref().map(hash).transpose()?.as_ref()!=Some(&self.identity.configuration) + ||saved.candidate.artifact_sha256!=b.artifact_sha256||saved.candidate.artifact_kind!=b.artifact_kind{ + return Err("temporal_actual_fit_source_binding_changed".into()); + } + self.actual_candidate=Some(saved.candidate.clone()); + self.actual_summary=match &saved.fit_diagnostics.summary{ + Some(summary)=>full_run::FitSummaryReference::Recorded{sha256:hash(summary)?}, + None=>match saved.fit_diagnostics.state{ + crate::room_learner::artifact::FitDiagnosticState::NotRecorded=>full_run::FitSummaryReference::NotRecorded, + _=>full_run::FitSummaryReference::Unavailable, + } + }; + Ok(()) + } + pub(super) fn record_data(self,now:u64,floors:Vec<(u8,u64,u64)>)->Result{ + self.identity.check_deadline(now)?; + let candidate=self.actual_candidate.ok_or("temporal_actual_saved_fit_required")?; + let continuation=self.identity.continuation_policy.ok_or("temporal_full_run_no_upconversion")?; + let body=full_run::VerifiedFitRecordBody{ + owner_scope:self.identity.owner,installation_id:self.identity.installation,context_digest:self.identity.context,run:self.identity.run, + prefix_schema:self.identity.prefix_schema.into(),metadata_kind:self.identity.metadata_kind.into(),continuation_policy_sha256:continuation, + artifact_kind:candidate.artifact_kind.clone(),artifact_sha256:candidate.artifact_sha256.clone(), + input_policy_sha256:self.identity.policy,execution_policy_sha256:hash(&EXECUTION_POLICY)?,schedule_sha256:self.identity.schedule, + configured_fit_sha256:self.identity.configuration,training_data_sha256:self.train_data,input_evidence_sha256:self.evidence, + identity_evidence_sha256:self.identities,train_seal_chain_sha256:self.seals,accepted_train_prefix_sha256:self.identity.prefix, + accepted_train_commitment:self.identity.token.data(),saved_candidate:candidate,actual_summary:self.actual_summary, + freeze_prepare_receiver_ns:now,source_floors:floors,operator_profile:self.identity.operator_profile, + }; + let sha256=hash(&(full_run::FIT_RECORD_DOMAIN,&body))?; + Ok(full_run::VerifiedFitRecordData{schema:full_run::FIT_RECORD_DOMAIN.into(),body,sha256}) + } +} + +fn complete_train( + w: &TemporalWorkflow, + schedule: &ManagedScheduleContract, + now: u64, +) -> Result<(), String> { + let bridge = w + .bridge + .as_ref() + .ok_or("temporal_fit_actual_bridge_required")?; + let managed = w + .session + .managed + .as_ref() + .ok_or("managed_session_required")?; + let counts = schedule.counts(); + let prefit_episodes=if schedule.is_operator_eight_step(){schedule.episodes().len()}else{counts.train_episodes}; + let prefit_windows=if schedule.is_operator_eight_step(){counts.total_windows}else{counts.train_windows}; + if now < w.last_now_ns || now / 1_000_000 >= w.descriptor.expires_at_ms { + return Err("temporal_fit_original_deadline_or_clock".into()); + } + if bridge.disposition != bridge::Disposition::Complete + || bridge.seals.len() != prefit_episodes + || !bridge.parts.is_empty() + || w.episode != prefit_episodes + || w.mapping.is_some() + || w.pending_window.is_some() + || w.session.active + || w.session.failed + || w.session.fitted + || w.session.rows.len() != prefit_windows + || w.session.evidence.len() != prefit_windows + || managed.windows.len() != prefit_windows + || w.ledger.spans.len() != prefit_windows + || w.durable_windows != prefit_windows + || w.session.rows.iter().any(|row| row.role != DataRole::Train && !(schedule.is_operator_eight_step() && row.role == DataRole::Validation)) + || w.pending_intent.is_some() + || w.pending_prefix.is_some() + || w.stage == Stage::BlockedDurability + { + return Err("temporal_fit_confirmed_durable_train_required".into()); + } + Ok(()) +} +fn identity( + w: &TemporalWorkflow, + schedule: &ManagedScheduleContract, + token: TemporalToken, + prefix: String, +) -> Result { + Ok(TemporalFitCaptureIdentity { + owner: w.descriptor.owner_scope.clone(), + installation: w.descriptor.installation_id.clone(), + context: w.context().digest(), + run: w.session.run_id.clone(), + schedule: schedule.sha256().into(), + policy: w.session.plan_sha256().into(), + configuration: hash(&w.session.config)?, + descriptor: w.descriptor.clone(), + token, + prefix, + prefix_schema:if w.full_run.is_some(){full_run::PREFIX_SCHEMA}else{"ruview.room-learner.temporal-runtime-prefix.v2"}, + metadata_kind:if w.full_run.is_some(){full_run::META_KIND}else{"temporal-prefix-commitment-v1"}, + continuation_policy:w.full_run.as_ref().map(|m|m.policy.sha256().into()), + operator_profile:schedule.is_operator_eight_step().then(||"native-2s-three-source-56bin-operator-eight-step-v1".into()), + }) +} +pub(super) fn current_identity( + w: &TemporalWorkflow, + schedule: &ManagedScheduleContract, + now: u64, +) -> Result { + complete_train(w, schedule, now)?; + if w.full_run.as_ref().is_some_and(|m|m.owner.is_none()){return Err("temporal_full_run_owner_custody_required".into());} + // The async writer supplies a cached verified digest. Never hash a growing + // raw prefix under the receiver lock or silently fall back to a sync path. + let state = w + .async_state + .as_ref() + .ok_or("temporal_fit_async_durability_required")?; + if state.pending.is_some() + || state.busy_pause + || state.error.is_some() + || state.accepted_seals != if schedule.is_operator_eight_step(){schedule.episodes().len()}else{schedule.counts().train_episodes} + || w.durable.is_none() + { + return Err("temporal_fit_durability_unconfirmed".into()); + } + identity( + w, + schedule, + w.commit_token + .clone() + .ok_or("temporal_fit_commitment_required")?, + w.durable_digest().ok_or("temporal_fit_digest_required")?, + ) +} +impl PendingTemporalFitInput { + pub(super) fn capture( + w: &TemporalWorkflow, + schedule: &ManagedScheduleContract, + now: u64, + ) -> Result { + let identity = current_identity(w, schedule, now)?; + Ok(Self { + identity, + accepted_bytes: w + .durable + .as_ref() + .ok_or("temporal_fit_prefix_required")? + .clone(), + records: checkpoint::workflow_records(w), + not_before_ns: now, + }) + } + pub(crate) fn identity(&self) -> &TemporalFitCaptureIdentity { + &self.identity + } + pub(crate) fn validate( + self, + store: &RecoveryStore, + now: u64, + ) -> Result { + self.identity.check_deadline(now)?; + if now < self.not_before_ns { + return Err("temporal_fit_receiver_clock_reversed".into()); + } + let recertified = store.recertify_temporal(&self.identity.token, now)?; + if recertified.token != self.identity.token + || recertified.prefix.descriptor() != &self.identity.descriptor + || bytes_hash(recertified.prefix.bytes()) != self.identity.prefix + || recertified.prefix.bytes() != self.accepted_bytes.as_slice() + { + return Err("temporal_fit_accepted_prefix_changed".into()); + } + // The existing decoder validates all raw rows, temporal identities, + // source floors, native quality periods and the immutable Train seals. + let restored = checkpoint::decode_shared( + recertified.prefix.bytes(), + recertified.prefix.descriptor(), + now, + &self.records, + )?; + let schedule = restored.schedule()?; + complete_train(&restored, &schedule, now)?; + if identity( + &restored, + &schedule, + recertified.token, + self.identity.prefix.clone(), + )? != self.identity + { + return Err("temporal_fit_input_binding_changed".into()); + } + Ok(ValidatedTemporalFitInput { + identity: self.identity, + train_seals_sha256:hash(&restored.bridge.as_ref().ok_or("temporal_fit_actual_bridge_required")?.seals.iter().filter(|seal|schedule.episodes().get(seal.episode).is_some_and(|e|e.role==DataRole::Train)).collect::>())?, + session: restored.session, + }) + } +} +impl ValidatedTemporalFitInput { + /// SAME existing permit/hash formulas and SAME fit session implementation. + /// Consumption prevents a cloned generic permit from being reused on a + /// second fit job. A retry requires fresh owner-command admission. + pub(crate) fn into_session_and_admission( + self, + ) -> Result<(TemporalTrainingSession, ManagedFitAdmission), String> { + let(session,permit,_)=self.into_session_admission_and_source()?;Ok((session,permit)) + } + pub(crate) fn into_session_admission_and_source(mut self)->Result<(TemporalTrainingSession,ManagedFitAdmission,VerifiedTemporalFitSource),String>{ + self.session.retain_managed_train_only()?; + let managed = self + .session + .managed + .as_ref() + .ok_or("managed_session_required")?; + let dataset = self.session.dataset()?; + let permit = ManagedFitAdmission { + policy: hash(&managed.binding)?, + data: hash(&dataset.train())?, + evidence: hash(&self.session.evidence)?, + identities: hash(&managed.windows)?, + }; + // Strict seal validation happened in the recertified producer decoder; + // recover its exact immutable chain from the accepted prefix again on + // the worker, never hash growing source data under Runtime.inner. + let source=VerifiedTemporalFitSource{identity:self.identity,train_data:permit.data.clone(),evidence:permit.evidence.clone(), + identities:permit.identities.clone(),seals:self.train_seals_sha256,actual_candidate:None,actual_summary:full_run::FitSummaryReference::Unavailable}; + // Full validated data is intentionally dropped here, on the worker. + drop(dataset); + Ok((self.session, permit,source)) + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_fit_summary.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_fit_summary.rs new file mode 100644 index 0000000000..ccd208e566 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_fit_summary.rs @@ -0,0 +1,331 @@ +//! Optional ACTUAL aggregate fit diagnostics in the SAME owner ArtifactStore. +//! Reuses dca8's report projection and finite/config/count/delta validation. +//! Never reconstructs historical outcomes or establishes qualification authority. +use super::temporal_artifacts::{bounded_read, immutable}; +use super::*; +use crate::room_learner::{ + observation::CaptureContext, + temporal_model_input::{TemporalFitBinding, TemporalModel}, + training::{FitConfig, ObjectiveValues, TrainingReport}, +}; +const SCHEMA: &str = "ruview.room-learner.temporal-fit-summary.v1"; +const MAX_BYTES: usize = 16_384; +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct FitObjective { + pub total: f32, + pub contrastive: f32, + pub supervised: f32, +} +impl From for FitObjective { + fn from(v: ObjectiveValues) -> Self { + Self { + total: v.total, + contrastive: v.contrastive, + supervised: v.supervised, + } + } +} +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct FitMetrics { + pub initial: FitObjective, + pub final_objective: FitObjective, + pub completed_epochs: usize, + pub parameter_count: usize, + pub parameter_delta_l2: f32, + pub encoder_delta_l2: f32, + pub backbone_delta_l2: f32, + pub projection_delta_l2: f32, + pub head_delta_l2: f32, + pub objective_evaluations: usize, + pub training_cohorts: usize, + pub contrastive_trials: usize, + pub configured: FitConfig, + /// Actual TrainingReport has no elapsed measurement. Never copy its limit here. + pub elapsed_ms: Option, +} +impl FitMetrics { + fn from_report(r: &TrainingReport, c: &FitConfig) -> Self { + Self { + initial: r.initial.into(), + final_objective: r.final_objective.into(), + completed_epochs: r.completed_epochs, + parameter_count: r.parameter_count, + parameter_delta_l2: r.parameter_delta_l2, + encoder_delta_l2: r.encoder_delta_l2, + backbone_delta_l2: r.backbone_delta_l2, + projection_delta_l2: r.projection_delta_l2, + head_delta_l2: r.head_delta_l2, + objective_evaluations: r.objective_evaluations, + training_cohorts: r.training_cohorts, + contrastive_trials: r.contrastive_trials, + configured: c.clone(), + elapsed_ms: None, + } + } + fn validate(&self, b: &TemporalFitBinding, context: &CaptureContext) -> Result<(), String> { + let c = &self.configured; + c.validate_context(context)?; + let finite = |o: &FitObjective| { + [o.total, o.supervised, o.contrastive] + .iter() + .all(|v| v.is_finite() && *v >= 0.) + && (f64::from(o.total) + - f64::from(c.supervised_weight) * f64::from(o.supervised) + - f64::from(c.contrastive_weight) * f64::from(o.contrastive)) + .abs() + <= 1e-5 * (1. + f64::from(o.total).abs()) + }; + if self.training_cohorts != b.training_cohorts + || !(4..=64).contains(&self.training_cohorts) + || !(2..=self.training_cohorts).contains(&self.contrastive_trials) + || self.parameter_count != b.parameter_count + || self.parameter_count > c.max_parameters + || self.completed_epochs == 0 + || self.completed_epochs > c.epochs + || self.objective_evaluations < 2 + || self.objective_evaluations >= c.max_objective_evaluations + || c.hidden_dim != b.encoder.hidden_dim + || c.embedding_dim != b.encoder.embedding_dim + || c.temporal_tokens != b.encoder.temporal_tokens + || c.temperature != b.encoder.temperature + || self.elapsed_ms.is_some() + || !finite(&self.initial) + || !finite(&self.final_objective) + || self.final_objective.total >= self.initial.total + || ![ + self.parameter_delta_l2, + self.encoder_delta_l2, + self.backbone_delta_l2, + self.projection_delta_l2, + self.head_delta_l2, + ] + .iter() + .all(|v| v.is_finite() && *v >= 0.) + || self.parameter_delta_l2 <= 0. + || self.encoder_delta_l2 <= 0. + || self.backbone_delta_l2 <= 0. + || self.projection_delta_l2 <= 0. + { + return Err("temporal_fit_report_invalid".into()); + } + if let Some(bound) = &b.bound_fit_config { + if serde_json::to_vec(bound).map_err(|_| "fit_config_encoding")? + != serde_json::to_vec(c).map_err(|_| "fit_config_encoding")? + { + return Err("temporal_fit_configuration_changed".into()); + } + } + let norm = |whole: f32, a: f32, b: f32| { + let expected = f64::from(a).hypot(f64::from(b)); + (f64::from(whole) - expected).abs() <= 1e-4 * (1. + expected) + }; + if !norm( + self.parameter_delta_l2, + self.encoder_delta_l2, + self.head_delta_l2, + ) || !norm( + self.encoder_delta_l2, + self.backbone_delta_l2, + self.projection_delta_l2, + ) || f64::from(self.parameter_delta_l2) > 20_000. * (self.parameter_count as f64).sqrt() + { + return Err("temporal_fit_deltas_inconsistent".into()); + } + Ok(()) + } +} +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct TemporalFitSummary { + pub schema: String, + pub binding: TemporalFitBinding, + pub metrics: FitMetrics, + pub diagnostic_only: bool, + pub physically_validated: bool, + pub qualification_granted: bool, +} +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Envelope { + summary: TemporalFitSummary, + summary_sha256: String, +} +fn small(value: &T) -> Result, String> { + let b = serde_json::to_vec(value).map_err(|_| "temporal_fit_encoding")?; + if b.len() > MAX_BYTES { + return Err("temporal_fit_summary_budget".into()); + } + Ok(b) +} +impl TemporalFitSummary { + fn actual( + model: &TemporalModel, + report: &TrainingReport, + configured: &FitConfig, + ) -> Result { + let v = Self { + schema: SCHEMA.into(), + binding: model.fit_diagnostic_binding()?, + metrics: FitMetrics::from_report(report, configured), + diagnostic_only: true, + physically_validated: false, + qualification_granted: false, + }; + v.validate(model)?; + Ok(v) + } + fn validate(&self, model: &TemporalModel) -> Result<(), String> { + let expected = model.fit_diagnostic_binding()?; + if self.schema != SCHEMA + || !self.diagnostic_only + || self.physically_validated + || self.qualification_granted + || small(&self.binding)? != small(&expected)? + { + return Err("temporal_fit_summary_binding_changed".into()); + } + self.metrics.validate(&expected, model.context()) + } +} +#[derive(Debug, Clone, Copy, Serialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub(crate) enum FitDiagnosticState { + Recorded, + NotRecorded, + Unavailable, +} +#[derive(Debug, Clone, Serialize)] +pub(crate) struct TemporalFitDiagnostics { + pub state: FitDiagnosticState, + pub summary: Option, + pub warning: Option<&'static str>, + pub diagnostic_only: bool, + pub physically_validated: bool, + pub qualification_granted: bool, +} +impl TemporalFitDiagnostics { + fn state( + state: FitDiagnosticState, + summary: Option, + warning: Option<&'static str>, + ) -> Self { + Self { + state, + summary, + warning, + diagnostic_only: true, + physically_validated: false, + qualification_granted: false, + } + } +} +#[derive(Debug, Serialize)] +pub(crate) struct TemporalFitSaveResult { + pub candidate: TemporalCandidateReceipt, + pub fit_diagnostics: TemporalFitDiagnostics, +} +impl ArtifactStore { + fn temporal_fit_summary_path(&self, model: &TemporalModel) -> Result { + let binding = model.fit_diagnostic_binding()?; + self.validate_context(model.context())?; + // Model must already exist as the WHOLE verified codec. Never call the old + // inner-v1 loader or create a loose summary from a supplied hash. + let loaded = + self.load_exact_temporal_candidate(model.context(), &binding.artifact_sha256)?; + if small(&loaded.fit_diagnostic_binding()?)? != small(&binding)? { + return Err("temporal_fit_artifact_changed".into()); + } + let directory = self + .temporal_directory(model.context(), false)? + .ok_or("temporal_fit_model_directory_missing")?; + Ok(directory.join(format!("{}.fit-summary-v1.json", binding.artifact_sha256))) + } + fn save_actual_temporal_fit_summary( + &self, + model: &TemporalModel, + report: &TrainingReport, + configured: &FitConfig, + ) -> Result { + let summary = TemporalFitSummary::actual(model, report, configured)?; + let envelope = Envelope { + summary_sha256: digest(&small(&summary)?), + summary, + }; + let bytes = small(&envelope)?; + let path = self.temporal_fit_summary_path(model)?; + immutable(&path, &bytes)?; + let loaded = self + .load_temporal_fit_summary(model)? + .ok_or("temporal_fit_readback_missing")?; + if small(&loaded)? != small(&envelope.summary)? { + return Err("temporal_fit_readback_changed".into()); + } + Ok(loaded) + } + pub(crate) fn load_temporal_fit_summary( + &self, + model: &TemporalModel, + ) -> Result, String> { + let path = self.temporal_fit_summary_path(model)?; + match std::fs::symlink_metadata(&path) { + Err(e) if e.kind() == io::ErrorKind::NotFound => return Ok(None), + Err(_) => return Err("temporal_fit_summary_unavailable".into()), + Ok(_) => {} + } + let bytes = bounded_read(&path, MAX_BYTES)?; + let envelope: Envelope = + serde_json::from_slice(&bytes).map_err(|_| "temporal_fit_summary_malformed")?; + if digest(&small(&envelope.summary)?) != envelope.summary_sha256 { + return Err("temporal_fit_summary_integrity".into()); + } + envelope.summary.validate(model)?; + Ok(Some(envelope.summary)) + } + /// Coherent real-caller seam: a valid whole model is durable FIRST. A metrics + /// failure is explicit optional diagnostics failure, never failed model save. + pub(crate) fn save_temporal_candidate_with_fit_summary( + &self, + model: &TemporalModel, + report: &TrainingReport, + configured: &FitConfig, + ) -> Result { + self.save_temporal_candidate_with_fit_summary_using(model,report,configured, |_|{}) + } + #[cfg(test)] + pub(crate) fn save_temporal_candidate_with_fit_summary_probe( + &self,model:&TemporalModel,report:&TrainingReport,configured:&FitConfig, + model_saved:impl FnOnce(&TemporalCandidateReceipt), + )->Result{ + self.save_temporal_candidate_with_fit_summary_using(model,report,configured,model_saved) + } + /// One shared real store body. The production wrapper supplies a no-op; + /// the test seam can hold the genuine model-save/report-save boundary. + fn save_temporal_candidate_with_fit_summary_using( + &self,model:&TemporalModel,report:&TrainingReport,configured:&FitConfig, + model_saved:impl FnOnce(&TemporalCandidateReceipt), + )->Result{ + let candidate = self.save_temporal_candidate(model)?; + model_saved(&candidate); + let fit_diagnostics=match self.save_actual_temporal_fit_summary(model,report,configured){Ok(summary)=>TemporalFitDiagnostics::state(FitDiagnosticState::Recorded,Some(summary),None),Err(_)=>TemporalFitDiagnostics::state(FitDiagnosticState::NotRecorded,None,Some("The model was saved, but actual aggregate fit diagnostics were not recorded; existing files were retained."))}; + Ok(TemporalFitSaveResult { + candidate, + fit_diagnostics, + }) + } + /// Future OWNER-side status projection only. Missing history is unknown, never + /// zeros/defaults or an inferred TrainingReport; paired progress stays unchanged. + pub(crate) fn temporal_fit_diagnostics(&self, model: &TemporalModel) -> TemporalFitDiagnostics { + match self.load_temporal_fit_summary(model){Ok(Some(summary))=>TemporalFitDiagnostics::state(FitDiagnosticState::Recorded,Some(summary),None),Ok(None)=>TemporalFitDiagnostics::state(FitDiagnosticState::NotRecorded,None,None),Err(_)=>TemporalFitDiagnostics::state(FitDiagnosticState::Unavailable,None,Some("Saved fit diagnostics could not be verified; the model and existing files were retained."))} + } +} + +/// Typed rehash for tampering tests: exercise semantic validation AFTER integrity, +/// never merely hash a serde_json::Value with different key/float encoding. +#[cfg(test)] +pub(crate) fn rehash_fixture(bytes: &[u8]) -> Result, String> { + let mut envelope: Envelope = serde_json::from_slice(bytes).map_err(|_| "fixture_schema")?; + envelope.summary_sha256 = digest(&small(&envelope.summary)?); + small(&envelope) +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_fit_summary_managed_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_fit_summary_managed_tests.rs new file mode 100644 index 0000000000..c75dc6fc7a --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_fit_summary_managed_tests.rs @@ -0,0 +1,472 @@ +//! SYNTHETIC actual managed fitter -> combined SAME-store helper -> cold model +//! and actual aggregate report. No production issuer/runtime/paired API. +use super::*; +use crate::room_learner::{ + artifact::ArtifactStore, + managed_schedule_contract::{LeadInPolicy, ManagedScheduleContract}, + native::{canonicalize_esp32, NativePacketInput}, +}; +use serde_json::{json, Value}; +use std::{ + path::{Path, PathBuf}, + sync::OnceLock, +}; +use wifi_densepose_hardware::{Adr018Flags, PpduType}; +fn context() -> CaptureContext { + CaptureContext { + owner_scope: "a".repeat(64), + installation_id: "synthetic-fit-summary-v3".into(), + room_digest: "synthetic-room".into(), + revision: 1, + sources: (1..=3) + .map(|source_id| SourceSpec { + source_id, + device_id: format!("esp32-node-{source_id}"), + grid_id: "ant1-bins64-ppdu0-usable-drop2-linear-v2".into(), + feature_count: 56, + signal_kind: ruview_unified::RfModality::WifiCsi, + center_frequency_hz: 2_437_000_000, + bandwidth_hz: 20_000_000, + link_id: format!("native-{source_id}"), + link_index: 0, + geometry: None, + }) + .collect(), + } +} +fn actual_fit() -> (TemporalModel, TrainingReport, FitConfig) { + let schedule = + ManagedScheduleContract::build(context(), LeadInPolicy::Train5HeldOut15EngineeringV1) + .unwrap(); + let configured = FitConfig { + epochs: 2, + ..Default::default() + }; + let mut session = TemporalTrainingSession::managed( + schedule.clone(), + "actual-fit-summary".into(), + configured.clone(), + ) + .unwrap(); + let mut start = 10_000_000_000; + let mut index = 0u32; + for episode in schedule + .episodes() + .iter() + .filter(|e| e.role == DataRole::Train) + { + let bound = schedule + .bind_episode_start("actual-fit-summary", episode.index, 0, 0, None) + .unwrap(); + for slot in 0..episode.slots.len() { + let target = ManagedWindowTarget::from_runtime_episode( + &schedule, + &bound, + "actual-fit-summary", + slot, + start, + ) + .unwrap(); + session.begin_managed_window(&target).unwrap(); + 'window: for row in 0..100 { + for source in 1..=3 { + let gain = match episode.condition { + RoomLabel::Empty => 3, + RoomLabel::AdjacentOnly => 4, + RoomLabel::OccupiedStatic => 8, + RoomLabel::OccupiedMoving => 10 + (row % 2) as u8, + _ => unreachable!(), + }; + let iq = (0..64) + .flat_map(|b| [gain + source + (b % 2) as u8, 4]) + .collect::>(); + let at = target.target_ns() + + row as u64 * 1_000_000_000u64.div_ceil(33) + + u64::from(source) * 1000; + let packet = canonicalize_esp32(NativePacketInput { + node_id: source, + n_antennas: 1, + raw_bins: 64, + frequency_hz: 2_437_000_000, + ppdu_type: PpduType::HtLegacy, + sequence: index * 1000 + row as u32 + 1, + received_at_ns: at, + receiver_sample_rate_hz: 33., + iq_bytes: &iq, + flags: Adr018Flags::default(), + provenance: CaptureProvenance::Synthetic, + }) + .unwrap(); + if session.ingest(&packet).unwrap() == WindowAdmission::Ready { + session.finish_window(at).unwrap(); + break 'window; + } + } + } + index += 1; + } + start += episode.target_active_recording_ms * 1_000_000 + 10_000_000_000; + } + let permit = session.synthetic_test_fit_admission().unwrap(); + let (model, report) = session + .fit_managed_with_cancel(permit, &AtomicBool::new(false)) + .unwrap(); + assert_eq!(report.training_cohorts, 64); + assert!(report.backbone_delta_l2 > 0.); + (model, report, configured) +} +fn fixture() -> (TemporalModel, TrainingReport, FitConfig) { + static FIT: OnceLock<(Vec, TrainingReport, FitConfig)> = OnceLock::new(); + let (bytes, report, c) = FIT.get_or_init(|| { + let (m, r, c) = actual_fit(); + (m.export().unwrap(), r, c) + }); + ( + TemporalModel::import(bytes).unwrap(), + report.clone(), + c.clone(), + ) +} +fn store(root: &Path) -> ArtifactStore { + ArtifactStore::new( + &root.join("a".repeat(64)).join("SensingServer"), + &"a".repeat(64), + "synthetic-fit-summary-v3", + ) + .unwrap() +} +fn sidecar(root: &Path, model: &TemporalModel) -> PathBuf { + let b = model.fit_diagnostic_binding().unwrap(); + root.join("a".repeat(64)) + .join("SensingServer/room-learner/temporal-models") + .join(b.context_digest) + .join(format!("{}.fit-summary-v1.json", b.artifact_sha256)) +} +fn json(v: &impl Serialize) -> Value { + serde_json::to_value(v).unwrap() +} +#[test] +fn actual_managed_fit_combined_model_first_save_and_cold_summary_preserve_the_returned_report() { + let root = tempfile::tempdir().unwrap(); + let storage = store(root.path()); + let (model, report, configured) = actual_fit(); + let before = model.export().unwrap(); + let result = storage + .save_temporal_candidate_with_fit_summary(&model, &report, &configured) + .unwrap(); + assert_eq!(json(&result.fit_diagnostics)["state"], "recorded"); + let summary = result.fit_diagnostics.summary.unwrap(); + assert_eq!( + json(&summary.metrics.initial), + json(&FitObjectiveValues(&report.initial)) + ); + assert_eq!(summary.metrics.completed_epochs, report.completed_epochs); + assert_eq!( + summary.metrics.objective_evaluations, + report.objective_evaluations + ); + assert_eq!(summary.metrics.training_cohorts, 64); + assert_eq!( + summary.metrics.contrastive_trials, + report.contrastive_trials + ); + assert_eq!( + summary.metrics.parameter_delta_l2, + report.parameter_delta_l2 + ); + assert_eq!(summary.metrics.backbone_delta_l2, report.backbone_delta_l2); + assert!(summary.metrics.elapsed_ms.is_none()); + assert_eq!( + summary.binding.artifact_sha256, + result.candidate.artifact_sha256 + ); + assert_eq!( + summary.binding.artifact_kind, + "ruview.room-learner.managed-temporal-input.v3" + ); + assert!( + summary.binding.schedule_sha256.is_some() + && summary.binding.execution_policy_sha256.is_some() + && summary.binding.identity_evidence_sha256.is_some() + ); + let cold = storage + .load_exact_temporal_candidate(model.context(), &result.candidate.artifact_sha256) + .unwrap(); + assert_eq!(cold.export().unwrap(), before); + assert_eq!(model.export().unwrap(), before); + let loaded = storage.temporal_fit_diagnostics(&cold); + assert_eq!(json(&loaded)["state"], "recorded"); + assert_eq!(json(&loaded.summary.unwrap()), json(&summary)); + assert!(!summary.physically_validated && !summary.qualification_granted); + assert!(sidecar(root.path(), &model).metadata().unwrap().len() < 16_384); +} +// Same names/types as ObjectiveValues serialization; no rounded/estimated loss. +struct FitObjectiveValues<'a>(&'a crate::room_learner::training::ObjectiveValues); +impl Serialize for FitObjectiveValues<'_> { + fn serialize(&self, s: S) -> Result { + self.0.serialize(s) + } +} +#[test] +fn missing_history_is_not_recorded_and_optional_write_failure_keeps_whole_model_available() { + let root = tempfile::tempdir().unwrap(); + let storage = store(root.path()); + let (model, report, c) = fixture(); + let receipt = storage.save_temporal_candidate(&model).unwrap(); + let before = model.export().unwrap(); + let missing = storage.temporal_fit_diagnostics(&model); + assert_eq!(json(&missing)["state"], "not_recorded"); + assert!(missing.summary.is_none()); + crate::companion::atomic_private_write( + &sidecar(root.path(), &model), + b"existing malformed diagnostics", + ) + .unwrap(); + let result = storage + .save_temporal_candidate_with_fit_summary(&model, &report, &c) + .unwrap(); + assert_eq!(json(&result.fit_diagnostics)["state"], "not_recorded"); + assert!(result.fit_diagnostics.warning.is_some()); + assert!(result.fit_diagnostics.summary.is_none()); + assert_eq!( + storage + .load_exact_temporal_candidate(model.context(), &receipt.artifact_sha256) + .unwrap() + .export() + .unwrap(), + before + ); + assert_eq!( + std::fs::read(sidecar(root.path(), &model)).unwrap(), + b"existing malformed diagnostics" + ); + assert_eq!( + json(&storage.temporal_fit_diagnostics(&model))["state"], + "unavailable" + ); +} +#[test] +fn exact_v3_bound_configuration_and_actual_numeric_report_are_required_without_invalidating_model() +{ + for mode in ["config", "objective", "delta", "count"] { + let root = tempfile::tempdir().unwrap(); + let storage = store(root.path()); + let (model, mut report, mut c) = fixture(); + match mode { + "config" => c.learning_rate = 0.3, + "objective" => report.final_objective.total = f32::NAN, + "delta" => report.parameter_delta_l2 = 0., + _ => report.training_cohorts = 4, + }; + let result = storage + .save_temporal_candidate_with_fit_summary(&model, &report, &c) + .unwrap(); + assert_eq!( + json(&result.fit_diagnostics)["state"], + "not_recorded", + "{mode}" + ); + assert!(result.fit_diagnostics.warning.is_some()); + assert!(storage + .load_exact_temporal_candidate(model.context(), &result.candidate.artifact_sha256) + .is_ok()); + } +} +#[test] +fn whole_identity_and_policy_tampering_even_with_recomputed_summary_hash_is_unavailable() { + for field in [ + "artifact_kind", + "artifact_sha256", + "context_digest", + "training_data_sha256", + "input_evidence_sha256", + "input_policy_sha256", + "execution_policy_sha256", + "schedule_sha256", + "identity_evidence_sha256", + ] { + let root = tempfile::tempdir().unwrap(); + let storage = store(root.path()); + let (model, r, c) = fixture(); + storage + .save_temporal_candidate_with_fit_summary(&model, &r, &c) + .unwrap(); + let path = sidecar(root.path(), &model); + let mut e: Value = serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap(); + e["summary"]["binding"][field] = json!("f".repeat(64)); + e["summary_sha256"] = json!(bytes_hash(&serde_json::to_vec(&e["summary"]).unwrap())); + crate::companion::atomic_private_write( + &path, + &crate::room_learner::artifact::rehash_temporal_fit_fixture( + &serde_json::to_vec(&e).unwrap(), + ) + .unwrap(), + ) + .unwrap(); + assert_eq!( + storage.load_temporal_fit_summary(&model).unwrap_err(), + "temporal_fit_summary_binding_changed", + "{field}" + ); + assert_eq!( + json(&storage.temporal_fit_diagnostics(&model))["state"], + "unavailable", + "{field}" + ); + assert!(storage.load_temporal_candidate(model.context()).is_ok()); + } +} +#[test] +fn aggregate_summary_contains_no_raw_rows_timestamps_credentials_or_authority() { + let root = tempfile::tempdir().unwrap(); + let storage = store(root.path()); + let (m, r, c) = fixture(); + let result = storage + .save_temporal_candidate_with_fit_summary(&m, &r, &c) + .unwrap(); + let v = json(&result.fit_diagnostics); + fn walk(v: &Value) { + match v { + Value::Object(o) => { + for (k, v) in o { + assert!( + ![ + "samples", + "frame_ids", + "sequence_ids", + "times_ns", + "captured_at", + "timestamp", + "secret", + "keys", + "raw_iq", + "teacher" + ] + .contains(&k.as_str()), + "{k}" + ); + walk(v) + } + } + Value::Array(a) => { + for v in a { + walk(v) + } + } + _ => {} + } + } + walk(&v); + assert_eq!(v["physically_validated"], false); + assert_eq!(v["qualification_granted"], false); +} + +#[test] +fn oversized_or_symlink_summary_is_optional_and_never_replaces_the_retained_bytes() { + for mode in ["oversize", "symlink"] { + let root = tempfile::tempdir().unwrap(); + let storage = store(root.path()); + let (m, r, c) = fixture(); + let receipt = storage.save_temporal_candidate(&m).unwrap(); + let path = sidecar(root.path(), &m); + if mode == "oversize" { + crate::companion::atomic_private_write(&path, &vec![b'x'; 16_385]).unwrap(); + } else { + #[cfg(unix)] + { + let target = root.path().join("unrelated.txt"); + std::fs::write(&target, b"keep unrelated").unwrap(); + std::os::unix::fs::symlink(&target, &path).unwrap(); + } + #[cfg(not(unix))] + { + continue; + } + } + let result = storage + .save_temporal_candidate_with_fit_summary(&m, &r, &c) + .unwrap(); + assert_eq!(json(&result.fit_diagnostics)["state"], "not_recorded"); + assert!(storage + .load_exact_temporal_candidate(m.context(), &receipt.artifact_sha256) + .is_ok()); + assert_eq!( + json(&storage.temporal_fit_diagnostics(&m))["state"], + "unavailable" + ); + if mode == "oversize" { + assert_eq!(std::fs::read(&path).unwrap().len(), 16_385); + } else { + assert_eq!( + std::fs::read(root.path().join("unrelated.txt")).unwrap(), + b"keep unrelated" + ); + assert!(std::fs::symlink_metadata(path) + .unwrap() + .file_type() + .is_symlink()); + } + } +} +#[test] +#[cfg(unix)] +fn insecure_summary_and_foreign_store_cannot_expose_actual_private_metrics() { + use std::os::unix::fs::PermissionsExt; + let root = tempfile::tempdir().unwrap(); + let storage = store(root.path()); + let (m, r, c) = fixture(); + let result = storage + .save_temporal_candidate_with_fit_summary(&m, &r, &c) + .unwrap(); + let path = sidecar(root.path(), &m); + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o644)).unwrap(); + assert_eq!( + json(&storage.temporal_fit_diagnostics(&m))["state"], + "unavailable" + ); + let wrong = ArtifactStore::new( + &root.path().join("b".repeat(64)).join("SensingServer"), + &"b".repeat(64), + "synthetic-fit-summary-v3", + ) + .unwrap(); + assert!(wrong + .save_temporal_candidate_with_fit_summary(&m, &r, &c) + .is_err()); + assert!(wrong.load_temporal_fit_summary(&m).is_err()); + assert!(storage + .load_exact_temporal_candidate(m.context(), &result.candidate.artifact_sha256) + .is_ok()); +} +#[test] +fn rehashed_false_objectives_cannot_be_reported_as_actual_fit_diagnostics() { + let root = tempfile::tempdir().unwrap(); + let storage = store(root.path()); + let (m, r, c) = fixture(); + storage + .save_temporal_candidate_with_fit_summary(&m, &r, &c) + .unwrap(); + let path = sidecar(root.path(), &m); + let mut envelope: Value = serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap(); + envelope["summary"]["metrics"]["final_objective"]["total"] = json!(r.initial.total + 10.); + envelope["summary_sha256"] = json!(bytes_hash( + &serde_json::to_vec(&envelope["summary"]).unwrap() + )); + crate::companion::atomic_private_write( + &path, + &crate::room_learner::artifact::rehash_temporal_fit_fixture( + &serde_json::to_vec(&envelope).unwrap(), + ) + .unwrap(), + ) + .unwrap(); + assert_eq!( + storage.load_temporal_fit_summary(&m).unwrap_err(), + "temporal_fit_report_invalid" + ); + assert_eq!( + json(&storage.temporal_fit_diagnostics(&m))["state"], + "unavailable" + ); + assert!(storage.load_temporal_candidate(m.context()).is_ok()); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_full_run.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_full_run.rs new file mode 100644 index 0000000000..134d2c5ee3 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_full_run.rs @@ -0,0 +1,1132 @@ +//! Fresh-genesis full-run recovery DATA and private owner custody. +//! No public constructor, migration, HeldOut producer or qualification authority. +use super::*; +use crate::room_learner::artifact::ArtifactStore; +use crate::room_learner::temporal_acquisition::{ReceiverCadenceBand, WindowInputPolicy}; +use std::sync::Arc; + +pub(crate) const PREFIX_SCHEMA: &str = "ruview.room-learner.temporal-runtime-prefix.v3"; +pub(crate) const META_KIND: &str = "temporal-prefix-commitment-v2"; +pub(crate) const POLICY_DOMAIN: &str = "ruview.room-learner.temporal-full-run-policy.v1"; +pub(crate) const FIT_RECORD_DOMAIN: &str = "ruview.room-learner.temporal-verified-fit-record.v1"; +pub(crate) const MODEL_KIND: &str = "ruview.room-learner.managed-temporal-input.v3"; + +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct PolicyBody { + prefix_schema: String, + metadata_kind: String, + model_artifact_kind: String, + context_digest: String, + input_policy_sha256: String, + execution_policy_sha256: String, + schedule_sha256: String, + configured_fit_sha256: String, + native_profile: WindowInputPolicy, + train_episodes: usize, + held_out_episodes: usize, + train_windows: usize, + held_out_windows: usize, + total_windows: usize, + max_total_windows: usize, + max_numeric_values: usize, + max_recovery_bytes: usize, + max_global_gaps: usize, + max_quality_parts: usize, + transition_limit: usize, +} +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct ContinuationPolicy { + schema: String, + body: PolicyBody, + sha256: String, +} +impl ContinuationPolicy { + pub(super) fn build(binding: &ManagedBinding) -> Result { + let schedule = binding.contract()?; + let c = schedule.counts(); + let context = schedule.context(); + if context.sources.len() != 3 + || context.sources.iter().any(|s| s.feature_count != 56) + || binding.native.target_span_ms() != 2000 + || binding.native.source_policies().len() != 3 + || binding + .native + .source_policies() + .iter() + .any(|s| s.receiver_cadence != ReceiverCadenceBand::Hz30To36 || s.max_rows != 77) + || ( + c.train_episodes, + c.held_out_episodes, + c.train_windows, + c.held_out_windows, + c.total_windows, + ) != if schedule.is_operator_eight_step(){(4,4,64,16,80)}else{(7,8,64,64,128)} + || schedule.resources().max_numeric_values != if schedule.is_operator_eight_step(){1_034_880}else{1_655_808} + || schedule.resources().max_numeric_values > 2_000_000 + { + return Err("temporal_full_run_engineering_profile_unsupported".into()); + } + let episodes = c + .train_episodes + .checked_add(c.held_out_episodes) + .ok_or("temporal_full_run_capacity")?; + let segments = episodes + .checked_add(128) + .ok_or("temporal_full_run_capacity")?; + // STATES, revision0..432: genesis; each initial/resumed segment; at + // most one accepted pause per segment (including terminal/deferred + // pauses); windows; separate confirmed seals; Frozen; reserved final + // completion; retirement. Recertification/SAME-intent retry add zero. + let transition_limit = 1usize + .checked_add(segments) + .and_then(|n| n.checked_add(segments)) + .and_then(|n| n.checked_add(c.total_windows)) + .and_then(|n| n.checked_add(episodes)) + .and_then(|n| n.checked_add(3)) + .ok_or("temporal_full_run_capacity")?; + if transition_limit != if schedule.is_operator_eight_step(){364}else{433} || transition_limit > 512 { + return Err("temporal_full_run_transition_capacity".into()); + } + let body = PolicyBody { + prefix_schema: PREFIX_SCHEMA.into(), + metadata_kind: META_KIND.into(), + model_artifact_kind: MODEL_KIND.into(), + context_digest: context.digest(), + input_policy_sha256: hash(binding)?, + execution_policy_sha256: hash(&EXECUTION_POLICY)?, + schedule_sha256: schedule.sha256().into(), + configured_fit_sha256: hash(&binding.fit)?, + native_profile: binding.native.clone(), + train_episodes: c.train_episodes, + held_out_episodes: c.held_out_episodes, + train_windows: c.train_windows, + held_out_windows: c.held_out_windows, + total_windows: c.total_windows, + max_total_windows: 128, + max_numeric_values: 2_000_000, + max_recovery_bytes: crate::room_learner::recovery_store::MAX_RECOVERY_BYTES, + max_global_gaps: 128, + max_quality_parts: segments, + transition_limit, + }; + let sha256 = hash(&(POLICY_DOMAIN, &body))?; + Ok(Self { + schema: POLICY_DOMAIN.into(), + body, + sha256, + }) + } + pub(super) fn validate(&self, binding: &ManagedBinding) -> Result<(), String> { + let expected = Self::build(binding)?; + if serde_json::to_vec(self).map_err(|_| "temporal_full_run_policy_encoding")? + != serde_json::to_vec(&expected).map_err(|_| "temporal_full_run_policy_encoding")? + { + return Err("temporal_full_run_policy_changed".into()); + } + Ok(()) + } + pub(super) fn sha256(&self) -> &str { + &self.sha256 + } + pub(super) fn transition_limit(&self) -> usize { + self.body.transition_limit + } +} + +/// Non-serde custody is never reconstructed by a raw prefix decoder. +pub(super) struct FullRunOwnerCustody { + runtime_epoch: String, + generation: u64, +} +pub(super) struct FullRunState { + pub policy: ContinuationPolicy, + pub owner: Option, + // Only DATA after raw decode. Verified session installation is separate. + pub frozen: Option, + pub local_start_not_before_ns: Option, + pub held_out: Option, + pub complete: Option, + pending_complete: Option, + completed: Option, + pending: Option, + verified: Option, +} +impl FullRunState { + pub(super) fn fresh( + binding: &ManagedBinding, + run: &str, + witness: crate::room_learner::lifecycle::FullRunGenesisWitness, + ) -> Result { + let (epoch, generation) = witness.consume(binding.native.context(), run)?; + Ok(Self { + policy: ContinuationPolicy::build(binding)?, + owner: Some(FullRunOwnerCustody { + runtime_epoch: epoch, + generation, + }), + frozen: None, + local_start_not_before_ns: None, + held_out: None, + complete: None, + pending_complete: None, + completed: None, + pending: None, + verified: None, + }) + } + pub(super) fn structural( + policy: ContinuationPolicy, + frozen: Option, + ) -> Self { + Self { + policy, + owner: None, + frozen, + local_start_not_before_ns: None, + held_out: None, + complete: None, + pending_complete: None, + completed: None, + pending: None, + verified: None, + } + } + pub(super) fn snapshot_permit( + &self, + expected: Option, + ) -> Result { + let owner = self + .owner + .as_ref() + .ok_or("temporal_full_run_owner_custody_required")?; + if self.frozen.is_some() + && self.pending.is_none() + && ((!self.frozen.as_ref().is_some_and(|f|f.body.operator_profile.is_some()) && self.held_out.is_none()) || self.verified.is_none() || self.completed.is_some()) + { + return Err("temporal_frozen_write_requires_current_producer".into()); + } + if self.complete.is_some() && self.pending_complete.is_none() { + return Err("temporal_complete_write_requires_current_producer".into()); + } + Ok(FullRunSnapshotPermit { + runtime_epoch: owner.runtime_epoch.clone(), + generation: owner.generation, + expected, + model: self + .pending + .as_ref() + .map(|p| p.model.clone()) + .or_else(|| self.verified.as_ref().map(|v| v.verifier())), + }) + } +} +/// Producer-only snapshot permission; serialized DATA never contains this. +pub(super) struct FullRunSnapshotPermit { + runtime_epoch: String, + generation: u64, + expected: Option, + model: Option, +} +#[derive(Clone)] +pub(crate) struct FullRunWriteGrant { + payload_sha256: String, + expected: Option, + runtime_epoch: String, + generation: u64, + model: Option, +} +impl FullRunSnapshotPermit { + pub(super) fn encoded(self, bytes: &[u8]) -> FullRunWriteGrant { + FullRunWriteGrant { + payload_sha256: bytes_hash(bytes), + expected: self.expected, + runtime_epoch: self.runtime_epoch, + generation: self.generation, + model: self.model, + } + } +} +impl FullRunWriteGrant { + pub(crate) fn permits(&self, bytes: &[u8], expected: Option<&TemporalToken>) -> bool { + !self.runtime_epoch.is_empty() + && self.generation > 0 + && self.expected.as_ref() == expected + && self.payload_sha256 == bytes_hash(bytes) + } + pub(super) fn matches_fit_record(&self, sha: Option<&str>) -> bool { + match (&self.model, sha) { + (None, None) => true, + (Some(model), Some(sha)) => model.record.sha256 == sha, + _ => false, + } + } + pub(crate) fn verify_frozen_model(&self) -> Result<(), String> { + match &self.model { + Some(m) => m.verify().map(|_| ()), + None => Ok(()), + } + } +} + +#[derive(Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(tag = "state", rename_all = "snake_case", deny_unknown_fields)] +pub(super) enum FitSummaryReference { + Recorded { sha256: String }, + NotRecorded, + Unavailable, +} +#[derive(Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub(super) struct VerifiedFitRecordBody { + pub owner_scope: String, + pub installation_id: String, + pub context_digest: String, + pub run: String, + pub prefix_schema: String, + pub metadata_kind: String, + pub continuation_policy_sha256: String, + pub artifact_kind: String, + pub artifact_sha256: String, + pub input_policy_sha256: String, + pub execution_policy_sha256: String, + pub schedule_sha256: String, + pub configured_fit_sha256: String, + pub training_data_sha256: String, + pub input_evidence_sha256: String, + pub identity_evidence_sha256: String, + pub train_seal_chain_sha256: String, + pub accepted_train_prefix_sha256: String, + pub accepted_train_commitment: + crate::room_learner::recovery_store::temporal_commitment::TemporalCommitmentData, + pub saved_candidate: crate::room_learner::artifact::TemporalCandidateReceipt, + pub actual_summary: FitSummaryReference, + pub freeze_prepare_receiver_ns: u64, + pub source_floors: Vec<(u8, u64, u64)>, + #[serde(default,skip_serializing_if="Option::is_none")] + pub operator_profile: Option, +} +/// Serializable DATA, not a permit. SHA binds data, never producer authority. +#[derive(Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub(super) struct VerifiedFitRecordData { + pub schema: String, + pub body: VerifiedFitRecordBody, + pub sha256: String, +} +impl VerifiedFitRecordData { + pub(super) fn validate_data( + &self, + binding: &ManagedBinding, + policy: &ContinuationPolicy, + ) -> Result<(), String> { + policy.validate(binding)?; + let b = &self.body; + let p = &policy.body; + let c = binding.native.context(); + let sha = |s: &str| { + s.len() == 64 + && s.bytes() + .all(|x| x.is_ascii_digit() || (b'a'..=b'f').contains(&x)) + }; + if self.schema != FIT_RECORD_DOMAIN + || self.sha256 != hash(&(FIT_RECORD_DOMAIN, &self.body))? + || b.owner_scope != c.owner_scope + || b.installation_id != c.installation_id + || b.context_digest != c.digest() + || b.operator_profile.as_deref() != binding.contract()?.is_operator_eight_step().then_some("native-2s-three-source-56bin-operator-eight-step-v1") + || b.run.is_empty() + || b.run.len() > 256 + || !b.run.is_ascii() + || b.prefix_schema != PREFIX_SCHEMA + || b.metadata_kind != META_KIND + || b.continuation_policy_sha256 != policy.sha256 + || b.artifact_kind != MODEL_KIND + || b.input_policy_sha256 != p.input_policy_sha256 + || b.execution_policy_sha256 != p.execution_policy_sha256 + || b.schedule_sha256 != p.schedule_sha256 + || b.configured_fit_sha256 != p.configured_fit_sha256 + || [ + &b.artifact_sha256, + &b.training_data_sha256, + &b.input_evidence_sha256, + &b.identity_evidence_sha256, + &b.train_seal_chain_sha256, + &b.accepted_train_prefix_sha256, + ] + .iter() + .any(|s| !sha(s)) + || b.freeze_prepare_receiver_ns == 0 + || b.source_floors.len() != c.sources.len() + || b.saved_candidate.artifact_kind != MODEL_KIND + || b.saved_candidate.artifact_sha256 != b.artifact_sha256 + || b.saved_candidate.schema != "ruview.room-learner.temporal-candidate.v1" + || b.saved_candidate.owner_scope != b.owner_scope + || b.saved_candidate.installation_id != b.installation_id + || b.saved_candidate.context_digest != b.context_digest + || b.saved_candidate.room_digest != c.room_digest + || b.saved_candidate.input_policy_sha256 != b.input_policy_sha256 + || !(1..=512).contains(&b.saved_candidate.parameter_count) + || !b.saved_candidate.diagnostic_only + || b.saved_candidate.qualification_granted + || b.saved_candidate.physically_validated + { + return Err("temporal_verified_fit_record_binding".into()); + } + if matches!(&b.actual_summary,FitSummaryReference::Recorded{sha256}if !sha(sha256)) { + return Err("temporal_verified_fit_summary_binding".into()); + } + Ok(()) + } +} + +#[derive(Clone)] +pub(super) struct FrozenModelVerifier { + store: Arc, + record: VerifiedFitRecordData, + context: CaptureContext, + actual: Arc, +} +impl FrozenModelVerifier { + pub(super) fn verify(&self) -> Result, String> { + let loaded = self + .store + .load_exact_temporal_candidate(&self.context, &self.record.body.artifact_sha256)?; + verify_model_binding(&self.record, &loaded)?; + Ok(Arc::new(loaded)) + } +} +fn verify_model_binding( + record: &VerifiedFitRecordData, + model: &TemporalModel, +) -> Result<(), String> { + let b = model.fit_diagnostic_binding()?; + let r = &record.body; + if record.body.operator_profile.is_some() != model.managed_is_operator_eight_step()? + || b.artifact_kind != MODEL_KIND + || b.artifact_sha256 != r.artifact_sha256 + || b.context_digest != r.context_digest + || b.input_policy_sha256 != r.input_policy_sha256 + || b.execution_policy_sha256.as_ref() != Some(&r.execution_policy_sha256) + || b.schedule_sha256.as_ref() != Some(&r.schedule_sha256) + || b.training_data_sha256 != r.training_data_sha256 + || b.input_evidence_sha256 != r.input_evidence_sha256 + || b.identity_evidence_sha256.as_ref() != Some(&r.identity_evidence_sha256) + || b.bound_fit_config.as_ref().map(hash).transpose()?.as_ref() + != Some(&r.configured_fit_sha256) + || b.parameter_count != r.saved_candidate.parameter_count + { + return Err("temporal_frozen_exact_model_kind_or_binding".into()); + } + Ok(()) +} +/// One current owner operation, separate from a completed fitting Job.cancel. +pub(crate) struct PendingFrozenTransition { + operation: (String, u64, u64), + record: VerifiedFitRecordData, + model: FrozenModelVerifier, + rows: Vec>, + evidence: Vec>, +} +/// Neither raw DATA nor latest model pointers can construct this install value. +pub(crate) struct DurableFrozenTransition { + install: VerifiedFrozenSession, + admitted_at_ns: u64, +} +pub(crate) struct RecoveredVerifiedFrozen { + install: VerifiedFrozenSession, + recertified_at_ns: u64, +} +struct VerifiedFrozenSession { + store: Arc, + model: Arc, + record: VerifiedFitRecordData, + rows: Vec>, + evidence: Vec>, +} +impl VerifiedFrozenSession { + fn verifier(&self) -> FrozenModelVerifier { + FrozenModelVerifier { + store: self.store.clone(), + record: self.record.clone(), + context: self.model.context().clone(), + actual: self.model.clone(), + } + } + fn check_current_train_prefix(&self, session: &TemporalTrainingSession) -> Result<(), String> { + let count=if self.record.body.operator_profile.is_some(){80}else{64}; + if session.failed + || !session.fitted + || session.rows.len() < count + || session.evidence.len() < count + || session.rows[..count] + .iter() + .zip(&self.rows) + .any(|(a, b)| !a.same_record(b)) + || session.evidence[..count] + .iter() + .zip(&self.evidence) + .any(|(a, b)| !a.same_record(b)) + || session.run_id != self.record.body.run + || session + .managed + .as_ref() + .and_then(|m| m.frozen_sha256.as_ref()) + != Some(&self.record.body.artifact_sha256) + { + return Err("temporal_heldout_immutable_train_prefix_changed".into()); + } + Ok(()) + } + fn check(&self, session: &TemporalTrainingSession) -> Result<(), String> { + let count=if self.record.body.operator_profile.is_some(){80}else{64}; + if session.active + || session.failed + || session.fitted + || session.rows.len() != count + || session.evidence.len() != count + || session + .rows + .iter() + .zip(&self.rows) + .any(|(a, b)| !a.same_record(b)) + || session + .evidence + .iter() + .zip(&self.evidence) + .any(|(a, b)| !a.same_record(b)) + || session.plan_sha256() != self.record.body.input_policy_sha256 + || session.run_id != self.record.body.run + || session.binding.context.digest() != self.record.body.context_digest + || hash(&session.config)? != self.record.body.configured_fit_sha256 + { + return Err("temporal_verified_frozen_session_changed".into()); + } + Ok(()) + } + fn apply(&self, session: &mut TemporalTrainingSession) { + session.managed.as_mut().unwrap().frozen_sha256 = + Some(self.record.body.artifact_sha256.clone()); + session.fitted = true; + } +} +impl FullRunState { + pub(super) fn is_frozen(&self) -> bool { + self.verified.is_some() + } + pub(super) fn is_freezing(&self) -> bool { + self.pending.is_some() + } + pub(super) fn pending_record(&self) -> Option<&VerifiedFitRecordData> { + self.pending.as_ref().map(|p| &p.record) + } + pub(super) fn verified_model(&self) -> Option> { + self.verified.as_ref().map(|v| v.model.clone()) + } +} +/// The ONLY cold Frozen authority producer calls the real CURRENT store and +/// recertifies its barriers itself; no raw decoded object is accepted as input. +pub(super) fn recover_current( + store: &RecoveryStore, + artifact: Arc, + expected: &CaptureContext, + now: u64, + epoch: &str, + anchor: u64, + witness: crate::room_learner::lifecycle::FullRunRecoveryWitness, +) -> Result { + let mut w = TemporalWorkflow::cold_restore(store, expected, now, epoch, anchor)?; + let (owner_epoch, generation) = witness.consume(expected)?; + let mode = w + .full_run + .as_ref() + .ok_or("temporal_full_run_no_upconversion")?; + mode.policy.validate( + &w.session + .managed + .as_ref() + .ok_or("managed_session_required")? + .binding, + )?; + let record = mode.frozen.clone(); + let continued = mode.held_out.is_some(); + if let Some(record) = record { + if now < record.body.freeze_prepare_receiver_ns { + return Err("temporal_frozen_recovery_clock_reversed".into()); + } + let model = Arc::new( + artifact.load_exact_temporal_candidate(expected, &record.body.artifact_sha256)?, + ); + verify_model_binding(&record, &model)?; + if continued { + let token = w + .commit_token + .clone() + .ok_or("temporal_recovered_current_token_required")?; + let verifier = FrozenModelVerifier { + store: artifact.clone(), + record, + context: expected.clone(), + actual: model, + }; + verifier + .continued_witness(&w, &token)? + .ok_or("temporal_recovered_continued_witness_missing")? + .install_raw(&mut w)?; + } else { + let install = VerifiedFrozenSession { + store: artifact.clone(), + model, + record, + rows: w.session.rows.clone(), + evidence: w.session.evidence.clone(), + }; + install.check(&w.session)?; + let recovered = RecoveredVerifiedFrozen { + install, + recertified_at_ns: now, + }; + recovered.install.apply(&mut w.session); + w.full_run.as_mut().unwrap().verified = Some(recovered.install); + } + w.full_run.as_mut().unwrap().local_start_not_before_ns = Some(now); + } + let mode = w.full_run.as_mut().unwrap(); + mode.owner = Some(FullRunOwnerCustody { + runtime_epoch: owner_epoch, + generation, + }); + if let Some(record) = &mode.complete { + let token = w + .commit_token + .clone() + .ok_or("temporal_recovered_current_token_required")?; + mode.completed = Some(VerifiedFullComplete { + token, + record_sha256: record.sha256.clone(), + }); + } + Ok(w) +} +impl TemporalWorkflow { + pub(super) fn request_frozen( + &mut self, + accepted: crate::room_learner::lifecycle::AcceptedTemporalFit, + witness: crate::room_learner::lifecycle::FrozenOwnerWitness, + store: Arc, + now: u64, + ) -> Result<(), String> { + let identity = fit_admission::current_identity(self, &self.schedule()?, now)?; + let mode = self + .full_run + .as_ref() + .ok_or("temporal_full_run_no_upconversion")?; + let owner = mode + .owner + .as_ref() + .ok_or("temporal_full_run_owner_custody_required")?; + if mode.pending.is_some() || mode.frozen.is_some() || mode.verified.is_some() { + return Err("temporal_frozen_operation_already_present".into()); + } + let operation = witness.consume(self.context(), &self.session.run_id)?; + if operation.0 != owner.runtime_epoch || operation.1 != owner.generation { + return Err("temporal_frozen_owner_changed".into()); + } + let (model, source) = + accepted.consume_for_owner(&owner.runtime_epoch, owner.generation, &identity)?; + let record = source.record_data(now, self.source_high.clone())?; + record.validate_data( + &self.session.managed.as_ref().unwrap().binding, + &mode.policy, + )?; + if model.context() != self.context() { + return Err("temporal_frozen_exact_model_context".into()); + } + let verifier = FrozenModelVerifier { + store, + record: record.clone(), + context: self.context().clone(), + actual: model, + }; + let pending = PendingFrozenTransition { + operation, + record: record.clone(), + model: verifier, + rows: self.session.rows.clone(), + evidence: self.session.evidence.clone(), + }; + self.check_clock(now)?; + self.descriptor.saved_at_ms = now / 1_000_000; + let mode = self.full_run.as_mut().unwrap(); + mode.pending = Some(pending); + mode.frozen = Some(record); + self.submit_async_commit(false) + } + pub(super) fn prepare_frozen_ack( + &self, + candidate: &TemporalWorkflow, + now: u64, + ) -> Result, String> { + let Some(mode) = &self.full_run else { + return Ok(None); + }; + let Some(pending) = &mode.pending else { + return Ok(None); + }; + let owner = mode + .owner + .as_ref() + .ok_or("temporal_full_run_owner_custody_required")?; + let data = candidate + .full_run + .as_ref() + .and_then(|m| m.frozen.as_ref()) + .ok_or("temporal_frozen_ack_record_missing")?; + if data != &pending.record + || pending.operation.0 != owner.runtime_epoch + || pending.operation.1 != owner.generation + || now < data.body.freeze_prepare_receiver_ns + || now < self.last_now_ns + || now / 1_000_000 >= self.descriptor.expires_at_ms + || self.source_high.iter().any(|(id, seq, time)| { + data.body + .source_floors + .iter() + .any(|(s, q, t)| s == id && (seq < q || time < t)) + }) + { + return Err("temporal_frozen_ack_owner_cut_or_record".into()); + } + let install = VerifiedFrozenSession { + store: pending.model.store.clone(), + model: pending.model.actual.clone(), + record: data.clone(), + rows: pending.rows.clone(), + evidence: pending.evidence.clone(), + }; + // Model I/O must be done on the writer, not here. The exact same model + // Arc is carried by the pending capability after writer readback. + install.check(&self.session)?; + Ok(Some(DurableFrozenTransition { + install, + admitted_at_ns: now, + })) + } + pub(super) fn apply_frozen_ack(&mut self, transition: DurableFrozenTransition) { + transition.install.apply(&mut self.session); + let mode = self.full_run.as_mut().unwrap(); + mode.pending = None; + mode.local_start_not_before_ns = Some(transition.admitted_at_ns); + mode.verified = Some(transition.install); + self.stage = Stage::DurableReady; + } +} + +/// Off-writer DATA digests of an actually read-back prefix, never an issuer. +#[derive(Clone)] +pub(super) struct FullPrefixHashes { + pub raw: String, + pub evidence: String, + pub identities: String, + pub ledger: String, +} +impl FullPrefixHashes { + pub(super) fn from_workflow(w: &TemporalWorkflow) -> Result { + Ok(Self { + raw: hash(&w.session.rows)?, + evidence: hash(&w.session.evidence)?, + identities: hash( + &w.session + .managed + .as_ref() + .ok_or("managed_session_required")? + .windows, + )?, + ledger: hash(&w.ledger)?, + }) + } +} +/// Produced only by actual writer/current-store readback with a producer-owned +/// model verifier. No serde/Clone or DATA/hash-only install constructor. +pub(super) struct VerifiedContinuedSession { + model: Arc, + record: VerifiedFitRecordData, + store: Arc, + rows: Vec>, + evidence: Vec>, + token: TemporalToken, +} +impl VerifiedContinuedSession { + pub(super) fn check_owner( + &self, + current: &TemporalWorkflow, + token: &TemporalToken, + ) -> Result<(), String> { + let mode = current + .full_run + .as_ref() + .ok_or("temporal_full_run_required")?; + let verified = mode + .verified + .as_ref() + .ok_or("temporal_verified_frozen_required")?; + if &self.token != token + || mode.owner.is_none() + || self.record != verified.record + || !Arc::ptr_eq(&self.model, &verified.model) + || self.rows.len() < 64 + || self.rows.len() > 128 + || self.rows.len() > current.session.rows.len() + || self + .rows + .iter() + .zip(¤t.session.rows) + .any(|(a, b)| !a.same_record(b)) + || self + .evidence + .iter() + .zip(¤t.session.evidence) + .any(|(a, b)| !a.same_record(b)) + { + return Err("temporal_continued_ack_owner_or_lineage".into()); + } + Ok(()) + } + pub(super) fn install_raw(&self, target: &mut TemporalWorkflow) -> Result<(), String> { + if target.session.fitted + || target.session.active + || target.session.failed + || target.session.rows.len() != self.rows.len() + || target + .session + .rows + .iter() + .zip(&self.rows) + .any(|(a, b)| !a.same_record(b)) + || target + .session + .evidence + .iter() + .zip(&self.evidence) + .any(|(a, b)| !a.same_record(b)) + || target.full_run.as_ref().and_then(|m| m.frozen.as_ref()) != Some(&self.record) + { + return Err("temporal_continued_raw_install_changed".into()); + } + target.session.managed.as_mut().unwrap().frozen_sha256 = + Some(self.record.body.artifact_sha256.clone()); + target.session.fitted = true; + let mode = target.full_run.as_mut().unwrap(); + mode.verified = Some(VerifiedFrozenSession { + store: self.store.clone(), + model: self.model.clone(), + record: self.record.clone(), + rows: self.rows[..if self.record.body.operator_profile.is_some(){80}else{64}].to_vec(), + evidence: self.evidence[..if self.record.body.operator_profile.is_some(){80}else{64}].to_vec(), + }); + Ok(()) + } +} +impl FrozenModelVerifier { + pub(super) fn check_workflow_data(&self, w: &TemporalWorkflow) -> Result<(), String> { + if w.context() != &self.context + || w.full_run.as_ref().and_then(|m| m.frozen.as_ref()) != Some(&self.record) + { + return Err("temporal_current_model_recertification_binding".into()); + } + Ok(()) + } + pub(super) fn continued_witness( + &self, + w: &TemporalWorkflow, + token: &TemporalToken, + ) -> Result, String> { + let Some(mode) = &w.full_run else { + return Ok(None); + }; + if mode.held_out.is_none() { + return Ok(None); + } + if mode.frozen.as_ref() != Some(&self.record) + || w.session.rows.len() < 64 + || w.session.rows.len() > 128 + { + return Err("temporal_continued_model_record_changed".into()); + } + self.verify()?; + Ok(Some(VerifiedContinuedSession { + model: self.actual.clone(), + record: self.record.clone(), + store: self.store.clone(), + rows: w.session.rows.clone(), + evidence: w.session.evidence.clone(), + token: token.clone(), + })) + } +} +impl FullRunWriteGrant { + pub(super) fn continued_witness( + &self, + w: &TemporalWorkflow, + token: &TemporalToken, + ) -> Result, String> { + self.model + .as_ref() + .ok_or("temporal_continued_producer_model_required")? + .continued_witness(w, token) + } +} +struct PendingFullCompleteTransition { + record: held_out::FullCompleteRecordData, + predecessor: TemporalToken, + runtime_epoch: String, + generation: u64, +} +pub(super) struct VerifiedFullComplete { + token: TemporalToken, + record_sha256: String, +} +impl FullRunState { + pub(super) fn check_current_train_prefix( + &self, + session: &TemporalTrainingSession, + ) -> Result<(), String> { + if self.owner.is_none() { + return Err("temporal_full_run_owner_custody_required".into()); + } + let verified = self + .verified + .as_ref() + .ok_or("temporal_verified_frozen_required")?; + if self.frozen.as_ref() != Some(&verified.record) + || verified.record.body.continuation_policy_sha256 != self.policy.sha256() + { + return Err("temporal_current_verified_fit_record_changed".into()); + } + verified.check_current_train_prefix(session) + } + pub(super) fn model_verifier(&self) -> Option { + self.verified.as_ref().map(|v| v.verifier()) + } + pub(super) fn is_complete(&self) -> bool { + self.completed.is_some() + } + pub(super) fn held_out_seals(&self) -> usize { + self.held_out.as_ref().map_or(0, |h| h.seals.len()) + } +} +impl TemporalWorkflow { + pub(super) fn frozen_identity( + &self, + ) -> Result, String> + { + let schedule = self.schedule()?; + if schedule + .episodes() + .get(self.episode) + .is_none_or(|e| e.role != DataRole::HeldOut) + { + return Ok(None); + } + let mode = self + .full_run + .as_ref() + .ok_or("temporal_verified_frozen_required")?; + mode.check_current_train_prefix(&self.session)?; + let fit = mode + .frozen + .as_ref() + .ok_or("temporal_frozen_record_required")?; + Ok(Some(crate::room_learner::managed_schedule_contract::FrozenModelIdentity::from_runtime_verified_artifact(&schedule,fit.body.artifact_sha256.clone())?)) + } + pub(super) fn begin_held_out_role(&mut self, now: u64) -> Result<(), String> { + self.frozen_identity()? + .ok_or("temporal_heldout_role_required")?; + let mode = self.full_run.as_mut().unwrap(); + if mode.completed.is_some() || mode.complete.is_some() { + return Err("temporal_full_run_already_complete".into()); + } + let fit = mode.frozen.as_ref().unwrap(); + if now <= fit.body.freeze_prepare_receiver_ns + || mode + .local_start_not_before_ns + .is_none_or(|floor| now <= floor) + { + return Err("temporal_heldout_start_before_frozen_admission".into()); + } + if mode.held_out.is_none() { + if self.episode != 7 + || self.durable_windows != 64 + || self.session.rows.len() != 64 + || self + .async_state + .as_ref() + .is_none_or(|s| s.accepted_seals != 7 || s.accepted_held_out_seals != 0) + { + return Err("temporal_heldout_prior_seal_ack_required".into()); + } + mode.held_out = Some(held_out::HeldOutBridgeState::new()); + } + let held = mode.held_out.as_ref().unwrap(); + if self.episode != 7 + held.seals.len() + || self + .async_state + .as_ref() + .is_none_or(|s| s.accepted_held_out_seals != held.seals.len()) + { + return Err("temporal_heldout_prior_seal_ack_required".into()); + } + if self.mapping.is_none() + && (self.session.rows.len() != 64 + held.seals.len() * 8 || !held.parts.is_empty()) + { + return Err("temporal_heldout_initial_episode_not_confirmed".into()); + } + Ok(()) + } + pub(super) fn prepare_complete_from_actual_seal_ack( + &mut self, + now: u64, + ) -> Result { + let Some(mode) = &self.full_run else { + return Ok(false); + }; + let operator=self.schedule()?.is_operator_eight_step(); + let operator_ready=operator && mode.verified.is_some() + && self.bridge.as_ref().is_some_and(|b|b.seals.len()==8); + if (!operator_ready && mode.held_out_seals() != 8) || mode.complete.is_some() { + return Ok(false); + } + let state = self.async_state.as_ref().ok_or("temporal_async_required")?; + if state.pending.is_some() + || if operator {state.accepted_seals != 8}else{state.accepted_held_out_seals != 8} + || self.durable_windows != if operator{80}else{128} + || self.session.rows.len() != if operator{80}else{128} + || self.episode != if operator{8}else{15} + || self.mapping.is_some() + { + return Ok(false); + } + mode.check_current_train_prefix(&self.session)?; + let hashes = state + .full_hashes + .as_ref() + .ok_or("temporal_complete_prefix_hashes_unconfirmed")?; + let predecessor = self + .commit_token + .clone() + .ok_or("temporal_complete_current_token_required")?; + let fit = mode.frozen.as_ref().unwrap(); + let owner = mode.owner.as_ref().unwrap(); + let body = held_out::FullCompleteBody { + context_sha256: self.context().digest(), + run: self.session.run_id.clone(), + descriptor_id: self.descriptor.id.clone(), + created_at_ms: self.descriptor.created_at_ms, + expires_at_ms: self.descriptor.expires_at_ms, + continuation_policy_sha256: mode.policy.sha256().into(), + verified_fit_sha256: fit.sha256.clone(), + artifact_kind: fit.body.artifact_kind.clone(), + artifact_sha256: fit.body.artifact_sha256.clone(), + held_out_chain_sha256: if operator { + let schedule=self.schedule()?; + hash(&self.bridge.as_ref().ok_or("operator_checks_seals_missing")?.seals.iter() + .filter(|seal|schedule.episodes()[seal.episode].role==DataRole::Validation).collect::>())? + } else {mode.held_out.as_ref().unwrap().seals.last().unwrap().sha256.clone()}, + raw_sha256: hashes.raw.clone(), + evidence_sha256: hashes.evidence.clone(), + identity_sha256: hashes.identities.clone(), + ledger_sha256: hashes.ledger.clone(), + windows: if operator{80}else{128}, + held_out_windows: if operator{16}else{64}, + held_out_seals: if operator{4}else{8}, + accepted_held_out_commitment: predecessor.data(), + complete_prepare_receiver_ns: now, + source_floors: self.source_high.clone(), + }; + let record = held_out::FullCompleteRecordData::from_body(body)?; + record.validate_binding_data(fit, &self.descriptor, now)?; + let pending = PendingFullCompleteTransition { + record: record.clone(), + predecessor, + runtime_epoch: owner.runtime_epoch.clone(), + generation: owner.generation, + }; + self.check_clock(now)?; + self.descriptor.saved_at_ms = now / 1_000_000; + let mode = self.full_run.as_mut().unwrap(); + mode.pending_complete = Some(pending); + mode.complete = Some(record); + self.submit_async_commit(false)?; + Ok(true) + } + pub(super) fn prepare_complete_ack( + &self, + candidate: &TemporalWorkflow, + token: &TemporalToken, + ) -> Result, String> { + let Some(mode) = self.full_run.as_ref() else { + return Ok(None); + }; + let Some(pending) = mode.pending_complete.as_ref() else { + return Ok(None); + }; + let owner = mode + .owner + .as_ref() + .ok_or("temporal_full_run_owner_custody_required")?; + if candidate + .full_run + .as_ref() + .and_then(|m| m.complete.as_ref()) + != Some(&pending.record) + || owner.runtime_epoch != pending.runtime_epoch + || owner.generation != pending.generation + || self.commit_token.as_ref() != Some(&pending.predecessor) + { + return Err("temporal_complete_ack_operation_changed".into()); + } + Ok(Some(VerifiedFullComplete { + token: token.clone(), + record_sha256: pending.record.sha256.clone(), + })) + } + pub(super) fn apply_complete_ack(&mut self, complete: VerifiedFullComplete) { + let mode = self.full_run.as_mut().unwrap(); + mode.completed = Some(complete); + mode.pending_complete = None; + } +} + +/// Structural resource budget derived from the canonical Full2 contract only. +/// Not serde, a capture/write permission, or a caller-supplied frame number. +pub(crate) struct ReplayFenceBudget { max_frames:usize,native_policy_sha256:String } +impl ReplayFenceBudget { + pub(crate) fn matches_native_policy(&self,policy:&WindowInputPolicy)->Result<(),String>{ + if self.native_policy_sha256!=hash(policy)?{return Err("temporal_full2_fence_policy_changed".into());}Ok(()) + } + pub(crate) fn max_frames(&self)->usize{self.max_frames} +} +impl ContinuationPolicy { + pub(super) fn replay_fence_budget(&self,binding:&ManagedBinding)->Result{ + self.validate(binding)?; + // Current asynchronous Full2 actor adopts/SAME-repairs its queued + // reservation before resuming, with at most ONE consumed next tail. + // R<=128 and D<=128+current; at R128 no new window exists, so R+D<=256. + // Generic abort-to-parent is explicitly denied for Full2 below. + let attempts=self.body.max_total_windows.checked_add(self.body.max_global_gaps).ok_or("temporal_full2_fence_capacity")?; + let per_attempt=binding.native.source_policies().iter().try_fold(0usize,|n,s|n.checked_add(s.max_rows)).ok_or("temporal_full2_fence_capacity")?; + let max_frames=attempts.checked_mul(per_attempt).ok_or("temporal_full2_fence_capacity")?; + if attempts!=256||per_attempt!=231||max_frames!=59_136{return Err("temporal_full2_fence_capacity".into());} + Ok(ReplayFenceBudget{max_frames,native_policy_sha256:hash(&binding.native)?}) + } +} + +#[path="temporal_evaluation_projection.rs"] +mod evaluation_projection; +pub(crate) use evaluation_projection::{PendingTemporalEvidenceSource, TemporalEvaluationBinding, VerifiedTemporalTrainSnapshot, VerifiedTemporalEvaluationSnapshot, VerifiedTemporalEvaluationResult}; + +impl FullRunState { + pub(super) fn public_verified_fit(&self)->Option<&VerifiedFitRecordData>{let verified=self.verified.as_ref()?;let data=self.frozen.as_ref()?;if self.owner.is_none()||data!=&verified.record{return None}Some(data)} + pub(super) fn public_complete(&self,current:Option<&TemporalToken>)->Option<&held_out::FullCompleteRecordData>{let fact=self.completed.as_ref()?;let data=self.complete.as_ref()?;if current!=Some(&fact.token)||data.sha256!=fact.record_sha256{return None}Some(data)} +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_full_run_checkpoint.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_full_run_checkpoint.rs new file mode 100644 index 0000000000..a712fbe2e4 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_full_run_checkpoint.rs @@ -0,0 +1,1123 @@ +//! Structural v3 codec. Raw JSON never installs owner/fitted/write authority. +use super::*; +use full_run::{ContinuationPolicy, FullRunSnapshotPermit, FullRunState, VerifiedFitRecordData}; +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Common { + binding: ManagedBinding, + run: String, + descriptor: RecoveryDescriptor, + last_now_ns: u64, + source_high: Vec<(u8, u64, u64)>, + ledger: Ledger, + mapping: Option, + episode: usize, +} +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Accepted { + rows: Vec, + evidence: Vec, + identities: Vec, + fences: AssemblerCheckpoint, +} +#[derive(Clone, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +enum FullStage { + Train { + bridge: BridgeState, + }, + OperatorFrozen { fit: VerifiedFitRecordData, seals: Vec }, + OperatorComplete { fit: VerifiedFitRecordData, seals: Vec, complete: held_out::FullCompleteRecordData }, + Frozen { + fit: VerifiedFitRecordData, + train_seals: Vec, + }, + HeldOut { + fit: VerifiedFitRecordData, + train_seals: Vec, + held_out: held_out::HeldOutBridgeState, + }, + Complete { + fit: VerifiedFitRecordData, + train_seals: Vec, + held_out_seals: Vec, + complete: held_out::FullCompleteRecordData, + }, +} +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct FullData { + schema: String, + policy: ContinuationPolicy, + common: Common, + accepted: Accepted, + stage: FullStage, +} +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct FullEnvelope

{ + schema: String, + payload_sha256: String, + payload: P, +} +pub(super) struct FullSnapshot { + data: FullData, SharedRecord>, + pub permit: Option, +} +impl FullSnapshot { + pub(super) fn from_workflow(w: &TemporalWorkflow) -> Result { + let mode = w.full_run.as_ref().ok_or("temporal_full_run_required")?; + mode.policy.validate( + &w.session + .managed + .as_ref() + .ok_or("managed_session_required")? + .binding, + )?; + let permit = mode.snapshot_permit(w.commit_token.clone())?; + let operator=w.schedule()?.is_operator_eight_step(); + let old = if mode.held_out.is_some() || operator && mode.frozen.is_some() && mode.pending_record().is_none() { + mode.check_current_train_prefix(&w.session)?; + if w.session.failed || w.session.active || w.session.rows.len() != w.ledger.spans.len() + { + return Err("temporal_full_snapshot_incomplete".into()); + } + let managed = w + .session + .managed + .as_ref() + .ok_or("managed_session_required")?; + Prefix { + schema: BRIDGE_SCHEMA.into(), + binding: managed.binding.clone(), + run: w.session.run_id.clone(), + descriptor: w.descriptor.clone(), + last_now_ns: w.last_now_ns, + rows: w.session.rows.clone(), + evidence: w.session.evidence.clone(), + identities: managed.windows.clone(), + fences: w.session.assembler.fence_checkpoint()?, + source_high: w.source_high.clone(), + ledger: w.ledger.clone(), + mapping: w.mapping.clone(), + episode: w.episode, + frozen_model: None, + quality_issued: false, + confirmation_issued: false, + bridge: w.bridge.clone(), + } + } else { + super::snapshot_old(w)? + }; + let stage = if operator && mode.frozen.is_some() { + let fit=mode.frozen.as_ref().unwrap().clone(); + let seals=old.bridge.as_ref().ok_or("operator_bridge_required")?.seals.clone(); + if let Some(complete)=&mode.complete {FullStage::OperatorComplete{fit,seals,complete:complete.clone()}} + else {FullStage::OperatorFrozen{fit,seals}} + } else if let Some(held) = &mode.held_out { + let fit = mode + .frozen + .as_ref() + .ok_or("temporal_heldout_frozen_record_missing")? + .clone(); + let train_seals = old + .bridge + .as_ref() + .ok_or("temporal_full_run_bridge_required")? + .seals + .clone(); + if let Some(complete) = &mode.complete { + FullStage::Complete { + fit, + train_seals, + held_out_seals: held.seals.clone(), + complete: complete.clone(), + } + } else { + FullStage::HeldOut { + fit, + train_seals, + held_out: held.clone(), + } + } + } else { + match &mode.frozen { + None => FullStage::Train { + bridge: old + .bridge + .clone() + .ok_or("temporal_full_run_bridge_required")?, + }, + Some(fit) => { + if mode.pending_record() != Some(fit) { + return Err("temporal_frozen_current_producer_required".into()); + } + FullStage::Frozen { + fit: fit.clone(), + train_seals: old + .bridge + .as_ref() + .ok_or("temporal_full_run_bridge_required")? + .seals + .clone(), + } + } + } + }; + Ok(Self { + data: FullData { + schema: full_run::PREFIX_SCHEMA.into(), + policy: mode.policy.clone(), + common: Common { + binding: old.binding, + run: old.run, + descriptor: old.descriptor, + last_now_ns: old.last_now_ns, + source_high: old.source_high, + ledger: old.ledger, + mapping: old.mapping, + episode: old.episode, + }, + accepted: Accepted { + rows: old.rows, + evidence: old.evidence, + identities: old.identities, + fences: old.fences, + }, + stage, + }, + permit: Some(permit), + }) + } + pub(super) fn records(&self) -> SharedRecords { + SharedRecords { + rows: self.data.accepted.rows.clone(), + evidence: self.data.accepted.evidence.clone(), + } + } + pub(super) fn encode(self) -> Result, String> { + let envelope = FullEnvelope { + schema: full_run::PREFIX_SCHEMA.into(), + payload_sha256: hash(&self.data)?, + payload: self.data, + }; + let bytes = serde_json::to_vec(&envelope).map_err(|_| "temporal_full_run_encoding")?; + if bytes.len() > crate::room_learner::recovery_store::MAX_RECOVERY_BYTES { + return Err("temporal_checkpoint_budget".into()); + } + Ok(bytes) + } +} +pub(super) fn is_full(bytes: &[u8]) -> Result { + if bytes.len() > crate::room_learner::recovery_store::MAX_RECOVERY_BYTES { + return Err("temporal_checkpoint_budget".into()); + } + #[derive(Deserialize)] + struct Header { + schema: String, + } + // Header dispatch skips unknown fields without making a serde Value/raw + // copy. The selected strict decoder still checks every field/canonical byte. + let header: Header = serde_json::from_slice(bytes).map_err(|_| "temporal_checkpoint_schema")?; + Ok(header.schema == full_run::PREFIX_SCHEMA) +} +fn parse(bytes: &[u8]) -> Result { + if bytes.len() > crate::room_learner::recovery_store::MAX_RECOVERY_BYTES { + return Err("temporal_checkpoint_budget".into()); + } + let e: FullEnvelope = serde_json::from_slice(bytes).map_err(|_| "temporal_full_run_schema")?; + if e.schema != full_run::PREFIX_SCHEMA + || e.payload.schema != full_run::PREFIX_SCHEMA + || e.payload_sha256 != hash(&e.payload)? + || serde_json::to_vec(&e).map_err(|_| "temporal_full_run_encoding")? != bytes + { + return Err("temporal_full_run_integrity".into()); + } + e.payload.policy.validate(&e.payload.common.binding)?; + Ok(e) +} +fn old_projection( + data: FullData, +) -> Result<(Prefix, ContinuationPolicy, Option), String> { + let common = data.common; + let accepted = data.accepted; + let (bridge, frozen) = match data.stage { + FullStage::Train { bridge } => { + // Full-v3 genesis and model identity retain their ORIGINAL Train + // seal profile. Recognizing fresh343 for Kind1 must not widen this + // separately versioned Kind2 continuation contract. + let operator=common.binding.contract()?.is_operator_eight_step(); + if bridge.policy != if operator{bridge::OPERATOR_EIGHT_POLICY}else{bridge::BRIDGE_POLICY} { + return Err("temporal_full_run_train_profile_changed".into()); + } + (bridge, None) + } + FullStage::Frozen { fit, train_seals } => { + fit.validate_data(&common.binding, &data.policy)?; + if common.run != fit.body.run + || common.last_now_ns != fit.body.freeze_prepare_receiver_ns + || common.source_high != fit.body.source_floors + || common.mapping.is_some() + { + return Err("temporal_frozen_record_cut_or_floors".into()); + } + let b = BridgeState { + policy: bridge::BRIDGE_POLICY.into(), + disposition: bridge::Disposition::Complete, + seals: train_seals, + parts: vec![], + }; + (b, Some(fit)) + } + _ => return Err("temporal_post_frozen_requires_full_decoder".into()), + }; + Ok(( + Prefix { + schema: BRIDGE_SCHEMA.into(), + binding: common.binding, + run: common.run, + descriptor: common.descriptor, + last_now_ns: common.last_now_ns, + rows: accepted.rows, + evidence: accepted.evidence, + identities: accepted.identities, + fences: accepted.fences, + source_high: common.source_high, + ledger: common.ledger, + mapping: common.mapping, + episode: common.episode, + frozen_model: None, + quality_issued: false, + confirmation_issued: false, + bridge: Some(bridge), + }, + data.policy, + frozen, + )) +} +fn old_bytes(prefix: Prefix) -> Result, String> { + serde_json::to_vec(&Envelope { + schema: BRIDGE_SCHEMA.into(), + payload_sha256: hash(&prefix)?, + payload: prefix, + }) + .map_err(|_| "temporal_full_run_projection".into()) +} +pub(super) fn decode_structural( + bytes: &[u8], + descriptor: &RecoveryDescriptor, + now: u64, +) -> Result { + let e = parse(bytes)?; + if matches!(e.payload.stage,FullStage::OperatorFrozen{..}|FullStage::OperatorComplete{..}) { + return decode_operator(e.payload,descriptor,now); + } + if matches!( + e.payload.stage, + FullStage::HeldOut { .. } | FullStage::Complete { .. } + ) { + return decode_continued(e.payload, descriptor, now); + } + let (prefix, policy, frozen) = old_projection(e.payload)?; + // This slice reuses the exact existing Train/quality/native validator. No + // HeldOut/Complete variant is accepted before the actual C4B producer exists. + let replay_budget = policy.replay_fence_budget(&prefix.binding)?; + let mut w = + super::decode_full_train_projection(&old_bytes(prefix)?, descriptor, now, &replay_budget)?; + if let Some(record) = &frozen { + let managed = w + .session + .managed + .as_ref() + .ok_or("managed_session_required")?; + let bridge = w + .bridge + .as_ref() + .ok_or("temporal_full_run_bridge_required")?; + let body = &record.body; + if w.session.rows.len() != 64 + || w.session.rows.iter().any(|r| r.role != DataRole::Train) + || bridge.disposition != bridge::Disposition::Complete + || bridge.seals.len() != 7 + || body.training_data_sha256 != hash(&w.session.rows)? + || body.input_evidence_sha256 != hash(&w.session.evidence)? + || body.identity_evidence_sha256 != hash(&managed.windows)? + || body.train_seal_chain_sha256 != hash(&bridge.seals)? + || body.accepted_train_commitment.run != w.session.run_id + || body.accepted_train_commitment.id != descriptor.id + { + return Err("temporal_frozen_train_binding".into()); + } + } + w.full_run = Some(FullRunState::structural(policy, frozen)); + // Deliberately remains unfitted. Only authenticated CURRENT recovery plus + // exact whole-model verification can mint the install witness. + Ok(w) +} +pub(super) fn validated_structural( + bytes: &[u8], + descriptor: &RecoveryDescriptor, + now: u64, +) -> Result { + let w = decode_structural(bytes, descriptor, now)?; + let mode = w.full_run.as_ref().ok_or("temporal_full_run_required")?; + Ok(ValidatedTemporalPrefix { + bytes: bytes.to_vec(), + descriptor: descriptor.clone(), + run: w.session.run_id.clone(), + policy_sha256: mode.policy.sha256().into(), + window_count: w.session.rows.len(), + transition_limit: mode.policy.transition_limit(), + saved_receiver_ns: w.last_now_ns, + metadata_kind: full_run::META_KIND, + full_write_grant: None, + full_fit_record_sha256: mode.frozen.as_ref().map(|fit| fit.sha256.clone()), + full_stage: Some(if let Some(complete) = &mode.complete { + ValidatedFullStage::Complete(complete.body.accepted_held_out_commitment.clone()) + } else if mode.held_out.is_some() { + ValidatedFullStage::HeldOut + } else if let Some(fit) = &mode.frozen { + ValidatedFullStage::Frozen(fit.body.accepted_train_commitment.clone()) + } else { + ValidatedFullStage::Train + }), + }) +} +pub(super) fn is_new_full_run(p: &ValidatedTemporalPrefix) -> Result { + let e = parse(p.bytes())?; + if !matches!(e.payload.stage, FullStage::Train { .. }) { + return Ok(false); + } + let (prefix, _, _) = old_projection(e.payload)?; + let bytes = old_bytes(prefix)?; + let old = super::validated_prefix(&bytes, p.descriptor(), p.saved_receiver_ns())?; + super::is_new_empty_run(&old) +} +pub(super) fn validate_full_extension( + old: &ValidatedTemporalPrefix, + new: &ValidatedTemporalPrefix, +) -> Result<(), String> { + if old.run != new.run + || old.policy_sha256 != new.policy_sha256 + || old.transition_limit != new.transition_limit + || old.descriptor.id != new.descriptor.id + || old.descriptor.owner_scope != new.descriptor.owner_scope + || old.descriptor.installation_id != new.descriptor.installation_id + || old.descriptor.context_digest != new.descriptor.context_digest + || old.descriptor.created_at_ms != new.descriptor.created_at_ms + || old.descriptor.expires_at_ms != new.descriptor.expires_at_ms + || new.descriptor.saved_at_ms < old.descriptor.saved_at_ms + || new.saved_receiver_ns < old.saved_receiver_ns + { + return Err("temporal_commit_identity_or_time_changed".into()); + } + let a = parse(old.bytes())?; + let b = parse(new.bytes())?; + if serde_json::to_vec(&a.payload.policy).map_err(|_| "temporal_encoding")? + != serde_json::to_vec(&b.payload.policy).map_err(|_| "temporal_encoding")? + { + return Err("temporal_full_run_policy_changed".into()); + } + if old.bytes() == new.bytes() { + return Ok(()); + } + if matches!( + a.payload.stage, + FullStage::HeldOut { .. } | FullStage::Complete { .. } + ) || matches!( + b.payload.stage, + FullStage::HeldOut { .. } | FullStage::Complete { .. } + ) { + return validate_continued_extension(&a.payload, &b.payload); + } + match (&a.payload.stage, &b.payload.stage) { + (FullStage::Train { .. }, FullStage::Train { .. }) => { + let (ap, apolicy, _) = old_projection(a.payload)?; + let (bp, bpolicy, _) = old_projection(b.payload)?; + let a_budget = apolicy.replay_fence_budget(&ap.binding)?; + let b_budget = bpolicy.replay_fence_budget(&bp.binding)?; + let ab = old_bytes(ap)?; + let bb = old_bytes(bp)?; + let av = super::validated_full_train_projection( + &ab, + old.descriptor(), + old.saved_receiver_ns(), + &a_budget, + )?; + let bv = super::validated_full_train_projection( + &bb, + new.descriptor(), + new.saved_receiver_ns(), + &b_budget, + )?; + super::validate_extension(&av, &bv) + } + (FullStage::Train { bridge }, FullStage::Frozen { fit, train_seals }) + | (FullStage::Train { bridge }, FullStage::OperatorFrozen { fit, seals:train_seals }) => { + if bridge.disposition != bridge::Disposition::Complete + || bridge.seals != *train_seals + || fit.body.accepted_train_prefix_sha256 != bytes_hash(old.bytes()) + || serde_json::to_vec(&a.payload.accepted).map_err(|_| "temporal_encoding")? + != serde_json::to_vec(&b.payload.accepted).map_err(|_| "temporal_encoding")? + || a.payload.common.binding.schedule_sha256 + != b.payload.common.binding.schedule_sha256 + || a.payload.common.run != b.payload.common.run + || a.payload.common.ledger != b.payload.common.ledger + || a.payload.common.mapping.is_some() + || b.payload.common.mapping.is_some() + || a.payload.common.episode != b.payload.common.episode + || b.payload.common.last_now_ns < a.payload.common.last_now_ns + { + return Err("temporal_frozen_transition_binding".into()); + } + for (id, seq, time) in &a.payload.common.source_high { + if !b + .payload + .common + .source_high + .iter() + .any(|(s, q, t)| s == id && q >= seq && t >= time) + { + return Err("temporal_commit_floor_regressed".into()); + } + } + // General descriptor/policy identity is also enforced by store + // commitment preparation; the raw structure grants no write. + Ok(()) + } + (FullStage::OperatorFrozen{fit:a_fit,seals:a_seals},FullStage::OperatorComplete{fit:b_fit,seals:b_seals,complete}) => { + if a_fit != b_fit || a_seals != b_seals + || serde_json::to_vec(&a.payload.accepted).map_err(|_|"temporal_encoding")? != serde_json::to_vec(&b.payload.accepted).map_err(|_|"temporal_encoding")? + || a.payload.common.binding.schedule_sha256 != b.payload.common.binding.schedule_sha256 + || a.payload.common.run != b.payload.common.run || a.payload.common.ledger != b.payload.common.ledger + || a.payload.common.mapping.is_some() || b.payload.common.mapping.is_some() + || a.payload.common.episode!=8 || b.payload.common.episode!=8 + || complete.body.accepted_held_out_commitment.run!=a.payload.common.run + || complete.body.complete_prepare_receiver_ns!=b.payload.common.last_now_ns { + return Err("operator_complete_transition_changed".into()); + } + for(id,seq,time)in &a.payload.common.source_high{if !b.payload.common.source_high.iter().any(|(s,q,t)|s==id&&q>=seq&&t>=time){return Err("temporal_commit_floor_regressed".into());}} + Ok(()) + } + _ => Err("temporal_full_run_transition_unsupported".into()), + } +} + +fn decode_operator(data:FullData,descriptor:&RecoveryDescriptor,now:u64)->Result{ + let (fit,seals,complete)=match data.stage{ + FullStage::OperatorFrozen{fit,seals}=>(fit,seals,None), + FullStage::OperatorComplete{fit,seals,complete}=>(fit,seals,Some(complete)), + _=>return Err("operator_frozen_stage_required".into()), + }; + let c=data.common;let a=data.accepted; + let schedule=c.binding.contract()?; + if !schedule.is_operator_eight_step() || c.episode!=8 || c.mapping.is_some() || a.rows.len()!=80 + || a.evidence.len()!=80 || a.identities.len()!=80 || seals.len()!=8 || c.run!=fit.body.run + || c.last_now_ns=seq&&t>=time)) { + return Err("operator_frozen_prefix_changed".into()); + } + fit.validate_data(&c.binding,&data.policy)?; + let prefix=Prefix{schema:BRIDGE_SCHEMA.into(),binding:c.binding,run:c.run,descriptor:c.descriptor,last_now_ns:c.last_now_ns, + rows:a.rows,evidence:a.evidence,identities:a.identities,fences:a.fences,source_high:c.source_high,ledger:c.ledger, + mapping:c.mapping,episode:c.episode,frozen_model:None,quality_issued:false,confirmation_issued:false, + bridge:Some(BridgeState{policy:bridge::OPERATOR_EIGHT_POLICY.into(),disposition:bridge::Disposition::Complete,seals,parts:vec![]})}; + let budget=data.policy.replay_fence_budget(&prefix.binding)?; + let mut w=super::decode_full_train_projection(&old_bytes(prefix)?,descriptor,now,&budget)?; + validate_immutable_train(&w,&fit)?; + if let Some(record)=&complete{ + record.validate_binding_data(&fit,descriptor,now)?; + let hashes=full_run::FullPrefixHashes::from_workflow(&w)?; + let checkseals=w.bridge.as_ref().unwrap().seals.iter().filter(|seal|schedule.episodes()[seal.episode].role==DataRole::Validation).collect::>(); + if record.body.raw_sha256!=hashes.raw || record.body.evidence_sha256!=hashes.evidence + || record.body.identity_sha256!=hashes.identities || record.body.ledger_sha256!=hashes.ledger + || record.body.held_out_chain_sha256!=hash(&checkseals)? { + return Err("operator_complete_actual_prefix_changed".into()); + } + } + let mut mode=FullRunState::structural(data.policy,Some(fit));mode.complete=complete;w.full_run=Some(mode); + Ok(w) +} + +fn decode_continued( + data: FullData, + descriptor: &RecoveryDescriptor, + now: u64, +) -> Result { + let (fit, train_seals, held, complete) = match data.stage { + FullStage::HeldOut { + fit, + train_seals, + held_out, + } => (fit, train_seals, held_out, None), + FullStage::Complete { + fit, + train_seals, + held_out_seals, + complete, + } => { + let mut held = held_out::HeldOutBridgeState::new(); + held.disposition = bridge::Disposition::Complete; + held.seals = held_out_seals; + (fit, train_seals, held, Some(complete)) + } + _ => return Err("temporal_post_frozen_stage_required".into()), + }; + let common = data.common; + let accepted = data.accepted; + fit.validate_data(&common.binding, &data.policy)?; + let schedule = common.binding.contract()?; + if accepted.rows.len() < 64 + || accepted.rows.len() > 128 + || accepted.evidence.len() != accepted.rows.len() + || accepted.identities.len() != accepted.rows.len() + || common.ledger.spans.len() != accepted.rows.len() + || common.run != fit.body.run + || common.last_now_ns < fit.body.freeze_prepare_receiver_ns + || fit.body.source_floors.iter().any(|(id, seq, time)| { + !common + .source_high + .iter() + .any(|(s, q, t)| s == id && q >= seq && t >= time) + }) + { + return Err("temporal_heldout_prefix_or_freeze_floor".into()); + } + let train = BridgeState { + policy: bridge::BRIDGE_POLICY.into(), + disposition: bridge::Disposition::Complete, + seals: train_seals, + parts: vec![], + }; + let train_ledger = Ledger { + spans: common.ledger.spans[..64].to_vec(), + gaps: common + .ledger + .gaps + .iter() + .filter(|g| g.episode < 7) + .cloned() + .collect(), + }; + if train.seals.len() != 7 || held.seals.len() > 8 { + return Err("temporal_bridge_phase_order".into()); + } + let mut count = 0; + let mut cuts = train + .seals + .iter() + .enumerate() + .map(|(i, _)| { + count += schedule.episodes()[i].slots.len(); + count + }) + .collect::>(); + cuts.extend((0..held.seals.len()).map(|i| 64 + (i + 1) * 8)); + let digests = prefix_digests::canonical_tuple_prefixes( + &accepted.rows, + &accepted.evidence, + &common.ledger.spans, + &cuts, + )?; + train.validate_full_run_with_prefixes( + &schedule, + &common.run, + &accepted.rows[..64], + &accepted.evidence[..64], + &train_ledger, + 7, + None, + common.last_now_ns, + &digests, + )?; + let train_parts = train.seals.iter().map(|s| s.parts.len()).sum(); + held.validate_shape_data( + &schedule, + &common.run, + &fit, + common.episode, + accepted.rows.len(), + train_parts, + common.ledger.gaps.len(), + common.last_now_ns, + )?; + let frozen=crate::room_learner::managed_schedule_contract::FrozenModelIdentity::from_runtime_verified_artifact(&schedule,fit.body.artifact_sha256.clone())?; + for (index, seal) in held.seals.iter().enumerate() { + let count = 64 + (index + 1) * 8; + seal.validate_actual_prefix_digest(digests.at(count)?)?; + bridge::validate_parts_with_frozen( + &seal.body.parts, + &schedule, + &common.run, + 7 + index, + Some(30_000_000_000), + seal.body.confirmed_at_ns, + Some(&frozen), + )?; + bridge::validate_part_lineage( + &seal.body.parts, + 7 + index, + &common.ledger, + &accepted.rows, + None, + )?; + } + if held.seals.len() < 8 { + let accepted_ns = common + .ledger + .spans + .last() + .filter(|s| s.episode == common.episode) + .map_or(0, |s| s.observed_end_ns); + bridge::validate_parts_with_frozen( + &held.parts, + &schedule, + &common.run, + common.episode, + None, + common.last_now_ns, + Some(&frozen), + )?; + bridge::validate_part_lineage( + &held.parts, + common.episode, + &common.ledger, + &accepted.rows, + common.mapping.as_ref(), + )?; + if let Some(mapping) = &common.mapping { + if held.parts.last().map(|p| &p.mapping) != Some(mapping) + || held.parts.last().map_or(0, |p| p.observed_end_ns) != accepted_ns + { + return Err("temporal_heldout_current_mapping_or_frontier".into()); + } + } else if !held.parts.is_empty() || accepted.rows.len() != 64 + held.seals.len() * 8 { + return Err("temporal_heldout_mapping_missing".into()); + } + if matches!( + held.disposition, + bridge::Disposition::Observing + | bridge::Disposition::Paused + | bridge::Disposition::NeedsConfirmation + ) && common.mapping.is_none() + { + return Err("temporal_heldout_phase_without_mapping".into()); + } + if held.disposition == bridge::Disposition::NeedsConfirmation + && accepted_ns != 30_000_000_000 + { + return Err("temporal_heldout_confirmation_before_end".into()); + } + } else if common.mapping.is_some() { + return Err("temporal_heldout_done_mapping".into()); + } + for part in held + .seals + .iter() + .flat_map(|s| &s.body.parts) + .chain(&held.parts) + { + for observed in &part.sources { + if !common.source_high.iter().any(|(id, seq, time)| { + *id == observed.source_id + && *seq >= observed.last_sequence + && *time >= observed.last_ns + }) { + return Err("temporal_heldout_quality_floors_regressed".into()); + } + } + } + if let Some(record) = &complete { + record.validate_binding_data(&fit, &common.descriptor, now)?; + if held.seals.len() != 8 + || accepted.rows.len() != 128 + || common.mapping.is_some() + || record.body.held_out_chain_sha256 != held.seals.last().unwrap().sha256 + || record.body.complete_prepare_receiver_ns != common.last_now_ns + || record.body.source_floors != common.source_high + || record.body.raw_sha256 != hash(&accepted.rows)? + || record.body.evidence_sha256 != hash(&accepted.evidence)? + || record.body.identity_sha256 != hash(&accepted.identities)? + || record.body.ledger_sha256 != hash(&common.ledger)? + { + return Err("temporal_full_complete_prefix_changed".into()); + } + } + let replay_budget = data.policy.replay_fence_budget(&common.binding)?; + let prefix = Prefix { + schema: BRIDGE_SCHEMA.into(), + binding: common.binding, + run: common.run, + descriptor: common.descriptor, + last_now_ns: common.last_now_ns, + rows: accepted.rows, + evidence: accepted.evidence, + identities: accepted.identities, + fences: accepted.fences, + source_high: common.source_high, + ledger: common.ledger, + mapping: common.mapping, + episode: common.episode, + frozen_model: None, + quality_issued: false, + confirmation_issued: false, + bridge: Some(train), + }; + let mut workflow = super::decode_full_post_frozen( + prefix, + descriptor, + now, + &fit, + held.seals.len(), + &replay_budget, + )?; + validate_immutable_train(&workflow, &fit)?; + let mut mode = FullRunState::structural(data.policy, Some(fit)); + mode.held_out = Some(held); + mode.complete = complete; + workflow.full_run = Some(mode); + Ok(workflow) +} +pub(super) fn validate_immutable_train( + w: &TemporalWorkflow, + record: &VerifiedFitRecordData, +) -> Result<(), String> { + let managed = w + .session + .managed + .as_ref() + .ok_or("managed_session_required")?; + let bridge = w + .bridge + .as_ref() + .ok_or("temporal_full_run_bridge_required")?; + let body = &record.body; + let schedule=w.schedule()?; + let operator=schedule.is_operator_eight_step(); + let rows=w.session.rows.iter().filter(|r|r.role==DataRole::Train).collect::>(); + let evidence=w.session.evidence.iter().filter(|r|r.role==DataRole::Train).collect::>(); + let identities=managed.windows.iter().filter(|r|r.role==DataRole::Train).collect::>(); + let seals=bridge.seals.iter().filter(|seal|schedule.episodes()[seal.episode].role==DataRole::Train).collect::>(); + if rows.len()!=64 || evidence.len()!=64 || identities.len()!=64 + || bridge.disposition != bridge::Disposition::Complete + || bridge.seals.len()!=if operator{8}else{7} + || body.training_data_sha256 != hash(&rows)? + || body.input_evidence_sha256 != hash(&evidence)? + || body.identity_evidence_sha256 != hash(&identities)? + || body.train_seal_chain_sha256 != hash(&seals)? + || body.accepted_train_commitment.run != w.session.run_id + || body.accepted_train_commitment.id != w.descriptor.id { + return Err("temporal_frozen_train_binding".into()); + } + Ok(()) +} + +fn validate_continued_extension(a: &FullData, b: &FullData) -> Result<(), String> { + macro_rules! prefix { + ($old:expr,$new:expr) => { + if $new.len() < $old.len() + || serde_json::to_vec(&$new[..$old.len()]).map_err(|_| "temporal_encoding")? + != serde_json::to_vec(&$old).map_err(|_| "temporal_encoding")? + { + return Err("temporal_committed_prefix_changed".into()); + } + }; + } + prefix!(a.accepted.rows, b.accepted.rows); + prefix!(a.accepted.evidence, b.accepted.evidence); + prefix!(a.accepted.identities, b.accepted.identities); + prefix!(a.common.ledger.spans, b.common.ledger.spans); + prefix!(a.common.ledger.gaps, b.common.ledger.gaps); + for (id, seq, time) in &a.common.source_high { + if !b + .common + .source_high + .iter() + .any(|(s, q, t)| s == id && q >= seq && t >= time) + { + return Err("temporal_commit_floor_regressed".into()); + } + } + let fixed = |fit_a: &VerifiedFitRecordData, + train_a: &Vec, + fit_b: &VerifiedFitRecordData, + train_b: &Vec| + -> Result<(), String> { + if fit_a != fit_b || train_a != train_b { + return Err("temporal_heldout_frozen_identity_changed".into()); + } + Ok(()) + }; + match (&a.stage, &b.stage) { + ( + FullStage::Frozen { fit, train_seals }, + FullStage::HeldOut { + fit: next_fit, + train_seals: next_train, + held_out, + }, + ) => { + fixed(fit, train_seals, next_fit, next_train)?; + if a.accepted.rows.len() != 64 + || b.accepted.rows.len() != 64 + || a.common.episode != 7 + || b.common.episode != 7 + || a.common.mapping.is_some() + || b.common.mapping.as_ref().is_none_or(|m| { + m.segment_ordinal != 0 + || m.observed_start_ns != 0 + || m.receiver_start_ns <= fit.body.freeze_prepare_receiver_ns + }) + || a.common.ledger != b.common.ledger + || !held_out.seals.is_empty() + || held_out.parts.len() != 1 + || held_out.parts[0].mapping != *b.common.mapping.as_ref().unwrap() + || !held_out.parts[0].sources.is_empty() + || held_out.parts[0].observed_end_ns != 0 + || held_out.disposition != bridge::Disposition::Observing + { + return Err("temporal_first_heldout_segment_changed".into()); + } + Ok(()) + } + ( + FullStage::HeldOut { + fit, + train_seals, + held_out: old, + }, + FullStage::HeldOut { + fit: next_fit, + train_seals: next_train, + held_out: new, + }, + ) => { + fixed(fit, train_seals, next_fit, next_train)?; + prefix!(old.seals, new.seals); + let window = b.accepted.rows.len() == a.accepted.rows.len() + 1 + && b.common.ledger.gaps == a.common.ledger.gaps + && b.common.mapping == a.common.mapping; + let segment = b.accepted.rows.len() == a.accepted.rows.len() + && ((a.common.mapping.is_none() + && b.common.mapping.is_some() + && b.common.ledger.gaps == a.common.ledger.gaps) + || b.common.ledger.gaps.len() == a.common.ledger.gaps.len() + 1); + let seal = b.accepted.rows.len() == a.accepted.rows.len() + && b.common.episode == a.common.episode + 1 + && b.common.ledger.gaps == a.common.ledger.gaps + && b.common.mapping.is_none() + && new.seals.len() == old.seals.len() + 1 + && new.parts.is_empty() + && new.seals.last().is_some_and(|s| { + s.body.parts == old.parts && s.body.confirmed_at_ns >= a.common.last_now_ns + }); + let pause = b.accepted.rows.len() == a.accepted.rows.len() + && b.common.episode == a.common.episode + && b.common.mapping == a.common.mapping + && b.common.ledger.gaps == a.common.ledger.gaps + && new.seals == old.seals + && new.parts == old.parts + && new.disposition == bridge::Disposition::Paused; + if seal { + return Ok(()); + } + if new.seals != old.seals || b.common.episode != a.common.episode { + return Err("temporal_heldout_seal_transition".into()); + } + if window { + if new.parts.len() != old.parts.len() || old.parts.is_empty() { + return Err("temporal_quality_part_transition".into()); + } + let n = old.parts.len() - 1; + if old.parts[..n] != new.parts[..n] + || old.parts[n].mapping != new.parts[n].mapping + || new.parts[n].observed_end_ns < old.parts[n].observed_end_ns + { + return Err("temporal_quality_prefix_changed".into()); + } + for source in &old.parts[n].sources { + if !new.parts[n].sources.iter().any(|s| { + s.source_id == source.source_id + && s.first_ns == source.first_ns + && s.first_sequence == source.first_sequence + && s.packets >= source.packets + && s.last_ns >= source.last_ns + && s.last_sequence >= source.last_sequence + && s.maximum_gap_ns >= source.maximum_gap_ns + && s.minimum_interval_ns <= source.minimum_interval_ns + }) { + return Err("temporal_quality_counts_regressed".into()); + } + } + } else if segment { + if new.parts.len() != old.parts.len() + 1 + || new.parts[..old.parts.len()] != old.parts + || new.parts.last().is_none_or(|p| { + !p.sources.is_empty() || p.observed_end_ns != p.mapping.observed_start_ns + }) + { + return Err("temporal_quality_segment_changed".into()); + } + } else if !pause { + return Err("temporal_heldout_transition_invalid".into()); + } + Ok(()) + } + ( + FullStage::HeldOut { + fit, + train_seals, + held_out, + }, + FullStage::Complete { + fit: next_fit, + train_seals: next_train, + held_out_seals, + complete, + }, + ) => { + fixed(fit, train_seals, next_fit, next_train)?; + if held_out.seals.len() != 8 + || held_out.disposition != bridge::Disposition::Complete + || held_out.seals != *held_out_seals + || !held_out.parts.is_empty() + || a.common.mapping.is_some() + || b.common.mapping.is_some() + || a.common.episode != 15 + || b.common.episode != 15 + || a.common.ledger != b.common.ledger + || serde_json::to_vec(&a.accepted).map_err(|_| "temporal_encoding")? + != serde_json::to_vec(&b.accepted).map_err(|_| "temporal_encoding")? + || complete.body.complete_prepare_receiver_ns < b.common.last_now_ns + { + return Err("temporal_full_complete_transition_changed".into()); + } + Ok(()) + } + _ => Err("temporal_full_run_transition_unsupported".into()), + } +} + +#[cfg(test)] +pub(super) fn c4b_reference_batch_and_stale_seal_probe( + bytes: &[u8], + descriptor: &RecoveryDescriptor, + now: u64, +) -> Result { + let mut envelope = parse(bytes)?; + let data = &envelope.payload; + let (train, held) = match &data.stage { + FullStage::HeldOut { + train_seals, + held_out, + .. + } => (train_seals, &held_out.seals), + FullStage::Complete{train_seals,held_out_seals,..}=>(train_seals,held_out_seals), + _ => return Err("probe_requires_actual_heldout_or_complete_prefix".into()), + }; + if train.len() != 7 || held.is_empty() || data.accepted.rows.len() < 72 { + return Err("probe_requires_actual_sealed_suffix".into()); + } + let mut cuts = Vec::new(); + let schedule = data.common.binding.contract()?; + let mut count = 0; + for (index, _) in train.iter().enumerate() { + count += schedule.episodes()[index].slots.len(); + cuts.push(count); + } + cuts.extend((0..held.len()).map(|i| 64 + (i + 1) * 8)); + let batched = prefix_digests::canonical_tuple_prefixes( + &data.accepted.rows, + &data.accepted.evidence, + &data.common.ledger.spans, + &cuts, + )?; + for (index, cut) in cuts.iter().copied().enumerate() { + let reference = hash(&( + &data.accepted.rows[..cut], + &data.accepted.evidence[..cut], + &data.common.ledger.spans[..cut], + ))?; + if reference != batched.at(cut)? { + return Err("probe_reference_batch_mismatch".into()); + } + let stored = if index < 7 { + &train[index].row_prefix_sha256 + } else { + &held[index - 7].body.row_prefix_sha256 + }; + if stored != &reference { + return Err("probe_stored_reference_mismatch".into()); + } + } + // Valid-shape HeldOut magnitude mutation: immutable Train/model/identities + // unchanged, update the affected cohort/evidence and COMPLETE span chain. + // Keep every existing HeldOut seal's expected tuple digest stale. + let data = &mut envelope.payload; + let old_span_hashes = data + .common + .ledger + .spans + .iter() + .map(hash) + .collect::, _>>()?; + data.accepted.rows[64].views[0].samples[0][0] += 0.125; + let cohort = hash(&data.accepted.rows[64])?; + data.accepted.evidence[64].cohort_sha256 = cohort.clone(); + data.common.ledger.spans[64].window_sha256 = cohort; + let mut changed = std::collections::HashMap::new(); + let mut previous = None; + for (index, span) in data.common.ledger.spans.iter_mut().enumerate() { + span.previous_sha256 = previous.clone(); + let new = hash(span)?; + changed.insert(old_span_hashes[index].clone(), new.clone()); + previous = Some(new); + } + for gap in &mut data.common.ledger.gaps { + if let Some(parent) = &mut gap.after_prefix_sha256 { + *parent = changed + .get(parent) + .cloned() + .ok_or("probe_missing_span_parent")?; + } + } + envelope.payload_sha256 = hash(&envelope.payload)?; + let altered = serde_json::to_vec(&envelope).map_err(|_| "probe_encoding")?; + let error = decode_structural(&altered, descriptor, now) + .err() + .ok_or("probe_stale_heldout_seal_accepted")?; + if error != "temporal_heldout_seal_prefix_changed" { + return Err(format!("probe_negative_stopped_before_seal:{error}")); + } + Ok( + serde_json::json!({"same_immutable_payload_bytes":bytes.len(),"cuts":cuts,"old_new_stored_all_equal":true,"sameN_mutation_no_stale_table":true,"mutation_recomputed_cohort_evidence_entire_span_chain_and_outer_sha":true,"negative_exact_error":error,"train_model_identity_unchanged":true}), + ) +} + +#[cfg(test)] +pub(super) fn c4b_complete_fault_matrix(bytes:&[u8],descriptor:&RecoveryDescriptor,now:u64)->Result{ + let mut results=vec![]; + for case in ["heldout_role","heldout_phase","common_phase","complete_kind","complete_model","complete_run","complete_deadline","complete_count","complete_floor","live_floor"]{ + let mut envelope=parse(bytes)?;let data=&mut envelope.payload; + let FullStage::Complete{held_out_seals,complete,..}=&mut data.stage else{return Err("fault_requires_actual_complete".into())}; + match case{ + "heldout_role"=>{let mut body=held_out_seals[0].body.clone();body.role=DataRole::Train;held_out_seals[0]=held_out::HeldOutSealData::from_body(body)?;}, + "heldout_phase"=>{let mut body=held_out_seals[0].body.clone();body.episode=0;held_out_seals[0]=held_out::HeldOutSealData::from_body(body)?;}, + "common_phase"=>data.common.episode=0, + "complete_kind"=>complete.body.artifact_kind="ruview.experimental.temporal-model-input.v2".into(), + "complete_model"=>complete.body.artifact_sha256="0".repeat(64), + "complete_run"=>complete.body.run.push_str("-forged"), + "complete_deadline"=>complete.body.expires_at_ms+=1, + "complete_count"=>complete.body.windows=127, + "complete_floor"=>complete.body.source_floors[0].1=0, + "live_floor"=>{data.common.source_high[0].1=0;complete.body.source_floors=data.common.source_high.clone();}, + _=>unreachable!(), + } + *complete=held_out::FullCompleteRecordData::from_body(complete.body.clone())?; + envelope.payload_sha256=hash(&envelope.payload)?;let changed=serde_json::to_vec(&envelope).map_err(|_|"fault_encoding")?; + let error=decode_structural(&changed,descriptor,now).err().ok_or_else(||format!("forged_complete_accepted:{case}"))?; + results.push(serde_json::json!({"case":case,"error":error,"inner_and_outer_data_hashes_recomputed":true})); + } + Ok(serde_json::json!({"cases":results,"count":10,"raw_json_authority_issued":false})) +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_held_out_capture.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_held_out_capture.rs new file mode 100644 index 0000000000..ee3c126a99 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_held_out_capture.rs @@ -0,0 +1,350 @@ +//! Private C4B role DATA and bounded shape checks. +//! +//! Nothing here issues a conditions, fit, write, evaluation or +//! qualification permission. The capture owner must also run the unchanged +//! receiver-quality/native/ledger predicates and actual writer-ACK admission. +use super::bridge::{Disposition, QualityPart}; +use super::full_run::VerifiedFitRecordData; +use super::*; +use crate::room_learner::recovery_store::temporal_commitment::TemporalCommitmentData; + +pub(super) const HELD_OUT_BRIDGE_DOMAIN: &str = "ruview.room-learner.temporal-heldout-bridge.v1"; +pub(super) const HELD_OUT_SEAL_DOMAIN: &str = "ruview.room-learner.temporal-heldout-seal.v1"; +pub(super) const HELD_OUT_CONDITIONS_DOMAIN: &str = + "ruview.room-learner.temporal-heldout-conditions.v1"; +pub(super) const FULL_COMPLETE_DOMAIN: &str = "ruview.room-learner.temporal-full-complete.v1"; +pub(super) const OPERATOR_COMPLETE_DOMAIN: &str = "ruview.room-learner.temporal-full-complete.operator-eight-step.v1"; +const WHOLE_MODEL_KIND: &str = "ruview.room-learner.managed-temporal-input.v3"; +const TRAIN_EPISODES: usize = 7; +const TRAIN_WINDOWS: usize = 64; +const HELD_OUT_EPISODES: usize = 8; +const WINDOWS_PER_HELD_OUT_EPISODE: usize = 8; +const FULL_WINDOWS: usize = 128; +const MAX_GLOBAL_GAPS: usize = 128; +const MAX_GLOBAL_PARTS: usize = 143; +const MAX_PARTS_PER_EPISODE: usize = 128; + +fn digest64(value: &str) -> bool { + value.len() == 64 + && value + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) +} + +/// Checks fixed schedule DATA only. It does not assert user conditions, actual +/// fitting, source-clock precision, independent trials or storage custody. +fn fixed_schedule(schedule: &ManagedScheduleContract) -> Result<(), String> { + let counts = schedule.counts(); + if counts.train_episodes != TRAIN_EPISODES + || counts.train_windows != TRAIN_WINDOWS + || counts.held_out_episodes != HELD_OUT_EPISODES + || counts.held_out_windows != TRAIN_WINDOWS + || schedule.episodes().len() != TRAIN_EPISODES + HELD_OUT_EPISODES + || schedule.episodes()[TRAIN_EPISODES..].iter().any(|episode| { + episode.role != DataRole::HeldOut + || episode.target_active_recording_ms != 30_000 + || episode.lead_in_ms != 15_000 + || episode.slots.len() != WINDOWS_PER_HELD_OUT_EPISODE + }) + { + return Err("temporal_heldout_fixed_schedule_changed".into()); + } + Ok(()) +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub(super) struct HeldOutSealBody { + pub(super) role: DataRole, + pub(super) episode: usize, + pub(super) trial_id: String, + pub(super) context_sha256: String, + pub(super) schedule_sha256: String, + pub(super) continuation_policy_sha256: String, + pub(super) verified_fit_sha256: String, + pub(super) artifact_kind: String, + pub(super) artifact_sha256: String, + pub(super) row_prefix_sha256: String, + pub(super) quality_sha256: String, + pub(super) parts: Vec, + pub(super) confirmed_at_ns: u64, + pub(super) previous_sha256: Option, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub(super) struct HeldOutSealData { + schema: String, + pub(super) body: HeldOutSealBody, + pub(super) sha256: String, +} +impl HeldOutSealData { + /// Canonical DATA constructor, NOT a seal/conditions authority issuer. + pub(super) fn from_body(body: HeldOutSealBody) -> Result { + let sha256 = hash(&(HELD_OUT_SEAL_DOMAIN, &body))?; + Ok(Self { + schema: HELD_OUT_SEAL_DOMAIN.into(), + body, + sha256, + }) + } + /// Caller still has to verify exact native/raw prefixes and reused whole- + /// period quality, then authenticate the actual current writer barrier. + pub(super) fn validate_binding_data( + &self, + schedule: &ManagedScheduleContract, + run: &str, + fit: &VerifiedFitRecordData, + now: u64, + ) -> Result<(), String> { + fixed_schedule(schedule)?; + let body = &self.body; + if self.schema != HELD_OUT_SEAL_DOMAIN + || body.role != DataRole::HeldOut + || !(TRAIN_EPISODES..TRAIN_EPISODES + HELD_OUT_EPISODES).contains(&body.episode) + || fit.body.run != run + || body.context_sha256 != schedule.context().digest() + || body.context_sha256 != fit.body.context_digest + || body.schedule_sha256 != schedule.sha256() + || body.schedule_sha256 != fit.body.schedule_sha256 + || body.continuation_policy_sha256 != fit.body.continuation_policy_sha256 + || body.verified_fit_sha256 != fit.sha256 + || body.artifact_kind != WHOLE_MODEL_KIND + || body.artifact_kind != fit.body.artifact_kind + || body.artifact_sha256 != fit.body.artifact_sha256 + || !digest64(&body.row_prefix_sha256) + || body.parts.is_empty() + || body.parts.len() > MAX_PARTS_PER_EPISODE + || body.parts.iter().any(|part| { + part.sources.len() > schedule.context().sources.len() + || part.mapping.segment_ordinal >= MAX_PARTS_PER_EPISODE as u16 + }) + || body.quality_sha256 != hash(&body.parts)? + || body.confirmed_at_ns < fit.body.freeze_prepare_receiver_ns + || body.confirmed_at_ns > now + || body + .previous_sha256 + .as_ref() + .is_some_and(|sha| !digest64(sha)) + || body.trial_id.len() > 128 + || !digest64(&self.sha256) + || self.sha256 != hash(&(HELD_OUT_SEAL_DOMAIN, body))? + { + return Err("temporal_heldout_seal_data_binding".into()); + } + // This hash carrier is used ONLY to check expected identity DATA. + // Its existing contract explicitly grants no fitted/filesystem custody. + let frozen = crate::room_learner::managed_schedule_contract::FrozenModelIdentity::from_runtime_verified_artifact( + schedule, + body.artifact_sha256.clone(), + )?; + let expected = schedule.bind_episode_start(run, body.episode, 0, 0, Some(&frozen))?; + if body.trial_id != expected.trial_id() { + return Err("temporal_heldout_seal_trial_changed".into()); + } + Ok(()) + } + pub(super) fn validate_actual_prefix_digest(&self, actual: &str) -> Result<(), String> { + if self.body.row_prefix_sha256 != actual { + return Err("temporal_heldout_seal_prefix_changed".into()); + } + Ok(()) + } +} + +/// Role-local capture DATA in the existing owner, never a second actor. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub(super) struct HeldOutBridgeState { + policy: String, + pub(super) disposition: Disposition, + pub(super) seals: Vec, + pub(super) parts: Vec, +} +impl HeldOutBridgeState { + pub(super) fn new() -> Self { + Self { + policy: HELD_OUT_BRIDGE_DOMAIN.into(), + disposition: Disposition::Ready, + seals: vec![], + parts: vec![], + } + } + /// Bounded shape/binding check ONLY. The caller must separately validate + /// complete reservations, original native identity and whole-period quality. + pub(super) fn validate_shape_data( + &self, + schedule: &ManagedScheduleContract, + run: &str, + fit: &VerifiedFitRecordData, + episode: usize, + accepted_windows: usize, + train_parts: usize, + global_gaps: usize, + now: u64, + ) -> Result<(), String> { + fixed_schedule(schedule)?; + let sealed_windows = TRAIN_WINDOWS + self.seals.len() * WINDOWS_PER_HELD_OUT_EPISODE; + if self.policy != HELD_OUT_BRIDGE_DOMAIN + || self.seals.len() > HELD_OUT_EPISODES + || episode != TRAIN_EPISODES + self.seals.len() + || global_gaps > MAX_GLOBAL_GAPS + || self.parts.len() > MAX_PARTS_PER_EPISODE + || train_parts + .checked_add(self.parts.len()) + .and_then(|value| { + self.seals + .iter() + .try_fold(value, |sum, seal| sum.checked_add(seal.body.parts.len())) + }) + .is_none_or(|parts| parts > MAX_GLOBAL_PARTS) + || accepted_windows < sealed_windows + || accepted_windows > FULL_WINDOWS + || (self.seals.len() < HELD_OUT_EPISODES + && accepted_windows > sealed_windows + WINDOWS_PER_HELD_OUT_EPISODE) + { + return Err("temporal_heldout_bridge_shape".into()); + } + let mut previous = None; + let mut last_confirmation = fit.body.freeze_prepare_receiver_ns; + for (index, seal) in self.seals.iter().enumerate() { + seal.validate_binding_data(schedule, run, fit, now)?; + if seal.body.episode != TRAIN_EPISODES + index + || seal.body.previous_sha256 != previous + || seal.body.confirmed_at_ns < last_confirmation + { + return Err("temporal_heldout_seal_order".into()); + } + previous = Some(seal.sha256.clone()); + last_confirmation = seal.body.confirmed_at_ns; + } + if self.seals.len() == HELD_OUT_EPISODES { + if self.disposition != Disposition::Complete + || !self.parts.is_empty() + || accepted_windows != FULL_WINDOWS + { + return Err("temporal_heldout_last_seal_shape".into()); + } + } else if self.disposition == Disposition::Complete + || (self.disposition == Disposition::Ready + && (!self.parts.is_empty() || accepted_windows != sealed_windows)) + { + return Err("temporal_heldout_early_complete_or_ready".into()); + } + Ok(()) + } +} + +#[derive(Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub(super) struct FullCompleteBody { + pub(super) context_sha256: String, + pub(super) run: String, + pub(super) descriptor_id: String, + pub(super) created_at_ms: u64, + pub(super) expires_at_ms: u64, + pub(super) continuation_policy_sha256: String, + pub(super) verified_fit_sha256: String, + pub(super) artifact_kind: String, + pub(super) artifact_sha256: String, + pub(super) held_out_chain_sha256: String, + pub(super) raw_sha256: String, + pub(super) evidence_sha256: String, + pub(super) identity_sha256: String, + pub(super) ledger_sha256: String, + pub(super) windows: usize, + pub(super) held_out_windows: usize, + pub(super) held_out_seals: usize, + pub(super) accepted_held_out_commitment: TemporalCommitmentData, + pub(super) complete_prepare_receiver_ns: u64, + pub(super) source_floors: Vec<(u8, u64, u64)>, +} +#[derive(Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub(super) struct FullCompleteRecordData { + schema: String, + pub(super) body: FullCompleteBody, + pub(super) sha256: String, +} +impl FullCompleteRecordData { + /// Computes a DATA digest. Only actual last-SEAL ACK/current-store recovery + /// may later supply the separate non-serde Complete write/install witness. + pub(super) fn from_body(body: FullCompleteBody) -> Result { + let domain=if (body.windows,body.held_out_windows,body.held_out_seals)==(80,16,4){OPERATOR_COMPLETE_DOMAIN}else{FULL_COMPLETE_DOMAIN}; + let sha256 = hash(&(domain, &body))?; + Ok(Self { + schema: domain.into(), + body, + sha256, + }) + } + pub(super) fn validate_binding_data( + &self, + fit: &VerifiedFitRecordData, + descriptor: &RecoveryDescriptor, + now: u64, + ) -> Result<(), String> { + let body = &self.body; + let operator=fit.body.operator_profile.as_deref()==Some("native-2s-three-source-56bin-operator-eight-step-v1"); + if fit.body.operator_profile.is_some() && !operator{return Err("temporal_full_complete_profile_changed".into());} + let domain=if operator{OPERATOR_COMPLETE_DOMAIN}else{FULL_COMPLETE_DOMAIN}; + let hashes = [ + &body.continuation_policy_sha256, + &body.verified_fit_sha256, + &body.artifact_sha256, + &body.held_out_chain_sha256, + &body.raw_sha256, + &body.evidence_sha256, + &body.identity_sha256, + &body.ledger_sha256, + ]; + if self.schema != domain + || hashes.iter().any(|value| !digest64(value)) + || body.context_sha256 != descriptor.context_digest + || body.context_sha256 != fit.body.context_digest + || body.run != fit.body.run + || body.descriptor_id != descriptor.id + || body.descriptor_id != fit.body.accepted_train_commitment.id + || body.created_at_ms != descriptor.created_at_ms + || body.expires_at_ms != descriptor.expires_at_ms + || body.continuation_policy_sha256 != fit.body.continuation_policy_sha256 + || body.verified_fit_sha256 != fit.sha256 + || body.artifact_kind != WHOLE_MODEL_KIND + || body.artifact_kind != fit.body.artifact_kind + || body.artifact_sha256 != fit.body.artifact_sha256 + || body.windows != if operator{80}else{FULL_WINDOWS} + || body.held_out_windows != if operator{16}else{TRAIN_WINDOWS} + || body.held_out_seals != if operator{4}else{HELD_OUT_EPISODES} + || body.accepted_held_out_commitment.id != descriptor.id + || body.accepted_held_out_commitment.run != body.run + || !digest64(&body.accepted_held_out_commitment.commitment) + || body.accepted_held_out_commitment.revision > if operator{361}else{430} + || body.complete_prepare_receiver_ns < fit.body.freeze_prepare_receiver_ns + || body.complete_prepare_receiver_ns > now + || now / 1_000_000 >= descriptor.expires_at_ms + || body.source_floors.len() != fit.body.source_floors.len() + || !digest64(&self.sha256) + || self.sha256 != hash(&(domain, body))? + { + return Err("temporal_full_complete_data_binding".into()); + } + let mut seen = std::collections::HashSet::new(); + for (source, sequence, time) in &body.source_floors { + if !seen.insert(*source) + || *sequence == 0 + || *sequence > u64::from(u32::MAX) + || *time > body.complete_prepare_receiver_ns + || !fit + .body + .source_floors + .iter() + .any(|(id, before_sequence, before_time)| { + id == source && sequence >= before_sequence && time >= before_time + }) + { + return Err("temporal_full_complete_floor_regressed".into()); + } + } + Ok(()) + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_live_driver.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_live_driver.rs new file mode 100644 index 0000000000..7408618cb8 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_live_driver.rs @@ -0,0 +1,527 @@ +//! Default-inactive, bounded continuous INPUT diagnostic, not a runtime/selected +//! detector. Every window uses the same TemporalModel opaque builder/infer path. +use super::{ + model::RoomPrediction, + native::CanonicalPacket, + observation::MAX_AGE_NS, + temporal_acquisition::{WindowInputPolicy, WindowAdmission}, + temporal_model_input::{TemporalLiveInput, TemporalModel}, +}; +use serde::Serialize; +use sha2::{Digest, Sha256}; +use std::collections::HashMap; +#[derive(Debug, Clone, Copy, Serialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum LiveState { + Inactive, + Warming, + WindowComplete, + Withdrawn, +} +#[derive(Debug, Clone, Serialize)] +pub struct WindowTiming { + pub ordinal: u64, + pub target_ns: String, + pub completed_ns: String, + pub receiver_latency_ns: String, + pub min_observed_span_ns: String, + pub max_observed_span_ns: String, + pub interval_since_previous_ns: Option, + pub min_source_rows: usize, + pub max_source_rows: usize, +} +#[derive(Debug, Clone, Copy, Serialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum ObservationAvailability { + Unavailable, + Available, +} +#[derive(Debug, Clone, Serialize)] +pub struct ObservationStatus { + pub availability: ObservationAvailability, + pub reason: Option<&'static str>, + pub digest: Option, + pub first_receiver_ns: Option, + pub completed_ns: Option, + pub expires_ns: Option, + pub age_ns: Option, + pub source_count: usize, +} +impl ObservationStatus { + fn unavailable(reason: &'static str) -> Self { + Self { + availability: ObservationAvailability::Unavailable, + reason: Some(reason), + digest: None, + first_receiver_ns: None, + completed_ns: None, + expires_ns: None, + age_ns: None, + source_count: 0, + } + } +} +#[derive(Clone, Serialize)] +struct SourceBounds { + source_id: u8, + first_ns: u64, + last_ns: u64, +} +struct CompletedObservation { + prediction: RoomPrediction, + model_sha256: String, + policy_sha256: String, + context_digest: String, + source_bounds: Vec, + first_ns: u64, + completed_ns: u64, + expires_ns: u64, + digest: String, +} +#[derive(Debug, Clone, Serialize)] +pub struct LiveDiagnostic { + /// Buffer progress only; observation availability is separate. + pub buffer_state: LiveState, + pub reason: Option<&'static str>, + pub model_sha256: String, + pub policy_sha256: String, + pub target_horizon_ms: u64, + pub cadence: &'static str, + pub buffer_sources_with_rows: usize, + pub required_sources: usize, + pub observation: ObservationStatus, + pub current_prediction: Option, + pub last_completed: Option, + pub diagnostic_only: bool, + pub physically_validated: bool, + pub adoption_authorized: bool, +} +struct Fence { + sequence: u32, + time_ns: u64, + frame_id: u128, +} +#[derive(Default)] +struct WindowCount { + first_ns: u64, + last_ns: u64, + rows: usize, +} +/// No data-set retention, training schedule cursor, new persistence or clocks. +/// At most one <=4s opaque buffer, <=8 source counters/fences and one aggregate +/// completion are retained. Error withdrawal preserves accepted replay fences. +pub struct ContinuousTemporalInput { + model: TemporalModel, + policy: WindowInputPolicy, + status: LiveDiagnostic, + active: bool, + input: Option, + target_ns: Option, + end_ns: Option, + ordinal: u64, + fences: HashMap, + continuity: HashMap, + counts: HashMap, + last_now: Option, + observation: Option, + #[cfg(test)] last_input_error:Option, +} +impl ContinuousTemporalInput { + pub fn new(model: TemporalModel) -> Result { + let policy = model.window_input_policy()?; + let bytes = model.export()?; + let status = LiveDiagnostic { + buffer_state: LiveState::Inactive, + reason: Some("inactive"), + model_sha256: format!("{:x}", Sha256::digest(bytes)), + policy_sha256: model.plan_sha256().into(), + target_horizon_ms: policy.target_span_ms(), + cadence: "nonoverlap_full_native_horizon", + buffer_sources_with_rows: 0, + required_sources: policy.context().sources.len(), + observation: ObservationStatus::unavailable("inactive"), + current_prediction: None, + last_completed: None, + diagnostic_only: true, + physically_validated: false, + adoption_authorized: false, + }; + Ok(Self { + model, + policy, + status, + active: false, + input: None, + target_ns: None, + end_ns: None, + ordinal: 0, + fences: HashMap::new(), + continuity: HashMap::new(), + counts: HashMap::new(), + last_now: None, + observation: None, + #[cfg(test)] last_input_error:None, + }) + } + pub fn activate(&mut self) { + self.active = true; + self.withdraw("collecting_full_horizon"); + self.status.buffer_state = LiveState::Warming; + } + pub fn deactivate(&mut self) { + self.active = false; + self.withdraw("inactive"); + self.status.buffer_state = LiveState::Inactive; + } + fn withdraw(&mut self, reason: &'static str) { + self.input = None; + self.target_ns = None; + self.end_ns = None; + self.counts.clear(); + self.continuity.clear(); + self.observation = None; + self.status.observation = ObservationStatus::unavailable(reason); + self.status.current_prediction = None; + self.status.buffer_sources_with_rows = 0; + self.status.buffer_state = if self.active { + LiveState::Withdrawn + } else { + LiveState::Inactive + }; + self.status.reason = Some(reason); + } + fn refresh_observation(&mut self, now_ns: u64) { + let Some(observation) = &self.observation else { + self.status.current_prediction = None; + return; + }; + if !self.active + || observation.model_sha256 != self.status.model_sha256 + || observation.policy_sha256 != self.status.policy_sha256 + || observation.context_digest != self.policy.context().digest() + { + self.withdraw("observation_identity_changed"); + return; + } + // Required sources, not just whichever source most recently delivered. + if self.policy.source_policies().iter().any(|p| { + self.continuity + .get(&p.source_id) + .is_none_or(|last| now_ns < *last || now_ns - *last > p.max_gap_ns) + }) { + self.withdraw("source_gap"); + return; + } + let Some(age) = now_ns.checked_sub(observation.first_ns) else { + self.withdraw("receiver_clock_reversed"); + return; + }; + let available = now_ns <= observation.expires_ns; + self.status.current_prediction = if available { + Some(observation.prediction.clone()) + } else { + None + }; + self.status.observation = ObservationStatus { + availability: if available { + ObservationAvailability::Available + } else { + ObservationAvailability::Unavailable + }, + reason: if available { + None + } else { + Some("original_first_row_expired") + }, + digest: Some(observation.digest.clone()), + first_receiver_ns: Some(observation.first_ns.to_string()), + completed_ns: Some(observation.completed_ns.to_string()), + expires_ns: Some(observation.expires_ns.to_string()), + age_ns: Some(age.to_string()), + source_count: observation.source_bounds.len(), + }; + } + /// Explicit receiver-domain tick; a silent source cannot leave a last result + /// looking current. No missing input or failure is converted into absence. + pub fn poll(&mut self, now_ns: u64) -> LiveDiagnostic { + if !self.active { + return self.status.clone(); + } + if self.last_now.is_some_and(|t| now_ns < t) { + self.withdraw("receiver_clock_reversed"); + return self.status.clone(); + } + self.last_now = Some(now_ns); + if self.continuity.iter().any(|(id, last)| { + self.policy + .source_policies() + .iter() + .find(|p| p.source_id == *id) + .is_none_or(|p| now_ns.saturating_sub(*last) > p.max_gap_ns) + }) { + self.withdraw("source_gap"); + } else if self.target_ns.is_some_and(|target| { + self.counts.len() < self.policy.context().sources.len() + && now_ns.saturating_sub(target) + > self.policy.max_first_packet_delay_ms() * 1_000_000 + }) { + self.withdraw("missing_source_warmup"); + } else if self.end_ns.is_some_and(|end| now_ns >= end) { + self.withdraw("incomplete_horizon"); + } + self.refresh_observation(now_ns); + self.status.clone() + } + pub fn ingest(&mut self, p: &CanonicalPacket, now_ns: u64) -> LiveDiagnostic { + if !self.active { + return self.status.clone(); + } + if self.last_now.is_some_and(|t| now_ns < t) { + self.withdraw("receiver_clock_reversed"); + return self.status.clone(); + } + self.last_now = Some(now_ns); + let time = p.native_frame.timestamp_ns; + if time > now_ns || now_ns - time > MAX_AGE_NS { + self.withdraw("stale_or_future_packet"); + return self.status.clone(); + } + let Some(rule) = self + .policy + .source_policies() + .iter() + .find(|s| s.source_id == p.source_id) + else { + self.withdraw("unbound_source"); + return self.status.clone(); + }; + let max_gap = rule.max_gap_ns; + if self.fences.get(&p.source_id).is_some_and(|f| { + p.sequence <= f.sequence || time <= f.time_ns || p.native_frame.frame_id == f.frame_id + }) { + self.withdraw("replayed_or_reversed_packet"); + return self.status.clone(); + } + if self + .continuity + .get(&p.source_id) + .is_some_and(|last| time.saturating_sub(*last) > max_gap) + { + self.withdraw("source_gap"); + return self.status.clone(); + } + // Also check every OTHER source against the receiver tick. A busy source + // cannot hide a dropout while continuing to fill its own window. + if self.continuity.iter().any(|(id, last)| { + self.policy + .source_policies() + .iter() + .find(|s| s.source_id == *id) + .is_none_or(|s| now_ns.saturating_sub(*last) > s.max_gap_ns) + }) { + self.withdraw("source_gap"); + return self.status.clone(); + } + if self.input.is_none() { + let end = match time.checked_add( + self.policy.max_observed_span_ms() * 1_000_000 + + self.policy.max_first_packet_delay_ms() * 1_000_000 + + 1, + ) { + Some(end) => end, + None => { + self.withdraw("receiver_time_overflow"); + return self.status.clone(); + } + }; + let mut input = match self + .model + .live_input(format!("continuous-live-{}", self.ordinal)) + { + Ok(input) => input, + Err(_) => { + self.withdraw("model_input_unavailable"); + return self.status.clone(); + } + }; + if input.begin_window(time, end).is_err() { + self.withdraw("window_start_refused"); + return self.status.clone(); + } + self.input = Some(input); + self.target_ns = Some(time); + self.end_ns = Some(end); + self.counts.clear(); + } + self.status.buffer_state = LiveState::Warming; + self.status.reason = Some("collecting_full_horizon"); + let admission = match self.input.as_mut().unwrap().ingest(p) { + Ok(v) => v, + Err(error) => { + #[cfg(test)]{self.last_input_error=Some(error.clone());} + #[cfg(not(test))]let _=error; + self.withdraw("source_or_window_invalid"); + return self.status.clone(); + } + }; + self.fences.insert( + p.source_id, + Fence { + sequence: p.sequence, + time_ns: time, + frame_id: p.native_frame.frame_id, + }, + ); + self.continuity.insert(p.source_id, time); + let count = self.counts.entry(p.source_id).or_default(); + if count.rows == 0 { + count.first_ns = time; + } + count.last_ns = time; + count.rows += 1; + self.status.buffer_sources_with_rows = self.counts.len(); + if admission == WindowAdmission::Ready { + let target = self.target_ns.unwrap(); + let input = self.input.take().unwrap(); + let observation = match input.finish(now_ns) { + Ok(v) => v, + Err(_) => { + self.withdraw("window_evidence_invalid"); + return self.status.clone(); + } + }; + let prediction = match self.model.predict(observation, now_ns) { + Ok(v) => v, + Err(_) => { + self.withdraw("prediction_input_refused"); + return self.status.clone(); + } + }; + if prediction.physically_validated { + self.withdraw("unexpected_authority"); + return self.status.clone(); + } + let timing = WindowTiming { + ordinal: self.ordinal, + target_ns: target.to_string(), + completed_ns: now_ns.to_string(), + receiver_latency_ns: (now_ns - target).to_string(), + min_observed_span_ns: self + .counts + .values() + .map(|c| c.last_ns - c.first_ns) + .min() + .unwrap() + .to_string(), + max_observed_span_ns: self + .counts + .values() + .map(|c| c.last_ns - c.first_ns) + .max() + .unwrap() + .to_string(), + interval_since_previous_ns: self + .status + .last_completed + .as_ref() + .and_then(|w| w.completed_ns.parse::().ok()) + .map(|last| now_ns.saturating_sub(last).to_string()), + min_source_rows: self.counts.values().map(|c| c.rows).min().unwrap(), + max_source_rows: self.counts.values().map(|c| c.rows).max().unwrap(), + }; + let Some(next) = self.ordinal.checked_add(1) else { + self.withdraw("window_counter_exhausted"); + return self.status.clone(); + }; + let mut source_bounds = Vec::with_capacity(self.policy.context().sources.len()); + let mut first = u64::MAX; + let mut expires = u64::MAX; + for spec in &self.policy.context().sources { + let count = self.counts.get(&spec.source_id).unwrap(); + let Some(deadline) = count.first_ns.checked_add(MAX_AGE_NS) else { + self.withdraw("observation_time_overflow"); + return self.status.clone(); + }; + first = first.min(count.first_ns); + expires = expires.min(deadline); + source_bounds.push(SourceBounds { + source_id: spec.source_id, + first_ns: count.first_ns, + last_ns: count.last_ns, + }); + } + let encoded = match serde_json::to_vec(&( + &self.status.model_sha256, + &self.status.policy_sha256, + &source_bounds, + &timing, + &prediction, + )) { + Ok(v) => v, + Err(_) => { + self.withdraw("observation_encoding_failed"); + return self.status.clone(); + } + }; + self.observation = Some(CompletedObservation { + prediction, + model_sha256: self.status.model_sha256.clone(), + policy_sha256: self.status.policy_sha256.clone(), + context_digest: self.policy.context().digest(), + source_bounds, + first_ns: first, + completed_ns: now_ns, + expires_ns: expires, + digest: format!("{:x}", Sha256::digest(encoded)), + }); + self.ordinal = next; + self.target_ns = None; + self.end_ns = None; + self.counts.clear(); + self.status.buffer_state = LiveState::WindowComplete; + self.status.reason = None; + self.status.last_completed = Some(timing); + } + self.refresh_observation(now_ns); + self.status.clone() + } +} +#[cfg(test)] +#[path = "temporal_live_driver_tests.rs"] +mod tests; + +/// Bounded current observation projection. This never changes inference time. +pub(crate) struct SelectedTemporalObservation { + pub prediction:RoomPrediction,pub digest:String,pub first_ns:u64,pub end_ns:u64,pub expires_ns:u64,pub source_latest_ns:u64, +} +impl ContinuousTemporalInput { + pub(crate) fn from_selected(selected:&crate::room_learner::artifact::StoredTemporalSelection)->Result{ + let mut driver=Self::new(selected.model().clone())?; + let held=selected.held();held.validate_data()?; + for floor in &held.body.final_source_fences { + driver.fences.insert(floor.source_id,Fence{sequence:floor.sequence.try_into().map_err(|_|"selected_sequence_overflow")?,time_ns:floor.receiver_ns.parse().map_err(|_|"selected_receiver_clock")?,frame_id:floor.native_frame_id.parse().map_err(|_|"selected_native_frame")?}); + } + driver.last_now=Some(held.body.complete_persisted_receiver_ns.parse().map_err(|_|"selected_receiver_clock")?); + driver.activate();Ok(driver) + } + pub(crate) fn advance_selected_floors(&mut self,floors:&[(u8,u64,u64)],receiver_floor:u64)->Result<(),String>{ + if floors.len()!=3||floors.iter().any(|(id,seq,_)|*seq>u32::MAX as u64||!self.policy.context().sources.iter().any(|s|s.source_id==*id)){return Err("selected_source_floor_identity".into());} + for(id,seq,time)in floors{let frame=(u128::from(*time)<<40)|(u128::from(*id)<<32)|u128::from(*seq);if let Some(old)=self.fences.get(id){if *seqResult{ + self.poll(now); + if self.status.observation.availability!=ObservationAvailability::Available{return Err(self.status.observation.reason.unwrap_or("selected_native_input_unavailable").into());} + let o=self.observation.as_ref().ok_or("selected_native_input_unavailable")?; + if o.prediction.decision==super::model::PresenceDecision::Abstain{return Err("selected_model_abstained".into());} + let source_latest_ns=self.policy.context().sources.iter().map(|s|self.continuity.get(&s.source_id).copied().ok_or("selected_source_missing")).collect::,_>>()?.into_iter().min().ok_or("selected_source_missing")?; + if now1_000_000_000{return Err("selected_source_stale".into());} + Ok(SelectedTemporalObservation{prediction:o.prediction.clone(),digest:o.digest.clone(),first_ns:o.first_ns,end_ns:o.source_bounds.iter().map(|s|s.last_ns).max().ok_or("selected_source_missing")?,expires_ns:o.expires_ns,source_latest_ns}) + } +} + +#[cfg(test)] +impl ContinuousTemporalInput{pub(crate) fn c4c_replay_floor_probe(&self)->Vec<(u8,u32,u64,u128)>{let mut out=self.fences.iter().map(|(id,f)|(*id,f.sequence,f.time_ns,f.frame_id)).collect::>();out.sort_by_key(|v|v.0);out}} + +#[cfg(test)]impl ContinuousTemporalInput{pub(crate) fn c4c_last_input_failure(&self)->Option<&str>{self.last_input_error.as_deref()}} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_live_driver_prior_control_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_live_driver_prior_control_tests.rs new file mode 100644 index 0000000000..bac73f92f6 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_live_driver_prior_control_tests.rs @@ -0,0 +1,337 @@ +//! Default-inactive, bounded continuous INPUT diagnostic, not a runtime/selected +//! detector. Every window uses the same TemporalModel opaque builder/infer path. +use super::{ + model::RoomPrediction, + native::CanonicalPacket, + observation::MAX_AGE_NS, + temporal_acquisition::{AcquisitionPlan, WindowAdmission}, + temporal_model_input::{TemporalLiveInput, TemporalModel}, +}; +use serde::Serialize; +use sha2::{Digest, Sha256}; +use std::collections::HashMap; +#[derive(Debug, Clone, Copy, Serialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum LiveState { + Inactive, + Warming, + Current, + Withdrawn, +} +#[derive(Debug, Clone, Serialize)] +pub struct WindowTiming { + pub ordinal: u64, + pub target_ns: String, + pub completed_ns: String, + pub receiver_latency_ns: String, + pub min_observed_span_ns: String, + pub max_observed_span_ns: String, + pub interval_since_previous_ns: Option, + pub min_source_rows: usize, + pub max_source_rows: usize, +} +#[derive(Debug, Clone, Serialize)] +pub struct LiveDiagnostic { + pub state: LiveState, + pub reason: Option<&'static str>, + pub model_sha256: String, + pub policy_sha256: String, + pub target_horizon_ms: u64, + pub cadence: &'static str, + pub ready_sources: usize, + pub required_sources: usize, + pub current_prediction: Option, + pub last_completed: Option, + pub diagnostic_only: bool, + pub physically_validated: bool, + pub adoption_authorized: bool, +} +struct Fence { + sequence: u32, + time_ns: u64, + frame_id: u128, +} +#[derive(Default)] +struct WindowCount { + first_ns: u64, + last_ns: u64, + rows: usize, +} +/// No data-set retention, training schedule cursor, new persistence or clocks. +/// At most one <=4s opaque buffer, <=8 source counters/fences and one aggregate +/// completion are retained. Error withdrawal preserves accepted replay fences. +pub struct ContinuousTemporalInput { + model: TemporalModel, + policy: AcquisitionPlan, + status: LiveDiagnostic, + active: bool, + input: Option, + target_ns: Option, + end_ns: Option, + ordinal: u64, + fences: HashMap, + continuity: HashMap, + counts: HashMap, + last_now: Option, +} +impl ContinuousTemporalInput { + pub fn new(model: TemporalModel) -> Result { + let policy = model.input_policy()?; + let bytes = model.export()?; + let status = LiveDiagnostic { + state: LiveState::Inactive, + reason: Some("inactive"), + model_sha256: format!("{:x}", Sha256::digest(bytes)), + policy_sha256: model.plan_sha256().into(), + target_horizon_ms: policy.target_span_ms(), + cadence: "nonoverlap_full_native_horizon", + ready_sources: 0, + required_sources: policy.context().sources.len(), + current_prediction: None, + last_completed: None, + diagnostic_only: true, + physically_validated: false, + adoption_authorized: false, + }; + Ok(Self { + model, + policy, + status, + active: false, + input: None, + target_ns: None, + end_ns: None, + ordinal: 0, + fences: HashMap::new(), + continuity: HashMap::new(), + counts: HashMap::new(), + last_now: None, + }) + } + pub fn activate(&mut self) { + self.active = true; + self.withdraw("collecting_full_horizon"); + self.status.state = LiveState::Warming; + } + pub fn deactivate(&mut self) { + self.active = false; + self.withdraw("inactive"); + self.status.state = LiveState::Inactive; + } + fn withdraw(&mut self, reason: &'static str) { + self.input = None; + self.target_ns = None; + self.end_ns = None; + self.counts.clear(); + self.continuity.clear(); + self.status.current_prediction = None; + self.status.ready_sources = 0; + self.status.state = if self.active { + LiveState::Withdrawn + } else { + LiveState::Inactive + }; + self.status.reason = Some(reason); + } + /// Explicit receiver-domain tick; a silent source cannot leave a last result + /// looking current. No missing input or failure is converted into absence. + pub fn poll(&mut self, now_ns: u64) -> LiveDiagnostic { + if !self.active { + return self.status.clone(); + } + if self.last_now.is_some_and(|t| now_ns < t) { + self.withdraw("receiver_clock_reversed"); + return self.status.clone(); + } + self.last_now = Some(now_ns); + if self.continuity.iter().any(|(id, last)| { + self.policy + .source_policies() + .iter() + .find(|p| p.source_id == *id) + .is_none_or(|p| now_ns.saturating_sub(*last) > p.max_gap_ns) + }) { + self.withdraw("source_gap"); + } else if self.target_ns.is_some_and(|target| { + self.counts.len() < self.policy.context().sources.len() + && now_ns.saturating_sub(target) + > self.policy.max_first_packet_delay_ms() * 1_000_000 + }) { + self.withdraw("missing_source_warmup"); + } else if self.end_ns.is_some_and(|end| now_ns >= end) { + self.withdraw("incomplete_horizon"); + } + self.status.clone() + } + pub fn ingest(&mut self, p: &CanonicalPacket, now_ns: u64) -> LiveDiagnostic { + if !self.active { + return self.status.clone(); + } + if self.last_now.is_some_and(|t| now_ns < t) { + self.withdraw("receiver_clock_reversed"); + return self.status.clone(); + } + self.last_now = Some(now_ns); + let time = p.native_frame.timestamp_ns; + if time > now_ns || now_ns - time > MAX_AGE_NS { + self.withdraw("stale_or_future_packet"); + return self.status.clone(); + } + let Some(rule) = self + .policy + .source_policies() + .iter() + .find(|s| s.source_id == p.source_id) + else { + self.withdraw("unbound_source"); + return self.status.clone(); + }; + let max_gap = rule.max_gap_ns; + if self.fences.get(&p.source_id).is_some_and(|f| { + p.sequence <= f.sequence || time <= f.time_ns || p.native_frame.frame_id == f.frame_id + }) { + self.withdraw("replayed_or_reversed_packet"); + return self.status.clone(); + } + if self + .continuity + .get(&p.source_id) + .is_some_and(|last| time.saturating_sub(*last) > max_gap) + { + self.withdraw("source_gap"); + return self.status.clone(); + } + // Also check every OTHER source against the receiver tick. A busy source + // cannot hide a dropout while continuing to fill its own window. + if self.continuity.iter().any(|(id, last)| { + self.policy + .source_policies() + .iter() + .find(|s| s.source_id == *id) + .is_none_or(|s| now_ns.saturating_sub(*last) > s.max_gap_ns) + }) { + self.withdraw("source_gap"); + return self.status.clone(); + } + if self.input.is_none() { + let end = match time.checked_add( + self.policy.max_observed_span_ms() * 1_000_000 + + self.policy.max_first_packet_delay_ms() * 1_000_000 + + 1, + ) { + Some(end) => end, + None => { + self.withdraw("receiver_time_overflow"); + return self.status.clone(); + } + }; + let mut input = match self + .model + .live_input(format!("continuous-live-{}", self.ordinal)) + { + Ok(input) => input, + Err(_) => { + self.withdraw("model_input_unavailable"); + return self.status.clone(); + } + }; + if input.begin_window(time, end).is_err() { + self.withdraw("window_start_refused"); + return self.status.clone(); + } + self.input = Some(input); + self.target_ns = Some(time); + self.end_ns = Some(end); + self.counts.clear(); + } + self.status.state = LiveState::Warming; + self.status.reason = Some("collecting_full_horizon"); + self.status.current_prediction = None; + let admission = match self.input.as_mut().unwrap().ingest(p) { + Ok(v) => v, + Err(_) => { + self.withdraw("source_or_window_invalid"); + return self.status.clone(); + } + }; + self.fences.insert( + p.source_id, + Fence { + sequence: p.sequence, + time_ns: time, + frame_id: p.native_frame.frame_id, + }, + ); + self.continuity.insert(p.source_id, time); + let count = self.counts.entry(p.source_id).or_default(); + if count.rows == 0 { + count.first_ns = time; + } + count.last_ns = time; + count.rows += 1; + self.status.ready_sources = self.counts.len(); + if admission == WindowAdmission::Ready { + let target = self.target_ns.unwrap(); + let input = self.input.take().unwrap(); + let observation = match input.finish(now_ns) { + Ok(v) => v, + Err(_) => { + self.withdraw("window_evidence_invalid"); + return self.status.clone(); + } + }; + let prediction = match self.model.predict(observation, now_ns) { + Ok(v) => v, + Err(_) => { + self.withdraw("prediction_input_refused"); + return self.status.clone(); + } + }; + if prediction.physically_validated { + self.withdraw("unexpected_authority"); + return self.status.clone(); + } + let timing = WindowTiming { + ordinal: self.ordinal, + target_ns: target.to_string(), + completed_ns: now_ns.to_string(), + receiver_latency_ns: (now_ns - target).to_string(), + min_observed_span_ns: self + .counts + .values() + .map(|c| c.last_ns - c.first_ns) + .min() + .unwrap() + .to_string(), + max_observed_span_ns: self + .counts + .values() + .map(|c| c.last_ns - c.first_ns) + .max() + .unwrap() + .to_string(), + interval_since_previous_ns: self + .status + .last_completed + .as_ref() + .and_then(|w| w.completed_ns.parse::().ok()) + .map(|last| now_ns.saturating_sub(last).to_string()), + min_source_rows: self.counts.values().map(|c| c.rows).min().unwrap(), + max_source_rows: self.counts.values().map(|c| c.rows).max().unwrap(), + }; + let Some(next) = self.ordinal.checked_add(1) else { + self.withdraw("window_counter_exhausted"); + return self.status.clone(); + }; + self.ordinal = next; + self.target_ns = None; + self.end_ns = None; + self.counts.clear(); + self.status.state = LiveState::Current; + self.status.reason = None; + self.status.current_prediction = Some(prediction); + self.status.last_completed = Some(timing); + } + self.status.clone() + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_live_driver_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_live_driver_tests.rs new file mode 100644 index 0000000000..33e3fae162 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_live_driver_tests.rs @@ -0,0 +1,646 @@ +//! SYNTHETIC native sources; actual fitter/store/cold loader/opaque infer path. +use crate::room_learner::temporal_acquisition::AcquisitionPlan; +use super::*; +use crate::room_learner::{ + artifact::ArtifactStore, + native::{canonicalize_esp32, NativePacketInput}, + observation::*, + temporal_acquisition::{ExperimentalPlanId, ReceiverCadenceBand}, + temporal_model_input::TemporalTrainingSession, + training::FitConfig, +}; +use std::sync::OnceLock; +use wifi_densepose_hardware::{Adr018Flags, PpduType}; +const PERIOD: u64 = 30_303_031; +const FRESH: u64 = 1_000_000_000_000; +fn context() -> CaptureContext { + CaptureContext { + owner_scope: "a".repeat(64), + installation_id: "synthetic-continuous".into(), + room_digest: "synthetic-room".into(), + revision: 1, + sources: (1..=2) + .map(|source_id| SourceSpec { + source_id, + device_id: format!("esp32-node-{source_id}"), + grid_id: "ant1-bins64-ppdu0-usable-drop2-linear-v2".into(), + feature_count: 56, + signal_kind: ruview_unified::RfModality::WifiCsi, + center_frequency_hz: 2_437_000_000, + bandwidth_hz: 20_000_000, + link_id: format!("native-{source_id}"), + link_index: 0, + geometry: None, + }) + .collect(), + } +} +fn packet(source: u8, sequence: u32, time: u64, label: RoomLabel, row: usize) -> CanonicalPacket { + let gain = match label { + RoomLabel::Empty => 3, + RoomLabel::AdjacentOnly => 4, + RoomLabel::OccupiedStatic => 8, + RoomLabel::OccupiedMoving => 10 + (row % 2) as u8, + RoomLabel::Unknown => 8, + }; + let bytes = (0..64) + .flat_map(|bin| [gain + source + (bin % 2) as u8, 4]) + .collect::>(); + canonicalize_esp32(NativePacketInput { + node_id: source, + n_antennas: 1, + raw_bins: 64, + frequency_hz: 2_437_000_000, + ppdu_type: PpduType::HtLegacy, + sequence, + received_at_ns: time, + receiver_sample_rate_hz: 33., + iq_bytes: &bytes, + flags: Adr018Flags::default(), + provenance: CaptureProvenance::Synthetic, + }) + .unwrap() +} +fn fitted() -> TemporalModel { + fitted_id(ExperimentalPlanId::Native2000msSlots2) +} +fn fitted_id(id: ExperimentalPlanId) -> TemporalModel { + static SHORT: OnceLock> = OnceLock::new(); + static LONG: OnceLock> = OnceLock::new(); + let bytes = if id == ExperimentalPlanId::Native2000msSlots2 { + &SHORT + } else { + &LONG + }; + TemporalModel::import(bytes.get_or_init(|| { + let mut s = TemporalTrainingSession::new( + context(), + id, + &[ + (1, ReceiverCadenceBand::Hz30To36), + (2, ReceiverCadenceBand::Hz30To36), + ], + "synthetic-continuous-fit".into(), + FitConfig { + epochs: 2, + ..Default::default() + }, + ) + .unwrap(); + let plan = AcquisitionPlan::build( + context(), + id, + &[ + (1, ReceiverCadenceBand::Hz30To36), + (2, ReceiverCadenceBand::Hz30To36), + ], + ) + .unwrap(); + let mut sequence = 0; + for index in 0..16 { + let at = 10_000_000_000 + index * 5_000_000_000; + let label = plan.schedule()[index as usize / 2].label; + s.begin_window(at, at + 5_000_000_000).unwrap(); + 'window: for row in 0..256 { + sequence += 1; + for source in 1..=2 { + let now = at + row as u64 * PERIOD + u64::from(source) * 1000; + if s.ingest(&packet(source, sequence, now, label, row)) + .unwrap() + == WindowAdmission::Ready + { + s.finish_window(now).unwrap(); + break 'window; + } + } + } + } + let (model, report) = s.fit().unwrap(); + assert!(report.backbone_delta_l2 > 0.); + assert_eq!(report.training_cohorts, 16); + model.export().unwrap() + })) + .unwrap() +} +fn feed( + d: &mut ContinuousTemporalInput, + start: u64, + rows: usize, + sequence: u32, +) -> Vec { + let mut completed = vec![]; + for row in 0..rows { + for source in 1..=2 { + let now = start + row as u64 * PERIOD + u64::from(source) * 1000; + let status = d.ingest( + &packet(source, sequence + row as u32, now, RoomLabel::Unknown, row), + now, + ); + if status.buffer_state == LiveState::WindowComplete { + completed.push(status); + } + } + } + completed +} +fn driver() -> ContinuousTemporalInput { + ContinuousTemporalInput::new(fitted()).unwrap() +} +#[test] +fn default_inactive_and_short_inputs_never_produce_absence_or_prediction() { + let mut d = driver(); + let p = packet(1, 100_000, FRESH, RoomLabel::Unknown, 0); + assert_eq!(d.ingest(&p, FRESH).buffer_state, LiveState::Inactive); + assert!(d.fences.is_empty()); + d.activate(); + assert!(feed(&mut d, FRESH, 8, 100_000).is_empty()); + let status = d.poll(FRESH + 7 * PERIOD + 2000); + assert_eq!(status.buffer_state, LiveState::Warming); + assert!(status.current_prediction.is_none()); + assert!(status.diagnostic_only && !status.physically_validated && !status.adoption_authorized); +} +#[test] +#[cfg(any(target_os = "macos", target_os = "linux"))] +fn actual_fit_store_cold_load_then_continuous_windows_use_the_same_infer_path() { + let dir = tempfile::tempdir().unwrap(); + let ctx = context(); + let store = ArtifactStore::new( + &dir.path().join(&ctx.owner_scope).join("SensingServer"), + &ctx.owner_scope, + &ctx.installation_id, + ) + .unwrap(); + let model = fitted(); + let receipt = store.save_temporal_candidate(&model).unwrap(); + let cold = store + .load_exact_temporal_candidate(&ctx, &receipt.artifact_sha256) + .unwrap(); + assert_eq!(cold.export().unwrap(), model.export().unwrap()); + let mut d = ContinuousTemporalInput::new(cold).unwrap(); + assert_eq!(d.status.model_sha256, receipt.artifact_sha256); + d.activate(); + let results = feed(&mut d, FRESH, 201, 100_000); + assert_eq!(results.len(), 3); + assert_eq!(d.ordinal, 3); + assert_eq!(d.fences.len(), 2); + assert!(d.input.is_none()); + assert!(d.counts.is_empty()); + for (i, status) in results.iter().enumerate() { + let t = status.last_completed.as_ref().unwrap(); + assert_eq!(t.ordinal, i as u64); + assert!(t.min_observed_span_ns.parse::().unwrap() >= 2_000_000_000); + assert_eq!(t.min_source_rows, 67); + assert_eq!(t.max_source_rows, 67); + assert!(status + .current_prediction + .as_ref() + .is_some_and(|p| !p.physically_validated)); + } + // Independently call the SAME opaque builder and model infer on the exact + // first source interval, without access to the inner v1 model. + let mut input = model.live_input("comparison-window".into()).unwrap(); + input + .begin_window(FRESH + 1000, FRESH + 3_000_000_000) + .unwrap(); + let mut now = FRESH; + 'ready: for row in 0..67 { + for source in 1..=2 { + now = FRESH + row as u64 * PERIOD + u64::from(source) * 1000; + if input + .ingest(&packet( + source, + 100_000 + row as u32, + now, + RoomLabel::Unknown, + row, + )) + .unwrap() + == WindowAdmission::Ready + { + break 'ready; + } + } + } + let expected = model.predict(input.finish(now).unwrap(), now).unwrap(); + assert_eq!( + serde_json::to_value(expected).unwrap(), + serde_json::to_value(results[0].current_prediction.as_ref().unwrap()).unwrap() + ); + assert!( + results[1] + .last_completed + .as_ref() + .unwrap() + .interval_since_previous_ns + .as_ref() + .unwrap() + .parse::() + .unwrap() + >= 2_000_000_000 + ); + let next = FRESH + 201 * PERIOD + 1000; + let status = d.ingest(&packet(1, 100_201, next, RoomLabel::Unknown, 201), next); + assert_eq!(status.buffer_state, LiveState::Warming); + assert!(status.current_prediction.is_some()); + assert_eq!( + status.observation.availability, + ObservationAvailability::Available + ); + assert_eq!(status.last_completed.as_ref().unwrap().ordinal, 2); +} +#[test] +fn replay_clock_and_sequence_fences_survive_window_rollover_and_reactivation() { + let mut d = driver(); + d.activate(); + assert_eq!(feed(&mut d, FRESH, 67, 100_000).len(), 1); + let now = FRESH + 67 * PERIOD + 1000; + let status = d.ingest(&packet(1, 100_067, now, RoomLabel::Unknown, 67), now); + assert_eq!(status.buffer_state, LiveState::Warming); + let old = packet( + 2, + 100_066, + FRESH + 66 * PERIOD + 2000, + RoomLabel::Unknown, + 66, + ); + let status = d.ingest(&old, now); + assert_eq!(status.buffer_state, LiveState::Withdrawn); + assert_eq!(status.reason, Some("replayed_or_reversed_packet")); + assert!(status.current_prediction.is_none()); + d.deactivate(); + d.activate(); + let status = d.ingest(&old, now + 1); + assert_eq!(status.reason, Some("replayed_or_reversed_packet")); + assert_eq!(d.fences.len(), 2); +} +#[test] +fn source_gap_and_polling_withdraw_without_renewing_source_freshness_then_fresh_warmup_recovers() { + let mut d = driver(); + d.activate(); + feed(&mut d, FRESH, 10, 100_000); + let last = FRESH + 9 * PERIOD + 2000; + let status = d.poll(last + 100_000_000); + assert_eq!(status.buffer_state, LiveState::Withdrawn); + assert_eq!(status.reason, Some("source_gap")); + assert!(status.current_prediction.is_none()); + let fences = d.fences.len(); + d.poll(last + 200_000_000); + assert_eq!(d.fences.len(), fences); + let fresh = last + 300_000_000; + let results = feed(&mut d, fresh, 67, 100_010); + assert_eq!(results.len(), 1); + let at = fresh + 66 * PERIOD + 2000; + let status = d.poll(at + 100_000_000); + assert_eq!(status.buffer_state, LiveState::Withdrawn); + assert!(status.current_prediction.is_none()); + assert_eq!(status.last_completed.as_ref().unwrap().ordinal, 0); +} +#[test] +fn cold_training_fences_invalid_phy_stale_and_reversed_receiver_clock_never_emit_prediction() { + let mut d = driver(); + d.activate(); + let old = packet(1, 1, FRESH + 1000, RoomLabel::Unknown, 0); + let status = d.ingest(&old, FRESH + 1000); + assert_eq!(status.reason, Some("source_or_window_invalid")); + assert!(d.fences.is_empty()); + for kind in ["grid", "frequency", "source", "stale", "clock"] { + let mut d = driver(); + d.activate(); + feed(&mut d, FRESH, 8, 100_000); + let last = FRESH + 7 * PERIOD + 2000; + let mut p = packet(1, 100_008, FRESH + 8 * PERIOD + 1000, RoomLabel::Unknown, 8); + let mut now = p.native_frame.timestamp_ns; + match kind { + "grid" => p.grid_id = "wrong-grid".into(), + "frequency" => p.frequency_hz = 5_180_000_000, + "source" => p.source_id = 99, + "stale" => now += 6_000_000_000, + "clock" => now = last - 1, + _ => unreachable!(), + } + let before = d.fences[&1].sequence; + let status = d.ingest(&p, now); + assert_eq!(status.buffer_state, LiveState::Withdrawn, "{kind}"); + assert!(status.current_prediction.is_none()); + assert_eq!( + d.fences[&1].sequence, before, + "invalid packet advanced fence: {kind}" + ); + } +} + +#[test] +fn repeated_poll_clock_reversal_cannot_lower_the_existing_receiver_floor() { + let mut d = driver(); + d.activate(); + feed(&mut d, FRESH, 8, 100_000); + let floor = FRESH + 7 * PERIOD + 2000; + for now in [floor - 1, floor - 100_000_000, floor - 2] { + let s = d.poll(now); + assert_eq!(s.reason, Some("receiver_clock_reversed")); + assert_eq!(d.last_now, Some(floor)); + assert!(s.current_prediction.is_none()); + } + d.deactivate(); + d.activate(); + assert_eq!(d.poll(floor - 1).reason, Some("receiver_clock_reversed")); + assert_eq!(d.last_now, Some(floor)); + let now = floor + PERIOD; + let s = d.ingest(&packet(1, 100_008, now, RoomLabel::Unknown, 8), now); + assert_eq!(s.buffer_state, LiveState::Warming); + assert!(s.current_prediction.is_none()); +} +#[test] +fn source_dropout_across_a_completed_window_boundary_withdraws_the_busy_source() { + let mut d = driver(); + d.activate(); + assert_eq!(feed(&mut d, FRESH, 67, 100_000).len(), 1); + let next = FRESH + 67 * PERIOD + 1000; + assert_eq!( + d.ingest(&packet(1, 100_067, next, RoomLabel::Unknown, 67), next) + .buffer_state, + LiveState::Warming + ); + let later = FRESH + 68 * PERIOD + 1000; + let s = d.ingest(&packet(1, 100_068, later, RoomLabel::Unknown, 68), later); + assert_eq!(s.buffer_state, LiveState::Withdrawn); + assert_eq!(s.reason, Some("source_gap")); + assert!(s.current_prediction.is_none()); + assert_eq!(s.last_completed.as_ref().unwrap().ordinal, 0); + assert_eq!(d.fences[&1].sequence, 100_067); + assert_eq!(d.fences[&2].sequence, 100_066); +} +#[test] +fn fresh_timestamps_do_not_authorize_a_counter_reset_after_rollover() { + let mut d = driver(); + d.activate(); + assert_eq!(feed(&mut d, FRESH, 67, 100_000).len(), 1); + let now = FRESH + 67 * PERIOD + 1000; + let reset = packet(1, 100_000, now, RoomLabel::Unknown, 67); + let s = d.ingest(&reset, now); + assert_eq!(s.reason, Some("replayed_or_reversed_packet")); + assert_eq!(s.buffer_state, LiveState::Withdrawn); + assert!(s.current_prediction.is_none()); + assert_eq!(d.fences[&1].sequence, 100_066); +} +#[test] +fn invalid_waveform_values_or_shape_withdraw_for_each_bound_source_without_advancing_fences() { + for source in 1..=2 { + for kind in ["nan", "shape"] { + let mut d = driver(); + d.activate(); + feed(&mut d, FRESH, 8, 100_000); + let now = FRESH + 8 * PERIOD + u64::from(source) * 1000; + let mut bad = packet(source, 100_008, now, RoomLabel::Unknown, 8); + if kind == "nan" { + bad.amplitudes[0] = f64::NAN; + } else { + bad.amplitudes.truncate(8); + } + let before = d.fences[&source].sequence; + let s = d.ingest(&bad, now); + assert_eq!(s.reason, Some("source_or_window_invalid")); + assert!(s.current_prediction.is_none()); + assert_eq!(d.fences[&source].sequence, before); + } + } +} + +#[test] +fn continuous_rollover_exceeds_finite_training_schedule_without_retaining_dataset_rows() { + let mut d = driver(); + let bound = d.model.input_policy().unwrap().resources().cohorts; + assert_eq!(bound, 32); + d.activate(); + let results = feed(&mut d, FRESH, (bound + 2) * 67, 100_000); + assert_eq!(results.len(), bound + 2); + assert_eq!(d.ordinal, (bound + 2) as u64); + assert_eq!(d.fences.len(), 2); + assert_eq!(d.continuity.len(), 2); + assert!(d.counts.is_empty()); + assert!(d.input.is_none()); + assert_eq!( + d.status.last_completed.as_ref().unwrap().ordinal, + (bound + 1) as u64 + ); + assert_eq!( + format!("{:x}", Sha256::digest(d.model.export().unwrap())), + d.status.model_sha256 + ); +} + +// The production portion of the frozen072 driver, verbatim, is test control +// only. It calls the SAME opaque model/infer; no parallel trainer is introduced. +use crate::room_learner::{model, native, observation, temporal_acquisition, temporal_model_input}; +#[allow(dead_code)] +#[path = "temporal_live_driver_prior_control_tests.rs"] +mod prior_control; + +#[test] +fn healthy_two_second_stream_retains_original_observation_during_collection_and_improves_duty_cycle( +) { + let model = fitted(); + let mut d = ContinuousTemporalInput::new(model.clone()).unwrap(); + let mut old = prior_control::ContinuousTemporalInput::new(model).unwrap(); + d.activate(); + old.activate(); + let mut previous = None; + let mut elapsed_new = 0u64; + let mut elapsed_old = 0u64; + let mut first_complete = None; + let mut digest = None; + let mut deadline = None; + for row in 0..201 { + for source in 1..=2 { + let now = FRESH + row as u64 * PERIOD + u64::from(source) * 1000; + if let Some((time, new_available, old_available)) = previous { + if new_available { + elapsed_new += now - time; + } + if old_available { + elapsed_old += now - time; + } + } + let p = packet(source, 100_000 + row as u32, now, RoomLabel::Unknown, row); + let new = d.ingest(&p, now); + let control = old.ingest(&p, now); + if row < 66 { + assert!(new.current_prediction.is_none()); + } + if row == 66 && source == 2 { + first_complete = Some(now); + digest = new.observation.digest.clone(); + deadline = new.observation.expires_ns.clone(); + } + if row == 67 { + assert_eq!(new.buffer_state, LiveState::Warming); + assert_eq!( + new.observation.availability, + ObservationAvailability::Available + ); + assert_eq!(new.observation.digest, digest); + assert_eq!(new.observation.expires_ns, deadline); + assert!(new.current_prediction.is_some()); + } + previous = Some(( + now, + new.current_prediction.is_some(), + control.current_prediction.is_some(), + )); + } + } + let end = FRESH + 200 * PERIOD + 2000; + assert_eq!(elapsed_new, end - first_complete.unwrap()); + assert!(elapsed_old * 20 < elapsed_new); + assert_eq!(d.ordinal, 3); +} +#[test] +fn first_row_deadline_is_fixed_exact_and_expiry_preserves_a_healthy_long_window() { + let model = fitted_id(ExperimentalPlanId::Native3900msMax4000Slots2); + let mut d = ContinuousTemporalInput::new(model).unwrap(); + d.activate(); + let completed = feed(&mut d, FRESH, 130, 100_000); + assert_eq!(completed.len(), 1); + let initial = completed[0].observation.clone(); + let expires = initial.expires_ns.as_ref().unwrap().parse::().unwrap(); + let first = initial + .first_receiver_ns + .as_ref() + .unwrap() + .parse::() + .unwrap(); + assert_eq!(expires, first + MAX_AGE_NS); + // Keep all sources healthy, collecting the next3.9s window, through the old + // result's5s FIRST-row deadline; its LAST-row+5s bound would be much later. + for row in 130..165 { + for source in 1..=2 { + let now = FRESH + row as u64 * PERIOD + u64::from(source) * 1000; + let s = d.ingest( + &packet(source, 100_000 + row as u32, now, RoomLabel::Unknown, row), + now, + ); + assert_eq!(s.observation.digest, initial.digest); + assert_eq!(s.observation.expires_ns, initial.expires_ns); + } + } + assert!(d.input.is_some()); + let count = d.counts.values().map(|c| c.rows).sum::(); + let at = d.poll(expires); + assert_eq!( + at.observation.availability, + ObservationAvailability::Available + ); + assert_eq!(at.observation.age_ns.as_deref(), Some("5000000000")); + let expired = d.poll(expires + 1); + assert_eq!(expired.buffer_state, LiveState::Warming); + assert_eq!( + expired.observation.availability, + ObservationAvailability::Unavailable + ); + assert_eq!( + expired.observation.reason, + Some("original_first_row_expired") + ); + assert!(expired.current_prediction.is_none()); + assert!(d.input.is_some()); + assert_eq!(d.counts.values().map(|c| c.rows).sum::(), count); + assert_eq!(d.ordinal, 1); + for _ in 0..3 { + let same = d.poll(expires + 1); + assert_eq!(same.observation.digest, initial.digest); + assert_eq!(same.observation.expires_ns, initial.expires_ns); + assert!(same.current_prediction.is_none()); + } + let next = FRESH + 165 * PERIOD + 1000; + let after_packet = d.ingest(&packet(1, 100_165, next, RoomLabel::Unknown, 165), next); + assert_eq!(after_packet.observation.digest, initial.digest); + assert_eq!(after_packet.observation.expires_ns, initial.expires_ns); + assert!(after_packet.current_prediction.is_none()); + assert!( + expires + < initial + .completed_ns + .as_ref() + .unwrap() + .parse::() + .unwrap() + + MAX_AGE_NS, + "causal first-vs-last boundary missing" + ); +} +#[test] +fn gap_invalidity_and_deactivation_cannot_resurrect_a_retained_observation() { + for mode in ["gap", "invalid", "deactivate"] { + let mut d = driver(); + d.activate(); + let complete = feed(&mut d, FRESH, 67, 100_000); + assert!(complete[0].current_prediction.is_some()); + let now = FRESH + 67 * PERIOD + 1000; + let s = match mode { + "gap" => d.poll(now + 100_000_000), + "invalid" => { + let mut bad = packet(1, 100_067, now, RoomLabel::Unknown, 67); + bad.amplitudes[0] = f64::NAN; + d.ingest(&bad, now) + } + _ => { + d.deactivate(); + d.poll(now) + } + }; + assert!(s.current_prediction.is_none()); + assert!(s.observation.digest.is_none()); + assert!(d.observation.is_none()); + d.activate(); + let reactivated = d.poll(now + 100_000_001); + assert!(reactivated.current_prediction.is_none()); + assert!(reactivated.observation.digest.is_none()); + } +} + +#[test] +fn repeated_available_reads_age_the_same_observation_without_changing_identity_or_deadline() { + let mut d = driver(); + d.activate(); + let completed = feed(&mut d, FRESH, 67, 100_000); + let initial = completed[0].observation.clone(); + let now = FRESH + 66 * PERIOD + 2000; + for offset in [1, 10, 1000, 1_000_000] { + let status = d.poll(now + offset); + assert_eq!( + status.observation.availability, + ObservationAvailability::Available + ); + assert_eq!(status.observation.digest, initial.digest); + assert_eq!( + status.observation.first_receiver_ns, + initial.first_receiver_ns + ); + assert_eq!(status.observation.completed_ns, initial.completed_ns); + assert_eq!(status.observation.expires_ns, initial.expires_ns); + assert_eq!( + status + .observation + .age_ns + .as_ref() + .unwrap() + .parse::() + .unwrap(), + now + offset + - initial + .first_receiver_ns + .as_ref() + .unwrap() + .parse::() + .unwrap() + ); + assert_eq!(d.ordinal, 1); + assert!(!status.physically_validated && !status.adoption_authorized); + } + // A model replacement is a new inactive owner-scoped driver; it cannot borrow + // a cached result merely because the cold bytes/model policy are identical. + let replacement = ContinuousTemporalInput::new(fitted()).unwrap(); + assert!(replacement.observation.is_none()); + assert!(replacement.status.current_prediction.is_none()); + assert_eq!(replacement.status.buffer_state, LiveState::Inactive); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_model_input.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_model_input.rs new file mode 100644 index 0000000000..a04c0eff16 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_model_input.rs @@ -0,0 +1,878 @@ +//! Experimental input-policy adapter over the EXISTING RoomModel/fitter. +//! No API/runtime/storage/managed-proof/selection authority. Imported metadata +//! is integrity-checked historical input claims, not physical attestation. +use super::{ + model::{EncoderConfig, RoomModel, RoomPrediction}, + native::CanonicalPacket, + observation::*, + temporal_acquisition::{ + AcquisitionPlan, ExperimentalPlanId, NativeWindowAssembler, ReceiverCadenceBand, + TemporalWindow, WindowAdmission, + }, + training::{FitConfig, TrainingReport}, +}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::collections::{HashMap, HashSet}; +pub(crate) const SCHEMA: &str = "ruview.experimental.temporal-model-input.v2"; +const MAX_BYTES: usize = 4 * 1024 * 1024; +fn hash(v: &T) -> Result { + Ok(format!( + "{:x}", + Sha256::digest(serde_json::to_vec(v).map_err(|_| "temporal_encoding")?) + )) +} +fn bytes_hash(v: &[u8]) -> String { + format!("{:x}", Sha256::digest(v)) +} +fn sha(s: &str) -> bool { + s.len() == 64 + && s.bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) +} +fn integer(s: &str) -> Result { + let n = s.parse::().map_err(|_| "temporal_integer")?; + if n.to_string() != s { + return Err("temporal_noncanonical_integer".into()); + } + Ok(n) +} +fn plan_id(s: &str) -> Result { + use ExperimentalPlanId::*; + Ok(match s { + "Native1000msSlots8" => Native1000msSlots8, + "Native1000msSlots4" => Native1000msSlots4, + "Native1000msSlots2" => Native1000msSlots2, + "Native2000msSlots8" => Native2000msSlots8, + "Native2000msSlots4" => Native2000msSlots4, + "Native2000msSlots2" => Native2000msSlots2, + "Native3900msMax4000Slots8" => Native3900msMax4000Slots8, + "Native3900msMax4000Slots4" => Native3900msMax4000Slots4, + "Native3900msMax4000Slots2" => Native3900msMax4000Slots2, + _ => return Err("temporal_plan_id".into()), + }) +} +fn band(s: &str) -> Result { + match s { + "Hz30To36" => Ok(ReceiverCadenceBand::Hz30To36), + "Hz45To55" => Ok(ReceiverCadenceBand::Hz45To55), + _ => Err("temporal_cadence_band".into()), + } +} +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +struct SourcePolicy { + source_id: u8, + band: String, + min_rows: usize, + max_rows: usize, + min_interval_ns: u64, + max_gap_ns: u64, +} +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +struct Binding { + plan_id: String, + native_frame_identity: String, + plan_sha256: String, + context: CaptureContext, + encoder: EncoderConfig, + target_span_ms: u64, + max_span_ms: u64, + max_pairwise_skew_ms: u64, + max_first_delay_ms: u64, + sources: Vec, +} +impl Binding { + fn from_plan(plan: &AcquisitionPlan) -> Self { + Self { + plan_id: format!("{:?}", plan.id()), + native_frame_identity: "existing-esp32-receiver-ns-node-u32-sequence-v1".into(), + plan_sha256: plan.sha256().into(), + context: plan.context().clone(), + encoder: plan.encoder_config().clone(), + target_span_ms: plan.target_span_ms(), + max_span_ms: plan.max_observed_span_ms(), + max_pairwise_skew_ms: plan.max_pairwise_receiver_skew_ms(), + max_first_delay_ms: plan.max_first_packet_delay_ms(), + sources: plan + .source_policies() + .iter() + .map(|s| SourcePolicy { + source_id: s.source_id, + band: format!("{:?}", s.receiver_cadence), + min_rows: s.min_rows, + max_rows: s.max_rows, + min_interval_ns: s.min_packet_interval_ns, + max_gap_ns: s.max_gap_ns, + }) + .collect(), + } + } + fn rebuild(&self) -> Result { + let bands = self + .sources + .iter() + .map(|s| Ok((s.source_id, band(&s.band)?))) + .collect::, String>>()?; + let p = AcquisitionPlan::build(self.context.clone(), plan_id(&self.plan_id)?, &bands)?; + if *self != Self::from_plan(&p) { + return Err("temporal_policy_binding_changed".into()); + } + Ok(p) + } +} +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct SourceEvidence { + source_id: u8, + source_spec_sha256: String, + times_ns: Vec, + frame_ids: Vec, + sequence_ids: Vec, +} +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct WindowEvidence { + cohort_id: String, + trial_id: String, + session_id: String, + role: DataRole, + label: RoomLabel, + cohort_sha256: String, + target_ns: String, + sealed_at_ns: String, + provenance: CaptureProvenance, + sources: Vec, +} +impl WindowEvidence { + fn from_assembled( + plan: &AcquisitionPlan, + w: &TemporalWindow, + sealed_at: u64, + target_ns: u64, + ) -> Result { + Self::from_binding(&Binding::from_plan(plan), w, sealed_at, target_ns) + } + fn from_binding( + binding: &Binding, + w: &TemporalWindow, + sealed_at: u64, + target_ns: u64, + ) -> Result { + if w.policy_sha256 != binding.plan_sha256 { + return Err("temporal_window_policy_changed".into()); + } + validate_cohort(&binding.context, &w.cohort)?; + if w.row_times_ns.len() != w.cohort.views.len() || w.cohort.teacher.is_some() { + return Err("temporal_window_evidence_changed".into()); + } + let mut sources = vec![]; + for ((spec, view), (source, times)) in binding + .context + .sources + .iter() + .zip(&w.cohort.views) + .zip(&w.row_times_ns) + { + view.validate(spec)?; + if *source != spec.source_id + || view.source_id != *source + || times.len() != view.samples.len() + || times.first() != Some(&view.first_time_ns) + || times.last() != Some(&view.last_time_ns) + || view.clock.kind != ClockKind::ReceiverOnly + || view.clock.uncertainty_ns != u64::MAX + || view.phase != PhaseAvailability::MagnitudeOnly + { + return Err("temporal_source_evidence_changed".into()); + } + sources.push(SourceEvidence { + source_id: *source, + source_spec_sha256: hash(spec)?, + times_ns: times.iter().map(ToString::to_string).collect(), + frame_ids: view.frame_ids.iter().map(ToString::to_string).collect(), + sequence_ids: view.sequence_ids.clone(), + }); + } + let e = Self { + cohort_id: w.cohort.id.clone(), + trial_id: w.cohort.trial_id.clone(), + session_id: w.cohort.session_id.clone(), + role: w.cohort.role, + label: w.cohort.label, + cohort_sha256: hash(&w.cohort)?, + target_ns: target_ns.to_string(), + sealed_at_ns: sealed_at.to_string(), + provenance: w.cohort.provenance, + sources, + }; + e.validate_sources(binding)?; + Ok(e) + } + fn validate_sources(&self, b: &Binding) -> Result<(), String> { + if !sha(&self.cohort_sha256) || self.sources.len() != b.sources.len() { + return Err("temporal_evidence_shape".into()); + } + let sealed = integer::(&self.sealed_at_ns)?; + let target = integer::(&self.target_ns)?; + if target == 0 { + return Err("temporal_target_missing".into()); + } + let mut first_min = u64::MAX; + let mut first_max = 0; + let mut last_min = u64::MAX; + let mut last_max = 0; + for ((s, p), spec) in self.sources.iter().zip(&b.sources).zip(&b.context.sources) { + let n = s.times_ns.len(); + if s.source_id != p.source_id + || s.source_id != spec.source_id + || s.source_spec_sha256 != hash(spec)? + || n < p.min_rows + || n > p.max_rows + || s.frame_ids.len() != n + || s.sequence_ids.len() != n + { + return Err("temporal_source_shape_or_policy".into()); + } + let times = s + .times_ns + .iter() + .map(|s| integer::(s)) + .collect::, _>>()?; + let frames = s + .frame_ids + .iter() + .map(|s| integer::(s)) + .collect::, _>>()?; + if times.contains(&0) + || frames.contains(&0) + || frames.iter().collect::>().len() != n + || s.sequence_ids.iter().any(|s| *s > u32::MAX as u64) + || s.sequence_ids.windows(2).any(|w| w[1] <= w[0]) + { + return Err("temporal_frame_or_sequence".into()); + } + for w in times.windows(2) { + let gap = w[1].checked_sub(w[0]).ok_or("temporal_time_reversed")?; + if gap < p.min_interval_ns || gap > p.max_gap_ns { + return Err("temporal_gap_or_cadence".into()); + } + } + // Check the EXACT identity codec already used by native::canonicalize_esp32. + // This links original receiver timestamps/counters to their recorded IDs; + // it does not infer a source clock or mint replacement frame identities. + for ((time, frame), sequence) in times.iter().zip(&frames).zip(&s.sequence_ids) { + let expected = (u128::from(*time) << 40) + | (u128::from(s.source_id) << 32) + | u128::from(*sequence); + if *frame != expected { + return Err("temporal_native_frame_identity_changed".into()); + } + } + let first = times[0]; + let last = times[n - 1]; + let span = last - first; + if first < target || first - target > b.max_first_delay_ms * 1_000_000 { + return Err("temporal_first_arrival_changed".into()); + } + if span < b.target_span_ms * 1_000_000 + || span > b.max_span_ms * 1_000_000 + || sealed < last + || sealed - first > MAX_AGE_NS + { + return Err("temporal_horizon_or_age".into()); + } + let (lo, hi) = band(&p.band)?.rate_bounds_millihz(); + let (lo, hi) = (u128::from(lo), u128::from(hi)); + let rate = (n - 1) as u128 * 1_000_000_000_000u128; + if rate < lo * span as u128 || rate > hi * span as u128 { + return Err("temporal_observed_cadence".into()); + } + first_min = first_min.min(first); + first_max = first_max.max(first); + last_min = last_min.min(last); + last_max = last_max.max(last); + } + if first_max - first_min > b.max_pairwise_skew_ms * 1_000_000 + || last_max - last_min > b.max_pairwise_skew_ms * 1_000_000 + || first_max >= last_min + { + return Err("temporal_source_alignment".into()); + } + Ok(()) + } +} +#[derive(Clone, Default)] +struct Fences { + frames: HashSet<(u8, u128)>, + last: HashMap, +} +impl Fences { + fn add(&mut self, e: &WindowEvidence) -> Result<(), String> { + for s in &e.sources { + for ((t, f), seq) in s.times_ns.iter().zip(&s.frame_ids).zip(&s.sequence_ids) { + let t = integer::(t)?; + let f = integer::(f)?; + if self.frames.contains(&(s.source_id, f)) + || self + .last + .get(&s.source_id) + .is_some_and(|(last_seq, last_t)| *seq <= *last_seq || t <= *last_t) + { + return Err("temporal_cross_window_replay".into()); + } + self.frames.insert((s.source_id, f)); + self.last.insert(s.source_id, (*seq, t)); + } + } + Ok(()) + } + fn check_packet(&self, p: &CanonicalPacket) -> Result<(), String> { + if self + .frames + .contains(&(p.source_id, p.native_frame.frame_id)) + || self.last.get(&p.source_id).is_some_and(|(seq, time)| { + u64::from(p.sequence) <= *seq || p.native_frame.timestamp_ns <= *time + }) + { + return Err("temporal_training_frame_or_time_overlap".into()); + } + Ok(()) + } +} +/// Immutable retained records shared with one bounded durability snapshot. +/// Serialization delegates exactly to the existing record; no new wire shape. +struct SharedRecordValue { + value:T, + // In-memory integrity cache only. Never serialized or accepted from JSON. + sha256:std::sync::OnceLock, +} +#[derive(Clone)] +struct SharedRecord(std::sync::Arc>); +impl SharedRecord { + fn new(value:T)->Self {Self(std::sync::Arc::new(SharedRecordValue{value,sha256:std::sync::OnceLock::new()}))} + fn same_record(&self,other:&Self)->bool {std::sync::Arc::ptr_eq(&self.0,&other.0)} +} +impl SharedRecord { + fn canonical_sha256(&self)->Result<&str,String>{ + if self.0.sha256.get().is_none(){let digest=hash(&self.0.value)?;let _=self.0.sha256.set(digest);} + self.0.sha256.get().map(String::as_str).ok_or("temporal_record_digest_unavailable".into()) + } +} +impl std::ops::Deref for SharedRecord { + type Target=T; + fn deref(&self)->&T {&self.0.value} +} +impl std::borrow::Borrow for SharedRecord { + fn borrow(&self)->&T {&self.0.value} +} +impl Serialize for SharedRecord { + fn serialize(&self,serializer:S)->Result{self.0.value.serialize(serializer)} +} +fn owned_records(records: &[SharedRecord]) -> Vec { + records.iter().map(|record| (**record).clone()).collect() +} + +/// Raw cohorts are private and assembled here; there is intentionally no API to +/// fit a caller-supplied temporal-evidence flag or caller-created window. +pub struct TemporalTrainingSession { + plan: AcquisitionPlan, + binding: Binding, + run_id: String, + assembler: NativeWindowAssembler, + rows: Vec>, + evidence: Vec>, + config: FitConfig, + fitted: bool, + failed: bool, + active: bool, + target_ns: Option, + managed: Option, +} +fn preflight( + context: &CaptureContext, + plan: &AcquisitionPlan, + config: &FitConfig, +) -> Result { + if !(2..=8).contains(&context.sources.len()) { + return Err("temporal_model_supports_2_to_8_sources".into()); + } + let encoder = plan.encoder_config(); + if config.hidden_dim != encoder.hidden_dim + || config.embedding_dim != encoder.embedding_dim + || config.temporal_tokens != encoder.temporal_tokens + || config.temperature != encoder.temperature + { + return Err("temporal_encoder_policy_changed".into()); + } + config.validate_context(context) +} +impl TemporalTrainingSession { + pub fn new( + context: CaptureContext, + id: ExperimentalPlanId, + cadences: &[(u8, ReceiverCadenceBand)], + run_id: String, + config: FitConfig, + ) -> Result { + if !(2..=8).contains(&context.sources.len()) { + return Err("temporal_model_supports_2_to_8_sources".into()); + } + let plan = AcquisitionPlan::build(context, id, cadences)?; + preflight(plan.context(), &plan, &config)?; + let binding = Binding::from_plan(&plan); + let assembler = NativeWindowAssembler::new(plan.clone(), run_id.clone())?; + Ok(Self { + plan, + binding, + run_id, + assembler, + rows: vec![], + evidence: vec![], + config, + fitted: false, + failed: false, + active: false, + target_ns: None, + managed: None, + }) + } + pub fn begin_window(&mut self, target_ns: u64, end_ns: u64) -> Result<(), String> { + if self.managed.is_some() { + return Err("managed_window_target_required".into()); + } + if self.failed { + return Err("temporal_session_failed".into()); + } + let train_count = self.plan.resources().cohorts / 2; + if self.rows.len() >= train_count && !self.fitted { + return Err("temporal_fit_before_heldout".into()); + } + self.assembler.begin_window(target_ns, end_ns)?; + self.active = true; + self.target_ns = Some(target_ns); + Ok(()) + } + pub fn ingest(&mut self, p: &CanonicalPacket) -> Result { + if self.failed { + return Err("temporal_session_failed".into()); + } + let result = self.assembler.ingest(p); + if result.is_err() { + self.failed = true; + } + result + } + pub fn finish_window(&mut self, now_ns: u64) -> Result<(), String> { + if self.failed { + return Err("temporal_session_failed".into()); + } + // The assembler consumes/advances its window before adapter evidence is + // checked. Any subsequent failure must terminate this wrapper run; + // previous valid windows remain available for diagnostics, never credit + // an unvalidated window or let its cursor be silently reused. + let result = (|| { + let window = self.assembler.finish_window(now_ns)?; + let evidence = if let Some(managed) = &self.managed { + WindowEvidence::from_binding( + &managed.binding.validation_binding()?, + &window, + now_ns, + self.target_ns.ok_or("temporal_target_missing")?, + )? + } else { + WindowEvidence::from_assembled( + &self.plan, + &window, + now_ns, + self.target_ns.ok_or("temporal_target_missing")?, + )? + }; + let managed_identity = window.managed_identity.clone(); + if self.managed.is_some() && managed_identity.is_none() { + return Err("managed_window_identity_missing".into()); + } + Ok::<_, String>((window, evidence, managed_identity)) + })(); + self.active = false; + self.target_ns = None; + match result { + Ok((window, evidence, identity)) => { + if let Some(managed) = &mut self.managed { + managed + .windows + .push(identity.ok_or("managed_window_identity_missing")?); + } + self.rows.push(SharedRecord::new(window.cohort)); + self.evidence.push(SharedRecord::new(evidence)); + Ok(()) + } + Err(error) => { + self.failed = true; + Err(error) + } + } + } + pub fn fit(&mut self) -> Result<(TemporalModel, TrainingReport), String> { + if self.managed.is_some() { + return Err("managed_fit_admission_required".into()); + } + if self.failed + || self.active + || self.fitted + || self.rows.len() != self.plan.resources().cohorts / 2 + || self.rows.iter().any(|r| r.role != DataRole::Train) + { + return Err("temporal_complete_train_schedule_required".into()); + } + preflight(self.plan.context(), &self.plan, &self.config)?; + let dataset = ValidatedDataset::new(self.plan.context().clone(), owned_records(&self.rows))?; + let (model, report) = RoomModel::fit(&dataset, self.config.clone())?; + if model.encoder_config() != self.plan.encoder_config() { + self.failed = true; + return Err("temporal_fitter_encoder_changed".into()); + } + let input = TemporalModel { + binding: self.binding.clone(), + run_id: self.run_id.clone(), + model, + evidence: owned_records(&self.evidence), + managed: None, + }; + if let Err(error) = input.validate() { + self.failed = true; + return Err(error); + } + self.fitted = true; + Ok((input, report)) + } + pub fn dataset(&self) -> Result { + ValidatedDataset::new(self.plan.context().clone(), owned_records(&self.rows)) + } + pub fn plan_sha256(&self) -> &str { + &self.binding.plan_sha256 + } +} +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Payload { + binding: Binding, + run_id: String, + training_data_sha256: String, + input_evidence_sha256: String, + evidence: Vec, + model_sha256: String, + model_json: String, +} +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Envelope { + schema: String, + payload_sha256: String, + payload: Payload, +} +/// Crate-private aggregate diagnostic identity. Not provenance or an inner model +/// accessor; checked against the WHOLE validated temporal codec on every use. +#[derive(Debug,Clone,Serialize,Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct TemporalFitBinding { + pub artifact_kind:String,pub artifact_sha256:String,pub context_digest:String, + pub training_data_sha256:String,pub input_evidence_sha256:String, + pub identity_evidence_sha256:Option,pub input_policy_sha256:String, + pub execution_policy_sha256:Option,pub schedule_sha256:Option, + pub source_ids:Vec,pub parameter_count:usize,pub training_cohorts:usize, + pub encoder:EncoderConfig,pub bound_fit_config:Option, +} +/// Existing learned network, with a new, separate versioned INPUT envelope. +/// Neither a checksum nor historical imported metadata is managed provenance. +#[derive(Clone)] +pub struct TemporalModel { + managed: Option, + binding: Binding, + run_id: String, + model: RoomModel, + evidence: Vec, +} +impl TemporalModel { + fn validate(&self) -> Result { + if self.managed.is_some() { + return Err("managed_v3_requires_policy_dispatch".into()); + } + let p = self.binding.rebuild()?; + self.model.validate()?; + if !(2..=8).contains(&p.context().sources.len()) + || self.model.context() != p.context() + || self.model.encoder_config() != p.encoder_config() + || self.evidence.len() != p.resources().cohorts / 2 + || self.model.training_manifest().training_cohorts != self.evidence.len() + || self.model.training_manifest().clock_claims != [ClockKind::ReceiverOnly] + || self.model.training_manifest().measurement_clock_available + { + return Err("temporal_model_input_binding".into()); + } + let _ = NativeWindowAssembler::new(p.clone(), self.run_id.clone())?; + let mut provenance = self + .evidence + .iter() + .map(|e| e.provenance) + .collect::>(); + provenance.sort_by_key(|p| match p { + CaptureProvenance::Synthetic => 0, + CaptureProvenance::CapturedReplay => 1, + CaptureProvenance::LiveReceiverObserved => 2, + }); + provenance.dedup(); + if provenance != self.model.training_manifest().training_provenance { + return Err("temporal_training_provenance_changed".into()); + } + let mut fences = Fences::default(); + for (index, e) in self.evidence.iter().enumerate() { + let step = &p.schedule()[index / p.slots_per_trial()]; + let slot = index % p.slots_per_trial(); + let trial = format!("{}-{}", self.run_id, step.step_id); + if e.role != DataRole::Train + || e.role != step.role + || e.label != step.label + || e.trial_id != trial + || e.session_id != trial + || e.cohort_id != format!("{trial}-{slot}") + { + return Err("temporal_training_role_or_identity".into()); + } + e.validate_sources(&self.binding)?; + fences.add(e)?; + } + Ok(p) + } + pub fn export(&self) -> Result, String> { + if self.managed.is_some() { + return self.export_managed(); + } + self.validate()?; + let model = self.model.export()?; + let payload = Payload { + binding: self.binding.clone(), + run_id: self.run_id.clone(), + training_data_sha256: self.model.training_manifest().training_data_sha256.clone(), + input_evidence_sha256: hash(&self.evidence)?, + evidence: self.evidence.clone(), + model_sha256: bytes_hash(&model), + model_json: String::from_utf8(model).map_err(|_| "temporal_model_encoding")?, + }; + let envelope = Envelope { + schema: SCHEMA.into(), + payload_sha256: hash(&payload)?, + payload, + }; + let bytes = serde_json::to_vec(&envelope).map_err(|_| "temporal_encoding")?; + if bytes.len() > MAX_BYTES { + return Err("temporal_input_envelope_budget".into()); + } + Ok(bytes) + } + pub fn import(bytes: &[u8]) -> Result { + if bytes.len() > MAX_BYTES { + return Err("temporal_input_envelope_budget".into()); + } + let version: serde_json::Value = + serde_json::from_slice(bytes).map_err(|_| "temporal_input_envelope_schema")?; + if version["schema"] == managed::SCHEMA_V3 { + return Self::import_managed(bytes); + } + let envelope: Envelope = + serde_json::from_slice(bytes).map_err(|_| "temporal_input_envelope_schema")?; + if envelope.schema != SCHEMA + || envelope.payload_sha256 != hash(&envelope.payload)? + || serde_json::to_vec(&envelope).map_err(|_| "temporal_encoding")? != bytes + { + return Err("temporal_input_envelope_integrity".into()); + } + let payload = envelope.payload; + if payload.model_sha256 != bytes_hash(payload.model_json.as_bytes()) + || payload.input_evidence_sha256 != hash(&payload.evidence)? + { + return Err("temporal_model_or_evidence_hash".into()); + } + let model = RoomModel::import(payload.model_json.as_bytes())?; + if model.export()? != payload.model_json.as_bytes() + || model.training_manifest().training_data_sha256 != payload.training_data_sha256 + { + return Err("temporal_model_training_binding".into()); + } + let result = Self { + managed: None, + binding: payload.binding, + run_id: payload.run_id, + model, + evidence: payload.evidence, + }; + result.validate()?; + Ok(result) + } + pub fn live_input(&self, run_id: String) -> Result { + if self.managed.is_some() { + return self.managed_live_input(run_id); + } + let p = self.validate()?; + let mut fences = Fences::default(); + for e in &self.evidence { + fences.add(e)?; + } + Ok(TemporalLiveInput { + binding: self.binding.clone(), + assembler: NativeWindowAssembler::new(p, run_id)?, + fences, + failed: false, + target_ns: None, + }) + } + pub fn predict( + &self, + input: TemporalLiveObservation, + now_ns: u64, + ) -> Result { + if self.managed.is_some() { + return self.predict_managed(input, now_ns); + } + let plan = self.validate()?; + if input.window.cohort.role != DataRole::Inference + || input.window.cohort.label != RoomLabel::Unknown + || input.window.cohort.teacher.is_some() + { + return Err("temporal_live_must_be_unlabelled".into()); + } + let evidence = + WindowEvidence::from_assembled(&plan, &input.window, now_ns, input.target_ns)?; + let mut fences = Fences::default(); + for e in &self.evidence { + fences.add(e)?; + } + fences.add(&evidence)?; + if input.binding != self.binding || input.window.policy_sha256 != self.binding.plan_sha256 { + return Err("temporal_live_policy_changed".into()); + } + let cohort = input.window.cohort; + validate_cohort(&self.binding.context, &cohort)?; + for v in &cohort.views { + if now_ns < v.last_time_ns + || now_ns + .checked_sub(v.first_time_ns) + .filter(|a| *a <= MAX_AGE_NS) + .is_none() + { + return Err("temporal_live_input_stale".into()); + } + } + Ok(self.model.infer(&cohort)) + } + /// Read-only immutable input policy for a diagnostic window driver. + /// Does not expose the inner v1 network or issue managed provenance. + pub fn input_policy(&self) -> Result { + self.validate() + } + pub fn window_input_policy( + &self, + ) -> Result { + if let Some(managed) = &self.managed { + self.validate_managed()?; + Ok(managed.binding.native.clone()) + } else { + Ok(super::temporal_acquisition::WindowInputPolicy::from_seed( + &self.validate()?, + )) + } + } + pub fn artifact_kind(&self) -> &'static str { + if self.managed.is_some() { + managed::SCHEMA_V3 + } else { + SCHEMA + } + } + /// Read-only historical context for owner-scoped diagnostic persistence. + /// This does not expose the inner model or establish managed provenance. + pub fn context(&self) -> &CaptureContext { + &self.binding.context + } + pub(crate) fn fit_diagnostic_binding(&self)->Result{ + let bytes=self.export()?; + let (schedule,execution,identities,configured)=if self.managed.is_some(){ + let (s,e,i,c)=self.managed_fit_diagnostic_policy()?;(Some(s),Some(e),Some(i),Some(c)) + }else{(None,None,None,None)}; + Ok(TemporalFitBinding{artifact_kind:self.artifact_kind().into(),artifact_sha256:bytes_hash(&bytes),context_digest:self.context().digest(),training_data_sha256:self.model.training_manifest().training_data_sha256.clone(),input_evidence_sha256:hash(&self.evidence)?,identity_evidence_sha256:identities,input_policy_sha256:self.plan_sha256().into(),execution_policy_sha256:execution,schedule_sha256:schedule,source_ids:self.context().sources.iter().map(|s|s.source_id).collect(),parameter_count:self.parameter_count(),training_cohorts:self.model.training_manifest().training_cohorts,encoder:self.model.encoder_config().clone(),bound_fit_config:configured}) + } + pub fn parameter_count(&self) -> usize { + self.model.parameter_count() + } + pub fn plan_sha256(&self) -> &str { + &self.binding.plan_sha256 + } +} +/// Bounded one-window inference input seed. Runtime warmup/continuous replay +/// custody remains a future integration; this does not replace a live service. +pub struct TemporalLiveInput { + binding: Binding, + assembler: NativeWindowAssembler, + fences: Fences, + failed: bool, + target_ns: Option, +} +pub struct TemporalLiveObservation { + binding: Binding, + window: TemporalWindow, + target_ns: u64, +} +impl TemporalLiveInput { + pub fn begin_window(&mut self, target_ns: u64, end_ns: u64) -> Result<(), String> { + if self.failed { + return Err("temporal_live_failed".into()); + } + self.assembler.begin_window(target_ns, end_ns)?; + self.target_ns = Some(target_ns); + Ok(()) + } + pub fn ingest(&mut self, p: &CanonicalPacket) -> Result { + if self.failed { + return Err("temporal_live_failed".into()); + } + if let Err(e) = self.fences.check_packet(p) { + self.failed = true; + return Err(e); + } + let result = self.assembler.ingest(p); + if result.is_err() { + self.failed = true; + } + result + } + pub fn finish(self, now_ns: u64) -> Result { + if self.failed { + return Err("temporal_live_failed".into()); + } + let mut assembler = self.assembler; + let mut window = assembler.finish_window(now_ns)?; + // Fresh inference observations carry no operator schedule label. This + // never relabels retained Train/HeldOut rows or creates teacher truth. + window.cohort.role = DataRole::Inference; + window.cohort.label = RoomLabel::Unknown; + window.cohort.teacher = None; + WindowEvidence::from_binding( + &self.binding, + &window, + now_ns, + self.target_ns.ok_or("temporal_target_missing")?, + )?; + Ok(TemporalLiveObservation { + binding: self.binding, + window, + target_ns: self.target_ns.ok_or("temporal_target_missing")?, + }) + } +} +#[cfg(test)] +#[path = "temporal_model_input_tests.rs"] +mod tests; + +#[path = "temporal_model_input_managed.rs"] +pub(crate) mod managed; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_model_input_managed.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_model_input_managed.rs new file mode 100644 index 0000000000..26635b07b4 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_model_input_managed.rs @@ -0,0 +1,629 @@ +//! Managed-v3 INPUT integration inside the SAME opaque temporal model/session. +//! The private Runtime may issue fit-only admission from its exact durable Train +//! producer. Neither this INPUT codec nor its claims issue managed qualification. +use super::*; +use crate::room_learner::{ + managed_schedule_contract::{FrozenModelIdentity, ManagedScheduleContract}, + temporal_acquisition::{ManagedWindowIdentity, ManagedWindowTarget, WindowInputPolicy}, +}; +use std::sync::atomic::AtomicBool; +#[path = "temporal_execution.rs"] +pub(crate) mod execution; +pub(crate) const SCHEMA_V3: &str = "ruview.room-learner.managed-temporal-input.v3"; +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct LivePolicy { + id: String, + target_horizon_ms: u64, + maximum_first_row_age_ns: u64, + source_gap_policy: String, +} +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct ManagedBinding { + schedule_json: String, + schedule_sha256: String, + pub(super) native: WindowInputPolicy, + encoder: EncoderConfig, + fit: FitConfig, + live: LivePolicy, + target_mapping: String, +} +impl ManagedBinding { + fn build(schedule: &ManagedScheduleContract, fit: FitConfig) -> Result { + fit.validate_context(schedule.context())?; + let encoder = EncoderConfig { + bins: 56, + source_count: schedule.context().sources.len(), + hidden_dim: fit.hidden_dim, + embedding_dim: fit.embedding_dim, + temporal_tokens: fit.temporal_tokens, + temperature: fit.temperature, + }; + Ok(Self { + schedule_json: String::from_utf8(schedule.export()?) + .map_err(|_| "managed_binding_encoding")?, + schedule_sha256: schedule.sha256().into(), + native: WindowInputPolicy::from_managed(schedule), + encoder, + fit, + live: LivePolicy { + id: "nonoverlap-native-oldest-first-v1".into(), + target_horizon_ms: 2000, + maximum_first_row_age_ns: MAX_AGE_NS, + source_gap_policy: "existing-native-profile-every-source-max-gap".into(), + }, + target_mapping: execution::EXECUTION_POLICY.into(), + }) + } + fn contract(&self) -> Result { + let schedule = ManagedScheduleContract::import(self.schedule_json.as_bytes())?; + let expected = Self::build(&schedule, self.fit.clone())?; + if serde_json::to_vec(self).map_err(|_| "managed_binding_encoding")? + != serde_json::to_vec(&expected).map_err(|_| "managed_binding_encoding")? + { + return Err("managed_binding_policy_changed".into()); + } + Ok(schedule) + } + pub(super) fn validation_binding(&self) -> Result { + self.contract()?; + Ok(Binding { + plan_id: "managed-v3-native-validation-only".into(), + native_frame_identity: "existing-esp32-receiver-ns-node-u32-sequence-v1".into(), + plan_sha256: hash(self)?, + context: self.native.context().clone(), + encoder: self.encoder.clone(), + target_span_ms: self.native.target_span_ms(), + max_span_ms: self.native.max_observed_span_ms(), + max_pairwise_skew_ms: self.native.max_pairwise_receiver_skew_ms(), + max_first_delay_ms: self.native.max_first_packet_delay_ms(), + sources: self + .native + .source_policies() + .iter() + .map(|p| SourcePolicy { + source_id: p.source_id, + band: format!("{:?}", p.receiver_cadence), + min_rows: p.min_rows, + max_rows: p.max_rows, + min_interval_ns: p.min_packet_interval_ns, + max_gap_ns: p.max_gap_ns, + }) + .collect(), + }) + } +} +pub(super) struct ManagedSession { + pub(super) binding: ManagedBinding, + pub(super) windows: Vec, + frozen_sha256: Option, +} +#[derive(Clone)] +pub(super) struct ManagedModel { + pub(super) binding: ManagedBinding, + windows: Vec, +} +/// The private owner Runtime can obtain this only by consuming the producer's +/// exact confirmed/quality-checked/durable Train prefix. Permission to compute a +/// diagnostic candidate is not physical qualification or selection authority. +/// It cannot be uploaded/deserialized or synthesized from generic model metadata. +pub(crate) struct ManagedFitAdmission { + policy: String, + data: String, + evidence: String, + identities: String, +} +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Claims { + diagnostic_only: bool, + managed_proof_issued: bool, + whole_episode_quality_issued: bool, + conditions_confirmed_by_runtime: bool, +} +impl Claims { + fn diagnostic() -> Self { + Self { + diagnostic_only: true, + managed_proof_issued: false, + whole_episode_quality_issued: false, + conditions_confirmed_by_runtime: false, + } + } +} +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct PayloadV3 { + binding: ManagedBinding, + run_id: String, + training_data_sha256: String, + input_evidence_sha256: String, + identity_evidence_sha256: String, + evidence: Vec, + identities: Vec, + model_sha256: String, + model_json: String, + claims: Claims, +} +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct EnvelopeV3 { + schema: String, + payload_sha256: String, + payload: PayloadV3, +} +fn identity( + schedule: &ManagedScheduleContract, + run: &str, + record: &ManagedWindowIdentity, +) -> Result { + let frozen = record + .frozen_artifact_sha256 + .as_ref() + .map(|sha| FrozenModelIdentity::from_runtime_verified_artifact(schedule, sha.clone())) + .transpose()?; + let bound = schedule.bind_episode_start( + run, + record.episode_index, + record.attempt_ordinal, + record.segment_ordinal, + frozen.as_ref(), + )?; + if record.collection_started_ns != record.mapping.receiver_start_ns.to_string() { + return Err("managed_segment_start_changed".into()); + } + let expected = ManagedWindowTarget::from_observed_episode( + schedule, + &bound, + run, + record.slot_index, + record.mapping.clone(), + )?; + if serde_json::to_vec(expected.identity()).map_err(|_| "managed_identity_encoding")? + != serde_json::to_vec(record).map_err(|_| "managed_identity_encoding")? + { + return Err("managed_identity_changed".into()); + } + Ok(expected) +} +fn expected_position( + schedule: &ManagedScheduleContract, + index: usize, + train_only: bool, +) -> Result<(usize, usize), String> { + let mut cursor = 0; + for episode in schedule.episodes() { + if train_only && episode.role != DataRole::Train { continue; } + if index < cursor + episode.slots.len() { + return Ok((episode.index, index - cursor)); + } + cursor += episode.slots.len(); + } + Err("managed_window_schedule_exhausted".into()) +} +fn validate_records( + schedule: &ManagedScheduleContract, + run: &str, + evidence: &[impl std::borrow::Borrow], + records: &[ManagedWindowIdentity], + binding: &Binding, + train_only: bool, +) -> Result<(), String> { + if evidence.len() != records.len() { + return Err("managed_identity_count_changed".into()); + } + let mut fences = Fences::default(); + let mut previous: Option<&ManagedWindowIdentity> = None; + for (index, (e, r)) in evidence.iter().zip(records).enumerate() { + let e = e.borrow(); + let position = expected_position(schedule, index, train_only)?; + if (r.episode_index, r.slot_index) != position + || (train_only && (r.role != DataRole::Train || r.frozen_artifact_sha256.is_some())) + { + return Err("managed_window_role_or_order".into()); + } + let target = identity(schedule, run, r)?; + if e.role != r.role + || e.label != r.label + || e.trial_id != r.trial_id + || e.session_id != r.session_id + || e.target_ns != r.target_ns + { + return Err("managed_cohort_identity_changed".into()); + } + let expected_id = format!( + "window-{:x}", + Sha256::digest(serde_json::to_vec(r).map_err(|_| "managed_identity_encoding")?) + ); + if e.cohort_id != expected_id || integer::(&e.sealed_at_ns)? >= target.end_ns() { + return Err("managed_window_mapping_changed".into()); + } + if let Some(old) = previous { + if old.episode_index == r.episode_index { + if old.attempt_id != r.attempt_id { + return Err("managed_attempt_changed".into()); + } + if old.segment_ordinal == r.segment_ordinal { + if old.mapping != r.mapping || old.session_id != r.session_id { + return Err("managed_segment_changed".into()); + } + } else { + let last = schedule.episodes()[old.episode_index].slots[old.slot_index] + .deadline_offset_ms + * 1_000_000; + if r.segment_ordinal != old.segment_ordinal + 1 + || r.mapping.observed_start_ns != last + || r.mapping.receiver_start_ns < integer::(&old.end_ns)? + || r.mapping.clock_anchor_ns < integer::(&old.end_ns)? + { + return Err("managed_segment_overlap_or_clock".into()); + } + } + } else if r.mapping.observed_start_ns != 0 + || r.mapping.receiver_start_ns < integer::(&old.end_ns)? + { + return Err("managed_episode_time_overlap".into()); + } + } + e.validate_sources(binding)?; + fences.add(e)?; + previous = Some(r); + } + Ok(()) +} +impl TemporalTrainingSession { + pub(crate) fn bound_managed_fit_config(&self) -> Result { + let managed=self.managed.as_ref().ok_or("managed_session_required")?; + managed.binding.contract()?; + Ok(managed.binding.fit.clone()) + } + pub fn managed( + schedule: ManagedScheduleContract, + run_id: String, + config: FitConfig, + ) -> Result { + let binding = ManagedBinding::build(&schedule, config.clone())?; + let policy = hash(&binding)?; + // Only the unchanged legacy fields' native-validation carrier is built; + // managed identity/hash/schedule NEVER comes from this seed. + let bands = schedule + .context() + .sources + .iter() + .map(|s| (s.source_id, ReceiverCadenceBand::Hz30To36)) + .collect::>(); + let plan = AcquisitionPlan::build( + schedule.context().clone(), + ExperimentalPlanId::Native2000msSlots8, + &bands, + )?; + let assembler = NativeWindowAssembler::managed(schedule, policy, run_id.clone())?; + Ok(Self { + plan, + binding: binding.validation_binding()?, + run_id, + assembler, + rows: vec![], + evidence: vec![], + config, + fitted: false, + failed: false, + active: false, + target_ns: None, + managed: Some(ManagedSession { + binding, + windows: vec![], + frozen_sha256: None, + }), + }) + } + pub fn begin_managed_window(&mut self, target: &ManagedWindowTarget) -> Result<(), String> { + let managed = self.managed.as_ref().ok_or("managed_session_required")?; + let r = target.identity(); + if self.failed || self.active { + return Err("temporal_session_failed_or_active".into()); + } + if r.role == DataRole::HeldOut + && (!self.fitted || r.frozen_artifact_sha256.as_ref() != managed.frozen_sha256.as_ref()) + { + return Err("managed_frozen_whole_model_required".into()); + } + if r.role == DataRole::Train && self.fitted { + return Err("managed_training_is_frozen".into()); + } + self.assembler.begin_managed_window(target)?; + self.active = true; + self.target_ns = Some(target.target_ns()); + Ok(()) + } + /// Called only on the recertified fitting copy. Validation rows remain in + /// the durable capture and never enter training, normalization or its codec. + pub(crate) fn retain_managed_train_only(&mut self) -> Result<(), String> { + let managed = self.managed.as_mut().ok_or("managed_session_required")?; + let schedule = managed.binding.contract()?; + if !schedule.is_operator_eight_step() { return Ok(()); } + if self.active || self.fitted || self.failed || self.rows.len() != schedule.counts().total_windows + || self.rows.len() != self.evidence.len() || self.rows.len() != managed.windows.len() { + return Err("operator_check_train_filter_requires_complete_capture".into()); + } + validate_records(&schedule, &self.run_id, &self.evidence, &managed.windows, &self.binding, false)?; + for ((row,evidence),identity) in self.rows.iter().zip(&self.evidence).zip(&managed.windows) { + if row.role!=identity.role || row.label!=identity.label || row.id!=evidence.cohort_id + || row.canonical_sha256()?!=&evidence.cohort_sha256 { + return Err("operator_check_training_partition_changed".into()); + } + } + let keep = managed.windows.iter().map(|w| w.role == DataRole::Train).collect::>(); + let mut index=0;self.rows.retain(|_|{let k=keep[index];index+=1;k}); + let mut index=0;self.evidence.retain(|_|{let k=keep[index];index+=1;k}); + managed.windows.retain(|w|w.role==DataRole::Train); + if self.rows.len()!=schedule.counts().train_windows || self.rows.iter().any(|r|r.role!=DataRole::Train) { + return Err("operator_check_training_partition_changed".into()); + } + Ok(()) + } + pub(crate) fn fit_managed_with_cancel( + &mut self, + permit: ManagedFitAdmission, + cancel: &AtomicBool, + ) -> Result<(TemporalModel, TrainingReport), String> { + let managed = self.managed.as_ref().ok_or("managed_session_required")?; + let schedule = managed.binding.contract()?; + if self.failed + || self.active + || self.fitted + || self.rows.len() != schedule.counts().train_windows + || self.rows.iter().any(|r| r.role != DataRole::Train) + { + return Err("managed_complete_train_schedule_required".into()); + } + validate_records( + &schedule, + &self.run_id, + &self.evidence, + &managed.windows, + &self.binding, + true, + )?; + let dataset = ValidatedDataset::new(schedule.context().clone(), owned_records(&self.rows))?; + if permit.policy != hash(&managed.binding)? + || permit.data != hash(&dataset.train())? + || permit.evidence != hash(&self.evidence)? + || permit.identities != hash(&managed.windows)? + { + return Err("managed_fit_admission_mismatch".into()); + } + let (model, report) = RoomModel::fit_with_cancel(&dataset, self.config.clone(), cancel)?; + if model.encoder_config() != &managed.binding.encoder { + return Err("managed_encoder_changed".into()); + } + let result = TemporalModel { + binding: self.binding.clone(), + run_id: self.run_id.clone(), + model, + evidence: owned_records(&self.evidence), + managed: Some(ManagedModel { + binding: managed.binding.clone(), + windows: managed.windows.clone(), + }), + }; + result.validate_managed()?; + let frozen = bytes_hash(&result.export()?); + self.managed.as_mut().unwrap().frozen_sha256 = Some(frozen); + self.fitted = true; + Ok((result, report)) + } + #[cfg(test)] + fn synthetic_test_fit_admission(&self) -> Result { + let managed = self.managed.as_ref().ok_or("managed_session_required")?; + let dataset = self.dataset()?; + Ok(ManagedFitAdmission { + policy: hash(&managed.binding)?, + data: hash(&dataset.train())?, + evidence: hash(&self.evidence)?, + identities: hash(&managed.windows)?, + }) + } +} +impl TemporalModel { + pub(crate) fn managed_is_operator_eight_step(&self) -> Result { + Ok(self.validate_managed()?.is_operator_eight_step()) + } + pub(super) fn validate_managed(&self) -> Result { + let m = self.managed.as_ref().ok_or("managed_model_required")?; + let schedule = m.binding.contract()?; + self.model.validate()?; + if self.model.context() != schedule.context() + || self.model.encoder_config() != &m.binding.encoder + || self.model.training_manifest().training_cohorts != schedule.counts().train_windows + || self.evidence.len() != schedule.counts().train_windows + || self.model.training_manifest().measurement_clock_available + || self.model.training_manifest().clock_claims != [ClockKind::ReceiverOnly] + { + return Err("managed_model_binding_changed".into()); + } + let expected = m.binding.validation_binding()?; + if serde_json::to_vec(&self.binding).map_err(|_| "managed_binding_encoding")? + != serde_json::to_vec(&expected).map_err(|_| "managed_binding_encoding")? + { + return Err("managed_native_binding_changed".into()); + } + let mut provenance = self + .evidence + .iter() + .map(|e| e.provenance) + .collect::>(); + provenance.sort_by_key(|p| match p { + CaptureProvenance::Synthetic => 0, + CaptureProvenance::CapturedReplay => 1, + CaptureProvenance::LiveReceiverObserved => 2, + }); + provenance.dedup(); + if provenance != self.model.training_manifest().training_provenance { + return Err("managed_provenance_changed".into()); + } + validate_records( + &schedule, + &self.run_id, + &self.evidence, + &m.windows, + &self.binding, + true, + )?; + Ok(schedule) + } + pub(super) fn managed_fit_diagnostic_policy(&self)->Result<(String,String,String,FitConfig),String>{ + self.validate_managed()?; + let model=self.managed.as_ref().ok_or("managed_model_required")?; + Ok((model.binding.schedule_sha256.clone(),hash(&model.binding.target_mapping)?,hash(&model.windows)?,model.binding.fit.clone())) + } + pub(super) fn export_managed(&self) -> Result, String> { + self.validate_managed()?; + let managed = self.managed.as_ref().unwrap(); + let bytes = self.model.export()?; + let payload = PayloadV3 { + binding: managed.binding.clone(), + run_id: self.run_id.clone(), + training_data_sha256: self.model.training_manifest().training_data_sha256.clone(), + input_evidence_sha256: hash(&self.evidence)?, + identity_evidence_sha256: hash(&managed.windows)?, + evidence: self.evidence.clone(), + identities: managed.windows.clone(), + model_sha256: bytes_hash(&bytes), + model_json: String::from_utf8(bytes).map_err(|_| "managed_model_encoding")?, + claims: Claims::diagnostic(), + }; + let e = EnvelopeV3 { + schema: SCHEMA_V3.into(), + payload_sha256: hash(&payload)?, + payload, + }; + let bytes = serde_json::to_vec(&e).map_err(|_| "managed_model_encoding")?; + if bytes.len() > MAX_BYTES { + return Err("temporal_input_envelope_budget".into()); + } + Ok(bytes) + } + pub(super) fn import_managed(bytes: &[u8]) -> Result { + let e: EnvelopeV3 = serde_json::from_slice(bytes).map_err(|_| "managed_model_schema")?; + if e.schema != SCHEMA_V3 + || e.payload_sha256 != hash(&e.payload)? + || serde_json::to_vec(&e).map_err(|_| "managed_model_encoding")? != bytes + { + return Err("managed_model_integrity".into()); + } + let p = e.payload; + if !p.claims.diagnostic_only + || p.claims.managed_proof_issued + || p.claims.whole_episode_quality_issued + || p.claims.conditions_confirmed_by_runtime + { + return Err("managed_input_is_not_proof".into()); + } + if p.input_evidence_sha256 != hash(&p.evidence)? + || p.identity_evidence_sha256 != hash(&p.identities)? + || p.model_sha256 != bytes_hash(p.model_json.as_bytes()) + { + return Err("managed_evidence_hash_changed".into()); + } + let model = RoomModel::import(p.model_json.as_bytes())?; + if model.export()? != p.model_json.as_bytes() + || model.training_manifest().training_data_sha256 != p.training_data_sha256 + { + return Err("managed_training_hash_changed".into()); + } + let binding = p.binding.validation_binding()?; + let result = Self { + binding, + run_id: p.run_id, + model, + evidence: p.evidence, + managed: Some(ManagedModel { + binding: p.binding, + windows: p.identities, + }), + }; + result.validate_managed()?; + Ok(result) + } + pub(super) fn managed_live_input(&self, run_id: String) -> Result { + self.validate_managed()?; + let m = self.managed.as_ref().unwrap(); + let mut fences = Fences::default(); + for e in &self.evidence { + fences.add(e)?; + } + Ok(TemporalLiveInput { + binding: self.binding.clone(), + assembler: NativeWindowAssembler::inference( + m.binding.native.clone(), + hash(&m.binding)?, + run_id, + )?, + fences, + failed: false, + target_ns: None, + }) + } + pub(super) fn predict_managed( + &self, + input: TemporalLiveObservation, + now_ns: u64, + ) -> Result { + self.validate_managed()?; + if input.window.cohort.role != DataRole::Inference + || input.window.cohort.label != RoomLabel::Unknown + || input.window.cohort.teacher.is_some() + || input.window.managed_identity.is_some() + { + return Err("temporal_live_must_be_unlabelled".into()); + } + let e = + WindowEvidence::from_binding(&self.binding, &input.window, now_ns, input.target_ns)?; + let mut fences = Fences::default(); + for old in &self.evidence { + fences.add(old)?; + } + fences.add(&e)?; + if serde_json::to_vec(&input.binding).map_err(|_| "managed_binding_encoding")? + != serde_json::to_vec(&self.binding).map_err(|_| "managed_binding_encoding")? + { + return Err("temporal_live_policy_changed".into()); + } + Ok(self.model.infer(&input.window.cohort)) + } +} +#[cfg(test)] +#[path = "temporal_model_input_managed_tests.rs"] +mod tests; + +#[cfg(test)] +#[path="temporal_fit_summary_managed_tests.rs"] +mod fit_summary_tests; + +impl TemporalModel { + /// Whole-model evaluation only. The protected snapshot has authenticated + /// original native/HeldOut lineage; no imported diagnostic or inner model + /// accessor can enter this boundary. Existing infer/tokens/math is reused. + pub(crate) fn evaluate_managed_held_out( + &self, + source:&execution::full_run::VerifiedTemporalEvaluationSnapshot, + )->Result{ + source.require_same_model(self)?; + self.validate_managed()?; + source.held_out().validate_data_links(source.proof())?; + let binding=self.fit_diagnostic_binding()?; + let identity=&source.proof().body.identity; + if binding.artifact_kind!=SCHEMA_V3||binding.artifact_sha256!=identity.artifact_sha256 + ||binding.context_digest!=identity.context_digest + ||binding.training_data_sha256!=source.proof().body.training_data_sha256 + ||binding.input_evidence_sha256!=source.proof().body.training_evidence_sha256 + ||binding.identity_evidence_sha256.as_ref()!=Some(&source.proof().body.training_identities_sha256) + {return Err("temporal_evaluation_whole_model_lineage_changed".into());} + // This bounded64-row temporary copy exists only off ingress lock and + // is not retained in another datastore or used for fitting/normalizing. + let rows=source.evaluation_rows(); + crate::room_learner::training::EvaluationReport::compute(&self.model,&rows) + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_model_input_managed_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_model_input_managed_tests.rs new file mode 100644 index 0000000000..d0576b8b16 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_model_input_managed_tests.rs @@ -0,0 +1,541 @@ +//! SYNTHETIC source integration only: no whole-trial quality/confirmation proof. +use super::*; +use crate::room_learner::{ + artifact::ArtifactStore, + managed_schedule_contract::{LeadInPolicy, ManagedScheduleContract}, + native::{canonicalize_esp32, NativePacketInput}, + temporal_live_driver::{ContinuousTemporalInput, ObservationAvailability}, +}; +use std::sync::OnceLock; +use wifi_densepose_hardware::{Adr018Flags, PpduType}; +pub(super) fn context() -> CaptureContext { + CaptureContext { + owner_scope: "a".repeat(64), + installation_id: "synthetic-managed-v3".into(), + room_digest: "synthetic-room".into(), + revision: 1, + sources: (1..=3) + .map(|source_id| SourceSpec { + source_id, + device_id: format!("esp32-node-{source_id}"), + grid_id: "ant1-bins64-ppdu0-usable-drop2-linear-v2".into(), + feature_count: 56, + signal_kind: ruview_unified::RfModality::WifiCsi, + center_frequency_hz: 2_437_000_000, + bandwidth_hz: 20_000_000, + link_id: format!("native-{source_id}"), + link_index: 0, + geometry: None, + }) + .collect(), + } +} +fn schedule() -> ManagedScheduleContract { + ManagedScheduleContract::build(context(), LeadInPolicy::Train5HeldOut15EngineeringV1).unwrap() +} +pub(super) fn packet( + source: u8, + seq: u32, + at: u64, + label: RoomLabel, + row: usize, +) -> CanonicalPacket { + let gain = match label { + RoomLabel::Empty => 3, + RoomLabel::AdjacentOnly => 4, + RoomLabel::OccupiedStatic => 8, + RoomLabel::OccupiedMoving => 10 + (row % 2) as u8, + RoomLabel::Unknown => 8, + }; + let iq = (0..64) + .flat_map(|bin| [gain + source + (bin % 2) as u8, 4]) + .collect::>(); + canonicalize_esp32(NativePacketInput { + node_id: source, + n_antennas: 1, + raw_bins: 64, + frequency_hz: 2_437_000_000, + ppdu_type: PpduType::HtLegacy, + sequence: seq, + received_at_ns: at, + receiver_sample_rate_hz: 33., + iq_bytes: &iq, + flags: Adr018Flags::default(), + provenance: CaptureProvenance::Synthetic, + }) + .unwrap() +} +fn collect( + session: &mut TemporalTrainingSession, + target: &ManagedWindowTarget, + index: u32, + label: RoomLabel, +) { + session.begin_managed_window(target).unwrap(); + for row in 0..100usize { + for source in 1..=3u8 { + let at = target.target_ns() + + row as u64 * 1_000_000_000u64.div_ceil(33) + + u64::from(source) * 1000; + if session + .ingest(&packet( + source, + index * 1000 + row as u32 + 1, + at, + label, + row, + )) + .unwrap() + == WindowAdmission::Ready + { + session.finish_window(at).unwrap(); + return; + } + } + } + panic!("missing actual2s native window"); +} +fn train() -> (TemporalTrainingSession, TemporalModel) { + let plan = schedule(); + let mut session = TemporalTrainingSession::managed( + plan.clone(), + "synthetic-v3-run".into(), + FitConfig { + epochs: 2, + ..Default::default() + }, + ) + .unwrap(); + assert!(session.fit().is_err()); + let mut start = 10_000_000_000u64; + let mut index = 0; + for episode in plan.episodes().iter().filter(|e| e.role == DataRole::Train) { + let bound = plan + .bind_episode_start("synthetic-v3-run", episode.index, 0, 0, None) + .unwrap(); + for slot in 0..episode.slots.len() { + let target = ManagedWindowTarget::from_runtime_episode( + &plan, + &bound, + "synthetic-v3-run", + slot, + start, + ) + .unwrap(); + collect(&mut session, &target, index, episode.condition); + index += 1; + } + start += episode.target_active_recording_ms * 1_000_000 + 10_000_000_000; + } + assert_eq!(session.dataset().unwrap().train().len(), 64); + assert_eq!( + session + .dataset() + .unwrap() + .train() + .iter() + .map(|row| row.trial_id.as_str()) + .collect::>() + .len(), + 7 + ); + let permit = session.synthetic_test_fit_admission().unwrap(); + let (model, report) = session + .fit_managed_with_cancel(permit, &AtomicBool::new(false)) + .unwrap(); + assert_eq!(report.training_cohorts, 64); + assert_eq!(report.completed_epochs, 2); + assert!(report.backbone_delta_l2 > 0.); + (session, model) +} +fn model() -> TemporalModel { + static BYTES: OnceLock> = OnceLock::new(); + TemporalModel::import(BYTES.get_or_init(|| train().1.export().unwrap())).unwrap() +} +fn live(model: TemporalModel) -> serde_json::Value { + let mut driver = ContinuousTemporalInput::new(model).unwrap(); + driver.activate(); + let target = 10_000_000_000_000; + for row in 0..100usize { + for source in 1..=3u8 { + let at = target + row as u64 * 1_000_000_000u64.div_ceil(33) + u64::from(source) * 1000; + let d = driver.ingest( + &packet(source, 1_000_000 + row as u32, at, RoomLabel::Unknown, row), + at, + ); + if d.observation.availability == ObservationAvailability::Available { + return serde_json::to_value(d.current_prediction).unwrap(); + } + } + } + panic!("continuous v3 input did not complete") +} +fn seed_v2_same_context() -> TemporalModel { + let mut session = TemporalTrainingSession::new( + context(), + ExperimentalPlanId::Native2000msSlots2, + &(1..=3) + .map(|s| (s, ReceiverCadenceBand::Hz30To36)) + .collect::>(), + "synthetic-seed-compatible".into(), + FitConfig { + epochs: 2, + ..Default::default() + }, + ) + .unwrap(); + for index in 0..16u32 { + let target = 10_000_000_000 + u64::from(index) * 3_000_000_000; + let label = + session.plan.schedule()[session.rows.len() / session.plan.slots_per_trial()].label; + session + .begin_window(target, target + 3_000_000_000) + .unwrap(); + 'window: for row in 0..100usize { + for source in 1..=3u8 { + let at = + target + row as u64 * 1_000_000_000u64.div_ceil(33) + u64::from(source) * 1000; + if session + .ingest(&packet( + source, + index * 1000 + row as u32 + 1, + at, + label, + row, + )) + .unwrap() + == WindowAdmission::Ready + { + session.finish_window(at).unwrap(); + break 'window; + } + } + } + } + session.fit().unwrap().0 +} +#[test] +fn actual_native_managed_schedule_same_fitter_whole_v3_store_cold_and_same_driver() { + let m = model(); + let bytes = m.export().unwrap(); + assert_eq!(m.artifact_kind(), SCHEMA_V3); + assert!( + m.input_policy().is_err(), + "v3 must not expose seed-v2 plan/hash" + ); + let value: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(value["schema"], SCHEMA_V3); + assert_eq!(value["payload"]["evidence"].as_array().unwrap().len(), 64); + assert_eq!(value["payload"]["identities"].as_array().unwrap().len(), 64); + assert_eq!( + value["payload"]["claims"]["whole_episode_quality_issued"], + false + ); + assert_eq!( + value["payload"]["claims"]["conditions_confirmed_by_runtime"], + false + ); + let root = tempfile::tempdir().unwrap(); + let c = context(); + let path = root.path().join(&c.owner_scope).join("SensingServer"); + let store = ArtifactStore::new(&path, &c.owner_scope, &c.installation_id).unwrap(); + let seed = seed_v2_same_context(); + let seed_bytes = seed.export().unwrap(); + let seed_receipt = store.save_temporal_candidate(&seed).unwrap(); + let directory = path + .join("room-learner") + .join("temporal-models") + .join(&seed_receipt.context_digest); + let pointer = directory.join("candidate-temporal-v2.json"); + let before_pointer = std::fs::read(&pointer).unwrap(); + let receipt = store.save_temporal_candidate(&m).unwrap(); + assert_eq!(receipt.artifact_kind, SCHEMA_V3); + assert_eq!(receipt.artifact_sha256, bytes_hash(&bytes)); + assert!(receipt.diagnostic_only && !receipt.qualification_granted); + assert_eq!( + std::fs::read(&pointer).unwrap(), + before_pointer, + "V3 may not rewrite V2 pointer" + ); + assert_eq!( + store + .load_temporal_candidate(&c) + .unwrap() + .unwrap() + .export() + .unwrap(), + seed_bytes + ); + assert_eq!( + store + .load_exact_temporal_candidate(&c, &seed_receipt.artifact_sha256) + .unwrap() + .export() + .unwrap(), + seed_bytes + ); + let cold = ArtifactStore::new(&path, &c.owner_scope, &c.installation_id) + .unwrap() + .load_managed_temporal_candidate(&c) + .unwrap() + .unwrap(); + assert_eq!(cold.export().unwrap(), bytes); + assert_eq!(live(m), live(cold)); +} +#[test] +fn heldout_uses_frozen_whole_artifact_and_never_changes_train_codec_or_profiles() { + let (mut session, m) = train(); + let before = m.export().unwrap(); + let frozen = bytes_hash(&before); + let plan = schedule(); + let identity = + FrozenModelIdentity::from_runtime_verified_artifact(&plan, frozen.clone()).unwrap(); + let mut start = 2_000_000_000_000; + let mut index = 64; + for episode in plan + .episodes() + .iter() + .filter(|e| e.role == DataRole::HeldOut) + { + let bound = plan + .bind_episode_start("synthetic-v3-run", episode.index, 0, 0, Some(&identity)) + .unwrap(); + for slot in 0..episode.slots.len() { + let target = ManagedWindowTarget::from_runtime_episode( + &plan, + &bound, + "synthetic-v3-run", + slot, + start, + ) + .unwrap(); + collect(&mut session, &target, index, episode.condition); + index += 1; + } + start += episode.target_active_recording_ms * 1_000_000 + 20_000_000_000; + } + assert_eq!(session.dataset().unwrap().held_out().len(), 64); + assert_eq!(m.export().unwrap(), before); + let permit = session.synthetic_test_fit_admission().unwrap(); + assert!(session + .fit_managed_with_cancel(permit, &AtomicBool::new(false)) + .is_err()); + let v: serde_json::Value = serde_json::from_slice(&before).unwrap(); + assert!(v["payload"]["identities"] + .as_array() + .unwrap() + .iter() + .all(|r| r["role"] == "Train")); +} +#[test] +fn managed_cannot_take_seed_labels_skip_slots_or_admit_wrong_frozen_model() { + let plan = schedule(); + let mut session = TemporalTrainingSession::managed( + plan.clone(), + "guard-run".into(), + FitConfig { + epochs: 2, + ..Default::default() + }, + ) + .unwrap(); + assert!(session.begin_window(10, 3_000_000_010).is_err()); + let bound = plan.bind_episode_start("guard-run", 0, 0, 0, None).unwrap(); + let target = + ManagedWindowTarget::from_runtime_episode(&plan, &bound, "guard-run", 1, 10_000_000_000) + .unwrap(); + assert!(session.begin_managed_window(&target).is_err()); + let foreign = plan + .bind_episode_start("foreign-run", 0, 0, 0, None) + .unwrap(); + let foreign_target = ManagedWindowTarget::from_runtime_episode( + &plan, + &foreign, + "foreign-run", + 0, + 10_000_000_000, + ) + .unwrap(); + assert!(session.begin_managed_window(&foreign_target).is_err()); + assert!(ManagedWindowTarget::from_runtime_episode( + &plan, + &bound, + "other-run", + 0, + 10_000_000_000 + ) + .is_err()); + let frozen = + FrozenModelIdentity::from_runtime_verified_artifact(&plan, "b".repeat(64)).unwrap(); + let held = plan + .bind_episode_start("guard-run", 7, 0, 0, Some(&frozen)) + .unwrap(); + let target = + ManagedWindowTarget::from_runtime_episode(&plan, &held, "guard-run", 0, 1_000_000_000_000) + .unwrap(); + assert!(session.begin_managed_window(&target).is_err()); +} +#[test] +fn canonical_v3_schedule_role_encoder_frame_provenance_and_authority_tampering_fail() { + let original = model().export().unwrap(); + for case in [ + "schedule", + "role", + "encoder", + "frame", + "provenance", + "authority", + ] { + let mut e: EnvelopeV3 = serde_json::from_slice(&original).unwrap(); + match case { + "schedule" => { + let mut v: serde_json::Value = + serde_json::from_str(&e.payload.binding.schedule_json).unwrap(); + v["body"]["episodes"][0]["target_active_recording_ms"] = 300000.into(); + e.payload.binding.schedule_json = v.to_string(); + } + "role" => { + e.payload.identities[0].role = DataRole::HeldOut; + e.payload.identity_evidence_sha256 = hash(&e.payload.identities).unwrap(); + } + "encoder" => e.payload.binding.encoder.temporal_tokens = 8, + "frame" => { + e.payload.evidence[0].sources[0].frame_ids[0] = "0".into(); + e.payload.input_evidence_sha256 = hash(&e.payload.evidence).unwrap(); + } + "provenance" => { + e.payload.evidence[0].provenance = CaptureProvenance::LiveReceiverObserved; + e.payload.input_evidence_sha256 = hash(&e.payload.evidence).unwrap(); + } + "authority" => e.payload.claims.whole_episode_quality_issued = true, + _ => unreachable!(), + } + e.payload_sha256 = hash(&e.payload).unwrap(); + let bad = serde_json::to_vec(&e).unwrap(); + assert!(TemporalModel::import(&bad).is_err(), "{case}"); + } +} +#[test] +fn v2_codec_remains_unwrapped_and_managed_fit_permit_cannot_be_omitted_or_forged() { + // The exact old V2 codec structs/field order are unchanged. Existing V2 native + // fit/cold/import golden suite is also run with this integration. + let plan = schedule(); + let mut s = TemporalTrainingSession::managed( + plan, + "permit-run".into(), + FitConfig { + epochs: 2, + ..Default::default() + }, + ) + .unwrap(); + assert!(s.fit().is_err()); + let fake = ManagedFitAdmission { + policy: "a".repeat(64), + data: "b".repeat(64), + evidence: "c".repeat(64), + identities: "d".repeat(64), + }; + assert!(s + .fit_managed_with_cancel(fake, &AtomicBool::new(false)) + .is_err()); + assert!(RoomModel::import(&model().export().unwrap()).is_err()); +} + +#[test] +fn mode_correct_policy_identity_tracks_managed_targets_and_preserves_seed_v2_identity() { + let plan = schedule(); + let config = FitConfig { + epochs: 2, + ..Default::default() + }; + let expected = hash(&ManagedBinding::build(&plan, config.clone()).unwrap()).unwrap(); + let mut session = + TemporalTrainingSession::managed(plan.clone(), "identity-run".into(), config.clone()) + .unwrap(); + assert_eq!(session.plan_sha256(), expected, "managed session must expose the whole managed input binding, not its seed validation carrier"); + assert_ne!(session.plan_sha256(), session.plan.sha256()); + let bound = plan + .bind_episode_start("identity-run", 0, 0, 0, None) + .unwrap(); + let target = + ManagedWindowTarget::from_runtime_episode(&plan, &bound, "identity-run", 0, 10_000_000_000) + .unwrap(); + assert_eq!(target.identity().contract_sha256, plan.sha256()); + // Actual native target admission -> assembled window -> evidence conversion + // rejects any policy identity different from the session's binding. + collect(&mut session, &target, 0, RoomLabel::Empty); + assert_eq!(session.evidence.len(), 1); + assert_eq!(model().plan_sha256(), expected); + let bands = (1..=3) + .map(|id| (id, ReceiverCadenceBand::Hz30To36)) + .collect::>(); + let seed = TemporalTrainingSession::new( + context(), + ExperimentalPlanId::Native2000msSlots2, + &bands, + "seed-identity-run".into(), + config, + ) + .unwrap(); + let old = + AcquisitionPlan::build(context(), ExperimentalPlanId::Native2000msSlots2, &bands).unwrap(); + assert_eq!(seed.plan_sha256(), old.sha256()); + assert_eq!(seed.plan_sha256(), seed.binding.plan_sha256); +} + + +fn collect_operator(validation_changed:bool)->TemporalTrainingSession { + let plan=ManagedScheduleContract::build(context(),LeadInPolicy::OperatorEightStepV1).unwrap(); + let mut session=TemporalTrainingSession::managed(plan.clone(),"synthetic-eight-run".into(),FitConfig{epochs:2,..Default::default()}).unwrap(); + let mut start=10_000_000_000u64;let mut index=0; + for episode in plan.episodes(){ + let bound=plan.bind_episode_start("synthetic-eight-run",episode.index,0,0,None).unwrap(); + for slot in 0..episode.slots.len(){ + let target=ManagedWindowTarget::from_runtime_episode(&plan,&bound,"synthetic-eight-run",slot,start).unwrap(); + if validation_changed&&episode.role==DataRole::Validation { + session.begin_managed_window(&target).unwrap(); + 'window: for row in 0..100usize {for source in 1..=3u8 { + let at=target.target_ns()+row as u64*1_000_000_000u64.div_ceil(33)+u64::from(source)*1000; + let iq=vec![120u8;128]; + let p=canonicalize_esp32(NativePacketInput{node_id:source,n_antennas:1,raw_bins:64,frequency_hz:2_437_000_000, + ppdu_type:PpduType::HtLegacy,sequence:index*1000+row as u32+1,received_at_ns:at,receiver_sample_rate_hz:33., + iq_bytes:&iq,flags:Adr018Flags::default(),provenance:CaptureProvenance::Synthetic}).unwrap(); + if session.ingest(&p).unwrap()==WindowAdmission::Ready{session.finish_window(at).unwrap();break 'window;} + }} + } else {collect(&mut session,&target,index,episode.condition);} + index+=1; + } + start+=episode.target_active_recording_ms*1_000_000+10_000_000_000; + } + assert_eq!(session.rows.len(),80); + assert_eq!(session.rows.iter().filter(|r|r.role==DataRole::Validation).count(),16); + session +} +#[test] +fn operator_reserved_checks_never_change_actual_fitter_normalization_or_model_bytes(){ + let mut first=collect_operator(false);let mut changed=collect_operator(true); + assert_ne!(hash(&first.rows).unwrap(),hash(&changed.rows).unwrap()); + first.retain_managed_train_only().unwrap();changed.retain_managed_train_only().unwrap(); + assert_eq!(first.rows.len(),64);assert_eq!(hash(&first.rows).unwrap(),hash(&changed.rows).unwrap()); + assert_eq!(first.rows.iter().map(|r|&r.trial_id).collect::>().len(),4); + let a=first.synthetic_test_fit_admission().unwrap();let b=changed.synthetic_test_fit_admission().unwrap(); + let(ma,ra)=first.fit_managed_with_cancel(a,&AtomicBool::new(false)).unwrap(); + let(mb,rb)=changed.fit_managed_with_cancel(b,&AtomicBool::new(false)).unwrap(); + assert_eq!(ma.export().unwrap(),mb.export().unwrap()); + assert_eq!(serde_json::to_value(ra).unwrap(),serde_json::to_value(rb).unwrap()); + assert!(ma.managed_is_operator_eight_step().unwrap()); + assert_eq!(TemporalModel::import(&ma.export().unwrap()).unwrap().export().unwrap(),ma.export().unwrap()); +} +#[test] +fn operator_training_partition_rejects_relabelled_validation_before_fitting(){ + let mut session=collect_operator(false); + let index=session.rows.iter().position(|r|r.role==DataRole::Validation).unwrap(); + let original=(*session.rows[index]).clone(); + for label_or_role in [false,true] { + let mut forged=original.clone();if label_or_role{forged.role=DataRole::Train}else{forged.label=RoomLabel::OccupiedMoving} + session.rows[index]=SharedRecord::new(forged); + assert!(session.retain_managed_train_only().is_err()); + assert!(!session.fitted); + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_model_input_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_model_input_tests.rs new file mode 100644 index 0000000000..84b44bfcd1 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_model_input_tests.rs @@ -0,0 +1,486 @@ +//! SYNTHETIC complete native horizons through existing production fitter/infer. +use super::super::native::{canonicalize_esp32, NativePacketInput}; +use super::*; +use std::sync::OnceLock; +use wifi_densepose_hardware::{Adr018Flags, PpduType}; +fn context(count: u8) -> CaptureContext { + CaptureContext { + owner_scope: "a".repeat(64), + installation_id: "synthetic-temporal".into(), + room_digest: "synthetic-room".into(), + revision: 1, + sources: (1..=count) + .map(|source_id| SourceSpec { + source_id, + device_id: format!("esp32-node-{source_id}"), + grid_id: "ant1-bins64-ppdu0-usable-drop2-linear-v2".into(), + feature_count: 56, + signal_kind: ruview_unified::RfModality::WifiCsi, + center_frequency_hz: 2_437_000_000, + bandwidth_hz: 20_000_000, + link_id: format!("native-{source_id}"), + link_index: 0, + geometry: None, + }) + .collect(), + } +} +fn bands(n: u8) -> Vec<(u8, ReceiverCadenceBand)> { + (1..=n) + .map(|n| (n, ReceiverCadenceBand::Hz30To36)) + .collect() +} +fn config() -> FitConfig { + FitConfig { + epochs: 2, + ..Default::default() + } +} +fn packet(source: u8, seq: u32, time: u64, label: RoomLabel, row: usize) -> CanonicalPacket { + let gain = match label { + RoomLabel::Empty => 3, + RoomLabel::AdjacentOnly => 4, + RoomLabel::OccupiedStatic => 8, + RoomLabel::OccupiedMoving => 10 + (row % 2) as u8, + RoomLabel::Unknown => 8, + }; + let bytes = (0..64) + .flat_map(|bin| [gain + source + (bin % 2) as u8, 4u8]) + .collect::>(); + canonicalize_esp32(NativePacketInput { + node_id: source, + n_antennas: 1, + raw_bins: 64, + frequency_hz: 2_437_000_000, + ppdu_type: PpduType::HtLegacy, + sequence: seq, + received_at_ns: time, + receiver_sample_rate_hz: 33., + iq_bytes: &bytes, + flags: Adr018Flags::default(), + provenance: CaptureProvenance::Synthetic, + }) + .unwrap() +} +fn collect(session: &mut TemporalTrainingSession, target: u64, seq: &mut u32) { + session + .begin_window(target, target + 3_000_000_000) + .unwrap(); + let step = &session.plan.schedule()[session.rows.len() / session.plan.slots_per_trial()]; + let label = step.label; + let period = 1_000_000_000u64.div_ceil(33); + for row in 0..256 { + *seq += 1; + for source in 1..=2 { + let now = target + row as u64 * period + u64::from(source) * 1000; + if session + .ingest(&packet(source, *seq, now, label, row)) + .unwrap() + == WindowAdmission::Ready + { + session.finish_window(now).unwrap(); + return; + } + } + } + panic!("no native horizon"); +} +fn fitted_bytes() -> Vec { + static MODEL: OnceLock> = OnceLock::new(); + MODEL + .get_or_init(|| { + let mut s = TemporalTrainingSession::new( + context(2), + ExperimentalPlanId::Native2000msSlots2, + &bands(2), + "synthetic-fit".into(), + config(), + ) + .unwrap(); + let mut seq = 0; + assert!(s.fit().is_err()); + for index in 0..16 { + collect(&mut s, 10_000_000_000 + index * 3_000_000_000, &mut seq); + } + assert_eq!(s.dataset().unwrap().train().len(), 16); + assert!(s.begin_window(60_000_000_000, 63_000_000_000).is_err()); + let (model, report) = s.fit().unwrap(); + assert_eq!(report.training_cohorts, 16); + assert!(report.backbone_delta_l2 > 0.); + assert!(report.parameter_delta_l2 > 0.); + assert_eq!(report.completed_epochs, config().epochs); + assert!(report.objective_evaluations > 0); + assert!(report.initial.total.is_finite()); + assert!(report.final_objective.total.is_finite()); + assert_eq!(model.model.training_manifest().training_cohorts, 16); + assert_eq!(model.evidence.len(), 16); + for (index, evidence) in model.evidence.iter().enumerate() { + assert_eq!(evidence.role, DataRole::Train); + assert_eq!(evidence.label, s.plan.schedule()[index / 2].label); + assert_eq!(evidence.target_ns, + (10_000_000_000u64 + index as u64 * 3_000_000_000).to_string()); + assert_eq!(evidence.sources.len(), 2); + } + assert_eq!(model.model.encoder_config(), s.plan.encoder_config()); + // Frozen fit precedes new HeldOut windows. A later fit cannot consume them. + for index in 16..32 { + collect(&mut s, 10_000_000_000 + index * 3_000_000_000, &mut seq); + } + let d = s.dataset().unwrap(); + assert_eq!(d.held_out().len(), 16); + assert!(!d.measurement_clock_available()); + assert!(s.fit().is_err()); + model.export().unwrap() + }) + .clone() +} +fn model() -> TemporalModel { + TemporalModel::import(&fitted_bytes()).unwrap() +} +fn live( + model: &TemporalModel, + target: u64, + rows: usize, +) -> Result<(TemporalLiveObservation, u64), String> { + let mut input = model.live_input("fresh-live".into())?; + input.begin_window(target, target + 3_000_000_000)?; + let period = 1_000_000_000u64.div_ceil(33); + let mut last = target; + for row in 0..rows { + for source in 1..=2 { + last = target + row as u64 * period + u64::from(source) * 1000; + let state = input.ingest(&packet( + source, + 100_000 + row as u32, + last, + RoomLabel::Unknown, + row, + ))?; + if state == WindowAdmission::Ready { + return Ok((input.finish(last)?, last)); + } + } + } + Ok((input.finish(last)?, last)) +} +#[test] +fn parameter_source_and_frozen_encoder_preflight_run_before_collection_or_fit() { + for count in [1, 9] { + assert!(TemporalTrainingSession::new( + context(count), + ExperimentalPlanId::Native1000msSlots2, + &bands(count), + "bad".into(), + config() + ) + .is_err()); + } + let mut exact = config(); + let count = exact.validate_context(&context(2)).unwrap(); + exact.max_parameters = count - 1; + assert!(TemporalTrainingSession::new( + context(2), + ExperimentalPlanId::Native2000msSlots2, + &bands(2), + "actual-boundary".into(), + exact + ) + .is_err()); + let mut tiny = config(); + tiny.max_parameters = 1; + assert!(TemporalTrainingSession::new( + context(2), + ExperimentalPlanId::Native2000msSlots2, + &bands(2), + "budget".into(), + tiny + ) + .is_err()); + let mut changed = config(); + changed.temporal_tokens = 8; + assert!(TemporalTrainingSession::new( + context(2), + ExperimentalPlanId::Native2000msSlots2, + &bands(2), + "changed".into(), + changed + ) + .is_err()); +} +#[test] +fn actual_fitter_cold_roundtrip_and_live_forward_keep_old_room_model_bytes_unchanged() { + let m = model(); + let bytes = m.export().unwrap(); + assert_eq!(bytes, fitted_bytes()); + let cold = TemporalModel::import(&bytes).unwrap(); + assert_eq!(cold.export().unwrap(), bytes); + assert_eq!(cold.parameter_count(), m.parameter_count()); + let old = m.model.export().unwrap(); + assert_eq!(RoomModel::import(&old).unwrap().export().unwrap(), old); + assert!(RoomModel::import(&bytes).is_err()); + assert!(TemporalModel::import(&old).is_err()); + let (input, now) = live(&cold, 1_000_000_000_000, 100).unwrap(); + assert_eq!(input.window.cohort.role, DataRole::Inference); + assert_eq!(input.window.cohort.label, RoomLabel::Unknown); + assert_eq!(input.window.cohort.views[0].samples.len(), 67); + let plain = cold.model.infer(&input.window.cohort); + let prediction = cold.predict(input, now).unwrap(); + assert_eq!( + serde_json::to_value(plain).unwrap(), + serde_json::to_value(prediction).unwrap() + ); + let envelope: Envelope = serde_json::from_slice(&bytes).unwrap(); + assert!(!serde_json::to_string(&envelope.payload.evidence) + .unwrap() + .contains("samples")); + assert_eq!( + envelope.payload.training_data_sha256, + m.model.training_manifest().training_data_sha256 + ); +} +#[test] +fn live_eight_or_64_packets_cannot_substitute_for_two_seconds() { + let m = model(); + for rows in [8, 64] { + assert!(live(&m, 1_000_000_000_000, rows).is_err()); + } +} +#[test] +fn live_training_replay_foreign_grid_policy_times_and_labelled_roles_are_refused() { + let m = model(); + let mut input = m.live_input("replayed-live".into()).unwrap(); + input + .begin_window(1_000_000_000_000, 1_003_000_000_000) + .unwrap(); + assert!(input + .ingest(&packet(1, 1, 1_000_000_001_000, RoomLabel::Unknown, 0)) + .is_err()); + for mode in ["policy", "grid", "times", "role", "rows", "stale"] { + let (mut input, now) = live(&m, 1_000_000_000_000, 100).unwrap(); + match mode { + "policy" => input.binding.plan_sha256 = "b".repeat(64), + "grid" => input.window.cohort.views[0].grid_id = "wrong".into(), + "times" => input.window.row_times_ns[0].1[1] += 100_000_000, + "role" => { + input.window.cohort.role = DataRole::Train; + input.window.cohort.label = RoomLabel::Empty; + } + "rows" => { + input.window.cohort.views[0].samples.truncate(8); + input.window.cohort.views[0].frame_ids.truncate(8); + input.window.cohort.views[0].sequence_ids.truncate(8); + input.window.row_times_ns[0].1.truncate(8); + } + _ => {} + } + assert!( + m.predict( + input, + if mode == "stale" { + now + 6_000_000_000 + } else { + now + } + ) + .is_err(), + "{mode}" + ); + } +} +fn recode(mut e: Envelope) -> Vec { + e.payload.input_evidence_sha256 = hash(&e.payload.evidence).unwrap(); + e.payload_sha256 = hash(&e.payload).unwrap(); + serde_json::to_vec(&e).unwrap() +} + +#[test] +fn cold_import_rejects_hash_policy_timing_source_and_cross_role_tampering() { + let bytes = fitted_bytes(); + for mode in [ + "outer_hash", + "model_hash", + "plan_hash", + "plan_horizon", + "source", + "rows", + "gap", + "frame_overlap", + "role", + "provenance", + ] { + let mut e: Envelope = serde_json::from_slice(&bytes).unwrap(); + match mode { + "outer_hash" => e.payload_sha256 = "0".repeat(64), + "model_hash" => e.payload.model_sha256 = "0".repeat(64), + "plan_hash" => e.payload.binding.plan_sha256 = "0".repeat(64), + "plan_horizon" => { + let p = AcquisitionPlan::build( + context(2), + ExperimentalPlanId::Native1000msSlots2, + &bands(2), + ) + .unwrap(); + e.payload.binding = Binding::from_plan(&p); + } + "source" => e.payload.evidence[0].sources[0].source_spec_sha256 = "f".repeat(64), + "rows" => { + let s = &mut e.payload.evidence[0].sources[0]; + s.times_ns.truncate(8); + s.frame_ids.truncate(8); + s.sequence_ids.truncate(8); + } + "gap" => { + let s = &mut e.payload.evidence[0].sources[0]; + let t = integer::(&s.times_ns[1]).unwrap(); + s.times_ns[1] = (t + 500_000_000).to_string(); + } + "frame_overlap" => { + e.payload.evidence[1].sources[0].frame_ids[0] = + e.payload.evidence[0].sources[0].frame_ids[0].clone() + } + "role" => e.payload.evidence[0].role = DataRole::HeldOut, + "provenance" => { + e.payload.evidence[0].provenance = CaptureProvenance::LiveReceiverObserved + } + _ => unreachable!(), + } + // Even recomputing generic envelope checksums cannot make a structurally + // incompatible plan/role/time/source claim satisfy the real input policy. + let altered = if mode == "outer_hash" { + serde_json::to_vec(&e).unwrap() + } else { + recode(e) + }; + assert!(TemporalModel::import(&altered).is_err(), "{mode}"); + } + assert!(TemporalModel::import(&vec![b' '; MAX_BYTES + 1]).is_err()); +} +#[test] +fn caller_cannot_fit_incomplete_or_failed_acquisition_as_a_complete_policy() { + let mut s = TemporalTrainingSession::new( + context(2), + ExperimentalPlanId::Native2000msSlots2, + &bands(2), + "failure".into(), + config(), + ) + .unwrap(); + let t = 10_000_000_000; + s.begin_window(t, t + 3_000_000_000).unwrap(); + s.ingest(&packet(1, 1, t + 1000, RoomLabel::Empty, 0)) + .unwrap(); + assert!(s.finish_window(t + 1_000_000_000).is_err()); + assert!(s.fit().is_err()); + assert!(s + .begin_window(t + 4_000_000_000, t + 7_000_000_000) + .is_err()); +} + +#[test] +fn cold_consistent_rate_time_shift_cannot_detach_original_receiver_times_from_frame_ids() { + let mut e: Envelope = serde_json::from_slice(&fitted_bytes()).unwrap(); + // Keep duration, row count, gaps, average cadence and source skew valid. + // Recomputed envelope hashes cannot make the unchanged ORIGINAL frame IDs + // refer to the shifted receiver times or a new original wire counter. + for source in &mut e.payload.evidence[0].sources { + for time in &mut source.times_ns { + *time = (integer::(time).unwrap() + 1000).to_string(); + } + } + let seal = integer::(&e.payload.evidence[0].sealed_at_ns).unwrap(); + e.payload.evidence[0].sealed_at_ns = (seal + 1000).to_string(); + assert!(TemporalModel::import(&recode(e)).is_err()); +} + +#[test] +fn post_assembler_evidence_failure_is_terminal_and_preserves_previous_valid_windows() { + let mut session = TemporalTrainingSession::new( + context(2), + ExperimentalPlanId::Native2000msSlots2, + &bands(2), + "post-consume".into(), + config(), + ) + .unwrap(); + let mut sequence = 0; + collect(&mut session, 10_000_000_000, &mut sequence); + let prior = hash(&session.rows).unwrap(); + let target = 20_000_000_000; + session + .begin_window(target, target + 3_000_000_000) + .unwrap(); + let period = 1_000_000_000u64.div_ceil(33); + let mut now = target; + 'ready: for row in 0..256 { + sequence += 1; + for source in 1..=2 { + now = target + row as u64 * period + u64::from(source) * 1000; + let mut packet = packet(source, sequence, now, RoomLabel::Empty, row); + if row == 0 && source == 1 { + packet.native_frame.frame_id ^= 1; + assert_ne!(packet.native_frame.frame_id, 0); + } + if session.ingest(&packet).unwrap() == WindowAdmission::Ready { + break 'ready; + } + } + } + assert_eq!( + session.finish_window(now).err().as_deref(), + Some("temporal_native_frame_identity_changed") + ); + assert!(session.failed); + assert!(!session.active); + assert!(session.target_ns.is_none()); + assert_eq!(session.rows.len(), 1); + assert_eq!(session.evidence.len(), 1); + assert_eq!(hash(&session.rows).unwrap(), prior); + assert!(session + .begin_window(30_000_000_000, 33_000_000_000) + .is_err()); + assert!(session + .ingest(&packet( + 1, + sequence + 1, + 30_000_001_000, + RoomLabel::Empty, + 0 + )) + .is_err()); + assert!(session.finish_window(now).is_err()); + assert!(session.fit().is_err()); +} + +// Frozen SYNTHETIC pre-C1 artifact produced by the actual native-horizon fitter +// on base #2465. Compatibility means preserving its serialized bytes, not +// requiring newly trained floating-point weights to match across platforms. +const PUBLIC_SEED_V2_GOLDEN: &[u8] = + include_bytes!("../../tests/fixtures/temporal-seed-v2-synthetic-golden.json"); + +#[test] +fn public_seed_v2_frozen_artifact_import_reexports_exact_bytes() { + assert_eq!(PUBLIC_SEED_V2_GOLDEN.len(), 113_803); + assert_eq!( + bytes_hash(PUBLIC_SEED_V2_GOLDEN), + "e46e9aff391b0e41a8fe26cc46789ee65d130f109df444726893303960b5735f" + ); + let frozen = TemporalModel::import(PUBLIC_SEED_V2_GOLDEN).unwrap(); + assert_eq!(frozen.context(), &context(2)); + assert_eq!(frozen.model.training_manifest().training_cohorts, 16); + assert_eq!(frozen.model.training_manifest().training_provenance, + vec![CaptureProvenance::Synthetic]); + assert_eq!(frozen.evidence.len(), 16); + assert!(frozen.evidence.iter().all(|e| e.role == DataRole::Train)); + assert_eq!(frozen.export().unwrap(), PUBLIC_SEED_V2_GOLDEN); +} + +#[test] +fn public_seed_v2_frozen_artifact_rehashed_role_tamper_is_rejected() { + let mut envelope: Envelope = serde_json::from_slice(PUBLIC_SEED_V2_GOLDEN).unwrap(); + envelope.payload.evidence[0].role = DataRole::HeldOut; + // Recompute the generic checksums: the real policy must still reject an + // artifact whose historical Train evidence was relabelled as HeldOut. + envelope.payload.input_evidence_sha256 = hash(&envelope.payload.evidence).unwrap(); + let altered = recode(envelope); + assert!(TemporalModel::import(&altered).is_err()); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_prefix_digests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_prefix_digests.rs new file mode 100644 index 0000000000..2cd0f642a5 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_prefix_digests.rs @@ -0,0 +1,286 @@ +//! Ephemeral exact canonical tuple-prefix SHA256 computation, OFFLOCK only. +//! No persistent cache, imported claims, validation waiver or authority issuer. +use serde::Serialize; +use sha2::{Digest, Sha256}; +use std::{ + collections::{BTreeMap, BTreeSet}, + io::{self, Write}, +}; +const MAX_CUTS: usize = 15; +const MAX_ROWS: usize = 128; + +pub(super) struct CanonicalPrefixDigests { + digests: BTreeMap, +} +impl CanonicalPrefixDigests { + pub(super) fn at(&self, cut: usize) -> Result<&str, String> { + self.digests + .get(&cut) + .map(String::as_str) + .ok_or_else(|| "temporal_prefix_digest_missing_cut".into()) + } +} +struct HashWriter<'a>(&'a mut Sha256); +impl Write for HashWriter<'_> { + fn write(&mut self, bytes: &[u8]) -> io::Result { + self.0.update(bytes); + Ok(bytes.len()) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} +struct Fork { + cut: usize, + state: Sha256, +} +struct FanoutWriter<'a> { + forks: &'a mut [Fork], + index: usize, +} +impl Write for FanoutWriter<'_> { + fn write(&mut self, bytes: &[u8]) -> io::Result { + for fork in self.forks.iter_mut().filter(|f| self.index < f.cut) { + fork.state.update(bytes); + } + Ok(bytes.len()) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} +/// EXACTLY SHA256(serde_json::to_vec(&(rows[..n],evidence[..n],spans[..n]))). +/// Requested duplicate cuts collapse; order does not affect results. At most15 +/// temporary SHA states. Only requested prefixes are serialized, never a tail +/// outside the largest cut; an empty request does not touch any input value. +pub(super) fn canonical_tuple_prefixes( + rows: &[R], + evidence: &[E], + spans: &[S], + cuts: &[usize], +) -> Result { + if rows.len() != evidence.len() + || rows.len() != spans.len() + || rows.len() > MAX_ROWS + || cuts.len() > MAX_CUTS + || cuts.iter().any(|n| *n > rows.len()) + { + return Err("temporal_prefix_digest_shape".into()); + } + if cuts.is_empty() { + return Ok(CanonicalPrefixDigests { + digests: BTreeMap::new(), + }); + } + let cuts = cuts.iter().copied().collect::>(); + let max = *cuts.last().unwrap(); + let encoding = |error: serde_json::Error| format!("temporal_prefix_digest_encoding:{error}"); + let mut raw = Sha256::new(); + raw.update(b"[["); + let mut forks = Vec::with_capacity(cuts.len()); + if cuts.contains(&0) { + forks.push(Fork { + cut: 0, + state: raw.clone(), + }); + } + for (index, row) in rows[..max].iter().enumerate() { + if index > 0 { + raw.update(b","); + } + serde_json::to_writer(HashWriter(&mut raw), row).map_err(&encoding)?; + if cuts.contains(&(index + 1)) { + forks.push(Fork { + cut: index + 1, + state: raw.clone(), + }); + } + } + for fork in &mut forks { + fork.state.update(b"],["); + } + for (index, item) in evidence[..max].iter().enumerate() { + if index > 0 { + for fork in forks.iter_mut().filter(|f| index < f.cut) { + fork.state.update(b","); + } + } + serde_json::to_writer( + FanoutWriter { + forks: &mut forks, + index, + }, + item, + ) + .map_err(&encoding)?; + } + for fork in &mut forks { + fork.state.update(b"],["); + } + for (index, item) in spans[..max].iter().enumerate() { + if index > 0 { + for fork in forks.iter_mut().filter(|f| index < f.cut) { + fork.state.update(b","); + } + } + serde_json::to_writer( + FanoutWriter { + forks: &mut forks, + index, + }, + item, + ) + .map_err(&encoding)?; + } + let mut digests = BTreeMap::new(); + for mut fork in forks { + fork.state.update(b"]]"); + digests.insert(fork.cut, format!("{:x}", fork.state.finalize())); + } + Ok(CanonicalPrefixDigests { digests }) +} +#[cfg(test)] +mod tests { + use super::*; + #[derive(Serialize)] + struct Row { + source: u8, + label: String, + samples: Vec, + single: f32, + optional: Option, + } + fn rows() -> Vec { + (0..128) + .map(|i| Row { + source: if i % 2 == 0 { 255 } else { 1 }, + label: format!("源\\\"\n\u{1}雪-{i}"), + samples: vec![ + -0.0, + 0.0, + f64::MAX, + f64::MIN, + f64::MIN_POSITIVE / 2.0, + i as f64 / 7.0, + ], + single: if i % 2 == 0 { + f32::MIN_POSITIVE / 2.0 + } else { + -0.0 + }, + optional: if i % 2 == 0 { + None + } else { + Some("é\\\"\r\t".into()) + }, + }) + .collect() + } + fn expected( + r: &[R], + e: &[E], + s: &[S], + n: usize, + ) -> String { + format!( + "{:x}", + Sha256::digest(serde_json::to_vec(&(&r[..n], &e[..n], &s[..n])).unwrap()) + ) + } + #[test] + fn every_cut_zero_through128_matches_actual_rust_tuple_bytes() { + let rows = rows(); + let evidence=(0..128).map(|i|serde_json::json!({"frame":format!("{}",u128::MAX-i),"unicode":"雪\n\\\"","id":i})).collect::>(); + let spans = (0..128) + .map(|i| (i, Some(i * 9), [i + 1, i + 2])) + .collect::>(); + for cuts in (0..=128).collect::>().chunks(15) { + let got = canonical_tuple_prefixes(&rows, &evidence, &spans, cuts).unwrap(); + for n in cuts { + assert_eq!( + got.at(*n).unwrap(), + expected(&rows, &evidence, &spans, *n), + "cut{n}" + ); + } + } + } + #[test] + fn exact_seven_train_and_eight_heldout_boundaries_match_reference() { + let rows = rows(); + let evidence = (0..128).map(|i| format!("e-{i}")).collect::>(); + let spans = (0..128).collect::>(); + let cuts = [ + 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, + ]; + let got = canonical_tuple_prefixes(&rows, &evidence, &spans, &cuts).unwrap(); + for n in cuts { + assert_eq!(got.at(n).unwrap(), expected(&rows, &evidence, &spans, n)); + } + } + #[test] + fn duplicate_unsorted_shared_references_and_repeat_calls_are_exact() { + let rows = rows(); + let repeated = vec![&rows[0]; 128]; + let evidence = vec![Some("same\\\n雪"); 128]; + let spans = vec![None::; 128]; + let cuts = [128, 0, 64, 64, 1, 0]; + let first = canonical_tuple_prefixes(&repeated, &evidence, &spans, &cuts).unwrap(); + let second = canonical_tuple_prefixes(&repeated, &evidence, &spans, &cuts).unwrap(); + assert_eq!(first.digests.len(), 4); + assert_eq!(first.digests, second.digests); + for n in cuts { + assert_eq!( + first.at(n).unwrap(), + expected(&repeated, &evidence, &spans, n) + ); + } + } + #[test] + fn malformed_lengths_capacity_and_cuts_fail_without_partial_hashes() { + assert!(canonical_tuple_prefixes(&[1], &[2, 3], &[4], &[0]).is_err()); + assert!(canonical_tuple_prefixes(&[1], &[2], &[3], &[2]).is_err()); + assert!(canonical_tuple_prefixes(&[1], &[2], &[3], &[0; 16]).is_err()); + assert!(canonical_tuple_prefixes(&[0; 129], &[0; 129], &[0; 129], &[1]).is_err()); + } + struct ErrorValue; + impl Serialize for ErrorValue { + fn serialize(&self, _serializer: T) -> Result { + Err(serde::ser::Error::custom("deliberate serializer failure")) + } + } + #[test] + fn serialization_errors_in_any_component_return_error_not_partial_results() { + assert!(canonical_tuple_prefixes(&[ErrorValue], &[1], &[2], &[0, 1]).is_err()); + assert!(canonical_tuple_prefixes(&[1], &[ErrorValue], &[2], &[0, 1]).is_err()); + assert!(canonical_tuple_prefixes(&[1], &[2], &[ErrorValue], &[0, 1]).is_err()); + } + #[test] + fn empty_request_and_irrelevant_tail_are_never_serialized() { + assert!( + canonical_tuple_prefixes(&[ErrorValue], &[ErrorValue], &[ErrorValue], &[]) + .unwrap() + .digests + .is_empty() + ); + #[derive(Serialize)] + #[serde(untagged)] + enum MaybeError { + Good(u8), + Bad(ErrorValue), + } + let rows = [MaybeError::Good(7), MaybeError::Bad(ErrorValue)]; + let got = canonical_tuple_prefixes(&rows, &[1, 2], &[3, 4], &[0, 1]).unwrap(); + assert_eq!(got.at(1).unwrap(), expected(&rows, &[1, 2], &[3, 4], 1)); + assert!(canonical_tuple_prefixes(&rows, &[1, 2], &[3, 4], &[2]).is_err()); + } + #[test] + fn empty_dataset_cut_zero_is_exact_empty_three_tuple() { + let empty: Vec = vec![]; + let got = canonical_tuple_prefixes(&empty, &empty, &empty, &[0]).unwrap(); + assert_eq!( + got.at(0).unwrap(), + format!("{:x}", Sha256::digest(b"[[],[],[]]")) + ); + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_qualification.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_qualification.rs new file mode 100644 index 0000000000..7f65f965be --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_qualification.rs @@ -0,0 +1,367 @@ +//! Whole-kind LOCAL operational qualification. Existing numeric criteria are +//! reused; no four-cohort aliases, physical validity or automatic selection. +use super::{ + artifact::StoredTemporalEvaluation, + temporal_model_input::managed::execution::full_run::VerifiedTemporalEvaluationSnapshot, + temporal_wire::*, +}; +use serde::{Deserialize, Serialize}; +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub(crate) struct TemporalQualificationEnvelope { + pub receipt_sha256: String, + pub receipt: TemporalQualificationReceipt, +} +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub(crate) struct TemporalQualificationReceipt { + pub schema: String, + pub owner_scope: String, + pub installation_id: String, + pub qualified_at_unix_ms: u64, + pub summary: TemporalQualificationSummaryWire, +} +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub(crate) struct TemporalAssessmentEnvelope { + pub receipt_sha256: String, + pub receipt: TemporalAssessmentReceipt, +} +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub(crate) struct TemporalAssessmentReceipt { + pub schema: String, + pub owner_scope: String, + pub installation_id: String, + pub summary: TemporalAssessmentWire, +} +/// Constructed only against the current protected Complete snapshot and exact +/// current-source verified evaluation. Imported DATA is not this result. +pub(crate) struct ComputedTemporalAssessment { + pub(super) qualification: Option, + pub(super) assessment: TemporalAssessmentEnvelope, +} +fn h(v: &T) -> Result { + super::qualification::hash(v) +} +impl ComputedTemporalAssessment { + pub(crate) fn issue_current( + source: &VerifiedTemporalEvaluationSnapshot, + evaluation: &StoredTemporalEvaluation, + now: u64, + ) -> Result { + if source.binding() != evaluation.binding() { + return Err("temporal_assessment_current_complete_changed".into()); + } + let proof = evaluation.train(); + let held = evaluation.held_out(); + let eval = evaluation.receipt(); + eval.validate_data_links(proof, held)?; + let operator = proof.is_operator_eight_step(); + let check_windows = if operator { 4 } else { 16 }; + let check_episodes = if operator { 1 } else { 2 }; + let id = &proof.body.identity; + if now / 1_000_000 >= id.original_deadline.expires_at_unix_ms + || now + < eval + .body + .evaluated_at_receiver_ns + .parse::() + .map_err(|_| "temporal_evaluation_data_integer")? + { + return Err("temporal_assessment_clock_or_original_deadline".into()); + } + if proof.body.windows_per_condition != [16; 4] + || held.body.windows_per_condition != [check_windows; 4] + || eval.body.report.samples != usize::from(check_windows) * 4 + || eval.body.report.per_condition.len() != 4 + || eval + .body + .report + .per_condition + .iter() + .any(|c| c.samples != usize::from(check_windows) || c.declared_trial_count != check_episodes) + { + return Err("temporal_assessment_exact_native_evidence_required".into()); + } + let policy = TemporalQualificationPolicyWire::for_operator_eight_step(operator); + let policy_sha = h(&policy)?; + let criteria = native_criteria(&eval.body.report, operator); + // Authoritative pass/failure is the established integer criteria, + // with exact versioned 16- or 4-window input shapes enforced above. + // The smaller operator validation set is not equal-confidence evidence. + let outcome = super::qualification::criteria(&eval.body.report, &[16; 4]); + let qualification = if outcome.is_ok() { + let mut summary = TemporalQualificationSummaryWire { + schema: QUALIFICATION_SCHEMA.into(), + receipt_sha256: String::new(), + model_kind: WholeTemporalKind::ManagedTemporalV3, + model_sha256: id.artifact_sha256.clone(), + verified_fit_record_sha256: id.verified_fit_record_sha256.clone(), + training_lineage_basis: if operator { "managed_temporal_training_proof_operator_eight_step_v1" } else { "managed_temporal_training_proof_v1" }.into(), + training_proof_sha256: proof.sha256.clone(), + held_out_evidence_sha256: held.sha256.clone(), + complete_record_sha256: held.body.complete_record_sha256.clone(), + evaluation_receipt_sha256: eval.sha256.clone(), + context_digest: id.context_digest.clone(), + room_digest: id.room_digest.clone(), + evaluated_at_unix_ms: eval.body.evaluated_at_unix_ms, + policy: policy.clone(), + policy_sha256: policy_sha.clone(), + label_provenance: if operator { "temporal-training-proof-operator-eight-step-v1+reserved-native-validation-v1" } else { "temporal-training-proof-v1+sealed-native-heldout-v1" }.into(), + criteria: criteria.clone(), + authority_scope: "local_operator_binary_presence".into(), + window_unit: "native_receiver_2s_window".into(), + train_windows: 64, + held_out_windows: if operator { 16 } else { 64 }, + train_episodes: if operator { 4 } else { 7 }, + held_out_episodes: if operator { 4 } else { 8 }, + independence_verified: false, + physically_validated: false, + measurement_clock_available: false, + }; + let receipt = TemporalQualificationReceipt { + schema: QUALIFICATION_SCHEMA.into(), + owner_scope: id.owner_scope.clone(), + installation_id: id.installation_id.clone(), + qualified_at_unix_ms: now / 1_000_000, + summary: summary.clone(), + }; + let sha = h(&receipt)?; + summary.receipt_sha256 = sha.clone(); + Some(TemporalQualificationEnvelope { + receipt_sha256: sha, + receipt: TemporalQualificationReceipt { summary, ..receipt }, + }) + } else { + None + }; + let condition_failures = eval + .body + .report + .per_condition + .iter() + .filter_map(|c| { + let codes = super::qualification::native_condition_failure_codes(c); + (!codes.is_empty()).then_some(TemporalConditionFailureWire { + condition: c.condition, + codes, + }) + }) + .collect(); + let summary = TemporalAssessmentWire { + schema: ASSESSMENT_SCHEMA.into(), + receipt_sha256: String::new(), + model_kind: WholeTemporalKind::ManagedTemporalV3, + model_sha256: id.artifact_sha256.clone(), + verified_fit_record_sha256: id.verified_fit_record_sha256.clone(), + training_proof_sha256: proof.sha256.clone(), + held_out_evidence_sha256: held.sha256.clone(), + complete_record_sha256: held.body.complete_record_sha256.clone(), + evaluation_receipt_sha256: eval.sha256.clone(), + context_digest: id.context_digest.clone(), + room_digest: id.room_digest.clone(), + evaluated_at_unix_ms: eval.body.evaluated_at_unix_ms, + assessed_at_unix_ms: now / 1_000_000, + policy, + policy_sha256: policy_sha, + window_unit: "native_receiver_2s_window".into(), + outcome: if qualification.is_some() { + "qualified" + } else { + "not_qualified" + } + .into(), + failure_reason: outcome.err(), + qualification_sha256: qualification.as_ref().map(|q| q.receipt_sha256.clone()), + criteria, + condition_failures, + }; + let mut receipt = TemporalAssessmentReceipt { + schema: ASSESSMENT_SCHEMA.into(), + owner_scope: id.owner_scope.clone(), + installation_id: id.installation_id.clone(), + summary, + }; + let sha = h(&receipt)?; + receipt.summary.receipt_sha256 = sha.clone(); + Ok(Self { + qualification, + assessment: TemporalAssessmentEnvelope { + receipt_sha256: sha, + receipt, + }, + }) + } +} + +// The public summary echoes the envelope digest; that echo is excluded from +// the canonical receipt hash to avoid a self-referential commitment. +impl TemporalQualificationEnvelope { + fn canonical_hash(&self) -> Result { + let mut receipt = self.receipt.clone(); + receipt.summary.receipt_sha256.clear(); + h(&receipt) + } + pub(crate) fn validate(&self, evaluation: &StoredTemporalEvaluation) -> Result<(), String> { + self.validate_links( + evaluation.train(), + evaluation.held_out(), + evaluation.receipt(), + ) + } + pub(crate) fn validate_links( + &self, + proof: &super::temporal_evaluation::TemporalTrainProofData, + held: &super::temporal_evaluation::TemporalHeldOutEvidenceData, + e: &super::temporal_evaluation::TemporalEvaluationReceiptData, + ) -> Result<(), String> { + e.validate_data_links(proof, held)?; + let q = &self.receipt.summary; + let id = &e.body.identity; + if self.receipt_sha256 != self.canonical_hash()? + || q.receipt_sha256 != self.receipt_sha256 + || self.receipt.schema != QUALIFICATION_SCHEMA + || q.schema != QUALIFICATION_SCHEMA + || self.receipt.owner_scope != id.owner_scope + || self.receipt.installation_id != id.installation_id + || q.model_sha256 != id.artifact_sha256 + || q.verified_fit_record_sha256 != id.verified_fit_record_sha256 + || q.training_proof_sha256 != proof.sha256 + || q.held_out_evidence_sha256 != held.sha256 + || q.complete_record_sha256 != held.body.complete_record_sha256 + || q.evaluation_receipt_sha256 != e.sha256 + || q.context_digest != id.context_digest + || q.room_digest != id.room_digest + || q.policy != TemporalQualificationPolicyWire::for_operator_eight_step(proof.is_operator_eight_step()) + || q.train_windows != 64 + || q.held_out_windows != if proof.is_operator_eight_step() { 16 } else { 64 } + || q.train_episodes != if proof.is_operator_eight_step() { 4 } else { 7 } + || q.held_out_episodes != if proof.is_operator_eight_step() { 4 } else { 8 } + || q.training_lineage_basis != if proof.is_operator_eight_step() { "managed_temporal_training_proof_operator_eight_step_v1" } else { "managed_temporal_training_proof_v1" } + || q.label_provenance != if proof.is_operator_eight_step() { "temporal-training-proof-operator-eight-step-v1+reserved-native-validation-v1" } else { "temporal-training-proof-v1+sealed-native-heldout-v1" } + || q.independence_verified || q.physically_validated || q.measurement_clock_available + || q.model_kind != WholeTemporalKind::ManagedTemporalV3 + || q.authority_scope != "local_operator_binary_presence" + || q.window_unit != "native_receiver_2s_window" + || q.policy_sha256 != h(&q.policy)? + || q.criteria != native_criteria(&e.body.report, proof.is_operator_eight_step()) + || q.evaluated_at_unix_ms != e.body.evaluated_at_unix_ms + || self.receipt.qualified_at_unix_ms < e.body.evaluated_at_unix_ms + || self.receipt.qualified_at_unix_ms >= id.original_deadline.expires_at_unix_ms + { + return Err("temporal_qualification_identity_changed".into()); + } + super::qualification::criteria(&e.body.report, &[16; 4]).map(|_| ()) + } +} +impl TemporalAssessmentEnvelope { + fn canonical_hash(&self) -> Result { + let mut receipt = self.receipt.clone(); + receipt.summary.receipt_sha256.clear(); + h(&receipt) + } + pub(crate) fn validate( + &self, + evaluation: &StoredTemporalEvaluation, + qualification: Option<&TemporalQualificationEnvelope>, + ) -> Result<(), String> { + self.validate_links( + evaluation.train(), + evaluation.held_out(), + evaluation.receipt(), + qualification, + ) + } + pub(crate) fn validate_links( + &self, + proof: &super::temporal_evaluation::TemporalTrainProofData, + held: &super::temporal_evaluation::TemporalHeldOutEvidenceData, + e: &super::temporal_evaluation::TemporalEvaluationReceiptData, + qualification: Option<&TemporalQualificationEnvelope>, + ) -> Result<(), String> { + e.validate_data_links(proof, held)?; + let a = &self.receipt.summary; + let id = &e.body.identity; + if self.receipt_sha256 != self.canonical_hash()? + || a.receipt_sha256 != self.receipt_sha256 + || self.receipt.schema != ASSESSMENT_SCHEMA + || a.schema != ASSESSMENT_SCHEMA + || self.receipt.owner_scope != id.owner_scope + || self.receipt.installation_id != id.installation_id + || a.model_sha256 != id.artifact_sha256 + || a.verified_fit_record_sha256 != id.verified_fit_record_sha256 + || a.training_proof_sha256 != proof.sha256 + || a.held_out_evidence_sha256 != held.sha256 + || a.complete_record_sha256 != held.body.complete_record_sha256 + || a.evaluation_receipt_sha256 != e.sha256 + || a.context_digest != id.context_digest + || a.room_digest != id.room_digest + || a.model_kind != WholeTemporalKind::ManagedTemporalV3 + || a.window_unit != "native_receiver_2s_window" + || a.policy != TemporalQualificationPolicyWire::for_operator_eight_step(proof.is_operator_eight_step()) + || a.policy_sha256 != h(&a.policy)? + || a.criteria != native_criteria(&e.body.report, proof.is_operator_eight_step()) + || a.evaluated_at_unix_ms != e.body.evaluated_at_unix_ms + || a.assessed_at_unix_ms < e.body.evaluated_at_unix_ms + || a.assessed_at_unix_ms >= id.original_deadline.expires_at_unix_ms + { + return Err("temporal_assessment_identity_changed".into()); + } + match qualification { + Some(q) => { + q.validate_links(proof, held, e)?; + if a.outcome != "qualified" + || a.qualification_sha256.as_deref() != Some(&q.receipt_sha256) + { + return Err("temporal_assessment_outcome_changed".into()); + } + } + None => { + if a.outcome != "not_qualified" + || a.qualification_sha256.is_some() + || super::qualification::criteria(&e.body.report, &[16; 4]).is_ok() + { + return Err("temporal_assessment_outcome_changed".into()); + } + } + } + Ok(()) + } +} + +fn native_criteria( + report: &super::training::EvaluationReport, + operator: bool, +) -> Vec { + report + .per_condition + .iter() + .map(|c| { + let occupied = matches!( + c.condition, + super::observation::RoomLabel::OccupiedStatic + | super::observation::RoomLabel::OccupiedMoving + ); + TemporalConditionCriterionWire { + condition: c.condition, + train_windows: 16, + held_out_windows: c.samples as u16, + scored_windows: c.scored as u16, + abstained_windows: c.abstained as u16, + missing_windows: c.missing as u16, + declared_train_episodes: if operator || c.condition == super::observation::RoomLabel::Empty { + 1 + } else { + 2 + }, + declared_held_out_episodes: if operator { 1 } else { 2 }, + evaluated_window_coverage: c.scored as f64 / c.samples as f64, + unconditional_sensitivity: occupied + .then_some(c.confusion.true_occupied as f64 / c.samples as f64), + false_positive_rate: (!occupied) + .then_some(c.confusion.false_occupied as f64 / c.samples as f64), + } + }) + .collect() +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_recovery_commitment.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_recovery_commitment.rs new file mode 100644 index 0000000000..cf6d301c78 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_recovery_commitment.rs @@ -0,0 +1,1385 @@ +//! Bounded selective-file consistency for private temporal prefixes. +//! NOT a trusted monotonic anchor against coordinated whole-profile rollback. +//! Typed APIs take receiver nanoseconds; descriptor/retention fields stay ms. +use super::*; +use crate::room_learner::temporal_model_input::managed::execution::checkpoint::{ + is_new_empty_run, validate_extension, validated_prefix, ValidatedTemporalPrefix, +}; +use sha2::{Digest, Sha256}; +// Storage directory/AAD framing stay unchanged. The schema field below is +// authenticated version DATA; new kind2 never creates a parallel namespace. +const KIND: &str = "temporal-prefix-commitment-v1"; +const FULL_KIND: &str = "temporal-prefix-commitment-v2"; +const LOCK: &str = "temporal-prefix.lock"; +const HEAD: &str = "temporal-head.bin"; +const MAGIC: &[u8; 8] = b"RVTCPR01"; +const SMALL: usize = 4096; +const MAX_TRANSITIONS: usize = 512; +const MAX_WITNESSES: usize = 2 * MAX_TRANSITIONS; +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +struct Meta { + schema: String, + generation: u64, + revision: usize, + descriptor: RecoveryDescriptor, + run: String, + policy_sha256: String, + transition_limit: usize, + window_count: usize, + receiver_ns: u64, + slot: Option, + payload_sha256: Option, + previous_sha256: Option, + retired_at_ms: Option, +} +fn hash(bytes: &[u8]) -> String { + format!("{:x}", Sha256::digest(bytes)) +} +impl Meta { + fn sha(&self) -> Result { + Ok(hash( + &serde_json::to_vec(self).map_err(|_| "temporal_metadata_encoding")?, + )) + } + fn token(&self) -> Result { + Ok(TemporalToken { + generation: self.generation, + revision: self.revision, + run: self.run.clone(), + id: self.descriptor.id.clone(), + commitment: self.sha()?, + }) + } + fn witness(&self) -> String { + format!("witness-{:016x}-{:04x}.bin", self.generation, self.revision) + } +} +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct TemporalToken { + generation: u64, + revision: usize, + run: String, + id: String, + commitment: String, +} +/// Read-only commitment DATA. Deserializing this is never a TemporalToken or +/// store/producer permission. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub(crate) struct TemporalCommitmentData { + pub generation:u64,pub revision:usize,pub run:String,pub id:String,pub commitment:String, +} +impl TemporalToken { + pub(crate) fn data(&self)->TemporalCommitmentData{ + TemporalCommitmentData{generation:self.generation,revision:self.revision,run:self.run.clone(),id:self.id.clone(),commitment:self.commitment.clone()} + } +} +#[derive(Clone)] +pub(crate) struct TemporalIntent { + expected: Option, + meta: Meta, +} +/// Authenticated observation only; not a completed durability-barrier certificate. +pub(crate) struct TemporalLoaded { + pub(crate) token: TemporalToken, + pub(crate) prefix: Option, +} +// NEGATIVE shape only, not an authentication primitive. Caller must first +// authenticate/canonical-validate metadata and enforce owner/clock/chain rules. +// Never accepts a positive payload or a generic limit+1 terminal. +fn legacy_saturated_terminal_shape(m:&Meta)->bool{ + m.schema==KIND && m.transition_limit==329 && m.revision==329 + && m.slot.is_none() && m.payload_sha256.is_none() + && m.previous_sha256.as_ref().is_some_and(|s|hex(s)) + && m.window_count<=64 + && m.retired_at_ms.is_some_and(|at|at>=m.descriptor.saved_at_ms && at==m.receiver_ns/1_000_000) +} +struct Components { + head: Option, + latest: Option, + witnesses: Vec<(String, Meta)>, +} +struct Locked { + file: File, + directory: PathBuf, +} +impl Drop for Locked { + fn drop(&mut self) { + let _ = self.file.unlock(); + } +} +impl RecoveryStore { + fn temporal_dir(&self) -> PathBuf { + self.directory.join(KIND) + } + fn temporal_lock(&self,create:bool)->Result{self.temporal_lock_mode(create,true)} + fn temporal_lock_mode(&self,create:bool,create_lock:bool)->Result{ + if create_lock{self.verify_directory()?;}else{self.verify_directory_read_only()?;} + let path = self.directory.join(LOCK); + let mut options = OpenOptions::new(); + options.read(true).write(true).create(create_lock); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + #[cfg(target_os = "macos")] + options.custom_flags(0x100); + #[cfg(target_os = "linux")] + options.custom_flags(0x20000); + } + let file = options.open(&path).map_err(|_| "temporal_lock_open")?; + private_metadata( + &file.metadata().map_err(|_| "temporal_lock_metadata")?, + false, + )?; + file.lock().map_err(|_| "temporal_lock_failed")?; + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + let a = file.metadata().map_err(|_| "temporal_lock_metadata")?; + let b = fs::symlink_metadata(&path).map_err(|_| "temporal_lock_inode_changed")?; + if b.file_type().is_symlink() || a.ino() != b.ino() || a.dev() != b.dev() { + return Err("temporal_lock_inode_changed".into()); + } + } + let directory = self.temporal_dir(); + if create { + private_directory(&directory)?; + File::open(&self.directory) + .and_then(|f| f.sync_all()) + .map_err(|_| "temporal_namespace_parent_fsync")?; + } else if !create_lock { + let m=fs::symlink_metadata(&directory).map_err(|_|"temporal_directory_read")?; + if !m.is_dir()||m.file_type().is_symlink(){return Err("temporal_directory_unsafe".into());}private_metadata(&m,true)?; + } else if directory.exists() { + private_directory(&directory)?; + } + Ok(Locked { file, directory }) + } + fn temporal_read(&self, path: &Path, limit: usize) -> Result>, String> { + let meta = match fs::symlink_metadata(path) { + Ok(m) => m, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(_) => return Err("temporal_read_failed".into()), + }; + if meta.file_type().is_symlink() || !meta.is_file() || meta.len() > limit as u64 { + return Err("temporal_unsafe_file".into()); + } + private_metadata(&meta, false)?; + let mut options = OpenOptions::new(); + options.read(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + #[cfg(target_os = "macos")] + options.custom_flags(0x100); + #[cfg(target_os = "linux")] + options.custom_flags(0x20000); + } + let file = options.open(path).map_err(|_| "temporal_read_failed")?; + let opened = file.metadata().map_err(|_| "temporal_read_failed")?; + private_metadata(&opened, false)?; + if !opened.is_file() || opened.len() > limit as u64 { + return Err("temporal_unsafe_file".into()); + } + let mut bytes = Vec::new(); + file.take((limit + 1) as u64) + .read_to_end(&mut bytes) + .map_err(|_| "temporal_read_failed")?; + if bytes.len() > limit { + return Err("temporal_read_budget".into()); + } + Ok(Some(bytes)) + } + fn temporal_seal(&self, domain: &str, plain: &[u8]) -> Result, String> { + let key = self.key.as_ref().ok_or("recovery_unavailable")?; + let aad = serde_json::to_vec(&(KIND, domain, &self.owner, &self.installation)) + .map_err(|_| "temporal_aad")?; + let mut nonce = [0; 12]; + SystemRandom::new() + .fill(&mut nonce) + .map_err(|_| "recovery_random_unavailable")?; + let mut cipher = plain.to_vec(); + key.seal_in_place_append_tag( + aead::Nonce::assume_unique_for_key(nonce), + aead::Aad::from(&aad), + &mut cipher, + ) + .map_err(|_| "temporal_encrypt")?; + let mut out = MAGIC.to_vec(); + out.extend_from_slice(&nonce); + out.extend(cipher); + Ok(out) + } + fn temporal_open(&self, domain: &str, bytes: &[u8], limit: usize) -> Result, String> { + if bytes.len() < 36 || bytes.len() > limit + 36 || &bytes[..8] != MAGIC { + return Err("temporal_envelope".into()); + } + let key = self.key.as_ref().ok_or("recovery_unavailable")?; + let aad = serde_json::to_vec(&(KIND, domain, &self.owner, &self.installation)) + .map_err(|_| "temporal_aad")?; + let nonce = + aead::Nonce::try_assume_unique_for_key(&bytes[8..20]).map_err(|_| "temporal_nonce")?; + let mut cipher = bytes[20..].to_vec(); + let p = key + .open_in_place(nonce, aead::Aad::from(&aad), &mut cipher) + .map_err(|_| "temporal_authentication")?; + if p.len() > limit { + return Err("temporal_plaintext_budget".into()); + } + Ok(p.to_vec()) + } + fn temporal_meta(&self, path: &Path, domain: &str) -> Result, String> { + let Some(bytes) = self.temporal_read(path, SMALL + 36)? else { + return Ok(None); + }; + let p = self.temporal_open(domain, &bytes, SMALL)?; + let m: Meta = serde_json::from_slice(&p).map_err(|_| "temporal_metadata_schema")?; + if serde_json::to_vec(&m).map_err(|_| "temporal_metadata_encoding")? != p + || !matches!(m.schema.as_str(),KIND|FULL_KIND) + || m.transition_limit > MAX_TRANSITIONS + || m.transition_limit < 2 + || !id(&m.run) + || !hex(&m.policy_sha256) + || m.window_count > if m.schema==FULL_KIND{128}else{64} + { + return Err("temporal_metadata_binding".into()); + } + self.descriptor(&m.descriptor, None)?; + if m.descriptor.expires_at_ms - m.descriptor.created_at_ms != RECOVERY_TTL_MS { + return Err("temporal_original_deadline".into()); + } + if m.receiver_ns / 1_000_000 < m.descriptor.saved_at_ms { + return Err("temporal_metadata_receiver_clock".into()); + } + match (m.slot, m.payload_sha256.as_ref(), m.retired_at_ms) { + (Some(slot), Some(sha), None) + if slot < 2 + && hex(sha) + && m.receiver_ns / 1_000_000 == m.descriptor.saved_at_ms => {} + (None, None, Some(t)) + if t >= m.descriptor.saved_at_ms && t == m.receiver_ns / 1_000_000 => {} + _ => return Err("temporal_metadata_state".into()), + }; + // Only after AEAD/canonical metadata, known-kind/descriptor/owner and + // state/clock validation may the old-family NEGATIVE terminal occupy + // its one reserved boundary. Positive payloads stay strictly =m.transition_limit && !legacy_saturated_terminal_shape(&m){return Err("temporal_metadata_binding".into());} + Ok(Some(m)) + } + fn temporal_components(&self, dir: &Path) -> Result { + if !dir.exists() { + return Ok(Components { + head: None, + latest: None, + witnesses: vec![], + }); + } + let mut witnesses = Vec::new(); + let mut count = 0; + for entry in fs::read_dir(dir).map_err(|_| "temporal_directory_read")? { + count += 1; + if count > MAX_WITNESSES + 6 { + return Err("temporal_directory_budget".into()); + } + let e = entry.map_err(|_| "temporal_directory_read")?; + let name = e + .file_name() + .into_string() + .map_err(|_| "temporal_unknown_path")?; + let entry_meta = + fs::symlink_metadata(e.path()).map_err(|_| "temporal_directory_read")?; + if entry_meta.file_type().is_symlink() + || !entry_meta.is_file() + || entry_meta.len() + > (if name.starts_with("witness-") + || [HEAD, "head.stage", "witness.stage"].contains(&name.as_str()) + { + SMALL + 36 + } else { + MAX_RECOVERY_BYTES + SMALL + 48 + }) as u64 + { + return Err("temporal_unsafe_directory_entry".into()); + } + private_metadata(&entry_meta, false)?; + if name.starts_with("witness-") && name.ends_with(".bin") { + let m = self + .temporal_meta(&e.path(), "witness")? + .ok_or("temporal_witness_missing")?; + if name != m.witness() { + return Err("temporal_witness_name".into()); + } + witnesses.push((name, m)); + } else if ![ + HEAD, + "slot-0.bin", + "slot-1.bin", + "payload.stage", + "head.stage", + "witness.stage", + ] + .contains(&name.as_str()) + { + return Err("temporal_unknown_path".into()); + } + } + witnesses.sort_by_key(|(_, m)| (m.generation, m.revision)); + let generations = witnesses + .iter() + .map(|(_, m)| m.generation) + .collect::>(); + if generations.len() > 2 { + return Err("temporal_generation_capacity".into()); + } + // GC may already have removed any prefix of an OBSOLETE generation. + // Authenticate every remaining record above, but require a COMPLETE + // contiguous chain for the highest committed-witness generation. The + // head/payload must match its exact tip before any caller gets authority. + let highest = witnesses.last().map(|(_, m)| m.generation); + let mut previous: Option<&Meta> = None; + for (_, m) in witnesses + .iter() + .filter(|(_, m)| Some(m.generation) == highest) + { + if let Some(p) = previous { + if m.revision != p.revision + 1 || m.previous_sha256.as_ref() != Some(&p.sha()?) { + return Err("temporal_witness_chain".into()); + } + } else if m.revision != 0 + || (m.generation == 0 && m.previous_sha256.is_some()) + || (m.generation > 0 && !m.previous_sha256.as_ref().is_some_and(|p| hex(p))) + { + return Err("temporal_witness_genesis".into()); + } + previous = Some(m); + } + let latest = witnesses.last().map(|(_, m)| m.clone()); + let head = self.temporal_meta(&dir.join(HEAD), "head")?; + Ok(Components { + head, + latest, + witnesses, + }) + } + fn temporal_current(&self, dir: &Path, now_ns: u64) -> Result, String> { + let c = self.temporal_components(dir)?; + match (c.head, c.latest) { + (None, None) => Ok(None), + (Some(h), Some(w)) if h == w => { + if now_ns < h.receiver_ns + || now_ns / 1_000_000 < h.descriptor.saved_at_ms + || h.retired_at_ms.is_some_and(|t| now_ns / 1_000_000 < t) + { + return Err("recovery_clock_regression".into()); + } + let token = h.token()?; + if h.retired_at_ms.is_some() { + return Ok(Some(TemporalLoaded { + token, + prefix: None, + })); + } + if now_ns / 1_000_000 >= h.descriptor.expires_at_ms { + return Err("recovery_expired".into()); + } + let prefix = self.temporal_payload(dir, &h, now_ns)?; + Ok(Some(TemporalLoaded { + token, + prefix: Some(prefix), + })) + } + _ => Err("temporal_head_witness_mismatch_or_pending".into()), + } + } + fn temporal_payload_seal( + &self, + m: &Meta, + p: &ValidatedTemporalPrefix, + ) -> Result, String> { + let metadata = serde_json::to_vec(m).map_err(|_| "temporal_metadata_encoding")?; + if metadata.len() > SMALL { + return Err("temporal_metadata_budget".into()); + } + let mut out = b"RVTPAY01".to_vec(); + out.extend_from_slice(&(metadata.len() as u32).to_be_bytes()); + out.extend(metadata); + out.extend(self.temporal_seal(&format!("payload:{}", m.sha()?), p.bytes())?); + Ok(out) + } + fn temporal_payload_any( + &self, + path: &Path, + now_ns: u64, + ) -> Result, String> { + let Some(bytes) = self.temporal_read(path, MAX_RECOVERY_BYTES + SMALL + 48)? else { + return Ok(None); + }; + if bytes.len() < 12 || &bytes[..8] != b"RVTPAY01" { + return Err("temporal_payload_header".into()); + } + let n = u32::from_be_bytes( + bytes[8..12] + .try_into() + .map_err(|_| "temporal_payload_header")?, + ) as usize; + if n == 0 || n > SMALL || bytes.len() < 12 + n + 36 { + return Err("temporal_payload_header".into()); + } + let m: Meta = + serde_json::from_slice(&bytes[12..12 + n]).map_err(|_| "temporal_payload_metadata")?; + if serde_json::to_vec(&m).map_err(|_| "temporal_metadata_encoding")? != bytes[12..12 + n] + || !matches!(m.schema.as_str(),KIND|FULL_KIND) + || m.retired_at_ms.is_some() + { + return Err("temporal_payload_metadata".into()); + } + let plain = self.temporal_open( + &format!("payload:{}", m.sha()?), + &bytes[12 + n..], + MAX_RECOVERY_BYTES, + )?; + if Some(hash(&plain)).as_ref() != m.payload_sha256.as_ref() { + return Err("temporal_payload_hash".into()); + } + let p = validated_prefix(&plain, &m.descriptor, now_ns)?; + if p.run() != m.run + ||p.metadata_kind()!=m.schema + || p.policy_sha256() != m.policy_sha256 + || p.window_count() != m.window_count + || p.transition_limit() != m.transition_limit + || p.saved_receiver_ns() != m.receiver_ns + { + return Err("temporal_payload_metadata".into()); + } + Ok(Some((m, p))) + } + fn temporal_payload( + &self, + dir: &Path, + m: &Meta, + now_ns: u64, + ) -> Result { + let slot = m.slot.ok_or("temporal_retired")?; + let (actual, p) = self + .temporal_payload_any(&dir.join(format!("slot-{slot}.bin")), now_ns)? + .ok_or("temporal_payload_missing")?; + if actual != *m { + return Err("temporal_named_payload_changed".into()); + } + Ok(p) + } + pub(crate) fn load_temporal(&self, now_ns: u64) -> Result, String> { + let _serial = self + .serial + .lock() + .map_err(|_| "recovery_store_unavailable")?; + let locked = self.temporal_lock(false)?; + self.temporal_current(&locked.directory, now_ns) + } +} + +impl RecoveryStore { + pub(crate) fn prepare_temporal( + &self, + expected: Option<&TemporalToken>, + prefix: &ValidatedTemporalPrefix, + now_ns: u64, + ) -> Result { + if !prefix.authorizes_write(expected){return Err("temporal_full_run_producer_write_grant_required".into());} + let _serial = self + .serial + .lock() + .map_err(|_| "recovery_store_unavailable")?; + let locked = self.temporal_lock(true)?; + self.descriptor(prefix.descriptor(), Some(now_ns / 1_000_000))?; + if now_ns < prefix.saved_receiver_ns() { + return Err("recovery_receiver_clock_regression".into()); + } + let current = self.temporal_current(&locked.directory, now_ns)?; + let c = self.temporal_components(&locked.directory)?; + let (generation, revision, previous, slot) = match (current, expected) { + (None, None) => { + if fs::read_dir(&locked.directory) + .map_err(|_| "temporal_directory_read")? + .next() + .is_some() + || !is_new_empty_run(prefix)? + { + return Err("temporal_explicit_empty_genesis_required".into()); + } + (0, 0, None, 0) + } + (Some(loaded), Some(expected)) if &loaded.token == expected => { + let old = c.head.as_ref().ok_or("temporal_head_missing")?; + if let Some(previous_prefix) = loaded.prefix { + validate_extension(&previous_prefix, prefix)?; + if previous_prefix.bytes() == prefix.bytes() { + return Ok(TemporalIntent { + expected: Some(expected.clone()), + meta: old.clone(), + }); + } + if self.temporal_cleanup_pending(&locked.directory, &c, old)? { + return Err("temporal_cleanup_repair_required".into()); + } + ( + old.generation, + old.revision + .checked_add(1) + .ok_or("temporal_revision_overflow")?, + Some(old.sha()?), + 1 - old.slot.ok_or("temporal_retired")?, + ) + } else { + if self.temporal_cleanup_pending(&locked.directory, &c, old)? { + return Err("temporal_cleanup_repair_required".into()); + } + if !is_new_empty_run(prefix)? + || prefix.descriptor().id == old.descriptor.id + || prefix.run() == old.run + || prefix.descriptor().created_at_ms + < old.retired_at_ms.ok_or("temporal_retirement_missing")? + || prefix.saved_receiver_ns() < old.receiver_ns + { + return Err("temporal_explicit_new_empty_run_required".into()); + } + ( + old.generation + .checked_add(1) + .ok_or("temporal_generation_overflow")?, + 0, + Some(old.sha()?), + 0, + ) + } + } + _ => return Err("temporal_expected_token_changed".into()), + }; + if revision >= prefix.transition_limit() { + return Err("temporal_transition_budget".into()); + } + Ok(TemporalIntent { + expected: expected.cloned(), + meta: Meta { + schema: prefix.metadata_kind().into(), + generation, + revision, + descriptor: prefix.descriptor().clone(), + run: prefix.run().into(), + policy_sha256: prefix.policy_sha256().into(), + transition_limit: prefix.transition_limit(), + window_count: prefix.window_count(), + receiver_ns: prefix.saved_receiver_ns(), + slot: Some(slot), + payload_sha256: Some(hash(prefix.bytes())), + previous_sha256: previous, + retired_at_ms: None, + }, + }) + } + pub(crate) fn prepare_retire_temporal( + &self, + expected: &TemporalToken, + now_ns: u64, + expiry_only: bool, + ) -> Result { + let _serial = self + .serial + .lock() + .map_err(|_| "recovery_store_unavailable")?; + let locked = self.temporal_lock(false)?; + let c = self.temporal_components(&locked.directory)?; + let mut meta = c + .head + .filter(|h| Some(h) == c.latest.as_ref()) + .ok_or("temporal_head_witness_mismatch_or_pending")?; + if meta.token()? != *expected + || now_ns < meta.receiver_ns + || now_ns / 1_000_000 < meta.descriptor.saved_at_ms + || meta.retired_at_ms.is_some_and(|t| now_ns / 1_000_000 < t) + || expiry_only && now_ns / 1_000_000 < meta.descriptor.expires_at_ms + { + return Err("temporal_retirement_identity_or_clock".into()); + } + if meta.retired_at_ms.is_none() { + meta.previous_sha256 = Some(meta.sha()?); + meta.revision = meta + .revision + .checked_add(1) + .ok_or("temporal_revision_overflow")?; + meta.slot = None; + meta.payload_sha256 = None; + meta.retired_at_ms = Some(now_ns / 1_000_000); + meta.receiver_ns = now_ns; + if meta.revision >= meta.transition_limit && !legacy_saturated_terminal_shape(&meta) { + return Err("temporal_transition_budget".into()); + } + } + Ok(TemporalIntent { + expected: Some(expected.clone()), + meta, + }) + } + fn temporal_write_stage( + &self, + dir: &Path, + stage: &str, + bytes: &[u8], + hook: &mut impl FnMut(&str) -> Result<(), String>, + ) -> Result<(), String> { + hook(&format!("before:{stage}:write"))?; + let path = dir.join(stage); + if let Some(existing) = self.temporal_read(&path, MAX_RECOVERY_BYTES + SMALL + 48)? { + if existing != bytes { + return Err("temporal_conflicting_stage_requires_repair".into()); + } + let f = private_existing_file(&path, true)?; + f.sync_all().map_err(|_| "temporal_stage_fsync")?; + return Ok(()); + } + let mut options = OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + let mut file = options.open(&path).map_err(|_| "temporal_stage_create")?; + file.write_all(bytes).map_err(|_| "temporal_stage_write")?; + hook(&format!("after:{stage}:write"))?; + hook(&format!("before:{stage}:fsync"))?; + file.sync_all().map_err(|_| "temporal_stage_fsync")?; + hook(&format!("after:{stage}:fsync"))?; + Ok(()) + } + fn temporal_replace( + &self, + dir: &Path, + stage: &str, + name: &str, + hook: &mut impl FnMut(&str) -> Result<(), String>, + ) -> Result<(), String> { + hook(&format!("before:{stage}:rename"))?; + fs::rename(dir.join(stage), dir.join(name)).map_err(|_| "temporal_stage_rename")?; + hook(&format!("after:{stage}:rename"))?; + hook(&format!("before:{stage}:directory_fsync"))?; + File::open(dir) + .and_then(|f| f.sync_all()) + .map_err(|_| "temporal_directory_fsync")?; + hook(&format!("after:{stage}:directory_fsync"))?; + Ok(()) + } + fn temporal_cleanup_pending( + &self, + dir: &Path, + c: &Components, + current: &Meta, + ) -> Result { + if c.witnesses + .iter() + .any(|(_, m)| m.generation < current.generation) + { + return Ok(true); + } + let mut names = vec!["payload.stage", "head.stage", "witness.stage"]; + if current.retired_at_ms.is_some() { + names.extend(["slot-0.bin", "slot-1.bin"]); + } + for name in names { + match fs::symlink_metadata(dir.join(name)) { + Ok(_) => return Ok(true), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + Err(_) => return Err("temporal_cleanup_inspection".into()), + } + } + Ok(false) + } + fn temporal_cleanup(&self, dir: &Path, current: &Meta) -> Result<(), String> { + let c = self.temporal_components(dir)?; + for (name, m) in c.witnesses { + if m.generation < current.generation { + fs::remove_file(dir.join(name)).map_err(|_| "temporal_metadata_cleanup")?; + } + } + for name in ["payload.stage", "head.stage", "witness.stage"] { + if self + .temporal_read(&dir.join(name), MAX_RECOVERY_BYTES + SMALL + 48)? + .is_some() + { + fs::remove_file(dir.join(name)).map_err(|_| "temporal_stage_cleanup")?; + } + } + if current.retired_at_ms.is_some() { + for name in ["slot-0.bin", "slot-1.bin"] { + if self + .temporal_read(&dir.join(name), MAX_RECOVERY_BYTES + SMALL + 48)? + .is_some() + { + fs::remove_file(dir.join(name)).map_err(|_| "temporal_payload_cleanup")?; + } + } + } + File::open(dir) + .and_then(|f| f.sync_all()) + .map_err(|_| "temporal_cleanup_fsync")?; + Ok(()) + } + pub(crate) fn publish_temporal( + &self, + intent: &TemporalIntent, + prefix: Option<&ValidatedTemporalPrefix>, + now_ns: u64, + ) -> Result { + self.publish_temporal_with(intent, prefix, now_ns, false, &mut |_| Ok(())) + } + pub(crate) fn repair_same_temporal( + &self, + intent: &TemporalIntent, + prefix: Option<&ValidatedTemporalPrefix>, + now_ns: u64, + ) -> Result { + self.publish_temporal_with(intent, prefix, now_ns, true, &mut |_| Ok(())) + } + fn publish_temporal_with( + &self, + intent: &TemporalIntent, + prefix: Option<&ValidatedTemporalPrefix>, + now_ns: u64, + repair: bool, + hook: &mut impl FnMut(&str) -> Result<(), String>, + ) -> Result { + let _serial = self + .serial + .lock() + .map_err(|_| "recovery_store_unavailable")?; + let locked = self.temporal_lock(true)?; + let dir = &locked.directory; + let meta = &intent.meta; + if now_ns < meta.receiver_ns { + return Err("recovery_receiver_clock_regression".into()); + } + if let Some(p) = prefix { + self.descriptor(&meta.descriptor, Some(now_ns / 1_000_000))?; + if meta.retired_at_ms.is_some() + || p.descriptor() != &meta.descriptor + || p.run() != meta.run + ||p.metadata_kind()!=meta.schema + || p.policy_sha256() != meta.policy_sha256 + || Some(hash(p.bytes())).as_ref() != meta.payload_sha256.as_ref() + { + return Err("temporal_intended_payload_changed".into()); + } + } else if meta.retired_at_ms.is_none() || now_ns / 1_000_000 < meta.retired_at_ms.unwrap() { + return Err("temporal_intended_retirement_changed".into()); + } + let c = self.temporal_components(dir)?; + let latest = c.latest.as_ref().map(Meta::token).transpose()?; + let target = meta.token()?; + let head = c.head.as_ref().map(Meta::token).transpose()?; + let already = latest.as_ref() == Some(&target); + if already { + if !repair && intent.expected.as_ref() != Some(&target) { + return Err("temporal_exact_repair_required".into()); + } + if head.as_ref() != Some(&target) && !repair { + return Err("temporal_head_witness_mismatch_or_pending".into()); + } + } else { + if latest != intent.expected { + return Err("temporal_expected_token_changed".into()); + } + if head != intent.expected && !(repair && head.as_ref() == Some(&target)) { + return Err("temporal_exact_pending_repair_required".into()); + } + } + if already && !repair && head.as_ref() == Some(&target) && meta.retired_at_ms.is_none() { + if self.temporal_cleanup_pending(dir, &c, meta)? { + return Err("temporal_cleanup_repair_required".into()); + } + let committed = self + .temporal_current(dir, now_ns)? + .ok_or("temporal_committed_readback")?; + if let (Some(a), Some(b)) = (committed.prefix, prefix) { + if a.bytes() != b.bytes() { + return Err("temporal_noop_conflict".into()); + } + } + return Ok(target); + } + if !already { + if let (Some(old), Some(p)) = (c.latest.as_ref(), prefix) { + if old.retired_at_ms.is_none() { + let previous = self.temporal_payload(dir, old, now_ns)?; + validate_extension(&previous, p)?; + } else if !is_new_empty_run(p)? { + return Err("temporal_new_run_not_empty".into()); + } + } + } + if !already { + if let Some(slot) = meta.slot { + if let Some((pending, _)) = + self.temporal_payload_any(&dir.join(format!("slot-{slot}.bin")), now_ns)? + { + let candidate = pending.previous_sha256.as_ref() + == meta.previous_sha256.as_ref() + && pending.generation == meta.generation + && pending.revision == meta.revision; + if candidate && pending != *meta { + return Err("temporal_conflicting_pending_payload".into()); + } + } + } + } + // A repair removes only known staging artifacts under the stable lock; its + // immutable SAME intended metadata/payload is re-established through barriers. + if repair { + for name in ["payload.stage", "head.stage", "witness.stage"] { + if self + .temporal_read(&dir.join(name), MAX_RECOVERY_BYTES + SMALL + 48)? + .is_some() + { + let matching = if name == "payload.stage" { + self.temporal_payload_any(&dir.join(name), now_ns)? + .is_some_and(|(m, _)| m == *meta) + } else { + self.temporal_meta( + &dir.join(name), + if name == "head.stage" { + "head" + } else { + "witness" + }, + )? + .as_ref() + == Some(meta) + }; + if !matching { + return Err("temporal_conflicting_pending_stage".into()); + } + fs::remove_file(dir.join(name)).map_err(|_| "temporal_repair_stage_cleanup")?; + } + } + File::open(dir) + .and_then(|f| f.sync_all()) + .map_err(|_| "temporal_repair_directory_fsync")?; + } + if let Some(p) = prefix { + let cipher = self.temporal_payload_seal(meta, p)?; + self.temporal_write_stage(dir, "payload.stage", &cipher, hook)?; + self.temporal_replace( + dir, + "payload.stage", + &format!("slot-{}.bin", meta.slot.ok_or("temporal_slot_missing")?), + hook, + )?; + hook("before:payload:readback")?; + let actual = self.temporal_payload(dir, meta, now_ns)?; + if actual.bytes() != p.bytes() { + return Err("temporal_payload_readback".into()); + } + hook("after:payload:readback")?; + } + let plain = serde_json::to_vec(meta).map_err(|_| "temporal_metadata_encoding")?; + if plain.len() > SMALL { + return Err("temporal_metadata_budget".into()); + } + let head_cipher = self.temporal_seal("head", &plain)?; + self.temporal_write_stage(dir, "head.stage", &head_cipher, hook)?; + self.temporal_replace(dir, "head.stage", HEAD, hook)?; + hook("before:head:readback")?; + if self.temporal_meta(&dir.join(HEAD), "head")?.as_ref() != Some(meta) { + return Err("temporal_head_readback".into()); + } + hook("after:head:readback")?; + let witness = dir.join(meta.witness()); + if let Some(existing) = self.temporal_meta(&witness, "witness")? { + if existing != *meta { + return Err("temporal_conflicting_committed_witness".into()); + } + private_existing_file(&witness, false)? + .sync_all() + .map_err(|_| "temporal_witness_fsync")?; + File::open(dir) + .and_then(|f| f.sync_all()) + .map_err(|_| "temporal_witness_directory_fsync")?; + } else { + let cipher = self.temporal_seal("witness", &plain)?; + self.temporal_write_stage(dir, "witness.stage", &cipher, hook)?; + hook("before:witness:install")?; + fs::hard_link(dir.join("witness.stage"), &witness) + .map_err(|_| "temporal_witness_install")?; + hook("after:witness:install")?; + File::open(dir) + .and_then(|f| f.sync_all()) + .map_err(|_| "temporal_witness_directory_fsync")?; + hook("after:witness:directory_fsync")?; + fs::remove_file(dir.join("witness.stage")) + .map_err(|_| "temporal_witness_stage_cleanup")?; + } + hook("before:committed:readback")?; + let loaded = self + .temporal_current(dir, now_ns)? + .ok_or("temporal_committed_readback")?; + if loaded.token != target { + return Err("temporal_committed_readback".into()); + } + hook("after:committed:readback")?; + hook("before:cleanup")?; + self.temporal_cleanup(dir, meta)?; + hook("after:cleanup")?; + Ok(target) + } +} + +#[cfg(test)] +#[path = "temporal_recovery_commitment_tests.rs"] +mod tests; + +fn private_existing_file(path: &Path, write: bool) -> Result { + let mut options = OpenOptions::new(); + options.read(true).write(write); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + #[cfg(target_os = "macos")] + options.custom_flags(0x100); + #[cfg(target_os = "linux")] + options.custom_flags(0x20000); + } + let file = options.open(path).map_err(|_| "temporal_private_open")?; + let m = file.metadata().map_err(|_| "temporal_private_metadata")?; + if !m.is_file() { + return Err("temporal_private_file_required".into()); + } + private_metadata(&m, false)?; + Ok(file) +} + +#[derive(Clone, Debug, PartialEq, Eq)] +enum AbortArtifact { + PayloadStage, + HeadStage, + WitnessStage, + InactiveSlot(u8), +} +impl AbortArtifact { + fn name(&self) -> String { + match self { + Self::PayloadStage => "payload.stage".into(), + Self::HeadStage => "head.stage".into(), + Self::WitnessStage => "witness.stage".into(), + Self::InactiveSlot(slot) => format!("slot-{slot}.bin"), + } + } +} +/// Private explicit-abort snapshot, never a caller-provided path or grant. +#[derive(Clone)] +pub(crate) struct TemporalAbortTicket { + expected: TemporalToken, + artifact: AbortArtifact, + digest: String, +} +impl RecoveryStore { + fn temporal_abortable( + &self, + dir: &Path, + c: &Components, + artifact: &AbortArtifact, + bytes: &[u8], + now_ns: u64, + ) -> Result { + let head = c.head.as_ref().ok_or("temporal_head_missing")?; + if head.retired_at_ms.is_some() { + return Ok(false); + } + if let AbortArtifact::InactiveSlot(slot) = artifact { + if *slot > 1 || Some(*slot) == head.slot { + return Ok(false); + } + } + let digest = hash(bytes); + // Both exact ciphertext copies/hardlinks and independently encrypted records + // naming a committed semantic identity are protected from ABORT. + if self + .temporal_read(&dir.join(HEAD), SMALL + 36)? + .is_some_and(|b| hash(&b) == digest) + { + return Ok(false); + } + for (name, m) in &c.witnesses { + if self + .temporal_read(&dir.join(name), SMALL + 36)? + .is_some_and(|b| hash(&b) == digest) + { + return Ok(false); + } + if let Some(slot) = m.slot { + let file = dir.join(format!("slot-{slot}.bin")); + if let Ok(Some((actual, _))) = self.temporal_payload_any(&file, now_ns) { + if actual.sha()? == m.sha()? + && self + .temporal_read(&file, MAX_RECOVERY_BYTES + SMALL + 48)? + .is_some_and(|b| hash(&b) == digest) + { + return Ok(false); + } + } + } + } + let identity = match artifact { + AbortArtifact::PayloadStage | AbortArtifact::InactiveSlot(_) => self + .temporal_payload_any(&dir.join(artifact.name()), now_ns) + .ok() + .flatten() + .map(|(m, _)| m), + AbortArtifact::HeadStage => self + .temporal_meta(&dir.join(artifact.name()), "head") + .ok() + .flatten(), + AbortArtifact::WitnessStage => self + .temporal_meta(&dir.join(artifact.name()), "witness") + .ok() + .flatten(), + }; + if let Some(m) = identity { + if c.witnesses + .iter() + .any(|(_, w)| w.sha().ok() == m.sha().ok()) + { + return Ok(false); + } + if matches!(artifact, AbortArtifact::InactiveSlot(_)) + && (m.previous_sha256.as_ref() != Some(&head.sha()?) + || m.generation != head.generation + || m.revision != head.revision + 1) + { + return Ok(false); + } + } else if matches!(artifact, AbortArtifact::InactiveSlot(_)) { + return Ok(false); + } + Ok(true) + } + pub(crate) fn inspect_abortable_temporal( + &self, + expected: &TemporalToken, + now_ns: u64, + ) -> Result, String> { + let _serial = self + .serial + .lock() + .map_err(|_| "recovery_store_unavailable")?; + let locked = self.temporal_lock(false)?; + let loaded = self + .temporal_current(&locked.directory, now_ns)? + .ok_or("temporal_committed_prefix_missing")?; + if loaded.token != *expected || loaded.prefix.is_none() { + return Err("temporal_abort_expected_changed".into()); + } + let c = self.temporal_components(&locked.directory)?; + let slot = c + .head + .as_ref() + .and_then(|m| m.slot) + .ok_or("temporal_committed_slot_missing")?; + let mut out = Vec::new(); + for artifact in [ + AbortArtifact::PayloadStage, + AbortArtifact::HeadStage, + AbortArtifact::WitnessStage, + AbortArtifact::InactiveSlot(1 - slot), + ] { + if let Some(bytes) = self.temporal_read( + &locked.directory.join(artifact.name()), + MAX_RECOVERY_BYTES + SMALL + 48, + )? { + if self.temporal_abortable(&locked.directory, &c, &artifact, &bytes, now_ns)? { + out.push(TemporalAbortTicket { + expected: expected.clone(), + artifact, + digest: hash(&bytes), + }); + } + } + } + Ok(out) + } + pub(crate) fn abort_temporal_pending( + &self, + ticket: &TemporalAbortTicket, + now_ns: u64, + ) -> Result { + let _serial = self + .serial + .lock() + .map_err(|_| "recovery_store_unavailable")?; + let locked = self.temporal_lock(false)?; + let before = self + .temporal_current(&locked.directory, now_ns)? + .ok_or("temporal_committed_prefix_missing")?; + if before.token != ticket.expected || before.prefix.is_none() { + return Err("temporal_abort_expected_changed".into()); + } + let c = self.temporal_components(&locked.directory)?; + let path = locked.directory.join(ticket.artifact.name()); + if let Some(bytes) = self.temporal_read(&path, MAX_RECOVERY_BYTES + SMALL + 48)? { + if hash(&bytes) != ticket.digest { + return Err("temporal_abort_artifact_changed".into()); + } + if !self.temporal_abortable(&locked.directory, &c, &ticket.artifact, &bytes, now_ns)? { + return Err("temporal_abort_committed_reference".into()); + } + fs::remove_file(&path).map_err(|_| "temporal_abort_remove")?; + } + File::open(&locked.directory) + .and_then(|f| f.sync_all()) + .map_err(|_| "temporal_abort_directory_fsync")?; + let after = self + .temporal_current(&locked.directory, now_ns)? + .ok_or("temporal_abort_committed_readback")?; + if after.token != before.token + || after.prefix.as_ref().map(|p| p.bytes()) != before.prefix.as_ref().map(|p| p.bytes()) + || path.exists() + { + return Err("temporal_abort_committed_changed".into()); + } + Ok(after) + } + #[cfg(test)] + pub(crate) fn synthetic_partial_write( + &self, + intent: &TemporalIntent, + prefix: &ValidatedTemporalPrefix, + now_ns: u64, + ) -> Result { + let path = self.temporal_dir().join("payload.stage"); + self.publish_temporal_with(intent, Some(prefix), now_ns, false, &mut |point| { + if point == "after:payload.stage:write" { + let mut bytes = fs::read(&path).unwrap(); + bytes.truncate(bytes.len() / 2); + fs::write(&path, bytes).unwrap(); + Err("injected-partial-native-prefix-write".into()) + } else { + Ok(()) + } + }) + } +} +impl TemporalAbortTicket { + pub(crate) fn expected_token(&self) -> &TemporalToken { + &self.expected + } +} + +/// Metadata-only erasure capability: deliberately not a TemporalToken accepted +/// by active commit/resume. Neither expired sample bytes nor fit authority escape. +pub(crate) struct TemporalRetirementInspection { + metadata_kind:String, + retired:bool, + descriptor: RecoveryDescriptor, + run: String, + observed_ns: u64, + expired: bool, + token: TemporalToken, +} +impl TemporalRetirementInspection { + pub(crate) fn metadata_kind(&self)->&str{&self.metadata_kind} + pub(crate) fn retired(&self)->bool{self.retired} + + pub(crate) fn descriptor(&self) -> &RecoveryDescriptor { + &self.descriptor + } + pub(crate) fn expired(&self) -> bool { + self.expired + } + pub(crate) fn run(&self) -> &str { + &self.run + } +} +/// Only explicit recertification constructs this durability-bearing result. +pub(crate) struct TemporalRestored { + pub(crate) token: TemporalToken, + pub(crate) prefix: ValidatedTemporalPrefix, +} +impl RecoveryStore { + pub(crate) fn inspect_temporal_retirement( + &self, + now_ns: u64, + ) -> Result, String> { + let _serial = self + .serial + .lock() + .map_err(|_| "recovery_store_unavailable")?; + let locked = self.temporal_lock(false)?; + let c = self.temporal_components(&locked.directory)?; + match (c.head, c.latest) { + (None, None) => Ok(None), + (Some(head), Some(latest)) if head == latest => { + if now_ns < head.receiver_ns { + return Err("recovery_receiver_clock_regression".into()); + } + Ok(Some(TemporalRetirementInspection { + metadata_kind:head.schema.clone(), + retired:head.retired_at_ms.is_some(), + descriptor: head.descriptor.clone(), + run: head.run.clone(), + observed_ns: now_ns, + expired: now_ns / 1_000_000 >= head.descriptor.expires_at_ms, + token: head.token()?, + })) + } + _ => Err("temporal_head_witness_mismatch_or_pending".into()), + } + } + /// Owner status discovery performs no mkdir/open-create/chmod or payload + /// read. Missing/corrupt metadata never creates a replacement lock/head. + pub(crate) fn inspect_temporal_read_only(&self,now_ns:u64)->Result,String>{ + let _serial=self.serial.lock().map_err(|_|"recovery_store_unavailable")?; + self.verify_directory_read_only()?; + match fs::symlink_metadata(self.temporal_dir()){ + Err(e)if e.kind()==std::io::ErrorKind::NotFound=>return Ok(None), + Err(_)=>return Err("temporal_directory_read".into()), + Ok(m)if !m.is_dir()||m.file_type().is_symlink()=>return Err("temporal_directory_unsafe".into()), + Ok(_)=>{} + } + let locked=self.temporal_lock_mode(false,false)?; + let c=self.temporal_components(&locked.directory)?; + match (c.head, c.latest) { + (None, None) => Ok(None), + (Some(head), Some(latest)) if head == latest => { + if now_ns < head.receiver_ns { + return Err("recovery_receiver_clock_regression".into()); + } + Ok(Some(TemporalRetirementInspection { + metadata_kind:head.schema.clone(), + retired:head.retired_at_ms.is_some(), + descriptor: head.descriptor.clone(), + run: head.run.clone(), + observed_ns: now_ns, + expired: now_ns / 1_000_000 >= head.descriptor.expires_at_ms, + token: head.token()?, + })) + } + _ => Err("temporal_head_witness_mismatch_or_pending".into()), + } + } + pub(crate) fn prepare_inspected_retirement( + &self, + inspection: &TemporalRetirementInspection, + now_ns: u64, + expiry_only: bool, + ) -> Result { + if now_ns < inspection.observed_ns { + return Err("recovery_receiver_clock_regression".into()); + } + self.prepare_retire_temporal(&inspection.token, now_ns, expiry_only) + } + pub(crate) fn fresh_train_genesis_predecessor(&self,now_ns:u64)->Result,String>{ + let _serial=self.serial.lock().map_err(|_|"recovery_store_unavailable")?; + let locked=self.temporal_lock(false)?; + let c=self.temporal_components(&locked.directory)?; + match (c.head,c.latest){ + (None,None)=>Ok(None), + (Some(head),Some(latest)) if head==latest=>{ + if now_nsErr("temporal_head_witness_mismatch_or_pending".into()), + } + } + pub(crate) fn recertify_temporal( + &self, + expected: &TemporalToken, + now_ns: u64, + ) -> Result { + self.recertify_temporal_with(expected, now_ns, &mut |_| Ok(())) + } + fn recertify_temporal_with( + &self, + expected: &TemporalToken, + now_ns: u64, + hook: &mut impl FnMut(&str) -> Result<(), String>, + ) -> Result { + let _serial = self + .serial + .lock() + .map_err(|_| "recovery_store_unavailable")?; + let locked = self.temporal_lock(false)?; + let before = self + .temporal_current(&locked.directory, now_ns)? + .ok_or("temporal_committed_prefix_missing")?; + if before.token != *expected || before.prefix.is_none() { + return Err("temporal_restore_expected_changed".into()); + } + let components = self.temporal_components(&locked.directory)?; + let head = components.head.as_ref().ok_or("temporal_head_missing")?; + let mut names = vec![ + HEAD.to_string(), + format!("slot-{}.bin", head.slot.ok_or("temporal_retired")?), + ]; + names.extend( + components + .witnesses + .iter() + .filter(|(_, m)| m.generation == head.generation) + .map(|(n, _)| n.clone()), + ); + for name in names { + hook("before:restore:file_fsync")?; + private_existing_file(&locked.directory.join(name), false)? + .sync_all() + .map_err(|_| "temporal_restore_file_fsync")?; + hook("after:restore:file_fsync")?; + } + hook("before:restore:directory_fsync")?; + File::open(&locked.directory) + .and_then(|f| f.sync_all()) + .map_err(|_| "temporal_restore_directory_fsync")?; + File::open(&self.directory) + .and_then(|f| f.sync_all()) + .map_err(|_| "temporal_restore_parent_fsync")?; + hook("after:restore:directory_fsync")?; + hook("before:restore:readback")?; + let after = self + .temporal_current(&locked.directory, now_ns)? + .ok_or("temporal_restore_readback")?; + if after.token != before.token + || after.prefix.as_ref().map(|p| p.bytes()) != before.prefix.as_ref().map(|p| p.bytes()) + { + return Err("temporal_restore_readback_changed".into()); + } + hook("after:restore:readback")?; + Ok(TemporalRestored { + token: after.token, + prefix: after.prefix.ok_or("temporal_retired")?, + }) + } +} + +impl RecoveryStore { + #[cfg(test)] + pub(crate) fn synthetic_lost_witness_ack( + &self, + intent: &TemporalIntent, + prefix: &ValidatedTemporalPrefix, + now_ns: u64, + ) -> Result { + self.publish_temporal_with(intent, Some(prefix), now_ns, false, &mut |point| { + if point == "after:witness:install" { + Err("injected-lost-witness-directory-ack".into()) + } else { + Ok(()) + } + }) + } +} + +#[cfg(test)] +impl RecoveryStore { + pub(crate) fn synthetic_fail_before_head_publication(&self,intent:&TemporalIntent,prefix:&ValidatedTemporalPrefix,now:u64)->Result{ + self.publish_temporal_with(intent,Some(prefix),now,false,&mut|point|{if point=="before:head.stage:rename"{Err("synthetic_failed_head_publication".into())}else{Ok(())}}) + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_recovery_commitment_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_recovery_commitment_tests.rs new file mode 100644 index 0000000000..0c47ea1413 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_recovery_commitment_tests.rs @@ -0,0 +1,1155 @@ +use super::*; +use crate::room_learner::temporal_model_input::managed::execution::checkpoint::{ + synthetic_changed_old_row, synthetic_segment_fork, synthetic_storage_prefixes, +}; +fn setup() -> ( + tempfile::TempDir, + RecoveryStore, + Vec, +) { + static PREFIXES: std::sync::OnceLock> = std::sync::OnceLock::new(); + let p = PREFIXES.get_or_init(synthetic_storage_prefixes).clone(); + let d = p[0].descriptor(); + let t = tempfile::tempdir().unwrap(); + let store = RecoveryStore::new( + &t.path().join(&d.owner_scope).join("SensingServer"), + &d.owner_scope, + &d.installation_id, + Some(&[7; 32]), + ) + .unwrap(); + (t, store, p) +} +fn now(p: &ValidatedTemporalPrefix) -> u64 { + p.saved_receiver_ns() +} +fn commit( + store: &RecoveryStore, + expected: Option<&TemporalToken>, + p: &ValidatedTemporalPrefix, +) -> TemporalToken { + let i = store.prepare_temporal(expected, p, now(p)).unwrap(); + store.publish_temporal(&i, Some(p), now(p)).unwrap() +} +#[test] +fn actual_aead_prefix_commits_two_slots_and_preserves_v1() { + let (_t, store, p) = setup(); + store + .save(p[0].descriptor(), p[0].bytes(), now(&p[0]) / 1_000_000) + .unwrap(); + let v1 = fs::read(store.directory.join(FILE)).unwrap(); + let mut token = None; + for value in &p { + let t = commit(&store, token.as_ref(), value); + let loaded = store.load_temporal(now(value)).unwrap().unwrap(); + assert_eq!(loaded.token, t); + assert_eq!(loaded.prefix.unwrap().bytes(), value.bytes()); + token = Some(t); + } + assert_eq!(fs::read(store.directory.join(FILE)).unwrap(), v1); + let files = fs::read_dir(store.temporal_dir()) + .unwrap() + .map(|e| e.unwrap().file_name().to_string_lossy().into_owned()) + .collect::>(); + assert_eq!(files.iter().filter(|n| n.starts_with("slot-")).count(), 2); + assert_eq!( + files.iter().filter(|n| n.starts_with("witness-")).count(), + p.len() + ); +} +#[test] +fn selective_head_payload_witness_replacement_fails_closed() { + for which in ["head", "payload", "witness"] { + let (_t, store, p) = setup(); + let t0 = commit(&store, None, &p[0]); + let dir = store.temporal_dir(); + let old_head = fs::read(dir.join(HEAD)).unwrap(); + let old_payload = fs::read(dir.join("slot-0.bin")).unwrap(); + let old_witness = fs::read(dir.join(format!( + "witness-{:016x}-{:04x}.bin", + t0.generation, t0.revision + ))) + .unwrap(); + let t1 = commit(&store, Some(&t0), &p[1]); + match which { + "head" => fs::write(dir.join(HEAD), old_head).unwrap(), + "payload" => fs::write(dir.join("slot-1.bin"), old_payload).unwrap(), + _ => fs::write( + dir.join(format!( + "witness-{:016x}-{:04x}.bin", + t1.generation, t1.revision + )), + old_witness, + ) + .unwrap(), + }; + assert!(store.load_temporal(now(&p[1])).is_err(), "{which}"); + } +} +#[test] +fn stale_token_changed_old_row_and_same_revision_conflict_reject() { + let (_t, store, p) = setup(); + let t0 = commit(&store, None, &p[0]); + let a = store + .prepare_temporal(Some(&t0), &p[1], now(&p[1])) + .unwrap(); + let fork = synthetic_segment_fork(&p[0]); + let b = store + .prepare_temporal(Some(&t0), &fork, now(&fork)) + .unwrap(); + let t1 = store.publish_temporal(&a, Some(&p[1]), now(&p[1])).unwrap(); + assert!(store.publish_temporal(&b, Some(&fork), now(&fork)).is_err()); + assert!(store + .prepare_temporal(Some(&t0), &p[2], now(&p[2])) + .is_err()); + let t2 = commit(&store, Some(&t1), &p[2]); + let changed = synthetic_changed_old_row(&p[3]); + assert!(store + .prepare_temporal(Some(&t2), &changed, now(&changed)) + .is_err()); + let same = store + .prepare_temporal(Some(&t2), &p[2], now(&p[2])) + .unwrap(); + assert_eq!( + store + .publish_temporal(&same, Some(&p[2]), now(&p[2])) + .unwrap(), + t2 + ); +} +#[test] +fn known_pending_same_revision_repair_never_promotes_other_intent() { + let (_t, store, p) = setup(); + let t0 = commit(&store, None, &p[0]); + let intent = store + .prepare_temporal(Some(&t0), &p[1], now(&p[1])) + .unwrap(); + let mut hook = |point: &str| { + if point == "after:head.stage:directory_fsync" { + Err("injected-head-ambiguous".into()) + } else { + Ok(()) + } + }; + assert!(store + .publish_temporal_with(&intent, Some(&p[1]), now(&p[1]), false, &mut hook) + .is_err()); + assert!(store.load_temporal(now(&p[1])).is_err()); + let token = store + .repair_same_temporal(&intent, Some(&p[1]), now(&p[1])) + .unwrap(); + assert_eq!(token.revision, 1); + assert_eq!( + store + .load_temporal(now(&p[1])) + .unwrap() + .unwrap() + .prefix + .unwrap() + .bytes(), + p[1].bytes() + ); +} + +#[test] +fn fault_every_barrier_preserves_known_commit_or_blocks_then_exact_repair() { + let (_t, store, p) = setup(); + let token = commit(&store, None, &p[0]); + let intent = store + .prepare_temporal(Some(&token), &p[1], now(&p[1])) + .unwrap(); + let mut points = Vec::new(); + store + .publish_temporal_with(&intent, Some(&p[1]), now(&p[1]), false, &mut |s| { + points.push(s.to_owned()); + Ok(()) + }) + .unwrap(); + for point in points { + let (_t, store, p) = setup(); + let token = commit(&store, None, &p[0]); + let intent = store + .prepare_temporal(Some(&token), &p[1], now(&p[1])) + .unwrap(); + let mut hook = |s: &str| { + if s == point { + Err(format!("injected:{s}")) + } else { + Ok(()) + } + }; + assert!( + store + .publish_temporal_with(&intent, Some(&p[1]), now(&p[1]), false, &mut hook) + .is_err(), + "{point}" + ); + if let Ok(Some(loaded)) = store.load_temporal(now(&p[1])) { + let b = loaded.prefix.unwrap(); + assert!( + b.bytes() == p[0].bytes() || b.bytes() == p[1].bytes(), + "{point}" + ); + } + let repaired = store + .repair_same_temporal(&intent, Some(&p[1]), now(&p[1])) + .unwrap_or_else(|e| panic!("repair {point}: {e}")); + assert_eq!(repaired.revision, 1); + assert_eq!( + store + .load_temporal(now(&p[1])) + .unwrap() + .unwrap() + .prefix + .unwrap() + .bytes(), + p[1].bytes() + ); + assert!(!store.temporal_dir().join("payload.stage").exists()); + } +} +#[test] +fn simultaneous_independent_handles_allow_only_one_expected_revision() { + let (t, store, p) = setup(); + let token = commit(&store, None, &p[0]); + let a = store + .prepare_temporal(Some(&token), &p[1], now(&p[1])) + .unwrap(); + let fork = synthetic_segment_fork(&p[0]); + let b = store + .prepare_temporal(Some(&token), &fork, now(&fork)) + .unwrap(); + let d = p[0].descriptor(); + let data = t.path().join(&d.owner_scope).join("SensingServer"); + let other = + RecoveryStore::new(&data, &d.owner_scope, &d.installation_id, Some(&[7; 32])).unwrap(); + let barrier = std::sync::Arc::new(std::sync::Barrier::new(2)); + let c = barrier.clone(); + let p1 = p[1].clone(); + let first = std::thread::spawn(move || { + c.wait(); + store.publish_temporal(&a, Some(&p1), now(&p1)) + }); + let second = std::thread::spawn(move || { + barrier.wait(); + other.publish_temporal(&b, Some(&fork), now(&fork)) + }); + let outcomes = [ + first.join().unwrap().is_ok(), + second.join().unwrap().is_ok(), + ]; + assert_eq!(outcomes.into_iter().filter(|x| *x).count(), 1); +} +#[test] +fn discard_expiry_and_explicit_new_generation_keep_stale_tokens_fenced() { + use crate::room_learner::temporal_model_input::managed::execution::checkpoint::synthetic_new_empty_run; + let (_t, store, p) = setup(); + let token = commit(&store, None, &p[0]); + let lock = fs::metadata(store.directory.join(LOCK)).unwrap(); + assert!(store + .load_temporal(p[0].descriptor().expires_at_ms * 1_000_000) + .is_err()); + let retired = store + .prepare_retire_temporal(&token, p[0].descriptor().expires_at_ms * 1_000_000, true) + .unwrap(); + let tombstone = store + .publish_temporal(&retired, None, p[0].descriptor().expires_at_ms * 1_000_000) + .unwrap(); + assert!(store + .load_temporal(p[0].descriptor().expires_at_ms * 1_000_000) + .unwrap() + .unwrap() + .prefix + .is_none()); + assert!(!store.temporal_dir().join("slot-0.bin").exists()); + assert!(store + .prepare_temporal(Some(&token), &p[1], now(&p[1])) + .is_err()); + let fresh = synthetic_new_empty_run(&p[0], p[0].descriptor().expires_at_ms + 1); + assert!(store.prepare_temporal(None, &fresh, now(&fresh)).is_err()); + let next = store + .prepare_temporal(Some(&tombstone), &fresh, now(&fresh)) + .unwrap(); + let next_token = store + .publish_temporal(&next, Some(&fresh), now(&fresh)) + .unwrap(); + assert_eq!(next_token.generation, tombstone.generation + 1); + assert_eq!(next_token.revision, 0); + assert!(store + .prepare_temporal(Some(&tombstone), &fresh, now(&fresh)) + .is_err()); + assert_eq!( + fs::read_dir(store.temporal_dir()) + .unwrap() + .filter(|e| e + .as_ref() + .unwrap() + .file_name() + .to_string_lossy() + .starts_with("witness-")) + .count(), + 1 + ); + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + assert_eq!( + lock.ino(), + fs::metadata(store.directory.join(LOCK)).unwrap().ino() + ); + } +} +#[test] +fn coordinated_head_and_witness_rollback_is_documented_limit_not_safety_success() { + let (_t, store, p) = setup(); + let t0 = commit(&store, None, &p[0]); + let dir = store.temporal_dir(); + let old_head = fs::read(dir.join(HEAD)).unwrap(); + let t1 = commit(&store, Some(&t0), &p[1]); + fs::write(dir.join(HEAD), old_head).unwrap(); + fs::remove_file(dir.join(format!( + "witness-{:016x}-{:04x}.bin", + t1.generation, t1.revision + ))) + .unwrap(); + let old = store.load_temporal(now(&p[1])).unwrap().unwrap(); + assert_eq!( + old.token, t0, + "deliberately demonstrates excluded coordinated rollback scope" + ); +} +#[test] +fn clock_regression_unknown_entries_and_metadata_capacity_fail_closed() { + let (_t, store, p) = setup(); + let t0 = commit(&store, None, &p[0]); + assert!(store.load_temporal(now(&p[0]) - 1).is_err()); + assert!(store + .prepare_temporal(Some(&t0), &p[1], now(&p[1]) - 1) + .is_err()); + assert_eq!(p[0].transition_limit(), 329); + assert!(p[0].transition_limit() <= MAX_TRANSITIONS); + let keep = store.temporal_dir().join("unrelated.db"); + fs::write(&keep, b"preserve").unwrap(); + assert!(store.load_temporal(now(&p[0])).is_err()); + assert!(keep.exists()); +} +#[test] +fn os_lock_is_shared_with_an_actual_child_process() { + let (t, store, p) = setup(); + let locked = store.temporal_lock(true).unwrap(); + let marker = t.path().join("child-acquired"); + let ready = t.path().join("child-ready"); + let d = p[0].descriptor(); + let child = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "room_learner::recovery_store::temporal_commitment::tests::os_lock_child_probe", + "--nocapture", + ]) + .env( + "RUVIEW_SYNTH_LOCK_DATA", + t.path().join(&d.owner_scope).join("SensingServer"), + ) + .env("RUVIEW_SYNTH_LOCK_OWNER", &d.owner_scope) + .env("RUVIEW_SYNTH_LOCK_INSTALL", &d.installation_id) + .env("RUVIEW_SYNTH_LOCK_READY", &ready) + .env("RUVIEW_SYNTH_LOCK_ACQUIRED", &marker) + .spawn() + .unwrap(); + let started = std::time::Instant::now(); + while !ready.exists() && started.elapsed() < std::time::Duration::from_secs(5) { + std::thread::yield_now(); + } + assert!(ready.exists()); + assert!( + !marker.exists(), + "child cannot acquire while parent holds same stable inode" + ); + drop(locked); + let output = child.wait_with_output().unwrap(); + assert!(output.status.success()); + assert!(marker.exists()); +} +#[test] +fn os_lock_child_probe() { + let Some(data) = std::env::var_os("RUVIEW_SYNTH_LOCK_DATA") else { + return; + }; + let owner = std::env::var("RUVIEW_SYNTH_LOCK_OWNER").unwrap(); + let installation = std::env::var("RUVIEW_SYNTH_LOCK_INSTALL").unwrap(); + let store = + RecoveryStore::new(Path::new(&data), &owner, &installation, Some(&[7; 32])).unwrap(); + fs::write( + std::env::var_os("RUVIEW_SYNTH_LOCK_READY").unwrap(), + b"ready", + ) + .unwrap(); + let _lock = store.temporal_lock(true).unwrap(); + fs::write( + std::env::var_os("RUVIEW_SYNTH_LOCK_ACQUIRED").unwrap(), + b"acquired", + ) + .unwrap(); +} + +#[test] +fn original_native_receiver_nanoseconds_are_never_rounded_for_cold_validation() { + let (_t, store, p) = setup(); + let t0 = commit(&store, None, &p[0]); + let t1 = commit(&store, Some(&t0), &p[1]); + let _t2 = commit(&store, Some(&t1), &p[2]); + let exact = p[2].saved_receiver_ns(); + let truncated = p[2].descriptor().saved_at_ms * 1_000_000; + assert!(exact > truncated); + assert!(store.load_temporal(truncated).is_err()); + let cold = store.load_temporal(exact).unwrap().unwrap().prefix.unwrap(); + assert_eq!(cold.saved_receiver_ns(), exact); + assert_eq!(cold.bytes(), p[2].bytes()); +} + +#[test] +fn valid_looking_lower_background_floor_cannot_extend_committed_prefix() { + use crate::room_learner::temporal_model_input::managed::execution::checkpoint::synthetic_floor_regression; + let (_t, store, p) = setup(); + let t0 = commit(&store, None, &p[0]); + let t1 = commit(&store, Some(&t0), &p[1]); + let t2 = commit(&store, Some(&t1), &p[2]); + let regressed = synthetic_floor_regression(&p[2]); + let err = store + .prepare_temporal(Some(&t2), ®ressed, now(®ressed)) + .err() + .expect("floor must not regress despite internal prefix validity"); + assert_eq!(err, "temporal_commit_floor_regressed"); +} +#[test] +fn conflicting_pending_payload_cannot_be_overwritten_by_other_same_revision_repair() { + let (_t, store, p) = setup(); + let t0 = commit(&store, None, &p[0]); + let a = store + .prepare_temporal(Some(&t0), &p[1], now(&p[1])) + .unwrap(); + let fork = synthetic_segment_fork(&p[0]); + let b = store + .prepare_temporal(Some(&t0), &fork, now(&fork)) + .unwrap(); + let mut hook = |s: &str| { + if s == "after:payload.stage:directory_fsync" { + Err("injected-before-head".into()) + } else { + Ok(()) + } + }; + assert!(store + .publish_temporal_with(&a, Some(&p[1]), now(&p[1]), false, &mut hook) + .is_err()); + let before = fs::read(store.temporal_dir().join("slot-1.bin")).unwrap(); + assert!(store + .repair_same_temporal(&b, Some(&fork), now(&fork)) + .is_err()); + assert_eq!( + fs::read(store.temporal_dir().join("slot-1.bin")).unwrap(), + before + ); + assert!(store + .repair_same_temporal(&a, Some(&p[1]), now(&p[1])) + .is_ok()); +} +#[test] +fn incomplete_unauthenticated_stage_blocks_repair_but_keeps_prior_commit_loadable() { + let (_t, store, p) = setup(); + let t0 = commit(&store, None, &p[0]); + let intent = store + .prepare_temporal(Some(&t0), &p[1], now(&p[1])) + .unwrap(); + let file = store.temporal_dir().join("payload.stage"); + let mut hook = |s: &str| { + if s == "after:payload.stage:write" { + let mut b = fs::read(&file).unwrap(); + b.truncate(b.len() / 2); + fs::write(&file, b).unwrap(); + Err("injected-partial-storage-write".into()) + } else { + Ok(()) + } + }; + assert!(store + .publish_temporal_with(&intent, Some(&p[1]), now(&p[1]), false, &mut hook) + .is_err()); + let old = store.load_temporal(now(&p[1])).unwrap().unwrap(); + assert_eq!(old.token, t0); + assert!( + store + .repair_same_temporal(&intent, Some(&p[1]), now(&p[1])) + .is_err(), + "unverifiable stage cannot be relabeled as same intent" + ); + let retire = store + .prepare_retire_temporal(&t0, now(&p[1]), false) + .unwrap(); + store.publish_temporal(&retire, None, now(&p[1])).unwrap(); + assert!(!file.exists()); +} + +#[test] +fn missing_head_or_witness_never_reinitializes_from_stale_ram() { + for target in ["head", "witness"] { + let (_t, store, p) = setup(); + let t0 = commit(&store, None, &p[0]); + let path = if target == "head" { + store.temporal_dir().join(HEAD) + } else { + store.temporal_dir().join(format!( + "witness-{:016x}-{:04x}.bin", + t0.generation, t0.revision + )) + }; + fs::remove_file(path).unwrap(); + assert!(store.load_temporal(now(&p[0])).is_err()); + assert!(store.prepare_temporal(None, &p[0], now(&p[0])).is_err()); + assert!(store + .prepare_temporal(Some(&t0), &p[1], now(&p[1])) + .is_err()); + } +} +#[test] +fn no_op_commit_does_not_rewrite_payload_or_consume_revision() { + let (_t, store, p) = setup(); + let t0 = commit(&store, None, &p[0]); + let snapshot = |store: &RecoveryStore| { + let mut out = std::collections::BTreeMap::new(); + for e in fs::read_dir(store.temporal_dir()).unwrap() { + let e = e.unwrap(); + out.insert(e.file_name(), fs::read(e.path()).unwrap()); + } + out + }; + let before = snapshot(&store); + let same = store + .prepare_temporal(Some(&t0), &p[0], now(&p[0])) + .unwrap(); + assert_eq!( + store + .publish_temporal(&same, Some(&p[0]), now(&p[0])) + .unwrap(), + t0 + ); + assert_eq!(snapshot(&store), before); +} + +#[test] +fn cleanup_ambiguity_cannot_be_skipped_by_new_generation_or_noop_ack() { + use crate::room_learner::temporal_model_input::managed::execution::checkpoint::synthetic_new_empty_run; + let (_t, store, p) = setup(); + let t0 = commit(&store, None, &p[0]); + let retirement = store + .prepare_retire_temporal(&t0, now(&p[1]), false) + .unwrap(); + let mut hook = |point: &str| { + if point == "before:cleanup" { + Err("injected-cleanup-ambiguity".into()) + } else { + Ok(()) + } + }; + assert!(store + .publish_temporal_with(&retirement, None, now(&p[1]), false, &mut hook) + .is_err()); + assert!(store.temporal_dir().join("slot-0.bin").exists()); + let retired = store.load_temporal(now(&p[1])).unwrap().unwrap().token; + let fresh = synthetic_new_empty_run(&p[0], now(&p[1]) / 1_000_000 + 1); + assert!(store + .prepare_temporal(Some(&retired), &fresh, now(&fresh)) + .is_err()); + store + .repair_same_temporal(&retirement, None, now(&p[1])) + .unwrap(); + let new = store + .prepare_temporal(Some(&retired), &fresh, now(&fresh)) + .unwrap(); + let mut hook = |point: &str| { + if point == "before:cleanup" { + Err("injected-new-generation-cleanup".into()) + } else { + Ok(()) + } + }; + assert!(store + .publish_temporal_with(&new, Some(&fresh), now(&fresh), false, &mut hook) + .is_err()); + let loaded = store.load_temporal(now(&fresh)).unwrap().unwrap(); + let noop = store + .prepare_temporal(Some(&loaded.token), &fresh, now(&fresh)) + .unwrap(); + assert!(store + .publish_temporal(&noop, Some(&fresh), now(&fresh)) + .is_err()); + store + .repair_same_temporal(&new, Some(&fresh), now(&fresh)) + .unwrap(); + assert_eq!( + fs::read_dir(store.temporal_dir()) + .unwrap() + .filter(|e| e + .as_ref() + .unwrap() + .file_name() + .to_string_lossy() + .starts_with("witness-")) + .count(), + 1 + ); +} + +#[test] +fn explicit_abort_only_uncommitted_stage_preserves_exact_prefix_token_and_expiry() { + let (_t, store, p) = setup(); + let t0 = commit(&store, None, &p[0]); + let intent = store + .prepare_temporal(Some(&t0), &p[1], now(&p[1])) + .unwrap(); + assert!(store + .synthetic_partial_write(&intent, &p[1], now(&p[1])) + .is_err()); + let tickets = store.inspect_abortable_temporal(&t0, now(&p[1])).unwrap(); + let ticket = tickets + .iter() + .find(|t| t.artifact == AbortArtifact::PayloadStage) + .unwrap(); + let returned = store.abort_temporal_pending(ticket, now(&p[1])).unwrap(); + assert_eq!(returned.token, t0); + let restored = returned.prefix.unwrap(); + assert_eq!(restored.bytes(), p[0].bytes()); + assert_eq!( + restored.descriptor().expires_at_ms, + p[0].descriptor().expires_at_ms + ); + assert!(!store.temporal_dir().join("payload.stage").exists()); + let next = commit(&store, Some(&t0), &p[1]); + assert_eq!(next.revision, t0.revision + 1); +} +#[test] +fn abort_rejects_stale_digest_token_and_any_committed_reference() { + let (_t, store, p) = setup(); + let t0 = commit(&store, None, &p[0]); + let intent = store + .prepare_temporal(Some(&t0), &p[1], now(&p[1])) + .unwrap(); + assert!(store + .synthetic_partial_write(&intent, &p[1], now(&p[1])) + .is_err()); + let ticket = store + .inspect_abortable_temporal(&t0, now(&p[1])) + .unwrap() + .into_iter() + .find(|t| t.artifact == AbortArtifact::PayloadStage) + .unwrap(); + let file = store.temporal_dir().join("payload.stage"); + let mut changed = fs::read(&file).unwrap(); + changed.push(0); + fs::write(&file, &changed).unwrap(); + assert!(store.abort_temporal_pending(&ticket, now(&p[1])).is_err()); + assert_eq!(fs::read(&file).unwrap(), changed); + let fresh = store + .inspect_abortable_temporal(&t0, now(&p[1])) + .unwrap() + .into_iter() + .find(|t| t.artifact == AbortArtifact::PayloadStage) + .unwrap(); + store.abort_temporal_pending(&fresh, now(&p[1])).unwrap(); + let t1 = commit(&store, Some(&t0), &p[1]); + assert!(store.abort_temporal_pending(&fresh, now(&p[1])).is_err()); + let dir = store.temporal_dir(); + fs::copy(dir.join("slot-1.bin"), dir.join("payload.stage")).unwrap(); + assert!(!store + .inspect_abortable_temporal(&t1, now(&p[1])) + .unwrap() + .iter() + .any(|t| t.artifact == AbortArtifact::PayloadStage)); + let forged = TemporalAbortTicket { + expected: t1, + artifact: AbortArtifact::InactiveSlot(1), + digest: hash(&fs::read(dir.join("slot-1.bin")).unwrap()), + }; + assert!(store.abort_temporal_pending(&forged, now(&p[1])).is_err()); +} +#[test] +fn post_head_ambiguity_cannot_abort_to_a_guessed_prior_revision() { + let (_t, store, p) = setup(); + let t0 = commit(&store, None, &p[0]); + let intent = store + .prepare_temporal(Some(&t0), &p[1], now(&p[1])) + .unwrap(); + let mut hook = |s: &str| { + if s == "after:head.stage:directory_fsync" { + Err("injected-pending-head".into()) + } else { + Ok(()) + } + }; + assert!(store + .publish_temporal_with(&intent, Some(&p[1]), now(&p[1]), false, &mut hook) + .is_err()); + assert!(store.inspect_abortable_temporal(&t0, now(&p[1])).is_err()); + assert!(store + .repair_same_temporal(&intent, Some(&p[1]), now(&p[1])) + .is_ok()); +} + +#[test] +fn independent_interrupted_old_generation_gc_is_same_intent_repairable() { + use crate::room_learner::temporal_model_input::managed::execution::checkpoint::synthetic_new_empty_run; + let (_t, store, p) = setup(); + let t0 = commit(&store, None, &p[0]); + let retirement = store + .prepare_retire_temporal(&t0, now(&p[1]), false) + .unwrap(); + let retired = store + .publish_temporal(&retirement, None, now(&p[1])) + .unwrap(); + let fresh = synthetic_new_empty_run(&p[0], now(&p[1]) / 1_000_000 + 1); + let new = store + .prepare_temporal(Some(&retired), &fresh, now(&fresh)) + .unwrap(); + let mut hook = |point: &str| { + if point == "before:cleanup" { + Err("stop-before-gc".into()) + } else { + Ok(()) + } + }; + assert!(store + .publish_temporal_with(&new, Some(&fresh), now(&fresh), false, &mut hook) + .is_err()); + assert_eq!( + store + .load_temporal(now(&fresh)) + .unwrap() + .unwrap() + .prefix + .unwrap() + .bytes(), + fresh.bytes() + ); + fs::remove_file(store.temporal_dir().join(format!( + "witness-{:016x}-{:04x}.bin", + t0.generation, t0.revision + ))) + .unwrap(); + let repaired = store.repair_same_temporal(&new, Some(&fresh), now(&fresh)); + assert!( + repaired.is_ok(), + "same intent repair after one completed GC unlink: {:?}", + repaired.err() + ); +} + +#[test] +fn each_completed_old_gc_unlink_keeps_current_prefix_loadable_and_repairable() { + use crate::room_learner::temporal_model_input::managed::execution::checkpoint::synthetic_new_empty_run; + for cut in 0..=5 { + let (_t, store, p) = setup(); + let mut token = commit(&store, None, &p[0]); + for value in &p[1..] { + token = commit(&store, Some(&token), value); + } + let retired_intent = store + .prepare_retire_temporal(&token, now(&p[3]), false) + .unwrap(); + let retired = store + .publish_temporal(&retired_intent, None, now(&p[3])) + .unwrap(); + let original = synthetic_new_empty_run(&p[0], now(&p[3]) / 1_000_000 + 1); + let intent = store + .prepare_temporal(Some(&retired), &original, now(&original)) + .unwrap(); + let mut hook = |point: &str| { + if point == "before:cleanup" { + Err("injected-before-obsolete-unlinks".into()) + } else { + Ok(()) + } + }; + assert!(store + .publish_temporal_with(&intent, Some(&original), now(&original), false, &mut hook) + .is_err()); + let components = store.temporal_components(&store.temporal_dir()).unwrap(); + let old = components + .witnesses + .iter() + .filter(|(_, m)| m.generation == retired.generation) + .map(|(n, _)| n.clone()) + .collect::>(); + assert_eq!(old.len(), 5); + for name in old.iter().take(cut) { + fs::remove_file(store.temporal_dir().join(name)).unwrap(); + } + let current = store.load_temporal(now(&original)).unwrap().unwrap(); + assert_eq!( + current.prefix.as_ref().unwrap().bytes(), + original.bytes(), + "cut{cut}" + ); + let repaired = store + .repair_same_temporal(&intent, Some(&original), now(&original)) + .unwrap(); + assert_eq!(repaired, current.token); + let loaded = store + .load_temporal(now(&original)) + .unwrap() + .unwrap() + .prefix + .unwrap(); + assert_eq!(loaded.descriptor(), original.descriptor()); + assert_eq!(loaded.bytes(), original.bytes()); + assert_eq!( + store + .temporal_components(&store.temporal_dir()) + .unwrap() + .witnesses + .len(), + 1 + ); + } +} +#[test] +fn obsolete_gc_fragments_never_authorize_old_head_or_incomplete_current_chain() { + use crate::room_learner::temporal_model_input::managed::execution::checkpoint::synthetic_new_empty_run; + let (_t, store, p) = setup(); + let t0 = commit(&store, None, &p[0]); + let retire = store + .prepare_retire_temporal(&t0, now(&p[1]), false) + .unwrap(); + let retired = store.publish_temporal(&retire, None, now(&p[1])).unwrap(); + let old_head = fs::read(store.temporal_dir().join(HEAD)).unwrap(); + let fresh = synthetic_new_empty_run(&p[0], now(&p[1]) / 1_000_000 + 1); + let intent = store + .prepare_temporal(Some(&retired), &fresh, now(&fresh)) + .unwrap(); + assert!(store + .publish_temporal_with(&intent, Some(&fresh), now(&fresh), false, &mut |s| if s + == "before:cleanup" + { + Err("pause-gc".into()) + } else { + Ok(()) + }) + .is_err()); + fs::remove_file( + store + .temporal_dir() + .join(format!("witness-{:016x}-{:04x}.bin", t0.generation, 0)), + ) + .unwrap(); + fs::write(store.temporal_dir().join(HEAD), old_head).unwrap(); + assert!(store.load_temporal(now(&fresh)).is_err()); + assert!(store.inspect_temporal_retirement(now(&fresh)).is_err()); + store + .repair_same_temporal(&intent, Some(&fresh), now(&fresh)) + .unwrap(); + let current = store.load_temporal(now(&fresh)).unwrap().unwrap().token; + fs::remove_file( + store + .temporal_dir() + .join(format!("witness-{:016x}-{:04x}.bin", current.generation, 0)), + ) + .unwrap(); + assert!(store.load_temporal(now(&fresh)).is_err()); + assert!(store.inspect_temporal_retirement(now(&fresh)).is_err()); +} +#[test] +fn cold_expired_handle_can_inspect_only_metadata_retire_and_begin_new_empty_run() { + use crate::room_learner::temporal_model_input::managed::execution::checkpoint::synthetic_new_empty_run; + let (t, store, p) = setup(); + commit(&store, None, &p[0]); + let d = p[0].descriptor().clone(); + drop(store); + let cold = RecoveryStore::new( + &t.path().join(&d.owner_scope).join("SensingServer"), + &d.owner_scope, + &d.installation_id, + Some(&[7; 32]), + ) + .unwrap(); + let expired = d.expires_at_ms * 1_000_000; + assert!(cold.load_temporal(expired).is_err()); + let inspection = cold.inspect_temporal_retirement(expired).unwrap().unwrap(); + assert!(inspection.expired()); + assert_eq!(inspection.descriptor(), &d); + assert_eq!(inspection.run(), p[0].run()); + assert!(cold + .prepare_inspected_retirement(&inspection, expired - 1, true) + .is_err()); + let retire = cold + .prepare_inspected_retirement(&inspection, expired, true) + .unwrap(); + let retired = cold.publish_temporal(&retire, None, expired).unwrap(); + assert!(cold + .load_temporal(expired) + .unwrap() + .unwrap() + .prefix + .is_none()); + let fresh = synthetic_new_empty_run(&p[0], d.expires_at_ms + 1); + commit(&cold, Some(&retired), &fresh); + assert!(cold + .prepare_inspected_retirement(&inspection, now(&fresh), false) + .is_err()); +} +#[test] +fn metadata_inspection_never_blesses_missing_corrupt_or_foreign_head() { + for target in ["missing", "corrupt"] { + let (_t, store, p) = setup(); + commit(&store, None, &p[0]); + let file = store.temporal_dir().join(HEAD); + if target == "missing" { + fs::remove_file(file).unwrap(); + } else { + let mut b = fs::read(&file).unwrap(); + let i = b.len() - 1; + b[i] ^= 1; + fs::write(file, b).unwrap(); + } + assert!(store + .inspect_temporal_retirement(p[0].descriptor().expires_at_ms * 1_000_000) + .is_err()); + } +} +#[test] +fn visible_lost_ack_requires_real_recertification_before_durable_result() { + let (_t, store, p) = setup(); + let t0 = commit(&store, None, &p[0]); + let t1 = commit(&store, Some(&t0), &p[1]); + let intent = store + .prepare_temporal(Some(&t1), &p[2], now(&p[2])) + .unwrap(); + let mut hook = |point: &str| { + if point == "after:witness:install" { + Err("injected-lost-witness-directory-ack".into()) + } else { + Ok(()) + } + }; + assert!(store + .publish_temporal_with(&intent, Some(&p[2]), now(&p[2]), false, &mut hook) + .is_err()); + let observed = store.load_temporal(now(&p[2])).unwrap().unwrap(); + assert_eq!(observed.prefix.as_ref().unwrap().bytes(), p[2].bytes()); + for fail in [ + "before:restore:file_fsync", + "after:restore:file_fsync", + "before:restore:directory_fsync", + "after:restore:directory_fsync", + "before:restore:readback", + "after:restore:readback", + ] { + assert!( + store + .recertify_temporal_with(&observed.token, now(&p[2]), &mut |point| if point == fail + { + Err("injected-restore-barrier".into()) + } else { + Ok(()) + }) + .is_err(), + "{fail}" + ); + } + let certified = store + .recertify_temporal(&observed.token, now(&p[2])) + .unwrap(); + assert_eq!(certified.token, observed.token); + assert_eq!(certified.prefix.bytes(), p[2].bytes()); +} + +#[test] +fn metadata_only_cold_retirement_rejects_copied_foreign_owner_records() { + let (t, store, p) = setup(); + commit(&store, None, &p[0]); + let owner = "b".repeat(64); + let foreign = RecoveryStore::new( + &t.path().join(&owner).join("SensingServer"), + &owner, + &p[0].descriptor().installation_id, + Some(&[7; 32]), + ) + .unwrap(); + drop(foreign.temporal_lock(true).unwrap()); + for entry in fs::read_dir(store.temporal_dir()).unwrap() { + let entry = entry.unwrap(); + let name = entry.file_name(); + if name == HEAD || name.to_string_lossy().starts_with("witness-") { + fs::copy(entry.path(), foreign.temporal_dir().join(name)).unwrap(); + } + } + assert!(foreign + .inspect_temporal_retirement(p[0].descriptor().expires_at_ms * 1_000_000) + .is_err()); +} + +#[test] +fn legacy_terminal_shape_is_known329_negative_only_and_not_a_loader_grant(){ + let descriptor=RecoveryDescriptor{schema:RECOVERY_SCHEMA.into(),id:"synthetic-terminal".into(),owner_scope:"a".repeat(64),installation_id:"synthetic-install".into(),context_digest:"b".repeat(64),created_at_ms:1,saved_at_ms:2,expires_at_ms:1+RECOVERY_TTL_MS}; + let terminal=Meta{schema:KIND.into(),generation:0,revision:329,descriptor,run:"synthetic-run".into(),policy_sha256:"c".repeat(64),transition_limit:329,window_count:59,receiver_ns:10_000_000,slot:None,payload_sha256:None,previous_sha256:Some("d".repeat(64)),retired_at_ms:Some(10)}; + assert!(legacy_saturated_terminal_shape(&terminal)); + for mode in 0..12{ + let mut bad=terminal.clone();match mode{ + 0=>bad.schema="unknown-kind".into(),1=>bad.transition_limit=343,2=>bad.transition_limit=512, + 3=>bad.revision=328,4=>bad.revision=330,5=>bad.slot=Some(0),6=>bad.payload_sha256=Some("e".repeat(64)), + 7=>bad.previous_sha256=None,8=>bad.previous_sha256=Some("invalid".into()),9=>bad.window_count=65, + 10=>bad.retired_at_ms=None,_=>bad.receiver_ns=11_000_000, + } + assert!(!legacy_saturated_terminal_shape(&bad),"shape mode{mode}"); + } + // Shape alone grants nothing. The wired loader/producer also enforce + // canonical AEAD metadata, owner, clock, exact head/token and witness chain. +} +// SYNTHETIC authenticated chain fixture for storage failpoints ONLY. Its +// counters do not establish positive sample admission; the independent actual +// Runtime128-gap companion suite is the positive reachability oracle. +fn synthetic_saturated_metadata_chain() -> (tempfile::TempDir,RecoveryStore,TemporalToken,u64){ + let(t,store,p)=setup();commit(&store,None,&p[0]); + let mut m=store.temporal_components(&store.temporal_dir()).unwrap().head.unwrap(); + assert_eq!(m.transition_limit,329); + for rev in 1..=328{ + m.previous_sha256=Some(m.sha().unwrap());m.revision=rev; + let plain=serde_json::to_vec(&m).unwrap(); + let path=store.temporal_dir().join(m.witness()); + let mut file=OpenOptions::new().write(true).create_new(true).open(&path).unwrap(); + #[cfg(unix)]{use std::os::unix::fs::PermissionsExt;file.set_permissions(fs::Permissions::from_mode(0o600)).unwrap();} + file.write_all(&store.temporal_seal("witness",&plain).unwrap()).unwrap();file.sync_all().unwrap(); + } + fs::write(store.temporal_dir().join(format!("slot-{}.bin",m.slot.unwrap())),store.temporal_payload_seal(&m,&p[0]).unwrap()).unwrap(); + fs::write(store.temporal_dir().join(HEAD),store.temporal_seal("head",&serde_json::to_vec(&m).unwrap()).unwrap()).unwrap(); + assert_eq!(store.temporal_components(&store.temporal_dir()).unwrap().head.as_ref(),Some(&m)); + let at=m.receiver_ns;(t,store,m.token().unwrap(),at) +} +#[test] +fn saturated_terminal_authenticated_writer_faults_require_exact_same_intent(){ + for point in ["before:head.stage:write","after:head.stage:write","after:head.stage:directory_fsync","after:witness:install","before:cleanup"]{ + let(_t,store,token,at)=synthetic_saturated_metadata_chain(); + let old=store.temporal_read(&store.temporal_dir().join(format!("witness-{:016x}-{:04x}.bin",token.generation,token.revision)),SMALL+36).unwrap().unwrap(); + let intent=store.prepare_retire_temporal(&token,at,false).unwrap();assert_eq!(intent.meta.revision,329);assert!(intent.meta.slot.is_none()); + let mut hook=|seen:&str|if seen==point{Err("synthetic-terminal-interrupted".into())}else{Ok(())}; + assert!(store.publish_temporal_with(&intent,None,at,false,&mut hook).is_err(),"{point}"); + let repaired=store.repair_same_temporal(&intent,None,at).unwrap();assert_eq!(repaired.revision,329); + assert!(store.load_temporal(at).unwrap().unwrap().prefix.is_none()); + assert_eq!(store.repair_same_temporal(&intent,None,at).unwrap(),repaired); + let same=store.prepare_retire_temporal(&repaired,at,false).unwrap();assert_eq!(same.meta.revision,329); + assert_eq!(store.temporal_read(&store.temporal_dir().join(format!("witness-{:016x}-{:04x}.bin",token.generation,token.revision)),SMALL+36).unwrap().unwrap(),old); + let mut forbidden=intent.clone();forbidden.meta.revision=330; + assert!(store.publish_temporal(&forbidden,None,at).is_err()); + } +} +#[test] +fn saturated_terminal_prehead_abort_and_corrupt_stage_never_become_acknowledgment(){ + let(_t,store,token,at)=synthetic_saturated_metadata_chain();let intent=store.prepare_retire_temporal(&token,at,false).unwrap(); + let mut hook=|point:&str|if point=="before:head.stage:rename"{Err("synthetic-pre-head-stop".into())}else{Ok(())}; + assert!(store.publish_temporal_with(&intent,None,at,false,&mut hook).is_err()); + let tickets=store.inspect_abortable_temporal(&token,at).unwrap();assert_eq!(tickets.len(),1); + assert_eq!(store.abort_temporal_pending(&tickets[0],at).unwrap().token,token); + // The abort only removes the uncommitted stage; authenticated tip is328. + assert_eq!(store.temporal_components(&store.temporal_dir()).unwrap().head.unwrap().revision,328); + assert!(store.publish_temporal_with(&intent,None,at,false,&mut hook).is_err()); + fs::write(store.temporal_dir().join("head.stage"),b"synthetic-corruption").unwrap(); + assert!(store.repair_same_temporal(&intent,None,at).is_err()); + assert!(store.fresh_train_genesis_predecessor(at).is_err()); + // Existing explicit negative abort may discard a corrupt UNCOMMITTED + // named stage. It is not a successful retirement/recovery ACK. The old + // stage ticket cannot erase changed bytes; a new hash-bound ticket is needed. + assert!(store.abort_temporal_pending(&tickets[0],at).is_err()); + let corrupt=store.inspect_abortable_temporal(&token,at).unwrap();assert_eq!(corrupt.len(),1); + let preserved=store.abort_temporal_pending(&corrupt[0],at).unwrap();assert_eq!(preserved.token,token); + assert!(preserved.prefix.is_some()); + assert_eq!(store.temporal_components(&store.temporal_dir()).unwrap().head.unwrap().revision,328); +} + +// Optional export is an independent OLD-binary compatibility test fixture, +// generated solely from this synthetic temp owner/key. No live profile reads. +#[test] +fn saturated_terminal_fixture_for_frozen_old_runtime_consumer(){ + let(t,store,token,at)=synthetic_saturated_metadata_chain(); + let intent=store.prepare_retire_temporal(&token,at,false).unwrap(); + let terminal=store.publish_temporal(&intent,None,at).unwrap();assert_eq!(terminal.revision,329); + assert!(store.load_temporal(at).unwrap().unwrap().prefix.is_none()); + if let Some(output)=std::env::var_os("RUVIEW_2473_SYNTHETIC_FIXTURE_OUTPUT"){ + fn copy_private(from:&Path,to:&Path){ + fs::create_dir(to).unwrap(); + #[cfg(unix)]{use std::os::unix::fs::PermissionsExt;fs::set_permissions(to,fs::Permissions::from_mode(0o700)).unwrap();} + for e in fs::read_dir(from).unwrap(){let e=e.unwrap();let m=fs::symlink_metadata(e.path()).unwrap();assert!(!m.file_type().is_symlink()); + if m.is_dir(){copy_private(&e.path(),&to.join(e.file_name()));}else{assert!(m.is_file());fs::copy(e.path(),to.join(e.file_name())).unwrap(); + #[cfg(unix)]{use std::os::unix::fs::PermissionsExt;fs::set_permissions(to.join(e.file_name()),fs::Permissions::from_mode(0o600)).unwrap();} + } + } + } + let output=PathBuf::from(output);fs::create_dir_all(&output).unwrap();let dest=output.join("terminal329");copy_private(t.path(),&dest); + fs::write(output.join("terminal329.json"),serde_json::to_vec(&serde_json::json!({"now_ns":at,"owner_scope":intent.meta.descriptor.owner_scope,"installation_id":intent.meta.descriptor.installation_id})).unwrap()).unwrap(); + } +} + +#[test] +fn authenticated_boundary_metadata_rejects_positive329_and_negative330(){ + for positive in [true,false]{ + let(_t,store,token,at)=synthetic_saturated_metadata_chain(); + let intent=store.prepare_retire_temporal(&token,at,false).unwrap(); + let terminal=store.publish_temporal(&intent,None,at).unwrap();assert_eq!(terminal.revision,329); + let mut inadmissible=intent.meta.clone(); + if positive{inadmissible.slot=Some(0);inadmissible.payload_sha256=Some("a".repeat(64));inadmissible.retired_at_ms=None;} + else{inadmissible.revision=330;inadmissible.previous_sha256=Some(intent.meta.sha().unwrap());} + // Actual canonical encryption under the synthetic owner key. This is + // not an outer-hash-only/corrupted-cipher rejection control. + let bytes=store.temporal_seal("head",&serde_json::to_vec(&inadmissible).unwrap()).unwrap(); + fs::write(store.temporal_dir().join(HEAD),bytes).unwrap(); + assert!(store.load_temporal(at).is_err());assert!(store.inspect_temporal_retirement(at).is_err()); + assert!(store.fresh_train_genesis_predecessor(at).is_err()); + } +} + +#[test] +fn boundary329_to_fresh343_interrupted_obsolete_gc_keeps_exact_new_chain(){ + use crate::room_learner::temporal_model_input::managed::execution::checkpoint::synthetic_fresh_train343_empty_run; + for cut in [1usize,329]{ + let(_t,store,token,at)=synthetic_saturated_metadata_chain(); + let initial=store.load_temporal(at).unwrap().unwrap().prefix.unwrap(); + let retirement=store.prepare_retire_temporal(&token,at,false).unwrap(); + let retired=store.publish_temporal(&retirement,None,at).unwrap();assert_eq!(retired.revision,329); + let old_head=fs::read(store.temporal_dir().join(HEAD)).unwrap(); + let fresh=synthetic_fresh_train343_empty_run(&initial,at/1_000_000+1);assert_eq!(fresh.transition_limit(),343); + let intent=store.prepare_temporal(Some(&retired),&fresh,now(&fresh)).unwrap(); + let mut hook=|point:&str|if point=="before:cleanup"{Err("synthetic-boundary-GC-interruption".into())}else{Ok(())}; + assert!(store.publish_temporal_with(&intent,Some(&fresh),now(&fresh),false,&mut hook).is_err()); + let components=store.temporal_components(&store.temporal_dir()).unwrap(); + let obsolete=components.witnesses.iter().filter(|(_,m)|m.generation==retired.generation).map(|(name,_)|name.clone()).collect::>(); + assert_eq!(obsolete.len(),330); + for name in obsolete.iter().take(cut){fs::remove_file(store.temporal_dir().join(name)).unwrap();} + // With cut329 the ONLY remaining obsolete record is negative329. + // It is authenticated and parsed but cannot outrank the complete new chain. + let loaded=store.load_temporal(now(&fresh)).unwrap().unwrap();assert_eq!(loaded.token.generation,1);assert_eq!(loaded.token.revision,0); + assert_eq!(loaded.prefix.as_ref().unwrap().bytes(),fresh.bytes()); + let repaired=store.repair_same_temporal(&intent,Some(&fresh),now(&fresh)).unwrap();assert_eq!(repaired,loaded.token); + assert_eq!(store.temporal_components(&store.temporal_dir()).unwrap().witnesses.len(),1); + // Selective replay of the old authentic terminal still fails closed. + fs::write(store.temporal_dir().join(HEAD),old_head).unwrap();assert!(store.load_temporal(now(&fresh)).is_err()); + assert_eq!(store.repair_same_temporal(&intent,Some(&fresh),now(&fresh)).unwrap(),repaired); + assert_eq!(store.load_temporal(now(&fresh)).unwrap().unwrap().prefix.unwrap().bytes(),fresh.bytes()); + } +} +#[test] +fn boundary329_to_fresh343_never_acknowledges_or_drops_malformed_or_old_stage(){ + use crate::room_learner::temporal_model_input::managed::execution::checkpoint::synthetic_fresh_train343_empty_run; + for corrupt in [false,true]{ + let(_t,store,token,at)=synthetic_saturated_metadata_chain();let initial=store.load_temporal(at).unwrap().unwrap().prefix.unwrap(); + let retirement=store.prepare_retire_temporal(&token,at,false).unwrap();let retired=store.publish_temporal(&retirement,None,at).unwrap(); + let old_head=fs::read(store.temporal_dir().join(HEAD)).unwrap();let fresh=synthetic_fresh_train343_empty_run(&initial,at/1_000_000+1); + let intent=store.prepare_temporal(Some(&retired),&fresh,now(&fresh)).unwrap(); + assert!(store.publish_temporal_with(&intent,Some(&fresh),now(&fresh),false,&mut |point|if point=="before:cleanup"{Err("synthetic-before-GC".into())}else{Ok(())}).is_err()); + let bytes=if corrupt{b"synthetic-corrupt-stage".to_vec()}else{old_head}; + let path=store.temporal_dir().join("head.stage"); + let mut file=OpenOptions::new().write(true).create_new(true).open(&path).unwrap(); + #[cfg(unix)]{use std::os::unix::fs::PermissionsExt;file.set_permissions(fs::Permissions::from_mode(0o600)).unwrap();} + file.write_all(&bytes).unwrap();file.sync_all().unwrap();drop(file); + assert!(store.repair_same_temporal(&intent,Some(&fresh),now(&fresh)).is_err()); + assert_eq!(fs::read(path).unwrap(),bytes,"failed repair removed untrusted pending bytes"); + let loaded=store.load_temporal(now(&fresh)).unwrap().unwrap();assert_eq!(loaded.token.generation,1);assert_eq!(loaded.token.revision,0); + assert_eq!(loaded.prefix.unwrap().bytes(),fresh.bytes()); + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_selection_store.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_selection_store.rs new file mode 100644 index 0000000000..173d7f6ff4 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_selection_store.rs @@ -0,0 +1,524 @@ +//! Version2 of the SAME protected selection pointer and immutable journal. +//! Only current protected Complete assessment writes it. Cold loading trusts +//! that owner-private journal, then checks every exact immutable dependency; +//! it never creates a new recovery/Complete/evaluation authority after expiry. +use super::*; +use crate::room_learner::temporal_model_input::TemporalModel; +use crate::room_learner::{ + temporal_evaluation as data, temporal_qualification::*, temporal_wire::*, +}; +const POINTER: &str = "model-selection-v1.json"; +const SCHEMA: &str = "ruview.room-learner.model-selection.v2"; +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub(crate) struct TemporalSelectionTarget { + pub context: CaptureContext, + pub model: TemporalModelIdentityWire, + pub training_proof_sha256: String, + pub held_out_evidence_sha256: String, + pub evaluation_lookup_sha256: String, + pub evaluation_receipt_sha256: String, + pub assessment_lookup_sha256: String, + pub assessment_sha256: String, + pub qualification_sha256: String, +} +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub(crate) struct TemporalSelectionReceipt { + pub schema: String, + pub owner_scope: String, + pub installation_id: String, + pub revision: u64, + pub selected_at_unix_ms: u64, + pub selected: TemporalSelectionTarget, +} +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub(crate) struct TemporalSelectionEnvelope { + pub receipt_sha256: String, + pub receipt: TemporalSelectionReceipt, +} +/// No serde/Clone constructor; obtained only through this store's protected +/// current pointer+same immutable journal validation or a current write. +pub(crate) struct StoredTemporalSelection { + pub(super) envelope: TemporalSelectionEnvelope, + model: TemporalModel, + pub(super) proof: data::TemporalTrainProofData, + pub(super) held: data::TemporalHeldOutEvidenceData, + pub(super) evaluation: data::TemporalEvaluationReceiptData, + pub(super) qualification: TemporalQualificationEnvelope, + pub(super) assessment: TemporalAssessmentEnvelope, +} +impl StoredTemporalSelection { + pub(crate) fn envelope(&self) -> &TemporalSelectionEnvelope { + &self.envelope + } + pub(crate) fn model(&self) -> &TemporalModel { + &self.model + } + pub(crate) fn held(&self) -> &data::TemporalHeldOutEvidenceData { + &self.held + } + pub(crate) fn qualification(&self) -> &TemporalQualificationEnvelope { + &self.qualification + } + pub(crate) fn capture_generation_id(&self) -> Result { + let id = &self.proof.body.identity; + crate::room_learner::qualification::hash(&( + "ruview.room-learner.capture-generation.v2", + &id.original_deadline.id, + &id.run, + &id.context_digest, + )) + } + pub(crate) fn identity(&self) -> TemporalSelectedIdentityWire { + let r = &self.envelope.receipt; + let q = &self.qualification.receipt.summary; + TemporalSelectedIdentityWire { + selection_sha256: self.envelope.receipt_sha256.clone(), + selection_revision: r.revision, + model_kind: WholeTemporalKind::ManagedTemporalV3, + model_sha256: q.model_sha256.clone(), + verified_fit_record_sha256: q.verified_fit_record_sha256.clone(), + training_proof_sha256: q.training_proof_sha256.clone(), + held_out_evidence_sha256: q.held_out_evidence_sha256.clone(), + complete_record_sha256: q.complete_record_sha256.clone(), + evaluation_receipt_sha256: q.evaluation_receipt_sha256.clone(), + qualification_sha256: q.receipt_sha256.clone(), + policy_sha256: q.policy_sha256.clone(), + context_digest: q.context_digest.clone(), + room_digest: q.room_digest.clone(), + source_node_ids: r + .selected + .context + .sources + .iter() + .map(|s| s.source_id) + .collect(), + selected_at_unix_ms: r.selected_at_unix_ms, + evaluated_at_unix_ms: q.evaluated_at_unix_ms, + authority_scope: q.authority_scope.clone(), + } + } +} +impl ArtifactStore { + pub(crate) fn temporal_selection_present(&self) -> Result { + self.verify_owner()?; + let path = self.root.join(POINTER); + if !regular_or_missing(&path)? { + return Ok(false); + } + let value: serde_json::Value = serde_json::from_slice( + &private_read(&path).map_err(|_| "selection pointer unavailable")?, + ) + .map_err(|_| "selection pointer invalid")?; + Ok(value + .pointer("/receipt/schema") + .and_then(serde_json::Value::as_str) + == Some(SCHEMA)) + } + pub(crate) fn selection_revision_any(&self) -> Result { + if self.temporal_selection_present()? { + let bytes = private_read(&self.root.join(POINTER)) + .map_err(|_| "temporal_selection_pointer_unavailable")?; + let e: TemporalSelectionEnvelope = + serde_json::from_slice(&bytes).map_err(|_| "temporal_selection_pointer_schema")?; + if e.receipt.schema != SCHEMA + || e.receipt.owner_scope != self.owner_scope + || e.receipt.installation_id != self.installation_id + || e.receipt.revision == 0 + || e.receipt_sha256 != crate::room_learner::qualification::hash(&e.receipt)? + || private_read( + &self + .root + .join("selections") + .join(format!("{}.json", e.receipt_sha256)), + ) + .map_err(|_| "temporal_selection_journal_unavailable")? + != bytes + { + return Err("temporal_selection_pointer_changed".into()); + } + // CAS identity only; a withdrawn old assessment must not prevent an + // explicitly authorized replacement from fresh protected evidence. + return Ok(e.receipt.revision); + } + Ok(self.load_selection()?.map_or(0, |e| e.receipt.revision)) + } + /// Final CAS for THIS already admitted write, not a history loader. + /// The first operation itself just created one immutable journal entry; + /// missing pointer+that exact entry is still revision0. Every other orphan + /// or concurrent history remains a conservative refusal. + fn revision_for_prepared_temporal_selection( + &self, + prepared: &TemporalSelectionEnvelope, + bytes: &[u8], + ) -> Result { + if regular_or_missing(&self.root.join(POINTER))? { + return self.selection_revision_any(); + } + if prepared.receipt.revision != 1 + || prepared.receipt.schema != SCHEMA + || prepared.receipt.owner_scope != self.owner_scope + || prepared.receipt.installation_id != self.installation_id + || prepared.receipt_sha256 + != crate::room_learner::qualification::hash(&prepared.receipt)? + || serde_json::to_vec(prepared).map_err(|_| "selection_encoding")? != bytes + { + return Err("selection_pointer_changed".into()); + } + let history = self.root.join("selections"); + if !temporal_artifacts::directory(&history, false)? { + return Err("selection_pending_history_missing".into()); + } + let mut entries = + std::fs::read_dir(&history).map_err(|_| "selection_pending_history_unavailable")?; + let only = entries + .next() + .ok_or("selection_pending_history_missing")? + .map_err(|_| "selection_pending_history_unavailable")?; + if only.path() != history.join(format!("{}.json", prepared.receipt_sha256)) + || entries.next().is_some() + || temporal_artifacts::bounded_read(&only.path(), LIMIT)? != bytes + { + return Err("selection_foreign_or_changed_history".into()); + } + Ok(0) + } + pub(crate) fn load_temporal_selection( + &self, + ) -> Result, String> { + if !self.temporal_selection_present()? { + return Ok(None); + } + let bytes = + private_read(&self.root.join(POINTER)).map_err(|_| "temporal_selection_unavailable")?; + let e: TemporalSelectionEnvelope = + serde_json::from_slice(&bytes).map_err(|_| "temporal_selection_schema")?; + let r = &e.receipt; + let t = &r.selected; + if r.schema != SCHEMA + || e.receipt_sha256 != crate::room_learner::qualification::hash(r)? + || r.owner_scope != self.owner_scope + || r.installation_id != self.installation_id + || r.revision == 0 + || r.selected_at_unix_ms == 0 + || r.selected_at_unix_ms > 253402300799999 + { + return Err("temporal_selection_identity".into()); + } + self.validate_context(&t.context)?; + if [ + &t.model.model_sha256, + &t.model.verified_fit_record_sha256, + &t.training_proof_sha256, + &t.held_out_evidence_sha256, + &t.evaluation_lookup_sha256, + &t.evaluation_receipt_sha256, + &t.assessment_lookup_sha256, + &t.assessment_sha256, + &t.qualification_sha256, + ] + .iter() + .any(|s| !hex_digest(s)) + { + return Err("temporal_selection_digest".into()); + } + if private_read( + &self + .root + .join("selections") + .join(format!("{}.json", e.receipt_sha256)), + ) + .map_err(|_| "temporal_selection_journal_missing")? + != bytes + { + return Err("temporal_selection_journal_changed".into()); + } + let model = self.load_exact_temporal_candidate(&t.context, &t.model.model_sha256)?; + if model.artifact_kind() != crate::room_learner::temporal_model_input::managed::SCHEMA_V3 { + return Err("temporal_selection_model_kind".into()); + } + let dir = self + .temporal_evaluation_directory(&t.context, false)? + .ok_or("temporal_selection_metadata_missing")?; + let proof = data::TemporalTrainProofData::decode_data(&temporal_artifacts::bounded_read( + &dir.join(format!("train-proof-{}.json", t.training_proof_sha256)), + data::MAX_METADATA_BYTES, + )?)?; + let held = + data::TemporalHeldOutEvidenceData::decode_data(&temporal_artifacts::bounded_read( + &dir.join(format!( + "heldout-evidence-{}.json", + t.held_out_evidence_sha256 + )), + data::MAX_METADATA_BYTES, + )?)?; + let evaluation = + data::TemporalEvaluationReceiptData::decode_data(&temporal_artifacts::bounded_read( + &dir.join(format!("evaluation-{}.json", t.evaluation_lookup_sha256)), + data::MAX_METADATA_BYTES, + )?)?; + evaluation.validate_data_links(&proof, &held)?; + let qualification: TemporalQualificationEnvelope = + serde_json::from_slice(&temporal_artifacts::bounded_read( + &dir.join(format!("qualification-{}.json", t.qualification_sha256)), + data::MAX_METADATA_BYTES, + )?) + .map_err(|_| "temporal_selection_qualification_schema")?; + let assessment: TemporalAssessmentEnvelope = + serde_json::from_slice(&temporal_artifacts::bounded_read( + &dir.join(format!("assessment-{}.json", t.assessment_lookup_sha256)), + data::MAX_METADATA_BYTES, + )?) + .map_err(|_| "temporal_selection_assessment_schema")?; + assessment.validate_links(&proof, &held, &evaluation, Some(&qualification))?; + #[derive(Deserialize)] + #[serde(deny_unknown_fields)] + struct CurrentAssessment { + schema: String, + model_sha256: String, + context_digest: String, + assessment_sha256: String, + assessment_lookup_sha256: String, + } + let current: CurrentAssessment = serde_json::from_slice(&temporal_artifacts::bounded_read( + &dir.join(format!("latest-assessment-{}.json", t.model.model_sha256)), + data::MAX_METADATA_BYTES, + )?) + .map_err(|_| "temporal_selection_latest_assessment_unavailable")?; + if current.schema != "ruview.room-learner.temporal-current-assessment.v1" + || current.model_sha256 != t.model.model_sha256 + || current.context_digest != t.context.digest() + || current.assessment_sha256 != t.assessment_sha256 + || current.assessment_lookup_sha256 != t.assessment_lookup_sha256 + { + return Err("newer_qualification_failed_or_requires_selection".into()); + } + + let id = &proof.body.identity; + if model.managed_is_operator_eight_step()? != proof.is_operator_eight_step() + || proof.sha256 != t.training_proof_sha256 + || held.sha256 != t.held_out_evidence_sha256 + || evaluation.sha256 != t.evaluation_receipt_sha256 + || qualification.receipt_sha256 != t.qualification_sha256 + || assessment.receipt_sha256 != t.assessment_sha256 + || id.artifact_sha256 != t.model.model_sha256 + || id.verified_fit_record_sha256 != t.model.verified_fit_record_sha256 + || id.context_digest != t.context.digest() + || id.owner_scope != self.owner_scope + || id.installation_id != self.installation_id + || r.selected_at_unix_ms < qualification.receipt.qualified_at_unix_ms + || r.selected_at_unix_ms >= id.original_deadline.expires_at_unix_ms + { + return Err("temporal_selection_chain_changed".into()); + } + Ok(Some(StoredTemporalSelection { + envelope: e, + model, + proof, + held, + evaluation, + qualification, + assessment, + })) + } + pub(crate) fn save_current_temporal_selection( + &self, + source:&crate::room_learner::temporal_model_input::managed::execution::full_run::VerifiedTemporalEvaluationSnapshot, + assessment: &StoredTemporalAssessment, + expected_revision: u64, + now: u64, + cancel: &std::sync::atomic::AtomicBool, + clock: &mut dyn FnMut() -> Result, + before_install: &mut dyn FnMut() -> Result<(), String>, + after_rename: &mut dyn FnMut() -> Result<(), String>, + ) -> Result { + use std::sync::atomic::Ordering; + source.require_same_store(self)?; + if source.binding() != assessment.binding() { + return Err("temporal_selection_current_complete_changed".into()); + } + let q = assessment + .qualification + .as_ref() + .ok_or("temporal_candidate_not_qualified")?; + assessment + .assessment + .validate(&assessment.evaluation, Some(q))?; + if now / 1_000_000 + >= source + .proof() + .body + .identity + .original_deadline + .expires_at_unix_ms + { + return Err("recovery_expired".into()); + } + if self.selection_revision_any()? != expected_revision { + return Err("selection pointer changed".into()); + } + let t = TemporalSelectionTarget { + context: source.binding().context().clone(), + model: TemporalModelIdentityWire { + model_kind: WholeTemporalKind::ManagedTemporalV3, + model_sha256: source.proof().body.identity.artifact_sha256.clone(), + verified_fit_record_sha256: source + .proof() + .body + .identity + .verified_fit_record_sha256 + .clone(), + }, + training_proof_sha256: source.proof().sha256.clone(), + held_out_evidence_sha256: source.held_out().sha256.clone(), + evaluation_lookup_sha256: assessment.evaluation.lookup_sha256().into(), + evaluation_receipt_sha256: assessment.evaluation.receipt().sha256.clone(), + assessment_lookup_sha256: crate::room_learner::qualification::hash(&( + crate::room_learner::temporal_wire::ASSESSMENT_SCHEMA, + &assessment.evaluation.receipt().sha256, + TemporalQualificationPolicyWire::for_operator_eight_step(source.proof().is_operator_eight_step()), + ))?, + assessment_sha256: assessment.assessment.receipt_sha256.clone(), + qualification_sha256: q.receipt_sha256.clone(), + }; + let receipt = TemporalSelectionReceipt { + schema: SCHEMA.into(), + owner_scope: self.owner_scope.clone(), + installation_id: self.installation_id.clone(), + revision: expected_revision + .checked_add(1) + .ok_or("selection revision exhausted")?, + selected_at_unix_ms: now / 1_000_000, + selected: t, + }; + let envelope = TemporalSelectionEnvelope { + receipt_sha256: crate::room_learner::qualification::hash(&receipt)?, + receipt, + }; + let bytes = serde_json::to_vec(&envelope).map_err(|_| "temporal_selection_encoding")?; + if cancel.load(Ordering::Acquire) { + return Err("temporal_selection_cancelled".into()); + } + clock()?; + let history = self.root.join("selections"); + private_dir(&history).map_err(|_| "selection history unavailable")?; + let history_path = history.join(format!("{}.json", envelope.receipt_sha256)); + temporal_artifacts::immutable(&history_path, &bytes)?; + temporal_artifacts::recertify_immutable(&history_path, &bytes, LIMIT, &mut |_| { + if cancel.load(Ordering::Acquire) { + return Err("temporal_selection_cancelled".into()); + } + clock()?; + Ok(()) + })?; + if cancel.load(Ordering::Acquire) { + return Err("temporal_selection_cancelled".into()); + } + clock()?; + // The existing atomic pointer replacement is the linearization point. + // Revalidate the current Complete witness/owner/deadline after all + // preceding potentially slow IO, immediately before installing it. + crate::companion::atomic_private_write_checked( + &self.root.join(POINTER), + &bytes, + &mut || { + before_install().map_err(std::io::Error::other)?; + if self + .revision_for_prepared_temporal_selection(&envelope, &bytes) + .map_err(std::io::Error::other)? + != expected_revision + { + return Err(std::io::Error::other("selection pointer changed")); + } + if cancel.load(Ordering::Acquire) { + return Err(std::io::Error::other("temporal_selection_cancelled")); + } + clock().map_err(std::io::Error::other)?; + Ok(()) + }, + &mut || after_rename().map_err(std::io::Error::other), + ) + .map_err(|_| "temporal_selection_pointer_write_unknown")?; + let read = self + .load_temporal_selection()? + .ok_or("temporal_selection_readback_missing")?; + if read.envelope != envelope { + return Err("temporal_selection_readback_changed".into()); + } + Ok(read) + } +} + +impl ArtifactStore { + pub(crate) fn withdraw_temporal_selection( + &self, + sha: &str, + revision: u64, + now: u64, + allowed: &mut dyn FnMut() -> Result<(), String>, + before_install: &mut dyn FnMut() -> Result<(), String>, + after_rename: &mut dyn FnMut() -> Result<(), String>, + ) -> Result { + // Negative CAS needs only the authenticated pointer/journal identity; + // raw expiry, unavailable RF or a failed assessment cannot block Off. + self.verify_owner()?; + let bytes = private_read(&self.root.join(POINTER)) + .map_err(|_| "temporal_selection_pointer_unavailable")?; + let old: TemporalSelectionEnvelope = + serde_json::from_slice(&bytes).map_err(|_| "temporal_selection_pointer_schema")?; + if old.receipt_sha256 != sha + || old.receipt.revision != revision + || old.receipt.owner_scope != self.owner_scope + || old.receipt.installation_id != self.installation_id + || old.receipt.schema != SCHEMA + || old.receipt_sha256 != crate::room_learner::qualification::hash(&old.receipt)? + || self.selection_revision_any()? != revision + { + return Err("selected_pointer_changed".into()); + } + let receipt = crate::room_learner::selection::SelectionReceipt { + schema: "ruview.room-learner.model-selection.v1".into(), + owner_scope: self.owner_scope.clone(), + installation_id: self.installation_id.clone(), + revision: revision + .checked_add(1) + .ok_or("selection_revision_exhausted")?, + selected_at_unix_ms: now / 1_000_000, + selected: None, + previous: None, + }; + let envelope = crate::room_learner::selection::SelectionEnvelope { + receipt_sha256: crate::room_learner::qualification::hash(&receipt)?, + receipt, + }; + let bytes = serde_json::to_vec(&envelope).map_err(|_| "selection_encoding")?; + let history = self.root.join("selections"); + private_dir(&history).map_err(|_| "selection_history_unavailable")?; + let path = history.join(format!("{}.json", envelope.receipt_sha256)); + temporal_artifacts::immutable(&path, &bytes)?; + temporal_artifacts::recertify_immutable(&path, &bytes, LIMIT, &mut |_| allowed())?; + crate::companion::atomic_private_write_checked( + &self.root.join(POINTER), + &bytes, + &mut || { + before_install().map_err(std::io::Error::other)?; + if self + .selection_revision_any() + .map_err(std::io::Error::other)? + != revision + { + return Err(std::io::Error::other("selected_pointer_changed")); + } + allowed().map_err(std::io::Error::other) + }, + &mut || after_rename().map_err(std::io::Error::other), + ) + .map_err(|_| "selection_pointer_write_unknown")?; + if self.load_selection()?.as_ref() != Some(&envelope) { + return Err("selection_pointer_readback_changed".into()); + } + Ok(envelope) + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_wire.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_wire.rs new file mode 100644 index 0000000000..799411bfba --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_wire.rs @@ -0,0 +1,572 @@ +//! Strict whole-Temporal public DTOs. These are read models/command inputs, +//! never Complete/evaluation/selection capabilities. V1 producers are unchanged. +use super::{ + lifecycle::MutationGuard, + observation::{DataRole, RoomLabel}, + training::EvaluationReport, +}; +use serde::{Deserialize, Serialize}; +pub const STATUS_SCHEMA: &str = "ruview.room-learner.status.v2"; +pub const SELECTION_SCHEMA: &str = "ruview.room-learner.selection.v2"; +pub const SELECTION_STATUS_SCHEMA: &str = "ruview.room-learner.model-selection-status.v2"; +pub const SELECTED_BINARY_SCHEMA: &str = "ruview.room-learner.selected-binary-evidence.v2"; +pub const QUALIFICATION_SCHEMA: &str = "ruview.local-operational-qualification.v3"; +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +pub enum WholeTemporalKind { + #[serde(rename = "ruview.room-learner.managed-temporal-input.v3")] + ManagedTemporalV3, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct TemporalModelIdentityWire { + pub model_kind: WholeTemporalKind, + pub model_sha256: String, + pub verified_fit_record_sha256: String, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct TemporalCompleteRefsWire { + pub model: TemporalModelIdentityWire, + pub training_proof_sha256: String, + pub complete_record_sha256: String, + pub complete_commitment_sha256: String, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct TemporalEvaluationRefsWire { + pub complete: TemporalCompleteRefsWire, + pub held_out_evidence_sha256: String, + pub evaluation_receipt_sha256: String, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct TemporalQualificationRefsWire { + pub evaluation: TemporalEvaluationRefsWire, + pub qualification_sha256: String, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct TemporalOriginalDeadlineWire { + pub created_at_unix_ms: u64, + pub expires_at_unix_ms: u64, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct TemporalEpisodeWire { + pub index: u8, + pub role: DataRole, + pub label: RoomLabel, + pub target_ms: u64, + pub lead_in_ms: u64, + pub native_windows: u16, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct TemporalProfileWire { + pub profile_id: String, + pub window_unit: String, + pub native_window_ms: u16, + pub train_windows: u16, + pub held_out_windows: u16, + pub train_episodes: u8, + pub held_out_episodes: u8, + pub windows_per_condition: [u16; 4], + pub train_episodes_per_condition: [u8; 4], + pub held_out_episodes_per_condition: [u8; 4], + pub episodes: Vec, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct TemporalCaptureWire { + pub capture_generation_id: String, + pub schedule_sha256: String, + pub continuation_policy_sha256: String, + pub durable_commitment_sha256: Option, + pub original_deadline: TemporalOriginalDeadlineWire, + pub receiver_epoch: String, + pub stage: String, + pub current_episode: Option, + pub role: Option, + pub label: Option, + pub target_ms: Option, + pub lead_in_remaining_ms: u64, + pub observed_ms: u64, + pub retained_windows: u16, + pub durable_windows: u16, + pub sealed_train_episodes: u8, + pub sealed_held_out_episodes: u8, + pub durability_pending: bool, + pub awaiting_ground_truth: bool, + pub confirmation_challenge: Option, + pub failure: Option, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct TemporalEvaluationWire { + pub refs: TemporalEvaluationRefsWire, + pub window_unit: String, + pub evaluated_at_unix_ms: u64, + pub report: EvaluationReport, + pub durable: bool, + pub independence_verified: bool, + pub physically_validated: bool, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum WorkflowKindWire { + Idle, + Legacy, + ManagedTemporalV3, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct TemporalLearnerStatusWire { + pub schema: String, + pub enabled: bool, + pub available_model_kinds: Vec, + pub temporal_profile: TemporalProfileWire, + pub workflow_kind: WorkflowKindWire, + pub mutation_guard: MutationGuard, + pub context_digest: Option, + pub room_digest: Option, + pub source_node_ids: Vec, + pub recovery: Option, + pub capture: Option, + pub candidate: Option, + pub training_proof_sha256: Option, + pub complete: Option, + pub evaluation: Option, + pub assessment: Option, + pub model_selection: Option, + pub job_state: String, + pub job_operation: Option, + pub error_code: Option, + pub error: Option, + pub legacy_status: Option, +} +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct TemporalConfigureRequestV2 { + pub schema: String, + pub model_kind: WholeTemporalKind, + pub expected: MutationGuard, + pub binding_digest: String, + pub revision: u64, + pub source_node_ids: Vec, + pub new_training: bool, + pub recovery_consent: bool, + #[serde(default)] + pub profile_id: Option, +} + +pub const RECOVERY_RETRY_REQUEST_SCHEMA: &str = "ruview.room-learner.recovery-retry.v2"; +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct TemporalRetryRequestV2 { + pub schema: String, + pub model_kind: WholeTemporalKind, + pub expected: MutationGuard, + pub capture_generation_id: String, + pub retry_confirmed: bool, +} +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct TemporalPhaseStartRequestV2 { + pub schema: String, + pub expected: MutationGuard, + pub capture_generation_id: String, + pub schedule_sha256: String, + pub episode_index: u8, + pub conditions_confirmed: bool, +} +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct TemporalPhaseFinishRequestV2 { + pub schema: String, + pub expected: MutationGuard, + pub capture_generation_id: String, + pub schedule_sha256: String, + pub episode_index: u8, + pub confirmation_challenge: String, + pub conditions_confirmed: bool, +} +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct TemporalEvaluateRequestV2 { + pub schema: String, + pub expected: MutationGuard, + pub capture_generation_id: String, + pub refs: TemporalCompleteRefsWire, +} +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct TemporalQualifyRequestV2 { + pub schema: String, + pub expected: MutationGuard, + pub capture_generation_id: String, + pub refs: TemporalEvaluationRefsWire, +} +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct TemporalSelectRequestV2 { + pub schema: String, + pub expected: MutationGuard, + pub capture_generation_id: String, + pub refs: TemporalQualificationRefsWire, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct TemporalCandidateIdentityWire { + pub model_kind: WholeTemporalKind, + pub model_sha256: String, + pub verified_fit_record_sha256: String, + pub context_digest: String, + pub room_digest: String, + pub persisted: bool, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct TemporalQualificationPolicyWire { + pub id: String, + pub window_unit: String, + pub minimum_windows_per_condition: u16, + pub minimum_unconditional_sensitivity: f64, + pub maximum_false_positive_rate: f64, + pub minimum_evaluated_window_coverage: f64, + pub coverage_scope: String, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct TemporalConditionCriterionWire { + pub condition: RoomLabel, + pub train_windows: u16, + pub held_out_windows: u16, + pub scored_windows: u16, + pub abstained_windows: u16, + /// Subset of abstained, not an additional exclusion from its denominator. + pub missing_windows: u16, + pub declared_train_episodes: u8, + pub declared_held_out_episodes: u8, + pub evaluated_window_coverage: f64, + pub unconditional_sensitivity: Option, + pub false_positive_rate: Option, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct TemporalQualificationSummaryWire { + pub schema: String, + pub receipt_sha256: String, + pub model_kind: WholeTemporalKind, + pub model_sha256: String, + pub verified_fit_record_sha256: String, + pub training_lineage_basis: String, + pub training_proof_sha256: String, + pub held_out_evidence_sha256: String, + pub complete_record_sha256: String, + pub evaluation_receipt_sha256: String, + pub context_digest: String, + pub room_digest: String, + pub evaluated_at_unix_ms: u64, + pub policy: TemporalQualificationPolicyWire, + pub policy_sha256: String, + pub label_provenance: String, + pub criteria: Vec, + pub authority_scope: String, + pub window_unit: String, + pub train_windows: u16, + pub held_out_windows: u16, + pub train_episodes: u8, + pub held_out_episodes: u8, + pub independence_verified: bool, + pub physically_validated: bool, + pub measurement_clock_available: bool, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct TemporalSelectedIdentityWire { + pub selection_sha256: String, + pub selection_revision: u64, + pub model_kind: WholeTemporalKind, + pub model_sha256: String, + pub verified_fit_record_sha256: String, + pub training_proof_sha256: String, + pub held_out_evidence_sha256: String, + pub complete_record_sha256: String, + pub evaluation_receipt_sha256: String, + pub qualification_sha256: String, + pub policy_sha256: String, + pub context_digest: String, + pub room_digest: String, + pub source_node_ids: Vec, + pub selected_at_unix_ms: u64, + pub evaluated_at_unix_ms: u64, + pub authority_scope: String, +} + +pub const CONFIGURE_REQUEST_SCHEMA: &str = "ruview.room-learner.configure.v2"; +pub const PHASE_START_REQUEST_SCHEMA: &str = "ruview.room-learner.phase-start.v2"; +pub const PHASE_FINISH_REQUEST_SCHEMA: &str = "ruview.room-learner.phase-finish.v2"; +pub const TRAIN_REQUEST_SCHEMA: &str = "ruview.room-learner.train.v2"; +pub const EVALUATE_REQUEST_SCHEMA: &str = "ruview.room-learner.evaluate.v2"; +pub const QUALIFY_REQUEST_SCHEMA: &str = "ruview.room-learner.qualify.v2"; +pub const SELECT_REQUEST_SCHEMA: &str = "ruview.room-learner.select.v2"; +pub const RECOVERY_REQUEST_SCHEMA: &str = "ruview.room-learner.recovery.v2"; +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct TemporalTrainRequestV2 { + pub schema: String, + pub expected: MutationGuard, + pub capture_generation_id: String, +} +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct TemporalRecoveryRequestV2 { + pub schema: String, + pub model_kind: WholeTemporalKind, + pub expected: MutationGuard, + pub binding_digest: String, + pub revision: u64, + pub source_node_ids: Vec, + pub capture_generation_id: String, + #[serde(default)] + pub profile_id: Option, +} + +pub const TEMPORAL_POLICY_ID: &str = "ruview.local-operational-temporal-policy.v1"; +pub const OPERATOR_EIGHT_STEP_TEMPORAL_POLICY_ID: &str = "ruview.local-operational-temporal-policy.operator-eight-step.v1"; +impl TemporalQualificationPolicyWire { + pub(crate) fn for_operator_eight_step(operator: bool) -> Self { + let mut policy = Self::current(); + if operator { + policy.id = OPERATOR_EIGHT_STEP_TEMPORAL_POLICY_ID.into(); + policy.minimum_windows_per_condition = 4; + } + policy + } + pub(crate) fn current() -> Self { + let numeric = super::qualification::QualificationPolicy::default(); + Self { + id: TEMPORAL_POLICY_ID.into(), + window_unit: "native_receiver_2s_window".into(), + minimum_windows_per_condition: 16, + minimum_unconditional_sensitivity: numeric.minimum_unconditional_sensitivity, + maximum_false_positive_rate: numeric.maximum_false_positive_rate, + minimum_evaluated_window_coverage: numeric.minimum_evaluated_cohort_coverage, + coverage_scope: "declared_native_windows_not_wall_time".into(), + } + } +} +#[cfg(test)] +#[path = "temporal_wire_tests.rs"] +mod tests; + +pub const PHASE_CANCEL_REQUEST_SCHEMA: &str = "ruview.room-learner.phase-cancel.v2"; +pub const TRAIN_CANCEL_REQUEST_SCHEMA: &str = "ruview.room-learner.train-cancel.v2"; +pub const RECOVERY_RESTORE_REQUEST_SCHEMA: &str = "ruview.room-learner.recovery-restore.v2"; +pub const RECOVERY_DISCARD_REQUEST_SCHEMA: &str = "ruview.room-learner.recovery-discard.v2"; +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct TemporalCurrentActionRequestV2 { + pub schema: String, + pub expected: MutationGuard, + pub capture_generation_id: String, +} +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct TemporalDiscardRequestV2 { + pub schema: String, + pub model_kind: WholeTemporalKind, + pub expected: MutationGuard, + pub capture_generation_id: String, +} + +impl TemporalProfileWire { + pub(crate) fn operator_eight_step() -> Self { + let labels=[RoomLabel::Empty,RoomLabel::OccupiedStatic,RoomLabel::OccupiedMoving,RoomLabel::AdjacentOnly]; + Self {profile_id:super::managed_schedule_contract::OPERATOR_EIGHT_PROFILE.into(),window_unit:"native_receiver_2s_window".into(),native_window_ms:2000, + train_windows:64,held_out_windows:16,train_episodes:4,held_out_episodes:4,windows_per_condition:[16;4],train_episodes_per_condition:[1;4],held_out_episodes_per_condition:[1;4], + episodes:labels.into_iter().flat_map(|label| [DataRole::Train,DataRole::Validation].into_iter().map(move |role|(label,role))).enumerate().map(|(index,(label,role))|TemporalEpisodeWire{ + index:index as u8,role,label,target_ms:if role==DataRole::Validation{15_000}else if label==RoomLabel::Empty{600_000}else{60_000},lead_in_ms:if matches!(label,RoomLabel::Empty|RoomLabel::AdjacentOnly){15_000}else{5_000},native_windows:if role==DataRole::Train{16}else{4} + }).collect()} + } + pub(crate) fn engineering() -> Self { + let labels = [ + RoomLabel::Empty, + RoomLabel::OccupiedStatic, + RoomLabel::OccupiedMoving, + RoomLabel::AdjacentOnly, + ]; + let order = [0, 1, 1, 2, 2, 3, 3, 0, 0, 1, 1, 2, 2, 3, 3]; + Self { + profile_id: "native-2s-three-source-56bin-option-a-engineering-v1".into(), + window_unit: "native_receiver_2s_window".into(), + native_window_ms: 2000, + train_windows: 64, + held_out_windows: 64, + train_episodes: 7, + held_out_episodes: 8, + windows_per_condition: [16; 4], + train_episodes_per_condition: [1, 2, 2, 2], + held_out_episodes_per_condition: [2; 4], + episodes: order + .iter() + .enumerate() + .map(|(i, n)| TemporalEpisodeWire { + index: i as u8, + role: if i < 7 { + DataRole::Train + } else { + DataRole::HeldOut + }, + label: labels[*n], + target_ms: if i == 0 { 600000 } else { 30000 }, + lead_in_ms: if i < 7 { 5000 } else { 15000 }, + native_windows: if i == 0 { 16 } else { 8 }, + }) + .collect(), + } + } +} + +pub const ASSESSMENT_SCHEMA: &str = "ruview.local-operational-assessment.v2"; +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct TemporalConditionFailureWire { + pub condition: RoomLabel, + pub codes: Vec, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct TemporalAssessmentWire { + pub schema: String, + pub receipt_sha256: String, + pub model_kind: WholeTemporalKind, + pub model_sha256: String, + pub verified_fit_record_sha256: String, + pub training_proof_sha256: String, + pub held_out_evidence_sha256: String, + pub complete_record_sha256: String, + pub evaluation_receipt_sha256: String, + pub context_digest: String, + pub room_digest: String, + pub evaluated_at_unix_ms: u64, + pub assessed_at_unix_ms: u64, + pub policy: TemporalQualificationPolicyWire, + pub policy_sha256: String, + pub window_unit: String, + pub outcome: String, + pub failure_reason: Option, + pub qualification_sha256: Option, + pub criteria: Vec, + pub condition_failures: Vec, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(tag = "status", rename_all = "snake_case", deny_unknown_fields)] +pub enum TemporalCurrentEvidenceWire { + None, + Unavailable { + identity: Option, + reason: String, + boot_epoch: Option, + receiver_epoch: Option, + }, + Ready { + identity: TemporalSelectedIdentityWire, + decision: super::model::PresenceDecision, + presence: bool, + occupied_probability: f32, + observed_at_unix_ms: u64, + observation_digest: String, + boot_epoch: String, + receiver_epoch: String, + window_start_unix_ms: u64, + window_end_unix_ms: u64, + valid_until_unix_ms: u64, + source_observed_at_unix_ms: u64, + }, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct TemporalSelectionStatusWire { + pub schema: String, + pub revision: u64, + pub selected: Option, + pub previous_available: bool, + pub qualification: Option, + pub qualification_error: Option, + pub integrity_error: Option, + pub current: TemporalCurrentEvidenceWire, +} + +pub const PAIRED_PROGRESS_SCHEMA: &str = "ruview.room-learner.progress.v2"; +#[derive(Debug, Clone, Serialize)] +pub struct TemporalPairedCaptureProgressWire { + pub capture_generation_id: String, + pub schedule_sha256: String, + pub stage: String, + pub current_episode: Option, + pub role: Option, + pub label: Option, + pub target_ms: Option, + pub lead_in_remaining_ms: u64, + pub observed_ms: u64, + pub retained_windows: u16, + pub durable_windows: u16, + pub sealed_train_episodes: u8, + pub sealed_held_out_episodes: u8, + pub durability_pending: bool, + pub awaiting_ground_truth: bool, + pub original_deadline: TemporalOriginalDeadlineWire, +} +#[derive(Debug, Clone, Serialize)] +pub struct TemporalPairedSourcesWire { + pub required: u8, + pub ready: u8, +} +#[derive(Debug, Clone, Serialize)] +pub struct TemporalPairedProgressWire { + pub schema: String, + pub success: bool, + pub installation_id: String, + /// Existing public calibration boot identity, never owner MutationGuard. + pub runtime_epoch: String, + pub observed_at_unix_ms: u64, + pub expires_at_unix_ms: u64, + pub room: Option, + pub context_match: bool, + pub stage: String, + pub state: String, + pub model_kind: Option, + pub profile: Option, + pub capture: Option, + pub candidate: Option, + pub evaluation: Option, + pub qualification: Option, + pub selection: Option, + pub sources: Option, + pub save_status: String, + pub blocker: Option, +} + +pub const DESELECT_REQUEST_SCHEMA: &str = "ruview.room-learner.deselect.v2"; +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct TemporalDeselectRequestV2 { + pub schema: String, + pub model_kind: WholeTemporalKind, + pub expected: MutationGuard, + pub selection_sha256: String, + pub selection_revision: u64, +} + +pub const RECOVERY_STATUS_SCHEMA: &str = "ruview.room-learner.recovery-status.v2"; +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct TemporalRecoveryStatusWire { + pub schema: String, + pub state: String, + pub capture_generation_id: Option, + pub context_digest: Option, + pub original_deadline: Option, +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_wire_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_wire_tests.rs new file mode 100644 index 0000000000..a9643b53b6 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_wire_tests.rs @@ -0,0 +1,322 @@ +//! Structural wire examples only: fake digest strings do not grant authority. +use super::*; +fn h(c: char) -> String { + std::iter::repeat(c).take(64).collect() +} +fn guard() -> MutationGuard { + MutationGuard { + runtime_epoch: "synthetic-wire-epoch".into(), + generation: 0, + workflow_revision: 0, + context_digest: None, + phase: None, + } +} +fn model() -> TemporalModelIdentityWire { + TemporalModelIdentityWire { + model_kind: WholeTemporalKind::ManagedTemporalV3, + model_sha256: h('a'), + verified_fit_record_sha256: h('b'), + } +} +fn complete() -> TemporalCompleteRefsWire { + TemporalCompleteRefsWire { + model: model(), + training_proof_sha256: h('c'), + complete_record_sha256: h('d'), + complete_commitment_sha256: h('e'), + } +} +fn eval() -> TemporalEvaluationRefsWire { + TemporalEvaluationRefsWire { + complete: complete(), + held_out_evidence_sha256: h('f'), + evaluation_receipt_sha256: h('1'), + } +} +fn profile() -> TemporalProfileWire { + let conditions = [ + RoomLabel::Empty, + RoomLabel::OccupiedStatic, + RoomLabel::OccupiedMoving, + RoomLabel::AdjacentOnly, + ]; + let order = [0, 1, 1, 2, 2, 3, 3, 0, 0, 1, 1, 2, 2, 3, 3]; + TemporalProfileWire { + profile_id: "native-2s-three-source-56bin-option-a-engineering-v1".into(), + window_unit: "native_receiver_2s_window".into(), + native_window_ms: 2000, + train_windows: 64, + held_out_windows: 64, + train_episodes: 7, + held_out_episodes: 8, + windows_per_condition: [16; 4], + train_episodes_per_condition: [1, 2, 2, 2], + held_out_episodes_per_condition: [2; 4], + episodes: order + .iter() + .enumerate() + .map(|(i, c)| TemporalEpisodeWire { + index: i as u8, + role: if i < 7 { + DataRole::Train + } else { + DataRole::HeldOut + }, + label: conditions[*c], + target_ms: if i == 0 { 600000 } else { 30000 }, + lead_in_ms: if i < 7 { 5000 } else { 15000 }, + native_windows: if i == 0 { 16 } else { 8 }, + }) + .collect(), + } +} +#[test] +fn public_identity_kind_is_required_and_legacy_numeric_version_is_not_an_alias() { + let mut value = serde_json::to_value(model()).unwrap(); + assert_eq!( + value["model_kind"], + "ruview.room-learner.managed-temporal-input.v3" + ); + value.as_object_mut().unwrap().remove("model_kind"); + assert!(serde_json::from_value::(value.clone()).is_err()); + value["model_kind"] = serde_json::json!("unknown"); + assert!(serde_json::from_value::(value.clone()).is_err()); + value["model_kind"] = serde_json::json!("ruview.room-learner.managed-temporal-input.v3"); + value["model_version"] = serde_json::json!(1); + assert!(serde_json::from_value::(value).is_err()); +} +#[test] +fn command_inputs_reject_rows_reports_and_verified_booleans() { + let command = TemporalEvaluateRequestV2 { + schema: EVALUATE_REQUEST_SCHEMA.into(), + expected: guard(), + capture_generation_id: h('0'), + refs: complete(), + }; + let original = serde_json::to_value(command).unwrap(); + for key in ["rows", "report", "verified", "complete_witness"] { + let mut value = original.clone(); + value[key] = serde_json::json!(true); + assert!( + serde_json::from_value::(value).is_err(), + "{key}" + ); + } +} +#[test] +fn export_canonical_structural_wire_examples_without_runtime_authority() { + let selected = TemporalSelectedIdentityWire { + selection_sha256: h('2'), + selection_revision: 1, + model_kind: WholeTemporalKind::ManagedTemporalV3, + model_sha256: h('a'), + verified_fit_record_sha256: h('b'), + training_proof_sha256: h('c'), + held_out_evidence_sha256: h('f'), + complete_record_sha256: h('d'), + evaluation_receipt_sha256: h('1'), + qualification_sha256: h('3'), + policy_sha256: super::super::qualification::hash( + &TemporalQualificationPolicyWire::current(), + ) + .unwrap(), + context_digest: h('4'), + room_digest: h('5'), + source_node_ids: vec![15, 16, 17], + selected_at_unix_ms: 1_000_200, + evaluated_at_unix_ms: 1_000_100, + authority_scope: "local_operator_binary_presence".into(), + }; + let policy = TemporalQualificationPolicyWire::current(); + let criteria = [ + RoomLabel::Empty, + RoomLabel::OccupiedStatic, + RoomLabel::OccupiedMoving, + RoomLabel::AdjacentOnly, + ] + .iter() + .map(|label| { + let positive = matches!(label, RoomLabel::OccupiedStatic | RoomLabel::OccupiedMoving); + TemporalConditionCriterionWire { + condition: *label, + train_windows: 16, + held_out_windows: 16, + scored_windows: 16, + abstained_windows: 0, + missing_windows: 0, + declared_train_episodes: if *label == RoomLabel::Empty { 1 } else { 2 }, + declared_held_out_episodes: 2, + evaluated_window_coverage: 1., + unconditional_sensitivity: positive.then_some(1.), + false_positive_rate: (!positive).then_some(0.), + } + }) + .collect(); + let qualification = TemporalQualificationSummaryWire { + schema: QUALIFICATION_SCHEMA.into(), + receipt_sha256: h('3'), + model_kind: WholeTemporalKind::ManagedTemporalV3, + model_sha256: h('a'), + verified_fit_record_sha256: h('b'), + training_lineage_basis: "managed_temporal_training_proof_v1".into(), + training_proof_sha256: h('c'), + held_out_evidence_sha256: h('f'), + complete_record_sha256: h('d'), + evaluation_receipt_sha256: h('1'), + context_digest: h('4'), + room_digest: h('5'), + evaluated_at_unix_ms: 1_000_100, + policy_sha256: super::super::qualification::hash(&policy).unwrap(), + policy, + label_provenance: "temporal-training-proof-v1+sealed-native-heldout-v1".into(), + criteria, + authority_scope: "local_operator_binary_presence".into(), + window_unit: "native_receiver_2s_window".into(), + train_windows: 64, + held_out_windows: 64, + train_episodes: 7, + held_out_episodes: 8, + independence_verified: false, + physically_validated: false, + measurement_clock_available: false, + }; + let idle = TemporalLearnerStatusWire { + schema: STATUS_SCHEMA.into(), + enabled: true, + available_model_kinds: vec![WholeTemporalKind::ManagedTemporalV3], + temporal_profile: profile(), + workflow_kind: WorkflowKindWire::Idle, + mutation_guard: guard(), + context_digest: None, + room_digest: None, + source_node_ids: vec![], + recovery: None, + capture: None, + candidate: None, + training_proof_sha256: None, + complete: None, + evaluation: None, + assessment: None, + model_selection: None, + job_state: "idle".into(), + job_operation: None, + error_code: None, + error: None, + legacy_status: None, + }; + let selection = TemporalSelectionStatusWire { + schema: SELECTION_STATUS_SCHEMA.into(), + revision: 1, + selected: Some(selected.clone()), + previous_available: false, + qualification: Some(qualification.clone()), + qualification_error: None, + integrity_error: None, + current: TemporalCurrentEvidenceWire::Unavailable { + identity: Some(selected.clone()), + reason: "collecting_full_horizon".into(), + boot_epoch: Some("synthetic-boot".into()), + receiver_epoch: Some("synthetic-receiver".into()), + }, + }; + let matching = TemporalPairedProgressWire { + schema: PAIRED_PROGRESS_SCHEMA.into(), + success: true, + installation_id: "synthetic-installation".into(), + runtime_epoch: "synthetic-boot".into(), + observed_at_unix_ms: 1_001_000, + expires_at_unix_ms: 1_011_000, + room: Some(super::super::lifecycle::PublishedProgressRoom { + binding_digest: h('5'), + revision: 1, + }), + context_match: true, + stage: "full_complete".into(), + state: "ready".into(), + model_kind: Some(WholeTemporalKind::ManagedTemporalV3), + profile: Some(profile()), + capture: Some(TemporalPairedCaptureProgressWire { + capture_generation_id: h('0'), + schedule_sha256: h('6'), + stage: "full_complete".into(), + current_episode: None, + role: None, + label: None, + target_ms: None, + lead_in_remaining_ms: 0, + observed_ms: 0, + retained_windows: 128, + durable_windows: 128, + sealed_train_episodes: 7, + sealed_held_out_episodes: 8, + durability_pending: false, + awaiting_ground_truth: false, + original_deadline: TemporalOriginalDeadlineWire { + created_at_unix_ms: 1_000_000, + expires_at_unix_ms: 87_400_000, + }, + }), + candidate: Some(TemporalCandidateIdentityWire { + model_kind: WholeTemporalKind::ManagedTemporalV3, + model_sha256: h('a'), + verified_fit_record_sha256: h('b'), + context_digest: h('4'), + room_digest: h('5'), + persisted: true, + }), + evaluation: None, + qualification: Some(qualification.clone()), + selection: Some(selection.clone()), + sources: Some(TemporalPairedSourcesWire { + required: 3, + ready: 3, + }), + save_status: "not_reported".into(), + blocker: None, + }; + let mismatch = TemporalPairedProgressWire { + schema: PAIRED_PROGRESS_SCHEMA.into(), + success: true, + installation_id: "synthetic-installation".into(), + runtime_epoch: "synthetic-boot".into(), + observed_at_unix_ms: 1_001_000, + expires_at_unix_ms: 1_011_000, + room: None, + context_match: false, + stage: "blocked".into(), + state: "idle".into(), + model_kind: None, + profile: None, + capture: None, + candidate: None, + evaluation: None, + qualification: None, + selection: None, + sources: None, + save_status: "not_reported".into(), + blocker: Some("context_mismatch".into()), + }; + println!("C4C_CANONICAL_PROGRESS_EXAMPLES={}",serde_json::to_string(&serde_json::json!({"fixture_kind":"SYNTHETIC_STRUCTURAL_SERIALIZATION_ONLY_NOT_AUTHORITY","matching_complete":matching,"context_mismatch_redacted":mismatch,"selected_temporal_idle":{"schema":SELECTION_SCHEMA,"success":true,"progress_schema":PAIRED_PROGRESS_SCHEMA,"model_selection":selection}})).unwrap()); + let examples = serde_json::json!({"fixture_kind":"SYNTHETIC_STRUCTURAL_SERIALIZATION_ONLY_NOT_AUTHORITY","idle_response":{"success":true,"status":idle},"candidate_identity":TemporalCandidateIdentityWire{model_kind:WholeTemporalKind::ManagedTemporalV3,model_sha256:h('a'),verified_fit_record_sha256:h('b'),context_digest:h('4'),room_digest:h('5'),persisted:false},"selected_identity":selected,"qualification_summary":qualification,"configure_request":TemporalConfigureRequestV2{schema:CONFIGURE_REQUEST_SCHEMA.into(),model_kind:WholeTemporalKind::ManagedTemporalV3,expected:guard(),binding_digest:h('4'),revision:1,source_node_ids:vec![15,16,17],new_training:true,recovery_consent:true,profile_id:None},"evaluate_request":TemporalEvaluateRequestV2{schema:EVALUATE_REQUEST_SCHEMA.into(),expected:guard(),capture_generation_id:h('0'),refs:complete()},"qualify_request":TemporalQualifyRequestV2{schema:QUALIFY_REQUEST_SCHEMA.into(),expected:guard(),capture_generation_id:h('0'),refs:eval()}}); + println!( + "C4C_CANONICAL_PUBLIC_WIRE_EXAMPLES={}", + serde_json::to_string(&examples).unwrap() + ); +} + + +#[test] +fn operator_eight_step_policy_is_explicit_and_preserves_numeric_quality_thresholds() { + let old = TemporalQualificationPolicyWire::current(); + assert_eq!(TemporalQualificationPolicyWire::for_operator_eight_step(false), old); + let new = TemporalQualificationPolicyWire::for_operator_eight_step(true); + assert_eq!(new.id, OPERATOR_EIGHT_STEP_TEMPORAL_POLICY_ID); + assert_eq!(new.minimum_windows_per_condition, 4); + assert_eq!(old.minimum_windows_per_condition, 16); + assert_eq!(new.window_unit, old.window_unit); + assert_eq!(new.minimum_unconditional_sensitivity, old.minimum_unconditional_sensitivity); + assert_eq!(new.maximum_false_positive_rate, old.maximum_false_positive_rate); + assert_eq!(new.minimum_evaluated_window_coverage, old.minimum_evaluated_window_coverage); + assert_ne!(super::super::qualification::hash(&new).unwrap(), super::super::qualification::hash(&old).unwrap()); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/training.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/training.rs new file mode 100644 index 0000000000..737d4ae721 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/training.rs @@ -0,0 +1,454 @@ +//! Bounded numerical optimization using the production trainer machinery. +use super::model::{ + center_empty_heads, extractor, initial_encoder, initial_projection_weights, EncoderConfig, + RoomModel, SourceProfile, TrainingManifest, +}; +use super::observation::{CaptureCohort, CaptureContext, RoomLabel, ValidatedDataset}; +use crate::embedding::info_nce_loss; +use serde::{Deserialize, Serialize}; +use std::cell::Cell; +use std::collections::HashSet; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::Instant; + +#[path = "optimization.rs"] +mod optimization; + +#[path = "fit_config.rs"] +mod fit_config; +pub use fit_config::FitConfig; + +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +pub struct ObjectiveValues { + pub total: f32, + pub contrastive: f32, + pub supervised: f32, +} +#[path = "evaluation_report.rs"] +mod evaluation_report; +pub use evaluation_report::{ + BinaryConfusion, ConditionEvaluation, EvaluationRates, EvaluationReport, +}; +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TrainingReport { + pub initial: ObjectiveValues, + pub final_objective: ObjectiveValues, + pub completed_epochs: usize, + pub parameter_count: usize, + pub parameter_delta_l2: f32, + pub encoder_delta_l2: f32, + pub backbone_delta_l2: f32, + pub projection_delta_l2: f32, + pub head_delta_l2: f32, + pub objective_evaluations: usize, + pub training_cohorts: usize, + pub contrastive_trials: usize, + pub validation: Option, + pub evidence: String, +} +struct Row { + // Train-only inputs are immutable for the complete fit. Conditioning and + // task skips do not depend on any optimized parameter, so compute them once + // rather than allocating/rescanning raw tokens for every numerical probe. + tokens: Vec>>, + skips: Vec<[f32; 2]>, + occupied: f32, + contrastive: bool, +} +impl Row { + fn prepare( + model: &RoomModel, + tokens: Vec>>, + occupied: f32, + contrastive: bool, + ) -> Result { + if tokens.len() != model.profiles.len() + || tokens.iter().any(|source| { + source.len() != model.config.temporal_tokens + || source.iter().any(|row| { + row.len() != model.config.bins + model.config.source_count + || row.iter().any(|value| !value.is_finite()) + }) + }) + { + return Err("invalid immutable training representation shape or value".into()); + } + let mut conditioned = Vec::with_capacity(tokens.len()); + let mut skips = Vec::with_capacity(tokens.len()); + for (index, source) in tokens.into_iter().enumerate() { + // Keep the exact operation/order used by model::represent_with_encoder. + let mean = source + .iter() + .flat_map(|row| row[..model.config.bins].iter()) + .sum::() + / (model.config.temporal_tokens * model.config.bins) as f32; + let temporal = source + .windows(2) + .flat_map(|w| { + w[0][..model.config.bins] + .iter() + .zip(&w[1][..model.config.bins]) + }) + .map(|(a, b)| (a - b).abs()) + .sum::() + / ((model.config.temporal_tokens - 1) * model.config.bins) as f32; + skips.push([mean, temporal]); + let mut source = source; + for row in &mut source { + for value in &mut row[..model.config.bins] { + *value /= model.profiles[index].max_normalized; + } + } + conditioned.push(source); + } + Ok(Self { + tokens: conditioned, + skips, + occupied, + contrastive, + }) + } +} +fn represent_row( + model: &RoomModel, + row: &Row, + encoder: &mut crate::embedding::EmbeddingExtractor, + projectors: &[crate::embedding::ProjectionHead], +) -> Result<(Vec, Vec>), String> { + let mut features = Vec::with_capacity(model.profiles.len() * (model.config.embedding_dim + 2)); + let mut embeddings = Vec::with_capacity(row.tokens.len()); + for (index, source) in row.tokens.iter().enumerate() { + encoder.projection = projectors[index].clone(); + let embedding = encoder.extract(source); + features.extend_from_slice(&embedding); + features.extend_from_slice(&row.skips[index]); + embeddings.push(embedding); + } + if !features.iter().all(|value| value.is_finite()) { + return Err("non-finite representation".into()); + } + Ok((features, embeddings)) +} +fn occupied(label: RoomLabel) -> f32 { + if matches!(label, RoomLabel::OccupiedStatic | RoomLabel::OccupiedMoving) { + 1.0 + } else { + 0.0 + } +} +fn bce(logit: f32, target: f32) -> f32 { + logit.max(0.0) - logit * target + (-logit.abs()).exp().ln_1p() +} +fn normalized(mut embedding: Vec) -> Vec { + let norm = embedding + .iter() + .map(|x| x * x) + .sum::() + .sqrt() + .max(1.0e-8); + embedding.iter_mut().for_each(|x| *x /= norm); + embedding +} +fn objective( + model: &RoomModel, + rows: &[Row], + params: &[f32], + config: &FitConfig, +) -> Result { + let mut supervised = 0.0; + let mut by_source = vec![Vec::new(); model.profiles.len()]; + let head = ¶ms[model.representation_parameter_count()..]; + let projectors = model.source_projectors(params)?; + let mut encoder = extractor(&model.config); + encoder + .transformer + .unflatten_weights(¶ms[..model.encoder_weights.len()])?; + for row in rows { + let (features, embeddings) = represent_row(model, row, &mut encoder, &projectors)?; + supervised += bce(model.logit(&features, head), row.occupied); + // One event per independent trial per batch. Sibling/overlapping windows + // in the same trial cannot accidentally become contrastive negatives. + if row.contrastive { + for (source, embedding) in by_source.iter_mut().zip(embeddings) { + source.push(normalized(embedding)); + } + } + } + supervised /= rows.len() as f32; + let mut contrastive = 0.0; + // A source-zero star aligns every source without O(source_count^2) pairs. + // Both directions are optimized; each row's other source is its positive. + for other in by_source.iter().skip(1) { + contrastive += (info_nce_loss(&by_source[0], other, config.temperature) + + info_nce_loss(other, &by_source[0], config.temperature)) + * 0.5; + } + contrastive /= (by_source.len() - 1) as f32; + let total = supervised * config.supervised_weight + contrastive * config.contrastive_weight; + if !total.is_finite() { + return Err("non-finite training objective".into()); + } + Ok(ObjectiveValues { + total, + supervised, + contrastive, + }) +} +fn profiles(dataset: &ValidatedDataset) -> Result, String> { + let mut profiles = Vec::new(); + for source in &dataset.context().sources { + let width = source.feature_count; + let mut mean = vec![0.0f64; width]; + let mut square = vec![0.0f64; width]; + let mut count = 0usize; + for event in dataset + .train() + .iter() + .filter(|c| c.label == RoomLabel::Empty) + { + let view = event + .views + .iter() + .find(|v| v.source_id == source.source_id) + .ok_or("missing baseline source")?; + for frame in &view.samples { + count += 1; + for (k, &v) in frame.iter().enumerate() { + mean[k] += v as f64; + square[k] += (v as f64).powi(2); + } + } + } + if count < 2 { + return Err("each source needs Train Empty baseline frames".into()); + } + let scale: Vec = mean + .iter() + .zip(&square) + .map(|(m, s)| { + (s / count as f64 - (m / count as f64).powi(2)) + .max(0.0) + .sqrt() + .max(0.1) as f32 + }) + .collect(); + let mean: Vec = mean.iter().map(|m| (m / count as f64) as f32).collect(); + let mut max_normalized = 1.0f32; + for event in dataset.train() { + let view = event + .views + .iter() + .find(|v| v.source_id == source.source_id) + .ok_or("missing training source")?; + for frame in &view.samples { + for (k, v) in frame.iter().enumerate() { + max_normalized = max_normalized.max(((v - mean[k]) / scale[k]).abs()); + } + } + } + profiles.push(SourceProfile { + source: source.clone(), + mean, + scale, + max_normalized, + }); + } + Ok(profiles) +} +impl RoomModel { + pub fn fit( + dataset: &ValidatedDataset, + config: FitConfig, + ) -> Result<(Self, TrainingReport), String> { + Self::fit_with_cancel(dataset, config, &AtomicBool::new(false)) + } + pub fn fit_with_cancel( + dataset: &ValidatedDataset, + config: FitConfig, + cancel: &AtomicBool, + ) -> Result<(Self, TrainingReport), String> { + let started = Instant::now(); + config.validate()?; + if cancel.load(Ordering::Relaxed) { + return Err("training cancelled".into()); + } + if !(4..=64).contains(&dataset.train().len()) { + return Err("training requires 4..64 cohorts".into()); + } + let trial_count = dataset + .train() + .iter() + .map(|c| &c.trial_id) + .collect::>() + .len(); + if trial_count < 2 + || !dataset.train().iter().any(|c| occupied(c.label) == 1.0) + || !dataset + .train() + .iter() + .any(|c| c.label == RoomLabel::AdjacentOnly) + { + return Err( + "training needs independent trials, occupied and adjacent-only controls".into(), + ); + } + let encoder_config = EncoderConfig { + source_count: dataset.context().sources.len(), + bins: dataset + .context() + .sources + .iter() + .map(|s| s.feature_count) + .max() + .unwrap_or(0), + hidden_dim: config.hidden_dim, + embedding_dim: config.embedding_dim, + temporal_tokens: config.temporal_tokens, + temperature: config.temperature, + }; + if !(1..=128).contains(&encoder_config.bins) { + return Err("source width exceeds bounded encoder".into()); + } + let encoder = initial_encoder(&encoder_config)?; + let mut model = Self { + version: 1, + algorithm: "aether_multiview_infonce_room_bce_v1".into(), + context: dataset.context().clone(), + training_manifest: TrainingManifest::from_training(dataset.train())?, + source_projection_weights: initial_projection_weights(&encoder_config), + config: encoder_config, + profiles: profiles(dataset)?, + encoder_weights: encoder.transformer.flatten_weights(), + head_weights: vec![ + 0.0; + dataset.context().sources.len() * (config.embedding_dim + 2) + 1 + ], + }; + model.validate()?; + if model.parameter_count() > config.max_parameters { + return Err("model exceeds parameter budget".into()); + } + center_empty_heads(&mut model, dataset.train())?; + model.validate()?; + let mut independent: Vec<&CaptureCohort> = Vec::new(); + let mut rows = Vec::new(); + for c in dataset.train() { + let contrastive = !independent.iter().any(|other| { + other.trial_id == c.trial_id + || c.views.iter().any(|v| { + other.views.iter().any(|w| { + v.source_id == w.source_id + && (v.frame_ids.iter().any(|id| w.frame_ids.contains(id)) + || (v.first_time_ns <= w.last_time_ns + && v.last_time_ns >= w.first_time_ns)) + }) + }) + }); + if contrastive { + independent.push(c); + } + rows.push(Row::prepare( + &model, + model.tokens(c, false)?, + occupied(c.label), + contrastive, + )?); + } + let contrastive_trials = independent.len(); + if contrastive_trials < 2 { + return Err("contrastive objective needs non-overlapping independent trials".into()); + } + let initial_params = model.parameters(); + let mut params = initial_params.clone(); + let initial = objective(&model, &rows, ¶ms, &config)?; + let mut current = initial; + let evaluations = Cell::new(1usize); + let check = || -> Result<(), String> { + if cancel.load(Ordering::Relaxed) { + Err("training cancelled".into()) + } else if started.elapsed().as_millis() >= config.max_duration_ms as u128 { + Err("training time budget exhausted".into()) + } else if evaluations.get() >= config.max_objective_evaluations { + Err("objective evaluation budget exhausted".into()) + } else { + Ok(()) + } + }; + let mut epochs = 0; + for _ in 0..config.epochs { + check()?; + let next = optimization::epoch_step( + &model, + &rows, + ¶ms, + &config, + current, + &check, + &evaluations, + )?; + if let Some((next, loss)) = next { + params = next; + current = loss; + } else { + break; + } + epochs += 1; + } + check()?; + if epochs == 0 || params == initial_params { + return Err("optimizer made no improving parameter update".into()); + } + optimization::require_rf_update(&model, ¶ms, &initial_params)?; + model.set_parameters(¶ms)?; + model.validate()?; + let delta = |a: &[f32], b: &[f32]| { + a.iter() + .zip(b) + .map(|(x, y)| (x - y).powi(2)) + .sum::() + .sqrt() + }; + let split = model.representation_parameter_count(); + let report = TrainingReport { + initial, + final_objective: current, + completed_epochs: epochs, + parameter_count: params.len(), + parameter_delta_l2: delta(¶ms, &initial_params), + encoder_delta_l2: delta(¶ms[..split], &initial_params[..split]), + backbone_delta_l2: delta( + ¶ms[..model.encoder_weights.len()], + &initial_params[..model.encoder_weights.len()], + ), + projection_delta_l2: delta( + ¶ms[model.encoder_weights.len()..split], + &initial_params[model.encoder_weights.len()..split], + ), + head_delta_l2: delta(¶ms[split..], &initial_params[split..]), + objective_evaluations: evaluations.get(), + training_cohorts: rows.len(), + contrastive_trials, + validation: if dataset.validation().is_empty() { + None + } else { + Some(model.evaluate(dataset.validation())?) + }, + evidence: "EXPERIMENTAL_RF_ONLY; physical accuracy and promotion are unqualified" + .into(), + }; + check()?; + Ok((model, report)) + } + /// Explicit evaluation only; fit never reads HeldOut observations. + pub fn evaluate(&self, cohorts: &[CaptureCohort]) -> Result { + EvaluationReport::compute(self, cohorts) + } +} + +#[cfg(test)] +#[path = "learning_tests.rs"] +pub(crate) mod learning_tests; + +#[cfg(test)] +#[path = "source_conditioning_tests.rs"] +mod source_conditioning_tests; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api.rs new file mode 100644 index 0000000000..ff95542388 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api.rs @@ -0,0 +1,484 @@ +//! Managed integration; raw RF enters only from the existing receiver, never +//! caller-supplied learning scores or uploaded synthetic examples. +use super::*; +use ruview_unified::RfModality; +use wifi_densepose_sensing_server::room_learner::{ + capture::PhaseRequest, + lifecycle::{MutationGuard, RoomLearningRuntime}, + native::{self, NativePacketInput}, + observation::{CaptureProvenance, DataRole, SourceSpec}, + training::FitConfig, +}; + +#[path = "room_learner_ingress.rs"] +mod ingress; +use ingress::native_candidates; +pub(super) use ingress::{observe, observe_with_receiver_timing, ReceiverIngressTiming, LiveSource, NativeSourceKey}; + +fn failure(code: &str, error: impl ToString) -> Json { + let message = error.to_string(); + let code = match message.as_str() { + "learner_state_changed" => "learner_state_changed", + "learner_recovery_pending" => "learner_recovery_pending", + _ => code, + }; + Json(serde_json::json!({"success":false,"error_code":code, + "error":error.to_string().chars().filter(|c|!c.is_control()).take(256).collect::()})) +} +async fn runtime(state: &SharedState) -> Result, Json> { + let s = state.read().await; + if s.privacy_mode { + return Err(failure( + "learner_privacy_mode", + "Room learning is unavailable in privacy-redacted mode.", + )); + } + s.room_learning.clone().ok_or_else(|| { + failure( + "learner_disabled", + "This helper does not enable the managed room learner.", + ) + }) +} +fn now_ns(runtime: &RoomLearningRuntime) -> u64 { + runtime.receiver_now_ns() +} +#[derive(serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct ConfigureRequest { + expected: MutationGuard, + binding_digest: String, + revision: u64, + source_node_ids: Vec, + new_training: bool, + #[serde(default)] + recovery_consent: bool, +} +fn source_ready(state: &AppStateInner, spec: &SourceSpec, now: std::time::Instant) -> bool { + native_candidates(state, spec.source_id, now) + .iter() + .any(|(key, meta)| { + native::grid_id( + key.n_antennas as usize, + key.raw_bins as usize, + wifi_densepose_hardware::PpduType::from_byte(key.ppdu_type), + ) + .ok() + .as_deref() + == Some(spec.grid_id.as_str()) + && meta.frequency_hz == spec.center_frequency_hz + && meta.bandwidth_hz == Some(spec.bandwidth_hz) + }) + && spec.feature_count == 56 + && spec.signal_kind == RfModality::WifiCsi + && spec.device_id == format!("esp32-node-{}", spec.source_id) +} +pub(super) async fn configure( + State(state): State, + Json(request): Json, +) -> Json { + let rt = match runtime(&state).await { + Ok(r) => r, + Err(e) => return e, + }; + if !valid_binding_digest(&request.binding_digest) + || !valid_source_node_ids(&request.source_node_ids) + || !(2..=8).contains(&request.source_node_ids.len()) + || request.revision == 0 + { + return failure( + "learner_context_invalid", + "Use a current room digest/revision and 2 to 8 sorted unique node IDs.", + ); + } + let now = std::time::Instant::now(); + let s = state.read().await; + if !request.new_training { + let catalog = match rt.saved_candidates(&request.binding_digest) { + Ok(c) => c, + Err(e) => return failure("learner_saved_models_unavailable", e), + }; + let found = catalog.candidates.iter().find(|entry| { + entry.context.revision == request.revision + && entry + .context + .sources + .iter() + .map(|spec| spec.source_id) + .collect::>() + == request.source_node_ids + && entry + .context + .sources + .iter() + .all(|spec| source_ready(&s, spec, now)) + }); + let Some(saved) = found else { + return failure( + "learner_saved_context_unavailable", + if catalog.candidates.is_empty() { + "No readable saved candidate for this exact room. Stored history is preserved; start a new capture or inspect model recovery." + } else { + "Saved candidates are retained, but their trained source/PHY context is not currently receiving. Restore those inputs or explicitly train a new model." + }, + ); + }; + let context = saved.context.clone(); + drop(s); + return match rt.configure_for_recovery( + &request.expected, + context, + opaque_calibration_id("learner-resume"), + false, + request.recovery_consent, + now_ns(&rt), + ) { + Ok(()) => Json(serde_json::json!({"success":true,"status":rt.status(now_ns(&rt))})), + Err(e) => failure("learner_resume_failed", e), + }; + } + let mut sources = Vec::new(); + for &id in &request.source_node_ids { + let Some(_) = s.node_states.get(&id) else { + return failure( + "learner_source_missing", + format!("Node {id} is not receiving."), + ); + }; + let Some((key, meta)) = + native_candidates(&s, id, now) + .into_iter() + .max_by(|(ka, a), (kb, b)| { + a.receiver_rate_hz + .total_cmp(&b.receiver_rate_hz) + .then(ka.raw_bins.cmp(&kb.raw_bins)) + }) + else { + return failure("learner_source_unready",format!("Node {id} needs a stable, supported exact native PHY cohort and a fresh eight-packet window.")); + }; + let bandwidth_hz = meta + .bandwidth_hz + .expect("eligible native cohort has supported bandwidth"); + sources.push(SourceSpec { + source_id: id, + device_id: format!("esp32-node-{id}"), + grid_id: match native::grid_id( + key.n_antennas as usize, + key.raw_bins as usize, + wifi_densepose_hardware::PpduType::from_byte(key.ppdu_type), + ) { + Ok(id) => id, + Err(error) => return failure("learner_grid_unsupported", error), + }, + feature_count: 56, + signal_kind: RfModality::WifiCsi, + center_frequency_hz: meta.frequency_hz, + bandwidth_hz, + link_id: format!("receiver-{id}-transmitter-unverified"), + link_index: 0, + geometry: None, + }); + } + drop(s); + let context = match rt.bind_context(request.binding_digest, request.revision, sources) { + Ok(c) => c, + Err(e) => return failure("learner_context_invalid", e), + }; + let run = opaque_calibration_id("learner-run"); + match rt.configure_for_recovery( + &request.expected, + context, + run, + request.new_training, + request.recovery_consent, + now_ns(&rt), + ) { + Ok(()) => Json(serde_json::json!({"success":true,"status":rt.status(now_ns(&rt))})), + Err(e) => failure("learner_configure_failed", e), + } +} +pub(super) async fn status(State(state): State) -> Json { + match runtime(&state).await { + Ok(rt) => { + let state = state.read().await; + Json(selection_routes::enriched_status(&state, &rt)) + } + Err(e) => e, + } +} +#[derive(serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct BeginRequest { + expected: MutationGuard, + phase: PhaseRequest, +} +#[derive(serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct GuardRequest { + expected: MutationGuard, +} +#[derive(serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct TrainRequest { + expected: MutationGuard, + config: FitConfig, +} +pub(super) async fn begin_phase( + State(state): State, + Json(request): Json, +) -> Json { + let state = state.write().await; + if state.privacy_mode { + return failure( + "learner_privacy_mode", + "Room learning is unavailable in privacy-redacted mode.", + ); + } + if state.field_model.as_ref().is_some_and(|f| { + matches!(f.status(), CalibrationStatus::Collecting) + || (state.calibration_session_id.is_some() + && matches!(f.status(), CalibrationStatus::Uncalibrated)) + }) { + return failure( + "legacy_empty_capture_active", + "Finish or cancel the legacy empty capture before learner example capture.", + ); + } + let Some(rt) = state.room_learning.as_ref() else { + return failure("learner_disabled", "Managed learner is not enabled."); + }; + if rt.context().as_ref().is_none_or(|context| { + context + .sources + .iter() + .any(|spec| !source_ready(&state, spec, std::time::Instant::now())) + }) { + return failure("learner_source_unready","A selected source/grid is no longer fresh. Restore its stream and refresh source readiness before capture."); + } + match rt.begin_phase(&request.expected, request.phase, now_ns(rt)) { + Ok(()) => Json(serde_json::json!({"success":true,"status":rt.status(now_ns(rt))})), + Err(e) => failure("learner_phase_rejected", e), + } +} +#[derive(serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct FinishRequest { + expected: MutationGuard, + conditions_held: bool, +} +pub(super) async fn finish_phase( + State(state): State, + Json(request): Json, +) -> Json { + let rt = match runtime(&state).await { + Ok(r) => r, + Err(e) => return e, + }; + match rt.finish_phase(&request.expected, now_ns(&rt), request.conditions_held) { + Ok(receipt) => Json(serde_json::json!({"success":true,"receipt":receipt})), + Err(e) => failure("learner_phase_incomplete", e), + } +} +pub(super) async fn cancel_phase( + State(state): State, + Json(request): Json, +) -> Json { + let rt = match runtime(&state).await { + Ok(r) => r, + Err(e) => return e, + }; + match rt.cancel_phase(&request.expected) { + Ok(()) => Json(serde_json::json!({"success":true})), + Err(e) => failure("learner_phase_cancel_failed", e), + } +} +pub(super) async fn train( + State(state): State, + Json(request): Json, +) -> Json { + let rt = match runtime(&state).await { + Ok(r) => r, + Err(e) => return e, + }; + match rt.start_training(&request.expected, request.config) { + Ok(generation) => Json( + serde_json::json!({"success":true,"generation":generation,"status":rt.status(now_ns(&rt))}), + ), + Err(e) => failure("learner_train_rejected", e), + } +} +pub(super) async fn cancel_training( + State(state): State, + Json(request): Json, +) -> Json { + let rt = match runtime(&state).await { + Ok(r) => r, + Err(e) => return e, + }; + match rt.cancel_training(&request.expected) { + Ok(phase) => Json(serde_json::json!({"success":true,"state":phase})), + Err(e) => failure("learner_cancel_failed", e), + } +} +#[derive(serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct EvaluationRequest { + expected: MutationGuard, + role: DataRole, +} +pub(super) async fn evaluate( + State(state): State, + Json(request): Json, +) -> Json { + let rt = match runtime(&state).await { + Ok(r) => r, + Err(e) => return e, + }; + match rt.evaluate(&request.expected, request.role) { + Ok(report) => Json(serde_json::json!({"success":true,"evaluation":report})), + Err(e) => failure("learner_evaluation_rejected", e), + } +} +pub(super) async fn prediction(State(state): State) -> Json { + let rt = match runtime(&state).await { + Ok(r) => r, + Err(e) => return e, + }; + match rt.prediction(now_ns(&rt)) { + Ok((mutation_guard, prediction)) => Json( + serde_json::json!({"success":true,"mutation_guard":mutation_guard,"candidate_prediction":prediction,"authority":"experimental_candidate_only"}), + ), + Err(e) => failure("learner_prediction_unavailable", e), + } +} + +#[derive(serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct RestoreCaptureRequest { + expected: MutationGuard, + recovery_id: String, + binding_digest: String, + revision: u64, + source_node_ids: Vec, +} +pub(super) async fn restore_capture( + State(state): State, + Json(request): Json, +) -> Json { + let rt = match runtime(&state).await { + Ok(r) => r, + Err(e) => return e, + }; + let now = now_ns(&rt); + let context = match rt.saved_recovery_context(&request.recovery_id, now) { + Ok(c) => c, + Err(e) => return failure("learner_recovery_restore_failed", e), + }; + if context.room_digest != request.binding_digest + || context.revision != request.revision + || context + .sources + .iter() + .map(|s| s.source_id) + .collect::>() + != request.source_node_ids + { + return failure( + "learner_recovery_context_mismatch", + "Saved capture does not match the requested room, revision and source set.", + ); + } + let s = state.read().await; + if s.field_model.as_ref().is_some_and(|f| { + matches!(f.status(), CalibrationStatus::Collecting) + || (s.calibration_session_id.is_some() + && matches!(f.status(), CalibrationStatus::Uncalibrated)) + }) { + return failure( + "learner_capture_conflict", + "Finish or cancel the active field calibration first.", + ); + } + if context + .sources + .iter() + .any(|spec| !source_ready(&s, spec, std::time::Instant::now())) + { + return failure( + "learner_recovery_source_unready", + "Restore the exact saved native source/PHY streams before recovering capture.", + ); + } + let restored = + rt.restore_capture_recovery(&request.expected, &request.recovery_id, &context, now); + drop(s); + match restored { + Ok(()) => Json(serde_json::json!({"success":true,"status":rt.status(now_ns(&rt))})), + Err(e) => failure("learner_recovery_restore_failed", e), + } +} +/// Owner-only: paired read/calibration grants do not include this mutation. +#[derive(serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct EnableCurrentRecoveryRequest { + schema: String, + expected: MutationGuard, + run_id: String, + consent: bool, +} +pub(super) async fn enable_current_recovery( + State(state): State, + Json(request): Json, +) -> Json { + if request.schema != "ruview.room-learner.enable-current-recovery.v1" || !request.consent { + return failure("learner_recovery_consent_required", "Explicit current-run recovery consent is required."); + } + let rt = match runtime(&state).await { Ok(rt) => rt, Err(e) => return e }; + match rt.enable_current_capture_recovery(&request.expected, &request.run_id, request.consent, now_ns(&rt)) { + Ok(()) => Json(serde_json::json!({"success":true,"status":rt.status(now_ns(&rt))})), + Err(e) => failure("learner_recovery_enable_failed", e), + } +} + +#[derive(serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct DiscardCaptureRequest { + expected: MutationGuard, + recovery_id: String, +} +pub(super) async fn discard_capture( + State(state): State, + Json(request): Json, +) -> Json { + let rt = match runtime(&state).await { + Ok(r) => r, + Err(e) => return e, + }; + match rt.discard_capture_recovery(&request.expected, &request.recovery_id) { + Ok(()) => Json(serde_json::json!({"success":true,"status":rt.status(now_ns(&rt))})), + Err(e) => failure("learner_recovery_discard_failed", e), + } +} + +#[cfg(test)] +#[path = "room_learner_api_tests.rs"] +mod tests; + + +#[path = "room_learner_api_publication.rs"] +mod publication; +#[cfg(feature = "mqtt")] +pub(super) use publication::legacy_sink_frame_allowed; +pub(super) use publication::{project_cached_selection, project_live_selection, refresh_queued_update, selection_owns_output}; + +#[path = "room_learner_api_selection.rs"] +mod selection_routes; +pub(super) use selection_routes::{owner_selection_routes, selection}; + +#[path = "room_learner_api_progress.rs"] +mod paired_progress; +pub(super) use paired_progress::{progress, progress_dispatch}; + +#[path = "room_learner_api_temporal.rs"] +mod temporal_routes; +pub(super) use temporal_routes::{status_dispatch, selection_dispatch, configure_dispatch, begin_dispatch, finish_dispatch, train_dispatch, evaluate_dispatch, restore_dispatch, retry_dispatch, discard_dispatch, pause_dispatch, cancel_operation_dispatch}; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_offline_metadata_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_offline_metadata_tests.rs new file mode 100644 index 0000000000..2126e54714 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_offline_metadata_tests.rs @@ -0,0 +1,205 @@ +//! Offline RF is not loss of authenticated owner metadata authority. +//! Synthetic temporary data only; no device or broker operations. +use super::*; + +async fn cancel_via_authenticated_router( + state: SharedState, + path: &str, + guard: MutationGuard, + token: &str, +) -> (StatusCode, serde_json::Value) { + let router = Router::new() + .route("/api/v1/room-learner/phase/cancel", post(cancel_phase)) + .route("/api/v1/room-learner/train/cancel", post(cancel_training)) + .with_state(state) + .layer(axum::middleware::from_fn_with_state( + wifi_densepose_sensing_server::bearer_auth::AuthState::from_token("synthetic-owner"), + wifi_densepose_sensing_server::bearer_auth::require_bearer, + )); + let response = router + .oneshot( + Request::builder() + .method(Method::POST) + .uri(path) + .header("content-type", "application/json") + .header("authorization", format!("Bearer {token}")) + .body(Body::from( + serde_json::to_vec(&serde_json::json!({"expected":guard})).unwrap(), + )) + .unwrap(), + ) + .await + .unwrap(); + let code = response.status(); + let bytes = axum::body::to_bytes(response.into_body(), 65536) + .await + .unwrap(); + ( + code, + serde_json::from_slice(&bytes).unwrap_or(serde_json::Value::Null), + ) +} + +#[tokio::test] +async fn offline_owner_can_cancel_then_deselect_but_cannot_reset_corrupt_pointer() { + let (dir, state, rt, _binding) = fitted().await; + let started = begin_phase( + State(state.clone()), + Json(BeginRequest { + expected: rt.mutation_guard().unwrap(), + phase: PhaseRequest { + trial_id: "synthetic-offline-cancel".into(), + session_id: "synthetic-offline-session".into(), + role: DataRole::HeldOut, + label: RoomLabel::Empty, + duration_ms: 12000, + lead_in_ms: 0, + conditions_confirmed: true, + }, + }), + ) + .await + .0; + assert_eq!(started["success"], true, "{started}"); + assert!(rt.capture_active().unwrap()); + // Readiness requires a native arrival within two seconds. Let the actual + // observed metadata become stale; do not replace its clock or forge flags. + tokio::time::sleep(std::time::Duration::from_millis(2100)).await; + { + let mut inner = state.write().await; + inner.last_esp32_frame = None; + assert!(rt.context().unwrap().sources.iter().all(|s| !source_ready( + &inner, + s, + std::time::Instant::now() + ))); + } + let grid = rt.context().unwrap().sources[0].grid_id.clone(); + rt.record_native_rejection(16, &grid, "synthetic RF interruption"); + let metadata = status(State(state.clone())).await.0; + assert_eq!(metadata["enabled"], true); + assert!(metadata["mutation_guard"].is_object()); + assert!(metadata["model_selection"]["selected"].is_object()); + assert_eq!( + metadata["model_selection"]["current"]["status"], + "unavailable" + ); + let guard = rt.mutation_guard().unwrap(); + let (code, _) = cancel_via_authenticated_router( + state.clone(), + "/api/v1/room-learner/phase/cancel", + guard.clone(), + "synthetic-peer", + ) + .await; + assert_ne!( + code, + StatusCode::OK, + "loss of RF must not broaden actor rights" + ); + assert!(rt.capture_active().unwrap()); + let (code, cancelled) = cancel_via_authenticated_router( + state.clone(), + "/api/v1/room-learner/phase/cancel", + guard, + "synthetic-owner", + ) + .await; + assert_eq!(code, StatusCode::OK); + assert_eq!(cancelled["success"], true, "{cancelled}"); + assert!(!rt.capture_active().unwrap()); + let known = status(State(state.clone())).await.0; + let deselected = owner_post( + state.clone(), + "/api/v1/room-learner/deselect", + serde_json::json!({"expected":known["mutation_guard"]}), + ) + .await; + assert_eq!(deselected["success"], true, "{deselected}"); + assert!(deselected["selection"]["receipt"]["selected"].is_null()); + assert_eq!(deselected["selection"]["receipt"]["revision"], 2); + + // Unknown disk state is different from absent RF. Do not guess a pointer + // revision or overwrite the file merely to acknowledge a metadata action. + let pointer = dir + .path() + .join("a".repeat(64)) + .join("SensingServer/room-learner/model-selection-v1.json"); + assert!(pointer.is_file()); + let corrupt = b"{ synthetic corrupt selection pointer"; + std::fs::write(&pointer, corrupt).unwrap(); + let refused = owner_post( + state.clone(), + "/api/v1/room-learner/deselect", + serde_json::json!({"expected":rt.mutation_guard().unwrap()}), + ) + .await; + assert_eq!(refused["success"], false, "{refused}"); + assert_eq!(refused["error_code"], "learner_deselection_rejected"); + assert_eq!(std::fs::read(pointer).unwrap(), corrupt); +} + +#[tokio::test] +async fn authenticated_training_cancel_does_not_require_live_native_ingress() { + let (_dir, state) = fixture(); + let configured=json_request(app(state.clone()),"/configure",Method::POST,serde_json::json!({ + "binding_digest":"b".repeat(64),"revision":7001,"source_node_ids":[15,16],"new_training":true})).await; + assert_eq!(configured["success"], true, "{configured}"); + let rt = state.read().await.room_learning.as_ref().unwrap().clone(); + let grid = rt.context().unwrap().sources[0].grid_id.clone(); + let base = rt.receiver_now_ns() - 100_000_000_000; + for (i, label) in [ + RoomLabel::Empty, + RoomLabel::OccupiedStatic, + RoomLabel::OccupiedMoving, + RoomLabel::AdjacentOnly, + ] + .into_iter() + .enumerate() + { + fixture_support::phase(&rt, &grid, DataRole::Train, label, i as u32, base); + } + rt.start_training( + &rt.mutation_guard().unwrap(), + FitConfig { + epochs: 8, + learning_rate: 0.5, + hidden_dim: 2, + embedding_dim: 2, + temporal_tokens: 2, + ..FitConfig::default() + }, + ) + .unwrap(); + state.write().await.room_learning_sources.clear(); + let guard = rt.mutation_guard().unwrap(); + let (code, _) = cancel_via_authenticated_router( + state.clone(), + "/api/v1/room-learner/train/cancel", + guard.clone(), + "synthetic-peer", + ) + .await; + assert_ne!(code, StatusCode::OK); + let (code, body) = cancel_via_authenticated_router( + state.clone(), + "/api/v1/room-learner/train/cancel", + guard, + "synthetic-owner", + ) + .await; + assert_eq!(code, StatusCode::OK); + assert_eq!(body["success"], true, "{body}"); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + loop { + let snapshot = rt.status(rt.receiver_now_ns()); + if snapshot["state"] == "cancelled" { + break; + } + assert!( + std::time::Instant::now() < deadline, + "cancelled training did not settle: {snapshot}" + ); + tokio::time::sleep(std::time::Duration::from_millis(5)).await; + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_progress.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_progress.rs new file mode 100644 index 0000000000..d4255edb40 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_progress.rs @@ -0,0 +1,89 @@ +//! Pair-read calibration progress only; never expose the owner's status object. +use super::*; +use axum::response::{IntoResponse, Response}; +use wifi_densepose_sensing_server::room_learner::lifecycle::PublishedProgressRoom; +const SCHEMA:&str="ruview.room-learner.progress.v1"; +fn denied(status:StatusCode,code:&'static str)->Response { + (status,Json(serde_json::json!({"schema":SCHEMA,"success":false,"error_code":code}))).into_response() +} +pub(crate) async fn progress(State(state):State, grant:Option>)->Response { + let state=state.read().await; + if state.privacy_mode {return denied(StatusCode::FORBIDDEN,"privacy_redacted");} + let Some(runtime)=state.room_learning.as_ref() else {return denied(StatusCode::SERVICE_UNAVAILABLE,"learner_unavailable");}; + let Some(store)=state.current_room_store.as_ref() else {return denied(StatusCode::SERVICE_UNAVAILABLE,"publication_unavailable");}; + let Some(publication)=store.snapshot() else {return denied(StatusCode::SERVICE_UNAVAILABLE,"publication_unavailable");}; + if state.installation_id.as_deref()!=Some(publication.installation_id.as_str()) {return denied(StatusCode::SERVICE_UNAVAILABLE,"publication_unavailable");} + let identity=publication.binding_digest.as_ref().map(|binding|PublishedProgressRoom{binding_digest:binding.clone(),revision:publication.revision}); + // The binding hashes installation, published revision and complete room. + // Hold its lock through projection; reject a publication changed since the + // initial envelope snapshot instead of mixing its identity with old counts. + match store.with_current_binding(|binding| { + if binding!=publication.binding_digest.as_deref() {return Err("publication_changed");} + let context=runtime.context(); + let ready:Vec<_>=context.as_ref().map(|c|c.sources.iter().filter(|s|source_ready(&state,s,std::time::Instant::now())).map(|s|s.source_id).collect()).unwrap_or_default(); + let ready_context=context.as_ref().map(|c|c.digest()); + runtime.paired_progress_snapshot(&publication.installation_id,&state.calibration_boot_epoch,identity,&ready,ready_context.as_deref(),runtime.receiver_now_ns()) + .map_err(|_|"progress_unavailable") + }) { + Ok(Ok(mut snapshot))=>{ + if let Some(axum::Extension(grant))=grant { + let Some(until)=grant.expires_at.checked_mul(1000) else {return StatusCode::UNAUTHORIZED.into_response();}; + let now=chrono::Utc::now().timestamp_millis().max(0) as u64; + if *grant.revoked.borrow() || until<=snapshot.observed_at_unix_ms || until<=now {return StatusCode::UNAUTHORIZED.into_response();} + snapshot.expires_at_unix_ms=snapshot.expires_at_unix_ms.min(until); + } + Json(snapshot).into_response() + }, + Ok(Err("publication_changed"))=>denied(StatusCode::CONFLICT,"publication_changed"), + _=>denied(StatusCode::SERVICE_UNAVAILABLE,"progress_unavailable"), + } +} + + +#[derive(Default, serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct ProgressQuery {schema:Option} +fn temporal_denied(status:StatusCode,code:&'static str)->Response { + (status,Json(serde_json::json!({"schema":"ruview.room-learner.progress.v2","success":false,"error_code":code}))).into_response() +} +pub(crate) async fn progress_dispatch(State(state):State, Query(query):Query, grant:Option>)->Response { + if let Some(schema)=query.schema.as_deref() { + if schema!="ruview.room-learner.progress.v2" { return temporal_denied(StatusCode::BAD_REQUEST,"learner_schema_unsupported"); } + } else { + let actual_temporal=state.read().await.room_learning.as_ref().is_some_and(|rt|rt.temporal_v2_state_present()); + if !actual_temporal {return progress(State(state),grant).await;} + } + temporal_progress(State(state),grant).await +} +async fn temporal_progress(State(state):State, grant:Option>)->Response { + let state=state.read().await; + if state.privacy_mode {return temporal_denied(StatusCode::FORBIDDEN,"privacy_redacted");} + let Some(runtime)=state.room_learning.as_ref() else {return temporal_denied(StatusCode::SERVICE_UNAVAILABLE,"learner_unavailable");}; + let Some(store)=state.current_room_store.as_ref() else {return temporal_denied(StatusCode::SERVICE_UNAVAILABLE,"publication_unavailable");}; + let Some(publication)=store.snapshot() else {return temporal_denied(StatusCode::SERVICE_UNAVAILABLE,"publication_unavailable");}; + if state.installation_id.as_deref()!=Some(publication.installation_id.as_str()) {return temporal_denied(StatusCode::SERVICE_UNAVAILABLE,"publication_unavailable");} + let identity=publication.binding_digest.as_ref().map(|binding|PublishedProgressRoom{binding_digest:binding.clone(),revision:publication.revision}); + // The binding hashes installation, published revision and complete room. + // Hold its lock through projection; reject a publication changed since the + // initial envelope snapshot instead of mixing its identity with old counts. + match store.with_current_binding(|binding| { + if binding!=publication.binding_digest.as_deref() {return Err("publication_changed");} + let context=runtime.temporal_progress_context(); + let ready:Vec<_>=context.as_ref().map(|c|c.sources.iter().filter(|s|source_ready(&state,s,std::time::Instant::now())).map(|s|s.source_id).collect()).unwrap_or_default(); + let ready_context=context.as_ref().map(|c|c.digest()); + runtime.paired_temporal_progress_snapshot(&publication.installation_id,&state.calibration_boot_epoch,identity,&ready,ready_context.as_deref(),runtime.receiver_now_ns()) + .map_err(|_|"progress_unavailable") + }) { + Ok(Ok(mut snapshot))=>{ + if let Some(axum::Extension(grant))=grant { + let Some(until)=grant.expires_at.checked_mul(1000) else {return StatusCode::UNAUTHORIZED.into_response();}; + let now=chrono::Utc::now().timestamp_millis().max(0) as u64; + if *grant.revoked.borrow() || until<=snapshot.observed_at_unix_ms || until<=now {return StatusCode::UNAUTHORIZED.into_response();} + snapshot.expires_at_unix_ms=snapshot.expires_at_unix_ms.min(until); + } + Json(snapshot).into_response() + }, + Ok(Err("publication_changed"))=>temporal_denied(StatusCode::CONFLICT,"publication_changed"), + _=>temporal_denied(StatusCode::SERVICE_UNAVAILABLE,"progress_unavailable"), + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_publication.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_publication.rs new file mode 100644 index 0000000000..ddc36f045b --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_publication.rs @@ -0,0 +1,318 @@ +//! One final binary-only projection for live publication and cached readback. +//! Legacy classification remains a labelled comparison, never a fallback for +//! an unavailable selected model. No count, pose, motion or Vitals authority is +//! inferred from the binary model. +use super::*; +use wifi_densepose_sensing_server::room_learner::{ + model::PresenceDecision, selection::SelectedBinaryEvidence, +}; + +const SCHEMA: &str = "ruview.room-learner.selected-binary-evidence.v1"; + +fn unavailable(evidence: SelectedBinaryEvidence, reason: &str) -> SelectedBinaryEvidence { + let identity = match evidence { + SelectedBinaryEvidence::None => return SelectedBinaryEvidence::None, + SelectedBinaryEvidence::Unavailable { identity, .. } => identity, + SelectedBinaryEvidence::Ready { identity, .. } => Some(identity), + }; + SelectedBinaryEvidence::Unavailable { + identity, + reason: reason.into(), + } +} + +fn temporal_current(state: &AppStateInner) -> Option { + let rt = state.room_learning.as_ref()?; + if !rt.temporal_v2_state_present() { + return None; + } + let unavailable = |reason: &str| serde_json::json!({"schema":"ruview.room-learner.selected-binary-evidence.v2","status":"unavailable","identity":null,"reason":reason,"boot_epoch":null,"receiver_epoch":null}); + if state.privacy_mode { + return Some(unavailable("selected_learning_unavailable_in_privacy_mode")); + } + let now = now_ns(rt); + let value = match state.current_room_store.as_ref() { + Some(room) => room + .with_current_binding(|binding| { + rt.temporal_selected_binary_v2(now, binding, Some(&state.calibration_boot_epoch)) + }) + .unwrap_or_else(|_| { + rt.temporal_selected_binary_v2(now, None, Some(&state.calibration_boot_epoch)) + }), + None => rt.temporal_selected_binary_v2(now, None, Some(&state.calibration_boot_epoch)), + } + .unwrap_or_else(|_| unavailable("selected_projection_unavailable")); + if value["status"] == "none" { + None + } else { + Some(value) + } +} +fn temporal_unavailable(mut value: serde_json::Value, reason: &str) -> serde_json::Value { + let identity = value + .get("identity") + .cloned() + .unwrap_or(serde_json::Value::Null); + let boot = value + .get("boot_epoch") + .cloned() + .unwrap_or(serde_json::Value::Null); + let receiver = value + .get("receiver_epoch") + .cloned() + .unwrap_or(serde_json::Value::Null); + value = serde_json::json!({"schema":"ruview.room-learner.selected-binary-evidence.v2","status":"unavailable","identity":identity,"reason":reason,"boot_epoch":boot,"receiver_epoch":receiver}); + value +} +fn temporal_dispatch(state: &AppStateInner) -> bool { + state + .room_learning + .as_ref() + .is_some_and(|rt| rt.temporal_v2_state_present()) +} + +pub(crate) fn selection_owns_output(state: &AppStateInner) -> bool { + if temporal_dispatch(state) { + return temporal_current(state).is_some(); + } + state.room_learning.as_ref().is_some_and(|runtime| { + !matches!( + runtime.selected_binary_evidence(now_ns(runtime), None), + SelectedBinaryEvidence::None + ) + }) +} + +fn current(state: &AppStateInner) -> SelectedBinaryEvidence { + let Some(rt) = state.room_learning.as_ref() else { + return SelectedBinaryEvidence::None; + }; + let now = now_ns(rt); + let evidence = match state.current_room_store.as_ref() { + Some(room) => room + .with_current_binding(|binding| rt.selected_binary_evidence(now, binding)) + .unwrap_or_else(|_| rt.selected_binary_evidence(now, None)), + None => rt.selected_binary_evidence(now, None), + }; + if state.privacy_mode { + return match evidence { + SelectedBinaryEvidence::None => SelectedBinaryEvidence::None, + _ => SelectedBinaryEvidence::Unavailable { + identity: None, + reason: "selected_learning_unavailable_in_privacy_mode".into(), + }, + }; + } + match &evidence { + SelectedBinaryEvidence::Ready { + decision, + presence, + occupied_probability, + .. + } if !occupied_probability.is_finite() + || !(0.0..=1.0).contains(occupied_probability) + || !matches!( + (decision, presence), + (PresenceDecision::Empty, false) | (PresenceDecision::Occupied, true) + ) => + { + unavailable(evidence, "invalid_selected_binary_decision") + } + _ => evidence, + } +} + +fn envelope(state: &AppStateInner, evidence: SelectedBinaryEvidence) -> Option { + if matches!(evidence, SelectedBinaryEvidence::None) { + return None; + } + let mut value = + serde_json::to_value(evidence).expect("bounded selected binary proof serializes"); + let object = value + .as_object_mut() + .expect("tagged selected binary proof is an object"); + object.insert("schema".into(), serde_json::json!(SCHEMA)); + object.insert( + "boot_epoch".into(), + if state.privacy_mode { + serde_json::Value::Null + } else { + serde_json::json!(state.calibration_boot_epoch) + }, + ); + Some(value) +} + +fn apply(update: &mut SensingUpdate, proof: Option) { + update.selected_binary_evidence = proof; + if update.selected_binary_evidence.is_none() { + update.classification_source = None; + return; + } + update.classification_source = Some("legacy_control_comparison".into()); + update.calibrated_presence_evidence = None; + update.detector_diagnostics = None; + update.estimated_persons = None; + update.persons = None; + update.pose_keypoints = None; + update.vital_signs = None; + update.vital_evidence = None; + update.enhanced_breathing = None; + update.enhanced_motion = None; + update.posture = None; + // Node and room classification diagnostics retain their own legacy source; + // they are not selected binary claims. None carries a qualified count. +} + +pub(crate) fn project_live_selection(state: &AppStateInner, update: &mut SensingUpdate) { + if temporal_dispatch(state) { + let proof = temporal_current(state).map(|value| { + if update.source == "esp32" { + value + } else { + temporal_unavailable(value, "selected_native_source_update_required") + } + }); + apply(update, proof); + return; + } + let mut evidence = current(state); + if update.source != "esp32" { + evidence = unavailable(evidence, "selected_native_source_update_required"); + } + apply(update, envelope(state, evidence)); +} + +pub(crate) fn project_cached_selection(state: &AppStateInner, update: &mut SensingUpdate) { + if temporal_dispatch(state) { + let proof = temporal_current(state).map(|value| { + if value["status"] != "ready" { + return value; + } + let now = state + .room_learning + .as_ref() + .map(|rt| now_ns(rt) / 1_000_000); + let cached = update.selected_binary_evidence.as_ref(); + let valid = update.source == "esp32" + && cached == Some(&value) + && value["observed_at_unix_ms"] + .as_u64() + .zip(now) + .and_then(|(observed, now)| now.checked_sub(observed)) + .is_some_and(|age| { + age <= wifi_densepose_sensing_server::room_learner::observation::MAX_AGE_NS + / 1_000_000 + }) + && value["valid_until_unix_ms"] + .as_u64() + .zip(now) + .is_some_and(|(until, now)| now < until); + if valid { + cached.cloned().expect("matched admitted proof") + } else { + temporal_unavailable(value, "selected_cached_observation_unavailable") + } + }); + apply(update, proof); + return; + } + let mut evidence = current(state); + if let SelectedBinaryEvidence::Ready { identity, .. } = &evidence { + let cached = update.selected_binary_evidence.as_ref(); + let now_ms = state + .room_learning + .as_ref() + .map(|rt| now_ns(rt) / 1_000_000); + let identity = + serde_json::to_value(identity).expect("bounded selected identity serializes"); + let matches = cached.is_some_and(|proof| { + let fresh = proof["observed_at_unix_ms"] + .as_u64() + .zip(now_ms) + .and_then(|(observed, now)| now.checked_sub(observed)) + .is_some_and(|age| { + age <= wifi_densepose_sensing_server::room_learner::observation::MAX_AGE_NS + / 1_000_000 + }); + proof["schema"] == SCHEMA + && proof["boot_epoch"] == state.calibration_boot_epoch + && proof["status"] == "ready" + && proof["identity"] == identity + && proof["observation_digest"].as_str().is_some_and(|digest| { + digest.len() == 64 && digest.bytes().all(|b| b.is_ascii_hexdigit()) + }) + && fresh + }); + if update.source != "esp32" || !matches { + evidence = unavailable(evidence, "selected_cached_observation_unavailable"); + } else { + // A cache read cannot manufacture a newly observed measurement or + // advance its timestamp. Keep the original exact proof. + let cached = cached.cloned(); + apply(update, cached); + return; + } + } + apply(update, envelope(state, evidence)); +} + +/// A queued frame may predate selection, rollback, a room change or source +/// withdrawal. Revalidate its authority at the subscriber boundary without +/// converting it into a new measurement. Non-sensing control messages remain +/// byte-identical; legacy edge-vitals messages are withheld while selected. +pub(crate) fn refresh_queued_update(state: &AppStateInner, json: &str) -> Option { + if !selection_owns_output(state) && !json.contains("\"selected_binary_evidence\"") { + return Some(json.to_owned()); + } + let value: serde_json::Value = serde_json::from_str(json).ok()?; + match value.get("type").and_then(serde_json::Value::as_str) { + Some("sensing_update") => { + let mut update: SensingUpdate = serde_json::from_value(value).ok()?; + project_cached_selection(state, &mut update); + serde_json::to_string(&update).ok() + } + Some("edge_vitals" | "edge_fused_vitals") if selection_owns_output(state) => None, + _ => Some(json.to_owned()), + } +} + +/// Legacy MQTT/automation bridges cannot reinterpret selected-model frames, +/// even after deselection or when an older unmarked frame was queued before +/// selection. This is an admission guard, not new MQTT presence authority. +#[cfg(any(feature = "mqtt", test))] +pub(crate) fn legacy_sink_frame_allowed(selection_owned: bool, frame: &serde_json::Value) -> bool { + !selection_owned + && frame.get("selected_binary_evidence").is_none() + && frame.get("classification_source").is_none() +} + +#[cfg(test)] +mod legacy_sink_tests { + use super::*; + #[test] + fn selected_current_state_and_queued_markers_both_withhold_legacy_bridges() { + let legacy = + serde_json::json!({"type":"sensing_update","classification":{"presence":true}}); + assert!(legacy_sink_frame_allowed(false, &legacy)); + assert!( + !legacy_sink_frame_allowed(true, &legacy), + "pre-selection queued legacy data must be withheld after selection" + ); + for status in ["ready", "unavailable", "none"] { + let mut selected = legacy.clone(); + selected["selected_binary_evidence"] = serde_json::json!({"status":status}); + assert!( + !legacy_sink_frame_allowed(false, &selected), + "old selected frame must not regain legacy authority after deselection" + ); + assert!(!legacy_sink_frame_allowed(true, &selected)); + } + let mut malformed = legacy.clone(); + malformed["selected_binary_evidence"] = serde_json::Value::Null; + assert!(!legacy_sink_frame_allowed(false, &malformed)); + let mut comparison = legacy; + comparison["classification_source"] = serde_json::json!("legacy_control_comparison"); + assert!(!legacy_sink_frame_allowed(false, &comparison)); + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_recovery_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_recovery_tests.rs new file mode 100644 index 0000000000..3688a5a8a3 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_recovery_tests.rs @@ -0,0 +1,160 @@ +//! SYNTHETIC actual HTTP handlers; no listeners, user data or hardware. +use super::*; +use wifi_densepose_sensing_server::room_learner::recovery_store::RecoveryStore; +async fn recovery_fixture() -> (tempfile::TempDir, SharedState) { + let (dir, state) = fixture(); + replace_runtime(dir.path(), &state).await; + (dir, state) +} +async fn replace_runtime(root: &std::path::Path, state: &SharedState) { + let owner = "a".repeat(64); + let path = root.join(&owner).join("SensingServer"); + let store = RecoveryStore::new(&path, &owner, "synthetic-install", Some(&[7; 32])).unwrap(); + state.write().await.room_learning = Some(Arc::new( + RoomLearningRuntime::new_with_recovery(&path, &owner, "synthetic-install", store).unwrap(), + )); +} +fn recovery_app(state: SharedState) -> Router { + Router::new() + .route("/configure", post(configure)) + .route("/status", get(status)) + .route("/phase", post(begin_phase)) + .route("/finish", post(finish_phase)) + .route("/recovery/restore", post(restore_capture)) + .route("/recovery/discard", post(discard_capture)) + .with_state(state) +} +fn recovery_config() -> serde_json::Value { + let mut c = config(); + c["recovery_consent"] = true.into(); + c +} +#[tokio::test] +async fn http_recovery_is_explicit_guarded_exact_and_metadata_only() { + let (dir, state) = recovery_fixture().await; + let router = recovery_app(state.clone()); + let configured = json_request( + router.clone(), + "/configure", + Method::POST, + recovery_config(), + ) + .await; + assert_eq!(configured["success"], true, "{configured}"); + assert_eq!(configured["status"]["derived_samples_persisted"], false); + let id = configured["status"]["recovery"]["discard_id"].clone(); + let oldguard = configured["status"]["mutation_guard"].clone(); + assert_eq!( + json_request( + router.clone(), + "/configure", + Method::POST, + recovery_config() + ) + .await["error_code"], + "learner_recovery_pending" + ); + replace_runtime(dir.path(), &state).await; + let cold = json_request_status(router.clone()).await; + assert!(cold["context_digest"].is_null()); + assert_eq!(cold["recovery"]["state"], "restore_required"); + let mut restore = serde_json::json!({"recovery_id":id,"binding_digest":"b".repeat(64),"revision":1,"source_node_ids":[15,16,17]}); + restore["expected"] = oldguard; + assert_eq!( + json_request( + router.clone(), + "/recovery/restore", + Method::POST, + restore.clone() + ) + .await["error_code"], + "learner_state_changed" + ); + restore["expected"] = cold["mutation_guard"].clone(); + restore["revision"] = 2.into(); + assert_eq!( + json_request( + router.clone(), + "/recovery/restore", + Method::POST, + restore.clone() + ) + .await["error_code"], + "learner_recovery_context_mismatch" + ); + restore["revision"] = 1.into(); + let recovered = json_request(router.clone(), "/recovery/restore", Method::POST, restore).await; + assert_eq!(recovered["success"], true, "{recovered}"); + assert_eq!(recovered["status"]["recovery"]["automatic_resume"], false); + assert!(recovered["status"]["capture"]["sources"] + .as_array() + .unwrap() + .iter() + .all(|s| s["live_buffer_frames"] == 0)); + let encoded = serde_json::to_string(&recovered).unwrap(); + for forbidden in [ + "amplitudes", + "plaintext", + "source_ordinals", + "cohort_id", + "signing_seed", + ] { + assert!(!encoded.contains(forbidden), "{forbidden}"); + } + let guard = recovered["status"]["mutation_guard"].clone(); + assert_eq!( + json_request( + router.clone(), + "/recovery/discard", + Method::POST, + serde_json::json!({"expected":guard,"recovery_id":"wrong"}) + ) + .await["success"], + false + ); + let discarded = json_request( + router.clone(), + "/recovery/discard", + Method::POST, + serde_json::json!({"recovery_id":id}), + ) + .await; + assert_eq!(discarded["success"], true, "{discarded}"); + assert!(discarded["status"]["recovery"]["saved"].is_null()); +} +#[tokio::test] +async fn http_recovery_rejects_unready_native_context_and_privacy_without_deleting_slot() { + let (_dir, state) = recovery_fixture().await; + let router = recovery_app(state.clone()); + let configured = json_request( + router.clone(), + "/configure", + Method::POST, + recovery_config(), + ) + .await; + let id = configured["status"]["recovery"]["discard_id"].clone(); + state.write().await.room_learning_sources.clear(); + let restore = serde_json::json!({"recovery_id":id,"binding_digest":"b".repeat(64),"revision":1,"source_node_ids":[15,16,17]}); + assert_eq!( + json_request(router.clone(), "/recovery/restore", Method::POST, restore).await + ["error_code"], + "learner_recovery_source_unready" + ); + state.write().await.privacy_mode = true; + assert_eq!( + json_request( + router.clone(), + "/recovery/discard", + Method::POST, + serde_json::json!({"recovery_id":id}) + ) + .await["error_code"], + "learner_privacy_mode" + ); + state.write().await.privacy_mode = false; + assert_eq!( + json_request_status(router).await["recovery"]["discard_id"], + id + ); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_selection.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_selection.rs new file mode 100644 index 0000000000..f42e6731a0 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_selection.rs @@ -0,0 +1,156 @@ +//! Owner-only learner controls, model selection and bounded read projection. +use super::*; + +/// Selection mutations are restricted to the local owner, even if ordinary +/// API bearer authentication is disabled or a peer has calibration access. +pub(crate) fn owner_selection_routes( + auth: wifi_densepose_sensing_server::bearer_auth::AuthState, +) -> Router { + Router::new() + .route("/api/v1/room-learner/recovery/enable-current", post(enable_current_recovery)) + .route("/api/v1/room-learner/recovery/retry", post(super::temporal_routes::retry_dispatch).layer(axum::middleware::from_fn(retry_requires_unpaired_owner))) + .route("/api/v1/room-learner/qualify", post(super::temporal_routes::qualify_dispatch)) + .route("/api/v1/room-learner/select", post(super::temporal_routes::select_dispatch)) + .route("/api/v1/room-learner/deselect", post(super::temporal_routes::deselect_dispatch)) + .route_layer(axum::middleware::from_fn_with_state( + auth, + companion_integration::require_owner, + )) +} + +#[derive(serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct SelectRequest { + expected: MutationGuard, + model_sha256: String, + qualification_sha256: String, +} + +pub(super) fn selection_snapshot( + state: &AppStateInner, + rt: &RoomLearningRuntime, + now: u64, +) -> wifi_densepose_sensing_server::room_learner::selection::ModelSelectionStatus { + let snapshot = match state.current_room_store.as_ref() { + Some(room) => room + .with_current_binding(|binding| rt.selection_snapshot(now, binding)) + .unwrap_or_else(|_| rt.selection_snapshot(now, None)), + None => rt.selection_snapshot(now, None), + }; + snapshot +} + +pub(super) fn enriched_status( + state: &AppStateInner, + rt: &RoomLearningRuntime, +) -> serde_json::Value { + let now = now_ns(rt); + let mut status = rt.status(now); + status["model_selection"] = serde_json::to_value(selection_snapshot(state, rt, now)) + .expect("bounded model selection status serializes"); + status +} + +pub(super) fn with_current_room( + state: &AppStateInner, + rt: &RoomLearningRuntime, + operation: impl FnOnce() -> Result, +) -> Result { + let room = state + .current_room_store + .as_ref() + .ok_or("current published room unavailable")?; + room.with_current_binding(|binding| { + let context = rt.context().ok_or("learner context unavailable")?; + if binding != Some(context.room_digest.as_str()) + || !context + .sources + .iter() + .all(|source| source_ready(state, source, std::time::Instant::now())) + { + return Err("current room or trained source binding mismatch".into()); + } + operation() + }) + .map_err(|_| "current published room unavailable".to_string())? +} + +pub(crate) async fn qualify( + State(state): State, + Json(request): Json, +) -> Json { + let rt = match runtime(&state).await { + Ok(rt) => rt, + Err(e) => return e, + }; + // Serialize authority transitions with live producers and legacy sink admission. + let state = state.write().await; + match with_current_room(&state, &rt, || rt.qualify(&request.expected)) { + Ok(receipt) => Json( + serde_json::json!({"success":true,"qualification":receipt,"status":enriched_status(&state, &rt)}), + ), + Err(e) => failure("learner_qualification_rejected", e), + } +} + +pub(crate) async fn select_model( + State(state): State, + Json(request): Json, +) -> Json { + let rt = match runtime(&state).await { + Ok(rt) => rt, + Err(e) => return e, + }; + // Serialize authority transitions with live producers and legacy sink admission. + let state = state.write().await; + match with_current_room(&state, &rt, || { + rt.select_model( + &request.expected, + &request.model_sha256, + &request.qualification_sha256, + ) + }) { + Ok(receipt) => Json( + serde_json::json!({"success":true,"selection":receipt,"status":enriched_status(&state, &rt)}), + ), + Err(e) => failure("learner_selection_rejected", e), + } +} + +pub(crate) async fn deselect_model( + State(state): State, + Json(request): Json, +) -> Json { + let rt = match runtime(&state).await { + Ok(rt) => rt, + Err(e) => return e, + }; + // Serialize authority transitions with live producers and legacy sink admission. + let state = state.write().await; + match rt.deselect_model(&request.expected) { + Ok(receipt) => Json( + serde_json::json!({"success":true,"selection":receipt,"status":enriched_status(&state, &rt)}), + ), + Err(e) => failure("learner_deselection_rejected", e), + } +} + +/// Read-scoped phone projection: no owner scope, mutation guard, raw context, +/// training examples or model weights are returned. +pub(crate) async fn selection(State(state): State) -> Json { + let rt = match runtime(&state).await { + Ok(rt) => rt, + Err(e) => return e, + }; + let state = state.read().await; + Json( + serde_json::json!({"schema":"ruview.room-learner.selection.v1","success":true, + "progress_schema":"ruview.room-learner.progress.v1", + "model_selection":selection_snapshot(&state, &rt, now_ns(&rt))}), + ) +} + +async fn retry_requires_unpaired_owner(request:axum::extract::Request,next:axum::middleware::Next)->axum::response::Response{ + if request.extensions().get::().is_some(){return axum::response::IntoResponse::into_response(StatusCode::FORBIDDEN);} + next.run(request).await +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_selection_composer_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_selection_composer_tests.rs new file mode 100644 index 0000000000..5ad6f1b575 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_selection_composer_tests.rs @@ -0,0 +1,385 @@ +//! SYNTHETIC public-runtime training and actual publication/API composition. +//! These tests grant no hardware accuracy or real-room qualification evidence. +use super::*; +use wifi_densepose_sensing_server::room_learner::{ + capture::{RawSourceFrame, MIN_PHASE_MS, WINDOW_PACKETS}, + observation::RoomLabel, +}; + +#[path = "room_learner_api_selection_fixture.rs"] +mod fixture_support; +use fixture_support::{feed, fitted, owner_post, update}; + +#[tokio::test] +async fn actual_selected_composer_withdraws_without_fallback_and_never_grants_auxiliary_authority() +{ + let (_dir, state, rt, binding) = fitted().await; + let mut out = update(); + { + let inner = state.read().await; + project_live_selection(&inner, &mut out); + } + let proof = out.selected_binary_evidence.as_ref().unwrap(); + assert_eq!(proof["status"], "ready", "{proof}"); + assert_eq!(proof["decision"], "occupied"); + assert_eq!(proof["presence"], true); + assert_eq!( + out.classification_source.as_deref(), + Some("legacy_control_comparison") + ); + assert!( + out.estimated_persons.is_none() && out.persons.is_none() && out.pose_keypoints.is_none() + ); + assert!( + out.vital_signs.is_none() + && out.vital_evidence.is_none() + && out.enhanced_breathing.is_none() + && out.posture.is_none() + ); + assert!(derive_pose_from_sensing(&out).is_empty()); + // Exercise the actual RuField emitter with a valid governed cycle: the + // legacy positive control emits, but the selected binary update cannot be + // relabelled as node motion/location/count evidence. + { + let mut inner = state.write().await; + let mut engine_nodes = HashMap::new(); + for id in [15u8, 16] { + let mut node = NodeState::new(); + node.frame_history = std::collections::VecDeque::from(vec![(0..56) + .map(|bin| 1.0 + f64::from(id) * 0.01 + bin as f64 * 0.1) + .collect()]); + node.last_frame_time = Some(std::time::Instant::now()); + engine_nodes.insert(id, node); + } + assert!(inner + .engine_bridge + .observe_cycle(&engine_nodes, 1000) + .is_some()); + emit_rufield_event(&inner, &update(), 15); + let emitted = inner.field_surface.read().await.recent().len(); + assert!( + emitted > 0, + "valid legacy positive control must reach RuField" + ); + emit_rufield_event(&inner, &out, 15); + assert_eq!(inner.field_surface.read().await.recent().len(), emitted); + } + let observed = proof["observed_at_unix_ms"].clone(); + { + let mut inner = state.write().await; + inner.latest_update = Some(out.clone()); + } + let cached = current_sensing_snapshot(&*state.read().await).unwrap(); + assert_eq!( + cached.selected_binary_evidence.as_ref().unwrap()["observed_at_unix_ms"], + observed + ); + // A fresh Empty model result overrides no legacy fields by implication: + // the authoritative binary proof is false while the explicitly labelled + // comparison still contains its previous positive heuristic. + let grid = rt.context().unwrap().sources[0].grid_id.clone(); + feed(&rt, &grid, RoomLabel::Empty, 2000, rt.receiver_now_ns()); + tokio::time::sleep(std::time::Duration::from_millis(12)).await; + let mut empty = update(); + project_live_selection(&*state.read().await, &mut empty); + let evidence = empty.selected_binary_evidence.as_ref().unwrap(); + assert_eq!(evidence["status"], "ready", "{evidence}"); + assert_eq!(evidence["decision"], "empty"); + assert_eq!(evidence["presence"], false); + assert!( + empty.classification.presence, + "legacy comparator stays source-labelled" + ); + let queued_original = serde_json::to_string(&out).unwrap(); + let retained = refresh_queued_update(&*state.read().await, &queued_original).unwrap(); + let retained: serde_json::Value = serde_json::from_str(&retained).unwrap(); + assert_eq!( + retained["selected_binary_evidence"]["observed_at_unix_ms"], + observed + ); + assert_eq!( + retained["selected_binary_evidence"]["decision"], "occupied", + "rolling cohort advancement must not relabel an earlier still-fresh measured decision" + ); + let mut expired: serde_json::Value = serde_json::from_str(&queued_original).unwrap(); + expired["selected_binary_evidence"]["observed_at_unix_ms"] = + serde_json::json!(rt.receiver_now_ns() / 1_000_000 - 6000); + let expired = refresh_queued_update(&*state.read().await, &expired.to_string()).unwrap(); + assert_eq!( + serde_json::from_str::(&expired).unwrap()["selected_binary_evidence"] + ["status"], + "unavailable" + ); + let boot = state.read().await.calibration_boot_epoch.clone(); + state.write().await.calibration_boot_epoch = "synthetic-other-boot".into(); + let withdrawn_boot = refresh_queued_update(&*state.read().await, &queued_original).unwrap(); + assert_eq!( + serde_json::from_str::(&withdrawn_boot).unwrap() + ["selected_binary_evidence"]["status"], + "unavailable" + ); + state.write().await.calibration_boot_epoch = boot; + state.write().await.latest_update = Some(empty); + + // Legacy preferred-grid bookkeeping is not the learner input authority. + // Its stale/missing node cache must not erase a valid native learner cohort. + let removed = state.write().await.node_states.remove(&16).unwrap(); + let last_seen = state + .write() + .await + .node_states + .get_mut(&15) + .unwrap() + .last_frame_time + .take(); + let cached = current_sensing_snapshot(&*state.read().await).unwrap(); + assert_eq!(cached.selected_binary_evidence.unwrap()["status"], "ready"); + state.write().await.node_states.insert(16, removed); + state + .write() + .await + .node_states + .get_mut(&15) + .unwrap() + .last_frame_time = last_seen; + let wrong_room = + companion_room::SharedRoomStore::load(&_dir.path().join("other-room"), "synthetic-install") + .unwrap(); + let original = state + .write() + .await + .current_room_store + .replace(wrong_room) + .unwrap(); + let mut withdrawn = update(); + project_live_selection(&*state.read().await, &mut withdrawn); + assert_eq!( + withdrawn.selected_binary_evidence.as_ref().unwrap()["status"], + "unavailable" + ); + state.write().await.current_room_store = Some(original); + assert_eq!( + rt.selection_snapshot(rt.receiver_now_ns(), Some(&binding)) + .selected + .unwrap() + .room_digest, + binding + ); + // Every read-only composer withholds auxiliary authority while selected. + let vitals = vital_signs_endpoint(State(state.clone())).await.0; + assert!(vitals["vital_signs"]["heart_rate_bpm"].is_null()); + assert_eq!(vitals["authority"], "abstained"); + let pose = pose_current( + State(state.clone()), + Query(PoseCurrentQuery { + view: pose_physics::PoseView::Raw, + }), + ) + .await + .into_response(); + assert_eq!(pose.status(), StatusCode::CONFLICT); + + assert!(edge_vitals_endpoint(State(state.clone())).await.0["edge_vitals"].is_null()); + assert_eq!( + pose_zones_summary(State(state.clone())).await.0["authority"], + "withheld" + ); + let read = selection(State(state.clone())).await.0; + let serialized = serde_json::to_string(&read).unwrap(); + for forbidden in [ + "owner_scope", + "mutation_guard", + "encoder_weights", + "training_samples", + "head_weights", + "geometry", + "last_accepted_sequence", + "last_observed_sequence", + "sequence_rejection_count", + "sequence_rejection_reason", + ] { + assert!( + !serialized.contains(forbidden), + "paired read leaked {forbidden}" + ); + } + let old_json = + serde_json::to_string(state.read().await.latest_update.as_ref().unwrap()).unwrap(); + { + let inner = state.read().await; + assert!( + refresh_queued_update(&inner, r#"{"type":"edge_vitals","heart_rate_bpm":62}"#) + .is_none() + ); + assert!(refresh_queued_update( + &inner, + r#"{"type":"edge_fused_vitals","heartrate_bpm":62}"# + ) + .is_none()); + } + state.write().await.privacy_mode = true; + let privacy = refresh_queued_update(&*state.read().await, &old_json).unwrap(); + let privacy: serde_json::Value = serde_json::from_str(&privacy).unwrap(); + let private_proof = &privacy["selected_binary_evidence"]; + assert_eq!(private_proof["status"], "unavailable"); + assert!(private_proof["identity"].is_null() && private_proof["boot_epoch"].is_null()); + assert!(private_proof.get("presence").is_none() && private_proof.get("decision").is_none()); + assert_eq!(selection(State(state.clone())).await.0["success"], false); + state.write().await.privacy_mode = false; + + // A selected native source failure clears the actual cohort. Refilling only + // one of the required inputs cannot borrow the old last-good other source. + rt.record_native_rejection(16, &grid, "synthetic selected source interruption"); + let at = rt.receiver_now_ns(); + for sample in 0..WINDOW_PACKETS { + let values: Vec<_> = (0..56) + .map(|bin| 2.3 + bin as f64 * 0.04 + sample as f64 * 0.003) + .collect(); + rt.ingest( + RawSourceFrame { + source_id: 15, + grid_id: &grid, + frequency_hz: 2_437_000_000, + bandwidth_hz: 20_000_000, + sequence: 4000 + sample as u32, + amplitudes: &values, + }, + at + sample as u64 * 1_000_000 + 15, + ) + .unwrap(); + } + tokio::time::sleep(std::time::Duration::from_millis(12)).await; + let mut missing = update(); + project_live_selection(&*state.read().await, &mut missing); + let missing = missing.selected_binary_evidence.unwrap(); + assert_eq!(missing["status"], "unavailable"); + assert!(missing.get("presence").is_none()); + assert!( + rt.selection_snapshot(rt.receiver_now_ns(), Some(&binding)) + .selected + .is_some(), + "withdrawal preserves selection history" + ); + + // Explicit owner deselection restores legacy control; it is never automatic. + let response = owner_post( + state.clone(), + "/api/v1/room-learner/deselect", + serde_json::json!({"expected":rt.mutation_guard().unwrap()}), + ) + .await; + assert_eq!(response["success"], true, "{response}"); + let mut legacy = update(); + project_live_selection(&*state.read().await, &mut legacy); + assert!(legacy.selected_binary_evidence.is_none()); + assert_eq!(legacy.estimated_persons, Some(3)); + let queued = refresh_queued_update(&*state.read().await, &old_json).unwrap(); + let queued: serde_json::Value = serde_json::from_str(&queued).unwrap(); + assert!(queued.get("selected_binary_evidence").is_none()); + assert!(queued.get("classification_source").is_none()); +} + +/// Explicit opt-in producer fixture for the actual TypeScript wire consumer. +/// All input, training and labels are SYNTHETIC; no device/user state is read. +#[tokio::test] +async fn export_actual_selected_contract_cases() { + async fn capture_case( + name: &str, + state: SharedState, + rt: &RoomLearningRuntime, + ) -> serde_json::Value { + let mut sensing = update(); + project_live_selection(&*state.read().await, &mut sensing); + let response = Router::new() + .route("/api/v1/room-learner/selection", get(selection)) + .with_state(state.clone()) + .oneshot( + Request::builder() + .uri("/api/v1/room-learner/selection") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let body = axum::body::to_bytes(response.into_body(), 65536) + .await + .unwrap(); + let paired_selection: serde_json::Value = serde_json::from_slice(&body).unwrap(); + let owner_status = status(State(state)).await.0; + serde_json::json!({ + "owner_status": owner_status, + "name": name, + "captured_at_unix_ms": rt.receiver_now_ns() / 1_000_000, + "sensing_update": sensing, + "paired_selection": paired_selection, + }) + } + let (_dir, state, rt, _binding) = fitted().await; + let grid = rt.context().unwrap().sources[0].grid_id.clone(); + let occupied = capture_case("ready_occupied", state.clone(), &rt).await; + assert_eq!( + occupied["sensing_update"]["selected_binary_evidence"]["decision"], + "occupied" + ); + feed(&rt, &grid, RoomLabel::Empty, 2000, rt.receiver_now_ns()); + tokio::time::sleep(std::time::Duration::from_millis(12)).await; + let empty = capture_case("ready_empty", state.clone(), &rt).await; + assert_eq!( + empty["sensing_update"]["selected_binary_evidence"]["decision"], + "empty" + ); + rt.record_native_rejection(16, &grid, "synthetic missing selected source"); + let unavailable = capture_case("unavailable_missing_selected_source", state.clone(), &rt).await; + assert_eq!( + unavailable["sensing_update"]["selected_binary_evidence"]["status"], + "unavailable" + ); + state.write().await.privacy_mode = true; + let privacy = capture_case("privacy_withheld", state.clone(), &rt).await; + assert_eq!(privacy["paired_selection"]["success"], false); + assert!(privacy["sensing_update"]["selected_binary_evidence"]["identity"].is_null()); + let producer_sha = std::env::var("RUVIEW_TEST_PRODUCER_SHA").ok(); + let fixture = serde_json::json!({ + "schema": "ruview.test.selected-contract-cases.v1", + "evidence": "SYNTHETIC_SOFTWARE_COMPOSITION_NOT_PHYSICAL_QUALIFICATION", + "producer_commit": producer_sha, + "producer_path": "real public runtime capture/train/evaluate/qualify/select; actual GET selection handler and final SensingUpdate projector", + "clock_contract": "Use each captured_at_unix_ms as an injected parser clock; stored fixtures are not fresh live observations.", + "authentication_scope": "owner and paired-read policy tested separately; this fixture exercises the real bounded GET response producer", + "cases": [occupied, empty, unavailable, privacy], + }); + if let Ok(output) = std::env::var("RUVIEW_TEST_SELECTED_CONTRACT_OUTPUT") { + use std::io::Write; + #[cfg(unix)] + use std::os::unix::fs::OpenOptionsExt; + let sha = producer_sha + .as_deref() + .expect("explicit fixture export requires producer commit"); + assert!(sha.len() == 40 && sha.bytes().all(|b| b.is_ascii_hexdigit())); + let path = std::path::PathBuf::from(output); + assert!(path.is_absolute()); + let parent = path.parent().unwrap().canonicalize().unwrap(); + let system_tmp = std::env::temp_dir().canonicalize().unwrap(); + assert!( + parent.starts_with(&system_tmp) + || parent == std::path::Path::new("/private/tmp") + || parent == std::path::Path::new("/tmp"), + "test export is confined to temporary storage" + ); + let mut options = std::fs::OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + options.mode(0o600); + let mut file = options + .open(&path) + .expect("never overwrite an existing fixture"); + serde_json::to_writer_pretty(&mut file, &fixture).unwrap(); + file.write_all(b"\n").unwrap(); + file.sync_all().unwrap(); + println!("RUVIEW_SELECTED_CONTRACT_FILE={}", path.display()); + } +} + +#[path = "room_learner_api_offline_metadata_tests.rs"] +mod offline_metadata_tests; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_selection_fixture.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_selection_fixture.rs new file mode 100644 index 0000000000..ec75852de6 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_selection_fixture.rs @@ -0,0 +1,258 @@ +//! SYNTHETIC public API fixture; no manufactured model/qualification receipts. +use super::*; + +pub(super) fn update() -> SensingUpdate { + serde_json::from_value(serde_json::json!({ + "type":"sensing_update","timestamp":chrono::Utc::now().timestamp_millis() as f64/1000.0, + "source":"esp32","tick":321,"nodes":[], + "features":{"mean_rssi":-50.0,"variance":1.0,"motion_band_power":2.0, + "breathing_band_power":1.0,"dominant_freq_hz":0.2,"change_points":0,"spectral_power":3.0}, + "classification":{"motion_level":"present_moving","presence":true,"confidence":0.99}, + "signal_field":{"grid_size":[1,1,1],"values":[1.0]}, + "estimated_persons":3, + "vital_signs":{"breathing_rate_bpm":14.0,"heart_rate_bpm":62.0, + "breathing_confidence":0.8,"heartbeat_confidence":0.7,"signal_quality":0.9}, + "pose_keypoints":[[0.1,0.2,0.0,0.9]],"enhanced_motion":{"level":"moving"}, + "enhanced_breathing":{"rate_bpm":14.0},"posture":"standing" + })).unwrap() +} + +fn gain(label: RoomLabel) -> f64 { + match label { + RoomLabel::Empty => 2.0, + RoomLabel::OccupiedStatic => 6.0, + RoomLabel::OccupiedMoving => 7.0, + RoomLabel::AdjacentOnly => 2.3, + RoomLabel::Unknown => unreachable!(), + } +} +pub(super) fn feed(rt: &RoomLearningRuntime, grid: &str, label: RoomLabel, sequence: u32, at: u64) { + for sample in 0..WINDOW_PACKETS { + for source in [15u8, 16] { + let values: Vec<_> = (0..56) + .map(|bin| { + gain(label) + + f64::from(source - 14) * 0.3 + + bin as f64 * 0.04 + + sample as f64 * 0.003 + }) + .collect(); + rt.ingest( + RawSourceFrame { + source_id: source, + grid_id: grid, + frequency_hz: 2_437_000_000, + bandwidth_hz: 20_000_000, + sequence: sequence + sample as u32, + amplitudes: &values, + }, + at + sample as u64 * 1_000_000 + u64::from(source), + ) + .unwrap(); + } + } +} +pub(super) fn phase( + rt: &RoomLearningRuntime, + grid: &str, + role: DataRole, + label: RoomLabel, + index: u32, + base: u64, +) { + let at = base + u64::from(index) * 13_000_000_000; + rt.begin_phase( + &rt.mutation_guard().unwrap(), + PhaseRequest { + trial_id: format!("synthetic-composer-trial-{index}"), + session_id: format!("synthetic-composer-session-{index}"), + role, + label, + duration_ms: MIN_PHASE_MS, + lead_in_ms: 0, + conditions_confirmed: true, + }, + at, + ) + .unwrap(); + for quarter in 0..4 { + feed( + rt, + grid, + label, + index * 32 + quarter * 8 + 1, + at + u64::from(quarter) * 3_000_000_000 + 1_000_000, + ); + } + rt.finish_phase( + &rt.mutation_guard().unwrap(), + at + MIN_PHASE_MS * 1_000_000, + true, + ) + .unwrap(); +} +pub(super) async fn owner_post( + state: SharedState, + path: &str, + body: serde_json::Value, +) -> serde_json::Value { + let app = owner_selection_routes( + wifi_densepose_sensing_server::bearer_auth::AuthState::from_token("synthetic-owner"), + ) + .with_state(state); + let mut request = Request::builder() + .method(Method::POST) + .uri(path) + .header("content-type", "application/json") + .header("authorization", "Bearer synthetic-owner") + .body(Body::from(serde_json::to_vec(&body).unwrap())) + .unwrap(); + request.extensions_mut().insert(axum::extract::ConnectInfo( + "127.0.0.1:41000".parse::().unwrap(), + )); + let response = app.oneshot(request).await.unwrap(); + assert_eq!(response.status(), StatusCode::OK); + serde_json::from_slice( + &axum::body::to_bytes(response.into_body(), 65536) + .await + .unwrap(), + ) + .unwrap() +} + +pub(super) async fn fitted() -> ( + tempfile::TempDir, + SharedState, + Arc, + String, +) { + let (dir, state) = fixture(); + let room = + companion_room::SharedRoomStore::load(&dir.path().join("companion"), "synthetic-install") + .unwrap(); + let room_json = serde_json::json!({"schema":"ruview.companion.room.v1","id":"synthetic-room","name":"Test room", + "widthM":2.5,"lengthM":2.5,"heightM":2.8,"nodes":[ + {"nodeId":15,"slot":"north_west","xM":0,"yM":0,"role":"receiver","headingDeg":90,"orientationConfirmed":true,"confirmedAtUnixMs":1}, + {"nodeId":16,"slot":"north_east","xM":2.5,"yM":0,"role":"receiver","headingDeg":180,"orientationConfirmed":true,"confirmedAtUnixMs":1}],"accessPoint":{"slot":"south_center","xM":1.25,"yM":2.5,"provenance":"operator_entered","confirmedAtUnixMs":1}}); + let response = companion_room::owner_router(room.clone()) + .oneshot( + Request::builder() + .method(Method::PUT) + .uri("/api/v1/companion/room") + .header("content-type", "application/json") + .body(Body::from( + serde_json::to_vec( + &serde_json::json!({"expected_revision":0,"room":room_json}), + ) + .unwrap(), + )) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let published: serde_json::Value = serde_json::from_slice( + &axum::body::to_bytes(response.into_body(), 65536) + .await + .unwrap(), + ) + .unwrap(); + assert_eq!(published["revision"], 1); + let binding = room.current_binding_digest().unwrap().unwrap(); + state.write().await.current_room_store = Some(room); + let configured = json_request( + app(state.clone()), + "/configure", + Method::POST, + serde_json::json!({ + "binding_digest":binding,"revision":7001,"source_node_ids":[15,16],"new_training":true}), + ) + .await; + assert_eq!(configured["success"], true, "{configured}"); + // Learner config generation and companion publication revision are separate + // domains. Only the full shared binding digest joins their authority. + assert_eq!(configured["status"]["revision"], 7001); + + let rt = state.read().await.room_learning.as_ref().unwrap().clone(); + let grid = rt.context().unwrap().sources[0].grid_id.clone(); + let base = rt.receiver_now_ns() - 200_000_000_000; + let labels = [ + RoomLabel::Empty, + RoomLabel::OccupiedStatic, + RoomLabel::OccupiedMoving, + RoomLabel::AdjacentOnly, + ]; + for (i, label) in labels.into_iter().enumerate() { + phase(&rt, &grid, DataRole::Train, label, i as u32, base) + } + // Small real optimizer fixture; production defaults remain unchanged. + rt.start_training( + &rt.mutation_guard().unwrap(), + FitConfig { + epochs: 8, + learning_rate: 0.5, + hidden_dim: 2, + embedding_dim: 2, + temporal_tokens: 2, + ..FitConfig::default() + }, + ) + .unwrap(); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(90); + loop { + let status = rt.status(rt.receiver_now_ns()); + if status["state"] == "ready" { + break; + } + assert!(status["state"] != "failed", "{status}"); + assert!( + std::time::Instant::now() < deadline, + "bounded synthetic fit timed out" + ); + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + for (i, label) in labels.into_iter().enumerate() { + phase(&rt, &grid, DataRole::HeldOut, label, i as u32 + 4, base) + } + rt.evaluate(&rt.mutation_guard().unwrap(), DataRole::HeldOut) + .unwrap(); + // Refresh the actual parser/readiness cache. Then admit labelled-value + // synthetic live frames with fresh sequence IDs through the public runtime. + let now = std::time::Instant::now(); + { + let mut inner = state.write().await; + for row in 0..120 { + for node in [15u8, 16] { + let packet = wire(node, 500 + row); + let frame = parse_esp32_frame(&packet).unwrap(); + let at = now - std::time::Duration::from_millis((120 - row) as u64 * 100); + observe(&mut inner, &frame, &packet, at); + } + } + for node in [15u8, 16] { + inner.node_states.get_mut(&node).unwrap().last_frame_time = + Some(std::time::Instant::now()); + } + inner.source = "esp32".into(); + inner.last_esp32_frame = Some(std::time::Instant::now()); + } + feed( + &rt, + &grid, + RoomLabel::OccupiedStatic, + 1000, + rt.receiver_now_ns(), + ); + tokio::time::sleep(std::time::Duration::from_millis(12)).await; + let q = owner_post( + state.clone(), + "/api/v1/room-learner/qualify", + serde_json::json!({"expected":rt.mutation_guard().unwrap()}), + ) + .await; + assert_eq!(q["success"], true, "{q}"); + let selected=owner_post(state.clone(),"/api/v1/room-learner/select",serde_json::json!({"expected":rt.mutation_guard().unwrap(), + "model_sha256":q["qualification"]["receipt"]["model_sha256"],"qualification_sha256":q["qualification"]["receipt_sha256"]})).await; + assert_eq!(selected["success"], true, "{selected}"); + (dir, state, rt, binding) +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_selection_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_selection_tests.rs new file mode 100644 index 0000000000..e20c1641a4 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_selection_tests.rs @@ -0,0 +1,107 @@ +use super::*; + +#[tokio::test] +async fn selection_mutations_are_local_owner_only_even_without_outer_bearer_gate() { + let (_dir, state) = fixture(); + let routes = owner_selection_routes( + wifi_densepose_sensing_server::bearer_auth::AuthState::from_token("synthetic-owner"), + ) + .with_state(state.clone()); + for path in [ + "/api/v1/room-learner/recovery/enable-current", + "/api/v1/room-learner/qualify", + "/api/v1/room-learner/select", + "/api/v1/room-learner/deselect", + ] { + for (peer, token) in [ + ("127.0.0.1:41000", None), + ("127.0.0.1:41000", Some("synthetic-peer-read")), + ("192.0.2.1:41000", Some("synthetic-owner")), + ] { + let mut request = Request::builder() + .method(Method::POST) + .uri(path) + .header("content-type", "application/json"); + if let Some(token) = token { + request = request.header("authorization", format!("Bearer {token}")); + } + let mut request = request.body(Body::from("{}")).unwrap(); + request.extensions_mut().insert(axum::extract::ConnectInfo( + peer.parse::().unwrap(), + )); + let response = routes.clone().oneshot(request).await.unwrap(); + assert_eq!( + response.status(), + StatusCode::FORBIDDEN, + "{path} {peer} {token:?}" + ); + } + } + // A legitimate owner reaches the typed handler; no geometry is published + // and no data is erased by this negative missing-context operation. + let expected = state + .read() + .await + .room_learning + .as_ref() + .unwrap() + .mutation_guard() + .unwrap(); + let mut request = Request::builder() + .method(Method::POST) + .uri("/api/v1/room-learner/qualify") + .header("content-type", "application/json") + .header("authorization", "Bearer synthetic-owner") + .body(Body::from( + serde_json::to_vec(&serde_json::json!({"expected":expected})).unwrap(), + )) + .unwrap(); + request.extensions_mut().insert(axum::extract::ConnectInfo( + "127.0.0.1:41000".parse::().unwrap(), + )); + let response = routes.oneshot(request).await.unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let body = axum::body::to_bytes(response.into_body(), 65536) + .await + .unwrap(); + let body: serde_json::Value = serde_json::from_slice(&body).unwrap(); + assert_eq!(body["success"], false); + assert_eq!(body["error_code"], "learner_qualification_rejected"); +} + +#[tokio::test] +async fn selection_readback_does_not_expose_owner_or_mutation_authority() { + let (_dir, state) = fixture(); + let response = Router::new() + .route("/api/v1/room-learner/selection", get(selection)) + .with_state(state) + .oneshot( + Request::builder() + .uri("/api/v1/room-learner/selection") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let bytes = axum::body::to_bytes(response.into_body(), 65536) + .await + .unwrap(); + let body: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(body["schema"], "ruview.room-learner.selection.v1"); + assert_eq!(body["model_selection"]["current"]["status"], "none"); + let text = String::from_utf8(bytes.to_vec()).unwrap(); + for forbidden in [ + "owner_scope", + "mutation_guard", + "encoder_weights", + "training_samples", + "head_weights", + "runtime_epoch", + ] { + assert!( + !text.contains(forbidden), + "read-scoped selection leaked {forbidden}" + ); + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_temporal.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_temporal.rs new file mode 100644 index 0000000000..409b43d11a --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_temporal.rs @@ -0,0 +1,704 @@ +//! Version-negotiated adapters. V1 handlers remain the default; malformed V2 +//! requests never enter the legacy workflow. Authority stays in Runtime. +use super::*; +use wifi_densepose_sensing_server::room_learner::temporal_wire::*; + +#[derive(Default, serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct SchemaQuery { + schema: Option, +} +fn schema_matches(actual: &str, expected: &str) -> Result<(), Json> { + if actual == expected { + Ok(()) + } else { + Err(failure( + "learner_schema_unsupported", + "Unsupported learner request schema.", + )) + } +} +macro_rules! command { + ($name:ident, $legacy:ty, $temporal:ty) => { + #[derive(serde::Deserialize)] + #[serde(untagged)] + pub(crate) enum $name { + Temporal($temporal), + Legacy($legacy), + } + }; +} +command!( + ConfigureCommand, + ConfigureRequest, + TemporalConfigureRequestV2 +); +command!(StartCommand, BeginRequest, TemporalPhaseStartRequestV2); +command!(FinishCommand, FinishRequest, TemporalPhaseFinishRequestV2); +command!(TrainCommand, TrainRequest, TemporalTrainRequestV2); +command!( + EvaluateCommand, + EvaluationRequest, + TemporalEvaluateRequestV2 +); +command!(QualifyCommand, GuardRequest, TemporalQualifyRequestV2); +command!( + SelectCommand, + selection_routes::SelectRequest, + TemporalSelectRequestV2 +); +command!( + RestoreCommand, + RestoreCaptureRequest, + TemporalRecoveryRequestV2 +); +command!( + DiscardCommand, + DiscardCaptureRequest, + TemporalDiscardRequestV2 +); + +command!(DeselectCommand, GuardRequest, TemporalDeselectRequestV2); +command!(PauseCommand, GuardRequest, TemporalCurrentActionRequestV2); +command!( + CancelOperationCommand, + GuardRequest, + TemporalCurrentActionRequestV2 +); + +pub(crate) async fn status_dispatch( + State(state): State, + Query(query): Query, +) -> Json { + let Some(schema) = query.schema else { + return status(State(state)).await; + }; + if let Err(error) = schema_matches(&schema, STATUS_SCHEMA) { + return error; + } + let rt = match runtime(&state).await { + Ok(rt) => rt, + Err(e) => return e, + }; + if let Err(e)=rt.temporal_default_operator_profile_v2(){return failure("learner_profile_unavailable",e);} + temporal_status(&state, &rt).await +} +fn temporal_selection( + state: &AppStateInner, + rt: &RoomLearningRuntime, + now: u64, +) -> Result { + let projection = match state.current_room_store.as_ref() { + Some(room) => room + .with_current_binding(|binding| { + rt.temporal_selection_v2(now, binding, Some(&state.calibration_boot_epoch)) + }) + .unwrap_or_else(|_| { + rt.temporal_selection_v2(now, None, Some(&state.calibration_boot_epoch)) + }), + None => rt.temporal_selection_v2(now, None, Some(&state.calibration_boot_epoch)), + }?; + validate_temporal_selection(projection) +} +fn validate_temporal_selection(projection: serde_json::Value) -> Result { + if projection["schema"] != SELECTION_SCHEMA + || projection["success"] != true + || projection["model_selection"]["schema"] != SELECTION_STATUS_SCHEMA + { + return Err("temporal_selection_projection_invalid".into()); + } + Ok(projection) +} +async fn temporal_status( + state: &SharedState, + rt: &Arc, +) -> Json { + temporal_status_with_reader(state, rt, |reader, now| reader.temporal_status_snapshot_v2(now)).await +} +// The reader seam keeps the blocking boundary explicit and allows a +// deterministic concurrency regression without a production runtime hook. +pub(super) async fn temporal_status_with_reader( + state: &SharedState, + rt: &Arc, + read_snapshot: F, +) -> Json +where + F: FnOnce(Arc, u64) -> Result + Send + 'static, +{ + // Concurrent polls wait as async futures, without owning an RF lock or + // spawning another blocking worker. Explicit cancellation removes a waiter. + let permit = match rt.acquire_temporal_status_reader().await { + Ok(permit) => permit, + Err(error) => return failure("learner_status_unavailable", error), + }; + // Clock advancement stays ordered with native ingress, but recovery disk + // discovery must never hold the shared RF lock while waiting on fsync. + let (now, expected, room_store, room_binding, boot_epoch) = { + let current = state.write().await; + if current.privacy_mode { + return failure("learner_privacy_mode", "Room learning unavailable in privacy mode."); + } + if !current.room_learning.as_ref().is_some_and(|live| Arc::ptr_eq(live, rt)) { + return failure("learner_state_changed", "learner_state_changed"); + } + let now = now_ns(rt); + if let Err(error) = rt.temporal_poll_status_v2(now) { + return failure("learner_status_unavailable", error); + } + let expected = match rt.mutation_guard() { + Ok(expected) => expected, + Err(error) => return failure("learner_status_unavailable", error), + }; + let room_binding = match current.current_room_store.as_ref().map(|room| room.current_binding_digest()).transpose() { + Ok(binding) => binding, + Err(_) => return failure("learner_state_changed", "current_room_binding_unavailable"), + }; + (now, expected, current.current_room_store.clone(), room_binding, current.calibration_boot_epoch.clone()) + }; + let reader = rt.clone(); + let read_binding = room_binding.clone(); + let read_epoch = boot_epoch.clone(); + let snapshot = tokio::task::spawn_blocking(move || { + // Cancellation of the awaiting HTTP request cannot drop this permit. + let _permit = permit; + let mut status = read_snapshot(reader.clone(), now)?; + let projection = reader.temporal_selection_v2(now, read_binding.as_ref().and_then(|binding| binding.as_deref()), Some(&read_epoch))?; + status["model_selection"] = validate_temporal_selection(projection)?["model_selection"].clone(); + Ok::<_, String>(status) + }).await; + let current = state.write().await; + if current.privacy_mode { + return failure("learner_privacy_mode", "Room learning unavailable in privacy mode."); + } + let same_room_store = match (&room_store, ¤t.current_room_store) { + (None, None) => true, + (Some(before), Some(after)) => Arc::ptr_eq(before, after), + _ => false, + }; + if !current.room_learning.as_ref().is_some_and(|live| Arc::ptr_eq(live, rt)) + || !same_room_store + || current.calibration_boot_epoch != boot_epoch + || current.current_room_store.as_ref().map(|room| room.current_binding_digest()).transpose().ok().as_ref() != Some(&room_binding) + || rt.mutation_guard().as_ref() != Ok(&expected) + { + return failure("learner_state_changed", "learner_state_changed"); + } + let result = match snapshot { + Ok(result) => result, + Err(_) => return failure("learner_status_unavailable", "learner_status_worker_unavailable"), + }; + match result.and_then(|status| { + if status["mutation_guard"] != serde_json::to_value(&expected).map_err(|_| "learner_status_guard_invalid")? { + return Err("learner_state_changed".into()); + } + Ok(status) + }) { + Ok(status) => Json(serde_json::json!({"success":true,"status":status})), + Err(error) => failure("learner_status_unavailable", error), + } +} +async fn operation_status( + state: &SharedState, + rt: &Arc, + id: String, + operation: &'static str, +) -> Json { + if id + .parse::() + .ok() + .is_none_or(|value| value.to_string() != id) + { + return failure( + "learner_operation_invalid", + "Operation identity is unavailable.", + ); + } + let mut result = temporal_status(state, rt).await; + if result.0["success"] == true { + result.0["operation_id"] = serde_json::json!(id); + result.0["operation"] = serde_json::json!(operation); + result.0["effect"] = serde_json::json!("started"); + } + result +} +fn field_capture_active(state: &AppStateInner) -> bool { + state.field_model.as_ref().is_some_and(|f| { + matches!(f.status(), CalibrationStatus::Collecting) + || (state.calibration_session_id.is_some() + && matches!(f.status(), CalibrationStatus::Uncalibrated)) + }) +} +fn require_live_context(state: &AppStateInner, rt: &RoomLearningRuntime) -> Result<(), String> { + let context = rt.context().ok_or("learner_source_unready")?; + if context + .sources + .iter() + .all(|spec| source_ready(state, spec, std::time::Instant::now())) + { + Ok(()) + } else { + Err("learner_source_unready".into()) + } +} + +pub(crate) async fn configure_dispatch( + State(state): State, + Json(command): Json, +) -> Json { + let ConfigureCommand::Temporal(mut request) = command else { + let ConfigureCommand::Legacy(request) = command else { + unreachable!() + }; + return configure(State(state), Json(request)).await; + }; + if let Err(e) = schema_matches(&request.schema, CONFIGURE_REQUEST_SCHEMA) { + return e; + } + if !request.new_training || !request.recovery_consent { + return failure( + "learner_recovery_consent_required", + "New Temporal training requires explicit encrypted recovery consent.", + ); + } + if !valid_binding_digest(&request.binding_digest) + || request.revision == 0 + || !valid_source_node_ids(&request.source_node_ids) + || request.source_node_ids.len() != 3 + { + return failure("learner_context_invalid", "Temporal engineering profile requires a current room revision and exactly three sorted unique sources."); + } + let rt = match runtime(&state).await { + Ok(rt) => rt, + Err(e) => return e, + }; + request.expected=match rt.temporal_select_profile_guarded_v2(request.profile_id.as_deref(),&request.expected){Ok(guard)=>guard,Err(e)=>return failure("learner_profile_rejected",e)}; + let s = state.read().await; + if field_capture_active(&s) { + return failure( + "learner_capture_conflict", + "Finish or cancel active field calibration first.", + ); + } + let now = std::time::Instant::now(); + let mut sources = Vec::new(); + for &id in &request.source_node_ids { + if !s.node_states.contains_key(&id) { + return failure( + "learner_source_missing", + "A selected node is not receiving.", + ); + } + let Some((key, meta)) = + native_candidates(&s, id, now) + .into_iter() + .max_by(|(ka, a), (kb, b)| { + a.receiver_rate_hz + .total_cmp(&b.receiver_rate_hz) + .then(ka.raw_bins.cmp(&kb.raw_bins)) + }) + else { + return failure( + "learner_source_unready", + "A selected native source/PHY stream is not ready.", + ); + }; + let grid_id = match native::grid_id( + key.n_antennas as usize, + key.raw_bins as usize, + wifi_densepose_hardware::PpduType::from_byte(key.ppdu_type), + ) { + Ok(grid) => grid, + Err(e) => return failure("learner_grid_unsupported", e), + }; + let Some(bandwidth_hz) = meta.bandwidth_hz else { + return failure("learner_source_unready", "Native bandwidth is unavailable."); + }; + sources.push(SourceSpec { + source_id: id, + device_id: format!("esp32-node-{id}"), + grid_id, + feature_count: 56, + signal_kind: RfModality::WifiCsi, + center_frequency_hz: meta.frequency_hz, + bandwidth_hz, + link_id: format!("receiver-{id}-transmitter-unverified"), + link_index: 0, + geometry: None, + }); + } + let context = match rt.bind_context(request.binding_digest, request.revision, sources) { + Ok(context) => context, + Err(e) => return failure("learner_context_invalid", e), + }; + let result = rt.temporal_configure_v2( + &request.expected, + context, + opaque_calibration_id("learner-temporal-run"), + request.recovery_consent, + now_ns(&rt), + ); + drop(s); + match result { + Ok(()) => temporal_status(&state, &rt).await, + Err(e) => failure("learner_configure_failed", e), + } +} + +pub(crate) async fn begin_dispatch( + State(state): State, + Json(command): Json, +) -> Json { + match command { + StartCommand::Legacy(request) => begin_phase(State(state), Json(request)).await, + StartCommand::Temporal(request) => { + if let Err(e) = schema_matches(&request.schema, PHASE_START_REQUEST_SCHEMA) { + return e; + } + let rt = match runtime(&state).await { + Ok(rt) => rt, + Err(e) => return e, + }; + let s = state.write().await; + if s.privacy_mode { + return failure( + "learner_privacy_mode", + "Room learning unavailable in privacy mode.", + ); + } + if field_capture_active(&s) { + return failure( + "legacy_empty_capture_active", + "Finish or cancel active field calibration first.", + ); + } + if let Err(e) = require_live_context(&s, &rt) { + return failure("learner_source_unready", e); + } + let result = rt.temporal_begin_phase_v2(request, now_ns(&rt)); + drop(s); + match result { + Ok(()) => temporal_status(&state, &rt).await, + Err(e) => failure("learner_phase_rejected", e), + } + } + } +} +pub(crate) async fn finish_dispatch( + State(state): State, + Json(command): Json, +) -> Json { + match command { + FinishCommand::Legacy(request) => finish_phase(State(state), Json(request)).await, + FinishCommand::Temporal(request) => { + if let Err(e) = schema_matches(&request.schema, PHASE_FINISH_REQUEST_SCHEMA) { + return e; + } + let rt = match runtime(&state).await { + Ok(rt) => rt, + Err(e) => return e, + }; + match rt.temporal_finish_phase_v2(request, now_ns(&rt)) { + Ok(()) => temporal_status(&state, &rt).await, + Err(e) => failure("learner_phase_incomplete", e), + } + } + } +} +macro_rules! operation_dispatch { + ($handler:ident, $command:ident, $legacy:ident, $schema:ident, $method:ident, $failure:literal) => { + pub(crate) async fn $handler( + State(state): State, + Json(command): Json<$command>, + ) -> Json { + match command { + $command::Legacy(request) => $legacy(State(state), Json(request)).await, + $command::Temporal(request) => { + if let Err(e) = schema_matches(&request.schema, $schema) { + return e; + } + let rt = match runtime(&state).await { + Ok(rt) => rt, + Err(e) => return e, + }; + match rt.$method(request, now_ns(&rt)) { + Ok(id) => operation_status(&state, &rt, id, stringify!($legacy)).await, + Err(e) => failure($failure, e), + } + } + } + } + }; +} +operation_dispatch!( + train_dispatch, + TrainCommand, + train, + TRAIN_REQUEST_SCHEMA, + temporal_train_v2, + "learner_train_rejected" +); +operation_dispatch!( + evaluate_dispatch, + EvaluateCommand, + evaluate, + EVALUATE_REQUEST_SCHEMA, + temporal_evaluate_v2, + "learner_evaluation_rejected" +); + +pub(crate) async fn qualify_dispatch( + State(state): State, + Json(command): Json, +) -> Json { + match command { + QualifyCommand::Legacy(request) => { + selection_routes::qualify(State(state), Json(request)).await + } + QualifyCommand::Temporal(request) => { + if let Err(e) = schema_matches(&request.schema, QUALIFY_REQUEST_SCHEMA) { + return e; + } + let rt = match runtime(&state).await { + Ok(rt) => rt, + Err(e) => return e, + }; + let s = state.write().await; + let result = selection_routes::with_current_room(&s, &rt, || { + rt.temporal_qualify_v2(request, now_ns(&rt)) + }); + drop(s); + match result { + Ok(id) => operation_status(&state, &rt, id, "qualify").await, + Err(e) => failure("learner_qualification_rejected", e), + } + } + } +} +pub(crate) async fn select_dispatch( + State(state): State, + Json(command): Json, +) -> Json { + match command { + SelectCommand::Legacy(request) => { + selection_routes::select_model(State(state), Json(request)).await + } + SelectCommand::Temporal(request) => { + if let Err(e) = schema_matches(&request.schema, SELECT_REQUEST_SCHEMA) { + return e; + } + let rt = match runtime(&state).await { + Ok(rt) => rt, + Err(e) => return e, + }; + let s = state.write().await; + let result = selection_routes::with_current_room(&s, &rt, || { + rt.temporal_select_v2(request, now_ns(&rt)) + }); + match result { + Ok(value) => Json(value), + Err(e) => failure("learner_selection_rejected", e), + } + } + } +} +pub(crate) async fn selection_dispatch( + State(state): State, + Query(query): Query, +) -> Json { + if let Some(schema) = query.schema.as_deref() { + if let Err(e) = schema_matches(schema, SELECTION_SCHEMA) { + return e; + } + } else { + let actual_temporal = state + .read() + .await + .room_learning + .as_ref() + .is_some_and(|rt| rt.temporal_v2_state_present()); + if !actual_temporal { + return selection_routes::selection(State(state)).await; + } + } + let rt = match runtime(&state).await { + Ok(rt) => rt, + Err(e) => return e, + }; + let s = state.read().await; + if s.privacy_mode { + return failure( + "learner_privacy_mode", + "Room learning unavailable in privacy mode.", + ); + } + match temporal_selection(&s, &rt, now_ns(&rt)) { + Ok(mut value) => { + value["progress_schema"] = serde_json::json!(PAIRED_PROGRESS_SCHEMA); + Json(value) + } + Err(e) => failure("learner_selection_unavailable", e), + } +} +pub(crate) async fn restore_dispatch( + State(state): State, + Json(command): Json, +) -> Json { + match command { + RestoreCommand::Legacy(request) => restore_capture(State(state), Json(request)).await, + RestoreCommand::Temporal(mut request) => { + if let Err(e) = schema_matches(&request.schema, RECOVERY_RESTORE_REQUEST_SCHEMA) { + return e; + } + let rt = match runtime(&state).await { + Ok(rt) => rt, + Err(e) => return e, + }; + request.expected=match rt.temporal_select_profile_guarded_v2(request.profile_id.as_deref(),&request.expected){Ok(guard)=>guard,Err(e)=>return failure("learner_profile_rejected",e)}; + let now = now_ns(&rt); + let context = match rt.temporal_recovery_context_v2( + &request.binding_digest, + request.revision, + &request.source_node_ids, + now, + ) { + Ok(context) => context, + Err(e) => return failure("learner_recovery_restore_failed", e), + }; + let s = state.read().await; + if field_capture_active(&s) { + return failure( + "learner_capture_conflict", + "Finish or cancel active field calibration first.", + ); + } + if context + .sources + .iter() + .any(|spec| !source_ready(&s, spec, std::time::Instant::now())) + { + return failure( + "learner_recovery_source_unready", + "Restore the exact native source/PHY streams before recovery.", + ); + } + let result = rt.temporal_restore_v2( + &request.expected, + &context, + &request.capture_generation_id, + now, + ); + drop(s); + match result { + Ok(()) => temporal_status(&state, &rt).await, + Err(e) => failure("learner_recovery_restore_failed", e), + } + } + } +} +pub(crate) async fn discard_dispatch( + State(state): State, + Json(command): Json, +) -> Json { + match command { + DiscardCommand::Legacy(request) => discard_capture(State(state), Json(request)).await, + DiscardCommand::Temporal(request) => { + if let Err(e) = schema_matches(&request.schema, RECOVERY_DISCARD_REQUEST_SCHEMA) { + return e; + } + let rt = match runtime(&state).await { + Ok(rt) => rt, + Err(e) => return e, + }; + match rt.temporal_discard_v2( + &request.expected, + &request.capture_generation_id, + now_ns(&rt), + ) { + Ok(()) => temporal_status(&state, &rt).await, + Err(e) => failure("learner_recovery_discard_failed", e), + } + } + } +} + +pub(crate) async fn pause_dispatch( + State(state): State, + Json(command): Json, +) -> Json { + match command { + PauseCommand::Legacy(request) => cancel_phase(State(state), Json(request)).await, + PauseCommand::Temporal(request) => { + if let Err(e) = schema_matches(&request.schema, PHASE_CANCEL_REQUEST_SCHEMA) { + return e; + } + let rt = match runtime(&state).await { + Ok(rt) => rt, + Err(e) => return e, + }; + match rt.temporal_pause_phase_v2(request, now_ns(&rt)) { + Ok(()) => temporal_status(&state, &rt).await, + Err(e) => failure("learner_phase_cancel_failed", e), + } + } + } +} +pub(crate) async fn cancel_operation_dispatch( + State(state): State, + Json(command): Json, +) -> Json { + match command { + CancelOperationCommand::Legacy(request) => { + cancel_training(State(state), Json(request)).await + } + CancelOperationCommand::Temporal(request) => { + if let Err(e) = schema_matches(&request.schema, TRAIN_CANCEL_REQUEST_SCHEMA) { + return e; + } + let rt = match runtime(&state).await { + Ok(rt) => rt, + Err(e) => return e, + }; + match rt.temporal_cancel_operation_v2(request, now_ns(&rt)) { + Ok(()) => temporal_status(&state, &rt).await, + Err(e) => failure("learner_cancel_failed", e), + } + } + } +} + +pub(crate) async fn deselect_dispatch( + State(state): State, + Json(command): Json, +) -> Json { + match command { + DeselectCommand::Legacy(request) => { + selection_routes::deselect_model(State(state), Json(request)).await + } + DeselectCommand::Temporal(request) => { + if let Err(e) = schema_matches(&request.schema, DESELECT_REQUEST_SCHEMA) { + return e; + } + let rt = match runtime(&state).await { + Ok(rt) => rt, + Err(e) => return e, + }; + let _state = state.write().await; + match rt.temporal_deselect_v2(request, now_ns(&rt)) { + Ok(value) => Json(value), + Err(e) => failure("learner_deselection_rejected", e), + } + } + } +} + + +pub(crate) async fn retry_dispatch(State(state):State,Json(request):Json)->Json{ + if let Err(e)=schema_matches(&request.schema,RECOVERY_RETRY_REQUEST_SCHEMA){return e;} + let rt=match runtime(&state).await{Ok(rt)=>rt,Err(e)=>return e}; + {let s=state.read().await;if field_capture_active(&s){return failure("learner_capture_conflict","Finish active field calibration first.");}} + // Retry is metadata recovery, not capture. Its Runtime CAS still fences the + // owner/context/profile; never hold the UDP producer's SharedState lock + // across writer drain, filesystem barriers or cryptographic readback. + let result=rt.temporal_retry_v2(request,now_ns(&rt)); + match result{Ok(())=>temporal_status(&state,&rt).await,Err(e)=>failure("learner_recovery_retry_failed",e)} +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_temporal_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_temporal_tests.rs new file mode 100644 index 0000000000..2b777a0a0b --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_temporal_tests.rs @@ -0,0 +1,781 @@ +//! Actual Axum extraction/dispatch controls; synthetic state only. +use super::super::temporal_routes::*; +use super::*; +use wifi_densepose_sensing_server::room_learner::temporal_wire::*; +fn temporal_app(state: SharedState) -> Router { + Router::new() + .route("/status", get(status_dispatch)) + .route("/selection", get(selection_dispatch)) + .route("/configure", post(configure_dispatch)) + .route("/phase/start", post(begin_dispatch)) + .route("/phase/finish", post(finish_dispatch)) + .route("/train", post(train_dispatch)) + .route("/evaluate", post(evaluate_dispatch)) + .route("/restore", post(restore_dispatch)) + .route("/retry", post(retry_dispatch)) + .route("/discard", post(discard_dispatch)) + .route("/phase/cancel", post(pause_dispatch)) + .route("/train/cancel", post(cancel_operation_dispatch)) + .with_state(state) +} +async fn request_raw( + app: Router, + path: &str, + body: Option, +) -> (StatusCode, Vec) { + let mut request = Request::builder().uri(path); + let body = if let Some(body) = body { + request = request + .method(Method::POST) + .header("content-type", "application/json"); + Body::from(serde_json::to_vec(&body).unwrap()) + } else { + Body::empty() + }; + let response = app.oneshot(request.body(body).unwrap()).await.unwrap(); + let status = response.status(); + let bytes = axum::body::to_bytes(response.into_body(), 65536) + .await + .unwrap(); + (status, bytes.to_vec()) +} +async fn decoded(app: Router, path: &str, body: Option) -> serde_json::Value { + let (status, bytes) = request_raw(app, path, body).await; + assert_eq!( + status, + StatusCode::OK, + "{}", + String::from_utf8_lossy(&bytes) + ); + serde_json::from_slice(&bytes).unwrap() +} +fn configure_v2(expected: MutationGuard) -> serde_json::Value { + serde_json::json!({"schema":CONFIGURE_REQUEST_SCHEMA,"model_kind":"ruview.room-learner.managed-temporal-input.v3", + "expected":expected,"binding_digest":"b".repeat(64),"revision":1,"source_node_ids":[15,16,17], + "new_training":true,"recovery_consent":true}) +} +#[tokio::test] +async fn negotiated_idle_capability_preserves_default_status_and_selection() { + let (_dir, state) = fixture(); + let app = temporal_app(state); + let legacy = decoded(app.clone(), "/status", None).await; + assert!(legacy.get("mutation_guard").is_some()); + assert!(legacy.get("available_model_kinds").is_none()); + let v2 = decoded( + app.clone(), + &format!("/status?schema={STATUS_SCHEMA}"), + None, + ) + .await; + assert_eq!(v2["success"], true); + assert_eq!(v2["status"]["schema"], STATUS_SCHEMA); + assert_eq!(v2["status"]["workflow_kind"], "idle"); + assert_eq!( + v2["status"]["model_selection"]["schema"], + SELECTION_STATUS_SCHEMA + ); + assert!(v2["status"]["model_selection"].get("success").is_none()); + assert_eq!( + v2["status"]["available_model_kinds"], + serde_json::json!([]), + "Runtime without encrypted recovery must not advertise Temporal activation" + ); + let selection = decoded(app, "/selection", None).await; + assert_eq!(selection["schema"], "ruview.room-learner.selection.v1"); + assert_eq!( + selection["progress_schema"], + "ruview.room-learner.progress.v1" + ); +} +#[tokio::test] +async fn unknown_status_schema_fails_without_starting_or_reconfiguring() { + let (_dir, state) = fixture(); + let rt = state.read().await.room_learning.clone().unwrap(); + let before = rt.mutation_guard().unwrap(); + let result = decoded( + temporal_app(state), + "/status?schema=ruview.room-learner.status.v999", + None, + ) + .await; + assert_eq!(result["success"], false); + assert_eq!(result["error_code"], "learner_schema_unsupported"); + assert_eq!( + serde_json::to_value(rt.mutation_guard().unwrap()).unwrap(), + serde_json::to_value(before).unwrap() + ); + assert!(rt.context().is_none()); +} +#[tokio::test] +async fn temporal_configure_unknown_kind_or_injected_authority_cannot_fall_back_to_legacy() { + let (_dir, state) = fixture(); + let rt = state.read().await.room_learning.clone().unwrap(); + let app = temporal_app(state); + let valid = configure_v2(rt.mutation_guard().unwrap()); + for (key, value) in [ + ("model_kind", serde_json::json!("legacy")), + ("report", serde_json::json!({"passed":true})), + ("verified", serde_json::json!(true)), + ("fit_config", serde_json::json!({"epochs":1})), + ] { + let mut body = valid.clone(); + body[key] = value; + let (status, _) = request_raw(app.clone(), "/configure", Some(body)).await; + assert_eq!(status, StatusCode::UNPROCESSABLE_ENTITY, "{key}"); + assert!(rt.context().is_none()); + } + let mut unknown = valid; + unknown["schema"] = serde_json::json!("configure.v999"); + let result = decoded(app, "/configure", Some(unknown)).await; + assert_eq!(result["error_code"], "learner_schema_unsupported"); + assert!(rt.context().is_none()); +} +#[tokio::test] +async fn temporal_configure_requires_exact_profile_and_current_explicit_consent() { + let (_dir, state) = fixture(); + let rt = state.read().await.room_learning.clone().unwrap(); + let app = temporal_app(state); + for (key, value, code) in [ + ( + "recovery_consent", + serde_json::json!(false), + "learner_recovery_consent_required", + ), + ( + "new_training", + serde_json::json!(false), + "learner_recovery_consent_required", + ), + ( + "source_node_ids", + serde_json::json!([15, 16]), + "learner_context_invalid", + ), + ( + "source_node_ids", + serde_json::json!([17, 16, 15]), + "learner_context_invalid", + ), + ] { + let mut body = configure_v2(rt.mutation_guard().unwrap()); + body[key] = value; + assert_eq!( + decoded(app.clone(), "/configure", Some(body)).await["error_code"], + code + ); + assert!(rt.context().is_none()); + } +} +#[tokio::test] +async fn temporal_commands_reject_role_rows_scores_and_unknown_schema_without_legacy_fallback() { + let (_dir, state) = fixture(); + let rt = state.read().await.room_learning.clone().unwrap(); + let app = temporal_app(state); + let start = serde_json::json!({"schema":PHASE_START_REQUEST_SCHEMA,"expected":rt.mutation_guard().unwrap(), + "capture_generation_id":"c".repeat(64),"schedule_sha256":"d".repeat(64),"episode_index":0,"conditions_confirmed":true}); + for (key, value) in [ + ("role", serde_json::json!("HeldOut")), + ("label", serde_json::json!("Empty")), + ("rows", serde_json::json!([])), + ("verified", serde_json::json!(true)), + ] { + let mut body = start.clone(); + body[key] = value; + assert_eq!( + request_raw(app.clone(), "/phase/start", Some(body)).await.0, + StatusCode::UNPROCESSABLE_ENTITY + ); + } + let mut bad = start; + bad["schema"] = serde_json::json!("start.v999"); + assert_eq!( + decoded(app.clone(), "/phase/start", Some(bad)).await["error_code"], + "learner_schema_unsupported" + ); + let train = serde_json::json!({"schema":TRAIN_REQUEST_SCHEMA,"expected":rt.mutation_guard().unwrap(),"capture_generation_id":"c".repeat(64)}); + for key in ["config", "rows", "report", "score"] { + let mut bad = train.clone(); + bad[key] = serde_json::json!({}); + assert_eq!( + request_raw(app.clone(), "/train", Some(bad)).await.0, + StatusCode::UNPROCESSABLE_ENTITY + ); + } + assert!(rt.context().is_none()); +} +#[tokio::test] +async fn negotiated_status_still_obeys_privacy_and_disabled_runtime() { + let (_dir, state) = fixture(); + state.write().await.privacy_mode = true; + let path = format!("/status?schema={STATUS_SCHEMA}"); + let result = decoded(temporal_app(state.clone()), &path, None).await; + assert_eq!(result["error_code"], "learner_privacy_mode"); + assert!(result.get("status").is_none()); + let mut s = state.write().await; + s.privacy_mode = false; + s.room_learning = None; + drop(s); + assert_eq!( + decoded(temporal_app(state), &path, None).await["error_code"], + "learner_disabled" + ); +} + +#[tokio::test] +async fn temporal_selection_is_read_scoped_and_unknown_schema_does_not_use_v1() { + let (_dir, state) = fixture(); + let app = temporal_app(state); + let result = decoded( + app.clone(), + &format!("/selection?schema={SELECTION_SCHEMA}"), + None, + ) + .await; + assert_eq!(result["schema"], SELECTION_SCHEMA); + let text = serde_json::to_string(&result).unwrap(); + for forbidden in [ + "owner_scope", + "mutation_guard", + "runtime_epoch", + "encoder_weights", + "raw_frames", + "source_watermarks", + ] { + assert!( + !text.contains(forbidden), + "read projection leaked {forbidden}" + ); + } + let bad = decoded(app, "/selection?schema=selection.v999", None).await; + assert_eq!(bad["error_code"], "learner_schema_unsupported"); + assert!(bad.get("model_selection").is_none()); +} + +#[tokio::test] +async fn temporal_selection_mutations_keep_the_existing_local_owner_gate() { + let (_dir, state) = fixture(); + let app = owner_selection_routes( + wifi_densepose_sensing_server::bearer_auth::AuthState::from_token("synthetic-owner"), + ) + .with_state(state); + for path in [ + "/api/v1/room-learner/qualify", + "/api/v1/room-learner/select", + "/api/v1/room-learner/recovery/retry", + ] { + for (peer, token) in [ + ("127.0.0.1:41000", "synthetic-peer-read"), + ("192.0.2.1:41000", "synthetic-owner"), + ] { + let mut request = Request::builder() + .uri(path) + .method(Method::POST) + .header("authorization", format!("Bearer {token}")) + .header("content-type", "application/json") + .body(Body::from("{}")) + .unwrap(); + request.extensions_mut().insert(axum::extract::ConnectInfo( + peer.parse::().unwrap(), + )); + assert_eq!( + app.clone().oneshot(request).await.unwrap().status(), + StatusCode::FORBIDDEN + ); + } + } +} + +#[tokio::test] +async fn discard_and_pause_do_not_accept_ignored_context_or_new_phase_authority() { + let (_dir, state) = fixture(); + let rt = state.read().await.room_learning.clone().unwrap(); + let app = temporal_app(state); + let discard = serde_json::json!({"schema":RECOVERY_DISCARD_REQUEST_SCHEMA,"model_kind":"ruview.room-learner.managed-temporal-input.v3","expected":rt.mutation_guard().unwrap(),"capture_generation_id":"c".repeat(64)}); + for key in [ + "binding_digest", + "revision", + "source_node_ids", + "consent", + "rows", + ] { + let mut bad = discard.clone(); + bad[key] = serde_json::json!(null); + assert_eq!( + request_raw(app.clone(), "/discard", Some(bad)).await.0, + StatusCode::UNPROCESSABLE_ENTITY, + "discard:{key}" + ); + } + for (path, schema) in [ + ("/phase/cancel", PHASE_CANCEL_REQUEST_SCHEMA), + ("/train/cancel", TRAIN_CANCEL_REQUEST_SCHEMA), + ] { + let action = serde_json::json!({"schema":schema,"expected":rt.mutation_guard().unwrap(),"capture_generation_id":"c".repeat(64)}); + for key in ["phase", "role", "new_training", "consent"] { + let mut bad = action.clone(); + bad[key] = serde_json::json!(true); + assert_eq!( + request_raw(app.clone(), path, Some(bad)).await.0, + StatusCode::UNPROCESSABLE_ENTITY, + "{path}:{key}" + ); + } + } + assert!(rt.context().is_none()); +} + +#[tokio::test] +async fn actual_temporal_configure_dispatches_queryless_phone_reads_without_owner_status() { + use wifi_densepose_sensing_server::room_learner::recovery_store::RecoveryStore; + let (dir, state) = fixture(); + let owner = "a".repeat(64); + let path = dir.path().join(&owner).join("SensingServer"); + let store = RecoveryStore::new(&path, &owner, "synthetic-install", Some(&[7; 32])).unwrap(); + let rt = Arc::new( + RoomLearningRuntime::new_with_recovery(&path, &owner, "synthetic-install", store).unwrap(), + ); + state.write().await.room_learning = Some(rt.clone()); + let app = temporal_app(state.clone()); + let capable = decoded( + app.clone(), + &format!("/status?schema={STATUS_SCHEMA}"), + None, + ) + .await; + assert_eq!( + capable["status"]["available_model_kinds"], + serde_json::json!(["ruview.room-learner.managed-temporal-input.v3"]) + ); + let configured = decoded( + app.clone(), + "/configure", + Some(configure_v2(rt.mutation_guard().unwrap())), + ) + .await; + assert_eq!(configured["success"], true, "{configured}"); + for _ in 0..200 { + let _ = decoded( + app.clone(), + &format!("/status?schema={STATUS_SCHEMA}"), + None, + ) + .await; + if rt.temporal_v2_state_present() { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + assert!( + rt.temporal_v2_state_present(), + "actual full run must be admitted" + ); + let selected = decoded(app.clone(), "/selection", None).await; + assert_eq!(selected["schema"], SELECTION_SCHEMA); + assert_eq!(selected["progress_schema"], PAIRED_PROGRESS_SCHEMA); + assert!(selected.get("status").is_none()); + assert!(!selected.to_string().contains("mutation_guard")); + let progress = Router::new() + .route( + "/progress", + get(super::super::paired_progress::progress_dispatch), + ) + .with_state(state); + let (status, body) = request_raw(progress, "/progress", None).await; + assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE); + let body: serde_json::Value = serde_json::from_slice(&body).unwrap(); + assert_eq!(body["schema"], PAIRED_PROGRESS_SCHEMA); + assert_eq!(body["error_code"], "publication_unavailable"); + assert!(body.get("mutation_guard").is_none()); +} + +#[tokio::test] +async fn queryless_idle_progress_remains_v1_and_unknown_query_cannot_activate() { + let (_dir, state) = fixture(); + let rt = state.read().await.room_learning.clone().unwrap(); + let app = Router::new() + .route( + "/progress", + get(super::super::paired_progress::progress_dispatch), + ) + .with_state(state); + let (status, body) = request_raw(app.clone(), "/progress", None).await; + assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE); + assert_eq!( + serde_json::from_slice::(&body).unwrap()["schema"], + "ruview.room-learner.progress.v1" + ); + let (status, body) = request_raw(app, "/progress?schema=progress.v999", None).await; + assert_eq!(status, StatusCode::BAD_REQUEST); + assert_eq!( + serde_json::from_slice::(&body).unwrap()["error_code"], + "learner_schema_unsupported" + ); + assert!(rt.context().is_none()); +} + +#[tokio::test] +async fn configured_owner_status_serializes_its_capture_clock_tick_before_sampling_time() { + use wifi_densepose_sensing_server::room_learner::recovery_store::RecoveryStore; + let (dir,state)=fixture(); + let owner="a".repeat(64); + let path=dir.path().join(&owner).join("SensingServer"); + let store=RecoveryStore::new(&path,&owner,"synthetic-install",Some(&[7;32])).unwrap(); + let rt=Arc::new(RoomLearningRuntime::new_with_recovery(&path,&owner,"synthetic-install",store).unwrap()); + state.write().await.room_learning=Some(rt.clone()); + let app=temporal_app(state.clone()); + let configured=decoded(app.clone(),"/configure",Some(configure_v2(rt.mutation_guard().unwrap()))).await; + assert_eq!(configured["success"],true,"{configured}"); + for _ in 0..100 { + let s=decoded(app.clone(),&format!("/status?schema={STATUS_SCHEMA}"),None).await; + if s["status"]["capture"]["stage"]=="ready"{break;} + tokio::time::sleep(std::time::Duration::from_millis(2)).await; + } + let shared_reader=state.read().await; + let task=tokio::spawn(async move{decoded(app,&format!("/status?schema={STATUS_SCHEMA}"),None).await}); + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + let ran_concurrently=task.is_finished(); + drop(shared_reader); + let reply=tokio::time::timeout(std::time::Duration::from_secs(5),task).await.unwrap().unwrap(); + assert_eq!(reply["status"]["capture"]["stage"],"ready","{reply}"); + assert_eq!(reply["status"]["capture"]["retained_windows"],0); + assert_eq!(reply["status"]["capture"]["observed_ms"],0); + assert!(!ran_concurrently,"clock-mutating owner status must not sample/tick concurrently under a shared read lock"); +} + +#[tokio::test(flavor="multi_thread", worker_threads=2)] +async fn concurrent_actual_owner_status_requests_keep_unstarted_full_run_ready() { + use wifi_densepose_sensing_server::room_learner::recovery_store::RecoveryStore; + let (dir,state)=fixture();let owner="a".repeat(64);let path=dir.path().join(&owner).join("SensingServer"); + let store=RecoveryStore::new(&path,&owner,"synthetic-install",Some(&[7;32])).unwrap(); + let rt=Arc::new(RoomLearningRuntime::new_with_recovery(&path,&owner,"synthetic-install",store).unwrap()); + state.write().await.room_learning=Some(rt.clone());let app=temporal_app(state.clone()); + let configured=decoded(app.clone(),"/configure",Some(configure_v2(rt.mutation_guard().unwrap()))).await;assert_eq!(configured["success"],true,"{configured}"); + let mut ready=false;for _ in 0..100{let s=decoded(app.clone(),&format!("/status?schema={STATUS_SCHEMA}"),None).await;if s["status"]["capture"]["stage"]=="ready"{ready=true;break;}tokio::time::sleep(std::time::Duration::from_millis(2)).await;}assert!(ready); + let barrier=Arc::new(tokio::sync::Barrier::new(2));let mut tasks=Vec::new(); + for _ in 0..2 {let app=app.clone();let barrier=barrier.clone();tasks.push(tokio::spawn(async move{let mut failed=None;for _ in 0..200{barrier.wait().await;let s=decoded(app.clone(),&format!("/status?schema={STATUS_SCHEMA}"),None).await;if s["status"]["capture"]["stage"]!="ready"&&failed.is_none(){failed=Some(s);}}failed}));} + let mut errors=Vec::new();for task in tasks{if let Some(error)=tokio::time::timeout(std::time::Duration::from_secs(30),task).await.unwrap().unwrap(){errors.push(error);}} + if let Some(error)=errors.first(){println!("SYNTHETIC_ACTUAL_PARALLEL_OWNER_STATUS_FAILURE={error}");} + assert!(errors.is_empty(),"actual concurrent owner GETs failed an unstarted run: {errors:?}"); + let last=decoded(app,&format!("/status?schema={STATUS_SCHEMA}"),None).await;assert_eq!(last["status"]["capture"]["retained_windows"],0);assert_eq!(last["status"]["capture"]["durable_windows"],0);assert_eq!(last["status"]["capture"]["observed_ms"],0); +} + +#[tokio::test] +async fn actual_client_publication_and_learner_revision_domains_pair_without_rewriting_context() { + use wifi_densepose_sensing_server::room_learner::recovery_store::RecoveryStore; + let client:serde_json::Value=serde_json::from_str(include_str!("testdata/paired_temporal_client_revision.json")).unwrap(); + let (dir,state)=fixture();let owner="a".repeat(64);let installation=client["installation_id"].as_str().unwrap(); + let path=dir.path().join(&owner).join("SensingServer");let recovery=RecoveryStore::new(&path,&owner,installation,Some(&[7;32])).unwrap(); + let rt=Arc::new(RoomLearningRuntime::new_with_recovery(&path,&owner,installation,recovery).unwrap()); + let publication=crate::companion_room::SharedRoomStore::load(&dir.path().join("publication"),installation).unwrap(); + let publish=|body:serde_json::Value|{let router=crate::companion_room::owner_router(publication.clone());async move{ + let reply=router.oneshot(Request::builder().method(Method::PUT).uri("/api/v1/companion/room").header("content-type","application/json").body(Body::from(body.to_string())).unwrap()).await.unwrap();assert_eq!(reply.status(),StatusCode::OK);let bytes=axum::body::to_bytes(reply.into_body(),65536).await.unwrap();serde_json::from_slice::(&bytes).unwrap() + }}; + let initial=publish(client["publication_request"]["body"].clone()).await; + assert_eq!(initial["binding_digest"],client["binding_digest"]);assert_eq!(initial["revision"],1); + {let mut s=state.write().await;s.room_learning=Some(rt.clone());s.current_room_store=Some(publication.clone());s.installation_id=Some(installation.into());s.calibration_boot_epoch="synthetic-paired-domain-boot".into();} + let mut command=client["configure_request"]["body"].clone();command["expected"]=serde_json::to_value(rt.mutation_guard().unwrap()).unwrap(); + assert_eq!(command["revision"],211247076768791u64); + let app=temporal_app(state.clone());let configured=decoded(app.clone(),"/configure",Some(command)).await;assert_eq!(configured["success"],true,"{configured}"); + let mut ready=false;for _ in 0..100{let s=decoded(app.clone(),&format!("/status?schema={STATUS_SCHEMA}"),None).await;if s["status"]["capture"]["stage"]=="ready"{ready=true;break;}tokio::time::sleep(std::time::Duration::from_millis(2)).await;}assert!(ready); + let context=rt.context().unwrap();assert_eq!(context.revision,211247076768791);assert_eq!(context.room_digest,client["binding_digest"].as_str().unwrap()); + let before_guard=rt.mutation_guard().unwrap();let before=decoded(app.clone(),&format!("/status?schema={STATUS_SCHEMA}"),None).await; + let progress=Router::new().route("/progress",get(super::super::paired_progress::progress_dispatch)).with_state(state.clone()); + let current=decoded(progress.clone(),"/progress",None).await; + println!("SYNTHETIC_ACTUAL_CLIENT_PAIRED_REVISION_SEAM={current}"); + assert_eq!(current["context_match"],true,"{current}");assert_eq!(current["room"]["revision"],1);assert_eq!(current["room"]["binding_digest"],client["binding_digest"]); + assert_eq!(current["sources"]["ready"],3);assert_eq!(current["capture"]["retained_windows"],0);assert_eq!(current["capture"]["durable_windows"],0); + assert_eq!(rt.mutation_guard().unwrap(),before_guard);assert_eq!(rt.context().unwrap(),context); + let after=decoded(app.clone(),&format!("/status?schema={STATUS_SCHEMA}"),None).await; + assert_eq!(before["status"]["capture"]["capture_generation_id"],after["status"]["capture"]["capture_generation_id"]);assert_eq!(before["status"]["capture"]["original_deadline"],after["status"]["capture"]["original_deadline"]); + let mut old_learner_context=context.clone();old_learner_context.revision=1; + assert_ne!(old_learner_context.digest(),context.digest(),"exact selected/capture context digest must retain the learner revision binding"); + let wrong_ready=rt.paired_temporal_progress_snapshot(installation,"synthetic-paired-domain-boot",Some(wifi_densepose_sensing_server::room_learner::lifecycle::PublishedProgressRoom{binding_digest:context.room_digest.clone(),revision:1}),&[15,16,17],Some(&"f".repeat(64)),rt.receiver_now_ns()); + assert_eq!(wrong_ready.unwrap_err(),"progress_context_changed"); + {state.write().await.privacy_mode=true;} + assert_eq!(request_raw(progress.clone(),"/progress",None).await.0,StatusCode::FORBIDDEN); + {let mut s=state.write().await;s.privacy_mode=false;s.installation_id=Some("foreign-installation".into());} + assert_eq!(request_raw(progress.clone(),"/progress",None).await.0,StatusCode::SERVICE_UNAVAILABLE); + {state.write().await.installation_id=Some(installation.into());} + let source_cache={let mut s=state.write().await;let old=s.room_learning_sources.clone();s.room_learning_sources.retain(|key,_|key.node_id!=17);old}; + let missing_source=decoded(progress.clone(),"/progress",None).await;assert_eq!(missing_source["context_match"],true);assert_eq!(missing_source["sources"]["ready"],2); + {state.write().await.room_learning_sources=source_cache;} + let (revoked,rx)=tokio::sync::watch::channel(false);let expiry=chrono::Utc::now().timestamp().max(0)as u64+8; + let grant=wifi_densepose_sensing_server::companion::Grant{device_id:"synthetic-paired-reader".into(),expires_at:expiry,revoked:rx}; + let guarded=Router::new().route("/progress",get(super::super::paired_progress::progress_dispatch)).with_state(state.clone()).layer(axum::Extension(grant)); + let authorized=decoded(guarded.clone(),"/progress",None).await;assert!(authorized["expires_at_unix_ms"].as_u64().unwrap()<=expiry*1000); + revoked.send(true).unwrap();assert_eq!(request_raw(guarded,"/progress",None).await.0,StatusCode::UNAUTHORIZED); + let mut revision=1; + for change in ["same_geometry_new_publication","dimensions","nodes","access_point"] { + let mut room=client["canonical_room"].clone(); + match change{"dimensions"=>room["widthM"]=serde_json::json!(2.6),"nodes"=>room["nodes"][0]["xM"]=serde_json::json!(0.5),"access_point"=>room["accessPoint"]=serde_json::json!({"slot":"north_center","xM":1.25,"yM":0.4,"provenance":"operator_entered","confirmedAtUnixMs":1}),_=>{}} + let changed=publish(serde_json::json!({"expected_revision":revision,"room":room})).await;revision+=1;assert_ne!(changed["binding_digest"],client["binding_digest"]); + let blocked=decoded(progress.clone(),"/progress",None).await;assert_eq!(blocked["context_match"],false,"{change}:{blocked}");assert_eq!(blocked["blocker"],"context_mismatch"); + for key in ["room","model_kind","profile","capture","candidate","evaluation","qualification","selection","sources"]{assert!(blocked[key].is_null(),"{change}:{key}");} + assert_eq!(rt.context().unwrap(),context);assert_eq!(rt.mutation_guard().unwrap(),before_guard); + } +} + +async fn cadence_priming_fixture(cold_restore:bool)->(tempfile::TempDir,SharedState,Arc,Router,serde_json::Value) { + use wifi_densepose_sensing_server::room_learner::recovery_store::RecoveryStore; + let (dir,state)=fixture();let owner="a".repeat(64);let path=dir.path().join(&owner).join("SensingServer"); + let store=RecoveryStore::new(&path,&owner,"synthetic-install",Some(&[7;32])).unwrap(); + let rt=Arc::new(RoomLearningRuntime::new_with_recovery(&path,&owner,"synthetic-install",store).unwrap()); + state.write().await.room_learning=Some(rt.clone());let app=temporal_app(state.clone()); + let configured=decoded(app.clone(),"/configure",Some(configure_v2(rt.mutation_guard().unwrap()))).await;assert_eq!(configured["success"],true,"{configured}"); + let mut before=serde_json::Value::Null;for _ in 0..100{let s=decoded(app.clone(),&format!("/status?schema={STATUS_SCHEMA}"),None).await;if s["status"]["capture"]["stage"]=="ready"{before=s;break;}tokio::time::sleep(std::time::Duration::from_millis(2)).await;}assert!(before.is_object()); + if !cold_restore{return(dir,state,rt,app,before);} + let context=rt.context().unwrap();let capture_id=before["status"]["capture"]["capture_generation_id"].as_str().unwrap().to_string(); + state.write().await.room_learning=None;drop(rt); + let store=RecoveryStore::new(&path,&owner,"synthetic-install",Some(&[7;32])).unwrap(); + let restored=Arc::new(RoomLearningRuntime::new_with_recovery(&path,&owner,"synthetic-install",store).unwrap()); + restored.temporal_restore_v2(&restored.mutation_guard().unwrap(),&context,&capture_id,restored.receiver_now_ns()).unwrap(); + state.write().await.room_learning=Some(restored.clone()); + let after=decoded(app.clone(),&format!("/status?schema={STATUS_SCHEMA}"),None).await;assert_eq!(after["status"]["capture"]["stage"],"paused","{after}"); + assert_eq!(after["status"]["capture"]["capture_generation_id"],before["status"]["capture"]["capture_generation_id"]);assert_eq!(after["status"]["capture"]["original_deadline"],before["status"]["capture"]["original_deadline"]); + (dir,state,restored,app,after) +} +async fn actual_header_observe(state:&SharedState,sequence:u32) -> f64 { + let packet=wire(15,sequence);let frame=parse_esp32_frame(&packet).unwrap();let mut s=state.write().await; + observe(&mut s,&frame,&packet,std::time::Instant::now()); + s.room_learning_sources.iter().find(|(k,_)|k.node_id==15).unwrap().1.receiver_rate_hz +} +async fn assert_nonobserving_cadence_does_not_fail(cold:bool) { + let (_dir,state,rt,app,before)=cadence_priming_fixture(cold).await; + let rate=actual_header_observe(&state,121).await;assert!(rate>0.&&rate<30.,"fixture must expose a real moving-arrival estimate below recording band: {rate}"); + let after=decoded(app,&format!("/status?schema={STATUS_SCHEMA}"),None).await; + println!("SYNTHETIC_ACTUAL_INGRESS_PRIMING_CADENCE cold={cold} measured_receiver_hz={rate} capture={}",after["status"]["capture"]); + assert_eq!(after["status"]["capture"]["stage"],if cold{"paused"}else{"ready"},"{after}"); + for key in ["capture_generation_id","original_deadline","schedule_sha256","retained_windows","durable_windows","sealed_train_episodes","sealed_held_out_episodes","observed_ms","durable_commitment_sha256","continuation_policy_sha256"]{assert_eq!(after["status"]["capture"][key],before["status"]["capture"][key],"{key}");} + assert_eq!(rt.context().unwrap().revision,1); +} +#[tokio::test] +async fn actual_ingress_ready_zero_window_rate_estimate_does_not_record_or_fail(){assert_nonobserving_cadence_does_not_fail(false).await;} +#[tokio::test] +async fn actual_ingress_cold_paused_zero_window_rate_estimate_does_not_record_or_fail(){assert_nonobserving_cadence_does_not_fail(true).await;} + +#[tokio::test] +async fn actual_ingress_low_cadence_lead_in_has_zero_credit_then_observing_rejects() { + let (_dir,state,rt,app,before)=cadence_priming_fixture(false).await; + let capture=&before["status"]["capture"]; + let start=decoded(app.clone(),"/phase/start",Some(serde_json::json!({"schema":PHASE_START_REQUEST_SCHEMA,"expected":rt.mutation_guard().unwrap(),"capture_generation_id":capture["capture_generation_id"],"schedule_sha256":capture["schedule_sha256"],"episode_index":0,"conditions_confirmed":true}))).await; + assert_eq!(start["success"],true,"{start}"); + let wait=std::time::Instant::now();let mut saw_lead=false;let mut failed=false; + for sequence in 121..221 { + {let mut s=state.write().await;for node in [15,16,17]{let packet=wire(node,sequence);let frame=parse_esp32_frame(&packet).unwrap();observe(&mut s,&frame,&packet,std::time::Instant::now());}} + let current=decoded(app.clone(),&format!("/status?schema={STATUS_SCHEMA}"),None).await; + assert_eq!(current["status"]["capture"]["retained_windows"],0);assert_eq!(current["status"]["capture"]["durable_windows"],0); + if current["status"]["capture"]["stage"]=="failed" { + assert!(saw_lead,"must first prove non-observing lead-in accepted without credit"); + assert_eq!(current["status"]["capture"]["failure"],"acquisition_cadence_changed","{current}");failed=true;break; + } + if current["status"]["capture"]["lead_in_remaining_ms"].as_u64().unwrap()>0 { + saw_lead=true;assert_eq!(current["status"]["capture"]["observed_ms"],0); + } + assert!(wait.elapsed()native::CanonicalPacket { + let bytes=wire(15,sequence);let frame=parse_esp32_frame(&bytes).unwrap(); + native::canonicalize_esp32(native::NativePacketInput{node_id:15,n_antennas:1,raw_bins:64,frequency_hz:2_437_000_000,ppdu_type:frame.ppdu_type,sequence,received_at_ns:rt.receiver_now_ns(),receiver_sample_rate_hz:10.,iq_bytes:&bytes[20..],flags:frame.adr018_flags,provenance:wifi_densepose_sensing_server::room_learner::observation::CaptureProvenance::LiveReceiverObserved}).unwrap() +} +#[tokio::test] +async fn fullrun_priming_still_rejects_replay_and_changed_phy_identity() { + {let (_dir,state,_rt,app,_)=cadence_priming_fixture(false).await;actual_header_observe(&state,121).await;actual_header_observe(&state,121).await; + let s=decoded(app,&format!("/status?schema={STATUS_SCHEMA}"),None).await;assert_eq!(s["status"]["capture"]["failure"],"temporal_priming_source_or_fence");assert_eq!(s["status"]["capture"]["retained_windows"],0);} + {let (_dir,_state,rt,app,_)=cadence_priming_fixture(false).await;let mut p=low_rate_native_packet(&rt,121);p.frequency_hz=2_462_000_000; + assert_eq!(rt.ingest_native(&p,rt.receiver_now_ns()),Err("acquisition_native_source_changed".into())); + let s=decoded(app,&format!("/status?schema={STATUS_SCHEMA}"),None).await;assert_eq!(s["status"]["capture"]["retained_windows"],0);} +} +#[tokio::test] +async fn fullrun_priming_still_rejects_zero_nan_and_nonfinite_rate_metadata() { + for rate in [0.,-1.,f64::NAN,f64::INFINITY] { + let (_dir,_state,rt,app,_)=cadence_priming_fixture(false).await; + let mut p=low_rate_native_packet(&rt,121);p.receiver_sample_rate_hz=rate; + assert_eq!(rt.ingest_native(&p,rt.receiver_now_ns()),Err("acquisition_cadence_changed".into())); + let s=decoded(app,&format!("/status?schema={STATUS_SCHEMA}"),None).await;assert_eq!(s["status"]["capture"]["retained_windows"],0);assert_eq!(s["status"]["capture"]["observed_ms"],0); + } +} + +#[tokio::test] +async fn retry_owner_router_denies_anonymous_and_paired_grants_before_mutating_runtime(){ + let (_dir,state)=fixture();let rt=state.read().await.room_learning.as_ref().unwrap().clone(); + let auth=wifi_densepose_sensing_server::bearer_auth::AuthState::from_token("synthetic-owner"); + let app=owner_selection_routes(auth).with_state(state); + let guard=rt.mutation_guard().unwrap(); + let body=serde_json::json!({"schema":RECOVERY_RETRY_REQUEST_SCHEMA,"model_kind":"ruview.room-learner.managed-temporal-input.v3","expected":guard,"capture_generation_id":"c".repeat(64),"retry_confirmed":true}); + for (token,paired) in [(None,false),(Some("synthetic-peer-read"),false),(Some("synthetic-owner"),true)] { + let mut builder=Request::builder().method(Method::POST).uri("/api/v1/room-learner/recovery/retry").header("content-type","application/json"); + if let Some(token)=token{builder=builder.header("authorization",format!("Bearer {token}"));} + let mut request=builder.body(Body::from(serde_json::to_vec(&body).unwrap())).unwrap(); + request.extensions_mut().insert(axum::extract::ConnectInfo("127.0.0.1:41000".parse::().unwrap())); + if paired {let(_,rx)=tokio::sync::watch::channel(false);request.extensions_mut().insert(wifi_densepose_sensing_server::companion::Grant{device_id:"synthetic-paired-readonly".into(),expires_at:chrono::Utc::now().timestamp().max(0)as u64+30,revoked:rx});} + assert_eq!(app.clone().oneshot(request).await.unwrap().status(),StatusCode::FORBIDDEN); + assert_eq!(rt.mutation_guard().unwrap(),guard); + } +} + +#[tokio::test] +async fn retry_command_refuses_imported_rows_and_wrong_cas_at_the_actual_dispatch(){ + let (_dir,state)=fixture();let rt=state.read().await.room_learning.as_ref().unwrap().clone();let app=temporal_app(state); + let guard=rt.mutation_guard().unwrap(); + let body=serde_json::json!({"schema":RECOVERY_RETRY_REQUEST_SCHEMA,"model_kind":"ruview.room-learner.managed-temporal-input.v3","expected":guard,"capture_generation_id":"c".repeat(64),"retry_confirmed":true}); + for field in ["rows","recording_start","new_training","deadline","committed"]{ + let mut invalid=body.clone();invalid[field]=serde_json::json!(true); + assert_eq!(request_raw(app.clone(),"/retry",Some(invalid)).await.0,StatusCode::UNPROCESSABLE_ENTITY); + assert_eq!(rt.mutation_guard().unwrap(),guard); + } + let mut wrong=body;wrong["expected"]["runtime_epoch"]=serde_json::json!("wrong-runtime"); + let response=decoded(app,"/retry",Some(wrong)).await; + assert_eq!(response["success"],false);assert_eq!(rt.mutation_guard().unwrap(),guard); +} + +#[tokio::test] +async fn actual_owner_retry_http_returns_paused_same_capture_without_starting_recording(){ + use wifi_densepose_sensing_server::room_learner::{observation::SourceSpec,recovery_store::RecoveryStore}; + let(dir,state)=fixture();let owner="a".repeat(64);let path=dir.path().join(&owner).join("SensingServer"); + let store=RecoveryStore::new(&path,&owner,"synthetic-install",Some(&[7;32])).unwrap(); + let rt=Arc::new(RoomLearningRuntime::new_with_recovery(&path,&owner,"synthetic-install",store).unwrap()); + let profile="native-2s-three-source-56bin-operator-eight-step-v1"; + rt.temporal_select_profile_guarded_v2(Some(profile),&rt.mutation_guard().unwrap()).unwrap(); + let sources=(15..=17u8).map(|id|SourceSpec{source_id:id,device_id:format!("esp32-node-{id}"),grid_id:"ant1-bins64-ppdu0-usable-drop2-linear-v2".into(),feature_count:56,signal_kind:ruview_unified::RfModality::WifiCsi,center_frequency_hz:2_437_000_000,bandwidth_hz:20_000_000,link_id:format!("synthetic-{id}"),link_index:0,geometry:None}).collect(); + let context=rt.bind_context("b".repeat(64),1,sources).unwrap(); + rt.temporal_configure_v2(&rt.mutation_guard().unwrap(),context,"synthetic-http-retry".into(),true,rt.receiver_now_ns()).unwrap(); + let started=std::time::Instant::now(); + loop{let status=rt.temporal_status_v2(rt.receiver_now_ns()).unwrap();if status["capture"]["stage"]=="ready"{break;}assert!(started.elapsed()().unwrap())); + let response=router.oneshot(request).await.unwrap();assert_eq!(response.status(),StatusCode::OK); + let bytes=axum::body::to_bytes(response.into_body(),65536).await.unwrap();let reply:serde_json::Value=serde_json::from_slice(&bytes).unwrap(); + assert_eq!(reply["success"],true,"{reply}");let after=&reply["status"]["capture"]; + assert_eq!(after["stage"],"paused");assert_eq!(after["durability_pending"],false);assert!(after["failure"].is_null()); + assert_eq!(after["capture_generation_id"],before["capture"]["capture_generation_id"]);assert_eq!(after["original_deadline"],before["capture"]["original_deadline"]); + assert_eq!(after["retained_windows"],0);assert_eq!(after["durable_windows"],0);assert_eq!(after["observed_ms"],0); +} + +/// SYNTHETIC real recovery file-lock contention through the production status +/// composer. The handshake runs inside the actual blocking reader; there is no +/// timing sleep or production Runtime hook standing in for contention. +#[cfg(unix)] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn temporal_status_recovery_wait_releases_ingress_and_rechecks_authority() { + use std::os::unix::fs::{OpenOptionsExt, PermissionsExt}; + use wifi_densepose_sensing_server::room_learner::recovery_store::RecoveryStore; + for change in ["none", "privacy", "runtime", "profile", "epoch"] { + let (dir, state) = fixture(); + let owner = "a".repeat(64); + let path = dir.path().join(&owner).join("SensingServer"); + let recovery = RecoveryStore::new(&path, &owner, "synthetic-install", Some(&[7; 32])).unwrap(); + let rt = Arc::new(RoomLearningRuntime::new_with_recovery(&path, &owner, "synthetic-install", recovery).unwrap()); + state.write().await.room_learning = Some(rt.clone()); + let recovery_path = path.join("room-learner-recovery"); + let prefix_path = recovery_path.join("temporal-prefix-commitment-v1"); + std::fs::create_dir(&prefix_path).unwrap(); + std::fs::set_permissions(&prefix_path, std::fs::Permissions::from_mode(0o700)).unwrap(); + let held = std::fs::OpenOptions::new().read(true).write(true).create(true).truncate(false) + .mode(0o600).open(recovery_path.join("temporal-prefix.lock")).unwrap(); + held.lock().unwrap(); + let (entered_tx, entered_rx) = tokio::sync::oneshot::channel(); + let reader_state = state.clone(); + let reader_runtime = rt.clone(); + let task = tokio::spawn(async move { + temporal_status_with_reader(&reader_state, &reader_runtime, move |reader, now| { + entered_tx.send(()).unwrap(); + reader.temporal_status_snapshot_v2(now) + }).await.0 + }); + tokio::time::timeout(std::time::Duration::from_secs(2), entered_rx).await.unwrap().unwrap(); + // This is exactly the shared write acquisition the live UDP path needs. + // Before the repair it could not complete until held was unlocked. + let mut ingress = tokio::time::timeout(std::time::Duration::from_secs(2), state.write()).await + .expect("recovery file I/O must not hold the RF ingress lock"); + assert!(!task.is_finished(), "the real recovery reader must still be blocked"); + match change { + "none" => {}, + "privacy" => ingress.privacy_mode = true, + "runtime" => ingress.room_learning = None, + "profile" => { rt.temporal_select_profile_guarded_v2(Some("native-2s-three-source-56bin-operator-eight-step-v1"), &rt.mutation_guard().unwrap()).unwrap(); }, + "epoch" => ingress.calibration_boot_epoch.push_str("-changed"), + _ => unreachable!(), + } + drop(ingress); + held.unlock().unwrap(); + let result = tokio::time::timeout(std::time::Duration::from_secs(2), task).await.unwrap().unwrap(); + if change == "none" { + assert_eq!(result["success"], true, "{result}"); + assert_eq!(result["status"]["mutation_guard"], serde_json::to_value(rt.mutation_guard().unwrap()).unwrap()); + } else { + assert_eq!(result["success"], false, "{change}: {result}"); + assert!(result.get("status").is_none(), "{change}: stale snapshot must not escape"); + assert_eq!(result["error_code"], if change == "privacy" {"learner_privacy_mode"} else {"learner_state_changed"}); + } + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn cancelled_status_retains_one_reader_permit_until_actual_work_finishes() { + let (_dir, state) = fixture(); + let rt = state.read().await.room_learning.clone().unwrap(); + let (entered_tx, entered_rx) = tokio::sync::oneshot::channel(); + let (release_tx, release_rx) = std::sync::mpsc::channel(); + let blocked_state = state.clone(); + let blocked_rt = rt.clone(); + let blocked = tokio::spawn(async move { + temporal_status_with_reader(&blocked_state, &blocked_rt, move |reader, now| { + entered_tx.send(()).unwrap(); + release_rx.recv().unwrap(); + reader.temporal_status_snapshot_v2(now) + }).await + }); + tokio::time::timeout(std::time::Duration::from_secs(2), entered_rx).await.unwrap().unwrap(); + blocked.abort(); + assert!(blocked.await.unwrap_err().is_cancelled()); + // Poll a real waiter once: the aborted HTTP task must not have released + // the permit still owned by its actual blocking reader. + let mut direct_wait = Box::pin(rt.acquire_temporal_status_reader()); + std::future::poll_fn(|cx| { + assert!(std::future::Future::poll(direct_wait.as_mut(), cx).is_pending()); + std::task::Poll::Ready(()) + }).await; + drop(direct_wait); + let invoked = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let mut waiters = Vec::new(); + for _ in 0..3 { + let (started_tx, started_rx) = tokio::sync::oneshot::channel(); + let waiter_state = state.clone(); + let waiter_rt = rt.clone(); + let calls = invoked.clone(); + waiters.push(tokio::spawn(async move { + started_tx.send(()).unwrap(); + temporal_status_with_reader(&waiter_state, &waiter_rt, move |reader, now| { + calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + reader.temporal_status_snapshot_v2(now) + }).await + })); + tokio::time::timeout(std::time::Duration::from_secs(2), started_rx).await.unwrap().unwrap(); + } + assert_eq!(invoked.load(std::sync::atomic::Ordering::SeqCst), 0, + "concurrent async waiters must not spawn more blocking readers"); + for waiter in waiters { + assert!(!waiter.is_finished()); + waiter.abort(); + assert!(waiter.await.unwrap_err().is_cancelled()); + } + // Cancellation did not retain the RF ingress lock either. + drop(tokio::time::timeout(std::time::Duration::from_secs(2), state.write()).await.unwrap()); + release_tx.send(()).unwrap(); + let permit = tokio::time::timeout(std::time::Duration::from_secs(2), rt.acquire_temporal_status_reader()).await + .expect("permit must return only after real blocking work exits").unwrap(); + drop(permit); + assert_eq!(invoked.load(std::sync::atomic::Ordering::SeqCst), 0, + "explicitly canceled async waiters must never run later"); + let response = temporal_status_with_reader(&state, &rt, |reader, now| reader.temporal_status_snapshot_v2(now)).await.0; + assert_eq!(response["success"], true, "{response}"); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_tests.rs new file mode 100644 index 0000000000..75d63b0a7a --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_tests.rs @@ -0,0 +1,561 @@ +//! @tier durable — SYNTHETIC datagrams through real configure/parser/observe. +//! Receiver-observed fixtures are not physical hardware or accuracy evidence. +use super::*; +use axum::{ + body::Body, + http::{Method, Request}, +}; +use tower::ServiceExt; +fn wire(node: u8, seq: u32) -> Vec { + let mut bytes = Vec::new(); + bytes.extend(0xC5110001u32.to_le_bytes()); + bytes.extend([node, 1]); + bytes.extend(64u16.to_le_bytes()); + bytes.extend(2437u32.to_le_bytes()); + bytes.extend(seq.to_le_bytes()); + bytes.extend([(-40i8) as u8, (-90i8) as u8, 0, 0x20]); + for _ in 0..64 { + bytes.extend([3, 4]); + } + bytes +} +fn fixture() -> (tempfile::TempDir, SharedState) { + let dir = tempfile::tempdir().unwrap(); + let owner = "a".repeat(64); + let runtime = Arc::new( + RoomLearningRuntime::new( + &dir.path().join(&owner).join("SensingServer"), + &owner, + "synthetic-install", + ) + .unwrap(), + ); + let mut inner = AppStateInner::minimal(); + inner.room_learning = Some(runtime); + let now = std::time::Instant::now(); + for row in 0..120 { + for node in [15, 16, 17] { + let packet = wire(node, row + 1); + let frame = parse_esp32_frame(&packet).unwrap(); + let at = now - std::time::Duration::from_millis((120 - row) as u64 * 100); + inner + .node_states + .entry(node) + .or_insert_with(NodeState::new) + .observe_raw_grid(CsiGridKey::from_frame(&frame), at); + observe(&mut inner, &frame, &packet, at); + } + } + (dir, Arc::new(RwLock::new(inner))) +} +fn app(state: SharedState) -> Router { + Router::new() + .route("/configure", post(configure)) + .route("/status", get(status)) + .route("/phase", post(begin_phase)) + .route("/finish", post(finish_phase)) + .route("/enable-current", post(enable_current_recovery)) + .with_state(state) +} +async fn json_request( + app: Router, + path: &str, + method: Method, + mut body: serde_json::Value, +) -> serde_json::Value { + if method == Method::POST && body.get("expected").is_none() { + let current = json_request_status(app.clone()).await; + if path == "/phase" { + body = serde_json::json!({"phase":body}); + } + body["expected"] = if current["mutation_guard"].is_object() { + current["mutation_guard"].clone() + } else { + serde_json::json!({"runtime_epoch":"synthetic-disabled","generation":0,"workflow_revision":0,"context_digest":null,"phase":null}) + }; + } + let response = app + .oneshot( + Request::builder() + .method(method) + .uri(path) + .header("content-type", "application/json") + .body(Body::from(serde_json::to_vec(&body).unwrap())) + .unwrap(), + ) + .await + .unwrap(); + let bytes = axum::body::to_bytes(response.into_body(), 65536) + .await + .unwrap(); + serde_json::from_slice(&bytes).unwrap() +} +async fn json_request_status(app: Router) -> serde_json::Value { + let response = app + .oneshot( + Request::builder() + .uri("/status") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + let bytes = axum::body::to_bytes(response.into_body(), 65536) + .await + .unwrap(); + serde_json::from_slice(&bytes).unwrap() +} +fn config() -> serde_json::Value { + serde_json::json!({"binding_digest":"b".repeat(64),"revision":1,"source_node_ids":[15,16,17],"new_training":true}) +} +#[tokio::test] +async fn configure_and_actual_native_ingress_share_grid_projection_and_device_identity() { + let (_dir, state) = fixture(); + let router = app(state.clone()); + let configured = json_request(router.clone(), "/configure", Method::POST, config()).await; + assert_eq!(configured["success"], true, "{configured}"); + let sources = configured["status"]["sources"].as_array().unwrap(); + assert_eq!(sources.len(), 3); + assert_eq!(sources[0]["device_id"], "esp32-node-15"); + assert_eq!( + sources[0]["grid_id"], + "ant1-bins64-ppdu0-usable-drop2-linear-v2" + ); + let begun=json_request(router.clone(),"/phase",Method::POST,serde_json::json!({ + "trial_id":"synthetic-empty","session_id":"synthetic-train-session","role":"Train","label":"Empty", + "duration_ms":12000,"conditions_confirmed":true})).await; + assert_eq!(begun["success"], true, "{begun}"); + for row in 0..8 { + for node in [15, 16, 17] { + let packet = wire(node, 121 + row); + let frame = parse_esp32_frame(&packet).unwrap(); + let mut inner = state.write().await; + observe(&mut inner, &frame, &packet, std::time::Instant::now()); + } + } + let read = json_request( + router.clone(), + "/status", + Method::GET, + serde_json::Value::Null, + ) + .await; + assert_eq!(read["required_source_ids"], serde_json::json!([15, 16, 17])); + assert_eq!(read["trained_source_ids"], serde_json::json!([])); + assert_eq!(read["scored_source_ids"], serde_json::json!([])); + for source in read["capture"]["sources"].as_array().unwrap() { + assert_eq!(source["admitted_packets"], 8, "{source}"); + assert_eq!(source["live_buffer_frames"], 8); + } + assert_eq!(read["physically_validated"], false); + assert!( + state.read().await.field_model.is_none(), + "new learner must not silently create/change control calibration" + ); + let incomplete = json_request( + router, + "/finish", + Method::POST, + serde_json::json!({"conditions_held":true}), + ) + .await; + assert_eq!( + incomplete["success"], false, + "wall-clock duration cannot be bypassed by fast synthetic packets" + ); +} +#[tokio::test] +async fn privacy_disabled_source_drop_and_unconfirmed_conditions_are_explicit() { + let empty = Arc::new(RwLock::new(AppStateInner::minimal())); + let disabled = json_request(app(empty), "/status", Method::GET, serde_json::Value::Null).await; + assert_eq!(disabled["error_code"], "learner_disabled"); + let (_dir, state) = fixture(); + state.write().await.privacy_mode = true; + let hidden = json_request(app(state.clone()), "/configure", Method::POST, config()).await; + assert_eq!(hidden["error_code"], "learner_privacy_mode"); + assert!(!hidden.to_string().contains("owner_scope")); + state.write().await.privacy_mode = false; + state.write().await.node_states.remove(&17); + let missing = json_request(app(state.clone()), "/configure", Method::POST, config()).await; + assert_eq!(missing["error_code"], "learner_source_missing"); + assert!(state + .read() + .await + .room_learning + .as_ref() + .unwrap() + .context() + .is_none()); +} +#[test] +fn every_new_mutation_requires_admin_and_default_cli_does_not_enable_capture() { + use wifi_densepose_sensing_server::bearer_auth::required_scope_for; + for suffix in [ + "configure", + "phase/start", + "phase/finish", + "phase/cancel", + "train", + "train/cancel", + "evaluate", + "recovery/restore", + "recovery/retry", + "recovery/discard", + "recovery/enable-current", + ] { + assert_eq!( + required_scope_for(&Method::POST, &format!("/api/v1/room-learner/{suffix}")), + "sensing:admin" + ); + } + assert_eq!( + required_scope_for(&Method::GET, "/api/v1/room-learner/status"), + "sensing:read" + ); + assert!( + !Args::try_parse_from(["sensing-server"]) + .unwrap() + .room_learner + ); +} + +#[tokio::test] +async fn learner_examples_and_legacy_empty_capture_cannot_contaminate_each_other() { + let (_dir, state) = fixture(); + let router = app(state.clone()); + assert_eq!( + json_request(router.clone(), "/configure", Method::POST, config()).await["success"], + true + ); + { + let mut s = state.write().await; + s.field_model = Some(FieldModel::new(field_bridge::single_link_config()).unwrap()); + s.calibration_session_id = Some("cal-session-synthetic".into()); + } + let request = serde_json::json!({"trial_id":"occupied","session_id":"train","role":"Train", + "label":"OccupiedStatic","duration_ms":12000,"lead_in_ms":5000,"conditions_confirmed":true}); + let blocked = json_request(router.clone(), "/phase", Method::POST, request.clone()).await; + assert_eq!(blocked["error_code"], "legacy_empty_capture_active"); + { + let mut s = state.write().await; + s.field_model = None; + s.calibration_session_id = None; + } + assert_eq!( + json_request(router, "/phase", Method::POST, request).await["success"], + true + ); + let Json(blocked) = calibration_start( + State(state.clone()), + Query(CalibrationStartQuery { + source_node_id: None, + }), + None, + None, + Json(CalibrationBindingRequest { + binding_digest: "b".repeat(64), + source_node_ids: vec![15, 16, 17], + }), + ) + .await; + assert_eq!(blocked["error_code"], "learner_capture_active"); + assert!(state.read().await.field_model.is_none()); +} + +#[tokio::test] +async fn stale_http_start_is_rejected_after_another_client_changes_room() { + let (_dir, state) = fixture(); + let router = app(state); + let a = json_request(router.clone(), "/configure", Method::POST, config()).await; + assert_eq!(a["success"], true); + let guard_a = a["status"]["mutation_guard"].clone(); + let mut desired_b = config(); + desired_b["binding_digest"] = serde_json::json!("c".repeat(64)); + desired_b["expected"] = guard_a.clone(); + let b = json_request(router.clone(), "/configure", Method::POST, desired_b).await; + assert_eq!(b["success"], true, "{b}"); + let delayed = serde_json::json!({"expected":guard_a,"phase":{ + "trial_id":"delayed-room-a","session_id":"room-a","role":"Train","label":"Empty", + "duration_ms":12000,"conditions_confirmed":true}}); + let rejected = json_request(router.clone(), "/phase", Method::POST, delayed).await; + assert_eq!( + rejected["error_code"], "learner_state_changed", + "{rejected}" + ); + let current = json_request_status(router).await; + assert_eq!(current["mutation_guard"], b["status"]["mutation_guard"]); + assert!(current["capture"]["phase"].is_null()); +} + +#[tokio::test] +async fn native_adapter_rejection_is_visible_without_admitting_or_labelling_packets() { + let (_dir, state) = fixture(); + let router = app(state.clone()); + let configured = json_request(router.clone(), "/configure", Method::POST, config()).await; + assert_eq!(configured["success"], true); + let mut packet = wire(15, 121); + packet[19] |= 0x01; // Actual ADR-018 40 MHz bit. + let frame = parse_esp32_frame(&packet).unwrap(); + observe( + &mut *state.write().await, + &frame, + &packet, + std::time::Instant::now(), + ); + let status = json_request_status(router).await; + assert_eq!(status["input_error"],"Node 15 native input rejected: native capture does not support the reported 40 MHz bandwidth"); + assert_eq!(status["capture"]["sources"][0]["admitted_packets"], 0); + assert_eq!( + status["mutation_guard"], + configured["status"]["mutation_guard"] + ); + assert_eq!(status["physically_validated"], false); +} + +#[tokio::test] +async fn minority_supported_bandwidth_cannot_borrow_wide_grid_eligibility() { + let (_dir, state) = fixture(); + { + let mut inner = state.write().await; + inner.node_states.clear(); + inner.room_learning_sources.clear(); + let now = std::time::Instant::now(); + for row in 0..120 { + for node in [15, 16, 17] { + let mut packet = wire(node, row + 1); + if row < 119 { + packet[19] |= 0x01; + } + let frame = parse_esp32_frame(&packet).unwrap(); + let at = now - std::time::Duration::from_millis((120 - row) as u64 * 100); + inner + .node_states + .entry(node) + .or_insert_with(NodeState::new) + .observe_raw_grid(CsiGridKey::from_frame(&frame), at); + observe(&mut inner, &frame, &packet, at); + } + } + } + let result = json_request(app(state), "/configure", Method::POST, config()).await; + assert_eq!( + result["success"], false, + "one 20MHz packet must not inherit sustained 40MHz cadence: {result}" + ); + assert_eq!(result["error_code"], "learner_source_unready"); +} + +#[tokio::test] +async fn rare_foreign_frequency_neither_relabels_selection_nor_invalidates_a_phase() { + let (_dir, state) = fixture(); + let mut foreign = wire(15, 121); + foreign[8..12].copy_from_slice(&2462u32.to_le_bytes()); + let frame = parse_esp32_frame(&foreign).unwrap(); + observe( + &mut *state.write().await, + &frame, + &foreign, + std::time::Instant::now(), + ); + let router = app(state.clone()); + let configured = json_request(router.clone(), "/configure", Method::POST, config()).await; + assert_eq!(configured["success"], true, "{configured}"); + assert_eq!( + configured["status"]["sources"][0]["center_frequency_hz"], + 2_437_000_000u64 + ); + let begun = json_request( + router.clone(), + "/phase", + Method::POST, + serde_json::json!({ + "trial_id":"synthetic-phy","session_id":"synthetic-phy","role":"Train","label":"Empty", + "duration_ms":12000,"conditions_confirmed":true}), + ) + .await; + assert_eq!(begun["success"], true, "{begun}"); + observe( + &mut *state.write().await, + &frame, + &foreign, + std::time::Instant::now(), + ); + let packet = wire(15, 122); + let frame = parse_esp32_frame(&packet).unwrap(); + observe( + &mut *state.write().await, + &frame, + &packet, + std::time::Instant::now(), + ); + let status = json_request_status(router).await; + assert_eq!(status["capture"]["phase"]["trial_id"], "synthetic-phy"); + assert_eq!(status["capture"]["sources"][0]["admitted_packets"], 1); + assert_eq!(status["mutation_guard"], begun["status"]["mutation_guard"]); +} + +// SYNTHETIC cache lifecycle regressions through the real parser and ingress. +fn observe_wire_at(inner: &mut AppStateInner, packet: &[u8], at: std::time::Instant) { + let frame = parse_esp32_frame(packet).expect("complete synthetic packet"); + inner + .node_states + .entry(frame.node_id) + .or_insert_with(NodeState::new) + .observe_raw_grid(CsiGridKey::from_frame(&frame), at); + observe(inner, &frame, packet, at); +} +#[tokio::test] +async fn unsupported_class_churn_cannot_exhaust_readiness_for_healthy_nodes() { + let (_dir, state) = fixture(); + let now = std::time::Instant::now(); + { + let mut inner = state.write().await; + inner.room_learning_sources.clear(); + for bins in 1u16..=1024 { + let mut packet = wire(200, 1); + packet[6..8].copy_from_slice(&bins.to_le_bytes()); + packet.resize(20 + usize::from(bins) * 2, 3); + observe_wire_at(&mut inner, &packet, now); + } + for frequency in 1u32..=1024 { + let mut packet = wire(200, 1); + packet[8..12].copy_from_slice(&frequency.to_le_bytes()); + observe_wire_at(&mut inner, &packet, now); + } + for row in 0..120 { + for node in [30, 31] { + let at = now - std::time::Duration::from_millis((120 - row) as u64 * 100); + observe_wire_at(&mut inner, &wire(node, row + 1), at); + } + } + } + let mut request = config(); + request["source_node_ids"] = serde_json::json!([30, 31]); + let response = json_request(app(state.clone()), "/configure", Method::POST, request).await; + assert_eq!(response["success"], true, "{response}"); + // Only the supported 64-bin class at 2437 MHz plus the two healthy nodes. + assert_eq!(state.read().await.room_learning_sources.len(), 3); +} +#[tokio::test] +async fn expired_supported_classes_are_reclaimed_without_manufacturing_readiness() { + let (_dir, state) = fixture(); + let now = std::time::Instant::now(); + let mut inner = state.write().await; + inner.room_learning_sources.clear(); + let old = now - CALIBRATION_GRID_EVIDENCE_WINDOW - std::time::Duration::from_secs(1); + // Supported one-antenna HT20 classes, all unique by frequency. + for frequency in 4900u32..5924 { + let mut packet = wire(200, 1); + packet[8..12].copy_from_slice(&frequency.to_le_bytes()); + // 5901..5924 falls outside the supported range; use a different node + // at the beginning of the same supported band for those slots. + if frequency > 5900 { + packet[4] = 201; + packet[8..12].copy_from_slice(&(frequency - 1001).to_le_bytes()); + } + observe_wire_at(&mut inner, &packet, old); + } + assert_eq!(inner.room_learning_sources.len(), 1024); + observe_wire_at(&mut inner, &wire(30, 1), now); + assert!(inner.room_learning_sources.len() <= 1024); + assert!( + native_candidates(&inner, 30, now).is_empty(), + "one fresh packet is not readiness" + ); + for row in 1..120 { + observe_wire_at( + &mut inner, + &wire(30, row + 1), + now + std::time::Duration::from_millis(row as u64 * 100), + ); + } + let later = now + std::time::Duration::from_millis(11900); + assert_eq!( + native_candidates(&inner, 30, later).len(), + 1, + "expired unrelated classes must not permanently block a new supported stream" + ); + assert_eq!(inner.room_learning_sources.len(), 1); +} +#[tokio::test] +async fn cache_pressure_preserves_selected_active_class_and_freshness() { + let (_dir, state) = fixture(); + let configured = json_request(app(state.clone()), "/configure", Method::POST, config()).await; + assert_eq!(configured["success"], true); + let now = std::time::Instant::now(); + let mut inner = state.write().await; + // Fill the remaining capacity with unique supported classes. Existing + // selected classes must retain their measured 120-packet readiness. + for frequency in 4900u32..5921 { + let mut packet = wire(200, 1); + packet[8..12].copy_from_slice(&frequency.to_le_bytes()); + if frequency > 5900 { + packet[4] = 201; + packet[8..12].copy_from_slice(&(frequency - 1001).to_le_bytes()); + } + observe_wire_at(&mut inner, &packet, now); + } + assert_eq!(inner.room_learning_sources.len(), 1024); + observe_wire_at(&mut inner, &wire(30, 1), now); + assert_eq!( + inner.room_learning_sources.len(), + 1024, + "fresh cache stays bounded" + ); + assert!(native_candidates(&inner, 30, now).is_empty()); + for node in [15, 16, 17] { + assert_eq!( + native_candidates(&inner, node, now).len(), + 1, + "pressure must not evict a selected ready source" + ); + } + let later = now + CALIBRATION_GRID_EVIDENCE_WINDOW + std::time::Duration::from_secs(1); + observe_wire_at(&mut inner, &wire(30, 2), later); + assert_eq!( + inner.room_learning_sources.len(), + 4, + "reclaim only unrelated expired classes" + ); + for node in [15, 16, 17] { + assert!( + native_candidates(&inner, node, later).is_empty(), + "retaining a selected class must not preserve expired readiness" + ); + } + assert!(native_candidates(&inner, 30, later).is_empty()); + let status = inner + .room_learning + .as_ref() + .unwrap() + .status(now_ns(inner.room_learning.as_ref().unwrap())); + assert_eq!( + status["mutation_guard"], + configured["status"]["mutation_guard"] + ); +} + +#[cfg(unix)] +#[path = "room_learner_api_recovery_tests.rs"] +mod recovery_tests; + +#[path = "room_learner_api_selection_tests.rs"] +mod selection_tests; + +#[path = "room_learner_api_selection_composer_tests.rs"] +mod selection_composer_tests; + +#[tokio::test] +async fn enable_current_recovery_requires_explicit_versioned_consent_and_does_not_configure() { + let (_dir,state)=fixture(); + let rt=state.read().await.room_learning.clone().unwrap(); + for body in [serde_json::json!({"schema":"wrong","run_id":"old","consent":true}),serde_json::json!({"schema":"ruview.room-learner.enable-current-recovery.v1","run_id":"old","consent":false})] { + let result=json_request(app(state.clone()),"/enable-current",Method::POST,body).await; + assert_eq!(result["success"],false); + assert_eq!(result["error_code"],"learner_recovery_consent_required"); + assert!(rt.status(rt.receiver_now_ns())["context_digest"].is_null()); + } +} + +#[path = "room_learner_api_temporal_tests.rs"] +mod temporal_contract_tests; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner_ingress.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner_ingress.rs new file mode 100644 index 0000000000..1796b722c5 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner_ingress.rs @@ -0,0 +1,280 @@ +//! Header-only readiness is keyed to the exact PHY class admitted by the learner. +use super::*; +use std::collections::VecDeque; + +/// Receiver-only pipeline coordinates. Deliberately not Serialize and never an admission clock. +#[derive(Clone, Copy)] +pub(crate) struct ReceiverIngressTiming { + dequeued_at: std::time::Instant, + parse_us: Option, + lock_us: Option, +} +fn diagnostic_micros(duration: Option) -> Option { + duration.and_then(|d| d.as_micros().try_into().ok()) +} +impl ReceiverIngressTiming { + pub(crate) fn new(dequeued_at: std::time::Instant, parse: Option, lock_wait: Option) -> Self { + Self { dequeued_at, parse_us: diagnostic_micros(parse), lock_us: diagnostic_micros(lock_wait) } + } +} + +#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq)] +pub(crate) struct NativeSourceKey { + pub(super) node_id: u8, + pub(super) n_antennas: u8, + pub(super) raw_bins: u16, + pub(super) ppdu_type: u8, + pub(super) frequency_mhz: u16, + pub(super) bw40: bool, +} +impl NativeSourceKey { + fn from_frame(frame: &Esp32Frame) -> Self { + Self { + node_id: frame.node_id, + n_antennas: frame.n_antennas, + raw_bins: frame.n_subcarriers, + ppdu_type: frame.ppdu_type.to_byte(), + frequency_mhz: frame.freq_mhz, + bw40: frame.adr018_flags.bw40, + } + } +} +#[derive(Debug, Clone)] +pub(crate) struct LiveSource { + pub(super) frequency_hz: u64, + pub(super) bandwidth_hz: Option, + pub(super) receiver_rate_hz: f64, + arrivals: VecDeque, + // Separate from arrivals/readiness/model input; one relative diagnostic predecessor. + last_dequeued: Option, +} +impl LiveSource { + fn diagnostic_dequeue_interval(&mut self, at: std::time::Instant) -> Option { + let previous = self.last_dequeued.replace(at); + diagnostic_micros(previous.and_then(|before| at.checked_duration_since(before))) + } + fn record(&mut self, now: std::time::Instant) { + while self + .arrivals + .front() + .is_some_and(|t| now.saturating_duration_since(*t) > CALIBRATION_GRID_EVIDENCE_WINDOW) + { + self.arrivals.pop_front(); + } + if self.arrivals.len() == CALIBRATION_GRID_EVIDENCE_CAPACITY { + self.arrivals.pop_front(); + } + if self.arrivals.back().is_none_or(|t| now > *t) { + self.arrivals.push_back(now); + } + self.receiver_rate_hz = self.evidence(now).map_or(0.0, |e| e.rate_hz); + } + fn evidence(&self, now: std::time::Instant) -> Option { + let arrivals: Vec<_> = self + .arrivals + .iter() + .copied() + .filter(|t| now.saturating_duration_since(*t) <= CALIBRATION_GRID_EVIDENCE_WINDOW) + .collect(); + CalibrationGridEvidence::from_arrivals(&arrivals, now) + } + fn eligible(&self, now: std::time::Instant) -> bool { + self.bandwidth_hz == Some(20_000_000) + && native::is_supported_frequency_hz(self.frequency_hz) + && self.evidence(now).is_some_and(|e| { + e.span_s >= CALIBRATION_GRID_MIN_SPAN_S + && e.rate_hz >= CALIBRATION_GRID_MIN_RATE_HZ + && e.max_gap_s < CALIBRATION_GRID_MAX_GAP_S + && e.latest_age_s < 2.0 + }) + && self + .arrivals + .iter() + .rev() + .nth(wifi_densepose_sensing_server::room_learner::capture::WINDOW_PACKETS - 1) + .is_some_and(|t| { + now.saturating_duration_since(*t).as_nanos() + <= u128::from( + wifi_densepose_sensing_server::room_learner::observation::MAX_AGE_NS, + ) + }) + } +} +pub(super) fn native_candidates( + state: &AppStateInner, + node: u8, + now: std::time::Instant, +) -> Vec<(&NativeSourceKey, &LiveSource)> { + state + .room_learning_sources + .iter() + .filter(|(key, meta)| { + key.node_id == node + && meta.eligible(now) + && native::grid_id( + key.n_antennas as usize, + key.raw_bins as usize, + wifi_densepose_hardware::PpduType::from_byte(key.ppdu_type), + ) + .is_ok() + }) + .collect() +} +pub(crate) fn observe( + state: &mut AppStateInner, + frame: &Esp32Frame, + packet: &[u8], + now: std::time::Instant, +) { + observe_with_receiver_timing(state, frame, packet, now, None); +} +pub(crate) fn observe_with_receiver_timing( + state: &mut AppStateInner, + frame: &Esp32Frame, + packet: &[u8], + now: std::time::Instant, + receiver_timing: Option, +) { + let Some(runtime) = state.room_learning.clone() else { + return; + }; + // Unsupported input must never consume the bounded readiness cache. Keep + // the selected-source rejection diagnostic without admitting any evidence. + let Ok(id) = native::grid_id( + frame.n_antennas as usize, + frame.n_subcarriers as usize, + frame.ppdu_type, + ) else { + return; + }; + let frequency_hz = u64::from(frame.freq_mhz) * 1_000_000; + let rejection = if frame.adr018_flags.bw40 { + Some("native capture does not support the reported 40 MHz bandwidth") + } else if !native::is_supported_frequency_hz(frequency_hz) { + Some("unknown or unsupported ESP32 centre frequency") + } else { + None + }; + if let Some(reason) = rejection { + runtime.record_native_rejection(frame.node_id, &id, reason); + return; + } + let bytes = usize::from(frame.n_antennas) * usize::from(frame.n_subcarriers) * 2; + let Some(iq) = packet.get(20..20 + bytes) else { + return; + }; + let key = NativeSourceKey::from_frame(frame); + if state.room_learning_sources.len() >= 1024 && !state.room_learning_sources.contains_key(&key) + { + // Reclaim only expired unrelated classes. Selected contexts keep their + // identity, but eligible() still expires their readiness by receiver + // time; retaining a key cannot manufacture a fresh source window. + let context = runtime.context(); + state.room_learning_sources.retain(|key, meta| { + let fresh = meta.arrivals.back().is_some_and(|last| { + now.saturating_duration_since(*last) <= CALIBRATION_GRID_EVIDENCE_WINDOW + }); + fresh + || context.as_ref().is_some_and(|context| { + context.sources.iter().any(|source| { + source.source_id == key.node_id + && source.center_frequency_hz == meta.frequency_hz + && Some(source.bandwidth_hz) == meta.bandwidth_hz + && native::grid_id( + key.n_antennas as usize, + key.raw_bins as usize, + wifi_densepose_hardware::PpduType::from_byte(key.ppdu_type), + ) + .ok() + .as_deref() + == Some(source.grid_id.as_str()) + }) + }) + }); + if state.room_learning_sources.len() >= 1024 { + return; + } + } + let meta = state + .room_learning_sources + .entry(key) + .or_insert_with(|| LiveSource { + frequency_hz, + bandwidth_hz: Some(20_000_000), + receiver_rate_hz: 0.0, + arrivals: VecDeque::new(), + last_dequeued: None, + }); + meta.record(now); + let rate = meta.receiver_rate_hz; + if !runtime.selects_grid(frame.node_id, &id) { + return; + } + let dequeue_interval_us = receiver_timing.and_then(|t| meta.diagnostic_dequeue_interval(t.dequeued_at)); + let native_stamp_at = receiver_timing.map(|_| std::time::Instant::now()); + let ns = runtime.receiver_now_ns(); + let projected = match native::canonicalize_esp32(NativePacketInput { + node_id: frame.node_id, + n_antennas: frame.n_antennas as usize, + raw_bins: frame.n_subcarriers as usize, + frequency_hz, + ppdu_type: frame.ppdu_type, + sequence: frame.sequence, + received_at_ns: ns, + receiver_sample_rate_hz: rate, + iq_bytes: iq, + flags: frame.adr018_flags, + provenance: CaptureProvenance::LiveReceiverObserved, + }) { + Ok(p) => p, + Err(reason) => { + runtime.record_native_rejection(frame.node_id, &id, &reason); + return; + } + }; + let timing = receiver_timing.map(|t| { + wifi_densepose_sensing_server::room_learner::lifecycle::ReceiverPipelineTiming::for_packet( + &projected, dequeue_interval_us, t.parse_us, t.lock_us, + diagnostic_micros(native_stamp_at.and_then(|at| at.checked_duration_since(t.dequeued_at)))) + }); + let _ = runtime.ingest_native_with_receiver_timing(&projected, ns, timing); +} + + +#[cfg(test)] +mod receiver_timing_tests { + use super::*; + use std::time::{Duration, Instant}; + fn source() -> LiveSource { LiveSource { frequency_hz:2_437_000_000, bandwidth_hz:Some(20_000_000), receiver_rate_hz:33.0, arrivals:VecDeque::new(), last_dequeued:None } } + #[test] + fn diagnostic_cache_is_separate_from_readiness_and_exact_phy_keys_do_not_mix() { + let t = Instant::now(); + let ht_key = NativeSourceKey { node_id:15, n_antennas:1, raw_bins:64, ppdu_type:0, frequency_mhz:2437, bw40:false }; + let he_key = NativeSourceKey { raw_bins:256, ppdu_type:1, ..ht_key }; + let mut cache = std::collections::HashMap::from([(ht_key, source()), (he_key, source())]); + let ht = cache.get_mut(&ht_key).unwrap(); ht.record(t); + let prior = (ht.arrivals.len(), ht.receiver_rate_hz, ht.eligible(t)); + assert_eq!(ht.diagnostic_dequeue_interval(t), None); + assert_eq!(ht.diagnostic_dequeue_interval(t + Duration::from_micros(30_303)), Some(30_303)); + assert_eq!((ht.arrivals.len(), ht.receiver_rate_hz, ht.eligible(t)), prior); + let he = cache.get_mut(&he_key).unwrap(); + assert_eq!(he.diagnostic_dequeue_interval(t + Duration::from_micros(30_304)), None); + } + #[test] + fn invalid_or_oversized_durations_stay_unknown_instead_of_clipping() { + assert_eq!(diagnostic_micros(None), None); + assert_eq!(diagnostic_micros(Some(Duration::from_micros(u64::from(u32::MAX) + 1))), None); + assert_eq!(diagnostic_micros(Some(Duration::from_micros(12))), Some(12)); + let t = Instant::now(); let mut s = source(); + s.diagnostic_dequeue_interval(t + Duration::from_millis(1)); + assert_eq!(s.diagnostic_dequeue_interval(t), None); + } + #[test] + fn synthetic_post_dequeue_delay_does_not_become_a_radio_timestamp() { + let t = Instant::now(); + let timing = ReceiverIngressTiming::new(t, Some(Duration::from_micros(20)), Some(Duration::from_millis(60))); + assert_eq!(timing.parse_us, Some(20)); assert_eq!(timing.lock_us, Some(60_000)); + let before_native_stamp = t + Duration::from_micros(60_025); + assert_eq!(diagnostic_micros(before_native_stamp.checked_duration_since(timing.dequeued_at)), Some(60_025)); + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/source_bank_shadow.rs b/v2/crates/wifi-densepose-sensing-server/src/source_bank_shadow.rs new file mode 100644 index 0000000000..e1e42ac33f --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/source_bank_shadow.rs @@ -0,0 +1,375 @@ +//! Opt-in, memory-only per-source calibration diagnostics. Never used by the +//! authoritative model, receipts, persistence, person count or vital publication. +use serde_json::{json, Value}; +use std::collections::BTreeMap; +use wifi_densepose_signal::hardware_norm::HardwareNormalizer; +use wifi_densepose_signal::ruvsense::{ + field_model::FieldModelConfig, + source_field_bank::{SourceBankPhase, SourceFieldBank, SourceFieldSpec, SourceFieldWindow}, +}; + +pub(crate) struct ShadowCapture { + bank: SourceFieldBank, + grid: String, + ordinals: BTreeMap, + latest: BTreeMap, + windows: BTreeMap>>, + failed: bool, + input_width: usize, + canonical_width: usize, + normalizer: HardwareNormalizer, +} + +impl ShadowCapture { + pub(crate) fn new(ids: &[u8], width: usize, grid: String) -> Option { + let mut capture = Self::with_config(ids, grid, super::field_bridge::single_link_config())?; + capture.input_width = width; + Some(capture) + } + + fn with_config(ids: &[u8], grid: String, config: FieldModelConfig) -> Option { + let specs = ids + .iter() + .map(|&source_id| SourceFieldSpec { + source_id, + grid_id: grid.clone(), + config: config.clone(), + }) + .collect(); + let bank = SourceFieldBank::new(specs, 5_000_000).ok()?; + Some(Self { + bank, + grid, + ordinals: ids.iter().map(|&id| (id, 0)).collect(), + latest: BTreeMap::new(), + windows: BTreeMap::new(), + failed: false, + input_width: config.n_subcarriers, + canonical_width: config.n_subcarriers, + normalizer: HardwareNormalizer::new(), + }) + } + + /// Only called after the existing source/grid/sequence admission succeeds. + /// Receiver timestamps and validated ordinals never come from an HTTP client. + pub(crate) fn observe(&mut self, id: u8, timestamp_us: u64, frame: &[f64]) { + if self.failed { + return; + } + let Some(ordinal) = self.ordinals.get_mut(&id) else { + self.failed = true; + return; + }; + if frame.len() != self.input_width + || frame.iter().any(|v| !v.is_finite() || v.abs() > 1_000_000.) + || timestamp_us == 0 + || self + .latest + .get(&id) + .is_some_and(|at| timestamp_us <= *at || timestamp_us - *at > 5_000_000) + { + self.failed = true; + return; + } + let canonical = if self.canonical_width == 56 { + self.normalizer.resample_to_canonical(frame) + } else { + frame.to_vec() + }; + let frame = canonical.as_slice(); + *ordinal = ordinal.saturating_add(1); + if self.bank.phase() == SourceBankPhase::Collecting { + if self + .bank + .feed(id, &self.grid, *ordinal, timestamp_us, frame) + .is_err() + { + self.failed = true; + return; + } + } else if self.bank.phase() != SourceBankPhase::Frozen { + self.failed = true; + return; + } + // Keep at most one runtime-sized window per configured source. + self.latest.insert(id, timestamp_us); + let window = self.windows.entry(id).or_default(); + window.push(frame.to_vec()); + if window.len() > 50 { + window.remove(0); + } + } + + pub(crate) fn invalidate(&mut self) { + self.failed = true; + } + + pub(crate) fn freeze(&mut self, timestamp_us: u64) { + if self.failed || self.bank.freeze(timestamp_us, 0).is_err() { + self.failed = true; + } + // Held-out windows must contain no training frames. + self.windows.clear(); + } + + pub(crate) fn diagnostics(&self, timestamp_us: u64) -> Value { + let phase = if self.failed { + "invalid" + } else { + match self.bank.phase() { + SourceBankPhase::Collecting => "collecting", + SourceBankPhase::Frozen => "frozen", + SourceBankPhase::Invalid => "invalid", + } + }; + let comparisons = if !self.failed + && self.bank.phase() == SourceBankPhase::Frozen + && self.windows.values().all(|frames| frames.len() == 50) + { + let windows: Vec<_> = self + .windows + .iter() + .map(|(&id, frames)| SourceFieldWindow { + source_id: id, + grid_id: &self.grid, + observed_at_us: self.latest[&id], + frames, + }) + .collect(); + self.bank + .compare_empty_windows(timestamp_us, &windows) + .ok() + .map(|result| { + result + .into_iter() + .map(|(id, score)| { + ( + id.to_string(), + json!({ + "matches_empty": score.matches_empty, + "residual_energy": score.residual_energy, + "threshold": score.residual_energy_threshold, + "reliable": score.reliable, + "maturity": score.maturity, + "reference_window_count": score.reference_window_count, + "window_size": score.window_size, + }), + ) + }) + .collect::>() + }) + } else { + None + }; + json!({ "schema": "ruview.calibration.source-bank-shadow.v1", "phase": phase, + "frame_counts": self.bank.frame_counts(), "comparisons": comparisons, + "authority": "diagnostic_only", "persisted": false, + "calibrated_presence_authorized": false, "numeric_vitals_authorized": false }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + fn capture() -> ShadowCapture { + ShadowCapture::with_config( + &[15, 16, 17], + "ht64".into(), + FieldModelConfig { + n_links: 1, + n_subcarriers: 4, + n_modes: 1, + min_calibration_frames: 1_000, + min_calibration_duration_s: 0.0, + ..Default::default() + }, + ) + .unwrap() + } + #[test] + fn incomplete_capture_never_freezes_or_exposes_comparisons() { + let mut s = capture(); + for i in 1..=1_000 { + s.observe(15, i * 1_000, &[1., 2., 3., 4.]); + } + s.freeze(1_000_001); + let d = s.diagnostics(1_000_002); + assert_eq!(d["phase"], "invalid"); + assert!(d["comparisons"].is_null()); + assert_eq!(d["numeric_vitals_authorized"], false); + } + #[test] + fn freeze_requires_new_complete_held_out_windows_and_never_exports_raw_frames() { + let mut s = capture(); + for i in 1..=1_000 { + for id in [15, 16, 17] { + s.observe(id, i * 1_000, &[id as f64, 2., 3., 4.]); + } + } + s.freeze(1_000_001); + assert_eq!(s.diagnostics(1_000_002)["phase"], "frozen"); + assert!(s.diagnostics(1_000_002)["comparisons"].is_null()); + for i in 1..=50 { + for id in [15, 16, 17] { + s.observe(id, 1_001_000 + i * 1_000, &[id as f64, 2., 3., 4.]); + } + } + let d = s.diagnostics(1_052_000); + assert_eq!(d["comparisons"].as_object().unwrap().len(), 3); + assert_eq!(d["frame_counts"]["15"], 1_000); + assert_eq!(d["calibrated_presence_authorized"], false); + assert!(d.get("windows").is_none()); + assert!(s.diagnostics(7_000_000)["comparisons"].is_null()); + } + #[test] + fn production_resampling_matches_the_single_source_control() { + use wifi_densepose_signal::ruvsense::field_model::FieldModel; + let config = FieldModelConfig { + min_calibration_frames: 1_000, + min_calibration_duration_s: 0.0, + ..super::super::field_bridge::single_link_config() + }; + let mut control = FieldModel::new(config.clone()).unwrap(); + let mut shadow = ShadowCapture::with_config(&[15, 16, 17], "ht64".into(), config).unwrap(); + shadow.input_width = 64; + for i in 1..=1_000 { + for id in [15, 16, 17] { + let frame: Vec<_> = (0..64) + .map(|tone| id as f64 + tone as f64 + (i % 13) as f64 * 0.01) + .collect(); + if id == 15 { + assert!(super::super::field_bridge::maybe_feed_calibration( + &mut control, + &frame + )); + } + shadow.observe(id, i * 1_000, &frame); + } + } + control.finalize_calibration(1_000_001, 0).unwrap(); + shadow.freeze(1_000_001); + let models = shadow.bank.frozen_models().unwrap(); + assert_eq!( + models[0].snapshot.modes.baseline, + control.export_snapshot().unwrap().modes.baseline + ); + assert_ne!( + models[0].snapshot.modes.baseline, + models[1].snapshot.modes.baseline + ); + } + #[test] + fn unknown_or_changed_runtime_sources_invalidate_only_the_shadow() { + let mut s = capture(); + s.observe(99, 1_000, &[1., 2., 3., 4.]); + assert_eq!(s.diagnostics(1_001)["phase"], "invalid"); + assert_eq!(s.bank.frame_counts().len(), 3); + let mut s = capture(); + s.observe(15, 1_000, &[1., 2., 3., 4.]); + s.observe(15, 1_000, &[1., 2., 3., 4.]); + assert_eq!(s.diagnostics(1_001)["phase"], "invalid"); + } + #[tokio::test] + async fn status_is_opt_in_privacy_redacted_and_cleared_with_binding() { + use super::super::{calibration_status, AppStateInner}; + use axum::extract::State; + use std::sync::Arc; + use tokio::sync::RwLock; + let state = Arc::new(RwLock::new(AppStateInner::minimal())); + let axum::Json(status) = calibration_status(State(state.clone())).await; + assert!(status.get("source_bank_shadow").is_none()); + state.write().await.source_bank_shadow = Some(capture()); + let axum::Json(status) = calibration_status(State(state.clone())).await; + assert_eq!(status["source_bank_shadow"]["authority"], "diagnostic_only"); + state.write().await.privacy_mode = true; + let axum::Json(status) = calibration_status(State(state.clone())).await; + assert!(status.get("source_bank_shadow").is_none()); + state.write().await.clear_field_model_binding(); + assert!(state.read().await.source_bank_shadow.is_none()); + } + #[test] + fn opt_in_admission_trains_each_shadow_without_pooling_the_primary() { + use super::super::{AppStateInner, CalibrationGridBinding, CsiGridKey}; + use wifi_densepose_signal::ruvsense::field_model::FieldModel; + let mut state = AppStateInner::minimal(); + let grid = CsiGridKey { + n_subcarriers: 64, + ppdu_type: 0, + }; + state.calibration_grid_binding = Some(CalibrationGridBinding { + source_node_id: 15, + grid, + evidence: None, + }); + state.calibration_source_node_ids.extend([15, 16, 17]); + state.field_model = + Some(FieldModel::new(super::super::field_bridge::single_link_config()).unwrap()); + state.source_bank_shadow = ShadowCapture::new(&[15, 16, 17], 64, "ht64".into()); + for sequence in 1..=100 { + for id in [15, 16, 17] { + assert!(state.maybe_feed_calibration_frame( + id, + grid, + sequence, + &vec![id as f64; 64], + std::time::Instant::now() + )); + } + } + assert_eq!( + state + .field_model + .as_ref() + .unwrap() + .calibration_frame_count(), + 100 + ); + let shadow = state.source_bank_shadow.as_ref().unwrap(); + assert_eq!( + shadow.bank.frame_counts(), + BTreeMap::from([(15, 100), (16, 100), (17, 100)]) + ); + assert!(state.calibration_model_receipt.is_none()); + assert_eq!( + shadow.diagnostics(u64::MAX)["numeric_vitals_authorized"], + false + ); + assert!(!state.maybe_feed_calibration_frame( + 15, + grid, + 0, + &vec![15.; 64], + std::time::Instant::now() + )); + assert_eq!( + state + .source_bank_shadow + .as_ref() + .unwrap() + .diagnostics(u64::MAX)["phase"], + "invalid" + ); + assert_eq!( + state + .field_model + .as_ref() + .unwrap() + .calibration_frame_count(), + 100 + ); + } + #[test] + fn command_line_shadow_is_disabled_by_default() { + use clap::Parser; + assert!( + !super::super::Args::try_parse_from(["sensing-server"]) + .unwrap() + .source_bank_shadow + ); + assert!( + super::super::Args::try_parse_from(["sensing-server", "--source-bank-shadow"]) + .unwrap() + .source_bank_shadow + ); + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/testdata/paired_temporal_client_revision.json b/v2/crates/wifi-densepose-sensing-server/src/testdata/paired_temporal_client_revision.json new file mode 100644 index 0000000000..54a117db7d --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/testdata/paired_temporal_client_revision.json @@ -0,0 +1,127 @@ +{ + "schema": "ruview.test.actual-client-paired-revision-seam.v1", + "evidence": "SYNTHETIC_ACTUAL_PRODUCTION_CLIENT_FUNCTIONS_NO_RUNTIME_AUTHORITY", + "source_commit": "c3b89011a42154f8baef6c2ce08b113a3926584c", + "provenance": { + "publication": "actual companionService.publishRoom captured by API mock before network/ack/store import", + "binding": "actual sanitizeSharedRoom/sharedRoomDigest/createRoomCalibrationBinding; Node SHA256 at Expo boundary", + "configure": "actual roomLearnerService.temporalConfigure captured by API mock before network; expected guard is Rust structural idle fixture, NOT a live server permit" + }, + "installation_id": "synthetic-installation", + "published_revision": 1, + "binding_digest": "c020caf87c161f0bf595e1f603aef139045c9362f0d933341537d73225bca30d", + "computed_structural_revision": 211247076768791, + "canonical_room": { + "schema": "ruview.companion.room.v1", + "id": "synthetic-revision-room", + "name": "SYNTHETIC revision probe", + "widthM": 2.5, + "lengthM": 2.5, + "heightM": 2.8, + "nodes": [ + { + "nodeId": 15, + "slot": "north_west", + "xM": 0.4, + "yM": 0.4, + "role": "base_station", + "headingDeg": 45, + "orientationConfirmed": true, + "confirmedAtUnixMs": 1 + }, + { + "nodeId": 16, + "slot": "south_east", + "xM": 2, + "yM": 2, + "role": "receiver", + "headingDeg": 225, + "orientationConfirmed": true, + "confirmedAtUnixMs": 1 + }, + { + "nodeId": 17, + "slot": "south_west", + "xM": 0.4, + "yM": 2, + "role": "receiver", + "headingDeg": 315, + "orientationConfirmed": true, + "confirmedAtUnixMs": 1 + } + ], + "accessPoint": null + }, + "publication_request": { + "origin": "http://127.0.0.1:3000", + "path": "/api/v1/companion/room", + "body": { + "expected_revision": 0, + "room": { + "schema": "ruview.companion.room.v1", + "id": "synthetic-revision-room", + "name": "SYNTHETIC revision probe", + "widthM": 2.5, + "lengthM": 2.5, + "heightM": 2.8, + "nodes": [ + { + "nodeId": 15, + "slot": "north_west", + "xM": 0.4, + "yM": 0.4, + "role": "base_station", + "headingDeg": 45, + "orientationConfirmed": true, + "confirmedAtUnixMs": 1 + }, + { + "nodeId": 16, + "slot": "south_east", + "xM": 2, + "yM": 2, + "role": "receiver", + "headingDeg": 225, + "orientationConfirmed": true, + "confirmedAtUnixMs": 1 + }, + { + "nodeId": 17, + "slot": "south_west", + "xM": 0.4, + "yM": 2, + "role": "receiver", + "headingDeg": 315, + "orientationConfirmed": true, + "confirmedAtUnixMs": 1 + } + ], + "accessPoint": null + } + } + }, + "configure_request": { + "origin": "http://127.0.0.1:3000", + "path": "/api/v1/room-learner/configure", + "body": { + "schema": "ruview.room-learner.configure.v2", + "model_kind": "ruview.room-learner.managed-temporal-input.v3", + "expected": { + "runtime_epoch": "synthetic-wire-epoch", + "generation": 0, + "workflow_revision": 0, + "context_digest": null, + "phase": null + }, + "binding_digest": "c020caf87c161f0bf595e1f603aef139045c9362f0d933341537d73225bca30d", + "source_node_ids": [ + 15, + 16, + 17 + ], + "revision": 211247076768791, + "new_training": true, + "recovery_consent": true + } + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/vital_publication.rs b/v2/crates/wifi-densepose-sensing-server/src/vital_publication.rs new file mode 100644 index 0000000000..e94ca6dce5 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/vital_publication.rs @@ -0,0 +1,141 @@ +//! Bounded vital publication provenance and read-only diagnostic DTOs. +//! Candidate rates stay private; diagnostics contain gate outcomes only. + +use crate::vital_signs::VitalSigns; +use serde::{Deserialize, Serialize}; + +pub const DIAGNOSTICS_SCHEMA: &str = "ruview.vital-publication-diagnostics.v1"; +pub const CANDIDATE_MAX_AGE_MS: u64 = 3_000; + +#[derive(Debug, Clone, Copy, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum CandidateKind { + RawCsiSpectral, + FirmwareEdge, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CalibrationIdentity { + pub boot_epoch: String, + pub session_id: String, + pub model_id: String, + pub binding_digest: String, + pub source_node_ids: Vec, + pub completed_at_unix_ms: u64, +} + +#[derive(Debug, Clone)] +pub struct NodeVitalCandidate { + pub kind: CandidateKind, + pub tick: u64, + pub observed_at_unix_ms: u64, + pub received_at: std::time::Instant, + pub rates: VitalSigns, + pub calibration_identity: Option, +} + +#[derive(Debug, Clone, Copy, Serialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum Reason { + Published, + PrivacyRedacted, + NoCandidate, + CandidateStale, + CandidateCalibrationChanged, + NotPublishedForSourceTick, + RoomNotSingleOccupant, + SequenceFault, + ExplicitCalibrationRequired, + BoundModelUnavailable, + BoundWindowNotReady, + CalibratedPresenceUnavailable, + CalibratedNotSingleOccupant, + LowSignalQuality, + NoCandidateRate, + InvalidCandidateRate, + LowRateConfidence, +} + +#[derive(Debug, Clone, Serialize)] +pub struct BufferStatus { + pub breathing_samples: usize, + pub breathing_capacity: usize, + pub heartbeat_samples: usize, + pub heartbeat_capacity: usize, +} +impl From<(usize, usize, usize, usize)> for BufferStatus { + fn from( + (breathing_samples, breathing_capacity, heartbeat_samples, heartbeat_capacity): ( + usize, + usize, + usize, + usize, + ), + ) -> Self { + Self { + breathing_samples, + breathing_capacity, + heartbeat_samples, + heartbeat_capacity, + } + } +} + +#[derive(Debug, Clone, Serialize)] +pub struct Decision { + pub node_id: u8, + pub candidate_kind: Option, + pub source_tick: Option, + pub observed_at_unix_ms: Option, + pub candidate_age_ms: Option, + pub occupancy_source_node_id: Option, + pub room_person_count: usize, + pub calibrated_person_count: Option, + pub explicit_calibration_fresh: bool, + pub sequence_continuity_ok: bool, + pub bound_window_ready: bool, + pub breathing_rate_present: bool, + pub heart_rate_present: bool, + pub signal_quality_passed: bool, + pub breathing_confidence_passed: bool, + pub heartbeat_confidence_passed: bool, + pub buffers: BufferStatus, + pub sample_rate_hz: f64, + pub observed_sample_rate_hz: f64, + pub sample_rate_resets: u64, + pub reason: Reason, +} + +/// Exact v1 envelope already consumed by the native clients. Inventory and +/// occupancy-source IDs are deliberately not used as rate contributors. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct VitalEvidence { + pub schema: String, + pub capability: String, + pub source: String, + pub source_tick: u64, + pub source_node_ids: Vec, + pub observed_at_unix_ms: u64, + #[serde(flatten)] + pub rates: VitalSigns, +} + +pub fn evidence( + node_id: u8, + candidate: &NodeVitalCandidate, + published: &VitalSigns, +) -> Option { + Some(VitalEvidence { + schema: "ruview.vital-evidence.v1".into(), + capability: match candidate.kind { + CandidateKind::RawCsiSpectral => "esp32_csi_spectral_vitals_v1", + CandidateKind::FirmwareEdge => "esp32_firmware_edge_vitals_v1", + } + .into(), + source: "esp32".into(), + source_tick: candidate.tick, + source_node_ids: vec![node_id], + observed_at_unix_ms: candidate.observed_at_unix_ms, + rates: published.clone(), + }) +} diff --git a/v2/crates/wifi-densepose-sensing-server/tests/auth_wiring.rs b/v2/crates/wifi-densepose-sensing-server/tests/auth_wiring.rs index 79e8eb2e45..533008f116 100644 --- a/v2/crates/wifi-densepose-sensing-server/tests/auth_wiring.rs +++ b/v2/crates/wifi-densepose-sensing-server/tests/auth_wiring.rs @@ -203,7 +203,24 @@ const WS_PATHS: &[&str] = &["/ws/sensing", "/ws/introspection", "/api/v1/stream/ /// protected `/api/v1/*` by prefix. `/ws/field` was gated in this PR and its /// REST twin, one path segment over, returned 200 to an anonymous caller on /// both listeners. That is why the gate is now deny-by-default. -const PROTECTED_REST_PATHS: &[&str] = &["/api/field", "/api/v1/models"]; +const PROTECTED_REST_PATHS: &[&str] = &["/api/field", "/api/v1/models", "/api/v1/vital-signs"]; + +#[test] +fn actual_vital_diagnostics_require_the_existing_bearer_boundary() { + let server = Server::start(&[("RUVIEW_API_TOKEN", TOKEN)]); + assert_eq!(status(server.http, "GET", "/api/v1/vital-signs", &[]), 401); + let mut connection = TcpStream::connect(("127.0.0.1", server.http)).unwrap(); + connection.set_read_timeout(Some(Duration::from_secs(10))).unwrap(); + write!(connection, "GET /api/v1/vital-signs HTTP/1.1\r\nHost: 127.0.0.1:{}\r\nAuthorization: Bearer {}\r\nConnection: close\r\n\r\n", server.http, TOKEN).unwrap(); + let mut response = String::new(); + connection.read_to_string(&mut response).unwrap(); + assert!(response.starts_with("HTTP/1.1 200")); + let (_, body) = response.split_once("\r\n\r\n").unwrap(); + let body: serde_json::Value = serde_json::from_str(body).unwrap(); + assert_eq!(body["diagnostics"]["schema"], "ruview.vital-publication-diagnostics.v1"); + assert_eq!(body["authority"], "abstained"); + assert!(body["vital_signs"]["heart_rate_bpm"].is_null()); +} #[test] fn with_auth_on_no_listener_serves_sensing_data_anonymously() { diff --git a/v2/crates/wifi-densepose-sensing-server/tests/fixtures/managed-worldgraph-v1.json b/v2/crates/wifi-densepose-sensing-server/tests/fixtures/managed-worldgraph-v1.json new file mode 100644 index 0000000000..a5e4fc87b7 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/tests/fixtures/managed-worldgraph-v1.json @@ -0,0 +1,51 @@ +{ + "protocol_version": 2, + "stream_epoch": "wg-1111111111111111111111111111111111111111111111111111111111111111", + "seq": 1, + "topology": { + "schema": "ruview.worldgraph.managed-topology.v1", + "coordinate_frame": "managed_room_local", + "axis_convention": "x_right_y_down_metres", + "provenance": "operator_entered", + "registration": null, + "installation_id": "installation-1", + "room_revision": 7, + "binding_digest": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "status": "configured", + "room": { + "kind": "room", + "id": 1, + "room_id": "room-1", + "name": "Configured room", + "width_m": 2.5, + "length_m": 3.0, + "height_m": 2.4 + }, + "nodes": [ + { + "kind": "sensor", + "id": 272, + "node_id": 15, + "x_m": 1.25, + "y_m": 2.0, + "height_m": null, + "role": "receiver", + "heading_deg": 150, + "slot": "north_west" + } + ], + "edges": [ + { + "from": 272, + "to": 1, + "rel": "located_in" + } + ], + "access_point": { + "slot": "north_east", + "x_m": 0, + "y_m": 0, + "provenance": "operator_entered" + } + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/tests/fixtures/paired-calibration-progress-v1.json b/v2/crates/wifi-densepose-sensing-server/tests/fixtures/paired-calibration-progress-v1.json new file mode 100644 index 0000000000..015d2341fd --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/tests/fixtures/paired-calibration-progress-v1.json @@ -0,0 +1,173 @@ +{ + "cases": [ + { + "name": "configured", + "payload": { + "blocker": null, + "conditions": [ + { + "held_out_recorded": 0, + "label": "Empty", + "train_recorded": 0 + }, + { + "held_out_recorded": 0, + "label": "OccupiedStatic", + "train_recorded": 0 + }, + { + "held_out_recorded": 0, + "label": "OccupiedMoving", + "train_recorded": 0 + }, + { + "held_out_recorded": 0, + "label": "AdjacentOnly", + "train_recorded": 0 + } + ], + "context_match": true, + "expires_at_unix_ms": 1793805643128, + "installation_id": "synthetic-install", + "observed_at_unix_ms": 1793805633128, + "phase": null, + "qualification": { + "model_sha256": null, + "receipt_sha256": null, + "state": "not_reported" + }, + "room": { + "binding_digest": "c396e10e54432dc5403dbb7719c40902464ad558568b947017b75c0fa5b28a30", + "revision": 1 + }, + "runtime_epoch": "cal-boot-synthetic", + "save_status": "not_reported", + "schema": "ruview.room-learner.progress.v1", + "selection": { + "current_usable": false, + "model_sha256": null, + "selection_sha256": null, + "state": "not_selected" + }, + "sources": { + "ready": 2, + "required": 2 + }, + "stage": "train", + "state": "idle", + "success": true + } + }, + { + "name": "preparing_empty", + "payload": { + "blocker": null, + "conditions": [ + { + "held_out_recorded": 0, + "label": "Empty", + "train_recorded": 0 + }, + { + "held_out_recorded": 0, + "label": "OccupiedStatic", + "train_recorded": 0 + }, + { + "held_out_recorded": 0, + "label": "OccupiedMoving", + "train_recorded": 0 + }, + { + "held_out_recorded": 0, + "label": "AdjacentOnly", + "train_recorded": 0 + } + ], + "context_match": true, + "expires_at_unix_ms": 1793805643129, + "installation_id": "synthetic-install", + "observed_at_unix_ms": 1793805633129, + "phase": { + "capture_elapsed_ms": 0, + "label": "Empty", + "lead_in_remaining_ms": 14999, + "representative_cohorts": 0, + "required_cohorts": 4, + "role": "Train", + "stage": "lead_in", + "target_ms": 600000 + }, + "qualification": { + "model_sha256": null, + "receipt_sha256": null, + "state": "not_reported" + }, + "room": { + "binding_digest": "c396e10e54432dc5403dbb7719c40902464ad558568b947017b75c0fa5b28a30", + "revision": 1 + }, + "runtime_epoch": "cal-boot-synthetic", + "save_status": "not_reported", + "schema": "ruview.room-learner.progress.v1", + "selection": { + "current_usable": false, + "model_sha256": null, + "selection_sha256": null, + "state": "not_selected" + }, + "sources": { + "ready": 2, + "required": 2 + }, + "stage": "train", + "state": "idle", + "success": true + } + }, + { + "name": "context_mismatch", + "payload": { + "blocker": "context_mismatch", + "conditions": null, + "context_match": false, + "expires_at_unix_ms": 1793805643136, + "installation_id": "synthetic-install", + "observed_at_unix_ms": 1793805633136, + "phase": null, + "qualification": { + "model_sha256": null, + "receipt_sha256": null, + "state": "not_reported" + }, + "room": { + "binding_digest": "d2c78eec26671ef83d4431456746ab6786b2c994517aecbb3587e81ad0051353", + "revision": 2 + }, + "runtime_epoch": "cal-boot-synthetic", + "save_status": "not_reported", + "schema": "ruview.room-learner.progress.v1", + "selection": { + "current_usable": false, + "model_sha256": null, + "selection_sha256": null, + "state": "not_reported" + }, + "sources": null, + "stage": "blocked", + "state": "idle", + "success": true + } + }, + { + "name": "privacy_redacted", + "payload": { + "error_code": "privacy_redacted", + "schema": "ruview.room-learner.progress.v1", + "success": false + } + } + ], + "evidence": "SYNTHETIC_ACTUAL_HTTP_PRODUCER_NOT_PHYSICAL_QUALIFICATION", + "schema": "ruview.test.paired-progress-producer.v1" +} \ No newline at end of file diff --git a/v2/crates/wifi-densepose-sensing-server/tests/fixtures/temporal-seed-v2-synthetic-golden.json b/v2/crates/wifi-densepose-sensing-server/tests/fixtures/temporal-seed-v2-synthetic-golden.json new file mode 100644 index 0000000000..f417b81076 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/tests/fixtures/temporal-seed-v2-synthetic-golden.json @@ -0,0 +1 @@ +{"schema":"ruview.experimental.temporal-model-input.v2","payload_sha256":"040f7ca8174a6338f329ed7591c6c6a39441463fcb2a530e35d05605313cf405","payload":{"binding":{"plan_id":"Native2000msSlots2","native_frame_identity":"existing-esp32-receiver-ns-node-u32-sequence-v1","plan_sha256":"db70ffda37461a5ae5f46dfb4dd21cf200bb202d8d88e189e0716c6caa35b1c5","context":{"owner_scope":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","installation_id":"synthetic-temporal","room_digest":"synthetic-room","revision":1,"sources":[{"source_id":1,"device_id":"esp32-node-1","grid_id":"ant1-bins64-ppdu0-usable-drop2-linear-v2","feature_count":56,"signal_kind":"WifiCsi","center_frequency_hz":2437000000,"bandwidth_hz":20000000,"link_id":"native-1","link_index":0,"geometry":null},{"source_id":2,"device_id":"esp32-node-2","grid_id":"ant1-bins64-ppdu0-usable-drop2-linear-v2","feature_count":56,"signal_kind":"WifiCsi","center_frequency_hz":2437000000,"bandwidth_hz":20000000,"link_id":"native-2","link_index":0,"geometry":null}]},"encoder":{"bins":56,"source_count":2,"temporal_tokens":4,"hidden_dim":3,"embedding_dim":3,"temperature":0.3},"target_span_ms":2000,"max_span_ms":2050,"max_pairwise_skew_ms":50,"max_first_delay_ms":100,"sources":[{"source_id":1,"band":"Hz30To36","min_rows":61,"max_rows":77,"min_interval_ns":27777777,"max_gap_ns":50000000},{"source_id":2,"band":"Hz30To36","min_rows":61,"max_rows":77,"min_interval_ns":27777777,"max_gap_ns":50000000}]},"run_id":"synthetic-fit","training_data_sha256":"fe5579c3ff4e5853f1885dba20f1adc996e198d4c506c95a971e6afda6e6d9f0","input_evidence_sha256":"d8dcf49baea50b2f55b42576df3cf6333b6f33af191815d060381f023c4a7420","evidence":[{"cohort_id":"synthetic-fit-v2-r0-c0-t0-0","trial_id":"synthetic-fit-v2-r0-c0-t0","session_id":"synthetic-fit-v2-r0-c0-t0","role":"Train","label":"Empty","cohort_sha256":"74c046635820116696ad6c3262b74c115131f9bd33feee46a6caeecdd023e319","target_ns":"10000000000","sealed_at_ns":"12000002046","provenance":"Synthetic","sources":[{"source_id":1,"source_spec_sha256":"ea991b6de43cd497ae7a09091d30f7a4a181ce6ced05b217bca4dc3a396e8bdf","times_ns":["10000001000","10030304031","10060607062","10090910093","10121213124","10151516155","10181819186","10212122217","10242425248","10272728279","10303031310","10333334341","10363637372","10393940403","10424243434","10454546465","10484849496","10515152527","10545455558","10575758589","10606061620","10636364651","10666667682","10696970713","10727273744","10757576775","10787879806","10818182837","10848485868","10878788899","10909091930","10939394961","10969697992","11000001023","11030304054","11060607085","11090910116","11121213147","11151516178","11181819209","11212122240","11242425271","11272728302","11303031333","11333334364","11363637395","11393940426","11424243457","11454546488","11484849519","11515152550","11545455581","11575758612","11606061643","11636364674","11666667705","11696970736","11727273767","11757576798","11787879829","11818182860","11848485891","11878788922","11909091953","11939394984","11969698015","12000001046"],"frame_ids":["10995117377275922743297","11028435912217279332354","11061754447158635921411","11095072982099992510468","11128391517041349099525","11161710051982705688582","11195028586924062277639","11228347121865418866696","11261665656806775455753","11294984191748132044810","11328302726689488633867","11361621261630845222924","11394939796572201811981","11428258331513558401038","11461576866454914990095","11494895401396271579152","11528213936337628168209","11561532471278984757266","11594851006220341346323","11628169541161697935380","11661488076103054524437","11694806611044411113494","11728125145985767702551","11761443680927124291608","11794762215868480880665","11828080750809837469722","11861399285751194058779","11894717820692550647836","11928036355633907236893","11961354890575263825950","11994673425516620415007","12027991960457977004064","12061310495399333593121","12094629030340690182178","12127947565282046771235","12161266100223403360292","12194584635164759949349","12227903170106116538406","12261221705047473127463","12294540239988829716520","12327858774930186305577","12361177309871542894634","12394495844812899483691","12427814379754256072748","12461132914695612661805","12494451449636969250862","12527769984578325839919","12561088519519682428976","12594407054461039018033","12627725589402395607090","12661044124343752196147","12694362659285108785204","12727681194226465374261","12760999729167821963318","12794318264109178552375","12827636799050535141432","12860955333991891730489","12894273868933248319546","12927592403874604908603","12960910938815961497660","12994229473757318086717","13027548008698674675774","13060866543640031264831","13094185078581387853888","13127503613522744442945","13160822148464101032002","13194140683405457621059"],"sequence_ids":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67]},{"source_id":2,"source_spec_sha256":"ec550d93545ee35041bd6e85e79292ebf40dddb726353c1701a5aeeb232761a5","times_ns":["10000002000","10030305031","10060608062","10090911093","10121214124","10151517155","10181820186","10212123217","10242426248","10272729279","10303032310","10333335341","10363638372","10393941403","10424244434","10454547465","10484850496","10515153527","10545456558","10575759589","10606062620","10636365651","10666668682","10696971713","10727274744","10757577775","10787880806","10818183837","10848486868","10878789899","10909092930","10939395961","10969698992","11000002023","11030305054","11060608085","11090911116","11121214147","11151517178","11181820209","11212123240","11242426271","11272729302","11303032333","11333335364","11363638395","11393941426","11424244457","11454547488","11484850519","11515153550","11545456581","11575759612","11606062643","11636365674","11666668705","11696971736","11727274767","11757577798","11787880829","11818183860","11848486891","11878789922","11909092953","11939395984","11969699015","12000002046"],"frame_ids":["10995118476791845486593","11028437011733202075650","11061755546674558664707","11095074081615915253764","11128392616557271842821","11161711151498628431878","11195029686439985020935","11228348221381341609992","11261666756322698199049","11294985291264054788106","11328303826205411377163","11361622361146767966220","11394940896088124555277","11428259431029481144334","11461577965970837733391","11494896500912194322448","11528215035853550911505","11561533570794907500562","11594852105736264089619","11628170640677620678676","11661489175618977267733","11694807710560333856790","11728126245501690445847","11761444780443047034904","11794763315384403623961","11828081850325760213018","11861400385267116802075","11894718920208473391132","11928037455149829980189","11961355990091186569246","11994674525032543158303","12027993059973899747360","12061311594915256336417","12094630129856612925474","12127948664797969514531","12161267199739326103588","12194585734680682692645","12227904269622039281702","12261222804563395870759","12294541339504752459816","12327859874446109048873","12361178409387465637930","12394496944328822226987","12427815479270178816044","12461134014211535405101","12494452549152891994158","12527771084094248583215","12561089619035605172272","12594408153976961761329","12627726688918318350386","12661045223859674939443","12694363758801031528500","12727682293742388117557","12761000828683744706614","12794319363625101295671","12827637898566457884728","12860956433507814473785","12894274968449171062842","12927593503390527651899","12960912038331884240956","12994230573273240830013","13027549108214597419070","13060867643155954008127","13094186178097310597184","13127504713038667186241","13160823247980023775298","13194141782921380364355"],"sequence_ids":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67]}]},{"cohort_id":"synthetic-fit-v2-r0-c0-t0-1","trial_id":"synthetic-fit-v2-r0-c0-t0","session_id":"synthetic-fit-v2-r0-c0-t0","role":"Train","label":"Empty","cohort_sha256":"af7cec0c758b8ece2556511a956eaa1223ff759854f1a45e81b50af4db7e4dc9","target_ns":"13000000000","sealed_at_ns":"15000002046","provenance":"Synthetic","sources":[{"source_id":1,"source_spec_sha256":"ea991b6de43cd497ae7a09091d30f7a4a181ce6ced05b217bca4dc3a396e8bdf","times_ns":["13000001000","13030304031","13060607062","13090910093","13121213124","13151516155","13181819186","13212122217","13242425248","13272728279","13303031310","13333334341","13363637372","13393940403","13424243434","13454546465","13484849496","13515152527","13545455558","13575758589","13606061620","13636364651","13666667682","13696970713","13727273744","13757576775","13787879806","13818182837","13848485868","13878788899","13909091930","13939394961","13969697992","14000001023","14030304054","14060607085","14090910116","14121213147","14151516178","14181819209","14212122240","14242425271","14272728302","14303031333","14333334364","14363637395","14393940426","14424243457","14454546488","14484849519","14515152550","14545455581","14575758612","14606061643","14636364674","14666667705","14696970736","14727273767","14757576798","14787879829","14818182860","14848485891","14878788922","14909091953","14939394984","14969698015","15000001046"],"frame_ids":["14293652260603922743364","14326970795545279332421","14360289330486635921478","14393607865427992510535","14426926400369349099592","14460244935310705688649","14493563470252062277706","14526882005193418866763","14560200540134775455820","14593519075076132044877","14626837610017488633934","14660156144958845222991","14693474679900201812048","14726793214841558401105","14760111749782914990162","14793430284724271579219","14826748819665628168276","14860067354606984757333","14893385889548341346390","14926704424489697935447","14960022959431054524504","14993341494372411113561","15026660029313767702618","15059978564255124291675","15093297099196480880732","15126615634137837469789","15159934169079194058846","15193252704020550647903","15226571238961907236960","15259889773903263826017","15293208308844620415074","15326526843785977004131","15359845378727333593188","15393163913668690182245","15426482448610046771302","15459800983551403360359","15493119518492759949416","15526438053434116538473","15559756588375473127530","15593075123316829716587","15626393658258186305644","15659712193199542894701","15693030728140899483758","15726349263082256072815","15759667798023612661872","15792986332964969250929","15826304867906325839986","15859623402847682429043","15892941937789039018100","15926260472730395607157","15959579007671752196214","15992897542613108785271","16026216077554465374328","16059534612495821963385","16092853147437178552442","16126171682378535141499","16159490217319891730556","16192808752261248319613","16226127287202604908670","16259445822143961497727","16292764357085318086784","16326082892026674675841","16359401426968031264898","16392719961909387853955","16426038496850744443012","16459357031792101032069","16492675566733457621126"],"sequence_ids":[68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134]},{"source_id":2,"source_spec_sha256":"ec550d93545ee35041bd6e85e79292ebf40dddb726353c1701a5aeeb232761a5","times_ns":["13000002000","13030305031","13060608062","13090911093","13121214124","13151517155","13181820186","13212123217","13242426248","13272729279","13303032310","13333335341","13363638372","13393941403","13424244434","13454547465","13484850496","13515153527","13545456558","13575759589","13606062620","13636365651","13666668682","13696971713","13727274744","13757577775","13787880806","13818183837","13848486868","13878789899","13909092930","13939395961","13969698992","14000002023","14030305054","14060608085","14090911116","14121214147","14151517178","14181820209","14212123240","14242426271","14272729302","14303032333","14333335364","14363638395","14393941426","14424244457","14454547488","14484850519","14515153550","14545456581","14575759612","14606062643","14636365674","14666668705","14696971736","14727274767","14757577798","14787880829","14818183860","14848486891","14878789922","14909092953","14939395984","14969699015","15000002046"],"frame_ids":["14293653360119845486660","14326971895061202075717","14360290430002558664774","14393608964943915253831","14426927499885271842888","14460246034826628431945","14493564569767985021002","14526883104709341610059","14560201639650698199116","14593520174592054788173","14626838709533411377230","14660157244474767966287","14693475779416124555344","14726794314357481144401","14760112849298837733458","14793431384240194322515","14826749919181550911572","14860068454122907500629","14893386989064264089686","14926705524005620678743","14960024058946977267800","14993342593888333856857","15026661128829690445914","15059979663771047034971","15093298198712403624028","15126616733653760213085","15159935268595116802142","15193253803536473391199","15226572338477829980256","15259890873419186569313","15293209408360543158370","15326527943301899747427","15359846478243256336484","15393165013184612925541","15426483548125969514598","15459802083067326103655","15493120618008682692712","15526439152950039281769","15559757687891395870826","15593076222832752459883","15626394757774109048940","15659713292715465637997","15693031827656822227054","15726350362598178816111","15759668897539535405168","15792987432480891994225","15826305967422248583282","15859624502363605172339","15892943037304961761396","15926261572246318350453","15959580107187674939510","15992898642129031528567","16026217177070388117624","16059535712011744706681","16092854246953101295738","16126172781894457884795","16159491316835814473852","16192809851777171062909","16226128386718527651966","16259446921659884241023","16292765456601240830080","16326083991542597419137","16359402526483954008194","16392721061425310597251","16426039596366667186308","16459358131308023775365","16492676666249380364422"],"sequence_ids":[68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134]}]},{"cohort_id":"synthetic-fit-v2-r0-c0-t1-0","trial_id":"synthetic-fit-v2-r0-c0-t1","session_id":"synthetic-fit-v2-r0-c0-t1","role":"Train","label":"Empty","cohort_sha256":"74d05eb68924ac64637190cc64638fa9711205748b166c5d3c2d4e02f5745ac7","target_ns":"16000000000","sealed_at_ns":"18000002046","provenance":"Synthetic","sources":[{"source_id":1,"source_spec_sha256":"ea991b6de43cd497ae7a09091d30f7a4a181ce6ced05b217bca4dc3a396e8bdf","times_ns":["16000001000","16030304031","16060607062","16090910093","16121213124","16151516155","16181819186","16212122217","16242425248","16272728279","16303031310","16333334341","16363637372","16393940403","16424243434","16454546465","16484849496","16515152527","16545455558","16575758589","16606061620","16636364651","16666667682","16696970713","16727273744","16757576775","16787879806","16818182837","16848485868","16878788899","16909091930","16939394961","16969697992","17000001023","17030304054","17060607085","17090910116","17121213147","17151516178","17181819209","17212122240","17242425271","17272728302","17303031333","17333334364","17363637395","17393940426","17424243457","17454546488","17484849519","17515152550","17545455581","17575758612","17606061643","17636364674","17666667705","17696970736","17727273767","17757576798","17787879829","17818182860","17848485891","17878788922","17909091953","17939394984","17969698015","18000001046"],"frame_ids":["17592187143931922743431","17625505678873279332488","17658824213814635921545","17692142748755992510602","17725461283697349099659","17758779818638705688716","17792098353580062277773","17825416888521418866830","17858735423462775455887","17892053958404132044944","17925372493345488634001","17958691028286845223058","17992009563228201812115","18025328098169558401172","18058646633110914990229","18091965168052271579286","18125283702993628168343","18158602237934984757400","18191920772876341346457","18225239307817697935514","18258557842759054524571","18291876377700411113628","18325194912641767702685","18358513447583124291742","18391831982524480880799","18425150517465837469856","18458469052407194058913","18491787587348550647970","18525106122289907237027","18558424657231263826084","18591743192172620415141","18625061727113977004198","18658380262055333593255","18691698796996690182312","18725017331938046771369","18758335866879403360426","18791654401820759949483","18824972936762116538540","18858291471703473127597","18891610006644829716654","18924928541586186305711","18958247076527542894768","18991565611468899483825","19024884146410256072882","19058202681351612661939","19091521216292969250996","19124839751234325840053","19158158286175682429110","19191476821117039018167","19224795356058395607224","19258113890999752196281","19291432425941108785338","19324750960882465374395","19358069495823821963452","19391388030765178552509","19424706565706535141566","19458025100647891730623","19491343635589248319680","19524662170530604908737","19557980705471961497794","19591299240413318086851","19624617775354674675908","19657936310296031264965","19691254845237387854022","19724573380178744443079","19757891915120101032136","19791210450061457621193"],"sequence_ids":[135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201]},{"source_id":2,"source_spec_sha256":"ec550d93545ee35041bd6e85e79292ebf40dddb726353c1701a5aeeb232761a5","times_ns":["16000002000","16030305031","16060608062","16090911093","16121214124","16151517155","16181820186","16212123217","16242426248","16272729279","16303032310","16333335341","16363638372","16393941403","16424244434","16454547465","16484850496","16515153527","16545456558","16575759589","16606062620","16636365651","16666668682","16696971713","16727274744","16757577775","16787880806","16818183837","16848486868","16878789899","16909092930","16939395961","16969698992","17000002023","17030305054","17060608085","17090911116","17121214147","17151517178","17181820209","17212123240","17242426271","17272729302","17303032333","17333335364","17363638395","17393941426","17424244457","17454547488","17484850519","17515153550","17545456581","17575759612","17606062643","17636365674","17666668705","17696971736","17727274767","17757577798","17787880829","17818183860","17848486891","17878789922","17909092953","17939395984","17969699015","18000002046"],"frame_ids":["17592188243447845486727","17625506778389202075784","17658825313330558664841","17692143848271915253898","17725462383213271842955","17758780918154628432012","17792099453095985021069","17825417988037341610126","17858736522978698199183","17892055057920054788240","17925373592861411377297","17958692127802767966354","17992010662744124555411","18025329197685481144468","18058647732626837733525","18091966267568194322582","18125284802509550911639","18158603337450907500696","18191921872392264089753","18225240407333620678810","18258558942274977267867","18291877477216333856924","18325196012157690445981","18358514547099047035038","18391833082040403624095","18425151616981760213152","18458470151923116802209","18491788686864473391266","18525107221805829980323","18558425756747186569380","18591744291688543158437","18625062826629899747494","18658381361571256336551","18691699896512612925608","18725018431453969514665","18758336966395326103722","18791655501336682692779","18824974036278039281836","18858292571219395870893","18891611106160752459950","18924929641102109049007","18958248176043465638064","18991566710984822227121","19024885245926178816178","19058203780867535405235","19091522315808891994292","19124840850750248583349","19158159385691605172406","19191477920632961761463","19224796455574318350520","19258114990515674939577","19291433525457031528634","19324752060398388117691","19358070595339744706748","19391389130281101295805","19424707665222457884862","19458026200163814473919","19491344735105171062976","19524663270046527652033","19557981804987884241090","19591300339929240830147","19624618874870597419204","19657937409811954008261","19691255944753310597318","19724574479694667186375","19757893014636023775432","19791211549577380364489"],"sequence_ids":[135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201]}]},{"cohort_id":"synthetic-fit-v2-r0-c0-t1-1","trial_id":"synthetic-fit-v2-r0-c0-t1","session_id":"synthetic-fit-v2-r0-c0-t1","role":"Train","label":"Empty","cohort_sha256":"2062593b0dda4a6ede2f5a0e6dd771596faa229472fe6d0656ec31c2877450fd","target_ns":"19000000000","sealed_at_ns":"21000002046","provenance":"Synthetic","sources":[{"source_id":1,"source_spec_sha256":"ea991b6de43cd497ae7a09091d30f7a4a181ce6ced05b217bca4dc3a396e8bdf","times_ns":["19000001000","19030304031","19060607062","19090910093","19121213124","19151516155","19181819186","19212122217","19242425248","19272728279","19303031310","19333334341","19363637372","19393940403","19424243434","19454546465","19484849496","19515152527","19545455558","19575758589","19606061620","19636364651","19666667682","19696970713","19727273744","19757576775","19787879806","19818182837","19848485868","19878788899","19909091930","19939394961","19969697992","20000001023","20030304054","20060607085","20090910116","20121213147","20151516178","20181819209","20212122240","20242425271","20272728302","20303031333","20333334364","20363637395","20393940426","20424243457","20454546488","20484849519","20515152550","20545455581","20575758612","20606061643","20636364674","20666667705","20696970736","20727273767","20757576798","20787879829","20818182860","20848485891","20878788922","20909091953","20939394984","20969698015","21000001046"],"frame_ids":["20890722027259922743498","20924040562201279332555","20957359097142635921612","20990677632083992510669","21023996167025349099726","21057314701966705688783","21090633236908062277840","21123951771849418866897","21157270306790775455954","21190588841732132045011","21223907376673488634068","21257225911614845223125","21290544446556201812182","21323862981497558401239","21357181516438914990296","21390500051380271579353","21423818586321628168410","21457137121262984757467","21490455656204341346524","21523774191145697935581","21557092726087054524638","21590411261028411113695","21623729795969767702752","21657048330911124291809","21690366865852480880866","21723685400793837469923","21757003935735194058980","21790322470676550648037","21823641005617907237094","21856959540559263826151","21890278075500620415208","21923596610441977004265","21956915145383333593322","21990233680324690182379","22023552215266046771436","22056870750207403360493","22090189285148759949550","22123507820090116538607","22156826355031473127664","22190144889972829716721","22223463424914186305778","22256781959855542894835","22290100494796899483892","22323419029738256072949","22356737564679612662006","22390056099620969251063","22423374634562325840120","22456693169503682429177","22490011704445039018234","22523330239386395607291","22556648774327752196348","22589967309269108785405","22623285844210465374462","22656604379151821963519","22689922914093178552576","22723241449034535141633","22756559983975891730690","22789878518917248319747","22823197053858604908804","22856515588799961497861","22889834123741318086918","22923152658682674675975","22956471193624031265032","22989789728565387854089","23023108263506744443146","23056426798448101032203","23089745333389457621260"],"sequence_ids":[202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,256,257,258,259,260,261,262,263,264,265,266,267,268]},{"source_id":2,"source_spec_sha256":"ec550d93545ee35041bd6e85e79292ebf40dddb726353c1701a5aeeb232761a5","times_ns":["19000002000","19030305031","19060608062","19090911093","19121214124","19151517155","19181820186","19212123217","19242426248","19272729279","19303032310","19333335341","19363638372","19393941403","19424244434","19454547465","19484850496","19515153527","19545456558","19575759589","19606062620","19636365651","19666668682","19696971713","19727274744","19757577775","19787880806","19818183837","19848486868","19878789899","19909092930","19939395961","19969698992","20000002023","20030305054","20060608085","20090911116","20121214147","20151517178","20181820209","20212123240","20242426271","20272729302","20303032333","20333335364","20363638395","20393941426","20424244457","20454547488","20484850519","20515153550","20545456581","20575759612","20606062643","20636365674","20666668705","20696971736","20727274767","20757577798","20787880829","20818183860","20848486891","20878789922","20909092953","20939395984","20969699015","21000002046"],"frame_ids":["20890723126775845486794","20924041661717202075851","20957360196658558664908","20990678731599915253965","21023997266541271843022","21057315801482628432079","21090634336423985021136","21123952871365341610193","21157271406306698199250","21190589941248054788307","21223908476189411377364","21257227011130767966421","21290545546072124555478","21323864081013481144535","21357182615954837733592","21390501150896194322649","21423819685837550911706","21457138220778907500763","21490456755720264089820","21523775290661620678877","21557093825602977267934","21590412360544333856991","21623730895485690446048","21657049430427047035105","21690367965368403624162","21723686500309760213219","21757005035251116802276","21790323570192473391333","21823642105133829980390","21856960640075186569447","21890279175016543158504","21923597709957899747561","21956916244899256336618","21990234779840612925675","22023553314781969514732","22056871849723326103789","22090190384664682692846","22123508919606039281903","22156827454547395870960","22190145989488752460017","22223464524430109049074","22256783059371465638131","22290101594312822227188","22323420129254178816245","22356738664195535405302","22390057199136891994359","22423375734078248583416","22456694269019605172473","22490012803960961761530","22523331338902318350587","22556649873843674939644","22589968408785031528701","22623286943726388117758","22656605478667744706815","22689924013609101295872","22723242548550457884929","22756561083491814473986","22789879618433171063043","22823198153374527652100","22856516688315884241157","22889835223257240830214","22923153758198597419271","22956472293139954008328","22989790828081310597385","23023109363022667186442","23056427897964023775499","23089746432905380364556"],"sequence_ids":[202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,256,257,258,259,260,261,262,263,264,265,266,267,268]}]},{"cohort_id":"synthetic-fit-v2-r0-c1-t0-0","trial_id":"synthetic-fit-v2-r0-c1-t0","session_id":"synthetic-fit-v2-r0-c1-t0","role":"Train","label":"OccupiedStatic","cohort_sha256":"ea0572c8925666583d43d4e6efb4f1028a4c6041f403593b99d887faf0d5e196","target_ns":"22000000000","sealed_at_ns":"24000002046","provenance":"Synthetic","sources":[{"source_id":1,"source_spec_sha256":"ea991b6de43cd497ae7a09091d30f7a4a181ce6ced05b217bca4dc3a396e8bdf","times_ns":["22000001000","22030304031","22060607062","22090910093","22121213124","22151516155","22181819186","22212122217","22242425248","22272728279","22303031310","22333334341","22363637372","22393940403","22424243434","22454546465","22484849496","22515152527","22545455558","22575758589","22606061620","22636364651","22666667682","22696970713","22727273744","22757576775","22787879806","22818182837","22848485868","22878788899","22909091930","22939394961","22969697992","23000001023","23030304054","23060607085","23090910116","23121213147","23151516178","23181819209","23212122240","23242425271","23272728302","23303031333","23333334364","23363637395","23393940426","23424243457","23454546488","23484849519","23515152550","23545455581","23575758612","23606061643","23636364674","23666667705","23696970736","23727273767","23757576798","23787879829","23818182860","23848485891","23878788922","23909091953","23939394984","23969698015","24000001046"],"frame_ids":["24189256910587922743565","24222575445529279332622","24255893980470635921679","24289212515411992510736","24322531050353349099793","24355849585294705688850","24389168120236062277907","24422486655177418866964","24455805190118775456021","24489123725060132045078","24522442260001488634135","24555760794942845223192","24589079329884201812249","24622397864825558401306","24655716399766914990363","24689034934708271579420","24722353469649628168477","24755672004590984757534","24788990539532341346591","24822309074473697935648","24855627609415054524705","24888946144356411113762","24922264679297767702819","24955583214239124291876","24988901749180480880933","25022220284121837469990","25055538819063194059047","25088857354004550648104","25122175888945907237161","25155494423887263826218","25188812958828620415275","25222131493769977004332","25255450028711333593389","25288768563652690182446","25322087098594046771503","25355405633535403360560","25388724168476759949617","25422042703418116538674","25455361238359473127731","25488679773300829716788","25521998308242186305845","25555316843183542894902","25588635378124899483959","25621953913066256073016","25655272448007612662073","25688590982948969251130","25721909517890325840187","25755228052831682429244","25788546587773039018301","25821865122714395607358","25855183657655752196415","25888502192597108785472","25921820727538465374529","25955139262479821963586","25988457797421178552643","26021776332362535141700","26055094867303891730757","26088413402245248319814","26121731937186604908871","26155050472127961497928","26188369007069318086985","26221687542010674676042","26255006076952031265099","26288324611893387854156","26321643146834744443213","26354961681776101032270","26388280216717457621327"],"sequence_ids":[269,270,271,272,273,274,275,276,277,278,279,280,281,282,283,284,285,286,287,288,289,290,291,292,293,294,295,296,297,298,299,300,301,302,303,304,305,306,307,308,309,310,311,312,313,314,315,316,317,318,319,320,321,322,323,324,325,326,327,328,329,330,331,332,333,334,335]},{"source_id":2,"source_spec_sha256":"ec550d93545ee35041bd6e85e79292ebf40dddb726353c1701a5aeeb232761a5","times_ns":["22000002000","22030305031","22060608062","22090911093","22121214124","22151517155","22181820186","22212123217","22242426248","22272729279","22303032310","22333335341","22363638372","22393941403","22424244434","22454547465","22484850496","22515153527","22545456558","22575759589","22606062620","22636365651","22666668682","22696971713","22727274744","22757577775","22787880806","22818183837","22848486868","22878789899","22909092930","22939395961","22969698992","23000002023","23030305054","23060608085","23090911116","23121214147","23151517178","23181820209","23212123240","23242426271","23272729302","23303032333","23333335364","23363638395","23393941426","23424244457","23454547488","23484850519","23515153550","23545456581","23575759612","23606062643","23636365674","23666668705","23696971736","23727274767","23757577798","23787880829","23818183860","23848486891","23878789922","23909092953","23939395984","23969699015","24000002046"],"frame_ids":["24189258010103845486861","24222576545045202075918","24255895079986558664975","24289213614927915254032","24322532149869271843089","24355850684810628432146","24389169219751985021203","24422487754693341610260","24455806289634698199317","24489124824576054788374","24522443359517411377431","24555761894458767966488","24589080429400124555545","24622398964341481144602","24655717499282837733659","24689036034224194322716","24722354569165550911773","24755673104106907500830","24788991639048264089887","24822310173989620678944","24855628708930977268001","24888947243872333857058","24922265778813690446115","24955584313755047035172","24988902848696403624229","25022221383637760213286","25055539918579116802343","25088858453520473391400","25122176988461829980457","25155495523403186569514","25188814058344543158571","25222132593285899747628","25255451128227256336685","25288769663168612925742","25322088198109969514799","25355406733051326103856","25388725267992682692913","25422043802934039281970","25455362337875395871027","25488680872816752460084","25521999407758109049141","25555317942699465638198","25588636477640822227255","25621955012582178816312","25655273547523535405369","25688592082464891994426","25721910617406248583483","25755229152347605172540","25788547687288961761597","25821866222230318350654","25855184757171674939711","25888503292113031528768","25921821827054388117825","25955140361995744706882","25988458896937101295939","26021777431878457884996","26055095966819814474053","26088414501761171063110","26121733036702527652167","26155051571643884241224","26188370106585240830281","26221688641526597419338","26255007176467954008395","26288325711409310597452","26321644246350667186509","26354962781292023775566","26388281316233380364623"],"sequence_ids":[269,270,271,272,273,274,275,276,277,278,279,280,281,282,283,284,285,286,287,288,289,290,291,292,293,294,295,296,297,298,299,300,301,302,303,304,305,306,307,308,309,310,311,312,313,314,315,316,317,318,319,320,321,322,323,324,325,326,327,328,329,330,331,332,333,334,335]}]},{"cohort_id":"synthetic-fit-v2-r0-c1-t0-1","trial_id":"synthetic-fit-v2-r0-c1-t0","session_id":"synthetic-fit-v2-r0-c1-t0","role":"Train","label":"OccupiedStatic","cohort_sha256":"de47ee5a29b1628776efa122c9b1f8bc1dfa506e5b406a93ccab04503ffaf6de","target_ns":"25000000000","sealed_at_ns":"27000002046","provenance":"Synthetic","sources":[{"source_id":1,"source_spec_sha256":"ea991b6de43cd497ae7a09091d30f7a4a181ce6ced05b217bca4dc3a396e8bdf","times_ns":["25000001000","25030304031","25060607062","25090910093","25121213124","25151516155","25181819186","25212122217","25242425248","25272728279","25303031310","25333334341","25363637372","25393940403","25424243434","25454546465","25484849496","25515152527","25545455558","25575758589","25606061620","25636364651","25666667682","25696970713","25727273744","25757576775","25787879806","25818182837","25848485868","25878788899","25909091930","25939394961","25969697992","26000001023","26030304054","26060607085","26090910116","26121213147","26151516178","26181819209","26212122240","26242425271","26272728302","26303031333","26333334364","26363637395","26393940426","26424243457","26454546488","26484849519","26515152550","26545455581","26575758612","26606061643","26636364674","26666667705","26696970736","26727273767","26757576798","26787879829","26818182860","26848485891","26878788922","26909091953","26939394984","26969698015","27000001046"],"frame_ids":["27487791793915922743632","27521110328857279332689","27554428863798635921746","27587747398739992510803","27621065933681349099860","27654384468622705688917","27687703003564062277974","27721021538505418867031","27754340073446775456088","27787658608388132045145","27820977143329488634202","27854295678270845223259","27887614213212201812316","27920932748153558401373","27954251283094914990430","27987569818036271579487","28020888352977628168544","28054206887918984757601","28087525422860341346658","28120843957801697935715","28154162492743054524772","28187481027684411113829","28220799562625767702886","28254118097567124291943","28287436632508480881000","28320755167449837470057","28354073702391194059114","28387392237332550648171","28420710772273907237228","28454029307215263826285","28487347842156620415342","28520666377097977004399","28553984912039333593456","28587303446980690182513","28620621981922046771570","28653940516863403360627","28687259051804759949684","28720577586746116538741","28753896121687473127798","28787214656628829716855","28820533191570186305912","28853851726511542894969","28887170261452899484026","28920488796394256073083","28953807331335612662140","28987125866276969251197","29020444401218325840254","29053762936159682429311","29087081471101039018368","29120400006042395607425","29153718540983752196482","29187037075925108785539","29220355610866465374596","29253674145807821963653","29286992680749178552710","29320311215690535141767","29353629750631891730824","29386948285573248319881","29420266820514604908938","29453585355455961497995","29486903890397318087052","29520222425338674676109","29553540960280031265166","29586859495221387854223","29620178030162744443280","29653496565104101032337","29686815100045457621394"],"sequence_ids":[336,337,338,339,340,341,342,343,344,345,346,347,348,349,350,351,352,353,354,355,356,357,358,359,360,361,362,363,364,365,366,367,368,369,370,371,372,373,374,375,376,377,378,379,380,381,382,383,384,385,386,387,388,389,390,391,392,393,394,395,396,397,398,399,400,401,402]},{"source_id":2,"source_spec_sha256":"ec550d93545ee35041bd6e85e79292ebf40dddb726353c1701a5aeeb232761a5","times_ns":["25000002000","25030305031","25060608062","25090911093","25121214124","25151517155","25181820186","25212123217","25242426248","25272729279","25303032310","25333335341","25363638372","25393941403","25424244434","25454547465","25484850496","25515153527","25545456558","25575759589","25606062620","25636365651","25666668682","25696971713","25727274744","25757577775","25787880806","25818183837","25848486868","25878789899","25909092930","25939395961","25969698992","26000002023","26030305054","26060608085","26090911116","26121214147","26151517178","26181820209","26212123240","26242426271","26272729302","26303032333","26333335364","26363638395","26393941426","26424244457","26454547488","26484850519","26515153550","26545456581","26575759612","26606062643","26636365674","26666668705","26696971736","26727274767","26757577798","26787880829","26818183860","26848486891","26878789922","26909092953","26939395984","26969699015","27000002046"],"frame_ids":["27487792893431845486928","27521111428373202075985","27554429963314558665042","27587748498255915254099","27621067033197271843156","27654385568138628432213","27687704103079985021270","27721022638021341610327","27754341172962698199384","27787659707904054788441","27820978242845411377498","27854296777786767966555","27887615312728124555612","27920933847669481144669","27954252382610837733726","27987570917552194322783","28020889452493550911840","28054207987434907500897","28087526522376264089954","28120845057317620679011","28154163592258977268068","28187482127200333857125","28220800662141690446182","28254119197083047035239","28287437732024403624296","28320756266965760213353","28354074801907116802410","28387393336848473391467","28420711871789829980524","28454030406731186569581","28487348941672543158638","28520667476613899747695","28553986011555256336752","28587304546496612925809","28620623081437969514866","28653941616379326103923","28687260151320682692980","28720578686262039282037","28753897221203395871094","28787215756144752460151","28820534291086109049208","28853852826027465638265","28887171360968822227322","28920489895910178816379","28953808430851535405436","28987126965792891994493","29020445500734248583550","29053764035675605172607","29087082570616961761664","29120401105558318350721","29153719640499674939778","29187038175441031528835","29220356710382388117892","29253675245323744706949","29286993780265101296006","29320312315206457885063","29353630850147814474120","29386949385089171063177","29420267920030527652234","29453586454971884241291","29486904989913240830348","29520223524854597419405","29553542059795954008462","29586860594737310597519","29620179129678667186576","29653497664620023775633","29686816199561380364690"],"sequence_ids":[336,337,338,339,340,341,342,343,344,345,346,347,348,349,350,351,352,353,354,355,356,357,358,359,360,361,362,363,364,365,366,367,368,369,370,371,372,373,374,375,376,377,378,379,380,381,382,383,384,385,386,387,388,389,390,391,392,393,394,395,396,397,398,399,400,401,402]}]},{"cohort_id":"synthetic-fit-v2-r0-c1-t1-0","trial_id":"synthetic-fit-v2-r0-c1-t1","session_id":"synthetic-fit-v2-r0-c1-t1","role":"Train","label":"OccupiedStatic","cohort_sha256":"3565c70576c88230e846845cf90b17851a9ca34b850eb859b8bd3d6afb30c84d","target_ns":"28000000000","sealed_at_ns":"30000002046","provenance":"Synthetic","sources":[{"source_id":1,"source_spec_sha256":"ea991b6de43cd497ae7a09091d30f7a4a181ce6ced05b217bca4dc3a396e8bdf","times_ns":["28000001000","28030304031","28060607062","28090910093","28121213124","28151516155","28181819186","28212122217","28242425248","28272728279","28303031310","28333334341","28363637372","28393940403","28424243434","28454546465","28484849496","28515152527","28545455558","28575758589","28606061620","28636364651","28666667682","28696970713","28727273744","28757576775","28787879806","28818182837","28848485868","28878788899","28909091930","28939394961","28969697992","29000001023","29030304054","29060607085","29090910116","29121213147","29151516178","29181819209","29212122240","29242425271","29272728302","29303031333","29333334364","29363637395","29393940426","29424243457","29454546488","29484849519","29515152550","29545455581","29575758612","29606061643","29636364674","29666667705","29696970736","29727273767","29757576798","29787879829","29818182860","29848485891","29878788922","29909091953","29939394984","29969698015","30000001046"],"frame_ids":["30786326677243922743699","30819645212185279332756","30852963747126635921813","30886282282067992510870","30919600817009349099927","30952919351950705688984","30986237886892062278041","31019556421833418867098","31052874956774775456155","31086193491716132045212","31119512026657488634269","31152830561598845223326","31186149096540201812383","31219467631481558401440","31252786166422914990497","31286104701364271579554","31319423236305628168611","31352741771246984757668","31386060306188341346725","31419378841129697935782","31452697376071054524839","31486015911012411113896","31519334445953767702953","31552652980895124292010","31585971515836480881067","31619290050777837470124","31652608585719194059181","31685927120660550648238","31719245655601907237295","31752564190543263826352","31785882725484620415409","31819201260425977004466","31852519795367333593523","31885838330308690182580","31919156865250046771637","31952475400191403360694","31985793935132759949751","32019112470074116538808","32052431005015473127865","32085749539956829716922","32119068074898186305979","32152386609839542895036","32185705144780899484093","32219023679722256073150","32252342214663612662207","32285660749604969251264","32318979284546325840321","32352297819487682429378","32385616354429039018435","32418934889370395607492","32452253424311752196549","32485571959253108785606","32518890494194465374663","32552209029135821963720","32585527564077178552777","32618846099018535141834","32652164633959891730891","32685483168901248319948","32718801703842604909005","32752120238783961498062","32785438773725318087119","32818757308666674676176","32852075843608031265233","32885394378549387854290","32918712913490744443347","32952031448432101032404","32985349983373457621461"],"sequence_ids":[403,404,405,406,407,408,409,410,411,412,413,414,415,416,417,418,419,420,421,422,423,424,425,426,427,428,429,430,431,432,433,434,435,436,437,438,439,440,441,442,443,444,445,446,447,448,449,450,451,452,453,454,455,456,457,458,459,460,461,462,463,464,465,466,467,468,469]},{"source_id":2,"source_spec_sha256":"ec550d93545ee35041bd6e85e79292ebf40dddb726353c1701a5aeeb232761a5","times_ns":["28000002000","28030305031","28060608062","28090911093","28121214124","28151517155","28181820186","28212123217","28242426248","28272729279","28303032310","28333335341","28363638372","28393941403","28424244434","28454547465","28484850496","28515153527","28545456558","28575759589","28606062620","28636365651","28666668682","28696971713","28727274744","28757577775","28787880806","28818183837","28848486868","28878789899","28909092930","28939395961","28969698992","29000002023","29030305054","29060608085","29090911116","29121214147","29151517178","29181820209","29212123240","29242426271","29272729302","29303032333","29333335364","29363638395","29393941426","29424244457","29454547488","29484850519","29515153550","29545456581","29575759612","29606062643","29636365674","29666668705","29696971736","29727274767","29757577798","29787880829","29818183860","29848486891","29878789922","29909092953","29939395984","29969699015","30000002046"],"frame_ids":["30786327776759845486995","30819646311701202076052","30852964846642558665109","30886283381583915254166","30919601916525271843223","30952920451466628432280","30986238986407985021337","31019557521349341610394","31052876056290698199451","31086194591232054788508","31119513126173411377565","31152831661114767966622","31186150196056124555679","31219468730997481144736","31252787265938837733793","31286105800880194322850","31319424335821550911907","31352742870762907500964","31386061405704264090021","31419379940645620679078","31452698475586977268135","31486017010528333857192","31519335545469690446249","31552654080411047035306","31585972615352403624363","31619291150293760213420","31652609685235116802477","31685928220176473391534","31719246755117829980591","31752565290059186569648","31785883825000543158705","31819202359941899747762","31852520894883256336819","31885839429824612925876","31919157964765969514933","31952476499707326103990","31985795034648682693047","32019113569590039282104","32052432104531395871161","32085750639472752460218","32119069174414109049275","32152387709355465638332","32185706244296822227389","32219024779238178816446","32252343314179535405503","32285661849120891994560","32318980384062248583617","32352298919003605172674","32385617453944961761731","32418935988886318350788","32452254523827674939845","32485573058769031528902","32518891593710388117959","32552210128651744707016","32585528663593101296073","32618847198534457885130","32652165733475814474187","32685484268417171063244","32718802803358527652301","32752121338299884241358","32785439873241240830415","32818758408182597419472","32852076943123954008529","32885395478065310597586","32918714013006667186643","32952032547948023775700","32985351082889380364757"],"sequence_ids":[403,404,405,406,407,408,409,410,411,412,413,414,415,416,417,418,419,420,421,422,423,424,425,426,427,428,429,430,431,432,433,434,435,436,437,438,439,440,441,442,443,444,445,446,447,448,449,450,451,452,453,454,455,456,457,458,459,460,461,462,463,464,465,466,467,468,469]}]},{"cohort_id":"synthetic-fit-v2-r0-c1-t1-1","trial_id":"synthetic-fit-v2-r0-c1-t1","session_id":"synthetic-fit-v2-r0-c1-t1","role":"Train","label":"OccupiedStatic","cohort_sha256":"e82fe125bb5ba2c70db70581f63cf3aab683bf25b3da0deae1f415b73d5e9974","target_ns":"31000000000","sealed_at_ns":"33000002046","provenance":"Synthetic","sources":[{"source_id":1,"source_spec_sha256":"ea991b6de43cd497ae7a09091d30f7a4a181ce6ced05b217bca4dc3a396e8bdf","times_ns":["31000001000","31030304031","31060607062","31090910093","31121213124","31151516155","31181819186","31212122217","31242425248","31272728279","31303031310","31333334341","31363637372","31393940403","31424243434","31454546465","31484849496","31515152527","31545455558","31575758589","31606061620","31636364651","31666667682","31696970713","31727273744","31757576775","31787879806","31818182837","31848485868","31878788899","31909091930","31939394961","31969697992","32000001023","32030304054","32060607085","32090910116","32121213147","32151516178","32181819209","32212122240","32242425271","32272728302","32303031333","32333334364","32363637395","32393940426","32424243457","32454546488","32484849519","32515152550","32545455581","32575758612","32606061643","32636364674","32666667705","32696970736","32727273767","32757576798","32787879829","32818182860","32848485891","32878788922","32909091953","32939394984","32969698015","33000001046"],"frame_ids":["34084861560571922743766","34118180095513279332823","34151498630454635921880","34184817165395992510937","34218135700337349099994","34251454235278705689051","34284772770220062278108","34318091305161418867165","34351409840102775456222","34384728375044132045279","34418046909985488634336","34451365444926845223393","34484683979868201812450","34518002514809558401507","34551321049750914990564","34584639584692271579621","34617958119633628168678","34651276654574984757735","34684595189516341346792","34717913724457697935849","34751232259399054524906","34784550794340411113963","34817869329281767703020","34851187864223124292077","34884506399164480881134","34917824934105837470191","34951143469047194059248","34984462003988550648305","35017780538929907237362","35051099073871263826419","35084417608812620415476","35117736143753977004533","35151054678695333593590","35184373213636690182647","35217691748578046771704","35251010283519403360761","35284328818460759949818","35317647353402116538875","35350965888343473127932","35384284423284829716989","35417602958226186306046","35450921493167542895103","35484240028108899484160","35517558563050256073217","35550877097991612662274","35584195632932969251331","35617514167874325840388","35650832702815682429445","35684151237757039018502","35717469772698395607559","35750788307639752196616","35784106842581108785673","35817425377522465374730","35850743912463821963787","35884062447405178552844","35917380982346535141901","35950699517287891730958","35984018052229248320015","36017336587170604909072","36050655122111961498129","36083973657053318087186","36117292191994674676243","36150610726936031265300","36183929261877387854357","36217247796818744443414","36250566331760101032471","36283884866701457621528"],"sequence_ids":[470,471,472,473,474,475,476,477,478,479,480,481,482,483,484,485,486,487,488,489,490,491,492,493,494,495,496,497,498,499,500,501,502,503,504,505,506,507,508,509,510,511,512,513,514,515,516,517,518,519,520,521,522,523,524,525,526,527,528,529,530,531,532,533,534,535,536]},{"source_id":2,"source_spec_sha256":"ec550d93545ee35041bd6e85e79292ebf40dddb726353c1701a5aeeb232761a5","times_ns":["31000002000","31030305031","31060608062","31090911093","31121214124","31151517155","31181820186","31212123217","31242426248","31272729279","31303032310","31333335341","31363638372","31393941403","31424244434","31454547465","31484850496","31515153527","31545456558","31575759589","31606062620","31636365651","31666668682","31696971713","31727274744","31757577775","31787880806","31818183837","31848486868","31878789899","31909092930","31939395961","31969698992","32000002023","32030305054","32060608085","32090911116","32121214147","32151517178","32181820209","32212123240","32242426271","32272729302","32303032333","32333335364","32363638395","32393941426","32424244457","32454547488","32484850519","32515153550","32545456581","32575759612","32606062643","32636365674","32666668705","32696971736","32727274767","32757577798","32787880829","32818183860","32848486891","32878789922","32909092953","32939395984","32969699015","33000002046"],"frame_ids":["34084862660087845487062","34118181195029202076119","34151499729970558665176","34184818264911915254233","34218136799853271843290","34251455334794628432347","34284773869735985021404","34318092404677341610461","34351410939618698199518","34384729474560054788575","34418048009501411377632","34451366544442767966689","34484685079384124555746","34518003614325481144803","34551322149266837733860","34584640684208194322917","34617959219149550911974","34651277754090907501031","34684596289032264090088","34717914823973620679145","34751233358914977268202","34784551893856333857259","34817870428797690446316","34851188963739047035373","34884507498680403624430","34917826033621760213487","34951144568563116802544","34984463103504473391601","35017781638445829980658","35051100173387186569715","35084418708328543158772","35117737243269899747829","35151055778211256336886","35184374313152612925943","35217692848093969515000","35251011383035326104057","35284329917976682693114","35317648452918039282171","35350966987859395871228","35384285522800752460285","35417604057742109049342","35450922592683465638399","35484241127624822227456","35517559662566178816513","35550878197507535405570","35584196732448891994627","35617515267390248583684","35650833802331605172741","35684152337272961761798","35717470872214318350855","35750789407155674939912","35784107942097031528969","35817426477038388118026","35850745011979744707083","35884063546921101296140","35917382081862457885197","35950700616803814474254","35984019151745171063311","36017337686686527652368","36050656221627884241425","36083974756569240830482","36117293291510597419539","36150611826451954008596","36183930361393310597653","36217248896334667186710","36250567431276023775767","36283885966217380364824"],"sequence_ids":[470,471,472,473,474,475,476,477,478,479,480,481,482,483,484,485,486,487,488,489,490,491,492,493,494,495,496,497,498,499,500,501,502,503,504,505,506,507,508,509,510,511,512,513,514,515,516,517,518,519,520,521,522,523,524,525,526,527,528,529,530,531,532,533,534,535,536]}]},{"cohort_id":"synthetic-fit-v2-r0-c2-t0-0","trial_id":"synthetic-fit-v2-r0-c2-t0","session_id":"synthetic-fit-v2-r0-c2-t0","role":"Train","label":"OccupiedMoving","cohort_sha256":"c0fd00a4fb80c266d52fa8fb422be1e059488287061eadec8616af79350e0d5d","target_ns":"34000000000","sealed_at_ns":"36000002046","provenance":"Synthetic","sources":[{"source_id":1,"source_spec_sha256":"ea991b6de43cd497ae7a09091d30f7a4a181ce6ced05b217bca4dc3a396e8bdf","times_ns":["34000001000","34030304031","34060607062","34090910093","34121213124","34151516155","34181819186","34212122217","34242425248","34272728279","34303031310","34333334341","34363637372","34393940403","34424243434","34454546465","34484849496","34515152527","34545455558","34575758589","34606061620","34636364651","34666667682","34696970713","34727273744","34757576775","34787879806","34818182837","34848485868","34878788899","34909091930","34939394961","34969697992","35000001023","35030304054","35060607085","35090910116","35121213147","35151516178","35181819209","35212122240","35242425271","35272728302","35303031333","35333334364","35363637395","35393940426","35424243457","35454546488","35484849519","35515152550","35545455581","35575758612","35606061643","35636364674","35666667705","35696970736","35727273767","35757576798","35787879829","35818182860","35848485891","35878788922","35909091953","35939394984","35969698015","36000001046"],"frame_ids":["37383396443899922743833","37416714978841279332890","37450033513782635921947","37483352048723992511004","37516670583665349100061","37549989118606705689118","37583307653548062278175","37616626188489418867232","37649944723430775456289","37683263258372132045346","37716581793313488634403","37749900328254845223460","37783218863196201812517","37816537398137558401574","37849855933078914990631","37883174468020271579688","37916493002961628168745","37949811537902984757802","37983130072844341346859","38016448607785697935916","38049767142727054524973","38083085677668411114030","38116404212609767703087","38149722747551124292144","38183041282492480881201","38216359817433837470258","38249678352375194059315","38282996887316550648372","38316315422257907237429","38349633957199263826486","38382952492140620415543","38416271027081977004600","38449589562023333593657","38482908096964690182714","38516226631906046771771","38549545166847403360828","38582863701788759949885","38616182236730116538942","38649500771671473127999","38682819306612829717056","38716137841554186306113","38749456376495542895170","38782774911436899484227","38816093446378256073284","38849411981319612662341","38882730516260969251398","38916049051202325840455","38949367586143682429512","38982686121085039018569","39016004656026395607626","39049323190967752196683","39082641725909108785740","39115960260850465374797","39149278795791821963854","39182597330733178552911","39215915865674535141968","39249234400615891731025","39282552935557248320082","39315871470498604909139","39349190005439961498196","39382508540381318087253","39415827075322674676310","39449145610264031265367","39482464145205387854424","39515782680146744443481","39549101215088101032538","39582419750029457621595"],"sequence_ids":[537,538,539,540,541,542,543,544,545,546,547,548,549,550,551,552,553,554,555,556,557,558,559,560,561,562,563,564,565,566,567,568,569,570,571,572,573,574,575,576,577,578,579,580,581,582,583,584,585,586,587,588,589,590,591,592,593,594,595,596,597,598,599,600,601,602,603]},{"source_id":2,"source_spec_sha256":"ec550d93545ee35041bd6e85e79292ebf40dddb726353c1701a5aeeb232761a5","times_ns":["34000002000","34030305031","34060608062","34090911093","34121214124","34151517155","34181820186","34212123217","34242426248","34272729279","34303032310","34333335341","34363638372","34393941403","34424244434","34454547465","34484850496","34515153527","34545456558","34575759589","34606062620","34636365651","34666668682","34696971713","34727274744","34757577775","34787880806","34818183837","34848486868","34878789899","34909092930","34939395961","34969698992","35000002023","35030305054","35060608085","35090911116","35121214147","35151517178","35181820209","35212123240","35242426271","35272729302","35303032333","35333335364","35363638395","35393941426","35424244457","35454547488","35484850519","35515153550","35545456581","35575759612","35606062643","35636365674","35666668705","35696971736","35727274767","35757577798","35787880829","35818183860","35848486891","35878789922","35909092953","35939395984","35969699015","36000002046"],"frame_ids":["37383397543415845487129","37416716078357202076186","37450034613298558665243","37483353148239915254300","37516671683181271843357","37549990218122628432414","37583308753063985021471","37616627288005341610528","37649945822946698199585","37683264357888054788642","37716582892829411377699","37749901427770767966756","37783219962712124555813","37816538497653481144870","37849857032594837733927","37883175567536194322984","37916494102477550912041","37949812637418907501098","37983131172360264090155","38016449707301620679212","38049768242242977268269","38083086777184333857326","38116405312125690446383","38149723847067047035440","38183042382008403624497","38216360916949760213554","38249679451891116802611","38282997986832473391668","38316316521773829980725","38349635056715186569782","38382953591656543158839","38416272126597899747896","38449590661539256336953","38482909196480612926010","38516227731421969515067","38549546266363326104124","38582864801304682693181","38616183336246039282238","38649501871187395871295","38682820406128752460352","38716138941070109049409","38749457476011465638466","38782776010952822227523","38816094545894178816580","38849413080835535405637","38882731615776891994694","38916050150718248583751","38949368685659605172808","38982687220600961761865","39016005755542318350922","39049324290483674939979","39082642825425031529036","39115961360366388118093","39149279895307744707150","39182598430249101296207","39215916965190457885264","39249235500131814474321","39282554035073171063378","39315872570014527652435","39349191104955884241492","39382509639897240830549","39415828174838597419606","39449146709779954008663","39482465244721310597720","39515783779662667186777","39549102314604023775834","39582420849545380364891"],"sequence_ids":[537,538,539,540,541,542,543,544,545,546,547,548,549,550,551,552,553,554,555,556,557,558,559,560,561,562,563,564,565,566,567,568,569,570,571,572,573,574,575,576,577,578,579,580,581,582,583,584,585,586,587,588,589,590,591,592,593,594,595,596,597,598,599,600,601,602,603]}]},{"cohort_id":"synthetic-fit-v2-r0-c2-t0-1","trial_id":"synthetic-fit-v2-r0-c2-t0","session_id":"synthetic-fit-v2-r0-c2-t0","role":"Train","label":"OccupiedMoving","cohort_sha256":"68e6653674d4c0a2768fe35fa743e03bfd561e593d24f7ff5e4529dce9c4f8a9","target_ns":"37000000000","sealed_at_ns":"39000002046","provenance":"Synthetic","sources":[{"source_id":1,"source_spec_sha256":"ea991b6de43cd497ae7a09091d30f7a4a181ce6ced05b217bca4dc3a396e8bdf","times_ns":["37000001000","37030304031","37060607062","37090910093","37121213124","37151516155","37181819186","37212122217","37242425248","37272728279","37303031310","37333334341","37363637372","37393940403","37424243434","37454546465","37484849496","37515152527","37545455558","37575758589","37606061620","37636364651","37666667682","37696970713","37727273744","37757576775","37787879806","37818182837","37848485868","37878788899","37909091930","37939394961","37969697992","38000001023","38030304054","38060607085","38090910116","38121213147","38151516178","38181819209","38212122240","38242425271","38272728302","38303031333","38333334364","38363637395","38393940426","38424243457","38454546488","38484849519","38515152550","38545455581","38575758612","38606061643","38636364674","38666667705","38696970736","38727273767","38757576798","38787879829","38818182860","38848485891","38878788922","38909091953","38939394984","38969698015","39000001046"],"frame_ids":["40681931327227922743900","40715249862169279332957","40748568397110635922014","40781886932051992511071","40815205466993349100128","40848524001934705689185","40881842536876062278242","40915161071817418867299","40948479606758775456356","40981798141700132045413","41015116676641488634470","41048435211582845223527","41081753746524201812584","41115072281465558401641","41148390816406914990698","41181709351348271579755","41215027886289628168812","41248346421230984757869","41281664956172341346926","41314983491113697935983","41348302026055054525040","41381620560996411114097","41414939095937767703154","41448257630879124292211","41481576165820480881268","41514894700761837470325","41548213235703194059382","41581531770644550648439","41614850305585907237496","41648168840527263826553","41681487375468620415610","41714805910409977004667","41748124445351333593724","41781442980292690182781","41814761515234046771838","41848080050175403360895","41881398585116759949952","41914717120058116539009","41948035654999473128066","41981354189940829717123","42014672724882186306180","42047991259823542895237","42081309794764899484294","42114628329706256073351","42147946864647612662408","42181265399588969251465","42214583934530325840522","42247902469471682429579","42281221004413039018636","42314539539354395607693","42347858074295752196750","42381176609237108785807","42414495144178465374864","42447813679119821963921","42481132214061178552978","42514450749002535142035","42547769283943891731092","42581087818885248320149","42614406353826604909206","42647724888767961498263","42681043423709318087320","42714361958650674676377","42747680493592031265434","42780999028533387854491","42814317563474744443548","42847636098416101032605","42880954633357457621662"],"sequence_ids":[604,605,606,607,608,609,610,611,612,613,614,615,616,617,618,619,620,621,622,623,624,625,626,627,628,629,630,631,632,633,634,635,636,637,638,639,640,641,642,643,644,645,646,647,648,649,650,651,652,653,654,655,656,657,658,659,660,661,662,663,664,665,666,667,668,669,670]},{"source_id":2,"source_spec_sha256":"ec550d93545ee35041bd6e85e79292ebf40dddb726353c1701a5aeeb232761a5","times_ns":["37000002000","37030305031","37060608062","37090911093","37121214124","37151517155","37181820186","37212123217","37242426248","37272729279","37303032310","37333335341","37363638372","37393941403","37424244434","37454547465","37484850496","37515153527","37545456558","37575759589","37606062620","37636365651","37666668682","37696971713","37727274744","37757577775","37787880806","37818183837","37848486868","37878789899","37909092930","37939395961","37969698992","38000002023","38030305054","38060608085","38090911116","38121214147","38151517178","38181820209","38212123240","38242426271","38272729302","38303032333","38333335364","38363638395","38393941426","38424244457","38454547488","38484850519","38515153550","38545456581","38575759612","38606062643","38636365674","38666668705","38696971736","38727274767","38757577798","38787880829","38818183860","38848486891","38878789922","38909092953","38939395984","38969699015","39000002046"],"frame_ids":["40681932426743845487196","40715250961685202076253","40748569496626558665310","40781888031567915254367","40815206566509271843424","40848525101450628432481","40881843636391985021538","40915162171333341610595","40948480706274698199652","40981799241216054788709","41015117776157411377766","41048436311098767966823","41081754846040124555880","41115073380981481144937","41148391915922837733994","41181710450864194323051","41215028985805550912108","41248347520746907501165","41281666055688264090222","41314984590629620679279","41348303125570977268336","41381621660512333857393","41414940195453690446450","41448258730395047035507","41481577265336403624564","41514895800277760213621","41548214335219116802678","41581532870160473391735","41614851405101829980792","41648169940043186569849","41681488474984543158906","41714807009925899747963","41748125544867256337020","41781444079808612926077","41814762614749969515134","41848081149691326104191","41881399684632682693248","41914718219574039282305","41948036754515395871362","41981355289456752460419","42014673824398109049476","42047992359339465638533","42081310894280822227590","42114629429222178816647","42147947964163535405704","42181266499104891994761","42214585034046248583818","42247903568987605172875","42281222103928961761932","42314540638870318350989","42347859173811674940046","42381177708753031529103","42414496243694388118160","42447814778635744707217","42481133313577101296274","42514451848518457885331","42547770383459814474388","42581088918401171063445","42614407453342527652502","42647725988283884241559","42681044523225240830616","42714363058166597419673","42747681593107954008730","42781000128049310597787","42814318662990667186844","42847637197932023775901","42880955732873380364958"],"sequence_ids":[604,605,606,607,608,609,610,611,612,613,614,615,616,617,618,619,620,621,622,623,624,625,626,627,628,629,630,631,632,633,634,635,636,637,638,639,640,641,642,643,644,645,646,647,648,649,650,651,652,653,654,655,656,657,658,659,660,661,662,663,664,665,666,667,668,669,670]}]},{"cohort_id":"synthetic-fit-v2-r0-c2-t1-0","trial_id":"synthetic-fit-v2-r0-c2-t1","session_id":"synthetic-fit-v2-r0-c2-t1","role":"Train","label":"OccupiedMoving","cohort_sha256":"9416b1c5d2c266b3cef633a824cb8705a7c4b657086e3db0eea6b555e0ee19f5","target_ns":"40000000000","sealed_at_ns":"42000002046","provenance":"Synthetic","sources":[{"source_id":1,"source_spec_sha256":"ea991b6de43cd497ae7a09091d30f7a4a181ce6ced05b217bca4dc3a396e8bdf","times_ns":["40000001000","40030304031","40060607062","40090910093","40121213124","40151516155","40181819186","40212122217","40242425248","40272728279","40303031310","40333334341","40363637372","40393940403","40424243434","40454546465","40484849496","40515152527","40545455558","40575758589","40606061620","40636364651","40666667682","40696970713","40727273744","40757576775","40787879806","40818182837","40848485868","40878788899","40909091930","40939394961","40969697992","41000001023","41030304054","41060607085","41090910116","41121213147","41151516178","41181819209","41212122240","41242425271","41272728302","41303031333","41333334364","41363637395","41393940426","41424243457","41454546488","41484849519","41515152550","41545455581","41575758612","41606061643","41636364674","41666667705","41696970736","41727273767","41757576798","41787879829","41818182860","41848485891","41878788922","41909091953","41939394984","41969698015","42000001046"],"frame_ids":["43980466210555922743967","44013784745497279333024","44047103280438635922081","44080421815379992511138","44113740350321349100195","44147058885262705689252","44180377420204062278309","44213695955145418867366","44247014490086775456423","44280333025028132045480","44313651559969488634537","44346970094910845223594","44380288629852201812651","44413607164793558401708","44446925699734914990765","44480244234676271579822","44513562769617628168879","44546881304558984757936","44580199839500341346993","44613518374441697936050","44646836909383054525107","44680155444324411114164","44713473979265767703221","44746792514207124292278","44780111049148480881335","44813429584089837470392","44846748119031194059449","44880066653972550648506","44913385188913907237563","44946703723855263826620","44980022258796620415677","45013340793737977004734","45046659328679333593791","45079977863620690182848","45113296398562046771905","45146614933503403360962","45179933468444759950019","45213252003386116539076","45246570538327473128133","45279889073268829717190","45313207608210186306247","45346526143151542895304","45379844678092899484361","45413163213034256073418","45446481747975612662475","45479800282916969251532","45513118817858325840589","45546437352799682429646","45579755887741039018703","45613074422682395607760","45646392957623752196817","45679711492565108785874","45713030027506465374931","45746348562447821963988","45779667097389178553045","45812985632330535142102","45846304167271891731159","45879622702213248320216","45912941237154604909273","45946259772095961498330","45979578307037318087387","46012896841978674676444","46046215376920031265501","46079533911861387854558","46112852446802744443615","46146170981744101032672","46179489516685457621729"],"sequence_ids":[671,672,673,674,675,676,677,678,679,680,681,682,683,684,685,686,687,688,689,690,691,692,693,694,695,696,697,698,699,700,701,702,703,704,705,706,707,708,709,710,711,712,713,714,715,716,717,718,719,720,721,722,723,724,725,726,727,728,729,730,731,732,733,734,735,736,737]},{"source_id":2,"source_spec_sha256":"ec550d93545ee35041bd6e85e79292ebf40dddb726353c1701a5aeeb232761a5","times_ns":["40000002000","40030305031","40060608062","40090911093","40121214124","40151517155","40181820186","40212123217","40242426248","40272729279","40303032310","40333335341","40363638372","40393941403","40424244434","40454547465","40484850496","40515153527","40545456558","40575759589","40606062620","40636365651","40666668682","40696971713","40727274744","40757577775","40787880806","40818183837","40848486868","40878789899","40909092930","40939395961","40969698992","41000002023","41030305054","41060608085","41090911116","41121214147","41151517178","41181820209","41212123240","41242426271","41272729302","41303032333","41333335364","41363638395","41393941426","41424244457","41454547488","41484850519","41515153550","41545456581","41575759612","41606062643","41636365674","41666668705","41696971736","41727274767","41757577798","41787880829","41818183860","41848486891","41878789922","41909092953","41939395984","41969699015","42000002046"],"frame_ids":["43980467310071845487263","44013785845013202076320","44047104379954558665377","44080422914895915254434","44113741449837271843491","44147059984778628432548","44180378519719985021605","44213697054661341610662","44247015589602698199719","44280334124544054788776","44313652659485411377833","44346971194426767966890","44380289729368124555947","44413608264309481145004","44446926799250837734061","44480245334192194323118","44513563869133550912175","44546882404074907501232","44580200939016264090289","44613519473957620679346","44646838008898977268403","44680156543840333857460","44713475078781690446517","44746793613723047035574","44780112148664403624631","44813430683605760213688","44846749218547116802745","44880067753488473391802","44913386288429829980859","44946704823371186569916","44980023358312543158973","45013341893253899748030","45046660428195256337087","45079978963136612926144","45113297498077969515201","45146616033019326104258","45179934567960682693315","45213253102902039282372","45246571637843395871429","45279890172784752460486","45313208707726109049543","45346527242667465638600","45379845777608822227657","45413164312550178816714","45446482847491535405771","45479801382432891994828","45513119917374248583885","45546438452315605172942","45579756987256961761999","45613075522198318351056","45646394057139674940113","45679712592081031529170","45713031127022388118227","45746349661963744707284","45779668196905101296341","45812986731846457885398","45846305266787814474455","45879623801729171063512","45912942336670527652569","45946260871611884241626","45979579406553240830683","46012897941494597419740","46046216476435954008797","46079535011377310597854","46112853546318667186911","46146172081260023775968","46179490616201380365025"],"sequence_ids":[671,672,673,674,675,676,677,678,679,680,681,682,683,684,685,686,687,688,689,690,691,692,693,694,695,696,697,698,699,700,701,702,703,704,705,706,707,708,709,710,711,712,713,714,715,716,717,718,719,720,721,722,723,724,725,726,727,728,729,730,731,732,733,734,735,736,737]}]},{"cohort_id":"synthetic-fit-v2-r0-c2-t1-1","trial_id":"synthetic-fit-v2-r0-c2-t1","session_id":"synthetic-fit-v2-r0-c2-t1","role":"Train","label":"OccupiedMoving","cohort_sha256":"2278fc454bc1504bc787c44cbee4e25957e32cc3659e4cda073dd9d93d735932","target_ns":"43000000000","sealed_at_ns":"45000002046","provenance":"Synthetic","sources":[{"source_id":1,"source_spec_sha256":"ea991b6de43cd497ae7a09091d30f7a4a181ce6ced05b217bca4dc3a396e8bdf","times_ns":["43000001000","43030304031","43060607062","43090910093","43121213124","43151516155","43181819186","43212122217","43242425248","43272728279","43303031310","43333334341","43363637372","43393940403","43424243434","43454546465","43484849496","43515152527","43545455558","43575758589","43606061620","43636364651","43666667682","43696970713","43727273744","43757576775","43787879806","43818182837","43848485868","43878788899","43909091930","43939394961","43969697992","44000001023","44030304054","44060607085","44090910116","44121213147","44151516178","44181819209","44212122240","44242425271","44272728302","44303031333","44333334364","44363637395","44393940426","44424243457","44454546488","44484849519","44515152550","44545455581","44575758612","44606061643","44636364674","44666667705","44696970736","44727273767","44757576798","44787879829","44818182860","44848485891","44878788922","44909091953","44939394984","44969698015","45000001046"],"frame_ids":["47279001093883922744034","47312319628825279333091","47345638163766635922148","47378956698707992511205","47412275233649349100262","47445593768590705689319","47478912303532062278376","47512230838473418867433","47545549373414775456490","47578867908356132045547","47612186443297488634604","47645504978238845223661","47678823513180201812718","47712142048121558401775","47745460583062914990832","47778779118004271579889","47812097652945628168946","47845416187886984758003","47878734722828341347060","47912053257769697936117","47945371792711054525174","47978690327652411114231","48012008862593767703288","48045327397535124292345","48078645932476480881402","48111964467417837470459","48145283002359194059516","48178601537300550648573","48211920072241907237630","48245238607183263826687","48278557142124620415744","48311875677065977004801","48345194212007333593858","48378512746948690182915","48411831281890046771972","48445149816831403361029","48478468351772759950086","48511786886714116539143","48545105421655473128200","48578423956596829717257","48611742491538186306314","48645061026479542895371","48678379561420899484428","48711698096362256073485","48745016631303612662542","48778335166244969251599","48811653701186325840656","48844972236127682429713","48878290771069039018770","48911609306010395607827","48944927840951752196884","48978246375893108785941","49011564910834465374998","49044883445775821964055","49078201980717178553112","49111520515658535142169","49144839050599891731226","49178157585541248320283","49211476120482604909340","49244794655423961498397","49278113190365318087454","49311431725306674676511","49344750260248031265568","49378068795189387854625","49411387330130744443682","49444705865072101032739","49478024400013457621796"],"sequence_ids":[738,739,740,741,742,743,744,745,746,747,748,749,750,751,752,753,754,755,756,757,758,759,760,761,762,763,764,765,766,767,768,769,770,771,772,773,774,775,776,777,778,779,780,781,782,783,784,785,786,787,788,789,790,791,792,793,794,795,796,797,798,799,800,801,802,803,804]},{"source_id":2,"source_spec_sha256":"ec550d93545ee35041bd6e85e79292ebf40dddb726353c1701a5aeeb232761a5","times_ns":["43000002000","43030305031","43060608062","43090911093","43121214124","43151517155","43181820186","43212123217","43242426248","43272729279","43303032310","43333335341","43363638372","43393941403","43424244434","43454547465","43484850496","43515153527","43545456558","43575759589","43606062620","43636365651","43666668682","43696971713","43727274744","43757577775","43787880806","43818183837","43848486868","43878789899","43909092930","43939395961","43969698992","44000002023","44030305054","44060608085","44090911116","44121214147","44151517178","44181820209","44212123240","44242426271","44272729302","44303032333","44333335364","44363638395","44393941426","44424244457","44454547488","44484850519","44515153550","44545456581","44575759612","44606062643","44636365674","44666668705","44696971736","44727274767","44757577798","44787880829","44818183860","44848486891","44878789922","44909092953","44939395984","44969699015","45000002046"],"frame_ids":["47279002193399845487330","47312320728341202076387","47345639263282558665444","47378957798223915254501","47412276333165271843558","47445594868106628432615","47478913403047985021672","47512231937989341610729","47545550472930698199786","47578869007872054788843","47612187542813411377900","47645506077754767966957","47678824612696124556014","47712143147637481145071","47745461682578837734128","47778780217520194323185","47812098752461550912242","47845417287402907501299","47878735822344264090356","47912054357285620679413","47945372892226977268470","47978691427168333857527","48012009962109690446584","48045328497051047035641","48078647031992403624698","48111965566933760213755","48145284101875116802812","48178602636816473391869","48211921171757829980926","48245239706699186569983","48278558241640543159040","48311876776581899748097","48345195311523256337154","48378513846464612926211","48411832381405969515268","48445150916347326104325","48478469451288682693382","48511787986230039282439","48545106521171395871496","48578425056112752460553","48611743591054109049610","48645062125995465638667","48678380660936822227724","48711699195878178816781","48745017730819535405838","48778336265760891994895","48811654800702248583952","48844973335643605173009","48878291870584961762066","48911610405526318351123","48944928940467674940180","48978247475409031529237","49011566010350388118294","49044884545291744707351","49078203080233101296408","49111521615174457885465","49144840150115814474522","49178158685057171063579","49211477219998527652636","49244795754939884241693","49278114289881240830750","49311432824822597419807","49344751359763954008864","49378069894705310597921","49411388429646667186978","49444706964588023776035","49478025499529380365092"],"sequence_ids":[738,739,740,741,742,743,744,745,746,747,748,749,750,751,752,753,754,755,756,757,758,759,760,761,762,763,764,765,766,767,768,769,770,771,772,773,774,775,776,777,778,779,780,781,782,783,784,785,786,787,788,789,790,791,792,793,794,795,796,797,798,799,800,801,802,803,804]}]},{"cohort_id":"synthetic-fit-v2-r0-c3-t0-0","trial_id":"synthetic-fit-v2-r0-c3-t0","session_id":"synthetic-fit-v2-r0-c3-t0","role":"Train","label":"AdjacentOnly","cohort_sha256":"770cb4ac22965a33d9895af740220efe788c6e79724d2c3cd8d4184235de6204","target_ns":"46000000000","sealed_at_ns":"48000002046","provenance":"Synthetic","sources":[{"source_id":1,"source_spec_sha256":"ea991b6de43cd497ae7a09091d30f7a4a181ce6ced05b217bca4dc3a396e8bdf","times_ns":["46000001000","46030304031","46060607062","46090910093","46121213124","46151516155","46181819186","46212122217","46242425248","46272728279","46303031310","46333334341","46363637372","46393940403","46424243434","46454546465","46484849496","46515152527","46545455558","46575758589","46606061620","46636364651","46666667682","46696970713","46727273744","46757576775","46787879806","46818182837","46848485868","46878788899","46909091930","46939394961","46969697992","47000001023","47030304054","47060607085","47090910116","47121213147","47151516178","47181819209","47212122240","47242425271","47272728302","47303031333","47333334364","47363637395","47393940426","47424243457","47454546488","47484849519","47515152550","47545455581","47575758612","47606061643","47636364674","47666667705","47696970736","47727273767","47757576798","47787879829","47818182860","47848485891","47878788922","47909091953","47939394984","47969698015","48000001046"],"frame_ids":["50577535977211922744101","50610854512153279333158","50644173047094635922215","50677491582035992511272","50710810116977349100329","50744128651918705689386","50777447186860062278443","50810765721801418867500","50844084256742775456557","50877402791684132045614","50910721326625488634671","50944039861566845223728","50977358396508201812785","51010676931449558401842","51043995466390914990899","51077314001332271579956","51110632536273628169013","51143951071214984758070","51177269606156341347127","51210588141097697936184","51243906676039054525241","51277225210980411114298","51310543745921767703355","51343862280863124292412","51377180815804480881469","51410499350745837470526","51443817885687194059583","51477136420628550648640","51510454955569907237697","51543773490511263826754","51577092025452620415811","51610410560393977004868","51643729095335333593925","51677047630276690182982","51710366165218046772039","51743684700159403361096","51777003235100759950153","51810321770042116539210","51843640304983473128267","51876958839924829717324","51910277374866186306381","51943595909807542895438","51976914444748899484495","52010232979690256073552","52043551514631612662609","52076870049572969251666","52110188584514325840723","52143507119455682429780","52176825654397039018837","52210144189338395607894","52243462724279752196951","52276781259221108786008","52310099794162465375065","52343418329103821964122","52376736864045178553179","52410055398986535142236","52443373933927891731293","52476692468869248320350","52510011003810604909407","52543329538751961498464","52576648073693318087521","52609966608634674676578","52643285143576031265635","52676603678517387854692","52709922213458744443749","52743240748400101032806","52776559283341457621863"],"sequence_ids":[805,806,807,808,809,810,811,812,813,814,815,816,817,818,819,820,821,822,823,824,825,826,827,828,829,830,831,832,833,834,835,836,837,838,839,840,841,842,843,844,845,846,847,848,849,850,851,852,853,854,855,856,857,858,859,860,861,862,863,864,865,866,867,868,869,870,871]},{"source_id":2,"source_spec_sha256":"ec550d93545ee35041bd6e85e79292ebf40dddb726353c1701a5aeeb232761a5","times_ns":["46000002000","46030305031","46060608062","46090911093","46121214124","46151517155","46181820186","46212123217","46242426248","46272729279","46303032310","46333335341","46363638372","46393941403","46424244434","46454547465","46484850496","46515153527","46545456558","46575759589","46606062620","46636365651","46666668682","46696971713","46727274744","46757577775","46787880806","46818183837","46848486868","46878789899","46909092930","46939395961","46969698992","47000002023","47030305054","47060608085","47090911116","47121214147","47151517178","47181820209","47212123240","47242426271","47272729302","47303032333","47333335364","47363638395","47393941426","47424244457","47454547488","47484850519","47515153550","47545456581","47575759612","47606062643","47636365674","47666668705","47696971736","47727274767","47757577798","47787880829","47818183860","47848486891","47878789922","47909092953","47939395984","47969699015","48000002046"],"frame_ids":["50577537076727845487397","50610855611669202076454","50644174146610558665511","50677492681551915254568","50710811216493271843625","50744129751434628432682","50777448286375985021739","50810766821317341610796","50844085356258698199853","50877403891200054788910","50910722426141411377967","50944040961082767967024","50977359496024124556081","51010678030965481145138","51043996565906837734195","51077315100848194323252","51110633635789550912309","51143952170730907501366","51177270705672264090423","51210589240613620679480","51243907775554977268537","51277226310496333857594","51310544845437690446651","51343863380379047035708","51377181915320403624765","51410500450261760213822","51443818985203116802879","51477137520144473391936","51510456055085829980993","51543774590027186570050","51577093124968543159107","51610411659909899748164","51643730194851256337221","51677048729792612926278","51710367264733969515335","51743685799675326104392","51777004334616682693449","51810322869558039282506","51843641404499395871563","51876959939440752460620","51910278474382109049677","51943597009323465638734","51976915544264822227791","52010234079206178816848","52043552614147535405905","52076871149088891994962","52110189684030248584019","52143508218971605173076","52176826753912961762133","52210145288854318351190","52243463823795674940247","52276782358737031529304","52310100893678388118361","52343419428619744707418","52376737963561101296475","52410056498502457885532","52443375033443814474589","52476693568385171063646","52510012103326527652703","52543330638267884241760","52576649173209240830817","52609967708150597419874","52643286243091954008931","52676604778033310597988","52709923312974667187045","52743241847916023776102","52776560382857380365159"],"sequence_ids":[805,806,807,808,809,810,811,812,813,814,815,816,817,818,819,820,821,822,823,824,825,826,827,828,829,830,831,832,833,834,835,836,837,838,839,840,841,842,843,844,845,846,847,848,849,850,851,852,853,854,855,856,857,858,859,860,861,862,863,864,865,866,867,868,869,870,871]}]},{"cohort_id":"synthetic-fit-v2-r0-c3-t0-1","trial_id":"synthetic-fit-v2-r0-c3-t0","session_id":"synthetic-fit-v2-r0-c3-t0","role":"Train","label":"AdjacentOnly","cohort_sha256":"760fc88fee72cff507da886a62ef718dd5f125ad9dd03ba30e672c17f5aad7c7","target_ns":"49000000000","sealed_at_ns":"51000002046","provenance":"Synthetic","sources":[{"source_id":1,"source_spec_sha256":"ea991b6de43cd497ae7a09091d30f7a4a181ce6ced05b217bca4dc3a396e8bdf","times_ns":["49000001000","49030304031","49060607062","49090910093","49121213124","49151516155","49181819186","49212122217","49242425248","49272728279","49303031310","49333334341","49363637372","49393940403","49424243434","49454546465","49484849496","49515152527","49545455558","49575758589","49606061620","49636364651","49666667682","49696970713","49727273744","49757576775","49787879806","49818182837","49848485868","49878788899","49909091930","49939394961","49969697992","50000001023","50030304054","50060607085","50090910116","50121213147","50151516178","50181819209","50212122240","50242425271","50272728302","50303031333","50333334364","50363637395","50393940426","50424243457","50454546488","50484849519","50515152550","50545455581","50575758612","50606061643","50636364674","50666667705","50696970736","50727273767","50757576798","50787879829","50818182860","50848485891","50878788922","50909091953","50939394984","50969698015","51000001046"],"frame_ids":["53876070860539922744168","53909389395481279333225","53942707930422635922282","53976026465363992511339","54009345000305349100396","54042663535246705689453","54075982070188062278510","54109300605129418867567","54142619140070775456624","54175937675012132045681","54209256209953488634738","54242574744894845223795","54275893279836201812852","54309211814777558401909","54342530349718914990966","54375848884660271580023","54409167419601628169080","54442485954542984758137","54475804489484341347194","54509123024425697936251","54542441559367054525308","54575760094308411114365","54609078629249767703422","54642397164191124292479","54675715699132480881536","54709034234073837470593","54742352769015194059650","54775671303956550648707","54808989838897907237764","54842308373839263826821","54875626908780620415878","54908945443721977004935","54942263978663333593992","54975582513604690183049","55008901048546046772106","55042219583487403361163","55075538118428759950220","55108856653370116539277","55142175188311473128334","55175493723252829717391","55208812258194186306448","55242130793135542895505","55275449328076899484562","55308767863018256073619","55342086397959612662676","55375404932900969251733","55408723467842325840790","55442042002783682429847","55475360537725039018904","55508679072666395607961","55541997607607752197018","55575316142549108786075","55608634677490465375132","55641953212431821964189","55675271747373178553246","55708590282314535142303","55741908817255891731360","55775227352197248320417","55808545887138604909474","55841864422079961498531","55875182957021318087588","55908501491962674676645","55941820026904031265702","55975138561845387854759","56008457096786744443816","56041775631728101032873","56075094166669457621930"],"sequence_ids":[872,873,874,875,876,877,878,879,880,881,882,883,884,885,886,887,888,889,890,891,892,893,894,895,896,897,898,899,900,901,902,903,904,905,906,907,908,909,910,911,912,913,914,915,916,917,918,919,920,921,922,923,924,925,926,927,928,929,930,931,932,933,934,935,936,937,938]},{"source_id":2,"source_spec_sha256":"ec550d93545ee35041bd6e85e79292ebf40dddb726353c1701a5aeeb232761a5","times_ns":["49000002000","49030305031","49060608062","49090911093","49121214124","49151517155","49181820186","49212123217","49242426248","49272729279","49303032310","49333335341","49363638372","49393941403","49424244434","49454547465","49484850496","49515153527","49545456558","49575759589","49606062620","49636365651","49666668682","49696971713","49727274744","49757577775","49787880806","49818183837","49848486868","49878789899","49909092930","49939395961","49969698992","50000002023","50030305054","50060608085","50090911116","50121214147","50151517178","50181820209","50212123240","50242426271","50272729302","50303032333","50333335364","50363638395","50393941426","50424244457","50454547488","50484850519","50515153550","50545456581","50575759612","50606062643","50636365674","50666668705","50696971736","50727274767","50757577798","50787880829","50818183860","50848486891","50878789922","50909092953","50939395984","50969699015","51000002046"],"frame_ids":["53876071960055845487464","53909390494997202076521","53942709029938558665578","53976027564879915254635","54009346099821271843692","54042664634762628432749","54075983169703985021806","54109301704645341610863","54142620239586698199920","54175938774528054788977","54209257309469411378034","54242575844410767967091","54275894379352124556148","54309212914293481145205","54342531449234837734262","54375849984176194323319","54409168519117550912376","54442487054058907501433","54475805589000264090490","54509124123941620679547","54542442658882977268604","54575761193824333857661","54609079728765690446718","54642398263707047035775","54675716798648403624832","54709035333589760213889","54742353868531116802946","54775672403472473392003","54808990938413829981060","54842309473355186570117","54875628008296543159174","54908946543237899748231","54942265078179256337288","54975583613120612926345","55008902148061969515402","55042220683003326104459","55075539217944682693516","55108857752886039282573","55142176287827395871630","55175494822768752460687","55208813357710109049744","55242131892651465638801","55275450427592822227858","55308768962534178816915","55342087497475535405972","55375406032416891995029","55408724567358248584086","55442043102299605173143","55475361637240961762200","55508680172182318351257","55541998707123674940314","55575317242065031529371","55608635777006388118428","55641954311947744707485","55675272846889101296542","55708591381830457885599","55741909916771814474656","55775228451713171063713","55808546986654527652770","55841865521595884241827","55875184056537240830884","55908502591478597419941","55941821126419954008998","55975139661361310598055","56008458196302667187112","56041776731244023776169","56075095266185380365226"],"sequence_ids":[872,873,874,875,876,877,878,879,880,881,882,883,884,885,886,887,888,889,890,891,892,893,894,895,896,897,898,899,900,901,902,903,904,905,906,907,908,909,910,911,912,913,914,915,916,917,918,919,920,921,922,923,924,925,926,927,928,929,930,931,932,933,934,935,936,937,938]}]},{"cohort_id":"synthetic-fit-v2-r0-c3-t1-0","trial_id":"synthetic-fit-v2-r0-c3-t1","session_id":"synthetic-fit-v2-r0-c3-t1","role":"Train","label":"AdjacentOnly","cohort_sha256":"3f567fe2f7357e982b18a0eebe3958175e1443d7bc4590484f450e4ba57cb3c0","target_ns":"52000000000","sealed_at_ns":"54000002046","provenance":"Synthetic","sources":[{"source_id":1,"source_spec_sha256":"ea991b6de43cd497ae7a09091d30f7a4a181ce6ced05b217bca4dc3a396e8bdf","times_ns":["52000001000","52030304031","52060607062","52090910093","52121213124","52151516155","52181819186","52212122217","52242425248","52272728279","52303031310","52333334341","52363637372","52393940403","52424243434","52454546465","52484849496","52515152527","52545455558","52575758589","52606061620","52636364651","52666667682","52696970713","52727273744","52757576775","52787879806","52818182837","52848485868","52878788899","52909091930","52939394961","52969697992","53000001023","53030304054","53060607085","53090910116","53121213147","53151516178","53181819209","53212122240","53242425271","53272728302","53303031333","53333334364","53363637395","53393940426","53424243457","53454546488","53484849519","53515152550","53545455581","53575758612","53606061643","53636364674","53666667705","53696970736","53727273767","53757576798","53787879829","53818182860","53848485891","53878788922","53909091953","53939394984","53969698015","54000001046"],"frame_ids":["57174605743867922744235","57207924278809279333292","57241242813750635922349","57274561348691992511406","57307879883633349100463","57341198418574705689520","57374516953516062278577","57407835488457418867634","57441154023398775456691","57474472558340132045748","57507791093281488634805","57541109628222845223862","57574428163164201812919","57607746698105558401976","57641065233046914991033","57674383767988271580090","57707702302929628169147","57741020837870984758204","57774339372812341347261","57807657907753697936318","57840976442695054525375","57874294977636411114432","57907613512577767703489","57940932047519124292546","57974250582460480881603","58007569117401837470660","58040887652343194059717","58074206187284550648774","58107524722225907237831","58140843257167263826888","58174161792108620415945","58207480327049977005002","58240798861991333594059","58274117396932690183116","58307435931874046772173","58340754466815403361230","58374073001756759950287","58407391536698116539344","58440710071639473128401","58474028606580829717458","58507347141522186306515","58540665676463542895572","58573984211404899484629","58607302746346256073686","58640621281287612662743","58673939816228969251800","58707258351170325840857","58740576886111682429914","58773895421053039018971","58807213955994395608028","58840532490935752197085","58873851025877108786142","58907169560818465375199","58940488095759821964256","58973806630701178553313","59007125165642535142370","59040443700583891731427","59073762235525248320484","59107080770466604909541","59140399305407961498598","59173717840349318087655","59207036375290674676712","59240354910232031265769","59273673445173387854826","59306991980114744443883","59340310515056101032940","59373629049997457621997"],"sequence_ids":[939,940,941,942,943,944,945,946,947,948,949,950,951,952,953,954,955,956,957,958,959,960,961,962,963,964,965,966,967,968,969,970,971,972,973,974,975,976,977,978,979,980,981,982,983,984,985,986,987,988,989,990,991,992,993,994,995,996,997,998,999,1000,1001,1002,1003,1004,1005]},{"source_id":2,"source_spec_sha256":"ec550d93545ee35041bd6e85e79292ebf40dddb726353c1701a5aeeb232761a5","times_ns":["52000002000","52030305031","52060608062","52090911093","52121214124","52151517155","52181820186","52212123217","52242426248","52272729279","52303032310","52333335341","52363638372","52393941403","52424244434","52454547465","52484850496","52515153527","52545456558","52575759589","52606062620","52636365651","52666668682","52696971713","52727274744","52757577775","52787880806","52818183837","52848486868","52878789899","52909092930","52939395961","52969698992","53000002023","53030305054","53060608085","53090911116","53121214147","53151517178","53181820209","53212123240","53242426271","53272729302","53303032333","53333335364","53363638395","53393941426","53424244457","53454547488","53484850519","53515153550","53545456581","53575759612","53606062643","53636365674","53666668705","53696971736","53727274767","53757577798","53787880829","53818183860","53848486891","53878789922","53909092953","53939395984","53969699015","54000002046"],"frame_ids":["57174606843383845487531","57207925378325202076588","57241243913266558665645","57274562448207915254702","57307880983149271843759","57341199518090628432816","57374518053031985021873","57407836587973341610930","57441155122914698199987","57474473657856054789044","57507792192797411378101","57541110727738767967158","57574429262680124556215","57607747797621481145272","57641066332562837734329","57674384867504194323386","57707703402445550912443","57741021937386907501500","57774340472328264090557","57807659007269620679614","57840977542210977268671","57874296077152333857728","57907614612093690446785","57940933147035047035842","57974251681976403624899","58007570216917760213956","58040888751859116803013","58074207286800473392070","58107525821741829981127","58140844356683186570184","58174162891624543159241","58207481426565899748298","58240799961507256337355","58274118496448612926412","58307437031389969515469","58340755566331326104526","58374074101272682693583","58407392636214039282640","58440711171155395871697","58474029706096752460754","58507348241038109049811","58540666775979465638868","58573985310920822227925","58607303845862178816982","58640622380803535406039","58673940915744891995096","58707259450686248584153","58740577985627605173210","58773896520568961762267","58807215055510318351324","58840533590451674940381","58873852125393031529438","58907170660334388118495","58940489195275744707552","58973807730217101296609","59007126265158457885666","59040444800099814474723","59073763335041171063780","59107081869982527652837","59140400404923884241894","59173718939865240830951","59207037474806597420008","59240356009747954009065","59273674544689310598122","59306993079630667187179","59340311614572023776236","59373630149513380365293"],"sequence_ids":[939,940,941,942,943,944,945,946,947,948,949,950,951,952,953,954,955,956,957,958,959,960,961,962,963,964,965,966,967,968,969,970,971,972,973,974,975,976,977,978,979,980,981,982,983,984,985,986,987,988,989,990,991,992,993,994,995,996,997,998,999,1000,1001,1002,1003,1004,1005]}]},{"cohort_id":"synthetic-fit-v2-r0-c3-t1-1","trial_id":"synthetic-fit-v2-r0-c3-t1","session_id":"synthetic-fit-v2-r0-c3-t1","role":"Train","label":"AdjacentOnly","cohort_sha256":"fcd7005b9f0f277ba32119af2d48939fa885104ffb4efa5768bc8313eea2a1c0","target_ns":"55000000000","sealed_at_ns":"57000002046","provenance":"Synthetic","sources":[{"source_id":1,"source_spec_sha256":"ea991b6de43cd497ae7a09091d30f7a4a181ce6ced05b217bca4dc3a396e8bdf","times_ns":["55000001000","55030304031","55060607062","55090910093","55121213124","55151516155","55181819186","55212122217","55242425248","55272728279","55303031310","55333334341","55363637372","55393940403","55424243434","55454546465","55484849496","55515152527","55545455558","55575758589","55606061620","55636364651","55666667682","55696970713","55727273744","55757576775","55787879806","55818182837","55848485868","55878788899","55909091930","55939394961","55969697992","56000001023","56030304054","56060607085","56090910116","56121213147","56151516178","56181819209","56212122240","56242425271","56272728302","56303031333","56333334364","56363637395","56393940426","56424243457","56454546488","56484849519","56515152550","56545455581","56575758612","56606061643","56636364674","56666667705","56696970736","56727273767","56757576798","56787879829","56818182860","56848485891","56878788922","56909091953","56939394984","56969698015","57000001046"],"frame_ids":["60473140627195922744302","60506459162137279333359","60539777697078635922416","60573096232019992511473","60606414766961349100530","60639733301902705689587","60673051836844062278644","60706370371785418867701","60739688906726775456758","60773007441668132045815","60806325976609488634872","60839644511550845223929","60872963046492201812986","60906281581433558402043","60939600116374914991100","60972918651316271580157","61006237186257628169214","61039555721198984758271","61072874256140341347328","61106192791081697936385","61139511326023054525442","61172829860964411114499","61206148395905767703556","61239466930847124292613","61272785465788480881670","61306104000729837470727","61339422535671194059784","61372741070612550648841","61406059605553907237898","61439378140495263826955","61472696675436620416012","61506015210377977005069","61539333745319333594126","61572652280260690183183","61605970815202046772240","61639289350143403361297","61672607885084759950354","61705926420026116539411","61739244954967473128468","61772563489908829717525","61805882024850186306582","61839200559791542895639","61872519094732899484696","61905837629674256073753","61939156164615612662810","61972474699556969251867","62005793234498325840924","62039111769439682429981","62072430304381039019038","62105748839322395608095","62139067374263752197152","62172385909205108786209","62205704444146465375266","62239022979087821964323","62272341514029178553380","62305660048970535142437","62338978583911891731494","62372297118853248320551","62405615653794604909608","62438934188735961498665","62472252723677318087722","62505571258618674676779","62538889793560031265836","62572208328501387854893","62605526863442744443950","62638845398384101033007","62672163933325457622064"],"sequence_ids":[1006,1007,1008,1009,1010,1011,1012,1013,1014,1015,1016,1017,1018,1019,1020,1021,1022,1023,1024,1025,1026,1027,1028,1029,1030,1031,1032,1033,1034,1035,1036,1037,1038,1039,1040,1041,1042,1043,1044,1045,1046,1047,1048,1049,1050,1051,1052,1053,1054,1055,1056,1057,1058,1059,1060,1061,1062,1063,1064,1065,1066,1067,1068,1069,1070,1071,1072]},{"source_id":2,"source_spec_sha256":"ec550d93545ee35041bd6e85e79292ebf40dddb726353c1701a5aeeb232761a5","times_ns":["55000002000","55030305031","55060608062","55090911093","55121214124","55151517155","55181820186","55212123217","55242426248","55272729279","55303032310","55333335341","55363638372","55393941403","55424244434","55454547465","55484850496","55515153527","55545456558","55575759589","55606062620","55636365651","55666668682","55696971713","55727274744","55757577775","55787880806","55818183837","55848486868","55878789899","55909092930","55939395961","55969698992","56000002023","56030305054","56060608085","56090911116","56121214147","56151517178","56181820209","56212123240","56242426271","56272729302","56303032333","56333335364","56363638395","56393941426","56424244457","56454547488","56484850519","56515153550","56545456581","56575759612","56606062643","56636365674","56666668705","56696971736","56727274767","56757577798","56787880829","56818183860","56848486891","56878789922","56909092953","56939395984","56969699015","57000002046"],"frame_ids":["60473141726711845487598","60506460261653202076655","60539778796594558665712","60573097331535915254769","60606415866477271843826","60639734401418628432883","60673052936359985021940","60706371471301341610997","60739690006242698200054","60773008541184054789111","60806327076125411378168","60839645611066767967225","60872964146008124556282","60906282680949481145339","60939601215890837734396","60972919750832194323453","61006238285773550912510","61039556820714907501567","61072875355656264090624","61106193890597620679681","61139512425538977268738","61172830960480333857795","61206149495421690446852","61239468030363047035909","61272786565304403624966","61306105100245760214023","61339423635187116803080","61372742170128473392137","61406060705069829981194","61439379240011186570251","61472697774952543159308","61506016309893899748365","61539334844835256337422","61572653379776612926479","61605971914717969515536","61639290449659326104593","61672608984600682693650","61705927519542039282707","61739246054483395871764","61772564589424752460821","61805883124366109049878","61839201659307465638935","61872520194248822227992","61905838729190178817049","61939157264131535406106","61972475799072891995163","62005794334014248584220","62039112868955605173277","62072431403896961762334","62105749938838318351391","62139068473779674940448","62172387008721031529505","62205705543662388118562","62239024078603744707619","62272342613545101296676","62305661148486457885733","62338979683427814474790","62372298218369171063847","62405616753310527652904","62438935288251884241961","62472253823193240831018","62505572358134597420075","62538890893075954009132","62572209428017310598189","62605527962958667187246","62638846497900023776303","62672165032841380365360"],"sequence_ids":[1006,1007,1008,1009,1010,1011,1012,1013,1014,1015,1016,1017,1018,1019,1020,1021,1022,1023,1024,1025,1026,1027,1028,1029,1030,1031,1032,1033,1034,1035,1036,1037,1038,1039,1040,1041,1042,1043,1044,1045,1046,1047,1048,1049,1050,1051,1052,1053,1054,1055,1056,1057,1058,1059,1060,1061,1062,1063,1064,1065,1066,1067,1068,1069,1070,1071,1072]}]}],"model_sha256":"0439ec86ebd7b3ae0285cf89a768e0d70bdc37f581a48ca7a5626240a8e06db0","model_json":"{\"version\":1,\"algorithm\":\"aether_multiview_infonce_room_bce_v1\",\"context\":{\"owner_scope\":\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"installation_id\":\"synthetic-temporal\",\"room_digest\":\"synthetic-room\",\"revision\":1,\"sources\":[{\"source_id\":1,\"device_id\":\"esp32-node-1\",\"grid_id\":\"ant1-bins64-ppdu0-usable-drop2-linear-v2\",\"feature_count\":56,\"signal_kind\":\"WifiCsi\",\"center_frequency_hz\":2437000000,\"bandwidth_hz\":20000000,\"link_id\":\"native-1\",\"link_index\":0,\"geometry\":null},{\"source_id\":2,\"device_id\":\"esp32-node-2\",\"grid_id\":\"ant1-bins64-ppdu0-usable-drop2-linear-v2\",\"feature_count\":56,\"signal_kind\":\"WifiCsi\",\"center_frequency_hz\":2437000000,\"bandwidth_hz\":20000000,\"link_id\":\"native-2\",\"link_index\":0,\"geometry\":null}]},\"training_manifest\":{\"schema\":\"ruview.room-learner.train-manifest.v1\",\"training_provenance\":[\"Synthetic\"],\"training_data_sha256\":\"fe5579c3ff4e5853f1885dba20f1adc996e198d4c506c95a971e6afda6e6d9f0\",\"consumed_channels\":[\"magnitude\"],\"clock_claims\":[\"ReceiverOnly\"],\"measurement_clock_available\":false,\"training_cohorts\":16},\"config\":{\"bins\":56,\"source_count\":2,\"temporal_tokens\":4,\"hidden_dim\":3,\"embedding_dim\":3,\"temperature\":0.3},\"profiles\":[{\"source\":{\"source_id\":1,\"device_id\":\"esp32-node-1\",\"grid_id\":\"ant1-bins64-ppdu0-usable-drop2-linear-v2\",\"feature_count\":56,\"signal_kind\":\"WifiCsi\",\"center_frequency_hz\":2437000000,\"bandwidth_hz\":20000000,\"link_id\":\"native-1\",\"link_index\":0,\"geometry\":null},\"mean\":[5.656854,6.321713,5.819677,6.1588902,5.982499,5.996068,6.145322,5.8332453,6.3081446,5.670423,6.3352814,5.8061085,6.172459,5.9689307,6.0096364,6.1317534,5.8468137,6.2945757,5.6839914,6.3488503,5.7925396,6.1860275,5.9553623,6.023205,6.1181846,5.8603826,6.2810073,5.69756,6.3624187,5.778971,6.199596,5.941794,6.0367737,6.104616,5.873951,6.267439,5.711128,6.375987,5.765403,6.213165,5.928225,6.050342,6.091048,5.8875194,6.25387,5.724697,6.3895555,5.751834,6.226733,5.9146566,6.0639105,6.0774794,5.901088,6.2403016,5.7382655,6.4031243],\"scale\":[0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1],\"max_normalized\":71.98347},{\"source\":{\"source_id\":2,\"device_id\":\"esp32-node-2\",\"grid_id\":\"ant1-bins64-ppdu0-usable-drop2-linear-v2\",\"feature_count\":56,\"signal_kind\":\"WifiCsi\",\"center_frequency_hz\":2437000000,\"bandwidth_hz\":20000000,\"link_id\":\"native-2\",\"link_index\":0,\"geometry\":null},\"mean\":[6.4031243,7.1229596,6.5794106,6.9466734,6.755697,6.770387,6.931983,6.594101,7.1082687,6.4178147,7.13765,6.5647197,6.961364,6.741006,6.7850776,6.917292,6.6087914,7.0935783,6.432505,7.1523404,6.5500293,6.976054,6.7263155,6.799768,6.9026017,6.6234818,7.078888,6.447196,7.167031,6.535339,6.9907446,6.711625,6.814459,6.8879113,6.6381726,7.0641975,6.4618864,7.1817217,6.5206485,7.0054355,6.6969347,6.8291492,6.873221,6.652863,7.0495067,6.476577,7.196412,6.5059576,7.020126,6.682244,6.8438396,6.85853,6.6675534,7.0348163,6.491267,7.2111025],\"scale\":[0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1],\"max_normalized\":73.49117}],\"encoder_weights\":[-0.31405702,-0.15605538,-0.019411659,-0.23722431,0.14105025,0.12448881,-0.17316124,-0.28230542,0.25161946,0.23627274,0.083663724,-0.11510868,0.16076146,-0.20444109,-0.28943694,-0.03411386,-0.05305876,0.1268511,-0.07930206,-0.16969015,-0.122171946,-0.18971951,0.2779942,-0.20771396,0.09614456,-0.27768067,-0.083600804,0.060154844,-0.27317116,-0.25799847,0.054317452,-0.31186154,-0.021532401,-0.24256697,-0.28336817,0.05801737,0.23311336,-0.08284997,0.18425474,0.3019741,0.0971509,-0.014180818,0.077958286,-0.21174139,-0.1348683,-0.18674442,-0.087604724,-0.13767663,0.10686318,0.17436363,0.1432364,0.08204346,-0.11314485,0.191478,-0.18946445,0.08444698,-1.0963722,-0.40854564,0.010921259,0.1592275,-0.24847019,0.28674957,0.22478792,0.17557,0.017283216,-0.1662348,-0.12553196,-0.04142716,-0.3048477,-0.05052951,0.30713764,0.016659763,-0.29538757,-0.22913143,0.12262335,-0.21247649,0.02428204,-0.18386027,0.26374215,-0.24371208,0.31186777,0.16386072,0.16449225,0.08720322,-0.121891394,-0.17309855,0.291033,-0.25434098,0.04707865,0.30687234,-0.23781548,0.15851216,-0.14798075,-0.18100414,-0.19928756,0.16521227,0.28165337,-0.008154298,-0.29637957,0.113215655,-0.2083508,0.11165612,-0.28445366,-0.20599249,0.083394624,-0.20772244,0.03947674,0.025858924,-0.07172223,-0.097277224,-0.23376308,0.23801543,0.12796475,-0.1848585,0.8503,0.9065268,-0.13296387,0.2758319,-0.2584599,0.19948332,-0.1988909,-0.1624675,0.16195628,-0.11784503,-0.10497568,0.19418798,-0.17294578,0.18471056,-0.08037034,0.0400655,0.0510028,-0.22936594,-0.16656943,0.16831462,0.12635878,0.044187985,0.21214493,0.17698123,0.0735129,-0.08216295,-0.15006073,0.22780012,0.1660651,-0.056969505,0.016330061,-0.027255945,0.28889793,-0.17832844,-0.17903326,0.12734558,0.2609079,-0.066520445,-0.047734052,-0.21986984,-0.2541291,0.050483312,-0.14191192,0.24874592,-0.16762812,-0.22350667,-0.13995315,0.11064909,-0.096272446,0.021436704,-0.2508471,-0.057291474,0.21689415,0.046441052,0.060559656,-0.3120285,-0.10967854,-0.21053293,-0.55092996,-0.5127955,-0.0028662088,-0.0010812389,-0.0024997608,-0.5477225,-0.064292245,-0.020785768,-0.50536597,-0.41897824,-0.5421137,0.20898235,-0.1542391,0.0101150265,-0.4036921,-0.24830106,0.27814552,-0.14486068,0.30726537,0.0039984505,-0.047804985,0.23023543,0.06093011,-0.37791353,-0.5216652,-0.3231506,0.19728532,-0.07173542,0.38454613,-0.40621033,0.4701021,-0.12535347,0.42216882,0.17630844,-0.1076794,0.16181,-0.1960641,-0.44001916,-0.53976184,-0.52484024,0.25335217,-0.17232236,0.13170429,0.08339477,0.5360642,-0.42585057,0.08468615,-0.13617964,0.5370887,-0.47148478,-0.3414682,0.3196109,-0.030125791,-0.33323315,0.33207354,-0.19529352,-1.0,0.3759377,-0.18503237,-0.04400152,0.19037545,0.85269415,-0.18120205,0.5656487,-0.96143025,0.0,0.0,0.0,-1.0,0.50096095,0.32058,-0.9772421,0.8576602,-0.2860003,-0.7461792,-0.14932656,0.6489322,0.0,0.0,0.0,-0.99967045,0.6257085,-0.6084689,0.93868876,-0.16036174,0.36021298,0.021588584,0.33143902,0.3636718,-0.0003348433,-0.00334344,-0.004175199,-0.99997467,0.7509599,0.24381536,-0.07059898,-0.47224766,-0.8430445,0.054828934,0.672361,0.9807283,0.000014342368,-0.0061520804,-0.009535632,1.0,0.0,0.0,0.0150192715,0.9982554,0.0026730986,0.024910392,-0.0037568703,1.0054708,1.0,0.993806,0.99038875,-0.99999994,0.004587412,0.47660053,-0.87525564,0.010010958,0.35218668,0.6689781,-0.722959,0.6814921,0.0,0.0,0.0,-1.2247448,0.61892134,-0.07202221,0.0],\"source_projection_weights\":[[-0.9999976,-0.8407852,-0.3105338,0.75500476,-0.67136806,-0.8269221,0.995204,-0.40957043,0.66540647,1.445065,1.3654729,0.49059448,-1.0003865,-0.9710343,0.5249519,-0.8282895,0.3677307,0.7756716,-0.27217376,0.10625149,0.312992,0.1267061,-0.009233973,-0.15559104],[-0.9999976,-0.5963865,-0.14052066,0.20321238,0.052331902,-0.2891694,-0.32647085,0.90668213,0.9648173,0.89604247,0.25865722,-1.6689024,-1.0002123,-0.720038,0.9145221,-0.1301688,-0.9965646,0.34389147,0.90929794,-0.57949734,0.051250495,-0.0006814752,-0.00033052405,-0.0029046456]],\"head_weights\":[0.003513919,-0.0003797346,0.0039039822,0.020262968,0.0005923073,0.004028996,0.0034028515,0.0000056478602,0.015872136,0.00059702375,-0.026364833]}"}} \ No newline at end of file diff --git a/v2/crates/wifi-densepose-sensing-server/tests/managed_parent.rs b/v2/crates/wifi-densepose-sensing-server/tests/managed_parent.rs new file mode 100644 index 0000000000..48e99d569c --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/tests/managed_parent.rs @@ -0,0 +1,284 @@ +//! Real isolated child processes and sockets; no physical sensing claims. +#![cfg(unix)] + +use std::{ + io::{BufRead, BufReader, Read, Write}, + net::{TcpListener, TcpStream, UdpSocket}, + process::{Child, Command, Stdio}, + thread::sleep, + time::{Duration, Instant}, +}; + +struct Fixture { + parent: Child, + helper: u32, + http: u16, + ws: u16, + directory: tempfile::TempDir, +} + +fn signal(pid: u32, signal: &str) { + let _ = Command::new("kill") + .args([signal, &pid.to_string()]) + .status(); +} + +fn running(pid: u32) -> bool { + let output = Command::new("ps") + .args(["-p", &pid.to_string(), "-o", "stat="]) + .output() + .unwrap(); + let state = String::from_utf8_lossy(&output.stdout); + !state.trim().is_empty() && !state.trim().starts_with('Z') +} + +impl Fixture { + fn start(guarded: bool) -> Self { + let a = TcpListener::bind("127.0.0.1:0").unwrap(); + let b = TcpListener::bind("127.0.0.1:0").unwrap(); + let udp = UdpSocket::bind("127.0.0.1:0").unwrap(); + let (http, ws, udp_port) = ( + a.local_addr().unwrap().port(), + b.local_addr().unwrap().port(), + udp.local_addr().unwrap().port(), + ); + let directory = tempfile::tempdir().unwrap(); + std::fs::write( + directory.path().join("retained-record.json"), + b"synthetic-retained-record", + ) + .unwrap(); + let script = if guarded { + "\"$@\" --parent-pid \"$$\" >/dev/null & child=$!; printf '%s\\n' \"$child\"; wait \"$child\"" + } else { + "\"$@\" >/dev/null & child=$!; printf '%s\\n' \"$child\"; wait \"$child\"" + }; + drop((a, b, udp)); + let mut parent = Command::new("/bin/sh") + .args([ + "-c", + script, + "managed-parent-test", + env!("CARGO_BIN_EXE_sensing-server"), + ]) + .args([ + "--source", + "esp32", + "--bind-addr", + "127.0.0.1", + "--udp-bind", + "127.0.0.1", + "--no-mdns", + "--http-port", + &http.to_string(), + "--ws-port", + &ws.to_string(), + "--udp-port", + &udp_port.to_string(), + "--data-dir", + directory.path().to_str().unwrap(), + ]) + .env_clear() + .env("PATH", "/usr/bin:/bin") + .env("RUST_LOG", "warn") + .stdout(Stdio::piped()) + .stderr(std::fs::File::create(directory.path().join("helper.log")).unwrap()) + .spawn() + .unwrap(); + let mut line = String::new(); + BufReader::new(parent.stdout.take().unwrap()) + .read_line(&mut line) + .unwrap(); + let fixture = Self { + parent, + helper: line.trim().parse().unwrap(), + http, + ws, + directory, + }; + let deadline = Instant::now() + Duration::from_secs(8); + while !fixture.healthy() { + assert!( + Instant::now() < deadline, + "helper did not start: {}", + std::fs::read_to_string(fixture.directory.path().join("helper.log")).unwrap() + ); + sleep(Duration::from_millis(25)); + } + fixture + } + + fn healthy(&self) -> bool { + let Ok(mut stream) = TcpStream::connect(("127.0.0.1", self.http)) else { + return false; + }; + stream + .set_read_timeout(Some(Duration::from_millis(250))) + .unwrap(); + if stream + .write_all(b"GET /health HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n") + .is_err() + { + return false; + } + let mut response = [0; 64]; + stream + .read(&mut response) + .is_ok_and(|count| response[..count].starts_with(b"HTTP/1.1 200")) + } + + fn websocket(&self, port: u16) -> TcpStream { + let mut stream = TcpStream::connect(("127.0.0.1", port)).unwrap(); + stream + .set_read_timeout(Some(Duration::from_secs(2))) + .unwrap(); + stream.write_all(b"GET /ws/sensing HTTP/1.1\r\nHost: localhost\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Version: 13\r\nSec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\n\r\n").unwrap(); + let mut response = [0; 1024]; + let count = stream.read(&mut response).unwrap(); + assert!(response[..count].starts_with(b"HTTP/1.1 101")); + stream + } + + fn end_parent(&mut self, how: &str) { + signal(self.parent.id(), how); + self.parent.wait().unwrap(); + } + + fn assert_stopped(&self) { + let deadline = Instant::now() + Duration::from_secs(4); + while running(self.helper) { + assert!( + Instant::now() < deadline, + "managed helper survived its parent" + ); + sleep(Duration::from_millis(25)); + } + assert!(TcpStream::connect(("127.0.0.1", self.http)).is_err()); + assert!(TcpStream::connect(("127.0.0.1", self.ws)).is_err()); + assert_eq!( + std::fs::read(self.directory.path().join("retained-record.json")).unwrap(), + b"synthetic-retained-record" + ); + } +} + +impl Drop for Fixture { + fn drop(&mut self) { + // Only these fixture-created processes; never enumerate a user's ports. + if running(self.helper) { + signal(self.helper, "-KILL"); + } + if self.parent.try_wait().unwrap().is_none() { + let _ = self.parent.kill(); + } + let _ = self.parent.wait(); + } +} + +#[test] +fn managed_helper_stops_after_parent_force_quit_even_with_an_open_websocket() { + let mut fixture = Fixture::start(true); + let _open_socket = fixture.websocket(fixture.ws); + let _open_http_socket = fixture.websocket(fixture.http); + fixture.end_parent("-KILL"); + fixture.assert_stopped(); +} + +#[test] +fn managed_helper_stops_after_parent_termination() { + let mut fixture = Fixture::start(true); + fixture.end_parent("-TERM"); + fixture.assert_stopped(); +} + +#[test] +fn unmanaged_server_survives_parent_exit() { + let mut fixture = Fixture::start(false); + fixture.end_parent("-KILL"); + sleep(Duration::from_millis(500)); + assert!(running(fixture.helper)); + assert!(fixture.healthy()); +} + +#[test] +fn managed_helper_remains_available_while_its_parent_lives() { + let fixture = Fixture::start(true); + sleep(Duration::from_millis(500)); + assert!(running(fixture.helper)); + assert!(fixture.healthy()); +} + +#[test] +fn wrong_parent_identity_is_rejected_before_creating_state() { + let directory = tempfile::tempdir().unwrap(); + let state = directory.path().join("must-not-be-created"); + for parent in ["0", "1", "4294967295"] { + let result = Command::new(env!("CARGO_BIN_EXE_sensing-server")) + .args([ + "--parent-pid", + parent, + "--data-dir", + state.to_str().unwrap(), + ]) + .env_clear() + .env("PATH", "/usr/bin:/bin") + .output() + .unwrap(); + assert_eq!(result.status.code(), Some(2)); + assert!(!state.exists()); + } +} + +// Subprocess fixture for the production watchdog when its async receiver is +// never polled. Ordinary test execution does not arm or park this fixture. +#[test] +fn watchdog_fixture() { + let Ok(parent) = std::env::var("RUVIEW_TEST_PARENT_PID") else { + return; + }; + let (_guard, _unpolled_receiver) = + wifi_densepose_sensing_server::parent_lifetime::arm(Some(parent.parse().unwrap())).unwrap(); + std::fs::write(std::env::var("RUVIEW_TEST_READY_FILE").unwrap(), b"ready").unwrap(); + loop { + std::thread::park(); + } +} + +#[test] +fn watchdog_bounds_exit_when_graceful_shutdown_cannot_run() { + let directory = tempfile::tempdir().unwrap(); + std::fs::write( + directory.path().join("retained-record.json"), + b"synthetic-retained-record", + ) + .unwrap(); + let ready = directory.path().join("ready"); + let mut parent = Command::new("/bin/sh") + .args(["-c", "RUVIEW_TEST_PARENT_PID=\"$$\" \"$@\" --exact watchdog_fixture --nocapture >/dev/null & child=$!; printf '%s\\n' \"$child\"; wait \"$child\"", "watchdog-parent"]) + .arg(std::env::current_exe().unwrap()) + .env_clear().env("PATH", "/usr/bin:/bin").env("RUVIEW_TEST_READY_FILE", &ready) + .stdout(Stdio::piped()).stderr(Stdio::null()).spawn().unwrap(); + let mut line = String::new(); + BufReader::new(parent.stdout.take().unwrap()) + .read_line(&mut line) + .unwrap(); + let mut fixture = Fixture { + parent, + helper: line.trim().parse().unwrap(), + http: 0, + ws: 0, + directory, + }; + let deadline = Instant::now() + Duration::from_secs(4); + while !ready.exists() { + assert!(Instant::now() < deadline, "watchdog fixture did not arm"); + sleep(Duration::from_millis(25)); + } + let started = Instant::now(); + fixture.end_parent("-KILL"); + fixture.assert_stopped(); + assert!( + started.elapsed() >= Duration::from_secs(2), + "fixture must exercise the hard deadline, not async shutdown" + ); +} diff --git a/v2/crates/wifi-densepose-signal/src/hardware_norm.rs b/v2/crates/wifi-densepose-signal/src/hardware_norm.rs index 188dd76a2b..11d5873e2c 100644 --- a/v2/crates/wifi-densepose-signal/src/hardware_norm.rs +++ b/v2/crates/wifi-densepose-signal/src/hardware_norm.rs @@ -21,6 +21,8 @@ pub enum HardwareNormError { UnknownHardware(usize), #[error("Invalid canonical subcarrier count: {0}")] InvalidCanonical(usize), + #[error("Magnitude must be finite and non-negative at index {index}")] + InvalidMagnitude { index: usize }, } /// Known WiFi chipset families with their subcarrier counts and MIMO configs. @@ -183,6 +185,56 @@ impl HardwareNormalizer { pub fn resample_to_canonical(&self, raw: &[f64]) -> Vec { resample_cubic(raw, self.canonical_subcarriers) } + + /// Length-only magnitude projection using convex linear interpolation. + /// + /// Spectral nulls and guard bands can make the cubic control path overshoot + /// below zero. This opt-in path retains the canonical grid and amplitude + /// units, including constant DC and gain, without phase or z-score changes. + /// Invalid inputs are rejected before identity/singleton handling; they are + /// never clamped, replaced with zeros, or reinterpreted as measurements. + pub fn resample_magnitude_to_canonical( + &self, + raw: &[f64], + ) -> Result, HardwareNormError> { + if raw.is_empty() { + return Err(HardwareNormError::EmptyFrame { amp: 0, phase: 0 }); + } + if let Some(index) = raw.iter().position(|v| !v.is_finite() || *v < 0.0) { + return Err(HardwareNormError::InvalidMagnitude { index }); + } + let dst_len = self.canonical_subcarriers; + if raw.len() == dst_len { + return Ok(raw.to_vec()); + } + if raw.len() == 1 || dst_len == 1 { + return Ok(vec![raw[0]; dst_len]); + } + let last = raw.len() - 1; + let ratio = last as f64 / (dst_len - 1) as f64; + Ok((0..dst_len) + .map(|i| { + if i == dst_len - 1 { + return raw[last]; + } + let position = i as f64 * ratio; + let left = (position.floor() as usize).min(last); + let right = (left + 1).min(last); + let fraction = position - left as f64; + if raw[left] == raw[right] { + raw[left] + } else { + // Same-sign finite endpoints keep the difference finite; + // FMA avoids the overflowing sum of a weighted blend. + let interpolated = (raw[right] - raw[left]).mul_add(fraction, raw[left]); + // Bound rounding to the two valid endpoints, never the input. + interpolated + .max(raw[left].min(raw[right])) + .min(raw[left].max(raw[right])) + } + }) + .collect()) + } } impl Default for HardwareNormalizer { @@ -294,6 +346,117 @@ fn sanitize_phase(phase: &[f64]) -> Vec { mod tests { use super::*; + #[test] + fn magnitude_projection_rejects_invalid_inputs_even_on_identity_grid() { + let norm = HardwareNormalizer::new(); + assert!(matches!( + norm.resample_magnitude_to_canonical(&[]), + Err(HardwareNormError::EmptyFrame { .. }) + )); + for invalid in [-0.01, f64::NAN, f64::INFINITY, f64::NEG_INFINITY] { + for len in [1, 56, 62, 254] { + let mut raw = vec![2.0; len]; + raw[len - 1] = invalid; + assert!(matches!( + norm.resample_magnitude_to_canonical(&raw), + Err(HardwareNormError::InvalidMagnitude { index }) if index == len - 1 + )); + } + } + } + + #[test] + fn magnitude_projection_preserves_native_hull_at_sharp_guard_bands() { + let norm = HardwareNormalizer::new(); + for len in [62, 254] { + let raw: Vec = (0..len) + .map(|i| { + if (len / 3..2 * len / 3).contains(&i) { + 40.0 + } else { + 0.0 + } + }) + .collect(); + let cubic = norm.resample_to_canonical(&raw); + assert!( + cubic.iter().any(|v| *v < -1.0), + "fixture must expose cubic undershoot" + ); + let projected = norm.resample_magnitude_to_canonical(&raw).unwrap(); + assert_eq!(projected.len(), 56); + assert!(projected + .iter() + .all(|v| v.is_finite() && (0.0..=40.0).contains(v))); + assert_eq!(projected[0], raw[0]); + assert_eq!(projected[55], raw[len - 1]); + } + // A zero-valued notch also exposes cubic ringing on the 62-bin grid. + let raw: Vec = (0..62) + .map(|i| if (20..41).contains(&i) { 0.0 } else { 40.0 }) + .collect(); + assert!(norm.resample_to_canonical(&raw).iter().any(|v| *v < 0.0)); + assert!(norm + .resample_magnitude_to_canonical(&raw) + .unwrap() + .iter() + .all(|v| (0.0..=40.0).contains(v))); + } + + #[test] + fn magnitude_projection_preserves_dc_gain_and_configured_width() { + for len in [62, 254] { + let norm = HardwareNormalizer::new(); + assert_eq!( + norm.resample_magnitude_to_canonical(&vec![7.25; len]) + .unwrap(), + vec![7.25; 56] + ); + let raw: Vec = (0..len).map(|i| (i % 13) as f64 * 0.25).collect(); + let affine: Vec = raw.iter().map(|v| 3.5 * v + 2.75).collect(); + let base = norm.resample_magnitude_to_canonical(&raw).unwrap(); + let scaled = norm.resample_magnitude_to_canonical(&affine).unwrap(); + for (a, b) in base.iter().zip(scaled) { + assert!((b - (3.5 * a + 2.75)).abs() < 1e-12); + } + } + let custom = HardwareNormalizer::with_canonical_subcarriers(3).unwrap(); + assert_eq!( + custom + .resample_magnitude_to_canonical(&[2.0, 10.0]) + .unwrap(), + vec![2.0, 6.0, 10.0] + ); + assert_eq!( + custom.resample_magnitude_to_canonical(&[3.0]).unwrap(), + vec![3.0; 3] + ); + assert_eq!( + custom + .resample_magnitude_to_canonical(&[2.0, 0.0, 9.0]) + .unwrap(), + vec![2.0, 0.0, 9.0] + ); + let quarters = HardwareNormalizer::with_canonical_subcarriers(5).unwrap(); + assert_eq!( + quarters.resample_magnitude_to_canonical(&[0.0, 8.0]).unwrap(), + vec![0.0, 2.0, 4.0, 6.0, 8.0] + ); + let singleton = HardwareNormalizer::with_canonical_subcarriers(1).unwrap(); + assert_eq!( + singleton + .resample_magnitude_to_canonical(&[2.0, 10.0]) + .unwrap(), + vec![2.0] + ); + let extreme = HardwareNormalizer::new() + .resample_magnitude_to_canonical(&[0.0, f64::MAX]) + .unwrap(); + assert!(extreme + .iter() + .all(|v| v.is_finite() && *v >= 0.0 && *v <= f64::MAX)); + } + #[test] fn detect_hardware_and_properties() { assert_eq!( diff --git a/v2/crates/wifi-densepose-signal/src/ruvsense/mod.rs b/v2/crates/wifi-densepose-signal/src/ruvsense/mod.rs index e2f56af434..0c75bde21b 100644 --- a/v2/crates/wifi-densepose-signal/src/ruvsense/mod.rs +++ b/v2/crates/wifi-densepose-signal/src/ruvsense/mod.rs @@ -38,6 +38,7 @@ pub mod adversarial; pub mod cross_room; pub mod field_model; +pub mod source_field_bank; pub mod gesture; pub mod intention; pub mod longitudinal; diff --git a/v2/crates/wifi-densepose-signal/src/ruvsense/source_field_bank.rs b/v2/crates/wifi-densepose-signal/src/ruvsense/source_field_bank.rs new file mode 100644 index 0000000000..57d1ddbf08 --- /dev/null +++ b/v2/crates/wifi-densepose-signal/src/ruvsense/source_field_bank.rs @@ -0,0 +1,315 @@ +//! Experimental calibration-only bank: one existing FieldModel per source. +//! +//! No sensing-server caller or detection promotion is introduced here. A frozen +//! bank is learned statistics, not an owner/room receipt or occupied acceptance. +//! The caller must enforce the production capture duration, room binding and +//! independent physical qualification. Tests use explicitly synthetic inputs. + +use std::collections::BTreeMap; +use thiserror::Error; + +use super::field_model::{ + CalibrationStatus, EmptyRoomMatch, FieldModel, FieldModelConfig, FieldModelError, + FieldModelSnapshotV1, +}; + +const MAX_SOURCES: usize = 32; +const MAX_SUBCARRIERS: usize = 256; +const MAX_SOURCE_AGE_US: u64 = 5_000_000; +const MAX_AMPLITUDE: f64 = 1_000_000.0; + +pub struct SourceFieldSpec { + pub source_id: u8, + pub grid_id: String, + pub config: FieldModelConfig, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SourceBankPhase { + Collecting, + Frozen, + Invalid, +} + +#[derive(Debug, Error)] +pub enum SourceBankError { + #[error("invalid source-bank configuration: {0}")] + Configuration(&'static str), + #[error("duplicate source {0}")] + DuplicateSource(u8), + #[error("unknown source {0}")] + UnknownSource(u8), + #[error("source {0} changed its grid, sample order, timing or frame shape")] + SourceChanged(u8), + #[error("source {0} has not completed its independent calibration")] + IncompleteSource(u8), + #[error("source {0} is missing, stale or future-dated")] + StaleSource(u8), + #[error("bank is not in the required phase")] + Phase, + #[error("field-model calibration failed: {0}")] + Model(#[from] FieldModelError), +} + +pub struct FrozenSourceField { + pub source_id: u8, + pub grid_id: String, + pub snapshot: FieldModelSnapshotV1, +} + +/// A bounded, memory-only window from the caller's admitted/epoch-bound source +/// recorder. The caller must describe its actual transport trust; this type does +/// not authenticate a radio. Comparisons are empty-background tests, not person labels. +pub struct SourceFieldWindow<'a> { + pub source_id: u8, + pub grid_id: &'a str, + pub observed_at_us: u64, + pub frames: &'a [Vec], +} + +struct SourceTraining { + grid_id: String, + width: usize, + model: FieldModel, + last_sample: Option<(u64, u64)>, +} + +pub struct SourceFieldBank { + sources: BTreeMap, + phase: SourceBankPhase, + source_age_us: u64, + frozen_at_us: Option, +} + +impl SourceFieldBank { + pub fn new(specs: Vec, source_age_us: u64) -> Result { + if !(2..=MAX_SOURCES).contains(&specs.len()) + || source_age_us == 0 + || source_age_us > MAX_SOURCE_AGE_US + { + return Err(SourceBankError::Configuration( + "bounded source set and freshness window required", + )); + } + // Validate every allocation shape before constructing any model. + for spec in &specs { + let c = &spec.config; + if c.n_links != 1 + || !(1..=MAX_SUBCARRIERS).contains(&c.n_subcarriers) + || !(1..=5).contains(&c.n_modes) + || !(2..=1_000_000).contains(&c.min_calibration_frames) + || !c.min_calibration_duration_s.is_finite() + || !(0.0..=86_400.0).contains(&c.min_calibration_duration_s) + || !c.baseline_expiry_s.is_finite() + || c.baseline_expiry_s <= 0.0 + || c.baseline_expiry_s > 86_400.0 + || spec.grid_id.is_empty() + || spec.grid_id.len() > 128 + || !spec + .grid_id + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b"-_.".contains(&b)) + { + return Err(SourceBankError::Configuration( + "one bounded finite single-link model and grid per source required", + )); + } + } + let mut sources = BTreeMap::new(); + for spec in specs { + if sources.contains_key(&spec.source_id) { + return Err(SourceBankError::DuplicateSource(spec.source_id)); + } + sources.insert( + spec.source_id, + SourceTraining { + grid_id: spec.grid_id, + width: spec.config.n_subcarriers, + model: FieldModel::new(spec.config)?, + last_sample: None, + }, + ); + } + Ok(Self { + sources, + phase: SourceBankPhase::Collecting, + source_age_us, + frozen_at_us: None, + }) + } + + pub fn phase(&self) -> SourceBankPhase { + self.phase + } + + /// `ordinal` is the server's validated per-source ordinal, not an unchecked + /// raw device counter. Device wrap/transport authentication remain upstream. + pub fn feed( + &mut self, + source_id: u8, + grid_id: &str, + ordinal: u64, + observed_at_us: u64, + amplitudes: &[f64], + ) -> Result<(), SourceBankError> { + if self.phase != SourceBankPhase::Collecting { + return Err(SourceBankError::Phase); + } + let source = self + .sources + .get_mut(&source_id) + .ok_or(SourceBankError::UnknownSource(source_id))?; + let changed = grid_id != source.grid_id + || observed_at_us == 0 + || amplitudes.len() != source.width + || amplitudes + .iter() + .any(|x| !x.is_finite() || x.abs() > MAX_AMPLITUDE) + || source.last_sample.is_some_and(|(last_ordinal, last_at)| { + ordinal <= last_ordinal + || observed_at_us <= last_at + || observed_at_us.saturating_sub(last_at) > self.source_age_us + }); + if changed { + self.phase = SourceBankPhase::Invalid; + return Err(SourceBankError::SourceChanged(source_id)); + } + if let Err(error) = source.model.feed_calibration(&[amplitudes.to_vec()]) { + self.phase = SourceBankPhase::Invalid; + return Err(error.into()); + } + source.last_sample = Some((ordinal, observed_at_us)); + Ok(()) + } + + pub fn frame_counts(&self) -> BTreeMap { + self.sources + .iter() + .map(|(id, s)| (*id, s.model.calibration_frame_count())) + .collect() + } + + /// Check all members before mutating any model. A model-level failure + /// invalidates the whole generation; partially frozen members cannot escape. + pub fn freeze(&mut self, timestamp_us: u64, geometry_hash: u64) -> Result<(), SourceBankError> { + if self.phase != SourceBankPhase::Collecting { + return Err(SourceBankError::Phase); + } + for (id, source) in &self.sources { + if source.model.calibration_frame_count() < source.model.min_calibration_frames() as u64 + || source.model.calibration_elapsed_s() < source.model.min_calibration_duration_s() + { + return Err(SourceBankError::IncompleteSource(*id)); + } + if source + .last_sample + .is_none_or(|(_, at)| timestamp_us < at || timestamp_us - at > self.source_age_us) + { + return Err(SourceBankError::StaleSource(*id)); + } + } + for source in self.sources.values_mut() { + if let Err(error) = source + .model + .finalize_calibration(timestamp_us, geometry_hash) + { + self.phase = SourceBankPhase::Invalid; + return Err(error.into()); + } + } + self.phase = SourceBankPhase::Frozen; + self.frozen_at_us = Some(timestamp_us); + Ok(()) + } + + /// Statistics only. The server must wrap these in a new versioned, + /// account/installation/room-bound artifact before persistence or adoption. + pub fn frozen_models(&self) -> Result, SourceBankError> { + if self.phase != SourceBankPhase::Frozen { + return Err(SourceBankError::Phase); + } + self.sources + .iter() + .map(|(id, source)| { + Ok(FrozenSourceField { + source_id: *id, + grid_id: source.grid_id.clone(), + snapshot: source.model.export_snapshot()?, + }) + }) + .collect() + } + + /// Shadow diagnostics only: require every configured source and return its + /// individual empty-background conformance. No aggregate presence is emitted. + pub fn compare_empty_windows( + &self, + timestamp_us: u64, + windows: &[SourceFieldWindow<'_>], + ) -> Result, SourceBankError> { + let freshness = self.freshness(timestamp_us)?; + if windows.len() != self.sources.len() { + return Err(SourceBankError::Configuration( + "complete runtime window set required", + )); + } + let mut result = BTreeMap::new(); + for window in windows { + let id = window.source_id; + let source = self + .sources + .get(&id) + .ok_or(SourceBankError::UnknownSource(id))?; + if result.contains_key(&id) + || window.grid_id != source.grid_id + || window.frames.is_empty() + || window.frames.len() > 200 + || window.frames.iter().any(|frame| { + frame.len() != source.width + || frame + .iter() + .any(|v| !v.is_finite() || v.abs() > MAX_AMPLITUDE) + }) + { + return Err(SourceBankError::SourceChanged(id)); + } + if freshness[&id] != CalibrationStatus::Fresh + || timestamp_us < window.observed_at_us + || timestamp_us - window.observed_at_us > self.source_age_us + { + return Err(SourceBankError::StaleSource(id)); + } + let compared = source + .model + .empty_room_match(window.frames) + .ok_or(SourceBankError::IncompleteSource(id))?; + result.insert(id, compared); + } + if result.len() != self.sources.len() { + return Err(SourceBankError::Configuration( + "required source window missing", + )); + } + Ok(result) + } + + pub fn freshness( + &self, + timestamp_us: u64, + ) -> Result, SourceBankError> { + if self.phase != SourceBankPhase::Frozen + || self.frozen_at_us.is_none_or(|at| timestamp_us < at) + { + return Err(SourceBankError::Phase); + } + Ok(self + .sources + .iter() + .map(|(id, s)| (*id, s.model.check_freshness(timestamp_us))) + .collect()) + } +} + +#[cfg(test)] +#[path = "source_field_bank_tests.rs"] +mod tests; diff --git a/v2/crates/wifi-densepose-signal/src/ruvsense/source_field_bank_tests.rs b/v2/crates/wifi-densepose-signal/src/ruvsense/source_field_bank_tests.rs new file mode 100644 index 0000000000..947f4fa9c4 --- /dev/null +++ b/v2/crates/wifi-densepose-signal/src/ruvsense/source_field_bank_tests.rs @@ -0,0 +1,201 @@ +use super::*; + +// SYNTHETIC software fixtures, not measured RF or a physical qualification. +fn config() -> FieldModelConfig { + FieldModelConfig { + n_links: 1, + n_subcarriers: 4, + n_modes: 3, + min_calibration_frames: 16, + min_calibration_duration_s: 0.0, + baseline_expiry_s: 10.0, + } +} +fn spec(id: u8) -> SourceFieldSpec { + SourceFieldSpec { + source_id: id, + grid_id: format!("grid-{id}"), + config: config(), + } +} +fn bank() -> SourceFieldBank { + SourceFieldBank::new(vec![spec(15), spec(16)], 1000).unwrap() +} +fn frame(base: f64, i: u64) -> Vec { + (0..4) + .map(|j| base + j as f64 * 0.1 + i as f64 * 0.01) + .collect() +} +fn train(bank: &mut SourceFieldBank, id: u8, base: f64) { + for i in 0..16 { + bank.feed(id, &format!("grid-{id}"), i, i + 1, &frame(base, i)) + .unwrap(); + } +} + +#[test] +fn source_scale_does_not_change_the_other_sources_model() { + let mut bank = bank(); + let mut control = FieldModel::new(config()).unwrap(); + for i in 0..16 { + let f = frame(10.0, i); + control.feed_calibration(&[f.clone()]).unwrap(); + bank.feed(15, "grid-15", i, i + 1, &f).unwrap(); + bank.feed(16, "grid-16", i, i + 1, &frame(10000.0, i)) + .unwrap(); + } + control.finalize_calibration(100, 42).unwrap(); + bank.freeze(100, 42).unwrap(); + let frozen = bank.frozen_models().unwrap(); + assert_eq!( + frozen.iter().map(|s| s.source_id).collect::>(), + vec![15, 16] + ); + assert_eq!( + frozen[0].snapshot.modes, + control.export_snapshot().unwrap().modes + ); + assert!(frozen[1].snapshot.modes.baseline[0][0] > 9000.0); +} + +#[test] +fn observed_or_missing_members_cannot_borrow_another_members_frames() { + let mut b = bank(); + train(&mut b, 15, 10.0); + b.feed(16, "grid-16", 0, 1, &frame(50.0, 0)).unwrap(); + assert!(matches!( + b.freeze(100, 42), + Err(SourceBankError::IncompleteSource(16)) + )); + assert_eq!(b.phase(), SourceBankPhase::Collecting); + assert!(b.frozen_models().is_err()); + assert_eq!(b.frame_counts()[&16], 1); +} + +#[test] +fn invalid_shapes_and_nonfinite_frames_invalidate_the_generation() { + for bad in [ + vec![1.0], + vec![f64::NAN; 4], + vec![f64::INFINITY; 4], + vec![f64::MAX; 4], + ] { + let mut b = bank(); + assert!(matches!( + b.feed(15, "grid-15", 0, 1, &bad), + Err(SourceBankError::SourceChanged(15)) + )); + assert_eq!(b.phase(), SourceBankPhase::Invalid); + assert!(b.freeze(2, 42).is_err()); + } +} + +#[test] +fn grid_sequence_clock_and_continuity_changes_are_not_partial_success() { + for (grid, ordinal, time) in [ + ("other", 1, 2), + ("grid-15", 0, 2), + ("grid-15", 1, 1), + ("grid-15", 1, 1002), + ] { + let mut b = bank(); + b.feed(15, "grid-15", 0, 1, &frame(10.0, 0)).unwrap(); + assert!(b.feed(15, grid, ordinal, time, &frame(10.0, 1)).is_err()); + assert_eq!(b.phase(), SourceBankPhase::Invalid); + assert!(b.frozen_models().is_err()); + } +} + +#[test] +fn stale_and_future_source_windows_cannot_freeze() { + let mut b = bank(); + train(&mut b, 15, 10.0); + train(&mut b, 16, 20.0); + assert!(matches!( + b.freeze(1017, 42), + Err(SourceBankError::StaleSource(15)) + )); + assert!(matches!( + b.freeze(15, 42), + Err(SourceBankError::StaleSource(15)) + )); + assert_eq!(b.phase(), SourceBankPhase::Collecting); + assert!(b.frozen_models().is_err()); +} + +#[test] +fn freeze_is_immutable_and_does_not_renew_model_age() { + let mut b = bank(); + train(&mut b, 15, 10.0); + train(&mut b, 16, 20.0); + b.freeze(100, 42).unwrap(); + assert!(b.feed(15, "grid-15", 16, 101, &frame(10.0, 16)).is_err()); + assert!(b.freeze(200, 42).is_err()); + assert!(b.freshness(99).is_err()); + assert!(b + .freshness(6_000_100) + .unwrap() + .values() + .all(|x| *x == CalibrationStatus::Stale)); + assert!(b + .freshness(11_000_100) + .unwrap() + .values() + .all(|x| *x == CalibrationStatus::Expired)); + assert!(b + .frozen_models() + .unwrap() + .iter() + .all(|s| s.snapshot.modes.calibrated_at_us == 100)); +} + +#[test] +fn configuration_is_bounded_and_unknown_nodes_do_not_enter_training() { + assert!(SourceFieldBank::new(vec![spec(15)], 1000).is_err()); + assert!(SourceFieldBank::new(vec![spec(15), spec(15)], 1000).is_err()); + assert!(SourceFieldBank::new(vec![spec(15), spec(16)], 0).is_err()); + let mut wide = spec(16); + wide.config.n_subcarriers = 10000; + assert!(SourceFieldBank::new(vec![spec(15), wide], 1000).is_err()); + let mut b = bank(); + assert!(matches!( + b.feed(99, "grid-99", 0, 1, &frame(1.0, 0)), + Err(SourceBankError::UnknownSource(99)) + )); + assert_eq!(b.phase(), SourceBankPhase::Collecting); + assert!(b.frame_counts().values().all(|n| *n == 0)); +} + +#[test] +fn shadow_windows_compare_individually_and_abstain_on_incomplete_or_stale_sets() { + let mut b = bank(); + for i in 0..1000 { + b.feed(15, "grid-15", i, i + 1, &frame(10.0, i)).unwrap(); + b.feed(16, "grid-16", i, i + 1, &frame(10000.0, i)).unwrap(); + } + b.freeze(1500, 42).unwrap(); + let quiet: Vec> = (450..500).map(|i| frame(10.0, i)).collect(); + let changed: Vec> = (450..500).map(|i| frame(11000.0, i)).collect(); + let windows = [ + SourceFieldWindow { + source_id: 15, + grid_id: "grid-15", + observed_at_us: 1990, + frames: &quiet, + }, + SourceFieldWindow { + source_id: 16, + grid_id: "grid-16", + observed_at_us: 1990, + frames: &changed, + }, + ]; + let result = b.compare_empty_windows(2000, &windows).unwrap(); + assert!(result[&15].matches_empty); + assert!(!result[&16].matches_empty); + assert!(b.compare_empty_windows(2000, &windows[..1]).is_err()); + assert!(b.compare_empty_windows(4000, &windows).is_err()); + assert!(b.compare_empty_windows(1980, &windows).is_err()); + assert!(b.compare_empty_windows(6_001_500, &windows).is_err()); + assert_eq!(b.phase(), SourceBankPhase::Frozen); +} diff --git a/v2/vendor/security/wasmtime-internal-cache-36.0.20/RUVIEW-PROVENANCE.md b/v2/vendor/security/wasmtime-internal-cache-36.0.20/RUVIEW-PROVENANCE.md deleted file mode 100644 index cb51c691fb..0000000000 --- a/v2/vendor/security/wasmtime-internal-cache-36.0.20/RUVIEW-PROVENANCE.md +++ /dev/null @@ -1,13 +0,0 @@ -# Dependency compatibility patch - -Source: crates.io `wasmtime-internal-cache` 36.0.20. -Published archive SHA-256: `65f09c789d1bb2e7ab1618ac8336d783c1ff0babc38430c7f1940d01c546722b` (verified against crates.io version metadata). - -Executable source, tests, fixtures and licenses are byte-for-byte upstream. -Only the normalized Cargo.toml dependency requirement changes: -`zstd` from `0.13.0` to `0.14.3`, retaining the upstream feature selection. -Registry bookkeeping, the nested lockfile, the unconsumed original manifest -and upstream CI files are omitted. No advisory suppression is added. - -This preserves the parent API while selecting the patched registry dependency. -Remove this override when a compatible parent release includes the fixed dependency. diff --git a/v2/vendor/security/wasmtime-internal-cache-36.0.20/Cargo.toml b/v2/vendor/security/wasmtime-internal-cache-36.0.21/Cargo.toml similarity index 99% rename from v2/vendor/security/wasmtime-internal-cache-36.0.20/Cargo.toml rename to v2/vendor/security/wasmtime-internal-cache-36.0.21/Cargo.toml index 9b86a5661b..b41f628f52 100644 --- a/v2/vendor/security/wasmtime-internal-cache-36.0.20/Cargo.toml +++ b/v2/vendor/security/wasmtime-internal-cache-36.0.21/Cargo.toml @@ -13,7 +13,7 @@ edition = "2024" rust-version = "1.86.0" name = "wasmtime-internal-cache" -version = "36.0.20" +version = "36.0.21" authors = ["The Wasmtime Project Developers"] build = "build.rs" autolib = false diff --git a/v2/vendor/security/wasmtime-internal-cache-36.0.20/LICENSE b/v2/vendor/security/wasmtime-internal-cache-36.0.21/LICENSE similarity index 100% rename from v2/vendor/security/wasmtime-internal-cache-36.0.20/LICENSE rename to v2/vendor/security/wasmtime-internal-cache-36.0.21/LICENSE diff --git a/v2/vendor/security/wasmtime-internal-cache-36.0.21/RUVIEW-PROVENANCE.md b/v2/vendor/security/wasmtime-internal-cache-36.0.21/RUVIEW-PROVENANCE.md new file mode 100644 index 0000000000..e9d78a01ff --- /dev/null +++ b/v2/vendor/security/wasmtime-internal-cache-36.0.21/RUVIEW-PROVENANCE.md @@ -0,0 +1,13 @@ +# Dependency compatibility patch + +Source: crates.io `wasmtime-internal-cache` 36.0.21. +Published archive SHA-256: `1ff2377df26271c42d9649ac4ee13656ef89a51f02f0daa7917e67c635d9bc1d`. + +Executable source, tests, fixtures and licenses are byte-for-byte upstream. +Only the normalized Cargo.toml zstd requirement changes from0.13.0 to0.14.3, +retaining upstream features and the previously reviewed compatibility repair. +Registry bookkeeping, nested lockfile, unconsumed original manifest and upstream +CI files are omitted. No source backport or advisory ignore is added here. + +This version matches the compatible Wasmtime36.0.21 core security update. +Remove the override when a compatible parent selects the fixed zstd dependency. diff --git a/v2/vendor/security/wasmtime-internal-cache-36.0.20/build.rs b/v2/vendor/security/wasmtime-internal-cache-36.0.21/build.rs similarity index 100% rename from v2/vendor/security/wasmtime-internal-cache-36.0.20/build.rs rename to v2/vendor/security/wasmtime-internal-cache-36.0.21/build.rs diff --git a/v2/vendor/security/wasmtime-internal-cache-36.0.20/src/config.rs b/v2/vendor/security/wasmtime-internal-cache-36.0.21/src/config.rs similarity index 100% rename from v2/vendor/security/wasmtime-internal-cache-36.0.20/src/config.rs rename to v2/vendor/security/wasmtime-internal-cache-36.0.21/src/config.rs diff --git a/v2/vendor/security/wasmtime-internal-cache-36.0.20/src/config/tests.rs b/v2/vendor/security/wasmtime-internal-cache-36.0.21/src/config/tests.rs similarity index 100% rename from v2/vendor/security/wasmtime-internal-cache-36.0.20/src/config/tests.rs rename to v2/vendor/security/wasmtime-internal-cache-36.0.21/src/config/tests.rs diff --git a/v2/vendor/security/wasmtime-internal-cache-36.0.20/src/lib.rs b/v2/vendor/security/wasmtime-internal-cache-36.0.21/src/lib.rs similarity index 100% rename from v2/vendor/security/wasmtime-internal-cache-36.0.20/src/lib.rs rename to v2/vendor/security/wasmtime-internal-cache-36.0.21/src/lib.rs diff --git a/v2/vendor/security/wasmtime-internal-cache-36.0.20/src/tests.rs b/v2/vendor/security/wasmtime-internal-cache-36.0.21/src/tests.rs similarity index 100% rename from v2/vendor/security/wasmtime-internal-cache-36.0.20/src/tests.rs rename to v2/vendor/security/wasmtime-internal-cache-36.0.21/src/tests.rs diff --git a/v2/vendor/security/wasmtime-internal-cache-36.0.20/src/worker.rs b/v2/vendor/security/wasmtime-internal-cache-36.0.21/src/worker.rs similarity index 100% rename from v2/vendor/security/wasmtime-internal-cache-36.0.20/src/worker.rs rename to v2/vendor/security/wasmtime-internal-cache-36.0.21/src/worker.rs diff --git a/v2/vendor/security/wasmtime-internal-cache-36.0.20/src/worker/tests.rs b/v2/vendor/security/wasmtime-internal-cache-36.0.21/src/worker/tests.rs similarity index 100% rename from v2/vendor/security/wasmtime-internal-cache-36.0.20/src/worker/tests.rs rename to v2/vendor/security/wasmtime-internal-cache-36.0.21/src/worker/tests.rs diff --git a/v2/vendor/security/wasmtime-internal-cache-36.0.20/src/worker/tests/system_time_stub.rs b/v2/vendor/security/wasmtime-internal-cache-36.0.21/src/worker/tests/system_time_stub.rs similarity index 100% rename from v2/vendor/security/wasmtime-internal-cache-36.0.20/src/worker/tests/system_time_stub.rs rename to v2/vendor/security/wasmtime-internal-cache-36.0.21/src/worker/tests/system_time_stub.rs diff --git a/v2/vendor/security/wasmtime-internal-cache-36.0.20/tests/cache_write_default_config.rs b/v2/vendor/security/wasmtime-internal-cache-36.0.21/tests/cache_write_default_config.rs similarity index 100% rename from v2/vendor/security/wasmtime-internal-cache-36.0.20/tests/cache_write_default_config.rs rename to v2/vendor/security/wasmtime-internal-cache-36.0.21/tests/cache_write_default_config.rs diff --git a/v2/vendor/security/zip-7.2.0/.amazonq/rules/rules.md b/v2/vendor/security/zip-7.2.0/.amazonq/rules/rules.md new file mode 100644 index 0000000000..a96b3eab20 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/.amazonq/rules/rules.md @@ -0,0 +1,14 @@ +- Conform to the ZIP file specification at https://support.pkware.com/pkzip/appnote. +- PR titles must conform to [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/) and start + with one of the types specified by the [Angular convention](https://github.com/angular/angular/blob/22b96b9/CONTRIBUTING.md#type). + The type prefix will usually be `fix:`, `feat:`, `perf:`, `docs:` or `chore(deps):`. +- When adding a new compression or decompression method, add a feature flag for it. +- Make sure that if new code has a dependency that's gated on a feature flag, then so is the new code. +- Carefully consider the configurations of feature flags your change will affect, and test accordingly. +- Optimize your code's performance for archives that contain many files *and* archives that contain large files. +- Consider whether your change affects ZIP64 files. If so, test it on 4GiB files both with and without compression. + Generate these files by copying repeatedly to or from the same buffer, so that the test doesn't occupy or allocate + 4GiB of memory. +- Wait and see whether the tests for your PR pass in CI. Fix them if they fail. +- Rebase your PR whenever it has a merge conflict. If you can't fix the rebase conflicts, rewrite it from scratch. +- Always run `cargo fmt --all` and `cargo clippy --all-features --all-targets`. \ No newline at end of file diff --git a/v2/vendor/security/zip-7.2.0/.cargo_vcs_info.json b/v2/vendor/security/zip-7.2.0/.cargo_vcs_info.json new file mode 100644 index 0000000000..509f2182f7 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/.cargo_vcs_info.json @@ -0,0 +1,6 @@ +{ + "git": { + "sha1": "46dc29cda6749736ddb1ec502aff06a65ffe54f8" + }, + "path_in_vcs": "" +} \ No newline at end of file diff --git a/v2/vendor/security/zip-7.2.0/.gitattributes b/v2/vendor/security/zip-7.2.0/.gitattributes new file mode 100644 index 0000000000..2b83c1e9d1 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/.gitattributes @@ -0,0 +1,8 @@ +tests/data/** binary=true +tests/data/**/LICENSE.*.txt binary=false +tests/data/folder/** binary=true +tests/data/legacy/** binary=true +fuzz/corpus/** binary=true + +tests/data/folder/first.txt text eol=lf +tests/data/legacy/*.out text eol=crlf \ No newline at end of file diff --git a/v2/vendor/security/zip-7.2.0/.gitignore b/v2/vendor/security/zip-7.2.0/.gitignore new file mode 100644 index 0000000000..eded420b2c --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/.gitignore @@ -0,0 +1,6 @@ +Cargo.lock +target +.DS_Store +\.idea/ +/fuzz_read/out/ +/fuzz_write/out/ diff --git a/v2/vendor/security/zip-7.2.0/.whitesource b/v2/vendor/security/zip-7.2.0/.whitesource new file mode 100644 index 0000000000..9c7ae90b4e --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/.whitesource @@ -0,0 +1,14 @@ +{ + "scanSettings": { + "baseBranches": [] + }, + "checkRunSettings": { + "vulnerableCheckRunConclusionLevel": "failure", + "displayMode": "diff", + "useMendCheckNames": true + }, + "issueSettings": { + "minSeverityLevel": "LOW", + "issueType": "DEPENDENCY" + } +} \ No newline at end of file diff --git a/v2/vendor/security/zip-7.2.0/CHANGELOG.md b/v2/vendor/security/zip-7.2.0/CHANGELOG.md new file mode 100644 index 0000000000..92c6d4d457 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/CHANGELOG.md @@ -0,0 +1,860 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +## [7.2.0](https://github.com/zip-rs/zip2/compare/v7.1.0...v7.2.0) - 2026-01-20 + +### 🚀 Features + +- add read_zipfile_from_stream_with_compressed_size ([#70](https://github.com/zip-rs/zip2/pull/70)) +- Allow choosing bzip2 rust backend ([#329](https://github.com/zip-rs/zip2/pull/329)) + +### 🐛 Bug Fixes + +- Need to include zip64 extra field in central directory (fix #353) ([#360](https://github.com/zip-rs/zip2/pull/360)) +- Fails to extract file which might or might not be malformed ([#376](https://github.com/zip-rs/zip2/pull/376)) ([#426](https://github.com/zip-rs/zip2/pull/426)) +- *(aes)* Allow AES encryption while streaming ([#463](https://github.com/zip-rs/zip2/pull/463)) +- Default "platform" field in zip files should be set to the local platform, rather than always "Unix" ([#470](https://github.com/zip-rs/zip2/pull/470)) ([#471](https://github.com/zip-rs/zip2/pull/471)) + +### 🚜 Refactor + +- Define cfg_if! and cfg_if_expr! internal macros ([#438](https://github.com/zip-rs/zip2/pull/438)) + +### ⚡ Performance + +- Change an assert to debug_assert when encrypting/decrypting AES, and eliminate a fallible operation ([#521](https://github.com/zip-rs/zip2/pull/521)) +- eliminate a String clone per new file added to archive, and other related refactors ([#522](https://github.com/zip-rs/zip2/pull/522)) + +### ⚙️ Miscellaneous Tasks + +- Fix another merge error, this one affecting only builds with flate2 and not zopfli +- Fix more merge issues +- Fix merge +- Fix write_dir build errors on specific feature configs +- Fix clippy warning +- Fix --all-features build error +- Fix merge + +## [7.1.0](https://github.com/zip-rs/zip2/compare/v7.0.0...v7.1.0) - 2026-01-14 + +### 🚀 Features + +- display the underlying error in Display impl for ZipError::Display ([#483](https://github.com/zip-rs/zip2/pull/483)) +- Enable creation of `ZipArchive` without reparsing ([#485](https://github.com/zip-rs/zip2/pull/485)) + +### 🐛 Bug Fixes + +- Return InvalidPassword rather than panic when AES key is the wrong length ([#457](https://github.com/zip-rs/zip2/pull/457)) +- bench with auto zip64 comment ([#505](https://github.com/zip-rs/zip2/pull/505)) +- add condition for `getrandom` dependency ([#504](https://github.com/zip-rs/zip2/pull/504)) +- *(zipcrypto)* Support streaming ZipCrypto encryption, don't store entire file in memory ([#462](https://github.com/zip-rs/zip2/pull/462)) + +### 🚜 Refactor + +- Clean up imports and move types ([#461](https://github.com/zip-rs/zip2/pull/461)) +- Replace handwritten `Ord` and `PartialOrd` for `DateTime` ([#484](https://github.com/zip-rs/zip2/pull/484)) + +### ⚙️ Miscellaneous Tasks + +- Lock `lzma-rust2` to at least 0.15.5 ([#491](https://github.com/zip-rs/zip2/pull/491)) + +## [7.0.0](https://github.com/zip-rs/zip2/compare/v6.0.0...v7.0.0) - 2025-12-05 + +### ⚠️ Breaking Changes + +- Removed the following features: `getrandom`, `hmac`, `pbkdf2`, `sha1`, `zeroize`. +- Removed `lzma-static` and `xz-static` feature flags, which were deprecated synonyms of `lzma` and `xz`. ([#405](https://github.com/zip-rs/zip2/pull/405), [#425](https://github.com/zip-rs/zip2/pull/425)) + +### 🚀 Features + +- *(`SimpleFileOptions`)* const DEFAULT implementation ([#474](https://github.com/zip-rs/zip2/pull/474)) +- ZipWriter `set_auto_large_file()` method to enable large-file data descriptor when necessary ([#468](https://github.com/zip-rs/zip2/pull/468)) + +### 🐛 Bug Fixes + +- print previous error when failing to search another cde ([#460](https://github.com/zip-rs/zip2/pull/460)) +- cargo doc warnings ([#472](https://github.com/zip-rs/zip2/pull/472)) +- Write ZIP64 data descriptors when large_file option is true ([#467](https://github.com/zip-rs/zip2/pull/467)) +- Pin generic-array to an old version to work around https://github.com/RustCrypto/traits/issues/2036 until next RustCrypto & aes-crypto releases ([#458](https://github.com/zip-rs/zip2/pull/458)) + +### ⚙️ Miscellaneous Tasks + +- Revert version bump so that release-plz will trigger +- expose more flate2 feature flags ([#476](https://github.com/zip-rs/zip2/pull/476)) +- Next release will be 7.0.0 +- release v6.0.0 ([#442](https://github.com/zip-rs/zip2/pull/442)) + +### Deps + +- Bump lzma-rust2 to v0.15 ([#465](https://github.com/zip-rs/zip2/pull/465)) + +## [6.0.0](https://github.com/zip-rs/zip2/compare/v5.1.1...v6.0.0) - 2025-10-09 + +### 🚀 Features + +- Add by_index_with_options(), which can be used to ignore encryption in a file's metadata ([#439](https://github.com/zip-rs/zip2/pull/439)) and may be used for other file-specific overrides in the future. + +### ⚙️ Miscellaneous Tasks + +- [**breaking**] `FileOptions::add_extra_data` is now generic and accepts any `AsRef<[u8]>`. (#435) + +## [5.1.1](https://github.com/zip-rs/zip2/compare/v5.1.0...v5.1.1) - 2025-09-11 + +### 🐛 Bug Fixes + +- panic when reading empty extended-timestamp field ([#404](https://github.com/zip-rs/zip2/pull/404)) ([#422](https://github.com/zip-rs/zip2/pull/422)) +- Restore original file timestamp when unzipping with `chrono` ([#46](https://github.com/zip-rs/zip2/pull/46)) + +### ⚙️ Miscellaneous Tasks + +- Configure Amazon Q rules ([#421](https://github.com/zip-rs/zip2/pull/421)) + +## [5.1.0](https://github.com/zip-rs/zip2/compare/v5.0.1...v5.1.0) - 2025-09-10 + +### 🚀 Features + +- Add legacy shrink/reduce/implode compression ([#303](https://github.com/zip-rs/zip2/pull/303)) + +## [5.0.1](https://github.com/zip-rs/zip2/compare/v5.0.0...v5.0.1) - 2025-09-09 + +### 🐛 Bug Fixes + +- AES metadata was not copied correctly in raw copy methods, which could corrupt the copied file. ([#417](https://github.com/zip-rs/zip2/pull/417)) + +## [5.0.0](https://github.com/zip-rs/zip2/compare/v4.6.1...v5.0.0) - 2025-09-05 + +### 🚀 Features + +- Implement by_path*() methods on ZipArchive ([#382](https://github.com/zip-rs/zip2/pull/382)) + +## [4.6.1](https://github.com/zip-rs/zip2/compare/v4.6.0...v4.6.1) - 2025-09-03 + +### 🐛 Bug Fixes + +- Fixes an issue introduced by the swap from `lzma-rs` to `liblzma` ([#407](https://github.com/zip-rs/zip2/pull/407)) + +## [4.6.0](https://github.com/zip-rs/zip2/compare/v4.5.0...v4.6.0) - 2025-08-30 + +### 🚀 Features + +- Allow to read zip files with unsupported extended timestamps ([#400](https://github.com/zip-rs/zip2/pull/400)) + +### 🐛 Bug Fixes + +- enable clamp_opt for ppmd and xz ([#401](https://github.com/zip-rs/zip2/pull/401)) + +## [4.5.0](https://github.com/zip-rs/zip2/compare/v4.4.0...v4.5.0) - 2025-08-21 + +### 🚀 Features + +- Allow reading ZIP files where the central directory comes *before* the files ([#384](https://github.com/zip-rs/zip2/pull/384)) ([#396](https://github.com/zip-rs/zip2/pull/396)) + +## [4.4.0](https://github.com/zip-rs/zip2/compare/v4.3.0...v4.4.0) - 2025-08-21 + +### 🚀 Features + +- Add `lzma-static` and `xz-static` features that enable `liblzma/static` ([#393](https://github.com/zip-rs/zip2/pull/393)) + +### ⚙️ Miscellaneous Tasks + +- Move deprecated annotations to fix a Clippy warning ([#391](https://github.com/zip-rs/zip2/pull/391)) + +## [4.3.0](https://github.com/zip-rs/zip2/compare/v4.2.0...v4.3.0) - 2025-07-09 + +### 🚀 Features + +- Add support for PPMd ([#370](https://github.com/zip-rs/zip2/pull/370)) + +## [4.2.0](https://github.com/zip-rs/zip2/compare/v4.1.0...v4.2.0) - 2025-06-21 + +### 🚀 Features + +- Write ZIP file to stream ([#246](https://github.com/zip-rs/zip2/pull/246)) + +## [4.1.0](https://github.com/zip-rs/zip2/compare/v4.0.0...v4.1.0) - 2025-06-14 + +### 🚀 Features + +- Add has_overlapping_files method + +## [4.0.0](https://github.com/zip-rs/zip2/compare/v3.0.0...v4.0.0) - 2025-05-21 + +### 🐛 Bug Fixes + +- Allow extraction of Zip64 where "Version needed to extract" is higher than "Version made by" ([#356](https://github.com/zip-rs/zip2/pull/356)) + +### ⚙️ Miscellaneous Tasks + +- Revert nt-time upgrade (would increase MSRV) +- Revert constant_time_eq update (would increase MSRV) +- Update fully-qualified names of liblzma imports + +## [3.0.0](https://github.com/zip-rs/zip2/compare/v2.6.1...v3.0.0) - 2025-05-14 + +### 🐛 Bug Fixes + +- return correct offset in SeekableTake::seek ([#342](https://github.com/zip-rs/zip2/pull/342)) +- When only zopfli is available, decompression of deflate should not be possible ([#348](https://github.com/zip-rs/zip2/pull/348)) +- Specify `flate2` dependency of the `deflate-flate2` feature. ([#345](https://github.com/zip-rs/zip2/pull/345)) + +### ⚙️ Miscellaneous Tasks + +- drop unused crossbeam-utils dependency ([#339](https://github.com/zip-rs/zip2/pull/339)) +- fix typo +- remove `deflate-flate2` dependency on specific backend +- [**breaking**] Drop deprecated `deflate-miniz` feature flag ([#351](https://github.com/zip-rs/zip2/pull/351)) + +## [2.6.1](https://github.com/zip-rs/zip2/compare/v2.6.0...v2.6.1) - 2025-04-03 + +### 🐛 Bug Fixes + +- avoid scanning through all local file headers while opening an archive ([#281](https://github.com/zip-rs/zip2/pull/281)) + +## [2.5.0](https://github.com/zip-rs/zip2/compare/v2.4.2...v2.5.0) - 2025-03-23 + +### 🚀 Features + +- Add support for `time::PrimitiveDateTime` ([#322](https://github.com/zip-rs/zip2/pull/322)) +- Add `jiff` integration ([#323](https://github.com/zip-rs/zip2/pull/323)) + +### 🐛 Bug Fixes + +- improve error message for duplicated file ([#277](https://github.com/zip-rs/zip2/pull/277)) + +## [2.4.2](https://github.com/zip-rs/zip2/compare/v2.4.1...v2.4.2) - 2025-03-18 + +### 🐛 Bug Fixes + +- `deep_copy_file` produced a mangled file header on big-endian platforms (#309) + +## [2.4.1](https://github.com/zip-rs/zip2/compare/v2.4.0...v2.4.1) - 2025-03-17 + +### 🐛 Bug Fixes + +- type issue in test +- double as_ref().canonicalize()? +- CI failures +- Create directory for extraction if necessary ([#314](https://github.com/zip-rs/zip2/pull/314)) + +## [2.4.0](https://github.com/zip-rs/zip2/compare/v2.3.0...v2.4.0) - 2025-03-17 + +### 🚀 Features + +- `ZipArchive::root_dir` and `ZipArchive::extract_unwrapped_root_dir` ([#304](https://github.com/zip-rs/zip2/pull/304)) + +### 🐛 Bug Fixes + +- wasm build failure due to a missing use statement ([#313](https://github.com/zip-rs/zip2/pull/313)) + +## [2.3.0](https://github.com/zip-rs/zip2/compare/v2.2.3...v2.3.0) - 2025-03-16 + +### 🚀 Features + +- Add support for NTFS extra field ([#279](https://github.com/zip-rs/zip2/pull/279)) + +### 🐛 Bug Fixes + +- *(test)* Conditionalize a zip64 doctest ([#308](https://github.com/zip-rs/zip2/pull/308)) +- fix failing tests, remove symlink loop check +- Canonicalize output path to avoid false negatives +- Symlink handling in stream extraction +- Canonicalize output paths and symlink targets, and ensure they descend from the destination + +### ⚙️ Miscellaneous Tasks + +- Fix clippy and cargo fmt warnings ([#310](https://github.com/zip-rs/zip2/pull/310)) + +## [2.2.3](https://github.com/zip-rs/zip2/compare/v2.2.2...v2.2.3) - 2025-02-26 + +### 🚜 Refactor + +- Change the inner structure of `DateTime` (#267) + +### ⚙️ Miscellaneous Tasks + +- cargo fix --edition + +## [2.2.2](https://github.com/zip-rs/zip2/compare/v2.2.1...v2.2.2) - 2024-12-16 + +### 🐛 Bug Fixes + +- rewrite the EOCD/EOCD64 detection to fix extreme performance regression (#247) + +## [2.2.1](https://github.com/zip-rs/zip2/compare/v2.2.0...v2.2.1) - 2024-11-20 + +### 🐛 Bug Fixes + +- remove executable bit ([#238](https://github.com/zip-rs/zip2/pull/238)) +- *(lzma)* fixed panic in case of invalid lzma stream ([#259](https://github.com/zip-rs/zip2/pull/259)) +- resolve new clippy warnings on nightly ([#262](https://github.com/zip-rs/zip2/pull/262)) +- resolve clippy warning in nightly ([#252](https://github.com/zip-rs/zip2/pull/252)) + +### ⚡ Performance + +- Faster cde rejection ([#255](https://github.com/zip-rs/zip2/pull/255)) + +## [2.2.0](https://github.com/zip-rs/zip2/compare/v2.1.6...v2.2.0) - 2024-08-11 + +### 🚀 Features +- Expose `ZipArchive::central_directory_start` ([#232](https://github.com/zip-rs/zip2/pull/232)) + +## [2.1.6](https://github.com/zip-rs/zip2/compare/v2.1.5...v2.1.6) - 2024-07-29 + +### 🐛 Bug Fixes +- ([#33](https://github.com/zip-rs/zip2/pull/33)) Rare combination of settings could lead to writing a corrupt archive with overlength extra data, and data_start locations when reading the archive back were also wrong ([#221](https://github.com/zip-rs/zip2/pull/221)) + +### 🚜 Refactor +- Eliminate some magic numbers and unnecessary path prefixes ([#225](https://github.com/zip-rs/zip2/pull/225)) + +## [2.1.5](https://github.com/zip-rs/zip2/compare/v2.1.4...v2.1.5) - 2024-07-20 + +### 🚜 Refactor +- change invalid_state() return type to io::Result + +## [2.1.4](https://github.com/zip-rs/zip2/compare/v2.1.3...v2.1.4) - 2024-07-18 + +### 🐛 Bug Fixes +- fix([#215](https://github.com/zip-rs/zip2/pull/215)): Upgrade to deflate64 0.1.9 +- Panic when reading a file truncated in the middle of an XZ block header +- Some archives with over u16::MAX files were handled incorrectly or slowly ([#189](https://github.com/zip-rs/zip2/pull/189)) +- Check number of files when deciding whether a CDE is the real one +- Could still select a fake CDE over a real one in some cases +- May have to consider multiple CDEs before filtering for validity +- We now keep searching for a real CDE header after read an invalid one from the file comment +- Always search for data start when opening an archive for append, and reject the header if data appears to start after central directory +- `deep_copy_file` no longer allows overwriting an existing file, to match the behavior of `shallow_copy_file` +- File start position was wrong when extra data was present +- Abort file if central extra data is too large +- Overflow panic when central directory extra data is too large +- ZIP64 header was being written twice when copying a file +- ZIP64 header was being written to central header twice +- Start position was incorrect when file had no extra data +- Allow all reserved headers we can create +- Fix a bug where alignment padding interacts with other extra-data fields +- Fix bugs involving alignment padding and Unicode extra fields +- Incorrect header when adding AES-encrypted files +- Parse the extra field and reject it if invalid +- Incorrect behavior following a rare combination of `merge_archive`, `abort_file` and `deep_copy_file`. As well, we now return an error when a file is being copied to itself. +- path_to_string now properly handles the case of an empty path +- Implement `Debug` for `ZipWriter` even when it's not implemented for the inner writer's type +- Fix an issue where the central directory could be incorrectly detected +- `finish_into_readable()` would corrupt the archive if the central directory had moved + +### 🚜 Refactor +- Verify with debug assertions that no FixedSizeBlock expects a multi-byte alignment ([#198](https://github.com/zip-rs/zip2/pull/198)) +- Use new do_or_abort_file method + +### ⚡ Performance +- Speed up CRC when encrypting small files +- Limit the number of extra fields +- Refactor extra-data validation +- Store extra data in plain vectors until after validation +- Only build one IndexMap after choosing among the possible valid headers +- Simplify validation of empty extra-data fields +- Validate automatic extra-data fields only once, even if several are present +- Remove redundant `validate_extra_data()` call +- Skip searching for the ZIP32 header if a valid ZIP64 header is present ([#189](https://github.com/zip-rs/zip2/pull/189)) + +### ⚙️ Miscellaneous Tasks +- Fix a bug introduced by c934c824 +- Fix a failing unit test +- Fix build errors on older Rust versions +- Fix build +- Fix another fuzz failure +- Switch to `ok_or_abort_file`, and inline when that fails borrow checker +- Switch to `ok_or_abort_file`, and inline when that fails borrow checker +- Fix a build error +- Fix boxed_local warning (can borrow instead) +- Partial debug +- Fix more errors when parsing multiple extra fields +- Fix an error when decoding AES header +- Fix an error caused by not allowing 0xa11e field +- Bug fix: crypto_header was being counted toward extra_data_end +- Bug fix: revert a change where crypto_header was incorrectly treated as an extra field +- Fix a bug where a modulo of 0 was used +- Fix a bug when ZipCrypto, alignment *and* a custom header are used +- Fix a bug when both ZipCrypto and alignment are used +- Fix another bug: header_end vs extra_data_end +- Fix use of a stale value in a `debug_assert_eq!` +- Fix: may still get an incorrect size if opening an invalid file for append +- Fix: may need the absolute start as tiebreaker to ensure deterministic behavior + +## [2.1.3](https://github.com/zip-rs/zip2/compare/v2.1.2...v2.1.3) - 2024-06-04 + +### 🐛 Bug Fixes +- Some date/time filters were previously unreliable (i.e. later-pass filters had no earliest-pass or latest-fail, and vice-versa) +- Decode Zip-Info UTF8 name and comment fields ([#159](https://github.com/zip-rs/zip2/pull/159)) + +### 🚜 Refactor +- Return extended timestamp fields copied rather than borrowed ([#183](https://github.com/zip-rs/zip2/pull/183)) + +### ⚙️ Miscellaneous Tasks +- Fix a new Clippy warning +- Fix a bug and inline `deserialize` for safety +- Add check for wrong-length blocks, and incorporate fixed-size requirement into the trait name +- Fix a fuzz failure by using checked_sub +- Add feature gate for new unit test + +## [2.1.1](https://github.com/zip-rs/zip2/compare/v2.1.0...v2.1.1) - 2024-05-28 + +### 🐛 Bug Fixes +- Derive `Debug` for `ZipWriter` +- lower default version to 4.5 and use the version-needed-to-extract where feasible. + +### 🚜 Refactor +- use a MIN_VERSION constant + +### ⚙️ Miscellaneous Tasks +- Bug fixes for debug implementation +- Bug fixes for debug implementation +- Update unit tests +- Remove unused import + +## [2.1.0](https://github.com/zip-rs/zip2/compare/v2.0.0...v2.1.0) - 2024-05-25 + +### 🚀 Features +- Support mutual conversion between `DateTime` and MS-DOS pair + +### 🐛 Bug Fixes +- version-needed-to-extract was incorrect in central header, and version-made-by could be lower than that ([#100](https://github.com/zip-rs/zip2/pull/100)) +- version-needed-to-extract was incorrect in central header, and version-made-by could be lower than that ([#100](https://github.com/zip-rs/zip2/pull/100)) + +### ⚙️ Miscellaneous Tasks +- Another tweak to ensure `version_needed` is applied +- Tweaks to make `version_needed` and `version_made_by` work with recently-merged changes + +## [2.0.0](https://github.com/zip-rs/zip2/compare/v1.3.1...v2.0.0) - 2024-05-24 + +### 🚀 Features +- Add `fmt::Display` for `DateTime` +- Implement more traits for `DateTime` + +### 🚜 Refactor +- Change type of `last_modified_time` to `Option` +- [**breaking**] Rename `from_msdos` to `from_msdos_unchecked`, make it unsafe, and add `try_from_msdos` ([#145](https://github.com/zip-rs/zip2/pull/145)) + +### ⚙️ Miscellaneous Tasks +- Continue to accept archives with invalid DateTime, and use `now_utc()` as default only when writing, not reading + +## [1.3.1](https://github.com/zip-rs/zip2/compare/v1.3.0...v1.3.1) - 2024-05-21 + +### 🚜 Refactor +- Make `deflate` enable both default implementations +- Merge the hidden deflate-flate2 flag into the public one +- Rename _deflate-non-zopfli to _deflate-flate2 +- Reject encrypted and using_data_descriptor files slightly faster in read_zipfile_from_stream +- Convert `impl TryInto for DateTime` to `impl TryFrom for NaiveDateTime` ([#136](https://github.com/zip-rs/zip2/pull/136)) + +### ⚡ Performance +- Change default compression implementation to `flate2/zlib-ng` + +### ⚙️ Miscellaneous Tasks +- chore([#132](https://github.com/zip-rs/zip2/pull/132)): Attribution for some copied test data +- chore([#133](https://github.com/zip-rs/zip2/pull/133)): chmod -x src/result.rs + +## [1.3.0](https://github.com/zip-rs/zip2/compare/v1.2.3...v1.3.0) - 2024-05-17 + +### 🚀 Features +- Add `is_symlink` method + +### 🐛 Bug Fixes +- Extract symlinks into symlinks on Unix and Windows, and fix a bug that affected making directories writable on MacOS + +### 🚜 Refactor +- Eliminate deprecation warning when `--all-features` implicitly enables the deprecated feature +- Check if archive contains a symlink's target, without borrowing both at the same time +- Eliminate a clone that's no longer necessary +- is_dir only needs to look at the filename +- Remove unnecessary #[cfg] attributes + +### ⚙️ Miscellaneous Tasks +- Fix borrow-of-moved-value +- Box doesn't directly convert to PathBuf, so convert back to String first +- partial revert - only &str has chars(), but Box should auto-deref +- contains_key needs a `Box`, so generify `is_dir` to accept one +- Add missing `ZipFileData::is_dir()` method +- Fix another Windows-specific error +- More bug fixes for Windows-specific symlink code +- More bug fixes for Windows-specific symlink code +- Bug fix: variable name change +- Bug fix: need both internal and output path to determine whether to symlink_dir +- Another bug fix +- Fix another error-type conversion error +- Fix error-type conversion on Windows +- Fix conditionally-unused import +- Fix continued issues, and factor out the Vec-to-OsString conversion (cc: [#125](https://github.com/zip-rs/zip2/pull/125)) +- Fix CI failure involving conversion to OsString for symlinks (see my comments on [#125](https://github.com/zip-rs/zip2/pull/125)) +- Move path join into platform-independent code + +## [1.2.3](https://github.com/zip-rs/zip2/compare/v1.2.2...v1.2.3) - 2024-05-10 + +### 🐛 Bug Fixes +- Remove a window when an extracted directory might be unexpectedly listable and/or `cd`able by non-owners +- Extract directory contents on Unix even if the directory doesn't have write permission (https://github.com/zip-rs/zip-old/issues/423) + +### ⚙️ Miscellaneous Tasks +- More conditionally-unused imports + +## [1.2.2](https://github.com/zip-rs/zip2/compare/v1.2.1...v1.2.2) - 2024-05-09 + +### 🐛 Bug Fixes +- Failed to clear "writing_raw" before finishing a symlink, leading to dropped extra fields + +### ⚡ Performance +- Use boxed slice for archive comment, since it can't be concatenated +- Optimize for the fact that false signatures can't overlap with real ones + +## [1.2.1](https://github.com/zip-rs/zip2/compare/v1.2.0...v1.2.1) - 2024-05-06 + +### 🐛 Bug Fixes +- Prevent panic when trying to read a file with an unsupported compression method +- Prevent panic after reading an invalid LZMA file +- Make `Stored` the default compression method if `Deflated` isn't available, so that zip files are readable by as much software as possible +- version_needed was wrong when e.g. cfg(bzip2) but current file wasn't bzip2 ([#100](https://github.com/zip-rs/zip2/pull/100)) +- file paths shouldn't start with slashes ([#102](https://github.com/zip-rs/zip2/pull/102)) + +### 🚜 Refactor +- Overhaul `impl Arbitrary for FileOptions` +- Remove unused `atomic` module + +## [1.2.0](https://github.com/zip-rs/zip2/compare/v1.1.4...v1.2.0) - 2024-05-06 + +### 🚀 Features +- Add method `decompressed_size()` so non-recursive ZIP bombs can be detected + +### 🚜 Refactor +- Make `ZipWriter::finish()` consume the `ZipWriter` + +### ⚙️ Miscellaneous Tasks +- Use panic! rather than abort to ensure the fuzz harness can process the failure +- Update fuzz_write to use replace_with +- Remove a drop that can no longer be explicit +- Add `#![allow(unexpected_cfgs)]` in nightly + +## [1.1.4](https://github.com/zip-rs/zip2/compare/v1.1.3...v1.1.4) - 2024-05-04 + +### 🐛 Bug Fixes +- Build was failing with bzip2 enabled +- use is_dir in more places where Windows paths might be handled incorrectly + +### ⚡ Performance +- Quick filter for paths that contain "/../" or "/./" or start with "./" or "../" +- Fast handling for separator-free paths +- Speed up logic if main separator isn't '/' +- Drop `normalized_components` slightly sooner when not using it +- Speed up `path_to_string` in cases where the path is already in the proper format + +### ⚙️ Miscellaneous Tasks +- Refactor: can short-circuit handling of paths that start with MAIN_SEPARATOR, no matter what MAIN_SEPARATOR is +- Bug fix: non-canonical path detection when MAIN_SEPARATOR is not slash or occurs twice in a row +- Bug fix: must recreate if . or .. is a path element +- Bug fix + +### ◀️ Revert +- [#58](https://github.com/zip-rs/zip2/pull/58) (partial): `bzip2-rs` can't replace `bzip2` because it's decompress-only + +## [1.1.3](https://github.com/zip-rs/zip2/compare/v1.1.2...v1.1.3) - 2024-04-30 + +### 🐛 Bug Fixes +- Rare bug where find_and_parse would give up prematurely on detecting a false end-of-CDR header + +## [1.1.2](https://github.com/Pr0methean/zip/compare/v1.1.1...v1.1.2) - 2024-04-28 + +### 🐛 Bug Fixes +- Alignment was previously handled incorrectly ([#33](https://github.com/Pr0methean/zip/pull/33)) + +### 🚜 Refactor +- deprecate `deflate-miniz` feature since it's now equivalent to `deflate` ([#35](https://github.com/Pr0methean/zip/pull/35)) + +## [1.1.1] + +### Added + +- `index_for_name`, `index_for_path`, `name_for_index`: get the index of a file given its path or vice-versa, without + initializing metadata from the local-file header or needing to mutably borrow the `ZipArchive`. +- `add_symlink_from_path`, `shallow_copy_file_from_path`, `deep_copy_file_from_path`, `raw_copy_file_to_path`: copy a + file or create a symlink using `AsRef` arguments + +### Changed + +- `add_directory_from_path` and `start_file_from_path` are no longer deprecated, and they now normalize `..` as well as + `.`. + +## [1.1.0] + +### Added + +- Support for decoding LZMA. + +### Changed + +- Eliminated a custom `AtomicU64` type by replacing it with `OnceLock` in the only place it's used. +- `FileOptions` now has the subtype `SimpleFileOptions` which implements `Copy` but has no extra data. + +## [1.0.1] + +### Changed + +- The published package on crates.io no longer includes the tests or examples. + +## [1.0.0] + +### Changed + +- Now uses boxed slices rather than `String` or `Vec` for metadata fields that aren't likely to grow. + +## [0.11.0] + +### Added + +- Support for `DEFLATE64` (decompression only). +- Support for Zopfli compression levels up to `i64::MAX`. + +### Changed + +- `InvalidPassword` is now a kind of `ZipError` to eliminate the need for nested `Result` structs. +- Updated dependencies. + +## [0.10.3] + +### Changed + +- Updated dependencies. +- MSRV increased to `1.67`. + +### Fixed + +- Fixed some rare bugs that could cause panics when trying to read an invalid ZIP file or using an incorrect password. + +## [0.10.2] + +### Changed + +- Where possible, methods are now `const`. This improves performance, especially when reading. + +## [0.10.1] + +### Changed + +- Date and time conversion methods now return `DateTimeRangeError` rather than `()` on error. + +## [0.10.0] + +### Changed + +- Replaces the `flush_on_finish_file` parameter of `ZipWriter::new` and `ZipWriter::Append` with + a `set_flush_on_finish_file` method. + +### Fixed + +- Fixes build errors that occur when all default features are disabled. +- Fixes more cases of a bug when ZIP64 magic bytes occur in filenames. + +## [0.9.2] + +### Added + +- `zlib-ng` for fast Deflate compression. This is now the default for compression levels 0-9. +- `chrono` to convert zip::DateTime to and from chrono::NaiveDateTime + +## [0.9.1] + +### Added + +- Zopfli for aggressive Deflate compression. + +## [0.9.0] + +### Added + + - `flush_on_finish_file` parameter for `ZipWriter`. + +## [0.8.3] + +### Changed + +- Uses the `aes::cipher::KeyInit` trait from `aes` 0.8.2 where appropriate. + +### Fixed + +- Calling `abort_file()` no longer corrupts the archive if called on a + shallow copy of a remaining file, or on an archive whose CDR entries are out + of sequence. However, it may leave an unused entry in the archive. +- Calling `abort_file()` while writing a ZipCrypto-encrypted file no longer + causes a crash. +- Calling `abort_file()` on the last file before `finish()` no longer produces + an invalid ZIP file or garbage in the comment. + +### Added + +- `ZipWriter` methods `get_comment()` and `get_raw_comment()`. + +## [0.8.2] + +### Fixed + +- Fixed an issue where code might spuriously fail during write fuzzing. + +### Added + +- New method `with_alignment` on `FileOptions`. + +## [0.8.1] + +### Fixed + +- `ZipWriter` now once again implements `Send` if the underlying writer does. + +## [0.8.0] + +### Deleted + +- Methods `start_file_aligned`, `start_file_with_extra_data`, `end_local_start_central_extra_data` and + `end_extra_data` (see below). + +### Changed + +- Alignment and extra-data fields are now attributes of [`zip::unstable::write::FileOptions`], allowing them to be + specified for `add_directory` and `add_symlink`. +- Extra-data fields are now formatted by the `FileOptions` method `add_extra_data`. +- Improved performance, especially for `shallow_copy_file` and `deep_copy_file` on files with extra data. + +### Fixed + +- Fixes a rare bug where the size of the extra-data field could overflow when `large_file` was set. +- Fixes more cases of a bug when ZIP64 magic bytes occur in filenames. + +## [0.7.5] + +### Fixed + +- Fixed a bug that occurs when ZIP64 magic bytes occur twice in a filename or across two filenames. + +## [0.7.4] + +### Added + +- Added experimental [`zip::unstable::write::FileOptions::with_deprecated_encryption`] API to enable encrypting + files with PKWARE encryption. + +## [0.7.3] + +### Fixed + +- Fixed a bug that occurs when a filename in a ZIP32 file includes the ZIP64 magic bytes. + +## [0.7.2] + +### Added + +- Method `abort_file` - removes the current or most recently-finished file from the archive. + +### Fixed + +- Fixed a bug where a file could remain open for writing after validations failed. + +## [0.7.1] + +### Changed + +- Bumped the version number in order to upload an updated README to crates.io. + +## [0.7.0] + +### Fixed + +- Calling `start_file` with invalid parameters no longer closes the `ZipWriter`. +- Attempting to write a 4GiB file without calling `FileOptions::large_file(true)` now removes the file from the archive + but does not close the `ZipWriter`. +- Attempting to write a file with an unrepresentable or invalid last-modified date will instead add it with a date of + 1980-01-01 00:00:00. + +### Added + +- Method `is_writing_file` - indicates whether a file is open for writing. + +## [0.6.13] + +### Fixed + +- Fixed a possible bug in deep_copy_file. + +## [0.6.12] + +### Fixed + +- Fixed a Clippy warning that was missed during the last release. + +## [0.6.11] + +### Fixed + +- Fixed a bug that could cause later writes to fail after a `deep_copy_file` call. + +## [0.6.10] + +### Changed + +- Updated dependency versions. + +## [0.6.9] + +### Fixed + +- Fixed an issue that prevented `ZipWriter` from implementing `Send`. + +## [0.6.8] + +### Added + +- Detects duplicate filenames. + +### Fixed + +- `deep_copy_file` could set incorrect Unix permissions. +- `deep_copy_file` could handle files incorrectly if their compressed size was u32::MAX bytes or less but their + uncompressed size was not. +- Documented that `deep_copy_file` does not copy a directory's contents. + +### Changed + +- Improved performance of `deep_copy_file` by using a HashMap and eliminating a redundant search. + +## [0.6.7] + +### Added + +- `deep_copy_file` method: more standards-compliant way to copy a file from within the ZipWriter + +## [0.6.6] + +### Fixed + +- Unused flag `#![feature(read_buf)]` was breaking compatibility with stable compiler. + +### Changed + +- Updated `aes` dependency to `0.8.2` (https://github.com/zip-rs/zip/pull/354) +- Updated other dependency versions. + +## [0.6.5] +### Changed + +- Added experimental [`zip::unstable::write::FileOptions::with_deprecated_encryption`] API to enable encrypting files with PKWARE encryption. + +### Added + +- `shallow_copy_file` method: copy a file from within the ZipWriter + + +## [0.6.4] + +### Changed + +- [#333](https://github.com/zip-rs/zip/pull/333): disabled the default features of the `time` dependency, and also `formatting` and `macros`, as they were enabled by mistake. +- Deprecated [`DateTime::from_time`](https://docs.rs/zip/0.6/zip/struct.DateTime.html#method.from_time) in favor of [`DateTime::try_from`](https://docs.rs/zip/0.6/zip/struct.DateTime.html#impl-TryFrom-for-DateTime) diff --git a/v2/vendor/security/zip-7.2.0/CODE_OF_CONDUCT.md b/v2/vendor/security/zip-7.2.0/CODE_OF_CONDUCT.md new file mode 100644 index 0000000000..2290ec2bf8 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/CODE_OF_CONDUCT.md @@ -0,0 +1,76 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to make participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, ethnicity, sex characteristics, gender identity and expression, +level of experience, education, socio-economic status, nationality, personal +appearance, race, religion, or sexual identity and orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment +include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or + advances +* Trolling, insulting/derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or electronic + address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or +reject comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct, or to ban temporarily or +permanently any contributor for other behaviors that they deem inappropriate, +threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies within all project spaces, and it also applies when +an individual is representing the project or its community in public spaces. +Examples of representing a project or community include using an official +project e-mail address, posting via an official social media account, or acting +as an appointed representative at an online or offline event. Representation of +a project may be further defined and clarified by project maintainers. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported by contacting the project team at ryan.levick@gmail.com. All +complaints will be reviewed and investigated and will result in a response that +is deemed necessary and appropriate to the circumstances. The project team is +obligated to maintain confidentiality with regard to the reporter of an incident. +Further details of specific enforcement policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good +faith may face temporary or permanent repercussions as determined by other +members of the project's leadership. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, +available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html + +[homepage]: https://www.contributor-covenant.org + +For answers to common questions about this code of conduct, see +https://www.contributor-covenant.org/faq diff --git a/v2/vendor/security/zip-7.2.0/CONTRIBUTING.md b/v2/vendor/security/zip-7.2.0/CONTRIBUTING.md new file mode 100644 index 0000000000..8d638d9f7b --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/CONTRIBUTING.md @@ -0,0 +1 @@ +Pull requests are welcome, but they're subject to some requirements that a lot of them don't meet. See https://github.com/zip-rs/zip2/raw/master/pull_request_template.md for details. diff --git a/v2/vendor/security/zip-7.2.0/Cargo.lock b/v2/vendor/security/zip-7.2.0/Cargo.lock new file mode 100644 index 0000000000..342a95aa45 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/Cargo.lock @@ -0,0 +1,1300 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures", +] + +[[package]] +name = "anstream" +version = "0.6.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78" + +[[package]] +name = "anstyle-parse" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys", +] + +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" +dependencies = [ + "derive_arbitrary", +] + +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "autocfg" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + +[[package]] +name = "bencher" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dfdb4953a096c551ce9ace855a604d702e6e62d77fac690575ae347571717f5" + +[[package]] +name = "bitflags" +version = "2.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" + +[[package]] +name = "bitstream-io" +version = "4.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60d4bd9d1db2c6bdf285e223a7fa369d5ce98ec767dec949c6ca62863ce61757" +dependencies = [ + "core2", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bumpalo" +version = "3.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dd9dc738b7a8311c7ade152424974d8115f2cdad61e8dab8dac9f2362298510" + +[[package]] +name = "bzip2" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3a53fac24f34a81bc9954b5d6cfce0c21e18ec6959f44f56e8e90e4bb7c346c" +dependencies = [ + "bzip2-sys", + "libbz2-rs-sys", +] + +[[package]] +name = "bzip2-sys" +version = "0.1.13+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" +dependencies = [ + "cc", + "pkg-config", +] + +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + +[[package]] +name = "cc" +version = "1.2.53" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "755d2fce177175ffca841e9a06afdb2c4ab0f593d53b4dee48147dfaade85932" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "chrono" +version = "0.4.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fac4744fb15ae8337dc853fee7fb3f4e48c0fbaa23d0afe49c447b4fab126118" +dependencies = [ + "num-traits", +] + +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", +] + +[[package]] +name = "clap" +version = "4.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e578d6ec4194633722ccf9544794b71b1385c3c027efe0c55db226fc880865c" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4df4df40ec50c46000231c914968278b1eb05098cf8f1b3a518a95030e71d1c7" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf9804afaaf59a91e75b022a30fb7229a7901f60c755489cc61c9b423b836442" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1" + +[[package]] +name = "cmake" +version = "0.1.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75443c44cd6b379beb8c5b45d85d0773baf31cce901fe7bb252f4eff3008ef7d" +dependencies = [ + "cc", +] + +[[package]] +name = "colorchoice" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" + +[[package]] +name = "constant_time_eq" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" + +[[package]] +name = "core2" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b49ba7ef1ad6107f8824dbe97de947cbaac53c44e7f9756a1fba0d37c1eec505" +dependencies = [ + "memchr", +] + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crc" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9710d3b3739c2e349eb44fe848ad0b7c8cb1e42bd87ee49371df2f7acaf3e675" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "deflate64" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26bf8fc351c5ed29b5c2f0cbbac1b209b74f60ecd62e675a998df72c49af5204" + +[[package]] +name = "deranged" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ececcb659e7ba858fb4f10388c250a7252eb0a27373f1a72b8748afdd248e587" +dependencies = [ + "powerfmt", +] + +[[package]] +name = "derive_arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", + "subtle", +] + +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + +[[package]] +name = "find-msvc-tools" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8591b0bcc8a98a64310a2fae1bb3e9b8564dd10e381e6e28010fde8e8e8568db" + +[[package]] +name = "flate2" +version = "1.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b375d6465b98090a5f25b1c7703f3859783755aa9a80433b36e0379a3ec2f369" +dependencies = [ + "crc32fast", + "libz-ng-sys", + "libz-sys", + "miniz_oxide", + "zlib-rs", +] + +[[package]] +name = "futures-core" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" + +[[package]] +name = "futures-task" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" + +[[package]] +name = "futures-util" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi", + "wasip2", + "wasm-bindgen", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + +[[package]] +name = "indexmap" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "generic-array", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itoa" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2" + +[[package]] +name = "jiff" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e67e8da4c49d6d9909fe03361f9b620f58898859f5c7aded68351e85e71ecf50" +dependencies = [ + "jiff-static", + "jiff-tzdb-platform", + "log", + "portable-atomic", + "portable-atomic-util", + "serde_core", + "windows-sys", +] + +[[package]] +name = "jiff-static" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0c84ee7f197eca9a86c6fd6cb771e55eb991632f15f2bc3ca6ec838929e6e78" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "jiff-tzdb" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68971ebff725b9e2ca27a601c5eb38a4c5d64422c4cbab0c535f248087eda5c2" + +[[package]] +name = "jiff-tzdb-platform" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "875a5a69ac2bab1a891711cf5eccbec1ce0341ea805560dcd90b7a2e925132e8" +dependencies = [ + "jiff-tzdb", +] + +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.85" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c942ebf8e95485ca0d52d97da7c5a2c387d0e7f0ba4c35e93bfcaee045955b3" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "libbz2-rs-sys" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c4a545a15244c7d945065b5d392b2d2d7f21526fba56ce51467b06ed445e8f7" + +[[package]] +name = "libc" +version = "0.2.180" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bcc35a38544a891a5f7c865aca548a982ccb3b8650a5b06d0fd33a10283c56fc" + +[[package]] +name = "libm" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" + +[[package]] +name = "libz-ng-sys" +version = "1.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bf914b7dd154ca9193afec311d8e39345c1bd93b48b3faa77329f0db8f553c0" +dependencies = [ + "cmake", + "libc", +] + +[[package]] +name = "libz-sys" +version = "1.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15d118bbf3771060e7311cc7bb0545b01d08a8b4a7de949198dec1fa0ca1c0f7" +dependencies = [ + "cc", + "cmake", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "linux-raw-sys" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" + +[[package]] +name = "log" +version = "0.4.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" + +[[package]] +name = "lzma-rust2" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1670343e58806300d87950e3401e820b519b9384281bbabfb15e3636689ffd69" +dependencies = [ + "crc", + "sha2", +] + +[[package]] +name = "memchr" +version = "2.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" + +[[package]] +name = "minicov" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4869b6a491569605d66d3952bcdf03df789e5b536e5f0cf7758a7f08a55ae24d" +dependencies = [ + "cc", + "walkdir", +] + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "nt-time" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c367e8edaff1f8a871e56343eb5e03888f6e0d1c2861880ccf4b7dc830899ed" +dependencies = [ + "time", +] + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "once_cell" +version = "1.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + +[[package]] +name = "pbkdf2" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" +dependencies = [ + "digest", + "hmac", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkg-config" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" + +[[package]] +name = "portable-atomic" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f89776e4d69bb58bc6993e99ffa1d11f228b839984854c7daeb5d37f87cbe950" + +[[package]] +name = "portable-atomic-util" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8a2f0d8d040d7848a709caf78912debcc3f33ee4b3cac47d73d1e1069e83507" +dependencies = [ + "portable-atomic", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppmd-rust" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d558c559f0450f16f2a27a1f017ef38468c1090c9ce63c8e51366232d53717b4" + +[[package]] +name = "proc-macro2" +version = "1.0.105" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "535d180e0ecab6268a3e718bb9fd44db66bbbc256257165fc699dadf70d16fe7" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc74d9a594b72ae6656596548f56f667211f8a97b3d4c3d467150794690dc40a" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "rayon" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "rustix" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "146c9e247ccc180c1f61615433868c99f3de3ae256a30a43b49f67c2d9171f34" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.149" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "simd-adler32" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2" + +[[package]] +name = "slab" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589" + +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4d107df263a3013ef9b1879b0df87d706ff80f65a86ea879bd9c31f9b307c2a" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "655da9c7eb6305c55742045d5a8d2037996d61d8de95806335c7c86ce0f82e9c" +dependencies = [ + "fastrand", + "getrandom", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "time" +version = "0.3.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9e442fc33d7fdb45aa9bfeb312c095964abdf596f7567261062b2a7107aaabd" +dependencies = [ + "deranged", + "itoa", + "js-sys", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b36ee98fd31ec7426d599183e8fe26932a8dc1fb76ddb6214d05493377d34ca" + +[[package]] +name = "time-macros" +version = "0.2.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71e552d1249bf61ac2a52db88179fd0673def1e1ad8243a00d9ec9ed71fee3dd" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "typed-path" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7922f2cdc51280d47b491af9eafc41eb0cdab85eabcb390c854412fcbf26dbe8" + +[[package]] +name = "typenum" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" + +[[package]] +name = "unicode-ident" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "wasip2" +version = "1.0.2+wasi-0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.108" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64024a30ec1e37399cf85a7ffefebdb72205ca1c972291c51512360d90bd8566" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70a6e77fd0ae8029c9ea0063f87c46fde723e7d887703d74ad2616d792e51e6f" +dependencies = [ + "cfg-if", + "futures-util", + "js-sys", + "once_cell", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.108" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "008b239d9c740232e71bd39e8ef6429d27097518b6b30bdf9086833bd5b6d608" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.108" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5256bae2d58f54820e6490f9839c49780dff84c65aeab9e772f15d5f0e913a55" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.108" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f01b580c9ac74c8d8f0c0e4afb04eeef2acf145458e52c03845ee9cd23e3d12" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-bindgen-test" +version = "0.3.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45649196a53b0b7a15101d845d44d2dda7374fc1b5b5e2bbf58b7577ff4b346d" +dependencies = [ + "async-trait", + "cast", + "js-sys", + "libm", + "minicov", + "nu-ansi-term", + "num-traits", + "oorandom", + "serde", + "serde_json", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-bindgen-test-macro", + "wasm-bindgen-test-shared", +] + +[[package]] +name = "wasm-bindgen-test-macro" +version = "0.3.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f579cdd0123ac74b94e1a4a72bd963cf30ebac343f2df347da0b8df24cdebed2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "wasm-bindgen-test-shared" +version = "0.2.108" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8145dd1593bf0fb137dbfa85b8be79ec560a447298955877804640e40c2d6ea" + +[[package]] +name = "web-sys" +version = "0.3.85" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "312e32e551d92129218ea9a2452120f4aabc03529ef03e4d0d82fb2780608598" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "wit-bindgen" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" + +[[package]] +name = "zeroize" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85a5b4158499876c763cb03bc4e49185d3cccbabb15b33c627f7884f43db852e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zip" +version = "7.2.0" +dependencies = [ + "aes", + "arbitrary", + "bencher", + "bitstream-io", + "bzip2", + "chrono", + "clap", + "constant_time_eq", + "crc32fast", + "deflate64", + "flate2", + "generic-array", + "getrandom", + "hmac", + "indexmap", + "jiff", + "lzma-rust2", + "memchr", + "nt-time", + "pbkdf2", + "ppmd-rust", + "rayon", + "sha1", + "tempfile", + "time", + "typed-path", + "walkdir", + "wasm-bindgen-test", + "zeroize", + "zopfli", + "zstd", +] + +[[package]] +name = "zlib-rs" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40990edd51aae2c2b6907af74ffb635029d5788228222c4bb811e9351c0caad3" + +[[package]] +name = "zmij" +version = "1.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfcd145825aace48cff44a8844de64bf75feec3080e0aa5cdbde72961ae51a65" + +[[package]] +name = "zopfli" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249" +dependencies = [ + "bumpalo", + "crc32fast", + "log", + "simd-adler32", +] + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/v2/vendor/security/zip-7.2.0/Cargo.toml b/v2/vendor/security/zip-7.2.0/Cargo.toml new file mode 100644 index 0000000000..91548c5cf5 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/Cargo.toml @@ -0,0 +1,330 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + +[package] +edition = "2021" +rust-version = "1.83.0" +name = "zip" +version = "7.2.0" +authors = [ + "Mathijs van de Nes ", + "Marli Frost ", + "Ryan Levick ", + "Chris Hennick ", +] +build = false +exclude = [ + "tests/**", + "examples/**", + ".github/**", + "fuzz_read/**", + "fuzz_write/**", +] +autolib = false +autobins = false +autoexamples = false +autotests = false +autobenches = false +description = """ +Library to support the reading and writing of zip files. +""" +readme = "README.md" +keywords = [ + "zip", + "archive", + "compression", +] +categories = [ + "compression", + "filesystem", + "parser-implementations", +] +license = "MIT" +repository = "https://github.com/zip-rs/zip2.git" + +[package.metadata.docs.rs] +all-features = true +rustdoc-args = [ + "--cfg", + "docsrs", +] + +[features] +_all-features = [] +_arbitrary = ["dep:arbitrary"] +_bzip2_any = [] +_deflate-any = [] +aes-crypto = [ + "dep:aes", + "dep:constant_time_eq", + "dep:generic-array", + "getrandom/std", + "dep:hmac", + "dep:pbkdf2", + "dep:sha1", + "dep:zeroize", +] +bzip2 = [ + "dep:bzip2", + "bzip2/default", + "_bzip2_any", +] +bzip2-rs = [ + "dep:bzip2", + "bzip2/bzip2-sys", + "_bzip2_any", +] +chrono = ["dep:chrono"] +default = [ + "aes-crypto", + "bzip2", + "deflate64", + "deflate", + "lzma", + "ppmd", + "time", + "zstd", + "xz", +] +deflate = [ + "deflate-zopfli", + "deflate-flate2-zlib-rs", +] +deflate-flate2 = [ + "_deflate-any", + "dep:flate2", +] +deflate-flate2-zlib = [ + "deflate-flate2", + "flate2/zlib", +] +deflate-flate2-zlib-ng = [ + "deflate-flate2", + "flate2/zlib-ng", +] +deflate-flate2-zlib-ng-compat = [ + "deflate-flate2", + "flate2/zlib-ng-compat", +] +deflate-flate2-zlib-rs = [ + "deflate-flate2", + "flate2/zlib-rs", +] +deflate-zopfli = [ + "dep:zopfli", + "_deflate-any", +] +jiff-02 = ["dep:jiff"] +legacy-zip = ["bitstream-io"] +lzma = ["dep:lzma-rust2"] +nt-time = ["dep:nt-time"] +ppmd = ["dep:ppmd-rust"] +unreserved = [] +xz = ["dep:lzma-rust2"] + +[lib] +name = "zip" +path = "src/lib.rs" + +[[bench]] +name = "merge_archive" +path = "benches/merge_archive.rs" +harness = false + +[[bench]] +name = "read_entry" +path = "benches/read_entry.rs" +harness = false + +[[bench]] +name = "read_metadata" +path = "benches/read_metadata.rs" +harness = false + +[dependencies.aes] +version = "0.8" +optional = true + +[dependencies.arbitrary] +version = "1.4" +features = ["derive"] +optional = true + +[dependencies.bitstream-io] +version = "4.5" +optional = true + +[dependencies.bzip2] +version = "0.6" +optional = true +default-features = false + +[dependencies.chrono] +version = "^0.4.27" +features = ["std"] +optional = true +default-features = false + +[dependencies.constant_time_eq] +version = "0.3" +optional = true + +[dependencies.crc32fast] +version = "1.5" + +[dependencies.deflate64] +version = "0.1.10" +optional = true + +[dependencies.flate2] +version = "1.1" +optional = true +default-features = false + +[dependencies.generic-array] +version = "=0.14.7" +optional = true + +[dependencies.hmac] +version = "0.12" +features = ["reset"] +optional = true + +[dependencies.indexmap] +version = "2" + +[dependencies.jiff] +version = "0.2.4" +optional = true + +[dependencies.lzma-rust2] +version = "0.15" +features = [ + "std", + "encoder", + "optimization", + "xz", +] +optional = true +default-features = false + +[dependencies.memchr] +version = "2.7" + +[dependencies.nt-time] +version = "0.10.6" +optional = true +default-features = false + +[dependencies.pbkdf2] +version = "0.12" +optional = true + +[dependencies.ppmd-rust] +version = "1.3" +optional = true + +[dependencies.sha1] +version = "0.10" +optional = true + +[dependencies.time] +version = "0.3" +features = ["std"] +optional = true +default-features = false + +[dependencies.typed-path] +version = "0.12" + +[dependencies.zeroize] +version = "1.8" +features = ["zeroize_derive"] +optional = true + +[dependencies.zopfli] +version = "0.8" +optional = true + +[dependencies.zstd] +version = "^0.13.3" +optional = true +default-features = false + +[dev-dependencies.bencher] +version = "0.1" + +[dev-dependencies.clap] +version = "=4.4.18" +features = ["derive"] + +[dev-dependencies.getrandom] +version = "0.3" +features = [] +default-features = false + +[dev-dependencies.rayon] +version = "1.11" + +[dev-dependencies.tempfile] +version = "3.15" + +[dev-dependencies.time] +version = "0.3" +features = [ + "formatting", + "macros", +] + +[dev-dependencies.walkdir] +version = "2.5" + +[target.'cfg(not(target_arch = "wasm32"))'.dependencies.getrandom] +version = "0.3" +optional = true +default-features = false + +[target.'cfg(not(target_arch = "wasm32"))'.dependencies.time] +version = "0.3" +features = ["wasm-bindgen"] +optional = true +default-features = false + +[target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies.getrandom] +version = "0.3" +default-features = false + +[target.'cfg(target_arch = "wasm32")'.dependencies.getrandom] +version = "0.3" +features = ["wasm_js"] +optional = true +default-features = false + +[target.'cfg(target_arch = "wasm32")'.dependencies.time] +version = "0.3" +features = [ + "std", + "wasm-bindgen", +] +optional = true +default-features = false + +[target.'cfg(target_arch = "wasm32")'.dev-dependencies.getrandom] +version = "0.3" +features = ["wasm_js"] +default-features = false + +[target.'cfg(target_arch = "wasm32")'.dev-dependencies.wasm-bindgen-test] +version = "^0.3.56" + +[[test]] +name = "ruview_0429_backport" +path = "tests/ruview_0429_backport.rs" diff --git a/v2/vendor/security/zip-7.2.0/Cargo.toml.orig b/v2/vendor/security/zip-7.2.0/Cargo.toml.orig new file mode 100644 index 0000000000..761bf24a7e --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/Cargo.toml.orig @@ -0,0 +1,139 @@ +[package] +name = "zip" +version = "7.2.0" +authors = [ + "Mathijs van de Nes ", + "Marli Frost ", + "Ryan Levick ", + "Chris Hennick ", +] +license = "MIT" +repository = "https://github.com/zip-rs/zip2.git" +keywords = ["zip", "archive", "compression"] +# Any change to rust-version must be reflected also in `README.md` and `.github/workflows/ci.yaml`. +# The MSRV policy is documented in `README.md`. +rust-version = "1.83.0" +categories = ["compression", "filesystem", "parser-implementations"] +description = """ +Library to support the reading and writing of zip files. +""" +edition = "2021" +exclude = ["tests/**", "examples/**", ".github/**", "fuzz_read/**", "fuzz_write/**"] + +[package.metadata.docs.rs] +all-features = true +rustdoc-args = ["--cfg", "docsrs"] + +[dependencies] +arbitrary = { version = "1.4", features = ["derive"], optional = true } +aes = { version = "0.8", optional = true } +bzip2 = { version = "0.6", default-features = false, optional = true } +chrono = { version = "^0.4.27", optional = true, default-features = false, features = ["std"] } +constant_time_eq = { version = "0.3", optional = true } +crc32fast = "1.5" +flate2 = { version = "1.1", default-features = false, optional = true } +generic-array = { version = "=0.14.7", optional = true } +hmac = { version = "0.12", optional = true, features = ["reset"] } +indexmap = "2" +jiff = { version = "0.2.4", optional = true } +memchr = "2.7" +nt-time = { version = "0.10.6", default-features = false, optional = true } +ppmd-rust = { version = "1.3", optional = true } +pbkdf2 = { version = "0.12", optional = true } +sha1 = { version = "0.10", optional = true } +time = { version = "0.3", default-features = false, optional = true, features = [ + "std", +] } +zeroize = { version = "1.8", optional = true, features = ["zeroize_derive"] } +zstd = { version = "^0.13.3", optional = true, default-features = false } +zopfli = { version = "0.8", optional = true } +deflate64 = { version = "0.1.10", optional = true } +lzma-rust2 = { version = "0.15", optional = true, default-features = false, features = ["std", "encoder", "optimization", "xz"] } +bitstream-io = { version = "4.5", optional = true } +typed-path = "0.12" + +# native: +[target.'cfg(not(target_arch = "wasm32"))'.dependencies] +getrandom = { version = "0.3", default-features = false, optional = true } +time = { version = "0.3", default-features = false, optional = true, features = [ + "wasm-bindgen" +] } + +[target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies] +getrandom = { version = "0.3", default-features = false } + +# wasm32 +[target.'cfg(target_arch = "wasm32")'.dependencies] +getrandom = { version = "0.3", default-features = false, features = ["wasm_js"], optional = true } +time = { version = "0.3", default-features = false, optional = true, features = [ + "std", + "wasm-bindgen" +] } + +[target.'cfg(target_arch = "wasm32")'.dev-dependencies] +getrandom = { version = "0.3", default-features = false, features = ["wasm_js"] } +wasm-bindgen-test = "^0.3.56" + +[dev-dependencies] +bencher = "0.1" +getrandom = { version = "0.3", default-features = false, features = [] } +walkdir = "2.5" +time = { version = "0.3", features = ["formatting", "macros"] } +clap = { version = "=4.4.18", features = ["derive"] } +tempfile = "3.15" +rayon = "1.11" + +[features] +_arbitrary = ["dep:arbitrary"] +aes-crypto = ["dep:aes", "dep:constant_time_eq", "dep:generic-array", "getrandom/std", "dep:hmac", "dep:pbkdf2", "dep:sha1", "dep:zeroize"] +chrono = ["dep:chrono"] +_deflate-any = [] +_all-features = [] # Detect when --all-features is used +deflate = ["deflate-zopfli", "deflate-flate2-zlib-rs"] +# Pull in flate2, but don't choose a backend; useful if you want to choose your own flate2 backend +deflate-flate2 = ["_deflate-any", "dep:flate2"] +# Pull in flate2 and the fast zlib-rs backend; this is what most users will want +deflate-flate2-zlib-rs = ["deflate-flate2", "flate2/zlib-rs"] +# Pull in zopfli (write-only DEFLATE, slower than flate2 with better compression ratios) +deflate-zopfli = ["dep:zopfli", "_deflate-any"] +jiff-02 = ["dep:jiff"] +nt-time = ["dep:nt-time"] +lzma = ["dep:lzma-rust2"] +ppmd = ["dep:ppmd-rust"] +unreserved = [] +xz = ["dep:lzma-rust2"] +_bzip2_any = [] +bzip2 = ["dep:bzip2", "bzip2/default", "_bzip2_any"] +legacy-zip = ["bitstream-io"] +default = [ + "aes-crypto", + "bzip2", + "deflate64", + "deflate", + "lzma", + "ppmd", + "time", + "zstd", + "xz", +] + +# The following features are not supported on wasm +bzip2-rs = ["dep:bzip2", "bzip2/bzip2-sys", "_bzip2_any"] +# Pull in flate2 and the zlib backend; only use this if you need a dynamically linked system zlib +deflate-flate2-zlib = ["deflate-flate2", "flate2/zlib"] +# Pull in flate2 and the zlib-ng backend; a modern fork of zlib +deflate-flate2-zlib-ng = ["deflate-flate2", "flate2/zlib-ng"] +# Pull in flate2 and the zlib-ng backend with a zlib-compatible API +deflate-flate2-zlib-ng-compat = ["deflate-flate2", "flate2/zlib-ng-compat"] + +[[bench]] +name = "read_entry" +harness = false + +[[bench]] +name = "read_metadata" +harness = false + +[[bench]] +name = "merge_archive" +harness = false diff --git a/v2/vendor/security/zip-7.2.0/LICENSE b/v2/vendor/security/zip-7.2.0/LICENSE new file mode 100644 index 0000000000..4a45976f88 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/LICENSE @@ -0,0 +1,24 @@ +The MIT License (MIT) + +Copyright (c) 2014 Mathijs van de Nes + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +Some files in the "tests/data" subdirectory of this repository are under other +licences; see files named LICENSE.*.txt for details. \ No newline at end of file diff --git a/v2/vendor/security/zip-7.2.0/README.md b/v2/vendor/security/zip-7.2.0/README.md new file mode 100644 index 0000000000..1459e9b852 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/README.md @@ -0,0 +1,145 @@ +zip +=== + +[![Build Status](https://github.com/zip-rs/zip2/actions/workflows/ci.yaml/badge.svg)](https://github.com/zip-rs/zip2/actions?query=branch%3Amaster+workflow%3ACI) +[![Crates.io version](https://img.shields.io/crates/v/zip.svg)](https://crates.io/crates/zip) + +[Documentation](https://docs.rs/zip/latest/zip/) + +# Info + +A zip library for rust which supports reading and writing of simple ZIP files. Formerly hosted at +https://github.com/zip-rs/zip2. + +Supported compression formats: + +* stored (i.e. none) +* deflate +* deflate64 (decompression only) +* bzip2 +* zstd +* lzma (decompression only) +* xz +* ppmd + +Currently unsupported zip extensions: + +* Multi-disk + +# Features + +The features available are: + +* `aes-crypto`: Enables decryption of files which were encrypted with AES. Supports AE-1 and AE-2 methods. +* `deflate`: Enables compressing and decompressing an unspecified implementation (that may change in future versions) of + the deflate compression algorithm, which is the default for zip files. Supports compression quality 1..=264. +* `deflate-flate2`: Combine this with any `flate2` feature flag that enables a back-end, to support deflate compression + at quality 1..=9. +* `deflate-zopfli`: Enables deflating files with the `zopfli` library (used when compression quality is 10..=264). This + is the most effective `deflate` implementation available, but also among the slowest. If `flate2` isn't also enabled, + only compression will be supported and not decompression. +* `deflate64`: Enables the deflate64 compression algorithm. Only decompression is supported. +* `lzma`: Enables the LZMA compression algorithm. Only decompression is supported. +* `bzip2`: Enables the BZip2 compression algorithm. +* `ppmd`: Enables the PPMd compression algorithm. +* `time`: Enables features using the [time](https://github.com/rust-lang-deprecated/time) crate. +* `chrono`: Enables converting last-modified `zip::DateTime` to and from `chrono::NaiveDateTime`. +* `jiff-02`: Enables converting last-modified `zip::DateTime` to and from `jiff::civil::DateTime`. +* `nt-time`: Enables returning timestamps stored in the NTFS extra field as `nt_time::FileTime`. +* `xz`: Enables the XZ compression algorithm. +* `zstd`: Enables the Zstandard compression algorithm. + +By default `aes-crypto`, `bzip2`, `deflate`, `deflate64`, `lzma`, `ppmd`, `time`, `xz` and `zstd` are enabled. + +# MSRV + +Our current Minimum Supported Rust Version is **1.83**. When adding features, +we will follow these guidelines: + +- We will always support a minor Rust version that has been stable for at least 6 months. +- Any change to the MSRV will be accompanied with a **minor** version bump. + +# Examples + +See the [examples directory](examples) for: + +* How to write a file to a zip. +* How to write a directory of files to a zip (using [walkdir](https://github.com/BurntSushi/walkdir)). +* How to extract a zip file. +* How to extract a single file from a zip. +* How to read a zip from the standard input. +* How to append a directory to an existing archive + +# Fuzzing + +Fuzzing support is through [`cargo afl`](https://rust-fuzz.github.io/book/afl/tutorial.html). To install `cargo afl`: + +```bash +cargo install cargo-afl +``` + +To start fuzzing zip extraction: + +```bash +mkdir -vp fuzz-read-out +cargo afl build --manifest-path=fuzz/Cargo.toml --all-features -p fuzz_read +# Curated input corpus: +cargo afl fuzz -i fuzz/read/in -o fuzz-read-out fuzz/target/debug/fuzz_read +# Test data files: +cargo afl fuzz -i tests/data -e zip -o fuzz-read-out fuzz/target/debug/fuzz_read +``` + +To start fuzzing zip creation: + +```bash +mkdir -vp fuzz-write-out +cargo afl build --manifest-path=fuzz/Cargo.toml --all-features -p fuzz_write +# Curated input corpus and dictionary schema: +cargo afl fuzz -x fuzz/write/fuzz.dict -i fuzz/write/in -o fuzz-write-out fuzz/target/debug/fuzz_write +``` + +## Fuzzing stdio + +The read and write fuzzers can also receive input over stdin for one-off validation. Note here that the fuzzers can be configured to build in support for DEFLATE, or not: +```bash +# Success, no output: +cargo run --manifest-path=fuzz/Cargo.toml --quiet --all-features -p fuzz_read MAX_SYMLINK_TARGET_LEN { ++ return Err(invalid!( ++ "symlink target declares {} bytes, more than the {} byte limit", ++ declared_len, ++ MAX_SYMLINK_TARGET_LEN ++ )); ++ } ++ let mut target = Vec::with_capacity(declared_len as usize); + file.read_to_end(&mut target)?; + drop(file); + make_symlink(&outpath, &target, &self.shared.files)?; +--- a/src/read/stream.rs ++++ b/src/read/stream.rs +@@ -1,4 +1,7 @@ ++use crate::result::invalid; ++ + use super::{ ++ MAX_SYMLINK_TARGET_LEN, + central_header_to_zip_file_inner, make_symlink, read_zipfile_from_stream, ZipCentralEntryBlock, + ZipFile, ZipFileData, ZipResult, + }; +@@ -67,7 +70,15 @@ + file.safe_prepare_path(&self.0, &mut outpath, None::<&(_, fn(&Path) -> bool)>)?; + + if file.is_symlink() { +- let mut target = Vec::with_capacity(file.size() as usize); ++ let declared_len = file.size(); ++ if declared_len > MAX_SYMLINK_TARGET_LEN { ++ return Err(invalid!( ++ "symlink target declares {} bytes, more than the {} byte limit", ++ declared_len, ++ MAX_SYMLINK_TARGET_LEN ++ )); ++ } ++ let mut target = Vec::with_capacity(declared_len as usize); + file.read_to_end(&mut target)?; + make_symlink(&outpath, &target, &self.1)?; + return Ok(()); diff --git a/v2/vendor/security/zip-7.2.0/RUVIEW-BACKPORT.md b/v2/vendor/security/zip-7.2.0/RUVIEW-BACKPORT.md new file mode 100644 index 0000000000..612944437a --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/RUVIEW-BACKPORT.md @@ -0,0 +1,35 @@ +# RuView ZIP 7.2.0 security backport + +This is the official crates.io ZIP 7.2.0 source package with the final upstream +[ZIP #984](https://github.com/zip-rs/zip2/pull/984) fix adapted to its archive and +stream readers. Version and licenses remain unchanged. Source/archive/fix hashes +are recorded in `ruview-provenance.json`; the production diff is +`RUSTSEC-2026-0429.patch` (two files). The only additional manifest entry enables +our self-contained public-API regression target. No Candle/framework snapshot or +RuView submodule revision changes are required. + +The patch rejects symlink target lengths over4096 before allocation. The exact +112-byte RUSTSEC-2026-0429 example aborts pristine7.2.0 and is rejected with +InvalidArchive here. Dedicated tests cover hostile lengths, the stream flag, +ordinary files and valid relative symlinks. A removed-guard mutation fails. + +Run from the repository root: + +```sh +cargo test --manifest-path v2/vendor/security/zip-7.2.0/Cargo.toml --locked --no-default-features --features deflate --test ruview_0429_backport +``` + +The registry package omits large upstream test archives. Therefore this package +is excluded from the parent workspace's unit-test membership; the explicit +regression target above is mandatory in security CI. Full upstream library and +selected integration tests were separately executed with fixtures from package +VCS46dc29cda6749736ddb1ec502aff06a65ffe54f8; they are not claimed as part of this +small committed fixture set. + +cargo-audit0.22.2 excludes path sources from registry advisory matching. A green +audit alone does not establish this patch's correctness; executable regressions, +provenance and review do. No advisory policy or exception was added or broadened. +This addresses0429 only; existing0367/0368/0369 reachability decisions remain +unchanged. Retire this backport when a reviewed compatible upstream upgrade is +available. Current Candle0.9 requires ZIP7, while moving the swarm submodule to +its new upstream workspace would be a separate integration change. diff --git a/v2/vendor/security/zip-7.2.0/SECURITY.md b/v2/vendor/security/zip-7.2.0/SECURITY.md new file mode 100644 index 0000000000..031b4cf473 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/SECURITY.md @@ -0,0 +1,36 @@ +# Security Policy + +## Supported Versions + +Use this section to tell people about which versions of your project are +currently being supported with security updates. + +| Version | Supported | +| ------- | ------------------ | +| 2.4.x | :white_check_mark: | +| 2.3.x | :white_check_mark: | +| < 2.0 | :x: | + +## Reporting a Vulnerability + +To report a vulnerability, please go to https://github.com/zip-rs/zip2/security/advisories/new. We'll attempt to: + +* Close the report within 7 days if it's invalid, or if a fix has already been released but some old versions needed to be yanked. +* Provide progress reports at least every 7 days to the original reporter. +* Fix vulnerabilities within 30 days of the initial report. + +## Disclosure + +A vulnerability will only be publicly disclosed once a fix is released. At that point, the delay before full public disclosure +will be determined as follows: + +* If the proof-of-concept is very simple, or an exploit is already in the wild (whether or not it specifically targets `zip`, + all details will be made public right away. +* If the vulnerability is specific to `zip` and cannot easily be reverse-engineered from the code history, then the + proof-of-concept and most of the details will be withheld until 14 days after the fix is released and all vulnerable + versions are yanked with `cargo yank`. +* If a potential victim requests more time to deploy a fix based on a credible risk, then the withholding of details can + be extended up to 30 days. This may be extended to 90 days if the victim is high-value (e.g. manages over US$1 billion + worth of financial assets or intellectual property, or has evidence that they're a target of nation-state attackers) + and there's a valid reason why they cannot deploy the fix as fast as most users (e.g. heavy reliance on an old version's + interface, or infrastructure damage in a war zone). diff --git a/v2/vendor/security/zip-7.2.0/benches/merge_archive.rs b/v2/vendor/security/zip-7.2.0/benches/merge_archive.rs new file mode 100644 index 0000000000..9e5598f7dd --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/benches/merge_archive.rs @@ -0,0 +1,130 @@ +use bencher::{benchmark_group, benchmark_main}; + +use std::io::{Cursor, Read, Seek, Write}; + +use bencher::Bencher; +use zip::{result::ZipResult, write::SimpleFileOptions, ZipArchive, ZipWriter}; + +fn generate_random_archive( + num_entries: usize, + entry_size: usize, + options: SimpleFileOptions, +) -> ZipResult<(usize, ZipArchive>>)> { + let buf = Cursor::new(Vec::new()); + let mut zip = ZipWriter::new(buf); + + let mut bytes = vec![0u8; entry_size]; + for i in 0..num_entries { + let name = format!("random{i}.dat"); + zip.start_file(name, options)?; + getrandom::fill(&mut bytes).unwrap(); + zip.write_all(&bytes)?; + } + + let buf = zip.finish()?.into_inner(); + let len = buf.len(); + + Ok((len, ZipArchive::new(Cursor::new(buf))?)) +} + +fn perform_merge( + src: ZipArchive, + mut target: ZipWriter, +) -> ZipResult> { + target.merge_archive(src)?; + Ok(target) +} + +fn perform_raw_copy_file( + mut src: ZipArchive, + mut target: ZipWriter, +) -> ZipResult> { + for i in 0..src.len() { + let entry = src.by_index(i)?; + target.raw_copy_file(entry)?; + } + Ok(target) +} + +const NUM_ENTRIES: usize = 100; +const ENTRY_SIZE: usize = 1024; + +fn merge_archive_stored(bench: &mut Bencher) { + let options = SimpleFileOptions::default().compression_method(zip::CompressionMethod::Stored); + let (len, src) = generate_random_archive(NUM_ENTRIES, ENTRY_SIZE, options).unwrap(); + + bench.bytes = len as u64; + + bench.iter(|| { + let buf = Cursor::new(Vec::new()); + let zip = ZipWriter::new(buf); + let zip = perform_merge(src.clone(), zip).unwrap(); + let buf = zip.finish().unwrap().into_inner(); + assert_eq!(buf.len(), len); + }); +} + +#[cfg(feature = "_deflate-any")] +fn merge_archive_compressed(bench: &mut Bencher) { + let options = SimpleFileOptions::default().compression_method(zip::CompressionMethod::Deflated); + let (len, src) = generate_random_archive(NUM_ENTRIES, ENTRY_SIZE, options).unwrap(); + + bench.bytes = len as u64; + + bench.iter(|| { + let buf = Cursor::new(Vec::new()); + let zip = ZipWriter::new(buf); + let zip = perform_merge(src.clone(), zip).unwrap(); + let buf = zip.finish().unwrap().into_inner(); + assert_eq!(buf.len(), len); + }); +} + +fn merge_archive_raw_copy_file_stored(bench: &mut Bencher) { + let options = SimpleFileOptions::default().compression_method(zip::CompressionMethod::Stored); + let (len, src) = generate_random_archive(NUM_ENTRIES, ENTRY_SIZE, options).unwrap(); + + bench.bytes = len as u64; + + bench.iter(|| { + let buf = Cursor::new(Vec::new()); + let zip = ZipWriter::new(buf); + let zip = perform_raw_copy_file(src.clone(), zip).unwrap(); + let buf = zip.finish().unwrap().into_inner(); + assert_eq!(buf.len(), len); + }); +} + +#[cfg(feature = "_deflate-any")] +fn merge_archive_raw_copy_file_compressed(bench: &mut Bencher) { + let options = SimpleFileOptions::default().compression_method(zip::CompressionMethod::Deflated); + let (len, src) = generate_random_archive(NUM_ENTRIES, ENTRY_SIZE, options).unwrap(); + + bench.bytes = len as u64; + + bench.iter(|| { + let buf = Cursor::new(Vec::new()); + let zip = ZipWriter::new(buf); + let zip = perform_raw_copy_file(src.clone(), zip).unwrap(); + let buf = zip.finish().unwrap().into_inner(); + assert_eq!(buf.len(), len); + }); +} + +#[cfg(feature = "_deflate-any")] +benchmark_group!( + benches, + merge_archive_stored, + merge_archive_compressed, + merge_archive_raw_copy_file_stored, + merge_archive_raw_copy_file_compressed, +); + +#[cfg(not(feature = "_deflate-any"))] +benchmark_group!( + benches, + merge_archive_stored, + merge_archive_raw_copy_file_stored, +); + +benchmark_main!(benches); diff --git a/v2/vendor/security/zip-7.2.0/benches/read_entry.rs b/v2/vendor/security/zip-7.2.0/benches/read_entry.rs new file mode 100644 index 0000000000..8fd4ef3a5c --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/benches/read_entry.rs @@ -0,0 +1,41 @@ +use bencher::{benchmark_group, benchmark_main}; + +use std::io::{Cursor, Read, Write}; + +use bencher::Bencher; +use zip::{write::SimpleFileOptions, ZipArchive, ZipWriter}; + +fn generate_random_archive(size: usize) -> Vec { + let data = Vec::new(); + let mut writer = ZipWriter::new(Cursor::new(data)); + let options = SimpleFileOptions::default().compression_method(zip::CompressionMethod::Stored); + + writer.start_file("random.dat", options).unwrap(); + let mut bytes = vec![0u8; size]; + getrandom::fill(&mut bytes).unwrap(); + writer.write_all(&bytes).unwrap(); + + writer.finish().unwrap().into_inner() +} + +fn read_entry(bench: &mut Bencher) { + let size = 1024 * 1024; + let bytes = generate_random_archive(size); + let mut archive = ZipArchive::new(Cursor::new(bytes.as_slice())).unwrap(); + + bench.iter(|| { + let mut file = archive.by_name("random.dat").unwrap(); + let mut buf = [0u8; 1024]; + loop { + let n = file.read(&mut buf).unwrap(); + if n == 0 { + break; + } + } + }); + + bench.bytes = size as u64; +} + +benchmark_group!(benches, read_entry); +benchmark_main!(benches); diff --git a/v2/vendor/security/zip-7.2.0/benches/read_metadata.rs b/v2/vendor/security/zip-7.2.0/benches/read_metadata.rs new file mode 100644 index 0000000000..06b4aa397c --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/benches/read_metadata.rs @@ -0,0 +1,143 @@ +use bencher::{benchmark_group, benchmark_main}; + +use std::fs; +use std::io::{self, Cursor, Write}; + +use bencher::Bencher; +use tempfile::TempDir; +use zip::write::SimpleFileOptions; +use zip::{result::ZipResult, CompressionMethod, ZipArchive, ZipWriter}; + +const FILE_COUNT: usize = 15_000; +const FILE_SIZE: usize = 1024; + +fn generate_random_archive(count_files: usize, file_size: usize) -> ZipResult> { + let data = Vec::new(); + let mut writer = ZipWriter::new(Cursor::new(data)); + let options = SimpleFileOptions::default().compression_method(CompressionMethod::Stored); + + let mut bytes = vec![0u8; file_size]; + + for i in 0..count_files { + let name = format!("file_deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef_{i}.dat"); + writer.start_file(name, options)?; + getrandom::fill(&mut bytes).map_err(io::Error::from)?; + writer.write_all(&bytes)?; + } + + Ok(writer.finish()?.into_inner()) +} + +fn read_metadata(bench: &mut Bencher) { + let bytes = generate_random_archive(FILE_COUNT, FILE_SIZE).unwrap(); + + bench.iter(|| { + let archive = ZipArchive::new(Cursor::new(bytes.as_slice())).unwrap(); + archive.len() + }); + bench.bytes = bytes.len() as u64; +} + +const COMMENT_SIZE: usize = 50_000; + +fn generate_zip32_archive_with_random_comment(comment_length: usize) -> ZipResult> { + let data = Vec::new(); + let mut writer = ZipWriter::new(Cursor::new(data)); + let options = SimpleFileOptions::default().compression_method(CompressionMethod::Stored); + + let mut bytes = vec![0u8; comment_length]; + getrandom::fill(&mut bytes).unwrap(); + writer.set_raw_comment(bytes.into_boxed_slice()); + + writer.start_file("asdf.txt", options)?; + writer.write_all(b"asdf")?; + + Ok(writer.finish()?.into_inner()) +} + +fn parse_archive_with_comment(bench: &mut Bencher) { + let bytes = generate_zip32_archive_with_random_comment(COMMENT_SIZE).unwrap(); + + bench.bench_n(1, |_| { + let archive = ZipArchive::new(Cursor::new(bytes.as_slice())).unwrap(); + let _ = archive.comment().len(); + }); + bench.bytes = bytes.len() as u64; +} + +const COMMENT_SIZE_64: usize = 500_000; + +fn generate_zip64_archive_with_random_comment(comment_length: usize) -> ZipResult> { + let data = Vec::new(); + let mut writer = ZipWriter::new(Cursor::new(data)); + let options = SimpleFileOptions::default() + .compression_method(CompressionMethod::Stored) + .large_file(true); + + let mut bytes = vec![0u8; comment_length]; + getrandom::fill(&mut bytes).unwrap(); + // should use the line below but still works + // writer.set_raw_zip64_comment(Some(comment)); + writer.set_raw_comment(bytes.into_boxed_slice()); + + writer.start_file("asdf.txt", options)?; + writer.write_all(b"asdf")?; + + Ok(writer.finish()?.into_inner()) +} + +fn parse_zip64_archive_with_comment(bench: &mut Bencher) { + let bytes = generate_zip64_archive_with_random_comment(COMMENT_SIZE_64).unwrap(); + + bench.iter(|| { + let archive = ZipArchive::new(Cursor::new(bytes.as_slice())).unwrap(); + archive.comment().len() + }); + bench.bytes = bytes.len() as u64; +} + +fn parse_stream_archive(bench: &mut Bencher) { + const STREAM_ZIP_ENTRIES: usize = 5; + const STREAM_FILE_SIZE: usize = 5; + + let bytes = generate_random_archive(STREAM_ZIP_ENTRIES, STREAM_FILE_SIZE).unwrap(); + + /* Write to a temporary file path to incur some filesystem overhead from repeated reads */ + let dir = TempDir::with_prefix("stream-bench").unwrap(); + let out = dir.path().join("bench-out.zip"); + fs::write(&out, &bytes).unwrap(); + + bench.iter(|| { + let mut f = fs::File::open(&out).unwrap(); + while zip::read::read_zipfile_from_stream(&mut f) + .unwrap() + .is_some() + {} + }); + bench.bytes = bytes.len() as u64; +} + +fn parse_large_non_zip(bench: &mut Bencher) { + const FILE_SIZE: usize = 17_000_000; + + // Create a large file that doesn't have a zip header (generating random data _might_ make a zip magic + // number somewhere which is _not_ what we're trying to test). + let dir = TempDir::with_prefix("large-non-zip-bench").unwrap(); + let file = dir.path().join("zeros"); + let buf = vec![0u8; FILE_SIZE]; + fs::write(&file, &buf).unwrap(); + + bench.iter(|| { + assert!(zip::ZipArchive::new(std::fs::File::open(&file).unwrap()).is_err()); + }) +} + +benchmark_group!( + benches, + read_metadata, + parse_archive_with_comment, + parse_zip64_archive_with_comment, + parse_stream_archive, + parse_large_non_zip, +); +benchmark_main!(benches); diff --git a/v2/vendor/security/zip-7.2.0/cliff.toml b/v2/vendor/security/zip-7.2.0/cliff.toml new file mode 100644 index 0000000000..5c1ceed58a --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/cliff.toml @@ -0,0 +1,91 @@ +# git-cliff ~ default configuration file +# https://git-cliff.org/docs/configuration +# +# Lines starting with "#" are comments. +# Configuration options are organized into tables and keys. +# See documentation for more information on available options. + +[changelog] +# changelog header +header = """ +# Changelog\n +All notable changes to this project will be documented in this file.\n +""" +# template for the changelog body +# https://keats.github.io/tera/docs/#introduction +body = """ +{% if version %}\ + ## [{{ version | trim_start_matches(pat="v") }}] - {{ timestamp | date(format="%Y-%m-%d") }} +{% else %}\ + ## [unreleased] +{% endif %}\ +{% for group, commits in commits | group_by(attribute="group") %} + ### {{ group | striptags | trim | upper_first }} + {% for commit in commits %} + - {% if commit.scope %}*({{ commit.scope }})* {% endif %}\ + {% if commit.breaking %}[**breaking**] {% endif %}\ + {{ commit.message | upper_first }}\ + {% endfor %} +{% endfor %}\n +""" +# template for the changelog footer +footer = """ + +""" +# remove the leading and trailing s +trim = true +# postprocessors +postprocessors = [ + # { pattern = '', replace = "https://github.com/orhun/git-cliff" }, # replace repository URL +] + +[git] +# parse the commits based on https://www.conventionalcommits.org +conventional_commits = true +# filter out the commits that are not conventional +filter_unconventional = true +# process each line of a commit as an individual commit +split_commits = false +# regex for preprocessing the commit messages +commit_preprocessors = [ + # Replace issue numbers + #{ pattern = '\((\w+\s)?#([0-9]+)\)', replace = "([#${2}](/issues/${2}))"}, + # Check spelling of the commit with https://github.com/crate-ci/typos + # If the spelling is incorrect, it will be automatically fixed. + #{ pattern = '.*', replace_command = 'typos --write-changes -' }, +] +# regex for parsing and grouping commits +commit_parsers = [ + { message = "^feat", group = "🚀 Features" }, + { message = "^fix", group = "🐛 Bug Fixes" }, + { message = "^doc", skip = true }, + { message = "^perf", group = "⚡ Performance" }, + { message = "^refactor", group = "🚜 Refactor" }, + { message = "^style", skip = true }, + { message = "^test", skip = true }, + { message = "^build", skip = true }, + { message = "^ci", skip = true }, + { message = "^chore\\(release\\)", skip = true }, + { message = "^chore\\(deps.*\\)", skip = true }, + { message = "^chore\\(pr\\)", skip = true }, + { message = "^chore\\(pull\\)", skip = true }, + { message = "^chore", group = "⚙️ Miscellaneous Tasks" }, + { body = ".*security", group = "🛡️ Security" }, + { message = "^revert", group = "◀️ Revert" }, +] +# protect breaking changes from being skipped due to matching a skipping commit_parser +protect_breaking_commits = true +# filter out the commits that are not matched by commit parsers +filter_commits = false +# regex for matching git tags +# tag_pattern = "v[0-9].*" +# regex for skipping tags +# skip_tags = "" +# regex for ignoring tags +# ignore_tags = "" +# sort the tags topologically +topo_order = false +# sort the commits inside sections by oldest/newest order +sort_commits = "oldest" +# limit the number of commits included in the changelog. +# limit_commits = 42 diff --git a/v2/vendor/security/zip-7.2.0/pull_request_template.md b/v2/vendor/security/zip-7.2.0/pull_request_template.md new file mode 100644 index 0000000000..9b3e77c179 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/pull_request_template.md @@ -0,0 +1,38 @@ + diff --git a/v2/vendor/security/zip-7.2.0/release-plz.toml b/v2/vendor/security/zip-7.2.0/release-plz.toml new file mode 100644 index 0000000000..58e434cbb1 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/release-plz.toml @@ -0,0 +1,27 @@ +[workspace] +dependencies_update = true # update dependencies with `cargo update` +pr_labels = ["release"] # add the `release` label to the release Pull Request +release_commits = "^(feat|fix|perf|refactor):" # prepare release only if at least one commit matches a regex +git_release_type = "auto" +release_always = false + +[changelog] +protect_breaking_commits = true +commit_parsers = [ + { message = "^feat", group = "🚀 Features" }, + { message = "^fix", group = "🐛 Bug Fixes" }, + { message = "^doc", skip = true }, + { message = "^perf", group = "⚡ Performance" }, + { message = "^refactor", group = "🚜 Refactor" }, + { message = "^style", skip = true }, + { message = "^test", skip = true }, + { message = "^build", skip = true }, + { message = "^ci", skip = true }, + { message = "^chore\\(release\\)", skip = true }, + { message = "^chore\\(deps.*\\)", skip = true }, + { message = "^chore\\(pr\\)", skip = true }, + { message = "^chore\\(pull\\)", skip = true }, + { message = "^chore", group = "⚙️ Miscellaneous Tasks" }, + { body = ".*security", group = "🛡️ Security" }, + { message = "^revert", group = "◀️ Revert" }, +] \ No newline at end of file diff --git a/v2/vendor/security/zip-7.2.0/ruview-provenance.json b/v2/vendor/security/zip-7.2.0/ruview-provenance.json new file mode 100644 index 0000000000..c043d76c61 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/ruview-provenance.json @@ -0,0 +1,16 @@ +{ + "crate": "zip", + "version": "7.2.0", + "registrySHA256": "c42e33efc22a0650c311c2ef19115ce232583abbe80850bc8b66509ebef02de0", + "upstreamVCS": "46dc29cda6749736ddb1ec502aff06a65ffe54f8", + "fixPR": "https://github.com/zip-rs/zip2/pull/984", + "fixMerge": "a543a1fc8f3f3eef36f27bb137230295284cdb98", + "fixHead": "4d1c864bb62cc342147bf44c13a0a743fc844c4d", + "hostileSHA256": "0c68c3934f22651a6bd2726f7d58140124b1839196a1d0ae468864cdfda74676", + "licenseSHA256": "04dc52136d82400b11df5f474949863a70a5d05d2278dec45b99c479494450a6", + "securityPatchSHA256": "d7da70fb4223322f0915384589558fa732999eacc62f4eef09ca4cb58719280f", + "upstreamTestSourceCommit": "46dc29cda6749736ddb1ec502aff06a65ffe54f8", + "codeAdaptation": "final PR984 guards, read/mod.rs mapped to7.2 read.rs, stream invalid macro imported", + "tests": "original102lib+1symlink integration; upstream984 hostile test plus4097 boundary and stream-bit regression", + "manifestChanges": "only explicit ruview_0429_backport self-contained integration-test registration; version7.2.0 and licenses unchanged" +} diff --git a/v2/vendor/security/zip-7.2.0/src/aes.rs b/v2/vendor/security/zip-7.2.0/src/aes.rs new file mode 100644 index 0000000000..ec441f705c --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/src/aes.rs @@ -0,0 +1,407 @@ +//! Implementation of the AES decryption for zip files. +//! +//! This was implemented according to the [WinZip specification](https://www.winzip.com/win/en/aes_info.html). +//! Note that using CRC with AES depends on the used encryption specification, AE-1 or AE-2. +//! If the file is marked as encrypted with AE-2 the CRC field is ignored, even if it isn't set to 0. + +use crate::aes_ctr::AesCipher; +use crate::result::ZipResult; +use crate::types::AesMode; +use crate::{aes_ctr, result::ZipError}; +use constant_time_eq::constant_time_eq; +use hmac::{Hmac, Mac}; +use sha1::Sha1; +use std::io::{self, Error, ErrorKind, Read, Write}; +use zeroize::{Zeroize, Zeroizing}; + +/// The length of the password verification value in bytes +pub const PWD_VERIFY_LENGTH: usize = 2; +/// The length of the authentication code in bytes +const AUTH_CODE_LENGTH: usize = 10; +/// The number of iterations used with PBKDF2 +const ITERATION_COUNT: u32 = 1000; + +enum Cipher { + Aes128(Box>), + Aes192(Box>), + Aes256(Box>), +} + +impl Cipher { + /// Create a `Cipher` depending on the used `AesMode` and the given `key`. + /// + /// Returns `ZipError::InvalidPassword` if `key` doesn't have the correct size for the chosen + /// AES mode. + fn from_mode(aes_mode: AesMode, key: &[u8]) -> ZipResult { + Ok(match aes_mode { + AesMode::Aes128 => Cipher::Aes128(Box::new(aes_ctr::AesCtrZipKeyStream::< + aes_ctr::Aes128, + >::new(key)?)), + AesMode::Aes192 => Cipher::Aes192(Box::new(aes_ctr::AesCtrZipKeyStream::< + aes_ctr::Aes192, + >::new(key)?)), + AesMode::Aes256 => Cipher::Aes256(Box::new(aes_ctr::AesCtrZipKeyStream::< + aes_ctr::Aes256, + >::new(key)?)), + }) + } + + fn crypt_in_place(&mut self, target: &mut [u8]) { + match self { + Self::Aes128(cipher) => cipher.crypt_in_place(target), + Self::Aes192(cipher) => cipher.crypt_in_place(target), + Self::Aes256(cipher) => cipher.crypt_in_place(target), + } + } +} + +// An aes encrypted file starts with a salt, whose length depends on the used aes mode +// followed by a 2 byte password verification value +// then the variable length encrypted data +// and lastly a 10 byte authentication code +pub struct AesReader { + reader: R, + aes_mode: AesMode, + data_length: u64, +} + +impl AesReader { + pub const fn new(reader: R, aes_mode: AesMode, compressed_size: u64) -> AesReader { + let data_length = compressed_size + - (PWD_VERIFY_LENGTH + AUTH_CODE_LENGTH + aes_mode.salt_length()) as u64; + + Self { + reader, + aes_mode, + data_length, + } + } + + /// Read the AES header bytes and validate the password. + /// + /// Even if the validation succeeds, there is still a 1 in 65536 chance that an incorrect + /// password was provided. + /// It isn't possible to check the authentication code in this step. This will be done after + /// reading and decrypting the file. + pub fn validate(mut self, password: &[u8]) -> Result, ZipError> { + let salt_length = self.aes_mode.salt_length(); + let key_length = self.aes_mode.key_length(); + + let mut salt = vec![0; salt_length]; + self.reader.read_exact(&mut salt)?; + + // next are 2 bytes used for password verification + let mut pwd_verification_value = vec![0; PWD_VERIFY_LENGTH]; + self.reader.read_exact(&mut pwd_verification_value)?; + + // derive a key from the password and salt + // the length depends on the aes key length + let derived_key_len = 2 * key_length + PWD_VERIFY_LENGTH; + let mut derived_key: Box<[u8]> = vec![0; derived_key_len].into_boxed_slice(); + + // use PBKDF2 with HMAC-Sha1 to derive the key + pbkdf2::pbkdf2::>(password, &salt, ITERATION_COUNT, &mut derived_key) + .map_err(|e| Error::new(ErrorKind::InvalidInput, e))?; + let decrypt_key = &derived_key[0..key_length]; + let hmac_key = &derived_key[key_length..key_length * 2]; + let pwd_verify = &derived_key[derived_key_len - 2..]; + + // the last 2 bytes should equal the password verification value + if pwd_verification_value != pwd_verify { + // wrong password + return Err(ZipError::InvalidPassword); + } + + let cipher = Cipher::from_mode(self.aes_mode, decrypt_key)?; + let hmac = Hmac::::new_from_slice(hmac_key).unwrap(); + + Ok(AesReaderValid { + reader: self.reader, + data_remaining: self.data_length, + cipher, + hmac, + finalized: false, + }) + } + + /// Read the AES header bytes and returns the verification value and salt. + /// + /// # Returns + /// + /// the verification value and the salt + pub fn get_verification_value_and_salt( + mut self, + ) -> io::Result<([u8; PWD_VERIFY_LENGTH], Vec)> { + let salt_length = self.aes_mode.salt_length(); + + let mut salt = vec![0; salt_length]; + self.reader.read_exact(&mut salt)?; + + // next are 2 bytes used for password verification + let mut pwd_verification_value = [0; PWD_VERIFY_LENGTH]; + self.reader.read_exact(&mut pwd_verification_value)?; + Ok((pwd_verification_value, salt)) + } +} + +/// A reader for aes encrypted files, which has already passed the first password check. +/// +/// There is a 1 in 65536 chance that an invalid password passes that check. +/// After the data has been read and decrypted an HMAC will be checked and provide a final means +/// to check if either the password is invalid or if the data has been changed. +pub struct AesReaderValid { + reader: R, + data_remaining: u64, + cipher: Cipher, + hmac: Hmac, + finalized: bool, +} + +impl Read for AesReaderValid { + /// This implementation does not fulfill all requirements set in the trait documentation. + /// + /// ```txt + /// "If an error is returned then it must be guaranteed that no bytes were read." + /// ``` + /// + /// Whether this applies to errors that occur while reading the encrypted data depends on the + /// underlying reader. If the error occurs while verifying the HMAC, the reader might become + /// practically unusable, since its position after the error is not known. + fn read(&mut self, buf: &mut [u8]) -> io::Result { + if self.data_remaining == 0 { + return Ok(0); + } + + // get the number of bytes to read, compare as u64 to make sure we can read more than + // 2^32 bytes even on 32 bit systems. + let bytes_to_read = self.data_remaining.min(buf.len() as u64) as usize; + let read = self.reader.read(&mut buf[0..bytes_to_read])?; + self.data_remaining -= read as u64; + + // Update the hmac with the encrypted data + self.hmac.update(&buf[0..read]); + + // decrypt the data + self.cipher.crypt_in_place(&mut buf[0..read]); + + // if there is no data left to read, check the integrity of the data + if self.data_remaining == 0 { + assert!( + !self.finalized, + "Tried to use an already finalized HMAC. This is a bug!" + ); + self.finalized = true; + + // Zip uses HMAC-Sha1-80, which only uses the first half of the hash + // see https://www.winzip.com/win/en/aes_info.html#auth-faq + let mut read_auth_code = [0; AUTH_CODE_LENGTH]; + self.reader.read_exact(&mut read_auth_code)?; + let computed_auth_code = &self.hmac.finalize_reset().into_bytes()[0..AUTH_CODE_LENGTH]; + + // use constant time comparison to mitigate timing attacks + if !constant_time_eq(computed_auth_code, &read_auth_code) { + return Err( + Error::new( + ErrorKind::InvalidData, + "Invalid authentication code, this could be due to an invalid password or errors in the data" + ) + ); + } + } + + Ok(read) + } +} + +impl AesReaderValid { + /// Consumes this decoder, returning the underlying reader. + pub fn into_inner(self) -> R { + self.reader + } +} + +pub struct AesWriter { + writer: W, + cipher: Cipher, + hmac: Hmac, + buffer: Zeroizing>, + encrypted_file_header: Option>, +} + +impl AesWriter { + pub fn new(writer: W, aes_mode: AesMode, password: &[u8]) -> ZipResult { + let salt_length = aes_mode.salt_length(); + let key_length = aes_mode.key_length(); + + let mut encrypted_file_header = Vec::with_capacity(salt_length + 2); + + let mut salt = vec![0; salt_length]; + getrandom::fill(&mut salt).map_err(|e| ZipError::Io(e.into()))?; + encrypted_file_header.write_all(&salt)?; + + // Derive a key from the password and salt. The length depends on the aes key length + let derived_key_len = 2 * key_length + PWD_VERIFY_LENGTH; + let mut derived_key: Zeroizing> = Zeroizing::new(vec![0; derived_key_len]); + + // Use PBKDF2 with HMAC-Sha1 to derive the key. + pbkdf2::pbkdf2::>(password, &salt, ITERATION_COUNT, &mut derived_key) + .map_err(|e| Error::new(ErrorKind::InvalidInput, e))?; + let encryption_key = &derived_key[0..key_length]; + let hmac_key = &derived_key[key_length..key_length * 2]; + + let pwd_verify = derived_key[derived_key_len - 2..].to_vec(); + encrypted_file_header.write_all(&pwd_verify)?; + + let cipher = Cipher::from_mode(aes_mode, encryption_key)?; + let hmac = Hmac::::new_from_slice(hmac_key).unwrap(); + + Ok(Self { + writer, + cipher, + hmac, + buffer: Default::default(), + encrypted_file_header: Some(encrypted_file_header), + }) + } + + pub fn finish(mut self) -> io::Result { + self.write_encrypted_file_header()?; + + // Zip uses HMAC-Sha1-80, which only uses the first half of the hash + // see https://www.winzip.com/win/en/aes_info.html#auth-faq + let computed_auth_code = &self.hmac.finalize_reset().into_bytes()[0..AUTH_CODE_LENGTH]; + self.writer.write_all(computed_auth_code)?; + + Ok(self.writer) + } + + /// The AES encryption specification requires some metadata being written at the start of the + /// file data section, but this can only be done once the extra data writing has been finished + /// so we can't do it when the writer is constructed. + fn write_encrypted_file_header(&mut self) -> io::Result<()> { + if let Some(header) = self.encrypted_file_header.take() { + self.writer.write_all(&header)?; + } + + Ok(()) + } +} + +impl Write for AesWriter { + fn write(&mut self, buf: &[u8]) -> io::Result { + self.write_encrypted_file_header()?; + + // Fill the internal buffer and encrypt it in-place. + self.buffer.extend_from_slice(buf); + self.cipher.crypt_in_place(&mut self.buffer[..]); + + // Update the hmac with the encrypted data. + self.hmac.update(&self.buffer[..]); + + // Write the encrypted buffer to the inner writer. We need to use `write_all` here as if + // we only write parts of the data we can't easily reverse the keystream in the cipher + // implementation. + self.writer.write_all(&self.buffer[..])?; + + // Zeroize the backing memory before clearing the buffer to prevent cleartext data from + // being left in memory. + self.buffer.zeroize(); + self.buffer.clear(); + + Ok(buf.len()) + } + + fn flush(&mut self) -> io::Result<()> { + self.writer.flush() + } +} + +#[cfg(all(test, feature = "aes-crypto"))] +mod tests { + use std::io::{self, Read, Write}; + + use crate::{ + aes::{AesReader, AesWriter}, + result::ZipError, + types::AesMode, + }; + + /// Checks whether `AesReader` can successfully decrypt what `AesWriter` produces. + fn roundtrip(aes_mode: AesMode, password: &[u8], plaintext: &[u8]) -> Result { + let mut buf = io::Cursor::new(vec![]); + let mut read_buffer = vec![]; + + { + let mut writer = AesWriter::new(&mut buf, aes_mode, password)?; + writer.write_all(plaintext)?; + writer.finish()?; + } + + // Reset cursor position to the beginning. + buf.set_position(0); + + { + let compressed_length = buf.get_ref().len() as u64; + let mut reader = + AesReader::new(&mut buf, aes_mode, compressed_length).validate(password)?; + reader.read_to_end(&mut read_buffer)?; + } + + Ok(plaintext == read_buffer) + } + + #[test] + fn crypt_aes_256_0_byte() { + let plaintext = &[]; + let password = b"some super secret password"; + assert!(roundtrip(AesMode::Aes256, password, plaintext).expect("could encrypt and decrypt")); + } + + #[test] + fn crypt_aes_128_5_byte() { + let plaintext = b"asdf\n"; + let password = b"some super secret password"; + + assert!(roundtrip(AesMode::Aes128, password, plaintext).expect("could encrypt and decrypt")); + } + + #[test] + fn crypt_aes_192_5_byte() { + let plaintext = b"asdf\n"; + let password = b"some super secret password"; + + assert!(roundtrip(AesMode::Aes192, password, plaintext).expect("could encrypt and decrypt")); + } + + #[test] + fn crypt_aes_256_5_byte() { + let plaintext = b"asdf\n"; + let password = b"some super secret password"; + + assert!(roundtrip(AesMode::Aes256, password, plaintext).expect("could encrypt and decrypt")); + } + + #[test] + fn crypt_aes_128_40_byte() { + let plaintext = b"Lorem ipsum dolor sit amet, consectetur\n"; + let password = b"some super secret password"; + + assert!(roundtrip(AesMode::Aes128, password, plaintext).expect("could encrypt and decrypt")); + } + + #[test] + fn crypt_aes_192_40_byte() { + let plaintext = b"Lorem ipsum dolor sit amet, consectetur\n"; + let password = b"some super secret password"; + + assert!(roundtrip(AesMode::Aes192, password, plaintext).expect("could encrypt and decrypt")); + } + + #[test] + fn crypt_aes_256_40_byte() { + let plaintext = b"Lorem ipsum dolor sit amet, consectetur\n"; + let password = b"some super secret password"; + + assert!(roundtrip(AesMode::Aes256, password, plaintext).expect("could encrypt and decrypt")); + } +} diff --git a/v2/vendor/security/zip-7.2.0/src/aes_ctr.rs b/v2/vendor/security/zip-7.2.0/src/aes_ctr.rs new file mode 100644 index 0000000000..7b3a3c0c4d --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/src/aes_ctr.rs @@ -0,0 +1,277 @@ +//! A counter mode (CTR) for AES to work with the encryption used in zip files. +//! +//! This was implemented since the zip specification requires the mode to not use a nonce and uses a +//! different byte order (little endian) than NIST (big endian). +//! See [AesCtrZipKeyStream] for more information. + +use crate::result::{ZipError, ZipResult}; +use aes::cipher::{BlockEncrypt, KeyInit}; +use core::{any, fmt}; + +/// Internal block size of an AES cipher. +const AES_BLOCK_SIZE: usize = 16; + +/// AES-128. +#[derive(Debug)] +pub struct Aes128; +/// AES-192. +#[derive(Debug)] +pub struct Aes192; +/// AES-256. +#[derive(Debug)] +pub struct Aes256; + +/// An AES cipher kind. +pub trait AesKind { + /// Key type. + type Key: AsRef<[u8]>; + /// Cipher used to decrypt. + type Cipher: KeyInit; +} + +impl AesKind for Aes128 { + type Key = [u8; 16]; + type Cipher = aes::Aes128; +} + +impl AesKind for Aes192 { + type Key = [u8; 24]; + type Cipher = aes::Aes192; +} + +impl AesKind for Aes256 { + type Key = [u8; 32]; + type Cipher = aes::Aes256; +} + +/// An AES-CTR key stream generator. +/// +/// Implements the slightly non-standard AES-CTR variant used by WinZip AES encryption. +/// +/// Typical AES-CTR implementations combine a nonce with a 64 bit counter. WinZIP AES instead uses +/// no nonce and also uses a different byte order (little endian) than NIST (big endian). +/// +/// The stream implements the `Read` trait; encryption or decryption is performed by XOR-ing the +/// bytes from the key stream with the ciphertext/plaintext. +pub struct AesCtrZipKeyStream { + /// Current AES counter. + counter: u128, + /// AES cipher instance. + cipher: C::Cipher, + /// Stores the currently available keystream bytes. + buffer: [u8; AES_BLOCK_SIZE], + /// Number of bytes already used up from `buffer`. + pos: usize, +} + +impl fmt::Debug for AesCtrZipKeyStream +where + C: AesKind, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "AesCtrZipKeyStream<{}>(counter: {})", + any::type_name::(), + self.counter + ) + } +} + +impl AesCtrZipKeyStream +where + C: AesKind, + C::Cipher: KeyInit, +{ + /// Creates a new zip variant AES-CTR key stream. + /// + /// Returns `ZipError::InvalidPassword` if `key` doesn't have the correct size for cipher `C`. + pub fn new(key: &[u8]) -> ZipResult> { + Ok(AesCtrZipKeyStream { + counter: 1, + cipher: C::Cipher::new_from_slice(key).map_err(|_| ZipError::InvalidPassword)?, + buffer: [0u8; AES_BLOCK_SIZE], + pos: AES_BLOCK_SIZE, + }) + } +} + +impl AesCipher for AesCtrZipKeyStream +where + C: AesKind, + C::Cipher: BlockEncrypt, +{ + /// Decrypt or encrypt `target`. + #[inline] + fn crypt_in_place(&mut self, mut target: &mut [u8]) { + while !target.is_empty() { + if self.pos == AES_BLOCK_SIZE { + self.buffer = self.counter.to_le_bytes(); + self.cipher.encrypt_block(self.buffer.as_mut().into()); + self.counter += 1; + self.pos = 0; + } + + let target_len = target.len().min(AES_BLOCK_SIZE - self.pos); + + xor( + &mut target[0..target_len], + &self.buffer[self.pos..(self.pos + target_len)], + ); + target = &mut target[target_len..]; + self.pos += target_len; + } + } +} + +/// This trait allows using generic AES ciphers with different key sizes. +pub trait AesCipher { + fn crypt_in_place(&mut self, target: &mut [u8]); +} + +/// XORs a slice in place with another slice. +#[inline] +fn xor(dest: &mut [u8], src: &[u8]) { + debug_assert_eq!( + dest.len(), + src.len(), + "XOR operation requires destination and source slices to have equal length" + ); + + for (lhs, rhs) in dest.iter_mut().zip(src.iter()) { + *lhs ^= *rhs; + } +} + +#[cfg(test)] +mod tests { + use super::{Aes128, Aes192, Aes256, AesCipher, AesCtrZipKeyStream, AesKind}; + use aes::cipher::{BlockEncrypt, KeyInit}; + + /// Checks whether `crypt_in_place` produces the correct plaintext after one use and yields the + /// ciphertext again after applying it again. + fn roundtrip(key: &[u8], ciphertext: &[u8], expected_plaintext: &[u8]) + where + Aes: AesKind, + Aes::Cipher: KeyInit + BlockEncrypt, + { + let mut key_stream = AesCtrZipKeyStream::::new(key).unwrap(); + + let mut plaintext = ciphertext.to_vec().into_boxed_slice(); + key_stream.crypt_in_place(&mut plaintext); + assert_eq!(*plaintext, *expected_plaintext); + + // Round-tripping should yield the ciphertext again. + let mut key_stream = AesCtrZipKeyStream::::new(key).unwrap(); + key_stream.crypt_in_place(&mut plaintext); + assert_eq!(*plaintext, *ciphertext); + } + + #[test] + fn new_with_wrong_key_size() { + AesCtrZipKeyStream::::new(&[1, 2, 3, 4, 5]) + .expect_err("Wrong key size should fail"); + } + + // The data used in these tests was generated with p7zip without any compression. + // It's not possible to recreate the exact same data, since a random salt is used for encryption. + // `7z a -phelloworld -mem=AES256 -mx=0 aes256_40byte.zip 40byte_data.txt` + #[test] + fn crypt_aes_256_0_byte() { + let ciphertext = []; + let expected_plaintext = &[]; + let key = [ + 0x0b, 0xec, 0x2e, 0xf2, 0x46, 0xf0, 0x7e, 0x35, 0x16, 0x54, 0xe0, 0x98, 0x10, 0xb3, + 0x18, 0x55, 0x24, 0xa3, 0x9e, 0x0e, 0x40, 0xe7, 0x92, 0xad, 0xb2, 0x8a, 0x48, 0xf4, + 0x5c, 0xd0, 0xc0, 0x54, + ]; + + roundtrip::(&key, &ciphertext, expected_plaintext); + } + + #[test] + fn crypt_aes_128_5_byte() { + let ciphertext = [0x98, 0xa9, 0x8c, 0x26, 0x0e]; + let expected_plaintext = b"asdf\n"; + let key = [ + 0xe0, 0x25, 0x7b, 0x57, 0x97, 0x6a, 0xa4, 0x23, 0xab, 0x94, 0xaa, 0x44, 0xfd, 0x47, + 0x4f, 0xa5, + ]; + + roundtrip::(&key, &ciphertext, expected_plaintext); + } + + #[test] + fn crypt_aes_192_5_byte() { + let ciphertext = [0x36, 0x55, 0x5c, 0x61, 0x3c]; + let expected_plaintext = b"asdf\n"; + let key = [ + 0xe4, 0x4a, 0x88, 0x52, 0x8f, 0xf7, 0x0b, 0x81, 0x7b, 0x75, 0xf1, 0x74, 0x21, 0x37, + 0x8c, 0x90, 0xad, 0xbe, 0x4a, 0x65, 0xa8, 0x96, 0x0e, 0xcc, + ]; + + roundtrip::(&key, &ciphertext, expected_plaintext); + } + + #[test] + fn crypt_aes_256_5_byte() { + let ciphertext = [0xc2, 0x47, 0xc0, 0xdc, 0x56]; + let expected_plaintext = b"asdf\n"; + let key = [ + 0x79, 0x5e, 0x17, 0xf2, 0xc6, 0x3d, 0x28, 0x9b, 0x4b, 0x4b, 0xbb, 0xa9, 0xba, 0xc9, + 0xa5, 0xee, 0x3a, 0x4f, 0x0f, 0x4b, 0x29, 0xbd, 0xe9, 0xb8, 0x41, 0x9c, 0x41, 0xa5, + 0x15, 0xb2, 0x86, 0xab, + ]; + + roundtrip::(&key, &ciphertext, expected_plaintext); + } + + #[test] + fn crypt_aes_128_40_byte() { + let ciphertext = [ + 0xcf, 0x72, 0x6b, 0xa1, 0xb2, 0x0f, 0xdf, 0xaa, 0x10, 0xad, 0x9c, 0x7f, 0x6d, 0x1c, + 0x8d, 0xb5, 0x16, 0x7e, 0xbb, 0x11, 0x69, 0x52, 0x8c, 0x89, 0x80, 0x32, 0xaa, 0x76, + 0xa6, 0x18, 0x31, 0x98, 0xee, 0xdd, 0x22, 0x68, 0xb7, 0xe6, 0x77, 0xd2, + ]; + let expected_plaintext = b"Lorem ipsum dolor sit amet, consectetur\n"; + let key = [ + 0x43, 0x2b, 0x6d, 0xbe, 0x05, 0x76, 0x6c, 0x9e, 0xde, 0xca, 0x3b, 0xf8, 0xaf, 0x5d, + 0x81, 0xb6, + ]; + + roundtrip::(&key, &ciphertext, expected_plaintext); + } + + #[test] + fn crypt_aes_192_40_byte() { + let ciphertext = [ + 0xa6, 0xfc, 0x52, 0x79, 0x2c, 0x6c, 0xfe, 0x68, 0xb1, 0xa8, 0xb3, 0x07, 0x52, 0x8b, + 0x82, 0xa6, 0x87, 0x9c, 0x72, 0x42, 0x3a, 0xf8, 0xc6, 0xa9, 0xc9, 0xfb, 0x61, 0x19, + 0x37, 0xb9, 0x56, 0x62, 0xf4, 0xfc, 0x5e, 0x7a, 0xdd, 0x55, 0x0a, 0x48, + ]; + let expected_plaintext = b"Lorem ipsum dolor sit amet, consectetur\n"; + let key = [ + 0xac, 0x92, 0x41, 0xba, 0xde, 0xd9, 0x02, 0xfe, 0x40, 0x92, 0x20, 0xf6, 0x56, 0x03, + 0xfe, 0xae, 0x1b, 0xba, 0x01, 0x97, 0x97, 0x79, 0xbb, 0xa6, + ]; + + roundtrip::(&key, &ciphertext, expected_plaintext); + } + + #[test] + fn crypt_aes_256_40_byte() { + let ciphertext = [ + 0xa9, 0x99, 0xbd, 0xea, 0x82, 0x9b, 0x8f, 0x2f, 0xb7, 0x52, 0x2f, 0x6b, 0xd8, 0xf6, + 0xab, 0x0e, 0x24, 0x51, 0x9e, 0x18, 0x0f, 0xc0, 0x8f, 0x54, 0x15, 0x80, 0xae, 0xbc, + 0xa0, 0x5c, 0x8a, 0x11, 0x8d, 0x14, 0x7e, 0xc5, 0xb4, 0xae, 0xd3, 0x37, + ]; + let expected_plaintext = b"Lorem ipsum dolor sit amet, consectetur\n"; + let key = [ + 0x64, 0x7c, 0x7a, 0xde, 0xf0, 0xf2, 0x61, 0x49, 0x1c, 0xf1, 0xf1, 0xe3, 0x37, 0xfc, + 0xe1, 0x4d, 0x4a, 0x77, 0xd4, 0xeb, 0x9e, 0x3d, 0x75, 0xce, 0x9a, 0x3e, 0x10, 0x50, + 0xc2, 0x07, 0x36, 0xb6, + ]; + + roundtrip::(&key, &ciphertext, expected_plaintext); + } +} diff --git a/v2/vendor/security/zip-7.2.0/src/compression.rs b/v2/vendor/security/zip-7.2.0/src/compression.rs new file mode 100644 index 0000000000..2848f8749a --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/src/compression.rs @@ -0,0 +1,557 @@ +//! Possible ZIP compression methods. + +use crate::cfg_if_expr; +use core::fmt; +use std::io; + +#[allow(deprecated)] +/// Identifies the storage format used to compress a file within a ZIP archive. +/// +/// Each file's compression method is stored alongside it, allowing the +/// contents to be read without context. +/// +/// When creating ZIP files, you may choose the method to use with +/// [`crate::write::FileOptions::compression_method`] +#[derive(Copy, Clone, PartialEq, Eq, Debug)] +#[cfg_attr(feature = "_arbitrary", derive(arbitrary::Arbitrary))] +#[non_exhaustive] +pub enum CompressionMethod { + /// Store the file as is + Stored, + /// Compress the file using Deflate + #[cfg(feature = "_deflate-any")] + Deflated, + /// Compress the file using Deflate64. + /// Decoding deflate64 is supported but encoding deflate64 is not supported. + #[cfg(feature = "deflate64")] + Deflate64, + /// Compress the file using BZIP2 + #[cfg(feature = "_bzip2_any")] + Bzip2, + /// Encrypted using AES. + /// + /// The actual compression method has to be taken from the AES extra data field + /// or from `ZipFileData`. + #[cfg(feature = "aes-crypto")] + Aes, + /// Compress the file using ZStandard + #[cfg(feature = "zstd")] + Zstd, + /// Compress the file using LZMA + #[cfg(feature = "lzma")] + Lzma, + #[cfg(feature = "legacy-zip")] + /// Method 1 Shrink + Shrink, + #[cfg(feature = "legacy-zip")] + /// Reduce (Method 2-5) + Reduce(u8), + #[cfg(feature = "legacy-zip")] + /// Method 6 Implode/explode + Implode, + /// Compress the file using XZ + #[cfg(feature = "xz")] + Xz, + /// Compress the file using PPMd + #[cfg(feature = "ppmd")] + Ppmd, + /// Unsupported compression method + #[cfg_attr( + not(any(fuzzing, feature = "_arbitrary")), + deprecated(since = "0.5.7", note = "use the constants instead") + )] + Unsupported(u16), +} +#[allow(deprecated, missing_docs)] +/// All compression methods defined for the ZIP format +impl CompressionMethod { + pub const STORE: Self = CompressionMethod::Stored; + #[cfg(feature = "legacy-zip")] + pub const SHRINK: Self = CompressionMethod::Shrink; + #[cfg(not(feature = "legacy-zip"))] + /// Legacy compression method (enable feature `legacy-zip` to get support) + pub const SHRINK: Self = CompressionMethod::Unsupported(1); + #[cfg(feature = "legacy-zip")] + /// Legacy compression method + pub const REDUCE_1: Self = CompressionMethod::Reduce(1); + #[cfg(not(feature = "legacy-zip"))] + /// Legacy compression method (enable feature `legacy-zip` to get support) + pub const REDUCE_1: Self = CompressionMethod::Unsupported(2); + #[cfg(feature = "legacy-zip")] + /// Legacy compression method + pub const REDUCE_2: Self = CompressionMethod::Reduce(2); + #[cfg(not(feature = "legacy-zip"))] + /// Legacy compression method (enable feature `legacy-zip` to get support) + pub const REDUCE_2: Self = CompressionMethod::Unsupported(3); + #[cfg(feature = "legacy-zip")] + /// Legacy compression method + pub const REDUCE_3: Self = CompressionMethod::Reduce(3); + #[cfg(not(feature = "legacy-zip"))] + /// Legacy compression method (enable feature `legacy-zip` to get support) + pub const REDUCE_3: Self = CompressionMethod::Unsupported(4); + #[cfg(feature = "legacy-zip")] + /// Legacy compression method + pub const REDUCE_4: Self = CompressionMethod::Reduce(4); + #[cfg(not(feature = "legacy-zip"))] + /// Legacy compression method (enable feature `legacy-zip` to get support) + pub const REDUCE_4: Self = CompressionMethod::Unsupported(5); + #[cfg(feature = "legacy-zip")] + /// Legacy compression method + pub const IMPLODE: Self = CompressionMethod::Implode; + #[cfg(not(feature = "legacy-zip"))] + /// Legacy compression method (enable feature `legacy-zip` to get support) + pub const IMPLODE: Self = CompressionMethod::Unsupported(6); + #[cfg(feature = "_deflate-any")] + pub const DEFLATE: Self = CompressionMethod::Deflated; + #[cfg(not(feature = "_deflate-any"))] + pub const DEFLATE: Self = CompressionMethod::Unsupported(8); + #[cfg(feature = "deflate64")] + pub const DEFLATE64: Self = CompressionMethod::Deflate64; + #[cfg(not(feature = "deflate64"))] + pub const DEFLATE64: Self = CompressionMethod::Unsupported(9); + pub const PKWARE_IMPLODE: Self = CompressionMethod::Unsupported(10); + #[cfg(feature = "_bzip2_any")] + pub const BZIP2: Self = CompressionMethod::Bzip2; + #[cfg(not(feature = "_bzip2_any"))] + pub const BZIP2: Self = CompressionMethod::Unsupported(12); + #[cfg(not(feature = "lzma"))] + pub const LZMA: Self = CompressionMethod::Unsupported(14); + #[cfg(feature = "lzma")] + pub const LZMA: Self = CompressionMethod::Lzma; + pub const IBM_ZOS_CMPSC: Self = CompressionMethod::Unsupported(16); + pub const IBM_TERSE: Self = CompressionMethod::Unsupported(18); + pub const ZSTD_DEPRECATED: Self = CompressionMethod::Unsupported(20); + #[cfg(feature = "zstd")] + pub const ZSTD: Self = CompressionMethod::Zstd; + #[cfg(not(feature = "zstd"))] + pub const ZSTD: Self = CompressionMethod::Unsupported(93); + pub const MP3: Self = CompressionMethod::Unsupported(94); + #[cfg(feature = "xz")] + pub const XZ: Self = CompressionMethod::Xz; + #[cfg(not(feature = "xz"))] + pub const XZ: Self = CompressionMethod::Unsupported(95); + pub const JPEG: Self = CompressionMethod::Unsupported(96); + pub const WAVPACK: Self = CompressionMethod::Unsupported(97); + #[cfg(feature = "ppmd")] + pub const PPMD: Self = CompressionMethod::Ppmd; + #[cfg(not(feature = "ppmd"))] + pub const PPMD: Self = CompressionMethod::Unsupported(98); + #[cfg(feature = "aes-crypto")] + pub const AES: Self = CompressionMethod::Aes; + #[cfg(not(feature = "aes-crypto"))] + pub const AES: Self = CompressionMethod::Unsupported(99); + + #[cfg(feature = "_deflate-any")] + pub const DEFAULT: Self = CompressionMethod::Deflated; + + #[cfg(all(not(feature = "_deflate-any"), feature = "_bzip2_any"))] + pub const DEFAULT: Self = CompressionMethod::Bzip2; + + #[cfg(all(not(feature = "_deflate-any"), not(feature = "_bzip2_any")))] + pub const DEFAULT: Self = CompressionMethod::Stored; +} +impl CompressionMethod { + pub(crate) const fn parse_from_u16(val: u16) -> Self { + match val { + 0 => CompressionMethod::Stored, + #[cfg(feature = "legacy-zip")] + 1 => CompressionMethod::Shrink, + #[cfg(feature = "legacy-zip")] + 2 => CompressionMethod::Reduce(1), + #[cfg(feature = "legacy-zip")] + 3 => CompressionMethod::Reduce(2), + #[cfg(feature = "legacy-zip")] + 4 => CompressionMethod::Reduce(3), + #[cfg(feature = "legacy-zip")] + 5 => CompressionMethod::Reduce(4), + #[cfg(feature = "legacy-zip")] + 6 => CompressionMethod::Implode, + #[cfg(feature = "_deflate-any")] + 8 => CompressionMethod::Deflated, + #[cfg(feature = "deflate64")] + 9 => CompressionMethod::Deflate64, + #[cfg(feature = "_bzip2_any")] + 12 => CompressionMethod::Bzip2, + #[cfg(feature = "lzma")] + 14 => CompressionMethod::Lzma, + #[cfg(feature = "xz")] + 95 => CompressionMethod::Xz, + #[cfg(feature = "zstd")] + 93 => CompressionMethod::Zstd, + #[cfg(feature = "ppmd")] + 98 => CompressionMethod::Ppmd, + #[cfg(feature = "aes-crypto")] + 99 => CompressionMethod::Aes, + #[allow(deprecated)] + v => CompressionMethod::Unsupported(v), + } + } + + /// Converts a u16 to its corresponding CompressionMethod + #[deprecated( + since = "0.5.7", + note = "use a constant to construct a compression method" + )] + pub const fn from_u16(val: u16) -> CompressionMethod { + Self::parse_from_u16(val) + } + + pub(crate) const fn serialize_to_u16(self) -> u16 { + match self { + CompressionMethod::Stored => 0, + #[cfg(feature = "legacy-zip")] + CompressionMethod::Shrink => 1, + #[cfg(feature = "legacy-zip")] + CompressionMethod::Reduce(n) => 1 + n as u16, + #[cfg(feature = "legacy-zip")] + CompressionMethod::Implode => 6, + + #[cfg(feature = "_deflate-any")] + CompressionMethod::Deflated => 8, + #[cfg(feature = "deflate64")] + CompressionMethod::Deflate64 => 9, + #[cfg(feature = "_bzip2_any")] + CompressionMethod::Bzip2 => 12, + #[cfg(feature = "aes-crypto")] + CompressionMethod::Aes => 99, + #[cfg(feature = "zstd")] + CompressionMethod::Zstd => 93, + #[cfg(feature = "lzma")] + CompressionMethod::Lzma => 14, + #[cfg(feature = "xz")] + CompressionMethod::Xz => 95, + #[cfg(feature = "ppmd")] + CompressionMethod::Ppmd => 98, + #[allow(deprecated)] + CompressionMethod::Unsupported(v) => v, + } + } + + /// Converts a CompressionMethod to a u16 + #[deprecated( + since = "0.5.7", + note = "to match on other compression methods, use a constant" + )] + pub const fn to_u16(self) -> u16 { + self.serialize_to_u16() + } +} + +impl Default for CompressionMethod { + fn default() -> Self { + cfg_if_expr! { + #[cfg(feature = "_deflate-any")] => CompressionMethod::Deflated, + _ => CompressionMethod::Stored + } + } +} + +impl fmt::Display for CompressionMethod { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + // Just duplicate what the Debug format looks like, i.e, the enum key: + write!(f, "{self:?}") + } +} + +/// The compression methods which have been implemented. +pub const SUPPORTED_COMPRESSION_METHODS: &[CompressionMethod] = &[ + CompressionMethod::Stored, + #[cfg(feature = "_deflate-any")] + CompressionMethod::Deflated, + #[cfg(feature = "deflate64")] + CompressionMethod::Deflate64, + #[cfg(feature = "_bzip2_any")] + CompressionMethod::Bzip2, + #[cfg(feature = "zstd")] + CompressionMethod::Zstd, + #[cfg(feature = "xz")] + CompressionMethod::Xz, + #[cfg(feature = "ppmd")] + CompressionMethod::Ppmd, +]; + +pub(crate) enum Decompressor { + Stored(R), + #[cfg(feature = "deflate-flate2")] + Deflated(flate2::bufread::DeflateDecoder), + #[cfg(feature = "deflate64")] + Deflate64(deflate64::Deflate64Decoder), + #[cfg(feature = "_bzip2_any")] + Bzip2(bzip2::bufread::BzDecoder), + #[cfg(feature = "zstd")] + Zstd(zstd::Decoder<'static, R>), + #[cfg(feature = "lzma")] + Lzma(Lzma), + #[cfg(feature = "legacy-zip")] + Shrink(crate::legacy::shrink::ShrinkDecoder), + #[cfg(feature = "legacy-zip")] + Reduce(crate::legacy::reduce::ReduceDecoder), + #[cfg(feature = "legacy-zip")] + Implode(crate::legacy::implode::ImplodeDecoder), + #[cfg(feature = "xz")] + Xz(Box>), + #[cfg(feature = "ppmd")] + Ppmd(Ppmd), +} + +#[cfg(feature = "lzma")] +pub(crate) enum Lzma { + Uninitialized { + reader: Option, + uncompressed_size: u64, + }, + Initialized(Box>), +} + +#[cfg(feature = "ppmd")] +pub(crate) enum Ppmd { + Uninitialized(Option), + Initialized(Box>), +} + +impl io::Read for Decompressor { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + match self { + Decompressor::Stored(r) => r.read(buf), + #[cfg(feature = "deflate-flate2")] + Decompressor::Deflated(r) => r.read(buf), + #[cfg(feature = "deflate64")] + Decompressor::Deflate64(r) => r.read(buf), + #[cfg(feature = "_bzip2_any")] + Decompressor::Bzip2(r) => r.read(buf), + #[cfg(feature = "zstd")] + Decompressor::Zstd(r) => r.read(buf), + #[cfg(feature = "lzma")] + Decompressor::Lzma(r) => match r { + Lzma::Uninitialized { + reader, + uncompressed_size, + } => { + let mut reader = reader.take().ok_or_else(|| { + io::Error::other("Reader was not set while reading LZMA data") + })?; + + // 5.8.8.1 LZMA Version Information & 5.8.8.2 LZMA Properties Size + let mut header = [0; 4]; + reader.read_exact(&mut header)?; + let _version_information = u16::from_le_bytes(header[0..2].try_into().unwrap()); + let properties_size = u16::from_le_bytes(header[2..4].try_into().unwrap()); + if properties_size != 5 { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("unexpected LZMA properties size of {properties_size}"), + )); + } + + let mut props_data = [0; 5]; + reader.read_exact(&mut props_data)?; + let props = props_data[0]; + let dict_size = u32::from_le_bytes(props_data[1..5].try_into().unwrap()); + + // We don't need to handle the end-of-stream marker here, since the LZMA reader + // stops at the end-of-stream marker OR when it has decoded uncompressed_size bytes, whichever comes first. + let mut decompressor = lzma_rust2::LzmaReader::new_with_props( + reader, + *uncompressed_size, + props, + dict_size, + None, + )?; + + let read = decompressor.read(buf)?; + + *r = Lzma::Initialized(Box::new(decompressor)); + + Ok(read) + } + Lzma::Initialized(decompressor) => decompressor.read(buf), + }, + #[cfg(feature = "xz")] + Decompressor::Xz(r) => r.read(buf), + #[cfg(feature = "ppmd")] + Decompressor::Ppmd(r) => match r { + Ppmd::Uninitialized(reader) => { + let mut reader = reader.take().ok_or_else(|| { + io::Error::other("Reader was not set while reading PPMd data") + })?; + + let mut buffer = [0; 2]; + reader.read_exact(&mut buffer)?; + let parameters = u16::from_le_bytes(buffer); + + let order = ((parameters & 0x0F) + 1) as u32; + let memory_size = 1024 * 1024 * (((parameters >> 4) & 0xFF) + 1) as u32; + let restoration_method = (parameters >> 12) & 0x0F; + + let mut decompressor = ppmd_rust::Ppmd8Decoder::new( + reader, + order, + memory_size, + restoration_method.into(), + ) + .map_err(|error| match error { + ppmd_rust::Error::RangeDecoderInitialization => io::Error::new( + io::ErrorKind::InvalidData, + "PPMd range coder initialization failed", + ), + ppmd_rust::Error::InvalidParameter => { + io::Error::new(io::ErrorKind::InvalidInput, "Invalid PPMd parameter") + } + ppmd_rust::Error::IoError(io_error) => io_error, + ppmd_rust::Error::MemoryAllocation => { + io::Error::new(io::ErrorKind::OutOfMemory, "Memory allocation failed") + } + })?; + + let read = decompressor.read(buf)?; + + *r = Ppmd::Initialized(Box::new(decompressor)); + + Ok(read) + } + Ppmd::Initialized(decompressor) => decompressor.read(buf), + }, + #[cfg(feature = "legacy-zip")] + Decompressor::Shrink(r) => r.read(buf), + #[cfg(feature = "legacy-zip")] + Decompressor::Reduce(r) => r.read(buf), + #[cfg(feature = "legacy-zip")] + Decompressor::Implode(r) => r.read(buf), + } + } +} + +impl Decompressor { + pub fn new( + reader: R, + compression_method: CompressionMethod, + #[cfg(any(feature = "lzma", feature = "legacy-zip"))] uncompressed_size: u64, + #[cfg(not(any(feature = "lzma", feature = "legacy-zip")))] _uncompressed_size: u64, + #[cfg(feature = "legacy-zip")] flags: u16, + #[cfg(not(feature = "legacy-zip"))] _flags: u16, + ) -> crate::result::ZipResult { + Ok(match compression_method { + CompressionMethod::Stored => Decompressor::Stored(reader), + #[cfg(feature = "deflate-flate2")] + CompressionMethod::Deflated => { + Decompressor::Deflated(flate2::bufread::DeflateDecoder::new(reader)) + } + #[cfg(feature = "deflate64")] + CompressionMethod::Deflate64 => { + Decompressor::Deflate64(deflate64::Deflate64Decoder::with_buffer(reader)) + } + #[cfg(feature = "_bzip2_any")] + CompressionMethod::Bzip2 => Decompressor::Bzip2(bzip2::bufread::BzDecoder::new(reader)), + #[cfg(feature = "zstd")] + CompressionMethod::Zstd => Decompressor::Zstd(zstd::Decoder::with_buffer(reader)?), + #[cfg(feature = "lzma")] + CompressionMethod::Lzma => Decompressor::Lzma(Lzma::Uninitialized { + reader: Some(reader), + uncompressed_size, + }), + #[cfg(feature = "xz")] + CompressionMethod::Xz => { + Decompressor::Xz(Box::new(lzma_rust2::XzReader::new(reader, false))) + } + #[cfg(feature = "ppmd")] + CompressionMethod::Ppmd => Decompressor::Ppmd(Ppmd::Uninitialized(Some(reader))), + #[cfg(feature = "legacy-zip")] + CompressionMethod::Shrink => Decompressor::Shrink( + crate::legacy::shrink::ShrinkDecoder::new(reader, uncompressed_size), + ), + #[cfg(feature = "legacy-zip")] + CompressionMethod::Reduce(n) => Decompressor::Reduce( + crate::legacy::reduce::ReduceDecoder::new(reader, uncompressed_size, n), + ), + #[cfg(feature = "legacy-zip")] + CompressionMethod::Implode => Decompressor::Implode( + crate::legacy::implode::ImplodeDecoder::new(reader, uncompressed_size, flags), + ), + _ => { + return Err(crate::result::ZipError::UnsupportedArchive( + "Compression method not supported", + )) + } + }) + } + + /// Consumes this decoder, returning the underlying reader. + #[allow(clippy::infallible_destructuring_match)] + pub fn into_inner(self) -> io::Result { + let inner = match self { + Decompressor::Stored(r) => r, + #[cfg(feature = "deflate-flate2")] + Decompressor::Deflated(r) => r.into_inner(), + #[cfg(feature = "deflate64")] + Decompressor::Deflate64(r) => r.into_inner(), + #[cfg(feature = "_bzip2_any")] + Decompressor::Bzip2(r) => r.into_inner(), + #[cfg(feature = "zstd")] + Decompressor::Zstd(r) => r.finish(), + #[cfg(feature = "lzma")] + Decompressor::Lzma(r) => match r { + Lzma::Uninitialized { mut reader, .. } => reader + .take() + .ok_or_else(|| io::Error::other("Reader was not set"))?, + Lzma::Initialized(decoder) => decoder.into_inner(), + }, + #[cfg(feature = "legacy-zip")] + Decompressor::Shrink(r) => r.into_inner(), + #[cfg(feature = "legacy-zip")] + Decompressor::Reduce(r) => r.into_inner(), + #[cfg(feature = "legacy-zip")] + Decompressor::Implode(r) => r.into_inner(), + #[cfg(feature = "xz")] + Decompressor::Xz(r) => r.into_inner(), + #[cfg(feature = "ppmd")] + Decompressor::Ppmd(r) => match r { + Ppmd::Uninitialized(mut reader) => reader + .take() + .ok_or_else(|| io::Error::other("Reader was not set"))?, + Ppmd::Initialized(decoder) => decoder.into_inner(), + }, + }; + Ok(inner) + } +} + +#[cfg(test)] +mod test { + use super::{CompressionMethod, SUPPORTED_COMPRESSION_METHODS}; + + #[test] + fn from_eq_to() { + for v in 0..(u16::MAX as u32 + 1) { + let from = CompressionMethod::parse_from_u16(v as u16); + let to = from.serialize_to_u16() as u32; + assert_eq!(v, to); + } + } + + #[test] + fn to_eq_from() { + fn check_match(method: CompressionMethod) { + let to = method.serialize_to_u16(); + let from = CompressionMethod::parse_from_u16(to); + let back = from.serialize_to_u16(); + assert_eq!(to, back); + } + + for &method in SUPPORTED_COMPRESSION_METHODS { + check_match(method); + } + } + + #[test] + fn to_display_fmt() { + fn check_match(method: CompressionMethod) { + let debug_str = format!("{method:?}"); + let display_str = format!("{method}"); + assert_eq!(debug_str, display_str); + } + + for &method in SUPPORTED_COMPRESSION_METHODS { + check_match(method); + } + } +} diff --git a/v2/vendor/security/zip-7.2.0/src/cp437.rs b/v2/vendor/security/zip-7.2.0/src/cp437.rs new file mode 100644 index 0000000000..2c38063f32 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/src/cp437.rs @@ -0,0 +1,207 @@ +//! Convert a string in IBM codepage 437 to UTF-8 + +/// Trait to convert IBM codepage 437 to the target type +pub trait FromCp437 { + /// Target type + type Target; + + /// Function that does the conversion from cp437. + /// Generally allocations will be avoided if all data falls into the ASCII range. + #[allow(clippy::wrong_self_convention)] + fn from_cp437(self) -> Self::Target; +} + +impl<'a> FromCp437 for &'a [u8] { + type Target = ::std::borrow::Cow<'a, str>; + + fn from_cp437(self) -> Self::Target { + if self.iter().all(|c| *c < 0x80) { + ::std::str::from_utf8(self).unwrap().into() + } else { + self.iter().map(|c| to_char(*c)).collect::().into() + } + } +} + +impl FromCp437 for Box<[u8]> { + type Target = Box; + + fn from_cp437(self) -> Self::Target { + if self.iter().all(|c| *c < 0x80) { + String::from_utf8(self.into()).unwrap() + } else { + self.iter().copied().map(to_char).collect() + } + .into_boxed_str() + } +} + +fn to_char(input: u8) -> char { + let output = match input { + 0x00..=0x7f => input as u32, + 0x80 => 0x00c7, + 0x81 => 0x00fc, + 0x82 => 0x00e9, + 0x83 => 0x00e2, + 0x84 => 0x00e4, + 0x85 => 0x00e0, + 0x86 => 0x00e5, + 0x87 => 0x00e7, + 0x88 => 0x00ea, + 0x89 => 0x00eb, + 0x8a => 0x00e8, + 0x8b => 0x00ef, + 0x8c => 0x00ee, + 0x8d => 0x00ec, + 0x8e => 0x00c4, + 0x8f => 0x00c5, + 0x90 => 0x00c9, + 0x91 => 0x00e6, + 0x92 => 0x00c6, + 0x93 => 0x00f4, + 0x94 => 0x00f6, + 0x95 => 0x00f2, + 0x96 => 0x00fb, + 0x97 => 0x00f9, + 0x98 => 0x00ff, + 0x99 => 0x00d6, + 0x9a => 0x00dc, + 0x9b => 0x00a2, + 0x9c => 0x00a3, + 0x9d => 0x00a5, + 0x9e => 0x20a7, + 0x9f => 0x0192, + 0xa0 => 0x00e1, + 0xa1 => 0x00ed, + 0xa2 => 0x00f3, + 0xa3 => 0x00fa, + 0xa4 => 0x00f1, + 0xa5 => 0x00d1, + 0xa6 => 0x00aa, + 0xa7 => 0x00ba, + 0xa8 => 0x00bf, + 0xa9 => 0x2310, + 0xaa => 0x00ac, + 0xab => 0x00bd, + 0xac => 0x00bc, + 0xad => 0x00a1, + 0xae => 0x00ab, + 0xaf => 0x00bb, + 0xb0 => 0x2591, + 0xb1 => 0x2592, + 0xb2 => 0x2593, + 0xb3 => 0x2502, + 0xb4 => 0x2524, + 0xb5 => 0x2561, + 0xb6 => 0x2562, + 0xb7 => 0x2556, + 0xb8 => 0x2555, + 0xb9 => 0x2563, + 0xba => 0x2551, + 0xbb => 0x2557, + 0xbc => 0x255d, + 0xbd => 0x255c, + 0xbe => 0x255b, + 0xbf => 0x2510, + 0xc0 => 0x2514, + 0xc1 => 0x2534, + 0xc2 => 0x252c, + 0xc3 => 0x251c, + 0xc4 => 0x2500, + 0xc5 => 0x253c, + 0xc6 => 0x255e, + 0xc7 => 0x255f, + 0xc8 => 0x255a, + 0xc9 => 0x2554, + 0xca => 0x2569, + 0xcb => 0x2566, + 0xcc => 0x2560, + 0xcd => 0x2550, + 0xce => 0x256c, + 0xcf => 0x2567, + 0xd0 => 0x2568, + 0xd1 => 0x2564, + 0xd2 => 0x2565, + 0xd3 => 0x2559, + 0xd4 => 0x2558, + 0xd5 => 0x2552, + 0xd6 => 0x2553, + 0xd7 => 0x256b, + 0xd8 => 0x256a, + 0xd9 => 0x2518, + 0xda => 0x250c, + 0xdb => 0x2588, + 0xdc => 0x2584, + 0xdd => 0x258c, + 0xde => 0x2590, + 0xdf => 0x2580, + 0xe0 => 0x03b1, + 0xe1 => 0x00df, + 0xe2 => 0x0393, + 0xe3 => 0x03c0, + 0xe4 => 0x03a3, + 0xe5 => 0x03c3, + 0xe6 => 0x00b5, + 0xe7 => 0x03c4, + 0xe8 => 0x03a6, + 0xe9 => 0x0398, + 0xea => 0x03a9, + 0xeb => 0x03b4, + 0xec => 0x221e, + 0xed => 0x03c6, + 0xee => 0x03b5, + 0xef => 0x2229, + 0xf0 => 0x2261, + 0xf1 => 0x00b1, + 0xf2 => 0x2265, + 0xf3 => 0x2264, + 0xf4 => 0x2320, + 0xf5 => 0x2321, + 0xf6 => 0x00f7, + 0xf7 => 0x2248, + 0xf8 => 0x00b0, + 0xf9 => 0x2219, + 0xfa => 0x00b7, + 0xfb => 0x221a, + 0xfc => 0x207f, + 0xfd => 0x00b2, + 0xfe => 0x25a0, + 0xff => 0x00a0, + }; + ::std::char::from_u32(output).unwrap() +} + +#[cfg(test)] +mod test { + #[test] + fn to_char_valid() { + for i in 0x00_u32..0x100 { + super::to_char(i as u8); + } + } + + #[test] + fn ascii() { + for i in 0x00..0x80 { + assert_eq!(super::to_char(i), i as char); + } + } + + #[test] + #[allow(unknown_lints)] // invalid_from_utf8 was added in rust 1.72 + #[allow(invalid_from_utf8)] + fn example_slice() { + use super::FromCp437; + let data = b"Cura\x87ao"; + assert!(::std::str::from_utf8(data).is_err()); + assert_eq!(data.from_cp437(), "Curaçao"); + } + + #[test] + fn example_vec() { + use super::FromCp437; + let data = vec![0xCC, 0xCD, 0xCD, 0xB9]; + assert!(String::from_utf8(data.clone()).is_err()); + assert_eq!(&*data.from_cp437(), "╠══╣"); + } +} diff --git a/v2/vendor/security/zip-7.2.0/src/crc32.rs b/v2/vendor/security/zip-7.2.0/src/crc32.rs new file mode 100644 index 0000000000..4d1bd8e264 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/src/crc32.rs @@ -0,0 +1,131 @@ +//! Helper module to compute a CRC32 checksum + +use std::io::{self, Read}; + +use crc32fast::Hasher; + +/// Reader that validates the CRC32 when it reaches the EOF. +pub struct Crc32Reader { + inner: R, + hasher: Hasher, + check: u32, + /// Signals if `inner` stores aes encrypted data. + /// AE-2 encrypted data doesn't use crc and sets the value to 0. + enabled: bool, +} + +impl Crc32Reader { + /// Get a new Crc32Reader which checks the inner reader against checksum. + /// The check is disabled if `ae2_encrypted == true`. + pub(crate) fn new(inner: R, checksum: u32, ae2_encrypted: bool) -> Crc32Reader { + Crc32Reader { + inner, + hasher: Hasher::new(), + check: checksum, + enabled: !ae2_encrypted, + } + } + + fn check_matches(&self) -> bool { + self.check == self.hasher.clone().finalize() + } + + pub fn into_inner(self) -> R { + self.inner + } +} + +#[cold] +fn invalid_checksum() -> io::Error { + io::Error::new(io::ErrorKind::InvalidData, "Invalid checksum") +} + +impl Read for Crc32Reader { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + let count = self.inner.read(buf)?; + + if self.enabled { + if count == 0 && !buf.is_empty() && !self.check_matches() { + return Err(invalid_checksum()); + } + self.hasher.update(&buf[..count]); + } + Ok(count) + } + + fn read_to_end(&mut self, buf: &mut Vec) -> io::Result { + let start = buf.len(); + let n = self.inner.read_to_end(buf)?; + + if self.enabled { + self.hasher.update(&buf[start..]); + if !self.check_matches() { + return Err(invalid_checksum()); + } + } + + Ok(n) + } + + fn read_to_string(&mut self, buf: &mut String) -> io::Result { + let start = buf.len(); + let n = self.inner.read_to_string(buf)?; + + if self.enabled { + self.hasher.update(&buf.as_bytes()[start..]); + if !self.check_matches() { + return Err(invalid_checksum()); + } + } + + Ok(n) + } +} + +#[cfg(test)] +mod test { + use std::io::Read; + + use super::Crc32Reader; + + #[test] + fn test_empty_reader() { + let data: &[u8] = b""; + let mut buf = [0; 1]; + + let mut reader = Crc32Reader::new(data, 0, false); + assert_eq!(reader.read(&mut buf).unwrap(), 0); + + let mut reader = Crc32Reader::new(data, 1, false); + assert!(reader + .read(&mut buf) + .unwrap_err() + .to_string() + .contains("Invalid checksum")); + } + + #[test] + fn test_byte_by_byte() { + let data: &[u8] = b"1234"; + let mut buf = [0; 1]; + + let mut reader = Crc32Reader::new(data, 0x9be3e0a3, false); + assert_eq!(reader.read(&mut buf).unwrap(), 1); + assert_eq!(reader.read(&mut buf).unwrap(), 1); + assert_eq!(reader.read(&mut buf).unwrap(), 1); + assert_eq!(reader.read(&mut buf).unwrap(), 1); + assert_eq!(reader.read(&mut buf).unwrap(), 0); + // Can keep reading 0 bytes after the end + assert_eq!(reader.read(&mut buf).unwrap(), 0); + } + + #[test] + fn test_zero_read() { + let data: &[u8] = b"1234"; + let mut buf = [0; 5]; + + let mut reader = Crc32Reader::new(data, 0x9be3e0a3, false); + assert_eq!(reader.read(&mut buf[..0]).unwrap(), 0); + assert_eq!(reader.read(&mut buf).unwrap(), 4); + } +} diff --git a/v2/vendor/security/zip-7.2.0/src/extra_fields/extended_timestamp.rs b/v2/vendor/security/zip-7.2.0/src/extra_fields/extended_timestamp.rs new file mode 100644 index 0000000000..22ebacf23c --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/src/extra_fields/extended_timestamp.rs @@ -0,0 +1,111 @@ +use crate::result::invalid; +use crate::result::{ZipError, ZipResult}; +use crate::unstable::LittleEndianReadExt; +use std::io::Read; + +/// extended timestamp, as described in + +#[derive(Debug, Clone)] +pub struct ExtendedTimestamp { + mod_time: Option, + ac_time: Option, + cr_time: Option, +} + +impl ExtendedTimestamp { + /// creates an extended timestamp struct by reading the required bytes from the reader. + /// + /// This method assumes that the length has already been read, therefore + /// it must be passed as an argument + pub fn try_from_reader(reader: &mut R, len: u16) -> ZipResult + where + R: Read, + { + if len == 0 { + return Err(invalid!("Extended timestamp field is empty")); + } + let mut flags = [0u8]; + let mut bytes_to_read = len as usize; + reader.read_exact(&mut flags)?; + bytes_to_read -= flags.len(); + let flags = flags[0]; + + // the `flags` field refers to the local headers and might not correspond + // to the len field. If the length field is 1+4, we assume that only + // the modification time has been set + + // > Those times that are present will appear in the order indicated, but + // > any combination of times may be omitted. (Creation time may be + // > present without access time, for example.) TSize should equal + // > (1 + 4*(number of set bits in Flags)), as the block is currently + // > defined. + if len != 5 && len as u32 != 1 + 4 * flags.count_ones() { + //panic!("found len {len} and flags {flags:08b}"); + return Err(ZipError::UnsupportedArchive( + "flags and len don't match in extended timestamp field", + )); + } + + // allow unsupported/undocumented flags + + let mod_time = if (flags & 0b00000001u8 == 0b00000001u8) || len == 5 { + bytes_to_read -= size_of::(); + Some(reader.read_u32_le()?) + } else { + None + }; + + let ac_time = if flags & 0b00000010u8 == 0b00000010u8 && len > 5 { + bytes_to_read -= size_of::(); + Some(reader.read_u32_le()?) + } else { + None + }; + + let cr_time = if flags & 0b00000100u8 == 0b00000100u8 && len > 5 { + bytes_to_read -= size_of::(); + Some(reader.read_u32_le()?) + } else { + None + }; + + if bytes_to_read > 0 { + // ignore undocumented bytes + reader.read_exact(&mut vec![0; bytes_to_read])? + } + + Ok(Self { + mod_time, + ac_time, + cr_time, + }) + } + + /// returns the last modification timestamp, if defined, as UNIX epoch seconds + pub fn mod_time(&self) -> Option { + self.mod_time + } + + /// returns the last access timestamp, if defined, as UNIX epoch seconds + pub fn ac_time(&self) -> Option { + self.ac_time + } + + /// returns the creation timestamp, if defined, as UNIX epoch seconds + pub fn cr_time(&self) -> Option { + self.cr_time + } +} + +#[test] +/// Ensure we don't panic or read garbage data if the field body is empty +pub fn test_bad_extended_timestamp() -> ZipResult<()> { + use crate::ZipArchive; + use std::io::Cursor; + + assert!(ZipArchive::new(Cursor::new(include_bytes!( + "../../tests/data/extended_timestamp_bad.zip" + ))) + .is_err()); + Ok(()) +} diff --git a/v2/vendor/security/zip-7.2.0/src/extra_fields/mod.rs b/v2/vendor/security/zip-7.2.0/src/extra_fields/mod.rs new file mode 100644 index 0000000000..de6a3b6c72 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/src/extra_fields/mod.rs @@ -0,0 +1,35 @@ +//! types for extra fields + +/// marker trait to denote the place where this extra field has been stored +pub trait ExtraFieldVersion {} + +/// use this to mark extra fields specified in a local header + +#[derive(Debug, Clone)] +pub struct LocalHeaderVersion; + +/// use this to mark extra fields specified in the central header + +#[derive(Debug, Clone)] +pub struct CentralHeaderVersion; + +impl ExtraFieldVersion for LocalHeaderVersion {} +impl ExtraFieldVersion for CentralHeaderVersion {} + +mod extended_timestamp; +mod ntfs; +mod zipinfo_utf8; + +pub use extended_timestamp::*; +pub use ntfs::Ntfs; +pub use zipinfo_utf8::UnicodeExtraField; + +/// contains one extra field +#[derive(Debug, Clone)] +pub enum ExtraField { + /// NTFS extra field + Ntfs(Ntfs), + + /// extended timestamp, as described in + ExtendedTimestamp(ExtendedTimestamp), +} diff --git a/v2/vendor/security/zip-7.2.0/src/extra_fields/ntfs.rs b/v2/vendor/security/zip-7.2.0/src/extra_fields/ntfs.rs new file mode 100644 index 0000000000..d303735bac --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/src/extra_fields/ntfs.rs @@ -0,0 +1,97 @@ +use std::io::Read; + +use crate::{ + result::{ZipError, ZipResult}, + unstable::LittleEndianReadExt, +}; + +/// The NTFS extra field as described in [PKWARE's APPNOTE.TXT v6.3.9]. +/// +/// This field stores [Windows file times], which are 64-bit unsigned integer +/// values that represents the number of 100-nanosecond intervals that have +/// elapsed since "1601-01-01 00:00:00 UTC". +/// +/// [PKWARE's APPNOTE.TXT v6.3.9]: https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT +/// [Windows file times]: https://docs.microsoft.com/en-us/windows/win32/sysinfo/file-times +#[derive(Clone, Debug)] +pub struct Ntfs { + mtime: u64, + atime: u64, + ctime: u64, +} + +impl Ntfs { + /// Creates a NTFS extra field struct by reading the required bytes from the + /// reader. + /// + /// This method assumes that the length has already been read, therefore it + /// must be passed as an argument. + pub fn try_from_reader(reader: &mut R, len: u16) -> ZipResult + where + R: Read, + { + if len != 32 { + return Err(ZipError::UnsupportedArchive( + "NTFS extra field has an unsupported length", + )); + } + + // Read reserved for future use. + let _ = reader.read_u32_le()?; + + let tag = reader.read_u16_le()?; + if tag != 0x0001 { + return Err(ZipError::UnsupportedArchive( + "NTFS extra field has an unsupported attribute tag", + )); + } + let size = reader.read_u16_le()?; + if size != 24 { + return Err(ZipError::UnsupportedArchive( + "NTFS extra field has an unsupported attribute size", + )); + } + + let mtime = reader.read_u64_le()?; + let atime = reader.read_u64_le()?; + let ctime = reader.read_u64_le()?; + Ok(Self { + mtime, + atime, + ctime, + }) + } + + /// Returns the file last modification time as a file time. + pub fn mtime(&self) -> u64 { + self.mtime + } + + /// Returns the file last modification time as a file time. + #[cfg(feature = "nt-time")] + pub fn modified_file_time(&self) -> nt_time::FileTime { + nt_time::FileTime::new(self.mtime) + } + + /// Returns the file last access time as a file time. + pub fn atime(&self) -> u64 { + self.atime + } + + /// Returns the file last access time as a file time. + #[cfg(feature = "nt-time")] + pub fn accessed_file_time(&self) -> nt_time::FileTime { + nt_time::FileTime::new(self.atime) + } + + /// Returns the file creation time as a file time. + pub fn ctime(&self) -> u64 { + self.ctime + } + + /// Returns the file creation time as a file time. + #[cfg(feature = "nt-time")] + pub fn created_file_time(&self) -> nt_time::FileTime { + nt_time::FileTime::new(self.ctime) + } +} diff --git a/v2/vendor/security/zip-7.2.0/src/extra_fields/zipinfo_utf8.rs b/v2/vendor/security/zip-7.2.0/src/extra_fields/zipinfo_utf8.rs new file mode 100644 index 0000000000..c2c09e50cd --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/src/extra_fields/zipinfo_utf8.rs @@ -0,0 +1,40 @@ +use crate::result::{invalid, ZipResult}; +use crate::unstable::LittleEndianReadExt; +use core::mem::size_of; +use std::io::Read; + +/// Info-ZIP Unicode Path Extra Field (0x7075) or Unicode Comment Extra Field (0x6375), as +/// specified in APPNOTE 4.6.8 and 4.6.9 +#[derive(Clone, Debug)] +pub struct UnicodeExtraField { + crc32: u32, + content: Box<[u8]>, +} + +impl UnicodeExtraField { + /// Verifies the checksum and returns the content. + pub fn unwrap_valid(self, ascii_field: &[u8]) -> ZipResult> { + let mut crc32 = crc32fast::Hasher::new(); + crc32.update(ascii_field); + let actual_crc32 = crc32.finalize(); + if self.crc32 != actual_crc32 { + return Err(invalid!("CRC32 checksum failed on Unicode extra field")); + } + Ok(self.content) + } +} + +impl UnicodeExtraField { + pub(crate) fn try_from_reader(reader: &mut R, len: u16) -> ZipResult { + // Read and discard version byte + reader.read_exact(&mut [0u8])?; + + let crc32 = reader.read_u32_le()?; + let content_len = (len as usize) + .checked_sub(size_of::() + size_of::()) + .ok_or(invalid!("Unicode extra field is too small"))?; + let mut content = vec![0u8; content_len].into_boxed_slice(); + reader.read_exact(&mut content)?; + Ok(Self { crc32, content }) + } +} diff --git a/v2/vendor/security/zip-7.2.0/src/legacy/huffman.rs b/v2/vendor/security/zip-7.2.0/src/legacy/huffman.rs new file mode 100644 index 0000000000..e7b2d3c77a --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/src/legacy/huffman.rs @@ -0,0 +1,260 @@ +use std::io::{self, Error, Seek}; + +use bitstream_io::{BitRead, BitReader, Endianness}; + +#[derive(Default, Clone, Copy)] +pub struct TableEntry { + /// Wide enough to fit the max symbol nbr. + pub sym: u16, + /// 0 means no symbol. + pub len: u8, +} + +/// Deflate uses max 288 symbols. +const MAX_HUFFMAN_SYMBOLS: usize = 288; +/// Implode uses max 16-bit codewords. +const MAX_HUFFMAN_BITS: usize = 16; +/// Seems a good trade-off. +const HUFFMAN_LOOKUP_TABLE_BITS: u8 = 8; + +pub struct HuffmanDecoder { + /// Lookup table for fast decoding of short codewords. + pub table: [TableEntry; 1 << HUFFMAN_LOOKUP_TABLE_BITS], + /// "Sentinel bits" value for each codeword length. + pub sentinel_bits: [u32; MAX_HUFFMAN_BITS + 1], + /// First symbol index minus first codeword mod 2**16 for each length. + pub offset_first_sym_idx: [u16; MAX_HUFFMAN_BITS + 1], + /// Map from symbol index to symbol. + pub syms: [u16; MAX_HUFFMAN_SYMBOLS], + // num_syms:usize +} + +impl Default for HuffmanDecoder { + fn default() -> Self { + Self { + table: [TableEntry::default(); 1 << HUFFMAN_LOOKUP_TABLE_BITS], + sentinel_bits: [0; MAX_HUFFMAN_BITS + 1], + offset_first_sym_idx: [0; MAX_HUFFMAN_BITS + 1], + syms: [0; MAX_HUFFMAN_SYMBOLS], + } + } +} + +/// Reverse the n least significant bits of x. +/// The (16 - n) most significant bits of the result will be zero. +pub fn reverse_lsb(x: u16, n: usize) -> u16 { + debug_assert!(n > 0); + debug_assert!(n <= 16); + x.reverse_bits() >> (16 - n) +} + +/// Initialize huffman decoder d for a code defined by the n codeword lengths. +/// Returns `Err` if the codeword lengths do not correspond to a valid prefix +/// code. +impl HuffmanDecoder { + pub fn init(&mut self, lengths: &[u8], n: usize) -> std::io::Result<()> { + let mut count = [0; MAX_HUFFMAN_BITS + 1]; + let mut code = [0; MAX_HUFFMAN_BITS + 1]; + let mut sym_idx = [0u16; MAX_HUFFMAN_BITS + 1]; + + // Count the number of codewords of each length. + for i in 0..n { + debug_assert!(lengths[i] as usize <= MAX_HUFFMAN_BITS); + count[lengths[i] as usize] += 1; + } + count[0] = 0; // Ignore zero-length codewords. + // Compute sentinel_bits and offset_first_sym_idx for each length. + code[0] = 0; + sym_idx[0] = 0; + for l in 1..=MAX_HUFFMAN_BITS { + // First canonical codeword of this length. + code[l] = (code[l - 1] + count[l - 1]) << 1; + + if count[l] != 0 && code[l] as u32 + count[l] as u32 - 1 > (1u32 << l) - 1 { + // The last codeword is longer than l bits. + return Err(Error::new( + io::ErrorKind::InvalidData, + "the last codeword is longer than len bits", + )); + } + + let s = (code[l] as u32 + count[l] as u32) << (MAX_HUFFMAN_BITS - l); + self.sentinel_bits[l] = s; + debug_assert!(self.sentinel_bits[l] >= code[l] as u32, "No overflow!"); + + sym_idx[l] = sym_idx[l - 1] + count[l - 1]; + self.offset_first_sym_idx[l] = sym_idx[l].wrapping_sub(code[l]); + } + + // Zero-initialize the lookup table. + self.table.fill(TableEntry::default()); + + // Build mapping from index to symbol and populate the lookup table. + lengths + .iter() + .enumerate() + .take(n) + .for_each(|(i, code_len)| { + let l = *code_len as usize; + if l == 0 { + return; + } + + self.syms[sym_idx[l] as usize] = i as u16; + sym_idx[l] += 1; + + if l <= HUFFMAN_LOOKUP_TABLE_BITS as usize { + self.table_insert(i, l, code[l]); + code[l] += 1; + } + }); + + Ok(()) + } + + pub fn table_insert(&mut self, sym: usize, len: usize, codeword: u16) { + debug_assert!(len <= HUFFMAN_LOOKUP_TABLE_BITS as usize); + + let codeword = reverse_lsb(codeword, len); // Make it LSB-first. + let pad_len = HUFFMAN_LOOKUP_TABLE_BITS as usize - len; + + // Pad the pad_len upper bits with all bit combinations. + for padding in 0..(1 << pad_len) { + let index = (codeword | (padding << len)) as usize; + debug_assert!(sym <= u16::MAX as usize); + self.table[index].sym = sym as u16; + debug_assert!(len <= u8::MAX as usize); + self.table[index].len = len as u8; + } + } + + /// Use the decoder d to decode a symbol from the LSB-first zero-padded bits. + /// Returns the decoded symbol number or an error if no symbol could be decoded. + /// *num_used_bits will be set to the number of bits used to decode the symbol, + /// or zero if no symbol could be decoded. + pub fn huffman_decode( + &mut self, + length: u64, + is: &mut BitReader, + ) -> std::io::Result { + // First try the lookup table. + let read_bits1 = (HUFFMAN_LOOKUP_TABLE_BITS as u64).min(length - is.position_in_bits()?); + let lookup_bits = !is.read_var::(read_bits1 as u32)? as usize; + debug_assert!(lookup_bits < self.table.len()); + if self.table[lookup_bits].len != 0 { + debug_assert!(self.table[lookup_bits].len <= HUFFMAN_LOOKUP_TABLE_BITS); + is.seek_bits(io::SeekFrom::Current( + -(read_bits1 as i64) + self.table[lookup_bits].len as i64, + ))?; + return Ok(self.table[lookup_bits].sym); + } + + // Then do canonical decoding with the bits in MSB-first order. + let read_bits2 = (HUFFMAN_LOOKUP_TABLE_BITS as u64).min(length - is.position_in_bits()?); + let mut bits = reverse_lsb( + (lookup_bits | ((!is.read_var::(read_bits2 as u32)? as usize) << read_bits1)) + as u16, + MAX_HUFFMAN_BITS, + ); + + for l in (HUFFMAN_LOOKUP_TABLE_BITS as usize + 1)..=MAX_HUFFMAN_BITS { + if (bits as u32) < self.sentinel_bits[l] { + bits >>= MAX_HUFFMAN_BITS - l; + let sym_idx = (self.offset_first_sym_idx[l] as usize + bits as usize) & 0xFFFF; + //assert(sym_idx < self.num_syms); + is.seek_bits(io::SeekFrom::Current( + -(read_bits1 as i64 + read_bits2 as i64) + l as i64, + ))?; + return Ok(self.syms[sym_idx]); + } + } + Err(Error::new( + io::ErrorKind::InvalidData, + "huffman decode failed", + )) + } +} + +#[cfg(test)] +mod tests { + use std::io::Cursor; + + use bitstream_io::{BitReader, LittleEndian}; + + use super::HuffmanDecoder; + #[test] + fn test_huffman_decode_basic() { + let lens = [ + 3, // sym 0: 000 + 3, // sym 1: 001 + 3, // sym 2: 010 + 3, // sym 3: 011 + 3, // sym 4: 100 + 3, // sym 5: 101 + 4, // sym 6: 1100 + 4, // sym 7: 1101 + 0, // sym 8: + 0, // sym 9: + 0, // sym 10: + 0, // sym 11: + 0, // sym 12: + 0, // sym 13: + 0, // sym 14: + 0, // sym 15: + 6, // sym 16: 111110 + 5, // sym 17: 11110 + 4, // sym 18: 1110 + ]; + + let mut d = HuffmanDecoder::default(); + d.init(&lens, lens.len()).unwrap(); + + // 000 (msb-first) -> 000 (lsb-first) + assert_eq!( + d.huffman_decode( + 8, + &mut BitReader::endian(&mut Cursor::new(&[!0x0]), LittleEndian) + ) + .unwrap(), + 0 + ); + + /* 011 (msb-first) -> 110 (lsb-first)*/ + assert_eq!( + d.huffman_decode( + 8, + &mut BitReader::endian(&mut Cursor::new(&[!0b110]), LittleEndian) + ) + .unwrap(), + 0b011 + ); + + /* 11110 (msb-first) -> 01111 (lsb-first)*/ + assert_eq!( + d.huffman_decode( + 8, + &mut BitReader::endian(&mut Cursor::new(&[!0b1111]), LittleEndian) + ) + .unwrap(), + 0b10001 + ); + + /* 111110 (msb-first) -> 011111 (lsb-first)*/ + assert_eq!( + d.huffman_decode( + 8, + &mut BitReader::endian(&mut Cursor::new(&[!0b11111]), LittleEndian) + ) + .unwrap(), + 0b10000 + ); + + /* 1111111 (msb-first) -> 1111111 (lsb-first)*/ + assert!(d + .huffman_decode( + 8, + &mut BitReader::endian(&mut Cursor::new(&[!0x7f]), LittleEndian) + ) + .is_err()); + } +} diff --git a/v2/vendor/security/zip-7.2.0/src/legacy/implode.rs b/v2/vendor/security/zip-7.2.0/src/legacy/implode.rs new file mode 100644 index 0000000000..bd35ed1a6f --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/src/legacy/implode.rs @@ -0,0 +1,250 @@ +use super::huffman::HuffmanDecoder; +use bitstream_io::{BitRead, BitReader, Endianness, LittleEndian}; +use std::io::{self, Cursor, Error, Read, Result}; + +/// Maximum symbol value in the length Huffman table (6 bits) +/// When this value is decoded, an additional byte is read for extended length +const MAX_LEN_SYMBOL: u16 = 63; + +/// Maximum number of symbols in a Huffman table (256 for literals, 64 for lengths/distances) +const MAX_HUFFMAN_SYMBOLS: usize = 1 << 8; // 256 + +/// Maximum code length in bits for Huffman codes (per ZIP specification) +const MAX_CODE_LENGTH: usize = 16; + +/// Size of the code length count array (lengths are actually 1..=16) +const CODE_LENGTH_COUNT_SIZE: usize = MAX_CODE_LENGTH; + +/// Initialize the Huffman decoder d with num_lens codeword lengths read from is. +/// Returns false if the input is invalid. +fn read_huffman_code( + is: &mut BitReader, + num_lens: usize, +) -> std::io::Result> { + let mut lens = [0; MAX_HUFFMAN_SYMBOLS]; + let mut len_count = [0; CODE_LENGTH_COUNT_SIZE]; + + // Number of bytes representing the Huffman code. + let byte = is.read::<8, u8>()?; + let num_bytes = (byte + 1) as usize; + + let mut codeword_idx = 0; + for _byte_idx in 0..num_bytes { + let byte = is.read::<8, u16>()?; + + let codeword_len = (byte & 0xf) + 1; /* Low four bits plus one. */ + let run_length = (byte >> 4) + 1; /* High four bits plus one. */ + + debug_assert!((1..=16).contains(&codeword_len)); + len_count[codeword_len as usize - 1] += run_length; + + if (codeword_idx + run_length) as usize > num_lens { + return Err(Error::new( + io::ErrorKind::InvalidData, + "too many codeword lengths", + )); + } + for _ in 0..run_length { + debug_assert!((codeword_idx as usize) < num_lens); + lens[codeword_idx as usize] = codeword_len as u8; + codeword_idx += 1; + } + } + + debug_assert!(codeword_idx as usize <= num_lens); + if (codeword_idx as usize) < num_lens { + return Err(Error::new( + io::ErrorKind::InvalidData, + "not enough codeword lengths", + )); + } + + // Check that the Huffman tree is full. + let mut avail_codewords = 1; + for i in 1..=16 { + debug_assert!(avail_codewords >= 0); + avail_codewords *= 2; + avail_codewords -= len_count[i - 1] as i32; + if avail_codewords < 0 { + return Err(Error::new( + io::ErrorKind::InvalidData, + "huffman tree is not full", + )); + } + } + if avail_codewords != 0 { + // Not all codewords were used. + return Err(Error::new( + io::ErrorKind::InvalidData, + "not all codewords were used", + )); + } + + let mut d = Box::new(HuffmanDecoder::default()); + d.init(&lens, num_lens)?; + Ok(d) +} + +fn hwexplode( + src: &[u8], + uncomp_len: usize, + large_wnd: bool, + lit_tree: bool, + pk101_bug_compat: bool, + dst: &mut Vec, +) -> std::io::Result<()> { + // Pre-allocate capacity + dst.reserve(uncomp_len); + + let bit_length = src.len() as u64 * 8; + let mut is = BitReader::endian(Cursor::new(&src), LittleEndian); + let mut lit_decoder_opt = if lit_tree { + Some(read_huffman_code(&mut is, 256)?) + } else { + None + }; + let mut len_decoder = read_huffman_code(&mut is, 64)?; + let mut dist_decoder = read_huffman_code(&mut is, 64)?; + let min_len = if (pk101_bug_compat && large_wnd) || (!pk101_bug_compat && lit_tree) { + 3 + } else { + 2 + }; + let dist_low_bits = if large_wnd { 7 } else { 6 }; + + while dst.len() < uncomp_len { + let is_literal = is.read_bit()?; + if is_literal { + // Literal. + let sym; + if let Some(lit_decoder) = &mut lit_decoder_opt { + sym = lit_decoder.huffman_decode(bit_length, &mut is)?; + } else { + sym = is.read::<8, u8>()? as u16; + } + debug_assert!(sym <= u8::MAX as u16); + dst.push(sym as u8); + continue; + } + + // Read the low dist bits. + let mut dist = is.read_var::(dist_low_bits)?; + // Read the Huffman-encoded high dist bits. + let sym = dist_decoder.huffman_decode(bit_length, &mut is)?; + dist |= (sym as u16) << dist_low_bits; + dist += 1; + + // Read the Huffman-encoded len. + let sym = len_decoder.huffman_decode(bit_length, &mut is)?; + let mut len = (sym + min_len) as usize; + + if sym == MAX_LEN_SYMBOL { + // Read an extra len byte. + len += is.read::<8, u16>()? as usize; + } + let len = len.min(uncomp_len - dst.len()); + + // Optimize the copy loop using extend_from_within when possible + if dist as usize <= dst.len() { + if dist as usize >= len { + // No overlap in copy, can use extend_from_within + let start = dst.len() - dist as usize; + dst.extend_from_within(start..start + len); + } else { + // Overlapping copy + for _ in 0..len { + let byte = dst[dst.len() - dist as usize]; + dst.push(byte); + } + } + } else { + // Copy with implicit zeros + for _ in 0..len { + if dist as usize > dst.len() { + dst.push(0); + } else { + let byte = dst[dst.len() - dist as usize]; + dst.push(byte); + } + } + } + } + Ok(()) +} + +#[derive(Debug)] +pub struct ImplodeDecoder { + compressed_reader: R, + uncompressed_size: u64, + stream_read: bool, + large_wnd: bool, + lit_tree: bool, + stream: Vec, + read_pos: usize, // Add read position tracker +} + +impl ImplodeDecoder { + pub fn new(inner: R, uncompressed_size: u64, flags: u16) -> Self { + let large_wnd = (flags & 2) != 0; + let lit_tree = (flags & 4) != 0; + ImplodeDecoder { + compressed_reader: inner, + uncompressed_size, + stream_read: false, + large_wnd, + lit_tree, + stream: Vec::new(), + read_pos: 0, + } + } + + pub fn into_inner(self) -> R { + self.compressed_reader + } +} + +impl Read for ImplodeDecoder { + fn read(&mut self, buf: &mut [u8]) -> Result { + if !self.stream_read { + self.stream_read = true; + let mut compressed_bytes = Vec::new(); + self.compressed_reader.read_to_end(&mut compressed_bytes)?; + + // Pre-allocate stream buffer + self.stream.reserve(self.uncompressed_size as usize); + + hwexplode( + &compressed_bytes, + self.uncompressed_size as usize, + self.large_wnd, + self.lit_tree, + false, + &mut self.stream, + )?; + } + + let available = self.stream.len() - self.read_pos; + let bytes_to_read = available.min(buf.len()); + buf[..bytes_to_read] + .copy_from_slice(&self.stream[self.read_pos..self.read_pos + bytes_to_read]); + self.read_pos += bytes_to_read; + Ok(bytes_to_read) + } +} + +#[cfg(test)] +mod tests { + use super::hwexplode; + + const HAMLET_256: &[u8; 249] = include_bytes!("../../tests/data/legacy/implode_hamlet_256.bin"); + const HAMLET_256_OUT: &[u8; 256] = + include_bytes!("../../tests/data/legacy/implode_hamlet_256.out"); + + #[test] + fn test_explode_hamlet_256() { + let mut dst = Vec::new(); + hwexplode(HAMLET_256, 256, false, false, false, &mut dst).unwrap(); + assert_eq!(dst.len(), 256); + assert_eq!(&dst, &HAMLET_256_OUT); + } +} diff --git a/v2/vendor/security/zip-7.2.0/src/legacy/mod.rs b/v2/vendor/security/zip-7.2.0/src/legacy/mod.rs new file mode 100644 index 0000000000..80858e2110 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/src/legacy/mod.rs @@ -0,0 +1,4 @@ +mod huffman; +pub(crate) mod implode; +pub(crate) mod reduce; +pub(crate) mod shrink; diff --git a/v2/vendor/security/zip-7.2.0/src/legacy/reduce.rs b/v2/vendor/security/zip-7.2.0/src/legacy/reduce.rs new file mode 100644 index 0000000000..e90d06a0ef --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/src/legacy/reduce.rs @@ -0,0 +1,313 @@ +use std::io::{self, Read, Result}; + +use bitstream_io::{BitRead, BitReader, Endianness, LittleEndian}; + +/// Number of bits used to represent indices in a follower set of size n. +const fn follower_idx_bitlen(n: u8) -> u8 { + debug_assert!(n <= 32); + match n { + 0 => 0, + 1 => 1, + _ => 8 - (n - 1).leading_zeros() as u8, + } +} + +#[derive(Default, Clone, Copy)] +struct FollowerSet { + followers: [u8; 32], + size: u8, + idx_bitlen: u8, +} + +/// Read the follower sets from is into fsets. Returns true on success. +type FollowerSetArray = [FollowerSet; u8::MAX as usize + 1]; + +fn read_follower_sets( + is: &mut BitReader, +) -> io::Result { + let mut fsets = [FollowerSet::default(); u8::MAX as usize + 1]; + for i in (0..=u8::MAX as usize).rev() { + let n = is.read::<6, u8>()?; + if n > 32 { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "invalid follower set", + )); + } + fsets[i].size = n; + fsets[i].idx_bitlen = follower_idx_bitlen(n); + + for j in 0..fsets[i].size as usize { + fsets[i].followers[j] = is.read::<8, u8>()?; + } + } + + Ok(fsets) +} + +/// Read the next byte from is, decoded based on prev_byte and the follower sets. +/// The byte is returned in *out_byte. +/// +/// # Returns +/// +/// * `Ok` with the byte if it was successfully read. +/// * `Err(io::Error)` on bad data or end of input. +fn read_next_byte( + is: &mut BitReader, + prev_byte: u8, + fsets: &mut FollowerSetArray, +) -> io::Result { + if fsets[prev_byte as usize].size == 0 // No followers + || is.read::<1, u8>()? == 1 + // Indicates next symbol is a literal byte + { + return is.read::<8, u8>(); + } + + // The bits represent the index of a follower byte. + let idx_bitlen = fsets[prev_byte as usize].idx_bitlen; + let follower_idx = is.read_var::(idx_bitlen as u32)? as usize; + if follower_idx >= fsets[prev_byte as usize].size as usize { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "invalid follower index", + )); + } + Ok(fsets[prev_byte as usize].followers[follower_idx]) +} + +fn max_len(comp_factor: u8) -> usize { + debug_assert!((1..=4).contains(&comp_factor)); + let v_len_bits = (8 - comp_factor) as usize; + // Bits in V + extra len byte + implicit 3. + ((1 << v_len_bits) - 1) + u8::MAX as usize + 3 +} + +fn max_dist(comp_factor: u8) -> usize { + debug_assert!((1..=4).contains(&comp_factor)); + let v_dist_bits = comp_factor as usize; + // Bits in V * 256 + W byte + implicit 1. */ + 1 << (v_dist_bits + 8) +} + +const DLE_BYTE: u8 = 0x90; + +/// Get the n least significant bits of x. +fn lsb(x: u8, n: u8) -> u8 { + if n >= 8 { + return x; + } + x & ((1 << n) - 1) +} + +fn hwexpand(src: &[u8], uncomp_len: usize, comp_factor: u8, dst: &mut Vec) -> io::Result<()> { + debug_assert!((1..=4).contains(&comp_factor)); + + // Pre-allocate to avoid reallocations + dst.reserve(uncomp_len); + + let mut is = BitReader::endian(src, LittleEndian); + let mut fsets = read_follower_sets(&mut is)?; + + // Number of bits in V used for backref length. + let v_len_bits = 8 - comp_factor; + + let mut curr_byte = 0; // The first "previous byte" is implicitly zero. + + while dst.len() < uncomp_len { + // Read a literal byte or DLE marker. + curr_byte = read_next_byte(&mut is, curr_byte, &mut fsets)?; + if curr_byte != DLE_BYTE { + // Output a literal byte. + dst.push(curr_byte); + continue; + } + + // Read the V byte which determines the length. + curr_byte = read_next_byte(&mut is, curr_byte, &mut fsets)?; + if curr_byte == 0 { + // Output a literal DLE byte. + dst.push(DLE_BYTE); + continue; + } + let v = curr_byte; + let mut len = lsb(v, v_len_bits) as usize; + if len == (1 << v_len_bits) - 1 { + // Read an extra length byte. + curr_byte = read_next_byte(&mut is, curr_byte, &mut fsets)?; + len += curr_byte as usize; + } + len += 3; + + // Read the W byte, which together with V gives the distance. + curr_byte = read_next_byte(&mut is, curr_byte, &mut fsets)?; + let dist = (((v as usize) >> v_len_bits) << 8) + curr_byte as usize + 1; + + debug_assert!(len <= max_len(comp_factor)); + debug_assert!(dist <= max_dist(comp_factor)); + + // Output the back reference. + if dist <= dst.len() { + // Optimize for non-overlapping copies + if dist >= len { + // No overlap, can use extend_from_within + let start = dst.len() - dist; + dst.extend_from_within(start..start + len.min(uncomp_len - dst.len())); + } else { + // Overlapping copy + let copy_len = len.min(uncomp_len - dst.len()); + for _ in 0..copy_len { + let byte = dst[dst.len() - dist]; + dst.push(byte); + } + } + } else { + // Copy with implicit zeros + let copy_len = len.min(uncomp_len - dst.len()); + for _ in 0..copy_len { + if dist > dst.len() { + dst.push(0); + } else { + let byte = dst[dst.len() - dist]; + dst.push(byte); + } + } + } + } + Ok(()) +} + +#[derive(Debug)] +pub struct ReduceDecoder { + compressed_reader: R, + uncompressed_size: u64, + stream_read: bool, + comp_factor: u8, + stream: Vec, + read_pos: usize, // Add read position tracker +} + +impl ReduceDecoder { + pub fn new(inner: R, uncompressed_size: u64, comp_factor: u8) -> Self { + ReduceDecoder { + compressed_reader: inner, + uncompressed_size, + stream_read: false, + comp_factor, + stream: Vec::new(), + read_pos: 0, + } + } + + pub fn into_inner(self) -> R { + self.compressed_reader + } +} + +impl Read for ReduceDecoder { + fn read(&mut self, buf: &mut [u8]) -> Result { + if !self.stream_read { + self.stream_read = true; + let mut compressed_bytes = Vec::new(); + self.compressed_reader.read_to_end(&mut compressed_bytes)?; + hwexpand( + &compressed_bytes, + self.uncompressed_size as usize, + self.comp_factor, + &mut self.stream, + )?; + } + + let available = self.stream.len() - self.read_pos; + let bytes_to_read = available.min(buf.len()); + buf[..bytes_to_read] + .copy_from_slice(&self.stream[self.read_pos..self.read_pos + bytes_to_read]); + self.read_pos += bytes_to_read; + Ok(bytes_to_read) + } +} + +#[cfg(test)] +mod tests { + use super::hwexpand; + use crate::legacy::reduce::{follower_idx_bitlen, lsb, max_dist}; + const HAMLET_2048: &[u8; 1285] = + include_bytes!("../../tests/data/legacy/reduce_hamlet_2048.bin"); + const HAMLET_2048_OUT: &[u8; 2048] = + include_bytes!("../../tests/data/legacy/implode_hamlet_2048.out"); + + #[test] + fn test_lsb() { + assert_eq!(lsb(0xFF, 8), 0xFF); + for i in 0..7 { + assert_eq!(lsb(0xFF, i), (1 << i) - 1); + } + } + + #[test] + fn test_expand_hamlet2048() { + let mut dst = Vec::new(); + hwexpand(HAMLET_2048, 2048, 4, &mut dst).unwrap(); + assert_eq!(dst.len(), 2048); + assert_eq!(&dst, &HAMLET_2048_OUT); + } + + /* + Put some text first to make PKZIP actually use Reduce compression. + Target the code path which copies a zero when dist > current position. + + $ curl -O http://cd.textfiles.com/originalsw/25/pkz092.exe + $ dosbox -c "mount c ." -c "c:" -c "pkz092" -c "exit" + $ dd if=hamlet.txt bs=1 count=2048 > a + $ dd if=/dev/zero bs=1 count=1024 >> a + $ dosbox -c "mount c ." -c "c:" -c "pkzip -ea4 a.zip a" -c "exit" + $ xxd -i -s 31 -l $(expr $(find A.ZIP -printf %s) - 100) A.ZIP + */ + const ZEROS_REDUCED: &[u8; 1297] = + include_bytes!("../../tests/data/legacy/reduce_zero_reduced.bin"); + + #[test] + fn test_expand_zeros() { + let mut dst = Vec::new(); + hwexpand(ZEROS_REDUCED, 2048 + 1024, 4, &mut dst).unwrap(); + assert_eq!(dst.len(), 2048 + 1024); + for i in 0..(1 << 10) { + assert_eq!(dst[(1 << 11) + i], 0); + } + } + + fn orig_follower_idx_bitlen(n: u8) -> u8 { + if n > 16 { + return 5; + } + if n > 8 { + return 4; + } + if n > 4 { + return 3; + } + if n > 2 { + return 2; + } + if n > 0 { + return 1; + } + 0 + } + + #[test] + fn test_follower_idx_biten() { + for i in 0..=32 { + assert_eq!(orig_follower_idx_bitlen(i), follower_idx_bitlen(i)); + } + } + + #[test] + fn test_max_dist() { + for i in 1..=4 { + let v_dist_bits = i as usize; + let c = 1 << (v_dist_bits + 8); + assert_eq!(max_dist(i), c); + } + } +} diff --git a/v2/vendor/security/zip-7.2.0/src/legacy/shrink.rs b/v2/vendor/security/zip-7.2.0/src/legacy/shrink.rs new file mode 100644 index 0000000000..c8615a920a --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/src/legacy/shrink.rs @@ -0,0 +1,373 @@ +use std::io::{self, Error, Read}; + +use bitstream_io::{BitRead, BitReader, Endianness, LittleEndian}; + +const MIN_CODE_SIZE: u8 = 9; +const MAX_CODE_SIZE: u8 = 13; + +const MAX_CODE: usize = (1 << MAX_CODE_SIZE) - 1; +const CONTROL_CODE: usize = 256; +const INC_CODE_SIZE: u16 = 1; +const PARTIAL_CLEAR: u16 = 2; + +/// Number of codes available for the LZW dictionary (excluding control codes) +/// These are the codes from CONTROL_CODE+1 to MAX_CODE +const FREE_CODE_QUEUE_SIZE: usize = MAX_CODE - CONTROL_CODE + 1; + +// const HASH_BITS: usize = MAX_CODE_SIZE + 1; /* For a load factor of 0.5. */ +// const HASHTAB_SIZE: usize = 1 << HASH_BITS; +const UNKNOWN_LEN: u16 = u16::MAX; + +struct CodeQueue { + next_idx: usize, + codes: [Option; FREE_CODE_QUEUE_SIZE], +} + +impl CodeQueue { + fn new() -> Self { + let mut codes = [None; FREE_CODE_QUEUE_SIZE]; + for (i, code) in ((CONTROL_CODE as u16 + 1)..=(MAX_CODE as u16 - 1)).enumerate() { + codes[i] = Some(code); + } + Self { next_idx: 0, codes } + } + + // Return the next code in the queue, or INVALID_CODE if the queue is empty. + fn next(&self) -> Option { + // assert(q->next_idx < sizeof(q->codes) / sizeof(q->codes[0])); + if let Some(Some(next)) = self.codes.get(self.next_idx) { + Some(*next) + } else { + None + } + } + + /// Return and remove the next code from the queue, or return INVALID_CODE if + /// the queue is empty. + fn remove_next(&mut self) -> Option { + let res = self.next(); + if res.is_some() { + self.next_idx += 1; + } + res + } +} + +#[derive(Clone, Debug, Copy, Default)] +struct Codetab { + last_dst_pos: usize, + prefix_code: Option, + len: u16, + ext_byte: u8, +} + +impl Codetab { + pub fn create_new() -> [Self; MAX_CODE + 1] { + let mut codetab = [Codetab::default(); MAX_CODE + 1]; + for (i, code) in codetab.iter_mut().enumerate().take(u8::MAX as usize + 1) { + *code = Codetab { + prefix_code: Some(i as u16), + ext_byte: i as u8, + len: 1, + last_dst_pos: 0, + }; + } + codetab + } +} + +fn unshrink_partial_clear(codetab: &mut [Codetab], queue: &mut CodeQueue) { + let mut is_prefix = [false; MAX_CODE + 1]; + + // Scan for codes that have been used as a prefix. + for code in codetab.iter().take(MAX_CODE + 1).skip(CONTROL_CODE + 1) { + if let Some(prefix_code) = code.prefix_code { + is_prefix[prefix_code as usize] = true; + } + } + + // Clear "non-prefix" codes in the table; populate the code queue. + let mut code_queue_size = 0; + for i in (CONTROL_CODE + 1)..MAX_CODE { + if !is_prefix[i] { + codetab[i].prefix_code = None; + queue.codes[code_queue_size] = Some(i as u16); + code_queue_size += 1; + } + } + queue.codes[code_queue_size] = None; // End-of-queue marker. + queue.next_idx = 0; +} + +/// Read the next code from the input stream and return it in next_code. Returns +/// `Ok(None)` if the end of the stream is reached. If the stream contains invalid +/// data, returns `Err`. +fn read_code( + is: &mut BitReader, + code_size: &mut u8, + codetab: &mut [Codetab], + queue: &mut CodeQueue, +) -> io::Result> { + // assert(sizeof(code) * CHAR_BIT >= *code_size); + let code = is.read_var::(*code_size as u32)?; + + // Handle regular codes (the common case). + if code != CONTROL_CODE as u16 { + return Ok(Some(code)); + } + + // Handle control codes. + if let Ok(control_code) = is.read_var::(*code_size as u32) { + match control_code { + INC_CODE_SIZE => { + if *code_size >= MAX_CODE_SIZE { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "tried to increase code size when already at maximum", + )); + } + *code_size += 1; + } + PARTIAL_CLEAR => { + unshrink_partial_clear(codetab, queue); + } + _ => { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("Invalid control code {}", control_code), + )); + } + } + return read_code(is, code_size, codetab, queue); + } + Ok(None) +} + +/// Output the string represented by a code into dst at dst_pos. +/// +/// # Returns +/// new first byte of the string, or io::Error InvalidData on invalid prefix code. +fn output_code( + dst: &mut Vec, + code: u16, + prev_code: u16, + codetab: &mut [Codetab], + queue: &CodeQueue, + first_byte: u8, +) -> io::Result { + debug_assert!(code <= MAX_CODE as u16 && code != CONTROL_CODE as u16); + if code <= u8::MAX as u16 { + dst.push(code as u8); + return Ok(code as u8); + } + if codetab[code as usize].prefix_code.is_none() + || codetab[code as usize].prefix_code == Some(code) + { + // Reject invalid codes. Self-referential codes may exist in + // the table but cannot be used. + return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid code")); + } + if codetab[code as usize].len != UNKNOWN_LEN { + // Output string with known length (the common case). + let ct = &codetab[code as usize]; + for i in ct.last_dst_pos..ct.last_dst_pos + ct.len as usize { + dst.push(dst[i]); + } + return Ok(dst[ct.last_dst_pos]); + } + + // Output a string of unknown length. This happens when the prefix + // was invalid (due to partial clearing) when the code was inserted into + // the table. The prefix can then become valid when it's added to the + // table at a later point. + let prefix_code = codetab[code as usize].prefix_code.unwrap(); + if cfg!(debug_assertions) { + let tab_entry = codetab[code as usize]; + assert!(tab_entry.len == UNKNOWN_LEN); + assert!(tab_entry.prefix_code.unwrap() as usize > CONTROL_CODE); + } + + if Some(prefix_code) == queue.next() { + /* The prefix code hasn't been added yet, but we were just + about to: the KwKwK case. Add the previous string extended + with its first byte. */ + codetab[prefix_code as usize] = Codetab { + prefix_code: Some(prev_code), + ext_byte: first_byte, + len: codetab[prev_code as usize].len + 1, + last_dst_pos: codetab[prev_code as usize].last_dst_pos, + }; + dst.push(first_byte); + } else if codetab[prefix_code as usize].prefix_code.is_none() { + // The prefix code is still invalid. + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "invalid prefix code", + )); + } + + // Output the prefix string, then the extension byte. + let ct = codetab[prefix_code as usize]; + let last_dst_pos = dst.len(); + let start = ct.last_dst_pos; + let len = ct.len as usize; + let first = dst[start]; + dst.extend_from_within(start..start + len); + dst.push(codetab[code as usize].ext_byte); + // Update the code table now that the string has a length and pos. + debug_assert!(prev_code != code); + + codetab[code as usize].len = (len + 1) as u16; + codetab[code as usize].last_dst_pos = last_dst_pos; + + Ok(first) +} + +fn hwunshrink(src: &[u8], uncompressed_size: usize, dst: &mut Vec) -> io::Result<()> { + dst.reserve(uncompressed_size); + let mut codetab = Codetab::create_new(); + let mut queue = CodeQueue::new(); + let mut is = BitReader::endian(src, LittleEndian); + let mut code_size = MIN_CODE_SIZE; + + // Handle the first code separately since there is no previous code. + let Some(curr_code) = read_code(&mut is, &mut code_size, &mut codetab, &mut queue)? else { + return Ok(()); + }; + + debug_assert!(curr_code != CONTROL_CODE as u16); + if curr_code > u8::MAX as u16 { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "the first code must be a literal", + )); + } + let mut first_byte = curr_code as u8; // make mutable + codetab[curr_code as usize].last_dst_pos = dst.len(); + dst.push(first_byte); + let mut prev_code = curr_code; + while dst.len() < uncompressed_size { + let curr_opt = match read_code(&mut is, &mut code_size, &mut codetab, &mut queue) { + Ok(c) => c, + Err(_) => break, // treat read error as end-of-stream for now; validated after loop + }; + + let Some(curr_code) = curr_opt else { + return Err(Error::new(io::ErrorKind::InvalidData, "invalid code")); + }; + + let dst_pos = dst.len(); + // Handle KwKwK: next code used before being added. + if Some(curr_code) == queue.next() { + if codetab[prev_code as usize].prefix_code.is_none() { + return Err(Error::new( + io::ErrorKind::InvalidData, + "previous code no longer valid", + )); + } + // Extend the previous code with its first byte. + debug_assert!(curr_code != prev_code); + codetab[curr_code as usize] = Codetab { + prefix_code: Some(prev_code), + ext_byte: first_byte, + len: codetab[prev_code as usize].len + 1, + last_dst_pos: codetab[prev_code as usize].last_dst_pos, + }; + } + // Output the string represented by the current code. + first_byte = output_code(dst, curr_code, prev_code, &mut codetab, &queue, first_byte)?; + if let Some(new_code) = queue.remove_next() { + let prev_entry = codetab[prev_code as usize]; + codetab[new_code as usize] = Codetab { + prefix_code: Some(prev_code), + ext_byte: first_byte, + last_dst_pos: prev_entry.last_dst_pos, + len: if prev_entry.prefix_code.is_none() { + // prev_code was invalidated in a partial + // clearing. Until that code is re-used, the + // string represented by new_code is + // indeterminate. + UNKNOWN_LEN + } else { + prev_entry.len + 1 + }, + }; + } + codetab[curr_code as usize].last_dst_pos = dst_pos; + prev_code = curr_code; + } + + if dst.len() != uncompressed_size { + return Err(io::Error::new( + io::ErrorKind::UnexpectedEof, + "unexpected end of compressed stream", + )); + } + + Ok(()) +} + +#[derive(Debug)] +pub struct ShrinkDecoder { + compressed_reader: R, + stream_read: bool, + uncompressed_size: u64, + stream: Vec, + read_pos: usize, +} + +impl ShrinkDecoder { + pub fn new(inner: R, uncompressed_size: u64) -> Self { + Self { + compressed_reader: inner, + uncompressed_size, + stream_read: false, + stream: Vec::new(), + read_pos: 0, + } + } + pub fn into_inner(self) -> R { + self.compressed_reader + } +} + +impl Read for ShrinkDecoder { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + if !self.stream_read { + self.stream_read = true; + let mut compressed_bytes = Vec::new(); + self.compressed_reader.read_to_end(&mut compressed_bytes)?; + self.stream.reserve(self.uncompressed_size as usize); + hwunshrink( + &compressed_bytes, + self.uncompressed_size as usize, + &mut self.stream, + )?; + } + let available = self.stream.len().saturating_sub(self.read_pos); + if available == 0 { + return Ok(0); + } + let n = available.min(buf.len()); + buf[..n].copy_from_slice(&self.stream[self.read_pos..self.read_pos + n]); + self.read_pos += n; + Ok(n) + } +} + +#[cfg(test)] +mod tests { + use crate::legacy::shrink::hwunshrink; + + const LZW_FIG5: &[u8; 17] = b"ababcbababaaaaaaa"; + const LZW_FIG5_SHRUNK: [u8; 12] = [ + 0x61, 0xc4, 0x04, 0x1c, 0x23, 0xb0, 0x60, 0x98, 0x83, 0x08, 0xc3, 0x00, + ]; + + #[test] + fn test_unshrink_lzw_fig5() { + let mut dst = Vec::with_capacity(LZW_FIG5.len()); + hwunshrink(&LZW_FIG5_SHRUNK, LZW_FIG5.len(), &mut dst).unwrap(); + assert_eq!(dst.as_slice(), LZW_FIG5); + } +} diff --git a/v2/vendor/security/zip-7.2.0/src/lib.rs b/v2/vendor/security/zip-7.2.0/src/lib.rs new file mode 100644 index 0000000000..adc69dc1cb --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/src/lib.rs @@ -0,0 +1,76 @@ +//! A library for reading and writing ZIP archives. +//! ZIP is a format designed for cross-platform file "archiving". +//! That is, storing a collection of files in a single datastream +//! to make them easier to share between computers. +//! Additionally, ZIP is able to compress and encrypt files in its +//! archives. +//! +//! The current implementation is based on [PKWARE's APPNOTE.TXT v6.3.9](https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT) +//! +//! --- +//! +//! [`zip`](`crate`) has support for the most common ZIP archives found in common use. +//! However, in special cases, +//! there are some zip archives that are difficult to read or write. +//! +//! This is a list of supported features: +//! +//! | | Reading | Writing | +//! | ------------------------------- | ------------------------------------- | -------------------------------------------------------- | +//! | Stored | ✅ | ✅ | +//! | Deflate | ✅ [->](`crate::ZipArchive::by_name`) | ✅ [->](`crate::write::FileOptions::compression_method`) | +//! | Deflate64 | ✅ | | +//! | Bzip2 | ✅ | ✅ | +//! | ZStandard | ✅ | ✅ | +//! | LZMA | ✅ | | +//! | XZ | ✅ | ✅ | +//! | PPMd | ✅ | ✅ | +//! | AES encryption | ✅ | ✅ | +//! | ZipCrypto deprecated encryption | ✅ | ✅ | +//! + +#![cfg_attr(docsrs, feature(doc_cfg))] +#![warn(missing_docs)] +#![allow(unexpected_cfgs)] // Needed for cfg(fuzzing) +pub use crate::compression::{CompressionMethod, SUPPORTED_COMPRESSION_METHODS}; +pub use crate::read::HasZipMetadata; +pub use crate::read::{ZipArchive, ZipReadOptions}; +pub use crate::spec::{ZIP64_BYTES_THR, ZIP64_ENTRY_THR}; +pub use crate::types::{AesMode, DateTime}; +pub use crate::write::ZipWriter; + +#[cfg(feature = "aes-crypto")] +mod aes; +#[cfg(feature = "aes-crypto")] +mod aes_ctr; +mod compression; +mod cp437; +mod crc32; +pub mod extra_fields; +mod path; +pub mod read; +pub mod result; +mod spec; +mod types; +pub mod write; +mod zipcrypto; +pub use extra_fields::ExtraField; +#[cfg(feature = "legacy-zip")] +mod legacy; + +#[doc = "Unstable APIs\n\ +\ +All APIs accessible by importing this module are unstable; They may be changed in patch \ +releases. You MUST use an exact version specifier in `Cargo.toml`, to indicate the version of this \ +API you're using:\n\ +\ +```toml\n +[dependencies]\n +zip = \"="] +#[doc=env!("CARGO_PKG_VERSION")] +#[doc = "\"\n\ +```"] +pub mod unstable; + +#[doc(hidden)] +pub mod macros; diff --git a/v2/vendor/security/zip-7.2.0/src/macros.rs b/v2/vendor/security/zip-7.2.0/src/macros.rs new file mode 100644 index 0000000000..3d07efd7b8 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/src/macros.rs @@ -0,0 +1,162 @@ +//! Macros used internally. +//! +//! These may technically be exported, but that's only to make them available to internal +//! project dependencies. The `#[doc(hidden)]` mark indicates that these are not stable or supported +//! APIs, and should not be relied upon by external dependees. + +/// The single macro export of the [`cfg-if`](https://docs.rs/cfg-if) crate. +/// +/// It is packaged here to avoid pulling in another dependency. The stdlib does the same[^1]. +/// +/// [^1]: https://github.com/rust-lang/rust/blob/a2db9280539229a3b8a084a09886670a57bc7e9c/library/compiler-builtins/libm/src/math/support/macros.rs#L1 +#[doc(hidden)] +#[macro_export] +macro_rules! cfg_if { + // match if/else chains with a final `else` + ( + $( + if #[cfg( $i_meta:meta )] { $( $i_tokens:tt )* } + ) else+ + else { $( $e_tokens:tt )* } + ) => { + $crate::cfg_if! { + @__items () ; + $( + (( $i_meta ) ( $( $i_tokens )* )) , + )+ + (() ( $( $e_tokens )* )) , + }; + }; + + // match if/else chains lacking a final `else` + ( + if #[cfg( $i_meta:meta )] { $( $i_tokens:tt )* } + $( + else if #[cfg( $e_meta:meta )] { $( $e_tokens:tt )* } + )* + ) => { + $crate::cfg_if! { + @__items () ; + (( $i_meta ) ( $( $i_tokens )* )) , + $( + (( $e_meta ) ( $( $e_tokens )* )) , + )* + }; + }; + + // Internal and recursive macro to emit all the items + // + // Collects all the previous cfgs in a list at the beginning, so they can be + // negated. After the semicolon are all the remaining items. + (@__items ( $( $_:meta , )* ) ; ) => {}; + ( + @__items ( $( $no:meta , )* ) ; + (( $( $yes:meta )? ) ( $( $tokens:tt )* )) , + $( $rest:tt , )* + ) => { + // Emit all items within one block, applying an appropriate #[cfg]. The + // #[cfg] will require all `$yes` matchers specified and must also negate + // all previous matchers. + #[cfg(all( + $( $yes , )? + not(any( $( $no ),* )) + ))] + $crate::cfg_if! { @__identity $( $tokens )* } + + // Recurse to emit all other items in `$rest`, and when we do so add all + // our `$yes` matchers to the list of `$no` matchers as future emissions + // will have to negate everything we just matched as well. + $crate::cfg_if! { + @__items ( $( $no , )* $( $yes , )? ) ; + $( $rest , )* + }; + }; + + // Internal macro to make __apply work out right for different match types, + // because of how macros match/expand stuff. + (@__identity $( $tokens:tt )* ) => { + $( $tokens )* + }; +} + +/// Similar to [`cfg_if`](cfg_if), but accepts a list of expressions, and generates an internal +/// closure to return each value. +/// +/// The main reason this is necessary is because attaching `#[cfg(...)]` annotations to certain +/// types of statements requires a nightly feature, or `cfg_if` would be enough on its own. This +/// macro's restricted interface allows it to generate a closure as a circumlocution that is legal +/// on stable rust. +/// +/// Note that any `return` operation within the expressions provided to this macro will apply to the +/// generated closure, not the enclosing scope--it cannot be used to interfere with external +/// control flow. +/// +/// The generated closure is non-[`const`](const@keyword), so cannot be used inside `const` methods. +#[doc(hidden)] +#[macro_export] +macro_rules! cfg_if_expr { + // Match =>, chains, maybe with a final _ => catchall clause. + ( + $( $ret_ty:ty : )? + $( + #[cfg( $i_meta:meta )] => $i_val:expr + ),+ , + _ => $rem_val:expr $(,)? + ) => { + (|| $( -> $ret_ty )? { + $crate::cfg_if_expr! { + @__items (); + $( + (( $i_meta ) ( + #[allow(unreachable_code)] + return $i_val ; + )) , + )+ + (() ( + #[allow(unreachable_code)] + return $rem_val ; + )) , + } + })() + }; + // Match =>, chains *without* any _ => clause. + ( + $( $ret_ty:ty : )? + $( + #[cfg( $i_meta:meta )] => $i_val:expr + ),+ $(,)? + ) => { + (|| $( -> $ret_ty )? { + $crate::cfg_if_expr! { + @__items (); + $( + (( $i_meta ) ( + #[allow(unreachable_code)] + return $i_val ; + )) , + )+ + } + })() + }; + + (@__items ( $( $_:meta , )* ) ; ) => {}; + ( + @__items ( $( $no:meta , )* ) ; + (( $( $yes:meta )? ) ( $( $tokens:tt )* )) , + $( $rest:tt , )* + ) => { + #[cfg(all( + $( $yes , )? + not(any( $( $no ),* )) + ))] + $crate::cfg_if_expr! { @__identity $( $tokens )* } + + $crate::cfg_if_expr! { + @__items ( $( $no , )* $( $yes , )? ) ; + $( $rest , )* + }; + }; + (@__identity $( $tokens:tt )* ) => { + $( $tokens )* + }; +} diff --git a/v2/vendor/security/zip-7.2.0/src/path.rs b/v2/vendor/security/zip-7.2.0/src/path.rs new file mode 100644 index 0000000000..8f0c698a03 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/src/path.rs @@ -0,0 +1,110 @@ +//! Path manipulation utilities + +use std::{ffi::OsStr, path::Path}; +use typed_path::{Utf8WindowsComponent, Utf8WindowsPath}; + +/// Simplify a path by removing the prefix and parent directories and only return normal components +pub(crate) fn simplified_components(input: &Path) -> Option> { + let mut out = Vec::new(); + let input_str = input.to_str()?; + for component in Utf8WindowsPath::new(input_str).components() { + match component { + // Skip prefix and root directory components instead of rejecting the entire path + // This allows extraction of ZIP files with absolute paths, similar to other ZIP tools + Utf8WindowsComponent::Prefix(_) | Utf8WindowsComponent::RootDir => (), + Utf8WindowsComponent::ParentDir => { + out.pop()?; + } + Utf8WindowsComponent::Normal(s) => out.push(OsStr::new(s)), + Utf8WindowsComponent::CurDir => (), + } + } + Some(out) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::path::Path; + + #[test] + fn test_simplified_components_relative_path() { + let path = Path::new("foo/bar/baz.txt"); + let components = simplified_components(path).unwrap(); + assert_eq!(components.len(), 3); + assert_eq!(components[0], "foo"); + assert_eq!(components[1], "bar"); + assert_eq!(components[2], "baz.txt"); + } + + #[test] + fn test_simplified_components_absolute_unix_path() { + let path = Path::new("/foo/bar/baz.txt"); + let components = simplified_components(path).unwrap(); + assert_eq!(components.len(), 3); + assert_eq!(components[0], "foo"); + assert_eq!(components[1], "bar"); + assert_eq!(components[2], "baz.txt"); + } + + #[test] + fn test_simplified_components_with_parent_dirs() { + let path = Path::new("foo/../bar/baz.txt"); + let components = simplified_components(path).unwrap(); + assert_eq!(components.len(), 2); + assert_eq!(components[0], "bar"); + assert_eq!(components[1], "baz.txt"); + } + + #[test] + fn test_simplified_components_too_many_parent_dirs() { + let path = Path::new("foo/../../bar"); + let result = simplified_components(path); + assert!(result.is_none()); // Should still fail for directory traversal attacks + } + + #[test] + fn test_simplified_components_with_current_dir() { + let path = Path::new("foo/./bar/baz.txt"); + let components = simplified_components(path).unwrap(); + assert_eq!(components.len(), 3); + assert_eq!(components[0], "foo"); + assert_eq!(components[1], "bar"); + assert_eq!(components[2], "baz.txt"); + } + + #[test] + fn test_simplified_components_empty_path() { + let path = Path::new(""); + let components = simplified_components(path).unwrap(); + assert_eq!(components.len(), 0); + } + + #[test] + fn test_simplified_components_root_only() { + let path = Path::new("/"); + let components = simplified_components(path).unwrap(); + assert_eq!(components.len(), 0); + } + + #[cfg(windows)] + #[test] + fn test_simplified_components_windows_absolute_path() { + let path = Path::new(r"C:\foo\bar\baz.txt"); + let components = simplified_components(path).unwrap(); + assert_eq!(components.len(), 3); + assert_eq!(components[0], "foo"); + assert_eq!(components[1], "bar"); + assert_eq!(components[2], "baz.txt"); + } + + #[cfg(windows)] + #[test] + fn test_simplified_components_windows_unc_path() { + let path = Path::new(r"\\server\share\foo\bar.txt"); + let components = simplified_components(path).unwrap(); + assert_eq!(components.len(), 2); + assert_eq!(components[0], "foo"); + assert_eq!(components[1], "bar.txt"); + } +} diff --git a/v2/vendor/security/zip-7.2.0/src/read.rs b/v2/vendor/security/zip-7.2.0/src/read.rs new file mode 100644 index 0000000000..be430435c0 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/src/read.rs @@ -0,0 +1,2650 @@ +//! Types for reading ZIP archives + +/// Upper bound on the uncompressed size of a symlink entry. +/// +/// A symlink target is a filesystem path, so its length is bounded by the +/// platform (`PATH_MAX` is 4096 on Linux) no matter what the archive says. The +/// uncompressed size stored in the central directory is attacker-controlled, +/// so entries above this bound are rejected rather than pre-allocated. +pub(crate) const MAX_SYMLINK_TARGET_LEN: u64 = 4096; + +#[cfg(feature = "aes-crypto")] +use crate::aes::{AesReader, AesReaderValid}; +use crate::cfg_if; +use crate::compression::{CompressionMethod, Decompressor}; +use crate::cp437::FromCp437; +use crate::crc32::Crc32Reader; +use crate::extra_fields::{ExtendedTimestamp, ExtraField, Ntfs}; +use crate::result::{invalid, ZipError, ZipResult}; +use crate::spec::{ + self, CentralDirectoryEndInfo, DataAndPosition, FixedSizeBlock, Pod, ZIP64_BYTES_THR, +}; +use crate::types::{ + AesMode, AesVendorVersion, DateTime, SimpleFileOptions, System, ZipCentralEntryBlock, + ZipFileData, ZipLocalEntryBlock, +}; +use crate::zipcrypto::{ZipCryptoReader, ZipCryptoReaderValid, ZipCryptoValidator}; +use core::mem::{replace, size_of}; +use core::ops::{Deref, Range}; +use indexmap::IndexMap; +use std::borrow::Cow; +use std::ffi::OsStr; +use std::io::{self, copy, sink, Read, Seek, SeekFrom, Write}; +use std::path::{Component, Path, PathBuf}; +use std::sync::{Arc, OnceLock}; + +mod config; + +pub use config::{ArchiveOffset, Config}; + +/// Provides high level API for reading from a stream. +pub(crate) mod stream; + +pub(crate) mod magic_finder; + +/// Immutable metadata about a `ZipArchive`. +#[derive(Debug)] +pub struct ZipArchiveMetadata { + pub(crate) files: IndexMap, ZipFileData>, + pub(crate) offset: u64, + pub(crate) dir_start: u64, + // This isn't yet used anywhere, but it is here for use cases in the future. + #[allow(dead_code)] + pub(crate) config: Config, + pub(crate) comment: Box<[u8]>, + pub(crate) zip64_comment: Option>, +} + +pub(crate) mod zip_archive { + use crate::read::ZipArchiveMetadata; + use indexmap::IndexMap; + use std::sync::Arc; + + #[derive(Debug)] + pub(crate) struct SharedBuilder { + pub(crate) files: Vec, + pub(super) offset: u64, + pub(super) dir_start: u64, + // This isn't yet used anywhere, but it is here for use cases in the future. + #[allow(dead_code)] + pub(super) config: super::Config, + } + + impl SharedBuilder { + pub fn build( + self, + comment: Box<[u8]>, + zip64_comment: Option>, + ) -> ZipArchiveMetadata { + let mut index_map = IndexMap::with_capacity(self.files.len()); + self.files.into_iter().for_each(|file| { + index_map.insert(file.file_name.clone(), file); + }); + ZipArchiveMetadata { + files: index_map, + offset: self.offset, + dir_start: self.dir_start, + config: self.config, + comment, + zip64_comment, + } + } + } + + /// ZIP archive reader + /// + /// At the moment, this type is cheap to clone if this is the case for the + /// reader it uses. However, this is not guaranteed by this crate and it may + /// change in the future. + /// + /// ```no_run + /// use std::io::{Read, Seek}; + /// fn list_zip_contents(reader: impl Read + Seek) -> zip::result::ZipResult<()> { + /// use zip::HasZipMetadata; + /// let mut zip = zip::ZipArchive::new(reader)?; + /// + /// for i in 0..zip.len() { + /// let mut file = zip.by_index(i)?; + /// println!("Filename: {}", file.name()); + /// std::io::copy(&mut file, &mut std::io::stdout())?; + /// } + /// + /// Ok(()) + /// } + /// ``` + #[derive(Clone, Debug)] + pub struct ZipArchive { + pub(super) reader: R, + pub(super) shared: Arc, + } +} + +#[cfg(feature = "aes-crypto")] +use crate::aes::PWD_VERIFY_LENGTH; +use crate::extra_fields::UnicodeExtraField; +use crate::result::ZipError::InvalidPassword; +use crate::spec::is_dir; +use crate::types::ffi::{S_IFLNK, S_IFREG}; +use crate::unstable::{path_to_string, LittleEndianReadExt}; +pub use zip_archive::ZipArchive; + +#[allow(clippy::large_enum_variant)] +pub(crate) enum CryptoReader<'a, R: Read + ?Sized> { + Plaintext(io::Take<&'a mut R>), + ZipCrypto(ZipCryptoReaderValid>), + #[cfg(feature = "aes-crypto")] + Aes { + reader: AesReaderValid>, + vendor_version: AesVendorVersion, + }, +} + +impl Read for CryptoReader<'_, R> { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + match self { + CryptoReader::Plaintext(r) => r.read(buf), + CryptoReader::ZipCrypto(r) => r.read(buf), + #[cfg(feature = "aes-crypto")] + CryptoReader::Aes { reader: r, .. } => r.read(buf), + } + } + + fn read_to_end(&mut self, buf: &mut Vec) -> io::Result { + match self { + CryptoReader::Plaintext(r) => r.read_to_end(buf), + CryptoReader::ZipCrypto(r) => r.read_to_end(buf), + #[cfg(feature = "aes-crypto")] + CryptoReader::Aes { reader: r, .. } => r.read_to_end(buf), + } + } + + fn read_to_string(&mut self, buf: &mut String) -> io::Result { + match self { + CryptoReader::Plaintext(r) => r.read_to_string(buf), + CryptoReader::ZipCrypto(r) => r.read_to_string(buf), + #[cfg(feature = "aes-crypto")] + CryptoReader::Aes { reader: r, .. } => r.read_to_string(buf), + } + } +} + +impl<'a, R: Read + ?Sized> CryptoReader<'a, R> { + /// Consumes this decoder, returning the underlying reader. + pub fn into_inner(self) -> io::Take<&'a mut R> { + match self { + CryptoReader::Plaintext(r) => r, + CryptoReader::ZipCrypto(r) => r.into_inner(), + #[cfg(feature = "aes-crypto")] + CryptoReader::Aes { reader: r, .. } => r.into_inner(), + } + } + + /// Returns `true` if the data is encrypted using AE2. + #[allow(clippy::needless_return)] // can't use cfg_if_expr! because const + pub const fn is_ae2_encrypted(&self) -> bool { + cfg_if! { + if #[cfg(feature = "aes-crypto")] { + return matches!( + self, + CryptoReader::Aes { + vendor_version: AesVendorVersion::Ae2, + .. + } + ); + } else { + return false; + } + } + } +} + +#[cold] +fn invalid_state() -> io::Result { + Err(io::Error::other("ZipFileReader was in an invalid state")) +} + +pub(crate) enum ZipFileReader<'a, R: Read + ?Sized> { + NoReader, + Raw(io::Take<&'a mut R>), + Compressed(Box>>>>), +} + +impl Read for ZipFileReader<'_, R> { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + match self { + ZipFileReader::NoReader => invalid_state(), + ZipFileReader::Raw(r) => r.read(buf), + ZipFileReader::Compressed(r) => r.read(buf), + } + } + + fn read_exact(&mut self, buf: &mut [u8]) -> io::Result<()> { + match self { + ZipFileReader::NoReader => invalid_state(), + ZipFileReader::Raw(r) => r.read_exact(buf), + ZipFileReader::Compressed(r) => r.read_exact(buf), + } + } + + fn read_to_end(&mut self, buf: &mut Vec) -> io::Result { + match self { + ZipFileReader::NoReader => invalid_state(), + ZipFileReader::Raw(r) => r.read_to_end(buf), + ZipFileReader::Compressed(r) => r.read_to_end(buf), + } + } + + fn read_to_string(&mut self, buf: &mut String) -> io::Result { + match self { + ZipFileReader::NoReader => invalid_state(), + ZipFileReader::Raw(r) => r.read_to_string(buf), + ZipFileReader::Compressed(r) => r.read_to_string(buf), + } + } +} + +impl<'a, R: Read + ?Sized> ZipFileReader<'a, R> { + fn into_inner(self) -> io::Result> { + match self { + ZipFileReader::NoReader => invalid_state(), + ZipFileReader::Raw(r) => Ok(r), + ZipFileReader::Compressed(r) => { + Ok(r.into_inner().into_inner()?.into_inner().into_inner()) + } + } + } +} + +/// A struct for reading a zip file +pub struct ZipFile<'a, R: Read + ?Sized> { + pub(crate) data: Cow<'a, ZipFileData>, + pub(crate) reader: ZipFileReader<'a, R>, +} + +/// A struct for reading and seeking a zip file +pub struct ZipFileSeek<'a, R> { + data: Cow<'a, ZipFileData>, + reader: ZipFileSeekReader<'a, R>, +} + +enum ZipFileSeekReader<'a, R: ?Sized> { + Raw(SeekableTake<'a, R>), +} + +struct SeekableTake<'a, R: ?Sized> { + inner: &'a mut R, + inner_starting_offset: u64, + length: u64, + current_offset: u64, +} + +impl<'a, R: Seek + ?Sized> SeekableTake<'a, R> { + pub fn new(inner: &'a mut R, length: u64) -> io::Result { + let inner_starting_offset = inner.stream_position()?; + Ok(Self { + inner, + inner_starting_offset, + length, + current_offset: 0, + }) + } +} + +impl Seek for SeekableTake<'_, R> { + fn seek(&mut self, pos: SeekFrom) -> io::Result { + let offset = match pos { + SeekFrom::Start(offset) => Some(offset), + SeekFrom::End(offset) => self.length.checked_add_signed(offset), + SeekFrom::Current(offset) => self.current_offset.checked_add_signed(offset), + }; + match offset { + None => Err(io::Error::new( + io::ErrorKind::InvalidInput, + "invalid seek to a negative or overflowing position", + )), + Some(offset) => { + let clamped_offset = std::cmp::min(self.length, offset); + let new_inner_offset = self + .inner + .seek(SeekFrom::Start(self.inner_starting_offset + clamped_offset))?; + self.current_offset = new_inner_offset - self.inner_starting_offset; + Ok(self.current_offset) + } + } + } +} + +impl Read for SeekableTake<'_, R> { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + let written = self + .inner + .take(self.length - self.current_offset) + .read(buf)?; + self.current_offset += written as u64; + Ok(written) + } +} + +pub(crate) fn make_writable_dir_all>(outpath: T) -> Result<(), ZipError> { + use std::fs; + fs::create_dir_all(outpath.as_ref())?; + #[cfg(unix)] + { + // Dirs must be writable until all normal files are extracted + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions( + outpath.as_ref(), + std::fs::Permissions::from_mode( + 0o700 | std::fs::metadata(outpath.as_ref())?.permissions().mode(), + ), + )?; + } + Ok(()) +} + +pub(crate) fn find_content<'a, R: Read + Seek + ?Sized>( + data: &ZipFileData, + reader: &'a mut R, +) -> ZipResult> { + // TODO: use .get_or_try_init() once stabilized to provide a closure returning a Result! + let data_start = data.data_start(reader)?; + + reader.seek(SeekFrom::Start(data_start))?; + Ok(reader.take(data.compressed_size)) +} + +fn find_content_seek<'a, R: Read + Seek + ?Sized>( + data: &ZipFileData, + reader: &'a mut R, +) -> ZipResult> { + // Parse local header + let data_start = data.data_start(reader)?; + reader.seek(SeekFrom::Start(data_start))?; + + // Explicit Ok and ? are needed to convert io::Error to ZipError + Ok(SeekableTake::new(reader, data.compressed_size)?) +} + +pub(crate) fn find_data_start( + data: &ZipFileData, + reader: &mut (impl Read + Seek + ?Sized), +) -> Result { + // Go to start of data. + reader.seek(SeekFrom::Start(data.header_start))?; + + // Parse static-sized fields and check the magic value. + let block = ZipLocalEntryBlock::parse(reader)?; + + // Calculate the end of the local header from the fields we just parsed. + let variable_fields_len = + // Each of these fields must be converted to u64 before adding, as the result may + // easily overflow a u16. + block.file_name_length as u64 + block.extra_field_length as u64; + let data_start = + data.header_start + size_of::() as u64 + variable_fields_len; + + // Set the value so we don't have to read it again. + match data.data_start.set(data_start) { + Ok(()) => (), + // If the value was already set in the meantime, ensure it matches (this is probably + // unnecessary). + Err(_) => { + debug_assert_eq!(*data.data_start.get().unwrap(), data_start); + } + } + + Ok(data_start) +} + +#[allow(clippy::too_many_arguments)] +pub(crate) fn make_crypto_reader<'a, R: Read + ?Sized>( + data: &ZipFileData, + reader: io::Take<&'a mut R>, + password: Option<&[u8]>, + aes_info: Option<(AesMode, AesVendorVersion, CompressionMethod)>, +) -> ZipResult> { + #[allow(deprecated)] + { + if let CompressionMethod::Unsupported(_) = data.compression_method { + return unsupported_zip_error("Compression method not supported"); + } + } + + let reader = match (password, aes_info) { + #[cfg(not(feature = "aes-crypto"))] + (Some(_), Some(_)) => { + return Err(ZipError::UnsupportedArchive( + "AES encrypted files cannot be decrypted without the aes-crypto feature.", + )) + } + #[cfg(feature = "aes-crypto")] + (Some(password), Some((aes_mode, vendor_version, _))) => CryptoReader::Aes { + reader: AesReader::new(reader, aes_mode, data.compressed_size).validate(password)?, + vendor_version, + }, + (Some(password), None) => { + let validator = if data.using_data_descriptor { + ZipCryptoValidator::InfoZipMsdosTime( + data.last_modified_time.map_or(0, |x| x.timepart()), + ) + } else { + ZipCryptoValidator::PkzipCrc32(data.crc32) + }; + CryptoReader::ZipCrypto(ZipCryptoReader::new(reader, password).validate(validator)?) + } + (None, Some(_)) => return Err(InvalidPassword), + (None, None) => CryptoReader::Plaintext(reader), + }; + Ok(reader) +} + +pub(crate) fn make_reader<'a, R: Read + ?Sized>( + compression_method: CompressionMethod, + uncompressed_size: u64, + crc32: u32, + reader: CryptoReader<'a, R>, + #[cfg(feature = "legacy-zip")] flags: u16, +) -> ZipResult> { + let ae2_encrypted = reader.is_ae2_encrypted(); + #[cfg(not(feature = "legacy-zip"))] + let flags = 0; + Ok(ZipFileReader::Compressed(Box::new(Crc32Reader::new( + Decompressor::new( + io::BufReader::new(reader), + compression_method, + uncompressed_size, + flags, + )?, + crc32, + ae2_encrypted, + )))) +} + +pub(crate) fn make_symlink( + outpath: &Path, + target: &[u8], + #[cfg_attr(not(windows), allow(unused))] existing_files: &IndexMap, T>, +) -> ZipResult<()> { + #[cfg_attr(not(any(unix, windows)), allow(unused))] + let Ok(target_str) = std::str::from_utf8(target) else { + return Err(invalid!("Invalid UTF-8 as symlink target")); + }; + + cfg_if! { + if #[cfg(unix)] { + std::os::unix::fs::symlink(Path::new(&target_str), outpath)?; + } else if #[cfg(windows)] { + let target = Path::new(OsStr::new(&target_str)); + let target_is_dir_from_archive = + existing_files.contains_key(target_str) && is_dir(target_str); + let target_is_dir = if target_is_dir_from_archive { + true + } else if let Ok(meta) = std::fs::metadata(target) { + meta.is_dir() + } else { + false + }; + if target_is_dir { + std::os::windows::fs::symlink_dir(target, outpath)?; + } else { + std::os::windows::fs::symlink_file(target, outpath)?; + } + } else { + use std::fs::File; + let output = File::create(outpath); + output?.write_all(target)?; + } + } + + Ok(()) +} + +#[derive(Debug)] +pub(crate) struct CentralDirectoryInfo { + pub(crate) archive_offset: u64, + pub(crate) directory_start: u64, + pub(crate) number_of_files: usize, + pub(crate) disk_number: u32, + pub(crate) disk_with_central_directory: u32, +} + +impl<'a> TryFrom<&'a CentralDirectoryEndInfo> for CentralDirectoryInfo { + type Error = ZipError; + + fn try_from(value: &'a CentralDirectoryEndInfo) -> Result { + let (relative_cd_offset, number_of_files, disk_number, disk_with_central_directory) = + match &value.eocd64 { + Some(DataAndPosition { data: eocd64, .. }) => { + if eocd64.number_of_files_on_this_disk > eocd64.number_of_files { + return Err(invalid!("ZIP64 footer indicates more files on this disk than in the whole archive")); + } + ( + eocd64.central_directory_offset, + eocd64.number_of_files as usize, + eocd64.disk_number, + eocd64.disk_with_central_directory, + ) + } + _ => ( + value.eocd.data.central_directory_offset as u64, + value.eocd.data.number_of_files_on_this_disk as usize, + value.eocd.data.disk_number as u32, + value.eocd.data.disk_with_central_directory as u32, + ), + }; + + let directory_start = relative_cd_offset + .checked_add(value.archive_offset) + .ok_or(invalid!("Invalid central directory size or offset"))?; + + Ok(Self { + archive_offset: value.archive_offset, + directory_start, + number_of_files, + disk_number, + disk_with_central_directory, + }) + } +} + +/// Store all entries which specify a numeric "mode" which is familiar to POSIX operating systems. +#[cfg(unix)] +#[derive(Default, Debug)] +struct UnixFileModes { + map: std::collections::BTreeMap, +} + +#[cfg(unix)] +impl UnixFileModes { + #[cfg_attr(not(debug_assertions), allow(unused))] + pub fn add_mode(&mut self, path: PathBuf, mode: u32) { + // We don't print a warning or consider it remotely out of the ordinary to receive two + // separate modes for the same path: just take the later one. + let old_entry = self.map.insert(path, mode); + debug_assert_eq!(old_entry, None); + } + + // Child nodes will be sorted later lexicographically, so reversing the order puts them first. + pub fn all_perms_with_children_first( + self, + ) -> impl IntoIterator { + use std::os::unix::fs::PermissionsExt; + self.map + .into_iter() + .rev() + .map(|(p, m)| (p, std::fs::Permissions::from_mode(m))) + } +} + +impl ZipArchive { + pub(crate) fn from_finalized_writer( + files: IndexMap, ZipFileData>, + comment: Box<[u8]>, + zip64_comment: Option>, + reader: R, + central_start: u64, + ) -> ZipResult { + let initial_offset = match files.first() { + Some((_, file)) => file.header_start, + None => central_start, + }; + let shared = Arc::new(ZipArchiveMetadata { + files, + offset: initial_offset, + dir_start: central_start, + config: Config { + archive_offset: ArchiveOffset::Known(initial_offset), + }, + comment, + zip64_comment, + }); + Ok(Self { reader, shared }) + } + + /// Total size of the files in the archive, if it can be known. Doesn't include directories or + /// metadata. + pub fn decompressed_size(&self) -> Option { + let mut total = 0u128; + for file in self.shared.files.values() { + if file.using_data_descriptor { + return None; + } + total = total.checked_add(file.uncompressed_size as u128)?; + } + Some(total) + } +} + +impl ZipArchive { + pub(crate) fn merge_contents( + &mut self, + mut w: W, + ) -> ZipResult, ZipFileData>> { + if self.shared.files.is_empty() { + return Ok(IndexMap::new()); + } + let mut new_files = self.shared.files.clone(); + /* The first file header will probably start at the beginning of the file, but zip doesn't + * enforce that, and executable zips like PEX files will have a shebang line so will + * definitely be greater than 0. + * + * assert_eq!(0, new_files[0].header_start); // Avoid this. + */ + + let first_new_file_header_start = w.stream_position()?; + + /* Push back file header starts for all entries in the covered files. */ + new_files.values_mut().try_for_each(|f| { + /* This is probably the only really important thing to change. */ + f.header_start = f + .header_start + .checked_add(first_new_file_header_start) + .ok_or(invalid!( + "new header start from merge would have been too large" + ))?; + /* This is only ever used internally to cache metadata lookups (it's not part of the + * zip spec), and 0 is the sentinel value. */ + f.central_header_start = 0; + /* This is an atomic variable so it can be updated from another thread in the + * implementation (which is good!). */ + if let Some(old_data_start) = f.data_start.take() { + let new_data_start = old_data_start + .checked_add(first_new_file_header_start) + .ok_or(invalid!( + "new data start from merge would have been too large" + ))?; + f.data_start.get_or_init(|| new_data_start); + } + Ok::<_, ZipError>(()) + })?; + + /* Rewind to the beginning of the file. + * + * NB: we *could* decide to start copying from new_files[0].header_start instead, which + * would avoid copying over e.g. any pex shebangs or other file contents that start before + * the first zip file entry. However, zip files actually shouldn't care about garbage data + * in *between* real entries, since the central directory header records the correct start + * location of each, and keeping track of that math is more complicated logic that will only + * rarely be used, since most zips that get merged together are likely to be produced + * specifically for that purpose (and therefore are unlikely to have a shebang or other + * preface). Finally, this preserves any data that might actually be useful. + */ + self.reader.rewind()?; + /* Find the end of the file data. */ + let length_to_read = self.shared.dir_start; + /* Produce a Read that reads bytes up until the start of the central directory header. + * This "as &mut dyn Read" trick is used elsewhere to avoid having to clone the underlying + * handle, which it really shouldn't need to anyway. */ + let mut limited_raw = (&mut self.reader as &mut dyn Read).take(length_to_read); + /* Copy over file data from source archive directly. */ + io::copy(&mut limited_raw, &mut w)?; + + /* Return the files we've just written to the data stream. */ + Ok(new_files) + } + + /// Get the directory start offset and number of files. This is done in a + /// separate function to ease the control flow design. + pub(crate) fn get_metadata(config: Config, reader: &mut R) -> ZipResult { + // End of the probed region, initially set to the end of the file + let file_len = reader.seek(io::SeekFrom::End(0))?; + let mut end_exclusive = file_len; + let mut last_err = None; + + loop { + // Find the EOCD and possibly EOCD64 entries and determine the archive offset. + let cde = match spec::find_central_directory( + reader, + config.archive_offset, + end_exclusive, + file_len, + ) { + Ok(cde) => cde, + Err(e) => { + // return the previous error first (if there is) + return Err(last_err.unwrap_or(e)); + } + }; + + // Turn EOCD into internal representation. + match CentralDirectoryInfo::try_from(&cde) + .and_then(|info| Self::read_central_header(info, config, reader)) + { + Ok(shared) => { + return Ok(shared.build( + cde.eocd.data.zip_file_comment, + cde.eocd64.map(|v| v.data.extensible_data_sector), + )); + } + Err(e) => { + last_err = Some(e); + } + }; + // Something went wrong while decoding the cde, try to find a new one + end_exclusive = cde.eocd.position; + continue; + } + } + + fn read_central_header( + dir_info: CentralDirectoryInfo, + config: Config, + reader: &mut R, + ) -> Result { + // If the parsed number of files is greater than the offset then + // something fishy is going on and we shouldn't trust number_of_files. + let file_capacity = if dir_info.number_of_files > dir_info.directory_start as usize { + 0 + } else { + dir_info.number_of_files + }; + + if dir_info.disk_number != dir_info.disk_with_central_directory { + return unsupported_zip_error("Support for multi-disk files is not implemented"); + } + + if file_capacity.saturating_mul(size_of::()) > isize::MAX as usize { + return unsupported_zip_error("Oversized central directory"); + } + + let mut files = Vec::with_capacity(file_capacity); + reader.seek(SeekFrom::Start(dir_info.directory_start))?; + for _ in 0..dir_info.number_of_files { + let file = central_header_to_zip_file(reader, &dir_info)?; + files.push(file); + } + + Ok(zip_archive::SharedBuilder { + files, + offset: dir_info.archive_offset, + dir_start: dir_info.directory_start, + config, + }) + } + + /// Returns the verification value and salt for the AES encryption of the file + /// + /// It fails if the file number is invalid. + /// + /// # Returns + /// + /// - None if the file is not encrypted with AES + #[cfg(feature = "aes-crypto")] + pub fn get_aes_verification_key_and_salt( + &mut self, + file_number: usize, + ) -> ZipResult> { + let (_, data) = self + .shared + .files + .get_index(file_number) + .ok_or(ZipError::FileNotFound)?; + + let limit_reader = find_content(data, &mut self.reader)?; + match data.aes_mode { + None => Ok(None), + Some((aes_mode, _, _)) => { + let (verification_value, salt) = + AesReader::new(limit_reader, aes_mode, data.compressed_size) + .get_verification_value_and_salt()?; + let aes_info = AesInfo { + aes_mode, + verification_value, + salt, + }; + Ok(Some(aes_info)) + } + } + } + + /// Read a ZIP archive, collecting the files it contains. + /// + /// This uses the central directory record of the ZIP file, and ignores local file headers. + /// + /// A default [`Config`] is used. + pub fn new(reader: R) -> ZipResult> { + Self::with_config(Default::default(), reader) + } + + /// Get the metadata associated with the ZIP archive. + /// + /// This can be used with [`Self::unsafe_new_with_metadata`] to create a new reader over the + /// same file without needing to reparse the metadata. + pub fn metadata(&self) -> Arc { + self.shared.clone() + } + + /// Read a ZIP archive using the given `metadata`. + /// + /// This is useful for creating multiple readers over the same file without + /// needing to reparse the metadata. + /// + /// # Safety + /// `unsafe` is used here to indicate that `reader` and `metadata` could + /// potentially be incompatible, and it is left to the user to ensure they are. + /// + /// # Example + /// + /// ```no_run + /// # use std::fs; + /// use rayon::prelude::*; + /// + /// const FILE_NAME: &str = "my_data.zip"; + /// + /// let file = fs::File::open(FILE_NAME).unwrap(); + /// let mut archive = zip::ZipArchive::new(file).unwrap(); + /// + /// let file_names = (0..archive.len()) + /// .into_par_iter() + /// .map_init({ + /// let metadata = archive.metadata().clone(); + /// move || { + /// let file = fs::File::open(FILE_NAME).unwrap(); + /// unsafe { zip::ZipArchive::unsafe_new_with_metadata(file, metadata.clone()) } + /// }}, + /// |archive, i| { + /// let mut file = archive.by_index(i).unwrap(); + /// file.enclosed_name() + /// } + /// ) + /// .filter_map(|name| name) + /// .collect::>(); + /// ``` + pub unsafe fn unsafe_new_with_metadata(reader: R, metadata: Arc) -> Self { + Self { + reader, + shared: metadata, + } + } + + /// Read a ZIP archive providing a read configuration, collecting the files it contains. + /// + /// This uses the central directory record of the ZIP file, and ignores local file headers. + pub fn with_config(config: Config, mut reader: R) -> ZipResult> { + let shared = Self::get_metadata(config, &mut reader)?; + + Ok(ZipArchive { + reader, + shared: shared.into(), + }) + } + + /// Extract a Zip archive into a directory, overwriting files if they + /// already exist. Paths are sanitized with [`ZipFile::enclosed_name`]. Symbolic links are only + /// created and followed if the target is within the destination directory (this is checked + /// conservatively using [`std::fs::canonicalize`]). + /// + /// Extraction is not atomic. If an error is encountered, some of the files + /// may be left on disk. However, on Unix targets, no newly-created directories with part but + /// not all of their contents extracted will be readable, writable or usable as process working + /// directories by any non-root user except you. + /// + /// On Unix and Windows, symbolic links are extracted correctly. On other platforms such as + /// WebAssembly, symbolic links aren't supported, so they're extracted as normal files + /// containing the target path in UTF-8. + pub fn extract>(&mut self, directory: P) -> ZipResult<()> { + self.extract_internal(directory, None:: bool>) + } + + /// Extracts a Zip archive into a directory in the same fashion as + /// [`ZipArchive::extract`], but detects a "root" directory in the archive + /// (a single top-level directory that contains the rest of the archive's + /// entries) and extracts its contents directly. + /// + /// For a sensible default `filter`, you can use [`root_dir_common_filter`]. + /// For a custom `filter`, see [`RootDirFilter`]. + /// + /// See [`ZipArchive::root_dir`] for more information on how the root + /// directory is detected and the meaning of the `filter` parameter. + /// + /// ## Example + /// + /// Imagine a Zip archive with the following structure: + /// + /// ```text + /// root/file1.txt + /// root/file2.txt + /// root/sub/file3.txt + /// root/sub/subsub/file4.txt + /// ``` + /// + /// If the archive is extracted to `foo` using [`ZipArchive::extract`], + /// the resulting directory structure will be: + /// + /// ```text + /// foo/root/file1.txt + /// foo/root/file2.txt + /// foo/root/sub/file3.txt + /// foo/root/sub/subsub/file4.txt + /// ``` + /// + /// If the archive is extracted to `foo` using + /// [`ZipArchive::extract_unwrapped_root_dir`], the resulting directory + /// structure will be: + /// + /// ```text + /// foo/file1.txt + /// foo/file2.txt + /// foo/sub/file3.txt + /// foo/sub/subsub/file4.txt + /// ``` + /// + /// ## Example - No Root Directory + /// + /// Imagine a Zip archive with the following structure: + /// + /// ```text + /// root/file1.txt + /// root/file2.txt + /// root/sub/file3.txt + /// root/sub/subsub/file4.txt + /// other/file5.txt + /// ``` + /// + /// Due to the presence of the `other` directory, + /// [`ZipArchive::extract_unwrapped_root_dir`] will extract this in the same + /// fashion as [`ZipArchive::extract`] as there is now no "root directory." + pub fn extract_unwrapped_root_dir>( + &mut self, + directory: P, + root_dir_filter: impl RootDirFilter, + ) -> ZipResult<()> { + self.extract_internal(directory, Some(root_dir_filter)) + } + + fn extract_internal>( + &mut self, + directory: P, + root_dir_filter: Option, + ) -> ZipResult<()> { + use std::fs; + + fs::create_dir_all(&directory)?; + let directory = directory.as_ref().canonicalize()?; + + let root_dir = root_dir_filter + .and_then(|filter| { + self.root_dir(&filter) + .transpose() + .map(|root_dir| root_dir.map(|root_dir| (root_dir, filter))) + }) + .transpose()?; + + // If we have a root dir, simplify the path components to be more + // appropriate for passing to `safe_prepare_path` + let root_dir = root_dir + .as_ref() + .map(|(root_dir, filter)| { + crate::path::simplified_components(root_dir) + .ok_or_else(|| { + // Should be unreachable + debug_assert!(false, "Invalid root dir path"); + + invalid!("Invalid root dir path") + }) + .map(|root_dir| (root_dir, filter)) + }) + .transpose()?; + + #[cfg(unix)] + let mut files_by_unix_mode = UnixFileModes::default(); + + for i in 0..self.len() { + let mut file = self.by_index(i)?; + + let mut outpath = directory.clone(); + /* TODO: the control flow of this method call and subsequent expectations about the + * values in this loop is extremely difficult to follow. It also appears to + * perform a nested loop upon extracting every single file entry? Why does it + * accept two arguments that point to the same directory path, one mutable? */ + file.safe_prepare_path(directory.as_ref(), &mut outpath, root_dir.as_ref())?; + + #[cfg(any(unix, windows))] + if file.is_symlink() { + let declared_len = file.size(); + if declared_len > MAX_SYMLINK_TARGET_LEN { + return Err(invalid!( + "symlink target declares {} bytes, more than the {} byte limit", + declared_len, + MAX_SYMLINK_TARGET_LEN + )); + } + let mut target = Vec::with_capacity(declared_len as usize); + file.read_to_end(&mut target)?; + drop(file); + make_symlink(&outpath, &target, &self.shared.files)?; + continue; + } else if file.is_dir() { + crate::read::make_writable_dir_all(&outpath)?; + continue; + } + let mut outfile = fs::File::create(&outpath)?; + io::copy(&mut file, &mut outfile)?; + + // Check for real permissions, which we'll set in a second pass. + #[cfg(unix)] + if let Some(mode) = file.unix_mode() { + files_by_unix_mode.add_mode(outpath, mode); + } + + // Set original timestamp. + #[cfg(feature = "chrono")] + if let Some(last_modified) = file.last_modified() { + if let Some(t) = datetime_to_systemtime(&last_modified) { + outfile.set_modified(t)?; + } + } + } + + // Ensure we update children's permissions before making a parent unwritable. + #[cfg(unix)] + for (path, perms) in files_by_unix_mode.all_perms_with_children_first() { + std::fs::set_permissions(path, perms)?; + } + + Ok(()) + } + + /// Number of files contained in this zip. + pub fn len(&self) -> usize { + self.shared.files.len() + } + + /// Get the starting offset of the zip central directory. + pub fn central_directory_start(&self) -> u64 { + self.shared.dir_start + } + + /// Whether this zip archive contains no files + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Get the offset from the beginning of the underlying reader that this zip begins at, in bytes. + /// + /// Normally this value is zero, but if the zip has arbitrary data prepended to it, then this value will be the size + /// of that prepended data. + pub fn offset(&self) -> u64 { + self.shared.offset + } + + /// Get the comment of the zip archive. + pub fn comment(&self) -> &[u8] { + &self.shared.comment + } + + /// Get the ZIP64 comment of the zip archive, if it is ZIP64. + pub fn zip64_comment(&self) -> Option<&[u8]> { + self.shared.zip64_comment.as_deref() + } + + /// Returns an iterator over all the file and directory names in this archive. + pub fn file_names(&self) -> impl Iterator { + self.shared.files.keys().map(|s| s.as_ref()) + } + + /// Returns Ok(true) if any compressed data in this archive belongs to more than one file. This + /// doesn't make the archive invalid, but some programs will refuse to decompress it because the + /// copies would take up space independently in the destination. + pub fn has_overlapping_files(&mut self) -> ZipResult { + let mut ranges = Vec::>::with_capacity(self.shared.files.len()); + for file in self.shared.files.values() { + if file.compressed_size == 0 { + continue; + } + let start = file.data_start(&mut self.reader)?; + let end = start + file.compressed_size; + if ranges + .iter() + .any(|range| range.start <= end && start <= range.end) + { + return Ok(true); + } + ranges.push(start..end); + } + Ok(false) + } + + /// Search for a file entry by name, decrypt with given password + /// + /// # Warning + /// + /// The implementation of the cryptographic algorithms has not + /// gone through a correctness review, and you should assume it is insecure: + /// passwords used with this API may be compromised. + /// + /// This function sometimes accepts wrong password. This is because the ZIP spec only allows us + /// to check for a 1/256 chance that the password is correct. + /// There are many passwords out there that will also pass the validity checks + /// we are able to perform. This is a weakness of the ZipCrypto algorithm, + /// due to its fairly primitive approach to cryptography. + pub fn by_name_decrypt(&mut self, name: &str, password: &[u8]) -> ZipResult> { + self.by_name_with_optional_password(name, Some(password)) + } + + /// Search for a file entry by name + pub fn by_name(&mut self, name: &str) -> ZipResult> { + self.by_name_with_optional_password(name, None) + } + + /// Get the index of a file entry by name, if it's present. + #[inline(always)] + pub fn index_for_name(&self, name: &str) -> Option { + self.shared.files.get_index_of(name) + } + + /// Search for a file entry by path, decrypt with given password + /// + /// # Warning + /// + /// The implementation of the cryptographic algorithms has not + /// gone through a correctness review, and you should assume it is insecure: + /// passwords used with this API may be compromised. + /// + /// This function sometimes accepts wrong password. This is because the ZIP spec only allows us + /// to check for a 1/256 chance that the password is correct. + /// There are many passwords out there that will also pass the validity checks + /// we are able to perform. This is a weakness of the ZipCrypto algorithm, + /// due to its fairly primitive approach to cryptography. + pub fn by_path_decrypt>( + &mut self, + path: T, + password: &[u8], + ) -> ZipResult> { + self.index_for_path(path) + .ok_or(ZipError::FileNotFound) + .and_then(|index| { + self.by_index_with_options(index, ZipReadOptions::new().password(Some(password))) + }) + } + + /// Search for a file entry by path + pub fn by_path>(&mut self, path: T) -> ZipResult> { + self.index_for_path(path) + .ok_or(ZipError::FileNotFound) + .and_then(|index| self.by_index_with_options(index, ZipReadOptions::new())) + } + + /// Get the index of a file entry by path, if it's present. + #[inline(always)] + pub fn index_for_path>(&self, path: T) -> Option { + self.index_for_name(&path_to_string(path)) + } + + /// Get the name of a file entry, if it's present. + #[inline(always)] + pub fn name_for_index(&self, index: usize) -> Option<&str> { + self.shared + .files + .get_index(index) + .map(|(name, _)| name.as_ref()) + } + + /// Search for a file entry by name and return a seekable object. + pub fn by_name_seek(&mut self, name: &str) -> ZipResult> { + self.by_index_seek(self.index_for_name(name).ok_or(ZipError::FileNotFound)?) + } + + /// Search for a file entry by index and return a seekable object. + pub fn by_index_seek(&mut self, index: usize) -> ZipResult> { + let reader = &mut self.reader; + self.shared + .files + .get_index(index) + .ok_or(ZipError::FileNotFound) + .and_then(move |(_, data)| { + let seek_reader = match data.compression_method { + CompressionMethod::Stored => { + ZipFileSeekReader::Raw(find_content_seek(data, reader)?) + } + _ => { + return Err(ZipError::UnsupportedArchive( + "Seekable compressed files are not yet supported", + )) + } + }; + Ok(ZipFileSeek { + reader: seek_reader, + data: Cow::Borrowed(data), + }) + }) + } + + fn by_name_with_optional_password<'a>( + &'a mut self, + name: &str, + password: Option<&[u8]>, + ) -> ZipResult> { + let Some(index) = self.shared.files.get_index_of(name) else { + return Err(ZipError::FileNotFound); + }; + self.by_index_with_options(index, ZipReadOptions::new().password(password)) + } + + /// Get a contained file by index, decrypt with given password + /// + /// # Warning + /// + /// The implementation of the cryptographic algorithms has not + /// gone through a correctness review, and you should assume it is insecure: + /// passwords used with this API may be compromised. + /// + /// This function sometimes accepts wrong password. This is because the ZIP spec only allows us + /// to check for a 1/256 chance that the password is correct. + /// There are many passwords out there that will also pass the validity checks + /// we are able to perform. This is a weakness of the ZipCrypto algorithm, + /// due to its fairly primitive approach to cryptography. + pub fn by_index_decrypt( + &mut self, + file_number: usize, + password: &[u8], + ) -> ZipResult> { + self.by_index_with_options(file_number, ZipReadOptions::new().password(Some(password))) + } + + /// Get a contained file by index + pub fn by_index(&mut self, file_number: usize) -> ZipResult> { + self.by_index_with_options(file_number, ZipReadOptions::new()) + } + + /// Get a contained file by index without decompressing it + pub fn by_index_raw(&mut self, file_number: usize) -> ZipResult> { + let reader = &mut self.reader; + let (_, data) = self + .shared + .files + .get_index(file_number) + .ok_or(ZipError::FileNotFound)?; + Ok(ZipFile { + reader: ZipFileReader::Raw(find_content(data, reader)?), + data: Cow::Borrowed(data), + }) + } + + /// Get a contained file by index with options. + pub fn by_index_with_options( + &mut self, + file_number: usize, + mut options: ZipReadOptions<'_>, + ) -> ZipResult> { + let (_, data) = self + .shared + .files + .get_index(file_number) + .ok_or(ZipError::FileNotFound)?; + + if options.ignore_encryption_flag { + // Always use no password when we're ignoring the encryption flag. + options.password = None; + } else { + // Require and use the password only if the file is encrypted. + match (options.password, data.encrypted) { + (None, true) => { + return Err(ZipError::UnsupportedArchive(ZipError::PASSWORD_REQUIRED)) + } + // Password supplied, but none needed! Discard. + (Some(_), false) => options.password = None, + _ => {} + } + } + let limit_reader = find_content(data, &mut self.reader)?; + + let crypto_reader = + make_crypto_reader(data, limit_reader, options.password, data.aes_mode)?; + + Ok(ZipFile { + data: Cow::Borrowed(data), + reader: make_reader( + data.compression_method, + data.uncompressed_size, + data.crc32, + crypto_reader, + #[cfg(feature = "legacy-zip")] + data.flags, + )?, + }) + } + + /// Find the "root directory" of an archive if it exists, filtering out + /// irrelevant entries when searching. + /// + /// Our definition of a "root directory" is a single top-level directory + /// that contains the rest of the archive's entries. This is useful for + /// extracting archives that contain a single top-level directory that + /// you want to "unwrap" and extract directly. + /// + /// For a sensible default filter, you can use [`root_dir_common_filter`]. + /// For a custom filter, see [`RootDirFilter`]. + pub fn root_dir(&self, filter: impl RootDirFilter) -> ZipResult> { + let mut root_dir: Option = None; + + for i in 0..self.len() { + let (_, file) = self + .shared + .files + .get_index(i) + .ok_or(ZipError::FileNotFound)?; + + let path = match file.enclosed_name() { + Some(path) => path, + None => return Ok(None), + }; + + if !filter(&path) { + continue; + } + + macro_rules! replace_root_dir { + ($path:ident) => { + match &mut root_dir { + Some(root_dir) => { + if *root_dir != $path { + // We've found multiple root directories, + // abort. + return Ok(None); + } else { + continue; + } + } + + None => { + root_dir = Some($path.into()); + continue; + } + } + }; + } + + // If this entry is located at the root of the archive... + if path.components().count() == 1 { + if file.is_dir() { + // If it's a directory, it could be the root directory. + replace_root_dir!(path); + } else { + // If it's anything else, this archive does not have a + // root directory. + return Ok(None); + } + } + + // Find the root directory for this entry. + let mut path = path.as_path(); + while let Some(parent) = path.parent().filter(|path| *path != Path::new("")) { + path = parent; + } + + replace_root_dir!(path); + } + + Ok(root_dir) + } + + /// Unwrap and return the inner reader object + /// + /// The position of the reader is undefined. + pub fn into_inner(self) -> R { + self.reader + } +} + +/// Holds the AES information of a file in the zip archive +#[derive(Debug)] +#[cfg(feature = "aes-crypto")] +pub struct AesInfo { + /// The AES encryption mode + pub aes_mode: AesMode, + /// The verification key + pub verification_value: [u8; PWD_VERIFY_LENGTH], + /// The salt + pub salt: Vec, +} + +const fn unsupported_zip_error(detail: &'static str) -> ZipResult { + Err(ZipError::UnsupportedArchive(detail)) +} + +/// Parse a central directory entry to collect the information for the file. +pub(crate) fn central_header_to_zip_file( + reader: &mut R, + central_directory: &CentralDirectoryInfo, +) -> ZipResult { + let central_header_start = reader.stream_position()?; + + // Parse central header + let block = ZipCentralEntryBlock::parse(reader)?; + + let file = central_header_to_zip_file_inner( + reader, + central_directory.archive_offset, + central_header_start, + block, + )?; + + let central_header_end = reader.stream_position()?; + + reader.seek(SeekFrom::Start(central_header_end))?; + Ok(file) +} + +#[inline] +fn read_variable_length_byte_field(reader: &mut R, len: usize) -> ZipResult> { + let mut data = vec![0; len].into_boxed_slice(); + if let Err(e) = reader.read_exact(&mut data) { + if e.kind() == io::ErrorKind::UnexpectedEof { + return Err(invalid!( + "Variable-length field extends beyond file boundary" + )); + } + return Err(e.into()); + } + Ok(data) +} + +/// Parse a central directory entry to collect the information for the file. +fn central_header_to_zip_file_inner( + reader: &mut R, + archive_offset: u64, + central_header_start: u64, + block: ZipCentralEntryBlock, +) -> ZipResult { + let ZipCentralEntryBlock { + // magic, + version_made_by, + // version_to_extract, + flags, + compression_method, + last_mod_time, + last_mod_date, + crc32, + compressed_size, + uncompressed_size, + file_name_length, + extra_field_length, + file_comment_length, + // disk_number, + // internal_file_attributes, + external_file_attributes, + offset, + .. + } = block; + + let encrypted = flags & 1 == 1; + let is_utf8 = flags & (1 << 11) != 0; + let using_data_descriptor = flags & (1 << 3) != 0; + + let file_name_raw = read_variable_length_byte_field(reader, file_name_length as usize)?; + let extra_field = read_variable_length_byte_field(reader, extra_field_length as usize)?; + let file_comment_raw = read_variable_length_byte_field(reader, file_comment_length as usize)?; + let file_name: Box = match is_utf8 { + true => String::from_utf8_lossy(&file_name_raw).into(), + false => file_name_raw.clone().from_cp437(), + }; + let file_comment: Box = match is_utf8 { + true => String::from_utf8_lossy(&file_comment_raw).into(), + false => file_comment_raw.from_cp437(), + }; + + // Construct the result + let mut result = ZipFileData { + system: System::from((version_made_by >> 8) as u8), + /* NB: this strips the top 8 bits! */ + version_made_by: version_made_by as u8, + encrypted, + using_data_descriptor, + is_utf8, + compression_method: CompressionMethod::parse_from_u16(compression_method), + compression_level: None, + last_modified_time: DateTime::try_from_msdos(last_mod_date, last_mod_time).ok(), + crc32, + compressed_size: compressed_size.into(), + uncompressed_size: uncompressed_size.into(), + flags, + file_name, + file_name_raw, + extra_field: Some(Arc::new(extra_field.to_vec())), + central_extra_field: None, + file_comment, + header_start: offset.into(), + extra_data_start: None, + central_header_start, + data_start: OnceLock::new(), + external_attributes: external_file_attributes, + large_file: false, + aes_mode: None, + aes_extra_data_start: 0, + extra_fields: Vec::new(), + }; + parse_extra_field(&mut result)?; + + let aes_enabled = result.compression_method == CompressionMethod::AES; + if aes_enabled && result.aes_mode.is_none() { + return Err(invalid!("AES encryption without AES extra data field")); + } + + // Account for shifted zip offsets. + result.header_start = result + .header_start + .checked_add(archive_offset) + .ok_or(invalid!("Archive header is too large"))?; + + Ok(result) +} + +pub(crate) fn parse_extra_field(file: &mut ZipFileData) -> ZipResult<()> { + let mut extra_field = file.extra_field.clone(); + let mut central_extra_field = file.central_extra_field.clone(); + for field_group in [&mut extra_field, &mut central_extra_field] { + let Some(extra_field) = field_group else { + continue; + }; + let mut modified = false; + let mut processed_extra_field = vec![]; + let len = extra_field.len(); + let mut reader = io::Cursor::new(&**extra_field); + + let mut position = reader.position(); + while position < len as u64 { + let old_position = position; + let remove = parse_single_extra_field(file, &mut reader, position, false)?; + position = reader.position(); + if remove { + modified = true; + } else { + let field_len = (position - old_position) as usize; + let write_start = processed_extra_field.len(); + reader.seek(SeekFrom::Start(old_position))?; + processed_extra_field.extend_from_slice(&vec![0u8; field_len]); + if let Err(e) = reader + .read_exact(&mut processed_extra_field[write_start..(write_start + field_len)]) + { + if e.kind() == io::ErrorKind::UnexpectedEof { + return Err(invalid!("Extra field content exceeds declared length")); + } + return Err(e.into()); + } + } + } + if modified { + *field_group = Some(Arc::new(processed_extra_field)); + } + } + file.extra_field = extra_field; + file.central_extra_field = central_extra_field; + Ok(()) +} + +pub(crate) fn parse_single_extra_field( + file: &mut ZipFileData, + reader: &mut R, + bytes_already_read: u64, + disallow_zip64: bool, +) -> ZipResult { + let kind = match reader.read_u16_le() { + Ok(kind) => kind, + Err(e) if e.kind() == io::ErrorKind::UnexpectedEof => return Ok(false), + Err(e) => return Err(e.into()), + }; + let len = match reader.read_u16_le() { + Ok(len) => len, + Err(e) if e.kind() == io::ErrorKind::UnexpectedEof => { + return Err(invalid!("Extra field header truncated")) + } + Err(e) => return Err(e.into()), + }; + match kind { + // Zip64 extended information extra field + 0x0001 => { + if disallow_zip64 { + return Err(invalid!("Can't write a custom field using the ZIP64 ID")); + } + file.large_file = true; + let mut consumed_len = 0; + if len >= 24 || file.uncompressed_size == spec::ZIP64_BYTES_THR { + file.uncompressed_size = match reader.read_u64_le() { + Ok(v) => v, + Err(e) if e.kind() == io::ErrorKind::UnexpectedEof => { + return Err(invalid!("ZIP64 extra field truncated")) + } + Err(e) => return Err(e.into()), + }; + consumed_len += size_of::(); + } + if len >= 24 || file.compressed_size == spec::ZIP64_BYTES_THR { + file.compressed_size = match reader.read_u64_le() { + Ok(v) => v, + Err(e) if e.kind() == io::ErrorKind::UnexpectedEof => { + return Err(invalid!("ZIP64 extra field truncated")) + } + Err(e) => return Err(e.into()), + }; + consumed_len += size_of::(); + } + if len >= 24 || file.header_start == spec::ZIP64_BYTES_THR { + file.header_start = match reader.read_u64_le() { + Ok(v) => v, + Err(e) if e.kind() == io::ErrorKind::UnexpectedEof => { + return Err(invalid!("ZIP64 extra field truncated")) + } + Err(e) => return Err(e.into()), + }; + consumed_len += size_of::(); + } + let Some(leftover_len) = (len as usize).checked_sub(consumed_len) else { + return Err(invalid!("ZIP64 extra-data field is the wrong length")); + }; + if let Err(e) = reader.read_exact(&mut vec![0u8; leftover_len]) { + if e.kind() == io::ErrorKind::UnexpectedEof { + return Err(invalid!("ZIP64 extra field truncated")); + } + return Err(e.into()); + } + return Ok(true); + } + 0x000a => { + // NTFS extra field + file.extra_fields + .push(ExtraField::Ntfs(Ntfs::try_from_reader(reader, len)?)); + } + 0x9901 => { + // AES + if len != 7 { + return Err(ZipError::UnsupportedArchive( + "AES extra data field has an unsupported length", + )); + } + let vendor_version = reader.read_u16_le()?; + let vendor_id = reader.read_u16_le()?; + let mut out = [0u8]; + if let Err(e) = reader.read_exact(&mut out) { + if e.kind() == io::ErrorKind::UnexpectedEof { + return Err(invalid!("AES extra field truncated")); + } + return Err(e.into()); + } + let aes_mode = out[0]; + let compression_method = CompressionMethod::parse_from_u16(reader.read_u16_le()?); + + if vendor_id != 0x4541 { + return Err(invalid!("Invalid AES vendor")); + } + let vendor_version = match vendor_version { + 0x0001 => AesVendorVersion::Ae1, + 0x0002 => AesVendorVersion::Ae2, + _ => return Err(invalid!("Invalid AES vendor version")), + }; + match aes_mode { + 0x01 => file.aes_mode = Some((AesMode::Aes128, vendor_version, compression_method)), + 0x02 => file.aes_mode = Some((AesMode::Aes192, vendor_version, compression_method)), + 0x03 => file.aes_mode = Some((AesMode::Aes256, vendor_version, compression_method)), + _ => return Err(invalid!("Invalid AES encryption strength")), + }; + file.compression_method = compression_method; + file.aes_extra_data_start = bytes_already_read; + } + 0x5455 => { + // extended timestamp + // https://libzip.org/specifications/extrafld.txt + + file.extra_fields.push(ExtraField::ExtendedTimestamp( + ExtendedTimestamp::try_from_reader(reader, len)?, + )); + } + 0x6375 => { + // Info-ZIP Unicode Comment Extra Field + // APPNOTE 4.6.8 and https://libzip.org/specifications/extrafld.txt + file.file_comment = String::from_utf8( + UnicodeExtraField::try_from_reader(reader, len)? + .unwrap_valid(file.file_comment.as_bytes())? + .into_vec(), + )? + .into(); + } + 0x7075 => { + // Info-ZIP Unicode Path Extra Field + // APPNOTE 4.6.9 and https://libzip.org/specifications/extrafld.txt + file.file_name_raw = UnicodeExtraField::try_from_reader(reader, len)? + .unwrap_valid(&file.file_name_raw)?; + file.file_name = + String::from_utf8(file.file_name_raw.clone().into_vec())?.into_boxed_str(); + file.is_utf8 = true; + } + _ => { + if let Err(e) = reader.read_exact(&mut vec![0u8; len as usize]) { + if e.kind() == io::ErrorKind::UnexpectedEof { + return Err(invalid!("Extra field content truncated")); + } + return Err(e.into()); + } + // Other fields are ignored + } + } + Ok(false) +} + +/// A trait for exposing file metadata inside the zip. +pub trait HasZipMetadata { + /// Get the file metadata + fn get_metadata(&self) -> &ZipFileData; +} + +/// Options for reading a file from an archive. +#[derive(Default)] +pub struct ZipReadOptions<'a> { + /// The password to use when decrypting the file. This is ignored if not required. + password: Option<&'a [u8]>, + + /// Ignore the value of the encryption flag and proceed as if the file were plaintext. + ignore_encryption_flag: bool, +} + +impl<'a> ZipReadOptions<'a> { + /// Create a new set of options with the default values. + #[must_use] + pub fn new() -> Self { + Self::default() + } + + /// Set the password, if any, to use. Return for chaining. + #[must_use] + pub fn password(mut self, password: Option<&'a [u8]>) -> Self { + self.password = password; + self + } + + /// Set the ignore encryption flag. Return for chaining. + #[must_use] + pub fn ignore_encryption_flag(mut self, ignore: bool) -> Self { + self.ignore_encryption_flag = ignore; + self + } +} + +/// Methods for retrieving information on zip files +impl<'a, R: Read + ?Sized> ZipFile<'a, R> { + pub(crate) fn take_raw_reader(&mut self) -> io::Result> { + replace(&mut self.reader, ZipFileReader::NoReader).into_inner() + } + + /// Get the version of the file + pub fn version_made_by(&self) -> (u8, u8) { + ( + self.get_metadata().version_made_by / 10, + self.get_metadata().version_made_by % 10, + ) + } + + /// Get the name of the file + /// + /// # Warnings + /// + /// It is dangerous to use this name directly when extracting an archive. + /// It may contain an absolute path (`/etc/shadow`), or break out of the + /// current directory (`../runtime`). Carelessly writing to these paths + /// allows an attacker to craft a ZIP archive that will overwrite critical + /// files. + /// + /// You can use the [`ZipFile::enclosed_name`] method to validate the name + /// as a safe path. + pub fn name(&self) -> &str { + &self.get_metadata().file_name + } + + /// Get the name of the file, in the raw (internal) byte representation. + /// + /// The encoding of this data is currently undefined. + pub fn name_raw(&self) -> &[u8] { + &self.get_metadata().file_name_raw + } + + /// Get the name of the file in a sanitized form. It truncates the name to the first NULL byte, + /// removes a leading '/' and removes '..' parts. + #[deprecated( + since = "0.5.7", + note = "by stripping `..`s from the path, the meaning of paths can change. + `mangled_name` can be used if this behaviour is desirable" + )] + pub fn sanitized_name(&self) -> PathBuf { + self.mangled_name() + } + + /// Rewrite the path, ignoring any path components with special meaning. + /// + /// - Absolute paths are made relative + /// - [`ParentDir`]s are ignored + /// - Truncates the filename at a NULL byte + /// + /// This is appropriate if you need to be able to extract *something* from + /// any archive, but will easily misrepresent trivial paths like + /// `foo/../bar` as `foo/bar` (instead of `bar`). Because of this, + /// [`ZipFile::enclosed_name`] is the better option in most scenarios. + /// + /// [`ParentDir`]: `Component::ParentDir` + pub fn mangled_name(&self) -> PathBuf { + self.get_metadata().file_name_sanitized() + } + + /// Ensure the file path is safe to use as a [`Path`]. + /// + /// - It can't contain NULL bytes + /// - It can't resolve to a path outside the current directory + /// > `foo/../bar` is fine, `foo/../../bar` is not. + /// - It can't be an absolute path + /// + /// This will read well-formed ZIP files correctly, and is resistant + /// to path-based exploits. It is recommended over + /// [`ZipFile::mangled_name`]. + pub fn enclosed_name(&self) -> Option { + self.get_metadata().enclosed_name() + } + + pub(crate) fn simplified_components(&self) -> Option> { + self.get_metadata().simplified_components() + } + + /// Prepare the path for extraction by creating necessary missing directories and checking for symlinks to be contained within the base path. + /// + /// `base_path` parameter is assumed to be canonicalized. + pub(crate) fn safe_prepare_path( + &self, + base_path: &Path, + outpath: &mut PathBuf, + root_dir: Option<&(Vec<&OsStr>, impl RootDirFilter)>, + ) -> ZipResult<()> { + let components = self + .simplified_components() + .ok_or(invalid!("Invalid file path"))?; + + let components = match root_dir { + Some((root_dir, filter)) => match components.strip_prefix(&**root_dir) { + Some(components) => components, + + // In this case, we expect that the file was not in the root + // directory, but was filtered out when searching for the + // root directory. + None => { + // We could technically find ourselves at this code + // path if the user provides an unstable or + // non-deterministic `filter` function. + // + // If debug assertions are on, we should panic here. + // Otherwise, the safest thing to do here is to just + // extract as-is. + debug_assert!( + !filter(&PathBuf::from_iter(components.iter())), + "Root directory filter should not match at this point" + ); + + // Extract as-is. + &components[..] + } + }, + + None => &components[..], + }; + + let components_len = components.len(); + + for (is_last, component) in components + .iter() + .copied() + .enumerate() + .map(|(i, c)| (i == components_len - 1, c)) + { + // we can skip the target directory itself because the base path is assumed to be "trusted" (if the user say extract to a symlink we can follow it) + outpath.push(component); + + // check if the path is a symlink, the target must be _inherently_ within the directory + for limit in (0..5u8).rev() { + let meta = match std::fs::symlink_metadata(&outpath) { + Ok(meta) => meta, + Err(e) if e.kind() == io::ErrorKind::NotFound => { + if !is_last { + crate::read::make_writable_dir_all(&outpath)?; + } + break; + } + Err(e) => return Err(e.into()), + }; + + if !meta.is_symlink() { + break; + } + + if limit == 0 { + return Err(invalid!("Extraction followed a symlink too deep")); + } + + // note that we cannot accept links that do not inherently resolve to a path inside the directory to prevent: + // - disclosure of unrelated path exists (no check for a path exist and then ../ out) + // - issues with file-system specific path resolution (case sensitivity, etc) + let target = std::fs::read_link(&outpath)?; + + if !crate::path::simplified_components(&target) + .ok_or(invalid!("Invalid symlink target path"))? + .starts_with( + &crate::path::simplified_components(base_path) + .ok_or(invalid!("Invalid base path"))?, + ) + { + let is_absolute_enclosed = base_path + .components() + .map(Some) + .chain(std::iter::once(None)) + .zip(target.components().map(Some).chain(std::iter::repeat(None))) + .all(|(a, b)| match (a, b) { + // both components are normal + (Some(Component::Normal(a)), Some(Component::Normal(b))) => a == b, + // both components consumed fully + (None, None) => true, + // target consumed fully but base path is not + (Some(_), None) => false, + // base path consumed fully but target is not (and normal) + (None, Some(Component::CurDir | Component::Normal(_))) => true, + _ => false, + }); + + if !is_absolute_enclosed { + return Err(invalid!("Symlink is not inherently safe")); + } + } + + outpath.push(target); + } + } + Ok(()) + } + + /// Get the comment of the file + pub fn comment(&self) -> &str { + &self.get_metadata().file_comment + } + + /// Get the compression method used to store the file + pub fn compression(&self) -> CompressionMethod { + self.get_metadata().compression_method + } + + /// Get if the files is encrypted or not + pub fn encrypted(&self) -> bool { + self.data.encrypted + } + + /// Get the size of the file, in bytes, in the archive + pub fn compressed_size(&self) -> u64 { + self.get_metadata().compressed_size + } + + /// Get the size of the file, in bytes, when uncompressed + pub fn size(&self) -> u64 { + self.get_metadata().uncompressed_size + } + + /// Get the time the file was last modified + pub fn last_modified(&self) -> Option { + self.data.last_modified_time + } + /// Returns whether the file is actually a directory + pub fn is_dir(&self) -> bool { + is_dir(self.name()) + } + + /// Returns whether the file is actually a symbolic link + pub fn is_symlink(&self) -> bool { + self.unix_mode() + .is_some_and(|mode| mode & S_IFLNK == S_IFLNK) + } + + /// Returns whether the file is a normal file (i.e. not a directory or symlink) + pub fn is_file(&self) -> bool { + !self.is_dir() && !self.is_symlink() + } + + /// Get unix mode for the file + pub fn unix_mode(&self) -> Option { + self.get_metadata().unix_mode() + } + + /// Get the CRC32 hash of the original file + pub fn crc32(&self) -> u32 { + self.get_metadata().crc32 + } + + /// Get the extra data of the zip header for this file + pub fn extra_data(&self) -> Option<&[u8]> { + self.get_metadata() + .extra_field + .as_ref() + .map(|v| v.deref().deref()) + } + + /// Get the starting offset of the data of the compressed file + pub fn data_start(&self) -> u64 { + *self.data.data_start.get().unwrap() + } + + /// Get the starting offset of the zip header for this file + pub fn header_start(&self) -> u64 { + self.get_metadata().header_start + } + /// Get the starting offset of the zip header in the central directory for this file + pub fn central_header_start(&self) -> u64 { + self.get_metadata().central_header_start + } + + /// Get the [`SimpleFileOptions`] that would be used to write this file to + /// a new zip archive. + pub fn options(&self) -> SimpleFileOptions { + let mut options = SimpleFileOptions::default() + .large_file(self.compressed_size().max(self.size()) > ZIP64_BYTES_THR) + .compression_method(self.compression()) + .unix_permissions(self.unix_mode().unwrap_or(0o644) | S_IFREG) + .last_modified_time( + self.last_modified() + .filter(|m| m.is_valid()) + .unwrap_or_else(DateTime::default_for_write), + ); + + options.normalize(); + #[cfg(feature = "aes-crypto")] + if let Some(aes) = self.get_metadata().aes_mode { + // Preserve AES metadata in options for downstream writers. + // This is metadata-only and does not trigger encryption. + options.aes_mode = Some(aes); + } + options + } +} + +/// Methods for retrieving information on zip files +impl ZipFile<'_, R> { + /// iterate through all extra fields + pub fn extra_data_fields(&self) -> impl Iterator { + self.data.extra_fields.iter() + } +} + +impl HasZipMetadata for ZipFile<'_, R> { + fn get_metadata(&self) -> &ZipFileData { + self.data.as_ref() + } +} + +impl Read for ZipFile<'_, R> { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + self.reader.read(buf) + } + + fn read_exact(&mut self, buf: &mut [u8]) -> io::Result<()> { + self.reader.read_exact(buf) + } + + fn read_to_end(&mut self, buf: &mut Vec) -> io::Result { + self.reader.read_to_end(buf) + } + + fn read_to_string(&mut self, buf: &mut String) -> io::Result { + self.reader.read_to_string(buf) + } +} + +impl Read for ZipFileSeek<'_, R> { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + match &mut self.reader { + ZipFileSeekReader::Raw(r) => r.read(buf), + } + } +} + +impl Seek for ZipFileSeek<'_, R> { + fn seek(&mut self, pos: SeekFrom) -> io::Result { + match &mut self.reader { + ZipFileSeekReader::Raw(r) => r.seek(pos), + } + } +} + +impl HasZipMetadata for ZipFileSeek<'_, R> { + fn get_metadata(&self) -> &ZipFileData { + self.data.as_ref() + } +} + +impl Drop for ZipFile<'_, R> { + fn drop(&mut self) { + // self.data is Owned, this reader is constructed by a streaming reader. + // In this case, we want to exhaust the reader so that the next file is accessible. + if let Cow::Owned(_) = self.data { + // Get the inner `Take` reader so all decryption, decompression and CRC calculation is skipped. + if let Ok(mut inner) = self.take_raw_reader() { + let _ = copy(&mut inner, &mut sink()); + } + } + } +} + +/// Read ZipFile structures from a non-seekable reader. +/// +/// This is an alternative method to read a zip file. If possible, use the ZipArchive functions +/// as some information will be missing when reading this manner. +/// +/// Reads a file header from the start of the stream. Will return `Ok(Some(..))` if a file is +/// present at the start of the stream. Returns `Ok(None)` if the start of the central directory +/// is encountered. No more files should be read after this. +/// +/// The Drop implementation of ZipFile ensures that the reader will be correctly positioned after +/// the structure is done. +/// +/// Missing fields are: +/// * `comment`: set to an empty string +/// * `data_start`: set to 0 +/// * `external_attributes`: `unix_mode()`: will return None +pub fn read_zipfile_from_stream(reader: &mut R) -> ZipResult>> { + // We can't use the typical ::parse() method, as we follow separate code paths depending on the + // "magic" value (since the magic value will be from the central directory header if we've + // finished iterating over all the actual files). + /* TODO: smallvec? */ + + let mut block = ZipLocalEntryBlock::zeroed(); + reader.read_exact(block.as_bytes_mut())?; + + match block.magic().from_le() { + spec::Magic::LOCAL_FILE_HEADER_SIGNATURE => (), + spec::Magic::CENTRAL_DIRECTORY_HEADER_SIGNATURE => return Ok(None), + _ => return Err(ZipLocalEntryBlock::WRONG_MAGIC_ERROR), + } + + let block = block.from_le(); + + let mut result = ZipFileData::from_local_block(block, reader)?; + + match parse_extra_field(&mut result) { + Ok(..) | Err(ZipError::Io(..)) => {} + Err(e) => return Err(e), + } + + let limit_reader = reader.take(result.compressed_size); + let crypto_reader = make_crypto_reader(&result, limit_reader, None, None)?; + let ZipFileData { + crc32, + uncompressed_size, + compression_method, + #[cfg(feature = "legacy-zip")] + flags, + .. + } = result; + + Ok(Some(ZipFile { + data: Cow::Owned(result), + reader: make_reader( + compression_method, + uncompressed_size, + crc32, + crypto_reader, + #[cfg(feature = "legacy-zip")] + flags, + )?, + })) +} + +/// A filter that determines whether an entry should be ignored when searching +/// for the root directory of a Zip archive. +/// +/// Returns `true` if the entry should be considered, and `false` if it should +/// be ignored. +/// +/// See [`root_dir_common_filter`] for a sensible default filter. +pub trait RootDirFilter: Fn(&Path) -> bool {} +impl bool> RootDirFilter for F {} + +/// Common filters when finding the root directory of a Zip archive. +/// +/// This filter is a sensible default for most use cases and filters out common +/// system files that are usually irrelevant to the contents of the archive. +/// +/// Currently, the filter ignores: +/// - `/__MACOSX/` +/// - `/.DS_Store` +/// - `/Thumbs.db` +/// +/// **This function is not guaranteed to be stable and may change in future versions.** +/// +/// # Example +/// +/// ```rust +/// # use std::path::Path; +/// assert!(zip::read::root_dir_common_filter(Path::new("foo.txt"))); +/// assert!(!zip::read::root_dir_common_filter(Path::new(".DS_Store"))); +/// assert!(!zip::read::root_dir_common_filter(Path::new("Thumbs.db"))); +/// assert!(!zip::read::root_dir_common_filter(Path::new("__MACOSX"))); +/// assert!(!zip::read::root_dir_common_filter(Path::new("__MACOSX/foo.txt"))); +/// ``` +pub fn root_dir_common_filter(path: &Path) -> bool { + const COMMON_FILTER_ROOT_FILES: &[&str] = &[".DS_Store", "Thumbs.db"]; + + if path.starts_with("__MACOSX") { + return false; + } + + if path.components().count() == 1 + && path.file_name().is_some_and(|file_name| { + COMMON_FILTER_ROOT_FILES + .iter() + .map(OsStr::new) + .any(|cmp| cmp == file_name) + }) + { + return false; + } + + true +} + +#[cfg(feature = "chrono")] +/// Generate a `SystemTime` from a `DateTime`. +fn datetime_to_systemtime(datetime: &DateTime) -> Option { + if let Some(t) = generate_chrono_datetime(datetime) { + let time = chrono::DateTime::::from_naive_utc_and_offset(t, chrono::Utc); + return Some(time.into()); + } + None +} + +#[cfg(feature = "chrono")] +/// Generate a `NaiveDateTime` from a `DateTime`. +fn generate_chrono_datetime(datetime: &DateTime) -> Option { + if let Some(d) = chrono::NaiveDate::from_ymd_opt( + datetime.year().into(), + datetime.month().into(), + datetime.day().into(), + ) { + if let Some(d) = d.and_hms_opt( + datetime.hour().into(), + datetime.minute().into(), + datetime.second().into(), + ) { + return Some(d); + } + } + None +} + +/// Read ZipFile from a non-seekable reader like [read_zipfile_from_stream] does, but assume the +/// given compressed size and don't read any further ahead than that. +pub fn read_zipfile_from_stream_with_compressed_size( + reader: &mut R, + compressed_size: u64, +) -> ZipResult>> { + let mut block = ZipLocalEntryBlock::zeroed(); + reader.read_exact(block.as_bytes_mut())?; + + match block.magic().from_le() { + spec::Magic::LOCAL_FILE_HEADER_SIGNATURE => (), + spec::Magic::CENTRAL_DIRECTORY_HEADER_SIGNATURE => return Ok(None), + _ => return Err(ZipLocalEntryBlock::WRONG_MAGIC_ERROR), + } + + let block = block.from_le(); + + let mut result = ZipFileData::from_local_block(block, reader)?; + result.compressed_size = compressed_size; + + if result.encrypted { + return unsupported_zip_error("Encrypted files are not supported"); + } + + let limit_reader = reader.take(result.compressed_size); + let crypto_reader = make_crypto_reader(&result, limit_reader, None, None)?; + let ZipFileData { + crc32, + compression_method, + uncompressed_size, + #[cfg(feature = "legacy-zip")] + flags, + .. + } = result; + + Ok(Some(ZipFile { + data: Cow::Owned(result), + reader: make_reader( + compression_method, + uncompressed_size, + crc32, + crypto_reader, + #[cfg(feature = "legacy-zip")] + flags, + )?, + })) +} + +#[cfg(test)] +mod test { + use crate::read::ZipReadOptions; + use crate::result::ZipResult; + use crate::types::SimpleFileOptions; + use crate::CompressionMethod::Stored; + use crate::{ZipArchive, ZipWriter}; + use std::io::{Cursor, Read, Write}; + use tempfile::TempDir; + + #[test] + fn invalid_offset() { + use super::ZipArchive; + + let reader = ZipArchive::new(Cursor::new(include_bytes!( + "../tests/data/invalid_offset.zip" + ))); + assert!(reader.is_err()); + } + + #[test] + fn invalid_offset2() { + use super::ZipArchive; + + let reader = ZipArchive::new(Cursor::new(include_bytes!( + "../tests/data/invalid_offset2.zip" + ))); + assert!(reader.is_err()); + } + + #[test] + fn zip64_with_leading_junk() { + use super::ZipArchive; + + let reader = + ZipArchive::new(Cursor::new(include_bytes!("../tests/data/zip64_demo.zip"))).unwrap(); + assert_eq!(reader.len(), 1); + } + + #[test] + fn zip_contents() { + use super::ZipArchive; + + let mut reader = + ZipArchive::new(Cursor::new(include_bytes!("../tests/data/mimetype.zip"))).unwrap(); + assert_eq!(reader.comment(), b""); + assert_eq!(reader.by_index(0).unwrap().central_header_start(), 77); + } + + #[test] + fn zip_read_streaming() { + use super::read_zipfile_from_stream; + + let mut reader = Cursor::new(include_bytes!("../tests/data/mimetype.zip")); + loop { + if read_zipfile_from_stream(&mut reader).unwrap().is_none() { + break; + } + } + } + + #[test] + fn zip_clone() { + use super::ZipArchive; + use std::io::Read; + + let mut reader1 = + ZipArchive::new(Cursor::new(include_bytes!("../tests/data/mimetype.zip"))).unwrap(); + let mut reader2 = reader1.clone(); + + let mut file1 = reader1.by_index(0).unwrap(); + let mut file2 = reader2.by_index(0).unwrap(); + + let t = file1.last_modified().unwrap(); + assert_eq!( + ( + t.year(), + t.month(), + t.day(), + t.hour(), + t.minute(), + t.second() + ), + (1980, 1, 1, 0, 0, 0) + ); + + let mut buf1 = [0; 5]; + let mut buf2 = [0; 5]; + let mut buf3 = [0; 5]; + let mut buf4 = [0; 5]; + + file1.read_exact(&mut buf1).unwrap(); + file2.read_exact(&mut buf2).unwrap(); + file1.read_exact(&mut buf3).unwrap(); + file2.read_exact(&mut buf4).unwrap(); + + assert_eq!(buf1, buf2); + assert_eq!(buf3, buf4); + assert_ne!(buf1, buf3); + } + + #[test] + fn file_and_dir_predicates() { + use super::ZipArchive; + + let mut zip = ZipArchive::new(Cursor::new(include_bytes!( + "../tests/data/files_and_dirs.zip" + ))) + .unwrap(); + + for i in 0..zip.len() { + let zip_file = zip.by_index(i).unwrap(); + let full_name = zip_file.enclosed_name().unwrap(); + let file_name = full_name.file_name().unwrap().to_str().unwrap(); + assert!( + (file_name.starts_with("dir") && zip_file.is_dir()) + || (file_name.starts_with("file") && zip_file.is_file()) + ); + } + } + + #[test] + fn zip64_magic_in_filenames() { + let files = vec![ + include_bytes!("../tests/data/zip64_magic_in_filename_1.zip").to_vec(), + include_bytes!("../tests/data/zip64_magic_in_filename_2.zip").to_vec(), + include_bytes!("../tests/data/zip64_magic_in_filename_3.zip").to_vec(), + include_bytes!("../tests/data/zip64_magic_in_filename_4.zip").to_vec(), + include_bytes!("../tests/data/zip64_magic_in_filename_5.zip").to_vec(), + ]; + // Although we don't allow adding files whose names contain the ZIP64 CDB-end or + // CDB-end-locator signatures, we still read them when they aren't genuinely ambiguous. + for file in files { + ZipArchive::new(Cursor::new(file)).unwrap(); + } + } + + /// test case to ensure we don't preemptively over allocate based on the + /// declared number of files in the CDE of an invalid zip when the number of + /// files declared is more than the alleged offset in the CDE + #[test] + fn invalid_cde_number_of_files_allocation_smaller_offset() { + use super::ZipArchive; + + let reader = ZipArchive::new(Cursor::new(include_bytes!( + "../tests/data/invalid_cde_number_of_files_allocation_smaller_offset.zip" + ))); + assert!(reader.is_err() || reader.unwrap().is_empty()); + } + + /// test case to ensure we don't preemptively over allocate based on the + /// declared number of files in the CDE of an invalid zip when the number of + /// files declared is less than the alleged offset in the CDE + #[test] + fn invalid_cde_number_of_files_allocation_greater_offset() { + use super::ZipArchive; + + let reader = ZipArchive::new(Cursor::new(include_bytes!( + "../tests/data/invalid_cde_number_of_files_allocation_greater_offset.zip" + ))); + assert!(reader.is_err()); + } + + #[cfg(feature = "deflate64")] + #[test] + fn deflate64_index_out_of_bounds() -> std::io::Result<()> { + let mut reader = ZipArchive::new(Cursor::new(include_bytes!( + "../tests/data/raw_deflate64_index_out_of_bounds.zip" + )))?; + std::io::copy(&mut reader.by_index(0)?, &mut std::io::sink()).expect_err("Invalid file"); + Ok(()) + } + + #[cfg(feature = "deflate64")] + #[test] + fn deflate64_not_enough_space() { + ZipArchive::new(Cursor::new(include_bytes!( + "../tests/data/deflate64_issue_25.zip" + ))) + .expect_err("Invalid file"); + } + + #[cfg(feature = "deflate-flate2")] + #[test] + fn test_read_with_data_descriptor() { + use std::io::Read; + + let mut reader = ZipArchive::new(Cursor::new(include_bytes!( + "../tests/data/data_descriptor.zip" + ))) + .unwrap(); + let mut decompressed = [0u8; 16]; + let mut file = reader.by_index(0).unwrap(); + assert_eq!(file.read(&mut decompressed).unwrap(), 12); + } + + #[test] + fn test_is_symlink() -> std::io::Result<()> { + let mut reader = ZipArchive::new(Cursor::new(include_bytes!("../tests/data/symlink.zip")))?; + assert!(reader.by_index(0)?.is_symlink()); + let tempdir = TempDir::with_prefix("test_is_symlink")?; + reader.extract(&tempdir)?; + assert!(tempdir.path().join("bar").is_symlink()); + Ok(()) + } + + #[test] + #[cfg(feature = "deflate-flate2")] + fn test_utf8_extra_field() { + let mut reader = + ZipArchive::new(Cursor::new(include_bytes!("../tests/data/chinese.zip"))).unwrap(); + reader.by_name("七个房间.txt").unwrap(); + } + + #[test] + fn test_utf8() { + let mut reader = + ZipArchive::new(Cursor::new(include_bytes!("../tests/data/linux-7z.zip"))).unwrap(); + reader.by_name("你好.txt").unwrap(); + } + + #[test] + fn test_utf8_2() { + let mut reader = ZipArchive::new(Cursor::new(include_bytes!( + "../tests/data/windows-7zip.zip" + ))) + .unwrap(); + reader.by_name("你好.txt").unwrap(); + } + + #[test] + fn test_64k_files() -> ZipResult<()> { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + let options = SimpleFileOptions { + compression_method: Stored, + ..Default::default() + }; + for i in 0..=u16::MAX { + let file_name = format!("{i}.txt"); + writer.start_file(&*file_name, options)?; + writer.write_all(i.to_string().as_bytes())?; + } + + let mut reader = ZipArchive::new(writer.finish()?)?; + for i in 0..=u16::MAX { + let expected_name = format!("{i}.txt"); + let expected_contents = i.to_string(); + let expected_contents = expected_contents.as_bytes(); + let mut file = reader.by_name(&expected_name)?; + let mut contents = Vec::with_capacity(expected_contents.len()); + file.read_to_end(&mut contents)?; + assert_eq!(contents, expected_contents); + drop(file); + contents.clear(); + let mut file = reader.by_index(i as usize)?; + file.read_to_end(&mut contents)?; + assert_eq!(contents, expected_contents); + } + Ok(()) + } + + /// Symlinks being extracted shouldn't be followed out of the destination directory. + #[test] + fn test_cannot_symlink_outside_destination() -> ZipResult<()> { + use std::fs::create_dir; + + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.add_symlink("symlink/", "../dest-sibling/", SimpleFileOptions::default())?; + writer.start_file("symlink/dest-file", SimpleFileOptions::default())?; + let mut reader = writer.finish_into_readable()?; + let dest_parent = TempDir::with_prefix("read__test_cannot_symlink_outside_destination")?; + let dest_sibling = dest_parent.path().join("dest-sibling"); + create_dir(&dest_sibling)?; + let dest = dest_parent.path().join("dest"); + create_dir(&dest)?; + assert!(reader.extract(dest).is_err()); + assert!(!dest_sibling.join("dest-file").exists()); + Ok(()) + } + + #[test] + fn test_can_create_destination() -> ZipResult<()> { + let mut reader = + ZipArchive::new(Cursor::new(include_bytes!("../tests/data/mimetype.zip")))?; + let dest = TempDir::with_prefix("read__test_can_create_destination")?; + reader.extract(&dest)?; + assert!(dest.path().join("mimetype").exists()); + Ok(()) + } + + #[test] + fn test_central_directory_not_at_end() -> ZipResult<()> { + let mut reader = ZipArchive::new(Cursor::new(include_bytes!("../tests/data/omni.ja")))?; + let mut file = reader.by_name("chrome.manifest")?; + let mut contents = String::new(); + file.read_to_string(&mut contents)?; // ensures valid UTF-8 + assert!(!contents.is_empty(), "chrome.manifest should not be empty"); + drop(file); + for i in 0..reader.len() { + let mut file = reader.by_index(i)?; + // Attempt to read a small portion or all of each file to ensure it's accessible + let mut buffer = Vec::new(); + file.read_to_end(&mut buffer)?; + assert_eq!( + buffer.len(), + file.size() as usize, + "File size mismatch for {}", + file.name() + ); + } + Ok(()) + } + + #[test] + fn test_ignore_encryption_flag() -> ZipResult<()> { + let mut reader = ZipArchive::new(Cursor::new(include_bytes!( + "../tests/data/ignore_encryption_flag.zip" + )))?; + + // Get the file entry by ignoring its encryption flag. + let mut file = + reader.by_index_with_options(0, ZipReadOptions::new().ignore_encryption_flag(true))?; + let mut contents = String::new(); + assert_eq!(file.name(), "plaintext.txt"); + + // The file claims it is encrypted, but it is not. + assert!(file.encrypted()); + file.read_to_string(&mut contents)?; // ensures valid UTF-8 + assert_eq!(contents, "This file is not encrypted.\n"); + Ok(()) + } +} diff --git a/v2/vendor/security/zip-7.2.0/src/read/config.rs b/v2/vendor/security/zip-7.2.0/src/read/config.rs new file mode 100644 index 0000000000..b00c58a49a --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/src/read/config.rs @@ -0,0 +1,23 @@ +/// Configuration for reading ZIP archives. +#[repr(transparent)] +#[derive(Debug, Default, Clone, Copy)] +pub struct Config { + /// An offset into the reader to use to find the start of the archive. + pub archive_offset: ArchiveOffset, +} + +/// The offset of the start of the archive from the beginning of the reader. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Hash)] +pub enum ArchiveOffset { + /// Try to detect the archive offset automatically. + /// + /// This will look at the central directory specified by `FromCentralDirectory` for a header. + /// If missing, this will behave as if `None` were specified. + #[default] + Detect, + /// Use the central directory length and offset to determine the start of the archive. + #[deprecated(since = "2.3.0", note = "use `Detect` instead")] + FromCentralDirectory, + /// Specify a fixed archive offset. + Known(u64), +} diff --git a/v2/vendor/security/zip-7.2.0/src/read/magic_finder.rs b/v2/vendor/security/zip-7.2.0/src/read/magic_finder.rs new file mode 100644 index 0000000000..b6e8c7d8c9 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/src/read/magic_finder.rs @@ -0,0 +1,291 @@ +use std::io::{Read, Seek, SeekFrom}; + +use memchr::memmem::{Finder, FinderRev}; + +use crate::result::ZipResult; + +pub trait FinderDirection<'a> { + fn new(needle: &'a [u8]) -> Self; + fn reset_cursor(bounds: (u64, u64), window_size: usize) -> u64; + fn scope_window(window: &[u8], mid_window_offset: usize) -> (&[u8], usize); + + fn needle(&self) -> &[u8]; + fn find(&self, haystack: &[u8]) -> Option; + fn move_cursor(&self, cursor: u64, bounds: (u64, u64), window_size: usize) -> Option; + fn move_scope(&self, offset: usize) -> usize; +} + +pub struct Forward<'a>(Finder<'a>); +impl<'a> FinderDirection<'a> for Forward<'a> { + fn new(needle: &'a [u8]) -> Self { + Self(Finder::new(needle)) + } + + fn reset_cursor((start_inclusive, _): (u64, u64), _: usize) -> u64 { + start_inclusive + } + + fn scope_window(window: &[u8], mid_window_offset: usize) -> (&[u8], usize) { + (&window[mid_window_offset..], mid_window_offset) + } + + fn find(&self, haystack: &[u8]) -> Option { + self.0.find(haystack) + } + + fn needle(&self) -> &[u8] { + self.0.needle() + } + + fn move_cursor(&self, cursor: u64, bounds: (u64, u64), window_size: usize) -> Option { + let magic_overlap = self.needle().len().saturating_sub(1) as u64; + let next = cursor.saturating_add(window_size as u64 - magic_overlap); + + if next >= bounds.1 { + None + } else { + Some(next) + } + } + + fn move_scope(&self, offset: usize) -> usize { + offset + self.needle().len() + } +} + +pub struct Backwards<'a>(FinderRev<'a>); +impl<'a> FinderDirection<'a> for Backwards<'a> { + fn new(needle: &'a [u8]) -> Self { + Self(FinderRev::new(needle)) + } + + fn reset_cursor(bounds: (u64, u64), window_size: usize) -> u64 { + bounds + .1 + .saturating_sub(window_size as u64) + .clamp(bounds.0, bounds.1) + } + + fn scope_window(window: &[u8], mid_window_offset: usize) -> (&[u8], usize) { + (&window[..mid_window_offset], 0) + } + + fn find(&self, haystack: &[u8]) -> Option { + self.0.rfind(haystack) + } + + fn needle(&self) -> &[u8] { + self.0.needle() + } + + fn move_cursor(&self, cursor: u64, bounds: (u64, u64), window_size: usize) -> Option { + let magic_overlap = self.needle().len().saturating_sub(1) as u64; + + if cursor <= bounds.0 { + None + } else { + Some( + cursor + .saturating_add(magic_overlap) + .saturating_sub(window_size as u64) + .clamp(bounds.0, bounds.1), + ) + } + } + + fn move_scope(&self, offset: usize) -> usize { + offset + } +} + +/// A utility for finding magic symbols from the end of a seekable reader. +/// +/// Can be repurposed to recycle the internal buffer. +pub struct MagicFinder { + buffer: Box<[u8]>, + pub(self) finder: Direction, + cursor: u64, + mid_buffer_offset: Option, + bounds: (u64, u64), +} + +impl<'a, T: FinderDirection<'a>> MagicFinder { + /// Create a new magic bytes finder to look within specific bounds. + pub fn new(magic_bytes: &'a [u8], start_inclusive: u64, end_exclusive: u64) -> Self { + const BUFFER_SIZE: usize = 2048; + + // Smaller buffer size would be unable to locate bytes. + // Equal buffer size would stall (the window could not be moved). + debug_assert!(BUFFER_SIZE >= magic_bytes.len()); + + Self { + buffer: vec![0; BUFFER_SIZE].into_boxed_slice(), + finder: T::new(magic_bytes), + cursor: T::reset_cursor((start_inclusive, end_exclusive), BUFFER_SIZE), + mid_buffer_offset: None, + bounds: (start_inclusive, end_exclusive), + } + } + + /// Repurpose the finder for different bytes or bounds. + pub fn repurpose(&mut self, magic_bytes: &'a [u8], bounds: (u64, u64)) -> &mut Self { + debug_assert!(self.buffer.len() >= magic_bytes.len()); + + self.finder = T::new(magic_bytes); + self.cursor = T::reset_cursor(bounds, self.buffer.len()); + self.bounds = bounds; + + // Reset the mid-buffer offset, to invalidate buffer content. + self.mid_buffer_offset = None; + + self + } + + /// Find the next magic bytes in the direction specified in the type. + pub fn next(&mut self, reader: &mut R) -> ZipResult> { + loop { + if self.cursor < self.bounds.0 || self.cursor >= self.bounds.1 { + // The finder is consumed + break; + } + + /* Position the window and ensure correct length */ + let window_start = self.cursor; + let window_end = self + .cursor + .saturating_add(self.buffer.len() as u64) + .min(self.bounds.1); + + if window_end <= window_start { + // Short-circuit on zero-sized windows to prevent loop + break; + } + + let window = &mut self.buffer[..(window_end - window_start) as usize]; + + if self.mid_buffer_offset.is_none() { + reader.seek(SeekFrom::Start(window_start))?; + if let Err(e) = reader.read_exact(window) { + if e.kind() == std::io::ErrorKind::UnexpectedEof { + // If we can't read the whole window, it means the bounds or file length are wrong. + break; + } + return Err(e.into()); + } + } + + let (window, window_start_offset) = match self.mid_buffer_offset { + Some(mid_buffer_offset) => T::scope_window(window, mid_buffer_offset), + None => (&*window, 0usize), + }; + + if let Some(offset) = self.finder.find(window) { + let magic_pos = window_start + window_start_offset as u64 + offset as u64; + reader.seek(SeekFrom::Start(magic_pos))?; + + self.mid_buffer_offset = Some(self.finder.move_scope(window_start_offset + offset)); + + return Ok(Some(magic_pos)); + } + + self.mid_buffer_offset = None; + + match self + .finder + .move_cursor(self.cursor, self.bounds, self.buffer.len()) + { + Some(new_cursor) => { + self.cursor = new_cursor; + } + None => { + // Destroy the finder when we've reached the end of the bounds. + self.bounds.0 = self.bounds.1; + break; + } + } + } + + Ok(None) + } +} + +/// A magic bytes finder with an optimistic guess that is tried before +/// the inner finder begins searching from end. This enables much faster +/// lookup in files without appended junk, because the magic bytes will be +/// found directly. +/// +/// The guess can be marked as mandatory to produce an error. This is useful +/// if the ArchiveOffset is known and auto-detection is not desired. +pub struct OptimisticMagicFinder { + inner: MagicFinder, + initial_guess: Option<(u64, bool)>, +} + +/// This is a temporary restriction, to avoid heap allocation in [`Self::next_back`]. +/// +/// We only use magic bytes of size 4 at the moment. +const STACK_BUFFER_SIZE: usize = 8; + +impl<'a, Direction: FinderDirection<'a>> OptimisticMagicFinder { + /// Create a new empty optimistic magic bytes finder. + pub fn new_empty() -> Self { + Self { + inner: MagicFinder::new(&[], 0, 0), + initial_guess: None, + } + } + + /// Repurpose the finder for different bytes, bounds and initial guesses. + pub fn repurpose( + &mut self, + magic_bytes: &'a [u8], + bounds: (u64, u64), + initial_guess: Option<(u64, bool)>, + ) -> &mut Self { + debug_assert!(magic_bytes.len() <= STACK_BUFFER_SIZE); + + self.inner.repurpose(magic_bytes, bounds); + self.initial_guess = initial_guess; + + self + } + + /// Equivalent to `next_back`, with an optional initial guess attempted before + /// proceeding with reading from the back of the reader. + pub fn next(&mut self, reader: &mut R) -> ZipResult> { + if let Some((v, mandatory)) = self.initial_guess { + reader.seek(SeekFrom::Start(v))?; + + let mut buffer = [0; STACK_BUFFER_SIZE]; + let buffer = &mut buffer[..self.inner.finder.needle().len()]; + + // Attempt to match only if there's enough space for the needle + if v.saturating_add(buffer.len() as u64) <= self.inner.bounds.1 { + if let Err(e) = reader.read_exact(buffer) { + if e.kind() == std::io::ErrorKind::UnexpectedEof { + if mandatory { + return Ok(None); + } + } else { + return Err(e.into()); + } + } else if self.inner.finder.needle() == buffer { + // If a match is found, yield it. + self.initial_guess.take(); + reader.seek(SeekFrom::Start(v))?; + return Ok(Some(v)); + } + } + + // If a match is not found, but the initial guess was mandatory, return an error. + if mandatory { + return Ok(None); + } + + // If the initial guess was not mandatory, remove it, as it was not found. + self.initial_guess.take(); + } + + self.inner.next(reader) + } +} diff --git a/v2/vendor/security/zip-7.2.0/src/read/stream.rs b/v2/vendor/security/zip-7.2.0/src/read/stream.rs new file mode 100644 index 0000000000..7b9dfada56 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/src/read/stream.rs @@ -0,0 +1,415 @@ +use crate::result::invalid; + +use super::{ + MAX_SYMLINK_TARGET_LEN, + central_header_to_zip_file_inner, make_symlink, read_zipfile_from_stream, ZipCentralEntryBlock, + ZipFile, ZipFileData, ZipResult, +}; +use crate::spec::FixedSizeBlock; +use indexmap::IndexMap; +use std::io::{self, Read}; +use std::path::{Path, PathBuf}; + +/// Stream decoder for zip. +#[derive(Debug)] +pub struct ZipStreamReader(R); + +impl ZipStreamReader { + /// Create a new ZipStreamReader + pub const fn new(reader: R) -> Self { + Self(reader) + } +} + +impl ZipStreamReader { + fn parse_central_directory(&mut self) -> ZipResult { + // Give archive_offset and central_header_start dummy value 0, since + // they are not used in the output. + let archive_offset = 0; + let central_header_start = 0; + + // Parse central header + let block = ZipCentralEntryBlock::parse(&mut self.0)?; + let file = central_header_to_zip_file_inner( + &mut self.0, + archive_offset, + central_header_start, + block, + )?; + Ok(ZipStreamFileMetadata(file)) + } + + /// Iterate over the stream and extract all file and their + /// metadata. + pub fn visit(mut self, visitor: &mut V) -> ZipResult<()> { + while let Some(mut file) = read_zipfile_from_stream(&mut self.0)? { + visitor.visit_file(&mut file)?; + } + + while let Ok(metadata) = self.parse_central_directory() { + visitor.visit_additional_metadata(&metadata)?; + } + + Ok(()) + } + + /// Extract a Zip archive into a directory, overwriting files if they + /// already exist. Paths are sanitized with [`ZipFile::enclosed_name`]. + /// + /// Extraction is not atomic; If an error is encountered, some of the files + /// may be left on disk. + pub fn extract>(self, directory: P) -> ZipResult<()> { + use std::fs; + fs::create_dir_all(&directory)?; + let directory = directory.as_ref().canonicalize()?; + struct Extractor(PathBuf, IndexMap, ()>); + impl ZipStreamVisitor for Extractor { + fn visit_file(&mut self, file: &mut ZipFile<'_, R>) -> ZipResult<()> { + self.1.insert(file.name().into(), ()); + let mut outpath = self.0.clone(); + file.safe_prepare_path(&self.0, &mut outpath, None::<&(_, fn(&Path) -> bool)>)?; + + if file.is_symlink() { + let declared_len = file.size(); + if declared_len > MAX_SYMLINK_TARGET_LEN { + return Err(invalid!( + "symlink target declares {} bytes, more than the {} byte limit", + declared_len, + MAX_SYMLINK_TARGET_LEN + )); + } + let mut target = Vec::with_capacity(declared_len as usize); + file.read_to_end(&mut target)?; + make_symlink(&outpath, &target, &self.1)?; + return Ok(()); + } + + if file.is_dir() { + fs::create_dir_all(&outpath)?; + } else { + let mut outfile = fs::File::create(&outpath)?; + io::copy(file, &mut outfile)?; + } + + Ok(()) + } + + #[allow(unused)] + fn visit_additional_metadata( + &mut self, + metadata: &ZipStreamFileMetadata, + ) -> ZipResult<()> { + #[cfg(unix)] + { + use super::ZipError; + let filepath = metadata + .enclosed_name() + .ok_or(crate::result::invalid!("Invalid file path"))?; + + let outpath = self.0.join(filepath); + + use std::os::unix::fs::PermissionsExt; + if let Some(mode) = metadata.unix_mode() { + fs::set_permissions(outpath, fs::Permissions::from_mode(mode))?; + } + } + + Ok(()) + } + } + + self.visit(&mut Extractor(directory, IndexMap::new())) + } +} + +/// Visitor for ZipStreamReader +pub trait ZipStreamVisitor { + /// * `file` - contains the content of the file and most of the metadata, + /// except: + /// - `comment`: set to an empty string + /// - `data_start`: set to 0 + /// - `external_attributes`: `unix_mode()`: will return None + fn visit_file(&mut self, file: &mut ZipFile<'_, R>) -> ZipResult<()>; + + /// This function is guaranteed to be called after all `visit_file`s. + /// + /// * `metadata` - Provides missing metadata in `visit_file`. + fn visit_additional_metadata(&mut self, metadata: &ZipStreamFileMetadata) -> ZipResult<()>; +} + +/// Additional metadata for the file. +#[derive(Debug)] +pub struct ZipStreamFileMetadata(ZipFileData); + +impl ZipStreamFileMetadata { + /// Get the name of the file + /// + /// # Warnings + /// + /// It is dangerous to use this name directly when extracting an archive. + /// It may contain an absolute path (`/etc/shadow`), or break out of the + /// current directory (`../runtime`). Carelessly writing to these paths + /// allows an attacker to craft a ZIP archive that will overwrite critical + /// files. + /// + /// You can use the [`ZipFile::enclosed_name`] method to validate the name + /// as a safe path. + pub fn name(&self) -> &str { + &self.0.file_name + } + + /// Get the name of the file, in the raw (internal) byte representation. + /// + /// The encoding of this data is currently undefined. + pub fn name_raw(&self) -> &[u8] { + &self.0.file_name_raw + } + + /// Rewrite the path, ignoring any path components with special meaning. + /// + /// - Absolute paths are made relative + /// - [std::path::Component::ParentDir]s are ignored + /// - Truncates the filename at a NULL byte + /// + /// This is appropriate if you need to be able to extract *something* from + /// any archive, but will easily misrepresent trivial paths like + /// `foo/../bar` as `foo/bar` (instead of `bar`). Because of this, + /// [`ZipFile::enclosed_name`] is the better option in most scenarios. + pub fn mangled_name(&self) -> PathBuf { + self.0.file_name_sanitized() + } + + /// Ensure the file path is safe to use as a [`Path`]. + /// + /// - It can't contain NULL bytes + /// - It can't resolve to a path outside the current directory + /// > `foo/../bar` is fine, `foo/../../bar` is not. + /// - It can't be an absolute path + /// + /// This will read well-formed ZIP files correctly, and is resistant + /// to path-based exploits. It is recommended over + /// [`ZipFile::mangled_name`]. + pub fn enclosed_name(&self) -> Option { + self.0.enclosed_name() + } + + /// Returns whether the file is actually a directory + pub fn is_dir(&self) -> bool { + self.name() + .chars() + .next_back() + .is_some_and(|c| c == '/' || c == '\\') + } + + /// Returns whether the file is a regular file + pub fn is_file(&self) -> bool { + !self.is_dir() + } + + /// Get the comment of the file + pub fn comment(&self) -> &str { + &self.0.file_comment + } + + /// Get unix mode for the file + pub const fn unix_mode(&self) -> Option { + self.0.unix_mode() + } +} + +#[cfg(test)] +mod test { + use tempfile::TempDir; + + use crate::read::stream::{ZipStreamFileMetadata, ZipStreamReader, ZipStreamVisitor}; + use crate::read::ZipFile; + use crate::result::ZipResult; + use crate::write::SimpleFileOptions; + use crate::ZipWriter; + use std::collections::BTreeSet; + use std::io::{Cursor, Read}; + + struct DummyVisitor; + impl ZipStreamVisitor for DummyVisitor { + fn visit_file(&mut self, _file: &mut ZipFile<'_, R>) -> ZipResult<()> { + Ok(()) + } + + fn visit_additional_metadata( + &mut self, + _metadata: &ZipStreamFileMetadata, + ) -> ZipResult<()> { + Ok(()) + } + } + + #[allow(dead_code)] + #[derive(Default, Debug, Eq, PartialEq)] + struct CounterVisitor(u64, u64); + impl ZipStreamVisitor for CounterVisitor { + fn visit_file(&mut self, _file: &mut ZipFile<'_, R>) -> ZipResult<()> { + self.0 += 1; + Ok(()) + } + + fn visit_additional_metadata( + &mut self, + _metadata: &ZipStreamFileMetadata, + ) -> ZipResult<()> { + self.1 += 1; + Ok(()) + } + } + + #[test] + fn invalid_offset() { + ZipStreamReader::new(Cursor::new(include_bytes!( + "../../tests/data/invalid_offset.zip" + ))) + .visit(&mut DummyVisitor) + .unwrap_err(); + } + + #[test] + fn invalid_offset2() { + ZipStreamReader::new(Cursor::new(include_bytes!( + "../../tests/data/invalid_offset2.zip" + ))) + .visit(&mut DummyVisitor) + .unwrap_err(); + } + + #[test] + fn zip_read_streaming() { + let reader = + ZipStreamReader::new(Cursor::new(include_bytes!("../../tests/data/mimetype.zip"))); + + #[derive(Default)] + struct V { + filenames: BTreeSet>, + } + impl ZipStreamVisitor for V { + fn visit_file(&mut self, file: &mut ZipFile<'_, R>) -> ZipResult<()> { + if file.is_file() { + self.filenames.insert(file.name().into()); + } + + Ok(()) + } + fn visit_additional_metadata( + &mut self, + metadata: &ZipStreamFileMetadata, + ) -> ZipResult<()> { + if metadata.is_file() { + assert!( + self.filenames.contains(metadata.name()), + "{} is missing its file content", + metadata.name() + ); + } + + Ok(()) + } + } + + reader.visit(&mut V::default()).unwrap(); + } + + #[test] + fn file_and_dir_predicates() { + let reader = ZipStreamReader::new(Cursor::new(include_bytes!( + "../../tests/data/files_and_dirs.zip" + ))); + + #[derive(Default)] + struct V { + filenames: BTreeSet>, + } + impl ZipStreamVisitor for V { + fn visit_file(&mut self, file: &mut ZipFile<'_, R>) -> ZipResult<()> { + let full_name = file.enclosed_name().unwrap(); + let file_name = full_name.file_name().unwrap().to_str().unwrap(); + assert!( + (file_name.starts_with("dir") && file.is_dir()) + || (file_name.starts_with("file") && file.is_file()) + ); + + if file.is_file() { + self.filenames.insert(file.name().into()); + } + + Ok(()) + } + fn visit_additional_metadata( + &mut self, + metadata: &ZipStreamFileMetadata, + ) -> ZipResult<()> { + if metadata.is_file() { + assert!( + self.filenames.contains(metadata.name()), + "{} is missing its file content", + metadata.name() + ); + } + + Ok(()) + } + } + + reader.visit(&mut V::default()).unwrap(); + } + + /// test case to ensure we don't preemptively over allocate based on the + /// declared number of files in the CDE of an invalid zip when the number of + /// files declared is more than the alleged offset in the CDE + #[test] + fn invalid_cde_number_of_files_allocation_smaller_offset() { + ZipStreamReader::new(Cursor::new(include_bytes!( + "../../tests/data/invalid_cde_number_of_files_allocation_smaller_offset.zip" + ))) + .visit(&mut DummyVisitor) + .unwrap_err(); + } + + /// test case to ensure we don't preemptively over allocate based on the + /// declared number of files in the CDE of an invalid zip when the number of + /// files declared is less than the alleged offset in the CDE + #[test] + fn invalid_cde_number_of_files_allocation_greater_offset() { + ZipStreamReader::new(Cursor::new(include_bytes!( + "../../tests/data/invalid_cde_number_of_files_allocation_greater_offset.zip" + ))) + .visit(&mut DummyVisitor) + .unwrap_err(); + } + + /// Symlinks being extracted shouldn't be followed out of the destination directory. + #[test] + fn test_cannot_symlink_outside_destination() -> ZipResult<()> { + use std::fs::create_dir; + + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.add_symlink("symlink/", "../dest-sibling/", SimpleFileOptions::default())?; + writer.start_file("symlink/dest-file", SimpleFileOptions::default())?; + let reader = ZipStreamReader::new(writer.finish()?); + let dest_parent = TempDir::with_prefix("stream__cannot_symlink_outside_destination")?; + let dest_sibling = dest_parent.path().join("dest-sibling"); + create_dir(&dest_sibling)?; + let dest = dest_parent.path().join("dest"); + create_dir(&dest)?; + assert!(reader.extract(dest).is_err()); + assert!(!dest_sibling.join("dest-file").exists()); + Ok(()) + } + + #[test] + fn test_can_create_destination() -> ZipResult<()> { + let mut v = Vec::new(); + v.extend_from_slice(include_bytes!("../../tests/data/mimetype.zip")); + let reader = ZipStreamReader::new(v.as_slice()); + let dest = TempDir::with_prefix("stream_test_can_create_destination").unwrap(); + reader.extract(&dest)?; + assert!(dest.path().join("mimetype").exists()); + Ok(()) + } +} diff --git a/v2/vendor/security/zip-7.2.0/src/result.rs b/v2/vendor/security/zip-7.2.0/src/result.rs new file mode 100644 index 0000000000..2217f20eaa --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/src/result.rs @@ -0,0 +1,143 @@ +#![allow(unknown_lints)] // non_local_definitions isn't in Rust 1.70 +#![allow(non_local_definitions)] +//! Error types that can be emitted from this library + +use core::error::Error; +use core::fmt::{self, Display, Formatter}; +use core::num::TryFromIntError; +use std::borrow::Cow; +use std::io; + +/// Generic result type with ZipError as its error variant +pub type ZipResult = Result; + +/// Error type for Zip +#[derive(Debug)] +#[non_exhaustive] +pub enum ZipError { + /// i/o error + Io(io::Error), + + /// invalid Zip archive + InvalidArchive(Cow<'static, str>), + + /// unsupported Zip archive + UnsupportedArchive(&'static str), + + /// specified file not found in archive + FileNotFound, + + /// provided password is incorrect + InvalidPassword, +} + +impl ZipError { + /// The text used as an error when a password is required and not supplied + /// + /// ```rust,no_run + /// # use zip::result::ZipError; + /// # let mut archive = zip::ZipArchive::new(std::io::Cursor::new(&[])).unwrap(); + /// match archive.by_index(1) { + /// Err(ZipError::UnsupportedArchive(ZipError::PASSWORD_REQUIRED)) => eprintln!("a password is needed to unzip this file"), + /// _ => (), + /// } + /// # () + /// ``` + pub const PASSWORD_REQUIRED: &'static str = "Password required to decrypt file"; +} + +impl Display for ZipError { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + match self { + Self::Io(e) => write!(f, "i/o error: {e}"), + Self::InvalidArchive(e) => write!(f, "invalid Zip archive: {e}"), + Self::UnsupportedArchive(e) => write!(f, "unsupported Zip archive: {e}"), + Self::FileNotFound => f.write_str("specified file not found in archive"), + Self::InvalidPassword => f.write_str("provided password is incorrect"), + } + } +} + +impl Error for ZipError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Io(e) => Some(e), + Self::InvalidArchive(_) + | Self::UnsupportedArchive(_) + | Self::FileNotFound + | Self::InvalidPassword => None, + } + } +} + +impl From for io::Error { + fn from(err: ZipError) -> io::Error { + let kind = match &err { + ZipError::Io(err) => err.kind(), + ZipError::InvalidArchive(_) => io::ErrorKind::InvalidData, + ZipError::UnsupportedArchive(_) => io::ErrorKind::Unsupported, + ZipError::FileNotFound => io::ErrorKind::NotFound, + ZipError::InvalidPassword => io::ErrorKind::InvalidInput, + }; + + io::Error::new(kind, err) + } +} + +impl From for ZipError { + fn from(value: io::Error) -> Self { + Self::Io(value) + } +} + +impl From for ZipError { + fn from(_: DateTimeRangeError) -> Self { + invalid!("Invalid date or time") + } +} + +impl From for ZipError { + fn from(_: std::string::FromUtf8Error) -> Self { + invalid!("Invalid UTF-8") + } +} + +/// Error type for time parsing +#[derive(Debug)] +pub struct DateTimeRangeError; + +// TryFromIntError is also an out-of-range error. +impl From for DateTimeRangeError { + fn from(_value: TryFromIntError) -> Self { + DateTimeRangeError + } +} + +impl fmt::Display for DateTimeRangeError { + fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + write!( + fmt, + "a date could not be represented within the bounds the MS-DOS date range (1980-2107)" + ) + } +} + +impl Error for DateTimeRangeError {} + +pub(crate) fn invalid_archive>>(message: M) -> ZipError { + ZipError::InvalidArchive(message.into()) +} + +pub(crate) const fn invalid_archive_const(message: &'static str) -> ZipError { + ZipError::InvalidArchive(Cow::Borrowed(message)) +} + +macro_rules! invalid { + ($message:literal) => { + crate::result::invalid_archive_const($message) + }; + ($($arg:tt)*) => { + crate::result::invalid_archive(format!($($arg)*)) + }; +} +pub(crate) use invalid; diff --git a/v2/vendor/security/zip-7.2.0/src/spec.rs b/v2/vendor/security/zip-7.2.0/src/spec.rs new file mode 100644 index 0000000000..61107d0f79 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/src/spec.rs @@ -0,0 +1,873 @@ +#![macro_use] + +use crate::read::magic_finder::{Backwards, Forward, MagicFinder, OptimisticMagicFinder}; +use crate::read::ArchiveOffset; +use crate::result::{invalid, ZipError, ZipResult}; +use core::any::type_name; +use core::mem; +use core::slice; +use std::io::{self, Read, Seek, Write}; + +/// "Magic" header values used in the zip spec to locate metadata records. +/// +/// These values currently always take up a fixed four bytes, so we can parse and wrap them in this +/// struct to enforce some small amount of type safety. +#[derive(Copy, Clone, Debug, PartialOrd, Ord, PartialEq, Eq, Hash)] +#[repr(transparent)] +pub(crate) struct Magic(u32); + +impl Magic { + pub const fn literal(x: u32) -> Self { + Self(x) + } + + #[inline(always)] + #[allow(dead_code)] + pub const fn from_le_bytes(bytes: [u8; 4]) -> Self { + Self(u32::from_le_bytes(bytes)) + } + + #[inline(always)] + pub const fn to_le_bytes(self) -> [u8; 4] { + self.0.to_le_bytes() + } + + #[allow(clippy::wrong_self_convention)] + #[inline(always)] + pub fn from_le(self) -> Self { + Self(u32::from_le(self.0)) + } + + #[allow(clippy::wrong_self_convention)] + #[inline(always)] + pub fn to_le(self) -> Self { + Self(u32::to_le(self.0)) + } + + pub const LOCAL_FILE_HEADER_SIGNATURE: Self = Self::literal(0x04034b50); + pub const CENTRAL_DIRECTORY_HEADER_SIGNATURE: Self = Self::literal(0x02014b50); + pub const CENTRAL_DIRECTORY_END_SIGNATURE: Self = Self::literal(0x06054b50); + pub const ZIP64_CENTRAL_DIRECTORY_END_SIGNATURE: Self = Self::literal(0x06064b50); + pub const ZIP64_CENTRAL_DIRECTORY_END_LOCATOR_SIGNATURE: Self = Self::literal(0x07064b50); + pub const DATA_DESCRIPTOR_SIGNATURE: Self = Self::literal(0x08074b50); +} + +/// Similar to [`Magic`], but used for extra field tags as per section 4.5.3 of APPNOTE.TXT. +#[derive(Copy, Clone, Debug, PartialOrd, Ord, PartialEq, Eq, Hash)] +#[repr(transparent)] +pub(crate) struct ExtraFieldMagic(u16); + +/* TODO: maybe try to use this for parsing extra fields as well as writing them? */ +#[allow(dead_code)] +impl ExtraFieldMagic { + pub const fn literal(x: u16) -> Self { + Self(x) + } + + #[inline(always)] + pub const fn from_le_bytes(bytes: [u8; 2]) -> Self { + Self(u16::from_le_bytes(bytes)) + } + + #[inline(always)] + pub const fn to_le_bytes(self) -> [u8; 2] { + self.0.to_le_bytes() + } + + #[allow(clippy::wrong_self_convention)] + #[inline(always)] + pub fn from_le(self) -> Self { + Self(u16::from_le(self.0)) + } + + #[allow(clippy::wrong_self_convention)] + #[inline(always)] + pub fn to_le(self) -> Self { + Self(u16::to_le(self.0)) + } + + pub const ZIP64_EXTRA_FIELD_TAG: Self = Self::literal(0x0001); +} + +/// The file size at which a ZIP64 record becomes necessary. +/// +/// If a file larger than this threshold attempts to be written, compressed or uncompressed, and +/// [`FileOptions::large_file()`](crate::write::FileOptions::large_file) was not true, then [`crate::ZipWriter`] will +/// raise an [`io::Error`] with [`io::ErrorKind::Other`]. +/// +/// If the zip file itself is larger than this value, then a zip64 central directory record will be +/// written to the end of the file. +/// +///``` +/// # fn main() -> Result<(), zip::result::ZipError> { +/// # #[cfg(target_pointer_width = "64")] +/// # { +/// use std::io::{self, Cursor, Write}; +/// use std::error::Error; +/// use zip::{ZipWriter, write::SimpleFileOptions}; +/// +/// let mut zip = ZipWriter::new(Cursor::new(Vec::new())); +/// // Writing an extremely large file for this test is faster without compression. +/// +/// let big_len: usize = (zip::ZIP64_BYTES_THR as usize) + 1; +/// let big_buf = vec![0u8; big_len]; +/// { +/// let options = SimpleFileOptions::default() +/// .compression_method(zip::CompressionMethod::Stored); +/// zip.start_file("zero.dat", options)?; +/// // This is too big! +/// let res = zip.write_all(&big_buf[..]).err().unwrap(); +/// assert_eq!(res.kind(), io::ErrorKind::Other); +/// let description = format!("{}", &res); +/// assert_eq!(description, "Large file option has not been set"); +/// // Attempting to write anything further to the same zip will still succeed, but the previous +/// // failing entry has been removed. +/// zip.start_file("one.dat", options)?; +/// let zip = zip.finish_into_readable()?; +/// let names: Vec<_> = zip.file_names().collect(); +/// assert_eq!(&names, &["one.dat"]); +/// } +/// +/// // Create a new zip output. +/// let mut zip = ZipWriter::new(Cursor::new(Vec::new())); +/// // This time, create a zip64 record for the file. +/// let options = SimpleFileOptions::default() +/// .compression_method(zip::CompressionMethod::Stored) +/// .large_file(true); +/// zip.start_file("zero.dat", options)?; +/// // This succeeds because we specified that it could be a large file. +/// assert!(zip.write_all(&big_buf[..]).is_ok()); +/// # } +/// # Ok(()) +/// # } +///``` +pub const ZIP64_BYTES_THR: u64 = u32::MAX as u64; +/// The number of entries within a single zip necessary to allocate a zip64 central +/// directory record. +/// +/// If more than this number of entries is written to a [`crate::ZipWriter`], then [`crate::ZipWriter::finish()`] +/// will write out extra zip64 data to the end of the zip file. +pub const ZIP64_ENTRY_THR: usize = u16::MAX as usize; + +/// # Safety +/// +/// - No padding/uninit bytes +/// - All bytes patterns must be valid +/// - No cell, pointers +/// +/// See `bytemuck::Pod` for more details. +pub(crate) unsafe trait Pod: Copy + 'static { + #[inline] + fn zeroed() -> Self { + unsafe { mem::zeroed() } + } + + #[inline] + fn as_bytes(&self) -> &[u8] { + unsafe { slice::from_raw_parts(self as *const Self as *const u8, mem::size_of::()) } + } + + #[inline] + fn as_bytes_mut(&mut self) -> &mut [u8] { + unsafe { slice::from_raw_parts_mut(self as *mut Self as *mut u8, mem::size_of::()) } + } +} + +pub(crate) trait FixedSizeBlock: Pod { + const MAGIC: Magic; + + fn magic(self) -> Magic; + + const WRONG_MAGIC_ERROR: ZipError; + + #[allow(clippy::wrong_self_convention)] + fn from_le(self) -> Self; + + fn parse(reader: &mut R) -> ZipResult { + let mut block = Self::zeroed(); + if let Err(e) = reader.read_exact(block.as_bytes_mut()) { + if e.kind() == io::ErrorKind::UnexpectedEof { + return Err(invalid!("Unexpected end of {}", type_name::())); + } + return Err(e.into()); + } + let block = Self::from_le(block); + + if block.magic() != Self::MAGIC { + return Err(Self::WRONG_MAGIC_ERROR); + } + Ok(block) + } + + fn to_le(self) -> Self; + + fn write(self, writer: &mut T) -> ZipResult<()> { + let block = self.to_le(); + writer.write_all(block.as_bytes())?; + Ok(()) + } +} + +/// Convert all the fields of a struct *from* little-endian representations. +macro_rules! from_le { + ($obj:ident, $field:ident, $type:ty) => { + $obj.$field = <$type>::from_le($obj.$field); + }; + ($obj:ident, [($field:ident, $type:ty) $(,)?]) => { + from_le![$obj, $field, $type]; + }; + ($obj:ident, [($field:ident, $type:ty), $($rest:tt),+ $(,)?]) => { + from_le![$obj, $field, $type]; + from_le!($obj, [$($rest),+]); + }; +} + +/// Convert all the fields of a struct *into* little-endian representations. +macro_rules! to_le { + ($obj:ident, $field:ident, $type:ty) => { + $obj.$field = <$type>::to_le($obj.$field); + }; + ($obj:ident, [($field:ident, $type:ty) $(,)?]) => { + to_le![$obj, $field, $type]; + }; + ($obj:ident, [($field:ident, $type:ty), $($rest:tt),+ $(,)?]) => { + to_le![$obj, $field, $type]; + to_le!($obj, [$($rest),+]); + }; +} + +/* TODO: derive macro to generate these fields? */ +/// Implement `from_le()` and `to_le()`, providing the field specification to both macros +/// and methods. +macro_rules! to_and_from_le { + ($($args:tt),+ $(,)?) => { + #[inline(always)] + fn from_le(mut self) -> Self { + from_le![self, [$($args),+]]; + self + } + #[inline(always)] + fn to_le(mut self) -> Self { + to_le![self, [$($args),+]]; + self + } + }; +} + +#[derive(Copy, Clone, Debug)] +#[repr(packed, C)] +pub(crate) struct Zip32CDEBlock { + magic: Magic, + pub disk_number: u16, + pub disk_with_central_directory: u16, + pub number_of_files_on_this_disk: u16, + pub number_of_files: u16, + pub central_directory_size: u32, + pub central_directory_offset: u32, + pub zip_file_comment_length: u16, +} + +unsafe impl Pod for Zip32CDEBlock {} + +impl FixedSizeBlock for Zip32CDEBlock { + const MAGIC: Magic = Magic::CENTRAL_DIRECTORY_END_SIGNATURE; + + #[inline(always)] + fn magic(self) -> Magic { + self.magic + } + + const WRONG_MAGIC_ERROR: ZipError = invalid!("Invalid digital signature header"); + + to_and_from_le![ + (magic, Magic), + (disk_number, u16), + (disk_with_central_directory, u16), + (number_of_files_on_this_disk, u16), + (number_of_files, u16), + (central_directory_size, u32), + (central_directory_offset, u32), + (zip_file_comment_length, u16) + ]; +} + +#[derive(Debug)] +pub(crate) struct Zip32CentralDirectoryEnd { + pub disk_number: u16, + pub disk_with_central_directory: u16, + pub number_of_files_on_this_disk: u16, + pub number_of_files: u16, + pub central_directory_size: u32, + pub central_directory_offset: u32, + pub zip_file_comment: Box<[u8]>, +} + +impl Zip32CentralDirectoryEnd { + fn into_block_and_comment(self) -> (Zip32CDEBlock, Box<[u8]>) { + let Self { + disk_number, + disk_with_central_directory, + number_of_files_on_this_disk, + number_of_files, + central_directory_size, + central_directory_offset, + zip_file_comment, + } = self; + let block = Zip32CDEBlock { + magic: Zip32CDEBlock::MAGIC, + disk_number, + disk_with_central_directory, + number_of_files_on_this_disk, + number_of_files, + central_directory_size, + central_directory_offset, + zip_file_comment_length: zip_file_comment.len() as u16, + }; + + (block, zip_file_comment) + } + + pub fn parse(reader: &mut T) -> ZipResult { + let Zip32CDEBlock { + // magic, + disk_number, + disk_with_central_directory, + number_of_files_on_this_disk, + number_of_files, + central_directory_size, + central_directory_offset, + zip_file_comment_length, + .. + } = Zip32CDEBlock::parse(reader)?; + + let mut zip_file_comment = vec![0u8; zip_file_comment_length as usize].into_boxed_slice(); + if let Err(e) = reader.read_exact(&mut zip_file_comment) { + if e.kind() == io::ErrorKind::UnexpectedEof { + return Err(invalid!("EOCD comment exceeds file boundary")); + } + + return Err(e.into()); + } + + Ok(Zip32CentralDirectoryEnd { + disk_number, + disk_with_central_directory, + number_of_files_on_this_disk, + number_of_files, + central_directory_size, + central_directory_offset, + zip_file_comment, + }) + } + + pub fn write(self, writer: &mut T) -> ZipResult<()> { + let (block, comment) = self.into_block_and_comment(); + + if comment.len() > u16::MAX as usize { + return Err(invalid!("EOCD comment length exceeds u16::MAX")); + } + + block.write(writer)?; + writer.write_all(&comment)?; + Ok(()) + } + + pub fn may_be_zip64(&self) -> bool { + self.number_of_files == u16::MAX || self.central_directory_offset == u32::MAX + } +} + +#[derive(Copy, Clone)] +#[repr(packed, C)] +pub(crate) struct Zip64CDELocatorBlock { + magic: Magic, + pub disk_with_central_directory: u32, + pub end_of_central_directory_offset: u64, + pub number_of_disks: u32, +} + +unsafe impl Pod for Zip64CDELocatorBlock {} + +impl FixedSizeBlock for Zip64CDELocatorBlock { + const MAGIC: Magic = Magic::ZIP64_CENTRAL_DIRECTORY_END_LOCATOR_SIGNATURE; + + #[inline(always)] + fn magic(self) -> Magic { + self.magic + } + + const WRONG_MAGIC_ERROR: ZipError = invalid!("Invalid zip64 locator digital signature header"); + + to_and_from_le![ + (magic, Magic), + (disk_with_central_directory, u32), + (end_of_central_directory_offset, u64), + (number_of_disks, u32), + ]; +} + +pub(crate) struct Zip64CentralDirectoryEndLocator { + pub disk_with_central_directory: u32, + pub end_of_central_directory_offset: u64, + pub number_of_disks: u32, +} + +impl Zip64CentralDirectoryEndLocator { + pub fn parse(reader: &mut T) -> ZipResult { + let Zip64CDELocatorBlock { + // magic, + disk_with_central_directory, + end_of_central_directory_offset, + number_of_disks, + .. + } = Zip64CDELocatorBlock::parse(reader)?; + + Ok(Zip64CentralDirectoryEndLocator { + disk_with_central_directory, + end_of_central_directory_offset, + number_of_disks, + }) + } + + pub fn block(self) -> Zip64CDELocatorBlock { + let Self { + disk_with_central_directory, + end_of_central_directory_offset, + number_of_disks, + } = self; + Zip64CDELocatorBlock { + magic: Zip64CDELocatorBlock::MAGIC, + disk_with_central_directory, + end_of_central_directory_offset, + number_of_disks, + } + } + + pub fn write(self, writer: &mut T) -> ZipResult<()> { + self.block().write(writer) + } +} + +#[derive(Copy, Clone)] +#[repr(packed, C)] +pub(crate) struct Zip64CDEBlock { + magic: Magic, + pub record_size: u64, + pub version_made_by: u16, + pub version_needed_to_extract: u16, + pub disk_number: u32, + pub disk_with_central_directory: u32, + pub number_of_files_on_this_disk: u64, + pub number_of_files: u64, + pub central_directory_size: u64, + pub central_directory_offset: u64, +} + +unsafe impl Pod for Zip64CDEBlock {} + +impl FixedSizeBlock for Zip64CDEBlock { + const MAGIC: Magic = Magic::ZIP64_CENTRAL_DIRECTORY_END_SIGNATURE; + + fn magic(self) -> Magic { + self.magic + } + + const WRONG_MAGIC_ERROR: ZipError = invalid!("Invalid digital signature header"); + + to_and_from_le![ + (magic, Magic), + (record_size, u64), + (version_made_by, u16), + (version_needed_to_extract, u16), + (disk_number, u32), + (disk_with_central_directory, u32), + (number_of_files_on_this_disk, u64), + (number_of_files, u64), + (central_directory_size, u64), + (central_directory_offset, u64), + ]; +} + +pub(crate) struct Zip64CentralDirectoryEnd { + pub record_size: u64, + pub version_made_by: u16, + pub version_needed_to_extract: u16, + pub disk_number: u32, + pub disk_with_central_directory: u32, + pub number_of_files_on_this_disk: u64, + pub number_of_files: u64, + pub central_directory_size: u64, + pub central_directory_offset: u64, + pub extensible_data_sector: Box<[u8]>, +} + +impl Zip64CentralDirectoryEnd { + pub fn parse( + reader: &mut T, + max_size: u64, + ) -> ZipResult { + let Zip64CDEBlock { + record_size, + version_made_by, + version_needed_to_extract, + disk_number, + disk_with_central_directory, + number_of_files_on_this_disk, + number_of_files, + central_directory_size, + central_directory_offset, + .. + } = Zip64CDEBlock::parse(reader)?; + + if record_size < 44 { + return Err(invalid!("Low EOCD64 record size")); + } else if record_size.saturating_add(12) > max_size { + return Err(invalid!("EOCD64 extends beyond EOCD64 locator")); + } + + let mut zip_file_comment = vec![0u8; record_size as usize - 44].into_boxed_slice(); + if let Err(e) = reader.read_exact(&mut zip_file_comment) { + if e.kind() == io::ErrorKind::UnexpectedEof { + return Err(invalid!( + "EOCD64 extensible data sector exceeds file boundary" + )); + } + return Err(e.into()); + } + + Ok(Self { + record_size, + version_made_by, + version_needed_to_extract, + disk_number, + disk_with_central_directory, + number_of_files_on_this_disk, + number_of_files, + central_directory_size, + central_directory_offset, + extensible_data_sector: zip_file_comment, + }) + } + + pub fn into_block_and_comment(self) -> (Zip64CDEBlock, Box<[u8]>) { + let Self { + record_size, + version_made_by, + version_needed_to_extract, + disk_number, + disk_with_central_directory, + number_of_files_on_this_disk, + number_of_files, + central_directory_size, + central_directory_offset, + extensible_data_sector, + } = self; + + ( + Zip64CDEBlock { + magic: Zip64CDEBlock::MAGIC, + record_size, + version_made_by, + version_needed_to_extract, + disk_number, + disk_with_central_directory, + number_of_files_on_this_disk, + number_of_files, + central_directory_size, + central_directory_offset, + }, + extensible_data_sector, + ) + } + + pub fn write(self, writer: &mut T) -> ZipResult<()> { + let (block, comment) = self.into_block_and_comment(); + block.write(writer)?; + writer.write_all(&comment)?; + Ok(()) + } +} + +pub(crate) struct DataAndPosition { + pub data: T, + #[allow(dead_code)] + pub position: u64, +} + +impl From<(T, u64)> for DataAndPosition { + fn from(value: (T, u64)) -> Self { + Self { + data: value.0, + position: value.1, + } + } +} + +pub(crate) struct CentralDirectoryEndInfo { + pub eocd: DataAndPosition, + pub eocd64: Option>, + + pub archive_offset: u64, +} + +/// Finds the EOCD and possibly the EOCD64 block and determines the archive offset. +/// +/// In the best case scenario (no prepended junk), this function will not backtrack +/// in the reader. +pub(crate) fn find_central_directory( + reader: &mut R, + archive_offset: ArchiveOffset, + end_exclusive: u64, + file_len: u64, +) -> ZipResult { + const EOCD_SIG_BYTES: [u8; mem::size_of::()] = + Magic::CENTRAL_DIRECTORY_END_SIGNATURE.to_le_bytes(); + + const EOCD64_SIG_BYTES: [u8; mem::size_of::()] = + Magic::ZIP64_CENTRAL_DIRECTORY_END_SIGNATURE.to_le_bytes(); + + const CDFH_SIG_BYTES: [u8; mem::size_of::()] = + Magic::CENTRAL_DIRECTORY_HEADER_SIGNATURE.to_le_bytes(); + + // Instantiate the mandatory finder + let mut eocd_finder = MagicFinder::>::new(&EOCD_SIG_BYTES, 0, end_exclusive); + let mut subfinder: Option>> = None; + + // Keep the last errors for cases of improper EOCD instances. + let mut parsing_error = None; + + while let Some(eocd_offset) = eocd_finder.next(reader)? { + // Attempt to parse the EOCD block + let eocd = match Zip32CentralDirectoryEnd::parse(reader) { + Ok(eocd) => eocd, + Err(e) => { + if parsing_error.is_none() { + parsing_error = Some(e); + } + continue; + } + }; + + // ! Relaxed (inequality) due to garbage-after-comment Python files + // Consistency check: the EOCD comment must terminate before the end of file + if eocd.zip_file_comment.len() as u64 + eocd_offset + 22 > file_len { + parsing_error = Some(invalid!("Invalid EOCD comment length")); + continue; + } + + let zip64_metadata = if eocd.may_be_zip64() { + fn try_read_eocd64_locator( + reader: &mut (impl Read + Seek + ?Sized), + eocd_offset: u64, + ) -> ZipResult<(u64, Zip64CentralDirectoryEndLocator)> { + if eocd_offset < mem::size_of::() as u64 { + return Err(invalid!("EOCD64 Locator does not fit in file")); + } + + let locator64_offset = eocd_offset - mem::size_of::() as u64; + + reader.seek(io::SeekFrom::Start(locator64_offset))?; + let locator64 = Zip64CentralDirectoryEndLocator::parse(reader); + Ok((locator64_offset, locator64?)) + } + + try_read_eocd64_locator(reader, eocd_offset).ok() + } else { + None + }; + + let Some((locator64_offset, locator64)) = zip64_metadata else { + // Branch out for zip32 + let relative_cd_offset = eocd.central_directory_offset as u64; + + // If the archive is empty, there is nothing more to be checked, the archive is correct. + if eocd.number_of_files == 0 { + return Ok(CentralDirectoryEndInfo { + eocd: (eocd, eocd_offset).into(), + eocd64: None, + archive_offset: eocd_offset.saturating_sub(relative_cd_offset), + }); + } + + // Consistency check: the CD relative offset cannot be after the EOCD + if relative_cd_offset >= eocd_offset { + parsing_error = Some(invalid!("Invalid CDFH offset in EOCD")); + continue; + } + + // Attempt to find the first CDFH + let subfinder = subfinder + .get_or_insert_with(OptimisticMagicFinder::new_empty) + .repurpose( + &CDFH_SIG_BYTES, + // The CDFH must be before the EOCD and after the relative offset, + // because prepended junk can only move it forward. + (relative_cd_offset, eocd_offset), + match archive_offset { + ArchiveOffset::Known(n) => { + Some((relative_cd_offset.saturating_add(n).min(eocd_offset), true)) + } + _ => Some((relative_cd_offset, false)), + }, + ); + + // Consistency check: find the first CDFH + if let Some(cd_offset) = subfinder.next(reader)? { + // The first CDFH will define the archive offset + let archive_offset = cd_offset - relative_cd_offset; + + return Ok(CentralDirectoryEndInfo { + eocd: (eocd, eocd_offset).into(), + eocd64: None, + archive_offset, + }); + } + + parsing_error = Some(invalid!("No CDFH found")); + continue; + }; + + // Consistency check: the EOCD64 offset must be before EOCD64 Locator offset */ + if locator64.end_of_central_directory_offset >= locator64_offset { + parsing_error = Some(invalid!("Invalid EOCD64 Locator CD offset")); + continue; + } + + if locator64.number_of_disks > 1 { + parsing_error = Some(invalid!("Multi-disk ZIP files are not supported")); + continue; + } + + // This was hidden inside a function to collect errors in a single place. + // Once try blocks are stabilized, this can go away. + fn try_read_eocd64( + reader: &mut R, + locator64: &Zip64CentralDirectoryEndLocator, + expected_length: u64, + ) -> ZipResult { + let z64 = Zip64CentralDirectoryEnd::parse(reader, expected_length)?; + + // Consistency check: EOCD64 locator should agree with the EOCD64 + if z64.disk_with_central_directory != locator64.disk_with_central_directory { + return Err(invalid!("Invalid EOCD64: inconsistency with Locator data")); + } + + // Consistency check: the EOCD64 must have the expected length + if z64.record_size + 12 != expected_length { + return Err(invalid!("Invalid EOCD64: inconsistent length")); + } + + Ok(z64) + } + + // Attempt to find the EOCD64 with an initial guess + let subfinder = subfinder + .get_or_insert_with(OptimisticMagicFinder::new_empty) + .repurpose( + &EOCD64_SIG_BYTES, + (locator64.end_of_central_directory_offset, locator64_offset), + match archive_offset { + ArchiveOffset::Known(n) => Some(( + locator64 + .end_of_central_directory_offset + .saturating_add(n) + .min(locator64_offset), + true, + )), + _ => Some((locator64.end_of_central_directory_offset, false)), + }, + ); + + // Consistency check: Find the EOCD64 + let mut local_error = None; + while let Some(eocd64_offset) = subfinder.next(reader)? { + let archive_offset = eocd64_offset - locator64.end_of_central_directory_offset; + + match try_read_eocd64( + reader, + &locator64, + locator64_offset.saturating_sub(eocd64_offset), + ) { + Ok(eocd64) => { + if eocd64_offset + < eocd64 + .number_of_files + .saturating_mul( + mem::size_of::() as u64 + ) + .saturating_add(eocd64.central_directory_offset) + { + local_error = + Some(invalid!("Invalid EOCD64: inconsistent number of files")); + continue; + } + + return Ok(CentralDirectoryEndInfo { + eocd: (eocd, eocd_offset).into(), + eocd64: Some((eocd64, eocd64_offset).into()), + archive_offset, + }); + } + Err(e) => { + local_error = Some(e); + } + } + } + + parsing_error = local_error.or(Some(invalid!("Could not find EOCD64"))); + } + + Err(parsing_error.unwrap_or(invalid!("Could not find EOCD"))) +} + +pub(crate) fn is_dir(filename: &str) -> bool { + filename + .chars() + .next_back() + .is_some_and(|c| c == '/' || c == '\\') +} + +#[cfg(test)] +mod test { + use std::io::Cursor; + + use crate::{ + result::{invalid, ZipError}, + spec::{FixedSizeBlock, Magic, Pod}, + }; + + #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] + #[repr(packed, C)] + pub struct TestBlock { + magic: Magic, + pub file_name_length: u16, + } + + unsafe impl Pod for TestBlock {} + + impl FixedSizeBlock for TestBlock { + const MAGIC: Magic = Magic::literal(0x01111); + + fn magic(self) -> Magic { + self.magic + } + + const WRONG_MAGIC_ERROR: ZipError = invalid!("unreachable"); + + to_and_from_le![(magic, Magic), (file_name_length, u16)]; + } + + /// Demonstrate that a block object can be safely written to memory and deserialized back out. + #[test] + fn block_serde() { + let block = TestBlock { + magic: TestBlock::MAGIC, + file_name_length: 3, + }; + let mut c = Cursor::new(Vec::new()); + block.write(&mut c).unwrap(); + c.set_position(0); + let block2 = TestBlock::parse(&mut c).unwrap(); + assert_eq!(block, block2); + } +} diff --git a/v2/vendor/security/zip-7.2.0/src/types.rs b/v2/vendor/security/zip-7.2.0/src/types.rs new file mode 100644 index 0000000000..cb906c1175 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/src/types.rs @@ -0,0 +1,1768 @@ +//! Types that specify what is contained in a ZIP. +use crate::cfg_if_expr; +use crate::cp437::FromCp437; +use crate::write::FileOptionExtension; +use crate::zipcrypto::EncryptWith; +use core::fmt::{self, Debug, Formatter}; +use core::mem; +use std::ffi::OsStr; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, OnceLock}; +use typed_path::{Utf8WindowsComponent, Utf8WindowsPath}; + +#[cfg(feature = "chrono")] +use chrono::{Datelike, NaiveDate, NaiveDateTime, NaiveTime, Timelike}; +#[cfg(feature = "jiff-02")] +use jiff::civil; + +use crate::result::{invalid, ZipError, ZipResult}; +use crate::spec::{self, FixedSizeBlock, Pod}; + +pub(crate) mod ffi { + pub const S_IFDIR: u32 = 0o0040000; + pub const S_IFREG: u32 = 0o0100000; + pub const S_IFLNK: u32 = 0o0120000; +} + +use crate::extra_fields::ExtraField; +use crate::read::find_data_start; +use crate::result::DateTimeRangeError; +use crate::spec::is_dir; +use crate::types::ffi::S_IFDIR; +use crate::{CompressionMethod, ZIP64_BYTES_THR}; +use std::io::{Read, Seek}; +#[cfg(feature = "time")] +use time::{error::ComponentRange, Date, Month, OffsetDateTime, PrimitiveDateTime, Time}; + +pub(crate) struct ZipRawValues { + pub(crate) crc32: u32, + pub(crate) compressed_size: u64, + pub(crate) uncompressed_size: u64, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Default)] +#[repr(u8)] +pub enum System { + Dos = 0, + Unix = 3, + #[default] + Unknown, +} + +impl From for System { + fn from(system: u8) -> Self { + match system { + 0 => Self::Dos, + 3 => Self::Unix, + _ => Self::Unknown, + } + } +} + +impl From for u8 { + fn from(system: System) -> Self { + match system { + System::Dos => 0, + System::Unix => 3, + System::Unknown => 4, + } + } +} + +/// Metadata for a file to be written +#[derive(Clone, Debug, Copy, Eq, PartialEq)] +pub struct FileOptions<'k, T: FileOptionExtension> { + pub(crate) compression_method: CompressionMethod, + pub(crate) compression_level: Option, + pub(crate) last_modified_time: DateTime, + pub(crate) permissions: Option, + pub(crate) large_file: bool, + pub(crate) encrypt_with: Option>, + pub(crate) extended_options: T, + pub(crate) alignment: u16, + #[cfg(feature = "deflate-zopfli")] + pub(super) zopfli_buffer_size: Option, + #[cfg(feature = "aes-crypto")] + pub(crate) aes_mode: Option<(AesMode, AesVendorVersion, CompressionMethod)>, +} +/// Simple File Options. Can be copied and good for simple writing zip files +pub type SimpleFileOptions = FileOptions<'static, ()>; + +/// Representation of a moment in time. +/// +/// Zip files use an old format from DOS to store timestamps, +/// with its own set of peculiarities. +/// For example, it has a resolution of 2 seconds! +/// +/// A [`DateTime`] can be stored directly in a zipfile with [`FileOptions::last_modified_time`], +/// or read from one with [`ZipFile::last_modified`](crate::read::ZipFile::last_modified). +/// +/// # Warning +/// +/// Because there is no timezone associated with the [`DateTime`], they should ideally only +/// be used for user-facing descriptions. +/// +/// Modern zip files store more precise timestamps; see [`crate::extra_fields::ExtendedTimestamp`] +/// for details. +#[derive(Clone, Copy, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct DateTime { + datepart: u16, + timepart: u16, +} + +impl Debug for DateTime { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + if *self == Self::default() { + return f.write_str("DateTime::default()"); + } + f.write_fmt(format_args!( + "DateTime::from_date_and_time({}, {}, {}, {}, {}, {})?", + self.year(), + self.month(), + self.day(), + self.hour(), + self.minute(), + self.second() + )) + } +} + +impl DateTime { + /// Constructs a default datetime of 1980-01-01 00:00:00. + pub const DEFAULT: Self = DateTime { + datepart: 0b0000000000100001, + timepart: 0, + }; + + /// Returns the current time if possible, otherwise the default of 1980-01-01. + #[cfg(feature = "time")] + pub fn default_for_write() -> Self { + let now = OffsetDateTime::now_utc(); + PrimitiveDateTime::new(now.date(), now.time()) + .try_into() + .unwrap_or_else(|_| DateTime::default()) + } + + /// Returns the current time if possible, otherwise the default of 1980-01-01. + #[cfg(not(feature = "time"))] + pub fn default_for_write() -> Self { + DateTime::default() + } +} + +#[cfg(feature = "_arbitrary")] +impl arbitrary::Arbitrary<'_> for DateTime { + fn arbitrary(u: &mut arbitrary::Unstructured) -> arbitrary::Result { + let year: u16 = u.int_in_range(1980..=2107)?; + let month: u16 = u.int_in_range(1..=12)?; + let day: u16 = u.int_in_range(1..=31)?; + let datepart = day | (month << 5) | ((year - 1980) << 9); + let hour: u16 = u.int_in_range(0..=23)?; + let minute: u16 = u.int_in_range(0..=59)?; + let second: u16 = u.int_in_range(0..=58)?; + let timepart = (second >> 1) | (minute << 5) | (hour << 11); + Ok(DateTime { datepart, timepart }) + } +} + +#[cfg(feature = "chrono")] +impl TryFrom for DateTime { + type Error = DateTimeRangeError; + + fn try_from(value: NaiveDateTime) -> Result { + DateTime::from_date_and_time( + value.year().try_into()?, + value.month().try_into()?, + value.day().try_into()?, + value.hour().try_into()?, + value.minute().try_into()?, + value.second().try_into()?, + ) + } +} + +#[cfg(feature = "chrono")] +impl TryFrom for NaiveDateTime { + type Error = DateTimeRangeError; + + fn try_from(value: DateTime) -> Result { + let date = NaiveDate::from_ymd_opt( + value.year().into(), + value.month().into(), + value.day().into(), + ) + .ok_or(DateTimeRangeError)?; + let time = NaiveTime::from_hms_opt( + value.hour().into(), + value.minute().into(), + value.second().into(), + ) + .ok_or(DateTimeRangeError)?; + Ok(NaiveDateTime::new(date, time)) + } +} + +#[cfg(feature = "jiff-02")] +impl TryFrom for DateTime { + type Error = DateTimeRangeError; + + fn try_from(value: civil::DateTime) -> Result { + Self::from_date_and_time( + value.year().try_into()?, + value.month() as u8, + value.day() as u8, + value.hour() as u8, + value.minute() as u8, + value.second() as u8, + ) + } +} + +#[cfg(feature = "jiff-02")] +impl TryFrom for civil::DateTime { + type Error = jiff::Error; + + fn try_from(value: DateTime) -> Result { + Self::new( + value.year() as i16, + value.month() as i8, + value.day() as i8, + value.hour() as i8, + value.minute() as i8, + value.second() as i8, + 0, + ) + } +} + +impl TryFrom<(u16, u16)> for DateTime { + type Error = DateTimeRangeError; + + #[inline] + fn try_from(values: (u16, u16)) -> Result { + Self::try_from_msdos(values.0, values.1) + } +} + +impl From for (u16, u16) { + #[inline] + fn from(dt: DateTime) -> Self { + (dt.datepart(), dt.timepart()) + } +} + +impl Default for DateTime { + /// Constructs an 'default' datetime of 1980-01-01 00:00:00 + fn default() -> DateTime { + DateTime::DEFAULT + } +} + +impl fmt::Display for DateTime { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "{:04}-{:02}-{:02} {:02}:{:02}:{:02}", + self.year(), + self.month(), + self.day(), + self.hour(), + self.minute(), + self.second() + ) + } +} + +impl DateTime { + /// Converts an msdos (u16, u16) pair to a DateTime object + /// + /// # Safety + /// The caller must ensure the date and time are valid. + pub const unsafe fn from_msdos_unchecked(datepart: u16, timepart: u16) -> DateTime { + DateTime { datepart, timepart } + } + + /// Converts an msdos (u16, u16) pair to a DateTime object if it represents a valid date and + /// time. + pub fn try_from_msdos(datepart: u16, timepart: u16) -> Result { + let seconds = (timepart & 0b0000000000011111) << 1; + let minutes = (timepart & 0b0000011111100000) >> 5; + let hours = (timepart & 0b1111100000000000) >> 11; + let days = datepart & 0b0000000000011111; + let months = (datepart & 0b0000000111100000) >> 5; + let years = (datepart & 0b1111111000000000) >> 9; + Self::from_date_and_time( + years.checked_add(1980).ok_or(DateTimeRangeError)?, + months.try_into()?, + days.try_into()?, + hours.try_into()?, + minutes.try_into()?, + seconds.try_into()?, + ) + } + + /// Constructs a DateTime from a specific date and time + /// + /// The bounds are: + /// * year: [1980, 2107] + /// * month: [1, 12] + /// * day: [1, 28..=31] + /// * hour: [0, 23] + /// * minute: [0, 59] + /// * second: [0, 58] + pub fn from_date_and_time( + year: u16, + month: u8, + day: u8, + hour: u8, + minute: u8, + second: u8, + ) -> Result { + fn is_leap_year(year: u16) -> bool { + (year % 4 == 0) && ((year % 25 != 0) || (year % 16 == 0)) + } + + if (1980..=2107).contains(&year) + && (1..=12).contains(&month) + && (1..=31).contains(&day) + && hour <= 23 + && minute <= 59 + && second <= 60 + { + let second = second.min(58); // exFAT can't store leap seconds + let max_day = match month { + 1 | 3 | 5 | 7 | 8 | 10 | 12 => 31, + 4 | 6 | 9 | 11 => 30, + 2 if is_leap_year(year) => 29, + 2 => 28, + _ => unreachable!(), + }; + if day > max_day { + return Err(DateTimeRangeError); + } + let datepart = (day as u16) | ((month as u16) << 5) | ((year - 1980) << 9); + let timepart = ((second as u16) >> 1) | ((minute as u16) << 5) | ((hour as u16) << 11); + Ok(DateTime { datepart, timepart }) + } else { + Err(DateTimeRangeError) + } + } + + /// Indicates whether this date and time can be written to a zip archive. + pub fn is_valid(&self) -> bool { + Self::try_from_msdos(self.datepart, self.timepart).is_ok() + } + + #[cfg(feature = "time")] + /// Converts a OffsetDateTime object to a DateTime + /// + /// Returns `Err` when this object is out of bounds + #[deprecated(since = "0.6.4", note = "use `DateTime::try_from()` instead")] + pub fn from_time(dt: OffsetDateTime) -> Result { + dt.try_into() + } + + /// Gets the time portion of this datetime in the msdos representation + pub const fn timepart(&self) -> u16 { + self.timepart + } + + /// Gets the date portion of this datetime in the msdos representation + pub const fn datepart(&self) -> u16 { + self.datepart + } + + #[cfg(feature = "time")] + /// Converts the DateTime to a OffsetDateTime structure + #[deprecated(since = "1.3.1", note = "use `OffsetDateTime::try_from()` instead")] + pub fn to_time(&self) -> Result { + (*self).try_into() + } + + /// Get the year. There is no epoch, i.e. 2018 will be returned as 2018. + pub const fn year(&self) -> u16 { + (self.datepart >> 9) + 1980 + } + + /// Get the month, where 1 = january and 12 = december + /// + /// # Warning + /// + /// When read from a zip file, this may not be a reasonable value + pub const fn month(&self) -> u8 { + ((self.datepart & 0b0000000111100000) >> 5) as u8 + } + + /// Get the day + /// + /// # Warning + /// + /// When read from a zip file, this may not be a reasonable value + pub const fn day(&self) -> u8 { + (self.datepart & 0b0000000000011111) as u8 + } + + /// Get the hour + /// + /// # Warning + /// + /// When read from a zip file, this may not be a reasonable value + pub const fn hour(&self) -> u8 { + (self.timepart >> 11) as u8 + } + + /// Get the minute + /// + /// # Warning + /// + /// When read from a zip file, this may not be a reasonable value + pub const fn minute(&self) -> u8 { + ((self.timepart & 0b0000011111100000) >> 5) as u8 + } + + /// Get the second + /// + /// # Warning + /// + /// When read from a zip file, this may not be a reasonable value + pub const fn second(&self) -> u8 { + ((self.timepart & 0b0000000000011111) << 1) as u8 + } +} + +#[cfg(feature = "time")] +impl TryFrom for DateTime { + type Error = DateTimeRangeError; + + fn try_from(dt: OffsetDateTime) -> Result { + Self::try_from(PrimitiveDateTime::new(dt.date(), dt.time())) + } +} + +#[cfg(feature = "time")] +impl TryFrom for DateTime { + type Error = DateTimeRangeError; + + fn try_from(dt: PrimitiveDateTime) -> Result { + Self::from_date_and_time( + dt.year().try_into()?, + dt.month().into(), + dt.day(), + dt.hour(), + dt.minute(), + dt.second(), + ) + } +} + +#[cfg(feature = "time")] +impl TryFrom for OffsetDateTime { + type Error = ComponentRange; + + fn try_from(dt: DateTime) -> Result { + PrimitiveDateTime::try_from(dt).map(PrimitiveDateTime::assume_utc) + } +} + +#[cfg(feature = "time")] +impl TryFrom for PrimitiveDateTime { + type Error = ComponentRange; + + fn try_from(dt: DateTime) -> Result { + let date = + Date::from_calendar_date(dt.year() as i32, Month::try_from(dt.month())?, dt.day())?; + let time = Time::from_hms(dt.hour(), dt.minute(), dt.second())?; + Ok(PrimitiveDateTime::new(date, time)) + } +} + +pub const MIN_VERSION: u8 = 10; +pub const DEFAULT_VERSION: u8 = 45; + +/// Structure representing a ZIP file. +#[derive(Debug, Clone, Default)] +pub struct ZipFileData { + /// Compatibility of the file attribute information + pub system: System, + /// Specification version + pub version_made_by: u8, + /// ZIP flags + pub flags: u16, + /// True if the file is encrypted. + pub encrypted: bool, + /// True if file_name and file_comment are UTF8 + pub is_utf8: bool, + /// True if the file uses a data-descriptor section + pub using_data_descriptor: bool, + /// Compression method used to store the file + pub compression_method: crate::compression::CompressionMethod, + /// Compression level to store the file + pub compression_level: Option, + /// Last modified time. This will only have a 2 second precision. + pub last_modified_time: Option, + /// CRC32 checksum + pub crc32: u32, + /// Size of the file in the ZIP + pub compressed_size: u64, + /// Size of the file when extracted + pub uncompressed_size: u64, + /// Name of the file + pub file_name: Box, + /// Raw file name. To be used when file_name was incorrectly decoded. + pub file_name_raw: Box<[u8]>, + /// Extra field usually used for storage expansion + pub extra_field: Option>>, + /// Extra field only written to central directory + pub central_extra_field: Option>>, + /// File comment + pub file_comment: Box, + /// Specifies where the local header of the file starts + pub header_start: u64, + /// Specifies where the extra data of the file starts + pub extra_data_start: Option, + /// Specifies where the central header of the file starts + /// + /// Note that when this is not known, it is set to 0 + pub central_header_start: u64, + /// Specifies where the compressed data of the file starts + pub data_start: OnceLock, + /// External file attributes + pub external_attributes: u32, + /// Reserve local ZIP64 extra field + pub large_file: bool, + /// AES mode if applicable + pub aes_mode: Option<(AesMode, AesVendorVersion, CompressionMethod)>, + /// Specifies where in the extra data the AES metadata starts + pub aes_extra_data_start: u64, + + /// extra fields, see + pub extra_fields: Vec, +} + +impl ZipFileData { + /// Get the starting offset of the data of the compressed file + pub fn data_start(&self, reader: &mut (impl Read + Seek + ?Sized)) -> ZipResult { + match self.data_start.get() { + Some(data_start) => Ok(*data_start), + None => Ok(find_data_start(self, reader)?), + } + } + + #[allow(dead_code)] + pub fn is_dir(&self) -> bool { + is_dir(&self.file_name) + } + + pub fn file_name_sanitized(&self) -> PathBuf { + let no_null_filename = match self.file_name.find('\0') { + Some(index) => &self.file_name[0..index], + None => &self.file_name, + }; + + Utf8WindowsPath::new(no_null_filename) + .components() + .filter(|component| matches!(*component, Utf8WindowsComponent::Normal(..))) + .fold(PathBuf::new(), |mut path, cur| { + if let Utf8WindowsComponent::Normal(s) = cur { + path.push(s); + } + path + }) + } + + /// Simplify the file name by removing the prefix and parent directories and only return normal components + pub(crate) fn simplified_components(&self) -> Option> { + if self.file_name.contains('\0') { + return None; + } + let input = Path::new(OsStr::new(&*self.file_name)); + crate::path::simplified_components(input) + } + + pub(crate) fn enclosed_name(&self) -> Option { + if self.file_name.contains('\0') { + return None; + } + let mut depth = 0usize; + let mut out_path = PathBuf::new(); + for component in Utf8WindowsPath::new(&self.file_name).components() { + match component { + Utf8WindowsComponent::Prefix(_) | Utf8WindowsComponent::RootDir => { + if depth > 0 { + return None; + } + } + Utf8WindowsComponent::ParentDir => { + depth = depth.checked_sub(1)?; + out_path.pop(); + } + Utf8WindowsComponent::Normal(s) => { + depth += 1; + out_path.push(s); + } + Utf8WindowsComponent::CurDir => (), + } + } + Some(out_path) + } + + /// Get unix mode for the file + pub(crate) const fn unix_mode(&self) -> Option { + if self.external_attributes == 0 { + return None; + } + let unix_mode = self.external_attributes >> 16; + if unix_mode != 0 { + // If the high 16 bits are non-zero, they probably contain Unix permissions. + // This happens for archives created on Windows by this crate or other tools, + // and is the only way to identify symlinks in such archives. + return Some(unix_mode); + } + match self.system { + System::Unix => Some(unix_mode), + System::Dos => { + // Interpret MS-DOS directory bit + let mut mode = if 0x10 == (self.external_attributes & 0x10) { + ffi::S_IFDIR | 0o0775 + } else { + ffi::S_IFREG | 0o0664 + }; + if 0x01 == (self.external_attributes & 0x01) { + // Read-only bit; strip write permissions + mode &= !0o222; + } + Some(mode) + } + _ => None, + } + } + + /// PKZIP version needed to open this file (from APPNOTE 4.4.3.2). + pub fn version_needed(&self) -> u16 { + let compression_version: u16 = match self.compression_method { + CompressionMethod::Stored => MIN_VERSION.into(), + #[cfg(feature = "_deflate-any")] + CompressionMethod::Deflated => 20, + #[cfg(feature = "_bzip2_any")] + CompressionMethod::Bzip2 => 46, + #[cfg(feature = "deflate64")] + CompressionMethod::Deflate64 => 21, + #[cfg(feature = "lzma")] + CompressionMethod::Lzma => 63, + #[cfg(feature = "xz")] + CompressionMethod::Xz => 63, + // APPNOTE doesn't specify a version for Zstandard + _ => DEFAULT_VERSION as u16, + }; + let crypto_version: u16 = if self.aes_mode.is_some() { + 51 + } else if self.encrypted { + 20 + } else { + 10 + }; + let misc_feature_version: u16 = if self.large_file { + 45 + } else if self + .unix_mode() + .is_some_and(|mode| mode & S_IFDIR == S_IFDIR) + { + // file is directory + 20 + } else { + 10 + }; + compression_version + .max(crypto_version) + .max(misc_feature_version) + } + #[inline(always)] + pub(crate) fn extra_field_len(&self) -> usize { + self.extra_field + .as_ref() + .map(|v| v.len()) + .unwrap_or_default() + } + #[inline(always)] + pub(crate) fn central_extra_field_len(&self) -> usize { + self.central_extra_field + .as_ref() + .map(|v| v.len()) + .unwrap_or_default() + } + + #[allow(clippy::too_many_arguments)] + pub(crate) fn initialize_local_block( + name: S, + options: &FileOptions, + raw_values: ZipRawValues, + header_start: u64, + extra_data_start: Option, + aes_extra_data_start: u64, + compression_method: crate::compression::CompressionMethod, + aes_mode: Option<(AesMode, AesVendorVersion, CompressionMethod)>, + extra_field: &[u8], + ) -> Self + where + S: ToString, + { + let permissions = options.permissions.unwrap_or(0o100644); + let file_name: Box = name.to_string().into_boxed_str(); + let file_name_raw: Box<[u8]> = file_name.bytes().collect(); + let mut external_attributes = permissions << 16; + let system = if (permissions & ffi::S_IFLNK) == ffi::S_IFLNK { + System::Unix + } else if cfg!(windows) { + if is_dir(&file_name) { + // DOS directory bit + external_attributes |= 0x10; + } + if options + .permissions + .is_some_and(|permissions| permissions & 0o444 == 0) + { + // DOS read-only bit + external_attributes |= 0x01; + } + System::Dos + } else { + System::Unix + }; + let mut local_block = ZipFileData { + system, + version_made_by: DEFAULT_VERSION, + flags: 0, + encrypted: options.encrypt_with.is_some() + || cfg_if_expr! { + #[cfg(feature = "aes-crypto")] => options.aes_mode.is_some(), + _ => false + }, + using_data_descriptor: false, + is_utf8: !file_name.is_ascii(), + compression_method, + compression_level: options.compression_level, + last_modified_time: Some(options.last_modified_time), + crc32: raw_values.crc32, + compressed_size: raw_values.compressed_size, + uncompressed_size: raw_values.uncompressed_size, + file_name, // Never used for saving, but used as map key in insert_file_data() + file_name_raw, + extra_field: Some(extra_field.to_vec().into()), + central_extra_field: options.extended_options.central_extra_data().cloned(), + file_comment: String::with_capacity(0).into_boxed_str(), + header_start, + data_start: OnceLock::new(), + central_header_start: 0, + external_attributes, + large_file: options.large_file, + aes_mode, + extra_fields: Vec::new(), + extra_data_start, + aes_extra_data_start, + }; + local_block.version_made_by = local_block.version_needed() as u8; + local_block + } + + pub(crate) fn from_local_block( + block: ZipLocalEntryBlock, + reader: &mut R, + ) -> ZipResult { + let ZipLocalEntryBlock { + // magic, + version_made_by, + flags, + compression_method, + last_mod_time, + last_mod_date, + crc32, + compressed_size, + uncompressed_size, + file_name_length, + extra_field_length, + .. + } = block; + + let encrypted: bool = flags & 1 == 1; + if encrypted { + return Err(ZipError::UnsupportedArchive( + "Encrypted files are not supported", + )); + } + + /* FIXME: these were previously incorrect: add testing! */ + /* flags & (1 << 3) != 0 */ + let using_data_descriptor: bool = flags & (1 << 3) == 1 << 3; + if using_data_descriptor { + return Err(ZipError::UnsupportedArchive( + "The file length is not available in the local header", + )); + } + + /* flags & (1 << 1) != 0 */ + let is_utf8: bool = flags & (1 << 11) != 0; + let compression_method = crate::CompressionMethod::parse_from_u16(compression_method); + let file_name_length: usize = file_name_length.into(); + let extra_field_length: usize = extra_field_length.into(); + + let mut file_name_raw = vec![0u8; file_name_length]; + if let Err(e) = reader.read_exact(&mut file_name_raw) { + if e.kind() == std::io::ErrorKind::UnexpectedEof { + return Err(invalid!("File name extends beyond file boundary")); + } + return Err(e.into()); + } + let mut extra_field = vec![0u8; extra_field_length]; + if let Err(e) = reader.read_exact(&mut extra_field) { + if e.kind() == std::io::ErrorKind::UnexpectedEof { + return Err(invalid!("Extra field extends beyond file boundary")); + } + return Err(e.into()); + } + + let file_name: Box = match is_utf8 { + true => String::from_utf8_lossy(&file_name_raw).into(), + false => file_name_raw.clone().from_cp437().into(), + }; + + let system: u8 = (version_made_by >> 8).try_into().unwrap(); + Ok(ZipFileData { + system: System::from(system), + /* NB: this strips the top 8 bits! */ + version_made_by: version_made_by as u8, + flags, + encrypted, + using_data_descriptor, + is_utf8, + compression_method, + compression_level: None, + last_modified_time: DateTime::try_from_msdos(last_mod_date, last_mod_time).ok(), + crc32, + compressed_size: compressed_size.into(), + uncompressed_size: uncompressed_size.into(), + file_name, + file_name_raw: file_name_raw.into(), + extra_field: Some(Arc::new(extra_field)), + central_extra_field: None, + file_comment: String::with_capacity(0).into_boxed_str(), // file comment is only available in the central directory + // header_start and data start are not available, but also don't matter, since seeking is + // not available. + header_start: 0, + data_start: OnceLock::new(), + central_header_start: 0, + // The external_attributes field is only available in the central directory. + // We set this to zero, which should be valid as the docs state 'If input came + // from standard input, this field is set to zero.' + external_attributes: 0, + large_file: false, + aes_mode: None, + extra_fields: Vec::new(), + extra_data_start: None, + aes_extra_data_start: 0, + }) + } + + fn is_utf8(&self) -> bool { + std::str::from_utf8(&self.file_name_raw).is_ok() + } + + fn is_ascii(&self) -> bool { + self.file_name_raw.is_ascii() + } + + fn flags(&self) -> u16 { + let utf8_bit: u16 = if self.is_utf8() && !self.is_ascii() { + 1u16 << 11 + } else { + 0 + }; + + let using_data_descriptor_bit = if self.using_data_descriptor { + 1u16 << 3 + } else { + 0 + }; + + let encrypted_bit: u16 = if self.encrypted { 1u16 << 0 } else { 0 }; + + utf8_bit | using_data_descriptor_bit | encrypted_bit + } + + fn clamp_size_field(&self, field: u64) -> u32 { + if self.large_file { + spec::ZIP64_BYTES_THR as u32 + } else { + field.min(spec::ZIP64_BYTES_THR).try_into().unwrap() + } + } + + pub(crate) fn local_block(&self) -> ZipResult { + let (compressed_size, uncompressed_size) = if self.using_data_descriptor { + (0, 0) + } else { + ( + self.clamp_size_field(self.compressed_size), + self.clamp_size_field(self.uncompressed_size), + ) + }; + let extra_field_length: u16 = self + .extra_field_len() + .try_into() + .map_err(|_| invalid!("Extra data field is too large"))?; + + let last_modified_time = self + .last_modified_time + .unwrap_or_else(DateTime::default_for_write); + Ok(ZipLocalEntryBlock { + magic: ZipLocalEntryBlock::MAGIC, + version_made_by: self.version_needed(), + flags: self.flags(), + compression_method: self.compression_method.serialize_to_u16(), + last_mod_time: last_modified_time.timepart(), + last_mod_date: last_modified_time.datepart(), + crc32: self.crc32, + compressed_size, + uncompressed_size, + file_name_length: self.file_name_raw.len().try_into().unwrap(), + extra_field_length, + }) + } + + pub(crate) fn block(&self) -> ZipResult { + let extra_field_len: u16 = self.extra_field_len().try_into().unwrap(); + let central_extra_field_len: u16 = self.central_extra_field_len().try_into().unwrap(); + let last_modified_time = self + .last_modified_time + .unwrap_or_else(DateTime::default_for_write); + let version_to_extract = self.version_needed(); + let version_made_by = (self.version_made_by as u16).max(version_to_extract); + Ok(ZipCentralEntryBlock { + magic: ZipCentralEntryBlock::MAGIC, + version_made_by: ((self.system as u16) << 8) | version_made_by, + version_to_extract, + flags: self.flags(), + compression_method: self.compression_method.serialize_to_u16(), + last_mod_time: last_modified_time.timepart(), + last_mod_date: last_modified_time.datepart(), + crc32: self.crc32, + compressed_size: self + .compressed_size + .min(spec::ZIP64_BYTES_THR) + .try_into() + .unwrap(), + uncompressed_size: self + .uncompressed_size + .min(spec::ZIP64_BYTES_THR) + .try_into() + .unwrap(), + file_name_length: self.file_name_raw.len().try_into().unwrap(), + extra_field_length: extra_field_len.checked_add(central_extra_field_len).ok_or( + invalid!("Extra field length in central directory exceeds 64KiB"), + )?, + file_comment_length: self.file_comment.len().try_into().unwrap(), + disk_number: 0, + internal_file_attributes: 0, + external_file_attributes: self.external_attributes, + offset: self + .header_start + .min(spec::ZIP64_BYTES_THR) + .try_into() + .unwrap(), + }) + } + + pub(crate) fn zip64_extra_field_block(&self) -> Option { + Zip64ExtraFieldBlock::maybe_new( + self.large_file, + self.uncompressed_size, + self.compressed_size, + self.header_start, + ) + } + + pub(crate) fn write_data_descriptor( + &self, + writer: &mut W, + auto_large_file: bool, + ) -> Result<(), ZipError> { + if self.large_file { + return self.zip64_data_descriptor_block().write(writer); + } + if self.compressed_size > spec::ZIP64_BYTES_THR + || self.uncompressed_size > spec::ZIP64_BYTES_THR + { + if auto_large_file { + return self.zip64_data_descriptor_block().write(writer); + } + return Err(ZipError::Io(std::io::Error::other( + "Large file option has not been set - use .large_file(true) in options", + ))); + } + self.data_descriptor_block().write(writer) + } + + pub(crate) fn data_descriptor_block(&self) -> ZipDataDescriptorBlock { + ZipDataDescriptorBlock { + magic: ZipDataDescriptorBlock::MAGIC, + crc32: self.crc32, + compressed_size: self.compressed_size as u32, + uncompressed_size: self.uncompressed_size as u32, + } + } + + pub(crate) fn zip64_data_descriptor_block(&self) -> Zip64DataDescriptorBlock { + Zip64DataDescriptorBlock { + magic: Zip64DataDescriptorBlock::MAGIC, + crc32: self.crc32, + compressed_size: self.compressed_size, + uncompressed_size: self.uncompressed_size, + } + } +} + +#[derive(Copy, Clone, Debug)] +#[repr(packed, C)] +pub(crate) struct ZipCentralEntryBlock { + magic: spec::Magic, + pub version_made_by: u16, + pub version_to_extract: u16, + pub flags: u16, + pub compression_method: u16, + pub last_mod_time: u16, + pub last_mod_date: u16, + pub crc32: u32, + pub compressed_size: u32, + pub uncompressed_size: u32, + pub file_name_length: u16, + pub extra_field_length: u16, + pub file_comment_length: u16, + pub disk_number: u16, + pub internal_file_attributes: u16, + pub external_file_attributes: u32, + pub offset: u32, +} + +unsafe impl Pod for ZipCentralEntryBlock {} + +impl FixedSizeBlock for ZipCentralEntryBlock { + const MAGIC: spec::Magic = spec::Magic::CENTRAL_DIRECTORY_HEADER_SIGNATURE; + + #[inline(always)] + fn magic(self) -> spec::Magic { + self.magic + } + + const WRONG_MAGIC_ERROR: ZipError = invalid!("Invalid Central Directory header"); + + to_and_from_le![ + (magic, spec::Magic), + (version_made_by, u16), + (version_to_extract, u16), + (flags, u16), + (compression_method, u16), + (last_mod_time, u16), + (last_mod_date, u16), + (crc32, u32), + (compressed_size, u32), + (uncompressed_size, u32), + (file_name_length, u16), + (extra_field_length, u16), + (file_comment_length, u16), + (disk_number, u16), + (internal_file_attributes, u16), + (external_file_attributes, u32), + (offset, u32), + ]; +} + +#[derive(Copy, Clone, Debug)] +#[repr(packed, C)] +pub(crate) struct ZipLocalEntryBlock { + magic: spec::Magic, + pub version_made_by: u16, + pub flags: u16, + pub compression_method: u16, + pub last_mod_time: u16, + pub last_mod_date: u16, + pub crc32: u32, + pub compressed_size: u32, + pub uncompressed_size: u32, + pub file_name_length: u16, + pub extra_field_length: u16, +} + +unsafe impl Pod for ZipLocalEntryBlock {} + +impl FixedSizeBlock for ZipLocalEntryBlock { + const MAGIC: spec::Magic = spec::Magic::LOCAL_FILE_HEADER_SIGNATURE; + + #[inline(always)] + fn magic(self) -> spec::Magic { + self.magic + } + + const WRONG_MAGIC_ERROR: ZipError = invalid!("Invalid local file header"); + + to_and_from_le![ + (magic, spec::Magic), + (version_made_by, u16), + (flags, u16), + (compression_method, u16), + (last_mod_time, u16), + (last_mod_date, u16), + (crc32, u32), + (compressed_size, u32), + (uncompressed_size, u32), + (file_name_length, u16), + (extra_field_length, u16), + ]; +} + +#[derive(Copy, Clone, Debug)] +pub(crate) struct Zip64ExtraFieldBlock { + magic: spec::ExtraFieldMagic, + size: u16, + uncompressed_size: Option, + compressed_size: Option, + header_start: Option, + // Excluded fields: + // u32: disk start number +} + +impl Zip64ExtraFieldBlock { + pub(crate) fn maybe_new( + large_file: bool, + uncompressed_size: u64, + compressed_size: u64, + header_start: u64, + ) -> Option { + let mut size: u16 = 0; + let uncompressed_size = if uncompressed_size >= ZIP64_BYTES_THR || large_file { + size += mem::size_of::() as u16; + Some(uncompressed_size) + } else { + None + }; + let compressed_size = if compressed_size >= ZIP64_BYTES_THR || large_file { + size += mem::size_of::() as u16; + Some(compressed_size) + } else { + None + }; + let header_start = if header_start >= ZIP64_BYTES_THR { + size += mem::size_of::() as u16; + Some(header_start) + } else { + None + }; + if size == 0 { + return None; + } + + Some(Zip64ExtraFieldBlock { + magic: spec::ExtraFieldMagic::ZIP64_EXTRA_FIELD_TAG, + size, + uncompressed_size, + compressed_size, + header_start, + }) + } +} + +impl Zip64ExtraFieldBlock { + pub fn full_size(&self) -> usize { + assert!(self.size > 0); + self.size as usize + mem::size_of::() + mem::size_of::() + } + + pub fn serialize(self) -> Box<[u8]> { + let Self { + magic, + size, + uncompressed_size, + compressed_size, + header_start, + } = self; + + let full_size = self.full_size(); + + let mut ret = Vec::with_capacity(full_size); + ret.extend(magic.to_le_bytes()); + ret.extend(u16::to_le_bytes(size)); + + if let Some(uncompressed_size) = uncompressed_size { + ret.extend(u64::to_le_bytes(uncompressed_size)); + } + if let Some(compressed_size) = compressed_size { + ret.extend(u64::to_le_bytes(compressed_size)); + } + if let Some(header_start) = header_start { + ret.extend(u64::to_le_bytes(header_start)); + } + debug_assert_eq!(ret.len(), full_size); + + ret.into_boxed_slice() + } +} + +#[derive(Copy, Clone, Debug)] +#[repr(packed, C)] +pub(crate) struct ZipDataDescriptorBlock { + magic: spec::Magic, + pub crc32: u32, + pub compressed_size: u32, + pub uncompressed_size: u32, +} + +unsafe impl Pod for ZipDataDescriptorBlock {} + +impl FixedSizeBlock for ZipDataDescriptorBlock { + const MAGIC: spec::Magic = spec::Magic::DATA_DESCRIPTOR_SIGNATURE; + + #[inline(always)] + fn magic(self) -> spec::Magic { + self.magic + } + + const WRONG_MAGIC_ERROR: ZipError = invalid!("Invalid data descriptor header"); + + to_and_from_le![ + (magic, spec::Magic), + (crc32, u32), + (compressed_size, u32), + (uncompressed_size, u32), + ]; +} + +#[derive(Copy, Clone, Debug)] +#[repr(packed, C)] +pub(crate) struct Zip64DataDescriptorBlock { + magic: spec::Magic, + pub crc32: u32, + pub compressed_size: u64, + pub uncompressed_size: u64, +} + +unsafe impl Pod for Zip64DataDescriptorBlock {} + +impl FixedSizeBlock for Zip64DataDescriptorBlock { + const MAGIC: spec::Magic = spec::Magic::DATA_DESCRIPTOR_SIGNATURE; + + #[inline(always)] + fn magic(self) -> spec::Magic { + self.magic + } + + const WRONG_MAGIC_ERROR: ZipError = invalid!("Invalid zip64 data descriptor header"); + + to_and_from_le![ + (magic, spec::Magic), + (crc32, u32), + (compressed_size, u64), + (uncompressed_size, u64), + ]; +} + +/// The encryption specification used to encrypt a file with AES. +/// +/// According to the [specification](https://www.winzip.com/win/en/aes_info.html#winzip11) AE-2 +/// does not make use of the CRC check. +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +#[repr(u16)] +pub enum AesVendorVersion { + Ae1 = 0x0001, + Ae2 = 0x0002, +} + +/// AES variant used. +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +#[cfg_attr(feature = "_arbitrary", derive(arbitrary::Arbitrary))] +#[repr(u8)] +pub enum AesMode { + /// 128-bit AES encryption. + Aes128 = 0x01, + /// 192-bit AES encryption. + Aes192 = 0x02, + /// 256-bit AES encryption. + Aes256 = 0x03, +} + +#[cfg(feature = "aes-crypto")] +impl AesMode { + /// Length of the salt for the given AES mode. + pub const fn salt_length(&self) -> usize { + self.key_length() / 2 + } + + /// Length of the key for the given AES mode. + pub const fn key_length(&self) -> usize { + match self { + Self::Aes128 => 16, + Self::Aes192 => 24, + Self::Aes256 => 32, + } + } +} + +#[cfg(test)] +mod test { + #[test] + fn system() { + use super::System; + assert_eq!(u8::from(System::Dos), 0u8); + assert_eq!(System::Dos as u8, 0u8); + assert_eq!(System::Unix as u8, 3u8); + assert_eq!(u8::from(System::Unix), 3u8); + assert_eq!(System::from(0), System::Dos); + assert_eq!(System::from(3), System::Unix); + assert_eq!(u8::from(System::Unknown), 4u8); + assert_eq!(System::Unknown as u8, 4u8); + } + + #[test] + fn unix_mode_robustness() { + use super::{System, ZipFileData}; + use crate::types::ffi::S_IFLNK; + let mut data = ZipFileData { + system: System::Dos, + external_attributes: (S_IFLNK | 0o777) << 16, + ..ZipFileData::default() + }; + assert_eq!(data.unix_mode(), Some(S_IFLNK | 0o777)); + + data.system = System::Unknown; + assert_eq!(data.unix_mode(), Some(S_IFLNK | 0o777)); + + data.external_attributes = 0x10; // DOS directory bit + data.system = System::Dos; + assert_eq!( + data.unix_mode().unwrap() & 0o170000, + crate::types::ffi::S_IFDIR + ); + } + + #[test] + fn sanitize() { + let file_name = "/path/../../../../etc/./passwd\0/etc/shadow".to_string(); + let data = ZipFileData { + system: System::Dos, + version_made_by: 0, + flags: 0, + encrypted: false, + using_data_descriptor: false, + is_utf8: true, + compression_method: crate::compression::CompressionMethod::Stored, + compression_level: None, + last_modified_time: None, + crc32: 0, + compressed_size: 0, + uncompressed_size: 0, + file_name: file_name.clone().into_boxed_str(), + file_name_raw: file_name.into_bytes().into_boxed_slice(), + extra_field: None, + central_extra_field: None, + file_comment: String::with_capacity(0).into_boxed_str(), + header_start: 0, + extra_data_start: None, + data_start: OnceLock::new(), + central_header_start: 0, + external_attributes: 0, + large_file: false, + aes_mode: None, + aes_extra_data_start: 0, + extra_fields: Vec::new(), + }; + assert_eq!(data.file_name_sanitized(), PathBuf::from("path/etc/passwd")); + } + + #[test] + #[allow(clippy::unusual_byte_groupings)] + fn datetime_default() { + use super::DateTime; + let dt = DateTime::default(); + assert_eq!(dt.timepart(), 0); + assert_eq!(dt.datepart(), 0b0000000_0001_00001); + } + + #[test] + #[allow(clippy::unusual_byte_groupings)] + fn datetime_max() { + use super::DateTime; + let dt = DateTime::from_date_and_time(2107, 12, 31, 23, 59, 58).unwrap(); + assert_eq!(dt.timepart(), 0b10111_111011_11101); + assert_eq!(dt.datepart(), 0b1111111_1100_11111); + } + + #[test] + fn datetime_equality() { + use super::DateTime; + + let dt = DateTime::from_date_and_time(2018, 11, 17, 10, 38, 30).unwrap(); + assert_eq!( + dt, + DateTime::from_date_and_time(2018, 11, 17, 10, 38, 30).unwrap() + ); + assert_ne!(dt, DateTime::default()); + } + + #[test] + fn datetime_order() { + use std::cmp::Ordering; + + use super::DateTime; + + let dt = DateTime::from_date_and_time(2018, 11, 17, 10, 38, 30).unwrap(); + assert_eq!( + dt.cmp(&DateTime::from_date_and_time(2018, 11, 17, 10, 38, 30).unwrap()), + Ordering::Equal + ); + // year + assert!(dt < DateTime::from_date_and_time(2019, 11, 17, 10, 38, 30).unwrap()); + assert!(dt > DateTime::from_date_and_time(2017, 11, 17, 10, 38, 30).unwrap()); + // month + assert!(dt < DateTime::from_date_and_time(2018, 12, 17, 10, 38, 30).unwrap()); + assert!(dt > DateTime::from_date_and_time(2018, 10, 17, 10, 38, 30).unwrap()); + // day + assert!(dt < DateTime::from_date_and_time(2018, 11, 18, 10, 38, 30).unwrap()); + assert!(dt > DateTime::from_date_and_time(2018, 11, 16, 10, 38, 30).unwrap()); + // hour + assert!(dt < DateTime::from_date_and_time(2018, 11, 17, 11, 38, 30).unwrap()); + assert!(dt > DateTime::from_date_and_time(2018, 11, 17, 9, 38, 30).unwrap()); + // minute + assert!(dt < DateTime::from_date_and_time(2018, 11, 17, 10, 39, 30).unwrap()); + assert!(dt > DateTime::from_date_and_time(2018, 11, 17, 10, 37, 30).unwrap()); + // second + assert!(dt < DateTime::from_date_and_time(2018, 11, 17, 10, 38, 32).unwrap()); + assert_eq!( + dt.cmp(&DateTime::from_date_and_time(2018, 11, 17, 10, 38, 31).unwrap()), + Ordering::Equal + ); + assert!(dt > DateTime::from_date_and_time(2018, 11, 17, 10, 38, 29).unwrap()); + assert!(dt > DateTime::from_date_and_time(2018, 11, 17, 10, 38, 28).unwrap()); + } + + #[test] + fn datetime_display() { + use super::DateTime; + + assert_eq!(format!("{}", DateTime::default()), "1980-01-01 00:00:00"); + assert_eq!( + format!( + "{}", + DateTime::from_date_and_time(2018, 11, 17, 10, 38, 30).unwrap() + ), + "2018-11-17 10:38:30" + ); + assert_eq!( + format!( + "{}", + DateTime::from_date_and_time(2107, 12, 31, 23, 59, 58).unwrap() + ), + "2107-12-31 23:59:58" + ); + } + + #[test] + fn datetime_bounds() { + use super::DateTime; + + assert!(DateTime::from_date_and_time(2000, 1, 1, 23, 59, 60).is_ok()); + assert!(DateTime::from_date_and_time(2000, 1, 1, 24, 0, 0).is_err()); + assert!(DateTime::from_date_and_time(2000, 1, 1, 0, 60, 0).is_err()); + assert!(DateTime::from_date_and_time(2000, 1, 1, 0, 0, 61).is_err()); + + assert!(DateTime::from_date_and_time(2107, 12, 31, 0, 0, 0).is_ok()); + assert!(DateTime::from_date_and_time(1980, 1, 1, 0, 0, 0).is_ok()); + assert!(DateTime::from_date_and_time(1979, 1, 1, 0, 0, 0).is_err()); + assert!(DateTime::from_date_and_time(1980, 0, 1, 0, 0, 0).is_err()); + assert!(DateTime::from_date_and_time(1980, 1, 0, 0, 0, 0).is_err()); + assert!(DateTime::from_date_and_time(2108, 12, 31, 0, 0, 0).is_err()); + assert!(DateTime::from_date_and_time(2107, 13, 31, 0, 0, 0).is_err()); + assert!(DateTime::from_date_and_time(2107, 12, 32, 0, 0, 0).is_err()); + + assert!(DateTime::from_date_and_time(2018, 1, 31, 0, 0, 0).is_ok()); + assert!(DateTime::from_date_and_time(2018, 2, 28, 0, 0, 0).is_ok()); + assert!(DateTime::from_date_and_time(2018, 2, 29, 0, 0, 0).is_err()); + assert!(DateTime::from_date_and_time(2018, 3, 31, 0, 0, 0).is_ok()); + assert!(DateTime::from_date_and_time(2018, 4, 30, 0, 0, 0).is_ok()); + assert!(DateTime::from_date_and_time(2018, 4, 31, 0, 0, 0).is_err()); + assert!(DateTime::from_date_and_time(2018, 5, 31, 0, 0, 0).is_ok()); + assert!(DateTime::from_date_and_time(2018, 6, 30, 0, 0, 0).is_ok()); + assert!(DateTime::from_date_and_time(2018, 6, 31, 0, 0, 0).is_err()); + assert!(DateTime::from_date_and_time(2018, 7, 31, 0, 0, 0).is_ok()); + assert!(DateTime::from_date_and_time(2018, 8, 31, 0, 0, 0).is_ok()); + assert!(DateTime::from_date_and_time(2018, 9, 30, 0, 0, 0).is_ok()); + assert!(DateTime::from_date_and_time(2018, 9, 31, 0, 0, 0).is_err()); + assert!(DateTime::from_date_and_time(2018, 10, 31, 0, 0, 0).is_ok()); + assert!(DateTime::from_date_and_time(2018, 11, 30, 0, 0, 0).is_ok()); + assert!(DateTime::from_date_and_time(2018, 11, 31, 0, 0, 0).is_err()); + assert!(DateTime::from_date_and_time(2018, 12, 31, 0, 0, 0).is_ok()); + + // leap year: divisible by 4 + assert!(DateTime::from_date_and_time(2024, 2, 29, 0, 0, 0).is_ok()); + // leap year: divisible by 100 and by 400 + assert!(DateTime::from_date_and_time(2000, 2, 29, 0, 0, 0).is_ok()); + // common year: divisible by 100 but not by 400 + assert!(DateTime::from_date_and_time(2100, 2, 29, 0, 0, 0).is_err()); + } + + use std::{path::PathBuf, sync::OnceLock}; + + #[cfg(feature = "time")] + use time::{format_description::well_known::Rfc3339, OffsetDateTime, PrimitiveDateTime}; + + use crate::types::{System, ZipFileData}; + + #[cfg(feature = "time")] + #[test] + fn datetime_try_from_offset_datetime() { + use time::macros::datetime; + + use super::DateTime; + + // 2018-11-17 10:38:30 + let dt = DateTime::try_from(datetime!(2018-11-17 10:38:30 UTC)).unwrap(); + assert_eq!(dt.year(), 2018); + assert_eq!(dt.month(), 11); + assert_eq!(dt.day(), 17); + assert_eq!(dt.hour(), 10); + assert_eq!(dt.minute(), 38); + assert_eq!(dt.second(), 30); + } + + #[cfg(feature = "time")] + #[test] + fn datetime_try_from_primitive_datetime() { + use time::macros::datetime; + + use super::DateTime; + + // 2018-11-17 10:38:30 + let dt = DateTime::try_from(datetime!(2018-11-17 10:38:30)).unwrap(); + assert_eq!(dt.year(), 2018); + assert_eq!(dt.month(), 11); + assert_eq!(dt.day(), 17); + assert_eq!(dt.hour(), 10); + assert_eq!(dt.minute(), 38); + assert_eq!(dt.second(), 30); + } + + #[cfg(feature = "time")] + #[test] + fn datetime_try_from_bounds() { + use super::DateTime; + use time::macros::datetime; + + // 1979-12-31 23:59:59 + assert!(DateTime::try_from(datetime!(1979-12-31 23:59:59)).is_err()); + + // 1980-01-01 00:00:00 + assert!(DateTime::try_from(datetime!(1980-01-01 00:00:00)).is_ok()); + + // 2107-12-31 23:59:59 + assert!(DateTime::try_from(datetime!(2107-12-31 23:59:59)).is_ok()); + + // 2108-01-01 00:00:00 + assert!(DateTime::try_from(datetime!(2108-01-01 00:00:00)).is_err()); + } + + #[cfg(feature = "time")] + #[test] + fn offset_datetime_try_from_datetime() { + use time::macros::datetime; + + use super::DateTime; + + // 2018-11-17 10:38:30 UTC + let dt = + OffsetDateTime::try_from(DateTime::try_from_msdos(0x4D71, 0x54CF).unwrap()).unwrap(); + assert_eq!(dt, datetime!(2018-11-17 10:38:30 UTC)); + } + + #[cfg(feature = "time")] + #[test] + fn primitive_datetime_try_from_datetime() { + use time::macros::datetime; + + use super::DateTime; + + // 2018-11-17 10:38:30 + let dt = + PrimitiveDateTime::try_from(DateTime::try_from_msdos(0x4D71, 0x54CF).unwrap()).unwrap(); + assert_eq!(dt, datetime!(2018-11-17 10:38:30)); + } + + #[cfg(feature = "time")] + #[test] + fn offset_datetime_try_from_bounds() { + use super::DateTime; + + // 1980-00-00 00:00:00 + assert!(OffsetDateTime::try_from(unsafe { + DateTime::from_msdos_unchecked(0x0000, 0x0000) + }) + .is_err()); + + // 2107-15-31 31:63:62 + assert!(OffsetDateTime::try_from(unsafe { + DateTime::from_msdos_unchecked(0xFFFF, 0xFFFF) + }) + .is_err()); + } + + #[cfg(feature = "time")] + #[test] + fn primitive_datetime_try_from_bounds() { + use super::DateTime; + + // 1980-00-00 00:00:00 + assert!(PrimitiveDateTime::try_from(unsafe { + DateTime::from_msdos_unchecked(0x0000, 0x0000) + }) + .is_err()); + + // 2107-15-31 31:63:62 + assert!(PrimitiveDateTime::try_from(unsafe { + DateTime::from_msdos_unchecked(0xFFFF, 0xFFFF) + }) + .is_err()); + } + + #[cfg(feature = "jiff-02")] + #[test] + fn datetime_try_from_civil_datetime() { + use jiff::civil; + + use super::DateTime; + + // 2018-11-17 10:38:30 + let dt = DateTime::try_from(civil::datetime(2018, 11, 17, 10, 38, 30, 0)).unwrap(); + assert_eq!(dt.year(), 2018); + assert_eq!(dt.month(), 11); + assert_eq!(dt.day(), 17); + assert_eq!(dt.hour(), 10); + assert_eq!(dt.minute(), 38); + assert_eq!(dt.second(), 30); + } + + #[cfg(feature = "jiff-02")] + #[test] + fn datetime_try_from_civil_datetime_bounds() { + use jiff::civil; + + use super::DateTime; + + // 1979-12-31 23:59:59 + assert!(DateTime::try_from(civil::datetime(1979, 12, 31, 23, 59, 59, 0)).is_err()); + + // 1980-01-01 00:00:00 + assert!(DateTime::try_from(civil::datetime(1980, 1, 1, 0, 0, 0, 0)).is_ok()); + + // 2107-12-31 23:59:59 + assert!(DateTime::try_from(civil::datetime(2107, 12, 31, 23, 59, 59, 0)).is_ok()); + + // 2108-01-01 00:00:00 + assert!(DateTime::try_from(civil::datetime(2108, 1, 1, 0, 0, 0, 0)).is_err()); + } + + #[cfg(feature = "jiff-02")] + #[test] + fn civil_datetime_try_from_datetime() { + use jiff::civil; + + use super::DateTime; + + // 2018-11-17 10:38:30 UTC + let dt = + civil::DateTime::try_from(DateTime::try_from_msdos(0x4D71, 0x54CF).unwrap()).unwrap(); + assert_eq!(dt, civil::datetime(2018, 11, 17, 10, 38, 30, 0)); + } + + #[cfg(feature = "jiff-02")] + #[test] + fn civil_datetime_try_from_datetime_bounds() { + use jiff::civil; + + use super::DateTime; + + // 1980-00-00 00:00:00 + assert!(civil::DateTime::try_from(unsafe { + DateTime::from_msdos_unchecked(0x0000, 0x0000) + }) + .is_err()); + + // 2107-15-31 31:63:62 + assert!(civil::DateTime::try_from(unsafe { + DateTime::from_msdos_unchecked(0xFFFF, 0xFFFF) + }) + .is_err()); + } + + #[test] + #[allow(deprecated)] + fn time_conversion() { + use super::DateTime; + let dt = DateTime::try_from_msdos(0x4D71, 0x54CF).unwrap(); + assert_eq!(dt.year(), 2018); + assert_eq!(dt.month(), 11); + assert_eq!(dt.day(), 17); + assert_eq!(dt.hour(), 10); + assert_eq!(dt.minute(), 38); + assert_eq!(dt.second(), 30); + + let dt = DateTime::try_from((0x4D71, 0x54CF)).unwrap(); + assert_eq!(dt.year(), 2018); + assert_eq!(dt.month(), 11); + assert_eq!(dt.day(), 17); + assert_eq!(dt.hour(), 10); + assert_eq!(dt.minute(), 38); + assert_eq!(dt.second(), 30); + + #[cfg(feature = "time")] + assert_eq!( + dt.to_time().unwrap().format(&Rfc3339).unwrap(), + "2018-11-17T10:38:30Z" + ); + + assert_eq!(<(u16, u16)>::from(dt), (0x4D71, 0x54CF)); + } + + #[test] + #[allow(deprecated)] + fn time_out_of_bounds() { + use super::DateTime; + let dt = unsafe { DateTime::from_msdos_unchecked(0xFFFF, 0xFFFF) }; + assert_eq!(dt.year(), 2107); + assert_eq!(dt.month(), 15); + assert_eq!(dt.day(), 31); + assert_eq!(dt.hour(), 31); + assert_eq!(dt.minute(), 63); + assert_eq!(dt.second(), 62); + + #[cfg(feature = "time")] + assert!(dt.to_time().is_err()); + + let dt = unsafe { DateTime::from_msdos_unchecked(0x0000, 0x0000) }; + assert_eq!(dt.year(), 1980); + assert_eq!(dt.month(), 0); + assert_eq!(dt.day(), 0); + assert_eq!(dt.hour(), 0); + assert_eq!(dt.minute(), 0); + assert_eq!(dt.second(), 0); + + #[cfg(feature = "time")] + assert!(dt.to_time().is_err()); + } + + #[cfg(feature = "time")] + #[test] + fn time_at_january() { + use super::DateTime; + + // 2020-01-01 00:00:00 + let clock = OffsetDateTime::from_unix_timestamp(1_577_836_800).unwrap(); + + assert!(DateTime::try_from(PrimitiveDateTime::new(clock.date(), clock.time())).is_ok()); + } +} diff --git a/v2/vendor/security/zip-7.2.0/src/unstable.rs b/v2/vendor/security/zip-7.2.0/src/unstable.rs new file mode 100644 index 0000000000..4bce5a6160 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/src/unstable.rs @@ -0,0 +1,121 @@ +#![allow(missing_docs)] + +use std::borrow::Cow; +use std::io; +use std::io::{Read, Write}; +use std::path::{Component, Path, MAIN_SEPARATOR}; + +/// Provides high level API for reading from a stream. +pub mod stream { + pub use crate::read::stream::*; +} +/// Types for creating ZIP archives. +pub mod write { + use crate::write::{FileOptionExtension, FileOptions}; + /// Unstable methods for [`FileOptions`]. + pub trait FileOptionsExt { + /// Write the file with the given password using the deprecated ZipCrypto algorithm. + /// + /// This is not recommended for new archives, as ZipCrypto is not secure. + fn with_deprecated_encryption(self, password: &[u8]) -> Self; + } + impl FileOptionsExt for FileOptions<'_, T> { + fn with_deprecated_encryption(self, password: &[u8]) -> FileOptions<'static, T> { + self.with_deprecated_encryption(password) + } + } +} + +/// Helper methods for writing unsigned integers in little-endian form. +pub trait LittleEndianWriteExt: Write { + fn write_u16_le(&mut self, input: u16) -> io::Result<()> { + self.write_all(&input.to_le_bytes()) + } + + fn write_u32_le(&mut self, input: u32) -> io::Result<()> { + self.write_all(&input.to_le_bytes()) + } + + fn write_u64_le(&mut self, input: u64) -> io::Result<()> { + self.write_all(&input.to_le_bytes()) + } + + fn write_u128_le(&mut self, input: u128) -> io::Result<()> { + self.write_all(&input.to_le_bytes()) + } +} + +impl LittleEndianWriteExt for W {} + +/// Helper methods for reading unsigned integers in little-endian form. +pub trait LittleEndianReadExt: Read { + fn read_u16_le(&mut self) -> io::Result { + let mut out = [0u8; 2]; + self.read_exact(&mut out)?; + Ok(u16::from_le_bytes(out)) + } + + fn read_u32_le(&mut self) -> io::Result { + let mut out = [0u8; 4]; + self.read_exact(&mut out)?; + Ok(u32::from_le_bytes(out)) + } + + fn read_u64_le(&mut self) -> io::Result { + let mut out = [0u8; 8]; + self.read_exact(&mut out)?; + Ok(u64::from_le_bytes(out)) + } +} + +impl LittleEndianReadExt for R {} + +/// Converts a path to the ZIP format (forward-slash-delimited and normalized). +pub fn path_to_string>(path: T) -> Box { + let mut maybe_original = None; + if let Some(original) = path.as_ref().to_str() { + if original.is_empty() || original == "." || original == ".." { + return String::new().into_boxed_str(); + } + if original.starts_with(MAIN_SEPARATOR) { + if original.len() == 1 { + return MAIN_SEPARATOR.to_string().into_boxed_str(); + } else if (MAIN_SEPARATOR == '/' || !original[1..].contains(MAIN_SEPARATOR)) + && !original.ends_with('.') + && !original.contains([MAIN_SEPARATOR, MAIN_SEPARATOR]) + && !original.contains([MAIN_SEPARATOR, '.', MAIN_SEPARATOR]) + && !original.contains([MAIN_SEPARATOR, '.', '.', MAIN_SEPARATOR]) + { + maybe_original = Some(&original[1..]); + } + } else if !original.contains(MAIN_SEPARATOR) { + return original.into(); + } + } + let mut recreate = maybe_original.is_none(); + let mut normalized_components = Vec::new(); + + for component in path.as_ref().components() { + match component { + Component::Normal(os_str) => match os_str.to_str() { + Some(valid_str) => normalized_components.push(Cow::Borrowed(valid_str)), + None => { + recreate = true; + normalized_components.push(os_str.to_string_lossy()); + } + }, + Component::ParentDir => { + recreate = true; + normalized_components.pop(); + } + _ => { + recreate = true; + } + } + } + if recreate { + normalized_components.join("/").into() + } else { + maybe_original.unwrap().into() + } +} diff --git a/v2/vendor/security/zip-7.2.0/src/write.rs b/v2/vendor/security/zip-7.2.0/src/write.rs new file mode 100644 index 0000000000..ff4932b7c1 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/src/write.rs @@ -0,0 +1,4077 @@ +//! Types for creating ZIP archives + +#[cfg(feature = "aes-crypto")] +use crate::aes::AesWriter; +use crate::compression::CompressionMethod; +use crate::read::{parse_single_extra_field, Config, ZipArchive, ZipFile}; +use crate::result::{invalid, ZipError, ZipResult}; +use crate::spec::{self, FixedSizeBlock, Zip32CDEBlock}; +use crate::types::ffi::S_IFLNK; +#[cfg(feature = "aes-crypto")] +use crate::types::AesMode; +use crate::types::{ + ffi, AesVendorVersion, DateTime, Zip64ExtraFieldBlock, ZipFileData, ZipLocalEntryBlock, + ZipRawValues, MIN_VERSION, +}; +use core::default::Default; +use core::fmt::{Debug, Formatter}; +use core::marker::PhantomData; +use core::mem::{self, size_of}; +#[cfg(feature = "deflate-zopfli")] +use core::num::NonZeroU64; +use core::str::{from_utf8, Utf8Error}; +use crc32fast::Hasher; +use indexmap::IndexMap; +use std::borrow::ToOwned; +use std::io::{self, Read, Seek, Write}; +use std::io::{BufReader, SeekFrom}; +use std::io::{Cursor, ErrorKind}; +use std::sync::Arc; + +#[cfg(feature = "deflate-flate2")] +use flate2::{write::DeflateEncoder, Compression}; + +#[cfg(feature = "_bzip2_any")] +use bzip2::write::BzEncoder; + +#[cfg(feature = "deflate-zopfli")] +use zopfli::Options; + +#[cfg(feature = "deflate-zopfli")] +use std::io::BufWriter; +use std::path::Path; +#[cfg(feature = "zstd")] +use zstd::stream::write::Encoder as ZstdEncoder; +#[cfg(feature = "_bzip2_any")] +use GenericZipWriter::Bzip2; +#[cfg(feature = "deflate-flate2")] +use GenericZipWriter::Deflater; +#[cfg(feature = "ppmd")] +use GenericZipWriter::Ppmd; +#[cfg(feature = "xz")] +use GenericZipWriter::Xz; +#[cfg(feature = "zstd")] +use GenericZipWriter::Zstd; +#[cfg(feature = "deflate-zopfli")] +use GenericZipWriter::{BufferedZopfliDeflater, ZopfliDeflater}; + +// re-export from types +pub use crate::types::{FileOptions, SimpleFileOptions}; + +#[cfg_attr(feature = "aes-crypto", allow(clippy::large_enum_variant))] +enum MaybeEncrypted { + Unencrypted(W), + #[cfg(feature = "aes-crypto")] + Aes(AesWriter), + ZipCrypto(crate::zipcrypto::ZipCryptoWriter), +} + +impl Debug for MaybeEncrypted { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + // Don't print W, since it may be a huge Vec + f.write_str(match self { + MaybeEncrypted::Unencrypted(_) => "Unencrypted", + #[cfg(feature = "aes-crypto")] + MaybeEncrypted::Aes(_) => "AES", + MaybeEncrypted::ZipCrypto(_) => "ZipCrypto", + }) + } +} + +impl Write for MaybeEncrypted { + fn write(&mut self, buf: &[u8]) -> io::Result { + match self { + MaybeEncrypted::Unencrypted(w) => w.write(buf), + #[cfg(feature = "aes-crypto")] + MaybeEncrypted::Aes(w) => w.write(buf), + MaybeEncrypted::ZipCrypto(w) => w.write(buf), + } + } + fn flush(&mut self) -> io::Result<()> { + match self { + MaybeEncrypted::Unencrypted(w) => w.flush(), + #[cfg(feature = "aes-crypto")] + MaybeEncrypted::Aes(w) => w.flush(), + MaybeEncrypted::ZipCrypto(w) => w.flush(), + } + } +} + +enum GenericZipWriter { + Closed, + Storer(MaybeEncrypted), + #[cfg(feature = "deflate-flate2")] + Deflater(DeflateEncoder>), + #[cfg(feature = "deflate-zopfli")] + ZopfliDeflater(zopfli::DeflateEncoder>), + #[cfg(feature = "deflate-zopfli")] + BufferedZopfliDeflater(BufWriter>>), + #[cfg(feature = "_bzip2_any")] + Bzip2(BzEncoder>), + #[cfg(feature = "zstd")] + Zstd(ZstdEncoder<'static, MaybeEncrypted>), + #[cfg(feature = "xz")] + Xz(Box>>), + #[cfg(feature = "ppmd")] + Ppmd(Box>>), +} + +impl Debug for GenericZipWriter { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + Closed => f.write_str("Closed"), + Storer(w) => f.write_fmt(format_args!("Storer({w:?})")), + #[cfg(feature = "deflate-flate2")] + Deflater(w) => f.write_fmt(format_args!("Deflater({:?})", w.get_ref())), + #[cfg(feature = "deflate-zopfli")] + ZopfliDeflater(_) => f.write_str("ZopfliDeflater"), + #[cfg(feature = "deflate-zopfli")] + BufferedZopfliDeflater(_) => f.write_str("BufferedZopfliDeflater"), + #[cfg(feature = "_bzip2_any")] + Bzip2(w) => f.write_fmt(format_args!("Bzip2({:?})", w.get_ref())), + #[cfg(feature = "zstd")] + Zstd(w) => f.write_fmt(format_args!("Zstd({:?})", w.get_ref())), + #[cfg(feature = "xz")] + Xz(w) => f.write_fmt(format_args!("Xz({:?})", w.inner())), + #[cfg(feature = "ppmd")] + Ppmd(_) => f.write_fmt(format_args!("Ppmd8Encoder")), + } + } +} + +// Put the struct declaration in a private module to convince rustdoc to display ZipWriter nicely +pub(crate) mod zip_writer { + use core::fmt::{Debug, Formatter}; + use std::io::{Seek, Write}; + + use indexmap::IndexMap; + + use crate::{ + types::ZipFileData, + write::{GenericZipWriter, ZipWriterStats}, + }; + + /// ZIP archive generator + /// + /// Handles the bookkeeping involved in building an archive, and provides an + /// API to edit its contents. + /// + /// ``` + /// # fn doit() -> zip::result::ZipResult<()> + /// # { + /// # use zip::ZipWriter; + /// use std::io::Write; + /// use zip::write::SimpleFileOptions; + /// + /// // We use a cursor + vec here, though you'd normally use a `File` + /// let mut cur = std::io::Cursor::new(Vec::new()); + /// let mut zip = ZipWriter::new(&mut cur); + /// + /// let options = SimpleFileOptions::default().compression_method(zip::CompressionMethod::Stored); + /// zip.start_file("hello_world.txt", options)?; + /// zip.write(b"Hello, World!")?; + /// + /// // Apply the changes you've made. + /// // Dropping the `ZipWriter` will have the same effect, but may silently fail + /// zip.finish()?; + /// + /// // raw zip data is available as a Vec + /// let zip_bytes = cur.into_inner(); + /// + /// # Ok(()) + /// # } + /// # doit().unwrap(); + /// ``` + pub struct ZipWriter { + pub(super) inner: GenericZipWriter, + pub(super) files: IndexMap, ZipFileData>, + pub(super) stats: ZipWriterStats, + pub(super) writing_to_file: bool, + pub(super) writing_raw: bool, + pub(super) comment: Box<[u8]>, + pub(super) zip64_comment: Option>, + pub(super) flush_on_finish_file: bool, + pub(super) seek_possible: bool, + pub(crate) auto_large_file: bool, + } + + impl Debug for ZipWriter { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.write_fmt(format_args!( + "ZipWriter {{files: {:?}, stats: {:?}, writing_to_file: {}, writing_raw: {}, comment: {:?}, flush_on_finish_file: {}}}", + self.files, self.stats, self.writing_to_file, self.writing_raw, + self.comment, self.flush_on_finish_file)) + } + } +} +#[doc(inline)] +pub use self::sealed::FileOptionExtension; +use crate::result::ZipError::UnsupportedArchive; +use crate::unstable::path_to_string; +use crate::unstable::LittleEndianWriteExt; +use crate::write::GenericZipWriter::{Closed, Storer}; +use crate::zipcrypto::{EncryptWith, ZipCryptoKeys}; +use crate::CompressionMethod::Stored; +pub use zip_writer::ZipWriter; + +#[derive(Default, Debug)] +struct ZipWriterStats { + hasher: Hasher, + start: u64, + bytes_written: u64, +} + +mod sealed { + use std::sync::Arc; + + use super::ExtendedFileOptions; + + pub trait Sealed {} + /// File options Extensions + #[doc(hidden)] + pub trait FileOptionExtension: Default + Sealed { + /// Extra Data + fn extra_data(&self) -> Option<&Arc>>; + /// Central Extra Data + fn central_extra_data(&self) -> Option<&Arc>>; + } + impl Sealed for () {} + impl FileOptionExtension for () { + fn extra_data(&self) -> Option<&Arc>> { + None + } + fn central_extra_data(&self) -> Option<&Arc>> { + None + } + } + impl Sealed for ExtendedFileOptions {} + + impl FileOptionExtension for ExtendedFileOptions { + fn extra_data(&self) -> Option<&Arc>> { + Some(&self.extra_data) + } + fn central_extra_data(&self) -> Option<&Arc>> { + Some(&self.central_extra_data) + } + } +} + +/// Adds Extra Data and Central Extra Data. It does not implement copy. +pub type FullFileOptions<'k> = FileOptions<'k, ExtendedFileOptions>; +/// The Extension for Extra Data and Central Extra Data +#[cfg_attr(feature = "_arbitrary", derive(arbitrary::Arbitrary))] +#[derive(Clone, Default, Eq, PartialEq)] +pub struct ExtendedFileOptions { + extra_data: Arc>, + central_extra_data: Arc>, +} + +impl ExtendedFileOptions { + /// Adds an extra data field, unless we detect that it's invalid. + pub fn add_extra_data>( + &mut self, + header_id: u16, + data: D, + central_only: bool, + ) -> ZipResult<()> { + let data = data.as_ref(); + let len = data.len() + 4; + if self.extra_data.len() + self.central_extra_data.len() + len > u16::MAX as usize { + Err(invalid!("Extra data field would be longer than allowed")) + } else { + let field = if central_only { + &mut self.central_extra_data + } else { + &mut self.extra_data + }; + let vec = Arc::get_mut(field); + let vec = match vec { + Some(exclusive) => exclusive, + None => { + *field = Arc::new(field.to_vec()); + Arc::get_mut(field).unwrap() + } + }; + Self::add_extra_data_unchecked(vec, header_id, data)?; + Self::validate_extra_data(vec, true)?; + Ok(()) + } + } + + pub(crate) fn add_extra_data_unchecked( + vec: &mut Vec, + header_id: u16, + data: &[u8], + ) -> Result<(), ZipError> { + vec.reserve_exact(data.len() + 4); + vec.write_u16_le(header_id)?; + vec.write_u16_le(data.len() as u16)?; + vec.write_all(data)?; + Ok(()) + } + + fn validate_extra_data(data: &[u8], disallow_zip64: bool) -> ZipResult<()> { + let len = data.len() as u64; + if len == 0 { + return Ok(()); + } + if len > u16::MAX as u64 { + return Err(ZipError::Io(io::Error::other( + "Extra-data field can't exceed u16::MAX bytes", + ))); + } + let mut data = Cursor::new(data); + let mut pos = data.position(); + while pos < len { + if len - data.position() < 4 { + return Err(ZipError::Io(io::Error::other( + "Extra-data field doesn't have room for ID and length", + ))); + } + #[cfg(not(feature = "unreserved"))] + { + use crate::unstable::LittleEndianReadExt; + let header_id = data.read_u16_le()?; + if EXTRA_FIELD_MAPPING.contains(&header_id) { + return Err(ZipError::Io(io::Error::other( + format!( + "Extra data header ID {header_id:#06} requires crate feature \"unreserved\"", + ), + ))); + } + data.seek(SeekFrom::Current(-2))?; + } + parse_single_extra_field(&mut ZipFileData::default(), &mut data, pos, disallow_zip64)?; + pos = data.position(); + } + Ok(()) + } +} + +impl Debug for ExtendedFileOptions { + fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), std::fmt::Error> { + f.write_fmt(format_args!("ExtendedFileOptions {{extra_data: vec!{:?}.into(), central_extra_data: vec!{:?}.into()}}", + self.extra_data, self.central_extra_data)) + } +} + +#[cfg(feature = "_arbitrary")] +impl<'a> arbitrary::Arbitrary<'a> for FileOptions<'a, ExtendedFileOptions> { + fn arbitrary(u: &mut arbitrary::Unstructured<'a>) -> arbitrary::Result { + let mut options = FullFileOptions { + compression_method: CompressionMethod::arbitrary(u)?, + compression_level: if bool::arbitrary(u)? { + Some(u.int_in_range(0..=24)?) + } else { + None + }, + last_modified_time: DateTime::arbitrary(u)?, + permissions: Option::::arbitrary(u)?, + large_file: bool::arbitrary(u)?, + encrypt_with: Option::::arbitrary(u)?, + alignment: u16::arbitrary(u)?, + #[cfg(feature = "deflate-zopfli")] + zopfli_buffer_size: None, + ..Default::default() + }; + #[cfg(feature = "deflate-zopfli")] + if options.compression_method == CompressionMethod::Deflated && bool::arbitrary(u)? { + options.zopfli_buffer_size = + Some(if bool::arbitrary(u)? { 2 } else { 3 } << u.int_in_range(8..=20)?); + } + u.arbitrary_loop(Some(0), Some(10), |u| { + options + .add_extra_data( + u.int_in_range(2..=u16::MAX)?, + Box::<[u8]>::arbitrary(u)?, + bool::arbitrary(u)?, + ) + .map_err(|_| arbitrary::Error::IncorrectFormat)?; + Ok(core::ops::ControlFlow::Continue(())) + })?; + ZipWriter::new(Cursor::new(Vec::new())) + .start_file("", options.clone()) + .map_err(|_| arbitrary::Error::IncorrectFormat)?; + Ok(options) + } +} + +impl FileOptions<'_, T> { + pub(crate) fn normalize(&mut self) { + if !self.last_modified_time.is_valid() { + self.last_modified_time = FileOptions::::default().last_modified_time; + } + + *self.permissions.get_or_insert(0o644) |= ffi::S_IFREG; + } + + /// Indicates whether this file will be encrypted (whether with AES or ZipCrypto). + pub const fn has_encryption(&self) -> bool { + #[cfg(feature = "aes-crypto")] + { + self.encrypt_with.is_some() || self.aes_mode.is_some() + } + #[cfg(not(feature = "aes-crypto"))] + { + self.encrypt_with.is_some() + } + } + + /// Set the compression method for the new file + /// + /// The default is [`CompressionMethod::Deflated`] if it is enabled. If not, + /// [`CompressionMethod::Bzip2`] is the default if it is enabled. If neither `bzip2` nor `deflate` + /// is enabled, [`CompressionMethod::Stored`] becomes the default and files are written uncompressed. + #[must_use] + pub const fn compression_method(mut self, method: CompressionMethod) -> Self { + self.compression_method = method; + self + } + + /// Set the compression level for the new file + /// + /// `None` value specifies default compression level. + /// + /// Range of values depends on compression method: + /// * `Deflated`: 10 - 264 for Zopfli, 0 - 9 for other encoders. Default is 24 if Zopfli is the + /// only encoder, or 6 otherwise. + /// * `Bzip2`: 0 - 9. Default is 6 + /// * `Zstd`: -7 - 22, with zero being mapped to default level. Default is 3 + /// * others: only `None` is allowed + #[must_use] + pub const fn compression_level(mut self, level: Option) -> Self { + self.compression_level = level; + self + } + + /// Set the last modified time + /// + /// The default is the current timestamp if the 'time' feature is enabled, and 1980-01-01 + /// otherwise + #[must_use] + pub const fn last_modified_time(mut self, mod_time: DateTime) -> Self { + self.last_modified_time = mod_time; + self + } + + /// Set the permissions for the new file. + /// + /// The format is represented with unix-style permissions. + /// The default is `0o644`, which represents `rw-r--r--` for files, + /// and `0o755`, which represents `rwxr-xr-x` for directories. + /// + /// This method only preserves the file permissions bits (via a `& 0o777`) and discards + /// higher file mode bits. So it cannot be used to denote an entry as a directory, + /// symlink, or other special file type. + #[must_use] + pub const fn unix_permissions(mut self, mode: u32) -> Self { + self.permissions = Some(mode & 0o777); + self + } + + /// Set whether the new file's compressed and uncompressed size is less than 4 GiB. + /// + /// If set to `false` and the file exceeds the limit, an I/O error is thrown and the file is + /// aborted. If set to `true`, readers will require ZIP64 support and if the file does not + /// exceed the limit, 20 B are wasted. The default is `false`. + #[must_use] + pub const fn large_file(mut self, large: bool) -> Self { + self.large_file = large; + self + } + + pub(crate) fn with_deprecated_encryption(self, password: &[u8]) -> FileOptions<'static, T> { + FileOptions { + encrypt_with: Some(EncryptWith::ZipCrypto( + ZipCryptoKeys::derive(password), + PhantomData, + )), + ..self + } + } + + /// Set the AES encryption parameters. + #[cfg(feature = "aes-crypto")] + pub fn with_aes_encryption(self, mode: AesMode, password: &str) -> FileOptions<'_, T> { + FileOptions { + encrypt_with: Some(EncryptWith::Aes { mode, password }), + ..self + } + } + + /// Sets the size of the buffer used to hold the next block that Zopfli will compress. The + /// larger the buffer, the more effective the compression, but the more memory is required. + /// A value of `None` indicates no buffer, which is recommended only when all non-empty writes + /// are larger than about 32 KiB. + #[must_use] + #[cfg(feature = "deflate-zopfli")] + pub const fn with_zopfli_buffer(mut self, size: Option) -> Self { + self.zopfli_buffer_size = size; + self + } + + /// Returns the compression level currently set. + pub const fn get_compression_level(&self) -> Option { + self.compression_level + } + /// Sets the alignment to the given number of bytes. + #[must_use] + pub const fn with_alignment(mut self, alignment: u16) -> Self { + self.alignment = alignment; + self + } +} +impl FileOptions<'_, ExtendedFileOptions> { + /// Adds an extra data field. + pub fn add_extra_data>( + &mut self, + header_id: u16, + data: D, + central_only: bool, + ) -> ZipResult<()> { + self.extended_options + .add_extra_data(header_id, data, central_only) + } + + /// Removes the extra data fields. + #[must_use] + pub fn clear_extra_data(mut self) -> Self { + if !self.extended_options.extra_data.is_empty() { + self.extended_options.extra_data = Arc::new(vec![]); + } + if !self.extended_options.central_extra_data.is_empty() { + self.extended_options.central_extra_data = Arc::new(vec![]); + } + self + } +} +impl FileOptions<'static, ()> { + /// Constructs a const FileOptions object. + /// + /// Note: This value is different than the return value of [`FileOptions::default()`]: + /// + /// - The `last_modified_time` is [`DateTime::DEFAULT`]. This corresponds to 1980-01-01 00:00:00 + pub const DEFAULT: Self = Self { + compression_method: CompressionMethod::DEFAULT, + compression_level: None, + last_modified_time: DateTime::DEFAULT, + large_file: false, + permissions: None, + encrypt_with: None, + extended_options: (), + alignment: 1, + #[cfg(feature = "deflate-zopfli")] + zopfli_buffer_size: Some(1 << 15), + #[cfg(feature = "aes-crypto")] + aes_mode: None, + }; +} + +impl Default for FileOptions<'_, T> { + /// Construct a new FileOptions object + fn default() -> Self { + Self { + compression_method: Default::default(), + compression_level: None, + last_modified_time: DateTime::default_for_write(), + permissions: None, + large_file: false, + encrypt_with: None, + extended_options: T::default(), + alignment: 1, + #[cfg(feature = "deflate-zopfli")] + zopfli_buffer_size: Some(1 << 15), + #[cfg(feature = "aes-crypto")] + aes_mode: None, + } + } +} + +impl Write for ZipWriter { + fn write(&mut self, buf: &[u8]) -> io::Result { + if !self.writing_to_file { + return Err(io::Error::other("No file has been started")); + } + if buf.is_empty() { + return Ok(0); + } + match self.inner.ref_mut() { + Some(ref mut w) => { + let write_result = w.write(buf); + if let Ok(count) = write_result { + self.stats.update(&buf[..count]); + if self.stats.bytes_written > spec::ZIP64_BYTES_THR + && !self.files.last_mut().unwrap().1.large_file + { + let _ = self.abort_file(); + return Err(io::Error::other("Large file option has not been set")); + } + } + write_result + } + None => Err(io::Error::new( + io::ErrorKind::BrokenPipe, + "write(): ZipWriter was already closed", + )), + } + } + + fn flush(&mut self) -> io::Result<()> { + match self.inner.ref_mut() { + Some(ref mut w) => w.flush(), + None => Err(io::Error::new( + io::ErrorKind::BrokenPipe, + "flush(): ZipWriter was already closed", + )), + } + } +} + +impl ZipWriterStats { + fn update(&mut self, buf: &[u8]) { + self.hasher.update(buf); + self.bytes_written += buf.len() as u64; + } +} + +impl ZipWriter { + /// Initializes the archive from an existing ZIP archive, making it ready for append. + /// + /// This uses a default configuration to initially read the archive. + pub fn new_append(readwriter: A) -> ZipResult> { + Self::new_append_with_config(Default::default(), readwriter) + } + + /// Initializes the archive from an existing ZIP archive, making it ready for append. + /// + /// This uses the given read configuration to initially read the archive. + pub fn new_append_with_config(config: Config, mut readwriter: A) -> ZipResult> { + readwriter.seek(SeekFrom::Start(0))?; + let shared = ZipArchive::get_metadata(config, &mut readwriter)?; + + Ok(ZipWriter { + inner: Storer(MaybeEncrypted::Unencrypted(readwriter)), + files: shared.files, + stats: Default::default(), + writing_to_file: false, + comment: shared.comment, + zip64_comment: shared.zip64_comment, + writing_raw: true, // avoid recomputing the last file's header + flush_on_finish_file: false, + seek_possible: true, + auto_large_file: false, + }) + } + + /// `flush_on_finish_file` is designed to support a streaming `inner` that may unload flushed + /// bytes. It flushes a file's header and body once it starts writing another file. A ZipWriter + /// will not try to seek back into where a previous file was written unless + /// either [`ZipWriter::abort_file`] is called while [`ZipWriter::is_writing_file`] returns + /// false, or [`ZipWriter::deep_copy_file`] is called. In the latter case, it will only need to + /// read previously-written files and not overwrite them. + /// + /// Note: when using an `inner` that cannot overwrite flushed bytes, do not wrap it in a + /// [BufWriter], because that has a [Seek::seek] method that implicitly calls + /// [BufWriter::flush], and ZipWriter needs to seek backward to update each file's header with + /// the size and checksum after writing the body. + /// + /// This setting is false by default. + pub fn set_flush_on_finish_file(&mut self, flush_on_finish_file: bool) { + self.flush_on_finish_file = flush_on_finish_file; + } +} + +impl ZipWriter { + /// Adds another copy of a file already in this archive. This will produce a larger but more + /// widely-compatible archive compared to [Self::shallow_copy_file]. Does not copy alignment. + pub fn deep_copy_file(&mut self, src_name: &str, dest_name: &str) -> ZipResult<()> { + self.finish_file()?; + if src_name == dest_name || self.files.contains_key(dest_name) { + return Err(invalid!("That file already exists")); + } + let write_position = self.inner.get_plain().stream_position()?; + let src_index = self.index_by_name(src_name)?; + let src_data = &mut self.files[src_index]; + let src_data_start = src_data.data_start(self.inner.get_plain())?; + debug_assert!(src_data_start <= write_position); + let mut compressed_size = src_data.compressed_size; + if compressed_size > (write_position - src_data_start) { + compressed_size = write_position - src_data_start; + src_data.compressed_size = compressed_size; + } + let mut reader = BufReader::new(self.inner.get_plain()); + reader.seek(SeekFrom::Start(src_data_start))?; + let mut copy = vec![0; compressed_size as usize]; + reader.take(compressed_size).read_exact(&mut copy)?; + self.inner + .get_plain() + .seek(SeekFrom::Start(write_position))?; + let mut new_data = src_data.clone(); + let dest_name_raw = dest_name.as_bytes(); + new_data.file_name = dest_name.into(); + new_data.file_name_raw = dest_name_raw.into(); + new_data.is_utf8 = !dest_name.is_ascii(); + new_data.header_start = write_position; + let extra_data_start = write_position + + size_of::() as u64 + + new_data.file_name_raw.len() as u64; + new_data.extra_data_start = Some(extra_data_start); + if let Some(extra) = &src_data.extra_field { + let stripped = strip_alignment_extra_field(extra); + if !stripped.is_empty() { + new_data.extra_field = Some(stripped.into()); + } else { + new_data.extra_field = None; + } + } + + let mut data_start = extra_data_start; + if let Some(extra) = &new_data.extra_field { + data_start += extra.len() as u64; + } + new_data.data_start.take(); + new_data.data_start.get_or_init(|| data_start); + new_data.central_header_start = 0; + let block = new_data.local_block()?; + let index = self.insert_file_data(new_data)?; + let new_data = &self.files[index]; + let result: io::Result<()> = { + let plain_writer = self.inner.get_plain(); + block.write(plain_writer)?; + plain_writer.write_all(&new_data.file_name_raw)?; + if let Some(data) = &new_data.extra_field { + plain_writer.write_all(data)?; + } + debug_assert_eq!(data_start, plain_writer.stream_position()?); + self.writing_to_file = true; + plain_writer.write_all(©)?; + if self.flush_on_finish_file { + plain_writer.flush()?; + } + Ok(()) + }; + self.ok_or_abort_file(result)?; + self.writing_to_file = false; + Ok(()) + } + + /// Like `deep_copy_file`, but uses Path arguments. + /// + /// This function ensures that the '/' path separator is used and normalizes `.` and `..`. It + /// ignores any `..` or Windows drive letter that would produce a path outside the ZIP file's + /// root. + pub fn deep_copy_file_from_path, U: AsRef>( + &mut self, + src_path: T, + dest_path: U, + ) -> ZipResult<()> { + let src = path_to_string(src_path); + let dest = path_to_string(dest_path); + self.deep_copy_file(&src, &dest) + } + + /// Write the zip file into the backing stream, then produce a readable archive of that data. + /// + /// This method avoids parsing the central directory records at the end of the stream for + /// a slight performance improvement over running [`ZipArchive::new()`] on the output of + /// [`Self::finish()`]. + /// + ///``` + /// # fn main() -> Result<(), zip::result::ZipError> { + /// # #[cfg(any(feature = "deflate-flate2", not(feature = "_deflate-any")))] + /// # { + /// use std::io::{Cursor, Read, Write}; + /// use zip::{ZipArchive, ZipWriter, write::SimpleFileOptions}; + /// + /// let buf = Cursor::new(Vec::new()); + /// let mut zip = ZipWriter::new(buf); + /// let options = SimpleFileOptions::default(); + /// zip.start_file("a.txt", options)?; + /// zip.write_all(b"hello\n")?; + /// + /// let mut zip = zip.finish_into_readable()?; + /// let mut s: String = String::new(); + /// zip.by_name("a.txt")?.read_to_string(&mut s)?; + /// assert_eq!(s, "hello\n"); + /// # } + /// # Ok(()) + /// # } + ///``` + pub fn finish_into_readable(mut self) -> ZipResult> { + let central_start = self.finalize()?; + let inner = mem::replace(&mut self.inner, Closed).unwrap(); + let comment = mem::take(&mut self.comment); + let zip64_comment = mem::take(&mut self.zip64_comment); + let files = mem::take(&mut self.files); + + let archive = + ZipArchive::from_finalized_writer(files, comment, zip64_comment, inner, central_start)?; + Ok(archive) + } +} + +impl ZipWriter { + /// Initializes the archive. + /// + /// Before writing to this object, the [`ZipWriter::start_file`] function should be called. + /// After a successful write, the file remains open for writing. After a failed write, call + /// [`ZipWriter::is_writing_file`] to determine if the file remains open. + pub fn new(inner: W) -> ZipWriter { + ZipWriter { + inner: Storer(MaybeEncrypted::Unencrypted(inner)), + files: IndexMap::new(), + stats: Default::default(), + writing_to_file: false, + writing_raw: false, + comment: Box::new([]), + zip64_comment: None, + flush_on_finish_file: false, + seek_possible: true, + auto_large_file: false, + } + } + + /// Set automatically large file to true if needed + pub fn set_auto_large_file(mut self) -> Self { + self.auto_large_file = true; + self + } + + /// Returns true if a file is currently open for writing. + pub const fn is_writing_file(&self) -> bool { + self.writing_to_file && !self.inner.is_closed() + } + + /// Set ZIP archive comment. + pub fn set_comment(&mut self, comment: S) + where + S: Into>, + { + self.set_raw_comment(comment.into().into_boxed_bytes()) + } + + /// Set ZIP archive comment. + /// + /// This sets the raw bytes of the comment. The comment + /// is typically expected to be encoded in UTF-8. + pub fn set_raw_comment(&mut self, comment: Box<[u8]>) { + let max_comment_len = u16::MAX as usize; // 65,535 + if comment.len() > max_comment_len { + self.set_raw_zip64_comment(Some(comment)); + self.comment = Box::new([]); + } else { + self.comment = comment; + self.set_raw_zip64_comment(None); + } + } + + /// Get ZIP archive comment. + pub fn get_comment(&mut self) -> Result<&str, Utf8Error> { + from_utf8(self.get_raw_comment()) + } + + /// Get ZIP archive comment. + /// + /// This returns the raw bytes of the comment. The comment + /// is typically expected to be encoded in UTF-8. + pub const fn get_raw_comment(&self) -> &[u8] { + &self.comment + } + + /// Set ZIP64 archive comment. + pub fn set_zip64_comment(&mut self, comment: Option) + where + S: Into>, + { + self.set_raw_zip64_comment(comment.map(|v| v.into().into_boxed_bytes())) + } + + /// Set ZIP64 archive comment. + /// + /// This sets the raw bytes of the comment. The comment + /// is typically expected to be encoded in UTF-8. + pub fn set_raw_zip64_comment(&mut self, comment: Option>) { + self.zip64_comment = comment; + } + + /// Get ZIP64 archive comment. + pub fn get_zip64_comment(&mut self) -> Option> { + self.get_raw_zip64_comment().map(from_utf8) + } + + /// Get ZIP archive comment. + /// + /// This returns the raw bytes of the comment. The comment + /// is typically expected to be encoded in UTF-8. + pub fn get_raw_zip64_comment(&self) -> Option<&[u8]> { + self.zip64_comment.as_deref() + } + + /// Set the file length and crc32 manually. + /// + /// # Safety + /// + /// This overwrites the internal crc32 calculation. It should only be used in case + /// the underlying [Write] is written independently and you need to adjust the zip metadata. + pub unsafe fn set_file_metadata(&mut self, length: u64, crc32: u32) -> ZipResult<()> { + if !self.writing_to_file { + return Err(ZipError::Io(io::Error::other("No file has been started"))); + } + self.stats.hasher = Hasher::new_with_initial_len(crc32, length); + self.stats.bytes_written = length; + Ok(()) + } + + fn ok_or_abort_file>(&mut self, result: Result) -> ZipResult { + match result { + Err(e) => { + let _ = self.abort_file(); + Err(e.into()) + } + Ok(t) => Ok(t), + } + } + + /// Start a new file for with the requested options. + fn start_entry( + &mut self, + name: S, + mut options: FileOptions, + raw_values: Option, + ) -> ZipResult<()> { + self.finish_file()?; + + let header_start = self.inner.get_plain().stream_position()?; + let raw_values = raw_values.unwrap_or(ZipRawValues { + crc32: 0, + compressed_size: 0, + uncompressed_size: 0, + }); + + // Check if we're close to the 4GB boundary and force ZIP64 if needed + // This ensures we properly handle appending to files close to 4GB + if header_start > spec::ZIP64_BYTES_THR { + // Files that start on or past the 4GiB boundary are always ZIP64 + options.large_file = true; + } + + let mut extra_data = match options.extended_options.extra_data() { + Some(data) => data.to_vec(), + None => vec![], + }; + let central_extra_data = options.extended_options.central_extra_data(); + if let Some(zip64_block) = + Zip64ExtraFieldBlock::maybe_new(options.large_file, 0, 0, header_start) + { + let mut new_extra_data = zip64_block.serialize().into_vec(); + new_extra_data.append(&mut extra_data); + extra_data = new_extra_data; + } + + // Figure out the underlying compression_method and aes mode when using + // AES encryption. + let (compression_method, aes_mode) = match options.encrypt_with { + // Preserve AES method for raw copies without needing a password + #[cfg(feature = "aes-crypto")] + None if options.aes_mode.is_some() => (CompressionMethod::Aes, options.aes_mode), + #[cfg(feature = "aes-crypto")] + Some(EncryptWith::Aes { mode, .. }) => ( + CompressionMethod::Aes, + Some((mode, AesVendorVersion::Ae2, options.compression_method)), + ), + _ => (options.compression_method, None), + }; + + // Write AES encryption extra data. + #[allow(unused_mut)] + let mut aes_extra_data_start = 0; + #[cfg(feature = "aes-crypto")] + if let Some((mode, vendor, underlying)) = aes_mode { + // For raw copies of AES entries, write the correct AES extra data immediately + let mut body = [0; 7]; + [body[0], body[1]] = (vendor as u16).to_le_bytes(); // vendor version (1 or 2) + [body[2], body[3]] = *b"AE"; // vendor id + body[4] = mode as u8; // strength + [body[5], body[6]] = underlying.serialize_to_u16().to_le_bytes(); // real compression method + aes_extra_data_start = extra_data.len() as u64; + ExtendedFileOptions::add_extra_data_unchecked(&mut extra_data, 0x9901, &body)?; + } + let header_end = + header_start + size_of::() as u64 + name.to_string().len() as u64; + + if options.alignment > 1 { + let extra_data_end = header_end + extra_data.len() as u64; + let align = options.alignment as u64; + let unaligned_header_bytes = extra_data_end % align; + if unaligned_header_bytes != 0 { + let mut pad_length = (align - unaligned_header_bytes) as usize; + while pad_length < 6 { + pad_length += align as usize; + } + let new_len = extra_data.len() + pad_length; + if new_len > u16::MAX as usize { + // Alignment is impossible without exceeding extra field size limits. + // Skip alignment. + } else { + // Add an extra field to the extra_data, per APPNOTE 4.6.11 + let mut pad_body = vec![0; pad_length - 4]; + debug_assert!(pad_body.len() >= 2); + [pad_body[0], pad_body[1]] = options.alignment.to_le_bytes(); + ExtendedFileOptions::add_extra_data_unchecked( + &mut extra_data, + 0xa11e, + &pad_body, + )?; + debug_assert_eq!((extra_data.len() as u64 + header_end) % align, 0); + } + } + } + let extra_data_len = extra_data.len(); + if let Some(data) = central_extra_data { + if extra_data_len + data.len() > u16::MAX as usize { + return Err(invalid!( + "Extra data and central extra data must be less than 64KiB when combined" + )); + } + ExtendedFileOptions::validate_extra_data(data, true)?; + } + let mut file = ZipFileData::initialize_local_block( + name, + &options, + raw_values, + header_start, + None, + aes_extra_data_start, + compression_method, + aes_mode, + &extra_data, + ); + file.using_data_descriptor = + !self.seek_possible || matches!(options.encrypt_with, Some(EncryptWith::ZipCrypto(..))); + file.version_made_by = file.version_made_by.max(file.version_needed() as u8); + file.extra_data_start = Some(header_end); + let index = self.insert_file_data(file)?; + self.writing_to_file = true; + let result: ZipResult<()> = { + ExtendedFileOptions::validate_extra_data(&extra_data, false)?; + let file = &mut self.files[index]; + let block = file.local_block()?; + let writer = self.inner.get_plain(); + block.write(writer)?; + // file name + writer.write_all(&file.file_name_raw)?; + if extra_data_len > 0 { + writer.write_all(&extra_data)?; + file.extra_field = Some(extra_data.into()); + } + Ok(()) + }; + self.ok_or_abort_file(result)?; + let writer = self.inner.get_plain(); + self.stats.start = writer.stream_position()?; + match options.encrypt_with { + #[cfg(feature = "aes-crypto")] + Some(EncryptWith::Aes { mode, password }) => { + let aeswriter = AesWriter::new( + mem::replace(&mut self.inner, Closed).unwrap(), + mode, + password.as_bytes(), + )?; + self.inner = Storer(MaybeEncrypted::Aes(aeswriter)); + } + Some(EncryptWith::ZipCrypto(keys, ..)) => { + let file = &mut self.files[index]; + // With ZipCrypto, we _need_ to use a data descriptor so that + // we can initialize the stream properly. + let mut zipwriter = crate::zipcrypto::ZipCryptoWriter { + writer: mem::replace(&mut self.inner, Closed).unwrap(), + keys, + }; + self.stats.start = zipwriter.writer.stream_position()?; + // crypto_header is counted as part of the data + let mut crypto_header = [0u8; 12]; + // The last two bytes of the header should either be the CRC of + // the file _OR_ the last timepart of the last modified time if + // a data descriptor is used. + // + // However, this header is encrypted, and ZipCrypto is not a + // seekable encryption algorithm - an earlier plaintext byte + // will have an impact on the later values. + // + // This makes it impossible to write the file without first + // calculating its CRC. To avoid having to read the file twice + // or keeping the entire file in memory, we force all ZipCrypto + // files to use a data descriptor. This way, we can use the + // local_modified_time as a password check byte instead of the + // CRC. + crypto_header[10..=11].copy_from_slice( + &file + .last_modified_time + .unwrap_or_else(DateTime::default_for_write) + .timepart() + .to_le_bytes(), + ); + let result = zipwriter.write_all(&crypto_header); + self.ok_or_abort_file(result)?; + self.inner = Storer(MaybeEncrypted::ZipCrypto(zipwriter)); + } + None => {} + } + let file = &mut self.files[index]; + debug_assert!(file.data_start.get().is_none()); + file.data_start.get_or_init(|| self.stats.start); + self.stats.bytes_written = 0; + self.stats.hasher = Hasher::new(); + Ok(()) + } + + fn insert_file_data(&mut self, file: ZipFileData) -> ZipResult { + if self.files.contains_key(&file.file_name) { + return Err(invalid!("Duplicate filename: {}", file.file_name)); + } + let (index, _) = self.files.insert_full(file.file_name.clone(), file); + Ok(index) + } + + fn finish_file(&mut self) -> ZipResult<()> { + if !self.writing_to_file { + return Ok(()); + } + + let make_plain_writer = self.inner.prepare_next_writer( + Stored, + None, + #[cfg(feature = "deflate-zopfli")] + None, + )?; + self.inner.switch_to(make_plain_writer)?; + self.switch_to_non_encrypting_writer()?; + let writer = self.inner.get_plain(); + + if !self.writing_raw { + let file = match self.files.last_mut() { + None => return Ok(()), + Some((_, f)) => f, + }; + file.uncompressed_size = self.stats.bytes_written; + + let file_end = writer.stream_position()?; + debug_assert!(file_end >= self.stats.start); + file.compressed_size = file_end - self.stats.start; + + if !file.using_data_descriptor { + // Not using a data descriptor means the underlying writer + // supports seeking, so we can go back and update the AES Extra + // Data header to use AE1 for large files. + update_aes_extra_data(writer, file, self.stats.bytes_written)?; + } + + let crc = !matches!(file.aes_mode, Some((_, AesVendorVersion::Ae2, _))); + + file.crc32 = if crc { + self.stats.hasher.clone().finalize() + } else { + 0 + }; + + if file.using_data_descriptor { + file.write_data_descriptor(writer, self.auto_large_file)?; + } else { + update_local_file_header(writer, file)?; + writer.seek(SeekFrom::Start(file_end))?; + } + } + if self.flush_on_finish_file { + let result = writer.flush(); + self.ok_or_abort_file(result)?; + } + + self.writing_to_file = false; + Ok(()) + } + + fn switch_to_non_encrypting_writer(&mut self) -> Result<(), ZipError> { + match mem::replace(&mut self.inner, Closed) { + #[cfg(feature = "aes-crypto")] + Storer(MaybeEncrypted::Aes(writer)) => { + self.inner = Storer(MaybeEncrypted::Unencrypted(writer.finish()?)); + } + Storer(MaybeEncrypted::ZipCrypto(writer)) => { + self.inner = Storer(MaybeEncrypted::Unencrypted(writer.finish()?)) + } + Storer(MaybeEncrypted::Unencrypted(w)) => { + self.inner = Storer(MaybeEncrypted::Unencrypted(w)) + } + _ => unreachable!(), + } + Ok(()) + } + + /// Removes the file currently being written from the archive if there is one, or else removes + /// the file most recently written. + pub fn abort_file(&mut self) -> ZipResult<()> { + let (_, last_file) = self.files.pop().ok_or(ZipError::FileNotFound)?; + let make_plain_writer = self.inner.prepare_next_writer( + Stored, + None, + #[cfg(feature = "deflate-zopfli")] + None, + )?; + self.inner.switch_to(make_plain_writer)?; + self.switch_to_non_encrypting_writer()?; + // Make sure this is the last file, and that no shallow copies of it remain; otherwise we'd + // overwrite a valid file and corrupt the archive + let rewind_safe: bool = match last_file.data_start.get() { + None => self.files.is_empty(), + Some(last_file_start) => self.files.values().all(|file| { + file.data_start + .get() + .is_some_and(|start| start < last_file_start) + }), + }; + if rewind_safe { + self.inner + .get_plain() + .seek(SeekFrom::Start(last_file.header_start))?; + } + self.writing_to_file = false; + Ok(()) + } + + /// Create a file in the archive and start writing its' contents. The file must not have the + /// same name as a file already in the archive. + /// + /// The data should be written using the [`Write`] implementation on this [`ZipWriter`] + pub fn start_file( + &mut self, + name: S, + mut options: FileOptions, + ) -> ZipResult<()> { + options.normalize(); + let make_new_self = self.inner.prepare_next_writer( + options.compression_method, + options.compression_level, + #[cfg(feature = "deflate-zopfli")] + options.zopfli_buffer_size, + )?; + self.start_entry(name, options, None)?; + let result = self.inner.switch_to(make_new_self); + self.ok_or_abort_file(result)?; + self.writing_raw = false; + Ok(()) + } + + /* TODO: link to/use Self::finish_into_readable() from https://github.com/zip-rs/zip/pull/400 in + * this docstring. */ + /// Copy over the entire contents of another archive verbatim. + /// + /// This method extracts file metadata from the `source` archive, then simply performs a single + /// big [`io::copy()`](io::copy) to transfer all the actual file contents without any + /// decompression or decryption. This is more performant than the equivalent operation of + /// calling [`Self::raw_copy_file()`] for each entry from the `source` archive in sequence. + /// + ///``` + /// # fn main() -> Result<(), zip::result::ZipError> { + /// # #[cfg(any(feature = "deflate-flate2", not(feature = "_deflate-any")))] + /// # { + /// use std::io::{Cursor, Write, Read}; + /// use zip::{ZipArchive, ZipWriter, write::SimpleFileOptions}; + /// + /// let buf = Cursor::new(Vec::new()); + /// let mut zip = ZipWriter::new(buf); + /// zip.start_file("a.txt", SimpleFileOptions::default())?; + /// zip.write_all(b"hello\n")?; + /// let src = ZipArchive::new(zip.finish()?)?; + /// + /// let buf = Cursor::new(Vec::new()); + /// let mut zip = ZipWriter::new(buf); + /// zip.start_file("b.txt", SimpleFileOptions::default())?; + /// zip.write_all(b"hey\n")?; + /// let src2 = ZipArchive::new(zip.finish()?)?; + /// + /// let buf = Cursor::new(Vec::new()); + /// + /// let mut zip = ZipWriter::new(buf); + /// zip.merge_archive(src)?; + /// zip.merge_archive(src2)?; + /// let mut result = ZipArchive::new(zip.finish()?)?; + /// + /// let mut s: String = String::new(); + /// result.by_name("a.txt")?.read_to_string(&mut s)?; + /// assert_eq!(s, "hello\n"); + /// s.clear(); + /// result.by_name("b.txt")?.read_to_string(&mut s)?; + /// assert_eq!(s, "hey\n"); + /// # } + /// # Ok(()) + /// # } + ///``` + pub fn merge_archive(&mut self, mut source: ZipArchive) -> ZipResult<()> + where + R: Read + Seek, + { + self.finish_file()?; + + /* Ensure we accept the file contents on faith (and avoid overwriting the data). + * See raw_copy_file_rename(). */ + self.writing_to_file = true; + self.writing_raw = true; + + let writer = self.inner.get_plain(); + /* Get the file entries from the source archive. */ + let new_files = source.merge_contents(writer)?; + + /* These file entries are now ours! */ + self.files.extend(new_files); + + Ok(()) + } + + /// Starts a file, taking a Path as argument. + /// + /// This function ensures that the '/' path separator is used and normalizes `.` and `..`. It + /// ignores any `..` or Windows drive letter that would produce a path outside the ZIP file's + /// root. + pub fn start_file_from_path>( + &mut self, + path: P, + options: FileOptions, + ) -> ZipResult<()> { + self.start_file(path_to_string(path), options) + } + + /// Add a new file using the already compressed data from a ZIP file being read and renames it, this + /// allows faster copies of the `ZipFile` since there is no need to decompress and compress it again. + /// Any `ZipFile` metadata is copied and not checked, for example the file CRC. + /// + /// ```no_run + /// use std::fs::File; + /// use std::io::{Read, Seek, Write}; + /// use zip::{ZipArchive, ZipWriter}; + /// + /// fn copy_rename( + /// src: &mut ZipArchive, + /// dst: &mut ZipWriter, + /// ) -> zip::result::ZipResult<()> + /// where + /// R: Read + Seek, + /// W: Write + Seek, + /// { + /// // Retrieve file entry by name + /// let file = src.by_name("src_file.txt")?; + /// + /// // Copy and rename the previously obtained file entry to the destination zip archive + /// dst.raw_copy_file_rename(file, "new_name.txt")?; + /// + /// Ok(()) + /// } + /// ``` + pub fn raw_copy_file_rename( + &mut self, + file: ZipFile, + name: S, + ) -> ZipResult<()> { + let options = file.options(); + self.raw_copy_file_rename_internal(file, name, options) + } + + fn raw_copy_file_rename_internal( + &mut self, + mut file: ZipFile, + name: S, + options: SimpleFileOptions, + ) -> ZipResult<()> { + let raw_values = ZipRawValues { + crc32: file.crc32(), + compressed_size: file.compressed_size(), + uncompressed_size: file.size(), + }; + + self.start_entry(name, options, Some(raw_values))?; + self.writing_to_file = true; + self.writing_raw = true; + + io::copy(&mut file.take_raw_reader()?, self)?; + self.finish_file() + } + + /// Like `raw_copy_file_to_path`, but uses Path arguments. + /// + /// This function ensures that the '/' path separator is used and normalizes `.` and `..`. It + /// ignores any `..` or Windows drive letter that would produce a path outside the ZIP file's + /// root. + pub fn raw_copy_file_to_path>( + &mut self, + file: ZipFile, + path: P, + ) -> ZipResult<()> { + self.raw_copy_file_rename(file, path_to_string(path)) + } + + /// Add a new file using the already compressed data from a ZIP file being read, this allows faster + /// copies of the `ZipFile` since there is no need to decompress and compress it again. Any `ZipFile` + /// metadata is copied and not checked, for example the file CRC. + /// + /// ```no_run + /// use std::fs::File; + /// use std::io::{Read, Seek, Write}; + /// use zip::{ZipArchive, ZipWriter}; + /// + /// fn copy(src: &mut ZipArchive, dst: &mut ZipWriter) -> zip::result::ZipResult<()> + /// where + /// R: Read + Seek, + /// W: Write + Seek, + /// { + /// // Retrieve file entry by name + /// let file = src.by_name("src_file.txt")?; + /// + /// // Copy the previously obtained file entry to the destination zip archive + /// dst.raw_copy_file(file)?; + /// + /// Ok(()) + /// } + /// ``` + pub fn raw_copy_file(&mut self, file: ZipFile) -> ZipResult<()> { + let name = file.name().to_owned(); + self.raw_copy_file_rename(file, name) + } + + /// Add a new file using the already compressed data from a ZIP file being read and set the last + /// modified date and unix mode. This allows faster copies of the `ZipFile` since there is no need + /// to decompress and compress it again. Any `ZipFile` metadata other than the last modified date + /// and the unix mode is copied and not checked, for example the file CRC. + /// + /// ```no_run + /// use std::io::{Read, Seek, Write}; + /// use zip::{DateTime, ZipArchive, ZipWriter}; + /// + /// fn copy(src: &mut ZipArchive, dst: &mut ZipWriter) -> zip::result::ZipResult<()> + /// where + /// R: Read + Seek, + /// W: Write + Seek, + /// { + /// // Retrieve file entry by name + /// let file = src.by_name("src_file.txt")?; + /// + /// // Copy the previously obtained file entry to the destination zip archive + /// dst.raw_copy_file_touch(file, DateTime::default(), Some(0o644))?; + /// + /// Ok(()) + /// } + /// ``` + pub fn raw_copy_file_touch( + &mut self, + file: ZipFile, + last_modified_time: DateTime, + unix_mode: Option, + ) -> ZipResult<()> { + let name = file.name().to_owned(); + + let mut options = file.options(); + + options = options.last_modified_time(last_modified_time); + + if let Some(perms) = unix_mode { + options = options.unix_permissions(perms); + } + + options.normalize(); + + self.raw_copy_file_rename_internal(file, name, options) + } + + /// Add a directory entry. + /// + /// As directories have no content, you must not call [`ZipWriter::write`] before adding a new file. + pub fn add_directory( + &mut self, + name: S, + mut options: FileOptions, + ) -> ZipResult<()> + where + S: Into, + { + if options.permissions.is_none() { + options.permissions = Some(0o755); + } + *options.permissions.as_mut().unwrap() |= 0o40000; + options.compression_method = Stored; + options.encrypt_with = None; + + let name_as_string = name.into(); + // Append a slash to the filename if it does not end with it. + let name_with_slash = match name_as_string.chars().last() { + Some('/') | Some('\\') => name_as_string, + _ => name_as_string + "/", + }; + + self.start_entry(name_with_slash, options, None)?; + self.writing_to_file = false; + self.switch_to_non_encrypting_writer()?; + Ok(()) + } + + /// Add a directory entry, taking a Path as argument. + /// + /// This function ensures that the '/' path separator is used and normalizes `.` and `..`. It + /// ignores any `..` or Windows drive letter that would produce a path outside the ZIP file's + /// root. + pub fn add_directory_from_path>( + &mut self, + path: P, + options: FileOptions, + ) -> ZipResult<()> { + self.add_directory(path_to_string(path), options) + } + + /// Finish the last file and write all other zip-structures + /// + /// This will return the writer, but one should normally not append any data to the end of the file. + /// Note that the zipfile will also be finished on drop. + pub fn finish(mut self) -> ZipResult { + let _central_start = self.finalize()?; + let inner = mem::replace(&mut self.inner, Closed); + Ok(inner.unwrap()) + } + + /// Add a symlink entry. + /// + /// The zip archive will contain an entry for path `name` which is a symlink to `target`. + /// + /// No validation or normalization of the paths is performed. For best results, + /// callers should normalize `\` to `/` and ensure symlinks are relative to other + /// paths within the zip archive. + /// + /// WARNING: not all zip implementations preserve symlinks on extract. Some zip + /// implementations may materialize a symlink as a regular file, possibly with the + /// content incorrectly set to the symlink target. For maximum portability, consider + /// storing a regular file instead. + pub fn add_symlink( + &mut self, + name: N, + target: T, + mut options: FileOptions, + ) -> ZipResult<()> { + if options.permissions.is_none() { + options.permissions = Some(0o777); + } + *options.permissions.as_mut().unwrap() |= S_IFLNK; + // The symlink target is stored as file content. And compressing the target path + // likely wastes space. So always store. + options.compression_method = Stored; + + self.start_entry(name, options, None)?; + self.writing_to_file = true; + let result = self.write_all(target.to_string().as_bytes()); + self.ok_or_abort_file(result)?; + self.writing_raw = false; + self.finish_file()?; + + Ok(()) + } + + /// Add a symlink entry, taking Paths to the location and target as arguments. + /// + /// This function ensures that the '/' path separator is used and normalizes `.` and `..`. It + /// ignores any `..` or Windows drive letter that would produce a path outside the ZIP file's + /// root. + pub fn add_symlink_from_path, T: AsRef, E: FileOptionExtension>( + &mut self, + path: P, + target: T, + options: FileOptions, + ) -> ZipResult<()> { + self.add_symlink(path_to_string(path), path_to_string(target), options) + } + + fn finalize(&mut self) -> ZipResult { + self.finish_file()?; + + let mut central_start = self.write_central_and_footer()?; + let writer = self.inner.get_plain(); + let footer_end = writer.stream_position()?; + let archive_end = writer.seek(SeekFrom::End(0))?; + if footer_end < archive_end { + // Data from an aborted file is past the end of the footer. + + // Overwrite the magic so the footer is no longer valid. + writer.seek(SeekFrom::Start(central_start))?; + writer.write_u32_le(0)?; + writer.seek(SeekFrom::Start( + footer_end - size_of::() as u64 - self.comment.len() as u64, + ))?; + writer.write_u32_le(0)?; + + // Rewrite the footer at the actual end. + let central_and_footer_size = footer_end - central_start; + writer.seek(SeekFrom::End(-(central_and_footer_size as i64)))?; + central_start = self.write_central_and_footer()?; + debug_assert!(self.inner.get_plain().stream_position()? == archive_end); + } + + Ok(central_start) + } + + fn write_central_and_footer(&mut self) -> Result { + let writer = self.inner.get_plain(); + + let mut version_needed = MIN_VERSION as u16; + let central_start = writer.stream_position()?; + for file in self.files.values() { + write_central_directory_header(writer, file)?; + version_needed = version_needed.max(file.version_needed()); + } + let central_size = writer.stream_position()? - central_start; + let is64 = self.files.len() > spec::ZIP64_ENTRY_THR + || central_size.max(central_start) > spec::ZIP64_BYTES_THR + || self.zip64_comment.is_some(); + + if is64 { + let comment = self.zip64_comment.clone().unwrap_or_default(); + + let zip64_footer = spec::Zip64CentralDirectoryEnd { + record_size: comment.len() as u64 + 44, + version_made_by: version_needed, + version_needed_to_extract: version_needed, + disk_number: 0, + disk_with_central_directory: 0, + number_of_files_on_this_disk: self.files.len() as u64, + number_of_files: self.files.len() as u64, + central_directory_size: central_size, + central_directory_offset: central_start, + extensible_data_sector: comment, + }; + + zip64_footer.write(writer)?; + + let zip64_footer = spec::Zip64CentralDirectoryEndLocator { + disk_with_central_directory: 0, + end_of_central_directory_offset: central_start + central_size, + number_of_disks: 1, + }; + + zip64_footer.write(writer)?; + } + + let number_of_files = self.files.len().min(spec::ZIP64_ENTRY_THR) as u16; + let footer = spec::Zip32CentralDirectoryEnd { + disk_number: 0, + disk_with_central_directory: 0, + zip_file_comment: self.comment.clone(), + number_of_files_on_this_disk: number_of_files, + number_of_files, + central_directory_size: central_size.min(spec::ZIP64_BYTES_THR) as u32, + central_directory_offset: central_start.min(spec::ZIP64_BYTES_THR) as u32, + }; + + footer.write(writer)?; + Ok(central_start) + } + + fn index_by_name(&self, name: &str) -> ZipResult { + self.files.get_index_of(name).ok_or(ZipError::FileNotFound) + } + + /// Adds another entry to the central directory referring to the same content as an existing + /// entry. The file's local-file header will still refer to it by its original name, so + /// unzipping the file will technically be unspecified behavior. [ZipArchive] ignores the + /// filename in the local-file header and treat the central directory as authoritative. However, + /// some other software (e.g. Minecraft) will refuse to extract a file copied this way. + pub fn shallow_copy_file(&mut self, src_name: &str, dest_name: &str) -> ZipResult<()> { + self.finish_file()?; + if src_name == dest_name { + return Err(invalid!("Trying to copy a file to itself")); + } + let src_index = self.index_by_name(src_name)?; + let mut dest_data = self.files[src_index].to_owned(); + dest_data.file_name = dest_name.to_string().into(); + dest_data.file_name_raw = dest_name.to_string().into_bytes().into(); + dest_data.central_header_start = 0; + self.insert_file_data(dest_data)?; + + Ok(()) + } + + /// Like `shallow_copy_file`, but uses Path arguments. + /// + /// This function ensures that the '/' path separator is used and normalizes `.` and `..`. It + /// ignores any `..` or Windows drive letter that would produce a path outside the ZIP file's + /// root. + pub fn shallow_copy_file_from_path, U: AsRef>( + &mut self, + src_path: T, + dest_path: U, + ) -> ZipResult<()> { + self.shallow_copy_file(&path_to_string(src_path), &path_to_string(dest_path)) + } +} + +impl ZipWriter> { + /// Creates a writer that doesn't require the inner writer to implement [Seek], but where + /// operations that would overwrite previously-written bytes or cause subsequent operations to + /// do so (such as `abort_file`) will always return an error. + pub fn new_stream(inner: W) -> ZipWriter> { + ZipWriter { + inner: Storer(MaybeEncrypted::Unencrypted(StreamWriter::new(inner))), + files: IndexMap::new(), + stats: Default::default(), + writing_to_file: false, + writing_raw: false, + comment: Box::new([]), + zip64_comment: None, + flush_on_finish_file: false, + seek_possible: false, + auto_large_file: false, + } + } +} + +impl Drop for ZipWriter { + fn drop(&mut self) { + if !self.inner.is_closed() { + if let Err(e) = self.finalize() { + let _ = write!( + io::stderr(), + "ZipWriter::drop: failed to finalize archive: {e:?}" + ); + } + } + } +} + +type SwitchWriterFunction = Box) -> ZipResult>>; + +impl GenericZipWriter { + fn prepare_next_writer( + &self, + compression: CompressionMethod, + compression_level: Option, + #[cfg(feature = "deflate-zopfli")] zopfli_buffer_size: Option, + ) -> ZipResult> { + if let Closed = self { + return Err( + io::Error::new(io::ErrorKind::BrokenPipe, "ZipWriter was already closed").into(), + ); + } + + match compression { + Stored => { + if compression_level.is_some() { + Err(UnsupportedArchive("Unsupported compression level")) + } else { + Ok(Box::new(|bare| Ok(Storer(bare)))) + } + } + #[allow(unreachable_code)] + #[cfg(feature = "_deflate-any")] + CompressionMethod::Deflated => { + let default = crate::cfg_if_expr! { + i64: + #[cfg(feature = "deflate-flate2")] => Compression::default().level() as i64, + #[cfg(feature = "deflate-zopfli")] => 24, + _ => compile_error!("could not calculate default: unknown deflate variant"), + }; + + let level = validate_value_in_range( + compression_level.unwrap_or(default), + deflate_compression_level_range(), + ) + .ok_or(UnsupportedArchive("Unsupported compression level"))? + as u32; + + #[cfg(feature = "deflate-zopfli")] + { + macro_rules! deflate_zopfli_and_return { + ($bare:expr, $best_non_zopfli:expr) => {{ + let options = Options { + iteration_count: NonZeroU64::try_from( + (level - $best_non_zopfli) as u64, + ) + .unwrap(), + ..Default::default() + }; + Ok(Box::new(move |bare| { + Ok(match zopfli_buffer_size { + Some(size) => BufferedZopfliDeflater(BufWriter::with_capacity( + size, + zopfli::DeflateEncoder::new( + options, + Default::default(), + bare, + ), + )), + None => ZopfliDeflater(zopfli::DeflateEncoder::new( + options, + Default::default(), + bare, + )), + }) + })) + }}; + } + + crate::cfg_if! { + if #[cfg(feature = "deflate-flate2")] { + let best_non_zopfli = Compression::best().level(); + if level > best_non_zopfli { + return deflate_zopfli_and_return!(bare, best_non_zopfli); + } + } else { + let best_non_zopfli = 9; + return deflate_zopfli_and_return!(bare, best_non_zopfli); + } + } + } + crate::cfg_if_expr! { + ZipResult>: + #[cfg(feature = "deflate-flate2")] => Ok(Box::new(move |bare| { + Ok(GenericZipWriter::Deflater(DeflateEncoder::new( + bare, + Compression::new(level), + ))) + })), + _ => unreachable!("deflate writer: have no fallback for this case") + } + } + #[cfg(feature = "deflate64")] + CompressionMethod::Deflate64 => { + Err(UnsupportedArchive("Compressing Deflate64 is not supported")) + } + #[cfg(feature = "_bzip2_any")] + CompressionMethod::Bzip2 => { + let level = validate_value_in_range( + compression_level.unwrap_or(bzip2::Compression::default().level() as i64), + bzip2_compression_level_range(), + ) + .ok_or(UnsupportedArchive("Unsupported compression level"))? + as u32; + Ok(Box::new(move |bare| { + Ok(Bzip2(BzEncoder::new(bare, bzip2::Compression::new(level)))) + })) + } + CompressionMethod::AES => Err(UnsupportedArchive( + "AES encryption is enabled through FileOptions::with_aes_encryption", + )), + #[cfg(feature = "zstd")] + CompressionMethod::Zstd => { + let level = validate_value_in_range( + compression_level.unwrap_or(zstd::DEFAULT_COMPRESSION_LEVEL as i64), + zstd::compression_level_range(), + ) + .ok_or(UnsupportedArchive("Unsupported compression level"))?; + Ok(Box::new(move |bare| { + Ok(Zstd( + ZstdEncoder::new(bare, level as i32).map_err(ZipError::Io)?, + )) + })) + } + #[cfg(feature = "legacy-zip")] + CompressionMethod::Shrink => Err(ZipError::UnsupportedArchive( + "Shrink compression unsupported", + )), + #[cfg(feature = "legacy-zip")] + CompressionMethod::Reduce(_) => Err(ZipError::UnsupportedArchive( + "Reduce compression unsupported", + )), + #[cfg(feature = "legacy-zip")] + CompressionMethod::Implode => Err(ZipError::UnsupportedArchive( + "Implode compression unsupported", + )), + #[cfg(feature = "lzma")] + CompressionMethod::Lzma => { + Err(UnsupportedArchive("LZMA isn't supported for compression")) + } + #[cfg(feature = "xz")] + CompressionMethod::Xz => { + let level = validate_value_in_range(compression_level.unwrap_or(6), 0..=9) + .ok_or(UnsupportedArchive("Unsupported compression level"))? + as u32; + Ok(Box::new(move |bare| { + Ok(Xz(Box::new( + lzma_rust2::XzWriter::new(bare, lzma_rust2::XzOptions::with_preset(level)) + .map_err(ZipError::Io)?, + ))) + })) + } + #[cfg(feature = "ppmd")] + CompressionMethod::Ppmd => { + const ORDERS: [u32; 10] = [0, 4, 5, 6, 7, 8, 9, 10, 11, 12]; + + let level = validate_value_in_range(compression_level.unwrap_or(7), 1..=9) + .ok_or(UnsupportedArchive("Unsupported compression level"))? + as u32; + + let order = ORDERS[level as usize]; + let memory_size = 1 << (level + 19); + let memory_size_mb = memory_size / 1024 / 1024; + + Ok(Box::new(move |mut bare| { + let parameter: u16 = (order as u16 - 1) + + ((memory_size_mb - 1) << 4) as u16 + + ((ppmd_rust::RestoreMethod::Restart as u16) << 12); + + bare.write_all(¶meter.to_le_bytes()) + .map_err(ZipError::Io)?; + + let encoder = ppmd_rust::Ppmd8Encoder::new( + bare, + order, + memory_size, + ppmd_rust::RestoreMethod::Restart, + ) + .map_err(|error| match error { + ppmd_rust::Error::RangeDecoderInitialization => ZipError::InvalidArchive( + "PPMd range coder initialization failed".into(), + ), + ppmd_rust::Error::InvalidParameter => { + ZipError::InvalidArchive("Invalid PPMd parameter".into()) + } + ppmd_rust::Error::IoError(io_error) => ZipError::Io(io_error), + ppmd_rust::Error::MemoryAllocation => ZipError::Io(io::Error::new( + ErrorKind::OutOfMemory, + "PPMd could not allocate memory", + )), + })?; + + Ok(Ppmd(Box::new(encoder))) + })) + } + #[allow(deprecated)] + CompressionMethod::Unsupported(..) => { + Err(UnsupportedArchive("Unsupported compression")) + } + } + } + + fn switch_to(&mut self, make_new_self: SwitchWriterFunction) -> ZipResult<()> { + let bare = match mem::replace(self, Closed) { + Storer(w) => w, + #[cfg(feature = "deflate-flate2")] + Deflater(w) => w.finish()?, + #[cfg(feature = "deflate-zopfli")] + ZopfliDeflater(w) => w.finish()?, + #[cfg(feature = "deflate-zopfli")] + BufferedZopfliDeflater(w) => w + .into_inner() + .map_err(|e| ZipError::Io(e.into_error()))? + .finish()?, + #[cfg(feature = "_bzip2_any")] + Bzip2(w) => w.finish()?, + #[cfg(feature = "zstd")] + Zstd(w) => w.finish()?, + #[cfg(feature = "xz")] + Xz(w) => w.finish()?, + #[cfg(feature = "ppmd")] + Ppmd(w) => { + // ZIP needs to encode an end marker (7z for example doesn't encode one). + w.finish(true)? + } + Closed => { + return Err(io::Error::new( + io::ErrorKind::BrokenPipe, + "ZipWriter was already closed", + ) + .into()); + } + }; + *self = make_new_self(bare)?; + Ok(()) + } + + fn ref_mut(&mut self) -> Option<&mut dyn Write> { + match self { + Storer(ref mut w) => Some(w as &mut dyn Write), + #[cfg(feature = "deflate-flate2")] + Deflater(ref mut w) => Some(w as &mut dyn Write), + #[cfg(feature = "deflate-zopfli")] + ZopfliDeflater(w) => Some(w as &mut dyn Write), + #[cfg(feature = "deflate-zopfli")] + BufferedZopfliDeflater(w) => Some(w as &mut dyn Write), + #[cfg(feature = "_bzip2_any")] + Bzip2(ref mut w) => Some(w as &mut dyn Write), + #[cfg(feature = "zstd")] + Zstd(ref mut w) => Some(w as &mut dyn Write), + #[cfg(feature = "xz")] + Xz(ref mut w) => Some(w as &mut dyn Write), + #[cfg(feature = "ppmd")] + Ppmd(ref mut w) => Some(w as &mut dyn Write), + Closed => None, + } + } + + const fn is_closed(&self) -> bool { + matches!(*self, Closed) + } + + fn get_plain(&mut self) -> &mut W { + match *self { + Storer(MaybeEncrypted::Unencrypted(ref mut w)) => w, + _ => panic!("Should have switched to stored and unencrypted beforehand"), + } + } + + fn unwrap(self) -> W { + match self { + Storer(MaybeEncrypted::Unencrypted(w)) => w, + _ => panic!("Should have switched to stored and unencrypted beforehand"), + } + } +} + +#[cfg(feature = "_deflate-any")] +fn deflate_compression_level_range() -> std::ops::RangeInclusive { + let min = crate::cfg_if_expr! { + i64: + #[cfg(feature = "deflate-flate2")] => Compression::fast().level() as i64, + #[cfg(feature = "deflate-zopfli")] => 1, + _ => compile_error!("min: unknown deflate variant"), + }; + + let max = crate::cfg_if_expr! { + i64: + #[cfg(feature = "deflate-zopfli")] => 264, + #[cfg(feature = "deflate-flate2")] => Compression::best().level() as i64, + _ => compile_error!("max: unknown deflate variant"), + }; + + min..=max +} + +#[cfg(feature = "_bzip2_any")] +fn bzip2_compression_level_range() -> std::ops::RangeInclusive { + let min = bzip2::Compression::fast().level() as i64; + let max = bzip2::Compression::best().level() as i64; + min..=max +} + +#[cfg(any( + feature = "_deflate-any", + feature = "_bzip2_any", + feature = "ppmd", + feature = "xz", + feature = "zstd", +))] +fn validate_value_in_range>( + value: T, + range: std::ops::RangeInclusive, +) -> Option { + if range.contains(&value.try_into().ok()?) { + Some(value) + } else { + None + } +} + +fn update_aes_extra_data( + writer: &mut W, + file: &mut ZipFileData, + bytes_written: u64, +) -> ZipResult<()> { + let Some((aes_mode, version, compression_method)) = &mut file.aes_mode else { + return Ok(()); + }; + + // We prefer using AE-1 which provides an extra CRC check, but for small files we + // switch to AE-2 to prevent being able to use the CRC value to to reconstruct the + // unencrypted contents. + // + // We can only do this when the underlying writer supports + // seek operations, so we gate this behind using_data_descriptor. + // + // C.f. https://www.winzip.com/en/support/aes-encryption/#crc-faq + *version = if bytes_written < 20 { + AesVendorVersion::Ae2 + } else { + AesVendorVersion::Ae1 + }; + + let extra_data_start = file.extra_data_start.unwrap(); + + writer.seek(SeekFrom::Start( + extra_data_start + file.aes_extra_data_start, + ))?; + + let mut buf = Vec::new(); + + /* TODO: implement this using the Block trait! */ + // Extra field header ID. + buf.write_u16_le(0x9901)?; + // Data size. + buf.write_u16_le(7)?; + // Integer version number. + buf.write_u16_le(*version as u16)?; + // Vendor ID. + buf.write_all(b"AE")?; + // AES encryption strength. + buf.write_all(&[*aes_mode as u8])?; + // Real compression method. + buf.write_u16_le(compression_method.serialize_to_u16())?; + + writer.write_all(&buf)?; + + let aes_extra_data_start = file.aes_extra_data_start as usize; + let extra_field = Arc::get_mut(file.extra_field.as_mut().unwrap()).unwrap(); + extra_field[aes_extra_data_start..aes_extra_data_start + buf.len()].copy_from_slice(&buf); + + Ok(()) +} + +fn update_local_file_header( + writer: &mut T, + file: &mut ZipFileData, +) -> ZipResult<()> { + const CRC32_OFFSET: u64 = 14; + writer.seek(SeekFrom::Start(file.header_start + CRC32_OFFSET))?; + writer.write_u32_le(file.crc32)?; + if file.large_file { + writer.write_u32_le(spec::ZIP64_BYTES_THR as u32)?; + writer.write_u32_le(spec::ZIP64_BYTES_THR as u32)?; + + update_local_zip64_extra_field(writer, file)?; + + file.compressed_size = spec::ZIP64_BYTES_THR; + file.uncompressed_size = spec::ZIP64_BYTES_THR; + } else { + // check compressed size as well as it can also be slightly larger than uncompressed size + if file.compressed_size > spec::ZIP64_BYTES_THR { + return Err(ZipError::Io(io::Error::other( + "Large file option has not been set", + ))); + } + writer.write_u32_le(file.compressed_size as u32)?; + // uncompressed size is already checked on write to catch it as soon as possible + writer.write_u32_le(file.uncompressed_size as u32)?; + } + Ok(()) +} + +fn write_central_directory_header(writer: &mut T, file: &ZipFileData) -> ZipResult<()> { + let mut block = file.block()?; + let stripped_extra = if let Some(extra) = &file.extra_field { + strip_alignment_extra_field(extra) + } else { + Vec::new() + }; + let central_len = file.central_extra_field_len(); + let zip64_len = if let Some(zip64) = file.zip64_extra_field_block() { + zip64.full_size() + } else { + 0 + }; + block.extra_field_length = (zip64_len + stripped_extra.len() + central_len) + .try_into() + .map_err(|_| invalid!("Extra field length in central directory exceeds 64KiB"))?; + + block.write(writer)?; + // file name + writer.write_all(&file.file_name_raw)?; + // extra field + if let Some(zip64_extra_field) = &file.zip64_extra_field_block() { + writer.write_all(&zip64_extra_field.serialize())?; + } + if !stripped_extra.is_empty() { + writer.write_all(&stripped_extra)?; + } + if let Some(central_extra_field) = &file.central_extra_field { + writer.write_all(central_extra_field)?; + } + // file comment + writer.write_all(file.file_comment.as_bytes())?; + + Ok(()) +} + +fn strip_alignment_extra_field(extra_field: &[u8]) -> Vec { + let mut new_extra = Vec::with_capacity(extra_field.len()); + let mut cursor = 0; + while cursor + 4 <= extra_field.len() { + let tag = u16::from_le_bytes([extra_field[cursor], extra_field[cursor + 1]]); + let len = u16::from_le_bytes([extra_field[cursor + 2], extra_field[cursor + 3]]) as usize; + if cursor + 4 + len > extra_field.len() { + new_extra.extend_from_slice(&extra_field[cursor..]); + break; + } + + if tag != 0xa11e { + new_extra.extend_from_slice(&extra_field[cursor..cursor + 4 + len]); + } + cursor += 4 + len; + } + if cursor < extra_field.len() { + new_extra.extend_from_slice(&extra_field[cursor..]); + } + new_extra +} + +fn update_local_zip64_extra_field( + writer: &mut T, + file: &mut ZipFileData, +) -> ZipResult<()> { + let block = file.zip64_extra_field_block().ok_or(invalid!( + "Attempted to update a nonexistent ZIP64 extra field" + ))?; + + let zip64_extra_field_start = file.header_start + + size_of::() as u64 + + file.file_name_raw.len() as u64; + + writer.seek(SeekFrom::Start(zip64_extra_field_start))?; + let block = block.serialize(); + writer.write_all(&block)?; + + let extra_field = Arc::get_mut(file.extra_field.as_mut().unwrap()).unwrap(); + extra_field[..block.len()].copy_from_slice(&block); + + Ok(()) +} + +/// Wrapper around a [Write] implementation that implements the [Seek] trait, but where seeking +/// returns an error unless it's a no-op. +pub struct StreamWriter { + inner: W, + bytes_written: u64, +} + +impl StreamWriter { + /// Creates an instance wrapping the provided inner writer. + pub fn new(inner: W) -> StreamWriter { + Self { + inner, + bytes_written: 0, + } + } + + /// Consumes this wrapper, returning the underlying writer. + pub fn into_inner(self) -> W { + self.inner + } +} + +impl Write for StreamWriter { + fn write(&mut self, buf: &[u8]) -> io::Result { + let bytes_written = self.inner.write(buf)?; + self.bytes_written += bytes_written as u64; + Ok(bytes_written) + } + + fn flush(&mut self) -> io::Result<()> { + self.inner.flush() + } +} + +impl Seek for StreamWriter { + fn seek(&mut self, pos: SeekFrom) -> io::Result { + match pos { + SeekFrom::Current(0) | SeekFrom::End(0) => return Ok(self.bytes_written), + SeekFrom::Start(x) => { + if x == self.bytes_written { + return Ok(self.bytes_written); + } + } + _ => {} + } + Err(io::Error::new( + ErrorKind::Unsupported, + "seek is not supported", + )) + } +} + +#[cfg(not(feature = "unreserved"))] +const EXTRA_FIELD_MAPPING: [u16; 43] = [ + 0x0007, 0x0008, 0x0009, 0x000a, 0x000c, 0x000d, 0x000e, 0x000f, 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x0020, 0x0021, 0x0022, 0x0023, 0x0065, 0x0066, 0x4690, 0x07c8, 0x2605, 0x2705, + 0x2805, 0x334d, 0x4341, 0x4453, 0x4704, 0x470f, 0x4b46, 0x4c41, 0x4d49, 0x4f4c, 0x5356, 0x554e, + 0x5855, 0x6542, 0x756e, 0x7855, 0xa220, 0xfd4a, 0x9902, +]; + +#[cfg(test)] +#[allow(unknown_lints)] // needless_update is new in clippy pre 1.29.0 +#[allow(clippy::needless_update)] // So we can use the same FileOptions decls with and without zopfli_buffer_size +#[allow(clippy::octal_escapes)] // many false positives in converted fuzz cases +mod test { + use super::{ExtendedFileOptions, FileOptions, FullFileOptions, ZipWriter}; + use crate::compression::CompressionMethod; + use crate::result::ZipResult; + use crate::types::{DateTime, System}; + use crate::write::EncryptWith::ZipCrypto; + use crate::write::SimpleFileOptions; + use crate::zipcrypto::ZipCryptoKeys; + use crate::CompressionMethod::Stored; + use crate::ZipArchive; + #[cfg(feature = "deflate-flate2")] + use std::io::Read; + use std::io::{Cursor, Write}; + use std::marker::PhantomData; + use std::path::PathBuf; + + const SYSTEM_BYTE: u8 = if cfg!(windows) { + System::Dos + } else { + System::Unix + } as u8; + + #[test] + fn write_empty_zip() { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_comment("ZIP"); + let result = writer.finish().unwrap(); + assert_eq!(result.get_ref().len(), 25); + assert_eq!( + *result.get_ref(), + [80, 75, 5, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 90, 73, 80] + ); + } + + #[test] + fn unix_permissions_bitmask() { + // unix_permissions() throws away upper bits. + let options = SimpleFileOptions::default().unix_permissions(0o120777); + assert_eq!(options.permissions, Some(0o777)); + } + + #[test] + fn write_zip_dir() { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer + .add_directory( + "test", + SimpleFileOptions::default().last_modified_time( + DateTime::from_date_and_time(2018, 8, 15, 20, 45, 6).unwrap(), + ), + ) + .unwrap(); + assert!(writer + .write(b"writing to a directory is not allowed, and will not write any data") + .is_err()); + let result = writer.finish().unwrap(); + assert_eq!(result.get_ref().len(), 108); + const DOS_DIR_ATTRIBUTES_BYTE: u8 = if cfg!(windows) { 0x10 } else { 0 }; + #[rustfmt::skip] + assert_eq!( + *result.get_ref(), + &[ + 80u8, 75, 3, 4, 20, 0, 0, 0, 0, 0, 163, 165, 15, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 5, 0, 0, 0, 116, 101, 115, 116, 47, 80, 75, 1, 2, 20, SYSTEM_BYTE, 20, 0, 0, 0, 0, 0, + 163, 165, 15, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, + DOS_DIR_ATTRIBUTES_BYTE, 0, 237, 65, 0, 0, 0, 0, 116, 101, 115, 116, 47, 80, 75, 5, 6, 0, 0, 0, 0, 1, 0, + 1, 0, 51, 0, 0, 0, 35, 0, 0, 0, 0, 0, + ] as &[u8] + ); + } + + #[test] + fn write_symlink_simple() { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer + .add_symlink( + "name", + "target", + SimpleFileOptions::default().last_modified_time( + DateTime::from_date_and_time(2018, 8, 15, 20, 45, 6).unwrap(), + ), + ) + .unwrap(); + assert!(writer + .write(b"writing to a symlink is not allowed and will not write any data") + .is_err()); + let result = writer.finish().unwrap(); + assert_eq!(result.get_ref().len(), 112); + #[rustfmt::skip] + assert_eq!( + *result.get_ref(), + &[ + 80u8, 75, 3, 4, 10, 0, 0, 0, 0, 0, 163, 165, 15, 77, 252, 47, 111, 70, 6, 0, 0, 0, + 6, 0, 0, 0, 4, 0, 0, 0, 110, 97, 109, 101, 116, 97, 114, 103, 101, 116, 80, 75, 1, + 2, 10, System::Unix as u8, 10, 0, 0, 0, 0, 0, 163, 165, 15, 77, 252, 47, 111, 70, 6, 0, 0, 0, 6, 0, + 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 161, 0, 0, 0, 0, 110, 97, 109, 101, + 80, 75, 5, 6, 0, 0, 0, 0, 1, 0, 1, 0, 50, 0, 0, 0, 40, 0, 0, 0, 0, 0 + ] as &[u8], + ); + } + + #[test] + fn test_path_normalization() { + let mut path = PathBuf::new(); + path.push("foo"); + path.push("bar"); + path.push(".."); + path.push("."); + path.push("example.txt"); + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer + .start_file_from_path(path, SimpleFileOptions::default()) + .unwrap(); + let archive = writer.finish_into_readable().unwrap(); + assert_eq!(Some("foo/example.txt"), archive.name_for_index(0)); + } + + #[test] + fn write_symlink_wonky_paths() { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer + .add_symlink( + "directory\\link", + "/absolute/symlink\\with\\mixed/slashes", + SimpleFileOptions::default().last_modified_time( + DateTime::from_date_and_time(2018, 8, 15, 20, 45, 6).unwrap(), + ), + ) + .unwrap(); + assert!(writer + .write(b"writing to a symlink is not allowed and will not write any data") + .is_err()); + let result = writer.finish().unwrap(); + assert_eq!(result.get_ref().len(), 162); + #[rustfmt::skip] + assert_eq!( + *result.get_ref(), + &[ + 80u8, 75, 3, 4, 10, 0, 0, 0, 0, 0, 163, 165, 15, 77, 95, 41, 81, 245, 36, 0, 0, 0, + 36, 0, 0, 0, 14, 0, 0, 0, 100, 105, 114, 101, 99, 116, 111, 114, 121, 92, 108, 105, + 110, 107, 47, 97, 98, 115, 111, 108, 117, 116, 101, 47, 115, 121, 109, 108, 105, + 110, 107, 92, 119, 105, 116, 104, 92, 109, 105, 120, 101, 100, 47, 115, 108, 97, + 115, 104, 101, 115, 80, 75, 1, 2, 10, System::Unix as u8, 10, 0, 0, 0, 0, 0, 163, 165, 15, 77, 95, + 41, 81, 245, 36, 0, 0, 0, 36, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, + 161, 0, 0, 0, 0, 100, 105, 114, 101, 99, 116, 111, 114, 121, 92, 108, 105, 110, + 107, 80, 75, 5, 6, 0, 0, 0, 0, 1, 0, 1, 0, 60, 0, 0, 0, 80, 0, 0, 0, 0, 0 + ] as &[u8], + ); + } + + #[test] + fn write_mimetype_zip() { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + let options = FileOptions { + compression_method: Stored, + compression_level: None, + last_modified_time: DateTime::default(), + permissions: Some(33188), + large_file: false, + encrypt_with: None, + extended_options: (), + alignment: 1, + #[cfg(feature = "deflate-zopfli")] + zopfli_buffer_size: None, + #[cfg(feature = "aes-crypto")] + aes_mode: None, + }; + writer.start_file("mimetype", options).unwrap(); + writer + .write_all(b"application/vnd.oasis.opendocument.text") + .unwrap(); + let result = writer.finish().unwrap(); + + assert_eq!(result.get_ref().len(), 153); + #[rustfmt::skip] + assert_eq!(result.get_ref(), &[80, 75, 3, 4, 10, 0, 0, 0, 0, 0, 0, 0, 33, 0, 94, 198, 50, + 12, 39, 0, 0, 0, 39, 0, 0, 0, 8, 0, 0, 0, 109, 105, 109, 101, 116, 121, 112, 101, 97, + 112, 112, 108, 105, 99, 97, 116, 105, 111, 110, 47, 118, 110, 100, 46, 111, 97, 115, + 105, 115, 46, 111, 112, 101, 110, 100, 111, 99, 117, 109, 101, 110, 116, 46, 116, 101, + 120, 116, 80, 75, 1, 2, 10, SYSTEM_BYTE, 10, 0, 0, 0, 0, 0, 0, 0, 33, 0, 94, 198, 50, + 12, 39, 0, 0, 0, 39, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 164, 129, 0, 0, 0, 0, + 109, 105, 109, 101, 116, 121, 112, 101, 80, 75, 5, 6, 0, 0, 0, 0, 1, 0, 1, 0, 54, 0, 0, + 0, 77, 0, 0, 0, 0, 0]); + } + + #[cfg(feature = "deflate-flate2")] + const RT_TEST_TEXT: &str = "And I can't stop thinking about the moments that I lost to you\ + And I can't stop thinking of things I used to do\ + And I can't stop making bad decisions\ + And I can't stop eating stuff you make me chew\ + I put on a smile like you wanna see\ + Another day goes by that I long to be like you"; + #[cfg(feature = "deflate-flate2")] + const RT_TEST_FILENAME: &str = "subfolder/sub-subfolder/can't_stop.txt"; + #[cfg(feature = "deflate-flate2")] + const SECOND_FILENAME: &str = "different_name.xyz"; + #[cfg(feature = "deflate-flate2")] + const THIRD_FILENAME: &str = "third_name.xyz"; + + #[test] + fn write_non_utf8() { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + let options = FileOptions { + compression_method: Stored, + compression_level: None, + last_modified_time: DateTime::default(), + permissions: Some(33188), + large_file: false, + encrypt_with: None, + extended_options: (), + alignment: 1, + #[cfg(feature = "deflate-zopfli")] + zopfli_buffer_size: None, + #[cfg(feature = "aes-crypto")] + aes_mode: None, + }; + + // GB18030 + // "中文" = [214, 208, 206, 196] + let filename = unsafe { String::from_utf8_unchecked(vec![214, 208, 206, 196]) }; + writer.start_file(filename, options).unwrap(); + writer.write_all(b"encoding GB18030").unwrap(); + + // SHIFT_JIS + // "日文" = [147, 250, 149, 182] + let filename = unsafe { String::from_utf8_unchecked(vec![147, 250, 149, 182]) }; + writer.start_file(filename, options).unwrap(); + writer.write_all(b"encoding SHIFT_JIS").unwrap(); + let result = writer.finish().unwrap(); + + assert_eq!(result.get_ref().len(), 224); + #[rustfmt::skip] + assert_eq!(result.get_ref(), &[80, 75, 3, 4, 10, 0, 0, 0, 0, 0, 0, 0, 33, 0, 29, 183, 125, + 75, 16, 0, 0, 0, 16, 0, 0, 0, 4, 0, 0, 0, 214, 208, 206, 196, 101, 110, 99, 111, 100, + 105, 110, 103, 32, 71, 66, 49, 56, 48, 51, 48, 80, 75, 3, 4, 10, 0, 0, 0, 0, 0, 0, 0, + 33, 0, 75, 110, 14, 242, 18, 0, 0, 0, 18, 0, 0, 0, 4, 0, 0, 0, 147, 250, 149, 182, 101, + 110, 99, 111, 100, 105, 110, 103, 32, 83, 72, 73, 70, 84, 95, 74, 73, 83, 80, 75, 1, 2, + 10, SYSTEM_BYTE, 10, 0, 0, 0, 0, 0, 0, 0, 33, 0, 29, 183, 125, 75, 16, 0, 0, 0, 16, 0, + 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 164, 129, 0, 0, 0, 0, 214, 208, 206, 196, 80, + 75, 1, 2, 10, SYSTEM_BYTE, 10, 0, 0, 0, 0, 0, 0, 0, 33, 0, 75, 110, 14, 242, 18, 0, 0, 0, 18, 0, + 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 164, 129, 50, 0, 0, 0, 147, 250, 149, 182, 80, + 75, 5, 6, 0, 0, 0, 0, 2, 0, 2, 0, 100, 0, 0, 0, 102, 0, 0, 0, 0, 0]); + } + + #[test] + fn path_to_string() { + let mut path = PathBuf::new(); + #[cfg(windows)] + path.push(r"C:\"); + #[cfg(unix)] + path.push("/"); + path.push("windows"); + path.push(".."); + path.push("."); + path.push("system32"); + let path_str = super::path_to_string(&path); + assert_eq!(&*path_str, "system32"); + } + + #[test] + #[cfg(feature = "deflate-flate2")] + fn test_shallow_copy() { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + let options = FileOptions { + compression_method: CompressionMethod::default(), + compression_level: None, + last_modified_time: DateTime::default(), + permissions: Some(33188), + large_file: false, + encrypt_with: None, + extended_options: (), + alignment: 0, + #[cfg(feature = "deflate-zopfli")] + zopfli_buffer_size: None, + #[cfg(feature = "aes-crypto")] + aes_mode: None, + }; + writer.start_file(RT_TEST_FILENAME, options).unwrap(); + writer.write_all(RT_TEST_TEXT.as_ref()).unwrap(); + writer + .shallow_copy_file(RT_TEST_FILENAME, SECOND_FILENAME) + .unwrap(); + writer + .shallow_copy_file(RT_TEST_FILENAME, SECOND_FILENAME) + .expect_err("Duplicate filename"); + let zip = writer.finish().unwrap(); + let mut writer = ZipWriter::new_append(zip).unwrap(); + writer + .shallow_copy_file(SECOND_FILENAME, SECOND_FILENAME) + .expect_err("Duplicate filename"); + let mut reader = writer.finish_into_readable().unwrap(); + let mut file_names: Vec<&str> = reader.file_names().collect(); + file_names.sort(); + let mut expected_file_names = vec![RT_TEST_FILENAME, SECOND_FILENAME]; + expected_file_names.sort(); + assert_eq!(file_names, expected_file_names); + let mut first_file_content = String::new(); + reader + .by_name(RT_TEST_FILENAME) + .unwrap() + .read_to_string(&mut first_file_content) + .unwrap(); + assert_eq!(first_file_content, RT_TEST_TEXT); + let mut second_file_content = String::new(); + reader + .by_name(SECOND_FILENAME) + .unwrap() + .read_to_string(&mut second_file_content) + .unwrap(); + assert_eq!(second_file_content, RT_TEST_TEXT); + } + + #[test] + #[cfg(feature = "deflate-flate2")] + fn test_deep_copy() { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + let options = FileOptions { + compression_method: CompressionMethod::default(), + compression_level: None, + last_modified_time: DateTime::default(), + permissions: Some(33188), + large_file: false, + encrypt_with: None, + extended_options: (), + alignment: 0, + #[cfg(feature = "deflate-zopfli")] + zopfli_buffer_size: None, + #[cfg(feature = "aes-crypto")] + aes_mode: None, + }; + writer.start_file(RT_TEST_FILENAME, options).unwrap(); + writer.write_all(RT_TEST_TEXT.as_ref()).unwrap(); + writer + .deep_copy_file(RT_TEST_FILENAME, SECOND_FILENAME) + .unwrap(); + let zip = writer.finish().unwrap().into_inner(); + zip.iter().copied().for_each(|x| print!("{x:02x}")); + println!(); + let mut writer = ZipWriter::new_append(Cursor::new(zip)).unwrap(); + writer + .deep_copy_file(RT_TEST_FILENAME, THIRD_FILENAME) + .unwrap(); + let zip = writer.finish().unwrap(); + let mut reader = ZipArchive::new(zip).unwrap(); + let mut file_names: Vec<&str> = reader.file_names().collect(); + file_names.sort(); + let mut expected_file_names = vec![RT_TEST_FILENAME, SECOND_FILENAME, THIRD_FILENAME]; + expected_file_names.sort(); + assert_eq!(file_names, expected_file_names); + let mut first_file_content = String::new(); + reader + .by_name(RT_TEST_FILENAME) + .unwrap() + .read_to_string(&mut first_file_content) + .unwrap(); + assert_eq!(first_file_content, RT_TEST_TEXT); + let mut second_file_content = String::new(); + reader + .by_name(SECOND_FILENAME) + .unwrap() + .read_to_string(&mut second_file_content) + .unwrap(); + assert_eq!(second_file_content, RT_TEST_TEXT); + } + + #[test] + fn duplicate_filenames() { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer + .start_file("foo/bar/test", SimpleFileOptions::default()) + .unwrap(); + writer + .write_all("The quick brown 🦊 jumps over the lazy 🐕".as_bytes()) + .unwrap(); + writer + .start_file("foo/bar/test", SimpleFileOptions::default()) + .expect_err("Expected duplicate filename not to be allowed"); + } + + #[test] + fn test_filename_looks_like_zip64_locator() { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer + .start_file( + "PK\u{6}\u{7}\0\0\0\u{11}\0\0\0\0\0\0\0\0\0\0\0\0", + SimpleFileOptions::default(), + ) + .unwrap(); + let zip = writer.finish().unwrap(); + let _ = ZipArchive::new(zip).unwrap(); + } + + #[test] + fn test_filename_looks_like_zip64_locator_2() { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer + .start_file( + "PK\u{6}\u{6}\0\0\0\0\0\0\0\0\0\0PK\u{6}\u{7}\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", + SimpleFileOptions::default(), + ) + .unwrap(); + let zip = writer.finish().unwrap(); + let _ = ZipArchive::new(zip).unwrap(); + } + + #[test] + fn test_filename_looks_like_zip64_locator_2a() { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer + .start_file( + "PK\u{6}\u{6}PK\u{6}\u{7}\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", + SimpleFileOptions::default(), + ) + .unwrap(); + let zip = writer.finish().unwrap(); + let _ = ZipArchive::new(zip).unwrap(); + } + + #[test] + fn test_filename_looks_like_zip64_locator_3() { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer + .start_file("\0PK\u{6}\u{6}", SimpleFileOptions::default()) + .unwrap(); + writer + .start_file( + "\0\u{4}\0\0PK\u{6}\u{7}\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\u{3}", + SimpleFileOptions::default(), + ) + .unwrap(); + let zip = writer.finish().unwrap(); + let _ = ZipArchive::new(zip).unwrap(); + } + + #[test] + fn test_filename_looks_like_zip64_locator_4() { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer + .start_file("PK\u{6}\u{6}", SimpleFileOptions::default()) + .unwrap(); + writer + .start_file("\0\0\0\0\0\0", SimpleFileOptions::default()) + .unwrap(); + writer + .start_file("\0", SimpleFileOptions::default()) + .unwrap(); + writer.start_file("", SimpleFileOptions::default()).unwrap(); + writer + .start_file("\0\0", SimpleFileOptions::default()) + .unwrap(); + writer + .start_file( + "\0\0\0PK\u{6}\u{7}\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", + SimpleFileOptions::default(), + ) + .unwrap(); + let zip = writer.finish().unwrap(); + let _ = ZipArchive::new(zip).unwrap(); + } + + #[test] + fn test_filename_looks_like_zip64_locator_5() -> ZipResult<()> { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer + .add_directory("", SimpleFileOptions::default().with_alignment(21)) + .unwrap(); + let mut writer = ZipWriter::new_append(writer.finish().unwrap()).unwrap(); + writer.shallow_copy_file("/", "").unwrap(); + writer.shallow_copy_file("", "\0").unwrap(); + writer.shallow_copy_file("\0", "PK\u{6}\u{6}").unwrap(); + let mut writer = ZipWriter::new_append(writer.finish().unwrap()).unwrap(); + writer + .start_file("\0\0\0\0\0\0", SimpleFileOptions::default()) + .unwrap(); + let mut writer = ZipWriter::new_append(writer.finish().unwrap()).unwrap(); + writer + .start_file( + "#PK\u{6}\u{7}\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", + SimpleFileOptions::default(), + ) + .unwrap(); + let zip = writer.finish().unwrap(); + let _ = ZipArchive::new(zip).unwrap(); + Ok(()) + } + + #[test] + #[cfg(feature = "deflate-flate2")] + fn remove_shallow_copy_keeps_original() -> ZipResult<()> { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer + .start_file("original", SimpleFileOptions::default()) + .unwrap(); + writer.write_all(RT_TEST_TEXT.as_bytes()).unwrap(); + writer + .shallow_copy_file("original", "shallow_copy") + .unwrap(); + writer.abort_file().unwrap(); + let mut zip = ZipArchive::new(writer.finish().unwrap()).unwrap(); + let mut file = zip.by_name("original").unwrap(); + let mut contents = Vec::new(); + file.read_to_end(&mut contents).unwrap(); + assert_eq!(RT_TEST_TEXT.as_bytes(), contents); + Ok(()) + } + + #[test] + fn remove_encrypted_file() -> ZipResult<()> { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + let first_file_options = SimpleFileOptions::default() + .with_alignment(65535) + .with_deprecated_encryption(b"Password"); + writer.start_file("", first_file_options).unwrap(); + writer.abort_file().unwrap(); + let zip = writer.finish().unwrap(); + let mut writer = ZipWriter::new(zip); + writer.start_file("", SimpleFileOptions::default()).unwrap(); + Ok(()) + } + + #[test] + fn remove_encrypted_aligned_symlink() -> ZipResult<()> { + let mut options = SimpleFileOptions::default(); + options = options.with_deprecated_encryption(b"Password"); + options.alignment = 65535; + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.add_symlink("", "s\t\0\0ggggg\0\0", options).unwrap(); + writer.abort_file().unwrap(); + let zip = writer.finish().unwrap(); + let mut writer = ZipWriter::new_append(zip).unwrap(); + writer.start_file("", SimpleFileOptions::default()).unwrap(); + Ok(()) + } + + #[cfg(feature = "deflate-zopfli")] + #[test] + fn zopfli_empty_write() -> ZipResult<()> { + let mut options = SimpleFileOptions::default(); + options = options + .compression_method(CompressionMethod::default()) + .compression_level(Some(264)); + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.start_file("", options).unwrap(); + writer.write_all(&[]).unwrap(); + writer.write_all(&[]).unwrap(); + Ok(()) + } + + #[test] + fn crash_with_no_features() -> ZipResult<()> { + const ORIGINAL_FILE_NAME: &str = "PK\u{6}\u{6}\0\0\0\0\0\0\0\0\0\u{2}g\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\u{1}\0\0\0\0\0\0\0\0\0\0PK\u{6}\u{7}\0\0\0\0\0\0\0\0\0\0\0\0\u{7}\0\t'"; + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + let mut options = SimpleFileOptions::default(); + options = options.with_alignment(3584).compression_method(Stored); + writer.start_file(ORIGINAL_FILE_NAME, options)?; + let archive = writer.finish()?; + let mut writer = ZipWriter::new_append(archive)?; + writer.shallow_copy_file(ORIGINAL_FILE_NAME, "\u{6}\\")?; + writer.finish()?; + Ok(()) + } + + #[test] + fn test_alignment() { + let page_size = 4096; + let options = SimpleFileOptions::default() + .compression_method(Stored) + .with_alignment(page_size); + let mut zip = ZipWriter::new(Cursor::new(Vec::new())); + let contents = b"sleeping"; + let () = zip.start_file("sleep", options).unwrap(); + let _count = zip.write(&contents[..]).unwrap(); + let mut zip = zip.finish_into_readable().unwrap(); + let file = zip.by_index(0).unwrap(); + assert_eq!(file.name(), "sleep"); + assert_eq!(file.data_start(), u64::from(page_size)); + } + + #[test] + fn test_alignment_2() { + let page_size = 4096; + let mut data = Vec::new(); + { + let options = SimpleFileOptions::default() + .compression_method(Stored) + .with_alignment(page_size); + let mut zip = ZipWriter::new(Cursor::new(&mut data)); + let contents = b"sleeping"; + let () = zip.start_file("sleep", options).unwrap(); + let _count = zip.write(&contents[..]).unwrap(); + } + assert_eq!(data[4096..4104], b"sleeping"[..]); + { + let mut zip = ZipArchive::new(Cursor::new(&mut data)).unwrap(); + let file = zip.by_index(0).unwrap(); + assert_eq!(file.name(), "sleep"); + assert_eq!(file.data_start(), u64::from(page_size)); + } + } + + #[test] + fn test_crash_short_read() { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + let comment = vec![ + 1, 80, 75, 5, 6, 237, 237, 237, 237, 237, 237, 237, 237, 44, 255, 191, 255, 255, 255, + 255, 255, 255, 255, 255, 16, + ] + .into_boxed_slice(); + writer.set_raw_comment(comment); + let options = SimpleFileOptions::default() + .compression_method(Stored) + .with_alignment(11823); + writer.start_file("", options).unwrap(); + writer.write_all(&[255, 255, 44, 255, 0]).unwrap(); + let written = writer.finish().unwrap(); + let _ = ZipWriter::new_append(written).unwrap(); + } + + #[cfg(all(feature = "_deflate-any", feature = "aes-crypto"))] + #[test] + fn test_fuzz_failure_2024_05_08() -> ZipResult<()> { + let mut first_writer = ZipWriter::new(Cursor::new(Vec::new())); + let mut second_writer = ZipWriter::new(Cursor::new(Vec::new())); + let options = SimpleFileOptions::default() + .compression_method(Stored) + .with_alignment(46036); + second_writer.add_symlink("\0", "", options)?; + let second_archive = second_writer.finish_into_readable()?.into_inner(); + let mut second_writer = ZipWriter::new_append(second_archive)?; + let options = SimpleFileOptions::default() + .compression_method(CompressionMethod::Deflated) + .large_file(true) + .with_alignment(46036) + .with_aes_encryption(crate::AesMode::Aes128, "\0\0"); + second_writer.add_symlink("", "", options)?; + let second_archive = second_writer.finish_into_readable()?.into_inner(); + let mut second_writer = ZipWriter::new_append(second_archive)?; + let options = SimpleFileOptions::default().compression_method(Stored); + second_writer.start_file(" ", options)?; + let second_archive = second_writer.finish_into_readable()?; + first_writer.merge_archive(second_archive)?; + let _ = ZipArchive::new(first_writer.finish()?)?; + Ok(()) + } + + #[cfg(all(feature = "_bzip2_any", not(miri)))] + #[test] + fn test_fuzz_failure_2024_06_08() -> ZipResult<()> { + use crate::write::ExtendedFileOptions; + use CompressionMethod::Bzip2; + + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + const SYMLINK_PATH: &str = "PK\u{6}\u{6}K\u{6}\u{6}\u{6}\0\0\0\0\u{18}\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\u{1}\0\0\0\0\0\0\0\0\u{1}\0\0PK\u{1}\u{2},\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\u{1}\0\0PK\u{1}\u{2},\0\0\0\0\0\0\0\0\0\0l\0\0\0\0\0\0PK\u{6}\u{7}P\0\0\0\0\0\0\0\0\0\0\0\0\u{1}\0\0"; + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let options = FileOptions { + compression_method: Bzip2, + compression_level: None, + last_modified_time: DateTime::from_date_and_time(1980, 5, 20, 21, 0, 57)?, + permissions: None, + large_file: false, + encrypt_with: None, + extended_options: ExtendedFileOptions { + extra_data: vec![].into(), + central_extra_data: vec![].into(), + }, + alignment: 2048, + ..Default::default() + }; + writer.add_symlink_from_path(SYMLINK_PATH, "||\0\0\0\0", options)?; + writer = ZipWriter::new_append(writer.finish_into_readable()?.into_inner())?; + writer.deep_copy_file_from_path(SYMLINK_PATH, "")?; + writer = ZipWriter::new_append(writer.finish_into_readable()?.into_inner())?; + writer.abort_file()?; + writer + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + writer = ZipWriter::new_append(writer.finish_into_readable()?.into_inner())?; + writer.deep_copy_file_from_path(SYMLINK_PATH, "foo")?; + let _ = writer.finish_into_readable()?; + Ok(()) + } + + #[test] + fn test_short_extra_data() { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let options = FileOptions { + extended_options: ExtendedFileOptions { + extra_data: vec![].into(), + central_extra_data: vec![99, 0, 15, 0, 207].into(), + }, + ..Default::default() + }; + assert!(writer.start_file_from_path("", options).is_err()); + } + + #[test] + #[cfg(not(feature = "unreserved"))] + fn test_invalid_extra_data() -> ZipResult<()> { + use crate::write::ExtendedFileOptions; + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let options = FileOptions { + compression_method: Stored, + compression_level: None, + last_modified_time: DateTime::from_date_and_time(1980, 1, 4, 6, 54, 0)?, + permissions: None, + large_file: false, + encrypt_with: None, + extended_options: ExtendedFileOptions { + extra_data: vec![].into(), + central_extra_data: vec![ + 7, 0, 15, 0, 207, 117, 177, 117, 112, 2, 0, 255, 255, 131, 255, 255, 255, 80, + 185, + ] + .into(), + }, + alignment: 32787, + ..Default::default() + }; + assert!(writer.start_file_from_path("", options).is_err()); + Ok(()) + } + + #[test] + #[cfg(not(feature = "unreserved"))] + fn test_invalid_extra_data_unreserved() { + use crate::write::ExtendedFileOptions; + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + let options = FileOptions { + compression_method: Stored, + compression_level: None, + last_modified_time: DateTime::from_date_and_time(2021, 8, 8, 1, 0, 29).unwrap(), + permissions: None, + large_file: true, + encrypt_with: None, + extended_options: ExtendedFileOptions { + extra_data: vec![].into(), + central_extra_data: vec![ + 1, 41, 4, 0, 1, 255, 245, 117, 117, 112, 5, 0, 80, 255, 149, 255, 247, + ] + .into(), + }, + alignment: 4103, + ..Default::default() + }; + assert!(writer.start_file_from_path("", options).is_err()); + } + + #[cfg(feature = "deflate64")] + #[test] + fn test_fuzz_crash_2024_06_13a() -> ZipResult<()> { + use crate::write::ExtendedFileOptions; + use CompressionMethod::Deflate64; + + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let options = FileOptions { + compression_method: Deflate64, + compression_level: None, + last_modified_time: DateTime::from_date_and_time(2039, 4, 17, 6, 18, 19)?, + permissions: None, + large_file: true, + encrypt_with: None, + extended_options: ExtendedFileOptions { + extra_data: vec![].into(), + central_extra_data: vec![].into(), + }, + alignment: 4, + ..Default::default() + }; + writer.add_directory_from_path("", options)?; + let _ = writer.finish_into_readable()?; + Ok(()) + } + + #[test] + fn test_fuzz_crash_2024_06_13b() -> ZipResult<()> { + use crate::write::ExtendedFileOptions; + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let options = FileOptions { + compression_method: Stored, + compression_level: None, + last_modified_time: DateTime::from_date_and_time(1980, 4, 14, 6, 11, 54)?, + permissions: None, + large_file: false, + encrypt_with: None, + extended_options: ExtendedFileOptions { + extra_data: vec![].into(), + central_extra_data: vec![].into(), + }, + alignment: 185, + ..Default::default() + }; + writer.add_symlink_from_path("", "", options)?; + writer + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + writer.deep_copy_file_from_path("", "_copy")?; + let _ = writer.finish_into_readable()?; + Ok(()) + } + + #[test] + fn test_fuzz_crash_2024_06_14() -> ZipResult<()> { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let options = FullFileOptions { + compression_method: Stored, + large_file: true, + alignment: 93, + ..Default::default() + }; + writer.start_file_from_path("\0", options)?; + writer = ZipWriter::new_append(writer.finish()?)?; + writer.deep_copy_file_from_path("\0", "")?; + writer + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + writer.deep_copy_file_from_path("", "copy")?; + let _ = writer.finish_into_readable()?; + Ok(()) + } + + #[test] + fn test_fuzz_crash_2024_06_14a() -> ZipResult<()> { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let options = FileOptions { + compression_method: Stored, + compression_level: None, + last_modified_time: DateTime::from_date_and_time(2083, 5, 30, 21, 45, 35)?, + permissions: None, + large_file: false, + encrypt_with: None, + extended_options: ExtendedFileOptions { + extra_data: vec![].into(), + central_extra_data: vec![].into(), + }, + alignment: 2565, + ..Default::default() + }; + writer.add_symlink_from_path("", "", options)?; + writer.abort_file()?; + let options = FileOptions { + compression_method: Stored, + compression_level: None, + last_modified_time: DateTime::default(), + permissions: None, + large_file: false, + encrypt_with: None, + extended_options: ExtendedFileOptions { + extra_data: vec![].into(), + central_extra_data: vec![].into(), + }, + alignment: 0, + ..Default::default() + }; + writer.start_file_from_path("", options)?; + let _ = writer.finish_into_readable()?; + Ok(()) + } + + #[allow(deprecated)] + #[test] + fn test_fuzz_crash_2024_06_14b() -> ZipResult<()> { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let options = FileOptions { + compression_method: Stored, + compression_level: None, + last_modified_time: DateTime::from_date_and_time(2078, 3, 6, 12, 48, 58)?, + permissions: None, + large_file: true, + encrypt_with: None, + extended_options: ExtendedFileOptions { + extra_data: vec![].into(), + central_extra_data: vec![].into(), + }, + alignment: 65521, + ..Default::default() + }; + writer.start_file_from_path("\u{4}\0@\n//\u{c}", options)?; + writer = ZipWriter::new_append(writer.finish()?)?; + writer.abort_file()?; + let options = FileOptions { + compression_method: CompressionMethod::Unsupported(65535), + compression_level: None, + last_modified_time: DateTime::from_date_and_time(2055, 10, 2, 11, 48, 49)?, + permissions: None, + large_file: true, + encrypt_with: None, + extended_options: ExtendedFileOptions { + extra_data: vec![255, 255, 1, 0, 255, 0, 0, 0, 0].into(), + central_extra_data: vec![].into(), + }, + alignment: 65535, + ..Default::default() + }; + writer.add_directory_from_path("", options)?; + let _ = writer.finish_into_readable()?; + Ok(()) + } + + #[test] + fn test_fuzz_crash_2024_06_14c() -> ZipResult<()> { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let options = FileOptions { + compression_method: Stored, + compression_level: None, + last_modified_time: DateTime::from_date_and_time(2060, 4, 6, 13, 13, 3)?, + permissions: None, + large_file: true, + encrypt_with: None, + extended_options: ExtendedFileOptions { + extra_data: vec![].into(), + central_extra_data: vec![].into(), + }, + alignment: 0, + ..Default::default() + }; + writer.start_file_from_path("\0", options)?; + writer.write_all(&([]))?; + writer = ZipWriter::new_append(writer.finish()?)?; + writer.deep_copy_file_from_path("\0", "")?; + writer + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + writer.deep_copy_file_from_path("", "_copy")?; + let _ = writer.finish_into_readable()?; + Ok(()) + } + + #[cfg(all(feature = "_deflate-any", feature = "aes-crypto"))] + #[test] + fn test_fuzz_crash_2024_06_14d() -> ZipResult<()> { + use crate::write::EncryptWith::Aes; + use crate::AesMode::Aes256; + use CompressionMethod::Deflated; + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let options = FileOptions { + compression_method: Deflated, + compression_level: Some(5), + last_modified_time: DateTime::from_date_and_time(2107, 4, 8, 15, 54, 19)?, + permissions: None, + large_file: true, + encrypt_with: Some(Aes { + mode: Aes256, + password: "", + }), + extended_options: ExtendedFileOptions { + extra_data: vec![2, 0, 1, 0, 0].into(), + central_extra_data: vec![ + 35, 229, 2, 0, 41, 41, 231, 44, 2, 0, 52, 233, 82, 201, 0, 0, 3, 0, 2, 0, 233, + 255, 3, 0, 2, 0, 26, 154, 38, 251, 0, 0, + ] + .into(), + }, + alignment: 65535, + ..Default::default() + }; + assert!(writer.add_directory_from_path("", options).is_err()); + Ok(()) + } + + #[test] + fn test_fuzz_crash_2024_06_14e() -> ZipResult<()> { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let options = FileOptions { + compression_method: Stored, + compression_level: None, + last_modified_time: DateTime::from_date_and_time(1988, 1, 1, 1, 6, 26)?, + permissions: None, + large_file: true, + encrypt_with: None, + extended_options: ExtendedFileOptions { + extra_data: vec![76, 0, 1, 0, 0, 2, 0, 0, 0].into(), + central_extra_data: vec![ + 1, 149, 1, 0, 255, 3, 0, 0, 0, 2, 255, 0, 0, 12, 65, 1, 0, 0, 67, 149, 0, 0, + 76, 149, 2, 0, 149, 149, 67, 149, 0, 0, + ] + .into(), + }, + alignment: 65535, + ..Default::default() + }; + assert!(writer.add_directory_from_path("", options).is_err()); + let _ = writer.finish_into_readable()?; + Ok(()) + } + + #[allow(deprecated)] + #[test] + fn test_fuzz_crash_2024_06_17() -> ZipResult<()> { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let sub_writer = { + let mut writer = + ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let options = FileOptions { + compression_method: CompressionMethod::Unsupported( + 65535, + ), + compression_level: Some(5), + last_modified_time: DateTime::from_date_and_time( + 2107, 2, 8, 15, 0, 0, + )?, + permissions: None, + large_file: true, + encrypt_with: Some(ZipCrypto( + ZipCryptoKeys::of( + 0x63ff, 0xc62d3103, 0xfffe00ea, + ), + PhantomData, + )), + extended_options: ExtendedFileOptions { + extra_data: vec![].into(), + central_extra_data: vec![].into(), + }, + alignment: 255, + ..Default::default() + }; + writer.add_symlink_from_path("1\0PK\u{6}\u{6}\u{b}\u{6}\u{6}\u{6}\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\u{1}\0\0\0\0\0\0\0\0\u{b}\0\0PK\u{1}\u{2},\0\0\0\0\0\0\0\0\0\0\0\u{10}\0\0\0K\u{6}\u{6}\0\0\0\0\0\0\0\0PK\u{2}\u{6}", "", options)?; + writer = ZipWriter::new_append( + writer.finish_into_readable()?.into_inner(), + )?; + writer + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + writer = ZipWriter::new_append( + writer.finish_into_readable()?.into_inner(), + )?; + let options = FileOptions { + compression_method: Stored, + compression_level: None, + last_modified_time: DateTime::from_date_and_time( + 1992, 7, 3, 0, 0, 0, + )?, + permissions: None, + large_file: true, + encrypt_with: None, + extended_options: ExtendedFileOptions { + extra_data: vec![].into(), + central_extra_data: vec![].into(), + }, + alignment: 43, + ..Default::default() + }; + writer.start_file_from_path( + "\0\0\0\u{3}\0\u{1a}\u{1a}\u{1a}\u{1a}\u{1a}\u{1a}", + options, + )?; + let options = FileOptions { + compression_method: Stored, + compression_level: None, + last_modified_time: DateTime::from_date_and_time( + 2006, 3, 27, 2, 24, 26, + )?, + permissions: None, + large_file: false, + encrypt_with: None, + extended_options: ExtendedFileOptions { + extra_data: vec![].into(), + central_extra_data: vec![].into(), + }, + alignment: 26, + ..Default::default() + }; + writer.start_file_from_path("\0K\u{6}\u{6}\0PK\u{6}\u{7}PK\u{6}\u{6}\0\0\0\0\0\0\0\0PK\u{2}\u{6}", options)?; + writer = ZipWriter::new_append( + writer.finish_into_readable()?.into_inner(), + )?; + let options = FileOptions { + compression_method: Stored, + compression_level: Some(17), + last_modified_time: DateTime::from_date_and_time( + 2103, 4, 10, 23, 15, 18, + )?, + permissions: Some(3284386755), + large_file: true, + encrypt_with: Some(ZipCrypto( + ZipCryptoKeys::of( + 0x8888c5bf, 0x88888888, 0xff888888, + ), + PhantomData, + )), + extended_options: ExtendedFileOptions { + extra_data: vec![3, 0, 1, 0, 255, 144, 136, 0, 0] + .into(), + central_extra_data: vec![].into(), + }, + alignment: 65535, + ..Default::default() + }; + writer.add_symlink_from_path("", "\nu", options)?; + writer = ZipWriter::new_append(writer.finish()?)?; + writer + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + writer = ZipWriter::new_append( + writer.finish_into_readable()?.into_inner(), + )?; + writer + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + writer = ZipWriter::new_append(writer.finish()?)?; + writer + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + writer = + ZipWriter::new_append(writer.finish_into_readable()?.into_inner())?; + writer.abort_file()?; + let options = FileOptions { + compression_method: CompressionMethod::Unsupported(49603), + compression_level: Some(20), + last_modified_time: DateTime::from_date_and_time( + 2047, 4, 14, 3, 15, 14, + )?, + permissions: Some(3284386755), + large_file: true, + encrypt_with: Some(ZipCrypto( + ZipCryptoKeys::of(0xc3, 0x0, 0x0), + PhantomData, + )), + extended_options: ExtendedFileOptions { + extra_data: vec![].into(), + central_extra_data: vec![].into(), + }, + alignment: 0, + ..Default::default() + }; + writer.add_directory_from_path("", options)?; + writer.deep_copy_file_from_path("/", "")?; + writer.shallow_copy_file_from_path("", "copy")?; + assert!(writer.shallow_copy_file_from_path("", "copy").is_err()); + assert!(writer.shallow_copy_file_from_path("", "copy").is_err()); + assert!(writer.shallow_copy_file_from_path("", "copy").is_err()); + assert!(writer.shallow_copy_file_from_path("", "copy").is_err()); + assert!(writer.shallow_copy_file_from_path("", "copy").is_err()); + assert!(writer.shallow_copy_file_from_path("", "copy").is_err()); + writer + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + writer + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + writer + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + writer + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + writer + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + let _ = writer.finish_into_readable()?; + Ok(()) + } + + #[test] + fn test_fuzz_crash_2024_06_17a() -> ZipResult<()> { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + const PATH_1: &str = "\0I\01\0P\0\0\u{2}\0\0\u{1a}\u{1a}\u{1a}\u{1a}\u{1b}\u{1a}UT\u{5}\0\0\u{1a}\u{1a}\u{1a}\u{1a}UT\u{5}\0\u{1}\0\u{1a}\u{1a}\u{1a}UT\t\0uc\u{5}\0\0\0\0\u{7f}\u{7f}\u{7f}\u{7f}PK\u{6}"; + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let options = FileOptions { + compression_method: Stored, + compression_level: None, + last_modified_time: DateTime::from_date_and_time(1981, 1, 1, 0, 24, 21)?, + permissions: Some(16908288), + large_file: false, + encrypt_with: None, + extended_options: ExtendedFileOptions { + extra_data: vec![].into(), + central_extra_data: vec![].into(), + }, + alignment: 20555, + ..Default::default() + }; + writer.start_file_from_path( + "\0\u{7}\u{1}\0\0\0\0\0\0\0\0\u{1}\0\0PK\u{1}\u{2};", + options, + )?; + writer.write_all( + &([ + 255, 255, 255, 255, 253, 253, 253, 203, 203, 203, 253, 253, 253, 253, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 249, 191, 225, 225, + 241, 197, + ]), + )?; + writer.write_all( + &([ + 197, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 75, 0, + ]), + )?; + writer + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + writer = ZipWriter::new_append(writer.finish_into_readable()?.into_inner())?; + let options = FileOptions { + compression_method: Stored, + compression_level: None, + last_modified_time: DateTime::from_date_and_time(1980, 11, 14, 10, 46, 47)?, + permissions: None, + large_file: false, + encrypt_with: None, + extended_options: ExtendedFileOptions { + extra_data: vec![].into(), + central_extra_data: vec![].into(), + }, + alignment: 0, + ..Default::default() + }; + writer.start_file_from_path(PATH_1, options)?; + writer.deep_copy_file_from_path(PATH_1, "eee\u{6}\0\0\0\0\0\0\0\0\0\0\0$\0\0\0\0\0\0\u{7f}\u{7f}PK\u{6}\u{6}K\u{6}\u{6}\u{6}\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\u{1}\0\0\0\0\0\0\0\0\u{1}\0\0PK\u{1}\u{1e},\0\0\0\0\0\0\0\0\0\0\0\u{8}\0*\0\0\u{1}PK\u{6}\u{7}PK\u{6}\u{6}\0\0\0\0\0\0\0\0}K\u{2}\u{6}")?; + writer + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + writer = ZipWriter::new_append(writer.finish_into_readable()?.into_inner())?; + writer.deep_copy_file_from_path(PATH_1, "")?; + writer = ZipWriter::new_append(writer.finish_into_readable()?.into_inner())?; + writer.shallow_copy_file_from_path("", "copy")?; + writer = ZipWriter::new_append(writer.finish_into_readable()?.into_inner())?; + let _ = writer.finish_into_readable()?; + Ok(()) + } + + #[test] + #[allow(clippy::octal_escapes)] + #[cfg(all(feature = "_bzip2_any", not(miri)))] + fn test_fuzz_crash_2024_06_17b() -> ZipResult<()> { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let options = FileOptions { + compression_method: Stored, + compression_level: None, + last_modified_time: DateTime::from_date_and_time( + 1981, 1, 1, 0, 0, 21, + )?, + permissions: Some(16908288), + large_file: false, + encrypt_with: None, + extended_options: ExtendedFileOptions { + extra_data: vec![].into(), + central_extra_data: vec![].into(), + }, + alignment: 20555, + ..Default::default() + }; + writer.start_file_from_path("\0\u{7}\u{1}\0\0\0\0\0\0\0\0\u{1}\0\0PK\u{1}\u{2};\u{1a}\u{18}\u{1a}UT\t.........................\0u", options)?; + writer + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + let options = FileOptions { + compression_method: CompressionMethod::Bzip2, + compression_level: Some(5), + last_modified_time: DateTime::from_date_and_time( + 2055, 7, 7, 3, 6, 6, + )?, + permissions: None, + large_file: false, + encrypt_with: None, + extended_options: ExtendedFileOptions { + extra_data: vec![].into(), + central_extra_data: vec![].into(), + }, + alignment: 0, + ..Default::default() + }; + writer.start_file_from_path("\0\0\0\0..\0\0\0\0\0\u{7f}\u{7f}PK\u{6}\u{6}K\u{6}\u{6}\u{6}\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\u{1}\0\0\0\0\0\0\0\0\u{1}\0\0PK\u{1}\u{1e},\0\0\0\0\0\0\0\0\0\0\0\u{8}\0*\0\0\u{1}PK\u{6}\u{7}PK\u{6}\u{6}\0\0\0\0\0\0\0\0}K\u{2}\u{6}", options)?; + writer = ZipWriter::new_append( + writer.finish_into_readable()?.into_inner(), + )?; + writer + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + writer = ZipWriter::new_append( + writer.finish_into_readable()?.into_inner(), + )?; + writer + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + writer = + ZipWriter::new_append(writer.finish_into_readable()?.into_inner())?; + writer + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + writer = + ZipWriter::new_append(writer.finish_into_readable()?.into_inner())?; + writer + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + writer + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + writer + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + writer + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + let _ = writer.finish_into_readable()?; + Ok(()) + } + + #[test] + fn test_fuzz_crash_2024_06_18() -> ZipResult<()> { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_raw_comment(Box::<[u8]>::from([ + 80, 75, 5, 6, 255, 255, 255, 255, 255, 255, 80, 75, 5, 6, 255, 255, 255, 255, 255, 255, + 13, 0, 13, 13, 13, 13, 13, 255, 255, 255, 255, 255, 255, 255, 255, + ])); + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + writer.set_raw_comment(Box::new([])); + writer + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + writer = ZipWriter::new_append(writer.finish()?)?; + let _ = writer.finish_into_readable()?; + Ok(()) + } + + #[test] + fn test_fuzz_crash_2024_06_18a() -> ZipResult<()> { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + writer.set_raw_comment(Box::<[u8]>::from([])); + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let options = FullFileOptions { + compression_method: Stored, + compression_level: None, + last_modified_time: DateTime::from_date_and_time(2107, 4, 8, 14, 0, 19)?, + permissions: None, + large_file: false, + encrypt_with: None, + extended_options: ExtendedFileOptions { + extra_data: vec![ + 182, 180, 1, 0, 180, 182, 74, 0, 0, 200, 0, 0, 0, 2, 0, 0, 0, + ] + .into(), + central_extra_data: vec![].into(), + }, + alignment: 1542, + ..Default::default() + }; + writer.start_file_from_path("\0\0PK\u{6}\u{6}K\u{6}PK\u{3}\u{4}\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\u{1}\0\0\0\0\0\0\0\0\u{1}\u{1}\0PK\u{1}\u{2},\0\0\0\0\0\0\0\0\0\0\0P\u{7}\u{4}/.\0KP\0\0;\0\0\0\u{1e}\0\0\0\0\0\0\0\0\0\0\0\0\0", options)?; + let finished = writer.finish_into_readable()?; + assert_eq!(1, finished.file_names().count()); + writer = ZipWriter::new_append(finished.into_inner())?; + let options = FullFileOptions { + compression_method: Stored, + compression_level: Some(5), + last_modified_time: DateTime::from_date_and_time(2107, 4, 1, 0, 0, 0)?, + permissions: None, + large_file: false, + encrypt_with: Some(ZipCrypto( + ZipCryptoKeys::of(0x0, 0x62e4b50, 0x100), + PhantomData, + )), + ..Default::default() + }; + writer.add_symlink_from_path( + "\0K\u{6}\0PK\u{6}\u{7}PK\u{6}\u{6}\0\0\0\0\0\0\0\0PK\u{2}\u{6}", + "\u{8}\0\0\0\0/\0", + options, + )?; + let finished = writer.finish_into_readable()?; + assert_eq!(2, finished.file_names().count()); + writer = ZipWriter::new_append(finished.into_inner())?; + assert_eq!(2, writer.files.len()); + writer + }; + let finished = sub_writer.finish_into_readable()?; + assert_eq!(2, finished.file_names().count()); + writer.merge_archive(finished)?; + writer = ZipWriter::new_append(writer.finish_into_readable()?.into_inner())?; + writer + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + writer + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + let _ = writer.finish_into_readable()?; + Ok(()) + } + + #[cfg(all(feature = "_bzip2_any", feature = "aes-crypto", not(miri)))] + #[test] + fn test_fuzz_crash_2024_06_18b() -> ZipResult<()> { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(true); + writer.set_raw_comment([0].into()); + writer = ZipWriter::new_append(writer.finish_into_readable()?.into_inner())?; + assert_eq!(writer.get_raw_comment()[0], 0); + let options = FileOptions { + compression_method: CompressionMethod::Bzip2, + compression_level: None, + last_modified_time: DateTime::from_date_and_time(2009, 6, 3, 13, 37, 39)?, + permissions: Some(2644352413), + large_file: true, + encrypt_with: Some(crate::write::EncryptWith::Aes { + mode: crate::AesMode::Aes256, + password: "", + }), + extended_options: ExtendedFileOptions { + extra_data: vec![].into(), + central_extra_data: vec![].into(), + }, + alignment: 255, + ..Default::default() + }; + writer.add_symlink_from_path("", "", options)?; + writer.deep_copy_file_from_path("", "PK\u{5}\u{6}\0\0\0\0\0\0\0\0\0\0\0\0\0\u{4}\0\0\0")?; + writer = ZipWriter::new_append(writer.finish_into_readable()?.into_inner())?; + assert_eq!(writer.get_raw_comment()[0], 0); + writer.deep_copy_file_from_path( + "PK\u{5}\u{6}\0\0\0\0\0\0\0\0\0\0\0\0\0\u{4}\0\0\0", + "\u{2}yy\u{5}qu\0", + )?; + let finished = writer.finish()?; + let archive = ZipArchive::new(finished.clone())?; + assert_eq!(archive.comment(), [0]); + writer = ZipWriter::new_append(finished)?; + assert_eq!(writer.get_raw_comment()[0], 0); + let _ = writer.finish_into_readable()?; + Ok(()) + } + + #[test] + fn test_fuzz_crash_2024_06_19() -> ZipResult<()> { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let options = FileOptions { + compression_method: Stored, + compression_level: None, + last_modified_time: DateTime::from_date_and_time(1980, 3, 1, 19, 55, 58)?, + permissions: None, + large_file: false, + encrypt_with: None, + extended_options: ExtendedFileOptions { + extra_data: vec![].into(), + central_extra_data: vec![].into(), + }, + alignment: 256, + ..Default::default() + }; + writer.start_file_from_path( + "\0\0\0PK\u{5}\u{6}\0\0\0\0\u{1}\0\u{12}\u{6}\0\0\0\0\0\u{1}\0\0\0\0\0\0\0\0\0", + options, + )?; + writer.set_flush_on_finish_file(false); + writer.shallow_copy_file_from_path( + "\0\0\0PK\u{5}\u{6}\0\0\0\0\u{1}\0\u{12}\u{6}\0\0\0\0\0\u{1}\0\0\0\0\0\0\0\0\0", + "", + )?; + writer.set_flush_on_finish_file(false); + writer.deep_copy_file_from_path("", "copy")?; + writer.abort_file()?; + writer.set_flush_on_finish_file(false); + writer.set_raw_comment([255, 0].into()); + writer.abort_file()?; + assert_eq!(writer.get_raw_comment(), [255, 0]); + writer = ZipWriter::new_append(writer.finish_into_readable()?.into_inner())?; + assert_eq!(writer.get_raw_comment(), [255, 0]); + writer.set_flush_on_finish_file(false); + let options = FileOptions { + compression_method: Stored, + compression_level: None, + last_modified_time: DateTime::default(), + permissions: None, + large_file: false, + encrypt_with: None, + extended_options: ExtendedFileOptions { + extra_data: vec![].into(), + central_extra_data: vec![].into(), + }, + ..Default::default() + }; + writer.start_file_from_path("", options)?; + assert_eq!(writer.get_raw_comment(), [255, 0]); + let archive = writer.finish_into_readable()?; + assert_eq!(archive.comment(), [255, 0]); + Ok(()) + } + + #[test] + fn fuzz_crash_2024_06_21() -> ZipResult<()> { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let options = FullFileOptions { + compression_method: Stored, + compression_level: None, + last_modified_time: DateTime::from_date_and_time(1980, 2, 1, 0, 0, 0)?, + permissions: None, + large_file: false, + encrypt_with: None, + ..Default::default() + }; + const LONG_PATH: &str = "\0@PK\u{6}\u{6}\u{7}\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0@/\0\0\00ΝPK\u{5}\u{6}O\0\u{10}\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0@PK\u{6}\u{7}\u{6}\0/@\0\0\0\0\0\0\0\0 \0\0"; + writer.start_file_from_path(LONG_PATH, options)?; + writer = ZipWriter::new_append(writer.finish()?)?; + writer.deep_copy_file_from_path(LONG_PATH, "oo\0\0\0")?; + writer.abort_file()?; + writer.set_raw_comment([33].into()); + let archive = writer.finish_into_readable()?; + writer = ZipWriter::new_append(archive.into_inner())?; + assert!(writer.get_raw_comment().starts_with(&[33])); + let archive = writer.finish_into_readable()?; + assert!(archive.comment().starts_with(&[33])); + Ok(()) + } + + #[test] + #[cfg(all(feature = "_bzip2_any", not(miri)))] + fn fuzz_crash_2024_07_17() -> ZipResult<()> { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let options = FileOptions { + compression_method: CompressionMethod::Bzip2, + compression_level: None, + last_modified_time: DateTime::from_date_and_time(2095, 2, 16, 21, 0, 1)?, + permissions: Some(84238341), + large_file: true, + encrypt_with: None, + extended_options: ExtendedFileOptions { + extra_data: vec![117, 99, 6, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 2, 0, 0, 0].into(), + central_extra_data: vec![].into(), + }, + alignment: 65535, + ..Default::default() + }; + writer.start_file_from_path("", options)?; + //writer = ZipWriter::new_append(writer.finish_into_readable()?.into_inner())?; + writer.deep_copy_file_from_path("", "copy")?; + let _ = ZipWriter::new_append(writer.finish_into_readable()?.into_inner())?; + Ok(()) + } + + #[test] + fn fuzz_crash_2024_07_19() -> ZipResult<()> { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let options = FileOptions { + compression_method: Stored, + compression_level: None, + last_modified_time: DateTime::from_date_and_time(1980, 6, 1, 0, 34, 47)?, + permissions: None, + large_file: true, + encrypt_with: None, + extended_options: ExtendedFileOptions { + extra_data: vec![].into(), + central_extra_data: vec![].into(), + }, + alignment: 45232, + ..Default::default() + }; + writer.add_directory_from_path("", options)?; + writer.deep_copy_file_from_path("/", "")?; + writer = ZipWriter::new_append(writer.finish_into_readable()?.into_inner())?; + writer.deep_copy_file_from_path("", "copy")?; + let _ = ZipWriter::new_append(writer.finish_into_readable()?.into_inner())?; + Ok(()) + } + + #[test] + #[cfg(feature = "aes-crypto")] + fn fuzz_crash_2024_07_19a() -> ZipResult<()> { + use crate::write::EncryptWith::Aes; + use crate::AesMode::Aes128; + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let options = FileOptions { + compression_method: Stored, + compression_level: None, + last_modified_time: DateTime::from_date_and_time(2107, 6, 5, 13, 0, 21)?, + permissions: None, + large_file: true, + encrypt_with: Some(Aes { + mode: Aes128, + password: "", + }), + extended_options: ExtendedFileOptions { + extra_data: vec![3, 0, 4, 0, 209, 53, 53, 8, 2, 61, 0, 0].into(), + central_extra_data: vec![].into(), + }, + alignment: 65535, + ..Default::default() + }; + writer.start_file_from_path("", options)?; + let _ = ZipWriter::new_append(writer.finish_into_readable()?.into_inner())?; + Ok(()) + } + + #[test] + fn fuzz_crash_2024_07_20() -> ZipResult<()> { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(true); + let options = FileOptions { + compression_method: Stored, + compression_level: None, + last_modified_time: DateTime::from_date_and_time(2041, 8, 2, 19, 38, 0)?, + permissions: None, + large_file: false, + encrypt_with: None, + extended_options: ExtendedFileOptions { + extra_data: vec![].into(), + central_extra_data: vec![].into(), + }, + alignment: 0, + ..Default::default() + }; + writer.add_directory_from_path("\0\0\0\0\0\0\07黻", options)?; + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let options = FileOptions { + compression_method: Stored, + compression_level: None, + last_modified_time: DateTime::default(), + permissions: None, + large_file: false, + encrypt_with: None, + extended_options: ExtendedFileOptions { + extra_data: vec![].into(), + central_extra_data: vec![].into(), + }, + alignment: 4, + ..Default::default() + }; + writer.add_directory_from_path("\0\0\0黻", options)?; + writer = ZipWriter::new_append(writer.finish_into_readable()?.into_inner())?; + writer.abort_file()?; + let options = FileOptions { + compression_method: Stored, + compression_level: None, + last_modified_time: DateTime::from_date_and_time(1980, 1, 1, 0, 7, 0)?, + permissions: Some(2663103419), + large_file: false, + encrypt_with: None, + extended_options: ExtendedFileOptions { + extra_data: vec![].into(), + central_extra_data: vec![].into(), + }, + alignment: 32256, + ..Default::default() + }; + writer.add_directory_from_path("\0", options)?; + writer = ZipWriter::new_append(writer.finish()?)?; + writer + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + let _ = ZipWriter::new_append(writer.finish_into_readable()?.into_inner())?; + Ok(()) + } + + #[test] + fn fuzz_crash_2024_07_21() -> ZipResult<()> { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.add_directory_from_path( + "", + FileOptions { + compression_method: Stored, + compression_level: None, + last_modified_time: DateTime::from_date_and_time(2105, 8, 1, 15, 0, 0)?, + permissions: None, + large_file: false, + encrypt_with: None, + extended_options: ExtendedFileOptions { + extra_data: vec![].into(), + central_extra_data: vec![].into(), + }, + alignment: 0, + ..Default::default() + }, + )?; + writer.abort_file()?; + let mut writer = ZipWriter::new_append(writer.finish()?)?; + writer.add_directory_from_path( + "", + FileOptions { + compression_method: Stored, + compression_level: None, + last_modified_time: DateTime::default(), + permissions: None, + large_file: false, + encrypt_with: None, + extended_options: ExtendedFileOptions { + extra_data: vec![].into(), + central_extra_data: vec![].into(), + }, + alignment: 16, + ..Default::default() + }, + )?; + ZipWriter::new_append(writer.finish()?)? + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + let writer = ZipWriter::new_append(writer.finish()?)?; + let _ = writer.finish_into_readable()?; + + Ok(()) + } +} diff --git a/v2/vendor/security/zip-7.2.0/src/zipcrypto.rs b/v2/vendor/security/zip-7.2.0/src/zipcrypto.rs new file mode 100644 index 0000000000..ee37b74b85 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/src/zipcrypto.rs @@ -0,0 +1,280 @@ +//! Implementation of the ZipCrypto algorithm +//! +//! The following paper was used to implement the ZipCrypto algorithm: +//! [https://courses.cs.ut.ee/MTAT.07.022/2015_fall/uploads/Main/dmitri-report-f15-16.pdf](https://courses.cs.ut.ee/MTAT.07.022/2015_fall/uploads/Main/dmitri-report-f15-16.pdf) + +use core::fmt::{Debug, Formatter}; +use core::hash::Hash; +use core::marker::PhantomData; +use core::num::Wrapping; + +use crate::cfg_if_expr; +use crate::result::ZipError; +#[cfg(feature = "aes-crypto")] +use crate::AesMode; + +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub(crate) enum EncryptWith<'k> { + #[cfg(feature = "aes-crypto")] + Aes { + mode: AesMode, + password: &'k str, + }, + ZipCrypto(ZipCryptoKeys, PhantomData<&'k ()>), +} + +#[cfg(feature = "_arbitrary")] +impl<'a> arbitrary::Arbitrary<'a> for EncryptWith<'a> { + fn arbitrary(u: &mut arbitrary::Unstructured<'a>) -> arbitrary::Result { + #[cfg(feature = "aes-crypto")] + if bool::arbitrary(u)? { + return Ok(EncryptWith::Aes { + mode: AesMode::arbitrary(u)?, + password: u.arbitrary::<&str>()?, + }); + } + + Ok(EncryptWith::ZipCrypto( + ZipCryptoKeys::arbitrary(u)?, + PhantomData, + )) + } +} + +/// A container to hold the current key state +#[cfg_attr(feature = "_arbitrary", derive(arbitrary::Arbitrary))] +#[derive(Clone, Copy, Hash, Ord, PartialOrd, Eq, PartialEq)] +pub(crate) struct ZipCryptoKeys { + key_0: Wrapping, + key_1: Wrapping, + key_2: Wrapping, +} + +impl Debug for ZipCryptoKeys { + #[allow(unreachable_code)] + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + cfg_if_expr! { + #[cfg(any(test, fuzzing))] => { + f.write_fmt(format_args!( + "ZipCryptoKeys::of({:#10x},{:#10x},{:#10x})", + self.key_0, self.key_1, self.key_2 + )) + }, + _ => { + use std::collections::hash_map::DefaultHasher; + use std::hash::Hasher; + let mut t = DefaultHasher::new(); + self.hash(&mut t); + f.write_fmt(format_args!("ZipCryptoKeys(hash {})", t.finish())) + } + } + } +} + +impl ZipCryptoKeys { + const fn new() -> ZipCryptoKeys { + ZipCryptoKeys { + key_0: Wrapping(0x12345678), + key_1: Wrapping(0x23456789), + key_2: Wrapping(0x34567890), + } + } + + #[allow(unused)] + pub const fn of(key_0: u32, key_1: u32, key_2: u32) -> ZipCryptoKeys { + ZipCryptoKeys { + key_0: Wrapping(key_0), + key_1: Wrapping(key_1), + key_2: Wrapping(key_2), + } + } + + fn update(&mut self, input: u8) { + self.key_0 = ZipCryptoKeys::crc32(self.key_0, input); + self.key_1 = + (self.key_1 + (self.key_0 & Wrapping(0xff))) * Wrapping(0x08088405) + Wrapping(1); + self.key_2 = ZipCryptoKeys::crc32(self.key_2, (self.key_1 >> 24).0 as u8); + } + + fn stream_byte(&mut self) -> u8 { + let temp: Wrapping = Wrapping(self.key_2.0 as u16) | Wrapping(3); + ((temp * (temp ^ Wrapping(1))) >> 8).0 as u8 + } + + fn decrypt_byte(&mut self, cipher_byte: u8) -> u8 { + let plain_byte: u8 = self.stream_byte() ^ cipher_byte; + self.update(plain_byte); + plain_byte + } + + #[allow(dead_code)] + fn encrypt_byte(&mut self, plain_byte: u8) -> u8 { + let cipher_byte: u8 = self.stream_byte() ^ plain_byte; + self.update(plain_byte); + cipher_byte + } + + fn crc32(crc: Wrapping, input: u8) -> Wrapping { + (crc >> 8) ^ Wrapping(CRCTABLE[((crc & Wrapping(0xff)).0 as u8 ^ input) as usize]) + } + pub(crate) fn derive(password: &[u8]) -> ZipCryptoKeys { + let mut keys = ZipCryptoKeys::new(); + for byte in password.iter() { + keys.update(*byte); + } + keys + } +} + +/// A ZipCrypto reader with unverified password +pub struct ZipCryptoReader { + file: R, + keys: ZipCryptoKeys, +} + +pub enum ZipCryptoValidator { + PkzipCrc32(u32), + InfoZipMsdosTime(u16), +} + +impl ZipCryptoReader { + /// Note: The password is `&[u8]` and not `&str` because the + /// [zip specification](https://pkware.cachefly.net/webdocs/APPNOTE/APPNOTE-6.3.3.TXT) + /// does not specify password encoding (see function `update_keys` in the specification). + /// Therefore, if `&str` was used, the password would be UTF-8 and it + /// would be impossible to decrypt files that were encrypted with a + /// password byte sequence that is unrepresentable in UTF-8. + pub fn new(file: R, password: &[u8]) -> ZipCryptoReader { + ZipCryptoReader { + file, + keys: ZipCryptoKeys::derive(password), + } + } + + /// Read the ZipCrypto header bytes and validate the password. + pub fn validate( + mut self, + validator: ZipCryptoValidator, + ) -> Result, ZipError> { + // ZipCrypto prefixes a file with a 12 byte header + let mut header_buf = [0u8; 12]; + self.file.read_exact(&mut header_buf)?; + for byte in header_buf.iter_mut() { + *byte = self.keys.decrypt_byte(*byte); + } + + match validator { + ZipCryptoValidator::PkzipCrc32(crc32_plaintext) => { + // PKZIP before 2.0 used 2 byte CRC check. + // PKZIP 2.0+ used 1 byte CRC check. It's more secure. + // We also use 1 byte CRC. + + if (crc32_plaintext >> 24) as u8 != header_buf[11] { + return Err(ZipError::InvalidPassword); + } + } + ZipCryptoValidator::InfoZipMsdosTime(last_mod_time) => { + // Info-ZIP modification to ZipCrypto format: + // If bit 3 of the general purpose bit flag is set + // (indicates that the file uses a data-descriptor section), + // it uses high byte of 16-bit File Time. + // Info-ZIP code probably writes 2 bytes of File Time. + // We check only 1 byte. + + if (last_mod_time >> 8) as u8 != header_buf[11] { + return Err(ZipError::InvalidPassword); + } + } + } + + Ok(ZipCryptoReaderValid { reader: self }) + } +} +#[allow(unused)] +pub(crate) struct ZipCryptoWriter { + pub(crate) writer: W, + pub(crate) keys: ZipCryptoKeys, +} +impl ZipCryptoWriter { + #[allow(unused)] + pub(crate) fn finish(mut self) -> std::io::Result { + Ok(self.writer) + } +} +impl std::io::Write for ZipCryptoWriter { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + const CHUNK_SIZE: usize = 4096; + let mut temp_buf = [0u8; CHUNK_SIZE]; + for chunk in buf.chunks(CHUNK_SIZE) { + let encrypted_chunk = &mut temp_buf[..chunk.len()]; + for (i, &byte) in chunk.iter().enumerate() { + encrypted_chunk[i] = self.keys.encrypt_byte(byte); + } + self.writer.write_all(encrypted_chunk)?; + } + Ok(buf.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + self.writer.flush() + } +} + +/// A ZipCrypto reader with verified password +pub struct ZipCryptoReaderValid { + reader: ZipCryptoReader, +} + +impl std::io::Read for ZipCryptoReaderValid { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + // Note: There might be potential for optimization. Inspiration can be found at: + // https://github.com/kornelski/7z/blob/master/CPP/7zip/Crypto/ZipCrypto.cpp + + let n = self.reader.file.read(buf)?; + for byte in buf.iter_mut().take(n) { + *byte = self.reader.keys.decrypt_byte(*byte); + } + Ok(n) + } +} + +impl ZipCryptoReaderValid { + /// Consumes this decoder, returning the underlying reader. + pub fn into_inner(self) -> R { + self.reader.file + } +} + +static CRCTABLE: [u32; 256] = [ + 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f, 0xe963a535, 0x9e6495a3, + 0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988, 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91, + 0x1db71064, 0x6ab020f2, 0xf3b97148, 0x84be41de, 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7, + 0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, 0x14015c4f, 0x63066cd9, 0xfa0f3d63, 0x8d080df5, + 0x3b6e20c8, 0x4c69105e, 0xd56041e4, 0xa2677172, 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b, + 0x35b5a8fa, 0x42b2986c, 0xdbbbc9d6, 0xacbcf940, 0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59, + 0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116, 0x21b4f4b5, 0x56b3c423, 0xcfba9599, 0xb8bda50f, + 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924, 0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d, + 0x76dc4190, 0x01db7106, 0x98d220bc, 0xefd5102a, 0x71b18589, 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433, + 0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818, 0x7f6a0dbb, 0x086d3d2d, 0x91646c97, 0xe6635c01, + 0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, 0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457, + 0x65b0d9c6, 0x12b7e950, 0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65, + 0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, 0x4adfa541, 0x3dd895d7, 0xa4d1c46d, 0xd3d6f4fb, + 0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0, 0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9, + 0x5005713c, 0x270241aa, 0xbe0b1010, 0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f, + 0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, 0x59b33d17, 0x2eb40d81, 0xb7bd5c3b, 0xc0ba6cad, + 0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a, 0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683, + 0xe3630b12, 0x94643b84, 0x0d6d6a3e, 0x7a6a5aa8, 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1, + 0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb, 0x196c3671, 0x6e6b06e7, + 0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc, 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5, + 0xd6d6a3e8, 0xa1d1937e, 0x38d8c2c4, 0x4fdff252, 0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b, + 0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55, 0x316e8eef, 0x4669be79, + 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236, 0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f, + 0xc5ba3bbe, 0xb2bd0b28, 0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d, + 0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a, 0x9c0906a9, 0xeb0e363f, 0x72076785, 0x05005713, + 0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, 0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21, + 0x86d3d2d4, 0xf1d4e242, 0x68ddb3f8, 0x1fda836e, 0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777, + 0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, 0x8f659eff, 0xf862ae69, 0x616bffd3, 0x166ccf45, + 0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2, 0xa7672661, 0xd06016f7, 0x4969474d, 0x3e6e77db, + 0xaed16a4a, 0xd9d65adc, 0x40df0b66, 0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9, + 0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605, 0xcdd70693, 0x54de5729, 0x23d967bf, + 0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94, 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d, +]; diff --git a/v2/vendor/security/zip-7.2.0/tests/ruview_0429_backport.rs b/v2/vendor/security/zip-7.2.0/tests/ruview_0429_backport.rs new file mode 100644 index 0000000000..747b45dc77 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/tests/ruview_0429_backport.rs @@ -0,0 +1,143 @@ +/// A symlink entry whose central directory declares an absurd uncompressed size +/// must be rejected, not pre-allocated. +/// +/// The declared size comes from the archive itself. Before the fix, `extract` +/// passed it straight to `Vec::with_capacity`, so a hostile entry could ask for +/// 2^62 bytes and abort the whole process ("memory allocation of ... failed"). +/// A symlink target is a path, so a size this large is malformed and the entry +/// must come back as an error. +#[cfg(not(miri))] +#[test] +fn extract_symlink_with_hostile_declared_size() -> zip::result::ZipResult<()> { + use std::io::Cursor; + use tempfile::TempDir; + use zip::ZipArchive; + + /// One entry, flagged as a symlink, whose Zip64 extra field claims + /// `declared_size` uncompressed bytes while storing only `payload`. + fn build(declared_size: u64, payload: &[u8]) -> Vec { + const S_IFLNK: u32 = 0o120_000; + const SENTINEL: u32 = 0xFFFF_FFFF; + let name = b"link"; + let mut o = Vec::new(); + + // local file header + o.extend_from_slice(&[0x50, 0x4b, 0x03, 0x04]); + o.extend_from_slice(&45u16.to_le_bytes()); // version needed: zip64 + o.extend_from_slice(&0u16.to_le_bytes()); // flags + o.extend_from_slice(&0u16.to_le_bytes()); // method: Stored + o.extend_from_slice(&0u16.to_le_bytes()); // mod time + o.extend_from_slice(&0x21u16.to_le_bytes()); // mod date + o.extend_from_slice(&0u32.to_le_bytes()); // crc32 + o.extend_from_slice(&SENTINEL.to_le_bytes()); + o.extend_from_slice(&SENTINEL.to_le_bytes()); + o.extend_from_slice(&(name.len() as u16).to_le_bytes()); + o.extend_from_slice(&20u16.to_le_bytes()); // zip64 extra field + o.extend_from_slice(name); + o.extend_from_slice(&0x0001u16.to_le_bytes()); // zip64 header id + o.extend_from_slice(&16u16.to_le_bytes()); + o.extend_from_slice(&declared_size.to_le_bytes()); + o.extend_from_slice(&(payload.len() as u64).to_le_bytes()); + o.extend_from_slice(payload); + + let cd_offset = o.len() as u32; + + // central directory header + o.extend_from_slice(&[0x50, 0x4b, 0x01, 0x02]); + o.extend_from_slice(&0x031Eu16.to_le_bytes()); // version made by: Unix + o.extend_from_slice(&45u16.to_le_bytes()); + o.extend_from_slice(&0u16.to_le_bytes()); + o.extend_from_slice(&0u16.to_le_bytes()); + o.extend_from_slice(&0u16.to_le_bytes()); + o.extend_from_slice(&0x21u16.to_le_bytes()); + o.extend_from_slice(&0u32.to_le_bytes()); // crc32 + o.extend_from_slice(&SENTINEL.to_le_bytes()); + o.extend_from_slice(&SENTINEL.to_le_bytes()); + o.extend_from_slice(&(name.len() as u16).to_le_bytes()); + o.extend_from_slice(&20u16.to_le_bytes()); + o.extend_from_slice(&0u16.to_le_bytes()); // comment len + o.extend_from_slice(&0u16.to_le_bytes()); // disk number start + o.extend_from_slice(&0u16.to_le_bytes()); // internal attrs + o.extend_from_slice(&((S_IFLNK | 0o777) << 16).to_le_bytes()); // symlink + o.extend_from_slice(&0u32.to_le_bytes()); // local header offset + o.extend_from_slice(name); + o.extend_from_slice(&0x0001u16.to_le_bytes()); + o.extend_from_slice(&16u16.to_le_bytes()); + o.extend_from_slice(&declared_size.to_le_bytes()); + o.extend_from_slice(&(payload.len() as u64).to_le_bytes()); + + let cd_size = o.len() as u32 - cd_offset; + + // end of central directory + o.extend_from_slice(&[0x50, 0x4b, 0x05, 0x06]); + o.extend_from_slice(&0u16.to_le_bytes()); + o.extend_from_slice(&0u16.to_le_bytes()); + o.extend_from_slice(&1u16.to_le_bytes()); + o.extend_from_slice(&1u16.to_le_bytes()); + o.extend_from_slice(&cd_size.to_le_bytes()); + o.extend_from_slice(&cd_offset.to_le_bytes()); + o.extend_from_slice(&0u16.to_le_bytes()); + o + } + + for declared_size in [4097, 1u64 << 40, 1u64 << 62, u64::MAX] { + let mut reader = ZipArchive::new(Cursor::new(build(declared_size, b"/etc/passwd")))?; + assert!(reader.by_index(0)?.is_symlink()); + let tempdir = TempDir::with_prefix("test_hostile_symlink_size")?; + + let err = reader + .extract(&tempdir) + .expect_err("oversized symlink target must be rejected"); + assert!( + matches!(err, zip::result::ZipError::InvalidArchive(_)), + "expected InvalidArchive for a {declared_size} byte symlink target, got {err:?}" + ); + } + + Ok(()) +} + + +#[test] +fn stream_does_not_expose_the_symlink_bit_backport() -> zip::result::ZipResult<()> { + use std::io::Cursor; + use zip::{ZipWriter, write::SimpleFileOptions}; + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.add_symlink("link", "/target", SimpleFileOptions::default())?; + let mut cur = writer.finish()?; + cur.set_position(0); + let file = zip::read::read_zipfile_from_stream(&mut cur)?.unwrap(); + assert!(!file.is_symlink()); + Ok(()) +} + +#[test] +fn valid_regular_file_roundtrip() -> zip::result::ZipResult<()> { + use std::io::{Cursor, Write}; + use zip::{ZipArchive, ZipWriter, write::SimpleFileOptions}; + let mut w = ZipWriter::new(Cursor::new(Vec::new())); + w.start_file("regular.txt", SimpleFileOptions::default())?; + w.write_all(b"ordinary archive payload")?; + let mut bytes = w.finish()?; bytes.set_position(0); + let dir = tempfile::TempDir::new()?; + ZipArchive::new(bytes)?.extract(dir.path())?; + assert_eq!(std::fs::read(dir.path().join("regular.txt"))?, b"ordinary archive payload"); + Ok(()) +} + +#[cfg(unix)] +#[test] +fn valid_relative_symlink_roundtrip() -> zip::result::ZipResult<()> { + use std::io::{Cursor, Write}; + use zip::{ZipArchive, ZipWriter, write::SimpleFileOptions}; + let mut w = ZipWriter::new(Cursor::new(Vec::new())); + w.start_file("target", SimpleFileOptions::default())?; + w.write_all(b"target payload")?; + w.add_symlink("link", "target", SimpleFileOptions::default())?; + let mut bytes = w.finish()?; bytes.set_position(0); + let dir = tempfile::TempDir::new()?; + ZipArchive::new(bytes)?.extract(dir.path())?; + assert_eq!(std::fs::read_link(dir.path().join("link"))?, std::path::PathBuf::from("target")); + assert_eq!(std::fs::read(dir.path().join("link"))?, b"target payload"); + Ok(()) +}