diff --git a/.gitattributes b/.gitattributes index 7597f9e..9ebbac2 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1 +1,13 @@ backend/spike_corpus/*.wav filter=lfs diff=lfs merge=lfs -text + +# Managed runtimes hash these exact UTF-8 bytes. Checkout conversion would +# otherwise make a release built on Windows disagree with the signed/pinned +# manifest digests while preserving semantically identical text. +backend/runtime-locks/*.txt text eol=lf +scripts/sa3-requirements.lock text eol=lf +scripts/sa3-tflite-requirements.lock text eol=lf +mrt2-pytorch-pin.json text eol=lf +mrt2-pytorch-wheels.json text eol=lf +sa3-pin.json text eol=lf +sa3-tflite-pin.json text eol=lf +sa3-tflite-wheels.json text eol=lf diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 3a92533..476b50b 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -4,6 +4,17 @@ /justfile @protocol-works/engineering /scripts/create-release.sh @protocol-works/engineering /scripts/build-macos-release.sh @protocol-works/engineering +/scripts/build-linux-appimage.sh @protocol-works/engineering +/scripts/assert-windows-release-rejects-unsigned.ps1 @protocol-works/engineering +/scripts/build-windows-installer.ps1 @protocol-works/engineering /scripts/freeze-sidecar.sh @protocol-works/engineering +/scripts/release_artifact.py @protocol-works/engineering +/scripts/sign-windows.ps1 @protocol-works/engineering +/scripts/test-windows-installer.ps1 @protocol-works/engineering +/scripts/verify-windows-release-install.ps1 @protocol-works/engineering +/scripts/verify-windows-signatures.ps1 @protocol-works/engineering +/scripts/verify_linux_appimage.py @protocol-works/engineering /src-tauri/entitlements.plist @protocol-works/engineering /src-tauri/tauri*.conf.json @protocol-works/engineering +/src-tauri/windows/English.nsh @protocol-works/engineering +/src-tauri/windows/installer-hooks.nsh @protocol-works/engineering diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..76c16d4 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,344 @@ +name: CI + +on: + pull_request: + push: + branches: + - main + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ci-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + windows-managed-runtime: + name: Managed runtime qualification (Windows) + runs-on: windows-2025 + timeout-minutes: 90 + + steps: + - name: Check out source and test corpus + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + lfs: true + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.13" + + - name: Set up Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: 24 + cache: npm + cache-dependency-path: frontend/package-lock.json + + - name: Build frontend assets for native shell + working-directory: frontend + run: npm ci && npm run build + + - name: Set up Rust + run: rustup toolchain install stable --profile minimal --no-self-update + + - name: Test managed runtime workspace + working-directory: src-tauri + run: cargo test --locked --workspace --features managed-runtime + + - name: Lint managed runtime workspace + working-directory: src-tauri + run: cargo clippy --locked --workspace --all-targets --features managed-runtime -- -D warnings + + - name: Build managed runtime release + working-directory: src-tauri + run: cargo build --locked --workspace --release --features managed-runtime + + shared: + name: Shared checks (${{ matrix.name }}) + runs-on: ${{ matrix.runner }} + timeout-minutes: 120 + strategy: + fail-fast: false + matrix: + include: + - name: macOS + runner: macos-15 + - name: Ubuntu + runner: ubuntu-24.04 + - name: Windows + runner: windows-2025 + + steps: + - name: Check out source and test corpus + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + lfs: true + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.13" + + - name: Install locked Python environment tool + run: python -m pip install uv==0.11.7 + + - name: Test release artifact contract + run: python -m unittest discover -s scripts/tests -v + + - name: Check release tooling formatting + run: >- + uv run --project backend --frozen --only-group ci ruff format --check + scripts/release_artifact.py scripts/verify_linux_appimage.py + scripts/tests/test_release_artifact.py + scripts/tests/test_verify_linux_appimage.py + scripts/tests/test_windows_packaging.py + + - name: Lint release tooling + run: >- + uv run --project backend --frozen --only-group ci ruff check + scripts/release_artifact.py scripts/verify_linux_appimage.py + scripts/tests/test_release_artifact.py + scripts/tests/test_verify_linux_appimage.py + scripts/tests/test_windows_packaging.py + + - name: Check portable Python formatting + working-directory: backend + run: uv run --frozen --only-group ci ruff format --check . + + - name: Lint portable Python + working-directory: backend + run: uv run --frozen --only-group ci ruff check . + + - name: Test portable Python services + working-directory: backend + run: >- + uv run --frozen --only-group ci python -m pytest + tests/test_loras.py + tests/test_worker.py + tests/test_sidecar.py + tests/test_controller.py + tests/test_frozen.py + + # Most DeckEngine behavior is model-independent. The excluded tests cross + # the Magenta/MLX import boundary and remain in the local full suite and + # the signed macOS release runtime check. + - name: Test portable Python deck behavior + working-directory: backend + run: >- + uv run --frozen --only-group ci python -m pytest tests/test_engine.py + -k "not constructor_uses_reference_sampling_defaults + and not embed_sample + and not sample_key_never_hits_the_text_embedder + and not sample_cache + and not failed_embed_does_not_evict" + + # SA3 uses a copied Python interpreter and fake CLI; no runtime, weights, + # accelerator, shell, or network is involved. The selected model-manager + # nodes exercise only SA3 readiness. Its Magenta discovery/download nodes + # remain behind the backend-specific runtime gate. + - name: Test model-free Python runtime contracts + working-directory: backend + run: >- + uv run --frozen --only-group ci python -m pytest + tests/test_sa3.py + tests/test_sa3_audio.py + tests/test_sa3_manifest.py + tests/test_gpu_broker.py + tests/test_mrt2_pytorch.py + tests/test_mrt2_runtime.py + tests/test_mrt2_runtime_locks.py + tests/test_render_sidecar.py + tests/test_runtime_paths.py + tests/test_sa3_cuda.py + tests/test_sa3_cuda_pins.py + tests/test_sa3_cuda_worker.py + tests/test_models.py::test_readiness_classifies_a_checkout + tests/test_models.py::test_readiness_missing_when_no_checkout + + - name: Set up Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: 24 + cache: npm + cache-dependency-path: frontend/package-lock.json + + - name: Install frontend dependencies + working-directory: frontend + run: npm ci + + - name: Lint frontend + working-directory: frontend + run: npm run lint + + - name: Type-check frontend + working-directory: frontend + run: npx tsc -b + + - name: Test frontend + working-directory: frontend + run: npm test + + - name: Build frontend assets for native shell + working-directory: frontend + run: npm run build + + - name: Install Ubuntu native build dependencies + if: runner.os == 'Linux' + run: >- + sudo apt-get update && sudo apt-get install --yes + build-essential + libasound2-dev + libayatana-appindicator3-dev + libgtk-3-dev + libssl-dev + libudev-dev + libwebkit2gtk-4.1-dev + libxdo-dev + librsvg2-dev + + - name: Set up Rust + run: rustup toolchain install stable --profile minimal --no-self-update + + - name: Test Rust workspace + run: cargo test --locked --workspace --manifest-path src-tauri/Cargo.toml + + - name: Lint Rust workspace + run: >- + cargo clippy --locked --workspace --all-targets + --manifest-path src-tauri/Cargo.toml -- -D warnings + + - name: Install pinned Tauri packaging CLI + if: runner.os == 'Windows' + run: cargo install tauri-cli --version '=2.11.2' --locked + + - name: Build older unsigned Windows installer + if: runner.os == 'Windows' + id: windows_older + shell: pwsh + run: ./scripts/build-windows-installer.ps1 -ReleaseTag v2026.08.1 -UnsignedDevelopment + + - name: Build newer unsigned Windows installer + if: runner.os == 'Windows' + id: windows_newer + shell: pwsh + run: ./scripts/build-windows-installer.ps1 -ReleaseTag v2026.08.2 -UnsignedDevelopment + + - name: Prove release verification rejects unsigned artifacts + if: runner.os == 'Windows' + shell: pwsh + run: >- + ./scripts/assert-windows-release-rejects-unsigned.ps1 + -Path '${{ steps.windows_newer.outputs.installer }}' + + - name: Test Windows installer lifecycle + if: runner.os == 'Windows' + shell: pwsh + run: >- + ./scripts/test-windows-installer.ps1 + -OlderInstaller '${{ steps.windows_older.outputs.installer }}' + -NewerInstaller '${{ steps.windows_newer.outputs.installer }}' + -OlderVersion '${{ steps.windows_older.outputs.version }}' + -NewerVersion '${{ steps.windows_newer.outputs.version }}' + + - name: Upload unsigned Windows development installer + if: runner.os == 'Windows' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: windows-x64-unsigned-development + path: ${{ steps.windows_newer.outputs.installer }} + if-no-files-found: error + retention-days: 7 + + linux_appimage: + name: Linux AppImage contract (Ubuntu 22.04) + runs-on: ubuntu-22.04 + timeout-minutes: 90 + + steps: + - name: Check out source and test corpus + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + lfs: true + persist-credentials: false + + - name: Set up Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: 24 + cache: npm + cache-dependency-path: frontend/package-lock.json + + - name: Install pinned native build dependencies + shell: bash + run: | + set -euo pipefail + sudo apt-get update + sudo apt-get install --yes \ + binutils \ + build-essential \ + libasound2-dev \ + libayatana-appindicator3-dev \ + libfuse2 \ + libgtk-3-dev \ + libssl-dev \ + libudev-dev \ + libwebkit2gtk-4.1-dev \ + libxdo-dev \ + librsvg2-dev \ + xvfb + rustup toolchain install stable --profile minimal --no-self-update + rustup default stable + cargo install tauri-cli --version '=2.11.2' --locked + npm ci --prefix frontend + + - name: Build and audit AppImage + env: + LSDJ_LINUX_AUDIT_PATH: ${{ runner.temp }}/linux-package-audit.json + shell: bash + run: scripts/build-linux-appimage.sh + + - name: Smoke AppImage with spaces, Unicode, and isolated XDG roots + shell: bash + run: | + set -euo pipefail + shopt -s nullglob + APPIMAGES=(src-tauri/target/release/bundle/appimage/*.AppImage) + [[ "${#APPIMAGES[@]}" -eq 1 ]] + + PROFILE="$RUNNER_TEMP/DJ Name 音楽" + export HOME="$PROFILE/home" + export XDG_CONFIG_HOME="$PROFILE/config 空間" + export XDG_DATA_HOME="$PROFILE/data 音楽" + export XDG_CACHE_HOME="$PROFILE/cache 音楽" + export XDG_RUNTIME_DIR="$RUNNER_TEMP/xdg-runtime" + mkdir -p \ + "$HOME" \ + "$XDG_CONFIG_HOME" \ + "$XDG_DATA_HOME" \ + "$XDG_CACHE_HOME" \ + "$XDG_RUNTIME_DIR" + chmod 700 "$XDG_RUNTIME_DIR" + + set +e + APPIMAGE_EXTRACT_AND_RUN=1 timeout --signal=TERM --kill-after=5s 15s \ + xvfb-run --auto-servernum "${APPIMAGES[0]}" + STATUS=$? + set -e + [[ "$STATUS" -eq 0 || "$STATUS" -eq 124 || "$STATUS" -eq 143 ]] + test -d "$XDG_CONFIG_HOME/lsdj" + test -d "$XDG_DATA_HOME/lsdj" + test -d "$XDG_CACHE_HOME/lsdj" + + - name: Upload package audit evidence + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: linux-package-audit + path: ${{ runner.temp }}/linux-package-audit.json + if-no-files-found: error + retention-days: 14 diff --git a/.github/workflows/macos-release.yml b/.github/workflows/macos-release.yml index 01d8608..9e26258 100644 --- a/.github/workflows/macos-release.yml +++ b/.github/workflows/macos-release.yml @@ -1,9 +1,9 @@ -name: macOS release +name: Release -run-name: macOS release from ${{ github.ref_name }} by @${{ github.actor }} +run-name: Release ${{ github.ref_name }} by @${{ github.actor }} # A protected release tag starts validation. Signing credentials remain behind -# the macos-release Environment's separate human approval gate. +# each platform release Environment's separate human approval gate. on: push: tags: @@ -13,7 +13,7 @@ permissions: contents: read concurrency: - group: macos-release + group: release cancel-in-progress: false jobs: @@ -24,6 +24,8 @@ jobs: startsWith(github.ref, 'refs/tags/v') runs-on: ubuntu-latest timeout-minutes: 10 + outputs: + revision: ${{ steps.release_identity.outputs.revision }} steps: - name: Check out the tagged commit @@ -33,6 +35,7 @@ jobs: persist-credentials: false - name: Verify release tag and ancestry + id: release_identity shell: bash run: | set -euo pipefail @@ -51,8 +54,10 @@ jobs: } echo "Validated $GITHUB_REF_NAME at $TAG_COMMIT" + echo "revision=$TAG_COMMIT" >> "$GITHUB_OUTPUT" - release: + produce_macos: + name: Produce macOS arm64 artifact needs: validate if: >- needs.validate.result == 'success' && @@ -87,6 +92,11 @@ jobs: cache: npm cache-dependency-path: frontend/package-lock.json + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.13" + - name: Check runner architecture run: | set -euo pipefail @@ -198,11 +208,32 @@ jobs: set -euo pipefail just tauri-release - - name: Upload verified DMG + - name: Package verified release artifact + env: + LSDJ_RELEASE_REVISION: ${{ needs.validate.outputs.revision }} + run: | + set -euo pipefail + shopt -s nullglob + DMG_FILES=(src-tauri/target/release/bundle/dmg/*.dmg) + [[ "${#DMG_FILES[@]}" -eq 1 ]] || { + echo "Expected exactly one verified DMG, found ${#DMG_FILES[@]}" >&2 + exit 1 + } + + python scripts/release_artifact.py create \ + --producer macos-arm64 \ + --release-tag "$GITHUB_REF_NAME" \ + --revision "$LSDJ_RELEASE_REVISION" \ + --asset "${DMG_FILES[0]}" \ + --output-dir release-artifacts/macos-arm64 + + # The publisher receives installers only through immutable per-run + # Actions artifacts, together with the tag/revision metadata and digest. + - name: Upload verified producer bundle uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: LSDJ-macOS-arm64 - path: src-tauri/target/release/bundle/dmg/*.dmg + name: release-macos-arm64 + path: release-artifacts/macos-arm64 if-no-files-found: error retention-days: 14 @@ -217,10 +248,246 @@ jobs: "${LSDJ_CERTIFICATE_PATH:-}" \ "${LSDJ_API_KEY_PATH:-}" + produce_linux: + name: Produce Linux x86_64 AppImage + needs: validate + if: >- + needs.validate.result == 'success' && + github.repository == 'protocol-works/lsdj' && + startsWith(github.ref, 'refs/tags/v') + # Ubuntu 22.04 (glibc 2.35) is the oldest supported base. Building here, + # rather than `ubuntu-latest`, prevents a newer host ABI from silently + # raising the AppImage floor. + runs-on: ubuntu-22.04 + timeout-minutes: 120 + + steps: + - name: Check out the approved release tag + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + lfs: true + persist-credentials: false + + - name: Set up Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: 24 + cache: npm + cache-dependency-path: frontend/package-lock.json + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.13" + + - name: Install pinned native build dependencies + shell: bash + run: | + set -euo pipefail + sudo apt-get update + sudo apt-get install --yes \ + binutils \ + build-essential \ + libasound2-dev \ + libayatana-appindicator3-dev \ + libfuse2 \ + libgtk-3-dev \ + libssl-dev \ + libudev-dev \ + libwebkit2gtk-4.1-dev \ + libxdo-dev \ + librsvg2-dev \ + xvfb + rustup toolchain install stable --profile minimal --no-self-update + rustup default stable + cargo install tauri-cli --version '=2.11.2' --locked + npm ci --prefix frontend + + - name: Test fail-closed portable runtime launch + shell: bash + run: >- + cargo test --locked --workspace --features managed-runtime + --manifest-path src-tauri/Cargo.toml + platform_diagnostics::tests::compiled_runtime_policy_matches_the_selected_release_feature + + - name: Build and audit AppImage + env: + LSDJ_RELEASE_VERSION: ${{ github.ref_name }} + LSDJ_LINUX_AUDIT_PATH: ${{ runner.temp }}/linux-package-audit.json + shell: bash + run: scripts/build-linux-appimage.sh + + - name: Smoke AppImage with isolated XDG paths + shell: bash + run: | + set -euo pipefail + shopt -s nullglob + APPIMAGES=(src-tauri/target/release/bundle/appimage/*.AppImage) + [[ "${#APPIMAGES[@]}" -eq 1 ]] + + PROFILE="$RUNNER_TEMP/DJ Name 音楽" + export HOME="$PROFILE/home" + export XDG_CONFIG_HOME="$PROFILE/config 空間" + export XDG_DATA_HOME="$PROFILE/data 音楽" + export XDG_CACHE_HOME="$PROFILE/cache 音楽" + export XDG_RUNTIME_DIR="$RUNNER_TEMP/xdg-runtime" + mkdir -p \ + "$HOME" \ + "$XDG_CONFIG_HOME" \ + "$XDG_DATA_HOME" \ + "$XDG_CACHE_HOME" \ + "$XDG_RUNTIME_DIR" + chmod 700 "$XDG_RUNTIME_DIR" + + set +e + APPIMAGE_EXTRACT_AND_RUN=1 timeout --signal=TERM --kill-after=5s 15s \ + xvfb-run --auto-servernum "${APPIMAGES[0]}" + STATUS=$? + set -e + [[ "$STATUS" -eq 0 || "$STATUS" -eq 124 || "$STATUS" -eq 143 ]] || { + echo "AppImage desktop smoke exited unexpectedly: $STATUS" >&2 + exit 1 + } + test -d "$XDG_CONFIG_HOME/lsdj" + test -d "$XDG_DATA_HOME/lsdj" + test -d "$XDG_CACHE_HOME/lsdj" + + - name: Package verified release artifact + env: + LSDJ_RELEASE_REVISION: ${{ needs.validate.outputs.revision }} + shell: bash + run: | + set -euo pipefail + shopt -s nullglob + APPIMAGES=(src-tauri/target/release/bundle/appimage/*.AppImage) + [[ "${#APPIMAGES[@]}" -eq 1 ]] + python scripts/release_artifact.py create \ + --producer linux-x64 \ + --release-tag "$GITHUB_REF_NAME" \ + --revision "$LSDJ_RELEASE_REVISION" \ + --asset "${APPIMAGES[0]}" \ + --output-dir release-artifacts/linux-x64 + + - name: Upload verified Linux producer bundle + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: release-linux-x64 + path: release-artifacts/linux-x64 + if-no-files-found: error + retention-days: 14 + + - name: Upload Linux package audit evidence + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: linux-package-audit + path: ${{ runner.temp }}/linux-package-audit.json + if-no-files-found: error + retention-days: 14 + + produce_windows: + name: Produce Windows x64 artifact + needs: validate + if: >- + needs.validate.result == 'success' && + github.repository == 'protocol-works/lsdj' && + startsWith(github.ref, 'refs/tags/v') + runs-on: windows-2025 + timeout-minutes: 180 + + # This protected Environment must require a separate human approval. The + # provider-specific identity provisioning step cannot be selected until the + # project chooses an Authenticode provider. These public identity values and + # the credentialed wrapper path stay unavailable before approval. + environment: + name: windows-release + + env: + LSDJ_WINDOWS_SIGN_COMMAND_PATH: ${{ vars.WINDOWS_SIGN_COMMAND_PATH }} + LSDJ_WINDOWS_EXPECTED_CERTIFICATE_SHA1: ${{ vars.WINDOWS_EXPECTED_CERTIFICATE_SHA1 }} + LSDJ_WINDOWS_EXPECTED_SUBJECT: ${{ vars.WINDOWS_EXPECTED_SUBJECT }} + + steps: + - name: Check out the approved release tag + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: 24 + cache: npm + cache-dependency-path: frontend/package-lock.json + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.13" + + - name: Set up Rust + shell: pwsh + run: | + rustup toolchain install stable --profile minimal --no-self-update + rustup default stable + + - name: Install build tools and frontend dependencies + shell: pwsh + run: | + cargo install tauri-cli --version '=2.11.2' --locked + npm ci --prefix frontend + npm run build --prefix frontend + + # The selected provider must insert a protected provisioning step before + # this preflight (for example, federated identity or an HSM-backed client). + # Until that reviewed decision exists, this job intentionally fails here. + - name: Require protected Authenticode identity + shell: pwsh + run: ./scripts/sign-windows.ps1 -Preflight + + - name: Build, sign, timestamp, and verify NSIS installer + id: windows_build + shell: pwsh + run: ./scripts/build-windows-installer.ps1 -ReleaseTag $env:GITHUB_REF_NAME -Release + + - name: Verify installed executable payloads and uninstall behavior + shell: pwsh + run: >- + ./scripts/verify-windows-release-install.ps1 + -Installer '${{ steps.windows_build.outputs.installer }}' + -ExpectedVersion '${{ steps.windows_build.outputs.version }}' + + - name: Package verified release artifact + shell: pwsh + env: + LSDJ_RELEASE_REVISION: ${{ needs.validate.outputs.revision }} + run: >- + python scripts/release_artifact.py create + --producer windows-x64 + --release-tag "$env:GITHUB_REF_NAME" + --revision "$env:LSDJ_RELEASE_REVISION" + --asset '${{ steps.windows_build.outputs.installer }}' + --output-dir release-artifacts/windows-x64 + + - name: Upload verified producer bundle + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: release-windows-x64 + path: release-artifacts/windows-x64 + if-no-files-found: error + retention-days: 14 + publish: - needs: release + name: Verify and publish complete release + needs: + - validate + - produce_macos + - produce_linux + - produce_windows if: >- - needs.release.result == 'success' && + needs.validate.result == 'success' && + needs.produce_macos.result == 'success' && + needs.produce_linux.result == 'success' && + needs.produce_windows.result == 'success' && github.repository == 'protocol-works/lsdj' && startsWith(github.ref, 'refs/tags/v') runs-on: ubuntu-latest @@ -229,26 +496,134 @@ jobs: contents: write steps: - - name: Download verified DMG + - name: Check out the approved release tag + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.13" + + - name: Download macOS producer bundle + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: release-macos-arm64 + path: release-input/macos-arm64 + + - name: Download Linux producer bundle uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: LSDJ-macOS-arm64 - path: dist + name: release-linux-x64 + path: release-input/linux-x64 - - name: Publish GitHub Release + - name: Download Windows producer bundle + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: release-windows-x64 + path: release-input/windows-x64 + + - name: Verify complete required producer set + env: + LSDJ_RELEASE_REVISION: ${{ needs.validate.outputs.revision }} + shell: bash + run: | + set -euo pipefail + python scripts/release_artifact.py verify \ + --input-root release-input \ + --required-producer macos-arm64 \ + --required-producer linux-x64 \ + --required-producer windows-x64 \ + --release-tag "$GITHUB_REF_NAME" \ + --revision "$LSDJ_RELEASE_REVISION" \ + --output-dir verified-release + + - name: Publish verified GitHub Release env: GH_TOKEN: ${{ github.token }} + LSDJ_RELEASE_REVISION: ${{ needs.validate.outputs.revision }} shell: bash run: | set -euo pipefail - mapfile -t DMG_FILES < <(find dist -maxdepth 1 -type f -name '*.dmg' -print) - [[ "${#DMG_FILES[@]}" -eq 1 ]] || { - echo "Expected exactly one verified DMG, found ${#DMG_FILES[@]}" >&2 + if gh release view "$GITHUB_REF_NAME" \ + --repo "$GITHUB_REPOSITORY" >/dev/null 2>&1; then + echo "A GitHub Release already exists for $GITHUB_REF_NAME" >&2 + exit 1 + fi + + mapfile -d '' -t RELEASE_FILES < <( + find verified-release -maxdepth 1 -type f -print0 | sort -z + ) + [[ "${#RELEASE_FILES[@]}" -gt 0 ]] || { + echo "The verified release contains no files" >&2 + exit 1 + } + + DRAFT_RELEASE_ID="" + PUBLISHED=0 + cleanup_draft() { + result=$? + if [[ -n "$DRAFT_RELEASE_ID" && "$PUBLISHED" -ne 1 ]]; then + CLEANUP_JSON="$RUNNER_TEMP/lsdj-cleanup-release.json" + if gh api \ + "repos/$GITHUB_REPOSITORY/releases/$DRAFT_RELEASE_ID" \ + > "$CLEANUP_JSON" 2>/dev/null && \ + python scripts/release_artifact.py verify-draft-identity \ + --release-json "$CLEANUP_JSON" \ + --release-tag "$GITHUB_REF_NAME" \ + --revision "$LSDJ_RELEASE_REVISION" \ + --expected-release-id "$DRAFT_RELEASE_ID" >/dev/null; then + echo "Publication failed; removing this run's unpublished draft" >&2 + gh api --method DELETE \ + "repos/$GITHUB_REPOSITORY/releases/$DRAFT_RELEASE_ID" || \ + echo "Could not remove this run's unpublished draft" >&2 + else + echo "Could not prove this run still owns the failed draft; leaving it in place" >&2 + fi + fi + exit "$result" + } + trap cleanup_draft EXIT + + git fetch --no-tags origin \ + "+$GITHUB_REF:refs/remotes/origin/lsdj-release-tag" + REMOTE_TAG_COMMIT="$(git rev-list -n 1 refs/remotes/origin/lsdj-release-tag)" + [[ "$REMOTE_TAG_COMMIT" == "$LSDJ_RELEASE_REVISION" ]] || { + echo "Release tag no longer resolves to the approved source revision" >&2 exit 1 } - gh release create "$GITHUB_REF_NAME" "${DMG_FILES[0]}" \ - --repo "$GITHUB_REPOSITORY" \ - --verify-tag \ - --generate-notes \ - --title "LSDJ $GITHUB_REF_NAME" + CREATE_RESPONSE="$RUNNER_TEMP/lsdj-created-release.json" + SOURCE_MARKER="Source revision: $LSDJ_RELEASE_REVISION" + gh api --method POST "repos/$GITHUB_REPOSITORY/releases" \ + --field "tag_name=$GITHUB_REF_NAME" \ + --field "target_commitish=$LSDJ_RELEASE_REVISION" \ + --field "name=LSDJ $GITHUB_REF_NAME" \ + --field "body=$SOURCE_MARKER" \ + --field draft=true \ + --field generate_release_notes=true \ + > "$CREATE_RESPONSE" + DRAFT_RELEASE_ID="$(python scripts/release_artifact.py \ + verify-draft-identity \ + --release-json "$CREATE_RESPONSE" \ + --release-tag "$GITHUB_REF_NAME" \ + --revision "$LSDJ_RELEASE_REVISION")" + + gh release upload "$GITHUB_REF_NAME" "${RELEASE_FILES[@]}" \ + --repo "$GITHUB_REPOSITORY" + + RELEASE_JSON="$RUNNER_TEMP/lsdj-draft-release.json" + gh api "repos/$GITHUB_REPOSITORY/releases/$DRAFT_RELEASE_ID" \ + > "$RELEASE_JSON" + python scripts/release_artifact.py verify-github-release \ + --release-json "$RELEASE_JSON" \ + --verified-dir verified-release \ + --release-tag "$GITHUB_REF_NAME" \ + --revision "$LSDJ_RELEASE_REVISION" \ + --expected-release-id "$DRAFT_RELEASE_ID" + + gh api --method PATCH \ + "repos/$GITHUB_REPOSITORY/releases/$DRAFT_RELEASE_ID" \ + --field draft=false >/dev/null + PUBLISHED=1 diff --git a/README.md b/README.md index bdb4ba6..f7d1ee7 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,12 @@ pads and finished tracks come from Stable Audio 3. See All common tasks live in the [`justfile`](justfile) — run `just` to list them. +Linux x86_64 AppImage support is under active qualification. Packaging and +hosted CI are available, but a public Linux release remains gated on the +portable MRT2/Stable Audio backends, licensing, and real NVIDIA/audio/FLX4 +evidence. See [the Linux support status](docs/linux.md); do not infer hardware +support from a green hosted build. + ## Setup ```sh @@ -50,7 +56,9 @@ combined selection looks tight for your RAM) and Stable Audio 3 (generated pads and tracks) land in the app-owned `~/Library/Application Support/LSDJ` (`MAGENTA_HOME` / `SA3_MLX_HOME` override the locations). `just migrate-models` relocates an existing install—including one under the previous app name—into -that folder without re-downloading. +that folder without re-downloading. The Rust host owns the cross-platform +config/data/cache/assets/staging mapping and passes it explicitly to Python; see +[the platform path contract](docs/platform-paths.md). ## Run diff --git a/backend/lsdj/controller.py b/backend/lsdj/controller.py index 3a9ec9f..5989510 100644 --- a/backend/lsdj/controller.py +++ b/backend/lsdj/controller.py @@ -10,28 +10,45 @@ import argparse import asyncio import contextlib -import io import json import logging import math import multiprocessing as mp import os import queue +import re +import secrets +import threading import time -import wave +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass, field import uvicorn from fastapi import FastAPI, HTTPException, Request -from fastapi.middleware.cors import CORSMiddleware -from fastapi.responses import Response +from fastapi.responses import JSONResponse, Response from starlette.datastructures import UploadFile from starlette.exceptions import HTTPException as StarletteHTTPException from . import engine, loras, sa3 +from .sa3_audio import AudioNormalizationCancelled from .worker import run_deck_worker logger = logging.getLogger(__name__) +API_CAPABILITY_ENV = "LSDJ_API_CAPABILITY" +API_CAPABILITY_HEADER = "x-lsdj-capability" +JOB_ID_HEADER = "x-lsdj-job-id" +SAFE_TAURI_ORIGINS = frozenset( + { + "tauri://localhost", + "http://tauri.localhost", + "https://tauri.localhost", + } +) +_PREFLIGHT_HEADERS = frozenset({"content-type", API_CAPABILITY_HEADER, JOB_ID_HEADER}) +_api_capability: str | None = None +_allowed_origins = SAFE_TAURI_ORIGINS + DEFAULT_MODEL = "mrt2_small" # Rough whole-process footprints (model + MusicCoCa + MLX runtime), used only @@ -39,7 +56,44 @@ MODEL_RAM_ESTIMATE_GB = {"mrt2_small": 2.0, "mrt2_base": 6.0} +def _windows_total_ram_bytes(kernel32=None) -> int: + """Read physical RAM through the Windows kernel API. + + ``os.sysconf`` is Unix-only. Keep this standard-library-only so the model + status endpoint remains available before any optional model runtime loads. + ``kernel32`` is injectable for a platform-independent contract test. + """ + import ctypes + + class MemoryStatusEx(ctypes.Structure): + _fields_ = [ + ("dwLength", ctypes.c_ulong), + ("dwMemoryLoad", ctypes.c_ulong), + ("ullTotalPhys", ctypes.c_ulonglong), + ("ullAvailPhys", ctypes.c_ulonglong), + ("ullTotalPageFile", ctypes.c_ulonglong), + ("ullAvailPageFile", ctypes.c_ulonglong), + ("ullTotalVirtual", ctypes.c_ulonglong), + ("ullAvailVirtual", ctypes.c_ulonglong), + ("ullAvailExtendedVirtual", ctypes.c_ulonglong), + ] + + status = MemoryStatusEx() + status.dwLength = ctypes.sizeof(status) + if kernel32 is None: + api = ctypes.windll.kernel32.GlobalMemoryStatusEx + api.argtypes = [ctypes.POINTER(MemoryStatusEx)] + api.restype = ctypes.c_int + else: + api = kernel32.GlobalMemoryStatusEx + if not api(ctypes.byref(status)): + raise OSError(ctypes.get_last_error(), "GlobalMemoryStatusEx failed") + return status.ullTotalPhys + + def _total_ram_gb() -> float: + if os.name == "nt": + return _windows_total_ram_bytes() / 1024**3 return os.sysconf("SC_PAGE_SIZE") * os.sysconf("SC_PHYS_PAGES") / 1024**3 @@ -60,6 +114,78 @@ async def _render_lifespan(_: FastAPI): app = FastAPI(lifespan=_render_lifespan) +def configure_local_api( + capability: str, allowed_origins: frozenset[str] = SAFE_TAURI_ORIGINS +) -> None: + """Install the in-memory launch capability used by the loopback API.""" + + global _api_capability, _allowed_origins + if not 32 <= len(capability) <= 256 or not capability.isascii(): + raise ValueError("the local API capability must be 32-256 ASCII characters") + if not allowed_origins or any("*" in origin for origin in allowed_origins): + raise ValueError("local API origins must be an exact non-wildcard allowlist") + _api_capability = capability + _allowed_origins = frozenset(allowed_origins) + + +def _cors_headers(origin: str) -> dict[str, str]: + return { + "access-control-allow-origin": origin, + "vary": "Origin", + } + + +@app.middleware("http") +async def secure_local_api(request: Request, call_next): + """Authenticate every local API request and reject foreign web origins.""" + + if not request.url.path.startswith("/api/"): + return await call_next(request) + origin = request.headers.get("origin") + if origin is not None and origin not in _allowed_origins: + return JSONResponse( + status_code=403, content={"detail": "origin is not allowed"} + ) + if request.method == "OPTIONS": + requested_method = request.headers.get( + "access-control-request-method", "" + ).upper() + requested_headers = { + value.strip().lower() + for value in request.headers.get( + "access-control-request-headers", "" + ).split(",") + if value.strip() + } + if ( + origin is None + or requested_method not in {"GET", "POST"} + or not requested_headers.issubset(_PREFLIGHT_HEADERS) + ): + return JSONResponse( + status_code=403, content={"detail": "preflight rejected"} + ) + return Response( + status_code=204, + headers={ + **_cors_headers(origin), + "access-control-allow-methods": "GET, POST", + "access-control-allow-headers": ", ".join(sorted(_PREFLIGHT_HEADERS)), + "access-control-max-age": "600", + }, + ) + supplied = request.headers.get(API_CAPABILITY_HEADER, "") + expected = _api_capability + if expected is None or not secrets.compare_digest(supplied, expected): + return JSONResponse( + status_code=401, content={"detail": "authentication required"} + ) + response = await call_next(request) + if origin is not None: + response.headers.update(_cors_headers(origin)) + return response + + # Worst case: a 32 s clip at a pessimistic ~1× real time, plus a cold # prompt embed; well past it the worker is wedged, not slow. RENDER_TIMEOUT_SECONDS = 90 @@ -75,6 +201,95 @@ async def _render_lifespan(_: FastAPI): MAX_MULTIPART_BODY_BYTES = ( sa3.MAX_INIT_AUDIO_BYTES + sa3.MAX_GENERATE_METADATA_BYTES + 128 * 1024 ) +MAX_RENDER_BODY_BYTES = 64 * 1024 + +_JOB_ID = re.compile(r"^[A-Za-z0-9_-]{16,80}$") +_normalization_pool = ThreadPoolExecutor( + max_workers=1, thread_name_prefix="sa3-normalize" +) + + +@dataclass +class GenerationJob: + job_id: str + loop: asyncio.AbstractEventLoop + cancel_event: asyncio.Event = field(default_factory=asyncio.Event) + normalization_cancel: threading.Event = field(default_factory=threading.Event) + state: str = "accepted" + progress: dict | None = None + detail: str | None = None + updated_at: float = field(default_factory=time.monotonic) + _lock: threading.Lock = field(default_factory=threading.Lock, repr=False) + + def cancel(self) -> None: + self.normalization_cancel.set() + with contextlib.suppress(RuntimeError): + self.loop.call_soon_threadsafe(self.cancel_event.set) + + def update( + self, state: str, progress: dict | None = None, detail: str | None = None + ) -> None: + with self._lock: + self.state = state + self.progress = progress + self.detail = detail + self.updated_at = time.monotonic() + + def public(self) -> dict: + with self._lock: + return { + "jobId": self.job_id, + "state": self.state, + "progress": self.progress, + "detail": self.detail, + } + + +class JobRegistry: + def __init__(self, retained: int = 32): + self._jobs: dict[str, GenerationJob] = {} + self._lock = threading.Lock() + self._retained = retained + + def create(self, requested: str | None) -> GenerationJob: + job_id = requested or secrets.token_hex(16) + if not _JOB_ID.fullmatch(job_id): + raise HTTPException(status_code=422, detail="invalid generation job id") + with self._lock: + if job_id in self._jobs: + raise HTTPException( + status_code=409, detail="generation job id already exists" + ) + job = GenerationJob(job_id=job_id, loop=asyncio.get_running_loop()) + self._jobs[job_id] = job + self._prune_locked() + return job + + def _prune_locked(self) -> None: + finished = sorted( + ( + job + for job in self._jobs.values() + if job.public()["state"] in {"succeeded", "failed", "cancelled"} + ), + key=lambda job: job.updated_at, + ) + for job in finished[: max(0, len(self._jobs) - self._retained)]: + self._jobs.pop(job.job_id, None) + + def get(self, job_id: str) -> GenerationJob | None: + with self._lock: + return self._jobs.get(job_id) + + def snapshots(self) -> list[dict]: + with self._lock: + jobs = sorted( + self._jobs.values(), key=lambda job: job.updated_at, reverse=True + ) + return [job.public() for job in jobs] + + +generation_jobs = JobRegistry() def render_timeout_for(seconds: float) -> float: @@ -197,34 +412,22 @@ def _generation_number( return float(value) -def _validate_init_wav(data: bytes) -> None: +def _normalize_init_wav( + data: bytes, + *, + cancel_event: threading.Event | None = None, + on_progress=None, +) -> bytes: try: - with wave.open(io.BytesIO(data), "rb") as source: - channels = source.getnchannels() - sample_width = source.getsampwidth() - sample_rate = source.getframerate() - frames = source.getnframes() - compression = source.getcomptype() - pcm_bytes = source.readframes(frames) - except (EOFError, wave.Error): - raise HTTPException( - status_code=422, detail="'init_audio' must be a valid WAV file" + return sa3.normalize_wav( + data, cancel_event=cancel_event, on_progress=on_progress + ).wav + except AudioNormalizationCancelled: + raise sa3.GenerationCancelled( + "generation cancelled during normalization" ) from None - if ( - compression != "NONE" - or channels not in (1, 2) - or sample_width != 2 - or sample_rate != 44_100 - or frames == 0 - or len(pcm_bytes) != frames * channels * sample_width - ): - raise HTTPException( - status_code=422, - detail=( - "'init_audio' must be non-empty 44.1 kHz 16-bit PCM WAV " - "with one or two channels" - ), - ) + except sa3.AudioFormatError as error: + raise HTTPException(status_code=422, detail=f"'init_audio' {error}") from None async def _read_init_audio(upload: UploadFile) -> bytes: @@ -241,10 +444,42 @@ async def _read_init_audio(upload: UploadFile) -> bytes: ) chunks.append(chunk) data = b"".join(chunks) - _validate_init_wav(data) return data +async def _normalize_init_wav_async(data: bytes, job: GenerationJob) -> bytes: + job.update( + "normalizing", + { + "stage": "normalizing", + "current": 0, + "total": 1, + "message": "normalizing input", + }, + ) + + def progress(current: int, total: int) -> None: + job.update( + "normalizing", + { + "stage": "normalizing", + "current": current, + "total": total, + "message": f"normalizing {current}/{total}", + }, + ) + + loop = asyncio.get_running_loop() + return await loop.run_in_executor( + _normalization_pool, + lambda: _normalize_init_wav( + data, + cancel_event=job.normalization_cancel, + on_progress=progress, + ), + ) + + async def _read_capped_body(request: Request, limit: int, detail: str) -> bytes: """Stream the request body, rejecting with 413 once it passes `limit`. @@ -409,6 +644,19 @@ def _validate_generate_request( ) options["apg"] = apg + if "steps" in parsed: + steps = parsed["steps"] + if ( + isinstance(steps, bool) + or not isinstance(steps, int) + or not sa3.MIN_STEPS <= steps <= sa3.MAX_STEPS + ): + raise HTTPException( + status_code=422, + detail=f"'steps' must be an integer from {sa3.MIN_STEPS}-{sa3.MAX_STEPS}", + ) + options["steps"] = steps + if "negative_prompt" in parsed: negative_prompt = parsed["negative_prompt"] if not isinstance(negative_prompt, str): @@ -547,8 +795,11 @@ async def render_clip(request: Request) -> Response: Returns the clip as a float32 WAV. """ try: - parsed = await request.json() - except json.JSONDecodeError: + body = await _read_capped_body( + request, MAX_RENDER_BODY_BYTES, "request body is too large" + ) + parsed = json.loads(body) + except (json.JSONDecodeError, UnicodeDecodeError): raise HTTPException(status_code=422, detail="body must be JSON") from None if not isinstance(parsed, dict): raise HTTPException(status_code=422, detail="body must be a JSON object") @@ -629,16 +880,81 @@ async def generate_audio(request: Request) -> Response: subprocess and is serialised, so a busy moment queues rather than stacking memory. """ - parsed, init_audio = await _parse_generate_body(request) - prompt, seconds, kind, options = _validate_generate_request(parsed, init_audio) + job = generation_jobs.create(request.headers.get(JOB_ID_HEADER)) try: - wav = await sa3.generate(prompt, seconds, kind, **options) + parsed, raw_init_audio = await _parse_generate_body(request) + init_audio = ( + None + if raw_init_audio is None + else await _normalize_init_wav_async(raw_init_audio, job) + ) + prompt, seconds, kind, options = _validate_generate_request(parsed, init_audio) + job.update("queued") + + def progress(event) -> None: + job.update("running", event.as_dict()) + + def state_changed(state: str) -> None: + job.update(state, job.public()["progress"]) + + wav = await sa3.generate( + prompt, + seconds, + kind, + **options, + cancel_event=job.cancel_event, + on_progress=progress, + on_state=state_changed, + ) except sa3.GenerationUnavailable as error: + job.update("failed", detail=str(error)) raise HTTPException(status_code=503, detail=str(error)) from None except sa3.GenerationFailed as error: + job.update("failed", detail=str(error)) logger.warning("generation failed: %s", error) raise HTTPException(status_code=502, detail=str(error)) from None - return Response(content=wav, media_type="audio/wav") + except sa3.GenerationCancelled as error: + job.update("cancelled", detail=str(error)) + raise HTTPException(status_code=499, detail=str(error)) from None + except HTTPException as error: + job.update("failed", detail=str(error.detail)) + raise + except asyncio.CancelledError: + job.cancel() + job.update("cancelled", detail="request disconnected") + raise + job.update("succeeded") + return Response( + content=wav, + media_type="audio/wav", + headers={JOB_ID_HEADER: job.job_id}, + ) + + +@app.get("/api/jobs/{job_id}") +def generation_job_status(job_id: str) -> dict: + job = generation_jobs.get(job_id) + if job is None: + raise HTTPException(status_code=404, detail="generation job not found") + return job.public() + + +@app.post("/api/jobs/{job_id}/cancel") +def cancel_generation_job(job_id: str) -> dict: + job = generation_jobs.get(job_id) + if job is None: + raise HTTPException(status_code=404, detail="generation job not found") + snapshot = job.public() + if snapshot["state"] not in {"succeeded", "failed", "cancelled"}: + job.cancel() + job.update("cancelling", progress=snapshot["progress"]) + return job.public() + + +@app.get("/api/sa3/status") +def stable_audio_status() -> dict: + """Selected runtime, feature matrix, limitations, and active generation.""" + return {**sa3.status(), "jobs": generation_jobs.snapshots()} @app.get("/api/models") @@ -670,15 +986,11 @@ def main(argv: list[str] | None = None) -> None: "--port", type=int, default=8000, help="loopback port to bind (default 8000)" ) args = parser.parse_args(argv) - - # The webview loads from the Tauri asset host and fetches this server - # cross-origin over loopback, so allow it. Loopback-bound; not exposed. - app.add_middleware( - CORSMiddleware, - allow_origins=["*"], - allow_methods=["*"], - allow_headers=["*"], - ) + capability = os.environ.pop(API_CAPABILITY_ENV, "") + try: + configure_local_api(capability) + except ValueError as error: + parser.error(str(error)) uvicorn.run(app, host="127.0.0.1", port=args.port) diff --git a/backend/lsdj/engine.py b/backend/lsdj/engine.py index 5f629f5..af4bdf7 100644 --- a/backend/lsdj/engine.py +++ b/backend/lsdj/engine.py @@ -5,6 +5,7 @@ depends on this interface instead of magenta_rt directly. """ +import importlib.metadata import math import numpy as np @@ -132,6 +133,7 @@ def __init__(self, model: str = "mrt2_small"): cfg_notes=CFG_NOTES, cfg_drums=CFG_DRUMS, ) + self._model = model self._state = None self._style = None self._notes: list[int] | None = None @@ -148,6 +150,29 @@ def __init__(self, model: str = "mrt2_small"): self._embed_cache: dict[str, np.ndarray] = {} self._samples: dict[str, np.ndarray] = {} + def diagnostics(self) -> dict[str, object]: + """Runtime facts matching the cross-platform worker ready contract.""" + + try: + version = importlib.metadata.version("magenta-rt") + except importlib.metadata.PackageNotFoundError: + version = "unknown" + return { + "runtime": "mlx", + "accelerator": "metal", + "model": self._model, + "magenta_rt_version": version, + "hardware_qualified": True, + "capabilities": { + "weighted_prompts": True, + "audio_style": True, + "notes": True, + "drums": True, + "negative_prompt": False, + "explicit_seed": False, + }, + } + def _embed_cached(self, text: str) -> np.ndarray: if text in self._embed_cache: # Refresh recency: dict order is the LRU order. diff --git a/backend/lsdj/gpu_broker.py b/backend/lsdj/gpu_broker.py new file mode 100644 index 0000000..9750412 --- /dev/null +++ b/backend/lsdj/gpu_broker.py @@ -0,0 +1,360 @@ +"""Cross-process NVIDIA work admission for MRT2 and Stable Audio 3. + +The audio decks have strict priority. A Stable Audio lease is admitted only +when no MRT2 generation is running or waiting and its measured reservation fits +inside the caller-provided VRAM budget. If MRT2 arrives while Stable Audio is +sampling, the SA3 callback observes the waiter and cancels its disposable child +process before MRT2 is admitted. + +State lives below the app-owned cache root and is guarded with an OS file lock; +no daemon, shell command, system Python, or third-party lock package is needed. +Dead-process records are pruned on every operation, so a killed worker cannot +leave the GPU permanently reserved. +""" + +from __future__ import annotations + +import contextlib +import dataclasses +import enum +import json +import os +import pathlib +import tempfile +import time +import uuid +from collections.abc import Callable, Iterator +from typing import Any + + +SCHEMA_VERSION = 1 +MAX_RECORDS = 32 +DEFAULT_POLL_SECONDS = 0.05 +PROCESS_QUERY_LIMITED_INFORMATION = 0x1000 +ERROR_INVALID_PARAMETER = 87 +STILL_ACTIVE = 259 + + +class Priority(enum.IntEnum): + SA3_BACKGROUND = 10 + MRT2_REALTIME = 100 + + +class BrokerError(RuntimeError): + """The broker state is invalid or work cannot be admitted safely.""" + + +class BrokerCancelled(BrokerError): + """The caller cancelled while waiting for the GPU.""" + + +class BrokerTimeout(BrokerError): + """The caller's bounded admission deadline expired.""" + + +@dataclasses.dataclass(frozen=True) +class Lease: + token: str + service: str + priority: Priority + reservation_bytes: int + pid: int + + +def _windows_pid_alive(pid: int) -> bool: + """Query one Windows process without treating signal emulation as truth. + + ``os.kill(pid, 0)`` is not a portable liveness probe on Windows. Open the + process with the least query privilege instead, and retain an indeterminate + broker record whenever access is denied or a query fails. False is safe + only when Windows positively reports an invalid PID or a completed process. + """ + + import ctypes + from ctypes import wintypes + + try: + kernel32 = ctypes.WinDLL("kernel32", use_last_error=True) + open_process = kernel32.OpenProcess + open_process.argtypes = [wintypes.DWORD, wintypes.BOOL, wintypes.DWORD] + open_process.restype = wintypes.HANDLE + get_exit_code = kernel32.GetExitCodeProcess + get_exit_code.argtypes = [wintypes.HANDLE, ctypes.POINTER(wintypes.DWORD)] + get_exit_code.restype = wintypes.BOOL + close_handle = kernel32.CloseHandle + close_handle.argtypes = [wintypes.HANDLE] + close_handle.restype = wintypes.BOOL + handle = open_process(PROCESS_QUERY_LIMITED_INFORMATION, False, pid) + except (AttributeError, OSError): + return True + if not handle: + return ctypes.get_last_error() != ERROR_INVALID_PARAMETER + exit_code = wintypes.DWORD() + try: + try: + queried = get_exit_code(handle, ctypes.byref(exit_code)) + except OSError: + return True + return not queried or exit_code.value == STILL_ACTIVE + finally: + close_handle(handle) + + +def _posix_pid_alive(pid: int) -> bool: + try: + os.kill(pid, 0) + except ProcessLookupError: + return False + except (PermissionError, OSError): + return True + return True + + +def _pid_alive(pid: int) -> bool: + if pid <= 0: + return False + if pid == os.getpid(): + return True + if os.name == "nt": + return _windows_pid_alive(pid) + return _posix_pid_alive(pid) + + +@contextlib.contextmanager +def _os_file_lock(path: pathlib.Path) -> Iterator[None]: + path.parent.mkdir(parents=True, exist_ok=True) + if path.is_symlink(): + raise BrokerError("GPU broker lock path must not be a symlink") + with path.open("a+b") as handle: + handle.seek(0, os.SEEK_END) + if handle.tell() == 0: + handle.write(b"\0") + handle.flush() + handle.seek(0) + if os.name == "nt": + import msvcrt + + msvcrt.locking(handle.fileno(), msvcrt.LK_LOCK, 1) + try: + yield + finally: + handle.seek(0) + msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1) + else: + import fcntl + + fcntl.flock(handle.fileno(), fcntl.LOCK_EX) + try: + yield + finally: + fcntl.flock(handle.fileno(), fcntl.LOCK_UN) + + +class GpuBroker: + def __init__( + self, + root: pathlib.Path, + *, + poll_seconds: float = DEFAULT_POLL_SECONDS, + clock: Callable[[], float] = time.monotonic, + sleeper: Callable[[float], None] = time.sleep, + pid_alive: Callable[[int], bool] = _pid_alive, + ) -> None: + if poll_seconds <= 0: + raise ValueError("poll_seconds must be positive") + self.root = root + self.state_path = root / "state.json" + self.lock_path = root / "state.lock" + self.poll_seconds = poll_seconds + self._clock = clock + self._sleep = sleeper + self._pid_alive = pid_alive + + def _empty_state(self) -> dict[str, Any]: + return {"schema_version": SCHEMA_VERSION, "waiters": [], "leases": []} + + def _read_state(self) -> dict[str, Any]: + if not self.state_path.exists(): + return self._empty_state() + if self.state_path.is_symlink(): + raise BrokerError("GPU broker state path must not be a symlink") + try: + parsed = json.loads(self.state_path.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as error: + raise BrokerError("GPU broker state is unreadable") from error + if ( + not isinstance(parsed, dict) + or parsed.get("schema_version") != SCHEMA_VERSION + ): + raise BrokerError("GPU broker state has an unsupported schema") + for field in ("waiters", "leases"): + records = parsed.get(field) + if not isinstance(records, list) or len(records) > MAX_RECORDS: + raise BrokerError(f"GPU broker {field} are invalid") + for record in records: + if not self._valid_record(record): + raise BrokerError(f"GPU broker {field} contain an invalid record") + return parsed + + @staticmethod + def _valid_record(record: Any) -> bool: + return ( + isinstance(record, dict) + and isinstance(record.get("token"), str) + and 1 <= len(record["token"]) <= 64 + and isinstance(record.get("service"), str) + and 1 <= len(record["service"]) <= 64 + and isinstance(record.get("priority"), int) + and not isinstance(record["priority"], bool) + and record["priority"] in {int(item) for item in Priority} + and isinstance(record.get("reservation_bytes"), int) + and not isinstance(record["reservation_bytes"], bool) + and record["reservation_bytes"] >= 0 + and isinstance(record.get("pid"), int) + and record["pid"] > 0 + ) + + def _write_state(self, state: dict[str, Any]) -> None: + self.root.mkdir(parents=True, exist_ok=True) + if self.state_path.is_symlink(): + raise BrokerError("GPU broker state path must not be a symlink") + descriptor, temporary_name = tempfile.mkstemp( + prefix="state.", suffix=".tmp", dir=self.root + ) + temporary = pathlib.Path(temporary_name) + try: + with os.fdopen(descriptor, "w", encoding="utf-8") as handle: + json.dump(state, handle, sort_keys=True, separators=(",", ":")) + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary, self.state_path) + finally: + with contextlib.suppress(FileNotFoundError): + temporary.unlink() + + def _prune(self, state: dict[str, Any]) -> None: + for field in ("waiters", "leases"): + state[field] = [ + record for record in state[field] if self._pid_alive(record["pid"]) + ] + + @contextlib.contextmanager + def _locked_state(self) -> Iterator[dict[str, Any]]: + with _os_file_lock(self.lock_path): + state = self._read_state() + self._prune(state) + yield state + self._write_state(state) + + @staticmethod + def _record(lease: Lease) -> dict[str, Any]: + return { + "token": lease.token, + "service": lease.service, + "priority": int(lease.priority), + "reservation_bytes": lease.reservation_bytes, + "pid": lease.pid, + } + + def acquire( + self, + service: str, + priority: Priority, + *, + reservation_bytes: int, + capacity_bytes: int, + timeout_seconds: float, + cancelled: Callable[[], bool] = lambda: False, + ) -> Lease: + if not service or len(service) > 64: + raise ValueError("service must contain 1-64 characters") + if reservation_bytes < 0 or capacity_bytes < 0: + raise ValueError("GPU byte counts must not be negative") + if timeout_seconds <= 0: + raise ValueError("timeout_seconds must be positive") + lease = Lease( + token=uuid.uuid4().hex, + service=service, + priority=priority, + reservation_bytes=reservation_bytes, + pid=os.getpid(), + ) + record = self._record(lease) + deadline = self._clock() + timeout_seconds + registered = False + try: + while True: + if cancelled(): + raise BrokerCancelled(f"{service} GPU request was cancelled") + if self._clock() >= deadline: + raise BrokerTimeout(f"{service} timed out waiting for the GPU") + with self._locked_state() as state: + if not registered: + if len(state["waiters"]) + len(state["leases"]) >= MAX_RECORDS: + raise BrokerError( + "GPU broker is at its bounded record limit" + ) + state["waiters"].append(record) + registered = True + higher_waiting = any( + item["token"] != lease.token + and item["priority"] > int(priority) + for item in state["waiters"] + ) + active_higher = any( + item["priority"] > int(priority) for item in state["leases"] + ) + active_lower = any( + item["priority"] < int(priority) for item in state["leases"] + ) + reserved = sum( + item["reservation_bytes"] for item in state["leases"] + ) + fits = reservation_bytes <= max(0, capacity_bytes - reserved) + if ( + not higher_waiting + and not active_higher + and not active_lower + and fits + ): + state["waiters"] = [ + item + for item in state["waiters"] + if item["token"] != lease.token + ] + state["leases"].append(record) + return lease + self._sleep(self.poll_seconds) + except Exception: + if registered: + self._remove(lease.token) + raise + + def _remove(self, token: str) -> None: + with self._locked_state() as state: + for field in ("waiters", "leases"): + state[field] = [item for item in state[field] if item["token"] != token] + + def release(self, lease: Lease) -> None: + self._remove(lease.token) + + @contextlib.contextmanager + def hold(self, *args: Any, **kwargs: Any) -> Iterator[Lease]: + lease = self.acquire(*args, **kwargs) + try: + yield lease + finally: + self.release(lease) + + def should_yield(self, lease: Lease) -> bool: + with self._locked_state() as state: + live = any(item["token"] == lease.token for item in state["leases"]) + if not live: + raise BrokerError("GPU lease is no longer live") + return any( + item["priority"] > int(lease.priority) for item in state["waiters"] + ) + + def diagnostics(self) -> dict[str, Any]: + with self._locked_state() as state: + return json.loads(json.dumps(state)) diff --git a/backend/lsdj/loras.py b/backend/lsdj/loras.py index 951c00d..45f8316 100644 --- a/backend/lsdj/loras.py +++ b/backend/lsdj/loras.py @@ -1,9 +1,9 @@ """Stable Audio 3 LoRA adapter registry — the read side (issue #66, ADR-0028). -Adapters live on disk under the app-owned data dir, one directory per -adapter, organised by the DiT family they ride: +Adapters live under the host-resolved asset root, one directory per adapter, +organised by the DiT family they ride: - ~/Library/Application Support/LSDJ/sa3-loras/// + $SA3_LORAS_HOME/// ``base`` is ``small`` (the 1024-wide sm-sfx / sm-music DiTs) or ``medium`` (the 1536-wide track DiT). An adapter directory holds its ``.safetensors`` @@ -17,6 +17,11 @@ import os import pathlib import re +import hashlib +import json +import stat + +from . import runtime_paths # The two DiT families an adapter can ride, and which generation kind uses # which. sm-sfx and sm-music share one architecture, so a "small" adapter @@ -47,29 +52,81 @@ class UnknownAdapter(Exception): def loras_dir( env: dict | None = None, home: pathlib.Path | None = None ) -> pathlib.Path: - """The registry root. $SA3_LORAS_HOME wins (tests, dev overrides); - otherwise the app-owned data dir, beside the SA3 checkout. Mirrors the - Rust `loras::loras_dir`.""" + """The registry root explicitly supplied by the Rust host.""" env = os.environ if env is None else env - home = pathlib.Path.home() if home is None else home - override = env.get("SA3_LORAS_HOME", "") - if override: - return pathlib.Path(override).expanduser() - return home / "Library" / "Application Support" / "LSDJ" / "sa3-loras" - - -def _adapter_file(adapter_dir: pathlib.Path) -> pathlib.Path | None: + del home # retained for API compatibility; platform paths come from Rust. + root = runtime_paths.loras_home(env) + if root is None: + raise RuntimeError("LSDJ asset roots were not supplied by the desktop host") + return root + + +def _is_linklike(path: pathlib.Path) -> bool: + try: + return path.is_symlink() or ( + hasattr(os.path, "isjunction") and os.path.isjunction(path) + ) + except OSError: + return True + + +def _contained(path: pathlib.Path, root: pathlib.Path, *, directory: bool) -> bool: + try: + if _is_linklike(path): + return False + mode = path.lstat().st_mode + if directory and not stat.S_ISDIR(mode): + return False + if not directory and not stat.S_ISREG(mode): + return False + path.resolve(strict=True).relative_to(root.resolve(strict=True)) + return True + except (OSError, ValueError): + return False + + +def _verified_manifest(adapter_dir: pathlib.Path, root: pathlib.Path) -> bool: + manifest_path = adapter_dir / "lora.json" + if not manifest_path.exists(): + return True # Preserve explicitly supported hand-placed adapters. + if not _contained(manifest_path, root, directory=False): + return False + try: + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + files = manifest.get("files", []) + if not files: + return True # Legacy imports predate the artifact inventory. + for record in files: + filename = record["filename"] + if not _SLUG.fullmatch(filename): + return False + artifact = adapter_dir / filename + if not _contained(artifact, root, directory=False): + return False + data = artifact.read_bytes() + if len(data) != record["size"]: + return False + if hashlib.sha256(data).hexdigest() != record["sha256"].lower(): + return False + return True + except (OSError, KeyError, TypeError, ValueError, json.JSONDecodeError): + return False + + +def _adapter_file(adapter_dir: pathlib.Path, root: pathlib.Path) -> pathlib.Path | None: """The adapter's .safetensors inside its directory, or None. The importer writes exactly one; tolerate a hand-placed dir the same way the runtime's `_resolve_path` does (one .safetensors, any name).""" - if not adapter_dir.is_dir(): + if not _contained(adapter_dir, root, directory=True): return None hits = sorted( entry for entry in adapter_dir.iterdir() - if entry.is_file() and entry.suffix == ".safetensors" + if _contained(entry, root, directory=False) and entry.suffix == ".safetensors" ) - return hits[0] if len(hits) == 1 else None + if len(hits) != 1 or not _verified_manifest(adapter_dir, root): + return None + return hits[0] def resolve( @@ -82,7 +139,14 @@ def resolve( base, _, slug = name.partition("/") if base not in BASES or not _SLUG.match(slug): raise UnknownAdapter(f"unknown adapter {name!r}") - adapter_dir = loras_dir(env, home) / base / slug - if _adapter_file(adapter_dir) is None: + root = loras_dir(env, home) + if not _contained(root, root, directory=True): + raise UnknownAdapter(f"unknown adapter {name!r}") + base_dir = root / base + adapter_dir = base_dir / slug + if ( + not _contained(base_dir, root, directory=True) + or _adapter_file(adapter_dir, root) is None + ): raise UnknownAdapter(f"unknown adapter {name!r}") return adapter_dir, base diff --git a/backend/lsdj/mrt2.py b/backend/lsdj/mrt2.py new file mode 100644 index 0000000..e344ad7 --- /dev/null +++ b/backend/lsdj/mrt2.py @@ -0,0 +1,244 @@ +"""Runtime-neutral Magenta RealTime 2 selection and engine contract. + +The Rust host always names a runtime explicitly. This module keeps the +platform policy and release qualification gate independent from either model +stack, so selecting PyTorch can never degrade silently to CPU inference. +""" + +from __future__ import annotations + +import os +import sys +from dataclasses import asdict, dataclass +from typing import Mapping, Protocol, runtime_checkable + + +MLX_RUNTIME = "mlx" +PYTORCH_CUDA_RUNTIME = "pytorch-cuda" +AUTO_RUNTIME = "auto" +RUNTIME_CHOICES = (AUTO_RUNTIME, MLX_RUNTIME, PYTORCH_CUDA_RUNTIME) + +# The #109 spike established API and packaging feasibility, but neither target +# OS has completed the required two-deck NVIDIA run. Keep that release fact in +# executable metadata instead of allowing an unqualified backend to look ready. +PYTORCH_HARDWARE_QUALIFIED = False +UNVERIFIED_OPT_IN = "LSDJ_ALLOW_UNVERIFIED_MRT2_CUDA" + +ADAPTER_REFERENCE = { + "repository": "https://github.com/multimodalart/magenta-realtime-torch.git", + "revision": "6d076baa3df3b10448876c400521a015a5137c59", + "license": "Apache-2.0", + "credit": "Apolinario", + "role": "implementation reference; not executed by LSDJ", +} +MODEL_SNAPSHOTS = { + "mrt2_base": { + "repository": "magenta-community/magenta-realtime-2", + "revision": "92087988d05d0fe38b11f021f0b0d00a75afb86b", + }, + "mrt2_small": { + "repository": "magenta-community/magenta-realtime-2-small", + "revision": "7037d99551c84ac5c6afb7f1a5e58c65e7233dbb", + }, +} +PROCESSOR_SNAPSHOT = { + "repository": "magenta-community/magenta-rt-musiccoca-torch", + "revision": "236c488e38aa98643805514996934d705668298b", +} +RUNTIME_CANDIDATE = { + "python": "3.12", + "torch": "2.12.1", + "transformers": "5.8.0", + "huggingface_hub": "1.5.0", + "numpy": "2.3.5", + "safetensors": "0.7.0", + "sentencepiece": "0.2.1", + "resampy": "0.4.3", + "cuda_wheel": "cu130", + "lock_status": "hash_locked_uninstalled", + "locks": { + "linux-x86_64": "runtime-locks/mrt2-pytorch-linux-x86_64.txt", + "windows-x86_64": "runtime-locks/mrt2-pytorch-windows-x86_64.txt", + }, +} + + +class RuntimeUnavailable(RuntimeError): + """The requested MRT2 runtime cannot be used safely on this host.""" + + +def public_startup_error(error: Exception) -> str: + """Return a bounded, non-sensitive startup diagnostic for the UI.""" + + if isinstance(error, RuntimeUnavailable): + return str(error)[:512] + return ( + f"{type(error).__name__}: MRT2 worker startup failed; " + "inspect the local application log for details" + ) + + +@dataclass(frozen=True) +class RuntimeSelection: + name: str + platform: str + accelerator: str + hardware_qualified: bool + experimental: bool + + +@runtime_checkable +class Mrt2Engine(Protocol): + """The model-independent contract consumed by ``run_deck_worker``.""" + + @property + def chunk_seconds(self) -> float: ... + + def set_style( + self, + prompts: list[tuple[str, float]], + sample_keys: frozenset[str] = frozenset(), + ) -> None: ... + + def embed_sample(self, sample_id: str, pcm: bytes) -> None: ... + + def set_notes(self, notes: list[int] | None) -> None: ... + + def set_drums(self, flag: int | None, cfg: float | None = None) -> None: ... + + def set_generation( + self, + temperature: float, + top_k: int, + cfg_musiccoca: float, + cfg_notes: float, + ) -> None: ... + + def set_chunk_frames(self, frames: int) -> None: ... + + def generate_chunk(self) -> bytes: ... + + def render_clip(self, prompt: str, seconds: float) -> bytes: ... + + def diagnostics(self) -> dict[str, object]: ... + + +def _platform_family(platform: str) -> str: + value = platform.lower() + if value.startswith("darwin"): + return "macos" + if value.startswith("linux"): + return "linux" + if value.startswith(("win32", "cygwin", "msys")): + return "windows" + return value + + +def _truthy(value: str | None) -> bool: + return value is not None and value.strip().lower() in {"1", "true", "yes", "on"} + + +def select_runtime( + requested: str = AUTO_RUNTIME, + *, + platform: str | None = None, + env: Mapping[str, str] | None = None, +) -> RuntimeSelection: + """Resolve a platform backend and enforce the #109 qualification gate.""" + + platform_name = _platform_family(sys.platform if platform is None else platform) + environment = os.environ if env is None else env + if requested not in RUNTIME_CHOICES: + raise RuntimeUnavailable( + f"unknown MRT2 runtime {requested!r}; expected one of {RUNTIME_CHOICES}" + ) + runtime = requested + if runtime == AUTO_RUNTIME: + if platform_name == "macos": + runtime = MLX_RUNTIME + elif platform_name in {"linux", "windows"}: + runtime = PYTORCH_CUDA_RUNTIME + else: + raise RuntimeUnavailable( + f"MRT2 has no runtime for unsupported platform {platform_name!r}" + ) + + if runtime == MLX_RUNTIME: + if platform_name != "macos": + raise RuntimeUnavailable( + f"the MLX MRT2 runtime is macOS-only, not {platform_name}" + ) + return RuntimeSelection(runtime, platform_name, "metal", True, False) + + if platform_name not in {"linux", "windows"}: + raise RuntimeUnavailable( + "the PyTorch CUDA MRT2 runtime is supported only on Linux and Windows" + ) + experimental = _truthy(environment.get(UNVERIFIED_OPT_IN)) + if not PYTORCH_HARDWARE_QUALIFIED and not experimental: + raise RuntimeUnavailable( + "the PyTorch CUDA MRT2 runtime is implemented but not release-qualified: " + "issue #109 still requires Linux and Windows NVIDIA two-deck hardware " + f"results; {UNVERIFIED_OPT_IN}=1 is reserved for that qualification run" + ) + return RuntimeSelection( + runtime, + platform_name, + "cuda", + PYTORCH_HARDWARE_QUALIFIED, + experimental, + ) + + +def runtime_manifest() -> dict[str, object]: + """Immutable dependency/install metadata exposed without importing a model.""" + + return { + "schema_version": 1, + "runtime": PYTORCH_CUDA_RUNTIME, + "release_ready": PYTORCH_HARDWARE_QUALIFIED, + "supported_platforms": ["linux", "windows"], + "accelerator": "nvidia-cuda", + "cpu_fallback": False, + "topology": "shared-worker-two-state", + "topology_implemented": True, + "adapter_reference": dict(ADAPTER_REFERENCE), + "executable_remote_code": { + name: dict(pin) for name, pin in MODEL_SNAPSHOTS.items() + }, + "models": {name: dict(pin) for name, pin in MODEL_SNAPSHOTS.items()}, + "processor": dict(PROCESSOR_SNAPSHOT), + "runtime_candidate": dict(RUNTIME_CANDIDATE), + "qualification_blockers": [ + "Linux NVIDIA two-deck 25-frame and 5-frame ten-minute results", + "Windows NVIDIA two-deck 25-frame and 5-frame ten-minute results", + "clean-host lock installation and minimum driver selection", + "issue #108 notices and download acknowledgement", + ], + } + + +def create_engine( + *, + model: str, + runtime: str, + platform: str | None = None, + env: Mapping[str, str] | None = None, +) -> Mrt2Engine: + """Construct exactly the selected backend; never fall back to another one.""" + + selection = select_runtime(runtime, platform=platform, env=env) + if selection.name == MLX_RUNTIME: + from .engine import DeckEngine + + return DeckEngine(model=model) + + from .mrt2_pytorch import PytorchMrt2Engine + + return PytorchMrt2Engine(model=model, selection=selection) + + +def selection_dict(selection: RuntimeSelection) -> dict[str, object]: + """Stable JSON-ready representation for diagnostics and tests.""" + + return asdict(selection) diff --git a/backend/lsdj/mrt2_pytorch.py b/backend/lsdj/mrt2_pytorch.py new file mode 100644 index 0000000..823869d --- /dev/null +++ b/backend/lsdj/mrt2_pytorch.py @@ -0,0 +1,531 @@ +"""Thin LSDJ adapter for Apolinario's pinned PyTorch MRT2 snapshots. + +All heavyweight imports and snapshot lookups happen in the supervised worker. +Acquisition is deliberately out of band: this adapter opens only immutable, +already-installed snapshots and never performs a network download at startup. +""" + +from __future__ import annotations + +import importlib.metadata +import math +import os +import threading +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import numpy as np + +from . import runtime_paths +from .gpu_broker import BrokerError, GpuBroker, Lease, Priority +from .engine import ( + CFG_MUSICCOCA, + CFG_NOTES, + CHANNELS, + EMBED_CACHE_SIZE, + FRAME_SECONDS, + FRAMES_PER_CHUNK, + MAX_CFG, + MAX_DRUM_CFG, + MAX_SAMPLE_SECONDS, + MAX_CHUNK_FRAMES, + MIN_CFG, + MIN_DRUM_CFG, + MIN_SAMPLE_SECONDS, + MIN_CHUNK_FRAMES, + MIN_TEMPERATURE, + MIN_TOP_K, + NOTE_ONSET, + NOTE_SLOTS, + NOTE_STATES, + NOTE_SUSTAIN, + SAMPLE_CACHE_SIZE, + SAMPLE_RATE, + TEMPERATURE, + TOP_K, +) +from .mrt2 import ( + MODEL_SNAPSHOTS, + PROCESSOR_SNAPSHOT, + PYTORCH_CUDA_RUNTIME, + RuntimeSelection, + RuntimeUnavailable, +) + +MAX_SEED = (1 << 63) - 1 +GPU_ADMISSION_TIMEOUT_SECONDS = 30.0 + + +@dataclass(frozen=True) +class PytorchBindings: + torch: Any + auto_model: Any + versions: dict[str, str] + + +def _package_version(name: str) -> str: + try: + return importlib.metadata.version(name) + except importlib.metadata.PackageNotFoundError: + return "unknown" + + +def load_bindings() -> PytorchBindings: + """Load the separately-installed CUDA runtime only inside its worker.""" + + try: + import torch + from transformers import AutoModel + except ImportError as error: + raise RuntimeUnavailable( + "the pinned PyTorch MRT2 runtime is not installed; use LSDJ's model " + "manager instead of installing system Python packages" + ) from error + return PytorchBindings( + torch=torch, + auto_model=AutoModel, + versions={ + "torch": _package_version("torch"), + "transformers": _package_version("transformers"), + "huggingface_hub": _package_version("huggingface-hub"), + }, + ) + + +def _driver_version(torch: Any) -> str | None: + getter = getattr(getattr(torch, "_C", None), "_cuda_getDriverVersion", None) + if not callable(getter): + return None + try: + value = int(getter()) + except (RuntimeError, TypeError, ValueError): + return None + # CUDA returns e.g. 13020 for 13.2. Preserve the raw value too imprecisely + # only when its layout is unexpected rather than inventing a version. + if value < 1000: + return str(value) + major = value // 1000 + minor = (value % 1000) // 10 + return f"{major}.{minor}" + + +def _verified_local_directory( + root: Path, child: Path, required: tuple[str, ...] +) -> Path: + """Reject links/reparse escapes before Transformers imports remote code.""" + + try: + canonical_root = root.resolve(strict=True) + canonical_child = child.resolve(strict=True) + root_link = root.is_symlink() or ( + hasattr(os.path, "isjunction") and os.path.isjunction(root) + ) + child_link = child.is_symlink() or ( + hasattr(os.path, "isjunction") and os.path.isjunction(child) + ) + if root_link or child_link or not child.is_dir(): + raise RuntimeUnavailable("managed MRT2 snapshot is not a regular directory") + canonical_child.relative_to(canonical_root) + for filename in required: + artifact = child / filename + artifact_link = artifact.is_symlink() or ( + hasattr(os.path, "isjunction") and os.path.isjunction(artifact) + ) + if artifact_link or not artifact.is_file(): + raise RuntimeUnavailable( + f"managed MRT2 snapshot is missing regular artifact {filename!r}" + ) + artifact.resolve(strict=True).relative_to(canonical_child) + except (OSError, ValueError) as error: + raise RuntimeUnavailable( + "the managed MRT2 snapshot escapes its verified service generation" + ) from error + return canonical_child + + +class PytorchMrt2Engine: + """LSDJ's model contract over an immutable Transformers snapshot.""" + + def __init__( + self, + model: str = "mrt2_small", + *, + selection: RuntimeSelection, + bindings: PytorchBindings | None = None, + cache_root: Path | None = None, + gpu_broker: GpuBroker | None = None, + ) -> None: + if selection.name != PYTORCH_CUDA_RUNTIME: + raise RuntimeUnavailable( + f"PyTorch adapter received the wrong runtime {selection.name!r}" + ) + if model not in MODEL_SNAPSHOTS: + raise ValueError(f"unknown pinned PyTorch MRT2 model {model!r}") + self._selection = selection + + if cache_root is None: + assets = runtime_paths.assets_home() + if assets is None: + raise RuntimeUnavailable( + "LSDJ_ASSETS_HOME is missing; the native host must supply the " + "app-owned model root" + ) + runtime_root = assets / "backend" / "services" / "mrt2" / "current" + else: + runtime_root = cache_root + model_pin = MODEL_SNAPSHOTS[model] + model_path = _verified_local_directory( + runtime_root, + runtime_root / "models" / model, + ("config.json", "model.safetensors", "modeling_magenta_rt2.py"), + ) + processor_path = _verified_local_directory( + runtime_root, + runtime_root / "models" / "musiccoca", + ( + "mel_params.npz", + "music_encoder.pt", + "quantizer.pt", + "spm.model", + "text_encoder.pt", + ), + ) + + broker_root = runtime_paths.cache_home() + if gpu_broker is None: + if broker_root is None: + raise RuntimeUnavailable( + "LSDJ_CACHE_HOME is missing; MRT2 cannot allocate CUDA without " + "the shared GPU broker" + ) + self._gpu_broker = GpuBroker(broker_root / "gpu-broker") + else: + self._gpu_broker = gpu_broker + self._gpu_lease: Lease | None = None + + # The lease covers the model's full CUDA lifetime, not just generate(). + # Acquire before importing/querying torch: cuda.is_available(), model.to(), + # and processor loading may all initialize a context or reserve VRAM. It + # intentionally lives until worker-process exit, the only reliable CUDA + # context teardown boundary; PID pruning then removes the broker record. + try: + self._gpu_lease = self._gpu_broker.acquire( + "mrt2", + priority=Priority.MRT2_REALTIME, + reservation_bytes=0, + capacity_bytes=0, + timeout_seconds=GPU_ADMISSION_TIMEOUT_SECONDS, + ) + self._bindings = bindings or load_bindings() + torch = self._bindings.torch + if not torch.cuda.is_available(): + raise RuntimeUnavailable( + "PyTorch reports no CUDA accelerator; MRT2 has no CPU fallback" + ) + if not getattr(torch.version, "cuda", None): + raise RuntimeUnavailable( + "the installed PyTorch build has no CUDA runtime; " + "MRT2 has no CPU fallback" + ) + + # `trust_remote_code` is safe only because model_path resolves the exact + # installer-verified revision above. Never pass a mutable repository ID. + upstream = self._bindings.auto_model.from_pretrained( + model_path, + trust_remote_code=True, + dtype=torch.bfloat16, + local_files_only=True, + ) + self._system = upstream.to("cuda").eval() + self._system.load_processor(processor_path, device="cuda") + except (BrokerError, RuntimeUnavailable): + raise + except Exception as error: + raise RuntimeUnavailable( + "the pinned PyTorch MRT2 snapshot could not initialize on CUDA" + ) from error + + self._model = model + self._model_pin = model_pin + self._model_lock = threading.RLock() + self._warmup_owner = True + self._init_deck_state() + + def _init_deck_state(self) -> None: + self._state: Any = None + self._style: list[int] | None = None + self._notes: list[int] | None = None + self._drums: int | None = None + self._drums_cfg: float | None = None + self._temperature = TEMPERATURE + self._top_k = TOP_K + self._cfg_musiccoca = CFG_MUSICCOCA + self._cfg_notes = CFG_NOTES + self._chunk_frames = FRAMES_PER_CHUNK + self._seed = 0 + self._embed_cache: dict[str, Any] = {} + self._samples: dict[str, Any] = {} + + def shared_deck(self) -> "PytorchMrt2Engine": + """A second deck state sharing this process's single loaded model.""" + + deck = self.__class__.__new__(self.__class__) + deck._selection = self._selection + deck._bindings = self._bindings + deck._system = self._system + deck._model = self._model + deck._model_pin = self._model_pin + deck._model_lock = self._model_lock + deck._gpu_broker = self._gpu_broker + deck._gpu_lease = self._gpu_lease + deck._warmup_owner = False + deck._init_deck_state() + return deck + + @property + def chunk_seconds(self) -> float: + return self._chunk_frames * FRAME_SECONDS + + def _embed_text(self, text: str) -> Any: + if text in self._embed_cache: + self._embed_cache[text] = self._embed_cache.pop(text) + else: + with self._model_lock: + embedding = self._system.processor.embed(text) + if len(self._embed_cache) >= EMBED_CACHE_SIZE: + self._embed_cache.pop(next(iter(self._embed_cache))) + self._embed_cache[text] = embedding + return self._embed_cache[text] + + def embed_sample(self, sample_id: str, pcm: bytes) -> None: + samples = np.frombuffer(pcm, dtype="= SAMPLE_CACHE_SIZE: + self._samples.pop(next(iter(self._samples))) + self._samples[sample_id] = embedding + + def set_style( + self, + prompts: list[tuple[str, float]], + sample_keys: frozenset[str] = frozenset(), + ) -> None: + weighted = [(key, float(weight)) for key, weight in prompts if weight > 0] + if not weighted: + raise ValueError("set_style needs at least one prompt with weight > 0") + total = sum(weight for _, weight in weighted) + blend: Any = None + for key, weight in weighted: + if key in sample_keys: + if key not in self._samples: + raise ValueError(f"unknown sample {key!r} — re-sample the deck") + embedding = self._samples.pop(key) + self._samples[key] = embedding + else: + embedding = self._embed_text(key) + term = embedding * (weight / total) + blend = term if blend is None else blend + term + with self._model_lock: + tokens = self._system.processor.tokenize(blend) + self._style = [int(token) for token in tokens] + + def set_notes(self, notes: list[int] | None) -> None: + if notes is not None: + if len(notes) != NOTE_SLOTS: + raise ValueError( + f"notes must hold {NOTE_SLOTS} slots, got {len(notes)}" + ) + if any(state not in NOTE_STATES for state in notes): + raise ValueError("note states must be -1, 0, 1, 2, or 3") + self._notes = None if notes is None else list(notes) + + def set_drums(self, flag: int | None, cfg: float | None = None) -> None: + if flag is not None and flag not in (0, 1): + raise ValueError("drum flag must be 0, 1, or None") + if cfg is not None and not MIN_DRUM_CFG <= cfg <= MAX_DRUM_CFG: + raise ValueError( + f"drum cfg must be in [{MIN_DRUM_CFG}, {MAX_DRUM_CFG}] or None" + ) + self._drums = flag + self._drums_cfg = cfg + + def set_generation( + self, + temperature: float, + top_k: int, + cfg_musiccoca: float, + cfg_notes: float, + ) -> None: + if not isinstance(top_k, int) or isinstance(top_k, bool) or top_k < MIN_TOP_K: + raise ValueError(f"top_k must be an int >= {MIN_TOP_K}") + for name, value in (("cfg_musiccoca", cfg_musiccoca), ("cfg_notes", cfg_notes)): + if not MIN_CFG <= value <= MAX_CFG: + raise ValueError(f"{name} must be in [{MIN_CFG}, {MAX_CFG}]") + self._temperature = max(MIN_TEMPERATURE, temperature) + self._top_k = top_k + self._cfg_musiccoca = cfg_musiccoca + self._cfg_notes = cfg_notes + + def set_chunk_frames(self, frames: int) -> None: + if ( + not isinstance(frames, int) + or isinstance(frames, bool) + or not MIN_CHUNK_FRAMES <= frames <= MAX_CHUNK_FRAMES + ): + raise ValueError( + f"chunk frames must be an int in [{MIN_CHUNK_FRAMES}, {MAX_CHUNK_FRAMES}]" + ) + self._chunk_frames = frames + + def set_seed(self, seed: int) -> None: + if ( + not isinstance(seed, int) + or isinstance(seed, bool) + or not 0 <= seed <= MAX_SEED + ): + raise ValueError(f"seed must be an int in [0, {MAX_SEED}]") + self._seed = seed + # Upstream creates the RNG only for a fresh state. Reset-to-reseed is + # explicit, never a misleading live seed change. + self._state = None + + def reset(self, *, seed: int | None = None) -> None: + if seed is not None: + self.set_seed(seed) + else: + self._state = None + + def _generate( + self, + *, + frames: int, + state: Any, + style: Any, + stream_conditioning: bool = True, + ) -> tuple[np.ndarray, Any]: + notes = self._notes if stream_conditioning else None + drums = self._drums if stream_conditioning else None + # The process-lifetime lease was acquired before CUDA initialization. + # The lock only serializes the two deck states sharing this loaded model. + with self._model_lock: + audio, state = self._system.generate( + style=style, + notes=notes, + drums=None if drums is None else [drums], + cfg_drums=self._drums_cfg if stream_conditioning else None, + temperature=self._temperature, + top_k=self._top_k, + cfg_musiccoca=self._cfg_musiccoca, + cfg_notes=self._cfg_notes, + frames=frames, + seed=self._seed, + state=state, + guidance=True, + ) + samples = np.asarray(audio) + expected = frames * round(SAMPLE_RATE * FRAME_SECONDS) + if samples.ndim != 2 or samples.shape != (expected, CHANNELS): + raise RuntimeError( + "upstream PyTorch MRT2 returned invalid audio shape " + f"{samples.shape!r}; expected {(expected, CHANNELS)!r}" + ) + if not np.isfinite(samples).all(): + raise RuntimeError("upstream PyTorch MRT2 returned non-finite audio") + return samples.astype(" None: + # Exercise model, CUDA kernels, and decoder before readiness, then clear + # every continuation/RNG state so the first audible stream is fresh. + if not self._warmup_owner: + return + self._generate( + frames=1, + state=None, + style=None, + stream_conditioning=False, + ) + self.reset() + + def generate_chunk(self) -> bytes: + samples, self._state = self._generate( + frames=self._chunk_frames, + state=self._state, + style=self._style, + ) + if self._notes is not None: + self._notes = [ + NOTE_SUSTAIN if state == NOTE_ONSET else state for state in self._notes + ] + return samples.tobytes() + + def render_clip(self, prompt: str, seconds: float) -> bytes: + embedding = self._embed_text(prompt) + with self._model_lock: + style = [int(token) for token in self._system.processor.tokenize(embedding)] + state: Any = None + pieces = [] + for _ in range(math.ceil(seconds / (FRAMES_PER_CHUNK * FRAME_SECONDS))): + samples, state = self._generate( + frames=FRAMES_PER_CHUNK, + state=state, + style=style, + stream_conditioning=False, + ) + pieces.append(samples) + return ( + np.concatenate(pieces)[: round(seconds * SAMPLE_RATE)] + .astype(" dict[str, object]: + torch = self._bindings.torch + device_index = torch.cuda.current_device() + props = torch.cuda.get_device_properties(device_index) + return { + "runtime": PYTORCH_CUDA_RUNTIME, + "accelerator": "cuda", + "acceleration_mode": "eager-guidance", + "topology": "shared-worker-two-state", + "gpu_broker": { + "enabled": self._gpu_broker is not None, + "priority": int(Priority.MRT2_REALTIME), + "preempts": "sa3-background", + }, + "hardware_qualified": self._selection.hardware_qualified, + "experimental": self._selection.experimental, + "model": self._model, + "model_repository": self._model_pin["repository"], + "model_revision": self._model_pin["revision"], + "processor_repository": PROCESSOR_SNAPSHOT["repository"], + "processor_revision": PROCESSOR_SNAPSHOT["revision"], + "remote_code_repository": self._model_pin["repository"], + "remote_code_revision": self._model_pin["revision"], + "torch_version": self._bindings.versions["torch"], + "transformers_version": self._bindings.versions["transformers"], + "huggingface_hub_version": self._bindings.versions["huggingface_hub"], + "torch_cuda_runtime": torch.version.cuda, + "nvidia_driver": _driver_version(torch), + "cuda_device": props.name, + "cuda_capability": list(torch.cuda.get_device_capability(device_index)), + "cuda_total_memory_bytes": int(props.total_memory), + "capabilities": { + "weighted_prompts": True, + "audio_style": True, + "notes": True, + "drums": True, + "negative_prompt": False, + "explicit_seed": True, + "reset_to_reseed": True, + }, + } diff --git a/backend/lsdj/runtime_paths.py b/backend/lsdj/runtime_paths.py new file mode 100644 index 0000000..db0f01b --- /dev/null +++ b/backend/lsdj/runtime_paths.py @@ -0,0 +1,69 @@ +"""Filesystem roots supplied by the native Rust host. + +The desktop host is the only component that knows platform conventions. Python +services consume these explicit values and never reconstruct Windows, XDG, or +macOS locations from a user home directory. Compatibility variables remain for +the current upstream runtimes, but they are populated by the same Rust contract. +""" + +import os +import pathlib +import sys +from collections.abc import Mapping + + +def _path(env: Mapping[str, str], name: str) -> pathlib.Path | None: + value = env.get(name, "") + return pathlib.Path(value) if value else None + + +def config_home(env: Mapping[str, str] | None = None) -> pathlib.Path | None: + return _path(os.environ if env is None else env, "LSDJ_CONFIG_HOME") + + +def data_home(env: Mapping[str, str] | None = None) -> pathlib.Path | None: + return _path(os.environ if env is None else env, "LSDJ_DATA_HOME") + + +def cache_home(env: Mapping[str, str] | None = None) -> pathlib.Path | None: + return _path(os.environ if env is None else env, "LSDJ_CACHE_HOME") + + +def assets_home(env: Mapping[str, str] | None = None) -> pathlib.Path | None: + return _path(os.environ if env is None else env, "LSDJ_ASSETS_HOME") + + +def staging_home(env: Mapping[str, str] | None = None) -> pathlib.Path | None: + return _path(os.environ if env is None else env, "LSDJ_STAGING_HOME") + + +def sa3_home(env: Mapping[str, str] | None = None) -> pathlib.Path | None: + env = os.environ if env is None else env + neutral_override = _path(env, "SA3_HOME") + if neutral_override is not None: + return neutral_override + tflite_override = _path(env, "SA3_TFLITE_HOME") + if tflite_override is not None: + return tflite_override + override = _path(env, "SA3_MLX_HOME") + if override is not None: + return override + assets = assets_home(env) + return None if assets is None else assets / "stable-audio-3" + + +def loras_home(env: Mapping[str, str] | None = None) -> pathlib.Path | None: + env = os.environ if env is None else env + override = _path(env, "SA3_LORAS_HOME") + if override is not None: + return override + assets = assets_home(env) + return None if assets is None else assets / "sa3-loras" + + +def venv_python(venv: pathlib.Path, *, platform: str | None = None) -> pathlib.Path: + """Return a venv interpreter as a path/argv item, never a shell string.""" + platform = sys.platform if platform is None else platform + if platform == "win32": + return venv / "Scripts" / "python.exe" + return venv / "bin" / "python" diff --git a/backend/lsdj/sa3.py b/backend/lsdj/sa3.py index 873c2c1..b32c23f 100644 --- a/backend/lsdj/sa3.py +++ b/backend/lsdj/sa3.py @@ -1,42 +1,50 @@ -"""Stable Audio 3 generation via a spawned sa3_mlx subprocess (ADR-0012). - -Nothing here imports sa3_mlx code: the checkout's own venv python runs its -CLI once per generation and the WAV comes back as bytes. The interpreter is -invoked directly — `uv run` would resolve the checkout's repo-root torch -project (measured), and the `./sa3` wrapper exists for humans and may -prompt. Generations are serialised so the transient ~1.5 GB peak never -stacks next to the two deck workers. +"""Runtime-neutral Stable Audio 3 service. + +LSDJ spawns an official, pinned upstream CLI for each generation. Apple +Silicon uses MLX; Linux and Windows use LiteRT/TFLite. Both adapters receive +the same request object, execute strictly offline against app-installed assets, +and return a validated canonical WAV. """ +from __future__ import annotations + import asyncio +import contextlib +import json import os import pathlib +import platform as host_platform +import re +import signal +import subprocess +import sys import tempfile -from collections.abc import Sequence +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass + +from . import runtime_paths, sa3_cuda +from .sa3_audio import AudioFormatError, inspect_canonical_wav, normalize_wav +from .sa3_audio import validate_output_wav as _validate_output_wav +from .sa3_contract import ( + BackendName, + GenerationRequest, + ProgressEvent, + capabilities_for, +) -# CLI vocabulary of scripts/sa3_mlx.py at the pinned commit (sa3-pin.json). -# Pads use the small DiTs with the SAME-S decoder; tracks (M19, ADR-0013) -# the medium DiT, which pairs with SAME-L. +# Both pinned official CLIs deliberately share these names. KINDS = {"sfx": "sm-sfx", "music": "sm-music", "track": "medium"} DECODERS = {"sfx": "same-s", "music": "same-s", "track": "same-l"} SAMPLER_STEPS = 8 +MIN_STEPS = 1 +MAX_STEPS = 100 MIN_SECONDS = 0.5 MAX_SECONDS = 32.0 -# Stability's published ceiling for the medium DiT (6:20). TRACK_MAX_SECONDS = 380.0 MAX_SECONDS_FOR = {"sfx": MAX_SECONDS, "music": MAX_SECONDS, "track": TRACK_MAX_SECONDS} -# A safety ceiling, not a UX limit: the prompt is passed to the sa3_mlx CLI as a -# single argv (see `generate`), so an unbounded prompt would blow the OS arg-length -# limit, and it guards the loopback endpoint against a pathological body. Set generous -# enough to hold a large structured/JSON prompt (a pasted song spec runs ~8 KB) with -# headroom, while staying far below the OS argv limit. The model's text encoder -# truncates beyond its own window anyway. -MAX_PROMPT_LENGTH = 32000 - -# Issue #54 generation controls. These are trust-boundary limits, mirrored by -# `controller.generate_audio`; the CLI itself is wider in places, but unbounded -# loopback input must not become unbounded argv, model guidance, or memory use. +MAX_PROMPT_LENGTH = 32_000 + MIN_INIT_NOISE_LEVEL = 0.01 MAX_INIT_NOISE_LEVEL = 5.0 MIN_CFG = -20.0 @@ -47,120 +55,748 @@ MAX_INIT_AUDIO_BYTES = 16 * 1024 * 1024 MAX_GENERATE_METADATA_BYTES = 64 * 1024 -# Measured small-DiT generation is ~1.5 s; the margin covers a cold -# filesystem cache and slower machines, not a first-ever weight download -# (see SETUP_HINT). TIMEOUT_SECONDS = 120 +TFLITE_THREADS_DEFAULT = 4 +TFLITE_THREADS_MAX = 8 +SA3_PREFERENCE_ENV = "LSDJ_SA3_PREFERENCE" + +STATE_MISSING = "missing" +STATE_VENV_MISSING = "venv_missing" +STATE_NOT_WARMED = "not_warmed" +STATE_READY = "ready" +STATE_UNSUPPORTED = "unsupported" +STATE_FAILED = "failed" -SETUP_HINT = ( - "sa3_mlx checkout not found - install Stable Audio 3 from the app's settings " - "drawer (the model manager), or point SA3_MLX_HOME at an existing checkout" +WARMED_STAMP = ".lsdj-warmed" +TFLITE_PROVENANCE_STAMP = ".lsdj-provenance.json" +TFLITE_RUNTIME_REPO = "https://github.com/Stability-AI/stable-audio-3" +TFLITE_RUNTIME_REVISION = "a0b57f5483c4588f827f3552b7d5c6ca2a9687be" +TFLITE_MODELS_REPO = "stabilityai/stable-audio-3-optimized" +TFLITE_MODELS_REVISION = "6736003cb57d06b7b1fdc36fad31b2a3709e4774" + +_MLX_SUBDIR = pathlib.Path("optimized/mlx") +_TFLITE_SUBDIR = pathlib.Path("optimized/tflite") +_MLX_SCRIPT = pathlib.Path("scripts/sa3_mlx.py") +_TFLITE_SCRIPT = pathlib.Path("scripts/sa3_tflite.py") + +# fp32 is the official TFLite default and the only precision for which upstream +# supports LoRA. The secure installer consumes sa3-tflite-pin.json and places +# these files before the worker is ever started. +_TFLITE_SHARED_ASSETS = ( + pathlib.Path("models/tokenizer.model"), + pathlib.Path("models/tflite/t5gemma/encoder_fp16.tflite"), ) +_TFLITE_MODEL_ASSETS = { + "sfx": ( + pathlib.Path("models/tflite/sa3-sm-sfx/dit_fp32.tflite"), + pathlib.Path("models/tflite/same-s/dec_fp32.tflite"), + ), + "music": ( + pathlib.Path("models/tflite/sa3-sm-music/dit_fp32.tflite"), + pathlib.Path("models/tflite/same-s/dec_fp32.tflite"), + ), + "track": ( + pathlib.Path("models/tflite/sa3-m/dit_fp32.tflite"), + pathlib.Path("models/tflite/same-l/dec_fp32.tflite"), + ), +} +_TFLITE_ENCODER_ASSET = { + "sfx": pathlib.Path("models/tflite/same-s/enc_fp32.tflite"), + "music": pathlib.Path("models/tflite/same-s/enc_fp32.tflite"), + "track": pathlib.Path("models/tflite/same-l/enc_fp32.tflite"), +} -def timeout_for(seconds: float) -> float: - """Deadline for one generation, scaled to the requested length. +class GenerationUnavailable(Exception): + """No supported, ready Stable Audio runtime exists on this machine.""" - The published medium benchmark is ~15 s wall for a 2-minute track on - M4-Pro-class hardware, so a second of deadline per second of audio is - ~8x slack on top of the flat base — a wedge kill-switch, not a UX - promise (ADR-0013).""" - return TIMEOUT_SECONDS + seconds +class GenerationFailed(Exception): + """The selected runtime failed or produced invalid audio.""" -_generation_lock = asyncio.Semaphore(1) +class GenerationCancelled(Exception): + """The caller cancelled a generation and the worker was stopped.""" -class GenerationUnavailable(Exception): - """No usable sa3_mlx checkout on this machine.""" +class UnsupportedCapability(GenerationUnavailable): + """A request names a capability the selected backend cannot honour.""" -class GenerationFailed(Exception): - """The CLI ran and did not produce a WAV.""" +@dataclass(frozen=True) +class RuntimeSelection: + backend: BackendName + checkout: pathlib.Path + runtime_dir: pathlib.Path + executable: pathlib.Path + script: pathlib.Path -# Canonical SA3 install states, shared verbatim with the Rust `model_status` -# and the model-manager UI (issue #43): the readiness contract is one of these. -STATE_MISSING = "missing" -STATE_VENV_MISSING = "venv_missing" -STATE_NOT_WARMED = "not_warmed" -STATE_READY = "ready" -WARMED_STAMP = ".lsdj-warmed" +def timeout_for(seconds: float) -> float: + """Wedge deadline, not a performance promise.""" + return TIMEOUT_SECONDS + seconds + + +def _normalise_arch(machine: str) -> str: + value = machine.strip().lower() + if value in {"arm64", "aarch64"}: + return "arm64" + if value in {"amd64", "x86_64"}: + return "x86_64" + return value -def _checkout_candidates(env: dict, home: pathlib.Path) -> list[pathlib.Path]: - """Checkout roots to probe, in order. $SA3_MLX_HOME wins (pointing at the - checkout root); otherwise the app-owned data dir, where the in-app installer - puts the checkout. Mirrors the Rust `models::sa3_candidates`.""" - candidates = [] - override = env.get("SA3_MLX_HOME", "") +def select_backend( + env: Mapping[str, str] | None = None, + *, + platform_name: str | None = None, + machine: str | None = None, +) -> BackendName: + """Select the backend deterministically; never fall back silently.""" + env = os.environ if env is None else env + platform_name = sys.platform if platform_name is None else platform_name + machine = host_platform.machine() if machine is None else machine + arch = _normalise_arch(machine) + override = env.get("LSDJ_SA3_BACKEND", "").strip().lower() if override: - candidates.append(pathlib.Path(override).expanduser()) - candidates.append( - home / "Library" / "Application Support" / "LSDJ" / "stable-audio-3" + try: + chosen = BackendName(override) + except ValueError: + raise GenerationUnavailable( + "LSDJ_SA3_BACKEND must be 'mlx' or 'tflite'" + ) from None + if chosen is BackendName.MLX and not ( + platform_name == "darwin" and arch == "arm64" + ): + raise GenerationUnavailable( + "the MLX Stable Audio backend requires Apple Silicon macOS" + ) + if chosen is BackendName.TFLITE and not ( + platform_name in {"linux", "win32"} and arch == "x86_64" + ): + raise GenerationUnavailable( + f"the TFLite Stable Audio backend does not support {platform_name}/{arch}" + ) + return chosen + if platform_name == "darwin" and arch == "arm64": + return BackendName.MLX + if platform_name in {"linux", "win32"} and arch == "x86_64": + return BackendName.TFLITE + raise GenerationUnavailable( + f"no Stable Audio backend supports {platform_name}/{arch}" ) - return candidates + + +def _checkout_candidates(env: Mapping[str, str]) -> list[pathlib.Path]: + checkout = runtime_paths.sa3_home(env) + return [] if checkout is None else [checkout] + + +def _layout(backend: BackendName) -> tuple[pathlib.Path, pathlib.Path]: + if backend is BackendName.MLX: + return _MLX_SUBDIR, _MLX_SCRIPT + return _TFLITE_SUBDIR, _TFLITE_SCRIPT + + +def _tflite_provenance_error(runtime_dir: pathlib.Path) -> str | None: + stamp = runtime_dir / TFLITE_PROVENANCE_STAMP + try: + parsed = json.loads(stamp.read_text()) + except (OSError, json.JSONDecodeError): + return "the verified TFLite provenance stamp is missing or unreadable" + expected = { + "runtime": { + "repo": TFLITE_RUNTIME_REPO, + "revision": TFLITE_RUNTIME_REVISION, + }, + "models": { + "repo": TFLITE_MODELS_REPO, + "revision": TFLITE_MODELS_REVISION, + }, + } + if parsed != expected: + return "the installed TFLite runtime/model revisions do not match LSDJ's pin" + return None + + +def resolve_runtime( + env: Mapping[str, str] | None = None, + *, + platform_name: str | None = None, + machine: str | None = None, +) -> RuntimeSelection | None: + env = os.environ if env is None else env + backend = select_backend(env, platform_name=platform_name, machine=machine) + subdir, script_rel = _layout(backend) + for checkout in _checkout_candidates(env): + runtime_dir = checkout / subdir + executable = runtime_paths.venv_python( + runtime_dir / ".venv", platform=platform_name + ) + script = runtime_dir / script_rel + if executable.is_file() and script.is_file(): + return RuntimeSelection( + backend=backend, + checkout=checkout, + runtime_dir=runtime_dir, + executable=executable, + script=script, + ) + return None def resolve_mlx_dir( - env: dict | None = None, home: pathlib.Path | None = None + env: Mapping[str, str] | None = None, home: pathlib.Path | None = None ) -> pathlib.Path | None: - """First checkout whose optimized/mlx has a venv and the CLI script.""" + """Compatibility probe used by existing model-manager tests.""" + del home env = os.environ if env is None else env - home = pathlib.Path.home() if home is None else home - for checkout in _checkout_candidates(env, home): - mlx_dir = checkout / "optimized" / "mlx" - python = mlx_dir / ".venv" / "bin" / "python" - script = mlx_dir / "scripts" / "sa3_mlx.py" - if python.is_file() and script.is_file(): - return mlx_dir + for checkout in _checkout_candidates(env): + runtime_dir = checkout / _MLX_SUBDIR + executable = runtime_paths.venv_python(runtime_dir / ".venv", platform="darwin") + if executable.is_file() and (runtime_dir / _MLX_SCRIPT).is_file(): + return runtime_dir return None -def readiness(env: dict | None = None, home: pathlib.Path | None = None) -> dict: - """The SA3 install state for the model manager (issue #43). - - Walks the same candidates as `resolve_mlx_dir` and classifies the first - checkout that has an `optimized/mlx` dir: +def resolve_tflite_dir( + env: Mapping[str, str] | None = None, + *, + platform_name: str | None = None, +) -> pathlib.Path | None: + env = os.environ if env is None else env + for checkout in _checkout_candidates(env): + runtime_dir = checkout / _TFLITE_SUBDIR + executable = runtime_paths.venv_python( + runtime_dir / ".venv", platform=platform_name + ) + if executable.is_file() and (runtime_dir / _TFLITE_SCRIPT).is_file(): + return runtime_dir + return None - - ``missing`` no checkout with an ``optimized/mlx`` dir - - ``venv_missing`` checkout present, but no ``.venv``/CLI script - - ``not_warmed`` venv present, but the ``.lsdj-warmed`` stamp is absent - - ``ready`` venv present and warmed - Returns ``{"state", "checkout", "mlx_dir"}`` (paths are str or None). The - Rust `model_status` mirrors this exact logic and these exact identifiers. - """ +def readiness( + env: Mapping[str, str] | None = None, + home: pathlib.Path | None = None, + *, + platform_name: str | None = None, + machine: str | None = None, +) -> dict: + del home env = os.environ if env is None else env - home = pathlib.Path.home() if home is None else home - - first_with_mlx: tuple[pathlib.Path, pathlib.Path] | None = None - for checkout in _checkout_candidates(env, home): - mlx_dir = checkout / "optimized" / "mlx" - if not mlx_dir.is_dir(): + try: + backend = select_backend(env, platform_name=platform_name, machine=machine) + except GenerationUnavailable as error: + return { + "state": STATE_UNSUPPORTED, + "backend": None, + "checkout": None, + "runtime_dir": None, + "mlx_dir": None, + "detail": str(error), + } + subdir, script_rel = _layout(backend) + first_runtime: tuple[pathlib.Path, pathlib.Path] | None = None + for checkout in _checkout_candidates(env): + runtime_dir = checkout / subdir + if not runtime_dir.is_dir(): continue - if first_with_mlx is None: - first_with_mlx = (checkout, mlx_dir) - python = mlx_dir / ".venv" / "bin" / "python" - script = mlx_dir / "scripts" / "sa3_mlx.py" - if not (python.is_file() and script.is_file()): + if first_runtime is None: + first_runtime = (checkout, runtime_dir) + executable = runtime_paths.venv_python( + runtime_dir / ".venv", platform=platform_name + ) + if not (executable.is_file() and (runtime_dir / script_rel).is_file()): continue - warmed = (mlx_dir / WARMED_STAMP).is_file() + warmed = (runtime_dir / WARMED_STAMP).is_file() + provenance_error = ( + _tflite_provenance_error(runtime_dir) + if backend is BackendName.TFLITE + else None + ) + state = ( + STATE_FAILED + if warmed and provenance_error is not None + else STATE_READY + if warmed + else STATE_NOT_WARMED + ) return { - "state": STATE_READY if warmed else STATE_NOT_WARMED, + "state": state, + "backend": backend.value, "checkout": str(checkout), - "mlx_dir": str(mlx_dir), + "runtime_dir": str(runtime_dir), + "mlx_dir": str(runtime_dir) if backend is BackendName.MLX else None, + "detail": provenance_error if state == STATE_FAILED else None, } - - if first_with_mlx is not None: - checkout, mlx_dir = first_with_mlx + if first_runtime is not None: + checkout, runtime_dir = first_runtime return { "state": STATE_VENV_MISSING, + "backend": backend.value, "checkout": str(checkout), - "mlx_dir": str(mlx_dir), + "runtime_dir": str(runtime_dir), + "mlx_dir": str(runtime_dir) if backend is BackendName.MLX else None, + "detail": None, } - return {"state": STATE_MISSING, "checkout": None, "mlx_dir": None} + return { + "state": STATE_MISSING, + "backend": backend.value, + "checkout": None, + "runtime_dir": None, + "mlx_dir": None, + "detail": None, + } + + +_generation_state: dict = { + "state": "idle", + "backend": None, + "mode": None, + "progress": None, +} + + +def status( + env: Mapping[str, str] | None = None, + *, + platform_name: str | None = None, + machine: str | None = None, +) -> dict: + env = os.environ if env is None else env + ready = readiness(env, platform_name=platform_name, machine=machine) + backend_value = ready["backend"] + capabilities = ( + None + if backend_value is None + else capabilities_for(BackendName(backend_value)).as_dict() + ) + platform_value = sys.platform if platform_name is None else platform_name + machine_value = host_platform.machine() if machine is None else machine + preference = env.get(SA3_PREFERENCE_ENV, sa3_cuda.BackendPreference.AUTO.value) + cuda = None + if platform_value == "win32" and _normalise_arch(machine_value) == "x86_64": + evidence = sa3_cuda.CudaEvidence( + platform=platform_value, + machine=machine_value, + runtime_ready=False, + provenance_complete=False, + packages={}, + cuda_available=False, + cuda_runtime=None, + driver=None, + device=None, + compute_capability=None, + total_vram_bytes=None, + free_vram_bytes=None, + estimated_vram_bytes={"music": None, "sfx": None, "track": None}, + ) + cuda = sa3_cuda.diagnostic_manifest( + evidence, tflite_ready=ready["state"] == STATE_READY, env=env + ) + return { + **ready, + "activeBackend": backend_value, + "preference": preference, + "preferenceChoices": [item.value for item in sa3_cuda.BackendPreference], + "cuda": cuda, + "capabilities": capabilities, + "generation": dict(_generation_state), + "maxSeconds": dict(MAX_SECONDS_FOR), + } + + +def _tflite_threads(env: Mapping[str, str]) -> int: + raw = env.get("LSDJ_SA3_TFLITE_THREADS", str(TFLITE_THREADS_DEFAULT)) + try: + threads = int(raw) + except ValueError: + raise GenerationUnavailable( + "LSDJ_SA3_TFLITE_THREADS must be an integer" + ) from None + if not 1 <= threads <= TFLITE_THREADS_MAX: + raise GenerationUnavailable( + f"LSDJ_SA3_TFLITE_THREADS must be 1-{TFLITE_THREADS_MAX}" + ) + return threads + + +def _required_tflite_assets(request: GenerationRequest) -> tuple[pathlib.Path, ...]: + paths = [*_TFLITE_SHARED_ASSETS, *_TFLITE_MODEL_ASSETS[request.kind]] + if request.init_audio is not None: + paths.append(_TFLITE_ENCODER_ASSET[request.kind]) + return tuple(paths) + + +def _preflight(selection: RuntimeSelection, request: GenerationRequest) -> None: + if request.inpaint_range is not None and request.init_audio is None: + raise UnsupportedCapability("inpainting requires init audio") + if request.negative_prompt is not None and ( + request.cfg is None or request.cfg == 1 + ): + raise UnsupportedCapability("negative prompt requires CFG other than 1") + if request.apg is not None and (request.cfg is None or request.cfg == 1): + raise UnsupportedCapability("APG requires CFG other than 1") + if request.lora_strengths is not None and len(request.lora_strengths) != len( + request.lora_dirs or () + ): + raise UnsupportedCapability("every LoRA must have exactly one aligned strength") + if not MIN_STEPS <= request.steps <= MAX_STEPS: + raise UnsupportedCapability(f"steps must be {MIN_STEPS}-{MAX_STEPS}") + if selection.backend is not BackendName.TFLITE: + return + if not (selection.runtime_dir / WARMED_STAMP).is_file(): + raise GenerationUnavailable( + "the TFLite runtime has not completed its verified warm-up" + ) + if provenance_error := _tflite_provenance_error(selection.runtime_dir): + raise GenerationUnavailable(provenance_error) + missing = [ + str(path) + for path in _required_tflite_assets(request) + if not (selection.runtime_dir / path).is_file() + ] + if missing: + names = ", ".join(missing) + raise GenerationUnavailable( + "the pinned TFLite model bundle is incomplete; install it from the " + f"model manager before generating (missing: {names})" + ) + + +def build_argv( + selection: RuntimeSelection, + request: GenerationRequest, + *, + out_path: pathlib.Path, + init_path: pathlib.Path | None, + env: Mapping[str, str] | None = None, +) -> list[str]: + """Translate the neutral request to an official CLI argument vector.""" + env = os.environ if env is None else env + _preflight(selection, request) + argv = [ + str(selection.executable), + str(selection.script), + "--prompt", + request.prompt, + "--dit", + KINDS[request.kind], + "--decoder", + DECODERS[request.kind], + "--seconds", + f"{request.seconds:g}", + "--steps", + str(request.steps), + "--out", + str(out_path), + ] + if selection.backend is BackendName.TFLITE: + argv.extend(("--precision", "fp32", "--threads", str(_tflite_threads(env)))) + if request.init_audio is not None: + if init_path is None: + raise UnsupportedCapability("init audio requires a normalized input path") + argv.extend(("--init-audio", str(init_path))) + if request.init_noise_level is not None: + argv.extend(("--init-noise-level", f"{request.init_noise_level:g}")) + if request.inpaint_range is not None: + start, end = request.inpaint_range + argv.extend(("--inpaint-range", f"{start:g},{end:g}")) + if request.negative_prompt is not None: + argv.extend(("--negative-prompt", request.negative_prompt)) + if request.cfg is not None: + argv.extend(("--cfg", f"{request.cfg:g}")) + if request.apg is not None: + argv.extend(("--apg", f"{request.apg:g}")) + if request.seed is not None: + argv.extend(("--seed", str(request.seed))) + for index, lora_dir in enumerate(request.lora_dirs or ()): + argv.extend(("--lora", lora_dir)) + if request.lora_strengths is not None: + argv.append(f"strength={request.lora_strengths[index]:g}") + return argv + + +_PROGRESS_PATTERNS = ( + ("sampling", re.compile(r"sampling step (\d+)/(\d+)")), + ("decode", re.compile(r"decode chunk (\d+)/(\d+)")), +) +_SENSITIVE_OUTPUT = re.compile( + r"(?i)(prompt|init audio|--lora|hf[_-]?token|hugging_face_hub_token|authorization)" +) + + +def _progress_from_line(line: str) -> ProgressEvent | None: + for stage, pattern in _PROGRESS_PATTERNS: + if match := pattern.search(line): + return ProgressEvent( + stage=stage, + current=int(match.group(1)), + total=int(match.group(2)), + message=f"{stage} {match.group(1)}/{match.group(2)}", + ) + return None + + +async def _drain_output( + stream: asyncio.StreamReader, + on_progress: Callable[[ProgressEvent], None] | None, +) -> str: + tail = bytearray() + pending = bytearray() + while chunk := await stream.read(4096): + tail.extend(chunk) + if len(tail) > 8192: + del tail[:-8192] + pending.extend(chunk) + while True: + newline = pending.find(b"\n") + if newline < 0: + break + line = bytes(pending[:newline]).decode(errors="replace") + del pending[: newline + 1] + if len(pending) > 8192: + del pending[:-8192] + event = _progress_from_line(line) + if event is not None and on_progress is not None: + on_progress(event) + if len(pending) > 8192: + del pending[:-8192] + return tail.decode(errors="replace") + + +def _safe_failure_tail(output: str, backend: BackendName) -> str: + lines = [ + line.strip() + for line in output.splitlines() + if line.strip() and not _SENSITIVE_OUTPUT.search(line) + ] + tail = "\n".join(lines[-8:])[-1000:] + return tail or f"the {backend.value} Stable Audio process failed" + + +class _WindowsJob: + """Per-generation Job Object so cancellation tears down the whole CLI tree.""" + + def __init__(self, pid: int) -> None: + import ctypes + from ctypes import wintypes + + class IoCounters(ctypes.Structure): + _fields_ = [ + (name, ctypes.c_ulonglong) + for name in ( + "ReadOperationCount", + "WriteOperationCount", + "OtherOperationCount", + "ReadTransferCount", + "WriteTransferCount", + "OtherTransferCount", + ) + ] + + class BasicLimitInformation(ctypes.Structure): + _fields_ = [ + ("PerProcessUserTimeLimit", ctypes.c_longlong), + ("PerJobUserTimeLimit", ctypes.c_longlong), + ("LimitFlags", wintypes.DWORD), + ("MinimumWorkingSetSize", ctypes.c_size_t), + ("MaximumWorkingSetSize", ctypes.c_size_t), + ("ActiveProcessLimit", wintypes.DWORD), + ("Affinity", ctypes.c_size_t), + ("PriorityClass", wintypes.DWORD), + ("SchedulingClass", wintypes.DWORD), + ] + + class ExtendedLimitInformation(ctypes.Structure): + _fields_ = [ + ("BasicLimitInformation", BasicLimitInformation), + ("IoInfo", IoCounters), + ("ProcessMemoryLimit", ctypes.c_size_t), + ("JobMemoryLimit", ctypes.c_size_t), + ("PeakProcessMemoryUsed", ctypes.c_size_t), + ("PeakJobMemoryUsed", ctypes.c_size_t), + ] + + kernel32 = ctypes.WinDLL("kernel32", use_last_error=True) + kernel32.CreateJobObjectW.argtypes = [ctypes.c_void_p, wintypes.LPCWSTR] + kernel32.CreateJobObjectW.restype = wintypes.HANDLE + kernel32.SetInformationJobObject.argtypes = [ + wintypes.HANDLE, + ctypes.c_int, + ctypes.c_void_p, + wintypes.DWORD, + ] + kernel32.SetInformationJobObject.restype = wintypes.BOOL + kernel32.OpenProcess.argtypes = [wintypes.DWORD, wintypes.BOOL, wintypes.DWORD] + kernel32.OpenProcess.restype = wintypes.HANDLE + kernel32.AssignProcessToJobObject.argtypes = [wintypes.HANDLE, wintypes.HANDLE] + kernel32.AssignProcessToJobObject.restype = wintypes.BOOL + kernel32.TerminateJobObject.argtypes = [wintypes.HANDLE, wintypes.UINT] + kernel32.TerminateJobObject.restype = wintypes.BOOL + kernel32.CloseHandle.argtypes = [wintypes.HANDLE] + kernel32.CloseHandle.restype = wintypes.BOOL + job = kernel32.CreateJobObjectW(None, None) + if not job: + raise ctypes.WinError(ctypes.get_last_error()) + info = ExtendedLimitInformation() + info.BasicLimitInformation.LimitFlags = 0x00002000 # KILL_ON_JOB_CLOSE + if not kernel32.SetInformationJobObject( + job, 9, ctypes.byref(info), ctypes.sizeof(info) + ): + error = ctypes.WinError(ctypes.get_last_error()) + kernel32.CloseHandle(job) + raise error + process = kernel32.OpenProcess(0x0001 | 0x0100 | 0x1000, False, pid) + if not process: + error = ctypes.WinError(ctypes.get_last_error()) + kernel32.CloseHandle(job) + raise error + try: + if not kernel32.AssignProcessToJobObject(job, process): + raise ctypes.WinError(ctypes.get_last_error()) + except Exception: + kernel32.CloseHandle(job) + raise + finally: + kernel32.CloseHandle(process) + self._kernel32 = kernel32 + self._handle = job + + def terminate(self) -> None: + if self._handle: + self._kernel32.TerminateJobObject(self._handle, 1) + + def close(self) -> None: + if self._handle: + self._kernel32.CloseHandle(self._handle) + self._handle = None + + +async def _stop_process( + process: asyncio.subprocess.Process, process_tree: _WindowsJob | None = None +) -> None: + if process.returncode is not None: + return + if os.name == "posix": + with contextlib.suppress(ProcessLookupError): + os.killpg(process.pid, signal.SIGTERM) + else: + if process_tree is not None: + process_tree.terminate() + else: + with contextlib.suppress(ProcessLookupError): + process.terminate() + try: + await asyncio.wait_for(process.wait(), timeout=1.0) + return + except TimeoutError: + pass + if os.name == "posix": + with contextlib.suppress(ProcessLookupError): + os.killpg(process.pid, signal.SIGKILL) + else: + if process_tree is not None: + process_tree.terminate() + else: + with contextlib.suppress(ProcessLookupError): + process.kill() + await process.wait() + + +def _child_environment(selection: RuntimeSelection) -> dict[str, str]: + env = dict(os.environ) + # Models are installed and verified by the app. Missing files must fail + # rather than trigger upstream's mutable first-run downloader. + env["HF_HUB_OFFLINE"] = "1" + env["HF_HUB_DISABLE_TELEMETRY"] = "1" + env["DO_NOT_TRACK"] = "1" + # Windows otherwise inherits a legacy console/filesystem encoding (often + # cp1252), which can make valid Unicode asset paths fail before inference. + env["PYTHONUTF8"] = "1" + env["PYTHONIOENCODING"] = "utf-8" + env.pop("HF_TOKEN", None) + env.pop("HUGGING_FACE_HUB_TOKEN", None) + if selection.backend is BackendName.TFLITE: + threads = str(_tflite_threads(os.environ)) + env["OMP_NUM_THREADS"] = threads + env["OPENBLAS_NUM_THREADS"] = threads + env["TF_NUM_INTRAOP_THREADS"] = threads + return env + + +async def _run_cli( + selection: RuntimeSelection, + argv: list[str], + *, + seconds: float, + cancel_event: asyncio.Event | None, + on_progress: Callable[[ProgressEvent], None] | None, +) -> tuple[int, str]: + spawn_options: dict = { + "cwd": selection.runtime_dir, + "env": _child_environment(selection), + "stdout": asyncio.subprocess.PIPE, + "stderr": asyncio.subprocess.STDOUT, + } + if os.name == "posix": + spawn_options["start_new_session"] = True + elif os.name == "nt": + flags = subprocess.CREATE_NEW_PROCESS_GROUP + if selection.backend is BackendName.TFLITE: + flags |= subprocess.BELOW_NORMAL_PRIORITY_CLASS + spawn_options["creationflags"] = flags + process = await asyncio.create_subprocess_exec(*argv, **spawn_options) + try: + process_tree = _WindowsJob(process.pid) if os.name == "nt" else None + except Exception: + # A process that escaped Job Object assignment could outlive cancellation + # or app exit. Fail closed and reap it before surfacing the setup error. + with contextlib.suppress(ProcessLookupError): + process.kill() + await process.wait() + raise + if selection.backend is BackendName.TFLITE and hasattr(os, "setpriority"): + with contextlib.suppress(OSError): + os.setpriority(os.PRIO_PROCESS, process.pid, 10) + assert process.stdout is not None + drain = asyncio.create_task(_drain_output(process.stdout, on_progress)) + wait = asyncio.create_task(process.wait()) + cancel = ( + asyncio.create_task(cancel_event.wait()) if cancel_event is not None else None + ) + watched = {wait} + if cancel is not None: + watched.add(cancel) + try: + done, _ = await asyncio.wait( + watched, timeout=timeout_for(seconds), return_when=asyncio.FIRST_COMPLETED + ) + if not done: + await _stop_process(process, process_tree) + raise GenerationFailed( + f"generation timed out after {timeout_for(seconds):g}s" + ) + if cancel is not None and cancel in done and cancel.result(): + await _stop_process(process, process_tree) + raise GenerationCancelled("generation cancelled") + return_code = await wait + return return_code, await drain + except asyncio.CancelledError: + await _stop_process(process, process_tree) + raise + finally: + if cancel is not None: + cancel.cancel() + if not drain.done(): + drain.cancel() + with contextlib.suppress(asyncio.CancelledError): + await drain + if process_tree is not None: + process_tree.close() + + +_generation_lock = asyncio.Semaphore(1) async def generate( @@ -175,82 +811,158 @@ async def generate( cfg: float | None = None, apg: float | None = None, seed: int | None = None, + steps: int = SAMPLER_STEPS, lora_dirs: Sequence[str] | None = None, lora_strengths: Sequence[float] | None = None, + cancel_event: asyncio.Event | None = None, + on_progress: Callable[[ProgressEvent], None] | None = None, + on_state: Callable[[str], None] | None = None, ) -> bytes: - """Run one generation and return the WAV bytes. - - Raises GenerationUnavailable when no checkout resolves and - GenerationFailed when the CLI errors, times out, or writes nothing. - Inputs are assumed validated at the trust boundary (controller). - """ - mlx_dir = resolve_mlx_dir() - if mlx_dir is None: - raise GenerationUnavailable(SETUP_HINT) - async with _generation_lock: - with tempfile.TemporaryDirectory(prefix="sa3-") as tmp: - out_path = pathlib.Path(tmp) / "out.wav" - argv = [ - str(mlx_dir / ".venv" / "bin" / "python"), - str(mlx_dir / "scripts" / "sa3_mlx.py"), - "--prompt", - prompt, - "--dit", - KINDS[kind], - "--decoder", - DECODERS[kind], - "--seconds", - f"{seconds:g}", - "--steps", - str(SAMPLER_STEPS), - "--out", - str(out_path), - ] - if init_audio is not None: - init_path = pathlib.Path(tmp) / "init.wav" - init_path.write_bytes(init_audio) - argv.extend(("--init-audio", str(init_path))) - if init_noise_level is not None: - argv.extend(("--init-noise-level", f"{init_noise_level:g}")) - if inpaint_range is not None: - start, end = inpaint_range - argv.extend(("--inpaint-range", f"{start:g},{end:g}")) - if negative_prompt is not None: - argv.extend(("--negative-prompt", negative_prompt)) - if cfg is not None: - argv.extend(("--cfg", f"{cfg:g}")) - if apg is not None: - argv.extend(("--apg", f"{apg:g}")) - if seed is not None: - argv.extend(("--seed", str(seed))) - if lora_dirs: - # One --lora group per adapter (upstream's PR #57/#65 syntax): - # the directory plus its strength=S option. The CLI resolves - # the .safetensors inside and merges all deltas at DiT load. - for index, lora_dir in enumerate(lora_dirs): - argv.extend(("--lora", lora_dir)) - if lora_strengths is not None: - argv.append(f"strength={lora_strengths[index]:g}") - process = await asyncio.create_subprocess_exec( - *argv, - cwd=mlx_dir, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.STDOUT, + """Generate one validated WAV through the platform-selected backend.""" + request = GenerationRequest( + prompt=prompt, + seconds=seconds, + kind=kind, + init_audio=init_audio, + init_noise_level=init_noise_level, + inpaint_range=inpaint_range, + negative_prompt=negative_prompt, + cfg=cfg, + apg=apg, + seed=seed, + steps=steps, + lora_dirs=lora_dirs, + lora_strengths=lora_strengths, + ) + preference_value = os.environ.get( + SA3_PREFERENCE_ENV, sa3_cuda.BackendPreference.AUTO.value + ) + try: + preference = sa3_cuda.BackendPreference(preference_value) + except ValueError: + raise GenerationUnavailable( + f"{SA3_PREFERENCE_ENV} must be auto, gpu, or cpu_tflite" + ) from None + if preference is sa3_cuda.BackendPreference.GPU: + platform_value, machine_value = sa3_cuda.host_identity() + evidence = sa3_cuda.CudaEvidence( + platform=platform_value, + machine=machine_value, + runtime_ready=False, + provenance_complete=False, + packages={}, + cuda_available=False, + cuda_runtime=None, + driver=None, + device=None, + compute_capability=None, + total_vram_bytes=None, + free_vram_bytes=None, + estimated_vram_bytes={kind: None}, + ) + try: + sa3_cuda.choose_backend( + preference, + kind=kind, + cuda=evidence, + tflite_ready=resolve_runtime() is not None, + ) + except sa3_cuda.CudaUnavailable as error: + fallback = ( + " Choose CPU/TFLite to confirm the fallback." + if error.fallback_available + else "" ) - timeout = timeout_for(seconds) + raise GenerationUnavailable(f"{error}{fallback}") from None + # The release gate currently makes this unreachable. Do not start the + # experimental worker through the public endpoint until the installer + # can produce a complete provenance stamp and hardware evidence flips + # HARDWARE_QUALIFIED in the same reviewed change. + raise GenerationUnavailable( + "the CUDA worker is not exposed until its release gates are complete" + ) + try: + selection = resolve_runtime() + except GenerationUnavailable: + raise + if selection is None: + backend = select_backend() + raise GenerationUnavailable( + f"the {backend.value} Stable Audio runtime is unavailable; " + "install the pinned runtime and model bundle from the model manager" + ) + + normalized = None + if init_audio is not None: + try: + normalized = inspect_canonical_wav(init_audio) + except AudioFormatError: try: - output, _ = await asyncio.wait_for( - process.communicate(), timeout=timeout + normalized = normalize_wav(init_audio) + except AudioFormatError as error: + raise GenerationFailed(str(error)) from None + + mode = request.mode( + input_seconds=None if normalized is None else normalized.seconds + ) + _generation_state.update( + state="queued", + backend=selection.backend.value, + mode=mode.value, + progress=None, + ) + if on_state is not None: + on_state("queued") + + def report(event: ProgressEvent) -> None: + _generation_state["progress"] = event.as_dict() + if on_progress is not None: + on_progress(event) + + async with _generation_lock: + _generation_state["state"] = "running" + if on_state is not None: + on_state("running") + staging = runtime_paths.staging_home() + if staging is not None: + staging.mkdir(parents=True, exist_ok=True) + try: + if cancel_event is not None and cancel_event.is_set(): + raise GenerationCancelled("generation cancelled while queued") + with tempfile.TemporaryDirectory(prefix="sa3-", dir=staging) as tmp: + tmp_path = pathlib.Path(tmp) + out_path = tmp_path / "out.wav" + init_path = None + if normalized is not None: + init_path = tmp_path / "init.wav" + init_path.write_bytes(normalized.wav) + argv = build_argv( + selection, + request, + out_path=out_path, + init_path=init_path, ) - except TimeoutError: - process.kill() - await process.wait() - raise GenerationFailed( - f"generation timed out after {timeout:g}s" - ) from None - if process.returncode != 0 or not out_path.is_file(): - # The CLI's last lines name the problem; progress bars and - # ANSI noise live further up. - tail = output.decode(errors="replace").strip()[-500:] - raise GenerationFailed(tail or "sa3_mlx produced no output") - return out_path.read_bytes() + return_code, output = await _run_cli( + selection, + argv, + seconds=seconds, + cancel_event=cancel_event, + on_progress=report, + ) + if return_code != 0 or not out_path.is_file(): + raise GenerationFailed( + _safe_failure_tail(output, selection.backend) + ) + max_output_bytes = round(seconds * 44_100) * 4 + 1024 * 1024 + if out_path.stat().st_size > max_output_bytes: + raise GenerationFailed( + "backend WAV is larger than the requested duration" + ) + try: + return _validate_output_wav(out_path.read_bytes(), seconds) + except AudioFormatError as error: + raise GenerationFailed(str(error)) from None + finally: + _generation_state.update( + state="idle", backend=None, mode=None, progress=None + ) diff --git a/backend/lsdj/sa3_audio.py b/backend/lsdj/sa3_audio.py new file mode 100644 index 0000000..72541d3 --- /dev/null +++ b/backend/lsdj/sa3_audio.py @@ -0,0 +1,212 @@ +"""Bounded WAV normalization and validation for Stable Audio 3. + +Only uncompressed integer PCM WAV is accepted at the HTTP boundary. LSDJ +converts sample width, channel layout, and sample rate itself, so neither SA3 +backend can fall through to its optional system-``ffmpeg`` path. +""" + +from __future__ import annotations + +import io +import math +import struct +import threading +import wave +from collections.abc import Callable +from dataclasses import dataclass + +SAMPLE_RATE = 44_100 +CHANNELS = 2 +SAMPLE_WIDTH = 2 +MAX_INPUT_SECONDS = 380.0 +MAX_DECODED_PCM_BYTES = 16 * 1024 * 1024 +MAX_NORMALIZED_FRAMES = round(MAX_INPUT_SECONDS * SAMPLE_RATE) +NORMALIZATION_CHECK_FRAMES = 16_384 + + +class AudioFormatError(ValueError): + """The WAV is corrupt or uses an encoding LSDJ does not accept.""" + + +class AudioNormalizationCancelled(Exception): + """The caller cancelled bounded input normalization.""" + + +@dataclass(frozen=True) +class NormalizedAudio: + wav: bytes + frames: int + seconds: float + + +def _decode_sample(raw: bytes, width: int) -> float: + if width == 1: + return (raw[0] - 128) / 128.0 + if width == 2: + return int.from_bytes(raw, "little", signed=True) / 32768.0 + if width == 3: + value = int.from_bytes(raw, "little", signed=False) + if value & 0x800000: + value -= 1 << 24 + return value / 8388608.0 + if width == 4: + return int.from_bytes(raw, "little", signed=True) / 2147483648.0 + raise AudioFormatError("PCM sample width must be 8, 16, 24, or 32 bits") + + +def _stereo_frame( + raw: bytes, index: int, channels: int, width: int +) -> tuple[float, float]: + start = index * channels * width + left = _decode_sample(raw[start : start + width], width) + if channels == 1: + return left, left + right_start = start + width + return left, _decode_sample(raw[right_start : right_start + width], width) + + +def _pcm16(value: float) -> int: + value = min(1.0, max(-1.0, value)) + if value <= -1.0: + return -32768 + return min(32767, max(-32768, round(value * 32767.0))) + + +def normalize_wav( + data: bytes, + *, + cancel_event: threading.Event | None = None, + on_progress: Callable[[int, int], None] | None = None, +) -> NormalizedAudio: + """Return canonical 44.1 kHz stereo PCM16 WAV bytes. + + The conversion is deterministic and bounded by ``MAX_INPUT_SECONDS``. + Multichannel input follows the official TFLite runtime's semantics and + retains the first two channels; mono is duplicated. + """ + try: + with wave.open(io.BytesIO(data), "rb") as source: + channels = source.getnchannels() + width = source.getsampwidth() + rate = source.getframerate() + frames = source.getnframes() + compression = source.getcomptype() + if compression != "NONE": + raise AudioFormatError("WAV must use uncompressed integer PCM") + if channels < 1 or channels > 32: + raise AudioFormatError("WAV must have between 1 and 32 channels") + if width not in (1, 2, 3, 4): + raise AudioFormatError("PCM sample width must be 8, 16, 24, or 32 bits") + if rate < 8_000 or rate > 384_000: + raise AudioFormatError( + "WAV sample rate must be between 8 kHz and 384 kHz" + ) + if frames < 1: + raise AudioFormatError("WAV must contain audio frames") + expected_bytes = frames * channels * width + if expected_bytes > MAX_DECODED_PCM_BYTES: + raise AudioFormatError( + f"decoded PCM must be at most {MAX_DECODED_PCM_BYTES} bytes" + ) + seconds = frames / rate + if not math.isfinite(seconds) or seconds > MAX_INPUT_SECONDS: + raise AudioFormatError( + f"WAV must be at most {MAX_INPUT_SECONDS:g} seconds" + ) + raw = source.readframes(frames) + except AudioFormatError: + raise + except (EOFError, wave.Error, OverflowError, struct.error): + raise AudioFormatError("init audio must be a valid PCM WAV file") from None + + if len(raw) != expected_bytes: + raise AudioFormatError("WAV sample data is truncated") + if cancel_event is not None and cancel_event.is_set(): + raise AudioNormalizationCancelled + if channels == CHANNELS and width == SAMPLE_WIDTH and rate == SAMPLE_RATE: + if on_progress is not None: + on_progress(frames, frames) + return NormalizedAudio(wav=data, frames=frames, seconds=seconds) + + target_frames = max(1, round(frames * SAMPLE_RATE / rate)) + if target_frames > MAX_NORMALIZED_FRAMES: + raise AudioFormatError( + f"normalized WAV must be at most {MAX_NORMALIZED_FRAMES} frames" + ) + pcm = bytearray(target_frames * CHANNELS * SAMPLE_WIDTH) + source_per_target = rate / SAMPLE_RATE + last_source_frame = frames - 1 + for index in range(target_frames): + if index % NORMALIZATION_CHECK_FRAMES == 0: + if cancel_event is not None and cancel_event.is_set(): + raise AudioNormalizationCancelled + if on_progress is not None: + on_progress(index, target_frames) + position = index * source_per_target + lower = min(int(position), last_source_frame) + upper = min(lower + 1, last_source_frame) + fraction = position - lower + left_lower, right_lower = _stereo_frame(raw, lower, channels, width) + left_upper, right_upper = _stereo_frame(raw, upper, channels, width) + left_sample = left_lower + (left_upper - left_lower) * fraction + right_sample = right_lower + (right_upper - right_lower) * fraction + offset = index * 4 + struct.pack_into(" NormalizedAudio: + """Validate a canonical backend WAV without copying its entire payload.""" + try: + with wave.open(io.BytesIO(data), "rb") as source: + channels = source.getnchannels() + width = source.getsampwidth() + rate = source.getframerate() + frames = source.getnframes() + compression = source.getcomptype() + if ( + compression != "NONE" + or channels != CHANNELS + or width != SAMPLE_WIDTH + or rate != SAMPLE_RATE + or frames < 1 + ): + raise AudioFormatError( + "backend output must be non-empty 44.1 kHz stereo PCM16 WAV" + ) + read_frames = 0 + while read_frames < frames: + chunk_frames = min(65_536, frames - read_frames) + chunk = source.readframes(chunk_frames) + if len(chunk) != chunk_frames * CHANNELS * SAMPLE_WIDTH: + raise AudioFormatError("backend WAV payload is truncated") + read_frames += chunk_frames + except AudioFormatError: + raise + except (EOFError, wave.Error, OverflowError): + raise AudioFormatError("backend produced a corrupt WAV") from None + return NormalizedAudio(wav=data, frames=frames, seconds=frames / SAMPLE_RATE) + + +def validate_output_wav(data: bytes, seconds: float) -> bytes: + output = inspect_canonical_wav(data) + expected_frames = round(seconds * SAMPLE_RATE) + if output.frames != expected_frames: + raise AudioFormatError( + f"backend produced {output.frames} frames; expected {expected_frames}" + ) + return data diff --git a/backend/lsdj/sa3_contract.py b/backend/lsdj/sa3_contract.py new file mode 100644 index 0000000..e4704ca --- /dev/null +++ b/backend/lsdj/sa3_contract.py @@ -0,0 +1,164 @@ +"""Runtime-neutral Stable Audio 3 service contract. + +The desktop app owns this contract. MLX and TFLite are implementation details: +they receive the same validated request and must either honour every populated +control or reject it explicitly. +""" + +from __future__ import annotations + +from collections.abc import Sequence +from dataclasses import dataclass +from enum import StrEnum + + +class BackendName(StrEnum): + MLX = "mlx" + TFLITE = "tflite" + PYTORCH_CUDA = "pytorch_cuda" + + +class GenerationMode(StrEnum): + TEXT_TO_AUDIO = "text_to_audio" + AUDIO_TO_AUDIO = "audio_to_audio" + INPAINT = "inpaint" + CONTINUATION = "continuation" + + +@dataclass(frozen=True) +class GenerationRequest: + prompt: str + seconds: float + kind: str + init_audio: bytes | None = None + init_noise_level: float | None = None + inpaint_range: tuple[float, float] | None = None + negative_prompt: str | None = None + cfg: float | None = None + apg: float | None = None + seed: int | None = None + steps: int = 8 + lora_dirs: Sequence[str] | None = None + lora_strengths: Sequence[float] | None = None + + def mode(self, *, input_seconds: float | None = None) -> GenerationMode: + if self.init_audio is None: + return GenerationMode.TEXT_TO_AUDIO + if self.inpaint_range is None: + return GenerationMode.AUDIO_TO_AUDIO + start, end = self.inpaint_range + one_sample = 1 / 44_100 + if ( + input_seconds is not None + and self.seconds > input_seconds + and abs(start - input_seconds) <= one_sample + and abs(end - self.seconds) <= one_sample + ): + return GenerationMode.CONTINUATION + return GenerationMode.INPAINT + + +@dataclass(frozen=True) +class BackendCapabilities: + backend: BackendName + modes: tuple[GenerationMode, ...] + controls: tuple[str, ...] + models: tuple[str, ...] + progress: bool + cancellation: bool + preview: bool + limitations: tuple[str, ...] + + def as_dict(self) -> dict: + return { + "backend": self.backend.value, + "modes": [mode.value for mode in self.modes], + "controls": list(self.controls), + "models": list(self.models), + "progress": self.progress, + "cancellation": self.cancellation, + "preview": self.preview, + "limitations": list(self.limitations), + } + + +@dataclass(frozen=True) +class ProgressEvent: + stage: str + current: int | None + total: int | None + message: str + + def as_dict(self) -> dict: + return { + "stage": self.stage, + "current": self.current, + "total": self.total, + "message": self.message, + } + + +COMMON_MODES = tuple(GenerationMode) +COMMON_CONTROLS = ( + "positive_prompt", + "negative_prompt", + "duration", + "steps", + "seed", + "init_noise_level", + "cfg", + "apg", + "inpaint_range", + "lora", +) +COMMON_MODELS = ("small_music", "small_sfx", "medium") + +MLX_CAPABILITIES = BackendCapabilities( + backend=BackendName.MLX, + modes=COMMON_MODES, + controls=COMMON_CONTROLS, + models=COMMON_MODELS, + progress=True, + cancellation=True, + preview=False, + limitations=("The pinned MLX CLI does not expose partial audio previews.",), +) + +TFLITE_CAPABILITIES = BackendCapabilities( + backend=BackendName.TFLITE, + modes=COMMON_MODES, + controls=COMMON_CONTROLS, + models=COMMON_MODELS, + progress=True, + cancellation=True, + preview=False, + limitations=( + "The official TFLite CLI does not expose partial audio previews.", + "Per-step LoRA gating is MLX-only; LSDJ supports TFLite LoRA strength but not step ranges.", + "The portable backend is CPU-only and does not use an NVIDIA GPU.", + ), +) + +PYTORCH_CUDA_CAPABILITIES = BackendCapabilities( + backend=BackendName.PYTORCH_CUDA, + modes=COMMON_MODES, + controls=COMMON_CONTROLS, + models=("small_music", "small_sfx"), + progress=True, + cancellation=True, + preview=False, + limitations=( + "Windows x64 with a qualified NVIDIA GPU, driver, runtime, and measured VRAM is required.", + "Stable Audio Medium remains on TFLite until an official Windows FlashAttention build is qualified.", + "There is no PyTorch CPU fallback; CPU mode uses the separate TFLite runtime.", + "The disposable worker releases its CUDA context after every generation.", + ), +) + + +def capabilities_for(backend: BackendName) -> BackendCapabilities: + return { + BackendName.MLX: MLX_CAPABILITIES, + BackendName.TFLITE: TFLITE_CAPABILITIES, + BackendName.PYTORCH_CUDA: PYTORCH_CUDA_CAPABILITIES, + }[backend] diff --git a/backend/lsdj/sa3_cuda.py b/backend/lsdj/sa3_cuda.py new file mode 100644 index 0000000..ee6e4dc --- /dev/null +++ b/backend/lsdj/sa3_cuda.py @@ -0,0 +1,318 @@ +"""Fail-closed policy for the optional Windows Stable Audio CUDA worker. + +The official TFLite backend remains the portable baseline. This module only +selects PyTorch when its immutable shared runtime, gated model provenance, +Windows NVIDIA driver, measured VRAM reservation, and release qualification are +all present. An explicit qualification opt-in may exercise unmeasured hardware, +but it never enables automatic selection or a PyTorch CPU fallback. +""" + +from __future__ import annotations + +import enum +import os +import platform as host_platform +import re +import sys +from dataclasses import asdict, dataclass +from typing import Mapping + + +BACKEND_NAME = "pytorch_cuda" +UNVERIFIED_OPT_IN = "LSDJ_ALLOW_UNVERIFIED_SA3_CUDA" +CUDA_RUNTIME = "12.6" +MIN_WINDOWS_DRIVER = (560, 76) +VRAM_HEADROOM_BYTES = 1024**3 + +# Flipped only in a PR that carries the completed physical-hardware evidence. +HARDWARE_QUALIFIED = False + +EXPECTED_PACKAGES = { + "torch": "2.7.1+cu126", + "torchaudio": "2.7.1+cu126", + "transformers": "5.8.0", + "huggingface-hub": "1.7.1", + "numpy": "2.3.5", + "safetensors": "0.7.0", + "sentencepiece": "0.2.1", + "resampy": "0.4.3", +} +SOURCE_REVISION = "a0b57f5483c4588f827f3552b7d5c6ca2a9687be" +RUNTIME_LOCK_SHA256 = "3c9bf7d79c3848ebe1da40fd14b26708b55d8157f008cb3a1944ddfb1cd597c4" +MODEL_PINS = { + "music": { + "repository": "stabilityai/stable-audio-3-small-music", + "revision": "0fef1392cd842149a2b6d445e181c97608faac06", + }, + "sfx": { + "repository": "stabilityai/stable-audio-3-small-sfx", + "revision": "ae12755283df9d62ca39a9b050a39a0b607b8c20", + }, +} + + +class BackendPreference(enum.StrEnum): + AUTO = "auto" + GPU = "gpu" + CPU_TFLITE = "cpu_tflite" + + +class CudaUnavailable(RuntimeError): + def __init__( + self, + message: str, + *, + reason: str, + fallback_available: bool, + ) -> None: + super().__init__(message) + self.reason = reason + self.fallback_available = fallback_available + + +@dataclass(frozen=True) +class CudaEvidence: + platform: str + machine: str + runtime_ready: bool + provenance_complete: bool + packages: Mapping[str, str] + cuda_available: bool + cuda_runtime: str | None + driver: str | None + device: str | None + compute_capability: tuple[int, int] | None + total_vram_bytes: int | None + free_vram_bytes: int | None + estimated_vram_bytes: Mapping[str, int | None] + source_revision: str | None = None + model_revision: str | None = None + + def as_dict(self) -> dict[str, object]: + value = asdict(self) + if self.compute_capability is not None: + value["compute_capability"] = list(self.compute_capability) + return value + + +@dataclass(frozen=True) +class BackendDecision: + backend: str + preference: BackendPreference + reason: str + fallback: bool + + def as_dict(self) -> dict[str, object]: + value = asdict(self) + value["preference"] = self.preference.value + return value + + +def _normalise_platform(platform_name: str) -> str: + value = platform_name.lower() + if value.startswith(("win32", "cygwin", "msys")): + return "windows" + return value + + +def _normalise_machine(machine: str) -> str: + value = machine.lower() + return "x86_64" if value in {"amd64", "x86_64"} else value + + +def _truthy(value: str | None) -> bool: + return value is not None and value.strip().lower() in {"1", "true", "yes", "on"} + + +def parse_driver_version(version: str | None) -> tuple[int, int] | None: + if version is None: + return None + match = re.fullmatch(r"\s*(\d{3,4})\.(\d{1,3})(?:\.\d+)?\s*", version) + if match is None: + return None + return int(match.group(1)), int(match.group(2)) + + +def runtime_errors( + evidence: CudaEvidence, + *, + kind: str, + allow_unmeasured_vram: bool = False, +) -> list[str]: + errors = [] + if ( + _normalise_platform(evidence.platform) != "windows" + or _normalise_machine(evidence.machine) != "x86_64" + ): + errors.append("the CUDA Stable Audio backend supports Windows x64 only") + if not evidence.runtime_ready: + errors.append("the shared app-owned PyTorch runtime is not ready") + if not evidence.provenance_complete: + errors.append("the gated Stable Audio model provenance is incomplete") + if evidence.source_revision != SOURCE_REVISION: + errors.append( + "the installed Stable Audio source revision does not match the pin" + ) + expected_model = MODEL_PINS.get(kind) + if ( + expected_model is not None + and evidence.model_revision != expected_model["revision"] + ): + errors.append( + "the installed Stable Audio model revision does not match the pin" + ) + mismatched = { + name: (evidence.packages.get(name), expected) + for name, expected in EXPECTED_PACKAGES.items() + if evidence.packages.get(name) != expected + } + if mismatched: + errors.append( + "the installed shared PyTorch dependency versions do not match the pin" + ) + if not evidence.cuda_available: + errors.append( + "PyTorch reports no CUDA device; there is no PyTorch CPU fallback" + ) + if evidence.cuda_runtime != CUDA_RUNTIME: + errors.append( + f"the installed PyTorch CUDA runtime is {evidence.cuda_runtime or 'unknown'}, " + f"not the pinned {CUDA_RUNTIME} runtime" + ) + driver = parse_driver_version(evidence.driver) + if driver is None: + errors.append("the NVIDIA display driver version could not be verified") + elif driver < MIN_WINDOWS_DRIVER: + errors.append( + "the NVIDIA driver is older than the provisional CUDA 12.6 floor " + f"{MIN_WINDOWS_DRIVER[0]}.{MIN_WINDOWS_DRIVER[1]}" + ) + if kind == "track": + errors.append( + "Stable Audio Medium requires FlashAttention 2; no official Windows " + "wheel has been qualified, so Medium remains on TFLite" + ) + estimate = evidence.estimated_vram_bytes.get(kind) + if estimate is None: + if not allow_unmeasured_vram: + errors.append(f"{kind} has no qualified VRAM reservation yet") + elif evidence.free_vram_bytes is None: + errors.append("free CUDA memory could not be measured") + elif evidence.free_vram_bytes < estimate + VRAM_HEADROOM_BYTES: + errors.append( + f"{kind} needs an estimated {estimate} bytes plus " + f"{VRAM_HEADROOM_BYTES} bytes headroom, but only " + f"{evidence.free_vram_bytes} bytes are free" + ) + return errors + + +def choose_backend( + preference: BackendPreference | str, + *, + kind: str, + cuda: CudaEvidence, + tflite_ready: bool, + env: Mapping[str, str] | None = None, +) -> BackendDecision: + try: + preference = BackendPreference(preference) + except ValueError: + raise CudaUnavailable( + "Stable Audio preference must be auto, gpu, or cpu_tflite", + reason="invalid_preference", + fallback_available=tflite_ready, + ) from None + environment = os.environ if env is None else env + + if preference is BackendPreference.CPU_TFLITE: + if not tflite_ready: + raise CudaUnavailable( + "the requested TFLite backend is not installed and ready", + reason="tflite_not_ready", + fallback_available=False, + ) + return BackendDecision("tflite", preference, "CPU/TFLite was selected", False) + + experimental = _truthy(environment.get(UNVERIFIED_OPT_IN)) + errors = runtime_errors( + cuda, + kind=kind, + allow_unmeasured_vram=(preference is BackendPreference.GPU and experimental), + ) + release_ready = HARDWARE_QUALIFIED and not errors + qualification_ready = experimental and not errors + + if preference is BackendPreference.GPU: + if not (release_ready or qualification_ready): + if not HARDWARE_QUALIFIED and not experimental: + errors.insert( + 0, + "the Windows CUDA backend is implemented but not release-qualified; " + f"{UNVERIFIED_OPT_IN}=1 is reserved for hardware qualification", + ) + raise CudaUnavailable( + "; ".join(errors) if errors else "the CUDA backend is unavailable", + reason="cuda_not_eligible", + fallback_available=tflite_ready, + ) + return BackendDecision( + BACKEND_NAME, + preference, + "explicit experimental GPU qualification" + if not HARDWARE_QUALIFIED + else "explicit GPU selection", + False, + ) + + if release_ready: + return BackendDecision( + BACKEND_NAME, preference, "qualified CUDA backend", False + ) + if tflite_ready: + return BackendDecision( + "tflite", + preference, + "; ".join(errors) + if errors + else "CUDA hardware qualification is incomplete", + True, + ) + raise CudaUnavailable( + "; ".join(errors + ["the TFLite fallback is not ready"]), + reason="no_ready_backend", + fallback_available=False, + ) + + +def diagnostic_manifest( + cuda: CudaEvidence, + *, + tflite_ready: bool, + env: Mapping[str, str] | None = None, +) -> dict[str, object]: + environment = os.environ if env is None else env + return { + "backend": BACKEND_NAME, + "release_ready": HARDWARE_QUALIFIED, + "qualification_opt_in": _truthy(environment.get(UNVERIFIED_OPT_IN)), + "cpu_fallback": False, + "tflite_fallback_ready": tflite_ready, + "cuda_runtime_pin": CUDA_RUNTIME, + "minimum_windows_driver_provisional": ( + f"{MIN_WINDOWS_DRIVER[0]}.{MIN_WINDOWS_DRIVER[1]}" + ), + "vram_headroom_bytes": VRAM_HEADROOM_BYTES, + "evidence": cuda.as_dict(), + "qualification_blockers": [ + "authenticated hashes for gated Stable Audio and T5Gemma artifacts", + "MRT2 parity on the shared torch 2.7.1/CUDA 12.6 runtime", + "measured Small Music and Small SFX VRAM reservations", + "Windows NVIDIA cancellation/OOM/crash/VRAM-release evidence", + "two active MRT2 decks for ten minutes at 25- and 5-frame scheduling", + ], + } + + +def host_identity() -> tuple[str, str]: + return sys.platform, host_platform.machine() diff --git a/backend/lsdj/sa3_cuda_worker.py b/backend/lsdj/sa3_cuda_worker.py new file mode 100644 index 0000000..24a0284 --- /dev/null +++ b/backend/lsdj/sa3_cuda_worker.py @@ -0,0 +1,634 @@ +"""Disposable Stable Audio 3 PyTorch/CUDA worker. + +The controller writes one bounded JSON request and starts this module with the +app-owned shared PyTorch interpreter. Heavyweight imports and model allocation +occur only in this child. Cancellation, an MRT2 priority waiter, CUDA OOM, a +driver reset, or any other failure ends the process, releasing its CUDA context +without affecting the deck workers or native audio callback. +""" + +from __future__ import annotations + +import argparse +import contextlib +import hashlib +import hmac +import importlib.metadata +import json +import os +import pathlib +import platform as host_platform +import re +import sys +import threading +import wave +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass +from typing import Any, Protocol + +import numpy as np + +from . import sa3_cuda +from .gpu_broker import GpuBroker, Lease, Priority + + +SCHEMA_VERSION = 1 +SAMPLE_RATE = 44_100 +CHANNELS = 2 +MAX_JSON_BYTES = 64 * 1024 +MAX_LAUNCH_TOKEN_BYTES = 512 +LAUNCH_TOKEN_ENV = "LSDJ_WORKER_LAUNCH_TOKEN" +MODEL_FOR_KIND = {"music": "small-music", "sfx": "small-sfx"} +_JOB_ID = re.compile(r"[A-Za-z0-9][A-Za-z0-9_-]{0,63}") +_SHA256 = re.compile(r"[0-9a-f]{64}") + + +class WorkerError(RuntimeError): + pass + + +class WorkerCancelled(WorkerError): + pass + + +class ModelProtocol(Protocol): + def load_lora(self, paths: Sequence[str]) -> None: ... + + def set_lora_strength( + self, strength: float, lora_index: int | None = None + ) -> None: ... + + def generate(self, **kwargs: Any) -> Any: ... + + +@dataclass(frozen=True) +class WorkerRequest: + job_id: str + launch_token_sha256: str + prompt: str + seconds: float + kind: str + steps: int + cfg: float | None + apg: float | None + seed: int | None + negative_prompt: str | None + init_noise_level: float | None + inpaint_range: tuple[float, float] | None + init_audio: pathlib.Path | None + lora_files: tuple[pathlib.Path, ...] + lora_strengths: tuple[float, ...] + model_dir: pathlib.Path + output: pathlib.Path + + @classmethod + def from_dict(cls, value: Mapping[str, Any]) -> "WorkerRequest": + if value.get("schema_version") != SCHEMA_VERSION: + raise WorkerError("unsupported CUDA worker request schema") + prompt = value.get("prompt") + job_id = value.get("job_id") + launch_token_sha256 = value.get("launch_token_sha256") + kind = value.get("kind") + seconds = value.get("seconds") + steps = value.get("steps") + if not isinstance(prompt, str) or not prompt or len(prompt) > 32_000: + raise WorkerError("prompt is invalid") + if not isinstance(job_id, str) or _JOB_ID.fullmatch(job_id) is None: + raise WorkerError("job_id is invalid") + if ( + not isinstance(launch_token_sha256, str) + or _SHA256.fullmatch(launch_token_sha256) is None + ): + raise WorkerError("launch_token_sha256 is invalid") + if kind not in MODEL_FOR_KIND: + raise WorkerError("CUDA supports only Small Music and Small SFX") + if ( + isinstance(seconds, bool) + or not isinstance(seconds, (int, float)) + or not 0.5 <= float(seconds) <= 32.0 + ): + raise WorkerError("seconds is invalid") + if ( + isinstance(steps, bool) + or not isinstance(steps, int) + or not 1 <= steps <= 100 + ): + raise WorkerError("steps is invalid") + inpaint = value.get("inpaint_range") + if inpaint is not None: + if ( + not isinstance(inpaint, list) + or len(inpaint) != 2 + or any( + isinstance(item, bool) or not isinstance(item, (int, float)) + for item in inpaint + ) + or not 0 <= float(inpaint[0]) < float(inpaint[1]) <= float(seconds) + ): + raise WorkerError("inpaint_range is invalid") + inpaint = (float(inpaint[0]), float(inpaint[1])) + lora_files = value.get("lora_files", []) + lora_strengths = value.get("lora_strengths", []) + if ( + not isinstance(lora_files, list) + or not isinstance(lora_strengths, list) + or len(lora_files) != len(lora_strengths) + or len(lora_files) > 4 + or any(not isinstance(item, str) for item in lora_files) + or any( + isinstance(item, bool) + or not isinstance(item, (int, float)) + or not 0 <= float(item) <= 4 + for item in lora_strengths + ) + ): + raise WorkerError("LoRA stack is invalid") + init_audio = value.get("init_audio") + if init_audio is not None and not isinstance(init_audio, str): + raise WorkerError("init_audio is invalid") + if inpaint is not None and init_audio is None: + raise WorkerError("inpainting requires init_audio") + return cls( + job_id=job_id, + launch_token_sha256=launch_token_sha256, + prompt=prompt, + seconds=float(seconds), + kind=kind, + steps=steps, + cfg=_optional_float(value, "cfg"), + apg=_optional_float(value, "apg"), + seed=_optional_int(value, "seed"), + negative_prompt=_optional_string(value, "negative_prompt"), + init_noise_level=_optional_float(value, "init_noise_level"), + inpaint_range=inpaint, + init_audio=None if init_audio is None else pathlib.Path(init_audio), + lora_files=tuple(pathlib.Path(item) for item in lora_files), + lora_strengths=tuple(float(item) for item in lora_strengths), + model_dir=pathlib.Path(_required_string(value, "model_dir")), + output=pathlib.Path(_required_string(value, "output")), + ) + + +def _required_string(value: Mapping[str, Any], field: str) -> str: + item = value.get(field) + if not isinstance(item, str) or not item: + raise WorkerError(f"{field} is invalid") + return item + + +def _optional_string(value: Mapping[str, Any], field: str) -> str | None: + item = value.get(field) + if item is None: + return None + if not isinstance(item, str) or not item or len(item) > 32_000: + raise WorkerError(f"{field} is invalid") + return item + + +def _optional_float(value: Mapping[str, Any], field: str) -> float | None: + item = value.get(field) + if item is None: + return None + if isinstance(item, bool) or not isinstance(item, (int, float)): + raise WorkerError(f"{field} is invalid") + result = float(item) + if not np.isfinite(result): + raise WorkerError(f"{field} is invalid") + return result + + +def _optional_int(value: Mapping[str, Any], field: str) -> int | None: + item = value.get(field) + if item is None: + return None + if isinstance(item, bool) or not isinstance(item, int): + raise WorkerError(f"{field} is invalid") + return item + + +def read_request(path: pathlib.Path) -> WorkerRequest: + if path.is_symlink() or not path.is_file(): + raise WorkerError("CUDA worker request must be a regular file") + if path.stat().st_size > MAX_JSON_BYTES: + raise WorkerError("CUDA worker request is too large") + try: + parsed = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as error: + raise WorkerError("CUDA worker request is unreadable") from error + if not isinstance(parsed, dict): + raise WorkerError("CUDA worker request must be an object") + return WorkerRequest.from_dict(parsed) + + +def verify_launch_token( + request: WorkerRequest, env: Mapping[str, str] | None = None +) -> None: + environment = os.environ if env is None else env + token = environment.get(LAUNCH_TOKEN_ENV) + if token is None or not 32 <= len(token.encode("utf-8")) <= MAX_LAUNCH_TOKEN_BYTES: + raise WorkerError("CUDA worker launch authorization is missing or invalid") + actual = hashlib.sha256(token.encode("utf-8")).hexdigest() + if not hmac.compare_digest(actual, request.launch_token_sha256): + raise WorkerError("CUDA worker launch authorization does not match the request") + + +def verify_provenance(path: pathlib.Path, request: WorkerRequest) -> dict[str, Any]: + if path.is_symlink() or not path.is_file() or path.stat().st_size > MAX_JSON_BYTES: + raise WorkerError("CUDA provenance must be a bounded regular file") + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as error: + raise WorkerError("CUDA provenance is unreadable") from error + expected = { + "schema_version": 1, + "backend": sa3_cuda.BACKEND_NAME, + "gated_artifacts_complete": True, + "source_revision": sa3_cuda.SOURCE_REVISION, + "runtime_lock_sha256": sa3_cuda.RUNTIME_LOCK_SHA256, + "packages": sa3_cuda.EXPECTED_PACKAGES, + "model": sa3_cuda.MODEL_PINS[request.kind], + } + if value != expected: + raise WorkerError( + "CUDA provenance does not match the immutable source/runtime/model pin" + ) + root = path.parent.resolve(strict=True) + model_dir = request.model_dir.resolve(strict=True) + expected_model_dir = (root / "models" / MODEL_FOR_KIND[request.kind]).resolve() + if model_dir != expected_model_dir: + raise WorkerError("CUDA model path is outside its verified runtime bundle") + return value + + +def _single_safetensors(path: pathlib.Path) -> pathlib.Path: + if path.suffix == ".safetensors" and path.is_file() and not path.is_symlink(): + return path + if path.is_dir() and not path.is_symlink(): + hits = [ + item + for item in path.iterdir() + if item.is_file() + and not item.is_symlink() + and item.suffix == ".safetensors" + ] + if len(hits) == 1: + return hits[0] + raise WorkerError("each LoRA must resolve to exactly one regular safetensors file") + + +def _load_pcm16(path: pathlib.Path) -> Any: + try: + with wave.open(str(path), "rb") as source: + if ( + source.getnchannels() != CHANNELS + or source.getsampwidth() != 2 + or source.getframerate() != SAMPLE_RATE + or source.getcomptype() != "NONE" + ): + raise WorkerError("init audio is not canonical PCM16") + frames = source.getnframes() + raw = source.readframes(frames) + except (EOFError, OSError, wave.Error) as error: + raise WorkerError("init audio is unreadable") from error + if frames < 1 or len(raw) != frames * CHANNELS * 2: + raise WorkerError("init audio is empty or truncated") + return ( + np.frombuffer(raw, dtype=" np.ndarray: + if isinstance(audio, np.ndarray): + return audio + value = audio.detach().to("cpu").float().numpy() + return np.asarray(value) + + +def write_pcm16(path: pathlib.Path, audio: Any, seconds: float) -> None: + samples = _to_numpy(audio) + if samples.ndim == 3 and samples.shape[0] == 1: + samples = samples[0] + if samples.ndim != 2 or samples.shape[0] != CHANNELS: + raise WorkerError(f"upstream returned invalid audio shape {samples.shape!r}") + frames = round(seconds * SAMPLE_RATE) + if samples.shape[1] < frames or not np.isfinite(samples[:, :frames]).all(): + raise WorkerError("upstream returned short or non-finite audio") + clipped = np.clip(samples[:, :frames], -1.0, 1.0) + pcm = np.where(clipped <= -1, -32768, np.rint(clipped * 32767)).astype(" dict[str, Any]: + init = None + if request.init_audio is not None: + waveform = torch_module.from_numpy(_load_pcm16(request.init_audio)) + init = (SAMPLE_RATE, waveform) + kwargs: dict[str, Any] = { + "prompt": request.prompt, + "negative_prompt": request.negative_prompt, + "duration": request.seconds, + "steps": request.steps, + "cfg_scale": 1.0 if request.cfg is None else request.cfg, + "apg_scale": 1.0 if request.apg is None else request.apg, + "seed": -1 if request.seed is None else request.seed, + "batch_size": 1, + "chunked_decode": True, + "callback": lambda info: progress(int(info["i"]) + 1, request.steps), + "disable_tqdm": True, + } + if request.inpaint_range is not None: + kwargs.update( + { + "inpaint_audio": init, + "inpaint_mask_start_seconds": request.inpaint_range[0], + "inpaint_mask_end_seconds": request.inpaint_range[1], + "init_audio": None, + } + ) + else: + kwargs.update( + { + "init_audio": init, + "init_noise_level": ( + 0.9 + if request.init_noise_level is None + else request.init_noise_level + ), + "inpaint_audio": None, + } + ) + return kwargs + + +def run_generation( + request: WorkerRequest, + *, + model: ModelProtocol, + torch_module: Any, + cancelled: Callable[[], bool], + broker: GpuBroker | None = None, + lease: Lease | None = None, + emit: Callable[[dict[str, object]], None] = lambda event: None, +) -> None: + lora_files = [_single_safetensors(path) for path in request.lora_files] + if lora_files: + model.load_lora([str(path) for path in lora_files]) + for index, strength in enumerate(request.lora_strengths): + model.set_lora_strength(strength, lora_index=index) + + def progress(current: int, total: int) -> None: + if cancelled(): + raise WorkerCancelled("Stable Audio generation was cancelled") + if broker is not None and lease is not None and broker.should_yield(lease): + raise WorkerCancelled("Stable Audio yielded to realtime MRT2 generation") + emit( + { + "event": "progress", + "stage": "sampling", + "current": current, + "total": total, + } + ) + + kwargs = generation_kwargs(request, torch_module=torch_module, progress=progress) + audio = model.generate(**kwargs) + emit({"event": "progress", "stage": "decoding", "current": None, "total": None}) + write_pcm16(request.output, audio, request.seconds) + emit({"event": "done"}) + + +def _package_versions() -> dict[str, str]: + versions = {} + for name in sa3_cuda.EXPECTED_PACKAGES: + try: + versions[name] = importlib.metadata.version(name) + except importlib.metadata.PackageNotFoundError: + versions[name] = "missing" + return versions + + +def _nvml_driver_version() -> str | None: + if os.name != "nt": + return None + import ctypes + + library = None + try: + library = ctypes.WinDLL("nvml.dll") + if library.nvmlInit_v2() != 0: + return None + buffer = ctypes.create_string_buffer(96) + if library.nvmlSystemGetDriverVersion(buffer, len(buffer)) != 0: + return None + return buffer.value.decode("ascii", "strict") + except (AttributeError, OSError, UnicodeDecodeError): + return None + finally: + if library is not None: + with contextlib.suppress(Exception): + library.nvmlShutdown() + + +def _load_production_runtime() -> tuple[Any, Callable[[WorkerRequest], ModelProtocol]]: + try: + import torch + from stable_audio_3.loading_utils import load_diffusion_cond + from stable_audio_3.model import StableAudioModel + except ImportError as error: + raise WorkerError( + "the pinned Stable Audio PyTorch dependency is missing" + ) from error + + def load(request: WorkerRequest) -> ModelProtocol: + config = request.model_dir / "model_config.json" + checkpoint = request.model_dir / "model.safetensors" + if any( + path.is_symlink() or not path.is_file() for path in (config, checkpoint) + ): + raise WorkerError("the verified Stable Audio model bundle is incomplete") + try: + model_config = json.loads(config.read_text(encoding="utf-8")) + upstream = load_diffusion_cond( + model_config, str(checkpoint), device="cuda", model_half=True + ) + upstream.use_lora = False + upstream.lora_names = [] + return StableAudioModel(upstream, model_config, "cuda", True) + except Exception as error: + raise WorkerError( + "the pinned Stable Audio model could not initialize" + ) from error + + return torch, load + + +def _emit(event: dict[str, object]) -> None: + sys.stdout.write(json.dumps(event, separators=(",", ":")) + "\n") + sys.stdout.flush() + + +def _job_emitter(job_id: str) -> Callable[[dict[str, object]], None]: + def emit(event: dict[str, object]) -> None: + _emit({"jobId": job_id, **event}) + + return emit + + +def start_broker_watchdog( + broker: GpuBroker, + lease: Lease, + emit: Callable[[dict[str, object]], None], + *, + poll_seconds: float = 0.05, + exit_process: Callable[[int], None] = os._exit, +) -> tuple[threading.Event, threading.Thread]: + """Hard-stop model loading/decoding when realtime MRT2 needs the GPU. + + The sampler callback provides graceful yield during diffusion. Loading and + decoding are upstream calls with no cancellation callback, so a daemon + watchdog terminates only this disposable worker. Process exit is the + reliable CUDA-context/VRAM release boundary. + """ + + stop = threading.Event() + + def watch() -> None: + while not stop.wait(poll_seconds): + try: + should_yield = broker.should_yield(lease) + except Exception: + should_yield = True + if should_yield: + emit( + { + "event": "cancelled", + "message": "Stable Audio yielded to realtime MRT2 generation", + } + ) + exit_process(2) + return + + thread = threading.Thread(target=watch, name="sa3-gpu-yield", daemon=True) + thread.start() + return stop, thread + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="LSDJ disposable SA3 CUDA worker") + parser.add_argument("--request", required=True) + parser.add_argument("--cancel-file", required=True) + parser.add_argument("--broker-root", required=True) + parser.add_argument("--provenance", required=True) + parser.add_argument("--reservation-bytes", required=True, type=int) + args = parser.parse_args(argv) + request: WorkerRequest | None = None + emit = _emit + model: ModelProtocol | None = None + torch: Any | None = None + try: + request = read_request(pathlib.Path(args.request)) + emit = _job_emitter(request.job_id) + verify_launch_token(request) + # The token authenticates this one request/child pairing. Upstream + # imports and any grandchildren must never inherit it. + os.environ.pop(LAUNCH_TOKEN_ENV, None) + provenance = verify_provenance(pathlib.Path(args.provenance), request) + cancel_file = pathlib.Path(args.cancel_file) + torch, load_model = _load_production_runtime() + versions = _package_versions() + if not torch.cuda.is_available(): + raise WorkerError( + "PyTorch reports no CUDA device; there is no PyTorch CPU fallback" + ) + free_bytes, total_bytes = torch.cuda.mem_get_info() + properties = torch.cuda.get_device_properties(torch.cuda.current_device()) + evidence = sa3_cuda.CudaEvidence( + platform=sys.platform, + machine=os.environ.get("PROCESSOR_ARCHITECTURE") or host_platform.machine(), + runtime_ready=True, + provenance_complete=True, + packages=versions, + cuda_available=torch.cuda.is_available(), + cuda_runtime=torch.version.cuda, + driver=_nvml_driver_version(), + device=properties.name, + compute_capability=tuple(torch.cuda.get_device_capability()), + total_vram_bytes=int(total_bytes), + free_vram_bytes=int(free_bytes), + estimated_vram_bytes={request.kind: args.reservation_bytes}, + source_revision=provenance["source_revision"], + model_revision=provenance["model"]["revision"], + ) + errors = sa3_cuda.runtime_errors(evidence, kind=request.kind) + if errors: + raise WorkerError("; ".join(errors)) + broker = GpuBroker(pathlib.Path(args.broker_root)) + capacity = max(0, int(free_bytes) - sa3_cuda.VRAM_HEADROOM_BYTES) + with broker.hold( + "sa3", + priority=Priority.SA3_BACKGROUND, + reservation_bytes=args.reservation_bytes, + capacity_bytes=capacity, + timeout_seconds=120, + cancelled=cancel_file.exists, + ) as lease: + emit( + { + "event": "progress", + "stage": "loading", + "current": None, + "total": None, + } + ) + watchdog_stop, watchdog = start_broker_watchdog(broker, lease, emit) + try: + model = load_model(request) + run_generation( + request, + model=model, + torch_module=torch, + cancelled=cancel_file.exists, + broker=broker, + lease=lease, + emit=emit, + ) + finally: + watchdog_stop.set() + watchdog.join(timeout=1) + return 0 + except WorkerCancelled as error: + emit({"event": "cancelled", "message": str(error)}) + return 2 + except Exception as error: + # Only our bounded, path-free errors cross the worker boundary. Unknown + # upstream/OS errors are intentionally reduced to their class name so a + # prompt, token, or app-owned filesystem path cannot leak into logs. + message = ( + str(error)[:512] + if isinstance(error, WorkerError) + else f"CUDA worker failed ({type(error).__name__})" + ) + emit({"event": "error", "message": message}) + return 1 + finally: + model = None + if torch is not None: + with contextlib.suppress(Exception): + torch.cuda.empty_cache() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/backend/lsdj/sidecar.py b/backend/lsdj/sidecar.py index 23f760e..b7def7b 100644 --- a/backend/lsdj/sidecar.py +++ b/backend/lsdj/sidecar.py @@ -1,10 +1,10 @@ -"""Per-deck inference sidecar (Phase 2 part 4, ADR-0019). +"""Native inference sidecar transport (ADR-0019 and ADR-0037). -The Rust shell (`src-tauri/src/sidecar.rs`) spawns this process per deck and -accepts its loopback-TCP connection. It runs the UNCHANGED `run_deck_worker` -generation loop (`worker.py`) with its `cmd_queue` / `out_queue` bridged to the -socket — so the inference loop is identical to the multiprocessing path; only the -transport differs. +The Rust shell (`src-tauri/src/sidecar.rs`) spawns one process per MLX deck on +macOS, or one shared PyTorch/CUDA process holding two independent deck states on +Linux and Windows. It accepts a loopback-TCP connection and runs the unchanged +`run_deck_worker` generation loop (`worker.py`) with its `cmd_queue` / `out_queue` +bridged to the socket. Only transport and process topology differ. Wire protocol (mirrors `src-tauri/src/sidecar.rs`): @@ -16,21 +16,44 @@ - CONTROL (engine → sidecar): a deck command (``play``/``stop``/``set_style``…) as UTF-8 JSON. +The dedicated MRT2 clip renderer uses the same authenticated connection but a +separate strict protocol: serial JSON RENDER_REQUEST messages with authoritative +integer frame counts and monotonically increasing sequence numbers (or a cancel +matching the active job/sequence), then exact RENDER_BEGIN metadata, bounded +aligned RENDER_CHUNK frames, and a RENDER_END carrying the exact identity, byte +count, and SHA-256. It never accepts deck control frames or unbounded PCM. + +Shared-worker frames prefix payloads with a single deck byte (0 or 1). The Rust +and Python transport tests cover both forms without loading either model stack. + The transport (framing + the queue adapters) is testable against a socketpair with a fake engine — no model, no Rust; see `tests/test_sidecar.py`. The model-loaded round-trip is a native-checklist item. """ import argparse +import hashlib import json +import math +import os import queue import re import socket import struct import sys import threading +import time +from dataclasses import dataclass +from typing import Any, BinaryIO, Callable, Mapping, MutableMapping -from .engine import DeckEngine +from .mrt2 import ( + AUTO_RUNTIME, + PYTORCH_CUDA_RUNTIME, + RUNTIME_CHOICES, + create_engine, + public_startup_error, + runtime_manifest, +) from .worker import run_deck_worker FRAME_PCM = 1 @@ -39,14 +62,120 @@ # Engine → sidecar: a style-sample embed (M15). Binary, not JSON, because it # carries raw PCM: [u32 LE id length][id utf-8][interleaved f32 LE PCM]. FRAME_EMBED = 4 +# First sidecar -> host frame. The per-launch token is delivered only through the +# child's scrubbed environment and proves that the connector is the process Rust +# just spawned, rather than another local process racing the loopback accept. +FRAME_AUTH = 5 +# Native host -> dedicated MRT2 render worker. The JSON payload is a single +# strict request; render workers never accept deck-control frames. +FRAME_RENDER_REQUEST = 6 +# Render worker -> native host. BEGIN fixes the expected audio identity and byte +# count before any payload; CHUNK carries aligned f32le PCM; END authenticates +# the completed byte stream with its exact count and SHA-256. +FRAME_RENDER_BEGIN = 7 +FRAME_RENDER_CHUNK = 8 +FRAME_RENDER_END = 9 +# Native host -> render worker. In-flight model calls are not cooperatively +# cancellable, so this frame makes the disposable worker exit and release CUDA. +FRAME_RENDER_CANCEL = 10 +# Render worker -> native host. Diagnostics are bounded and path-free. +FRAME_RENDER_ERROR = 11 + +MAX_FRAME_BYTES = 16 * 1024 * 1024 +MAX_EMBED_ID_BYTES = 4 * 1024 +WORKER_TOKEN_ENV = "LSDJ_WORKER_LAUNCH_TOKEN" + +RENDER_SCHEMA_VERSION = 1 +RENDER_SAMPLE_RATE = 48_000 +RENDER_CHANNELS = 2 +RENDER_SAMPLE_WIDTH = 4 +RENDER_BYTES_PER_FRAME = RENDER_CHANNELS * RENDER_SAMPLE_WIDTH +MIN_RENDER_SECONDS = 0.5 +MAX_RENDER_SECONDS = 180.0 +MIN_RENDER_FRAMES = 24_000 +MAX_RENDER_FRAMES = 8_640_000 +MAX_RENDER_PROMPT_CHARS = 32_000 +MAX_RENDER_REQUEST_BYTES = 64 * 1024 +MAX_RENDER_CONTROL_BYTES = 1024 +MAX_RENDER_PCM_BYTES = MAX_RENDER_FRAMES * RENDER_BYTES_PER_FRAME +RENDER_PCM_CHUNK_BYTES = 1024 * 1024 +MAX_RENDER_METADATA_BYTES = 8 * 1024 +RENDER_WRITE_POLL_SECONDS = 0.01 +RENDER_WRITE_TIMEOUT_SECONDS = 5.0 +MAX_U64 = (1 << 64) - 1 +_RENDER_JOB_ID = re.compile(r"^[A-Za-z0-9_-]{16,80}$") # u8 frame type, u32 little-endian payload length. _HEADER = struct.Struct(" float: + # Frames are authoritative on the wire. Seconds exist only at the + # upstream engine boundary, so Python never independently rounds the + # user's duration. + return self.frames / RENDER_SAMPLE_RATE + + @property + def pcm_bytes(self) -> int: + return self.frames * RENDER_BYTES_PER_FRAME + + +@dataclass(frozen=True) +class RenderCancel: + job_id: str + sequence: int + + +@dataclass(frozen=True) +class _RenderReaderFailure: + message: str + + +_RenderCommand = RenderRequest | RenderCancel | _RenderReaderFailure | None + + +class _RenderWorkerStopped(Exception): + """Internal control-flow marker after the disposable worker is stopped.""" + + +class _RenderWriteError(Exception): + """A response frame could not be committed within the bounded deadline.""" def write_frame(sock: socket.socket, frame_type: int, payload: bytes) -> None: """Send one framed message. `sendall` is atomic enough here: the worker loop is the only writer, so frames never interleave.""" + if frame_type not in _FRAME_TYPES: + raise ValueError(f"unknown sidecar frame type {frame_type}") + if len(payload) > MAX_FRAME_BYTES: + raise ValueError(f"sidecar frame length {len(payload)} exceeds the cap") sock.sendall(_HEADER.pack(frame_type, len(payload)) + payload) @@ -57,12 +186,414 @@ def read_frame(reader) -> tuple[int, bytes] | None: if len(head) < _HEADER.size: return None frame_type, length = _HEADER.unpack(head) + if length > MAX_FRAME_BYTES: + raise ValueError(f"sidecar frame length {length} exceeds the cap") payload = reader.read(length) if len(payload) < length: return None return frame_type, payload +def authenticate_to_host( + sock: socket.socket, env: MutableMapping[str, str] | None = None +) -> None: + """Consume and send the per-child capability before any worker traffic. + + The token is removed before the write, including on failure, so a child can + never reconnect or retry with the same capability. The native listener must + accept it as the exact first frame, compare it in constant time, and consume + its expected token after that one connection attempt. + """ + + env = os.environ if env is None else env + token = env.pop(WORKER_TOKEN_ENV, "") + if not 32 <= len(token) <= 256 or not token.isascii(): + raise RuntimeError("the authenticated sidecar launch token is missing") + write_frame(sock, FRAME_AUTH, token.encode("ascii")) + + +def _strict_json_object(payload: bytes) -> dict[str, object]: + def pairs(values: list[tuple[str, object]]) -> dict[str, object]: + result: dict[str, object] = {} + for name, value in values: + if name in result: + raise RenderProtocolError("render JSON contains a duplicate field") + result[name] = value + return result + + try: + value = json.loads(payload.decode("utf-8"), object_pairs_hook=pairs) + except RenderProtocolError: + raise + except ( + UnicodeDecodeError, + json.JSONDecodeError, + ValueError, + RecursionError, + OverflowError, + ): + raise RenderProtocolError("render JSON is invalid") from None + if not isinstance(value, dict): + raise RenderProtocolError("render JSON must be an object") + return value + + +def _read_bounded_render_frame( + reader: BinaryIO, limits: Mapping[int, int] +) -> tuple[int, bytes] | None: + head = reader.read(_HEADER.size) + if not head: + return None + if len(head) != _HEADER.size: + raise RenderProtocolError("render frame header is truncated") + frame_type, length = _HEADER.unpack(head) + limit = limits.get(frame_type) + if limit is None: + raise RenderProtocolError(f"render frame type {frame_type} is out of order") + if length > limit: + raise RenderProtocolError( + f"render frame type {frame_type} exceeds its {limit}-byte cap" + ) + payload = reader.read(length) + if len(payload) != length: + raise RenderProtocolError("render frame payload is truncated") + return frame_type, payload + + +def _validate_job_id(value: object) -> str: + if not isinstance(value, str) or _RENDER_JOB_ID.fullmatch(value) is None: + raise RenderProtocolError("render jobId is invalid") + return value + + +def _validate_exact_int(value: object, *, name: str, minimum: int, maximum: int) -> int: + if type(value) is not int or not minimum <= value <= maximum: + raise RenderProtocolError(f"render {name} is invalid") + return value + + +def render_frames_for_seconds(seconds: float) -> int: + """Reference conversion for the native gateway's user-facing duration. + + The gateway performs this once, before sending an integer frame count: + ``floor(seconds_f64 * 48000 + 0.5)``. This deliberately avoids Python's + ties-to-even ``round`` behavior at half-frame boundaries. + """ + + if ( + isinstance(seconds, bool) + or not isinstance(seconds, (int, float)) + or not math.isfinite(seconds) + or not MIN_RENDER_SECONDS <= float(seconds) <= MAX_RENDER_SECONDS + ): + raise RenderProtocolError( + f"render seconds must be {MIN_RENDER_SECONDS:g}-{MAX_RENDER_SECONDS:g}" + ) + frames = math.floor(float(seconds) * RENDER_SAMPLE_RATE + 0.5) + return _validate_exact_int( + frames, + name="frames", + minimum=MIN_RENDER_FRAMES, + maximum=MAX_RENDER_FRAMES, + ) + + +def read_render_command(reader: BinaryIO) -> RenderRequest | RenderCancel | None: + """Read one strict host command, distinguishing clean EOF from truncation.""" + + frame = _read_bounded_render_frame( + reader, + { + FRAME_RENDER_REQUEST: MAX_RENDER_REQUEST_BYTES, + FRAME_RENDER_CANCEL: MAX_RENDER_CONTROL_BYTES, + }, + ) + if frame is None: + return None + frame_type, payload = frame + value = _strict_json_object(payload) + schema_version = value.get("schemaVersion") + if type(schema_version) is not int or schema_version != RENDER_SCHEMA_VERSION: + raise RenderProtocolError("render command schema is unsupported") + job_id = _validate_job_id(value.get("jobId")) + sequence = _validate_exact_int( + value.get("sequence"), name="sequence", minimum=1, maximum=MAX_U64 + ) + if frame_type == FRAME_RENDER_CANCEL: + if set(value) != {"schemaVersion", "jobId", "sequence"}: + raise RenderProtocolError("render cancel contains unknown fields") + return RenderCancel(job_id=job_id, sequence=sequence) + + if set(value) != {"schemaVersion", "jobId", "sequence", "prompt", "frames"}: + raise RenderProtocolError("render request contains missing or unknown fields") + prompt = value.get("prompt") + if not isinstance(prompt, str) or not prompt.strip(): + raise RenderProtocolError("render prompt must be a non-empty string") + prompt = prompt.strip() + if len(prompt) > MAX_RENDER_PROMPT_CHARS: + raise RenderProtocolError("render prompt exceeds its character cap") + frames = _validate_exact_int( + value.get("frames"), + name="frames", + minimum=MIN_RENDER_FRAMES, + maximum=MAX_RENDER_FRAMES, + ) + return RenderRequest( + job_id=job_id, + sequence=sequence, + prompt=prompt, + frames=frames, + ) + + +def _render_json(value: Mapping[str, object]) -> bytes: + payload = json.dumps(value, sort_keys=True, separators=(",", ":")).encode() + if len(payload) > MAX_RENDER_METADATA_BYTES: + raise RenderProtocolError("render metadata exceeds its cap") + return payload + + +def write_render_error( + sock: socket.socket, + *, + job_id: str | None, + sequence: int, + code: str, + message: str, + send_frame: Callable[[int, bytes], None] | None = None, +) -> None: + sequence = _validate_exact_int( + sequence, name="sequence", minimum=0, maximum=MAX_U64 + ) + if job_id is not None: + _validate_job_id(job_id) + sender = ( + (lambda frame_type, payload: write_frame(sock, frame_type, payload)) + if send_frame is None + else send_frame + ) + sender( + FRAME_RENDER_ERROR, + _render_json( + { + "schemaVersion": RENDER_SCHEMA_VERSION, + "jobId": job_id, + "sequence": sequence, + "code": code[:64], + "message": message[:512], + } + ), + ) + + +def write_render_response( + sock: socket, + request: RenderRequest, + pcm: bytes, + *, + send_frame: Callable[[int, bytes], None] | None = None, + before_frame: Callable[[], None] | None = None, +) -> None: + """Write one complete, size-checked f32le render response.""" + + _validate_job_id(request.job_id) + _validate_exact_int(request.sequence, name="sequence", minimum=1, maximum=MAX_U64) + _validate_exact_int( + request.frames, + name="frames", + minimum=MIN_RENDER_FRAMES, + maximum=MAX_RENDER_FRAMES, + ) + if not isinstance(pcm, bytes): + raise RenderProtocolError("render engine returned a non-bytes payload") + if len(pcm) != request.pcm_bytes or len(pcm) > MAX_RENDER_PCM_BYTES: + raise RenderProtocolError( + f"render engine returned {len(pcm)} PCM bytes; expected {request.pcm_bytes}" + ) + sender = ( + (lambda frame_type, payload: write_frame(sock, frame_type, payload)) + if send_frame is None + else send_frame + ) + check = (lambda: None) if before_frame is None else before_frame + check() + sender( + FRAME_RENDER_BEGIN, + _render_json( + { + "schemaVersion": RENDER_SCHEMA_VERSION, + "jobId": request.job_id, + "sequence": request.sequence, + "sampleRate": RENDER_SAMPLE_RATE, + "channels": RENDER_CHANNELS, + "sampleFormat": "f32le", + "frames": request.frames, + "pcmBytes": request.pcm_bytes, + } + ), + ) + digest = hashlib.sha256() + for start in range(0, len(pcm), RENDER_PCM_CHUNK_BYTES): + check() + chunk = pcm[start : start + RENDER_PCM_CHUNK_BYTES] + digest.update(chunk) + sender(FRAME_RENDER_CHUNK, chunk) + check() + sender( + FRAME_RENDER_END, + _render_json( + { + "schemaVersion": RENDER_SCHEMA_VERSION, + "jobId": request.job_id, + "sequence": request.sequence, + "frames": request.frames, + "pcmBytes": request.pcm_bytes, + "sha256": digest.hexdigest(), + } + ), + ) + + +def _validate_render_error(value: Mapping[str, object], request: RenderRequest) -> str: + if ( + set(value) != {"schemaVersion", "jobId", "sequence", "code", "message"} + or type(value.get("schemaVersion")) is not int + or value.get("schemaVersion") != RENDER_SCHEMA_VERSION + or value.get("jobId") != request.job_id + or type(value.get("sequence")) is not int + or value.get("sequence") != request.sequence + or type(value.get("code")) is not str + or not 1 <= len(value["code"]) <= 64 + or type(value.get("message")) is not str + or len(value["message"]) > 512 + ): + raise RenderProtocolError("render error metadata is invalid") + return value["code"] + + +def read_render_response( + reader: BinaryIO, request: RenderRequest, *, require_eof: bool = False +) -> bytes: + """Validate and assemble one response; the native host mirrors this parser.""" + + _validate_job_id(request.job_id) + _validate_exact_int(request.sequence, name="sequence", minimum=1, maximum=MAX_U64) + _validate_exact_int( + request.frames, + name="frames", + minimum=MIN_RENDER_FRAMES, + maximum=MAX_RENDER_FRAMES, + ) + first = _read_bounded_render_frame( + reader, + { + FRAME_RENDER_BEGIN: MAX_RENDER_METADATA_BYTES, + FRAME_RENDER_ERROR: MAX_RENDER_METADATA_BYTES, + }, + ) + if first is None: + raise RenderProtocolError("render response ended before begin") + frame_type, payload = first + value = _strict_json_object(payload) + if frame_type == FRAME_RENDER_ERROR: + code = _validate_render_error(value, request) + raise RenderProtocolError(f"render worker returned {code}") + expected_fields = { + "schemaVersion", + "jobId", + "sequence", + "sampleRate", + "channels", + "sampleFormat", + "frames", + "pcmBytes", + } + if ( + set(value) != expected_fields + or type(value.get("schemaVersion")) is not int + or value.get("schemaVersion") != RENDER_SCHEMA_VERSION + ): + raise RenderProtocolError("render begin metadata is invalid") + if ( + value.get("jobId") != request.job_id + or type(value.get("sequence")) is not int + or value.get("sequence") != request.sequence + ): + raise RenderProtocolError("render response identity is out of turn") + frames = value.get("frames") + pcm_bytes = value.get("pcmBytes") + if ( + type(value.get("sampleRate")) is not int + or value.get("sampleRate") != RENDER_SAMPLE_RATE + or type(value.get("channels")) is not int + or value.get("channels") != RENDER_CHANNELS + or value.get("sampleFormat") != "f32le" + or type(frames) is not int + or frames != request.frames + or type(pcm_bytes) is not int + or pcm_bytes != request.pcm_bytes + ): + raise RenderProtocolError("render begin audio identity is invalid") + + output = bytearray() + digest = hashlib.sha256() + while True: + frame = _read_bounded_render_frame( + reader, + { + FRAME_RENDER_CHUNK: RENDER_PCM_CHUNK_BYTES, + FRAME_RENDER_END: MAX_RENDER_METADATA_BYTES, + FRAME_RENDER_ERROR: MAX_RENDER_METADATA_BYTES, + }, + ) + if frame is None: + raise RenderProtocolError("render response is truncated") + frame_type, payload = frame + if frame_type == FRAME_RENDER_CHUNK: + if not payload or len(payload) % RENDER_BYTES_PER_FRAME: + raise RenderProtocolError("render PCM chunk is empty or misaligned") + if len(output) + len(payload) > pcm_bytes: + raise RenderProtocolError("render response contains extra PCM bytes") + output.extend(payload) + digest.update(payload) + continue + + if frame_type == FRAME_RENDER_ERROR: + code = _validate_render_error(_strict_json_object(payload), request) + raise RenderProtocolError(f"render worker returned {code}") + + end = _strict_json_object(payload) + if ( + set(end) + != { + "schemaVersion", + "jobId", + "sequence", + "frames", + "pcmBytes", + "sha256", + } + or type(end.get("schemaVersion")) is not int + or end.get("schemaVersion") != RENDER_SCHEMA_VERSION + or end.get("jobId") != request.job_id + or type(end.get("sequence")) is not int + or end.get("sequence") != request.sequence + or type(end.get("frames")) is not int + or end.get("frames") != request.frames + or type(end.get("pcmBytes")) is not int + or end.get("pcmBytes") != pcm_bytes + or type(end.get("sha256")) is not str + or end.get("sha256") != digest.hexdigest() + or len(output) != pcm_bytes + ): + raise RenderProtocolError( + "render end metadata or exact byte total is invalid" + ) + if require_eof and reader.read(1): + raise RenderProtocolError("render response contains frames after end") + return bytes(output) + + class SocketOutQueue: """`run_deck_worker`'s `out_queue`, writing to the socket: ``('audio', bytes)`` → a PCM frame, ``('status', dict)`` → a status frame.""" @@ -96,7 +627,10 @@ def __init__(self, reader) -> None: def _pump(self) -> None: while True: - frame = read_frame(self._reader) + try: + frame = read_frame(self._reader) + except (OSError, ValueError): + frame = None if frame is None: self._queue.put({"type": "shutdown"}) return @@ -107,7 +641,14 @@ def _pump(self) -> None: if len(payload) < 4: continue id_len = int.from_bytes(payload[:4], "little") - sample_id = payload[4 : 4 + id_len].decode("utf-8", "replace") + if id_len > MAX_EMBED_ID_BYTES or id_len > len(payload) - 4: + continue + try: + sample_id = payload[4 : 4 + id_len].decode("utf-8") + except UnicodeDecodeError: + continue + if not sample_id: + continue pcm = bytes(payload[4 + id_len :]) self._queue.put({"type": "embed_sample", "id": sample_id, "pcm": pcm}) continue @@ -127,17 +668,495 @@ def get_nowait(self): return self._queue.get_nowait() +class SharedSocketOutQueue: + """Multiplex one worker process's per-deck output onto one socket.""" + + def __init__(self, sock: socket.socket, deck: int, lock: threading.Lock) -> None: + self._sock = sock + self._deck = deck + self._lock = lock + + def put(self, item: tuple[str, object]) -> None: + kind, payload = item + if kind == "audio": + frame_type = FRAME_PCM + encoded = payload + elif kind == "status": + frame_type = FRAME_STATUS + encoded = json.dumps(payload).encode("utf-8") + else: + return + with self._lock: + write_frame(self._sock, frame_type, bytes([self._deck]) + encoded) + + +class SharedSocketCmdQueues: + """Demultiplex deck-prefixed control/embed frames for a shared worker.""" + + def __init__(self, reader, deck_count: int = 2) -> None: + self.queues = [queue.Queue() for _ in range(deck_count)] + self._reader = reader + self._thread = threading.Thread(target=self._pump, daemon=True) + self._thread.start() + + def _pump(self) -> None: + while True: + try: + frame = read_frame(self._reader) + except (OSError, ValueError): + frame = None + if frame is None: + for target in self.queues: + target.put({"type": "shutdown"}) + return + frame_type, payload = frame + if not payload or payload[0] >= len(self.queues): + continue + target = self.queues[payload[0]] + body = payload[1:] + if frame_type == FRAME_EMBED: + if len(body) < 4: + continue + id_len = int.from_bytes(body[:4], "little") + if id_len > MAX_EMBED_ID_BYTES or id_len > len(body) - 4: + continue + try: + sample_id = body[4 : 4 + id_len].decode("utf-8") + except UnicodeDecodeError: + continue + if not sample_id: + continue + target.put( + { + "type": "embed_sample", + "id": sample_id, + "pcm": bytes(body[4 + id_len :]), + } + ) + continue + if frame_type != FRAME_CONTROL: + continue + try: + command = json.loads(body) + except json.JSONDecodeError: + continue + if isinstance(command, dict) and "type" in command: + target.put(command) + + def run_sidecar( - sock: socket.socket, deck_id: str, model: str, engine_factory=DeckEngine + sock: socket.socket, + deck_id: str, + model: str, + *, + runtime: str = AUTO_RUNTIME, + engine_factory=None, ) -> None: """Bridge `sock` to `run_deck_worker` for `deck_id` and run the generation loop until the socket closes. `engine_factory` is injectable for tests.""" reader = sock.makefile("rb") cmd_queue = SocketCmdQueue(reader) out_queue = SocketOutQueue(sock) + if engine_factory is None: + + def engine_factory(*, model): + return create_engine(model=model, runtime=runtime) + run_deck_worker(deck_id, model, cmd_queue, out_queue, engine_factory=engine_factory) +def run_shared_sidecar( + sock: socket.socket, + models: tuple[str, str], + *, + runtime: str, + engine_factory=None, +) -> None: + """Run both decks in one process, sharing a model when their pins match.""" + + reader = sock.makefile("rb") + commands = SharedSocketCmdQueues(reader) + send_lock = threading.Lock() + outputs = [SharedSocketOutQueue(sock, deck, send_lock) for deck in range(2)] + for deck, model in enumerate(models): + outputs[deck].put( + ( + "status", + {"event": "warming", "deck": "ab"[deck], "model": model}, + ) + ) + try: + if engine_factory is not None: + engines = [engine_factory(model=model) for model in models] + elif models[0] == models[1]: + primary = create_engine(model=models[0], runtime=runtime) + shared_deck = getattr(primary, "shared_deck", None) + if not callable(shared_deck): + raise RuntimeError( + f"runtime {runtime!r} does not implement shared two-state topology" + ) + engines = [primary, shared_deck()] + else: + # Preserve independent per-deck model selection. Two different + # models share one supervised process but necessarily load twice. + engines = [create_engine(model=model, runtime=runtime) for model in models] + + # Warm each distinct loaded model once before either deck can report + # readiness. The shared clone marks itself as a non-owner/no-op warmup. + for engine in engines: + warm_up = getattr(engine, "warm_up", None) + if callable(warm_up): + warm_up() + if hasattr(engine, "_warmup_owner"): + engine._warmup_owner = False + except Exception as error: + for deck, model in enumerate(models): + outputs[deck].put( + ( + "status", + { + "event": "startup_failed", + "deck": "ab"[deck], + "model": model, + "error": public_startup_error(error), + }, + ) + ) + return + + threads = [] + for deck, (model, engine) in enumerate(zip(models, engines, strict=True)): + thread = threading.Thread( + target=run_deck_worker, + args=("ab"[deck], model, commands.queues[deck], outputs[deck]), + kwargs={ + "engine_factory": lambda *, model, value=engine: value, + "perform_warmup": False, + }, + name=f"mrt2-deck-{'ab'[deck]}", + ) + thread.start() + threads.append(thread) + for thread in threads: + thread.join() + + +class _RenderCommandReader: + """Continuously read the control half so cancellation can preempt rendering.""" + + def __init__(self, reader: BinaryIO) -> None: + # Socket backpressure bounds requests that arrive faster than the single + # render slot can consume them; no unbounded JSON queue exists. + self.commands: queue.Queue[_RenderCommand] = queue.Queue(maxsize=1) + self._reader = reader + self._thread = threading.Thread( + target=self._pump, name="mrt2-render-control", daemon=True + ) + self._thread.start() + + def _pump(self) -> None: + while True: + try: + command = read_render_command(self._reader) + except RenderProtocolError as error: + message = str(error) + self.commands.put(_RenderReaderFailure(message[:512])) + return + except Exception: # noqa: BLE001 - never expose unexpected details + message = "render control connection failed" + self.commands.put(_RenderReaderFailure(message[:512])) + return + self.commands.put(command) + if command is None: + return + + +def _render_startup_error(error: Exception) -> str: + # `public_startup_error` preserves deliberately bounded RuntimeUnavailable + # diagnostics and collapses unknown exceptions to their class only. + return public_startup_error(error)[:512] + + +def _shutdown_render_socket(sock: socket.socket) -> None: + try: + sock.shutdown(socket.SHUT_RDWR) + except (AttributeError, OSError): + pass + + +def _write_render_frame_bounded( + sock: socket.socket, + frame_type: int, + payload: bytes, + *, + poll: Callable[[], None] | None = None, + timeout: float = RENDER_WRITE_TIMEOUT_SECONDS, +) -> None: + """Commit a frame without blocking the cancellation/control loop. + + ``sendall`` runs in a daemon because Python cannot portably make a socket's + send side nonblocking without also disturbing the concurrent buffered read + side. The caller remains live, polls control every 10 ms, and enforces one + absolute write deadline. Closing the socket before process termination + prevents a blocked writer from emitting late output in injected tests too. + """ + + result: queue.Queue[Exception | None] = queue.Queue(maxsize=1) + + def send() -> None: + try: + write_frame(sock, frame_type, payload) + except Exception as error: # noqa: BLE001 - sanitized below + result.put(error) + else: + result.put(None) + + if poll is not None: + poll() + threading.Thread( + target=send, + name=f"mrt2-render-write-{frame_type}", + daemon=True, + ).start() + deadline = time.monotonic() + timeout + while True: + if poll is not None: + poll() + remaining = deadline - time.monotonic() + if remaining <= 0: + _shutdown_render_socket(sock) + raise _RenderWriteError("render connection write timed out") + try: + error = result.get(timeout=min(RENDER_WRITE_POLL_SECONDS, remaining)) + except queue.Empty: + continue + if error is not None: + raise _RenderWriteError("render connection write failed") from None + return + + +def run_render_worker( + sock: socket.socket, + model: str, + *, + runtime: str = PYTORCH_CUDA_RUNTIME, + engine_factory=None, + terminate: Callable[[int], Any] = os._exit, +) -> None: + """Serve serial, authenticated MRT2 clip renders over one bounded socket. + + Authentication is emitted by :func:`main` before this function runs. The + loaded model remains warm across successful requests. A render call cannot + be interrupted safely inside upstream PyTorch, so cancellation or EOF while + it is active terminates this disposable process and releases its CUDA + context; the native supervisor may start a fresh worker for the next job. + + The supervisor owns an outer startup/job deadline and must kill and reap the + full child tree on cancellation, disconnect, deadline, or owner drop, + including before ``render_ready``. A child has one connection and one + consumed auth token; it never reconnects or retries a sequence. + """ + + try: + engine = ( + create_engine(model=model, runtime=runtime) + if engine_factory is None + else engine_factory(model=model) + ) + warm_up = getattr(engine, "warm_up", None) + if callable(warm_up): + warm_up() + except Exception as error: + try: + write_render_error( + sock, + job_id=None, + sequence=0, + code="startup_failed", + message=_render_startup_error(error), + send_frame=lambda frame_type, payload: _write_render_frame_bounded( + sock, frame_type, payload, timeout=1.0 + ), + ) + except _RenderWriteError: + pass + return + + try: + _write_render_frame_bounded( + sock, + FRAME_STATUS, + _render_json( + { + "schemaVersion": RENDER_SCHEMA_VERSION, + "event": "render_ready", + "model": model, + "runtime": runtime, + "nextSequence": 1, + } + ), + ) + except _RenderWriteError: + return + commands = _RenderCommandReader(sock.makefile("rb")) + expected_sequence: int | None = 1 + + def stop(code: int) -> None: + _shutdown_render_socket(sock) + terminate(code) + raise _RenderWorkerStopped + + def send_error( + *, job_id: str | None, sequence: int, code: str, message: str + ) -> None: + try: + write_render_error( + sock, + job_id=job_id, + sequence=sequence, + code=code, + message=message, + send_frame=lambda frame_type, payload: _write_render_frame_bounded( + sock, frame_type, payload, timeout=1.0 + ), + ) + except _RenderWriteError: + pass + + try: + while True: + command = commands.commands.get() + if command is None: + return + if isinstance(command, _RenderReaderFailure): + send_error( + job_id=None, + sequence=expected_sequence or MAX_U64, + code="protocol_error", + message=command.message, + ) + stop(2) + if isinstance(command, RenderCancel): + send_error( + job_id=command.job_id, + sequence=command.sequence, + code="no_active_job", + message="render job is not active", + ) + stop(2) + if expected_sequence is None or command.sequence != expected_sequence: + send_error( + job_id=command.job_id, + sequence=command.sequence, + code="sequence_error", + message="render sequence is duplicate or out of order", + ) + stop(2) + expected_sequence = ( + None if command.sequence == MAX_U64 else command.sequence + 1 + ) + + result: queue.Queue[tuple[bool, bytes | Exception]] = queue.Queue(maxsize=1) + + def render() -> None: + try: + result.put( + (True, engine.render_clip(command.prompt, command.seconds)) + ) + except Exception as error: # noqa: BLE001 - boundary collapse + result.put((False, error)) + + threading.Thread( + target=render, + name=f"mrt2-render-{command.job_id[:16]}", + daemon=True, + ).start() + + def poll_active(*, can_reply: bool = True) -> None: + try: + pending = commands.commands.get_nowait() + except queue.Empty: + return + + if pending is None: + stop(0) + if ( + isinstance(pending, RenderCancel) + and pending.job_id == command.job_id + and pending.sequence == command.sequence + ): + if can_reply: + send_error( + job_id=command.job_id, + sequence=command.sequence, + code="cancelled", + message="render job was cancelled", + ) + stop(2) + message = ( + pending.message + if isinstance(pending, _RenderReaderFailure) + else "render command is overlapping or out of turn" + ) + if can_reply: + send_error( + job_id=command.job_id, + sequence=command.sequence, + code="protocol_error", + message=message, + ) + stop(2) + + # Control always gets first look. Once the result arrives, poll + # again before BEGIN so a cancel already queued behind it wins. + while True: + poll_active() + try: + succeeded, value = result.get(timeout=RENDER_WRITE_POLL_SECONDS) + break + except queue.Empty: + continue + poll_active() + + if not succeeded: + send_error( + job_id=command.job_id, + sequence=command.sequence, + code="render_failed", + message="MRT2 render failed; the worker must be restarted", + ) + return + try: + if not isinstance(value, bytes): + raise RenderProtocolError( + "render engine returned a non-bytes payload" + ) + write_render_response( + sock, + command, + value, + before_frame=poll_active, + send_frame=lambda frame_type, payload: _write_render_frame_bounded( + sock, + frame_type, + payload, + poll=lambda: poll_active(can_reply=False), + ), + ) + except RenderProtocolError: + send_error( + job_id=command.job_id, + sequence=command.sequence, + code="invalid_audio", + message="MRT2 render returned an invalid PCM payload", + ) + return + except _RenderWriteError: + stop(2) + except _RenderWorkerStopped: + return + + # --- Model tooling (the in-app model manager, issue #43) ------------------- # # The Rust shell spawns this same binary to install Magenta assets without a @@ -234,6 +1253,23 @@ def main(argv=None) -> None: # model-tooling modes below (issue #43) without a deck/port. parser.add_argument("--deck", help="deck id (e.g. a or b)") parser.add_argument("--model", help="model name (e.g. mrt2_small)") + modes = parser.add_mutually_exclusive_group() + modes.add_argument( + "--shared", action="store_true", help="run both decks in one worker" + ) + modes.add_argument( + "--render-worker", + action="store_true", + help="run the dedicated authenticated MRT2 clip renderer", + ) + parser.add_argument("--model-a", help="shared-worker model for deck a") + parser.add_argument("--model-b", help="shared-worker model for deck b") + parser.add_argument( + "--runtime", + choices=RUNTIME_CHOICES, + default=AUTO_RUNTIME, + help="explicit MRT2 implementation selected by the native host", + ) parser.add_argument( "--port", type=int, @@ -249,8 +1285,17 @@ def main(argv=None) -> None: metavar="NAME", help="download an exported Magenta model, emit JSON progress, then exit", ) + parser.add_argument( + "--runtime-info", + action="store_true", + help="emit immutable PyTorch runtime/install metadata, then exit", + ) args = parser.parse_args(argv) + if args.runtime_info: + _emit(runtime_manifest()) + return + if args.init_resources or args.download_model: run_model_tooling( init_resources=args.init_resources, @@ -258,6 +1303,44 @@ def main(argv=None) -> None: ) return + if args.render_worker: + missing = [name for name in ("model", "port") if getattr(args, name) is None] + if missing: + parser.error( + "the following arguments are required in render-worker mode: " + + ", ".join("--" + name for name in missing) + ) + if args.runtime != PYTORCH_CUDA_RUNTIME: + parser.error( + "render-worker mode requires the explicit pytorch-cuda runtime" + ) + sock = socket.create_connection(("127.0.0.1", args.port)) + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + authenticate_to_host(sock) + run_render_worker(sock, args.model, runtime=args.runtime) + return + + if args.shared: + missing = [ + name + for name in ("model_a", "model_b", "port") + if getattr(args, name) is None + ] + if missing: + parser.error( + "the following arguments are required in shared mode: " + + ", ".join("--" + name.replace("_", "-") for name in missing) + ) + sock = socket.create_connection(("127.0.0.1", args.port)) + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + authenticate_to_host(sock) + run_shared_sidecar( + sock, + (args.model_a, args.model_b), + runtime=args.runtime, + ) + return + missing = [ name for name in ("deck", "model", "port") if getattr(args, name) is None ] @@ -269,7 +1352,8 @@ def main(argv=None) -> None: sock = socket.create_connection(("127.0.0.1", args.port)) sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) - run_sidecar(sock, args.deck, args.model) + authenticate_to_host(sock) + run_sidecar(sock, args.deck, args.model, runtime=args.runtime) if __name__ == "__main__": diff --git a/backend/lsdj/worker.py b/backend/lsdj/worker.py index 6e02852..c6ad97c 100644 --- a/backend/lsdj/worker.py +++ b/backend/lsdj/worker.py @@ -18,6 +18,7 @@ import time from .engine import DeckEngine +from .mrt2 import public_startup_error logger = logging.getLogger(__name__) @@ -32,11 +33,45 @@ def run_deck_worker( out_queue, engine_factory=DeckEngine, clip_queue=None, + perform_warmup=True, ) -> None: logging.basicConfig(level=logging.INFO) logger.info("deck %s: loading %s", deck_id, model) - engine = engine_factory(model=model) - out_queue.put(("status", {"event": "ready", "deck": deck_id, "model": model})) + try: + engine = engine_factory(model=model) + warm_up = getattr(engine, "warm_up", None) + if perform_warmup and callable(warm_up): + out_queue.put( + ("status", {"event": "warming", "deck": deck_id, "model": model}) + ) + warm_up() + diagnostics = getattr(engine, "diagnostics", None) + runtime = diagnostics() if callable(diagnostics) else {} + except Exception as error: + logger.exception("deck %s: startup failed", deck_id) + out_queue.put( + ( + "status", + { + "event": "startup_failed", + "deck": deck_id, + "model": model, + "error": public_startup_error(error), + }, + ) + ) + return + out_queue.put( + ( + "status", + { + "event": "ready", + "deck": deck_id, + "model": model, + "runtime": runtime, + }, + ) + ) playing = False style: dict | None = None @@ -189,6 +224,42 @@ def run_deck_worker( }, ) ) + elif kind == "reset": + reset = getattr(engine, "reset", None) + if not callable(reset): + out_queue.put( + ( + "status", + { + "event": "error", + "error": "reset is unsupported by this runtime", + }, + ) + ) + else: + try: + reset(seed=cmd.get("seed")) + except Exception: + logger.exception("deck %s: reset failed", deck_id) + out_queue.put( + ( + "status", + {"event": "error", "error": "reset failed"}, + ) + ) + else: + playing = False + pace_seconds = 0.0 + out_queue.put( + ( + "status", + { + "event": "reset", + "seed": cmd.get("seed"), + "effective_from_chunk": chunk_index, + }, + ) + ) elif kind in ("set_notes", "set_drums"): # Idempotent full-state conditioning (ADR-0023): the payload # replaces the held state wholesale; None returns to masked. @@ -300,13 +371,23 @@ def run_deck_worker( continue elapsed = time.monotonic() - started out_queue.put(("audio", pcm)) + try: + queue_depth = cmd_queue.qsize() + except (AttributeError, NotImplementedError, OSError): + queue_depth = None out_queue.put( ( "status", { "event": "chunk", "index": chunk_index, - "rtf": round(1.0 / elapsed, 2) if elapsed > 0 else None, + "generation_latency_ms": round(elapsed * 1000, 2), + "queue_depth": queue_depth, + "rtf": ( + round(engine.chunk_seconds / elapsed, 2) + if elapsed > 0 + else None + ), "style": style, }, ) diff --git a/backend/mrt2-pytorch-runtime.in b/backend/mrt2-pytorch-runtime.in new file mode 100644 index 0000000..edc285d --- /dev/null +++ b/backend/mrt2-pytorch-runtime.in @@ -0,0 +1,10 @@ +# Direct dependencies audited by issue #109. This file is not installed into +# the MLX backend environment; it is the input for target-specific bundled +# Python/CUDA runtime locks. +torch==2.12.1 +transformers==5.8.0 +huggingface-hub==1.5.0 +numpy==2.3.5 +safetensors==0.7.0 +sentencepiece==0.2.1 +resampy==0.4.3 diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 0f237ae..b3266e5 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -23,6 +23,18 @@ dev = [ # scripts/freeze-sidecar.sh). A build tool, not a runtime dep. "pyinstaller>=6.21", ] +# Model runtimes are platform-specific, but most controller, transport, and +# validation tests are not. CI installs this locked group on every OS so those +# shared contracts stay testable without pretending that MLX is portable. +ci = [ + "fastapi>=0.136.3", + "httpx>=0.27", + "numpy>=2.2", + "pytest>=8", + "python-multipart>=0.0.32", + "ruff>=0.8", + "uvicorn>=0.49.0", +] [build-system] requires = ["hatchling"] diff --git a/backend/runtime-locks/mrt2-pytorch-linux-x86_64.txt b/backend/runtime-locks/mrt2-pytorch-linux-x86_64.txt new file mode 100644 index 0000000..92b41e8 --- /dev/null +++ b/backend/runtime-locks/mrt2-pytorch-linux-x86_64.txt @@ -0,0 +1,826 @@ +# This file was autogenerated by uv via the following command: +# just lock-mrt2-pytorch +annotated-doc==0.0.5 \ + --hash=sha256:117bac03a25ede5df5440e855b32d556049ca169ead221505badf432fed4b101 \ + --hash=sha256:c7e58ce09192557605d8bbd92836d7e1d520ac9580096042c0bfd197efacf1bb + # via typer +anyio==4.14.2 \ + --hash=sha256:9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494 \ + --hash=sha256:cfa139f3ed1a23ee8f88a145ddb5ac7605b8bbfd8592baacd7ce3d8bb4313c7f + # via httpx +certifi==2026.7.22 \ + --hash=sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775 \ + --hash=sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55 + # via + # httpcore + # httpx +cuda-bindings==13.3.1 \ + --hash=sha256:04436a9364059c84b8f9636f359eccda1cf814341f5b670c71d80d2f79dbc708 \ + --hash=sha256:120fcc53d57903df529c3486962c56528cba5b7d6c57c99537320ed9922c8b86 \ + --hash=sha256:18c8c167c8907b8f02531ca810534315c458dabef31f7965095619bf647b9202 \ + --hash=sha256:1ab2f74ed65bfef4163ba07a8db16f1085e0729291db12a2423aff84ee8278b8 \ + --hash=sha256:2732904099e0a4d4db774a5fc6d91ee95fae065b4d2ecabb4968c5fe2406c9d7 \ + --hash=sha256:36febb7c1079d68a981dbbd8d5a67235b399802b82075c9388624719607e52b9 \ + --hash=sha256:507b0e19e7f934c5e30f30f0244ad70a75812619a7d3a0d742543caae1bd50f1 \ + --hash=sha256:61120b5e4f4a63f67efd7e7396914cb9ef871bb1f0021e990fb70277be240a4d \ + --hash=sha256:8de12ef60bf40756852cb62bbb40460609269f6ece522903d1cc93d73a3ececb \ + --hash=sha256:9851b0caa8bfd3bc6fa054eaf57bea7c8e9c3a62db2d2621224677f49f3c53d0 \ + --hash=sha256:9efb21c1ee64981e184b9e0ba5eb3179e5ba3d4b51665a6cb52b8ef3d01a7cbf \ + --hash=sha256:b134dd8c5c66ae4c4ad814f7aee88fd215353c077010cbc47e3b55ed35ec9eff \ + --hash=sha256:c0c4b1a995098c46695c24257a342dc97d6e6d3f3050b944c9f43bd26d734051 \ + --hash=sha256:c3c772dfff49681541d59630c90f858e173ac926b9c593a2b7123f2a1043cc76 \ + --hash=sha256:c5879712accf6e14bb01aa5e67440eb84998b8d104b509cc7a6dc0b8f656a474 \ + --hash=sha256:c7855c4868aabc0cfae28abbe83d56734bdfbd08f08fc234ac1912a12858bf49 \ + --hash=sha256:e32d08f71ebcdf00f0f41eab2eb37e8da94c8ed411cc9f7f7a019ce6b34abe3a \ + --hash=sha256:efd4c814d311ec08c981f6dded1dbe7d4b371067ee4f6c14cccec4bde9590f80 + # via torch +cuda-pathfinder==1.6.0 \ + --hash=sha256:1503af579d8379c24bdd65528379bc57039b0455be9f5f9686cf8e473a1fce51 + # via cuda-bindings +cuda-toolkit==13.0.2 \ + --hash=sha256:b198824cf2f54003f50d64ada3a0f184b42ca0846c1c94192fa269ecd97a66eb + # via torch +filelock==3.32.2 \ + --hash=sha256:87dd94cf281e586d135fa51132b8e3d9a598b316e90377a288663c9321036c82 \ + --hash=sha256:c33351e1f49cae33414acbc6d56784e6ecee82514ec90795da1161fc4836b5b8 + # via + # huggingface-hub + # torch +fsspec==2026.7.0 \ + --hash=sha256:b57ddbafedfaef7018c1ecab32aa200a9d7ca26b77965f64e48b70061249d279 \ + --hash=sha256:c803c40f4cf860b49dea58ee3e1c33cb9c790520e233537e1340049f89b82a88 + # via + # huggingface-hub + # torch +h11==0.16.0 \ + --hash=sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1 \ + --hash=sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86 + # via httpcore +hf-xet==1.6.0 \ + --hash=sha256:0e6e21fa3cdfcdcd76748564bf593870a5e013f47d97cf10aed63aa222cff5b7 \ + --hash=sha256:23379c2f9ec8696d952b16414a2bae72cad86a52df869b050698ba60f538c675 \ + --hash=sha256:2e58454a340b3556dfa4972d5451aff4fba8dd42a236600ba1a1d2b1514f0fef \ + --hash=sha256:35cec30d75c6f9eb9c16a77cef68e85a103b72e24d4b473714ec9ff06428bab9 \ + --hash=sha256:3dc3e35441ba395006af5aaacc40ef2e603c51ef46c3530b9156185f00935ea3 \ + --hash=sha256:4fc74352a17015bd0ee90038bc9efe38db894cde45f268b6712b04fce8cd0acb \ + --hash=sha256:5153e6bb103ad49d6ea9f1b2e230db5a2ea32551ad09a706d2f61d7c7c80d80e \ + --hash=sha256:5789835d7c6bc9436962853192082374297fb72d7eff7e7762ec25ceb7e25338 \ + --hash=sha256:633dc0cd71d32da58ab8c03ad38e2fac452c15c2b0a2866ebf6ededfe0a5061d \ + --hash=sha256:70cbb9c896901600128cb9b6f06e132954fbede1db30f31f7c6c63f84cb7c31d \ + --hash=sha256:75765820ce4700db3750c94acc8fe27c5fae4c9ec000a0dbac3ca082acf97765 \ + --hash=sha256:8fb4f71cba6129110c3374a33f919001ff130488fc23553698e34cc1c2a1198c \ + --hash=sha256:948f15d3a9545cfe5932f6bd8b440f6ae630aee108f14b7bd6c561f7c2dcc522 \ + --hash=sha256:d62671bb130879cef0ee4c9ebe47a14af6c66ec53e6d84dc15936e5ffdfac82f \ + --hash=sha256:f0906082d9932ae0c0057fa194041c22b4e2cdb46b2592ef3b91f020d62a081a \ + --hash=sha256:f2f7278c05c22fd60cb436cda1269649b3e81db65ecdc8496e5e164aa4143e7b \ + --hash=sha256:fb4fadde1b2b70bf4c0c14a6dccbe7194b1c28947fefd5bbe3fed9d940676c3b + # via huggingface-hub +httpcore==1.0.9 \ + --hash=sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55 \ + --hash=sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8 + # via httpx +httpx==0.28.1 \ + --hash=sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc \ + --hash=sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad + # via huggingface-hub +huggingface-hub==1.5.0 \ + --hash=sha256:c9c0b3ab95a777fc91666111f3b3ede71c0cdced3614c553a64e98920585c4ee \ + --hash=sha256:f281838db29265880fb543de7a23b0f81d3504675de82044307ea3c6c62f799d + # via + # -r backend/mrt2-pytorch-runtime.in + # tokenizers + # transformers +idna==3.18 \ + --hash=sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2 \ + --hash=sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848 + # via + # anyio + # httpx +jinja2==3.1.6 \ + --hash=sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d \ + --hash=sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67 + # via torch +llvmlite==0.48.0 \ + --hash=sha256:02853fe4214acb3780fc920c3fee10564b61d58a35e1b78afcc8a546c2deaba3 \ + --hash=sha256:054aa7d46595935565f276cf0c1659b4f10929c996dd4a606875fae26fba2a23 \ + --hash=sha256:05f0103c8f2f96a37441337e3643863c01b8e83e530aff38960dcb383c54a065 \ + --hash=sha256:13532f248960ba888ad5ab8150494e2f3a3d20e5f59f264e63741ea5b0ba844c \ + --hash=sha256:1d9f952dff6c350c529997423d4fa43abae9722a884ac7ffafc37a3af676e7db \ + --hash=sha256:2fe5cb59b2063bfa039dcb8ca6481c0181bf552f340d10dcf61d7996a665556e \ + --hash=sha256:321f1ac39b462603f0b589751aecf2d237d056f6d005749c1752b6f23ec3f074 \ + --hash=sha256:37d66fae72802175b0bfe1ea06e624b51e2d7aee6c3c34bbd09739b8f88e8e0b \ + --hash=sha256:3ee0c77685a18f5fca994ae21d0763007fca5c5c64b41de37accc78b69079176 \ + --hash=sha256:416fa4c2c66c2c6dc6d0a402648c19206e548efa0aa1eff01ad5cdad0af8217d \ + --hash=sha256:543b19f9ef8f3c7c60d1468191e4ee1b1537bf9f8a3d56f64c0ddd98de92edd2 \ + --hash=sha256:56a7e24607d3f02d7b1bae8d29c7e1e423d53143d68b072999777f19678fe77b \ + --hash=sha256:614aad57df707e3172efd5165f2aa7da6a0c6897e40dce590bf756396815ba76 \ + --hash=sha256:6fa532d6bb3fd3f0803567c736401c54aecfe1a396d3ad25d2440d220e09f0e7 \ + --hash=sha256:7a5c413317050a1d67c34708bde97707f9b2257ef1017f7532d21fe7d9a9ff30 \ + --hash=sha256:8761b9e522f55207e24424fcd98370289eec2710bf8e915c82d1053f642450dc \ + --hash=sha256:91c7e24e74cde3f02b88aa5acca678373f9e069f3b98531b3dbb3a142d9d10bb \ + --hash=sha256:966dcab0a598e2bd8fb5f2cc082cf7b07bae564fc485a3a8692393caf986facf \ + --hash=sha256:979a66a3f28a02565383ff463527dce78e9b856298872a361283132488e83591 \ + --hash=sha256:a83a99ef0c05b4ccddf9b6218ed9fe84b653a0caf7c1d9dbe148d6d16c67f518 \ + --hash=sha256:d0b3c61aac83b42fb48cc96bffbf57c81b82b2aa92276b7ed6420c814629a99a \ + --hash=sha256:d45c7541a80934ec6d8ab0defe67439494ecd2193cbf852a44ba827808976ac1 \ + --hash=sha256:d66c3beb4209087ddd4cf4ed2a0856b6887e6a913bdcf1aacfec9851cf2cba4e \ + --hash=sha256:efaee0276e5e17c2b99b92e0c974bd484ef5977cf5dbc9168e82b71578edb47f \ + --hash=sha256:f5e5a5131045b72345c71062ea1a91910dde913792b6c9b28ebb2c1c0a712e98 + # via numba +markdown-it-py==4.2.0 \ + --hash=sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49 \ + --hash=sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a + # via rich +markupsafe==3.0.3 \ + --hash=sha256:0303439a41979d9e74d18ff5e2dd8c43ed6c6001fd40e5bf2e43f7bd9bbc523f \ + --hash=sha256:068f375c472b3e7acbe2d5318dea141359e6900156b5b2ba06a30b169086b91a \ + --hash=sha256:0bf2a864d67e76e5c9a34dc26ec616a66b9888e25e7b9460e1c76d3293bd9dbf \ + --hash=sha256:0db14f5dafddbb6d9208827849fad01f1a2609380add406671a26386cdf15a19 \ + --hash=sha256:0eb9ff8191e8498cca014656ae6b8d61f39da5f95b488805da4bb029cccbfbaf \ + --hash=sha256:0f4b68347f8c5eab4a13419215bdfd7f8c9b19f2b25520968adfad23eb0ce60c \ + --hash=sha256:1085e7fbddd3be5f89cc898938f42c0b3c711fdcb37d75221de2666af647c175 \ + --hash=sha256:116bb52f642a37c115f517494ea5feb03889e04df47eeff5b130b1808ce7c219 \ + --hash=sha256:12c63dfb4a98206f045aa9563db46507995f7ef6d83b2f68eda65c307c6829eb \ + --hash=sha256:133a43e73a802c5562be9bbcd03d090aa5a1fe899db609c29e8c8d815c5f6de6 \ + --hash=sha256:1353ef0c1b138e1907ae78e2f6c63ff67501122006b0f9abad68fda5f4ffc6ab \ + --hash=sha256:15d939a21d546304880945ca1ecb8a039db6b4dc49b2c5a400387cdae6a62e26 \ + --hash=sha256:177b5253b2834fe3678cb4a5f0059808258584c559193998be2601324fdeafb1 \ + --hash=sha256:1872df69a4de6aead3491198eaf13810b565bdbeec3ae2dc8780f14458ec73ce \ + --hash=sha256:1b4b79e8ebf6b55351f0d91fe80f893b4743f104bff22e90697db1590e47a218 \ + --hash=sha256:1b52b4fb9df4eb9ae465f8d0c228a00624de2334f216f178a995ccdcf82c4634 \ + --hash=sha256:1ba88449deb3de88bd40044603fafffb7bc2b055d626a330323a9ed736661695 \ + --hash=sha256:1cc7ea17a6824959616c525620e387f6dd30fec8cb44f649e31712db02123dad \ + --hash=sha256:218551f6df4868a8d527e3062d0fb968682fe92054e89978594c28e642c43a73 \ + --hash=sha256:26a5784ded40c9e318cfc2bdb30fe164bdb8665ded9cd64d500a34fb42067b1c \ + --hash=sha256:2713baf880df847f2bece4230d4d094280f4e67b1e813eec43b4c0e144a34ffe \ + --hash=sha256:2a15a08b17dd94c53a1da0438822d70ebcd13f8c3a95abe3a9ef9f11a94830aa \ + --hash=sha256:2f981d352f04553a7171b8e44369f2af4055f888dfb147d55e42d29e29e74559 \ + --hash=sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa \ + --hash=sha256:3524b778fe5cfb3452a09d31e7b5adefeea8c5be1d43c4f810ba09f2ceb29d37 \ + --hash=sha256:3537e01efc9d4dccdf77221fb1cb3b8e1a38d5428920e0657ce299b20324d758 \ + --hash=sha256:35add3b638a5d900e807944a078b51922212fb3dedb01633a8defc4b01a3c85f \ + --hash=sha256:38664109c14ffc9e7437e86b4dceb442b0096dfe3541d7864d9cbe1da4cf36c8 \ + --hash=sha256:3a7e8ae81ae39e62a41ec302f972ba6ae23a5c5396c8e60113e9066ef893da0d \ + --hash=sha256:3b562dd9e9ea93f13d53989d23a7e775fdfd1066c33494ff43f5418bc8c58a5c \ + --hash=sha256:457a69a9577064c05a97c41f4e65148652db078a3a509039e64d3467b9e7ef97 \ + --hash=sha256:4bd4cd07944443f5a265608cc6aab442e4f74dff8088b0dfc8238647b8f6ae9a \ + --hash=sha256:4e885a3d1efa2eadc93c894a21770e4bc67899e3543680313b09f139e149ab19 \ + --hash=sha256:4faffd047e07c38848ce017e8725090413cd80cbc23d86e55c587bf979e579c9 \ + --hash=sha256:509fa21c6deb7a7a273d629cf5ec029bc209d1a51178615ddf718f5918992ab9 \ + --hash=sha256:5678211cb9333a6468fb8d8be0305520aa073f50d17f089b5b4b477ea6e67fdc \ + --hash=sha256:591ae9f2a647529ca990bc681daebdd52c8791ff06c2bfa05b65163e28102ef2 \ + --hash=sha256:5a7d5dc5140555cf21a6fefbdbf8723f06fcd2f63ef108f2854de715e4422cb4 \ + --hash=sha256:69c0b73548bc525c8cb9a251cddf1931d1db4d2258e9599c28c07ef3580ef354 \ + --hash=sha256:6b5420a1d9450023228968e7e6a9ce57f65d148ab56d2313fcd589eee96a7a50 \ + --hash=sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698 \ + --hash=sha256:729586769a26dbceff69f7a7dbbf59ab6572b99d94576a5592625d5b411576b9 \ + --hash=sha256:77f0643abe7495da77fb436f50f8dab76dbc6e5fd25d39589a0f1fe6548bfa2b \ + --hash=sha256:795e7751525cae078558e679d646ae45574b47ed6e7771863fcc079a6171a0fc \ + --hash=sha256:7be7b61bb172e1ed687f1754f8e7484f1c8019780f6f6b0786e76bb01c2ae115 \ + --hash=sha256:7c3fb7d25180895632e5d3148dbdc29ea38ccb7fd210aa27acbd1201a1902c6e \ + --hash=sha256:7e68f88e5b8799aa49c85cd116c932a1ac15caaa3f5db09087854d218359e485 \ + --hash=sha256:83891d0e9fb81a825d9a6d61e3f07550ca70a076484292a70fde82c4b807286f \ + --hash=sha256:8485f406a96febb5140bfeca44a73e3ce5116b2501ac54fe953e488fb1d03b12 \ + --hash=sha256:8709b08f4a89aa7586de0aadc8da56180242ee0ada3999749b183aa23df95025 \ + --hash=sha256:8f71bc33915be5186016f675cd83a1e08523649b0e33efdb898db577ef5bb009 \ + --hash=sha256:915c04ba3851909ce68ccc2b8e2cd691618c4dc4c4232fb7982bca3f41fd8c3d \ + --hash=sha256:949b8d66bc381ee8b007cd945914c721d9aba8e27f71959d750a46f7c282b20b \ + --hash=sha256:94c6f0bb423f739146aec64595853541634bde58b2135f27f61c1ffd1cd4d16a \ + --hash=sha256:9a1abfdc021a164803f4d485104931fb8f8c1efd55bc6b748d2f5774e78b62c5 \ + --hash=sha256:9b79b7a16f7fedff2495d684f2b59b0457c3b493778c9eed31111be64d58279f \ + --hash=sha256:a320721ab5a1aba0a233739394eb907f8c8da5c98c9181d1161e77a0c8e36f2d \ + --hash=sha256:a4afe79fb3de0b7097d81da19090f4df4f8d3a2b3adaa8764138aac2e44f3af1 \ + --hash=sha256:ad2cf8aa28b8c020ab2fc8287b0f823d0a7d8630784c31e9ee5edea20f406287 \ + --hash=sha256:b8512a91625c9b3da6f127803b166b629725e68af71f8184ae7e7d54686a56d6 \ + --hash=sha256:bc51efed119bc9cfdf792cdeaa4d67e8f6fcccab66ed4bfdd6bde3e59bfcbb2f \ + --hash=sha256:bdc919ead48f234740ad807933cdf545180bfbe9342c2bb451556db2ed958581 \ + --hash=sha256:bdd37121970bfd8be76c5fb069c7751683bdf373db1ed6c010162b2a130248ed \ + --hash=sha256:be8813b57049a7dc738189df53d69395eba14fb99345e0a5994914a3864c8a4b \ + --hash=sha256:c0c0b3ade1c0b13b936d7970b1d37a57acde9199dc2aecc4c336773e1d86049c \ + --hash=sha256:c47a551199eb8eb2121d4f0f15ae0f923d31350ab9280078d1e5f12b249e0026 \ + --hash=sha256:c4ffb7ebf07cfe8931028e3e4c85f0357459a3f9f9490886198848f4fa002ec8 \ + --hash=sha256:ccfcd093f13f0f0b7fdd0f198b90053bf7b2f02a3927a30e63f3ccc9df56b676 \ + --hash=sha256:d2ee202e79d8ed691ceebae8e0486bd9a2cd4794cec4824e1c99b6f5009502f6 \ + --hash=sha256:d53197da72cc091b024dd97249dfc7794d6a56530370992a5e1a08983ad9230e \ + --hash=sha256:d6dd0be5b5b189d31db7cda48b91d7e0a9795f31430b7f271219ab30f1d3ac9d \ + --hash=sha256:d88b440e37a16e651bda4c7c2b930eb586fd15ca7406cb39e211fcff3bf3017d \ + --hash=sha256:de8a88e63464af587c950061a5e6a67d3632e36df62b986892331d4620a35c01 \ + --hash=sha256:df2449253ef108a379b8b5d6b43f4b1a8e81a061d6537becd5582fba5f9196d7 \ + --hash=sha256:e1c1493fb6e50ab01d20a22826e57520f1284df32f2d8601fdd90b6304601419 \ + --hash=sha256:e1cf1972137e83c5d4c136c43ced9ac51d0e124706ee1c8aa8532c1287fa8795 \ + --hash=sha256:e2103a929dfa2fcaf9bb4e7c091983a49c9ac3b19c9061b6d5427dd7d14d81a1 \ + --hash=sha256:e56b7d45a839a697b5eb268c82a71bd8c7f6c94d6fd50c3d577fa39a9f1409f5 \ + --hash=sha256:e8afc3f2ccfa24215f8cb28dcf43f0113ac3c37c2f0f0806d8c70e4228c5cf4d \ + --hash=sha256:e8fc20152abba6b83724d7ff268c249fa196d8259ff481f3b1476383f8f24e42 \ + --hash=sha256:eaa9599de571d72e2daf60164784109f19978b327a3910d3e9de8c97b5b70cfe \ + --hash=sha256:ec15a59cf5af7be74194f7ab02d0f59a62bdcf1a537677ce67a2537c9b87fcda \ + --hash=sha256:f190daf01f13c72eac4efd5c430a8de82489d9cff23c364c3ea822545032993e \ + --hash=sha256:f34c41761022dd093b4b6896d4810782ffbabe30f2d443ff5f083e0cbbb8c737 \ + --hash=sha256:f3e98bb3798ead92273dc0e5fd0f31ade220f59a266ffd8a4f6065e0a3ce0523 \ + --hash=sha256:f42d0984e947b8adf7dd6dde396e720934d12c506ce84eea8476409563607591 \ + --hash=sha256:f71a396b3bf33ecaa1626c255855702aca4d3d9fea5e051b41ac59a9c1c41edc \ + --hash=sha256:f9e130248f4462aaa8e2552d547f36ddadbeaa573879158d721bbd33dfe4743a \ + --hash=sha256:fed51ac40f757d41b7c48425901843666a6677e3e8eb0abcff09e4ba6e664f50 + # via jinja2 +mdurl==0.1.2 \ + --hash=sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8 \ + --hash=sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba + # via markdown-it-py +mpmath==1.3.0 \ + --hash=sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f \ + --hash=sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c + # via sympy +networkx==3.6.1 \ + --hash=sha256:26b7c357accc0c8cde558ad486283728b65b6a95d85ee1cd66bafab4c8168509 \ + --hash=sha256:d47fbf302e7d9cbbb9e2555a0d267983d2aa476bac30e90dfbe5669bd57f3762 + # via torch +numba==0.66.0 \ + --hash=sha256:0999e3ee1b18c48e1fb51d11af35ef59852c7f4f50569c9550c25faef0616ad1 \ + --hash=sha256:2338cc0d43609fe448930848fd35a5bc688761b986f81b597a6f45cc0f8c9577 \ + --hash=sha256:27951c47e0def9bf8afe580eb961102902e2fd23cb77924b7d9d7cc0f8b444cb \ + --hash=sha256:380b2556a2019ccd1e956ae77dd257eaa39403f7520768b626d44b755112785e \ + --hash=sha256:46ae5f2b19e2af3c33c2df100306a90ea2f981c8158b0390f8bf6c20eee7357e \ + --hash=sha256:53ca5900b7cab15109796030113a6b28576bae5ad7bb507ad6dd1360ddd81ba4 \ + --hash=sha256:63b943eb2c9ba371908ce2cd6dfc643db51fc40f7966993376a1701bc922f537 \ + --hash=sha256:651a2b53298340956db26ecbe7ab043106b50a40c2807e66d617a4917245a4ab \ + --hash=sha256:7c14c044c06b453ec3fa7715dfe75425e2ba72c73377a7ffde6d9ec511dfd94c \ + --hash=sha256:8c1144ba1720ea59ad79f4f488ed54d149b2613b357f7e445678b7d0739c70e9 \ + --hash=sha256:939316d5d8619751207b8972a67852b5a7646665298cb4de693cd6bf135152f4 \ + --hash=sha256:aac69f3ccb8af100f5913c1241edc9692bad1cdd2508721713f426eb06c9a659 \ + --hash=sha256:b075a4e7ebc43dc6294f223e2821659656209fd5e0ce53245877c23d66d6e1a9 \ + --hash=sha256:b900e63a0e26c05ea9a6d5a3a5a0a177cb64c5011887bf43edb8c3ed2c38d363 \ + --hash=sha256:bbd531c327557a9004507fa6bff06c53ab51a7a5776b75261bb9cef1efe2b2ea \ + --hash=sha256:bd57790acd20f6a468e0ad333ef6b82355e309a92310fb7dff80e919f01a21a9 \ + --hash=sha256:c5bfe5350284509ab0474390321454c3a8627a188af5b68c910e83df3e2db4a7 \ + --hash=sha256:cc408c54b450f41582f4be1608f8981c1dcc44c7f40355cc150dd93015753407 \ + --hash=sha256:cdf506775d9f02eb92a87bf5c5b1e0d25506fd18cafd769f4ed914a8feac73e7 \ + --hash=sha256:d426178fb991a85714c43112a8ea7b9d9579ea856ad8dcdb9c1c3941903ba5be \ + --hash=sha256:db7735d15ea17a283d6485b9fa3504769f78fd86e5146638ad5e8da57c031b9e \ + --hash=sha256:e2b101f23b8b63978d334574d2039f27f0dccfe1d891756f33a2e2f3e4c88cf4 \ + --hash=sha256:efe0d2d5099790df945e0cb6e1b3104bd965d7bbfac50d62f1d5d1d6ade0825d \ + --hash=sha256:fb601841d9e02e6237bb6522e36d0741614be3cfe2b482a6f00a41b5ba209443 \ + --hash=sha256:fc6629becb21a867d85401ec89f426dd24c484a4193ade8a38309debfd1529ca + # via resampy +numpy==2.3.5 \ + --hash=sha256:00dc4e846108a382c5869e77c6ed514394bdeb3403461d25a829711041217d5b \ + --hash=sha256:0472f11f6ec23a74a906a00b48a4dcf3849209696dff7c189714511268d103ae \ + --hash=sha256:04822c00b5fd0323c8166d66c701dc31b7fbd252c100acd708c48f763968d6a3 \ + --hash=sha256:052e8c42e0c49d2575621c158934920524f6c5da05a1d3b9bab5d8e259e045f0 \ + --hash=sha256:09a1bea522b25109bf8e6f3027bd810f7c1085c64a0c7ce050c1676ad0ba010b \ + --hash=sha256:0cd00b7b36e35398fa2d16af7b907b65304ef8bb4817a550e06e5012929830fa \ + --hash=sha256:0d8163f43acde9a73c2a33605353a4f1bc4798745a8b1d73183b28e5b435ae28 \ + --hash=sha256:1062fde1dcf469571705945b0f221b73928f34a20c904ffb45db101907c3454e \ + --hash=sha256:11e06aa0af8c0f05104d56450d6093ee639e15f24ecf62d417329d06e522e017 \ + --hash=sha256:17531366a2e3a9e30762c000f2c43a9aaa05728712e25c11ce1dbe700c53ad41 \ + --hash=sha256:1978155dd49972084bd6ef388d66ab70f0c323ddee6f693d539376498720fb7e \ + --hash=sha256:1ed1ec893cff7040a02c8aa1c8611b94d395590d553f6b53629a4461dc7f7b63 \ + --hash=sha256:2dcd0808a421a482a080f89859a18beb0b3d1e905b81e617a188bd80422d62e9 \ + --hash=sha256:2e2eb32ddb9ccb817d620ac1d8dae7c3f641c1e5f55f531a33e8ab97960a75b8 \ + --hash=sha256:2feae0d2c91d46e59fcd62784a3a83b3fb677fead592ce51b5a6fbb4f95965ff \ + --hash=sha256:3095bdb8dd297e5920b010e96134ed91d852d81d490e787beca7e35ae1d89cf7 \ + --hash=sha256:30bc11310e8153ca664b14c5f1b73e94bd0503681fcf136a163de856f3a50139 \ + --hash=sha256:3101e5177d114a593d79dd79658650fe28b5a0d8abeb8ce6f437c0e6df5be1a4 \ + --hash=sha256:396084a36abdb603546b119d96528c2f6263921c50df3c8fd7cb28873a237748 \ + --hash=sha256:3997b5b3c9a771e157f9aae01dd579ee35ad7109be18db0e85dbdbe1de06e952 \ + --hash=sha256:414802f3b97f3c1eef41e530aaba3b3c1620649871d8cb38c6eaff034c2e16bd \ + --hash=sha256:51c1e14eb1e154ebd80e860722f9e6ed6ec89714ad2db2d3aa33c31d7c12179b \ + --hash=sha256:51c55fe3451421f3a6ef9a9c1439e82101c57a2c9eab9feb196a62b1a10b58ce \ + --hash=sha256:5ee6609ac3604fa7780e30a03e5e241a7956f8e2fcfe547d51e3afa5247ac47f \ + --hash=sha256:612a95a17655e213502f60cfb9bf9408efdc9eb1d5f50535cc6eb365d11b42b5 \ + --hash=sha256:6203fdf9f3dc5bdaed7319ad8698e685c7a3be10819f41d32a0723e611733b42 \ + --hash=sha256:63c0e9e7eea69588479ebf4a8a270d5ac22763cc5854e9a7eae952a3908103f7 \ + --hash=sha256:66f85ce62c70b843bab1fb14a05d5737741e74e28c7b8b5a064de10142fad248 \ + --hash=sha256:6cf9b429b21df6b99f4dee7a1218b8b7ffbbe7df8764dc0bd60ce8a0708fed1e \ + --hash=sha256:70b37199913c1bd300ff6e2693316c6f869c7ee16378faf10e4f5e3275b299c3 \ + --hash=sha256:727fd05b57df37dc0bcf1a27767a3d9a78cbbc92822445f32cc3436ba797337b \ + --hash=sha256:74ae7b798248fe62021dbf3c914245ad45d1a6b0cb4a29ecb4b31d0bfbc4cc3e \ + --hash=sha256:784db1dcdab56bf0517743e746dfb0f885fc68d948aba86eeec2cba234bdf1c0 \ + --hash=sha256:86945f2ee6d10cdfd67bcb4069c1662dd711f7e2a4343db5cecec06b87cf31aa \ + --hash=sha256:86d835afea1eaa143012a2d7a3f45a3adce2d7adc8b4961f0b362214d800846a \ + --hash=sha256:872a5cf366aec6bb1147336480fef14c9164b154aeb6542327de4970282cd2f5 \ + --hash=sha256:8b973c57ff8e184109db042c842423ff4f60446239bd585a5131cc47f06f789d \ + --hash=sha256:8cba086a43d54ca804ce711b2a940b16e452807acebe7852ff327f1ecd49b0d4 \ + --hash=sha256:8f7f0e05112916223d3f438f293abf0727e1181b5983f413dfa2fefc4098245c \ + --hash=sha256:900218e456384ea676e24ea6a0417f030a3b07306d29d7ad843957b40a9d8d52 \ + --hash=sha256:93eebbcf1aafdf7e2ddd44c2923e2672e1010bddc014138b229e49725b4d6be5 \ + --hash=sha256:9c75442b2209b8470d6d5d8b1c25714270686f14c749028d2199c54e29f20b4d \ + --hash=sha256:9ee2197ef8c4f0dfe405d835f3b6a14f5fee7782b5de51ba06fb65fc9b36e9f1 \ + --hash=sha256:a414504bef8945eae5f2d7cb7be2d4af77c5d1cb5e20b296c2c25b61dff2900c \ + --hash=sha256:a4b9159734b326535f4dd01d947f919c6eefd2d9827466a696c44ced82dfbc18 \ + --hash=sha256:a80afd79f45f3c4a7d341f13acbe058d1ca8ac017c165d3fa0d3de6bc1a079d7 \ + --hash=sha256:aa5bc7c5d59d831d9773d1170acac7893ce3a5e130540605770ade83280e7188 \ + --hash=sha256:acfd89508504a19ed06ef963ad544ec6664518c863436306153e13e94605c218 \ + --hash=sha256:aeffcab3d4b43712bb7a60b65f6044d444e75e563ff6180af8f98dd4b905dfd2 \ + --hash=sha256:afaffc4393205524af9dfa400fa250143a6c3bc646c08c9f5e25a9f4b4d6a903 \ + --hash=sha256:b0c7088a73aef3d687c4deef8452a3ac7c1be4e29ed8bf3b366c8111128ac60c \ + --hash=sha256:b46b4ec24f7293f23adcd2d146960559aaf8020213de8ad1909dba6c013bf89c \ + --hash=sha256:b501b5fa195cc9e24fe102f21ec0a44dffc231d2af79950b451e0d99cea02234 \ + --hash=sha256:bf06bc2af43fa8d32d30fae16ad965663e966b1a3202ed407b84c989c3221e82 \ + --hash=sha256:c804e3a5aba5460c73955c955bdbd5c08c354954e9270a2c1565f62e866bdc39 \ + --hash=sha256:c8a9958e88b65c3b27e22ca2a076311636850b612d6bbfb76e8d156aacde2aaf \ + --hash=sha256:cc0a57f895b96ec78969c34f682c602bf8da1a0270b09bc65673df2e7638ec20 \ + --hash=sha256:cc8920d2ec5fa99875b670bb86ddeb21e295cb07aa331810d9e486e0b969d946 \ + --hash=sha256:ccc933afd4d20aad3c00bcef049cb40049f7f196e0397f1109dba6fed63267b0 \ + --hash=sha256:ce581db493ea1a96c0556360ede6607496e8bf9b3a8efa66e06477267bc831e9 \ + --hash=sha256:d0f23b44f57077c1ede8c5f26b30f706498b4862d3ff0a7298b8411dd2f043ff \ + --hash=sha256:d21644de1b609825ede2f48be98dfde4656aefc713654eeee280e37cadc4e0ad \ + --hash=sha256:d6889ec4ec662a1a37eb4b4fb26b6100841804dac55bd9df579e326cdc146227 \ + --hash=sha256:de5672f4a7b200c15a4127042170a694d4df43c992948f5e1af57f0174beed10 \ + --hash=sha256:e6a0bc88393d65807d751a614207b7129a310ca4fe76a74e5c7da5fa5671417e \ + --hash=sha256:ed89927b86296067b4f81f108a2271d8926467a8868e554eaf370fc27fa3ccaf \ + --hash=sha256:ee3888d9ff7c14604052b2ca5535a30216aa0a58e948cdd3eeb8d3415f638769 \ + --hash=sha256:f0963b55cdd70fad460fa4c1341f12f976bb26cb66021a5580329bd498988310 \ + --hash=sha256:f16417ec91f12f814b10bafe79ef77e70113a2f5f7018640e7425ff979253425 \ + --hash=sha256:f28620fe26bee16243be2b7b874da327312240a7cdc38b769a697578d2100013 \ + --hash=sha256:f4255143f5160d0de972d28c8f9665d882b5f61309d8362fdd3e103cf7bf010c \ + --hash=sha256:ffac52f28a7849ad7576293c0cb7b9f08304e8f7d738a8cb8a90ec4c55a998eb \ + --hash=sha256:ffe22d2b05504f786c867c8395de703937f934272eb67586817b46188b4ded6d \ + --hash=sha256:fffe29a1ef00883599d1dc2c51aa2e5d80afe49523c261a74933df395c15c520 + # via + # -r backend/mrt2-pytorch-runtime.in + # numba + # resampy + # transformers +nvidia-cublas==13.1.1.3 \ + --hash=sha256:37936a16db8fe4ac1f065c2139360608a543a09275cb1a1af612e08cfa065436 \ + --hash=sha256:b6cdce694e47ff6aadf0a69df1cab6628d696f5ff56e8d16af50309d855fa20f \ + --hash=sha256:b7a210458267ac818974c53038fbec2e969d5c99f305ab15c72522fa9f001dd5 + # via + # nvidia-cudnn-cu13 + # nvidia-cusolver + # torch +nvidia-cuda-cupti==13.0.85 \ + --hash=sha256:4eb01c08e859bf924d222250d2e8f8b8ff6d3db4721288cf35d14252a4d933c8 \ + --hash=sha256:683f58d301548deeefcb8f6fac1b8d907691b9d8b18eccab417f51e362102f00 \ + --hash=sha256:796bd679890ee55fb14a94629b698b6db54bcfd833d391d5e94017dd9d7d3151 + # via cuda-toolkit +nvidia-cuda-nvrtc==13.0.88 \ + --hash=sha256:6bcd4e7f8e205cbe644f5a98f2f799bef9556fefc89dd786e79a16312ce49872 \ + --hash=sha256:ad9b6d2ead2435f11cbb6868809d2adeeee302e9bb94bcf0539c7a40d80e8575 \ + --hash=sha256:d27f20a0ca67a4bb34268a5e951033496c5b74870b868bacd046b1b8e0c3267b + # via + # cuda-toolkit + # nvidia-cublas +nvidia-cuda-runtime==13.0.96 \ + --hash=sha256:7f82250d7782aa23b6cfe765ecc7db554bd3c2870c43f3d1821f1d18aebf0548 \ + --hash=sha256:ef9bcbe90493a2b9d810e43d249adb3d02e98dd30200d86607d8d02687c43f55 \ + --hash=sha256:f79298c8a098cec150a597c8eba58ecdab96e3bdc4b9bc4f9983635031740492 + # via cuda-toolkit +nvidia-cudnn-cu13==9.20.0.48 \ + --hash=sha256:0c45dd8eeb50b603f07995b1b300c62ffe6a1980482b82b3bcf94a4ca9d49304 \ + --hash=sha256:af8139732b99c0118be65ea5aac97f0d46018f8c552889e49d2fb0c6261a4a24 \ + --hash=sha256:e31454ae00094b0c55319d9d15b6fa2fc50a9e1c0f5c8c80fb75258234e731e1 + # via torch +nvidia-cufft==12.0.0.61 \ + --hash=sha256:2708c852ef8cd89d1d2068bdbece0aa188813a0c934db3779b9b1faa8442e5f5 \ + --hash=sha256:2abce5b39d2f5ae12730fb7e5db6696533e36c26e2d3e8fd1750bdd2853364eb \ + --hash=sha256:6c44f692dce8fd5ffd3e3df134b6cdb9c2f72d99cf40b62c32dde45eea9ddad3 + # via cuda-toolkit +nvidia-cufile==1.15.1.6 \ + --hash=sha256:08a3ecefae5a01c7f5117351c64f17c7c62efa5fffdbe24fc7d298da19cd0b44 \ + --hash=sha256:bdc0deedc61f548bddf7733bdc216456c2fdb101d020e1ab4b88d232d5e2f6d1 + # via cuda-toolkit +nvidia-curand==10.4.0.35 \ + --hash=sha256:133df5a7509c3e292aaa2b477afd0194f06ce4ea24d714d616ff36439cee349a \ + --hash=sha256:1aee33a5da6e1db083fe2b90082def8915f30f3248d5896bcec36a579d941bfc \ + --hash=sha256:65b1710aa6961d326b411e314b374290904c5ddf41dc3f766ebc3f1d7d4ca69f + # via cuda-toolkit +nvidia-cusolver==12.0.4.66 \ + --hash=sha256:02c2457eaa9e39de20f880f4bd8820e6a1cfb9f9a34f820eb12a155aa5bc92d2 \ + --hash=sha256:0a759da5dea5c0ea10fd307de75cdeb59e7ea4fcb8add0924859b944babf1112 \ + --hash=sha256:16515bd33a8e76bb54d024cfa068fa68d30e80fc34b9e1090813ea9362e0cb65 + # via cuda-toolkit +nvidia-cusparse==12.6.3.3 \ + --hash=sha256:2b3c89c88d01ee0e477cb7f82ef60a11a4bcd57b6b87c33f789350b59759360b \ + --hash=sha256:80bcc4662f23f1054ee334a15c72b8940402975e0eab63178fc7e670aa59472c \ + --hash=sha256:cbcf42feb737bd7ec15b4c0a63e62351886bd3f975027b8815d7f720a2b5ea79 + # via + # cuda-toolkit + # nvidia-cusolver +nvidia-cusparselt-cu13==0.8.1 \ + --hash=sha256:4dca476c50bf4780d46cd0bfbd82e2bc10a08e4fef7950917ce8d7578d22a23f \ + --hash=sha256:786ce87568c303fadb5afcc7102d454cd3040d75f6f8626f5db460d1871f4dd0 \ + --hash=sha256:dccbd362f91a7b9024d1f55ee9f548ac065027ff15d8c8b0db889ab3a8f31215 + # via torch +nvidia-nccl-cu13==2.29.7 \ + --hash=sha256:674a12383e3c38a1bcccae7d4f3633b37852230b6047883cb2f4c2d1b36d9bf5 \ + --hash=sha256:edd81538446786ec3b73972543e53bb43bcaf0bfc8ef76cb679fcc390ffe136d + # via torch +nvidia-nvjitlink==13.0.88 \ + --hash=sha256:13a74f429e23b921c1109976abefacc69835f2f433ebd323d3946e11d804e47b \ + --hash=sha256:634e96e3da9ef845ae744097a1f289238ecf946ce0b82e93cdce14b9782e682f \ + --hash=sha256:e931536ccc7d467a98ba1d8b89ff7fa7f1fa3b13f2b0069118cd7f47bff07d0c + # via + # cuda-toolkit + # nvidia-cufft + # nvidia-cusolver + # nvidia-cusparse +nvidia-nvshmem-cu13==3.4.5 \ + --hash=sha256:290f0a2ee94c9f3687a02502f3b9299a9f9fe826e6d0287ee18482e78d495b80 \ + --hash=sha256:6dc2a197f38e5d0376ad52cd1a2a3617d3cdc150fd5966f4aee9bcebb1d68fe9 + # via torch +nvidia-nvtx==13.0.85 \ + --hash=sha256:4936d1d6780fbe68db454f5e72a42ff64d1fd6397df9f363ae786930fd5c1cd4 \ + --hash=sha256:cb7780edb6b14107373c835bf8b72e7a178bac7367e23da7acb108f973f157a6 \ + --hash=sha256:d66ea44254dd3c6eacc300047af6e1288d2269dd072b417e0adffbf479e18519 + # via cuda-toolkit +packaging==26.3 \ + --hash=sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79 \ + --hash=sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c + # via + # huggingface-hub + # transformers +pygments==2.20.0 \ + --hash=sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f \ + --hash=sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176 + # via rich +pyyaml==6.0.3 \ + --hash=sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c \ + --hash=sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a \ + --hash=sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3 \ + --hash=sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956 \ + --hash=sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6 \ + --hash=sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c \ + --hash=sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65 \ + --hash=sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a \ + --hash=sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0 \ + --hash=sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b \ + --hash=sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1 \ + --hash=sha256:22ba7cfcad58ef3ecddc7ed1db3409af68d023b7f940da23c6c2a1890976eda6 \ + --hash=sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7 \ + --hash=sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e \ + --hash=sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007 \ + --hash=sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310 \ + --hash=sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4 \ + --hash=sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9 \ + --hash=sha256:3ff07ec89bae51176c0549bc4c63aa6202991da2d9a6129d7aef7f1407d3f295 \ + --hash=sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea \ + --hash=sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0 \ + --hash=sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e \ + --hash=sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac \ + --hash=sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9 \ + --hash=sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7 \ + --hash=sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35 \ + --hash=sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb \ + --hash=sha256:5cf4e27da7e3fbed4d6c3d8e797387aaad68102272f8f9752883bc32d61cb87b \ + --hash=sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69 \ + --hash=sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5 \ + --hash=sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b \ + --hash=sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c \ + --hash=sha256:6344df0d5755a2c9a276d4473ae6b90647e216ab4757f8426893b5dd2ac3f369 \ + --hash=sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd \ + --hash=sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824 \ + --hash=sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198 \ + --hash=sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065 \ + --hash=sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c \ + --hash=sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c \ + --hash=sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764 \ + --hash=sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196 \ + --hash=sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b \ + --hash=sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00 \ + --hash=sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac \ + --hash=sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8 \ + --hash=sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e \ + --hash=sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28 \ + --hash=sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3 \ + --hash=sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5 \ + --hash=sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4 \ + --hash=sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b \ + --hash=sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf \ + --hash=sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5 \ + --hash=sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702 \ + --hash=sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8 \ + --hash=sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788 \ + --hash=sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da \ + --hash=sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d \ + --hash=sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc \ + --hash=sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c \ + --hash=sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba \ + --hash=sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f \ + --hash=sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917 \ + --hash=sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5 \ + --hash=sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26 \ + --hash=sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f \ + --hash=sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b \ + --hash=sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be \ + --hash=sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c \ + --hash=sha256:efd7b85f94a6f21e4932043973a7ba2613b059c4a000551892ac9f1d11f5baf3 \ + --hash=sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6 \ + --hash=sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926 \ + --hash=sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0 + # via + # huggingface-hub + # transformers +regex==2026.7.19 \ + --hash=sha256:062f8cb7a9739c4835d22bd96f370c59aba89f257adcfa53be3cc209e08d3ae0 \ + --hash=sha256:064f1760a5a4ade65c5419be23e782f29147528e8a66e0c42dd4cedb8d4e9fc6 \ + --hash=sha256:09523a592938aa9f587fb74467c63ff0cf88fc3df14c82ab0f0517dcf76aaa62 \ + --hash=sha256:09d3007fc76249a83cdd33de160d50e6cb77f54e09d8fa9e7148e10607ce24af \ + --hash=sha256:09f3e5287f94f17b709dc9a9e70865855feee835c861613be144218ce4ca82cc \ + --hash=sha256:0c41c63992bf1874cebb6e7f56fd7d3c007924659a604ae3d90e427d40d4fd13 \ + --hash=sha256:0e9554c8785eac5cffe6300f69a91f58ba72bc88a5f8d661235ad7c6aa5b8ccd \ + --hash=sha256:1123ef4211d763ee771d47916a1596e2f4915794f7aabdc1adcb20e4249a6951 \ + --hash=sha256:15b364b9b98d6d2fe1a85034c23a3180ff913f46caddc3895f6fd65186255ccc \ + --hash=sha256:1649eb39fcc9ea80c4d2f110fde2b8ab2aef3877b98f02ab9b14e961f418c511 \ + --hash=sha256:17ed5692f6acc4183e98331101a5f9e4f64d72fe58b753da4d444a2c77d05b12 \ + --hash=sha256:199535629f25caf89698039af3d1ad5fcae7f933e2112c73f1cdf49165c99518 \ + --hash=sha256:1c398716054621aa300b3d411f467dda903806c5da0df6945ab73982b8d115db \ + --hash=sha256:1d3372064506b94dd2c67c845f2db8062e9e9ba84d04e33cb96d7d33c11fe1ae \ + --hash=sha256:1d58561843f0ff7dc78b4c28b5e2dc388f3eff94ebc8a232a3adba961fc00009 \ + --hash=sha256:1d793a7988e04fcb1e2e135567443d82173225d657419ec09414a9b5a145b986 \ + --hash=sha256:1ebac3474b8589fce2f9b225b650afd61448f7c73a5d0255a10cc6366471aed1 \ + --hash=sha256:20568e182eb82d39a6bf7cff3fd58566f14c75c6f74b2c8c96537eecf9010e3a \ + --hash=sha256:22a992de9a0d91bda927bf02b94351d737a0302905432c88a53de7c4b9ce62e2 \ + --hash=sha256:2955907b7157a6660f27079edf7e0229e9c9c5325c77a2ef6a890cba91efa6f0 \ + --hash=sha256:2c4e61e2e1be56f63ec3cc618aa9e0de81ef6f43d177205451840022e24f5b78 \ + --hash=sha256:2cc3460cedf7579948486eab03bc9ad7089df4d7281c0f47f4afe03e8d13f02d \ + --hash=sha256:2ce9e679f776649746729b6c86382da519ef649c8e34cc41df0d2e5e0f6c36d4 \ + --hash=sha256:2ef7eeb108c47ce7bcc9513e51bcb1bf57e8f483d52fce68a8642e3527141ae0 \ + --hash=sha256:3080a7fd38ef049bd489e01c970c97dd84ff446a885b0f1f6b26d9b1ad13ce11 \ + --hash=sha256:343a4504e3fb688c47cad451221ca5d4814f42b1e16c0065bde9cbf7f473bd52 \ + --hash=sha256:36aacfb15faaff3ced55afbf35ec72f50d4aee22082c4f7fe0573a33e2fca92e \ + --hash=sha256:3d3143f159261b1ce5b24c261c590e5913370c3200c5e9ebbb92b5aa5e111902 \ + --hash=sha256:40b34dd88658e4fedd2fddbf0275ac970d00614b731357f425722a3ed1983d11 \ + --hash=sha256:4458124d71339f505bf1fb94f69fd1bb8fa9d2481eebfef27c10ef4f2b9e12f6 \ + --hash=sha256:4896db1f4ce0576765b8272aa922df324e0f5b9bb2c3d03044ff32a7234a9aba \ + --hash=sha256:4a0530bb1b8c1c985e7e2122e2b4d3aedd8a3c21c6bfddae6767c4405668b56e \ + --hash=sha256:4aa5435cdb3eb6f55fe98a171b05e3fbcd95fadaa4aa32acf62afd9b0cfdbcac \ + --hash=sha256:4c3501bfa814ab07b5580741f9bf78dfdfe146a04057f82df9e2402d2a975939 \ + --hash=sha256:4e5413bd5f13d3a4e3539ca98f70f75e7fca92518dd7f117f030ebedd10b60cb \ + --hash=sha256:4e6883a021db30511d9fb8cfb0f222ce1f2c369f7d4d8b0448f449a93ba0bdfc \ + --hash=sha256:52579c60a6078be70a0e49c81d6e56d677f34cd439af281a0083b8c7bc75c095 \ + --hash=sha256:555497390743af1a65045fa4527782d10ff5b88970359412baa4a1e628fe393b \ + --hash=sha256:56ad4d9f77df871a99e25c37091052a02528ec0eb059de928ee33956b854b45b \ + --hash=sha256:571fde9741eb0ccde23dd4e0c1d50fbae910e901fa7e629faf39b2dda740d220 \ + --hash=sha256:572fc57b0009c735ee56c175ea021b637a15551a312f56734277f923d6fd0f6c \ + --hash=sha256:59787bd5f8c70aa339084e961d2996b53fbdeab4d5393bba5c1fe1fc32e02bae \ + --hash=sha256:5a2721c8720e2cb3c209925dfb9200199b4b07361c9e01d321719404b21458b3 \ + --hash=sha256:5cc26a66e212fa5d6c6170c3a40d99d888db3020c6fdab1523250d4341382e44 \ + --hash=sha256:5ebee1ee89c39c953baac6924fcde08c5bb427c4057510862f9d7c7bdb3d8665 \ + --hash=sha256:60be8693a1dadc210bbcbc0db3e26da5f7d01d1d5a3da594e99b4fa42df404f5 \ + --hash=sha256:618a0aed532be87294c4477b0481f3aa0f1520f4014a4374dd4cf789b4cd2c97 \ + --hash=sha256:61bb1bd45520aacd56dd80943bd34991fb5350afdd1f36f2282230fd5154a218 \ + --hash=sha256:6383cd2ed53a646c659ba1fe65727db76437fdaa069e697a0b44a51d5843d864 \ + --hash=sha256:64729333167c2dcaaa56a331d40ee097bd9c5617ffd51dabb09eaddafb1b532e \ + --hash=sha256:64b6ca7391a1395c2638dd5c7456d67bea44fc6c5e8e92c5dc8aa6a8f23292b4 \ + --hash=sha256:65dcd28d3eba2ab7c2fd906485cc301392b47cc2234790d27d4e4814e02cdfda \ + --hash=sha256:65fa6cb38ed5e9c3637e68e544f598b39c3b86b808ed0627a67b68320384b459 \ + --hash=sha256:66bd62c59a5427746e8c44becae1d9b99d22fb13f30f492083dfb9ad7c45cc18 \ + --hash=sha256:6e44c0e7c5664be20aee92085153150c0a7967310a73a43c0f832b7cd35d0dd3 \ + --hash=sha256:6f8c6e7a1cfa3dc9d0ee2de0e65e834537fa29992cc3976ffec914afc35c5dd5 \ + --hash=sha256:7322ec6cc9fba9d49ab888bb82d67ac5625627aa168f0165139b17018df3fb8a \ + --hash=sha256:73b133a9e6fb512858e7f065e96f1180aa46646bc74a83aea62f1d314f3dd035 \ + --hash=sha256:73f272fba87b8ccfe70a137d02a54af386f6d27aa509fbffdd978f5947aae1aa \ + --hash=sha256:7e77b324909c1617cbb4c668677e2c6ae13f44d7c1de0d4f15f2e3c10f3315b5 \ + --hash=sha256:80115dd39481fd3a4b4080220799dbcacb921a844de4b827264ececacbe17c78 \ + --hash=sha256:87ccab0db8d5f4fbb0272642113c1adb2ffc698c16d3a0944580222331fa7a20 \ + --hash=sha256:89dfee3319f5ae3f75ebd5c2445a809bb320252ba5529ffdafea4ef25d79cf1a \ + --hash=sha256:8ac59a0900474a52b7c04af8196affc22bd9842acb0950df12f7b813e983609a \ + --hash=sha256:8cae6fd77a5b72dae505084b1a2ee0360139faf72fedbab667cd7cc65aae7a6a \ + --hash=sha256:8d3469c91dd92ee41b7c95280edbd975ef1ba9195086686623a1c6e8935ce965 \ + --hash=sha256:90c633e7e8d6bf4e992b8b36ce69e018f834b641dd6de8cea6d78c06ffa119c5 \ + --hash=sha256:93db40c8de0815baab96a06e08a984bac71f989d13bab789e382158c5d426797 \ + --hash=sha256:9724e6cb5e478cd7d8cabf027826178739cb18cf0e117d0e32814d479fa02276 \ + --hash=sha256:98c6ac18480fcdb33f35439183f1d2e79760ab41930309c6d951cb1f8e46694c \ + --hash=sha256:9a15e785f244f3e07847b984ce8773fc3da10a9f3c131cc49a4c5b4d672b4547 \ + --hash=sha256:9b60d7814174f059e5de4ab98271cc5ba9259cfea55273a81544dceea32dc8d9 \ + --hash=sha256:9be2a6647740dd3cca6acb24e87f03d7632cd280dbce9bbe40c26353a215a45d \ + --hash=sha256:9c7472192ebfad53a6be7c4a8bfb2d64b81c0e93a1fc8c57e1dd0b638297b5d1 \ + --hash=sha256:9dce8ec9695f531a1b8a6f314fd4b393adcccf2ea861db480cdf97a301d01a68 \ + --hash=sha256:9e50d748a32da622f256e8d505867f5d3c43a837c6a9f0efb149655fadd1042a \ + --hash=sha256:a81758ed242b861b72e778ba34d41366441a2e10b16b472784c88da2dea7e2dd \ + --hash=sha256:ac777001cdfc28b72477d93c8564bb7583081ea8fb45cdca3d568e0a4f87183c \ + --hash=sha256:b2b506b1788df5fecd270a10d5e70a95fe77b87ea2b370a318043f6f5f817ee6 \ + --hash=sha256:b2ea4a3e8357be8849e833beeae757ac3c7a6b3fc055c03c808a53c91ad30d82 \ + --hash=sha256:bf1516fe58fc104f39b2d1dbe2d5e27d0cd45c4be2e42ba6ee0cc763701ec3c7 \ + --hash=sha256:c0d702548d89d572b2929879bc883bb7a4c4709efafe4512cadee56c55c9bd15 \ + --hash=sha256:c10b82c2634df08dfb13b1f04e38fe310d086ee092f4f69c0c8da234251e556e \ + --hash=sha256:c42572142ed0b9d5d261ba727157c426510da78e20828b66bbb855098b8a4e38 \ + --hash=sha256:c4585c3e64b4f9e583b4d2683f18f5d5d872b3d71dcf24594b74ecc23602fa96 \ + --hash=sha256:c639ea314df70a7b2811e8020448c75af8c9445f5a60f8a4ced81c306a9380c2 \ + --hash=sha256:c670fe7be5b6020b76bc6e8d2196074657e1327595bca93a389e1a76ab130ad8 \ + --hash=sha256:cc1b2440423a851fad781309dd87843868f4f66a6bcd1ddb9225cf4ec2c84732 \ + --hash=sha256:cd3584591ea4429026cdb931b054342c2bcf189b44ff367f8d5c15bc092a2966 \ + --hash=sha256:d15df07081d91b76ff20d43f94592ee110330152d617b730fdbe5ef9fb680053 \ + --hash=sha256:d19662dbedbe783d323196312d38f5ba53cf56296378252171985da6899887d3 \ + --hash=sha256:d24ecb4f5e009ea0bd275ee37ad9953b32005e2e5e60f8bbae16da0dbbf0d3a0 \ + --hash=sha256:d446c6ac40bb6e05025ccee55b84d80fe9bf8e93010ffc4bb9484f13d498835f \ + --hash=sha256:d51ffd3427640fa2da6ade574ceba932f210ad095f65fcc450a2b0a0d454868e \ + --hash=sha256:d6ce43a0269d68cee79a7d1ade7def53c20f8f2a047b92d7b5d5bcc73ae88327 \ + --hash=sha256:d721e53758b2cca74990185eb0671dd466d7a388a1a45d0c6f4c13cef41a68ac \ + --hash=sha256:d7da47a0f248977f08e2cb659ff3c17ddc13a4d39b3a7baa0a81bf5b415430f6 \ + --hash=sha256:db47b561c9afd884baa1f96f797c9ca369872c4b65912bc691cfa99e68340af2 \ + --hash=sha256:dbe6493fbd27321b1d1f2dd4f5c7e5bd4d8b1d7cab7f32fd67db3d0b2ed8248a \ + --hash=sha256:dbece16025afda5e3031af0c4059207e61dcf73ef13af844964f57f387d1c435 \ + --hash=sha256:ddd67571c10869f65a5d7dde536d1e066e306cc90de57d7de4d5f34802428bb5 \ + --hash=sha256:de9208bb427130c82a5dbfd104f92c8876fc9559278c880b3002755bbbe9c83d \ + --hash=sha256:e30d40268a28d54ce0437031750497004c22602b8e3ab891f759b795a003b312 \ + --hash=sha256:e8b0abe7d870f53ca5143895fef7d1041a0c831a140d3dc2c760dd7ba25d4a8b \ + --hash=sha256:f035d9dc1d25eff9d361456572231c7d27b5ccd473ca7dc0adfce732bd006d40 \ + --hash=sha256:f04b9f56b0e0614c0126be12c2c2d9f8850c1e57af302bd0a63bed379d4af974 \ + --hash=sha256:f0fa4fa9c3632d708742baf2282f2055c11d888a790362670a403cbf48a2c404 \ + --hash=sha256:f2e7f8e2ab6c2922be02c7ec45185aa5bd771e2e57b95455ee343a44d8130dff \ + --hash=sha256:f8f6fa298bb4f7f58a33334406218ba74716e68feddf5e4e54cd5d8082705abf \ + --hash=sha256:fbf300e2070bb35038660b3be1be4b91b0024edb41517e6996320b49b92b4175 \ + --hash=sha256:fce7760bf283405b2c7999cab3da4e72f7deca6396013115e3f7a955db9760da \ + --hash=sha256:fcee38cd8e5089d6d4f048ba1233b3ad76e5954f545382180889112ff5cb712d \ + --hash=sha256:fe31f28c94402043161876a258a9c6f757cb485905c7614ce8d6cd40e6b7bdc1 \ + --hash=sha256:ffd8893ccc1c2fce6e0d6ca402d716fe1b29db70c7132609a05955e31b2aa8f2 + # via transformers +resampy==0.4.3 \ + --hash=sha256:a0d1c28398f0e55994b739650afef4e3974115edbe96cd4bb81968425e916e47 \ + --hash=sha256:ad2ed64516b140a122d96704e32bc0f92b23f45419e8b8f478e5a05f83edcebd + # via -r backend/mrt2-pytorch-runtime.in +rich==15.0.0 \ + --hash=sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb \ + --hash=sha256:edd07a4824c6b40189fb7ac9bc4c52536e9780fbbfbddf6f1e2502c31b068c36 + # via typer +safetensors==0.7.0 \ + --hash=sha256:0071bffba4150c2f46cae1432d31995d77acfd9f8db598b5d1a2ce67e8440ad2 \ + --hash=sha256:07663963b67e8bd9f0b8ad15bb9163606cd27cc5a1b96235a50d8369803b96b0 \ + --hash=sha256:12f49080303fa6bb424b362149a12949dfbbf1e06811a88f2307276b0c131afd \ + --hash=sha256:1d060c70284127fa805085d8f10fbd0962792aed71879d00864acda69dbab981 \ + --hash=sha256:42cb091236206bb2016d245c377ed383aa7f78691748f3bb6ee1bfa51ae2ce6a \ + --hash=sha256:473b32699f4200e69801bf5abf93f1a4ecd432a70984df164fc22ccf39c4a6f3 \ + --hash=sha256:54bef08bf00a2bff599982f6b08e8770e09cc012d7bba00783fc7ea38f1fb37d \ + --hash=sha256:5d72abdb8a4d56d4020713724ba81dac065fedb7f3667151c4a637f1d3fb26c0 \ + --hash=sha256:672132907fcad9f2aedcb705b2d7b3b93354a2aec1b2f706c4db852abe338f85 \ + --hash=sha256:6999421eb8ba9df4450a16d9184fcb7bef26240b9f98e95401f17af6c2210b71 \ + --hash=sha256:7b95a3fa7b3abb9b5b0e07668e808364d0d40f6bbbf9ae0faa8b5b210c97b140 \ + --hash=sha256:8469155f4cb518bafb4acf4865e8bb9d6804110d2d9bdcaa78564b9fd841e104 \ + --hash=sha256:94fd4858284736bb67a897a41608b5b0c2496c9bdb3bf2af1fa3409127f20d57 \ + --hash=sha256:b0f6d66c1c538d5a94a73aa9ddca8ccc4227e6c9ff555322ea40bdd142391dd4 \ + --hash=sha256:c74af94bf3ac15ac4d0f2a7c7b4663a15f8c2ab15ed0fc7531ca61d0835eccba \ + --hash=sha256:c82f4d474cf725255d9e6acf17252991c3c8aac038d6ef363a4bf8be2f6db517 \ + --hash=sha256:cdab83a366799fa730f90a4ebb563e494f28e9e92c4819e556152ad55e43591b \ + --hash=sha256:cfdead2f57330d76aa7234051dadfa7d4eedc0e5a27fd08e6f96714a92b00f09 \ + --hash=sha256:d1239932053f56f3456f32eb9625590cc7582e905021f94636202a864d470755 \ + --hash=sha256:dac7252938f0696ddea46f5e855dd3138444e82236e3be475f54929f0c510d48 \ + --hash=sha256:dc92bc2db7b45bda4510e4f51c59b00fe80b2d6be88928346e4294ce1c2abe7c \ + --hash=sha256:e07d91d0c92a31200f25351f4acb2bc6aff7f48094e13ebb1d0fb995b54b6542 \ + --hash=sha256:f4729811a6640d019a4b7ba8638ee2fd21fa5ca8c7e7bdf0fed62068fcaac737 + # via + # -r backend/mrt2-pytorch-runtime.in + # transformers +sentencepiece==0.2.1 \ + --hash=sha256:010f025a544ef770bb395091d57cb94deb9652d8972e0d09f71d85d5a0816c8c \ + --hash=sha256:017f97b274d4b0baa84b2dc743bf4517be81156f413bb24f12aacacde378e5ab \ + --hash=sha256:01e6912125cb45d3792f530a4d38f8e21bf884d6b4d4ade1b2de5cf7a8d2a52b \ + --hash=sha256:02593eca45440ef39247cee8c47322a34bdcc1d8ae83ad28ba5a899a2cf8d79a \ + --hash=sha256:097f3394e99456e9e4efba1737c3749d7e23563dd1588ce71a3d007f25475fff \ + --hash=sha256:0a0d15781a171d188b661ae4bde1d998c303f6bd8621498c50c671bd45a4798e \ + --hash=sha256:0a81799d0a68d618e89063fb423c3001a034c893069135ffe51fee439ae474d6 \ + --hash=sha256:0c0f672da370cc490e4c59d89e12289778310a0e71d176c541e4834759e1ae07 \ + --hash=sha256:0cdfecef430d985f1c2bcbfff3defd1d95dae876fbd0173376012d2d7d24044b \ + --hash=sha256:105e36e75cbac1292642045458e8da677b2342dcd33df503e640f0b457cb6751 \ + --hash=sha256:10ed3dab2044c47f7a2e7b4969b0c430420cdd45735d78c8f853191fa0e3148b \ + --hash=sha256:1855f57db07b51fb51ed6c9c452f570624d2b169b36f0f79ef71a6e6c618cd8b \ + --hash=sha256:2005242a16d2dc3ac5fe18aa7667549134d37854823df4c4db244752453b78a8 \ + --hash=sha256:22c4ebcb3c6ab1496ab1c37c79ef7bb563b8726f29548c30773b7a4cb152df1a \ + --hash=sha256:251874d720ac7f28024a168501f3c7bb15d1802245f6e66de565f18bbb9b5eaa \ + --hash=sha256:27e38eee653abc3d387862e67bc5c8b6f428cd604e688b85d29170b7e725c26c \ + --hash=sha256:2af5a1fb05013332ad94343b8b5f3973e006a2dde2dfba55a819549e054e2f0f \ + --hash=sha256:2f27ae6deea72efdb6f361750c92f6c21fd0ad087445082770cc34015213c526 \ + --hash=sha256:33f068c9382dc2e7c228eedfd8163b52baa86bb92f50d0488bf2b7da7032e484 \ + --hash=sha256:39f8651bd10974eafb9834ce30d9bcf5b73e1fc798a7f7d2528f9820ca86e119 \ + --hash=sha256:3d165fbb9bf8fba35f1946ba2617c3f9995679f07438325f07c026d53f33e746 \ + --hash=sha256:477c81505db072b3ab627e7eab972ea1025331bd3a92bacbf798df2b75ea86ec \ + --hash=sha256:4cdc7c36234fda305e85c32949c5211faaf8dd886096c7cea289ddc12a2d02de \ + --hash=sha256:4f5a3e0d9f445ed9d66c0fec47d4b23d12cfc858b407a03c194c1b26c2ac2a63 \ + --hash=sha256:56dd39a3c4d6493db3cdca7e8cc68c6b633f0d4195495cbadfcf5af8a22d05a6 \ + --hash=sha256:57cae326c8727de58c85977b175af132a7138d84c764635d7e71bbee7e774133 \ + --hash=sha256:5d0350b686c320068702116276cfb26c066dc7e65cfef173980b11bb4d606719 \ + --hash=sha256:5e4366c97b68218fd30ea72d70c525e6e78a6c0a88650f57ac4c43c63b234a9d \ + --hash=sha256:60937c959e6f44159fdd9f56fbdd302501f96114a5ba436829496d5f32d8de3f \ + --hash=sha256:6356d0986b8b8dc351b943150fcd81a1c6e6e4d439772e8584c64230e58ca987 \ + --hash=sha256:6d297a1748d429ba8534eebe5535448d78b8acc32d00a29b49acf28102eeb094 \ + --hash=sha256:733e59ff1794d26db706cd41fc2d7ca5f6c64a820709cb801dc0ea31780d64ab \ + --hash=sha256:8138cec27c2f2282f4a34d9a016e3374cd40e5c6e9cb335063db66a0a3b71fad \ + --hash=sha256:814978ac05130dd5812b4b03215c766bc6abaef13e7bd72bc534e4d1e12e9a4c \ + --hash=sha256:82d9ead6591015f009cb1be1cb1c015d5e6f04046dbb8c9588b931e869a29728 \ + --hash=sha256:881b2e44b14fc19feade3cbed314be37de639fc415375cefaa5bc81a4be137fd \ + --hash=sha256:891ade6503dd93d418c03993f7d6a8aa20260c422cefff5096b9068185e67642 \ + --hash=sha256:89a3ea015517c42c0341d0d962f3e6aaf2cf10d71b1932d475c44ba48d00aa2b \ + --hash=sha256:8dd4b477a7b069648d19363aad0cab9bad2f4e83b2d179be668efa672500dc94 \ + --hash=sha256:8f8ba89a3acb3dc1ae90f65ec1894b0b9596fdb98ab003ff38e058f898b39bc7 \ + --hash=sha256:9076430ac25dfa7147d9d05751dbc66a04bc1aaac371c07f84952979ea59f0d0 \ + --hash=sha256:92b3816aa2339355fda2c8c4e021a5de92180b00aaccaf5e2808972e77a4b22f \ + --hash=sha256:99f955df238021bf11f0fc37cdb54fd5e5b5f7fd30ecc3d93fb48b6815437167 \ + --hash=sha256:a19adcec27c524cb7069a1c741060add95f942d1cbf7ad0d104dffa0a7d28a2b \ + --hash=sha256:a483fd29a34c3e34c39ac5556b0a90942bec253d260235729e50976f5dba1068 \ + --hash=sha256:ac650534e2251083c5f75dde4ff28896ce7c8904133dc8fef42780f4d5588fcd \ + --hash=sha256:ad8493bea8432dae8d6830365352350f3b4144415a1d09c4c8cb8d30cf3b6c3c \ + --hash=sha256:afefe50a0cdcb4f2fd9733cb52001a2c164181ee2d82c32d38f5b1b326a8528c \ + --hash=sha256:b3616ad246f360e52c85781e47682d31abfb6554c779e42b65333d4b5f44ecc0 \ + --hash=sha256:b81a24733726e3678d2db63619acc5a8dccd074f7aa7a54ecd5ca33ca6d2d596 \ + --hash=sha256:c415c9de1447e0a74ae3fdb2e52f967cb544113a3a5ce3a194df185cbc1f962f \ + --hash=sha256:c6c8f42949f419ff8c7e9960dbadcfbc982d7b5efc2f6748210d3dd53a7de062 \ + --hash=sha256:c7f0fd2f2693309e6628aeeb2e2faf6edd221134dfccac3308ca0de01f8dab47 \ + --hash=sha256:c7f54a31cde6fa5cb030370566f68152a742f433f8d2be458463d06c208aef33 \ + --hash=sha256:c83b85ab2d6576607f31df77ff86f28182be4a8de6d175d2c33ca609925f5da1 \ + --hash=sha256:caa4e560c72c151da80036aecc2159e51a7fd8ae9efebefd96860460ce6bd025 \ + --hash=sha256:d3233770f78e637dc8b1fda2cd7c3b99ec77e7505041934188a4e7fe751de3b0 \ + --hash=sha256:d7b670879c370d350557edabadbad1f6561a9e6968126e6debca4029e5547820 \ + --hash=sha256:d8b1d91545578852f128650b8cce4ec20f93d39b378ff554ebe66290f2dabb92 \ + --hash=sha256:d9381351182ff9888cc80e41c632e7e274b106f450de33d67a9e8f6043da6f76 \ + --hash=sha256:daeb5e9e9fcad012324807856113708614d534f596d5008638eb9b40112cd9e4 \ + --hash=sha256:dcd8161eee7b41aae57ded06272905dbd680a0a04b91edd0f64790c796b2f706 \ + --hash=sha256:e10fa50bdbaa5e2445dbd387979980d391760faf0ec99a09bd7780ff37eaec44 \ + --hash=sha256:e37e4b4c4a11662b5db521def4e44d4d30ae69a1743241412a93ae40fdcab4bb \ + --hash=sha256:e52144670738b4b477fade6c2a9b6af71a8d0094514c9853ac9f6fc1fcfabae7 + # via -r backend/mrt2-pytorch-runtime.in +setuptools==81.0.0 \ + --hash=sha256:487b53915f52501f0a79ccfd0c02c165ffe06631443a886740b91af4b7a5845a \ + --hash=sha256:fdd925d5c5d9f62e4b74b30d6dd7828ce236fd6ed998a08d81de62ce5a6310d6 + # via torch +shellingham==1.5.4 \ + --hash=sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686 \ + --hash=sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de + # via typer +sympy==1.14.0 \ + --hash=sha256:d3d3fe8df1e5a0b42f0e7bdf50541697dbe7d23746e894990c030e2b05e72517 \ + --hash=sha256:e091cc3e99d2141a0ba2847328f5479b05d94a6635cb96148ccb3f34671bd8f5 + # via torch +tokenizers==0.22.2 \ + --hash=sha256:143b999bdc46d10febb15cbffb4207ddd1f410e2c755857b5a0797961bbdc113 \ + --hash=sha256:1a62ba2c5faa2dd175aaeed7b15abf18d20266189fb3406c5d0550dd34dd5f37 \ + --hash=sha256:1c774b1276f71e1ef716e5486f21e76333464f47bece56bbd554485982a9e03e \ + --hash=sha256:1e418a55456beedca4621dbab65a318981467a2b188e982a23e117f115ce5001 \ + --hash=sha256:1e50f8554d504f617d9e9d6e4c2c2884a12b388a97c5c77f0bc6cf4cd032feee \ + --hash=sha256:2249487018adec45d6e3554c71d46eb39fa8ea67156c640f7513eb26f318cec7 \ + --hash=sha256:25b85325d0815e86e0bac263506dd114578953b7b53d7de09a6485e4a160a7dd \ + --hash=sha256:29c30b83d8dcd061078b05ae0cb94d3c710555fbb44861139f9f83dcca3dc3e4 \ + --hash=sha256:319f659ee992222f04e58f84cbf407cfa66a65fe3a8de44e8ad2bc53e7d99012 \ + --hash=sha256:369cc9fc8cc10cb24143873a0d95438bb8ee257bb80c71989e3ee290e8d72c67 \ + --hash=sha256:37ae80a28c1d3265bb1f22464c856bd23c02a05bb211e56d0c5301a435be6c1a \ + --hash=sha256:38337540fbbddff8e999d59970f3c6f35a82de10053206a7562f1ea02d046fa5 \ + --hash=sha256:473b83b915e547aa366d1eee11806deaf419e17be16310ac0a14077f1e28f917 \ + --hash=sha256:544dd704ae7238755d790de45ba8da072e9af3eea688f698b137915ae959281c \ + --hash=sha256:64d94e84f6660764e64e7e0b22baa72f6cd942279fdbb21d46abd70d179f0195 \ + --hash=sha256:753d47ebd4542742ef9261d9da92cd545b2cacbb48349a1225466745bb866ec4 \ + --hash=sha256:791135ee325f2336f498590eb2f11dc5c295232f288e75c99a36c5dbce63088a \ + --hash=sha256:9ce725d22864a1e965217204946f830c37876eee3b2ba6fc6255e8e903d5fcbc \ + --hash=sha256:a6bf3f88c554a2b653af81f3204491c818ae2ac6fbc09e76ef4773351292bc92 \ + --hash=sha256:bfb88f22a209ff7b40a576d5324bf8286b519d7358663db21d6246fb17eea2d5 \ + --hash=sha256:c9ea31edff2968b44a88f97d784c2f16dc0729b8b143ed004699ebca91f05c48 \ + --hash=sha256:df6c4265b289083bf710dff49bc51ef252f9d5be33a45ee2bed151114a56207b \ + --hash=sha256:e10bf9113d209be7cd046d40fbabbaf3278ff6d18eb4da4c500443185dc1896c \ + --hash=sha256:f01a9c019878532f98927d2bacb79bbb404b43d3437455522a00a30718cdedb5 + # via transformers +torch==2.12.1+cu130 \ + --hash=sha256:046caae1457ec8a88256536958538dffb173c419320e3478f5b8d676038ec0d8 \ + --hash=sha256:0b4a8bdc9a89d1109bf4056d56e1c3ed05966273056d983a91f1cf597c25bea6 \ + --hash=sha256:1a4cd0c01ce52e4267147f0d860951ef8e937bafff613a9b5047010b5aacab11 \ + --hash=sha256:2a2bb858316615b90b14ff27d0c732d5af85d066f5ee7bf81fad2c9215839be7 \ + --hash=sha256:2d3e87d41ffb340ddf8c99e2a690a29feea9f5271459dd57621cd11317a434f2 \ + --hash=sha256:3b6e6e3ce55c3ebd688b00001cd44ff1a43fa30823f0394d20c8fd9910fb7087 \ + --hash=sha256:3d9cbeffed603075484270c34344e9506d86850af734d1e2eee13a8ba4bd690c \ + --hash=sha256:4bafc356fbb622e2756179406825c3a56c17b401196435a1487c5b40c657706c \ + --hash=sha256:52c5da6a0898d5d3473c02bd304b7a3bc0b72e351c6f3bfa0783e45ef9f4cd61 \ + --hash=sha256:5ff38932260cb4d5a52170d955642f6ede17f565de64e62eaca12a875851471b \ + --hash=sha256:6235f3a20b6094c7e8882c65a86a7f3c6e8c3ac8dbec0b57e575937d0166ed0f \ + --hash=sha256:76dd848312a40d29499614b714a4318841734ff309ad922f2365868395b6b054 \ + --hash=sha256:c84c5988b3e416669ed3790d772479af5934d1788119288d8bddd9c4cb207285 \ + --hash=sha256:d1cd8a4fd0556b2604db5447d9298323b8fba1ba67501fb3d8b22485c764a3a6 \ + --hash=sha256:d5e1840442d2182957b3d2f778cc325c90fa5cb42aa8b1ac949f029e9bdd7f06 \ + --hash=sha256:d6770341450d042a2998be19b5f7e431ab27281d356148a6eba659f5afff7c3e \ + --hash=sha256:f7f5ac061f7674917cacba9421f0a15e8df351057f90132cd46cccba9dfa7721 \ + --hash=sha256:fe0e04f34287aace6be33dd676307f9924108ad1a101c144e455a1eed0dc49e9 + # via -r backend/mrt2-pytorch-runtime.in +tqdm==4.70.0 \ + --hash=sha256:55b0b0dbd97462d06ebee91e4dac24ed4d4702be82b24f07e6c1d27e08cea220 \ + --hash=sha256:7f585706bfddbdebf89daac705b2dfcc16890130727d3197ca62c732b4310953 + # via + # huggingface-hub + # transformers +transformers==5.8.0 \ + --hash=sha256:6cc9a1f0291d16b1c1b735bad775e78ebefff7722701d4e28f98aaaa2bd6fb91 \ + --hash=sha256:e9d2cae6d195a7e1e05164c5ebf26142a7044e4dc4267274f4809204f92827e4 + # via -r backend/mrt2-pytorch-runtime.in +triton==3.7.1 \ + --hash=sha256:10ba85fa2cca4a2fbdeb36bf1cb082f2c252bda55bf9fccd74f65ec5bc647e68 \ + --hash=sha256:2020153b08280415ec0da6607834e79166442147e78e144df06b508c75b186d2 \ + --hash=sha256:3daf64305d6cea88d3334c65ebc9bcd0c64c9564a977084366aa768d57cbcf64 \ + --hash=sha256:58c0e131da05134a2a4788ccbcc0c1105cf0f54c8e98f19e34cd465396dc15eb \ + --hash=sha256:6744957e9fd610a29680ec2346057d0c86948ed3812468670719f391e94b44a5 \ + --hash=sha256:7e40869937a68206ec70d7f25bb7ec6433cb083f9135e1f36dbd318dc449a728 \ + --hash=sha256:9497f2e696ee368862a181a90b2dcc03ca978cc4f602abd67c7d81022a6988e1 \ + --hash=sha256:c58e4c61f0c73b5dba3b5d19b4a7093c32f90dc18b2a7f121a7c16ccd31107b7 \ + --hash=sha256:cdbfc09d9ec58bc5e68321525653220de7515c199e7a8097a97c85e62b52cd0a \ + --hash=sha256:d4a0e1cd4c4a76370ed74a8432a53cea28716827d19e40ffc732233e35ceb3f6 \ + --hash=sha256:ee89fbf782ec2ad50391dd1cf26cbea4f4467154c37f4773026da8fc31c0f58e \ + --hash=sha256:fe4ea396a06171f1f1f58cbd39c70b09294398f7dd7c620939bab54ad6f934fa + # via torch +typer==0.27.1 \ + --hash=sha256:53150287edd11baeb4e4722c8e394fcdf8181c0ae89485cba8d25c778d5edd56 \ + --hash=sha256:a79bef8469a79c45498e7b814ecf8d603cc7644e9acbd9e19cac0334240b18df + # via + # huggingface-hub + # transformers +typing-extensions==4.16.0 \ + --hash=sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8 \ + --hash=sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5 + # via + # anyio + # huggingface-hub + # torch diff --git a/backend/runtime-locks/mrt2-pytorch-windows-x86_64.txt b/backend/runtime-locks/mrt2-pytorch-windows-x86_64.txt new file mode 100644 index 0000000..14e2962 --- /dev/null +++ b/backend/runtime-locks/mrt2-pytorch-windows-x86_64.txt @@ -0,0 +1,709 @@ +# This file was autogenerated by uv via the following command: +# just lock-mrt2-pytorch +annotated-doc==0.0.5 \ + --hash=sha256:117bac03a25ede5df5440e855b32d556049ca169ead221505badf432fed4b101 \ + --hash=sha256:c7e58ce09192557605d8bbd92836d7e1d520ac9580096042c0bfd197efacf1bb + # via typer +anyio==4.14.2 \ + --hash=sha256:9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494 \ + --hash=sha256:cfa139f3ed1a23ee8f88a145ddb5ac7605b8bbfd8592baacd7ce3d8bb4313c7f + # via httpx +certifi==2026.7.22 \ + --hash=sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775 \ + --hash=sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55 + # via + # httpcore + # httpx +colorama==0.4.6 \ + --hash=sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44 \ + --hash=sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6 + # via + # tqdm + # typer +filelock==3.32.2 \ + --hash=sha256:87dd94cf281e586d135fa51132b8e3d9a598b316e90377a288663c9321036c82 \ + --hash=sha256:c33351e1f49cae33414acbc6d56784e6ecee82514ec90795da1161fc4836b5b8 + # via + # huggingface-hub + # torch +fsspec==2026.7.0 \ + --hash=sha256:b57ddbafedfaef7018c1ecab32aa200a9d7ca26b77965f64e48b70061249d279 \ + --hash=sha256:c803c40f4cf860b49dea58ee3e1c33cb9c790520e233537e1340049f89b82a88 + # via + # huggingface-hub + # torch +h11==0.16.0 \ + --hash=sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1 \ + --hash=sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86 + # via httpcore +hf-xet==1.6.0 \ + --hash=sha256:0e6e21fa3cdfcdcd76748564bf593870a5e013f47d97cf10aed63aa222cff5b7 \ + --hash=sha256:23379c2f9ec8696d952b16414a2bae72cad86a52df869b050698ba60f538c675 \ + --hash=sha256:2e58454a340b3556dfa4972d5451aff4fba8dd42a236600ba1a1d2b1514f0fef \ + --hash=sha256:35cec30d75c6f9eb9c16a77cef68e85a103b72e24d4b473714ec9ff06428bab9 \ + --hash=sha256:3dc3e35441ba395006af5aaacc40ef2e603c51ef46c3530b9156185f00935ea3 \ + --hash=sha256:4fc74352a17015bd0ee90038bc9efe38db894cde45f268b6712b04fce8cd0acb \ + --hash=sha256:5153e6bb103ad49d6ea9f1b2e230db5a2ea32551ad09a706d2f61d7c7c80d80e \ + --hash=sha256:5789835d7c6bc9436962853192082374297fb72d7eff7e7762ec25ceb7e25338 \ + --hash=sha256:633dc0cd71d32da58ab8c03ad38e2fac452c15c2b0a2866ebf6ededfe0a5061d \ + --hash=sha256:70cbb9c896901600128cb9b6f06e132954fbede1db30f31f7c6c63f84cb7c31d \ + --hash=sha256:75765820ce4700db3750c94acc8fe27c5fae4c9ec000a0dbac3ca082acf97765 \ + --hash=sha256:8fb4f71cba6129110c3374a33f919001ff130488fc23553698e34cc1c2a1198c \ + --hash=sha256:948f15d3a9545cfe5932f6bd8b440f6ae630aee108f14b7bd6c561f7c2dcc522 \ + --hash=sha256:d62671bb130879cef0ee4c9ebe47a14af6c66ec53e6d84dc15936e5ffdfac82f \ + --hash=sha256:f0906082d9932ae0c0057fa194041c22b4e2cdb46b2592ef3b91f020d62a081a \ + --hash=sha256:f2f7278c05c22fd60cb436cda1269649b3e81db65ecdc8496e5e164aa4143e7b \ + --hash=sha256:fb4fadde1b2b70bf4c0c14a6dccbe7194b1c28947fefd5bbe3fed9d940676c3b + # via huggingface-hub +httpcore==1.0.9 \ + --hash=sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55 \ + --hash=sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8 + # via httpx +httpx==0.28.1 \ + --hash=sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc \ + --hash=sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad + # via huggingface-hub +huggingface-hub==1.5.0 \ + --hash=sha256:c9c0b3ab95a777fc91666111f3b3ede71c0cdced3614c553a64e98920585c4ee \ + --hash=sha256:f281838db29265880fb543de7a23b0f81d3504675de82044307ea3c6c62f799d + # via + # -r backend/mrt2-pytorch-runtime.in + # tokenizers + # transformers +idna==3.18 \ + --hash=sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2 \ + --hash=sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848 + # via + # anyio + # httpx +jinja2==3.1.6 \ + --hash=sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d \ + --hash=sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67 + # via torch +llvmlite==0.48.0 \ + --hash=sha256:02853fe4214acb3780fc920c3fee10564b61d58a35e1b78afcc8a546c2deaba3 \ + --hash=sha256:054aa7d46595935565f276cf0c1659b4f10929c996dd4a606875fae26fba2a23 \ + --hash=sha256:05f0103c8f2f96a37441337e3643863c01b8e83e530aff38960dcb383c54a065 \ + --hash=sha256:13532f248960ba888ad5ab8150494e2f3a3d20e5f59f264e63741ea5b0ba844c \ + --hash=sha256:1d9f952dff6c350c529997423d4fa43abae9722a884ac7ffafc37a3af676e7db \ + --hash=sha256:2fe5cb59b2063bfa039dcb8ca6481c0181bf552f340d10dcf61d7996a665556e \ + --hash=sha256:321f1ac39b462603f0b589751aecf2d237d056f6d005749c1752b6f23ec3f074 \ + --hash=sha256:37d66fae72802175b0bfe1ea06e624b51e2d7aee6c3c34bbd09739b8f88e8e0b \ + --hash=sha256:3ee0c77685a18f5fca994ae21d0763007fca5c5c64b41de37accc78b69079176 \ + --hash=sha256:416fa4c2c66c2c6dc6d0a402648c19206e548efa0aa1eff01ad5cdad0af8217d \ + --hash=sha256:543b19f9ef8f3c7c60d1468191e4ee1b1537bf9f8a3d56f64c0ddd98de92edd2 \ + --hash=sha256:56a7e24607d3f02d7b1bae8d29c7e1e423d53143d68b072999777f19678fe77b \ + --hash=sha256:614aad57df707e3172efd5165f2aa7da6a0c6897e40dce590bf756396815ba76 \ + --hash=sha256:6fa532d6bb3fd3f0803567c736401c54aecfe1a396d3ad25d2440d220e09f0e7 \ + --hash=sha256:7a5c413317050a1d67c34708bde97707f9b2257ef1017f7532d21fe7d9a9ff30 \ + --hash=sha256:8761b9e522f55207e24424fcd98370289eec2710bf8e915c82d1053f642450dc \ + --hash=sha256:91c7e24e74cde3f02b88aa5acca678373f9e069f3b98531b3dbb3a142d9d10bb \ + --hash=sha256:966dcab0a598e2bd8fb5f2cc082cf7b07bae564fc485a3a8692393caf986facf \ + --hash=sha256:979a66a3f28a02565383ff463527dce78e9b856298872a361283132488e83591 \ + --hash=sha256:a83a99ef0c05b4ccddf9b6218ed9fe84b653a0caf7c1d9dbe148d6d16c67f518 \ + --hash=sha256:d0b3c61aac83b42fb48cc96bffbf57c81b82b2aa92276b7ed6420c814629a99a \ + --hash=sha256:d45c7541a80934ec6d8ab0defe67439494ecd2193cbf852a44ba827808976ac1 \ + --hash=sha256:d66c3beb4209087ddd4cf4ed2a0856b6887e6a913bdcf1aacfec9851cf2cba4e \ + --hash=sha256:efaee0276e5e17c2b99b92e0c974bd484ef5977cf5dbc9168e82b71578edb47f \ + --hash=sha256:f5e5a5131045b72345c71062ea1a91910dde913792b6c9b28ebb2c1c0a712e98 + # via numba +markdown-it-py==4.2.0 \ + --hash=sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49 \ + --hash=sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a + # via rich +markupsafe==3.0.3 \ + --hash=sha256:0303439a41979d9e74d18ff5e2dd8c43ed6c6001fd40e5bf2e43f7bd9bbc523f \ + --hash=sha256:068f375c472b3e7acbe2d5318dea141359e6900156b5b2ba06a30b169086b91a \ + --hash=sha256:0bf2a864d67e76e5c9a34dc26ec616a66b9888e25e7b9460e1c76d3293bd9dbf \ + --hash=sha256:0db14f5dafddbb6d9208827849fad01f1a2609380add406671a26386cdf15a19 \ + --hash=sha256:0eb9ff8191e8498cca014656ae6b8d61f39da5f95b488805da4bb029cccbfbaf \ + --hash=sha256:0f4b68347f8c5eab4a13419215bdfd7f8c9b19f2b25520968adfad23eb0ce60c \ + --hash=sha256:1085e7fbddd3be5f89cc898938f42c0b3c711fdcb37d75221de2666af647c175 \ + --hash=sha256:116bb52f642a37c115f517494ea5feb03889e04df47eeff5b130b1808ce7c219 \ + --hash=sha256:12c63dfb4a98206f045aa9563db46507995f7ef6d83b2f68eda65c307c6829eb \ + --hash=sha256:133a43e73a802c5562be9bbcd03d090aa5a1fe899db609c29e8c8d815c5f6de6 \ + --hash=sha256:1353ef0c1b138e1907ae78e2f6c63ff67501122006b0f9abad68fda5f4ffc6ab \ + --hash=sha256:15d939a21d546304880945ca1ecb8a039db6b4dc49b2c5a400387cdae6a62e26 \ + --hash=sha256:177b5253b2834fe3678cb4a5f0059808258584c559193998be2601324fdeafb1 \ + --hash=sha256:1872df69a4de6aead3491198eaf13810b565bdbeec3ae2dc8780f14458ec73ce \ + --hash=sha256:1b4b79e8ebf6b55351f0d91fe80f893b4743f104bff22e90697db1590e47a218 \ + --hash=sha256:1b52b4fb9df4eb9ae465f8d0c228a00624de2334f216f178a995ccdcf82c4634 \ + --hash=sha256:1ba88449deb3de88bd40044603fafffb7bc2b055d626a330323a9ed736661695 \ + --hash=sha256:1cc7ea17a6824959616c525620e387f6dd30fec8cb44f649e31712db02123dad \ + --hash=sha256:218551f6df4868a8d527e3062d0fb968682fe92054e89978594c28e642c43a73 \ + --hash=sha256:26a5784ded40c9e318cfc2bdb30fe164bdb8665ded9cd64d500a34fb42067b1c \ + --hash=sha256:2713baf880df847f2bece4230d4d094280f4e67b1e813eec43b4c0e144a34ffe \ + --hash=sha256:2a15a08b17dd94c53a1da0438822d70ebcd13f8c3a95abe3a9ef9f11a94830aa \ + --hash=sha256:2f981d352f04553a7171b8e44369f2af4055f888dfb147d55e42d29e29e74559 \ + --hash=sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa \ + --hash=sha256:3524b778fe5cfb3452a09d31e7b5adefeea8c5be1d43c4f810ba09f2ceb29d37 \ + --hash=sha256:3537e01efc9d4dccdf77221fb1cb3b8e1a38d5428920e0657ce299b20324d758 \ + --hash=sha256:35add3b638a5d900e807944a078b51922212fb3dedb01633a8defc4b01a3c85f \ + --hash=sha256:38664109c14ffc9e7437e86b4dceb442b0096dfe3541d7864d9cbe1da4cf36c8 \ + --hash=sha256:3a7e8ae81ae39e62a41ec302f972ba6ae23a5c5396c8e60113e9066ef893da0d \ + --hash=sha256:3b562dd9e9ea93f13d53989d23a7e775fdfd1066c33494ff43f5418bc8c58a5c \ + --hash=sha256:457a69a9577064c05a97c41f4e65148652db078a3a509039e64d3467b9e7ef97 \ + --hash=sha256:4bd4cd07944443f5a265608cc6aab442e4f74dff8088b0dfc8238647b8f6ae9a \ + --hash=sha256:4e885a3d1efa2eadc93c894a21770e4bc67899e3543680313b09f139e149ab19 \ + --hash=sha256:4faffd047e07c38848ce017e8725090413cd80cbc23d86e55c587bf979e579c9 \ + --hash=sha256:509fa21c6deb7a7a273d629cf5ec029bc209d1a51178615ddf718f5918992ab9 \ + --hash=sha256:5678211cb9333a6468fb8d8be0305520aa073f50d17f089b5b4b477ea6e67fdc \ + --hash=sha256:591ae9f2a647529ca990bc681daebdd52c8791ff06c2bfa05b65163e28102ef2 \ + --hash=sha256:5a7d5dc5140555cf21a6fefbdbf8723f06fcd2f63ef108f2854de715e4422cb4 \ + --hash=sha256:69c0b73548bc525c8cb9a251cddf1931d1db4d2258e9599c28c07ef3580ef354 \ + --hash=sha256:6b5420a1d9450023228968e7e6a9ce57f65d148ab56d2313fcd589eee96a7a50 \ + --hash=sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698 \ + --hash=sha256:729586769a26dbceff69f7a7dbbf59ab6572b99d94576a5592625d5b411576b9 \ + --hash=sha256:77f0643abe7495da77fb436f50f8dab76dbc6e5fd25d39589a0f1fe6548bfa2b \ + --hash=sha256:795e7751525cae078558e679d646ae45574b47ed6e7771863fcc079a6171a0fc \ + --hash=sha256:7be7b61bb172e1ed687f1754f8e7484f1c8019780f6f6b0786e76bb01c2ae115 \ + --hash=sha256:7c3fb7d25180895632e5d3148dbdc29ea38ccb7fd210aa27acbd1201a1902c6e \ + --hash=sha256:7e68f88e5b8799aa49c85cd116c932a1ac15caaa3f5db09087854d218359e485 \ + --hash=sha256:83891d0e9fb81a825d9a6d61e3f07550ca70a076484292a70fde82c4b807286f \ + --hash=sha256:8485f406a96febb5140bfeca44a73e3ce5116b2501ac54fe953e488fb1d03b12 \ + --hash=sha256:8709b08f4a89aa7586de0aadc8da56180242ee0ada3999749b183aa23df95025 \ + --hash=sha256:8f71bc33915be5186016f675cd83a1e08523649b0e33efdb898db577ef5bb009 \ + --hash=sha256:915c04ba3851909ce68ccc2b8e2cd691618c4dc4c4232fb7982bca3f41fd8c3d \ + --hash=sha256:949b8d66bc381ee8b007cd945914c721d9aba8e27f71959d750a46f7c282b20b \ + --hash=sha256:94c6f0bb423f739146aec64595853541634bde58b2135f27f61c1ffd1cd4d16a \ + --hash=sha256:9a1abfdc021a164803f4d485104931fb8f8c1efd55bc6b748d2f5774e78b62c5 \ + --hash=sha256:9b79b7a16f7fedff2495d684f2b59b0457c3b493778c9eed31111be64d58279f \ + --hash=sha256:a320721ab5a1aba0a233739394eb907f8c8da5c98c9181d1161e77a0c8e36f2d \ + --hash=sha256:a4afe79fb3de0b7097d81da19090f4df4f8d3a2b3adaa8764138aac2e44f3af1 \ + --hash=sha256:ad2cf8aa28b8c020ab2fc8287b0f823d0a7d8630784c31e9ee5edea20f406287 \ + --hash=sha256:b8512a91625c9b3da6f127803b166b629725e68af71f8184ae7e7d54686a56d6 \ + --hash=sha256:bc51efed119bc9cfdf792cdeaa4d67e8f6fcccab66ed4bfdd6bde3e59bfcbb2f \ + --hash=sha256:bdc919ead48f234740ad807933cdf545180bfbe9342c2bb451556db2ed958581 \ + --hash=sha256:bdd37121970bfd8be76c5fb069c7751683bdf373db1ed6c010162b2a130248ed \ + --hash=sha256:be8813b57049a7dc738189df53d69395eba14fb99345e0a5994914a3864c8a4b \ + --hash=sha256:c0c0b3ade1c0b13b936d7970b1d37a57acde9199dc2aecc4c336773e1d86049c \ + --hash=sha256:c47a551199eb8eb2121d4f0f15ae0f923d31350ab9280078d1e5f12b249e0026 \ + --hash=sha256:c4ffb7ebf07cfe8931028e3e4c85f0357459a3f9f9490886198848f4fa002ec8 \ + --hash=sha256:ccfcd093f13f0f0b7fdd0f198b90053bf7b2f02a3927a30e63f3ccc9df56b676 \ + --hash=sha256:d2ee202e79d8ed691ceebae8e0486bd9a2cd4794cec4824e1c99b6f5009502f6 \ + --hash=sha256:d53197da72cc091b024dd97249dfc7794d6a56530370992a5e1a08983ad9230e \ + --hash=sha256:d6dd0be5b5b189d31db7cda48b91d7e0a9795f31430b7f271219ab30f1d3ac9d \ + --hash=sha256:d88b440e37a16e651bda4c7c2b930eb586fd15ca7406cb39e211fcff3bf3017d \ + --hash=sha256:de8a88e63464af587c950061a5e6a67d3632e36df62b986892331d4620a35c01 \ + --hash=sha256:df2449253ef108a379b8b5d6b43f4b1a8e81a061d6537becd5582fba5f9196d7 \ + --hash=sha256:e1c1493fb6e50ab01d20a22826e57520f1284df32f2d8601fdd90b6304601419 \ + --hash=sha256:e1cf1972137e83c5d4c136c43ced9ac51d0e124706ee1c8aa8532c1287fa8795 \ + --hash=sha256:e2103a929dfa2fcaf9bb4e7c091983a49c9ac3b19c9061b6d5427dd7d14d81a1 \ + --hash=sha256:e56b7d45a839a697b5eb268c82a71bd8c7f6c94d6fd50c3d577fa39a9f1409f5 \ + --hash=sha256:e8afc3f2ccfa24215f8cb28dcf43f0113ac3c37c2f0f0806d8c70e4228c5cf4d \ + --hash=sha256:e8fc20152abba6b83724d7ff268c249fa196d8259ff481f3b1476383f8f24e42 \ + --hash=sha256:eaa9599de571d72e2daf60164784109f19978b327a3910d3e9de8c97b5b70cfe \ + --hash=sha256:ec15a59cf5af7be74194f7ab02d0f59a62bdcf1a537677ce67a2537c9b87fcda \ + --hash=sha256:f190daf01f13c72eac4efd5c430a8de82489d9cff23c364c3ea822545032993e \ + --hash=sha256:f34c41761022dd093b4b6896d4810782ffbabe30f2d443ff5f083e0cbbb8c737 \ + --hash=sha256:f3e98bb3798ead92273dc0e5fd0f31ade220f59a266ffd8a4f6065e0a3ce0523 \ + --hash=sha256:f42d0984e947b8adf7dd6dde396e720934d12c506ce84eea8476409563607591 \ + --hash=sha256:f71a396b3bf33ecaa1626c255855702aca4d3d9fea5e051b41ac59a9c1c41edc \ + --hash=sha256:f9e130248f4462aaa8e2552d547f36ddadbeaa573879158d721bbd33dfe4743a \ + --hash=sha256:fed51ac40f757d41b7c48425901843666a6677e3e8eb0abcff09e4ba6e664f50 + # via jinja2 +mdurl==0.1.2 \ + --hash=sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8 \ + --hash=sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba + # via markdown-it-py +mpmath==1.3.0 \ + --hash=sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f \ + --hash=sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c + # via sympy +networkx==3.6.1 \ + --hash=sha256:26b7c357accc0c8cde558ad486283728b65b6a95d85ee1cd66bafab4c8168509 \ + --hash=sha256:d47fbf302e7d9cbbb9e2555a0d267983d2aa476bac30e90dfbe5669bd57f3762 + # via torch +numba==0.66.0 \ + --hash=sha256:0999e3ee1b18c48e1fb51d11af35ef59852c7f4f50569c9550c25faef0616ad1 \ + --hash=sha256:2338cc0d43609fe448930848fd35a5bc688761b986f81b597a6f45cc0f8c9577 \ + --hash=sha256:27951c47e0def9bf8afe580eb961102902e2fd23cb77924b7d9d7cc0f8b444cb \ + --hash=sha256:380b2556a2019ccd1e956ae77dd257eaa39403f7520768b626d44b755112785e \ + --hash=sha256:46ae5f2b19e2af3c33c2df100306a90ea2f981c8158b0390f8bf6c20eee7357e \ + --hash=sha256:53ca5900b7cab15109796030113a6b28576bae5ad7bb507ad6dd1360ddd81ba4 \ + --hash=sha256:63b943eb2c9ba371908ce2cd6dfc643db51fc40f7966993376a1701bc922f537 \ + --hash=sha256:651a2b53298340956db26ecbe7ab043106b50a40c2807e66d617a4917245a4ab \ + --hash=sha256:7c14c044c06b453ec3fa7715dfe75425e2ba72c73377a7ffde6d9ec511dfd94c \ + --hash=sha256:8c1144ba1720ea59ad79f4f488ed54d149b2613b357f7e445678b7d0739c70e9 \ + --hash=sha256:939316d5d8619751207b8972a67852b5a7646665298cb4de693cd6bf135152f4 \ + --hash=sha256:aac69f3ccb8af100f5913c1241edc9692bad1cdd2508721713f426eb06c9a659 \ + --hash=sha256:b075a4e7ebc43dc6294f223e2821659656209fd5e0ce53245877c23d66d6e1a9 \ + --hash=sha256:b900e63a0e26c05ea9a6d5a3a5a0a177cb64c5011887bf43edb8c3ed2c38d363 \ + --hash=sha256:bbd531c327557a9004507fa6bff06c53ab51a7a5776b75261bb9cef1efe2b2ea \ + --hash=sha256:bd57790acd20f6a468e0ad333ef6b82355e309a92310fb7dff80e919f01a21a9 \ + --hash=sha256:c5bfe5350284509ab0474390321454c3a8627a188af5b68c910e83df3e2db4a7 \ + --hash=sha256:cc408c54b450f41582f4be1608f8981c1dcc44c7f40355cc150dd93015753407 \ + --hash=sha256:cdf506775d9f02eb92a87bf5c5b1e0d25506fd18cafd769f4ed914a8feac73e7 \ + --hash=sha256:d426178fb991a85714c43112a8ea7b9d9579ea856ad8dcdb9c1c3941903ba5be \ + --hash=sha256:db7735d15ea17a283d6485b9fa3504769f78fd86e5146638ad5e8da57c031b9e \ + --hash=sha256:e2b101f23b8b63978d334574d2039f27f0dccfe1d891756f33a2e2f3e4c88cf4 \ + --hash=sha256:efe0d2d5099790df945e0cb6e1b3104bd965d7bbfac50d62f1d5d1d6ade0825d \ + --hash=sha256:fb601841d9e02e6237bb6522e36d0741614be3cfe2b482a6f00a41b5ba209443 \ + --hash=sha256:fc6629becb21a867d85401ec89f426dd24c484a4193ade8a38309debfd1529ca + # via resampy +numpy==2.3.5 \ + --hash=sha256:00dc4e846108a382c5869e77c6ed514394bdeb3403461d25a829711041217d5b \ + --hash=sha256:0472f11f6ec23a74a906a00b48a4dcf3849209696dff7c189714511268d103ae \ + --hash=sha256:04822c00b5fd0323c8166d66c701dc31b7fbd252c100acd708c48f763968d6a3 \ + --hash=sha256:052e8c42e0c49d2575621c158934920524f6c5da05a1d3b9bab5d8e259e045f0 \ + --hash=sha256:09a1bea522b25109bf8e6f3027bd810f7c1085c64a0c7ce050c1676ad0ba010b \ + --hash=sha256:0cd00b7b36e35398fa2d16af7b907b65304ef8bb4817a550e06e5012929830fa \ + --hash=sha256:0d8163f43acde9a73c2a33605353a4f1bc4798745a8b1d73183b28e5b435ae28 \ + --hash=sha256:1062fde1dcf469571705945b0f221b73928f34a20c904ffb45db101907c3454e \ + --hash=sha256:11e06aa0af8c0f05104d56450d6093ee639e15f24ecf62d417329d06e522e017 \ + --hash=sha256:17531366a2e3a9e30762c000f2c43a9aaa05728712e25c11ce1dbe700c53ad41 \ + --hash=sha256:1978155dd49972084bd6ef388d66ab70f0c323ddee6f693d539376498720fb7e \ + --hash=sha256:1ed1ec893cff7040a02c8aa1c8611b94d395590d553f6b53629a4461dc7f7b63 \ + --hash=sha256:2dcd0808a421a482a080f89859a18beb0b3d1e905b81e617a188bd80422d62e9 \ + --hash=sha256:2e2eb32ddb9ccb817d620ac1d8dae7c3f641c1e5f55f531a33e8ab97960a75b8 \ + --hash=sha256:2feae0d2c91d46e59fcd62784a3a83b3fb677fead592ce51b5a6fbb4f95965ff \ + --hash=sha256:3095bdb8dd297e5920b010e96134ed91d852d81d490e787beca7e35ae1d89cf7 \ + --hash=sha256:30bc11310e8153ca664b14c5f1b73e94bd0503681fcf136a163de856f3a50139 \ + --hash=sha256:3101e5177d114a593d79dd79658650fe28b5a0d8abeb8ce6f437c0e6df5be1a4 \ + --hash=sha256:396084a36abdb603546b119d96528c2f6263921c50df3c8fd7cb28873a237748 \ + --hash=sha256:3997b5b3c9a771e157f9aae01dd579ee35ad7109be18db0e85dbdbe1de06e952 \ + --hash=sha256:414802f3b97f3c1eef41e530aaba3b3c1620649871d8cb38c6eaff034c2e16bd \ + --hash=sha256:51c1e14eb1e154ebd80e860722f9e6ed6ec89714ad2db2d3aa33c31d7c12179b \ + --hash=sha256:51c55fe3451421f3a6ef9a9c1439e82101c57a2c9eab9feb196a62b1a10b58ce \ + --hash=sha256:5ee6609ac3604fa7780e30a03e5e241a7956f8e2fcfe547d51e3afa5247ac47f \ + --hash=sha256:612a95a17655e213502f60cfb9bf9408efdc9eb1d5f50535cc6eb365d11b42b5 \ + --hash=sha256:6203fdf9f3dc5bdaed7319ad8698e685c7a3be10819f41d32a0723e611733b42 \ + --hash=sha256:63c0e9e7eea69588479ebf4a8a270d5ac22763cc5854e9a7eae952a3908103f7 \ + --hash=sha256:66f85ce62c70b843bab1fb14a05d5737741e74e28c7b8b5a064de10142fad248 \ + --hash=sha256:6cf9b429b21df6b99f4dee7a1218b8b7ffbbe7df8764dc0bd60ce8a0708fed1e \ + --hash=sha256:70b37199913c1bd300ff6e2693316c6f869c7ee16378faf10e4f5e3275b299c3 \ + --hash=sha256:727fd05b57df37dc0bcf1a27767a3d9a78cbbc92822445f32cc3436ba797337b \ + --hash=sha256:74ae7b798248fe62021dbf3c914245ad45d1a6b0cb4a29ecb4b31d0bfbc4cc3e \ + --hash=sha256:784db1dcdab56bf0517743e746dfb0f885fc68d948aba86eeec2cba234bdf1c0 \ + --hash=sha256:86945f2ee6d10cdfd67bcb4069c1662dd711f7e2a4343db5cecec06b87cf31aa \ + --hash=sha256:86d835afea1eaa143012a2d7a3f45a3adce2d7adc8b4961f0b362214d800846a \ + --hash=sha256:872a5cf366aec6bb1147336480fef14c9164b154aeb6542327de4970282cd2f5 \ + --hash=sha256:8b973c57ff8e184109db042c842423ff4f60446239bd585a5131cc47f06f789d \ + --hash=sha256:8cba086a43d54ca804ce711b2a940b16e452807acebe7852ff327f1ecd49b0d4 \ + --hash=sha256:8f7f0e05112916223d3f438f293abf0727e1181b5983f413dfa2fefc4098245c \ + --hash=sha256:900218e456384ea676e24ea6a0417f030a3b07306d29d7ad843957b40a9d8d52 \ + --hash=sha256:93eebbcf1aafdf7e2ddd44c2923e2672e1010bddc014138b229e49725b4d6be5 \ + --hash=sha256:9c75442b2209b8470d6d5d8b1c25714270686f14c749028d2199c54e29f20b4d \ + --hash=sha256:9ee2197ef8c4f0dfe405d835f3b6a14f5fee7782b5de51ba06fb65fc9b36e9f1 \ + --hash=sha256:a414504bef8945eae5f2d7cb7be2d4af77c5d1cb5e20b296c2c25b61dff2900c \ + --hash=sha256:a4b9159734b326535f4dd01d947f919c6eefd2d9827466a696c44ced82dfbc18 \ + --hash=sha256:a80afd79f45f3c4a7d341f13acbe058d1ca8ac017c165d3fa0d3de6bc1a079d7 \ + --hash=sha256:aa5bc7c5d59d831d9773d1170acac7893ce3a5e130540605770ade83280e7188 \ + --hash=sha256:acfd89508504a19ed06ef963ad544ec6664518c863436306153e13e94605c218 \ + --hash=sha256:aeffcab3d4b43712bb7a60b65f6044d444e75e563ff6180af8f98dd4b905dfd2 \ + --hash=sha256:afaffc4393205524af9dfa400fa250143a6c3bc646c08c9f5e25a9f4b4d6a903 \ + --hash=sha256:b0c7088a73aef3d687c4deef8452a3ac7c1be4e29ed8bf3b366c8111128ac60c \ + --hash=sha256:b46b4ec24f7293f23adcd2d146960559aaf8020213de8ad1909dba6c013bf89c \ + --hash=sha256:b501b5fa195cc9e24fe102f21ec0a44dffc231d2af79950b451e0d99cea02234 \ + --hash=sha256:bf06bc2af43fa8d32d30fae16ad965663e966b1a3202ed407b84c989c3221e82 \ + --hash=sha256:c804e3a5aba5460c73955c955bdbd5c08c354954e9270a2c1565f62e866bdc39 \ + --hash=sha256:c8a9958e88b65c3b27e22ca2a076311636850b612d6bbfb76e8d156aacde2aaf \ + --hash=sha256:cc0a57f895b96ec78969c34f682c602bf8da1a0270b09bc65673df2e7638ec20 \ + --hash=sha256:cc8920d2ec5fa99875b670bb86ddeb21e295cb07aa331810d9e486e0b969d946 \ + --hash=sha256:ccc933afd4d20aad3c00bcef049cb40049f7f196e0397f1109dba6fed63267b0 \ + --hash=sha256:ce581db493ea1a96c0556360ede6607496e8bf9b3a8efa66e06477267bc831e9 \ + --hash=sha256:d0f23b44f57077c1ede8c5f26b30f706498b4862d3ff0a7298b8411dd2f043ff \ + --hash=sha256:d21644de1b609825ede2f48be98dfde4656aefc713654eeee280e37cadc4e0ad \ + --hash=sha256:d6889ec4ec662a1a37eb4b4fb26b6100841804dac55bd9df579e326cdc146227 \ + --hash=sha256:de5672f4a7b200c15a4127042170a694d4df43c992948f5e1af57f0174beed10 \ + --hash=sha256:e6a0bc88393d65807d751a614207b7129a310ca4fe76a74e5c7da5fa5671417e \ + --hash=sha256:ed89927b86296067b4f81f108a2271d8926467a8868e554eaf370fc27fa3ccaf \ + --hash=sha256:ee3888d9ff7c14604052b2ca5535a30216aa0a58e948cdd3eeb8d3415f638769 \ + --hash=sha256:f0963b55cdd70fad460fa4c1341f12f976bb26cb66021a5580329bd498988310 \ + --hash=sha256:f16417ec91f12f814b10bafe79ef77e70113a2f5f7018640e7425ff979253425 \ + --hash=sha256:f28620fe26bee16243be2b7b874da327312240a7cdc38b769a697578d2100013 \ + --hash=sha256:f4255143f5160d0de972d28c8f9665d882b5f61309d8362fdd3e103cf7bf010c \ + --hash=sha256:ffac52f28a7849ad7576293c0cb7b9f08304e8f7d738a8cb8a90ec4c55a998eb \ + --hash=sha256:ffe22d2b05504f786c867c8395de703937f934272eb67586817b46188b4ded6d \ + --hash=sha256:fffe29a1ef00883599d1dc2c51aa2e5d80afe49523c261a74933df395c15c520 + # via + # -r backend/mrt2-pytorch-runtime.in + # numba + # resampy + # transformers +packaging==26.3 \ + --hash=sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79 \ + --hash=sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c + # via + # huggingface-hub + # transformers +pygments==2.20.0 \ + --hash=sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f \ + --hash=sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176 + # via rich +pyyaml==6.0.3 \ + --hash=sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c \ + --hash=sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a \ + --hash=sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3 \ + --hash=sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956 \ + --hash=sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6 \ + --hash=sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c \ + --hash=sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65 \ + --hash=sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a \ + --hash=sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0 \ + --hash=sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b \ + --hash=sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1 \ + --hash=sha256:22ba7cfcad58ef3ecddc7ed1db3409af68d023b7f940da23c6c2a1890976eda6 \ + --hash=sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7 \ + --hash=sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e \ + --hash=sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007 \ + --hash=sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310 \ + --hash=sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4 \ + --hash=sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9 \ + --hash=sha256:3ff07ec89bae51176c0549bc4c63aa6202991da2d9a6129d7aef7f1407d3f295 \ + --hash=sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea \ + --hash=sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0 \ + --hash=sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e \ + --hash=sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac \ + --hash=sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9 \ + --hash=sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7 \ + --hash=sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35 \ + --hash=sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb \ + --hash=sha256:5cf4e27da7e3fbed4d6c3d8e797387aaad68102272f8f9752883bc32d61cb87b \ + --hash=sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69 \ + --hash=sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5 \ + --hash=sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b \ + --hash=sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c \ + --hash=sha256:6344df0d5755a2c9a276d4473ae6b90647e216ab4757f8426893b5dd2ac3f369 \ + --hash=sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd \ + --hash=sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824 \ + --hash=sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198 \ + --hash=sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065 \ + --hash=sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c \ + --hash=sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c \ + --hash=sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764 \ + --hash=sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196 \ + --hash=sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b \ + --hash=sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00 \ + --hash=sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac \ + --hash=sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8 \ + --hash=sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e \ + --hash=sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28 \ + --hash=sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3 \ + --hash=sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5 \ + --hash=sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4 \ + --hash=sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b \ + --hash=sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf \ + --hash=sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5 \ + --hash=sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702 \ + --hash=sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8 \ + --hash=sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788 \ + --hash=sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da \ + --hash=sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d \ + --hash=sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc \ + --hash=sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c \ + --hash=sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba \ + --hash=sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f \ + --hash=sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917 \ + --hash=sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5 \ + --hash=sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26 \ + --hash=sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f \ + --hash=sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b \ + --hash=sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be \ + --hash=sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c \ + --hash=sha256:efd7b85f94a6f21e4932043973a7ba2613b059c4a000551892ac9f1d11f5baf3 \ + --hash=sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6 \ + --hash=sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926 \ + --hash=sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0 + # via + # huggingface-hub + # transformers +regex==2026.7.19 \ + --hash=sha256:062f8cb7a9739c4835d22bd96f370c59aba89f257adcfa53be3cc209e08d3ae0 \ + --hash=sha256:064f1760a5a4ade65c5419be23e782f29147528e8a66e0c42dd4cedb8d4e9fc6 \ + --hash=sha256:09523a592938aa9f587fb74467c63ff0cf88fc3df14c82ab0f0517dcf76aaa62 \ + --hash=sha256:09d3007fc76249a83cdd33de160d50e6cb77f54e09d8fa9e7148e10607ce24af \ + --hash=sha256:09f3e5287f94f17b709dc9a9e70865855feee835c861613be144218ce4ca82cc \ + --hash=sha256:0c41c63992bf1874cebb6e7f56fd7d3c007924659a604ae3d90e427d40d4fd13 \ + --hash=sha256:0e9554c8785eac5cffe6300f69a91f58ba72bc88a5f8d661235ad7c6aa5b8ccd \ + --hash=sha256:1123ef4211d763ee771d47916a1596e2f4915794f7aabdc1adcb20e4249a6951 \ + --hash=sha256:15b364b9b98d6d2fe1a85034c23a3180ff913f46caddc3895f6fd65186255ccc \ + --hash=sha256:1649eb39fcc9ea80c4d2f110fde2b8ab2aef3877b98f02ab9b14e961f418c511 \ + --hash=sha256:17ed5692f6acc4183e98331101a5f9e4f64d72fe58b753da4d444a2c77d05b12 \ + --hash=sha256:199535629f25caf89698039af3d1ad5fcae7f933e2112c73f1cdf49165c99518 \ + --hash=sha256:1c398716054621aa300b3d411f467dda903806c5da0df6945ab73982b8d115db \ + --hash=sha256:1d3372064506b94dd2c67c845f2db8062e9e9ba84d04e33cb96d7d33c11fe1ae \ + --hash=sha256:1d58561843f0ff7dc78b4c28b5e2dc388f3eff94ebc8a232a3adba961fc00009 \ + --hash=sha256:1d793a7988e04fcb1e2e135567443d82173225d657419ec09414a9b5a145b986 \ + --hash=sha256:1ebac3474b8589fce2f9b225b650afd61448f7c73a5d0255a10cc6366471aed1 \ + --hash=sha256:20568e182eb82d39a6bf7cff3fd58566f14c75c6f74b2c8c96537eecf9010e3a \ + --hash=sha256:22a992de9a0d91bda927bf02b94351d737a0302905432c88a53de7c4b9ce62e2 \ + --hash=sha256:2955907b7157a6660f27079edf7e0229e9c9c5325c77a2ef6a890cba91efa6f0 \ + --hash=sha256:2c4e61e2e1be56f63ec3cc618aa9e0de81ef6f43d177205451840022e24f5b78 \ + --hash=sha256:2cc3460cedf7579948486eab03bc9ad7089df4d7281c0f47f4afe03e8d13f02d \ + --hash=sha256:2ce9e679f776649746729b6c86382da519ef649c8e34cc41df0d2e5e0f6c36d4 \ + --hash=sha256:2ef7eeb108c47ce7bcc9513e51bcb1bf57e8f483d52fce68a8642e3527141ae0 \ + --hash=sha256:3080a7fd38ef049bd489e01c970c97dd84ff446a885b0f1f6b26d9b1ad13ce11 \ + --hash=sha256:343a4504e3fb688c47cad451221ca5d4814f42b1e16c0065bde9cbf7f473bd52 \ + --hash=sha256:36aacfb15faaff3ced55afbf35ec72f50d4aee22082c4f7fe0573a33e2fca92e \ + --hash=sha256:3d3143f159261b1ce5b24c261c590e5913370c3200c5e9ebbb92b5aa5e111902 \ + --hash=sha256:40b34dd88658e4fedd2fddbf0275ac970d00614b731357f425722a3ed1983d11 \ + --hash=sha256:4458124d71339f505bf1fb94f69fd1bb8fa9d2481eebfef27c10ef4f2b9e12f6 \ + --hash=sha256:4896db1f4ce0576765b8272aa922df324e0f5b9bb2c3d03044ff32a7234a9aba \ + --hash=sha256:4a0530bb1b8c1c985e7e2122e2b4d3aedd8a3c21c6bfddae6767c4405668b56e \ + --hash=sha256:4aa5435cdb3eb6f55fe98a171b05e3fbcd95fadaa4aa32acf62afd9b0cfdbcac \ + --hash=sha256:4c3501bfa814ab07b5580741f9bf78dfdfe146a04057f82df9e2402d2a975939 \ + --hash=sha256:4e5413bd5f13d3a4e3539ca98f70f75e7fca92518dd7f117f030ebedd10b60cb \ + --hash=sha256:4e6883a021db30511d9fb8cfb0f222ce1f2c369f7d4d8b0448f449a93ba0bdfc \ + --hash=sha256:52579c60a6078be70a0e49c81d6e56d677f34cd439af281a0083b8c7bc75c095 \ + --hash=sha256:555497390743af1a65045fa4527782d10ff5b88970359412baa4a1e628fe393b \ + --hash=sha256:56ad4d9f77df871a99e25c37091052a02528ec0eb059de928ee33956b854b45b \ + --hash=sha256:571fde9741eb0ccde23dd4e0c1d50fbae910e901fa7e629faf39b2dda740d220 \ + --hash=sha256:572fc57b0009c735ee56c175ea021b637a15551a312f56734277f923d6fd0f6c \ + --hash=sha256:59787bd5f8c70aa339084e961d2996b53fbdeab4d5393bba5c1fe1fc32e02bae \ + --hash=sha256:5a2721c8720e2cb3c209925dfb9200199b4b07361c9e01d321719404b21458b3 \ + --hash=sha256:5cc26a66e212fa5d6c6170c3a40d99d888db3020c6fdab1523250d4341382e44 \ + --hash=sha256:5ebee1ee89c39c953baac6924fcde08c5bb427c4057510862f9d7c7bdb3d8665 \ + --hash=sha256:60be8693a1dadc210bbcbc0db3e26da5f7d01d1d5a3da594e99b4fa42df404f5 \ + --hash=sha256:618a0aed532be87294c4477b0481f3aa0f1520f4014a4374dd4cf789b4cd2c97 \ + --hash=sha256:61bb1bd45520aacd56dd80943bd34991fb5350afdd1f36f2282230fd5154a218 \ + --hash=sha256:6383cd2ed53a646c659ba1fe65727db76437fdaa069e697a0b44a51d5843d864 \ + --hash=sha256:64729333167c2dcaaa56a331d40ee097bd9c5617ffd51dabb09eaddafb1b532e \ + --hash=sha256:64b6ca7391a1395c2638dd5c7456d67bea44fc6c5e8e92c5dc8aa6a8f23292b4 \ + --hash=sha256:65dcd28d3eba2ab7c2fd906485cc301392b47cc2234790d27d4e4814e02cdfda \ + --hash=sha256:65fa6cb38ed5e9c3637e68e544f598b39c3b86b808ed0627a67b68320384b459 \ + --hash=sha256:66bd62c59a5427746e8c44becae1d9b99d22fb13f30f492083dfb9ad7c45cc18 \ + --hash=sha256:6e44c0e7c5664be20aee92085153150c0a7967310a73a43c0f832b7cd35d0dd3 \ + --hash=sha256:6f8c6e7a1cfa3dc9d0ee2de0e65e834537fa29992cc3976ffec914afc35c5dd5 \ + --hash=sha256:7322ec6cc9fba9d49ab888bb82d67ac5625627aa168f0165139b17018df3fb8a \ + --hash=sha256:73b133a9e6fb512858e7f065e96f1180aa46646bc74a83aea62f1d314f3dd035 \ + --hash=sha256:73f272fba87b8ccfe70a137d02a54af386f6d27aa509fbffdd978f5947aae1aa \ + --hash=sha256:7e77b324909c1617cbb4c668677e2c6ae13f44d7c1de0d4f15f2e3c10f3315b5 \ + --hash=sha256:80115dd39481fd3a4b4080220799dbcacb921a844de4b827264ececacbe17c78 \ + --hash=sha256:87ccab0db8d5f4fbb0272642113c1adb2ffc698c16d3a0944580222331fa7a20 \ + --hash=sha256:89dfee3319f5ae3f75ebd5c2445a809bb320252ba5529ffdafea4ef25d79cf1a \ + --hash=sha256:8ac59a0900474a52b7c04af8196affc22bd9842acb0950df12f7b813e983609a \ + --hash=sha256:8cae6fd77a5b72dae505084b1a2ee0360139faf72fedbab667cd7cc65aae7a6a \ + --hash=sha256:8d3469c91dd92ee41b7c95280edbd975ef1ba9195086686623a1c6e8935ce965 \ + --hash=sha256:90c633e7e8d6bf4e992b8b36ce69e018f834b641dd6de8cea6d78c06ffa119c5 \ + --hash=sha256:93db40c8de0815baab96a06e08a984bac71f989d13bab789e382158c5d426797 \ + --hash=sha256:9724e6cb5e478cd7d8cabf027826178739cb18cf0e117d0e32814d479fa02276 \ + --hash=sha256:98c6ac18480fcdb33f35439183f1d2e79760ab41930309c6d951cb1f8e46694c \ + --hash=sha256:9a15e785f244f3e07847b984ce8773fc3da10a9f3c131cc49a4c5b4d672b4547 \ + --hash=sha256:9b60d7814174f059e5de4ab98271cc5ba9259cfea55273a81544dceea32dc8d9 \ + --hash=sha256:9be2a6647740dd3cca6acb24e87f03d7632cd280dbce9bbe40c26353a215a45d \ + --hash=sha256:9c7472192ebfad53a6be7c4a8bfb2d64b81c0e93a1fc8c57e1dd0b638297b5d1 \ + --hash=sha256:9dce8ec9695f531a1b8a6f314fd4b393adcccf2ea861db480cdf97a301d01a68 \ + --hash=sha256:9e50d748a32da622f256e8d505867f5d3c43a837c6a9f0efb149655fadd1042a \ + --hash=sha256:a81758ed242b861b72e778ba34d41366441a2e10b16b472784c88da2dea7e2dd \ + --hash=sha256:ac777001cdfc28b72477d93c8564bb7583081ea8fb45cdca3d568e0a4f87183c \ + --hash=sha256:b2b506b1788df5fecd270a10d5e70a95fe77b87ea2b370a318043f6f5f817ee6 \ + --hash=sha256:b2ea4a3e8357be8849e833beeae757ac3c7a6b3fc055c03c808a53c91ad30d82 \ + --hash=sha256:bf1516fe58fc104f39b2d1dbe2d5e27d0cd45c4be2e42ba6ee0cc763701ec3c7 \ + --hash=sha256:c0d702548d89d572b2929879bc883bb7a4c4709efafe4512cadee56c55c9bd15 \ + --hash=sha256:c10b82c2634df08dfb13b1f04e38fe310d086ee092f4f69c0c8da234251e556e \ + --hash=sha256:c42572142ed0b9d5d261ba727157c426510da78e20828b66bbb855098b8a4e38 \ + --hash=sha256:c4585c3e64b4f9e583b4d2683f18f5d5d872b3d71dcf24594b74ecc23602fa96 \ + --hash=sha256:c639ea314df70a7b2811e8020448c75af8c9445f5a60f8a4ced81c306a9380c2 \ + --hash=sha256:c670fe7be5b6020b76bc6e8d2196074657e1327595bca93a389e1a76ab130ad8 \ + --hash=sha256:cc1b2440423a851fad781309dd87843868f4f66a6bcd1ddb9225cf4ec2c84732 \ + --hash=sha256:cd3584591ea4429026cdb931b054342c2bcf189b44ff367f8d5c15bc092a2966 \ + --hash=sha256:d15df07081d91b76ff20d43f94592ee110330152d617b730fdbe5ef9fb680053 \ + --hash=sha256:d19662dbedbe783d323196312d38f5ba53cf56296378252171985da6899887d3 \ + --hash=sha256:d24ecb4f5e009ea0bd275ee37ad9953b32005e2e5e60f8bbae16da0dbbf0d3a0 \ + --hash=sha256:d446c6ac40bb6e05025ccee55b84d80fe9bf8e93010ffc4bb9484f13d498835f \ + --hash=sha256:d51ffd3427640fa2da6ade574ceba932f210ad095f65fcc450a2b0a0d454868e \ + --hash=sha256:d6ce43a0269d68cee79a7d1ade7def53c20f8f2a047b92d7b5d5bcc73ae88327 \ + --hash=sha256:d721e53758b2cca74990185eb0671dd466d7a388a1a45d0c6f4c13cef41a68ac \ + --hash=sha256:d7da47a0f248977f08e2cb659ff3c17ddc13a4d39b3a7baa0a81bf5b415430f6 \ + --hash=sha256:db47b561c9afd884baa1f96f797c9ca369872c4b65912bc691cfa99e68340af2 \ + --hash=sha256:dbe6493fbd27321b1d1f2dd4f5c7e5bd4d8b1d7cab7f32fd67db3d0b2ed8248a \ + --hash=sha256:dbece16025afda5e3031af0c4059207e61dcf73ef13af844964f57f387d1c435 \ + --hash=sha256:ddd67571c10869f65a5d7dde536d1e066e306cc90de57d7de4d5f34802428bb5 \ + --hash=sha256:de9208bb427130c82a5dbfd104f92c8876fc9559278c880b3002755bbbe9c83d \ + --hash=sha256:e30d40268a28d54ce0437031750497004c22602b8e3ab891f759b795a003b312 \ + --hash=sha256:e8b0abe7d870f53ca5143895fef7d1041a0c831a140d3dc2c760dd7ba25d4a8b \ + --hash=sha256:f035d9dc1d25eff9d361456572231c7d27b5ccd473ca7dc0adfce732bd006d40 \ + --hash=sha256:f04b9f56b0e0614c0126be12c2c2d9f8850c1e57af302bd0a63bed379d4af974 \ + --hash=sha256:f0fa4fa9c3632d708742baf2282f2055c11d888a790362670a403cbf48a2c404 \ + --hash=sha256:f2e7f8e2ab6c2922be02c7ec45185aa5bd771e2e57b95455ee343a44d8130dff \ + --hash=sha256:f8f6fa298bb4f7f58a33334406218ba74716e68feddf5e4e54cd5d8082705abf \ + --hash=sha256:fbf300e2070bb35038660b3be1be4b91b0024edb41517e6996320b49b92b4175 \ + --hash=sha256:fce7760bf283405b2c7999cab3da4e72f7deca6396013115e3f7a955db9760da \ + --hash=sha256:fcee38cd8e5089d6d4f048ba1233b3ad76e5954f545382180889112ff5cb712d \ + --hash=sha256:fe31f28c94402043161876a258a9c6f757cb485905c7614ce8d6cd40e6b7bdc1 \ + --hash=sha256:ffd8893ccc1c2fce6e0d6ca402d716fe1b29db70c7132609a05955e31b2aa8f2 + # via transformers +resampy==0.4.3 \ + --hash=sha256:a0d1c28398f0e55994b739650afef4e3974115edbe96cd4bb81968425e916e47 \ + --hash=sha256:ad2ed64516b140a122d96704e32bc0f92b23f45419e8b8f478e5a05f83edcebd + # via -r backend/mrt2-pytorch-runtime.in +rich==15.0.0 \ + --hash=sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb \ + --hash=sha256:edd07a4824c6b40189fb7ac9bc4c52536e9780fbbfbddf6f1e2502c31b068c36 + # via typer +safetensors==0.7.0 \ + --hash=sha256:0071bffba4150c2f46cae1432d31995d77acfd9f8db598b5d1a2ce67e8440ad2 \ + --hash=sha256:07663963b67e8bd9f0b8ad15bb9163606cd27cc5a1b96235a50d8369803b96b0 \ + --hash=sha256:12f49080303fa6bb424b362149a12949dfbbf1e06811a88f2307276b0c131afd \ + --hash=sha256:1d060c70284127fa805085d8f10fbd0962792aed71879d00864acda69dbab981 \ + --hash=sha256:42cb091236206bb2016d245c377ed383aa7f78691748f3bb6ee1bfa51ae2ce6a \ + --hash=sha256:473b32699f4200e69801bf5abf93f1a4ecd432a70984df164fc22ccf39c4a6f3 \ + --hash=sha256:54bef08bf00a2bff599982f6b08e8770e09cc012d7bba00783fc7ea38f1fb37d \ + --hash=sha256:5d72abdb8a4d56d4020713724ba81dac065fedb7f3667151c4a637f1d3fb26c0 \ + --hash=sha256:672132907fcad9f2aedcb705b2d7b3b93354a2aec1b2f706c4db852abe338f85 \ + --hash=sha256:6999421eb8ba9df4450a16d9184fcb7bef26240b9f98e95401f17af6c2210b71 \ + --hash=sha256:7b95a3fa7b3abb9b5b0e07668e808364d0d40f6bbbf9ae0faa8b5b210c97b140 \ + --hash=sha256:8469155f4cb518bafb4acf4865e8bb9d6804110d2d9bdcaa78564b9fd841e104 \ + --hash=sha256:94fd4858284736bb67a897a41608b5b0c2496c9bdb3bf2af1fa3409127f20d57 \ + --hash=sha256:b0f6d66c1c538d5a94a73aa9ddca8ccc4227e6c9ff555322ea40bdd142391dd4 \ + --hash=sha256:c74af94bf3ac15ac4d0f2a7c7b4663a15f8c2ab15ed0fc7531ca61d0835eccba \ + --hash=sha256:c82f4d474cf725255d9e6acf17252991c3c8aac038d6ef363a4bf8be2f6db517 \ + --hash=sha256:cdab83a366799fa730f90a4ebb563e494f28e9e92c4819e556152ad55e43591b \ + --hash=sha256:cfdead2f57330d76aa7234051dadfa7d4eedc0e5a27fd08e6f96714a92b00f09 \ + --hash=sha256:d1239932053f56f3456f32eb9625590cc7582e905021f94636202a864d470755 \ + --hash=sha256:dac7252938f0696ddea46f5e855dd3138444e82236e3be475f54929f0c510d48 \ + --hash=sha256:dc92bc2db7b45bda4510e4f51c59b00fe80b2d6be88928346e4294ce1c2abe7c \ + --hash=sha256:e07d91d0c92a31200f25351f4acb2bc6aff7f48094e13ebb1d0fb995b54b6542 \ + --hash=sha256:f4729811a6640d019a4b7ba8638ee2fd21fa5ca8c7e7bdf0fed62068fcaac737 + # via + # -r backend/mrt2-pytorch-runtime.in + # transformers +sentencepiece==0.2.1 \ + --hash=sha256:010f025a544ef770bb395091d57cb94deb9652d8972e0d09f71d85d5a0816c8c \ + --hash=sha256:017f97b274d4b0baa84b2dc743bf4517be81156f413bb24f12aacacde378e5ab \ + --hash=sha256:01e6912125cb45d3792f530a4d38f8e21bf884d6b4d4ade1b2de5cf7a8d2a52b \ + --hash=sha256:02593eca45440ef39247cee8c47322a34bdcc1d8ae83ad28ba5a899a2cf8d79a \ + --hash=sha256:097f3394e99456e9e4efba1737c3749d7e23563dd1588ce71a3d007f25475fff \ + --hash=sha256:0a0d15781a171d188b661ae4bde1d998c303f6bd8621498c50c671bd45a4798e \ + --hash=sha256:0a81799d0a68d618e89063fb423c3001a034c893069135ffe51fee439ae474d6 \ + --hash=sha256:0c0f672da370cc490e4c59d89e12289778310a0e71d176c541e4834759e1ae07 \ + --hash=sha256:0cdfecef430d985f1c2bcbfff3defd1d95dae876fbd0173376012d2d7d24044b \ + --hash=sha256:105e36e75cbac1292642045458e8da677b2342dcd33df503e640f0b457cb6751 \ + --hash=sha256:10ed3dab2044c47f7a2e7b4969b0c430420cdd45735d78c8f853191fa0e3148b \ + --hash=sha256:1855f57db07b51fb51ed6c9c452f570624d2b169b36f0f79ef71a6e6c618cd8b \ + --hash=sha256:2005242a16d2dc3ac5fe18aa7667549134d37854823df4c4db244752453b78a8 \ + --hash=sha256:22c4ebcb3c6ab1496ab1c37c79ef7bb563b8726f29548c30773b7a4cb152df1a \ + --hash=sha256:251874d720ac7f28024a168501f3c7bb15d1802245f6e66de565f18bbb9b5eaa \ + --hash=sha256:27e38eee653abc3d387862e67bc5c8b6f428cd604e688b85d29170b7e725c26c \ + --hash=sha256:2af5a1fb05013332ad94343b8b5f3973e006a2dde2dfba55a819549e054e2f0f \ + --hash=sha256:2f27ae6deea72efdb6f361750c92f6c21fd0ad087445082770cc34015213c526 \ + --hash=sha256:33f068c9382dc2e7c228eedfd8163b52baa86bb92f50d0488bf2b7da7032e484 \ + --hash=sha256:39f8651bd10974eafb9834ce30d9bcf5b73e1fc798a7f7d2528f9820ca86e119 \ + --hash=sha256:3d165fbb9bf8fba35f1946ba2617c3f9995679f07438325f07c026d53f33e746 \ + --hash=sha256:477c81505db072b3ab627e7eab972ea1025331bd3a92bacbf798df2b75ea86ec \ + --hash=sha256:4cdc7c36234fda305e85c32949c5211faaf8dd886096c7cea289ddc12a2d02de \ + --hash=sha256:4f5a3e0d9f445ed9d66c0fec47d4b23d12cfc858b407a03c194c1b26c2ac2a63 \ + --hash=sha256:56dd39a3c4d6493db3cdca7e8cc68c6b633f0d4195495cbadfcf5af8a22d05a6 \ + --hash=sha256:57cae326c8727de58c85977b175af132a7138d84c764635d7e71bbee7e774133 \ + --hash=sha256:5d0350b686c320068702116276cfb26c066dc7e65cfef173980b11bb4d606719 \ + --hash=sha256:5e4366c97b68218fd30ea72d70c525e6e78a6c0a88650f57ac4c43c63b234a9d \ + --hash=sha256:60937c959e6f44159fdd9f56fbdd302501f96114a5ba436829496d5f32d8de3f \ + --hash=sha256:6356d0986b8b8dc351b943150fcd81a1c6e6e4d439772e8584c64230e58ca987 \ + --hash=sha256:6d297a1748d429ba8534eebe5535448d78b8acc32d00a29b49acf28102eeb094 \ + --hash=sha256:733e59ff1794d26db706cd41fc2d7ca5f6c64a820709cb801dc0ea31780d64ab \ + --hash=sha256:8138cec27c2f2282f4a34d9a016e3374cd40e5c6e9cb335063db66a0a3b71fad \ + --hash=sha256:814978ac05130dd5812b4b03215c766bc6abaef13e7bd72bc534e4d1e12e9a4c \ + --hash=sha256:82d9ead6591015f009cb1be1cb1c015d5e6f04046dbb8c9588b931e869a29728 \ + --hash=sha256:881b2e44b14fc19feade3cbed314be37de639fc415375cefaa5bc81a4be137fd \ + --hash=sha256:891ade6503dd93d418c03993f7d6a8aa20260c422cefff5096b9068185e67642 \ + --hash=sha256:89a3ea015517c42c0341d0d962f3e6aaf2cf10d71b1932d475c44ba48d00aa2b \ + --hash=sha256:8dd4b477a7b069648d19363aad0cab9bad2f4e83b2d179be668efa672500dc94 \ + --hash=sha256:8f8ba89a3acb3dc1ae90f65ec1894b0b9596fdb98ab003ff38e058f898b39bc7 \ + --hash=sha256:9076430ac25dfa7147d9d05751dbc66a04bc1aaac371c07f84952979ea59f0d0 \ + --hash=sha256:92b3816aa2339355fda2c8c4e021a5de92180b00aaccaf5e2808972e77a4b22f \ + --hash=sha256:99f955df238021bf11f0fc37cdb54fd5e5b5f7fd30ecc3d93fb48b6815437167 \ + --hash=sha256:a19adcec27c524cb7069a1c741060add95f942d1cbf7ad0d104dffa0a7d28a2b \ + --hash=sha256:a483fd29a34c3e34c39ac5556b0a90942bec253d260235729e50976f5dba1068 \ + --hash=sha256:ac650534e2251083c5f75dde4ff28896ce7c8904133dc8fef42780f4d5588fcd \ + --hash=sha256:ad8493bea8432dae8d6830365352350f3b4144415a1d09c4c8cb8d30cf3b6c3c \ + --hash=sha256:afefe50a0cdcb4f2fd9733cb52001a2c164181ee2d82c32d38f5b1b326a8528c \ + --hash=sha256:b3616ad246f360e52c85781e47682d31abfb6554c779e42b65333d4b5f44ecc0 \ + --hash=sha256:b81a24733726e3678d2db63619acc5a8dccd074f7aa7a54ecd5ca33ca6d2d596 \ + --hash=sha256:c415c9de1447e0a74ae3fdb2e52f967cb544113a3a5ce3a194df185cbc1f962f \ + --hash=sha256:c6c8f42949f419ff8c7e9960dbadcfbc982d7b5efc2f6748210d3dd53a7de062 \ + --hash=sha256:c7f0fd2f2693309e6628aeeb2e2faf6edd221134dfccac3308ca0de01f8dab47 \ + --hash=sha256:c7f54a31cde6fa5cb030370566f68152a742f433f8d2be458463d06c208aef33 \ + --hash=sha256:c83b85ab2d6576607f31df77ff86f28182be4a8de6d175d2c33ca609925f5da1 \ + --hash=sha256:caa4e560c72c151da80036aecc2159e51a7fd8ae9efebefd96860460ce6bd025 \ + --hash=sha256:d3233770f78e637dc8b1fda2cd7c3b99ec77e7505041934188a4e7fe751de3b0 \ + --hash=sha256:d7b670879c370d350557edabadbad1f6561a9e6968126e6debca4029e5547820 \ + --hash=sha256:d8b1d91545578852f128650b8cce4ec20f93d39b378ff554ebe66290f2dabb92 \ + --hash=sha256:d9381351182ff9888cc80e41c632e7e274b106f450de33d67a9e8f6043da6f76 \ + --hash=sha256:daeb5e9e9fcad012324807856113708614d534f596d5008638eb9b40112cd9e4 \ + --hash=sha256:dcd8161eee7b41aae57ded06272905dbd680a0a04b91edd0f64790c796b2f706 \ + --hash=sha256:e10fa50bdbaa5e2445dbd387979980d391760faf0ec99a09bd7780ff37eaec44 \ + --hash=sha256:e37e4b4c4a11662b5db521def4e44d4d30ae69a1743241412a93ae40fdcab4bb \ + --hash=sha256:e52144670738b4b477fade6c2a9b6af71a8d0094514c9853ac9f6fc1fcfabae7 + # via -r backend/mrt2-pytorch-runtime.in +setuptools==81.0.0 \ + --hash=sha256:487b53915f52501f0a79ccfd0c02c165ffe06631443a886740b91af4b7a5845a \ + --hash=sha256:fdd925d5c5d9f62e4b74b30d6dd7828ce236fd6ed998a08d81de62ce5a6310d6 + # via torch +shellingham==1.5.4 \ + --hash=sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686 \ + --hash=sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de + # via typer +sympy==1.14.0 \ + --hash=sha256:d3d3fe8df1e5a0b42f0e7bdf50541697dbe7d23746e894990c030e2b05e72517 \ + --hash=sha256:e091cc3e99d2141a0ba2847328f5479b05d94a6635cb96148ccb3f34671bd8f5 + # via torch +tokenizers==0.22.2 \ + --hash=sha256:143b999bdc46d10febb15cbffb4207ddd1f410e2c755857b5a0797961bbdc113 \ + --hash=sha256:1a62ba2c5faa2dd175aaeed7b15abf18d20266189fb3406c5d0550dd34dd5f37 \ + --hash=sha256:1c774b1276f71e1ef716e5486f21e76333464f47bece56bbd554485982a9e03e \ + --hash=sha256:1e418a55456beedca4621dbab65a318981467a2b188e982a23e117f115ce5001 \ + --hash=sha256:1e50f8554d504f617d9e9d6e4c2c2884a12b388a97c5c77f0bc6cf4cd032feee \ + --hash=sha256:2249487018adec45d6e3554c71d46eb39fa8ea67156c640f7513eb26f318cec7 \ + --hash=sha256:25b85325d0815e86e0bac263506dd114578953b7b53d7de09a6485e4a160a7dd \ + --hash=sha256:29c30b83d8dcd061078b05ae0cb94d3c710555fbb44861139f9f83dcca3dc3e4 \ + --hash=sha256:319f659ee992222f04e58f84cbf407cfa66a65fe3a8de44e8ad2bc53e7d99012 \ + --hash=sha256:369cc9fc8cc10cb24143873a0d95438bb8ee257bb80c71989e3ee290e8d72c67 \ + --hash=sha256:37ae80a28c1d3265bb1f22464c856bd23c02a05bb211e56d0c5301a435be6c1a \ + --hash=sha256:38337540fbbddff8e999d59970f3c6f35a82de10053206a7562f1ea02d046fa5 \ + --hash=sha256:473b83b915e547aa366d1eee11806deaf419e17be16310ac0a14077f1e28f917 \ + --hash=sha256:544dd704ae7238755d790de45ba8da072e9af3eea688f698b137915ae959281c \ + --hash=sha256:64d94e84f6660764e64e7e0b22baa72f6cd942279fdbb21d46abd70d179f0195 \ + --hash=sha256:753d47ebd4542742ef9261d9da92cd545b2cacbb48349a1225466745bb866ec4 \ + --hash=sha256:791135ee325f2336f498590eb2f11dc5c295232f288e75c99a36c5dbce63088a \ + --hash=sha256:9ce725d22864a1e965217204946f830c37876eee3b2ba6fc6255e8e903d5fcbc \ + --hash=sha256:a6bf3f88c554a2b653af81f3204491c818ae2ac6fbc09e76ef4773351292bc92 \ + --hash=sha256:bfb88f22a209ff7b40a576d5324bf8286b519d7358663db21d6246fb17eea2d5 \ + --hash=sha256:c9ea31edff2968b44a88f97d784c2f16dc0729b8b143ed004699ebca91f05c48 \ + --hash=sha256:df6c4265b289083bf710dff49bc51ef252f9d5be33a45ee2bed151114a56207b \ + --hash=sha256:e10bf9113d209be7cd046d40fbabbaf3278ff6d18eb4da4c500443185dc1896c \ + --hash=sha256:f01a9c019878532f98927d2bacb79bbb404b43d3437455522a00a30718cdedb5 + # via transformers +torch==2.12.1+cu130 \ + --hash=sha256:046caae1457ec8a88256536958538dffb173c419320e3478f5b8d676038ec0d8 \ + --hash=sha256:0b4a8bdc9a89d1109bf4056d56e1c3ed05966273056d983a91f1cf597c25bea6 \ + --hash=sha256:1a4cd0c01ce52e4267147f0d860951ef8e937bafff613a9b5047010b5aacab11 \ + --hash=sha256:2a2bb858316615b90b14ff27d0c732d5af85d066f5ee7bf81fad2c9215839be7 \ + --hash=sha256:2d3e87d41ffb340ddf8c99e2a690a29feea9f5271459dd57621cd11317a434f2 \ + --hash=sha256:3b6e6e3ce55c3ebd688b00001cd44ff1a43fa30823f0394d20c8fd9910fb7087 \ + --hash=sha256:3d9cbeffed603075484270c34344e9506d86850af734d1e2eee13a8ba4bd690c \ + --hash=sha256:4bafc356fbb622e2756179406825c3a56c17b401196435a1487c5b40c657706c \ + --hash=sha256:52c5da6a0898d5d3473c02bd304b7a3bc0b72e351c6f3bfa0783e45ef9f4cd61 \ + --hash=sha256:5ff38932260cb4d5a52170d955642f6ede17f565de64e62eaca12a875851471b \ + --hash=sha256:6235f3a20b6094c7e8882c65a86a7f3c6e8c3ac8dbec0b57e575937d0166ed0f \ + --hash=sha256:76dd848312a40d29499614b714a4318841734ff309ad922f2365868395b6b054 \ + --hash=sha256:c84c5988b3e416669ed3790d772479af5934d1788119288d8bddd9c4cb207285 \ + --hash=sha256:d1cd8a4fd0556b2604db5447d9298323b8fba1ba67501fb3d8b22485c764a3a6 \ + --hash=sha256:d5e1840442d2182957b3d2f778cc325c90fa5cb42aa8b1ac949f029e9bdd7f06 \ + --hash=sha256:d6770341450d042a2998be19b5f7e431ab27281d356148a6eba659f5afff7c3e \ + --hash=sha256:f7f5ac061f7674917cacba9421f0a15e8df351057f90132cd46cccba9dfa7721 \ + --hash=sha256:fe0e04f34287aace6be33dd676307f9924108ad1a101c144e455a1eed0dc49e9 + # via -r backend/mrt2-pytorch-runtime.in +tqdm==4.70.0 \ + --hash=sha256:55b0b0dbd97462d06ebee91e4dac24ed4d4702be82b24f07e6c1d27e08cea220 \ + --hash=sha256:7f585706bfddbdebf89daac705b2dfcc16890130727d3197ca62c732b4310953 + # via + # huggingface-hub + # transformers +transformers==5.8.0 \ + --hash=sha256:6cc9a1f0291d16b1c1b735bad775e78ebefff7722701d4e28f98aaaa2bd6fb91 \ + --hash=sha256:e9d2cae6d195a7e1e05164c5ebf26142a7044e4dc4267274f4809204f92827e4 + # via -r backend/mrt2-pytorch-runtime.in +typer==0.27.1 \ + --hash=sha256:53150287edd11baeb4e4722c8e394fcdf8181c0ae89485cba8d25c778d5edd56 \ + --hash=sha256:a79bef8469a79c45498e7b814ecf8d603cc7644e9acbd9e19cac0334240b18df + # via + # huggingface-hub + # transformers +typing-extensions==4.16.0 \ + --hash=sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8 \ + --hash=sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5 + # via + # anyio + # huggingface-hub + # torch diff --git a/backend/runtime-locks/windows-gpu-pytorch.in b/backend/runtime-locks/windows-gpu-pytorch.in new file mode 100644 index 0000000..4f3c7f8 --- /dev/null +++ b/backend/runtime-locks/windows-gpu-pytorch.in @@ -0,0 +1,22 @@ +# Candidate shared Windows NVIDIA runtime for MRT2 and Stable Audio 3. +# +# Stable Audio 3's immutable upstream source pins torch/torchaudio 2.7.1 and +# requires huggingface-hub >=1.7.1. MRT2's adapter surface is model-free tested +# against this one environment, but the candidate remains release-blocked until +# both models complete the physical-GPU qualification matrix in issue #114. +--index-url https://pypi.org/simple +--extra-index-url https://download.pytorch.org/whl/cu126 + +einops==0.8.2 +einops-exts==0.0.4 +huggingface-hub==1.7.1 +numpy==2.3.5 +packaging==26.0 +resampy==0.4.3 +safetensors==0.7.0 +sentencepiece==0.2.1 +soundfile==0.13.1 +torch==2.7.1+cu126 +torchaudio==2.7.1+cu126 +tqdm==4.67.3 +transformers==5.8.0 diff --git a/backend/runtime-locks/windows-gpu-pytorch.txt b/backend/runtime-locks/windows-gpu-pytorch.txt new file mode 100644 index 0000000..4fc4ed5 --- /dev/null +++ b/backend/runtime-locks/windows-gpu-pytorch.txt @@ -0,0 +1,756 @@ +annotated-doc==0.0.5 \ + --hash=sha256:117bac03a25ede5df5440e855b32d556049ca169ead221505badf432fed4b101 \ + --hash=sha256:c7e58ce09192557605d8bbd92836d7e1d520ac9580096042c0bfd197efacf1bb + # via typer +anyio==4.14.2 \ + --hash=sha256:9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494 \ + --hash=sha256:cfa139f3ed1a23ee8f88a145ddb5ac7605b8bbfd8592baacd7ce3d8bb4313c7f + # via httpx +certifi==2026.7.22 \ + --hash=sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775 \ + --hash=sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55 + # via + # httpcore + # httpx +cffi==2.1.1 \ + --hash=sha256:046bfc24911b37851ee1b51aab8bffe713d89c68c6a057b09484ce9fd5f69b4e \ + --hash=sha256:06c72bb76605a4b0cd0aad6930b69d4baf7dd5d806cfc409b824191099700e66 \ + --hash=sha256:0beceaabe56af686895136a2de78db54ecd8e4046b236b8fd6d6cb61389e9bf2 \ + --hash=sha256:154852545011f779917b11c78db2358d095da62a9a172b78ad0a583ee5adc0d0 \ + --hash=sha256:194cffa889098ced9976c3fc6340305e43f6303657d298da55366907c05c22d6 \ + --hash=sha256:19ee6127ee34de7d83ce3d371ebc5ed91addbdcc39f9ab15ce4eb35a4e534971 \ + --hash=sha256:1a18a57b58cfb21fc28d72e876acf10eaed67a1ed96226f92af4df681d571c4c \ + --hash=sha256:1aa5645c30469b09530c4ebca77ebf8f17618293c58f8549cb1a543a50236e7d \ + --hash=sha256:1dea0e4d7d4f11f619fe8c1d76caf49e24405b4b5743c0e3be16a500ecd930c9 \ + --hash=sha256:208f941bb9d18e768138677f0a6d2ce01f590df56043dda1df1535ac57c88517 \ + --hash=sha256:210019b6c7cf07f081b4c54635c8cf744377001350e29cc0f81c4377b4797735 \ + --hash=sha256:246fa40ce8645a614ff682e0b70f37134e460eaf93a775e0cbe3cca585a67a80 \ + --hash=sha256:25792eac27877609e7bb06d42ff88278a6624fff2ba9bbb523c09616b117e80f \ + --hash=sha256:27350daa11d4f10c540e6e89dada4c54feb7256ad03e9a4dc075ebad7ba360d1 \ + --hash=sha256:28907ab9bfb6aa13184cfc17c6b8e1023c5ab6fd7076d8c20a35e59fe04f8f29 \ + --hash=sha256:2ae64be792b8966f2c69538199728b290e34726562896df1e5dc8ffd8d8188e8 \ + --hash=sha256:31348097ff5bbe827ccc41795d4dd099d9f0625e7def00ee653c137a490c2a6c \ + --hash=sha256:3143d81e29e1e20a9ce10901ec369012947876596f75a222235965f2b7ae832e \ + --hash=sha256:3222ba5d678f80a030e6afbcc33dc1ae5cb45facabb61cee2c7016b8432fde48 \ + --hash=sha256:3311ed60d36f83378794e1009ac6258bafbf81f7888b4caa7b35a521e3f95813 \ + --hash=sha256:334644fbac4eff73d985a17a91226df55d0f394160c4cfb880e084c8f7161cac \ + --hash=sha256:34e261f78cb6ceaaa36f42f2613f4380d94d9c759a9c73c769ee6e0247364632 \ + --hash=sha256:363e05fa78e15116c3c32c210ee36884fd6b9afa6d440e47112c3bd511d64cb6 \ + --hash=sha256:398aff33cee2767e3e781d2554c54bd0dff386bb437581e0d8011fde1a942ec1 \ + --hash=sha256:3d22a20b1fb1632cc72c22f95f7b0d2961c3e1c235f245ba4c606c4771035659 \ + --hash=sha256:42a494cee34437f05546455144f2b5d9ac09b1face62bcfce597d2e521066688 \ + --hash=sha256:42e2f76b9455f5a9a844f770bf3e200ed3da0e15f5df3db9c31fe80b04b3d004 \ + --hash=sha256:42f6930c31dc7f50732c9ae793c2786c7b6b044195967bbdde40bb9be81c4cc0 \ + --hash=sha256:456a61fa52d579ebf9df2e9552ead5129855dbaff6c1e5a9b1bc408809bdc062 \ + --hash=sha256:471cee653ae88de62096552e6d24ccb4a5adb8c8c9f10b5054d0122c15bf2779 \ + --hash=sha256:49cbc70e6542d4ccccb936558d1064a8012541e78f821f955cff24e357776c94 \ + --hash=sha256:4a7c934f7360e8cd64fe9efadcbd10c7c6364f531e432b9a4bf5ccbc9e0e8b50 \ + --hash=sha256:4be96343e422f2dfcd12ab5c9f5aebe03f82f737c6bffeca6830b3875cb44aab \ + --hash=sha256:4f42141fc14250de6dde5ee7ea4432be017252d91f19c5ad043c084cea629cac \ + --hash=sha256:507a24c282e0f42f8ed737cf048572cbf580468da5555764a8331735e9c736b6 \ + --hash=sha256:51b31d1c98274844cfd7838ce00bfc27c7423a4dc00fc0772fc3331c2cc90676 \ + --hash=sha256:58acb8ab8e295e6c5ea12f888cbb13cf21511ef2a3303a23f4325c29d17fe5c1 \ + --hash=sha256:5a59cc1c4442bc3d5c703bf720b51138d0bfc173618807c9ee2490a7541dd3d9 \ + --hash=sha256:5bb4e7ea95dcd6a014a6fef62e62467d67d8e582326443f3d68e71d6320a9fcf \ + --hash=sha256:5c58fe613dc5e5336357eff555824a314d8e43282600435c8d1cb6a7a2fedd13 \ + --hash=sha256:5e7cecbaadb83884793e05828cee59b210b24583b9c7425d0ba6a754fe22eb4e \ + --hash=sha256:616f097f2fe415bc92a247f02e11f634e1f9e9a83d327e3c915c15089c87869e \ + --hash=sha256:63bbfd5ded17c4840ac07cd8f1c21ba9d9708141f840b324f422f41b207e3973 \ + --hash=sha256:64faea20f4e2613363a1a9b9c7dd73058f3ecd00133a511e72ad7c511658f527 \ + --hash=sha256:661c298b4821edebead0c91edd2b00374d67ad7c5a1f7a91d4442633b79d6a72 \ + --hash=sha256:68e62fe11f30d5ca8289242866f0a5291402d8529ca2178ab8afc5c9694ae890 \ + --hash=sha256:6a8dddef476fab96d066d578fc88526767b836ab5ab21754e1d5bf3879c31c7c \ + --hash=sha256:6e192623c49c94421616a5778fba35cf0d5a8d000650c1967ef4448ee5cdd990 \ + --hash=sha256:7225e4514edb64eb6740324353e0da0711954fd8d7da4576755b1c6e09b697cd \ + --hash=sha256:75f80557d1389eddbd0de2681f6a390a0c5338c31ddaa821381c203fc3fd50d9 \ + --hash=sha256:770de9db11e84213beec501cfcaa013b019820ca881e03344dea5844f7876d94 \ + --hash=sha256:7750c6449dff7864bb9bb27ddfb0267756189201a3afc911d82b3caacd70dfc3 \ + --hash=sha256:7bde5e4cc5c10140859842b9d383af292b22639a4dffb725314baf45968cef80 \ + --hash=sha256:7ce713ace7c0e4520535b42b77eaa742c16dab813978064913e5a3cf82973b41 \ + --hash=sha256:7da0c5eff80f0197f3b3d1232ec5a682a9325f4ae9016a78f5f5ca35f9ced1f5 \ + --hash=sha256:7dbb61fe3a7699468030f71bbe5f8a0e326a151daa91beb11a6fc1f980c55e1c \ + --hash=sha256:811bd1e21d32de12efca32393a0ab3f5133b54fce9bd44b8bd77ab07da14bf6a \ + --hash=sha256:8ef53b2de9bcb9197d31854256575d59dbac0cba72ac627bb291ef5eceb74be4 \ + --hash=sha256:937c0052c05a31ca1daf18de3158eed4dbfcb9cc107adbea227728d647be701e \ + --hash=sha256:9d2055050ea716bd38b7f7f1579c275386646b4894c155a3e2f3cd62ed41b7c6 \ + --hash=sha256:9f8d177621de5cb38ee3e731eda45d421db093ec0739f46a5594babda7987a98 \ + --hash=sha256:a2d7755bef5a12ed488f4ef1f1b69ee9191d7396083b755a5d2295f6edb4768b \ + --hash=sha256:a48d62ab9d6f4f98c983223a547af44be6ca3691074c31cecced6facd3ba2dc1 \ + --hash=sha256:a4f00aa42f75d6e4595e8866e748cc1705adc0cddfeb2ca86d0d03993d63ba03 \ + --hash=sha256:a6e721d4b0e45d5b65e87534470e67b18dcd092c83f68fba09f152b9cbc061af \ + --hash=sha256:a730a083190634c65cca36ba5f489531576ebd79bcd5c8e172130f6453127231 \ + --hash=sha256:a931079504ecc49efed7744c476a5c343a92fabf66dec2db95edb1b2fdc770e2 \ + --hash=sha256:aa9511c62d14da7aacc9b4bf51f3f697a621e83b2d6919008243c3aad168eea3 \ + --hash=sha256:ab36d55f9ed2d067327667c2fea18dda018eb628dd6347aa01dda6cf1f5d3836 \ + --hash=sha256:ad2c86c495b899d862ea0f4b42891b8713a3bd45dd4105c7fd51c2a72f39f3a5 \ + --hash=sha256:aeae0e330c9f6acd681f647d46cefd30c29f93e3392882e792e82080c9691399 \ + --hash=sha256:b0431303acaea1089ad4b3e9ce4e6518193def1118d4073ca848635ee4ea2e96 \ + --hash=sha256:b5bdfd1c873d4e093aabc0ca84c4ca6dbc4f752afb5c86f146d9742580c9da2e \ + --hash=sha256:baed1e86cc735622097354b9d1281406caf42ff42a886d29faa8e8d1630333be \ + --hash=sha256:c1453022f490d2459a11819d83ad1d586e9ff65a12ac3e705ffebd46d3685dcf \ + --hash=sha256:c26608d2222fb1e94487e4a387d85f13eb55d5ed725cb25a0c589ac4ee60e7bc \ + --hash=sha256:c7659f22557c5a0bc4855cd635f55edec690cc008a40768527762cb9fb263455 \ + --hash=sha256:c8c69575568085ba0b1b10c0249d779a214aea6f6522e949a0fc9fb0fcb449d0 \ + --hash=sha256:c8d2c9fd1f2d16f780d15127abb050d13d1a76c03a4bd87d7e4980e45e511e12 \ + --hash=sha256:ca82be1a1d406ecfe1d25dc16cb33488e5a16bf4438c9fb590484ea29d92478b \ + --hash=sha256:cc572dace3f60ef98d7b12ff411d20f5362feb31a0439eab0085bbfd349982d7 \ + --hash=sha256:d18e5ac0f2f03f4f518d3e23db0f0cad7faa1da8620e9c09461d443bbf6e6692 \ + --hash=sha256:d28630f5854ab07ab1fd4aba756de52326c82e6be15d414b12793f1975048b54 \ + --hash=sha256:d9c275eaacd24aa73f94ffd6de08fc3f932424d8b6c376f4bed7cde376fe7bc3 \ + --hash=sha256:da0e573f9f97159390c89d9f1a9e41908b66d408cc5b58d08cf3847d844c531b \ + --hash=sha256:dd31f52ea1086513bb9df30f8fcee9b8918323ae067a3d5b78bc826a000712be \ + --hash=sha256:dddad92b554513a31f272570678ba307fb9f618f05e3d4a5eacafff9eae03e1d \ + --hash=sha256:df423d40ee8654634421812bc3b196da3f9bd7d32929da813f8394c4348a5358 \ + --hash=sha256:df913725b79db7bcf03448f36b7bf8815363417d5b58deecf9305e3e30f0f21a \ + --hash=sha256:e0bcb7e0f677f543555d2adff3bf19c05f66cdb4796e5ff602442ab2fe3c4ef7 \ + --hash=sha256:e2d65b31f36619cda3999b78b2aa9632e76b78448e7a56fc4240824200e7c4fc \ + --hash=sha256:e6e8cff14d6fb0be70a09c0bdc58096f501952d04624ebf867e0e56da2df8960 \ + --hash=sha256:f16c709686a78c727bbbf059f92b0bf41c6fc60deec706d2dc19f529175a6125 \ + --hash=sha256:f24fb43132a4c6b4cb4eb029492919b2db645be6808d738f244fd146c03c32cb \ + --hash=sha256:f53e442b08449d42821fa4a4fba000095af9f62742a500f978a9f557ec44339a \ + --hash=sha256:f5cfbc5fe74540d335175b656c725d74d90e3730c626d92575eea35029d9afaa \ + --hash=sha256:f81b3b8f3d4e343550fa4baa0e479bba9f2d29ce9c2e9b51d1ce1718d7442fcf \ + --hash=sha256:f8ec5e643a9a937f64e1999eb9f75d072263751912dc5cd06d3c85f8f44be7c3 \ + --hash=sha256:fb92203a88b3d3053034db775110081c49d28be6551923805e039924093761e4 \ + --hash=sha256:fcd22650c908d7b7da162bbfaab594a1227a15d1643a98c68b122ac642fa2264 + # via soundfile +colorama==0.4.6 \ + --hash=sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6 + # via + # tqdm + # typer +einops==0.8.2 \ + --hash=sha256:54058201ac7087911181bfec4af6091bb59380360f069276601256a76af08193 \ + --hash=sha256:609da665570e5e265e27283aab09e7f279ade90c4f01bcfca111f3d3e13f2827 + # via + # -r backend/runtime-locks/windows-gpu-pytorch.in + # einops-exts +einops-exts==0.0.4 \ + --hash=sha256:616f145b3411f8e9e3be5da5c968bbe372e55c249de11faa909c7a4b74580a6c \ + --hash=sha256:6d310a4c858e459ebff8288580f90255d354cfa3bde22a53b59baae64b48cb95 + # via -r backend/runtime-locks/windows-gpu-pytorch.in +filelock==3.32.2 \ + --hash=sha256:87dd94cf281e586d135fa51132b8e3d9a598b316e90377a288663c9321036c82 \ + --hash=sha256:c33351e1f49cae33414acbc6d56784e6ecee82514ec90795da1161fc4836b5b8 + # via + # huggingface-hub + # torch +fsspec==2026.7.0 \ + --hash=sha256:b57ddbafedfaef7018c1ecab32aa200a9d7ca26b77965f64e48b70061249d279 \ + --hash=sha256:c803c40f4cf860b49dea58ee3e1c33cb9c790520e233537e1340049f89b82a88 + # via + # huggingface-hub + # torch +h11==0.16.0 \ + --hash=sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1 \ + --hash=sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86 + # via httpcore +hf-xet==1.6.0 \ + --hash=sha256:0e6e21fa3cdfcdcd76748564bf593870a5e013f47d97cf10aed63aa222cff5b7 \ + --hash=sha256:23379c2f9ec8696d952b16414a2bae72cad86a52df869b050698ba60f538c675 \ + --hash=sha256:2e58454a340b3556dfa4972d5451aff4fba8dd42a236600ba1a1d2b1514f0fef \ + --hash=sha256:35cec30d75c6f9eb9c16a77cef68e85a103b72e24d4b473714ec9ff06428bab9 \ + --hash=sha256:3dc3e35441ba395006af5aaacc40ef2e603c51ef46c3530b9156185f00935ea3 \ + --hash=sha256:4fc74352a17015bd0ee90038bc9efe38db894cde45f268b6712b04fce8cd0acb \ + --hash=sha256:5153e6bb103ad49d6ea9f1b2e230db5a2ea32551ad09a706d2f61d7c7c80d80e \ + --hash=sha256:5789835d7c6bc9436962853192082374297fb72d7eff7e7762ec25ceb7e25338 \ + --hash=sha256:633dc0cd71d32da58ab8c03ad38e2fac452c15c2b0a2866ebf6ededfe0a5061d \ + --hash=sha256:70cbb9c896901600128cb9b6f06e132954fbede1db30f31f7c6c63f84cb7c31d \ + --hash=sha256:75765820ce4700db3750c94acc8fe27c5fae4c9ec000a0dbac3ca082acf97765 \ + --hash=sha256:8fb4f71cba6129110c3374a33f919001ff130488fc23553698e34cc1c2a1198c \ + --hash=sha256:948f15d3a9545cfe5932f6bd8b440f6ae630aee108f14b7bd6c561f7c2dcc522 \ + --hash=sha256:d62671bb130879cef0ee4c9ebe47a14af6c66ec53e6d84dc15936e5ffdfac82f \ + --hash=sha256:f0906082d9932ae0c0057fa194041c22b4e2cdb46b2592ef3b91f020d62a081a \ + --hash=sha256:f2f7278c05c22fd60cb436cda1269649b3e81db65ecdc8496e5e164aa4143e7b \ + --hash=sha256:fb4fadde1b2b70bf4c0c14a6dccbe7194b1c28947fefd5bbe3fed9d940676c3b + # via huggingface-hub +httpcore==1.0.9 \ + --hash=sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55 \ + --hash=sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8 + # via httpx +httpx==0.28.1 \ + --hash=sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc \ + --hash=sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad + # via huggingface-hub +huggingface-hub==1.7.1 \ + --hash=sha256:38c6cce7419bbde8caac26a45ed22b0cea24152a8961565d70ec21f88752bfaa \ + --hash=sha256:be38fe66e9b03c027ad755cb9e4b87ff0303c98acf515b5d579690beb0bf3048 + # via + # -r backend/runtime-locks/windows-gpu-pytorch.in + # tokenizers + # transformers +idna==3.18 \ + --hash=sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2 \ + --hash=sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848 + # via + # anyio + # httpx +jinja2==3.1.6 \ + --hash=sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67 + # via torch +llvmlite==0.48.0 \ + --hash=sha256:02853fe4214acb3780fc920c3fee10564b61d58a35e1b78afcc8a546c2deaba3 \ + --hash=sha256:054aa7d46595935565f276cf0c1659b4f10929c996dd4a606875fae26fba2a23 \ + --hash=sha256:05f0103c8f2f96a37441337e3643863c01b8e83e530aff38960dcb383c54a065 \ + --hash=sha256:13532f248960ba888ad5ab8150494e2f3a3d20e5f59f264e63741ea5b0ba844c \ + --hash=sha256:1d9f952dff6c350c529997423d4fa43abae9722a884ac7ffafc37a3af676e7db \ + --hash=sha256:2fe5cb59b2063bfa039dcb8ca6481c0181bf552f340d10dcf61d7996a665556e \ + --hash=sha256:321f1ac39b462603f0b589751aecf2d237d056f6d005749c1752b6f23ec3f074 \ + --hash=sha256:37d66fae72802175b0bfe1ea06e624b51e2d7aee6c3c34bbd09739b8f88e8e0b \ + --hash=sha256:3ee0c77685a18f5fca994ae21d0763007fca5c5c64b41de37accc78b69079176 \ + --hash=sha256:416fa4c2c66c2c6dc6d0a402648c19206e548efa0aa1eff01ad5cdad0af8217d \ + --hash=sha256:543b19f9ef8f3c7c60d1468191e4ee1b1537bf9f8a3d56f64c0ddd98de92edd2 \ + --hash=sha256:56a7e24607d3f02d7b1bae8d29c7e1e423d53143d68b072999777f19678fe77b \ + --hash=sha256:614aad57df707e3172efd5165f2aa7da6a0c6897e40dce590bf756396815ba76 \ + --hash=sha256:6fa532d6bb3fd3f0803567c736401c54aecfe1a396d3ad25d2440d220e09f0e7 \ + --hash=sha256:7a5c413317050a1d67c34708bde97707f9b2257ef1017f7532d21fe7d9a9ff30 \ + --hash=sha256:8761b9e522f55207e24424fcd98370289eec2710bf8e915c82d1053f642450dc \ + --hash=sha256:91c7e24e74cde3f02b88aa5acca678373f9e069f3b98531b3dbb3a142d9d10bb \ + --hash=sha256:966dcab0a598e2bd8fb5f2cc082cf7b07bae564fc485a3a8692393caf986facf \ + --hash=sha256:979a66a3f28a02565383ff463527dce78e9b856298872a361283132488e83591 \ + --hash=sha256:a83a99ef0c05b4ccddf9b6218ed9fe84b653a0caf7c1d9dbe148d6d16c67f518 \ + --hash=sha256:d0b3c61aac83b42fb48cc96bffbf57c81b82b2aa92276b7ed6420c814629a99a \ + --hash=sha256:d45c7541a80934ec6d8ab0defe67439494ecd2193cbf852a44ba827808976ac1 \ + --hash=sha256:d66c3beb4209087ddd4cf4ed2a0856b6887e6a913bdcf1aacfec9851cf2cba4e \ + --hash=sha256:efaee0276e5e17c2b99b92e0c974bd484ef5977cf5dbc9168e82b71578edb47f \ + --hash=sha256:f5e5a5131045b72345c71062ea1a91910dde913792b6c9b28ebb2c1c0a712e98 + # via numba +markdown-it-py==4.2.0 \ + --hash=sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49 \ + --hash=sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a + # via rich +markupsafe==3.0.3 \ + --hash=sha256:26a5784ded40c9e318cfc2bdb30fe164bdb8665ded9cd64d500a34fb42067b1c + # via jinja2 +mdurl==0.1.2 \ + --hash=sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8 \ + --hash=sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba + # via markdown-it-py +mpmath==1.3.0 \ + --hash=sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f \ + --hash=sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c + # via sympy +networkx==3.6.1 \ + --hash=sha256:26b7c357accc0c8cde558ad486283728b65b6a95d85ee1cd66bafab4c8168509 \ + --hash=sha256:d47fbf302e7d9cbbb9e2555a0d267983d2aa476bac30e90dfbe5669bd57f3762 + # via torch +numba==0.66.0 \ + --hash=sha256:0999e3ee1b18c48e1fb51d11af35ef59852c7f4f50569c9550c25faef0616ad1 \ + --hash=sha256:2338cc0d43609fe448930848fd35a5bc688761b986f81b597a6f45cc0f8c9577 \ + --hash=sha256:27951c47e0def9bf8afe580eb961102902e2fd23cb77924b7d9d7cc0f8b444cb \ + --hash=sha256:380b2556a2019ccd1e956ae77dd257eaa39403f7520768b626d44b755112785e \ + --hash=sha256:46ae5f2b19e2af3c33c2df100306a90ea2f981c8158b0390f8bf6c20eee7357e \ + --hash=sha256:53ca5900b7cab15109796030113a6b28576bae5ad7bb507ad6dd1360ddd81ba4 \ + --hash=sha256:63b943eb2c9ba371908ce2cd6dfc643db51fc40f7966993376a1701bc922f537 \ + --hash=sha256:651a2b53298340956db26ecbe7ab043106b50a40c2807e66d617a4917245a4ab \ + --hash=sha256:7c14c044c06b453ec3fa7715dfe75425e2ba72c73377a7ffde6d9ec511dfd94c \ + --hash=sha256:8c1144ba1720ea59ad79f4f488ed54d149b2613b357f7e445678b7d0739c70e9 \ + --hash=sha256:939316d5d8619751207b8972a67852b5a7646665298cb4de693cd6bf135152f4 \ + --hash=sha256:aac69f3ccb8af100f5913c1241edc9692bad1cdd2508721713f426eb06c9a659 \ + --hash=sha256:b075a4e7ebc43dc6294f223e2821659656209fd5e0ce53245877c23d66d6e1a9 \ + --hash=sha256:b900e63a0e26c05ea9a6d5a3a5a0a177cb64c5011887bf43edb8c3ed2c38d363 \ + --hash=sha256:bbd531c327557a9004507fa6bff06c53ab51a7a5776b75261bb9cef1efe2b2ea \ + --hash=sha256:bd57790acd20f6a468e0ad333ef6b82355e309a92310fb7dff80e919f01a21a9 \ + --hash=sha256:c5bfe5350284509ab0474390321454c3a8627a188af5b68c910e83df3e2db4a7 \ + --hash=sha256:cc408c54b450f41582f4be1608f8981c1dcc44c7f40355cc150dd93015753407 \ + --hash=sha256:cdf506775d9f02eb92a87bf5c5b1e0d25506fd18cafd769f4ed914a8feac73e7 \ + --hash=sha256:d426178fb991a85714c43112a8ea7b9d9579ea856ad8dcdb9c1c3941903ba5be \ + --hash=sha256:db7735d15ea17a283d6485b9fa3504769f78fd86e5146638ad5e8da57c031b9e \ + --hash=sha256:e2b101f23b8b63978d334574d2039f27f0dccfe1d891756f33a2e2f3e4c88cf4 \ + --hash=sha256:efe0d2d5099790df945e0cb6e1b3104bd965d7bbfac50d62f1d5d1d6ade0825d \ + --hash=sha256:fb601841d9e02e6237bb6522e36d0741614be3cfe2b482a6f00a41b5ba209443 \ + --hash=sha256:fc6629becb21a867d85401ec89f426dd24c484a4193ade8a38309debfd1529ca + # via resampy +numpy==2.3.5 \ + --hash=sha256:00dc4e846108a382c5869e77c6ed514394bdeb3403461d25a829711041217d5b \ + --hash=sha256:0472f11f6ec23a74a906a00b48a4dcf3849209696dff7c189714511268d103ae \ + --hash=sha256:04822c00b5fd0323c8166d66c701dc31b7fbd252c100acd708c48f763968d6a3 \ + --hash=sha256:052e8c42e0c49d2575621c158934920524f6c5da05a1d3b9bab5d8e259e045f0 \ + --hash=sha256:09a1bea522b25109bf8e6f3027bd810f7c1085c64a0c7ce050c1676ad0ba010b \ + --hash=sha256:0cd00b7b36e35398fa2d16af7b907b65304ef8bb4817a550e06e5012929830fa \ + --hash=sha256:0d8163f43acde9a73c2a33605353a4f1bc4798745a8b1d73183b28e5b435ae28 \ + --hash=sha256:1062fde1dcf469571705945b0f221b73928f34a20c904ffb45db101907c3454e \ + --hash=sha256:11e06aa0af8c0f05104d56450d6093ee639e15f24ecf62d417329d06e522e017 \ + --hash=sha256:17531366a2e3a9e30762c000f2c43a9aaa05728712e25c11ce1dbe700c53ad41 \ + --hash=sha256:1978155dd49972084bd6ef388d66ab70f0c323ddee6f693d539376498720fb7e \ + --hash=sha256:1ed1ec893cff7040a02c8aa1c8611b94d395590d553f6b53629a4461dc7f7b63 \ + --hash=sha256:2dcd0808a421a482a080f89859a18beb0b3d1e905b81e617a188bd80422d62e9 \ + --hash=sha256:2e2eb32ddb9ccb817d620ac1d8dae7c3f641c1e5f55f531a33e8ab97960a75b8 \ + --hash=sha256:2feae0d2c91d46e59fcd62784a3a83b3fb677fead592ce51b5a6fbb4f95965ff \ + --hash=sha256:3095bdb8dd297e5920b010e96134ed91d852d81d490e787beca7e35ae1d89cf7 \ + --hash=sha256:30bc11310e8153ca664b14c5f1b73e94bd0503681fcf136a163de856f3a50139 \ + --hash=sha256:3101e5177d114a593d79dd79658650fe28b5a0d8abeb8ce6f437c0e6df5be1a4 \ + --hash=sha256:396084a36abdb603546b119d96528c2f6263921c50df3c8fd7cb28873a237748 \ + --hash=sha256:3997b5b3c9a771e157f9aae01dd579ee35ad7109be18db0e85dbdbe1de06e952 \ + --hash=sha256:414802f3b97f3c1eef41e530aaba3b3c1620649871d8cb38c6eaff034c2e16bd \ + --hash=sha256:51c1e14eb1e154ebd80e860722f9e6ed6ec89714ad2db2d3aa33c31d7c12179b \ + --hash=sha256:51c55fe3451421f3a6ef9a9c1439e82101c57a2c9eab9feb196a62b1a10b58ce \ + --hash=sha256:5ee6609ac3604fa7780e30a03e5e241a7956f8e2fcfe547d51e3afa5247ac47f \ + --hash=sha256:612a95a17655e213502f60cfb9bf9408efdc9eb1d5f50535cc6eb365d11b42b5 \ + --hash=sha256:6203fdf9f3dc5bdaed7319ad8698e685c7a3be10819f41d32a0723e611733b42 \ + --hash=sha256:63c0e9e7eea69588479ebf4a8a270d5ac22763cc5854e9a7eae952a3908103f7 \ + --hash=sha256:66f85ce62c70b843bab1fb14a05d5737741e74e28c7b8b5a064de10142fad248 \ + --hash=sha256:6cf9b429b21df6b99f4dee7a1218b8b7ffbbe7df8764dc0bd60ce8a0708fed1e \ + --hash=sha256:70b37199913c1bd300ff6e2693316c6f869c7ee16378faf10e4f5e3275b299c3 \ + --hash=sha256:727fd05b57df37dc0bcf1a27767a3d9a78cbbc92822445f32cc3436ba797337b \ + --hash=sha256:74ae7b798248fe62021dbf3c914245ad45d1a6b0cb4a29ecb4b31d0bfbc4cc3e \ + --hash=sha256:784db1dcdab56bf0517743e746dfb0f885fc68d948aba86eeec2cba234bdf1c0 \ + --hash=sha256:86945f2ee6d10cdfd67bcb4069c1662dd711f7e2a4343db5cecec06b87cf31aa \ + --hash=sha256:86d835afea1eaa143012a2d7a3f45a3adce2d7adc8b4961f0b362214d800846a \ + --hash=sha256:872a5cf366aec6bb1147336480fef14c9164b154aeb6542327de4970282cd2f5 \ + --hash=sha256:8b973c57ff8e184109db042c842423ff4f60446239bd585a5131cc47f06f789d \ + --hash=sha256:8cba086a43d54ca804ce711b2a940b16e452807acebe7852ff327f1ecd49b0d4 \ + --hash=sha256:8f7f0e05112916223d3f438f293abf0727e1181b5983f413dfa2fefc4098245c \ + --hash=sha256:900218e456384ea676e24ea6a0417f030a3b07306d29d7ad843957b40a9d8d52 \ + --hash=sha256:93eebbcf1aafdf7e2ddd44c2923e2672e1010bddc014138b229e49725b4d6be5 \ + --hash=sha256:9c75442b2209b8470d6d5d8b1c25714270686f14c749028d2199c54e29f20b4d \ + --hash=sha256:9ee2197ef8c4f0dfe405d835f3b6a14f5fee7782b5de51ba06fb65fc9b36e9f1 \ + --hash=sha256:a414504bef8945eae5f2d7cb7be2d4af77c5d1cb5e20b296c2c25b61dff2900c \ + --hash=sha256:a4b9159734b326535f4dd01d947f919c6eefd2d9827466a696c44ced82dfbc18 \ + --hash=sha256:a80afd79f45f3c4a7d341f13acbe058d1ca8ac017c165d3fa0d3de6bc1a079d7 \ + --hash=sha256:aa5bc7c5d59d831d9773d1170acac7893ce3a5e130540605770ade83280e7188 \ + --hash=sha256:acfd89508504a19ed06ef963ad544ec6664518c863436306153e13e94605c218 \ + --hash=sha256:aeffcab3d4b43712bb7a60b65f6044d444e75e563ff6180af8f98dd4b905dfd2 \ + --hash=sha256:afaffc4393205524af9dfa400fa250143a6c3bc646c08c9f5e25a9f4b4d6a903 \ + --hash=sha256:b0c7088a73aef3d687c4deef8452a3ac7c1be4e29ed8bf3b366c8111128ac60c \ + --hash=sha256:b46b4ec24f7293f23adcd2d146960559aaf8020213de8ad1909dba6c013bf89c \ + --hash=sha256:b501b5fa195cc9e24fe102f21ec0a44dffc231d2af79950b451e0d99cea02234 \ + --hash=sha256:bf06bc2af43fa8d32d30fae16ad965663e966b1a3202ed407b84c989c3221e82 \ + --hash=sha256:c804e3a5aba5460c73955c955bdbd5c08c354954e9270a2c1565f62e866bdc39 \ + --hash=sha256:c8a9958e88b65c3b27e22ca2a076311636850b612d6bbfb76e8d156aacde2aaf \ + --hash=sha256:cc0a57f895b96ec78969c34f682c602bf8da1a0270b09bc65673df2e7638ec20 \ + --hash=sha256:cc8920d2ec5fa99875b670bb86ddeb21e295cb07aa331810d9e486e0b969d946 \ + --hash=sha256:ccc933afd4d20aad3c00bcef049cb40049f7f196e0397f1109dba6fed63267b0 \ + --hash=sha256:ce581db493ea1a96c0556360ede6607496e8bf9b3a8efa66e06477267bc831e9 \ + --hash=sha256:d0f23b44f57077c1ede8c5f26b30f706498b4862d3ff0a7298b8411dd2f043ff \ + --hash=sha256:d21644de1b609825ede2f48be98dfde4656aefc713654eeee280e37cadc4e0ad \ + --hash=sha256:d6889ec4ec662a1a37eb4b4fb26b6100841804dac55bd9df579e326cdc146227 \ + --hash=sha256:de5672f4a7b200c15a4127042170a694d4df43c992948f5e1af57f0174beed10 \ + --hash=sha256:e6a0bc88393d65807d751a614207b7129a310ca4fe76a74e5c7da5fa5671417e \ + --hash=sha256:ed89927b86296067b4f81f108a2271d8926467a8868e554eaf370fc27fa3ccaf \ + --hash=sha256:ee3888d9ff7c14604052b2ca5535a30216aa0a58e948cdd3eeb8d3415f638769 \ + --hash=sha256:f0963b55cdd70fad460fa4c1341f12f976bb26cb66021a5580329bd498988310 \ + --hash=sha256:f16417ec91f12f814b10bafe79ef77e70113a2f5f7018640e7425ff979253425 \ + --hash=sha256:f28620fe26bee16243be2b7b874da327312240a7cdc38b769a697578d2100013 \ + --hash=sha256:f4255143f5160d0de972d28c8f9665d882b5f61309d8362fdd3e103cf7bf010c \ + --hash=sha256:ffac52f28a7849ad7576293c0cb7b9f08304e8f7d738a8cb8a90ec4c55a998eb \ + --hash=sha256:ffe22d2b05504f786c867c8395de703937f934272eb67586817b46188b4ded6d \ + --hash=sha256:fffe29a1ef00883599d1dc2c51aa2e5d80afe49523c261a74933df395c15c520 + # via + # -r backend/runtime-locks/windows-gpu-pytorch.in + # numba + # resampy + # soundfile + # transformers +packaging==26.0 \ + --hash=sha256:00243ae351a257117b6a241061796684b084ed1c516a08c48a3f7e147a9d80b4 \ + --hash=sha256:b36f1fef9334a5588b4166f8bcd26a14e521f2b55e6b9de3aaa80d3ff7a37529 + # via + # -r backend/runtime-locks/windows-gpu-pytorch.in + # huggingface-hub + # transformers +pycparser==3.0 \ + --hash=sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29 \ + --hash=sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992 + # via cffi +pygments==2.20.0 \ + --hash=sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f \ + --hash=sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176 + # via rich +pyyaml==6.0.3 \ + --hash=sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c \ + --hash=sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a \ + --hash=sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3 \ + --hash=sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956 \ + --hash=sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6 \ + --hash=sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c \ + --hash=sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65 \ + --hash=sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a \ + --hash=sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0 \ + --hash=sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b \ + --hash=sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1 \ + --hash=sha256:22ba7cfcad58ef3ecddc7ed1db3409af68d023b7f940da23c6c2a1890976eda6 \ + --hash=sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7 \ + --hash=sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e \ + --hash=sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007 \ + --hash=sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310 \ + --hash=sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4 \ + --hash=sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9 \ + --hash=sha256:3ff07ec89bae51176c0549bc4c63aa6202991da2d9a6129d7aef7f1407d3f295 \ + --hash=sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea \ + --hash=sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0 \ + --hash=sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e \ + --hash=sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac \ + --hash=sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9 \ + --hash=sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7 \ + --hash=sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35 \ + --hash=sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb \ + --hash=sha256:5cf4e27da7e3fbed4d6c3d8e797387aaad68102272f8f9752883bc32d61cb87b \ + --hash=sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69 \ + --hash=sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5 \ + --hash=sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b \ + --hash=sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c \ + --hash=sha256:6344df0d5755a2c9a276d4473ae6b90647e216ab4757f8426893b5dd2ac3f369 \ + --hash=sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd \ + --hash=sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824 \ + --hash=sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198 \ + --hash=sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065 \ + --hash=sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c \ + --hash=sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c \ + --hash=sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764 \ + --hash=sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196 \ + --hash=sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b \ + --hash=sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00 \ + --hash=sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac \ + --hash=sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8 \ + --hash=sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e \ + --hash=sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28 \ + --hash=sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3 \ + --hash=sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5 \ + --hash=sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4 \ + --hash=sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b \ + --hash=sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf \ + --hash=sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5 \ + --hash=sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702 \ + --hash=sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8 \ + --hash=sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788 \ + --hash=sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da \ + --hash=sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d \ + --hash=sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc \ + --hash=sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c \ + --hash=sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba \ + --hash=sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f \ + --hash=sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917 \ + --hash=sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5 \ + --hash=sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26 \ + --hash=sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f \ + --hash=sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b \ + --hash=sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be \ + --hash=sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c \ + --hash=sha256:efd7b85f94a6f21e4932043973a7ba2613b059c4a000551892ac9f1d11f5baf3 \ + --hash=sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6 \ + --hash=sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926 \ + --hash=sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0 + # via + # huggingface-hub + # transformers +regex==2026.7.19 \ + --hash=sha256:062f8cb7a9739c4835d22bd96f370c59aba89f257adcfa53be3cc209e08d3ae0 \ + --hash=sha256:064f1760a5a4ade65c5419be23e782f29147528e8a66e0c42dd4cedb8d4e9fc6 \ + --hash=sha256:09523a592938aa9f587fb74467c63ff0cf88fc3df14c82ab0f0517dcf76aaa62 \ + --hash=sha256:09d3007fc76249a83cdd33de160d50e6cb77f54e09d8fa9e7148e10607ce24af \ + --hash=sha256:09f3e5287f94f17b709dc9a9e70865855feee835c861613be144218ce4ca82cc \ + --hash=sha256:0c41c63992bf1874cebb6e7f56fd7d3c007924659a604ae3d90e427d40d4fd13 \ + --hash=sha256:0e9554c8785eac5cffe6300f69a91f58ba72bc88a5f8d661235ad7c6aa5b8ccd \ + --hash=sha256:1123ef4211d763ee771d47916a1596e2f4915794f7aabdc1adcb20e4249a6951 \ + --hash=sha256:15b364b9b98d6d2fe1a85034c23a3180ff913f46caddc3895f6fd65186255ccc \ + --hash=sha256:1649eb39fcc9ea80c4d2f110fde2b8ab2aef3877b98f02ab9b14e961f418c511 \ + --hash=sha256:17ed5692f6acc4183e98331101a5f9e4f64d72fe58b753da4d444a2c77d05b12 \ + --hash=sha256:199535629f25caf89698039af3d1ad5fcae7f933e2112c73f1cdf49165c99518 \ + --hash=sha256:1c398716054621aa300b3d411f467dda903806c5da0df6945ab73982b8d115db \ + --hash=sha256:1d3372064506b94dd2c67c845f2db8062e9e9ba84d04e33cb96d7d33c11fe1ae \ + --hash=sha256:1d58561843f0ff7dc78b4c28b5e2dc388f3eff94ebc8a232a3adba961fc00009 \ + --hash=sha256:1d793a7988e04fcb1e2e135567443d82173225d657419ec09414a9b5a145b986 \ + --hash=sha256:1ebac3474b8589fce2f9b225b650afd61448f7c73a5d0255a10cc6366471aed1 \ + --hash=sha256:20568e182eb82d39a6bf7cff3fd58566f14c75c6f74b2c8c96537eecf9010e3a \ + --hash=sha256:22a992de9a0d91bda927bf02b94351d737a0302905432c88a53de7c4b9ce62e2 \ + --hash=sha256:2955907b7157a6660f27079edf7e0229e9c9c5325c77a2ef6a890cba91efa6f0 \ + --hash=sha256:2c4e61e2e1be56f63ec3cc618aa9e0de81ef6f43d177205451840022e24f5b78 \ + --hash=sha256:2cc3460cedf7579948486eab03bc9ad7089df4d7281c0f47f4afe03e8d13f02d \ + --hash=sha256:2ce9e679f776649746729b6c86382da519ef649c8e34cc41df0d2e5e0f6c36d4 \ + --hash=sha256:2ef7eeb108c47ce7bcc9513e51bcb1bf57e8f483d52fce68a8642e3527141ae0 \ + --hash=sha256:3080a7fd38ef049bd489e01c970c97dd84ff446a885b0f1f6b26d9b1ad13ce11 \ + --hash=sha256:343a4504e3fb688c47cad451221ca5d4814f42b1e16c0065bde9cbf7f473bd52 \ + --hash=sha256:36aacfb15faaff3ced55afbf35ec72f50d4aee22082c4f7fe0573a33e2fca92e \ + --hash=sha256:3d3143f159261b1ce5b24c261c590e5913370c3200c5e9ebbb92b5aa5e111902 \ + --hash=sha256:40b34dd88658e4fedd2fddbf0275ac970d00614b731357f425722a3ed1983d11 \ + --hash=sha256:4458124d71339f505bf1fb94f69fd1bb8fa9d2481eebfef27c10ef4f2b9e12f6 \ + --hash=sha256:4896db1f4ce0576765b8272aa922df324e0f5b9bb2c3d03044ff32a7234a9aba \ + --hash=sha256:4a0530bb1b8c1c985e7e2122e2b4d3aedd8a3c21c6bfddae6767c4405668b56e \ + --hash=sha256:4aa5435cdb3eb6f55fe98a171b05e3fbcd95fadaa4aa32acf62afd9b0cfdbcac \ + --hash=sha256:4c3501bfa814ab07b5580741f9bf78dfdfe146a04057f82df9e2402d2a975939 \ + --hash=sha256:4e5413bd5f13d3a4e3539ca98f70f75e7fca92518dd7f117f030ebedd10b60cb \ + --hash=sha256:4e6883a021db30511d9fb8cfb0f222ce1f2c369f7d4d8b0448f449a93ba0bdfc \ + --hash=sha256:52579c60a6078be70a0e49c81d6e56d677f34cd439af281a0083b8c7bc75c095 \ + --hash=sha256:555497390743af1a65045fa4527782d10ff5b88970359412baa4a1e628fe393b \ + --hash=sha256:56ad4d9f77df871a99e25c37091052a02528ec0eb059de928ee33956b854b45b \ + --hash=sha256:571fde9741eb0ccde23dd4e0c1d50fbae910e901fa7e629faf39b2dda740d220 \ + --hash=sha256:572fc57b0009c735ee56c175ea021b637a15551a312f56734277f923d6fd0f6c \ + --hash=sha256:59787bd5f8c70aa339084e961d2996b53fbdeab4d5393bba5c1fe1fc32e02bae \ + --hash=sha256:5a2721c8720e2cb3c209925dfb9200199b4b07361c9e01d321719404b21458b3 \ + --hash=sha256:5cc26a66e212fa5d6c6170c3a40d99d888db3020c6fdab1523250d4341382e44 \ + --hash=sha256:5ebee1ee89c39c953baac6924fcde08c5bb427c4057510862f9d7c7bdb3d8665 \ + --hash=sha256:60be8693a1dadc210bbcbc0db3e26da5f7d01d1d5a3da594e99b4fa42df404f5 \ + --hash=sha256:618a0aed532be87294c4477b0481f3aa0f1520f4014a4374dd4cf789b4cd2c97 \ + --hash=sha256:61bb1bd45520aacd56dd80943bd34991fb5350afdd1f36f2282230fd5154a218 \ + --hash=sha256:6383cd2ed53a646c659ba1fe65727db76437fdaa069e697a0b44a51d5843d864 \ + --hash=sha256:64729333167c2dcaaa56a331d40ee097bd9c5617ffd51dabb09eaddafb1b532e \ + --hash=sha256:64b6ca7391a1395c2638dd5c7456d67bea44fc6c5e8e92c5dc8aa6a8f23292b4 \ + --hash=sha256:65dcd28d3eba2ab7c2fd906485cc301392b47cc2234790d27d4e4814e02cdfda \ + --hash=sha256:65fa6cb38ed5e9c3637e68e544f598b39c3b86b808ed0627a67b68320384b459 \ + --hash=sha256:66bd62c59a5427746e8c44becae1d9b99d22fb13f30f492083dfb9ad7c45cc18 \ + --hash=sha256:6e44c0e7c5664be20aee92085153150c0a7967310a73a43c0f832b7cd35d0dd3 \ + --hash=sha256:6f8c6e7a1cfa3dc9d0ee2de0e65e834537fa29992cc3976ffec914afc35c5dd5 \ + --hash=sha256:7322ec6cc9fba9d49ab888bb82d67ac5625627aa168f0165139b17018df3fb8a \ + --hash=sha256:73b133a9e6fb512858e7f065e96f1180aa46646bc74a83aea62f1d314f3dd035 \ + --hash=sha256:73f272fba87b8ccfe70a137d02a54af386f6d27aa509fbffdd978f5947aae1aa \ + --hash=sha256:7e77b324909c1617cbb4c668677e2c6ae13f44d7c1de0d4f15f2e3c10f3315b5 \ + --hash=sha256:80115dd39481fd3a4b4080220799dbcacb921a844de4b827264ececacbe17c78 \ + --hash=sha256:87ccab0db8d5f4fbb0272642113c1adb2ffc698c16d3a0944580222331fa7a20 \ + --hash=sha256:89dfee3319f5ae3f75ebd5c2445a809bb320252ba5529ffdafea4ef25d79cf1a \ + --hash=sha256:8ac59a0900474a52b7c04af8196affc22bd9842acb0950df12f7b813e983609a \ + --hash=sha256:8cae6fd77a5b72dae505084b1a2ee0360139faf72fedbab667cd7cc65aae7a6a \ + --hash=sha256:8d3469c91dd92ee41b7c95280edbd975ef1ba9195086686623a1c6e8935ce965 \ + --hash=sha256:90c633e7e8d6bf4e992b8b36ce69e018f834b641dd6de8cea6d78c06ffa119c5 \ + --hash=sha256:93db40c8de0815baab96a06e08a984bac71f989d13bab789e382158c5d426797 \ + --hash=sha256:9724e6cb5e478cd7d8cabf027826178739cb18cf0e117d0e32814d479fa02276 \ + --hash=sha256:98c6ac18480fcdb33f35439183f1d2e79760ab41930309c6d951cb1f8e46694c \ + --hash=sha256:9a15e785f244f3e07847b984ce8773fc3da10a9f3c131cc49a4c5b4d672b4547 \ + --hash=sha256:9b60d7814174f059e5de4ab98271cc5ba9259cfea55273a81544dceea32dc8d9 \ + --hash=sha256:9be2a6647740dd3cca6acb24e87f03d7632cd280dbce9bbe40c26353a215a45d \ + --hash=sha256:9c7472192ebfad53a6be7c4a8bfb2d64b81c0e93a1fc8c57e1dd0b638297b5d1 \ + --hash=sha256:9dce8ec9695f531a1b8a6f314fd4b393adcccf2ea861db480cdf97a301d01a68 \ + --hash=sha256:9e50d748a32da622f256e8d505867f5d3c43a837c6a9f0efb149655fadd1042a \ + --hash=sha256:a81758ed242b861b72e778ba34d41366441a2e10b16b472784c88da2dea7e2dd \ + --hash=sha256:ac777001cdfc28b72477d93c8564bb7583081ea8fb45cdca3d568e0a4f87183c \ + --hash=sha256:b2b506b1788df5fecd270a10d5e70a95fe77b87ea2b370a318043f6f5f817ee6 \ + --hash=sha256:b2ea4a3e8357be8849e833beeae757ac3c7a6b3fc055c03c808a53c91ad30d82 \ + --hash=sha256:bf1516fe58fc104f39b2d1dbe2d5e27d0cd45c4be2e42ba6ee0cc763701ec3c7 \ + --hash=sha256:c0d702548d89d572b2929879bc883bb7a4c4709efafe4512cadee56c55c9bd15 \ + --hash=sha256:c10b82c2634df08dfb13b1f04e38fe310d086ee092f4f69c0c8da234251e556e \ + --hash=sha256:c42572142ed0b9d5d261ba727157c426510da78e20828b66bbb855098b8a4e38 \ + --hash=sha256:c4585c3e64b4f9e583b4d2683f18f5d5d872b3d71dcf24594b74ecc23602fa96 \ + --hash=sha256:c639ea314df70a7b2811e8020448c75af8c9445f5a60f8a4ced81c306a9380c2 \ + --hash=sha256:c670fe7be5b6020b76bc6e8d2196074657e1327595bca93a389e1a76ab130ad8 \ + --hash=sha256:cc1b2440423a851fad781309dd87843868f4f66a6bcd1ddb9225cf4ec2c84732 \ + --hash=sha256:cd3584591ea4429026cdb931b054342c2bcf189b44ff367f8d5c15bc092a2966 \ + --hash=sha256:d15df07081d91b76ff20d43f94592ee110330152d617b730fdbe5ef9fb680053 \ + --hash=sha256:d19662dbedbe783d323196312d38f5ba53cf56296378252171985da6899887d3 \ + --hash=sha256:d24ecb4f5e009ea0bd275ee37ad9953b32005e2e5e60f8bbae16da0dbbf0d3a0 \ + --hash=sha256:d446c6ac40bb6e05025ccee55b84d80fe9bf8e93010ffc4bb9484f13d498835f \ + --hash=sha256:d51ffd3427640fa2da6ade574ceba932f210ad095f65fcc450a2b0a0d454868e \ + --hash=sha256:d6ce43a0269d68cee79a7d1ade7def53c20f8f2a047b92d7b5d5bcc73ae88327 \ + --hash=sha256:d721e53758b2cca74990185eb0671dd466d7a388a1a45d0c6f4c13cef41a68ac \ + --hash=sha256:d7da47a0f248977f08e2cb659ff3c17ddc13a4d39b3a7baa0a81bf5b415430f6 \ + --hash=sha256:db47b561c9afd884baa1f96f797c9ca369872c4b65912bc691cfa99e68340af2 \ + --hash=sha256:dbe6493fbd27321b1d1f2dd4f5c7e5bd4d8b1d7cab7f32fd67db3d0b2ed8248a \ + --hash=sha256:dbece16025afda5e3031af0c4059207e61dcf73ef13af844964f57f387d1c435 \ + --hash=sha256:ddd67571c10869f65a5d7dde536d1e066e306cc90de57d7de4d5f34802428bb5 \ + --hash=sha256:de9208bb427130c82a5dbfd104f92c8876fc9559278c880b3002755bbbe9c83d \ + --hash=sha256:e30d40268a28d54ce0437031750497004c22602b8e3ab891f759b795a003b312 \ + --hash=sha256:e8b0abe7d870f53ca5143895fef7d1041a0c831a140d3dc2c760dd7ba25d4a8b \ + --hash=sha256:f035d9dc1d25eff9d361456572231c7d27b5ccd473ca7dc0adfce732bd006d40 \ + --hash=sha256:f04b9f56b0e0614c0126be12c2c2d9f8850c1e57af302bd0a63bed379d4af974 \ + --hash=sha256:f0fa4fa9c3632d708742baf2282f2055c11d888a790362670a403cbf48a2c404 \ + --hash=sha256:f2e7f8e2ab6c2922be02c7ec45185aa5bd771e2e57b95455ee343a44d8130dff \ + --hash=sha256:f8f6fa298bb4f7f58a33334406218ba74716e68feddf5e4e54cd5d8082705abf \ + --hash=sha256:fbf300e2070bb35038660b3be1be4b91b0024edb41517e6996320b49b92b4175 \ + --hash=sha256:fce7760bf283405b2c7999cab3da4e72f7deca6396013115e3f7a955db9760da \ + --hash=sha256:fcee38cd8e5089d6d4f048ba1233b3ad76e5954f545382180889112ff5cb712d \ + --hash=sha256:fe31f28c94402043161876a258a9c6f757cb485905c7614ce8d6cd40e6b7bdc1 \ + --hash=sha256:ffd8893ccc1c2fce6e0d6ca402d716fe1b29db70c7132609a05955e31b2aa8f2 + # via transformers +resampy==0.4.3 \ + --hash=sha256:a0d1c28398f0e55994b739650afef4e3974115edbe96cd4bb81968425e916e47 \ + --hash=sha256:ad2ed64516b140a122d96704e32bc0f92b23f45419e8b8f478e5a05f83edcebd + # via -r backend/runtime-locks/windows-gpu-pytorch.in +rich==15.0.0 \ + --hash=sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb \ + --hash=sha256:edd07a4824c6b40189fb7ac9bc4c52536e9780fbbfbddf6f1e2502c31b068c36 + # via typer +safetensors==0.7.0 \ + --hash=sha256:0071bffba4150c2f46cae1432d31995d77acfd9f8db598b5d1a2ce67e8440ad2 \ + --hash=sha256:07663963b67e8bd9f0b8ad15bb9163606cd27cc5a1b96235a50d8369803b96b0 \ + --hash=sha256:12f49080303fa6bb424b362149a12949dfbbf1e06811a88f2307276b0c131afd \ + --hash=sha256:1d060c70284127fa805085d8f10fbd0962792aed71879d00864acda69dbab981 \ + --hash=sha256:42cb091236206bb2016d245c377ed383aa7f78691748f3bb6ee1bfa51ae2ce6a \ + --hash=sha256:473b32699f4200e69801bf5abf93f1a4ecd432a70984df164fc22ccf39c4a6f3 \ + --hash=sha256:54bef08bf00a2bff599982f6b08e8770e09cc012d7bba00783fc7ea38f1fb37d \ + --hash=sha256:5d72abdb8a4d56d4020713724ba81dac065fedb7f3667151c4a637f1d3fb26c0 \ + --hash=sha256:672132907fcad9f2aedcb705b2d7b3b93354a2aec1b2f706c4db852abe338f85 \ + --hash=sha256:6999421eb8ba9df4450a16d9184fcb7bef26240b9f98e95401f17af6c2210b71 \ + --hash=sha256:7b95a3fa7b3abb9b5b0e07668e808364d0d40f6bbbf9ae0faa8b5b210c97b140 \ + --hash=sha256:8469155f4cb518bafb4acf4865e8bb9d6804110d2d9bdcaa78564b9fd841e104 \ + --hash=sha256:94fd4858284736bb67a897a41608b5b0c2496c9bdb3bf2af1fa3409127f20d57 \ + --hash=sha256:b0f6d66c1c538d5a94a73aa9ddca8ccc4227e6c9ff555322ea40bdd142391dd4 \ + --hash=sha256:c74af94bf3ac15ac4d0f2a7c7b4663a15f8c2ab15ed0fc7531ca61d0835eccba \ + --hash=sha256:c82f4d474cf725255d9e6acf17252991c3c8aac038d6ef363a4bf8be2f6db517 \ + --hash=sha256:cdab83a366799fa730f90a4ebb563e494f28e9e92c4819e556152ad55e43591b \ + --hash=sha256:cfdead2f57330d76aa7234051dadfa7d4eedc0e5a27fd08e6f96714a92b00f09 \ + --hash=sha256:d1239932053f56f3456f32eb9625590cc7582e905021f94636202a864d470755 \ + --hash=sha256:dac7252938f0696ddea46f5e855dd3138444e82236e3be475f54929f0c510d48 \ + --hash=sha256:dc92bc2db7b45bda4510e4f51c59b00fe80b2d6be88928346e4294ce1c2abe7c \ + --hash=sha256:e07d91d0c92a31200f25351f4acb2bc6aff7f48094e13ebb1d0fb995b54b6542 \ + --hash=sha256:f4729811a6640d019a4b7ba8638ee2fd21fa5ca8c7e7bdf0fed62068fcaac737 + # via + # -r backend/runtime-locks/windows-gpu-pytorch.in + # transformers +sentencepiece==0.2.1 \ + --hash=sha256:010f025a544ef770bb395091d57cb94deb9652d8972e0d09f71d85d5a0816c8c \ + --hash=sha256:017f97b274d4b0baa84b2dc743bf4517be81156f413bb24f12aacacde378e5ab \ + --hash=sha256:01e6912125cb45d3792f530a4d38f8e21bf884d6b4d4ade1b2de5cf7a8d2a52b \ + --hash=sha256:02593eca45440ef39247cee8c47322a34bdcc1d8ae83ad28ba5a899a2cf8d79a \ + --hash=sha256:097f3394e99456e9e4efba1737c3749d7e23563dd1588ce71a3d007f25475fff \ + --hash=sha256:0a0d15781a171d188b661ae4bde1d998c303f6bd8621498c50c671bd45a4798e \ + --hash=sha256:0a81799d0a68d618e89063fb423c3001a034c893069135ffe51fee439ae474d6 \ + --hash=sha256:0c0f672da370cc490e4c59d89e12289778310a0e71d176c541e4834759e1ae07 \ + --hash=sha256:0cdfecef430d985f1c2bcbfff3defd1d95dae876fbd0173376012d2d7d24044b \ + --hash=sha256:105e36e75cbac1292642045458e8da677b2342dcd33df503e640f0b457cb6751 \ + --hash=sha256:10ed3dab2044c47f7a2e7b4969b0c430420cdd45735d78c8f853191fa0e3148b \ + --hash=sha256:1855f57db07b51fb51ed6c9c452f570624d2b169b36f0f79ef71a6e6c618cd8b \ + --hash=sha256:2005242a16d2dc3ac5fe18aa7667549134d37854823df4c4db244752453b78a8 \ + --hash=sha256:22c4ebcb3c6ab1496ab1c37c79ef7bb563b8726f29548c30773b7a4cb152df1a \ + --hash=sha256:251874d720ac7f28024a168501f3c7bb15d1802245f6e66de565f18bbb9b5eaa \ + --hash=sha256:27e38eee653abc3d387862e67bc5c8b6f428cd604e688b85d29170b7e725c26c \ + --hash=sha256:2af5a1fb05013332ad94343b8b5f3973e006a2dde2dfba55a819549e054e2f0f \ + --hash=sha256:2f27ae6deea72efdb6f361750c92f6c21fd0ad087445082770cc34015213c526 \ + --hash=sha256:33f068c9382dc2e7c228eedfd8163b52baa86bb92f50d0488bf2b7da7032e484 \ + --hash=sha256:39f8651bd10974eafb9834ce30d9bcf5b73e1fc798a7f7d2528f9820ca86e119 \ + --hash=sha256:3d165fbb9bf8fba35f1946ba2617c3f9995679f07438325f07c026d53f33e746 \ + --hash=sha256:477c81505db072b3ab627e7eab972ea1025331bd3a92bacbf798df2b75ea86ec \ + --hash=sha256:4cdc7c36234fda305e85c32949c5211faaf8dd886096c7cea289ddc12a2d02de \ + --hash=sha256:4f5a3e0d9f445ed9d66c0fec47d4b23d12cfc858b407a03c194c1b26c2ac2a63 \ + --hash=sha256:56dd39a3c4d6493db3cdca7e8cc68c6b633f0d4195495cbadfcf5af8a22d05a6 \ + --hash=sha256:57cae326c8727de58c85977b175af132a7138d84c764635d7e71bbee7e774133 \ + --hash=sha256:5d0350b686c320068702116276cfb26c066dc7e65cfef173980b11bb4d606719 \ + --hash=sha256:5e4366c97b68218fd30ea72d70c525e6e78a6c0a88650f57ac4c43c63b234a9d \ + --hash=sha256:60937c959e6f44159fdd9f56fbdd302501f96114a5ba436829496d5f32d8de3f \ + --hash=sha256:6356d0986b8b8dc351b943150fcd81a1c6e6e4d439772e8584c64230e58ca987 \ + --hash=sha256:6d297a1748d429ba8534eebe5535448d78b8acc32d00a29b49acf28102eeb094 \ + --hash=sha256:733e59ff1794d26db706cd41fc2d7ca5f6c64a820709cb801dc0ea31780d64ab \ + --hash=sha256:8138cec27c2f2282f4a34d9a016e3374cd40e5c6e9cb335063db66a0a3b71fad \ + --hash=sha256:814978ac05130dd5812b4b03215c766bc6abaef13e7bd72bc534e4d1e12e9a4c \ + --hash=sha256:82d9ead6591015f009cb1be1cb1c015d5e6f04046dbb8c9588b931e869a29728 \ + --hash=sha256:881b2e44b14fc19feade3cbed314be37de639fc415375cefaa5bc81a4be137fd \ + --hash=sha256:891ade6503dd93d418c03993f7d6a8aa20260c422cefff5096b9068185e67642 \ + --hash=sha256:89a3ea015517c42c0341d0d962f3e6aaf2cf10d71b1932d475c44ba48d00aa2b \ + --hash=sha256:8dd4b477a7b069648d19363aad0cab9bad2f4e83b2d179be668efa672500dc94 \ + --hash=sha256:8f8ba89a3acb3dc1ae90f65ec1894b0b9596fdb98ab003ff38e058f898b39bc7 \ + --hash=sha256:9076430ac25dfa7147d9d05751dbc66a04bc1aaac371c07f84952979ea59f0d0 \ + --hash=sha256:92b3816aa2339355fda2c8c4e021a5de92180b00aaccaf5e2808972e77a4b22f \ + --hash=sha256:99f955df238021bf11f0fc37cdb54fd5e5b5f7fd30ecc3d93fb48b6815437167 \ + --hash=sha256:a19adcec27c524cb7069a1c741060add95f942d1cbf7ad0d104dffa0a7d28a2b \ + --hash=sha256:a483fd29a34c3e34c39ac5556b0a90942bec253d260235729e50976f5dba1068 \ + --hash=sha256:ac650534e2251083c5f75dde4ff28896ce7c8904133dc8fef42780f4d5588fcd \ + --hash=sha256:ad8493bea8432dae8d6830365352350f3b4144415a1d09c4c8cb8d30cf3b6c3c \ + --hash=sha256:afefe50a0cdcb4f2fd9733cb52001a2c164181ee2d82c32d38f5b1b326a8528c \ + --hash=sha256:b3616ad246f360e52c85781e47682d31abfb6554c779e42b65333d4b5f44ecc0 \ + --hash=sha256:b81a24733726e3678d2db63619acc5a8dccd074f7aa7a54ecd5ca33ca6d2d596 \ + --hash=sha256:c415c9de1447e0a74ae3fdb2e52f967cb544113a3a5ce3a194df185cbc1f962f \ + --hash=sha256:c6c8f42949f419ff8c7e9960dbadcfbc982d7b5efc2f6748210d3dd53a7de062 \ + --hash=sha256:c7f0fd2f2693309e6628aeeb2e2faf6edd221134dfccac3308ca0de01f8dab47 \ + --hash=sha256:c7f54a31cde6fa5cb030370566f68152a742f433f8d2be458463d06c208aef33 \ + --hash=sha256:c83b85ab2d6576607f31df77ff86f28182be4a8de6d175d2c33ca609925f5da1 \ + --hash=sha256:caa4e560c72c151da80036aecc2159e51a7fd8ae9efebefd96860460ce6bd025 \ + --hash=sha256:d3233770f78e637dc8b1fda2cd7c3b99ec77e7505041934188a4e7fe751de3b0 \ + --hash=sha256:d7b670879c370d350557edabadbad1f6561a9e6968126e6debca4029e5547820 \ + --hash=sha256:d8b1d91545578852f128650b8cce4ec20f93d39b378ff554ebe66290f2dabb92 \ + --hash=sha256:d9381351182ff9888cc80e41c632e7e274b106f450de33d67a9e8f6043da6f76 \ + --hash=sha256:daeb5e9e9fcad012324807856113708614d534f596d5008638eb9b40112cd9e4 \ + --hash=sha256:dcd8161eee7b41aae57ded06272905dbd680a0a04b91edd0f64790c796b2f706 \ + --hash=sha256:e10fa50bdbaa5e2445dbd387979980d391760faf0ec99a09bd7780ff37eaec44 \ + --hash=sha256:e37e4b4c4a11662b5db521def4e44d4d30ae69a1743241412a93ae40fdcab4bb \ + --hash=sha256:e52144670738b4b477fade6c2a9b6af71a8d0094514c9853ac9f6fc1fcfabae7 + # via -r backend/runtime-locks/windows-gpu-pytorch.in +setuptools==84.0.0 \ + --hash=sha256:51a52592b3b99e102b609654876bd65f19f999935166d1352678931132b0c670 \ + --hash=sha256:f4695c21257f0d9b537ec2692c941d02ee143b7cc1276941349a546573b2ef73 + # via torch +shellingham==1.5.4 \ + --hash=sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686 \ + --hash=sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de + # via typer +soundfile==0.13.1 \ + --hash=sha256:03267c4e493315294834a0870f31dbb3b28a95561b80b134f0bd3cf2d5f0e618 \ + --hash=sha256:1e70a05a0626524a69e9f0f4dd2ec174b4e9567f4d8b6c11d38b5c289be36ee9 \ + --hash=sha256:743f12c12c4054921e15736c6be09ac26b3b3d603aef6fd69f9dde68748f2593 \ + --hash=sha256:82dc664d19831933fe59adad199bf3945ad06d84bc111a5b4c0d3089a5b9ec33 \ + --hash=sha256:9c9e855f5a4d06ce4213f31918653ab7de0c5a8d8107cd2427e44b42df547deb \ + --hash=sha256:a23c717560da2cf4c7b5ae1142514e0fd82d6bbd9dfc93a50423447142f2c445 \ + --hash=sha256:b2c68dab1e30297317080a5b43df57e302584c49e2942defdde0acccc53f0e5b \ + --hash=sha256:c734564fab7c5ddf8e9be5bf70bab68042cd17e9c214c06e365e20d64f9a69d5 + # via -r backend/runtime-locks/windows-gpu-pytorch.in +sympy==1.14.0 \ + --hash=sha256:d3d3fe8df1e5a0b42f0e7bdf50541697dbe7d23746e894990c030e2b05e72517 \ + --hash=sha256:e091cc3e99d2141a0ba2847328f5479b05d94a6635cb96148ccb3f34671bd8f5 + # via torch +tokenizers==0.22.2 \ + --hash=sha256:143b999bdc46d10febb15cbffb4207ddd1f410e2c755857b5a0797961bbdc113 \ + --hash=sha256:1a62ba2c5faa2dd175aaeed7b15abf18d20266189fb3406c5d0550dd34dd5f37 \ + --hash=sha256:1c774b1276f71e1ef716e5486f21e76333464f47bece56bbd554485982a9e03e \ + --hash=sha256:1e418a55456beedca4621dbab65a318981467a2b188e982a23e117f115ce5001 \ + --hash=sha256:1e50f8554d504f617d9e9d6e4c2c2884a12b388a97c5c77f0bc6cf4cd032feee \ + --hash=sha256:2249487018adec45d6e3554c71d46eb39fa8ea67156c640f7513eb26f318cec7 \ + --hash=sha256:25b85325d0815e86e0bac263506dd114578953b7b53d7de09a6485e4a160a7dd \ + --hash=sha256:29c30b83d8dcd061078b05ae0cb94d3c710555fbb44861139f9f83dcca3dc3e4 \ + --hash=sha256:319f659ee992222f04e58f84cbf407cfa66a65fe3a8de44e8ad2bc53e7d99012 \ + --hash=sha256:369cc9fc8cc10cb24143873a0d95438bb8ee257bb80c71989e3ee290e8d72c67 \ + --hash=sha256:37ae80a28c1d3265bb1f22464c856bd23c02a05bb211e56d0c5301a435be6c1a \ + --hash=sha256:38337540fbbddff8e999d59970f3c6f35a82de10053206a7562f1ea02d046fa5 \ + --hash=sha256:473b83b915e547aa366d1eee11806deaf419e17be16310ac0a14077f1e28f917 \ + --hash=sha256:544dd704ae7238755d790de45ba8da072e9af3eea688f698b137915ae959281c \ + --hash=sha256:64d94e84f6660764e64e7e0b22baa72f6cd942279fdbb21d46abd70d179f0195 \ + --hash=sha256:753d47ebd4542742ef9261d9da92cd545b2cacbb48349a1225466745bb866ec4 \ + --hash=sha256:791135ee325f2336f498590eb2f11dc5c295232f288e75c99a36c5dbce63088a \ + --hash=sha256:9ce725d22864a1e965217204946f830c37876eee3b2ba6fc6255e8e903d5fcbc \ + --hash=sha256:a6bf3f88c554a2b653af81f3204491c818ae2ac6fbc09e76ef4773351292bc92 \ + --hash=sha256:bfb88f22a209ff7b40a576d5324bf8286b519d7358663db21d6246fb17eea2d5 \ + --hash=sha256:c9ea31edff2968b44a88f97d784c2f16dc0729b8b143ed004699ebca91f05c48 \ + --hash=sha256:df6c4265b289083bf710dff49bc51ef252f9d5be33a45ee2bed151114a56207b \ + --hash=sha256:e10bf9113d209be7cd046d40fbabbaf3278ff6d18eb4da4c500443185dc1896c \ + --hash=sha256:f01a9c019878532f98927d2bacb79bbb404b43d3437455522a00a30718cdedb5 + # via transformers +torch==2.7.1+cu126 \ + --hash=sha256:03b83a0f2c1e90afafd7a5728b956e211bb3e6c56ea3d7d8c7638a659e448d5f \ + --hash=sha256:27d396231f33dc6103ba26ec6ec2ec5939d9850b599e32da711b038af272954e \ + --hash=sha256:30119a54e1b4ccefe20dfe5d4b13f6aef76c17ec605b40e26d39789db00906f2 \ + --hash=sha256:49692cc24edb72ba247a6f37345572cb2371f125eda132bc2834fd842f16bb7e \ + --hash=sha256:63bce0590bc540fc16139e2be0177847585182b8c5e68d7f9213789d1d96c978 \ + --hash=sha256:7d897b5ff67e778de4a2a05d4528377003105e29854fd73ecbe965287533f08b \ + --hash=sha256:a05c0001fd1d0ceae9cda8c8c1b8a16ed5def858fe996c9237a28016559dad52 \ + --hash=sha256:a38a903c9b55cea1217100e0851b25659765b6bb8cd75e6de6bbf0063a2cd51e \ + --hash=sha256:d4e68a1aeb2a6272d0234b7575089fc70757a93d24dccde8e962a3b18aef77d1 \ + --hash=sha256:e1a8465165708c2e2e90786ade8a3e1b1d01eca1f022792cd397caad9d8c21bc \ + --hash=sha256:ef0d0b0bd96d2adb07a47da12426e60d91921dfcd7c1964eea309f41488c2462 \ + --hash=sha256:f3af23387ac106b5b01dbef0eb021883e0c00ff4073477b7ce1cbade5ef5038d + # via + # -r backend/runtime-locks/windows-gpu-pytorch.in + # torchaudio +torchaudio==2.7.1+cu126 \ + --hash=sha256:0c306e9f5ae1204dc8cd998912d0ad4b13420d00428aa5f5fcab01b180abe386 \ + --hash=sha256:1e9231eb156400a53d8041688f7567ac92c8758332225e364c35d68603cae2da \ + --hash=sha256:2c5d2d639aa466cbb3cf3f32d20009b6a6d472df8e07a88238248b1bcf208023 \ + --hash=sha256:33e67c2da5da68c075a065062cc39115b7af29890120c4773cfd24a9ea05a428 \ + --hash=sha256:38484cd84566b96a12e7371ea0f5a91ce35708c8c2aaa223342da833e094ee6d \ + --hash=sha256:560692b35b4c0325b4b11e793574e04565769f462a9146ad254d6d0411b8c7f4 \ + --hash=sha256:759de378d1fe4f4a5a56d58f51b4a05a9aa9681ae7b9469e638d9ed321a73e95 \ + --hash=sha256:9c6e00c79c09572a65eb54652d12f00fa79df63847552f61f10b7705fd0aca0b \ + --hash=sha256:c293d7e8d5d86d855313a940862c8d1506536a80ea44eca1ab020579b4f8bb0b \ + --hash=sha256:d3a5f160336aa7ec262f1361755c5da4e451571ed69f3cb8435b09bd7a2a0227 \ + --hash=sha256:d9a65c7f7100748802030257efd22535decd74eba310d2b2b9c41489c77cbe1d \ + --hash=sha256:fb9d04a635ab9856bf70c47926e3d47976c36aeb60779e6c4639ac2a630f9b0f + # via -r backend/runtime-locks/windows-gpu-pytorch.in +tqdm==4.67.3 \ + --hash=sha256:7d825f03f89244ef73f1d4ce193cb1774a8179fd96f31d7e1dcde62092b960bb \ + --hash=sha256:ee1e4c0e59148062281c49d80b25b67771a127c85fc9676d3be5f243206826bf + # via + # -r backend/runtime-locks/windows-gpu-pytorch.in + # huggingface-hub + # transformers +transformers==5.8.0 \ + --hash=sha256:6cc9a1f0291d16b1c1b735bad775e78ebefff7722701d4e28f98aaaa2bd6fb91 \ + --hash=sha256:e9d2cae6d195a7e1e05164c5ebf26142a7044e4dc4267274f4809204f92827e4 + # via -r backend/runtime-locks/windows-gpu-pytorch.in +typer==0.27.1 \ + --hash=sha256:53150287edd11baeb4e4722c8e394fcdf8181c0ae89485cba8d25c778d5edd56 \ + --hash=sha256:a79bef8469a79c45498e7b814ecf8d603cc7644e9acbd9e19cac0334240b18df + # via + # huggingface-hub + # transformers +typing-extensions==4.16.0 \ + --hash=sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8 \ + --hash=sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5 + # via + # anyio + # huggingface-hub + # torch diff --git a/backend/tests/test_controller.py b/backend/tests/test_controller.py index 484d54f..4186c79 100644 --- a/backend/tests/test_controller.py +++ b/backend/tests/test_controller.py @@ -8,13 +8,17 @@ import io import json import queue +import threading import wave +from concurrent.futures import ThreadPoolExecutor import pytest from fastapi.testclient import TestClient from lsdj import controller, sa3 +API_TOKEN = "test-api-capability-0123456789abcdef" + class FakeProcess: def __init__(self): @@ -53,10 +57,89 @@ def send(self, command): if command.get("type") == "render_clip" and self.render_response is not None: self.clip_queue.put((command["id"], self.render_response)) + def shutdown(self): + self.process.terminate() + @pytest.fixture -def client(): - return TestClient(controller.app) +def client(monkeypatch): + controller.configure_local_api(API_TOKEN) + monkeypatch.setattr(controller, "generation_jobs", controller.JobRegistry()) + with TestClient( + controller.app, headers={controller.API_CAPABILITY_HEADER: API_TOKEN} + ) as test_client: + yield test_client + + +def request_options(options): + """Remove controller-owned lifecycle hooks from a captured SA3 call.""" + + return { + key: value + for key, value in options.items() + if key not in {"cancel_event", "on_progress", "on_state"} + } + + +def test_local_api_rejects_missing_wrong_and_foreign_credentials(): + controller.configure_local_api(API_TOKEN) + with TestClient(controller.app) as unauthenticated: + assert unauthenticated.get("/api/models").status_code == 401 + assert ( + unauthenticated.get( + "/api/models", headers={controller.API_CAPABILITY_HEADER: "x" * 40} + ).status_code + == 401 + ) + assert ( + unauthenticated.get( + "/api/models", + headers={ + controller.API_CAPABILITY_HEADER: API_TOKEN, + "origin": "https://attacker.example", + }, + ).status_code + == 403 + ) + + +def test_local_api_preflight_allows_only_the_exact_tauri_origin_and_headers(): + controller.configure_local_api(API_TOKEN) + with TestClient(controller.app) as browser: + allowed = browser.options( + "/api/generate", + headers={ + "origin": "http://tauri.localhost", + "access-control-request-method": "POST", + "access-control-request-headers": ( + "content-type, x-lsdj-capability, x-lsdj-job-id" + ), + }, + ) + assert allowed.status_code == 204 + assert ( + allowed.headers["access-control-allow-origin"] == "http://tauri.localhost" + ) + assert "access-control-allow-credentials" not in allowed.headers + + foreign = browser.options( + "/api/generate", + headers={ + "origin": "https://attacker.example", + "access-control-request-method": "POST", + }, + ) + assert foreign.status_code == 403 + + extra_header = browser.options( + "/api/generate", + headers={ + "origin": "tauri://localhost", + "access-control-request-method": "POST", + "access-control-request-headers": "x-lsdj-capability, authorization", + }, + ) + assert extra_header.status_code == 403 # --- /api/generate (M18, ADR-0012) --------------------------------------- @@ -93,7 +176,7 @@ def generate_multipart(metadata, audio=None, extra=()): def test_generate_returns_wav_and_strips_the_prompt(client, monkeypatch): calls = [] - async def fake_generate(prompt, seconds, kind): + async def fake_generate(prompt, seconds, kind, **options): calls.append((prompt, seconds, kind)) return b"RIFFwav" @@ -111,7 +194,7 @@ def test_generate_forwards_optional_json_controls(client, monkeypatch): calls = [] async def fake_generate(prompt, seconds, kind, **options): - calls.append((prompt, seconds, kind, options)) + calls.append((prompt, seconds, kind, request_options(options))) return b"RIFFwav" monkeypatch.setattr(controller.sa3, "generate", fake_generate) @@ -123,6 +206,7 @@ async def fake_generate(prompt, seconds, kind, **options): cfg=4.5, apg=0.75, seed=12345, + steps=12, ), ) assert response.status_code == 200 @@ -137,6 +221,7 @@ async def fake_generate(prompt, seconds, kind, **options): "apg": 0.75, "negative_prompt": "vocals", "seed": 12345, + "steps": 12, }, ) ] @@ -164,7 +249,7 @@ def test_generate_forwards_the_lora_stack_with_aligned_strengths( calls = [] async def fake_generate(prompt, seconds, kind, **options): - calls.append(options) + calls.append(request_options(options)) return b"RIFFwav" monkeypatch.setattr(controller.sa3, "generate", fake_generate) @@ -197,7 +282,7 @@ def test_generate_accepts_the_lora_strength_boundaries( calls = [] async def fake_generate(prompt, seconds, kind, **options): - calls.append(options) + calls.append(request_options(options)) return b"RIFFwav" monkeypatch.setattr(controller.sa3, "generate", fake_generate) @@ -215,7 +300,7 @@ def test_generate_treats_an_empty_lora_stack_as_no_adapters( calls = [] async def fake_generate(prompt, seconds, kind, **options): - calls.append(options) + calls.append(request_options(options)) return b"RIFFwav" monkeypatch.setattr(controller.sa3, "generate", fake_generate) @@ -286,12 +371,13 @@ def test_generate_forwards_multipart_init_audio_and_inpaint( ) async def fake_generate(prompt, seconds, kind, **options): - calls.append((prompt, seconds, kind, options)) + calls.append((prompt, seconds, kind, request_options(options))) return b"RIFFwav" monkeypatch.setattr(controller.sa3, "generate", fake_generate) response = client.post("/api/generate", files=generate_multipart(metadata, source)) assert response.status_code == 200 + normalized = sa3.normalize_wav(source).wav assert calls == [ ( "vinyl spinback", @@ -301,7 +387,7 @@ async def fake_generate(prompt, seconds, kind, **options): "init_noise_level": 0.55, "seed": 7, "inpaint_range": (0.0, 3.0), - "init_audio": source, + "init_audio": normalized, }, ) ] @@ -311,7 +397,7 @@ def test_generate_accepts_the_optional_control_boundaries(client, monkeypatch): calls = [] async def fake_generate(prompt, seconds, kind, **options): - calls.append(options) + calls.append(request_options(options)) return b"RIFFwav" monkeypatch.setattr(controller.sa3, "generate", fake_generate) @@ -323,6 +409,7 @@ async def fake_generate(prompt, seconds, kind, **options): cfg=sa3.MIN_CFG, apg=sa3.MIN_APG, seed=sa3.MAX_SEED, + steps=sa3.MIN_STEPS, ), ) assert response.status_code == 200 @@ -333,6 +420,7 @@ async def fake_generate(prompt, seconds, kind, **options): "apg": sa3.MIN_APG, "negative_prompt": "kick", "seed": sa3.MAX_SEED, + "steps": sa3.MIN_STEPS, } ] @@ -341,7 +429,7 @@ def test_generate_accepts_the_optional_control_upper_boundaries(client, monkeypa calls = [] async def fake_generate(prompt, seconds, kind, **options): - calls.append(options) + calls.append(request_options(options)) return b"RIFFwav" monkeypatch.setattr(controller.sa3, "generate", fake_generate) @@ -353,6 +441,7 @@ async def fake_generate(prompt, seconds, kind, **options): cfg=sa3.MAX_CFG, apg=sa3.MAX_APG, seed=0, + steps=sa3.MAX_STEPS, ), ) assert response.status_code == 200 @@ -363,6 +452,7 @@ async def fake_generate(prompt, seconds, kind, **options): "apg": sa3.MAX_APG, "negative_prompt": "kick", "seed": 0, + "steps": sa3.MAX_STEPS, } ] @@ -386,7 +476,7 @@ async def fake_generate(prompt, seconds, kind, **options): ], ) def test_generate_validates_the_trust_boundary(client, monkeypatch, body): - async def fake_generate(prompt, seconds, kind): # pragma: no cover + async def fake_generate(prompt, seconds, kind, **options): # pragma: no cover raise AssertionError("invalid input must not reach generation") monkeypatch.setattr(controller.sa3, "generate", fake_generate) @@ -399,7 +489,7 @@ def test_generate_accepts_a_track_at_track_length(client, monkeypatch): # while pad kinds keep the small-model 32 s bound. calls = [] - async def fake_generate(prompt, seconds, kind): + async def fake_generate(prompt, seconds, kind, **options): calls.append((prompt, seconds, kind)) return b"RIFFwav" @@ -414,7 +504,7 @@ async def fake_generate(prompt, seconds, kind): def test_generate_rejects_nan_seconds(client, monkeypatch): # httpx's json= encoder refuses NaN, but Python's json.loads parses it — # so it can reach the server, and the boundary must catch it. - async def fake_generate(prompt, seconds, kind): # pragma: no cover + async def fake_generate(prompt, seconds, kind, **options): # pragma: no cover raise AssertionError("invalid input must not reach generation") monkeypatch.setattr(controller.sa3, "generate", fake_generate) @@ -455,6 +545,11 @@ async def fake_generate(prompt, seconds, kind): # pragma: no cover {"seed": 1.5}, {"seed": -1}, {"seed": sa3.MAX_SEED + 1}, + {"steps": None}, + {"steps": True}, + {"steps": 1.5}, + {"steps": sa3.MIN_STEPS - 1}, + {"steps": sa3.MAX_STEPS + 1}, {"inpaint_range": None}, {"inpaint_range": []}, {"inpaint_range": [0]}, @@ -484,7 +579,7 @@ def test_generate_treats_a_blank_negative_prompt_as_absent(client, monkeypatch): calls = [] async def fake_generate(prompt, seconds, kind, **options): - calls.append(options) + calls.append(request_options(options)) return b"RIFFwav" monkeypatch.setattr(controller.sa3, "generate", fake_generate) @@ -500,9 +595,6 @@ async def fake_generate(prompt, seconds, kind, **options): [ b"", b"not a wave", - pcm16_wav(sample_rate=48_000), - pcm16_wav(channels=3), - pcm16_wav(sample_width=1), pcm16_wav(frames=0), pcm16_wav()[:-4], ], @@ -518,6 +610,24 @@ async def fake_generate(prompt, seconds, kind, **options): # pragma: no cover assert response.status_code == 422 +def test_generate_normalizes_sample_rate_width_and_channel_layout(client, monkeypatch): + calls = [] + source = pcm16_wav(sample_rate=48_000, channels=3, sample_width=1, frames=48) + + async def fake_generate(prompt, seconds, kind, **options): + calls.append(options["init_audio"]) + return b"RIFFwav" + + monkeypatch.setattr(controller.sa3, "generate", fake_generate) + response = client.post( + "/api/generate", files=generate_multipart(generate_request(), source) + ) + assert response.status_code == 200 + normalized = sa3.inspect_canonical_wav(calls[0]) + assert normalized.frames == 44 + assert normalized.seconds == pytest.approx(44 / 44_100) + + def test_generate_rejects_an_oversized_init_file(client, monkeypatch): monkeypatch.setattr(sa3, "MAX_INIT_AUDIO_BYTES", 48) response = client.post( @@ -615,7 +725,7 @@ def test_generate_rejects_bad_content_types_and_malformed_bodies(client): def test_generate_maps_missing_checkout_to_503(client, monkeypatch): - async def fake_generate(prompt, seconds, kind): + async def fake_generate(prompt, seconds, kind, **options): raise controller.sa3.GenerationUnavailable("setup hint") monkeypatch.setattr(controller.sa3, "generate", fake_generate) @@ -625,7 +735,7 @@ async def fake_generate(prompt, seconds, kind): def test_generate_maps_cli_failure_to_502(client, monkeypatch): - async def fake_generate(prompt, seconds, kind): + async def fake_generate(prompt, seconds, kind, **options): raise controller.sa3.GenerationFailed("error: no DiT weights found") monkeypatch.setattr(controller.sa3, "generate", fake_generate) @@ -634,6 +744,64 @@ async def fake_generate(prompt, seconds, kind): assert "no DiT weights" in response.json()["detail"] +def test_generate_maps_cancellation_to_499(client, monkeypatch): + async def fake_generate(prompt, seconds, kind, **options): + raise controller.sa3.GenerationCancelled("generation cancelled") + + monkeypatch.setattr(controller.sa3, "generate", fake_generate) + response = client.post("/api/generate", json=generate_request()) + assert response.status_code == 499 + + +def test_job_cancel_is_responsive_and_does_not_cancel_another_job(client, monkeypatch): + started = {"cancel": threading.Event(), "keep": threading.Event()} + release_keep = threading.Event() + + async def fake_generate(prompt, seconds, kind, **options): + options["on_state"]("running") + started[prompt].set() + if prompt == "cancel": + await options["cancel_event"].wait() + raise controller.sa3.GenerationCancelled("generation cancelled") + await asyncio.to_thread(release_keep.wait) + return b"RIFFwav" + + monkeypatch.setattr(controller.sa3, "generate", fake_generate) + cancel_id = "cancel_job_0000000001" + keep_id = "keep_job_000000000001" + with ThreadPoolExecutor(max_workers=2) as pool: + cancelled = pool.submit( + client.post, + "/api/generate", + json=generate_request(prompt="cancel"), + headers={controller.JOB_ID_HEADER: cancel_id}, + ) + kept = pool.submit( + client.post, + "/api/generate", + json=generate_request(prompt="keep"), + headers={controller.JOB_ID_HEADER: keep_id}, + ) + assert started["cancel"].wait(2) + assert started["keep"].wait(2) + assert client.get(f"/api/jobs/{cancel_id}").json()["state"] == "running" + assert client.post(f"/api/jobs/{cancel_id}/cancel").status_code == 200 + assert cancelled.result(timeout=2).status_code == 499 + assert client.get(f"/api/jobs/{keep_id}").json()["state"] == "running" + release_keep.set() + assert kept.result(timeout=2).status_code == 200 + + assert client.get(f"/api/jobs/{cancel_id}").json()["state"] == "cancelled" + assert client.get(f"/api/jobs/{keep_id}").json()["state"] == "succeeded" + + +def test_sa3_status_exposes_the_runtime_contract(client, monkeypatch): + monkeypatch.setattr(controller.sa3, "status", lambda: {"backend": "tflite"}) + response = client.get("/api/sa3/status") + assert response.status_code == 200 + assert response.json() == {"backend": "tflite", "jobs": []} + + # --- /api/render (M18, the third Magenta engine) -------------------------- @@ -669,6 +837,16 @@ def test_render_returns_the_worker_clip_as_wav(client, render_worker): assert command["seconds"] == 2.0 +def test_render_rejects_oversized_body_before_worker_use(client, render_worker): + response = client.post( + "/api/render", + content=b"{" + b" " * controller.MAX_RENDER_BODY_BYTES + b"}", + headers={"content-type": "application/json"}, + ) + assert response.status_code == 413 + assert render_worker.ready_waits == 0 + + def test_render_maps_worker_failure_to_502(client, render_worker): render_worker.render_response = {"error": "render failed"} response = client.post("/api/render", json={"prompt": "air horn", "seconds": 2.0}) @@ -781,3 +959,13 @@ def test_models_endpoint_returns_list_and_ram(client, monkeypatch): assert body["sample_rate"] == 48000 assert body["total_ram_gb"] > 0 assert "mrt2_small" in body["model_ram_estimate_gb"] + + +def test_windows_ram_detection_uses_global_memory_status() -> None: + class FakeKernel32: + @staticmethod + def GlobalMemoryStatusEx(status_pointer): + status_pointer._obj.ullTotalPhys = 16 * 1024**3 + return 1 + + assert controller._windows_total_ram_bytes(FakeKernel32()) == 16 * 1024**3 diff --git a/backend/tests/test_engine.py b/backend/tests/test_engine.py index 8ba6166..e5a629d 100644 --- a/backend/tests/test_engine.py +++ b/backend/tests/test_engine.py @@ -50,6 +50,12 @@ def sample_pcm(seconds: float) -> bytes: return np.zeros(frames * 2, dtype=" 0 + return self.handle + + def _get_exit_code(self, handle, destination): + assert handle == self.handle + destination._obj.value = self.exit_code + return self.query_succeeds + + def _close_handle(self, handle): + self.closed.append(handle) + return True + + +def install_fake_windows(monkeypatch, kernel, *, last_error=0): + monkeypatch.setattr( + ctypes, "WinDLL", lambda *_args, **_kwargs: kernel, raising=False + ) + monkeypatch.setattr(ctypes, "get_last_error", lambda: last_error, raising=False) + + +def broker(tmp_path: pathlib.Path) -> GpuBroker: + return GpuBroker(tmp_path / "gpu-broker", poll_seconds=0.001) + + +@pytest.mark.parametrize( + ("exit_code", "expected"), + [(gpu_broker.STILL_ACTIVE, True), (0, False)], +) +def test_windows_liveness_queries_exit_state_and_closes_handle( + monkeypatch, exit_code, expected +): + kernel = FakeKernel32(exit_code=exit_code) + install_fake_windows(monkeypatch, kernel) + + assert gpu_broker._windows_pid_alive(1234) is expected + assert kernel.closed == [kernel.handle] + + +@pytest.mark.parametrize( + ("last_error", "expected"), + [(gpu_broker.ERROR_INVALID_PARAMETER, False), (5, True), (12345, True)], +) +def test_windows_open_failure_only_prunes_a_definitively_invalid_pid( + monkeypatch, last_error, expected +): + kernel = FakeKernel32(handle=0) + install_fake_windows(monkeypatch, kernel, last_error=last_error) + + assert gpu_broker._windows_pid_alive(1234) is expected + assert kernel.closed == [] + + +def test_windows_failed_exit_query_fails_closed_and_closes_handle(monkeypatch): + kernel = FakeKernel32(query_succeeds=False) + install_fake_windows(monkeypatch, kernel) + + assert gpu_broker._windows_pid_alive(1234) is True + assert kernel.closed == [kernel.handle] + + +@pytest.mark.parametrize( + ("error", "expected"), + [(ProcessLookupError(), False), (PermissionError(), True), (OSError(), True)], +) +def test_posix_liveness_semantics_are_preserved(monkeypatch, error, expected): + def fail(_pid, _signal): + raise error + + monkeypatch.setattr(gpu_broker.os, "kill", fail) + assert gpu_broker._posix_pid_alive(gpu_broker.os.getpid() + 1000) is expected + + +def test_pid_liveness_dispatches_to_the_host_platform(monkeypatch): + pid = gpu_broker.os.getpid() + 1000 + calls = [] + probe_name = ( + "_windows_pid_alive" if gpu_broker.os.name == "nt" else "_posix_pid_alive" + ) + + monkeypatch.setattr( + gpu_broker, + probe_name, + lambda candidate: calls.append(candidate) or True, + ) + + assert gpu_broker._pid_alive(pid) is True + assert calls == [pid] + + +def test_sa3_lease_is_bounded_by_measured_capacity(tmp_path): + service = broker(tmp_path) + with pytest.raises(BrokerTimeout): + service.acquire( + "sa3", + Priority.SA3_BACKGROUND, + reservation_bytes=8, + capacity_bytes=7, + timeout_seconds=0.01, + ) + assert service.diagnostics()["waiters"] == [] + + +def test_mrt2_waiter_preempts_sa3_at_the_next_callback(tmp_path): + service = broker(tmp_path) + sa3 = service.acquire( + "sa3", + Priority.SA3_BACKGROUND, + reservation_bytes=8, + capacity_bytes=16, + timeout_seconds=1, + ) + state = service.diagnostics() + state["waiters"].append( + { + "token": "mrt2-waiter", + "service": "mrt2", + "priority": int(Priority.MRT2_REALTIME), + "reservation_bytes": 0, + "pid": sa3.pid, + } + ) + service._write_state(state) + + assert service.should_yield(sa3) is True + service.release(sa3) + assert service.diagnostics()["leases"] == [] + + +def test_active_mrt2_blocks_background_generation(tmp_path): + service = broker(tmp_path) + realtime = service.acquire( + "mrt2", + Priority.MRT2_REALTIME, + reservation_bytes=0, + capacity_bytes=0, + timeout_seconds=1, + ) + with pytest.raises(BrokerTimeout): + service.acquire( + "sa3", + Priority.SA3_BACKGROUND, + reservation_bytes=1, + capacity_bytes=16, + timeout_seconds=0.01, + ) + service.release(realtime) + + +def test_cancelled_waiter_is_removed(tmp_path): + service = broker(tmp_path) + realtime = service.acquire( + "mrt2", + Priority.MRT2_REALTIME, + reservation_bytes=0, + capacity_bytes=0, + timeout_seconds=1, + ) + with pytest.raises(BrokerCancelled): + service.acquire( + "sa3", + Priority.SA3_BACKGROUND, + reservation_bytes=1, + capacity_bytes=16, + timeout_seconds=1, + cancelled=lambda: True, + ) + assert service.diagnostics()["waiters"] == [] + service.release(realtime) + + +def test_dead_process_records_are_recovered(tmp_path): + service = GpuBroker( + tmp_path / "gpu-broker", poll_seconds=0.001, pid_alive=lambda pid: pid != 99 + ) + service.root.mkdir(parents=True) + service.state_path.write_text( + json.dumps( + { + "schema_version": 1, + "waiters": [], + "leases": [ + { + "token": "dead", + "service": "sa3", + "priority": int(Priority.SA3_BACKGROUND), + "reservation_bytes": 8, + "pid": 99, + } + ], + } + ) + ) + lease = service.acquire( + "mrt2", + Priority.MRT2_REALTIME, + reservation_bytes=0, + capacity_bytes=0, + timeout_seconds=1, + ) + assert [item["token"] for item in service.diagnostics()["leases"]] == [lease.token] + + +def test_tampered_or_unbounded_state_fails_closed(tmp_path): + service = broker(tmp_path) + service.root.mkdir(parents=True) + service.state_path.write_text("{}") + with pytest.raises(BrokerError, match="unsupported schema"): + service.diagnostics() diff --git a/backend/tests/test_loras.py b/backend/tests/test_loras.py index 5debdcf..c435161 100644 --- a/backend/tests/test_loras.py +++ b/backend/tests/test_loras.py @@ -5,6 +5,10 @@ exercised against well-formed, malformed, and hostile names. """ +import hashlib +import json +import os + import pytest from lsdj import loras @@ -28,12 +32,16 @@ def test_env_override_wins(self, tmp_path): == tmp_path / "elsewhere" ) - def test_defaults_to_the_app_support_home(self, tmp_path): + def test_uses_the_host_supplied_assets_home(self, tmp_path): assert ( - loras.loras_dir(env={}, home=tmp_path) - == tmp_path / "Library" / "Application Support" / "LSDJ" / "sa3-loras" + loras.loras_dir(env={"LSDJ_ASSETS_HOME": str(tmp_path / "assets")}) + == tmp_path / "assets" / "sa3-loras" ) + def test_refuses_to_guess_a_platform_home(self): + with pytest.raises(RuntimeError, match="desktop host"): + loras.loras_dir(env={}) + class TestResolve: def test_resolves_an_installed_adapter(self, tmp_path): @@ -87,3 +95,61 @@ def test_rejects_a_directory_with_two_safetensors(self, tmp_path): loras.resolve( "medium/both", env={"SA3_LORAS_HOME": str(tmp_path)}, home=tmp_path ) + + @pytest.mark.skipif( + os.name == "nt", reason="symlink creation needs privileges on Windows" + ) + def test_rejects_symlinked_directory_and_weights(self, tmp_path): + outside = tmp_path / "outside" + install_adapter(outside, "small", "real") + (tmp_path / "small").mkdir() + (tmp_path / "small" / "linked-dir").symlink_to( + outside / "small" / "real", target_is_directory=True + ) + with pytest.raises(loras.UnknownAdapter): + loras.resolve( + "small/linked-dir", + env={"SA3_LORAS_HOME": str(tmp_path)}, + home=tmp_path, + ) + + linked = tmp_path / "small" / "linked-file" + linked.mkdir() + (linked / "adapter_model.safetensors").symlink_to( + outside / "small" / "real" / "adapter_model.safetensors" + ) + with pytest.raises(loras.UnknownAdapter): + loras.resolve( + "small/linked-file", + env={"SA3_LORAS_HOME": str(tmp_path)}, + home=tmp_path, + ) + + def test_rejects_tamper_against_import_provenance(self, tmp_path): + adapter_dir = install_adapter(tmp_path, "small", "sealed") + adapter = adapter_dir / "adapter_model.safetensors" + manifest = { + "source": "https://huggingface.co/friend/adapter", + "revision": "a" * 40, + "convention": "peft", + "adapterType": "lora", + "rank": 8, + "files": [ + { + "filename": adapter.name, + "size": len(ADAPTER), + "sha256": hashlib.sha256(ADAPTER).hexdigest(), + } + ], + } + (adapter_dir / "lora.json").write_text(json.dumps(manifest)) + loras.resolve( + "small/sealed", env={"SA3_LORAS_HOME": str(tmp_path)}, home=tmp_path + ) + adapter.write_bytes(b"same-size-tamper!!") + with pytest.raises(loras.UnknownAdapter): + loras.resolve( + "small/sealed", + env={"SA3_LORAS_HOME": str(tmp_path)}, + home=tmp_path, + ) diff --git a/backend/tests/test_models.py b/backend/tests/test_models.py index 15c8a34..3165a06 100644 --- a/backend/tests/test_models.py +++ b/backend/tests/test_models.py @@ -11,7 +11,7 @@ import pytest -from lsdj import engine, sa3, sidecar +from lsdj import engine, runtime_paths, sa3, sidecar # --- Dynamic Magenta discovery -------------------------------------------- @@ -45,8 +45,9 @@ def _checkout(root: pathlib.Path, *, venv: bool, warmed: bool) -> None: mlx = root / "optimized" / "mlx" mlx.mkdir(parents=True) if venv: - (mlx / ".venv" / "bin").mkdir(parents=True) - (mlx / ".venv" / "bin" / "python").write_text("") + python = runtime_paths.venv_python(mlx / ".venv", platform="darwin") + python.parent.mkdir(parents=True) + python.write_text("") (mlx / "scripts").mkdir() (mlx / "scripts" / "sa3_mlx.py").write_text("") if warmed: @@ -64,7 +65,12 @@ def _checkout(root: pathlib.Path, *, venv: bool, warmed: bool) -> None: def test_readiness_classifies_a_checkout(tmp_path, venv, warmed, expected): root = tmp_path / "co" _checkout(root, venv=venv, warmed=warmed) - result = sa3.readiness(env={"SA3_MLX_HOME": str(root)}, home=tmp_path / "home") + result = sa3.readiness( + env={"SA3_MLX_HOME": str(root)}, + home=tmp_path / "home", + platform_name="darwin", + machine="arm64", + ) assert result["state"] == expected assert result["checkout"] == str(root) diff --git a/backend/tests/test_mrt2_pytorch.py b/backend/tests/test_mrt2_pytorch.py new file mode 100644 index 0000000..0fe1a34 --- /dev/null +++ b/backend/tests/test_mrt2_pytorch.py @@ -0,0 +1,361 @@ +from pathlib import Path +from types import SimpleNamespace +import tempfile + +import numpy as np +import pytest + +from lsdj.engine import CHANNELS, FRAME_SECONDS, NOTE_SUSTAIN, SAMPLE_RATE +from lsdj.mrt2 import ( + MODEL_SNAPSHOTS, + PROCESSOR_SNAPSHOT, + RuntimeSelection, + RuntimeUnavailable, +) +from lsdj.mrt2_pytorch import ( + GPU_ADMISSION_TIMEOUT_SECONDS, + PytorchBindings, + PytorchMrt2Engine, +) +from lsdj.gpu_broker import BrokerCancelled, Priority + + +class RecordingBroker: + def __init__(self, events=None, *, error=None): + self.events = events if events is not None else [] + self.error = error + self.calls = [] + self.releases = [] + self.lease = object() + + def acquire(self, service, **kwargs): + self.events.append("broker_acquire") + self.calls.append((service, kwargs)) + if self.error is not None: + raise self.error + return self.lease + + def release(self, lease): + self.events.append("broker_release") + self.releases.append(lease) + + +DEFAULT_TEST_BROKER = object() + + +class FakeCuda: + def __init__(self, available=True, events=None): + self.available = available + self.events = events if events is not None else [] + + def is_available(self): + self.events.append("cuda_available") + return self.available + + def current_device(self): + return 0 + + def get_device_properties(self, _index): + return SimpleNamespace(name="Fake NVIDIA", total_memory=12 * 1024**3) + + def get_device_capability(self, _index): + return (9, 9) + + +class FakeTorch: + bfloat16 = "bf16" + + def __init__(self, available=True, events=None): + self.cuda = FakeCuda(available, events) + self.version = SimpleNamespace(cuda="13.0") + self._C = SimpleNamespace(_cuda_getDriverVersion=lambda: 13020) + + +class FakeProcessor: + def __init__(self): + self.embeds = [] + self.tokenizes = [] + + def embed(self, value): + self.embeds.append(value) + if isinstance(value, str): + return np.array([len(value), 2.0], dtype=np.float32) + return np.array([10.0, 4.0], dtype=np.float32) + + def tokenize(self, embedding): + self.tokenizes.append(np.asarray(embedding).copy()) + return np.arange(12, dtype=np.int64) + + +class FakeModel: + def __init__(self, events=None, *, fail_cuda_load=False): + self.processor = FakeProcessor() + self.calls = [] + self.processor_path = None + self.bad_shape = False + self.events = events if events is not None else [] + self.fail_cuda_load = fail_cuda_load + + def to(self, device): + assert device == "cuda" + self.events.append("model_to_cuda") + if self.fail_cuda_load: + raise RuntimeError("injected CUDA allocation failure") + return self + + def eval(self): + return self + + def load_processor(self, path, *, device): + self.events.append("processor_to_cuda") + self.processor_path = (path, device) + + def generate(self, **kwargs): + self.calls.append(kwargs) + frames = kwargs["frames"] + shape = (frames * 1920, CHANNELS) + if self.bad_shape: + shape = (frames * 1920, 1) + return np.zeros(shape, dtype=np.float32), {"call": len(self.calls)} + + +class FakeAutoModel: + def __init__(self, model, events=None): + self.model = model + self.calls = [] + self.events = events if events is not None else [] + + def from_pretrained(self, path, **kwargs): + self.events.append("from_pretrained") + self.calls.append((path, kwargs)) + return self.model + + +def make_engine(*, cuda=True, gpu_broker=DEFAULT_TEST_BROKER, events=None, model=None): + events = events if events is not None else [] + if gpu_broker is DEFAULT_TEST_BROKER: + gpu_broker = RecordingBroker(events) + model = model if model is not None else FakeModel(events) + auto_model = FakeAutoModel(model, events) + bindings = PytorchBindings( + torch=FakeTorch(cuda, events), + auto_model=auto_model, + versions={ + "torch": "2.12.1", + "transformers": "5.8.0", + "huggingface_hub": "1.5.0", + }, + ) + runtime_root = Path(tempfile.mkdtemp(prefix="lsdj-mrt2-local-")) + model_root = runtime_root / "models" / "mrt2_small" + processor_root = runtime_root / "models" / "musiccoca" + model_root.mkdir(parents=True) + processor_root.mkdir(parents=True) + for filename in ("config.json", "model.safetensors", "modeling_magenta_rt2.py"): + (model_root / filename).write_bytes(b"fixture") + for filename in ( + "mel_params.npz", + "music_encoder.pt", + "quantizer.pt", + "spm.model", + "text_encoder.pt", + ): + (processor_root / filename).write_bytes(b"fixture") + selection = RuntimeSelection("pytorch-cuda", "linux", "cuda", False, True) + engine = PytorchMrt2Engine( + selection=selection, + bindings=bindings, + cache_root=runtime_root, + gpu_broker=gpu_broker, + ) + return engine, model, auto_model, runtime_root + + +def test_loads_only_pinned_local_snapshots(): + engine, model, auto_model, runtime_root = make_engine() + assert auto_model.calls[0][0] == (runtime_root / "models" / "mrt2_small").resolve() + assert auto_model.calls[0][1] == { + "trust_remote_code": True, + "dtype": "bf16", + "local_files_only": True, + } + assert model.processor_path[1] == "cuda" + assert engine.diagnostics()["remote_code_revision"].startswith("7037d995") + + +def test_cuda_is_mandatory_and_never_falls_back_to_cpu(): + with pytest.raises(RuntimeUnavailable, match="no CPU fallback"): + make_engine(cuda=False) + + +def test_missing_broker_root_fails_before_cuda_is_touched(monkeypatch): + events = [] + monkeypatch.delenv("LSDJ_CACHE_HOME", raising=False) + + with pytest.raises(RuntimeUnavailable, match="shared GPU broker"): + make_engine(gpu_broker=None, events=events) + + assert events == [] + + +def test_weighted_style_and_controls_map_to_upstream_generate(): + engine, model, _, _ = make_engine() + engine.set_style([("funk", 3.0), ("dub", 1.0)]) + engine.set_generation(0.7, 20, 3.0, 1.0) + notes = [0] * 128 + notes[60] = 2 + engine.set_notes(notes) + engine.set_drums(0, 5.0) + engine.set_chunk_frames(5) + pcm = engine.generate_chunk() + + assert len(pcm) == round(5 * FRAME_SECONDS * SAMPLE_RATE) * CHANNELS * 4 + call = model.calls[-1] + assert call["style"] == list(range(12)) + assert call["notes"][60] == 2 + assert call["drums"] == [0] + assert (call["temperature"], call["top_k"]) == (0.7, 20) + assert (call["cfg_musiccoca"], call["cfg_notes"], call["cfg_drums"]) == ( + 3.0, + 1.0, + 5.0, + ) + assert call["guidance"] is True + assert engine._notes[60] == NOTE_SUSTAIN + np.testing.assert_allclose(model.processor.tokenizes[-1], [3.75, 2.0]) + + +def test_reset_to_reseed_discards_continuation_state(): + engine, model, _, _ = make_engine() + engine.generate_chunk() + assert engine._state is not None + engine.reset(seed=42) + engine.generate_chunk() + assert model.calls[-1]["seed"] == 42 + assert model.calls[-1]["state"] is None + + +def test_render_clip_does_not_carry_live_stream_conditioning(): + engine, model, _, _ = make_engine() + notes = [0] * 128 + notes[60] = 2 + engine.set_notes(notes) + engine.set_drums(1, 4.0) + + engine.render_clip("air horn", FRAME_SECONDS) + + assert model.calls[-1]["notes"] is None + assert model.calls[-1]["drums"] is None + assert model.calls[-1]["cfg_drums"] is None + + +def test_warmup_exercises_cuda_then_clears_state(): + engine, model, _, _ = make_engine() + engine.warm_up() + assert model.calls[-1]["frames"] == 1 + assert engine._state is None + + +def test_shared_deck_reuses_one_model_with_independent_continuation_state(): + first, model, _, _ = make_engine() + second = first.shared_deck() + assert first._system is second._system + assert first._model_lock is second._model_lock + assert first._gpu_broker is second._gpu_broker + assert first._gpu_lease is second._gpu_lease + + first.generate_chunk() + second.generate_chunk() + assert model.calls[0]["state"] is None + assert model.calls[1]["state"] is None + first.generate_chunk() + assert model.calls[2]["state"] == {"call": 1} + assert first._state != second._state + + +def test_invalid_upstream_audio_shape_fails_before_pcm_handoff(): + engine, model, _, _ = make_engine() + model.bad_shape = True + with pytest.raises(RuntimeError, match="invalid audio shape"): + engine.generate_chunk() + + +def test_diagnostics_disclose_unqualified_runtime_and_cuda_versions(): + engine, _, _, _ = make_engine() + diagnostics = engine.diagnostics() + model_pin = MODEL_SNAPSHOTS["mrt2_small"] + assert diagnostics["model_repository"] == model_pin["repository"] + assert diagnostics["model_revision"] == model_pin["revision"] + assert diagnostics["remote_code_repository"] == model_pin["repository"] + assert diagnostics["remote_code_revision"] == model_pin["revision"] + assert diagnostics["processor_repository"] == PROCESSOR_SNAPSHOT["repository"] + assert diagnostics["processor_revision"] == PROCESSOR_SNAPSHOT["revision"] + assert diagnostics["hardware_qualified"] is False + assert diagnostics["experimental"] is True + assert diagnostics["torch_cuda_runtime"] == "13.0" + assert diagnostics["nvidia_driver"] == "13.2" + assert diagnostics["cuda_device"] == "Fake NVIDIA" + assert diagnostics["capabilities"]["negative_prompt"] is False + + +def test_mrt2_model_lifetime_takes_realtime_priority_over_background_sa3(): + broker = RecordingBroker() + engine, _, _, _ = make_engine(gpu_broker=broker) + engine.generate_chunk() + + assert len(broker.calls) == 1, "generation must reuse the model-lifetime lease" + service, values = broker.calls[0] + assert service == "mrt2" + assert values["priority"] is Priority.MRT2_REALTIME + assert values["reservation_bytes"] == 0 + assert values["capacity_bytes"] == 0 + assert values["timeout_seconds"] == GPU_ADMISSION_TIMEOUT_SECONDS + assert broker.releases == [], "only worker-process exit may release CUDA ownership" + assert engine.diagnostics()["gpu_broker"] == { + "enabled": True, + "priority": 100, + "preempts": "sa3-background", + } + + +def test_gpu_lease_is_acquired_before_any_cuda_or_model_allocation(): + events = [] + broker = RecordingBroker(events) + + make_engine(gpu_broker=broker, events=events) + + assert events == [ + "broker_acquire", + "cuda_available", + "from_pretrained", + "model_to_cuda", + "processor_to_cuda", + ] + + +def test_cancelled_gpu_admission_never_touches_cuda_or_the_model(): + events = [] + broker = RecordingBroker(events, error=BrokerCancelled("injected cancellation")) + + with pytest.raises(BrokerCancelled, match="injected cancellation"): + make_engine(gpu_broker=broker, events=events) + + assert events == ["broker_acquire"] + assert broker.releases == [] + + +def test_cuda_load_failure_keeps_the_lease_until_process_cleanup(): + events = [] + broker = RecordingBroker(events) + model = FakeModel(events, fail_cuda_load=True) + + with pytest.raises(RuntimeUnavailable, match="could not initialize on CUDA"): + make_engine(gpu_broker=broker, events=events, model=model) + + assert events == [ + "broker_acquire", + "cuda_available", + "from_pretrained", + "model_to_cuda", + ] + assert broker.releases == [], "failed CUDA teardown is safe only at process exit" diff --git a/backend/tests/test_mrt2_runtime.py b/backend/tests/test_mrt2_runtime.py new file mode 100644 index 0000000..a7a3a45 --- /dev/null +++ b/backend/tests/test_mrt2_runtime.py @@ -0,0 +1,72 @@ +import json + +import pytest + +from lsdj.mrt2 import ( + MODEL_SNAPSHOTS, + PYTORCH_CUDA_RUNTIME, + UNVERIFIED_OPT_IN, + RuntimeUnavailable, + runtime_manifest, + select_runtime, +) +from lsdj.sidecar import main + + +def test_platform_default_is_explicit_and_never_cpu(): + mac = select_runtime("auto", platform="darwin", env={}) + assert (mac.name, mac.accelerator) == ("mlx", "metal") + + for platform in ("linux", "win32"): + selected = select_runtime( + "auto", platform=platform, env={UNVERIFIED_OPT_IN: "1"} + ) + assert selected.name == PYTORCH_CUDA_RUNTIME + assert selected.accelerator == "cuda" + assert selected.experimental is True + + +def test_pytorch_runtime_fails_closed_until_hardware_is_qualified(): + with pytest.raises(RuntimeUnavailable, match="two-deck hardware results"): + select_runtime("pytorch-cuda", platform="linux", env={}) + + +def test_runtime_platform_mismatches_are_clear(): + with pytest.raises(RuntimeUnavailable, match="macOS-only"): + select_runtime("mlx", platform="win32", env={}) + with pytest.raises(RuntimeUnavailable, match="Linux and Windows"): + select_runtime( + "pytorch-cuda", + platform="darwin", + env={UNVERIFIED_OPT_IN: "1"}, + ) + with pytest.raises(RuntimeUnavailable, match="unsupported platform"): + select_runtime("auto", platform="freebsd", env={}) + + +def test_manifest_keeps_every_external_dependency_immutable(): + manifest = runtime_manifest() + assert manifest["cpu_fallback"] is False + assert manifest["release_ready"] is False + assert manifest["topology"] == "shared-worker-two-state" + assert manifest["topology_implemented"] is True + pins = [ + manifest["adapter_reference"]["revision"], + manifest["processor"]["revision"], + ] + pins.extend(model["revision"] for model in manifest["models"].values()) + assert all(len(pin) == 40 for pin in pins) + assert manifest["models"] == MODEL_SNAPSHOTS + assert manifest["adapter_reference"]["role"].endswith("not executed by LSDJ") + assert manifest["runtime_candidate"]["lock_status"] == "hash_locked_uninstalled" + assert set(manifest["runtime_candidate"]["locks"]) == { + "linux-x86_64", + "windows-x86_64", + } + + +def test_runtime_info_cli_is_model_free_and_machine_readable(capsys): + main(["--runtime-info"]) + payload = json.loads(capsys.readouterr().out) + assert payload["runtime"] == "pytorch-cuda" + assert payload["release_ready"] is False diff --git a/backend/tests/test_mrt2_runtime_locks.py b/backend/tests/test_mrt2_runtime_locks.py new file mode 100644 index 0000000..da91c63 --- /dev/null +++ b/backend/tests/test_mrt2_runtime_locks.py @@ -0,0 +1,54 @@ +import re +from pathlib import Path + +import pytest + + +BACKEND_ROOT = Path(__file__).parents[1] +LOCKS = { + "linux": BACKEND_ROOT / "runtime-locks/mrt2-pytorch-linux-x86_64.txt", + "windows": BACKEND_ROOT / "runtime-locks/mrt2-pytorch-windows-x86_64.txt", +} +DIRECT_PINS = { + "huggingface-hub": "1.5.0", + "numpy": "2.3.5", + "resampy": "0.4.3", + "safetensors": "0.7.0", + "sentencepiece": "0.2.1", + "torch": "2.12.1+cu130", + "transformers": "5.8.0", +} +REQUIREMENT = re.compile(r"^([a-z0-9][a-z0-9_.-]*)==([^ \\]+) \\$", re.MULTILINE) + + +def _requirements(text: str) -> dict[str, str]: + return dict(REQUIREMENT.findall(text)) + + +@pytest.mark.parametrize("platform", LOCKS) +def test_target_runtime_lock_is_immutable_and_hashed(platform): + text = LOCKS[platform].read_text() + requirements = _requirements(text) + + assert requirements + assert DIRECT_PINS.items() <= requirements.items() + assert "git+" not in text + assert "http://" not in text + assert " @ " not in text + assert "--editable" not in text + assert text.count("--hash=sha256:") >= len(requirements) + + for match in REQUIREMENT.finditer(text): + next_requirement = REQUIREMENT.search(text, match.end()) + block_end = len(text) if next_requirement is None else next_requirement.start() + assert "--hash=sha256:" in text[match.end() : block_end] + + +def test_target_locks_capture_platform_specific_dependency_graphs(): + linux = _requirements(LOCKS["linux"].read_text()) + windows = _requirements(LOCKS["windows"].read_text()) + + assert linux["triton"] == "3.7.1" + assert "triton" not in windows + assert windows["colorama"] == "0.4.6" + assert "colorama" not in linux diff --git a/backend/tests/test_render_sidecar.py b/backend/tests/test_render_sidecar.py new file mode 100644 index 0000000..553ffcd --- /dev/null +++ b/backend/tests/test_render_sidecar.py @@ -0,0 +1,812 @@ +"""Bounded, authenticated protocol for the dedicated MRT2 render worker.""" + +import hashlib +import io +import json +import socket +import struct +import threading + +import pytest + +from lsdj.sidecar import ( + FRAME_AUTH, + FRAME_RENDER_BEGIN, + FRAME_RENDER_CANCEL, + FRAME_RENDER_CHUNK, + FRAME_RENDER_END, + FRAME_RENDER_ERROR, + FRAME_RENDER_REQUEST, + FRAME_STATUS, + MAX_RENDER_FRAMES, + MAX_RENDER_PCM_BYTES, + MAX_RENDER_PROMPT_CHARS, + MAX_RENDER_REQUEST_BYTES, + PYTORCH_CUDA_RUNTIME, + RENDER_BYTES_PER_FRAME, + RENDER_CHANNELS, + RENDER_PCM_CHUNK_BYTES, + RENDER_SAMPLE_RATE, + RENDER_SCHEMA_VERSION, + RenderProtocolError, + RenderRequest, + authenticate_to_host, + read_frame, + read_render_command, + read_render_response, + render_frames_for_seconds, + run_render_worker, + write_frame, + write_render_response, +) + +JOB_ID = "render-job-0123456789abcdef" + + +class RecordingSock: + def __init__(self): + self.buffer = bytearray() + + def sendall(self, data): + self.buffer.extend(data) + + def setsockopt(self, *_args): + pass + + +class FakeRenderEngine: + def __init__(self): + self.warmups = 0 + self.requests = [] + + def warm_up(self): + self.warmups += 1 + + def render_clip(self, prompt, seconds): + self.requests.append((prompt, seconds)) + frames = round(seconds * RENDER_SAMPLE_RATE) + return b"\0" * (frames * RENDER_BYTES_PER_FRAME) + + +class BlockingRenderEngine(FakeRenderEngine): + def __init__(self): + super().__init__() + self.started = threading.Event() + self.release = threading.Event() + + def render_clip(self, prompt, seconds): + self.requests.append((prompt, seconds)) + self.started.set() + self.release.wait(timeout=5) + return b"\0" * (round(seconds * RENDER_SAMPLE_RATE) * RENDER_BYTES_PER_FRAME) + + +class SignalAfterFirstChunkSock: + """Record frame writes and optionally signal after the first PCM chunk.""" + + def __init__(self, sock, signal=None): + self.sock = sock + self.signal = signal + self.signalled = False + self.frame_types = [] + + def sendall(self, data): + self.frame_types.append(data[0]) + self.sock.sendall(data) + if ( + data[0] == FRAME_RENDER_CHUNK + and not self.signalled + and self.signal is not None + ): + self.signalled = True + self.signal() + + def makefile(self, *args, **kwargs): + return self.sock.makefile(*args, **kwargs) + + def shutdown(self, how): + return self.sock.shutdown(how) + + +def request_payload( + *, job_id=JOB_ID, sequence=1, prompt="bright piano", frames=24_000, **extra +): + return json.dumps( + { + "schemaVersion": RENDER_SCHEMA_VERSION, + "jobId": job_id, + "sequence": sequence, + "prompt": prompt, + "frames": frames, + **extra, + }, + separators=(",", ":"), + ).encode() + + +def cancel_payload(job_id=JOB_ID, sequence=1): + return json.dumps( + { + "schemaVersion": RENDER_SCHEMA_VERSION, + "jobId": job_id, + "sequence": sequence, + }, + separators=(",", ":"), + ).encode() + + +def begin_payload(*, job_id=JOB_ID, sequence=1, frames=24_000, **extra): + return json.dumps( + { + "schemaVersion": RENDER_SCHEMA_VERSION, + "jobId": job_id, + "sequence": sequence, + "sampleRate": RENDER_SAMPLE_RATE, + "channels": RENDER_CHANNELS, + "sampleFormat": "f32le", + "frames": frames, + "pcmBytes": frames * RENDER_BYTES_PER_FRAME, + **extra, + }, + separators=(",", ":"), + ).encode() + + +def read_ready(reader): + frame_type, payload = read_frame(reader) + assert frame_type == FRAME_STATUS + assert json.loads(payload)["event"] == "render_ready" + + +def test_render_command_is_strict_and_bounded(): + valid = io.BytesIO( + struct.pack(" 1 + + +@pytest.mark.parametrize("delta", [-RENDER_BYTES_PER_FRAME, RENDER_BYTES_PER_FRAME]) +def test_render_response_writer_rejects_short_and_extra_pcm(delta): + request = RenderRequest(JOB_ID, 1, "piano", 24_000) + with pytest.raises(RenderProtocolError, match="expected"): + write_render_response( + RecordingSock(), request, b"\0" * (request.pcm_bytes + delta) + ) + + +def test_render_response_reader_rejects_out_of_order_oversized_and_extra_pcm(): + request = RenderRequest(JOB_ID, 1, "piano", 24_000) + out_of_order = RecordingSock() + write_frame(out_of_order, FRAME_RENDER_CHUNK, b"\0" * RENDER_BYTES_PER_FRAME) + with pytest.raises(RenderProtocolError, match="out of order"): + read_render_response(io.BytesIO(out_of_order.buffer), request) + + oversized = bytearray() + begin = begin_payload() + oversized.extend(struct.pack(" "$(dirname "$0")/argv.txt" -for arg in "$@"; do - if [ "$prev" = "--out" ]; then out="$arg"; fi - if [ "$prev" = "--init-audio" ]; then cp "$arg" "$(dirname "$0")/init.wav"; fi - printf '%s\\n' "$arg" >> "$(dirname "$0")/argv.txt" - prev="$arg" -done -printf 'RIFFfakewavdata' > "$out" -""" +from lsdj import runtime_paths, sa3 +from lsdj.sa3_contract import BackendName, GenerationMode, GenerationRequest -FAILURE_STUB = """#!/bin/sh -echo "error: no DiT weights found" -exit 3 + +SUCCESS_STUB = r"""import os +import pathlib +import shutil +import sys +import wave + +runtime_dir = pathlib.Path(__file__).resolve().parent.parent +(runtime_dir / "argv.txt").write_text("\n".join(sys.argv) + "\n") +(runtime_dir / "env.txt").write_text( + f"offline={os.environ.get('HF_HUB_OFFLINE')}\n" + f"token={os.environ.get('HF_TOKEN')}\n" + f"threads={os.environ.get('OMP_NUM_THREADS')}\n" + f"pythonutf8={os.environ.get('PYTHONUTF8')}\n" + f"stdio={os.environ.get('PYTHONIOENCODING')}\n" +) +if "--init-audio" in sys.argv: + source = pathlib.Path(sys.argv[sys.argv.index("--init-audio") + 1]) + shutil.copyfile(source, runtime_dir / "init.wav") +seconds = float(sys.argv[sys.argv.index("--seconds") + 1]) +frames = round(seconds * 44100) +out = pathlib.Path(sys.argv[sys.argv.index("--out") + 1]) +with wave.open(str(out), "wb") as target: + target.setnchannels(2) + target.setsampwidth(2) + target.setframerate(44100) + target.writeframes(b"\0" * frames * 4) """ -# Exits cleanly without writing the WAV. -SILENT_STUB = """#!/bin/sh -exit 0 +PROGRESS_STUB = SUCCESS_STUB.replace( + "seconds = float", 'print("sampling step 1/2", flush=True)\nseconds = float' +).replace("frames = round", 'print("sampling step 2/2", flush=True)\nframes = round') + +FAILURE_STUB = """import sys +print("prompt super secret user prompt") +print("error: no DiT weights found") +sys.exit(3) +""" +SILENT_STUB = "pass\n" +CORRUPT_STUB = """import pathlib, sys +pathlib.Path(sys.argv[sys.argv.index("--out") + 1]).write_bytes(b"RIFFbad") +""" +WRONG_DURATION_STUB = """import pathlib, sys, wave +out = pathlib.Path(sys.argv[sys.argv.index("--out") + 1]) +with wave.open(str(out), "wb") as target: + target.setnchannels(2); target.setsampwidth(2); target.setframerate(44100) + target.writeframes(b"\\0" * round(0.25 * 44100) * 4) """ +TIMEOUT_STUB = "import time\ntime.sleep(30)\n" +TREE_STUB = r"""import pathlib, subprocess, sys, time +runtime_dir = pathlib.Path(__file__).resolve().parent.parent +child = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(30)"]) +(runtime_dir / "grandchild.pid").write_text(str(child.pid)) +time.sleep(30) +""" + + +def pcm16_wav(seconds: float = 0.25) -> bytes: + output = io.BytesIO() + with wave.open(output, "wb") as target: + target.setnchannels(2) + target.setsampwidth(2) + target.setframerate(44_100) + target.writeframes(b"\0" * round(seconds * 44_100) * 4) + return output.getvalue() -def make_checkout(root: pathlib.Path, stub_body: str) -> pathlib.Path: - """Lay out /optimized/mlx with an executable python stub.""" - mlx_dir = root / "optimized" / "mlx" - (mlx_dir / ".venv" / "bin").mkdir(parents=True) - (mlx_dir / "scripts").mkdir() - (mlx_dir / "scripts" / "sa3_mlx.py").write_text("# stub CLI\n") - python = mlx_dir / ".venv" / "bin" / "python" - python.write_text(stub_body) - python.chmod(0o755) - return mlx_dir +def _install_interpreter(runtime_dir: pathlib.Path, platform_name: str) -> pathlib.Path: + executable = runtime_paths.venv_python( + runtime_dir / ".venv", platform=platform_name + ) + executable.parent.mkdir(parents=True) + (runtime_dir / ".venv" / "pyvenv.cfg").write_text( + f"home = {sys.base_prefix}\n" + "include-system-site-packages = false\n" + f"version = {sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}\n" + ) + if platform_name == sys.platform: + if os.name == "nt": + shutil.copyfile(sys.executable, executable) + else: + executable.symlink_to(sys.executable) + else: + executable.write_bytes(b"fake") + return executable -class TestResolveMlxDir: - def test_env_override_wins(self, tmp_path): - mlx_dir = make_checkout(tmp_path / "elsewhere", SUCCESS_STUB) - resolved = sa3.resolve_mlx_dir( - env={"SA3_MLX_HOME": str(tmp_path / "elsewhere")}, home=tmp_path / "home" +def make_runtime( + root: pathlib.Path, + backend: BackendName, + stub: str = SUCCESS_STUB, + *, + platform_name: str | None = None, + assets: bool = True, +) -> sa3.RuntimeSelection: + platform_name = sys.platform if platform_name is None else platform_name + subdir = "mlx" if backend is BackendName.MLX else "tflite" + script_name = "sa3_mlx.py" if backend is BackendName.MLX else "sa3_tflite.py" + runtime_dir = root / "optimized" / subdir + script = runtime_dir / "scripts" / script_name + script.parent.mkdir(parents=True) + script.write_text(stub) + executable = _install_interpreter(runtime_dir, platform_name) + (runtime_dir / sa3.WARMED_STAMP).write_text("ready\n") + if backend is BackendName.TFLITE: + (runtime_dir / sa3.TFLITE_PROVENANCE_STAMP).write_text( + json.dumps( + { + "runtime": { + "repo": sa3.TFLITE_RUNTIME_REPO, + "revision": sa3.TFLITE_RUNTIME_REVISION, + }, + "models": { + "repo": sa3.TFLITE_MODELS_REPO, + "revision": sa3.TFLITE_MODELS_REVISION, + }, + } + ) ) - assert resolved == mlx_dir - - def test_resolves_the_app_support_home(self, tmp_path): - # In-app installs (and `just setup-sa3`) put the checkout in the app-owned - # data dir — the only non-override candidate. - mlx_dir = make_checkout( - tmp_path / "Library" / "Application Support" / "LSDJ" / "stable-audio-3", - SUCCESS_STUB, + if backend is BackendName.TFLITE and assets: + request = GenerationRequest("probe", 0.5, "sfx", init_audio=pcm16_wav()) + for relative in sa3._required_tflite_assets(request): + path = runtime_dir / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(b"fixture") + for kind in ("music", "track"): + request = GenerationRequest("probe", 0.5, kind, init_audio=pcm16_wav()) + for relative in sa3._required_tflite_assets(request): + path = runtime_dir / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(b"fixture") + return sa3.RuntimeSelection(backend, root, runtime_dir, executable, script) + + +@pytest.fixture +def tflite_runtime(tmp_path, monkeypatch): + def install(stub=SUCCESS_STUB, *, assets=True): + selection = make_runtime( + tmp_path / "Stable Audio 模型", BackendName.TFLITE, stub, assets=assets ) - assert sa3.resolve_mlx_dir(env={}, home=tmp_path) == mlx_dir + monkeypatch.setenv("SA3_HOME", str(selection.checkout)) + monkeypatch.setenv("LSDJ_SA3_BACKEND", "tflite") + # The worker tests inject a fully resolved fake runtime so they remain + # model-free on every CI host without weakening production's target gate. + monkeypatch.setattr(sa3, "resolve_runtime", lambda: selection) + return selection + + return install - def test_checkout_without_venv_is_skipped(self, tmp_path): - checkout = ( - tmp_path / "Library" / "Application Support" / "LSDJ" / "stable-audio-3" + +@pytest.mark.parametrize( + ("platform_name", "machine", "expected"), + [ + ("darwin", "arm64", BackendName.MLX), + ("darwin", "aarch64", BackendName.MLX), + ("linux", "x86_64", BackendName.TFLITE), + ("win32", "AMD64", BackendName.TFLITE), + ], +) +def test_backend_selection_is_explicit(platform_name, machine, expected): + assert ( + sa3.select_backend({}, platform_name=platform_name, machine=machine) is expected + ) + + +def test_backend_override_is_validated(): + assert ( + sa3.select_backend( + {"LSDJ_SA3_BACKEND": "tflite"}, + platform_name="linux", + machine="x86_64", + ) + is BackendName.TFLITE + ) + with pytest.raises(sa3.GenerationUnavailable, match="does not support"): + sa3.select_backend( + {"LSDJ_SA3_BACKEND": "tflite"}, + platform_name="darwin", + machine="x86_64", + ) + with pytest.raises(sa3.GenerationUnavailable, match="requires Apple Silicon"): + sa3.select_backend( + {"LSDJ_SA3_BACKEND": "mlx"}, + platform_name="win32", + machine="AMD64", + ) + with pytest.raises(sa3.GenerationUnavailable, match="must be"): + sa3.select_backend( + {"LSDJ_SA3_BACKEND": "cuda"}, + platform_name="linux", + machine="x86_64", ) - (checkout / "optimized" / "mlx" / "scripts").mkdir(parents=True) - (checkout / "optimized" / "mlx" / "scripts" / "sa3_mlx.py").write_text("#") - assert sa3.resolve_mlx_dir(env={}, home=tmp_path) is None - def test_nothing_resolves_to_none(self, tmp_path): - assert sa3.resolve_mlx_dir(env={}, home=tmp_path) is None +def test_unsupported_platform_fails_instead_of_guessing(): + for platform_name, machine in (("freebsd", "x86_64"), ("linux", "aarch64")): + with pytest.raises(sa3.GenerationUnavailable, match="no Stable Audio backend"): + sa3.select_backend({}, platform_name=platform_name, machine=machine) -@pytest.fixture -def checkout(tmp_path, monkeypatch): - """Install a stub checkout, point SA3_MLX_HOME at it, return mlx dir.""" - def install(stub_body): - mlx_dir = make_checkout(tmp_path / "sa3", stub_body) - monkeypatch.setenv("SA3_MLX_HOME", str(tmp_path / "sa3")) - return mlx_dir +def test_runtime_resolution_uses_windows_venv_layout(tmp_path): + selection = make_runtime( + tmp_path / "Audio Runtime", + BackendName.TFLITE, + platform_name="win32", + ) + resolved = sa3.resolve_runtime( + {"SA3_HOME": str(selection.checkout)}, + platform_name="win32", + machine="AMD64", + ) + assert resolved is not None + assert resolved.executable.name == "python.exe" + assert resolved.executable.parent.name == "Scripts" - return install +def test_status_exposes_backend_capabilities_and_real_limitations(tmp_path): + selection = make_runtime( + tmp_path / "sa3", BackendName.TFLITE, platform_name="linux" + ) + result = sa3.status( + {"SA3_HOME": str(selection.checkout)}, + platform_name="linux", + machine="x86_64", + ) + assert result["state"] == sa3.STATE_READY + assert result["backend"] == "tflite" + assert result["capabilities"]["preview"] is False + assert result["capabilities"]["cancellation"] is True + assert any( + "Per-step LoRA" in item for item in result["capabilities"]["limitations"] + ) -class TestGenerate: - def test_returns_wav_bytes(self, checkout): - checkout(SUCCESS_STUB) - wav = asyncio.run(sa3.generate("vinyl spinback", 3.0, "sfx")) - assert wav == FAKE_WAV - - def test_default_cli_argv_is_unchanged(self, checkout): - mlx_dir = checkout(SUCCESS_STUB) - asyncio.run(sa3.generate("deep house loop", 7.74, "music")) - argv = (mlx_dir / ".venv" / "bin" / "argv.txt").read_text().splitlines() - assert argv[:-1] == [ - str(mlx_dir / "scripts" / "sa3_mlx.py"), - "--prompt", - "deep house loop", - "--dit", - "sm-music", - "--decoder", - "same-s", - "--seconds", - "7.74", - "--steps", - "8", - "--out", - ] - assert pathlib.Path(argv[-1]).name == "out.wav" - - def test_passes_the_full_generation_surface_and_init_bytes(self, checkout): - mlx_dir = checkout(SUCCESS_STUB) - init_audio = b"RIFFsource-WAVE" - asyncio.run( - sa3.generate( - "warm dub loop", - 8.0, - "music", - init_audio=init_audio, - init_noise_level=0.6, - inpaint_range=(1.25, 2.5), - negative_prompt="vocals", - cfg=4.5, - apg=0.75, - seed=12345, - ) + +def test_windows_status_exposes_conservative_backend_choices_and_cuda_blockers( + tmp_path, +): + selection = make_runtime( + tmp_path / "sa3", BackendName.TFLITE, platform_name="win32" + ) + result = sa3.status( + {"SA3_HOME": str(selection.checkout)}, + platform_name="win32", + machine="AMD64", + ) + + assert result["preference"] == "auto" + assert result["preferenceChoices"] == ["auto", "gpu", "cpu_tflite"] + assert result["activeBackend"] == "tflite" + assert result["cuda"]["release_ready"] is False + assert result["cuda"]["tflite_fallback_ready"] is True + assert any("gated" in item for item in result["cuda"]["qualification_blockers"]) + + +def test_explicit_gpu_fails_before_start_and_requires_confirmed_tflite_fallback( + tflite_runtime, monkeypatch +): + tflite_runtime() + monkeypatch.setenv(sa3.SA3_PREFERENCE_ENV, "gpu") + + with pytest.raises(sa3.GenerationUnavailable, match="Choose CPU/TFLite"): + asyncio.run(sa3.generate("kick", 0.5, "sfx")) + + +def test_status_fails_closed_for_unverified_tflite_provenance(tmp_path): + selection = make_runtime( + tmp_path / "sa3", BackendName.TFLITE, platform_name="linux" + ) + (selection.runtime_dir / sa3.TFLITE_PROVENANCE_STAMP).write_text("{}") + result = sa3.status( + {"SA3_HOME": str(selection.checkout)}, + platform_name="linux", + machine="x86_64", + ) + assert result["state"] == sa3.STATE_FAILED + assert "do not match" in result["detail"] + + +def test_generation_modes_include_continuation(): + request = GenerationRequest( + "continue", + 2.0, + "music", + init_audio=b"wav", + inpaint_range=(1.0, 2.0), + ) + assert request.mode(input_seconds=1.0) is GenerationMode.CONTINUATION + assert request.mode(input_seconds=0.5) is GenerationMode.INPAINT + + +def _option(argv: list[str], name: str) -> str: + return argv[argv.index(name) + 1] + + +def test_mlx_and_tflite_translate_the_same_service_controls(tmp_path): + mlx = make_runtime(tmp_path / "mlx-root", BackendName.MLX) + tflite = make_runtime(tmp_path / "tflite-root", BackendName.TFLITE) + request = GenerationRequest( + "warm dub loop", + 0.5, + "music", + init_audio=pcm16_wav(), + init_noise_level=0.6, + inpaint_range=(0.1, 0.4), + negative_prompt="vocals", + cfg=4.5, + apg=0.75, + seed=12345, + steps=12, + lora_dirs=["/adapters/one", "/adapters/two"], + lora_strengths=[0.75, 1.5], + ) + commands = [ + sa3.build_argv( + selection, + request, + out_path=tmp_path / f"{selection.backend}.wav", + init_path=tmp_path / "init.wav", + env={}, ) - argv = (mlx_dir / ".venv" / "bin" / "argv.txt").read_text().splitlines() - init_index = argv.index("--init-audio") - assert pathlib.Path(argv[init_index + 1]).name == "init.wav" - assert argv[init_index + 2 :] == [ - "--init-noise-level", - "0.6", - "--inpaint-range", - "1.25,2.5", - "--negative-prompt", - "vocals", - "--cfg", - "4.5", - "--apg", - "0.75", - "--seed", - "12345", - ] - assert (mlx_dir / ".venv" / "bin" / "init.wav").read_bytes() == init_audio - - def test_passes_one_lora_group_per_adapter_with_its_strength(self, checkout): - # Issue #66 (ADR-0028): each adapter rides the argv as its own - # --lora group — the directory plus a strength=S option (the - # upstream PR #57/#65 CLI syntax). - mlx_dir = checkout(SUCCESS_STUB) - asyncio.run( - sa3.generate( - "maqam phrasing", - 120.0, - "track", - lora_dirs=["/adapters/medium/maqam", "/adapters/medium/breaks"], - lora_strengths=[0.75, 1.5], - ) + for selection in (mlx, tflite) + ] + for flag in ( + "--prompt", + "--dit", + "--decoder", + "--seconds", + "--steps", + "--init-audio", + "--init-noise-level", + "--inpaint-range", + "--negative-prompt", + "--cfg", + "--apg", + "--seed", + ): + assert _option(commands[0], flag) == _option(commands[1], flag) + assert commands[1][commands[1].index("--precision") + 1] == "fp32" + assert commands[1][commands[1].index("--threads") + 1] == "4" + first_lora = commands[1].index("--lora") + assert commands[1][first_lora : first_lora + 6] == [ + "--lora", + "/adapters/one", + "strength=0.75", + "--lora", + "/adapters/two", + "strength=1.5", + ] + + +def test_long_medium_request_maps_to_the_official_model_without_allocating_audio( + tmp_path, +): + selection = make_runtime(tmp_path / "sa3", BackendName.TFLITE) + request = GenerationRequest("long-form dub", 380.0, "track", steps=8) + argv = sa3.build_argv( + selection, + request, + out_path=tmp_path / "out.wav", + init_path=None, + env={}, + ) + assert _option(argv, "--dit") == "medium" + assert _option(argv, "--decoder") == "same-l" + assert _option(argv, "--seconds") == "380" + assert sa3.timeout_for(380.0) == sa3.TIMEOUT_SECONDS + 380.0 + + +def test_generate_returns_a_validated_wav_and_runs_offline(tflite_runtime): + selection = tflite_runtime() + wav = asyncio.run(sa3.generate("vinyl spinback", 0.5, "sfx", seed=7)) + assert sa3.inspect_canonical_wav(wav).frames == 22_050 + env = (selection.runtime_dir / "env.txt").read_text() + assert "offline=1" in env + assert "token=None" in env + assert "threads=4" in env + assert "pythonutf8=1" in env + assert "stdio=utf-8" in env + + +def test_generate_passes_full_control_surface_and_normalized_input(tflite_runtime): + selection = tflite_runtime() + source = pcm16_wav() + asyncio.run( + sa3.generate( + "warm dub loop", + 0.5, + "music", + init_audio=source, + init_noise_level=0.6, + inpaint_range=(0.1, 0.4), + negative_prompt="vocals", + cfg=4.5, + apg=0.75, + seed=12345, + steps=12, ) - argv = (mlx_dir / ".venv" / "bin" / "argv.txt").read_text().splitlines() - first = argv.index("--lora") - assert argv[first : first + 6] == [ - "--lora", - "/adapters/medium/maqam", - "strength=0.75", - "--lora", - "/adapters/medium/breaks", - "strength=1.5", - ] - - def test_lora_without_strengths_omits_the_option(self, checkout): - # No strengths → bare --lora groups; the CLI's default (1.0) applies. - mlx_dir = checkout(SUCCESS_STUB) - asyncio.run( - sa3.generate( - "vinyl spinback", 3.0, "sfx", lora_dirs=["/adapters/small/crackle"] - ) + ) + argv = (selection.runtime_dir / "argv.txt").read_text().splitlines() + assert _option(argv, "--steps") == "12" + assert _option(argv, "--inpaint-range") == "0.1,0.4" + assert _option(argv, "--negative-prompt") == "vocals" + assert _option(argv, "--cfg") == "4.5" + assert _option(argv, "--apg") == "0.75" + assert (selection.runtime_dir / "init.wav").read_bytes() == source + + +def test_missing_pinned_asset_fails_before_spawn(tflite_runtime): + selection = tflite_runtime(assets=False) + with pytest.raises(sa3.GenerationUnavailable, match="bundle is incomplete"): + asyncio.run(sa3.generate("anything", 0.5, "sfx")) + assert not (selection.runtime_dir / "argv.txt").exists() + + +def test_cli_failure_is_bounded_and_redacts_the_prompt(tflite_runtime): + tflite_runtime(FAILURE_STUB) + with pytest.raises(sa3.GenerationFailed) as caught: + asyncio.run(sa3.generate("super secret user prompt", 0.5, "sfx")) + assert "no DiT weights" in str(caught.value) + assert "super secret" not in str(caught.value) + + +@pytest.mark.parametrize("stub", [SILENT_STUB, CORRUPT_STUB, WRONG_DURATION_STUB]) +def test_missing_corrupt_or_wrong_duration_output_fails(tflite_runtime, stub): + tflite_runtime(stub) + with pytest.raises(sa3.GenerationFailed): + asyncio.run(sa3.generate("anything", 0.5, "sfx")) + + +def test_timeout_stops_the_worker(tflite_runtime, monkeypatch): + tflite_runtime(TIMEOUT_STUB) + monkeypatch.setattr(sa3, "TIMEOUT_SECONDS", 0.05) + with pytest.raises(sa3.GenerationFailed, match="timed out"): + asyncio.run(sa3.generate("anything", 0.5, "sfx")) + assert sa3.status()["generation"]["state"] == "idle" + + +def test_explicit_cancellation_stops_the_worker(tflite_runtime): + tflite_runtime(TIMEOUT_STUB) + + async def run(): + cancelled = asyncio.Event() + task = asyncio.create_task( + sa3.generate("anything", 0.5, "sfx", cancel_event=cancelled) ) - argv = (mlx_dir / ".venv" / "bin" / "argv.txt").read_text().splitlines() - lora_index = argv.index("--lora") - assert argv[lora_index + 1] == "/adapters/small/crackle" - assert not any(arg.startswith("strength=") for arg in argv) - assert "--lora-strength" not in argv - - def test_tracks_run_the_medium_dit_with_its_decoder(self, checkout): - # M19 (ADR-0013): tracks pair the medium DiT with SAME-L; the - # pad kinds keep the small DiTs with SAME-S. - mlx_dir = checkout(SUCCESS_STUB) - asyncio.run(sa3.generate("late night dub techno", 120.0, "track")) - argv = (mlx_dir / ".venv" / "bin" / "argv.txt").read_text().splitlines() - assert argv[argv.index("--dit") + 1] == "medium" - assert argv[argv.index("--decoder") + 1] == "same-l" - assert argv[argv.index("--seconds") + 1] == "120" - - def test_timeout_scales_with_the_requested_length(self): - assert sa3.timeout_for(3.0) == sa3.TIMEOUT_SECONDS + 3.0 - assert sa3.timeout_for(380.0) == sa3.TIMEOUT_SECONDS + 380.0 - - def test_no_checkout_raises_unavailable(self, monkeypatch, tmp_path): - monkeypatch.delenv("SA3_MLX_HOME", raising=False) - monkeypatch.setattr(sa3.pathlib.Path, "home", staticmethod(lambda: tmp_path)) - with pytest.raises(sa3.GenerationUnavailable): - asyncio.run(sa3.generate("anything", 3.0, "sfx")) - - def test_cli_failure_raises_with_output_tail(self, checkout): - checkout(FAILURE_STUB) - with pytest.raises(sa3.GenerationFailed, match="no DiT weights"): - asyncio.run(sa3.generate("anything", 3.0, "sfx")) - - def test_clean_exit_without_wav_is_a_failure(self, checkout): - checkout(SILENT_STUB) - with pytest.raises(sa3.GenerationFailed): - asyncio.run(sa3.generate("anything", 3.0, "sfx")) - - def test_timeout_kills_and_raises(self, checkout, monkeypatch): - # The deadline is base + seconds (timeout_for), so a short clip - # keeps the test fast while exercising the real kill path. - checkout("#!/bin/sh\nsleep 30\n") - monkeypatch.setattr(sa3, "TIMEOUT_SECONDS", 0.2) - with pytest.raises(sa3.GenerationFailed, match="timed out"): - asyncio.run(sa3.generate("anything", 0.5, "sfx")) + await asyncio.sleep(0.1) + cancelled.set() + await task + + with pytest.raises(sa3.GenerationCancelled, match="cancelled"): + asyncio.run(run()) + assert sa3.status()["generation"]["state"] == "idle" + + +@pytest.mark.skipif(os.name != "posix", reason="Unix process-group contract") +def test_cancellation_stops_the_generation_grandchild(tflite_runtime): + selection = tflite_runtime(TREE_STUB) + + async def run(): + cancelled = asyncio.Event() + task = asyncio.create_task( + sa3.generate("anything", 0.5, "sfx", cancel_event=cancelled) + ) + pidfile = selection.runtime_dir / "grandchild.pid" + for _ in range(100): + if pidfile.exists(): + break + await asyncio.sleep(0.01) + assert pidfile.exists(), "stub did not launch its grandchild" + cancelled.set() + await task + + with pytest.raises(sa3.GenerationCancelled, match="cancelled"): + asyncio.run(run()) + pid = int((selection.runtime_dir / "grandchild.pid").read_text()) + for _ in range(100): + try: + os.kill(pid, 0) + except ProcessLookupError: + break + time.sleep(0.01) + else: + os.kill(pid, signal.SIGKILL) + pytest.fail(f"generation grandchild {pid} survived cancellation") + + +def test_progress_is_normalized_from_the_official_text_stream(tflite_runtime): + tflite_runtime(PROGRESS_STUB) + events = [] + asyncio.run(sa3.generate("anything", 0.5, "sfx", on_progress=events.append)) + assert [(event.stage, event.current, event.total) for event in events] == [ + ("sampling", 1, 2), + ("sampling", 2, 2), + ] + + +def test_no_runtime_raises_unavailable(monkeypatch, tmp_path): + monkeypatch.setenv("LSDJ_SA3_BACKEND", "tflite") + monkeypatch.setenv("SA3_HOME", str(tmp_path / "missing")) + monkeypatch.setattr(sa3, "resolve_runtime", lambda: None) + monkeypatch.setattr(sa3, "select_backend", lambda: BackendName.TFLITE) + with pytest.raises(sa3.GenerationUnavailable, match="tflite"): + asyncio.run(sa3.generate("anything", 0.5, "sfx")) diff --git a/backend/tests/test_sa3_audio.py b/backend/tests/test_sa3_audio.py new file mode 100644 index 0000000..d0c7aa1 --- /dev/null +++ b/backend/tests/test_sa3_audio.py @@ -0,0 +1,154 @@ +"""Deterministic, model-free Stable Audio WAV boundary tests.""" + +import io +import struct +import threading +import wave + +import pytest + +from lsdj import sa3_audio + + +def pcm_wav( + raw: bytes, + *, + sample_rate: int, + channels: int, + sample_width: int, +) -> bytes: + output = io.BytesIO() + with wave.open(output, "wb") as target: + target.setnchannels(channels) + target.setsampwidth(sample_width) + target.setframerate(sample_rate) + target.writeframes(raw) + return output.getvalue() + + +def test_canonical_wav_is_validated_without_reencoding(): + source = pcm_wav( + b"\0" * 100 * 4, + sample_rate=44_100, + channels=2, + sample_width=2, + ) + normalized = sa3_audio.normalize_wav(source) + assert normalized.wav is source + assert normalized.frames == 100 + + +def test_mono_8khz_pcm8_is_resampled_and_duplicated_to_stereo(): + source = pcm_wav( + bytes([128]) * 8_000, + sample_rate=8_000, + channels=1, + sample_width=1, + ) + normalized = sa3_audio.normalize_wav(source) + with wave.open(io.BytesIO(normalized.wav), "rb") as result: + assert result.getframerate() == 44_100 + assert result.getnchannels() == 2 + assert result.getsampwidth() == 2 + assert result.getnframes() == 44_100 + assert result.readframes(1) == b"\0\0\0\0" + + +def test_multichannel_input_keeps_the_first_two_channels(): + frame = struct.pack(" current + cancel.set() + + with pytest.raises(sa3_audio.AudioNormalizationCancelled): + sa3_audio.normalize_wav(source, cancel_event=cancel, on_progress=progress) + + +def test_decoded_work_and_normalized_frame_limits_are_enforced(monkeypatch): + source = pcm_wav( + b"\0" * 16, + sample_rate=8_000, + channels=1, + sample_width=1, + ) + monkeypatch.setattr(sa3_audio, "MAX_DECODED_PCM_BYTES", 8) + with pytest.raises(sa3_audio.AudioFormatError, match="decoded PCM"): + sa3_audio.normalize_wav(source) + + monkeypatch.setattr(sa3_audio, "MAX_DECODED_PCM_BYTES", 1024) + monkeypatch.setattr(sa3_audio, "MAX_NORMALIZED_FRAMES", 10) + with pytest.raises(sa3_audio.AudioFormatError, match="normalized WAV"): + sa3_audio.normalize_wav(source) diff --git a/backend/tests/test_sa3_cuda.py b/backend/tests/test_sa3_cuda.py new file mode 100644 index 0000000..c183a00 --- /dev/null +++ b/backend/tests/test_sa3_cuda.py @@ -0,0 +1,124 @@ +import pytest + +from lsdj import sa3_cuda + + +def evidence(**updates): + values = { + "platform": "win32", + "machine": "AMD64", + "runtime_ready": True, + "provenance_complete": True, + "packages": dict(sa3_cuda.EXPECTED_PACKAGES), + "cuda_available": True, + "cuda_runtime": "12.6", + "driver": "560.76", + "device": "NVIDIA test device", + "compute_capability": (8, 9), + "total_vram_bytes": 24 * 1024**3, + "free_vram_bytes": 16 * 1024**3, + "estimated_vram_bytes": {"music": 6 * 1024**3, "sfx": 6 * 1024**3}, + "source_revision": sa3_cuda.SOURCE_REVISION, + "model_revision": sa3_cuda.MODEL_PINS["music"]["revision"], + } + values.update(updates) + return sa3_cuda.CudaEvidence(**values) + + +def test_auto_keeps_tflite_until_hardware_is_release_qualified(): + decision = sa3_cuda.choose_backend( + "auto", kind="music", cuda=evidence(), tflite_ready=True, env={} + ) + assert decision.backend == "tflite" + assert decision.fallback is True + + +def test_explicit_gpu_never_silently_falls_back(): + with pytest.raises(sa3_cuda.CudaUnavailable) as caught: + sa3_cuda.choose_backend( + "gpu", kind="music", cuda=evidence(), tflite_ready=True, env={} + ) + assert caught.value.fallback_available is True + assert caught.value.reason == "cuda_not_eligible" + + +def test_qualification_opt_in_allows_small_models_but_not_auto(): + explicit = sa3_cuda.choose_backend( + "gpu", + kind="music", + cuda=evidence(estimated_vram_bytes={"music": None}), + tflite_ready=True, + env={sa3_cuda.UNVERIFIED_OPT_IN: "1"}, + ) + automatic = sa3_cuda.choose_backend( + "auto", + kind="music", + cuda=evidence(estimated_vram_bytes={"music": None}), + tflite_ready=True, + env={sa3_cuda.UNVERIFIED_OPT_IN: "1"}, + ) + assert explicit.backend == "pytorch_cuda" + assert automatic.backend == "tflite" + + +@pytest.mark.parametrize( + "updates, expected", + [ + ({"provenance_complete": False}, "provenance is incomplete"), + ({"source_revision": "0" * 40}, "source revision does not match"), + ({"model_revision": "0" * 40}, "model revision does not match"), + ({"cuda_available": False}, "no CUDA device"), + ({"cuda_runtime": "13.0"}, "not the pinned 12.6"), + ({"driver": None}, "driver version could not be verified"), + ({"driver": "528.33"}, "older than the provisional"), + ( + {"packages": {**sa3_cuda.EXPECTED_PACKAGES, "torch": "2.12.1+cu130"}}, + "dependency versions do not match", + ), + ], +) +def test_explicit_gpu_fails_closed_on_runtime_mismatch(updates, expected): + with pytest.raises(sa3_cuda.CudaUnavailable, match=expected): + sa3_cuda.choose_backend( + "gpu", + kind="music", + cuda=evidence(**updates), + tflite_ready=True, + env={sa3_cuda.UNVERIFIED_OPT_IN: "1"}, + ) + + +def test_medium_stays_on_tflite_without_an_official_windows_flashattention_build(): + with pytest.raises( + sa3_cuda.CudaUnavailable, match="Medium requires FlashAttention" + ): + sa3_cuda.choose_backend( + "gpu", + kind="track", + cuda=evidence(estimated_vram_bytes={"track": 12 * 1024**3}), + tflite_ready=True, + env={sa3_cuda.UNVERIFIED_OPT_IN: "1"}, + ) + + +def test_free_vram_is_advisory_but_still_a_conservative_admission_gate(): + errors = sa3_cuda.runtime_errors( + evidence(free_vram_bytes=6 * 1024**3), kind="music" + ) + assert any("headroom" in error for error in errors) + + +def test_cpu_choice_requires_the_portable_baseline(): + with pytest.raises( + sa3_cuda.CudaUnavailable, match="TFLite backend is not installed" + ): + sa3_cuda.choose_backend( + "cpu_tflite", kind="music", cuda=evidence(), tflite_ready=False + ) + + +def test_diagnostics_are_honest_about_hardware_and_gated_hashes(): + status = sa3_cuda.diagnostic_manifest(evidence(), tflite_ready=True) + assert status["release_ready"] is False + assert status["cpu_fallback"] is False + assert any("gated" in item for item in status["qualification_blockers"]) diff --git a/backend/tests/test_sa3_cuda_pins.py b/backend/tests/test_sa3_cuda_pins.py new file mode 100644 index 0000000..5403f99 --- /dev/null +++ b/backend/tests/test_sa3_cuda_pins.py @@ -0,0 +1,64 @@ +import hashlib +import json +import re +from pathlib import Path + +from lsdj import sa3_cuda + + +ROOT = Path(__file__).parents[2] +PIN_PATH = ROOT / "sa3-pytorch-cuda-pin.json" +LOCK_PATH = ROOT / "backend/runtime-locks/windows-gpu-pytorch.txt" +REQUIREMENT = re.compile(r"^([a-z0-9][a-z0-9_.-]*)==([^ \\]+) \\$", re.MULTILINE) + + +def test_shared_windows_runtime_is_hash_locked_and_matches_executable_policy(): + pin = json.loads(PIN_PATH.read_text()) + lock = LOCK_PATH.read_bytes() + runtime = pin["sharedRuntime"] + requirements = dict(REQUIREMENT.findall(lock.decode())) + + assert len(requirements) == 44 + assert runtime["packages"] == sa3_cuda.EXPECTED_PACKAGES + assert runtime["packages"].items() <= requirements.items() + assert runtime["requirementsLockSize"] == len(lock) + assert runtime["requirementsLockSha256"] == hashlib.sha256(lock).hexdigest() + assert runtime["requirementsLockSha256"] == sa3_cuda.RUNTIME_LOCK_SHA256 + assert lock.count(b"--hash=sha256:") >= len(requirements) + assert not any(token in lock for token in (b"git+", b"http://", b" @ ")) + + +def test_candidate_cannot_be_released_with_missing_gated_hashes(): + pin = json.loads(PIN_PATH.read_text()) + missing = [ + f"{model_name}/{artifact['path']}" + for model_name, model in pin["models"].items() + if model.get("required") or model.get("enabled") + for artifact in (model["weight"], model["config"]) + if artifact["sha256"] is None + ] + + assert missing == [ + "small-music/model_config.json", + "small-sfx/model_config.json", + ] + assert pin["gatedArtifactsComplete"] is False + assert pin["releaseReady"] is False + assert pin["releaseBlockers"] + + +def test_cuda_manifest_uses_upstream_as_an_external_immutable_dependency(): + pin = json.loads(PIN_PATH.read_text()) + source = pin["source"] + + assert source["repository"] == "https://github.com/Stability-AI/stable-audio-3" + assert source["revision"] == sa3_cuda.SOURCE_REVISION + assert source["revision"] in source["archiveUrl"] + assert source["license"] == "MIT" + pinned_models = { + (model["repository"], model["revision"]) for model in pin["models"].values() + } + assert { + (model["repository"], model["revision"]) + for model in sa3_cuda.MODEL_PINS.values() + } <= pinned_models diff --git a/backend/tests/test_sa3_cuda_worker.py b/backend/tests/test_sa3_cuda_worker.py new file mode 100644 index 0000000..5d827de --- /dev/null +++ b/backend/tests/test_sa3_cuda_worker.py @@ -0,0 +1,292 @@ +import json +import hashlib +import pathlib +import threading +import wave + +import numpy as np +import pytest + +from lsdj import sa3_cuda, sa3_cuda_worker as worker + + +class FakeTorch: + @staticmethod + def from_numpy(value): + return value + + +class FakeModel: + def __init__(self, seconds=0.5): + self.seconds = seconds + self.kwargs = None + self.loras = [] + self.strengths = [] + + def load_lora(self, paths): + self.loras = paths + + def set_lora_strength(self, strength, lora_index=None): + self.strengths.append((strength, lora_index)) + + def generate(self, **kwargs): + self.kwargs = kwargs + for index in range(kwargs["steps"]): + kwargs["callback"]({"i": index}) + return np.zeros( + (1, 2, round(self.seconds * worker.SAMPLE_RATE)), dtype=np.float32 + ) + + +def pcm16_wav(path, seconds=0.5): + frames = round(seconds * worker.SAMPLE_RATE) + with wave.open(str(path), "wb") as output: + output.setnchannels(2) + output.setsampwidth(2) + output.setframerate(worker.SAMPLE_RATE) + output.writeframes(b"\0" * frames * 4) + + +def request(tmp_path, **updates): + launch_token = "qualification-token-with-32-bytes-minimum" + value = { + "schema_version": 1, + "job_id": "sa3-job-123", + "launch_token_sha256": hashlib.sha256(launch_token.encode()).hexdigest(), + "prompt": "warm dub loop", + "seconds": 0.5, + "kind": "music", + "steps": 8, + "cfg": 4.5, + "apg": 0.75, + "seed": 123, + "negative_prompt": "vocals", + "init_noise_level": 0.6, + "inpaint_range": None, + "init_audio": None, + "lora_files": [], + "lora_strengths": [], + "model_dir": str(tmp_path / "model"), + "output": str(tmp_path / "out.wav"), + } + value.update(updates) + return worker.WorkerRequest.from_dict(value) + + +def test_maps_every_shared_control_to_the_pinned_python_api(tmp_path): + model = FakeModel() + events = [] + item = request(tmp_path) + worker.run_generation( + item, + model=model, + torch_module=FakeTorch, + cancelled=lambda: False, + emit=events.append, + ) + assert model.kwargs | {"callback": None} == { + "prompt": "warm dub loop", + "negative_prompt": "vocals", + "duration": 0.5, + "steps": 8, + "cfg_scale": 4.5, + "apg_scale": 0.75, + "seed": 123, + "batch_size": 1, + "chunked_decode": True, + "callback": None, + "disable_tqdm": True, + "init_audio": None, + "init_noise_level": 0.6, + "inpaint_audio": None, + } + assert events[-1] == {"event": "done"} + with wave.open(str(item.output), "rb") as output: + assert output.getparams()[:4] == (2, 2, 44_100, 22_050) + + +def test_maps_inpainting_and_continuation_to_upstream_inpaint_api(tmp_path): + init = tmp_path / "init.wav" + pcm16_wav(init) + item = request(tmp_path, init_audio=str(init), inpaint_range=[0.25, 0.5]) + model = FakeModel() + worker.run_generation( + item, + model=model, + torch_module=FakeTorch, + cancelled=lambda: False, + ) + assert model.kwargs["init_audio"] is None + assert model.kwargs["inpaint_audio"][0] == 44_100 + assert model.kwargs["inpaint_audio"][1].shape == (2, 22_050) + assert model.kwargs["inpaint_mask_start_seconds"] == 0.25 + assert model.kwargs["inpaint_mask_end_seconds"] == 0.5 + + +def test_stacked_lora_strengths_are_set_per_index(tmp_path): + adapters = [] + for name in ("one", "two"): + directory = tmp_path / name + directory.mkdir() + (directory / f"{name}.safetensors").write_bytes(b"fixture") + adapters.append(str(directory)) + item = request(tmp_path, lora_files=adapters, lora_strengths=[0.75, 1.5]) + model = FakeModel() + worker.run_generation( + item, + model=model, + torch_module=FakeTorch, + cancelled=lambda: False, + ) + assert [pathlib.Path(path).name for path in model.loras] == [ + "one.safetensors", + "two.safetensors", + ] + assert model.strengths == [(0.75, 0), (1.5, 1)] + + +def test_cancellation_is_observed_between_sampling_steps(tmp_path): + calls = 0 + + def cancelled(): + nonlocal calls + calls += 1 + return calls >= 2 + + with pytest.raises(worker.WorkerCancelled, match="cancelled"): + worker.run_generation( + request(tmp_path), + model=FakeModel(), + torch_module=FakeTorch, + cancelled=cancelled, + ) + assert not (tmp_path / "out.wav").exists() + + +def test_priority_waiter_cancels_the_disposable_worker(tmp_path): + class Broker: + @staticmethod + def should_yield(_lease): + return True + + with pytest.raises(worker.WorkerCancelled, match="yielded"): + worker.run_generation( + request(tmp_path), + model=FakeModel(), + torch_module=FakeTorch, + cancelled=lambda: False, + broker=Broker(), + lease=object(), + ) + + +@pytest.mark.parametrize( + "updates, message", + [ + ({"kind": "track"}, "Small Music and Small SFX"), + ({"steps": 0}, "steps"), + ({"inpaint_range": [0.1, 0.2]}, "requires init_audio"), + ({"lora_files": ["one"], "lora_strengths": []}, "LoRA stack"), + ], +) +def test_request_contract_fails_closed(updates, message, tmp_path): + with pytest.raises(worker.WorkerError, match=message): + request(tmp_path, **updates) + + +def test_launch_token_authenticates_one_private_request_without_storing_secret( + tmp_path, +): + item = request(tmp_path) + token = "qualification-token-with-32-bytes-minimum" + worker.verify_launch_token(item, {worker.LAUNCH_TOKEN_ENV: token}) + with pytest.raises(worker.WorkerError, match="does not match"): + worker.verify_launch_token(item, {worker.LAUNCH_TOKEN_ENV: "x" * 40}) + with pytest.raises(worker.WorkerError, match="missing or invalid"): + worker.verify_launch_token(item, {}) + + +def test_broker_watchdog_hard_stops_disposable_worker_during_model_load(): + exited = threading.Event() + events = [] + + class Broker: + @staticmethod + def should_yield(_lease): + return True + + stop, thread = worker.start_broker_watchdog( + Broker(), + object(), + events.append, + poll_seconds=0.001, + exit_process=lambda code: exited.set() if code == 2 else None, + ) + assert exited.wait(1) + thread.join(timeout=1) + stop.set() + assert events == [ + { + "event": "cancelled", + "message": "Stable Audio yielded to realtime MRT2 generation", + } + ] + + +def test_request_file_is_bounded_and_rejects_symlinks(tmp_path): + real = tmp_path / "request.json" + real.write_text(json.dumps({"schema_version": 1})) + link = tmp_path / "link.json" + link.symlink_to(real) + with pytest.raises(worker.WorkerError, match="regular file"): + worker.read_request(link) + + +def test_provenance_matches_exact_source_runtime_model_and_bundle_path(tmp_path): + root = tmp_path / "runtime" + model_dir = root / "models" / "small-music" + model_dir.mkdir(parents=True) + item = request(tmp_path, model_dir=str(model_dir)) + stamp = root / "provenance.json" + value = { + "schema_version": 1, + "backend": sa3_cuda.BACKEND_NAME, + "gated_artifacts_complete": True, + "source_revision": sa3_cuda.SOURCE_REVISION, + "runtime_lock_sha256": sa3_cuda.RUNTIME_LOCK_SHA256, + "packages": sa3_cuda.EXPECTED_PACKAGES, + "model": sa3_cuda.MODEL_PINS["music"], + } + stamp.write_text(json.dumps(value)) + + assert worker.verify_provenance(stamp, item) == value + + value["source_revision"] = "0" * 40 + stamp.write_text(json.dumps(value)) + with pytest.raises(worker.WorkerError, match="immutable"): + worker.verify_provenance(stamp, item) + + +def test_provenance_rejects_model_path_outside_verified_runtime(tmp_path): + root = tmp_path / "runtime" + model_dir = tmp_path / "other" / "small-music" + model_dir.mkdir(parents=True) + root.mkdir() + item = request(tmp_path, model_dir=str(model_dir)) + stamp = root / "provenance.json" + stamp.write_text( + json.dumps( + { + "schema_version": 1, + "backend": sa3_cuda.BACKEND_NAME, + "gated_artifacts_complete": True, + "source_revision": sa3_cuda.SOURCE_REVISION, + "runtime_lock_sha256": sa3_cuda.RUNTIME_LOCK_SHA256, + "packages": sa3_cuda.EXPECTED_PACKAGES, + "model": sa3_cuda.MODEL_PINS["music"], + } + ) + ) + + with pytest.raises(worker.WorkerError, match="outside"): + worker.verify_provenance(stamp, item) diff --git a/backend/tests/test_sa3_manifest.py b/backend/tests/test_sa3_manifest.py new file mode 100644 index 0000000..ca18567 --- /dev/null +++ b/backend/tests/test_sa3_manifest.py @@ -0,0 +1,93 @@ +"""Fail-closed checks for the pinned official TFLite runtime manifest.""" + +import json +import pathlib +import re + +from lsdj import sa3 +from lsdj.sa3_contract import GenerationRequest + +ROOT = pathlib.Path(__file__).resolve().parents[2] +MANIFEST = ROOT / "sa3-tflite-pin.json" +LOCK = ROOT / "scripts" / "sa3-tflite-requirements.lock" + + +def load_manifest() -> dict: + return json.loads(MANIFEST.read_text()) + + +def test_runtime_and_models_use_immutable_official_revisions(): + manifest = load_manifest() + runtime = manifest["runtime"] + models = manifest["models"] + assert runtime["repo"] == "https://github.com/Stability-AI/stable-audio-3" + assert re.fullmatch(r"[0-9a-f]{40}", runtime["revision"]) + assert models["repo"] == "stabilityai/stable-audio-3-optimized" + assert re.fullmatch(r"[0-9a-f]{40}", models["revision"]) + assert runtime["repo"] == sa3.TFLITE_RUNTIME_REPO + assert runtime["revision"] == sa3.TFLITE_RUNTIME_REVISION + assert models["repo"] == sa3.TFLITE_MODELS_REPO + assert models["revision"] == sa3.TFLITE_MODELS_REVISION + + +def test_every_model_asset_has_a_safe_path_exact_size_and_sha256(): + models = load_manifest()["models"] + assets = [*models["shared"]] + for bundle in models["bundles"].values(): + assets.extend(bundle) + for asset in assets: + assert asset["size"] > 0 + assert re.fullmatch(r"[0-9a-f]{64}", asset["sha256"]) + install_path = pathlib.PurePosixPath(asset["installPath"]) + assert not install_path.is_absolute() + assert ".." not in install_path.parts + assert asset["path"].startswith("tflite/") + + +def test_adapter_preflight_paths_match_the_pinned_manifest(): + manifest = load_manifest()["models"] + installed = {entry["installPath"] for entry in manifest["shared"]} + for entries in manifest["bundles"].values(): + installed.update(entry["installPath"] for entry in entries) + required = set() + for kind in ("sfx", "music", "track"): + request = GenerationRequest("fixture", 0.5, kind, init_audio=b"wav") + required.update( + path.as_posix() for path in sa3._required_tflite_assets(request) + ) + required.remove("models/tokenizer.model") + assert required == installed + + +def test_measured_bundle_storage_totals_are_stable(): + models = load_manifest()["models"] + shared = sum(entry["size"] for entry in models["shared"]) + totals = { + name: shared + sum(entry["size"] for entry in entries) + for name, entries in models["bundles"].items() + } + assert totals == { + "sm-music": 2_836_149_512, + "sm-sfx": 2_836_149_512, + "medium": 10_027_905_456, + } + + +def test_runtime_lock_is_hash_pinned_and_covers_official_direct_dependencies(): + lock = LOCK.read_text() + for package in ( + "ai-edge-litert", + "numpy", + "sentencepiece", + "soundfile", + "huggingface-hub", + ): + assert re.search(rf"(?m)^{re.escape(package)}==", lock) + assert "--hash=sha256:" in lock + requirements = [ + line + for line in lock.splitlines() + if line and not line[0].isspace() and not line.startswith("#") + ] + assert requirements + assert all("==" in requirement for requirement in requirements) diff --git a/backend/tests/test_sidecar.py b/backend/tests/test_sidecar.py index 630dec6..dc07798 100644 --- a/backend/tests/test_sidecar.py +++ b/backend/tests/test_sidecar.py @@ -9,15 +9,21 @@ import threading import time +import pytest + from lsdj.sidecar import ( FRAME_CONTROL, + FRAME_AUTH, FRAME_EMBED, FRAME_PCM, FRAME_STATUS, + SharedSocketCmdQueues, SocketCmdQueue, SocketOutQueue, + authenticate_to_host, read_frame, run_sidecar, + run_shared_sidecar, write_frame, ) @@ -76,6 +82,27 @@ def test_read_frame_returns_none_on_truncated_payload(): assert read_frame(reader) is None +def test_frames_are_bounded_in_both_directions(monkeypatch): + import lsdj.sidecar as sidecar_mod + + monkeypatch.setattr(sidecar_mod, "MAX_FRAME_BYTES", 4) + with pytest.raises(ValueError, match="exceeds the cap"): + sidecar_mod.write_frame(RecordingSock(), FRAME_PCM, b"12345") + with pytest.raises(ValueError, match="exceeds the cap"): + sidecar_mod.read_frame(io.BytesIO(struct.pack("= 0 + assert chunk["queue_depth"] is not None + assert chunk["rtf"] is not None + + +def test_ready_carries_runtime_diagnostics(): + harness = DeckHarness() + harness.thread.start() + ready = harness.next_event("ready") + assert ready["runtime"] == { + "runtime": "fake", + "hardware_qualified": False, + } + harness.send(type="shutdown") + harness.thread.join(timeout=2) + + +def test_startup_failure_is_structured_and_worker_exits(): + out_queue = queue.Queue() + + def fail(**_kwargs): + raise RuntimeError("CUDA unavailable") + + run_deck_worker("a", "mrt2_small", queue.Queue(), out_queue, engine_factory=fail) + kind, status = out_queue.get_nowait() + assert kind == "status" + assert status == { + "event": "startup_failed", + "deck": "a", + "model": "mrt2_small", + "error": ( + "RuntimeError: MRT2 worker startup failed; " + "inspect the local application log for details" + ), + } + + +def test_reset_to_reseed_stops_generation_and_reports_contract(deck): + deck.send(type="reset", seed=42) + status = deck.next_event("reset") + assert status["seed"] == 42 + assert deck.engine.resets == [42] def test_set_prompt_applies_as_single_prompt_style(deck): diff --git a/backend/uv.lock b/backend/uv.lock index a0d4a40..8daa307 100644 --- a/backend/uv.lock +++ b/backend/uv.lock @@ -825,6 +825,15 @@ dependencies = [ ] [package.dev-dependencies] +ci = [ + { name = "fastapi" }, + { name = "httpx" }, + { name = "numpy" }, + { name = "pytest" }, + { name = "python-multipart" }, + { name = "ruff" }, + { name = "uvicorn" }, +] dev = [ { name = "httpx" }, { name = "pyinstaller" }, @@ -842,6 +851,15 @@ requires-dist = [ ] [package.metadata.requires-dev] +ci = [ + { name = "fastapi", specifier = ">=0.136.3" }, + { name = "httpx", specifier = ">=0.27" }, + { name = "numpy", specifier = ">=2.2" }, + { name = "pytest", specifier = ">=8" }, + { name = "python-multipart", specifier = ">=0.0.32" }, + { name = "ruff", specifier = ">=0.8" }, + { name = "uvicorn", specifier = ">=0.49.0" }, +] dev = [ { name = "httpx", specifier = ">=0.27" }, { name = "pyinstaller", specifier = ">=6.21" }, diff --git a/compliance/README.md b/compliance/README.md new file mode 100644 index 0000000..b00afae --- /dev/null +++ b/compliance/README.md @@ -0,0 +1,26 @@ +# Model and runtime asset inventory + +`model-assets.json` is the machine-readable source of truth for the model and +model-runtime compliance work in issue #108. It records what upstream projects +say at exact revisions, how LSDJ obtains each asset, and which decisions still +require project-owner review. It does not approve a use or provide legal advice. + +Run the dependency-free validator and its mutation tests with: + +```sh +python3 compliance/validate_inventory.py +python3 -m unittest compliance.test_inventory +``` + +The validator fails for missing required fields, branch-like or short revisions, +revision URLs that do not contain the pinned hash, unsafe installer settings, +unresolved dependency/catalog IDs, and mutable runtime download behavior that is +not kept as a release gate. An upstream source revision may be recorded as +`unresolved_upstream` only with a null value, a canonical evidence URL, and an +explicit release gate; this avoids inventing precision the reviewed evidence does +not support. + +When a runtime or model revision changes, update this manifest, the human notice +document, and the release checklist in the same pull request. Evidence URLs for +versioned source/model artifacts should point at the exact commit or snapshot; +policy URLs may remain canonical because upstream policies are living documents. diff --git a/compliance/__init__.py b/compliance/__init__.py new file mode 100644 index 0000000..1624ffc --- /dev/null +++ b/compliance/__init__.py @@ -0,0 +1 @@ +"""Compliance inventory validation helpers.""" diff --git a/compliance/model-assets.json b/compliance/model-assets.json new file mode 100644 index 0000000..b8ee65f --- /dev/null +++ b/compliance/model-assets.json @@ -0,0 +1,1263 @@ +{ + "schema_version": 1, + "inventory_revision": "2026-08-08.4", + "audited_at": "2026-08-08", + "audit_base_revision": "c9cd822ef6cbb86711e72d35f0f7e50a126d666f", + "purpose": "Revision-specific technical provenance and notice inputs for LSDJ model and model-runtime assets. This inventory records upstream statements and unresolved owner-review gates; it is not legal advice or a project-use approval.", + "project_use": { + "reported_context": "GitHub issue #108 describes LSDJ as open-source and non-commercial.", + "owner_confirmation_status": "pending", + "future_distribution_or_commercial_change_is_release_gate": true, + "public_record_must_exclude": [ + "revenue details", + "account details", + "contracts", + "credentials" + ] + }, + "catalogs": { + "bundled_lora_ids": [], + "official_lora_ids": [], + "documented_reference_lora_ids": [ + "motif-maqam-lora" + ], + "note": "The application has no built-in or Stability-AI-official LoRA catalog at the audited revision. The Motif Maqam adapter is the sole public adapter named in LSDJ documentation and tests, so it is inventoried independently as a documented reference, not represented as an official Stability AI artifact." + }, + "assets": [ + { + "id": "lsdj-source", + "name": "LSDJ application source", + "family": "lsdj", + "asset_type": "application_code", + "support_status": "current", + "upstream": { + "project": "LSDJ", + "canonical_url": "https://github.com/protocol-works/lsdj" + }, + "revision": { + "kind": "git_commit", + "value": "c9cd822ef6cbb86711e72d35f0f7e50a126d666f", + "url": "https://github.com/protocol-works/lsdj/tree/c9cd822ef6cbb86711e72d35f0f7e50a126d666f" + }, + "licenses": { + "code": [ + { + "status": "unresolved", + "identifier": "NOASSERTION", + "name": "No project LICENSE or NOTICE file found at the audited revision", + "scope": "LSDJ-authored application code", + "terms_url": null, + "notice_url": null + } + ], + "weights": [ + { + "status": "not_applicable", + "identifier": "NONE", + "name": "Not applicable", + "scope": "This entry is application code, not model weights", + "terms_url": null, + "notice_url": null + } + ] + }, + "notices": { + "required_text": [], + "attribution": [], + "sources": [] + }, + "access": { + "gated": false, + "account_required": false, + "credential_required": false, + "terms_acceptance_required": false, + "privacy_url": null, + "acceptable_use_url": null + }, + "distribution": { + "mode": "bundled_in_application", + "source_url": "https://github.com/protocol-works/lsdj/tree/c9cd822ef6cbb86711e72d35f0f7e50a126d666f", + "installer_contains_asset": true, + "installer_contains_weights": false, + "redistribution_confirmed": false, + "immutable_reference_enforced": true, + "release_gate": true, + "notes": "Project owners must select and add the LSDJ code license before a public source or binary release can claim a license." + }, + "dependencies": [], + "owner_review": { + "required": true, + "status": "pending", + "question": "Select the LSDJ code license and copyright/notice text, then confirm how it is carried in source and packaged applications.", + "issue": "https://github.com/protocol-works/lsdj/issues/108" + }, + "evidence": [ + "https://github.com/protocol-works/lsdj/tree/c9cd822ef6cbb86711e72d35f0f7e50a126d666f" + ] + }, + { + "id": "google-mrt2-runtime-code", + "name": "Google Magenta RealTime 2 Python runtime", + "family": "magenta-realtime-2", + "asset_type": "runtime_code", + "support_status": "current", + "upstream": { + "project": "magenta/magenta-realtime", + "canonical_url": "https://github.com/magenta/magenta-realtime" + }, + "revision": { + "kind": "git_commit", + "value": "4bf995bdd9c29b818543574e1b3a6e67867c9a58", + "url": "https://github.com/magenta/magenta-realtime/tree/4bf995bdd9c29b818543574e1b3a6e67867c9a58" + }, + "package": { + "name": "magenta-rt", + "version": "2.0.2", + "url": "https://pypi.org/project/magenta-rt/2.0.2/", + "wheel_sha256": "92df64a8150a6bff85ab9a0db0b54929a69b2538949df72677af2dc25a6a3845", + "sdist_sha256": "4f6241955094e5c38142deef820a03ce8e64bd1b300223e0d78a9195056c9b98" + }, + "licenses": { + "code": [ + { + "status": "declared", + "identifier": "Apache-2.0", + "name": "Apache License 2.0", + "scope": "Magenta RealTime source and Python package", + "terms_url": "https://github.com/magenta/magenta-realtime/blob/4bf995bdd9c29b818543574e1b3a6e67867c9a58/LICENSE", + "notice_url": null + } + ], + "weights": [ + { + "status": "not_applicable", + "identifier": "NONE", + "name": "Not applicable", + "scope": "Weights are inventoried separately", + "terms_url": null, + "notice_url": null + } + ] + }, + "notices": { + "required_text": [ + "Include the Apache-2.0 license with redistributed code and preserve applicable copyright, patent, trademark, and attribution notices." + ], + "attribution": [ + "Copyright 2026 Google LLC" + ], + "sources": [ + "https://github.com/magenta/magenta-realtime/blob/4bf995bdd9c29b818543574e1b3a6e67867c9a58/LICENSE" + ] + }, + "access": { + "gated": false, + "account_required": false, + "credential_required": false, + "terms_acceptance_required": false, + "privacy_url": null, + "acceptable_use_url": null + }, + "distribution": { + "mode": "bundled_runtime_dependency", + "source_url": "https://github.com/magenta/magenta-realtime/tree/4bf995bdd9c29b818543574e1b3a6e67867c9a58", + "installer_contains_asset": true, + "installer_contains_weights": false, + "redistribution_confirmed": false, + "immutable_reference_enforced": true, + "release_gate": true, + "notes": "The package is hash-locked in backend/uv.lock. Owner review must confirm the packaged third-party notice location before release." + }, + "dependencies": [ + "google-mrt2-weights" + ], + "owner_review": { + "required": true, + "status": "pending", + "question": "Confirm the Apache-2.0 runtime notice is included in every packaged sidecar/application artifact.", + "issue": "https://github.com/protocol-works/lsdj/issues/108" + }, + "evidence": [ + "https://github.com/magenta/magenta-realtime/tree/4bf995bdd9c29b818543574e1b3a6e67867c9a58", + "https://pypi.org/project/magenta-rt/2.0.2/" + ] + }, + { + "id": "google-mrt2-weights", + "name": "Google Magenta RealTime 2 weights and shared resources", + "family": "magenta-realtime-2", + "asset_type": "model_weights", + "support_status": "current", + "upstream": { + "project": "google/magenta-realtime-2", + "canonical_url": "https://huggingface.co/google/magenta-realtime-2" + }, + "revision": { + "kind": "model_snapshot", + "value": "010aa0dcb0dfd27b24f0ad07b4dad63e8f9521cc", + "url": "https://huggingface.co/google/magenta-realtime-2/tree/010aa0dcb0dfd27b24f0ad07b4dad63e8f9521cc" + }, + "included_artifacts": [ + "models/mrt2_base/*", + "models/mrt2_small/*", + "resources/musiccoca/*", + "resources/spectrostream/*" + ], + "licenses": { + "code": [ + { + "status": "not_applicable", + "identifier": "NONE", + "name": "Not applicable", + "scope": "Runtime code is inventoried separately", + "terms_url": null, + "notice_url": null + } + ], + "weights": [ + { + "status": "declared", + "identifier": "CC-BY-4.0", + "name": "Creative Commons Attribution 4.0 International", + "scope": "MRT2 model weights and resource models in the Google snapshot", + "terms_url": "https://creativecommons.org/licenses/by/4.0/legalcode", + "notice_url": "https://huggingface.co/google/magenta-realtime-2/blob/010aa0dcb0dfd27b24f0ad07b4dad63e8f9521cc/README.md" + } + ] + }, + "notices": { + "required_text": [ + "Attribute Google DeepMind and link the CC-BY-4.0 license and revision-specific model card.", + "Retain the model card's responsible-use statement with user-facing model notices." + ], + "attribution": [ + "Magenta RealTime 2 — Authors: Google DeepMind", + "Copyright 2026 Google LLC" + ], + "sources": [ + "https://huggingface.co/google/magenta-realtime-2/blob/010aa0dcb0dfd27b24f0ad07b4dad63e8f9521cc/README.md", + "https://creativecommons.org/licenses/by/4.0/legalcode" + ] + }, + "access": { + "gated": false, + "account_required": false, + "credential_required": false, + "terms_acceptance_required": false, + "privacy_url": null, + "acceptable_use_url": null + }, + "distribution": { + "mode": "downloaded_from_upstream", + "source_url": "https://huggingface.co/google/magenta-realtime-2/tree/010aa0dcb0dfd27b24f0ad07b4dad63e8f9521cc", + "installer_contains_asset": false, + "installer_contains_weights": false, + "redistribution_confirmed": false, + "immutable_reference_enforced": false, + "release_gate": true, + "notes": "magenta-rt 2.0.2's Hugging Face downloader omits revision=, so the current application follows mutable repository state. A release must pin and verify this snapshot before download. Model weights must remain outside installers unless exact-revision redistribution is separately confirmed." + }, + "dependencies": [], + "owner_review": { + "required": true, + "status": "pending", + "question": "Confirm the project-use and attribution path for CC-BY-4.0 weights and whether user acknowledgement is required before first download.", + "issue": "https://github.com/protocol-works/lsdj/issues/108" + }, + "evidence": [ + "https://huggingface.co/google/magenta-realtime-2/tree/010aa0dcb0dfd27b24f0ad07b4dad63e8f9521cc", + "https://github.com/magenta/magenta-realtime/blob/4bf995bdd9c29b818543574e1b3a6e67867c9a58/MODEL.md" + ] + }, + { + "id": "pytorch-mrt2-port-code", + "name": "Apolinario / multimodalart Magenta RealTime 2 PyTorch port", + "family": "magenta-realtime-2-pytorch", + "asset_type": "runtime_code", + "support_status": "reference_only", + "upstream": { + "project": "multimodalart/magenta-realtime-torch", + "canonical_url": "https://github.com/multimodalart/magenta-realtime-torch" + }, + "revision": { + "kind": "git_commit", + "value": "6d076baa3df3b10448876c400521a015a5137c59", + "url": "https://github.com/multimodalart/magenta-realtime-torch/tree/6d076baa3df3b10448876c400521a015a5137c59" + }, + "licenses": { + "code": [ + { + "status": "declared", + "identifier": "Apache-2.0", + "name": "Apache License 2.0", + "scope": "PyTorch port source code", + "terms_url": "https://github.com/multimodalart/magenta-realtime-torch/blob/6d076baa3df3b10448876c400521a015a5137c59/LICENSE", + "notice_url": null + } + ], + "weights": [ + { + "status": "not_applicable", + "identifier": "NONE", + "name": "Not applicable", + "scope": "Converted weights are inventoried separately", + "terms_url": null, + "notice_url": null + } + ] + }, + "notices": { + "required_text": [ + "Include the Apache-2.0 license with redistributed port code and preserve applicable upstream notices." + ], + "attribution": [ + "PyTorch port by multimodalart / fffiloni (Apolinario)" + ], + "sources": [ + "https://github.com/multimodalart/magenta-realtime-torch/blob/6d076baa3df3b10448876c400521a015a5137c59/LICENSE" + ] + }, + "access": { + "gated": false, + "account_required": false, + "credential_required": false, + "terms_acceptance_required": false, + "privacy_url": null, + "acceptable_use_url": null + }, + "distribution": { + "mode": "reference_only_not_acquired", + "source_url": "https://github.com/multimodalart/magenta-realtime-torch/tree/6d076baa3df3b10448876c400521a015a5137c59", + "installer_contains_asset": false, + "installer_contains_weights": false, + "redistribution_confirmed": false, + "immutable_reference_enforced": true, + "release_gate": true, + "notes": "Issue #109 found no PyTorch-specific release/tag. Production does not clone, copy, or execute this GitHub repository; it acquires the exact remote-code files carried by the separately inventoried immutable model snapshots." + }, + "dependencies": [ + "pytorch-mrt2-base-weights", + "pytorch-mrt2-small-weights", + "pytorch-musiccoca-processor", + "google-mrt2-weights" + ], + "owner_review": { + "required": true, + "status": "pending", + "question": "Confirm the Apache-2.0 notice path for the snapshot-carried remote code; the GitHub port remains a credited implementation reference and is not an acquired runtime dependency.", + "issue": "https://github.com/protocol-works/lsdj/issues/108" + }, + "evidence": [ + "https://github.com/multimodalart/magenta-realtime-torch/tree/6d076baa3df3b10448876c400521a015a5137c59", + "https://github.com/multimodalart/magenta-realtime-torch/blob/6d076baa3df3b10448876c400521a015a5137c59/LICENSE" + ] + }, + { + "id": "pytorch-mrt2-base-weights", + "name": "Magenta community MRT2 base PyTorch snapshot", + "family": "magenta-realtime-2-pytorch", + "asset_type": "model_weights_and_remote_code", + "support_status": "current_conditional", + "upstream": { + "project": "magenta-community/magenta-realtime-2", + "canonical_url": "https://huggingface.co/magenta-community/magenta-realtime-2" + }, + "revision": { + "kind": "model_snapshot", + "value": "92087988d05d0fe38b11f021f0b0d00a75afb86b", + "url": "https://huggingface.co/magenta-community/magenta-realtime-2/tree/92087988d05d0fe38b11f021f0b0d00a75afb86b" + }, + "licenses": { + "code": [ + { + "status": "declared", + "identifier": "Apache-2.0", + "name": "Apache License 2.0", + "scope": "Snapshot remote-code files, per model-card metadata", + "terms_url": "https://github.com/multimodalart/magenta-realtime-torch/blob/6d076baa3df3b10448876c400521a015a5137c59/LICENSE", + "notice_url": "https://huggingface.co/magenta-community/magenta-realtime-2/blob/92087988d05d0fe38b11f021f0b0d00a75afb86b/README.md" + } + ], + "weights": [ + { + "status": "declared", + "identifier": "Apache-2.0", + "name": "Apache License 2.0", + "scope": "Converted snapshot model-card declaration", + "terms_url": "https://huggingface.co/magenta-community/magenta-realtime-2/blob/92087988d05d0fe38b11f021f0b0d00a75afb86b/README.md", + "notice_url": "https://huggingface.co/magenta-community/magenta-realtime-2/blob/92087988d05d0fe38b11f021f0b0d00a75afb86b/README.md" + }, + { + "status": "underlying", + "identifier": "CC-BY-4.0", + "name": "Creative Commons Attribution 4.0 International", + "scope": "Google source weights; the port card says these weights are re-keyed and numerically identical", + "terms_url": "https://creativecommons.org/licenses/by/4.0/legalcode", + "notice_url": "https://huggingface.co/google/magenta-realtime-2/blob/010aa0dcb0dfd27b24f0ad07b4dad63e8f9521cc/README.md" + }, + { + "status": "unresolved", + "identifier": "NOASSERTION", + "name": "Effective converted-weight license and attribution path requires owner review", + "scope": "Conflict between Apache-2.0 model-card metadata and CC-BY-4.0 underlying weights", + "terms_url": null, + "notice_url": null + } + ] + }, + "notices": { + "required_text": [ + "Until reviewed, display both the converted snapshot's Apache-2.0 declaration and the Google source-weight CC-BY-4.0 provenance; do not imply that re-keying relicensed the weights." + ], + "attribution": [ + "Magenta RealTime 2 — Google DeepMind; PyTorch conversion by magenta-community / multimodalart" + ], + "sources": [ + "https://huggingface.co/magenta-community/magenta-realtime-2/blob/92087988d05d0fe38b11f021f0b0d00a75afb86b/README.md", + "https://huggingface.co/google/magenta-realtime-2/blob/010aa0dcb0dfd27b24f0ad07b4dad63e8f9521cc/README.md" + ] + }, + "access": { + "gated": false, + "account_required": false, + "credential_required": false, + "terms_acceptance_required": false, + "privacy_url": null, + "acceptable_use_url": null + }, + "distribution": { + "mode": "downloaded_from_upstream", + "source_url": "https://huggingface.co/magenta-community/magenta-realtime-2/tree/92087988d05d0fe38b11f021f0b0d00a75afb86b", + "installer_contains_asset": false, + "installer_contains_weights": false, + "redistribution_confirmed": false, + "immutable_reference_enforced": true, + "release_gate": true, + "notes": "The native installer downloads only the exact snapshot and verifies every file size and SHA-256 before atomic promotion. Do not bundle or mirror until the Apache model-card versus underlying CC-BY-4.0 ambiguity is explicitly resolved by project owners." + }, + "dependencies": [ + "google-mrt2-weights", + "pytorch-musiccoca-processor" + ], + "owner_review": { + "required": true, + "status": "pending", + "question": "Resolve and record the effective redistribution, attribution, and acknowledgement path for Apache-labeled re-keyed weights derived from CC-BY-4.0 Google weights.", + "issue": "https://github.com/protocol-works/lsdj/issues/108" + }, + "evidence": [ + "https://huggingface.co/magenta-community/magenta-realtime-2/tree/92087988d05d0fe38b11f021f0b0d00a75afb86b", + "https://huggingface.co/google/magenta-realtime-2/tree/010aa0dcb0dfd27b24f0ad07b4dad63e8f9521cc" + ] + }, + { + "id": "pytorch-mrt2-small-weights", + "name": "Magenta community MRT2 small PyTorch snapshot", + "family": "magenta-realtime-2-pytorch", + "asset_type": "model_weights_and_remote_code", + "support_status": "current_conditional", + "upstream": { + "project": "magenta-community/magenta-realtime-2-small", + "canonical_url": "https://huggingface.co/magenta-community/magenta-realtime-2-small" + }, + "revision": { + "kind": "model_snapshot", + "value": "7037d99551c84ac5c6afb7f1a5e58c65e7233dbb", + "url": "https://huggingface.co/magenta-community/magenta-realtime-2-small/tree/7037d99551c84ac5c6afb7f1a5e58c65e7233dbb" + }, + "licenses": { + "code": [ + { + "status": "declared", + "identifier": "Apache-2.0", + "name": "Apache License 2.0", + "scope": "Snapshot remote-code files, per model-card metadata", + "terms_url": "https://github.com/multimodalart/magenta-realtime-torch/blob/6d076baa3df3b10448876c400521a015a5137c59/LICENSE", + "notice_url": "https://huggingface.co/magenta-community/magenta-realtime-2-small/blob/7037d99551c84ac5c6afb7f1a5e58c65e7233dbb/README.md" + } + ], + "weights": [ + { + "status": "declared", + "identifier": "Apache-2.0", + "name": "Apache License 2.0", + "scope": "Converted snapshot model-card declaration", + "terms_url": "https://huggingface.co/magenta-community/magenta-realtime-2-small/blob/7037d99551c84ac5c6afb7f1a5e58c65e7233dbb/README.md", + "notice_url": "https://huggingface.co/magenta-community/magenta-realtime-2-small/blob/7037d99551c84ac5c6afb7f1a5e58c65e7233dbb/README.md" + }, + { + "status": "underlying", + "identifier": "CC-BY-4.0", + "name": "Creative Commons Attribution 4.0 International", + "scope": "Google source weights; the port card says these weights are re-keyed and numerically identical", + "terms_url": "https://creativecommons.org/licenses/by/4.0/legalcode", + "notice_url": "https://huggingface.co/google/magenta-realtime-2/blob/010aa0dcb0dfd27b24f0ad07b4dad63e8f9521cc/README.md" + }, + { + "status": "unresolved", + "identifier": "NOASSERTION", + "name": "Effective converted-weight license and attribution path requires owner review", + "scope": "Conflict between Apache-2.0 model-card metadata and CC-BY-4.0 underlying weights", + "terms_url": null, + "notice_url": null + } + ] + }, + "notices": { + "required_text": [ + "Until reviewed, display both the converted snapshot's Apache-2.0 declaration and the Google source-weight CC-BY-4.0 provenance; do not imply that re-keying relicensed the weights." + ], + "attribution": [ + "Magenta RealTime 2 — Google DeepMind; PyTorch conversion by magenta-community / multimodalart" + ], + "sources": [ + "https://huggingface.co/magenta-community/magenta-realtime-2-small/blob/7037d99551c84ac5c6afb7f1a5e58c65e7233dbb/README.md", + "https://huggingface.co/google/magenta-realtime-2/blob/010aa0dcb0dfd27b24f0ad07b4dad63e8f9521cc/README.md" + ] + }, + "access": { + "gated": false, + "account_required": false, + "credential_required": false, + "terms_acceptance_required": false, + "privacy_url": null, + "acceptable_use_url": null + }, + "distribution": { + "mode": "downloaded_from_upstream", + "source_url": "https://huggingface.co/magenta-community/magenta-realtime-2-small/tree/7037d99551c84ac5c6afb7f1a5e58c65e7233dbb", + "installer_contains_asset": false, + "installer_contains_weights": false, + "redistribution_confirmed": false, + "immutable_reference_enforced": true, + "release_gate": true, + "notes": "The native installer downloads only the exact snapshot and verifies every file size and SHA-256 before atomic promotion. Do not bundle or mirror until the Apache model-card versus underlying CC-BY-4.0 ambiguity is explicitly resolved by project owners." + }, + "dependencies": [ + "google-mrt2-weights", + "pytorch-musiccoca-processor" + ], + "owner_review": { + "required": true, + "status": "pending", + "question": "Resolve and record the effective redistribution, attribution, and acknowledgement path for Apache-labeled re-keyed weights derived from CC-BY-4.0 Google weights.", + "issue": "https://github.com/protocol-works/lsdj/issues/108" + }, + "evidence": [ + "https://huggingface.co/magenta-community/magenta-realtime-2-small/tree/7037d99551c84ac5c6afb7f1a5e58c65e7233dbb", + "https://huggingface.co/google/magenta-realtime-2/tree/010aa0dcb0dfd27b24f0ad07b4dad63e8f9521cc" + ] + }, + { + "id": "pytorch-musiccoca-processor", + "name": "Magenta community MusicCoCa PyTorch processor", + "family": "magenta-realtime-2-pytorch", + "asset_type": "processor_weights", + "support_status": "current_conditional", + "upstream": { + "project": "magenta-community/magenta-rt-musiccoca-torch", + "canonical_url": "https://huggingface.co/magenta-community/magenta-rt-musiccoca-torch" + }, + "revision": { + "kind": "model_snapshot", + "value": "236c488e38aa98643805514996934d705668298b", + "url": "https://huggingface.co/magenta-community/magenta-rt-musiccoca-torch/tree/236c488e38aa98643805514996934d705668298b" + }, + "licenses": { + "code": [ + { + "status": "not_applicable", + "identifier": "NONE", + "name": "Not separately declared", + "scope": "This inventory treats the repository as processor artifacts; conversion scripts need owner review if bundled", + "terms_url": null, + "notice_url": null + } + ], + "weights": [ + { + "status": "declared", + "identifier": "CC-BY-4.0", + "name": "Creative Commons Attribution 4.0 International", + "scope": "Converted MusicCoCa text encoder, quantizer, and tokenizer artifacts", + "terms_url": "https://creativecommons.org/licenses/by/4.0/legalcode", + "notice_url": "https://huggingface.co/magenta-community/magenta-rt-musiccoca-torch/blob/236c488e38aa98643805514996934d705668298b/README.md" + } + ] + }, + "notices": { + "required_text": [ + "Attribute the Google MusicCoCa source and the magenta-community conversion; link CC-BY-4.0 and the exact processor card." + ], + "attribution": [ + "MusicCoCa from Google Magenta RealTime 2; PyTorch conversion by magenta-community" + ], + "sources": [ + "https://huggingface.co/magenta-community/magenta-rt-musiccoca-torch/blob/236c488e38aa98643805514996934d705668298b/README.md" + ] + }, + "access": { + "gated": false, + "account_required": false, + "credential_required": false, + "terms_acceptance_required": false, + "privacy_url": null, + "acceptable_use_url": null + }, + "distribution": { + "mode": "downloaded_from_upstream", + "source_url": "https://huggingface.co/magenta-community/magenta-rt-musiccoca-torch/tree/236c488e38aa98643805514996934d705668298b", + "installer_contains_asset": false, + "installer_contains_weights": false, + "redistribution_confirmed": false, + "immutable_reference_enforced": true, + "release_gate": true, + "notes": "The native installer overrides the port's mutable default, downloads this exact snapshot, verifies every pinned file, and resolves it locally with offline mode enabled." + }, + "dependencies": [ + "google-mrt2-weights" + ], + "owner_review": { + "required": true, + "status": "pending", + "question": "Confirm attribution and packaging rules for the CC-BY-4.0 converted processor artifacts and any bundled conversion code.", + "issue": "https://github.com/protocol-works/lsdj/issues/108" + }, + "evidence": [ + "https://huggingface.co/magenta-community/magenta-rt-musiccoca-torch/tree/236c488e38aa98643805514996934d705668298b" + ] + }, + { + "id": "stable-audio-3-code", + "name": "Stable Audio 3 inference code", + "family": "stable-audio-3", + "asset_type": "runtime_code", + "support_status": "current_and_planned", + "upstream": { + "project": "Stability-AI/stable-audio-3", + "canonical_url": "https://github.com/Stability-AI/stable-audio-3" + }, + "revision": { + "kind": "git_commit", + "value": "a0b57f5483c4588f827f3552b7d5c6ca2a9687be", + "url": "https://github.com/Stability-AI/stable-audio-3/tree/a0b57f5483c4588f827f3552b7d5c6ca2a9687be" + }, + "licenses": { + "code": [ + { + "status": "declared", + "identifier": "MIT", + "name": "MIT License", + "scope": "Stable Audio 3 source, including the MLX and TFLite runtime code at the pinned commit", + "terms_url": "https://github.com/Stability-AI/stable-audio-3/blob/a0b57f5483c4588f827f3552b7d5c6ca2a9687be/LICENSE", + "notice_url": "https://github.com/Stability-AI/stable-audio-3/blob/a0b57f5483c4588f827f3552b7d5c6ca2a9687be/LICENSE" + } + ], + "weights": [ + { + "status": "not_applicable", + "identifier": "NONE", + "name": "Not applicable", + "scope": "Weights are inventoried separately", + "terms_url": null, + "notice_url": null + } + ] + }, + "notices": { + "required_text": [ + "Include the MIT copyright and permission notice in copies or substantial portions of the Stable Audio 3 software." + ], + "attribution": [ + "Copyright (c) 2026 Stability AI" + ], + "sources": [ + "https://github.com/Stability-AI/stable-audio-3/blob/a0b57f5483c4588f827f3552b7d5c6ca2a9687be/LICENSE" + ] + }, + "access": { + "gated": false, + "account_required": false, + "credential_required": false, + "terms_acceptance_required": false, + "privacy_url": null, + "acceptable_use_url": null + }, + "distribution": { + "mode": "downloaded_from_upstream", + "source_url": "https://github.com/Stability-AI/stable-audio-3/archive/a0b57f5483c4588f827f3552b7d5c6ca2a9687be.tar.gz", + "installer_contains_asset": false, + "installer_contains_weights": false, + "redistribution_confirmed": false, + "immutable_reference_enforced": true, + "release_gate": true, + "notes": "The current model manager downloads the exact source archive recorded in sa3-pin.json. Packaged notices still need owner confirmation." + }, + "dependencies": [ + "stable-audio-3-optimized-weights" + ], + "owner_review": { + "required": true, + "status": "pending", + "question": "Confirm the MIT notice is present in source, About/Licenses, and any package that includes or downloads this runtime.", + "issue": "https://github.com/protocol-works/lsdj/issues/108" + }, + "evidence": [ + "https://github.com/Stability-AI/stable-audio-3/tree/a0b57f5483c4588f827f3552b7d5c6ca2a9687be", + "https://github.com/Stability-AI/stable-audio-3/blob/a0b57f5483c4588f827f3552b7d5c6ca2a9687be/LICENSE" + ] + }, + { + "id": "stable-audio-3-optimized-weights", + "name": "Stable Audio 3 optimized MLX and TFLite weights", + "family": "stable-audio-3", + "asset_type": "model_weights", + "support_status": "current_and_planned", + "upstream": { + "project": "stabilityai/stable-audio-3-optimized", + "canonical_url": "https://huggingface.co/stabilityai/stable-audio-3-optimized" + }, + "revision": { + "kind": "model_snapshot", + "value": "6736003cb57d06b7b1fdc36fad31b2a3709e4774", + "url": "https://huggingface.co/stabilityai/stable-audio-3-optimized/tree/6736003cb57d06b7b1fdc36fad31b2a3709e4774" + }, + "included_artifacts": [ + "MLX/dit_sm-music_f16.npz", + "MLX/dit_sm-sfx_f16.npz", + "MLX/dit_medium_f16.npz", + "MLX/same_s_encoder_f32.npz", + "MLX/same_s_decoder_f32.npz", + "MLX/same_l_encoder_f32.npz", + "MLX/same_l_decoder_f32.npz", + "MLX/t5gemma_f16.npz", + "tflite/sa3-sm-music/*", + "tflite/sa3-sm-sfx/*", + "tflite/sa3-m/*", + "tflite/same-s/*", + "tflite/same-l/*", + "tflite/t5gemma/encoder_fp16.tflite" + ], + "licenses": { + "code": [ + { + "status": "not_applicable", + "identifier": "NONE", + "name": "Not applicable", + "scope": "Runtime code is inventoried separately", + "terms_url": null, + "notice_url": null + } + ], + "weights": [ + { + "status": "underlying", + "identifier": "LicenseRef-Stability-AI-Community", + "name": "Stability AI Community License", + "scope": "Stable Audio 3 model, codec, conditioner, and optimized derivative weights", + "terms_url": "https://huggingface.co/stabilityai/stable-audio-3-optimized/blob/6736003cb57d06b7b1fdc36fad31b2a3709e4774/LICENSE.md", + "notice_url": "https://huggingface.co/stabilityai/stable-audio-3-optimized/blob/6736003cb57d06b7b1fdc36fad31b2a3709e4774/NOTICE" + }, + { + "status": "underlying", + "identifier": "LicenseRef-Gemma-Terms", + "name": "Gemma Terms of Use", + "scope": "Redistributed T5Gemma-derived encoder weights and tokenizer components inside the optimized snapshot", + "terms_url": "https://huggingface.co/stabilityai/stable-audio-3-optimized/blob/6736003cb57d06b7b1fdc36fad31b2a3709e4774/LICENSE_GEMMA.md", + "notice_url": "https://huggingface.co/stabilityai/stable-audio-3-optimized/blob/6736003cb57d06b7b1fdc36fad31b2a3709e4774/NOTICE" + } + ] + }, + "notices": { + "required_text": [ + "This Stability AI Model is licensed under the Stability AI Community License, Copyright © Stability AI Ltd. All Rights Reserved", + "Gemma is provided under and subject to the Gemma Terms of Use found at ai.google.dev/gemma/terms", + "Display “Powered by Stability AI” in an applicable user-facing or documentation surface." + ], + "attribution": [ + "Stable Audio 3 by Stability AI", + "T5Gemma text encoder from Google" + ], + "sources": [ + "https://huggingface.co/stabilityai/stable-audio-3-optimized/blob/6736003cb57d06b7b1fdc36fad31b2a3709e4774/LICENSE.md", + "https://huggingface.co/stabilityai/stable-audio-3-optimized/blob/6736003cb57d06b7b1fdc36fad31b2a3709e4774/LICENSE_GEMMA.md", + "https://huggingface.co/stabilityai/stable-audio-3-optimized/blob/6736003cb57d06b7b1fdc36fad31b2a3709e4774/NOTICE" + ] + }, + "access": { + "gated": false, + "account_required": false, + "credential_required": false, + "terms_acceptance_required": true, + "privacy_url": "https://stability.ai/privacy-policy", + "acceptable_use_url": "https://stability.ai/use-policy" + }, + "distribution": { + "mode": "downloaded_from_upstream", + "source_url": "https://huggingface.co/stabilityai/stable-audio-3-optimized/tree/6736003cb57d06b7b1fdc36fad31b2a3709e4774", + "installer_contains_asset": false, + "installer_contains_weights": false, + "redistribution_confirmed": false, + "immutable_reference_enforced": false, + "release_gate": true, + "notes": "The pinned runtime's weights.py calls hf_hub_download without revision=. Current downloads therefore follow mutable repository state even though this audit records a snapshot. Pin and verify before release. Non-gated anonymous access does not remove Stability Community or Gemma obligations. Do not put these weights in installers without exact-revision confirmation." + }, + "dependencies": [ + "t5gemma-b-b-ul2" + ], + "owner_review": { + "required": true, + "status": "pending", + "question": "Confirm the open-source/non-commercial Stability Community path, Gemma derivative distribution path, exact notices, first-download acknowledgement, and no-bundle policy for this snapshot.", + "issue": "https://github.com/protocol-works/lsdj/issues/108" + }, + "evidence": [ + "https://huggingface.co/stabilityai/stable-audio-3-optimized/tree/6736003cb57d06b7b1fdc36fad31b2a3709e4774", + "https://github.com/Stability-AI/stable-audio-3/blob/0385302ea26522f00c80392c4b708df5ebf1adf5/optimized/mlx/scripts/weights.py", + "https://github.com/Stability-AI/stable-audio-3/blob/0385302ea26522f00c80392c4b708df5ebf1adf5/optimized/tflite/scripts/weights.py" + ] + }, + { + "id": "stable-audio-3-small-music-cuda-weights", + "name": "Stable Audio 3 Small Music PyTorch/CUDA weights", + "family": "stable-audio-3", + "asset_type": "model_weights", + "support_status": "optional_fail_closed", + "upstream": { + "project": "stabilityai/stable-audio-3-small-music", + "canonical_url": "https://huggingface.co/stabilityai/stable-audio-3-small-music" + }, + "revision": { + "kind": "model_snapshot", + "value": "0fef1392cd842149a2b6d445e181c97608faac06", + "url": "https://huggingface.co/stabilityai/stable-audio-3-small-music/tree/0fef1392cd842149a2b6d445e181c97608faac06" + }, + "included_artifacts": [ + "model.safetensors", + "model_config.json" + ], + "artifact_integrity": { + "status": "incomplete", + "weight_size": 2270384940, + "weight_sha256": "da85866b11b01d0694d990785f6abbd79c8064df1b0e6f8aea52935e0ef84b64", + "config_size": 10341, + "config_sha256": null, + "notes": "Public metadata supplied the root weight hash, but the gated authenticated audit has not supplied the configuration and nested-component SHA-256 set." + }, + "licenses": { + "code": [ + { + "status": "not_applicable", + "identifier": "NONE", + "name": "Not applicable", + "scope": "Runtime code is inventoried separately", + "terms_url": null, + "notice_url": null + } + ], + "weights": [ + { + "status": "unresolved", + "identifier": "NOASSERTION", + "name": "Authenticated gated-model terms audit pending", + "scope": "Small Music checkpoint, configuration, and any nested model components at the pinned snapshot", + "terms_url": null, + "notice_url": null + } + ] + }, + "notices": { + "required_text": [ + "Do not enable acquisition or distribution until the authenticated issue #108 review records the exact terms, notices, and nested component obligations for this snapshot." + ], + "attribution": [ + "Stable Audio 3 Small Music by Stability AI" + ], + "sources": [ + "https://huggingface.co/stabilityai/stable-audio-3-small-music/tree/0fef1392cd842149a2b6d445e181c97608faac06" + ] + }, + "access": { + "gated": true, + "account_required": true, + "credential_required": true, + "terms_acceptance_required": true, + "privacy_url": "https://huggingface.co/privacy", + "acceptable_use_url": "https://stability.ai/use-policy" + }, + "distribution": { + "mode": "optional_gated_download", + "source_url": "https://huggingface.co/stabilityai/stable-audio-3-small-music/tree/0fef1392cd842149a2b6d445e181c97608faac06", + "installer_contains_asset": false, + "installer_contains_weights": false, + "redistribution_confirmed": false, + "immutable_reference_enforced": true, + "release_gate": true, + "notes": "The optional Windows CUDA manifest pins this revision and root weight hash, but releaseReady and gatedArtifactsComplete remain false. The native installer rejects the candidate while the config hash, nested artifacts, terms review, VRAM reservation, and hardware qualification are incomplete. TFLite remains the supported path." + }, + "dependencies": [ + "stable-audio-3-code", + "t5gemma-b-b-ul2" + ], + "owner_review": { + "required": true, + "status": "pending", + "question": "Complete the authenticated terms, notice, nested-artifact, and exact-hash audit for the pinned Small Music snapshot before enabling its optional download.", + "issue": "https://github.com/protocol-works/lsdj/issues/108" + }, + "evidence": [ + "https://huggingface.co/stabilityai/stable-audio-3-small-music/tree/0fef1392cd842149a2b6d445e181c97608faac06", + "https://huggingface.co/stabilityai/stable-audio-3-small-music/blob/0fef1392cd842149a2b6d445e181c97608faac06/model_config.json" + ] + }, + { + "id": "stable-audio-3-small-sfx-cuda-weights", + "name": "Stable Audio 3 Small SFX PyTorch/CUDA weights", + "family": "stable-audio-3", + "asset_type": "model_weights", + "support_status": "optional_fail_closed", + "upstream": { + "project": "stabilityai/stable-audio-3-small-sfx", + "canonical_url": "https://huggingface.co/stabilityai/stable-audio-3-small-sfx" + }, + "revision": { + "kind": "model_snapshot", + "value": "ae12755283df9d62ca39a9b050a39a0b607b8c20", + "url": "https://huggingface.co/stabilityai/stable-audio-3-small-sfx/tree/ae12755283df9d62ca39a9b050a39a0b607b8c20" + }, + "included_artifacts": [ + "model.safetensors", + "model_config.json" + ], + "artifact_integrity": { + "status": "incomplete", + "weight_size": 2270384940, + "weight_sha256": "ed9cf1b6172f1a8c2921a9560c21109ff3239524563ced9dce6dcdef41e2f515", + "config_size": 10454, + "config_sha256": null, + "notes": "Public metadata supplied the root weight hash, but the gated authenticated audit has not supplied the configuration and nested-component SHA-256 set." + }, + "licenses": { + "code": [ + { + "status": "not_applicable", + "identifier": "NONE", + "name": "Not applicable", + "scope": "Runtime code is inventoried separately", + "terms_url": null, + "notice_url": null + } + ], + "weights": [ + { + "status": "unresolved", + "identifier": "NOASSERTION", + "name": "Authenticated gated-model terms audit pending", + "scope": "Small SFX checkpoint, configuration, and any nested model components at the pinned snapshot", + "terms_url": null, + "notice_url": null + } + ] + }, + "notices": { + "required_text": [ + "Do not enable acquisition or distribution until the authenticated issue #108 review records the exact terms, notices, and nested component obligations for this snapshot." + ], + "attribution": [ + "Stable Audio 3 Small SFX by Stability AI" + ], + "sources": [ + "https://huggingface.co/stabilityai/stable-audio-3-small-sfx/tree/ae12755283df9d62ca39a9b050a39a0b607b8c20" + ] + }, + "access": { + "gated": true, + "account_required": true, + "credential_required": true, + "terms_acceptance_required": true, + "privacy_url": "https://huggingface.co/privacy", + "acceptable_use_url": "https://stability.ai/use-policy" + }, + "distribution": { + "mode": "optional_gated_download", + "source_url": "https://huggingface.co/stabilityai/stable-audio-3-small-sfx/tree/ae12755283df9d62ca39a9b050a39a0b607b8c20", + "installer_contains_asset": false, + "installer_contains_weights": false, + "redistribution_confirmed": false, + "immutable_reference_enforced": true, + "release_gate": true, + "notes": "The optional Windows CUDA manifest pins this revision and root weight hash, but releaseReady and gatedArtifactsComplete remain false. The native installer rejects the candidate while the config hash, nested artifacts, terms review, VRAM reservation, and hardware qualification are incomplete. TFLite remains the supported path." + }, + "dependencies": [ + "stable-audio-3-code", + "t5gemma-b-b-ul2" + ], + "owner_review": { + "required": true, + "status": "pending", + "question": "Complete the authenticated terms, notice, nested-artifact, and exact-hash audit for the pinned Small SFX snapshot before enabling its optional download.", + "issue": "https://github.com/protocol-works/lsdj/issues/108" + }, + "evidence": [ + "https://huggingface.co/stabilityai/stable-audio-3-small-sfx/tree/ae12755283df9d62ca39a9b050a39a0b607b8c20", + "https://huggingface.co/stabilityai/stable-audio-3-small-sfx/blob/ae12755283df9d62ca39a9b050a39a0b607b8c20/model_config.json" + ] + }, + { + "id": "t5gemma-b-b-ul2", + "name": "Google T5Gemma B-B UL2", + "family": "t5gemma", + "asset_type": "model_weights_and_tokenizer", + "support_status": "underlying_component", + "upstream": { + "project": "google/t5gemma-b-b-ul2", + "canonical_url": "https://huggingface.co/google/t5gemma-b-b-ul2" + }, + "revision": { + "kind": "unresolved_upstream", + "value": null, + "url": "https://huggingface.co/google/t5gemma-b-b-ul2" + }, + "licenses": { + "code": [ + { + "status": "not_applicable", + "identifier": "NONE", + "name": "Not applicable to the derived encoder artifact tracked here", + "scope": "T5Gemma model and tokenizer assets", + "terms_url": null, + "notice_url": null + } + ], + "weights": [ + { + "status": "declared", + "identifier": "LicenseRef-Gemma-Terms", + "name": "Gemma Terms of Use", + "scope": "T5Gemma model weights, tokenizer, and model derivatives", + "terms_url": "https://ai.google.dev/gemma/terms", + "notice_url": "https://huggingface.co/stabilityai/stable-audio-3-optimized/blob/6736003cb57d06b7b1fdc36fad31b2a3709e4774/LICENSE_GEMMA.md" + } + ] + }, + "notices": { + "required_text": [ + "Gemma is provided under and subject to the Gemma Terms of Use found at ai.google.dev/gemma/terms", + "Provide the Gemma terms and incorporated prohibited-use policy when distributing Gemma or a model derivative, subject to owner review of the exact path." + ], + "attribution": [ + "T5Gemma by Google" + ], + "sources": [ + "https://ai.google.dev/gemma/terms", + "https://ai.google.dev/gemma/prohibited_use_policy" + ] + }, + "access": { + "gated": true, + "account_required": true, + "credential_required": true, + "terms_acceptance_required": true, + "privacy_url": "https://policies.google.com/privacy", + "acceptable_use_url": "https://ai.google.dev/gemma/prohibited_use_policy" + }, + "distribution": { + "mode": "underlying_component_in_upstream_download", + "source_url": "https://huggingface.co/google/t5gemma-b-b-ul2", + "installer_contains_asset": false, + "installer_contains_weights": false, + "redistribution_confirmed": false, + "immutable_reference_enforced": false, + "release_gate": true, + "notes": "The exact T5Gemma source revision used to produce Stability's optimized derivative is not identified by the pinned optimized repository and remains unresolved. Direct Hugging Face access is manually gated and requires an account, credentials, and terms acceptance. LSDJ currently receives the derivative inside Stability AI's non-gated snapshot; that convenience does not remove the Gemma terms. No credential should be required for the optimized source, and no token may be logged or stored in plaintext app data." + }, + "dependencies": [], + "owner_review": { + "required": true, + "status": "pending", + "question": "Identify or confirm that no exact T5Gemma source revision is available, then confirm the Gemma model-derivative distribution and acknowledgement path for T5Gemma embedded in Stable Audio optimized weights.", + "issue": "https://github.com/protocol-works/lsdj/issues/108" + }, + "evidence": [ + "https://huggingface.co/google/t5gemma-b-b-ul2", + "https://huggingface.co/stabilityai/stable-audio-3-optimized/blob/6736003cb57d06b7b1fdc36fad31b2a3709e4774/LICENSE_GEMMA.md" + ] + }, + { + "id": "stable-audio-3-medium-source-weights", + "name": "Stable Audio 3 Medium upstream source family", + "family": "stable-audio-3", + "asset_type": "model_weights_provenance", + "support_status": "provenance_only", + "upstream": { + "project": "stabilityai/stable-audio-3-medium", + "canonical_url": "https://huggingface.co/stabilityai/stable-audio-3-medium" + }, + "revision": { + "kind": "unresolved_upstream", + "value": null, + "url": "https://huggingface.co/stabilityai/stable-audio-3-medium" + }, + "licenses": { + "code": [ + { + "status": "not_applicable", + "identifier": "NONE", + "name": "Not applicable", + "scope": "Model checkpoint provenance entry", + "terms_url": null, + "notice_url": null + } + ], + "weights": [ + { + "status": "declared", + "identifier": "LicenseRef-Stability-AI-Community", + "name": "Stability AI Community License", + "scope": "Base-family terms asserted by the pinned optimized model and relevant to the Maqam LoRA; exact conversion-source revision unresolved", + "terms_url": "https://huggingface.co/stabilityai/stable-audio-3-optimized/blob/6736003cb57d06b7b1fdc36fad31b2a3709e4774/LICENSE.md", + "notice_url": "https://huggingface.co/stabilityai/stable-audio-3-optimized/blob/6736003cb57d06b7b1fdc36fad31b2a3709e4774/NOTICE" + }, + { + "status": "underlying", + "identifier": "LicenseRef-Gemma-Terms", + "name": "Gemma Terms of Use", + "scope": "T5Gemma component terms carried by the pinned optimized snapshot", + "terms_url": "https://huggingface.co/stabilityai/stable-audio-3-optimized/blob/6736003cb57d06b7b1fdc36fad31b2a3709e4774/LICENSE_GEMMA.md", + "notice_url": "https://huggingface.co/stabilityai/stable-audio-3-optimized/blob/6736003cb57d06b7b1fdc36fad31b2a3709e4774/NOTICE" + } + ] + }, + "notices": { + "required_text": [ + "Show the Stability AI Community License, Stability privacy link, Gemma terms, and incorporated use restrictions before any direct gated download." + ], + "attribution": [ + "Stable Audio 3 by Stability AI; T5Gemma component by Google" + ], + "sources": [ + "https://huggingface.co/stabilityai/stable-audio-3-optimized/blob/6736003cb57d06b7b1fdc36fad31b2a3709e4774/README.md" + ] + }, + "access": { + "gated": true, + "account_required": true, + "credential_required": true, + "terms_acceptance_required": true, + "privacy_url": "https://stability.ai/privacy-policy", + "acceptable_use_url": "https://stability.ai/use-policy" + }, + "distribution": { + "mode": "not_directly_downloaded_by_lsdj", + "source_url": "https://huggingface.co/stabilityai/stable-audio-3-medium", + "installer_contains_asset": false, + "installer_contains_weights": false, + "redistribution_confirmed": false, + "immutable_reference_enforced": false, + "release_gate": true, + "notes": "The optimized model and Maqam LoRA name this upstream model family, but the reviewed evidence does not identify the exact source revision used for conversion or training. LSDJ downloads the separately pinned optimized snapshot instead." + }, + "dependencies": [ + "t5gemma-b-b-ul2" + ], + "owner_review": { + "required": true, + "status": "pending", + "question": "Identify or confirm that no exact Stable Audio 3 Medium source revision is available, then confirm that optimized weights and LoRA references carry the applicable base-model terms and notices.", + "issue": "https://github.com/protocol-works/lsdj/issues/108" + }, + "evidence": [ + "https://huggingface.co/stabilityai/stable-audio-3-medium", + "https://huggingface.co/stabilityai/stable-audio-3-optimized/blob/6736003cb57d06b7b1fdc36fad31b2a3709e4774/README.md" + ] + }, + { + "id": "motif-maqam-lora", + "name": "Motif Technologies Stable Audio 3 Maqam LoRA", + "family": "stable-audio-3-lora", + "asset_type": "lora", + "support_status": "documented_reference", + "upstream": { + "project": "motiftechnologies/stable-audio-3-maqam-lora", + "canonical_url": "https://huggingface.co/motiftechnologies/stable-audio-3-maqam-lora" + }, + "revision": { + "kind": "model_snapshot", + "value": "3e1d9aa6fcb72a619b4ced00a240c5039f76daf0", + "url": "https://huggingface.co/motiftechnologies/stable-audio-3-maqam-lora/tree/3e1d9aa6fcb72a619b4ced00a240c5039f76daf0" + }, + "licenses": { + "code": [ + { + "status": "not_applicable", + "identifier": "NONE", + "name": "Not applicable", + "scope": "Adapter weights and metadata only", + "terms_url": null, + "notice_url": null + } + ], + "weights": [ + { + "status": "unresolved", + "identifier": "NOASSERTION", + "name": "The model card says license: other but supplies no license text or redistribution grant", + "scope": "adapter_model.safetensors and adapter metadata", + "terms_url": null, + "notice_url": "https://huggingface.co/motiftechnologies/stable-audio-3-maqam-lora/blob/3e1d9aa6fcb72a619b4ced00a240c5039f76daf0/README.md" + }, + { + "status": "underlying", + "identifier": "LicenseRef-Stability-AI-Community", + "name": "Stability AI Community License", + "scope": "Stable Audio 3 Medium base-model obligations; exact LoRA applicability is an owner-review question", + "terms_url": "https://huggingface.co/stabilityai/stable-audio-3-medium/blob/27b5a21b791b1b033d193a9e1e3ce78493f102f9/LICENSE.md", + "notice_url": "https://huggingface.co/stabilityai/stable-audio-3-medium/blob/27b5a21b791b1b033d193a9e1e3ce78493f102f9/NOTICE" + } + ] + }, + "notices": { + "required_text": [ + "Do not call this a built-in or Stability-AI-official adapter and do not mirror or redistribute it until the adapter author supplies or confirms a license.", + "For a user-initiated upstream download, show adapter provenance, exact revision, unresolved license status, and the Stable Audio 3 Medium base-model terms." + ], + "attribution": [ + "Fine-tuned by Motif Technologies; base model Stable Audio 3 Medium by Stability AI" + ], + "sources": [ + "https://huggingface.co/motiftechnologies/stable-audio-3-maqam-lora/blob/3e1d9aa6fcb72a619b4ced00a240c5039f76daf0/README.md" + ] + }, + "access": { + "gated": false, + "account_required": false, + "credential_required": false, + "terms_acceptance_required": true, + "privacy_url": "https://huggingface.co/privacy", + "acceptable_use_url": "https://stability.ai/use-policy" + }, + "distribution": { + "mode": "user_initiated_download_from_upstream", + "source_url": "https://huggingface.co/motiftechnologies/stable-audio-3-maqam-lora/tree/3e1d9aa6fcb72a619b4ced00a240c5039f76daf0", + "installer_contains_asset": false, + "installer_contains_weights": false, + "redistribution_confirmed": false, + "immutable_reference_enforced": false, + "release_gate": true, + "notes": "The current importer fetches resolve/main and therefore does not enforce this audited snapshot. Pin a revision before treating this as an LSDJ-provided download. The generic user-import flow remains separate and must show a responsibility/provenance notice." + }, + "dependencies": [ + "stable-audio-3-medium-source-weights" + ], + "owner_review": { + "required": true, + "status": "pending", + "question": "Obtain or identify the adapter's license and specific download/mirroring permission; until then, keep it a documented user-directed upstream reference only.", + "issue": "https://github.com/protocol-works/lsdj/issues/108" + }, + "evidence": [ + "https://huggingface.co/motiftechnologies/stable-audio-3-maqam-lora/tree/3e1d9aa6fcb72a619b4ced00a240c5039f76daf0" + ] + } + ] +} diff --git a/compliance/test_inventory.py b/compliance/test_inventory.py new file mode 100644 index 0000000..83a4aa3 --- /dev/null +++ b/compliance/test_inventory.py @@ -0,0 +1,231 @@ +from __future__ import annotations + +import copy +import json +import unittest +from pathlib import Path + +from compliance.validate_inventory import load, validate + + +MANIFEST = Path(__file__).with_name("model-assets.json") + + +class InventoryValidationTests(unittest.TestCase): + def setUp(self) -> None: + self.data = load(MANIFEST) + + def test_repository_inventory_is_valid(self) -> None: + self.assertEqual(validate(self.data), []) + + def test_runtime_acquisition_pins_match_compliance_revisions(self) -> None: + root = MANIFEST.parent.parent + mrt2_pin = json.loads((root / "mrt2-pytorch-pin.json").read_text()) + sa3_pin = json.loads((root / "sa3-pin.json").read_text()) + sa3_cuda_pin = json.loads( + (root / "sa3-pytorch-cuda-pin.json").read_text() + ) + assets = {asset["id"]: asset for asset in self.data["assets"]} + + self.assertEqual( + assets["pytorch-mrt2-base-weights"]["revision"]["value"], + mrt2_pin["models"]["mrt2_base"]["revision"], + ) + self.assertEqual( + assets["pytorch-mrt2-small-weights"]["revision"]["value"], + mrt2_pin["models"]["mrt2_small"]["revision"], + ) + self.assertEqual( + assets["pytorch-musiccoca-processor"]["revision"]["value"], + mrt2_pin["processor"]["revision"], + ) + self.assertEqual( + assets["stable-audio-3-code"]["revision"]["value"], + sa3_pin["commit"], + ) + self.assertEqual( + assets["stable-audio-3-small-music-cuda-weights"]["revision"]["value"], + sa3_cuda_pin["models"]["small-music"]["revision"], + ) + self.assertEqual( + assets["stable-audio-3-small-sfx-cuda-weights"]["revision"]["value"], + sa3_cuda_pin["models"]["small-sfx"]["revision"], + ) + self.assertEqual( + assets["stable-audio-3-small-music-cuda-weights"]["artifact_integrity"][ + "weight_sha256" + ], + sa3_cuda_pin["models"]["small-music"]["weight"]["sha256"], + ) + self.assertEqual( + assets["stable-audio-3-small-sfx-cuda-weights"]["artifact_integrity"][ + "weight_sha256" + ], + sa3_cuda_pin["models"]["small-sfx"]["weight"]["sha256"], + ) + self.assertIsNone( + assets["stable-audio-3-small-music-cuda-weights"]["artifact_integrity"][ + "config_sha256" + ] + ) + self.assertIsNone( + assets["stable-audio-3-small-sfx-cuda-weights"]["artifact_integrity"][ + "config_sha256" + ] + ) + self.assertEqual( + assets["pytorch-mrt2-port-code"]["distribution"]["mode"], + "reference_only_not_acquired", + ) + + def test_missing_required_asset_field_is_rejected(self) -> None: + broken = copy.deepcopy(self.data) + del broken["assets"][0]["notices"] + self.assertTrue( + any("missing field 'notices'" in error for error in validate(broken)) + ) + + def test_mutable_revision_is_rejected(self) -> None: + broken = copy.deepcopy(self.data) + broken["assets"][0]["revision"] = { + "kind": "git_commit", + "value": "main", + "url": "https://github.com/protocol-works/lsdj/tree/main", + } + errors = validate(broken) + self.assertTrue(any("40-character hash" in error for error in errors)) + self.assertTrue(any("mutable branch" in error for error in errors)) + + def test_revision_url_must_name_exact_revision(self) -> None: + broken = copy.deepcopy(self.data) + broken["assets"][0]["revision"]["url"] = ( + "https://github.com/protocol-works/lsdj" + ) + self.assertTrue( + any( + "must contain the exact revision" in error for error in validate(broken) + ) + ) + + def test_unresolved_upstream_revision_is_explicit_and_gated(self) -> None: + broken = copy.deepcopy(self.data) + asset = next( + item for item in broken["assets"] if item["id"] == "t5gemma-b-b-ul2" + ) + asset["distribution"]["release_gate"] = False + errors = validate(broken) + self.assertTrue(any("must remain a release gate" in error for error in errors)) + + broken = copy.deepcopy(self.data) + asset = next( + item for item in broken["assets"] if item["id"] == "t5gemma-b-b-ul2" + ) + asset["revision"]["value"] = "97ea9b7e92738bb57437867277ae38e65345b8d7" + errors = validate(broken) + self.assertTrue(any("use null" in error for error in errors)) + + def test_non_object_asset_returns_errors_without_crashing(self) -> None: + broken = copy.deepcopy(self.data) + broken["assets"].append("not-an-object") + errors = validate(broken) + self.assertTrue(any("expected an object" in error for error in errors)) + + def test_non_object_nested_fields_return_errors_without_crashing(self) -> None: + broken = copy.deepcopy(self.data) + asset = next( + item for item in broken["assets"] if item["id"] == "t5gemma-b-b-ul2" + ) + asset["distribution"] = "not-an-object" + errors = validate(broken) + self.assertTrue( + any("distribution: expected an object" in error for error in errors) + ) + + broken = copy.deepcopy(self.data) + broken["assets"][0]["revision"] = "not-an-object" + errors = validate(broken) + self.assertTrue( + any("revision: expected an object" in error for error in errors) + ) + + def test_unhashable_nested_values_return_errors_without_crashing(self) -> None: + mutations = [ + ( + "project status", + lambda data: data["project_use"].__setitem__( + "owner_confirmation_status", {} + ), + "owner_confirmation_status: invalid status", + ), + ( + "revision kind", + lambda data: data["assets"][0]["revision"].__setitem__("kind", {}), + "revision.kind: invalid revision kind", + ), + ( + "license status", + lambda data: data["assets"][0]["licenses"]["code"][0].__setitem__( + "status", {} + ), + "status: invalid status", + ), + ( + "license identifier", + lambda data: data["assets"][1]["licenses"]["code"][0].__setitem__( + "identifier", {} + ), + "identifier: expected a string", + ), + ( + "owner status", + lambda data: data["assets"][0]["owner_review"].__setitem__( + "status", {} + ), + "owner_review.status: invalid status", + ), + ( + "dependency id", + lambda data: data["assets"][1]["dependencies"].append({}), + "dependencies: expected string asset ids", + ), + ( + "catalog id", + lambda data: data["catalogs"]["official_lora_ids"].append({}), + "official_lora_ids: expected string asset ids", + ), + ] + for label, mutate, expected in mutations: + with self.subTest(label=label): + broken = copy.deepcopy(self.data) + mutate(broken) + errors = validate(broken) + self.assertTrue(any(expected in error for error in errors), errors) + + def test_unconfirmed_weights_cannot_be_in_installer(self) -> None: + broken = copy.deepcopy(self.data) + asset = next( + item for item in broken["assets"] if item["id"] == "google-mrt2-weights" + ) + asset["distribution"]["installer_contains_weights"] = True + self.assertTrue( + any("unconfirmed weights" in error for error in validate(broken)) + ) + + def test_mutable_runtime_path_must_remain_release_gate(self) -> None: + broken = copy.deepcopy(self.data) + asset = next( + item for item in broken["assets"] if item["id"] == "google-mrt2-weights" + ) + asset["distribution"]["release_gate"] = False + self.assertTrue( + any("mutable runtime path" in error for error in validate(broken)) + ) + + def test_catalog_ids_must_resolve_to_loras(self) -> None: + broken = copy.deepcopy(self.data) + broken["catalogs"]["official_lora_ids"] = ["google-mrt2-weights"] + self.assertTrue(any("is not a LoRA" in error for error in validate(broken))) + + +if __name__ == "__main__": + unittest.main() diff --git a/compliance/validate_inventory.py b/compliance/validate_inventory.py new file mode 100644 index 0000000..c408c53 --- /dev/null +++ b/compliance/validate_inventory.py @@ -0,0 +1,496 @@ +#!/usr/bin/env python3 +"""Validate the revision-specific model/runtime compliance inventory. + +This is deliberately standard-library-only so release jobs can run it before +installing any application dependencies. +""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +from pathlib import Path +from typing import Any +from urllib.parse import urlparse + + +ROOT_REQUIRED = { + "schema_version", + "inventory_revision", + "audited_at", + "audit_base_revision", + "purpose", + "project_use", + "catalogs", + "assets", +} +ASSET_REQUIRED = { + "id", + "name", + "family", + "asset_type", + "support_status", + "upstream", + "revision", + "licenses", + "notices", + "access", + "distribution", + "dependencies", + "owner_review", + "evidence", +} +LICENSE_REQUIRED = { + "status", + "identifier", + "name", + "scope", + "terms_url", + "notice_url", +} +ACCESS_REQUIRED = { + "gated", + "account_required", + "credential_required", + "terms_acceptance_required", + "privacy_url", + "acceptable_use_url", +} +DISTRIBUTION_REQUIRED = { + "mode", + "source_url", + "installer_contains_asset", + "installer_contains_weights", + "redistribution_confirmed", + "immutable_reference_enforced", + "release_gate", + "notes", +} +NOTICE_REQUIRED = {"required_text", "attribution", "sources"} +OWNER_REVIEW_REQUIRED = {"required", "status", "question", "issue"} +REVISION_KINDS = {"git_commit", "model_snapshot", "unresolved_upstream"} +LICENSE_STATUSES = {"declared", "underlying", "unresolved", "not_applicable"} +OWNER_STATUSES = {"pending", "confirmed", "not_required"} +COMMIT_RE = re.compile(r"^[0-9a-f]{40}$") +DATE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$") +INVENTORY_REV_RE = re.compile(r"^\d{4}-\d{2}-\d{2}\.\d+$") +MUTABLE_URL_RE = re.compile(r"/(?:blob|tree|resolve)/(?:main|master|HEAD)(?:/|$)", re.I) + + +def _missing(value: dict[str, Any], required: set[str], path: str) -> list[str]: + return [ + f"{path}: missing field {field!r}" for field in sorted(required - value.keys()) + ] + + +def _is_https(value: Any) -> bool: + return ( + isinstance(value, str) + and urlparse(value).scheme == "https" + and bool(urlparse(value).netloc) + ) + + +def _url_error(value: Any, path: str, *, nullable: bool = False) -> list[str]: + if nullable and value is None: + return [] + return [] if _is_https(value) else [f"{path}: must be an https URL"] + + +def validate(data: Any) -> list[str]: + """Return all validation errors without stopping at the first one.""" + if not isinstance(data, dict): + return ["root: expected an object"] + errors = _missing(data, ROOT_REQUIRED, "root") + if errors: + return errors + + if data["schema_version"] != 1: + errors.append("root.schema_version: only version 1 is supported") + if not INVENTORY_REV_RE.fullmatch(str(data["inventory_revision"])): + errors.append("root.inventory_revision: expected YYYY-MM-DD.N") + if not DATE_RE.fullmatch(str(data["audited_at"])): + errors.append("root.audited_at: expected YYYY-MM-DD") + if not COMMIT_RE.fullmatch(str(data["audit_base_revision"])): + errors.append("root.audit_base_revision: expected a full 40-character commit") + + project_use = data["project_use"] + if not isinstance(project_use, dict): + errors.append("root.project_use: expected an object") + else: + required = { + "reported_context", + "owner_confirmation_status", + "future_distribution_or_commercial_change_is_release_gate", + "public_record_must_exclude", + } + errors.extend(_missing(project_use, required, "root.project_use")) + owner_confirmation_status = project_use.get("owner_confirmation_status") + if not isinstance( + owner_confirmation_status, str + ) or owner_confirmation_status not in {"pending", "confirmed"}: + errors.append("root.project_use.owner_confirmation_status: invalid status") + if ( + project_use.get("future_distribution_or_commercial_change_is_release_gate") + is not True + ): + errors.append( + "root.project_use: future use/distribution changes must be a release gate" + ) + + assets = data["assets"] + if not isinstance(assets, list) or not assets: + errors.append("root.assets: expected a non-empty list") + return errors + + ids: set[str] = set() + for index, asset in enumerate(assets): + path = f"root.assets[{index}]" + if not isinstance(asset, dict): + errors.append(f"{path}: expected an object") + continue + missing = _missing(asset, ASSET_REQUIRED, path) + errors.extend(missing) + if missing: + continue + + asset_id = asset["id"] + if not isinstance(asset_id, str) or not re.fullmatch( + r"[a-z0-9][a-z0-9-]*", asset_id + ): + errors.append(f"{path}.id: expected a lowercase kebab-case identifier") + elif asset_id in ids: + errors.append(f"{path}.id: duplicate identifier {asset_id!r}") + else: + ids.add(asset_id) + + upstream = asset["upstream"] + if not isinstance(upstream, dict): + errors.append(f"{path}.upstream: expected an object") + else: + errors.extend( + _missing(upstream, {"project", "canonical_url"}, f"{path}.upstream") + ) + errors.extend( + _url_error( + upstream.get("canonical_url"), f"{path}.upstream.canonical_url" + ) + ) + + revision = asset["revision"] + if not isinstance(revision, dict): + errors.append(f"{path}.revision: expected an object") + else: + errors.extend( + _missing(revision, {"kind", "value", "url"}, f"{path}.revision") + ) + kind = revision.get("kind") + value = revision.get("value") + url = revision.get("url") + distribution_for_revision = asset.get("distribution") + distribution_for_revision = ( + distribution_for_revision + if isinstance(distribution_for_revision, dict) + else {} + ) + if not isinstance(kind, str) or kind not in REVISION_KINDS: + errors.append(f"{path}.revision.kind: invalid revision kind") + if kind == "unresolved_upstream": + if value is not None: + errors.append( + f"{path}.revision.value: unresolved upstream revisions use null" + ) + if distribution_for_revision.get("immutable_reference_enforced"): + errors.append( + f"{path}.revision: unresolved upstream revision cannot be marked immutable" + ) + if not distribution_for_revision.get("release_gate"): + errors.append( + f"{path}.revision: unresolved upstream revision must remain a release gate" + ) + elif not isinstance(value, str) or not COMMIT_RE.fullmatch(value): + errors.append( + f"{path}.revision.value: expected a full immutable 40-character hash" + ) + errors.extend(_url_error(url, f"{path}.revision.url")) + if isinstance(url, str) and MUTABLE_URL_RE.search(url): + errors.append( + f"{path}.revision.url: mutable branch references are forbidden" + ) + if ( + kind != "unresolved_upstream" + and isinstance(url, str) + and isinstance(value, str) + and value not in url + ): + errors.append( + f"{path}.revision.url: must contain the exact revision value" + ) + + licenses = asset["licenses"] + if not isinstance(licenses, dict): + errors.append(f"{path}.licenses: expected an object") + else: + errors.extend(_missing(licenses, {"code", "weights"}, f"{path}.licenses")) + for license_kind in ("code", "weights"): + records = licenses.get(license_kind) + license_path = f"{path}.licenses.{license_kind}" + if not isinstance(records, list) or not records: + errors.append(f"{license_path}: expected a non-empty list") + continue + for license_index, record in enumerate(records): + record_path = f"{license_path}[{license_index}]" + if not isinstance(record, dict): + errors.append(f"{record_path}: expected an object") + continue + record_missing = _missing(record, LICENSE_REQUIRED, record_path) + errors.extend(record_missing) + if record_missing: + continue + status = record["status"] + identifier = record["identifier"] + if not isinstance(status, str) or status not in LICENSE_STATUSES: + errors.append(f"{record_path}.status: invalid status") + if not isinstance(identifier, str): + errors.append(f"{record_path}.identifier: expected a string") + if isinstance(status, str) and status in {"declared", "underlying"}: + if not isinstance(identifier, str) or identifier in { + "NONE", + "NOASSERTION", + }: + errors.append( + f"{record_path}.identifier: applicable license needs an identifier" + ) + errors.extend( + _url_error(record["terms_url"], f"{record_path}.terms_url") + ) + elif status == "unresolved" and identifier != "NOASSERTION": + errors.append( + f"{record_path}.identifier: unresolved licenses use NOASSERTION" + ) + elif status == "not_applicable" and identifier != "NONE": + errors.append( + f"{record_path}.identifier: non-applicable licenses use NONE" + ) + errors.extend( + _url_error( + record["notice_url"], + f"{record_path}.notice_url", + nullable=True, + ) + ) + + notices = asset["notices"] + if not isinstance(notices, dict): + errors.append(f"{path}.notices: expected an object") + else: + errors.extend(_missing(notices, NOTICE_REQUIRED, f"{path}.notices")) + for field in ("required_text", "attribution", "sources"): + if field in notices and not isinstance(notices[field], list): + errors.append(f"{path}.notices.{field}: expected a list") + for source_index, source in enumerate(notices.get("sources", [])): + errors.extend( + _url_error(source, f"{path}.notices.sources[{source_index}]") + ) + + access = asset["access"] + if not isinstance(access, dict): + errors.append(f"{path}.access: expected an object") + else: + errors.extend(_missing(access, ACCESS_REQUIRED, f"{path}.access")) + for field in ( + "gated", + "account_required", + "credential_required", + "terms_acceptance_required", + ): + if field in access and not isinstance(access[field], bool): + errors.append(f"{path}.access.{field}: expected a boolean") + for field in ("privacy_url", "acceptable_use_url"): + errors.extend( + _url_error( + access.get(field), f"{path}.access.{field}", nullable=True + ) + ) + if access.get("gated") and not access.get("terms_acceptance_required"): + errors.append( + f"{path}.access: gated assets must record terms acceptance" + ) + + distribution = asset["distribution"] + if not isinstance(distribution, dict): + errors.append(f"{path}.distribution: expected an object") + else: + errors.extend( + _missing(distribution, DISTRIBUTION_REQUIRED, f"{path}.distribution") + ) + source_url = distribution.get("source_url") + errors.extend(_url_error(source_url, f"{path}.distribution.source_url")) + if isinstance(source_url, str) and MUTABLE_URL_RE.search(source_url): + errors.append( + f"{path}.distribution.source_url: mutable branch references are forbidden" + ) + revision_for_distribution = asset.get("revision") + revision_for_distribution = ( + revision_for_distribution + if isinstance(revision_for_distribution, dict) + else {} + ) + revision_value = revision_for_distribution.get("value") + if ( + revision_for_distribution.get("kind") != "unresolved_upstream" + and isinstance(source_url, str) + and isinstance(revision_value, str) + and revision_value not in source_url + ): + errors.append( + f"{path}.distribution.source_url: must contain the exact revision value" + ) + for field in ( + "installer_contains_asset", + "installer_contains_weights", + "redistribution_confirmed", + "immutable_reference_enforced", + "release_gate", + ): + if field in distribution and not isinstance(distribution[field], bool): + errors.append(f"{path}.distribution.{field}: expected a boolean") + if distribution.get("installer_contains_weights") and not distribution.get( + "redistribution_confirmed" + ): + errors.append( + f"{path}.distribution: unconfirmed weights must not be placed in installers" + ) + if not distribution.get( + "immutable_reference_enforced" + ) and not distribution.get("release_gate"): + errors.append( + f"{path}.distribution: a mutable runtime path must remain a release gate" + ) + + owner_review = asset["owner_review"] + if not isinstance(owner_review, dict): + errors.append(f"{path}.owner_review: expected an object") + else: + errors.extend( + _missing(owner_review, OWNER_REVIEW_REQUIRED, f"{path}.owner_review") + ) + owner_status = owner_review.get("status") + if not isinstance(owner_status, str) or owner_status not in OWNER_STATUSES: + errors.append(f"{path}.owner_review.status: invalid status") + errors.extend( + _url_error(owner_review.get("issue"), f"{path}.owner_review.issue") + ) + if ( + owner_review.get("required") is True + and owner_review.get("status") == "not_required" + ): + errors.append( + f"{path}.owner_review: required review cannot be marked not_required" + ) + + if not isinstance(asset["dependencies"], list): + errors.append(f"{path}.dependencies: expected a list") + if not isinstance(asset["evidence"], list) or not asset["evidence"]: + errors.append(f"{path}.evidence: expected a non-empty list") + else: + for evidence_index, evidence in enumerate(asset["evidence"]): + errors.extend( + _url_error(evidence, f"{path}.evidence[{evidence_index}]") + ) + + for index, asset in enumerate(assets): + if not isinstance(asset, dict) or "dependencies" not in asset: + continue + for dependency in ( + asset["dependencies"] if isinstance(asset["dependencies"], list) else [] + ): + if not isinstance(dependency, str): + errors.append( + f"root.assets[{index}].dependencies: expected string asset ids" + ) + continue + if dependency not in ids: + errors.append( + f"root.assets[{index}].dependencies: unknown asset id {dependency!r}" + ) + if dependency == asset.get("id"): + errors.append( + f"root.assets[{index}].dependencies: self-dependency is forbidden" + ) + + assets_by_id = { + item["id"]: item + for item in assets + if isinstance(item, dict) and isinstance(item.get("id"), str) + } + catalogs = data["catalogs"] + if not isinstance(catalogs, dict): + errors.append("root.catalogs: expected an object") + else: + required = { + "bundled_lora_ids", + "official_lora_ids", + "documented_reference_lora_ids", + "note", + } + errors.extend(_missing(catalogs, required, "root.catalogs")) + for field in ( + "bundled_lora_ids", + "official_lora_ids", + "documented_reference_lora_ids", + ): + values = catalogs.get(field) + if not isinstance(values, list): + errors.append(f"root.catalogs.{field}: expected a list") + continue + for asset_id in values: + if not isinstance(asset_id, str): + errors.append(f"root.catalogs.{field}: expected string asset ids") + continue + if asset_id not in ids: + errors.append( + f"root.catalogs.{field}: unknown asset id {asset_id!r}" + ) + else: + asset = assets_by_id[asset_id] + if asset.get("asset_type") != "lora": + errors.append( + f"root.catalogs.{field}: {asset_id!r} is not a LoRA" + ) + + return errors + + +def load(path: Path) -> Any: + with path.open(encoding="utf-8") as handle: + return json.load(handle) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "path", + nargs="?", + type=Path, + default=Path(__file__).with_name("model-assets.json"), + ) + args = parser.parse_args(argv) + try: + data = load(args.path) + except (OSError, json.JSONDecodeError) as exc: + print(f"{args.path}: {exc}", file=sys.stderr) + return 2 + errors = validate(data) + if errors: + for error in errors: + print(error, file=sys.stderr) + return 1 + print(f"validated {len(data['assets'])} assets in {args.path}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/docs/adr/0037-platform-mrt2-runtime-contract.md b/docs/adr/0037-platform-mrt2-runtime-contract.md new file mode 100644 index 0000000..944af87 --- /dev/null +++ b/docs/adr/0037-platform-mrt2-runtime-contract.md @@ -0,0 +1,54 @@ +# ADR 0037: Platform MRT2 runtimes behind one worker contract + +## Status + +Accepted for implementation; PyTorch release qualification remains blocked by +issue #109's Linux and Windows NVIDIA hardware matrix. + +## Context + +LSDJ's deck worker contract predates platform support and constructed the MLX +implementation directly. Apolinario's PyTorch port provides the corresponding +CUDA implementation through immutable Hugging Face Transformers snapshots. It +is an external dependency: LSDJ does not copy, patch, or publish that runtime. + +The #109 spike validated the API surface but did not have Linux or Windows +NVIDIA hosts. It therefore could not establish a supported GPU/driver floor +or prove two-deck real-time performance. + +## Decision + +- The native host passes an explicit runtime on every deck-sidecar launch: + `mlx` on macOS and `pytorch-cuda` on Linux/Windows. +- Python resolves that name through the model-independent `Mrt2Engine` contract. + It never catches a backend failure and tries another implementation. +- PyTorch requires CUDA. Missing CUDA, a CPU-only torch build, missing pinned + assets, corrupt assets, and initialization failures are startup failures. +- The PyTorch adapter loads the exact model and MusicCoCa revisions recorded by + #109 with `local_files_only=True`. `trust_remote_code=True` receives only the + resolved local immutable snapshot path, never a mutable repository name. +- LSDJ owns input normalization, weighted-style caching, continuation state, + reset-to-reseed semantics, PCM validation, lifecycle integration, and + diagnostics. Upstream owns model and processor code. +- The PyTorch runtime remains fail-closed until the required hardware evidence + exists. `LSDJ_ALLOW_UNVERIFIED_MRT2_CUDA=1` is an explicit qualification-only + opt-in and is disclosed in diagnostics. +- The PyTorch topology is one supervised process with two deck loops. Equal + model selections share one loaded upstream model behind serialized inference + and keep independent continuation/style/control state. Different per-deck + model selections preserve existing behavior by loading both models in that + same process. macOS retains its two independent MLX workers. + +## Consequences + +macOS keeps its existing MLX behavior. Linux and Windows builds have an +explicit CUDA path and actionable failure diagnostics, but cannot be described +as supported releases yet. Platform-specific, hash-locked Python/CUDA inputs +exist for Linux x86_64 and Windows x86_64, but they remain installation +candidates until clean-host and NVIDIA qualification completes. Their +existence does not imply a supported minimum GPU or driver. + +To update upstream, change only immutable revisions and candidate versions in +`backend/lsdj/mrt2.py`, re-run the model-free contract suite, then repeat every +unchecked item in `docs/issue-110-hardware-checklist.md`. Never copy upstream +runtime sources into this repository. diff --git a/docs/adr/0038-windows-sa3-cuda-qualification-gate.md b/docs/adr/0038-windows-sa3-cuda-qualification-gate.md new file mode 100644 index 0000000..7c0e647 --- /dev/null +++ b/docs/adr/0038-windows-sa3-cuda-qualification-gate.md @@ -0,0 +1,113 @@ +# ADR-0038: Windows SA3 CUDA is a gated shared-runtime backend + +Status: proposed, implemented behind a release gate + +## Context + +Windows already has the official LiteRT/TFLite Stable Audio 3 backend. CUDA can +make Small Music and Small SFX practical on a capable NVIDIA GPU, but it adds a +second large model family beside the realtime MRT2 decks. The two projects' +original dependency pins do not match: the pinned SA3 source requires PyTorch +2.7.1 and Hugging Face Hub 1.7.1 or newer, while the MRT2 candidate from #110 +used PyTorch 2.12.1 and Hugging Face Hub 1.5.0. + +The SA3 model repositories are gated. Public Hugging Face metadata establishes +immutable revisions and the root weight hashes, but cannot establish all +configuration and nested T5Gemma hashes without the authenticated terms flow +owned by #108. No Windows NVIDIA qualification host was available for this +change. + +## Decision + +- Consume `Stability-AI/stable-audio-3` at the immutable upstream commit in + `sa3-pytorch-cuda-pin.json`. LSDJ owns a thin adapter; it does not fork or + vendor upstream runtime code. +- Test one shared Windows PyTorch environment for MRT2 and SA3. The resolved + candidate uses PyTorch/torchaudio 2.7.1+cu126 and Hugging Face Hub 1.7.1 in a + fully hashed 44-package lock. This is a candidate, not a supported upgrade: + MRT2 must be requalified on it. +- Keep the TFLite runtime separately installed and available. It is not placed + inside the PyTorch environment and remains the release/default backend. +- Run SA3 in a new disposable child for every generation. Heavy imports, model + allocation, and CUDA context creation occur only in that child. Completion, + cancellation, broker yield, OOM, crash, backend switch, and app teardown end + the process. +- Bind each private request to its supervised child with a per-launch secret: + only its SHA-256 enters the bounded request file, the secret arrives through + an allowlisted inherited environment value, and the worker removes it before + upstream imports. A bounded job ID is attached to every progress/terminal + event; neither value enters argv or diagnostics. +- Coordinate CUDA with a cross-process file-locked broker. MRT2 uses realtime +priority; background SA3 checks for a higher-priority waiter at every sampler +callback. A daemon watchdog covers model loading and decoding, whose upstream +calls have no callback, and hard-stops only the disposable SA3 process so the +MRT2 request can proceed. +- Expose Auto, GPU, and CPU/TFLite policy in the backend status contract. Auto + remains on TFLite until the hardware gate is flipped with evidence. Explicit + GPU fails before launch and asks for a confirmed TFLite fallback while the + candidate is blocked; it never silently changes backend or runs PyTorch on + CPU. +- Enable only Small Music and Small SFX in the CUDA capability contract. Medium + remains on TFLite unless an official Windows FlashAttention path and a + measured hardware tier are qualified. Unofficial wheels, custom extensions, + and private model forks are forbidden. + +## Trust and release gate + +The native installer compiles the candidate manifest but rejects it unless all +of these are true in one reviewed change: + +1. `releaseReady` and `gatedArtifactsComplete` are true and the blocker list is + empty. +2. Every required Small model artifact has an exact SHA-256 and byte count. +3. The embedded shared lock matches the manifest's exact size and SHA-256. +4. A worker provenance stamp matches the source revision, lock digest, package + versions, model repository, and model revision exactly. +5. The Windows NVIDIA matrix in the issue #114 checklist is complete. + +The worker independently rechecks package, CUDA runtime, driver, free-memory, +reservation, provenance, and model path facts before importing a model. A +reported free-memory value is only an admission snapshot; the process boundary +is still the recovery mechanism for unrelated VRAM pressure and CUDA failure. + +## Qualification no-go: resident MRT2 ownership + +Commit `5cbe842` is approved as broker safety behavior. On Windows it prunes a +lease only when the owner is positively known to have exited; access or query +uncertainty retains the record and fails closed. This approval is not NVIDIA +hardware or VRAM evidence. + +Commit `7fdbb32` deliberately acquires one shared, process-lifetime MRT2 lease +before CUDA availability is queried or any model is loaded. Keeping that lease +through model-load failure and for the lifetime of the worker is fail-closed: +SA3 cannot treat VRAM owned by a resident MRT2 model as free. The ordering must +remain until a measured replacement is available. + +The current single-level lease consequently prevents SA3 admission while the +MRT2 worker is alive, so it cannot satisfy the coexistence and dual-deck tests +required by issue #114. Releasing the lease between MRT2 generations would not +be safe because the model remains resident. It would also undercount ownership +when multiple models or workers are present. + +Issue #114 is a no-go until one of these paths is completed: + +1. Measure resident and active-generation VRAM on Windows NVIDIA hardware, then + implement a two-level broker. Long-lived reservations must account for every + resident model and worker; short active-generation priority leases must let + realtime MRT2 work interrupt or defer background SA3 work. +2. Obtain explicit product acceptance that MRT2 will unload before SA3 starts, + then re-load and warm up afterward, and qualify that lifecycle instead of + coexistence. + +The required VRAM and hardware evidence does not yet exist. Auto selection, +public SA3 CUDA availability, `HARDWARE_QUALIFIED`, and the release manifest +gate remain disabled. This record does not change runtime code, dependency +pins, qualification gates, or make a release-support claim. + +## Consequences + +The design and model-free failure behavior can merge without delaying the +TFLite Windows release. CUDA is not advertised as installable or selected by +Auto in this state. Completing #108's authenticated audit, resolving measured +VRAM tiers, and running both SA3 parity and MRT2 realtime qualification are +mandatory follow-ups, not release notes that can be waived. diff --git a/docs/adr/README.md b/docs/adr/README.md index cd77f8b..0973cfb 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -58,3 +58,5 @@ scaffolds the file from the template. | [0031](0031-native-midi-in-the-rust-core-superseding-web-midi.md) | Native MIDI in the Rust core, superseding Web MIDI | Accepted | | [0032](0032-standalone-midi-keyboard-window.md) | Standalone MIDI keyboard window, routing decoupled from steering | Accepted | | [0035](0035-dual-envelope-beat-detection-with-fast-change-invalidation.md) | Dual-envelope beat detection with fast change invalidation | Accepted | +| [0037](0037-platform-mrt2-runtime-contract.md) | Platform MRT2 runtimes behind one worker contract | Accepted for implementation | +| [0038](0038-windows-sa3-cuda-qualification-gate.md) | Windows SA3 CUDA is a gated shared-runtime backend | Proposed | diff --git a/docs/cross-platform-ci-and-release.md b/docs/cross-platform-ci-and-release.md new file mode 100644 index 0000000..691b363 --- /dev/null +++ b/docs/cross-platform-ci-and-release.md @@ -0,0 +1,121 @@ +# Cross-platform CI and release contract + +Issue #107 introduces two related but deliberately separate gates: + +1. shared software contracts that can run unattended on GitHub-hosted macOS, + Ubuntu, and Windows runners; and +2. qualification that requires real audio, MIDI, GPU, installer, or signing + hardware and credentials. + +Passing CI means a change is portable at the code and build boundary. It does +not by itself claim that a particular device, accelerator, or installer has +been qualified. + +## Automated shared checks + +`.github/workflows/ci.yml` runs the same three-stack checks on `macos-15`, +`ubuntu-24.04`, and `windows-2025`: + +- frontend lint, TypeScript checking, and the complete Vitest suite; +- Rust workspace tests and Clippy, including each OS's compiled platform code; +- Python lint plus controller, worker, transport, validation, model-free deck + behavior, SA3 subprocess, and SA3 readiness tests; and +- release artifact contract tests, including checksum, identity, completeness, + and pre-publication draft verification failures. + +The backend's locked `ci` dependency group intentionally omits Magenta, MLX, +PyTorch, TFLite, and model weights. The SA3 process tests use a copied Python +interpreter and fake CLI, so argument passing, output handling, failure, and +timeout behavior run on all three operating systems. Only tests that actually +import a model runtime stay in the full local `just check` suite and in +backend-specific qualification. This keeps a shared Python regression gate +honest without installing an unsupported accelerator on a runner. + +The Windows matrix uses runner-native `python`, `npm`, `rustup`, and `cargo` +commands. Shell-specific system package installation is restricted to the +Ubuntu step. + +## Hardware-only qualification + +The following evidence must be recorded in the platform issue or its linked +qualification run. It is not replaced by green hosted-runner CI: + +| Surface | Minimum real-system evidence | +| --- | --- | +| Audio output | Enumerate and play through representative mono, stereo, integer, and multichannel devices; verify master/cue routing, underrun behavior, device loss, and recovery. | +| MIDI | Connect supported controllers; verify input, LEDs, hot-plug, shutdown, and reconnect behavior with native drivers. | +| MRT2 | Run two simultaneous decks at both chunk sizes on the supported CPU/GPU backend; capture startup, p50/p95/p99 latency, RAM/VRAM, sustained playback, and teardown. | +| Stable Audio 3 | Generate every supported duration/kind on the supported CPU/GPU backend; verify cancellation, progress, failure cleanup, and output playback. | +| App lifecycle | Install on a clean user account, launch without developer tools, install/remove models, survive paths with spaces/non-ASCII text, and leave no worker processes after normal or forced shutdown. | +| Distribution | Exercise the native installer/uninstaller, OS trust prompts, signing where applicable, checksum validation, offline behavior after install, and a representative antivirus scan. | + +CI tests should use fakes or loopback devices only when they verify a shared +contract. A fake result must not be reported as hardware qualification. + +## Release producer/publisher boundary + +The tag workflow requires macOS arm64, Linux x64, and Windows x64 release +artifacts. It has five stages: + +1. `validate` accepts only a calendar-version `v*` tag whose commit is contained + in `main`. +2. `produce_macos` waits + behind the protected `macos-release` Environment, + freezes the backend, imports ephemeral signing material, builds, signs, + notarizes, staples, and verifies the app and DMG. It then uploads one Actions + artifact containing the DMG, `SHA256SUMS.txt`, and metadata binding the + producer to the tag and exact source revision. +3. `produce_linux` builds the x86_64 AppImage on Ubuntu 22.04 (glibc 2.35), + verifies its desktop/resource layout and ELF dependencies, performs an + isolated-XDG virtual-X11 smoke, and uploads the AppImage with the same + checksum/tag/revision contract. +4. `produce_windows` waits behind the protected `windows-release` Environment, + requires the selected provider's protected one-file signing interface, builds + the per-user NSIS installer, verifies the exact Authenticode identity and + timestamp on the installer and installed executable payloads, and exercises + preservation/removal before uploading its producer bundle. Until a provider + and CI identity are selected, this job intentionally fails at preflight. +5. `publish` is the only job with `contents: write`. It downloads every required + producer bundle, requires the producer set to match exactly, recomputes all + sizes and SHA-256 digests, and verifies tag/revision/platform metadata before + it creates a GitHub Release. + +Linux is fail-closed: a skipped or failed producer prevents the publisher from +running. This automated package smoke does not replace issue #112's NVIDIA, +Wayland/Xorg, audio, MIDI/FLX4, or suspend/resume hardware gate. + +The publisher creates an unpublished draft, uploads the complete verified file +set, checks GitHub's returned asset names, sizes, upload state, and SHA-256 +digest, and only then makes the release public. A missing digest fails closed. +The published checksum and release-index files provide the same cryptographic +verification surface to downloaders. Creation records the draft's immutable +numeric release ID plus its tag and exact source revision. Verification, +publication, and failure cleanup remain bound to that ID; cleanup rechecks that +the same release is still a draft with the expected tag and source marker before +deleting it. A tag lookup therefore cannot redirect cleanup to a collaborator's +replacement draft. A failure before publication keeps the release private and +attempts to remove only the draft created by that run. An existing release is +never overwritten. + +Signing and notarization secrets exist only in their protected platform +producer. The publisher receives no signing credentials, and producers never +receive `contents: write`. + +## Adding a release platform + +Linux or Windows artifacts become required only in the change that adds their +production installer. That change must, together: + +- add a named producer job with its platform-native build and trust checks; +- add the producer policy to `scripts/release_artifact.py`; +- upload its installer, checksum, and tag/revision metadata as one Actions + artifact; +- add the producer to the publisher's `needs` list and + `--required-producer` arguments; and +- extend the release contract tests and real-system qualification record. + +There must remain exactly one publisher job. It must wait for every required +producer and fail closed if any producer is absent, skipped, duplicated, +unexpected, or inconsistent. Optional best-effort release artifacts are not +published. The verifier treats every configured producer policy as required, +so the workflow's `--required-producer` list cannot silently omit a new policy. diff --git a/docs/issue-109-hardware-checklist.md b/docs/issue-109-hardware-checklist.md new file mode 100644 index 0000000..d1dafb6 --- /dev/null +++ b/docs/issue-109-hardware-checklist.md @@ -0,0 +1,83 @@ +# Issue #109 Linux/Windows NVIDIA qualification checklist + +Run every item once on representative Ubuntu 22.04+ and Windows 11 systems. +Attach the JSON and logs to #109; do not summarize a failing/missing run as a +pass. + +## Host record + +- [ ] Record OS/build, CPU, physical RAM, GPU model, VRAM, NVIDIA driver, power + mode, and whether the GPU drives a display. +- [ ] Save `nvidia-smi` output before and after each run. +- [ ] Record Python, PyTorch, Transformers, CUDA runtime, cuDNN, and exact + `provenance.json` revisions. +- [ ] Confirm `torch.cuda.is_available()` and the reported compute capability. +- [ ] Start from a clean runtime with no system Python, Git, shell, CUDA toolkit, + or compiler dependency in the packaged execution path. + +## Acquisition/offline proof + +- [ ] Acquire the small model at `7037d99551c84ac5c6afb7f1a5e58c65e7233dbb`. +- [ ] Acquire MusicCoCa at `236c488e38aa98643805514996934d705668298b`. +- [ ] Verify hashes, disconnect network (or set `HF_HUB_OFFLINE=1`), and confirm + the harness starts. A cache miss must fail clearly without a download. +- [ ] Repeat for base `92087988d05d0fe38b11f021f0b0d00a75afb86b` + only if base is a proposed supported model. + +## Required matrix + +For every proposed acceleration mode, run: + +```text +shared-worker × 25 frames × 600 seconds +shared-worker × 5 frames × 600 seconds +per-deck × 25 frames × 600 seconds +per-deck × 5 frames × 600 seconds +``` + +- [ ] Run the dry adapter first and retain it separately as synthetic evidence. +- [ ] Run `python -m spike.mrt2_pytorch.harness --backend upstream ...` for the + full matrix with parity guidance (the default). +- [ ] Repeat with a live app build feeding the native engine and capture its + engine-reported underrun counter; the harness proxy is not a substitute. +- [ ] Confirm both decks prime, generate/play continuously, and report zero + native engine underruns. +- [ ] Confirm p50/p95/p99/max latency, generated-audio/wall ratio, RSS peak, + PyTorch CUDA peak, process VRAM peak, driver, temperature, P-state, and + power rows are present. +- [ ] Inspect thermal clocks/temperature across the full ten minutes; record any + throttling or laptop power-mode dependency. + +## Controls and continuity + +- [ ] At the scheduled change, verify weighted prompt, temperature, top-k, + prompt CFG, note CFG, drum CFG, MIDI onset, and onset-to-sustain take effect + without resetting continuation state. +- [ ] Listen/inspect boundaries in both chunk modes for gaps, repeats, channel + swaps, clipping, or a sample-rate mismatch (required: float32, stereo, + 48 kHz). +- [ ] Run twice from a fresh state with the same seed and record determinism; + confirm changing seed with retained state does not falsely claim reseeding. +- [ ] Exercise text and captured-audio style inputs. +- [ ] Switch small/base models and verify reset/readiness behavior. + +## Supervision/topology + +- [ ] Record cold start, warm-up, readiness, and clean shutdown time. +- [ ] Kill deck A in the per-deck topology; deck B must continue and the parent + must report the failure. +- [ ] Kill the shared worker; both decks must stop/report failure coherently. +- [ ] Close the parent/app during generation and confirm no Python, helper, + compiler, or GPU process remains. +- [ ] Force one malformed control payload and one generation exception; no + silent hang or stale-playing state is allowed. + +## Release gate + +- [ ] Name the lowest GPU/VRAM/driver configuration that passed every required + run on both operating systems. +- [ ] Confirm packaged execution requires no runtime compiler; otherwise mark the + acceleration path no-go. +- [ ] #108 confirms code, derived-weight, MusicCoCa, and original-weight notices. +- [ ] Publish the topology recommendation and raw evidence on #109 before #110 + starts. diff --git a/docs/issue-110-hardware-checklist.md b/docs/issue-110-hardware-checklist.md new file mode 100644 index 0000000..fb80d69 --- /dev/null +++ b/docs/issue-110-hardware-checklist.md @@ -0,0 +1,84 @@ +# Issue #110 — PyTorch MRT2 production qualification + +This checklist is intentionally unchecked. The implementation was developed +without Linux or Windows NVIDIA hardware; unit tests are not performance or +driver evidence. + +## Immutable inputs + +- [ ] Confirm the source, model, and processor revisions printed by + `python -m lsdj.sidecar --runtime-info` match the release inventory. +- [x] Resolve the candidate Python/PyTorch/CUDA dependencies into separate, + hash-locked Linux x86_64 and Windows x86_64 requirement sets. Clean-host + installation remains unchecked below. +- [ ] Install each runtime through the atomic #107 installer on a clean host + without system Python, Git, shell tools, a compiler, or CUDA toolkit. +- [ ] Disconnect networking and prove startup and prompt/audio embedding use + only the verified app-owned snapshot cache. + +## Required hosts and diagnostics + +Record the app version, OS build, GPU, VRAM, NVIDIA driver, torch version, CUDA +runtime, model revision, processor revision, upstream source revision, and +acceleration mode from the worker's `ready.runtime` payload. + +- [ ] Ubuntu 22.04+ on the proposed minimum NVIDIA GPU. +- [ ] Windows 11 x64 on the proposed minimum NVIDIA GPU. +- [ ] Unsupported/no GPU fails at startup and never starts CPU inference. +- [ ] An insufficient-VRAM case fails cleanly without leaving a child process. + +Set `LSDJ_ALLOW_UNVERIFIED_MRT2_CUDA=1` only for these qualification runs. It +does not turn an unchecked configuration into a supported one. + +## Two-deck real-time matrix + +For every row, run two armed decks continuously for at least ten minutes. Save +the native engine telemetry and worker diagnostics; the #109 ring proxy alone +is not an underrun measurement. + +| OS | model | frames/chunk | duration | engine underruns | p50/p95/p99 latency | peak VRAM | result | +| --- | --- | ---: | ---: | ---: | --- | ---: | --- | +| Ubuntu | mrt2_small | 25 | 10 min | | | | [ ] | +| Ubuntu | mrt2_small | 5 | 10 min | | | | [ ] | +| Windows | mrt2_small | 25 | 10 min | | | | [ ] | +| Windows | mrt2_small | 5 | 10 min | | | | [ ] | + +- [ ] Repeat the matrix for `mrt2_base` if it will be offered on that support + floor; otherwise document its higher minimum VRAM separately. +- [ ] Change weighted prompts, temperature, top-k, prompt CFG, notes, drum state, + and drum CFG during each 5-frame run. +- [ ] Verify note onset decays to sustain after one chunk. +- [ ] Verify reset with a fixed seed starts a repeatable fresh stream and never + pretends to reseed existing continuation state. +- [ ] Compare prompt/note guidance behavior against MLX parity fixtures. +- [ ] Listen across every chunk boundary and verify continuous stereo 48 kHz + audio with the approximately 1.5-second playback safety ring. + +## Lifecycle and recovery + +- [ ] Readiness arrives only after CUDA/model/processor warm-up completes. +- [ ] Generation latency and command queue depth remain visible while playing. +- [ ] Kill the shared worker: both decks stop clearly and can recover. +- [ ] Corrupt or remove one snapshot: startup identifies the repair action. +- [ ] Cancel install/update at every stage; the previous verified runtime works. +- [ ] Exit and crash the app during warm-up and generation; no Python or GPU + process remains. +- [ ] Prove the production Rust host owns one model worker with independent deck + continuation states, as selected by #109. +- [ ] On the minimum-VRAM Windows and Linux hosts, switch the shared worker from + equal models to different models and back while both decks are active. Verify + both decks enter loading/unavailable together, the old process and CUDA + allocation are fully reaped before replacement allocation starts, and no + transient second generation appears in process/VRAM telemetry. +- [ ] Force the replacement launch and model load to fail after the old shared + worker is reaped. Verify both decks remain clearly unavailable and a later + valid selection recovers them. This serialized minimum-VRAM transition does + not promise live hardware rollback to the old worker. + +## Release gate + +- [ ] Issue #108 notices and download acknowledgement are complete. +- [ ] Both platform release checklists link the captured results above. +- [ ] The minimum GPU, VRAM, driver, and runtime are written from measured data. +- [ ] Flip the fail-closed qualification constant only in the PR containing all + evidence and the final target-specific locks. diff --git a/docs/issue-114-windows-sa3-cuda-checklist.md b/docs/issue-114-windows-sa3-cuda-checklist.md new file mode 100644 index 0000000..7b396c4 --- /dev/null +++ b/docs/issue-114-windows-sa3-cuda-checklist.md @@ -0,0 +1,163 @@ +# Issue #114 — Windows Stable Audio 3 CUDA qualification + +This checklist is intentionally unchecked. Unit tests prove policy, mapping, +broker behavior, and failure containment without weights; they are not NVIDIA +performance evidence. Until every release gate is complete, Auto and the model +manager continue to use the supported TFLite backend. + +Set `LSDJ_ALLOW_UNVERIFIED_SA3_CUDA=1` only on a dedicated qualification host. +It permits explicit GPU probes; it does not enable Auto or make a build +release-ready. + +## Current no-go: resident MRT2 ownership + +The following checks record reviewed code behavior, not Windows NVIDIA +qualification evidence: + +- [x] `5cbe842` retains Windows lease records when owner-process liveness is + uncertain and prunes only owners positively known to have exited. This + fail-closed safety behavior is approved. +- [x] `7fdbb32` acquires the shared MRT2 lease before CUDA or model load and + holds it for the worker lifetime, including after a CUDA load failure. This + ordering deliberately prevents resident MRT2 VRAM from being treated as + free. + +**NO-GO:** the process-lifetime MRT2 lease prevents SA3 admission while the +MRT2 worker is alive. Releasing it between generations is unsafe because model +VRAM remains resident. Issue #114 cannot pass its broker or dual-deck +acceptance sections until one of these alternatives is complete: + +- [ ] On Windows NVIDIA hardware, measure resident and active-generation VRAM + separately for each MRT2 model and worker, both decks/multiple model + combinations, and SA3 Small Music and Small SFX. +- [ ] Implement and validate a two-level broker: long-lived reservations for + every resident model/worker plus short active-generation priority leases for + realtime MRT2 versus background SA3 work. Prove that capacity is neither + double-counted nor inferred from VRAM that remains resident. +- [ ] Alternatively, obtain explicit product acceptance to unload MRT2 before + SA3 starts and re-load/warm it afterward, then replace the coexistence tests + below with evidence for that lifecycle. + +Until that decision and the required hardware evidence exist, Auto and public +SA3 CUDA remain disabled, `HARDWARE_QUALIFIED` and the release gate remain +unchanged, and no Windows SA3 CUDA release claim may be made. + +## Immutable inputs and shared runtime + +- [x] Pin the official upstream source commit without an LSDJ fork. +- [x] Resolve one 44-package, fully hash-locked Windows x64 candidate shared by + SA3 and MRT2: Python 3.12, PyTorch/torchaudio 2.7.1+cu126, Transformers 5.8.0, + Hugging Face Hub 1.7.1. +- [x] Record the source archive, Small Music/SFX root weights, optional Medium + root weight, exact immutable revisions, known hashes, and incomplete gates in + `sa3-pytorch-cuda-pin.json`. +- [ ] Through #108's authenticated terms flow, record SHA-256 and byte count for + every required Small config and nested T5Gemma artifact. +- [ ] Regenerate the lock with the release uv version on a clean Windows x64 + host and prove `--require-hashes --only-binary :all:` installation. +- [ ] Re-run all MRT2 functional fixtures on the shared PyTorch 2.7.1/CUDA 12.6 + runtime. The #110 PyTorch 2.12.1/CUDA 13.0 results do not transfer. +- [ ] Confirm the source/runtime/model provenance shown by diagnostics matches + the compiled manifest and that an altered stamp fails before model import. +- [x] Require a per-launch secret and bounded job ID before imports; echo only + the job ID on progress/terminal events and scrub the secret before upstream + code loads. +- [ ] Complete #108 acknowledgement, attribution, and terms UX before exposing + the CUDA download. + +Run the local candidate audit with: + +```console +python3 scripts/audit-sa3-cuda-pin.py --allow-incomplete +``` + +The same command without `--allow-incomplete` is the release gate and must fail +until the required gated hashes exist. + +## Required host inventory + +For each row, save the LSDJ version, Windows build, GPU, VRAM, NVIDIA driver, +PyTorch version, CUDA runtime, source revision, model revision, peak VRAM, and +generated WAV hash. CUDA 12.6's provisional minimum Windows driver is 560.76; +replace that value with the measured support floor before release. + +| Host tier | GPU / VRAM | driver | Small Music reserve | Small SFX reserve | result | +| --- | --- | --- | ---: | ---: | --- | +| proposed minimum | | | | | [ ] | +| mid-range | | | | | [ ] | +| high-end | | | | | [ ] | +| insufficient VRAM | | | n/a | n/a | [ ] clean failure | + +- [ ] Measure cold-load, sampling, decode, peak allocated/reserved VRAM, and + post-exit VRAM for Small Music. +- [ ] Repeat for Small SFX. +- [ ] Choose conservative per-model reservations and at least 1 GiB headroom + from results; never infer them from marketed card capacity. +- [ ] Add an unrelated VRAM consumer before and during admission. Free VRAM must + be treated as advisory and unsafe work must fail before start. +- [ ] Unsupported GPU, old driver, CUDA mismatch, and no GPU fail without a + PyTorch CPU attempt. + +## Shared-contract parity + +For both Small models, compare the same pinned fixtures against TFLite: + +- [ ] text-to-audio; +- [ ] audio-to-audio and init-noise level; +- [ ] inpainting and continuation; +- [ ] positive and negative prompt; +- [ ] fixed seed, duration, sampling steps, CFG, and APG; +- [ ] one LoRA and a stacked LoRA with independent strengths; +- [ ] normalized progress and cancellation; +- [ ] exact 44.1 kHz stereo PCM16 duration/output boundary. + +Record intentional numerical/performance differences. A populated control may +not be ignored. If the pinned API cannot represent it, coordinate upstream or +route it explicitly to TFLite. + +## Broker, isolation, and lifecycle + +- [ ] Start SA3, then request MRT2 work during loading, sampling, and decoding. + The watchdog/callback exits SA3, releases its lease/context, and MRT2 proceeds. +- [ ] With the two-level broker, queue SA3 while MRT2 holds an active-generation + priority lease. SA3 waits without disturbing either deck or the native audio + callback; MRT2's resident reservation remains accounted for. +- [ ] Cancel while waiting, loading, sampling, and decoding; no child or CUDA + allocation remains. +- [ ] Force CUDA OOM, worker exception, invalid output, and abrupt worker death; + MRT2 and deck audio continue. +- [ ] Switch to CPU/TFLite during/after generation and exit the app at every + worker stage; the Windows Job Object removes every descendant. +- [ ] Corrupt the broker state and provenance stamp; both fail closed with a + bounded, non-sensitive diagnostic. + +## Dual-deck realtime acceptance + +Run two active `mrt2_small` decks for at least ten minutes per row while +repeatedly queueing/running/cancelling alternating Small Music and Small SFX +jobs. Save native engine underrun telemetry; silence or ring occupancy is not a +substitute. + +| frames/chunk | duration | SA3 workload | engine underruns | MRT2 p50/p95/p99 | SA3 p50/p95 | peak VRAM | result | +| ---: | ---: | --- | ---: | --- | --- | ---: | --- | +| 25 | 10 min | alternating Small Music/SFX | | | | | [ ] | +| 5 | 10 min | alternating Small Music/SFX | | | | | [ ] | + +- [ ] Both rows have zero engine-reported underruns. +- [ ] Exercise weighted prompts, notes, drums, seed/reset, and both deck states + during the 5-frame run. +- [ ] Verify broker yield does not kill or reset the shared MRT2 worker. + +## Selection and release + +- [ ] Auto selects CUDA only on a fully qualified configuration and explains + why it chose TFLite otherwise. +- [ ] Explicit GPU shows requirements and fails or offers a user-confirmed + TFLite fallback before generation; it never silently falls back after start. +- [ ] CPU/TFLite always selects the independent verified portable runtime. +- [ ] Active backend, worker state, fallback reason, estimate/reservation, GPU, + VRAM, driver, CUDA, PyTorch, and immutable revisions are visible. +- [ ] Keep Medium on TFLite. Enable it only in a later evidence-bearing change + with an official Windows FlashAttention path; never ship an unofficial wheel. +- [ ] Flip `HARDWARE_QUALIFIED` and the manifest release gate only in the PR that + links all evidence above. diff --git a/docs/linux-qualification-checklist.md b/docs/linux-qualification-checklist.md new file mode 100644 index 0000000..86574f1 --- /dev/null +++ b/docs/linux-qualification-checklist.md @@ -0,0 +1,97 @@ +# Linux release qualification — issue #112 + +Hosted CI is not hardware qualification. Attach this completed record to issue +#112 for each proposed minimum configuration and for both a real Wayland and a +real Xorg session. + +## Exact environment + +- [ ] LSDJ tag and source revision: +- [ ] AppImage filename and SHA-256: +- [ ] Ubuntu version and kernel: +- [ ] Session type and desktop/compositor: +- [ ] CPU and RAM: +- [ ] GPU and VRAM: +- [ ] NVIDIA driver, PyTorch, CUDA runtime, MRT2 dependency/model revisions: +- [ ] Stable Audio/TFLite/model/LoRA revisions: +- [ ] Main/cue audio devices, sample formats/rates/channels, buffer frames: +- [ ] MIDI controller model, firmware, and raw ALSA port names: +- [ ] Relevant udev/session ACL configuration: + +## Clean install and desktop + +- [ ] On a clean Ubuntu 22.04+ x86_64 user account, checksum verification and + executable-bit setup lead to a normal AppImage launch without developer + tools, system Python, Git, a shell command, or a CUDA toolkit. +- [ ] Native titlebar/window behavior, file/folder dialogs, opener, Trash, and + every notification used by LSDJ behave correctly. +- [ ] Repeat on Wayland and Xorg, including paths/home names with spaces and + non-ASCII characters. +- [ ] Normal exit, forced app exit, and failed worker startup leave no worker + descendants. + +## Runtime/model installation + +- [ ] First download shows exact revisions, terms/links, storage, backend, and + driver compatibility before work begins. +- [ ] MRT2 and Stable Audio runtimes/models install from verified pins into XDG + assets/staging roots; corrupt, interrupted, cancelled, and failed updates + retain the previous verified version. +- [ ] Offline launch after successful installation does not invoke or require + system Python, `uv`, Git, shell tools, a CUDA toolkit, or network access. +- [ ] Insufficient disk, RAM/VRAM, incompatible driver, authentication, and + verification failures are actionable and redact credentials. + +## Audio and lifecycle + +- [ ] ALSA direct and PipeWire-ALSA paths enumerate and play the intended + devices; record which path each device used. +- [ ] Validate default-device selection/change, 48 kHz and non-48 kHz devices, + f32/i16/u16 where offered, mono/stereo/multichannel conversion, and + unsupported-layout errors. +- [ ] FLX4 combined routing sends master to channels 1/2 and cue to 3/4; split + main/cue routing also works. +- [ ] Unplug/replug, PipeWire restart, default-device change, suspend/resume, + and app restart recover clearly without callback stalls. + +## MIDI and FLX4 + +- [ ] FLX4 is usable without an unsafe blanket udev rule; record any required + group/session ACL or device-specific `uaccess` rule. +- [ ] ALSA port-name variants normalize for matching while the raw port remains + selectable and reconnects to the same device. +- [ ] Validate hotplug/reconnect, transport, mixer controls, jog wheels, pad + modes, LEDs, position-query SysEx, and the in-app MIDI monitor. +- [ ] DDJ-400 remains best-effort regression evidence, not a release blocker. + +## Sustained MRT2 performance + +- [ ] Run both decks for at least 10 minutes at 25 frames (approximately 1 s). +- [ ] Run both armed decks for at least 10 minutes at 5 frames (approximately + 200 ms). +- [ ] Both runs have zero **engine-reported** underruns. +- [ ] Capture p50/p95/p99 generation latency, queue depth, audio buffer settings, + CPU/RAM/VRAM, temperature, and throttling notes. + +## Stable Audio parity while decks remain live + +- [ ] Music and SFX generation. +- [ ] Audio-to-audio, continuation, and inpainting. +- [ ] Small and Medium models, positive/negative prompts, all exposed sampling + controls, LoRA selection/application, preview, output naming, and corrupt + output validation. +- [ ] Cancellation and long-duration validation, including the supported Medium + maximum, without blocking the audio callback or causing deck underruns. +- [ ] Record CPU/RAM use and the queue/constrain/pause policy used to protect + both live decks. + +## Release decision + +- [ ] #108 licensing/acknowledgement release gate complete. +- [ ] #110 production PyTorch MRT2 adapter complete and qualified. +- [ ] #111 Stable Audio TFLite adapter complete and qualified. +- [ ] Linux producer bundle, native dependency audit, checksum, and deterministic + tag/revision metadata pass the single-publisher verification. +- [ ] Known limitations and measured minimum CPU/RAM/GPU/VRAM/driver/storage + requirements are published. +- [ ] Issue #112 records an explicit go/no-go decision and links this evidence. diff --git a/docs/linux.md b/docs/linux.md new file mode 100644 index 0000000..f1a19aa --- /dev/null +++ b/docs/linux.md @@ -0,0 +1,145 @@ +# Linux AppImage support + +LSDJ's Linux target is Ubuntu 22.04 or newer on x86_64 with a supported NVIDIA +GPU. The AppImage packaging, desktop configuration, XDG storage contract, and +fail-closed release producer are implemented as part of issue #112. A public +Linux release remains gated on the production PyTorch MRT2 backend (#110), the +portable Stable Audio TFLite backend (#111), the model licensing flow (#108), +and the real-hardware checklist below. + +Passing hosted CI proves that the shell compiles, the AppImage extracts, its +desktop metadata/resources are present, and it can open in a virtual X11 +session with paths containing spaces and Unicode. It does **not** qualify an +NVIDIA driver, PipeWire/ALSA device, Wayland compositor, FLX4, suspend/resume, +or model performance. + +## Install and verify + +Download the `.AppImage` and the release's `SHA256SUMS.txt`/producer metadata +from the same GitHub Release. Verify the checksum before launch, then make the +file executable and open it: + +```sh +sha256sum --check linux-x64-SHA256SUMS.txt +chmod +x LSDJ_*.AppImage +./LSDJ_*.AppImage +``` + +The AppImage is the only supported Linux package. `.deb`, `.rpm`, Flatpak, +Snap, ARM64, AMD/Intel GPU acceleration, and JACK-specific integration are not +part of the supported target. Other distributions may work, but are community +configurations until separately qualified. + +The application package does not invoke or require a system Python, `uv`, Git, +a shell, or a CUDA toolkit. Model adapters are installed into app-owned storage +from pinned, checksum-verified artifacts. If a managed adapter is absent or +invalid, the corresponding service reports unavailable instead of falling back +to a command from `PATH`. MRT2 and Stable Audio are resolved independently from +their own verified service manifests, so one missing runtime does not redirect +or downgrade the other service. A compatible NVIDIA **driver** is still +required for MRT2; the minimum version and VRAM floor remain unset until the +#110 hardware qualification records measured results. + +If FUSE is unavailable, AppImage's standard extract-and-run mode is a useful +diagnostic fallback: + +```sh +APPIMAGE_EXTRACT_AND_RUN=1 ./LSDJ_*.AppImage +``` + +The release gate still tests ordinary AppImage packaging; extract-and-run is +not a substitute for the clean-machine qualification. + +## Storage and first run + +Rust resolves the roots once and passes them explicitly to every service. The +default Linux layout is: + +| Purpose | Default path | +| --- | --- | +| Configuration | `$XDG_CONFIG_HOME/lsdj` or `~/.config/lsdj` | +| Durable data | `$XDG_DATA_HOME/lsdj` or `~/.local/share/lsdj` | +| Models/runtimes | `$XDG_DATA_HOME/lsdj/assets` | +| Same-filesystem install staging | `$XDG_DATA_HOME/lsdj/staging` | +| Disposable cache | `$XDG_CACHE_HOME/lsdj` or `~/.cache/lsdj` | + +The model manager owns first download, verification, installation, update, +rollback, cancellation, and recovery. Downloads that require upstream terms or +credentials remain blocked until #108's current-revision acknowledgement flow +authorizes them. A failed or interrupted update must leave the prior verified +runtime usable. + +## Audio: ALSA and PipeWire + +The Rust audio host uses CPAL's ALSA backend. On a PipeWire desktop, the +distribution's PipeWire ALSA compatibility layer routes those streams; a +PulseAudio desktop follows the same ALSA-facing application path. LSDJ does not +invoke `pw-*`, `pactl`, `aplay`, or another external audio utility. + +The in-app `platform_diagnostics` response records whether `/dev/snd`, the +PipeWire socket, and the Pulse socket are visible. It reports evidence only; a +socket's presence is not a successful audio-device test. The following stable +advisory codes are intended for localized UI/support surfaces: + +- `linux.audio.alsaDevicesMissing` +- `linux.session.notDetected` +- `linux.distribution.notSupported` + +Default-device changes, 44.1/48 kHz conversion, stereo and FLX4 four-channel +routing, device removal, PipeWire restart, and suspend/resume must all be +verified on real systems before release. + +## MIDI and device permissions + +Linux MIDI uses ALSA sequencer through `midir`. Port matching preserves the raw +ALSA name used to open the device while normalizing case, punctuation, and ALSA +client/port suffixes for FLX4/DDJ-400 identification. + +`platform_diagnostics` reports `/dev/snd/seq` as `available`, +`permissionDenied`, or `missing` without opening a sequencer client. Its stable +advisory codes are: + +- `linux.midi.sequencerPermissionDenied` +- `linux.midi.sequencerMissing` + +Ubuntu desktop sessions normally grant sound-device access through logind/udev. +If access is denied, first reconnect the controller and sign out/in so the +session ACL can refresh. On a system administered through the traditional +`audio` group, an administrator may add the user to that group and require a +new login. Do not install a blanket world-writable udev rule. A custom rule, if +the distribution truly needs one, must match the controller's measured vendor +and product IDs and grant active-session `uaccess`; record that rule and the +`udevadm info` evidence in the qualification report. + +## Desktop integration + +The Linux overlay uses the desktop's normal decorated titlebar and produces a +single Audio/Music `.desktop` entry and icon. Tauri's native dialog, opener, and +trash integrations remain scoped through Rust; the webview receives no general +filesystem/opener permission. Hosted CI verifies the packaged entry point, +resource layout, executable bits, ELF dependency inventory, and a virtual-X11 +launch using isolated XDG roots with spaces and non-ASCII characters. + +Real Wayland and Xorg sessions must still validate window behavior, native file +and folder dialogs, opener/trash behavior, notifications used by the app, +multi-monitor/scale behavior, and clean shutdown. No notification behavior is +claimed merely because the package launches in Xvfb. + +## Diagnostics and support bundle facts + +The `platform_diagnostics` command exposes: + +- OS/architecture and Ubuntu support classification; +- detected Wayland/X11 session type; +- the resolved config/data/cache/assets/staging roots; +- ALSA, PipeWire/Pulse socket, and MIDI-sequencer evidence; +- `developerFallbackAllowed` (always `false` in the AppImage); and +- runtime mode (`managed` for the Linux package). + +These facts contain no tokens and do not execute external diagnostic tools. +Model/runtime revisions, NVIDIA driver/VRAM, generation latency, queue depth, +and underruns belong to the #110/#111 service diagnostics once those adapters +are integrated. + +See [the Linux qualification checklist](linux-qualification-checklist.md) for +the evidence required before calling the platform supported. diff --git a/docs/managed-runtimes.md b/docs/managed-runtimes.md new file mode 100644 index 0000000..e7a7fcf --- /dev/null +++ b/docs/managed-runtimes.md @@ -0,0 +1,33 @@ +# Managed runtimes on Linux and Windows + +Linux and Windows releases launch only application-managed backend generations. +Each generation is assembled in private staging, built exclusively from exact +URL, size, and SHA-256 pins, installed offline, validated, and then atomically +promoted. The prior generation remains available for rollback. At launch, LSDJ +revalidates the target, generation stamp, complete file inventory, executable, +working directory, and every file digest immediately before spawning an +absolute program with a fixed argument vector and a cleared environment. + +No production path invokes a shell, searches `PATH`, or falls back to system +Python, Git, or `uv`. The managed services are `mrt2`, `sa3-tflite`, and the +reserved `sa3-pytorch-cuda` service used by the Windows CUDA backend work. + +## Hugging Face access + +The MRT2 snapshot pins use immutable Hugging Face revisions. LSDJ can pass an +`HF_TOKEN` or `HUGGING_FACE_HUB_TOKEN` only to the native authenticated download +request; credentials are not written to provenance, manifests, command lines, +or logs. LSDJ does not bypass repository gates or accept model terms for a user. +If an upstream repository requires authentication or acceptance of its terms, +that remains an external acquisition prerequisite and the installer fails +closed until the user has completed it. + +## Launch-secret seam + +The runtime manifest declares two ephemeral launch-only keys: +`LSDJ_API_CAPABILITY` and `LSDJ_WORKER_LAUNCH_TOKEN`. They are accepted only by +the structured `VerifiedCommand::into_command(extra_args, ephemeral)` boundary, +are carried in the child environment, and are excluded from fixed arguments, +static manifest values, provenance, disk, and diagnostics. Issue #130 owns the +authenticated IPC protocol and token generation/verification that populate +this seam. diff --git a/docs/model-license-release-checklist.md b/docs/model-license-release-checklist.md new file mode 100644 index 0000000..e38b7d3 --- /dev/null +++ b/docs/model-license-release-checklist.md @@ -0,0 +1,84 @@ +# Model licensing release checklist + +Run this checklist for every public release and whenever a model, runtime, +download source, license, terms page, product distribution model, or commercial +status changes. This is a release-control checklist, not legal advice. + +## Automated inventory gate + +- [ ] `python3 compliance/validate_inventory.py` passes. +- [ ] `python3 -m unittest compliance.test_inventory` passes. +- [ ] `audit_base_revision` is updated to the release candidate's full commit. +- [ ] Each application runtime/package pin matches its manifest entry and lockfile. +- [ ] Every model/processor/LoRA download supplies the exact manifest revision; + no `main`, `master`, `HEAD`, latest tag, or omitted revision remains. +- [ ] Download verification rejects a source/revision mismatch and partial files. +- [ ] No installer contains a manifest weight/adapter whose + `redistribution_confirmed` value is false. + +## Project-owner record + +- [ ] Owners selected and committed the LSDJ code license and notice. +- [ ] Owners confirmed the current open-source/non-commercial project-use path + for Google MRT2, the PyTorch conversion, Stable Audio 3, and T5Gemma. +- [ ] Owners resolved the Apache model-card vs underlying CC-BY-4.0 treatment for + both re-keyed PyTorch MRT2 snapshots. +- [ ] Owners confirmed the Stable Audio Community and Gemma derivative path for + the exact optimized snapshot. +- [ ] Owners either identified specific permission for each official/downloadable + LoRA or removed it from the official catalog. No sensitive account, + contract, credential, or revenue information was put in the public record. +- [ ] Any change in product distribution or commercial status triggered a fresh + owner review before publication. + +## Application and package notices + +- [ ] About/Licenses lists each manifest asset name, exact revision, upstream + link, code license, weight/model terms, attribution, and owner-review state. +- [ ] The packaged notices include applicable Apache/MIT notices, Google MRT2 + CC-BY attribution, Stability Community attribution and “Powered by + Stability AI” display, and the Gemma notice/terms link. +- [ ] LSDJ's code license is visually separate from third-party model terms and + explicitly says it does not relicense model weights or LoRAs. +- [ ] Release notes link this notice document and state whether models are + downloaded rather than included. +- [ ] Platform packages (macOS, Linux, Windows) contain the same notice version. + +## Download acknowledgement and access + +- [ ] First download is blocked until the current asset/license revision is + acknowledged wherever the manifest says `terms_acceptance_required: true`. +- [ ] A changed asset revision or notice/terms version requires a fresh + acknowledgement; unrelated telemetry or marketing consent is separate. +- [ ] Gated-download cancellation, offline behavior, rejection, revoked access, + and expired/invalid credentials produce actionable errors without starting + a partial install. +- [ ] Tokens are redacted from command lines, logs, events, diagnostics, crash + reports, and UI; persistent tokens use the OS credential store and are + never written to plaintext application data. +- [ ] Anonymous optimized artifacts still show the underlying Stability/Gemma + terms; anonymous access is not treated as permission to relicense. + +## LoRA provenance + +- [ ] `bundled_lora_ids`, `official_lora_ids`, and documented references match + the actual application catalog and documentation exactly. +- [ ] Each LSDJ-provided LoRA download displays author, source, exact revision, + adapter license, compatible base, and base-model terms before download. +- [ ] No LoRA is mirrored or bundled without permission for that exact artifact. +- [ ] User imports show the responsibility/provenance notice and do not claim + that LSDJ reviewed the user's rights or private file contents. +- [ ] The Motif Maqam adapter remains a user-directed reference unless its + unresolved license/permission gate is closed. + +## Evidence captured for the release + +- [ ] Record release tag/commit, inventory version, reviewer, review date, and + the exact artifact revisions actually downloaded in the private release + record. +- [ ] Archive generated package file lists proving restricted weights are absent. +- [ ] Record passing acknowledgement-versioning, gated-download blocking, + offline/error, and credential-redaction tests from the follow-up UI/download + implementation. +- [ ] If any item above is not complete, block Linux and Windows publication and + link the unresolved owner decision without exposing sensitive information. diff --git a/docs/native-packaging.md b/docs/native-packaging.md index 0bb8db8..c3836ec 100644 --- a/docs/native-packaging.md +++ b/docs/native-packaging.md @@ -8,6 +8,16 @@ Spike C ([`docs/spike-c-midi.md`](spike-c-midi.md), the Tauri MIDI app) — so t steps below are reproducible on a Mac with an Apple Developer ID and are also enforced by the protected release workflow. +Linux uses the same shared Tauri base with +[`tauri.linux.conf.json`](../src-tauri/tauri.linux.conf.json), while macOS-only +window and bundle settings live in +[`tauri.macos.conf.json`](../src-tauri/tauri.macos.conf.json). The Linux +AppImage is built on Ubuntu 22.04 through `just tauri-linux-release`. Its +`managed-runtime` feature refuses every developer-tool fallback until #110/#111 +supply an explicit verified adapter executable. Packaging, XDG/audio/MIDI +diagnostics, and qualification details are in +[`linux.md`](linux.md). + ## 1. Freeze the backend runtime ```sh @@ -231,10 +241,14 @@ fresh install and later top-ups: huggingface_hub`, `--collect-all fsspec`, `--hidden-import click`). A missing collection only fails at runtime in the packaged app — hence the checklist item below, which static analysis cannot cover. -- Stable Audio 3 installs in-app too, into the app-owned data dir - (`~/Library/Application Support/LSDJ/stable-audio-3`, the resolver's first - candidate): the Rust shell fetches the pinned source - ([`sa3-pin.json`](../sa3-pin.json)) as a tarball (`curl`), extracts it (`tar`), - and runs [`scripts/sa3-install.sh`](../scripts/sa3-install.sh) — build+warm - steps with no git, no tty, no system Python 3.11 (`install.sh -y --python - 3.11`). Both families' weights move there with `just migrate-models`. +- Stable Audio 3 installs in-app too, into the app-owned assets dir. The Rust + worker treats [`sa3-pin.json`](../sa3-pin.json) as its trust root: native HTTPS + downloads the immutable source, `uv` runtime, and every model artifact; + application-controlled SHA-256 values are checked before bounded native + extraction or execution. Python dependencies come from the embedded + hash-locked requirements file. Pin provenance and the reproducible release + audit are recorded in [`docs/sa3-pin-audit.md`](sa3-pin-audit.md). The complete candidate is built and warmed in + the host-resolved same-filesystem staging root, then atomically promoted while + the previous verified install remains available for rollback. No in-app step + invokes `bash`, `curl`, `tar`, `chmod`, shell activation, or a shell command + string. Both families' weights move there with `just migrate-models`. diff --git a/docs/platform-paths.md b/docs/platform-paths.md new file mode 100644 index 0000000..fa8e4d5 --- /dev/null +++ b/docs/platform-paths.md @@ -0,0 +1,48 @@ +# Platform filesystem contract + +The Rust desktop host resolves LSDJ's filesystem roots once during Tauri setup, +before it starts the deck sidecars, generation server, watchers, or installers. +Python services receive the resolved paths through environment variables and do +not derive platform locations from a home directory. + +| Ownership | macOS | Windows | Linux | +| --- | --- | --- | --- | +| Configuration | `~/Library/Application Support/works.protocol.lsdj` | `%LOCALAPPDATA%\LSDJ\config` | `$XDG_CONFIG_HOME/lsdj` | +| Durable user data | `~/Documents/LSDJ` | `%LOCALAPPDATA%\LSDJ\data` | `$XDG_DATA_HOME/lsdj` | +| Disposable cache | `~/Library/Caches/works.protocol.lsdj` | `%LOCALAPPDATA%\LSDJ\cache` | `$XDG_CACHE_HOME/lsdj` | +| Downloaded assets | `~/Library/Application Support/LSDJ` | `%LOCALAPPDATA%\LSDJ\assets` | `$XDG_DATA_HOME/lsdj/assets` | +| Install staging | `~/Library/Application Support/LSDJ/.staging` | `%LOCALAPPDATA%\LSDJ\staging` | `$XDG_DATA_HOME/lsdj/staging` | + +On Linux, absent or invalid XDG variables use the standard fallbacks +`~/.config`, `~/.local/share`, and `~/.cache`. On Windows, every root is +non-roaming and the short `LSDJ` directory deliberately avoids consuming path +budget when long-path support is disabled. Staging and downloaded assets always +share a filesystem so a validated install can be promoted atomically. + +The host exports `LSDJ_CONFIG_HOME`, `LSDJ_DATA_HOME`, `LSDJ_CACHE_HOME`, +`LSDJ_ASSETS_HOME`, and `LSDJ_STAGING_HOME`. It also supplies the current +compatibility variables `MAGENTA_HOME`, `SA3_MLX_HOME`, and `SA3_LORAS_HOME`; +explicit developer/user values for those three are captured into the contract +at startup. Paths are passed as native process-environment values and executable +arguments, not interpolated into shell command strings. + +## macOS compatibility and migration + +The contract preserves the current visible locations: generated songs and +samples remain in Documents, model assets remain in Application Support, and +settings/MCP credentials remain under the bundle identifier. Startup retains +the existing one-time migrations from `LSDJai` and from +`~/Documents/Magenta/magenta-rt-v2`. + +Each migration is an atomic same-filesystem rename attempted only when the +destination does not exist. A restart sees the destination and does nothing. If +preparing or renaming fails, the process contract points the relevant backend at +the old directory for that run, so a migration failure cannot hide an installed +model or adapter. + +## Virtual environments + +Virtual-environment interpreters are resolved centrally as `bin/python` on +macOS/Linux and `Scripts/python.exe` on Windows. The interpreter and each +argument remain separate process arguments, including when a profile path +contains spaces or non-ASCII characters. diff --git a/docs/sa3-pin-audit.md b/docs/sa3-pin-audit.md new file mode 100644 index 0000000..8d4483c --- /dev/null +++ b/docs/sa3-pin-audit.md @@ -0,0 +1,55 @@ +# Stable Audio 3 pin provenance and audit + +`sa3-pin.json` and `sa3-tflite-pin.json` are executable supply-chain policy for +the in-app installer, not just version documentation. Every URL is HTTPS, every +revision is immutable, and every byte-bearing artifact has an +application-controlled SHA-256 and exact size. A pin change must update this +record in the same pull request. + +## Recorded provenance (2026-08-08) + +| Pin family | Immutable upstream evidence | How the manifest value was established | +| --- | --- | --- | +| SA3 source | Stability AI Git commit `a0b57f5483c4588f827f3552b7d5c6ca2a9687be` | Streamed the exact GitHub commit archive: 50,494,239 bytes, SHA-256 `98e206e061a3b64a4e65f50b2802bdb6965910ac1fab65da919808dfb4497e9f`. The 442-entry archive expands to 83,845,120 bytes. GitHub does not publish a signed checksum for generated source archives, so any byte change fails closed and requires explicit review. | +| uv runtime | Astral uv release `0.11.7` | The publisher checksums and downloaded bytes agree for `aarch64-apple-darwin` (20,839,135 bytes, `66e37d91…34ba8`), `x86_64-unknown-linux-gnu` (24,249,861 bytes, `6681d691…ea868`), and `x86_64-pc-windows-msvc` (23,572,531 bytes, `fe0c7815…a8b29`). The Windows artifact is securely extracted from ZIP; the other two are tarballs. | +| Python runtime | python-build-standalone release `20251007`, CPython `3.11.13` | Publisher metadata and downloaded bytes agree for Apple arm64 (18,949,778 bytes, `78bc6def…e8c`), Linux x64 (30,157,215 bytes, `43bfc425…f0c3`), and Windows x64 (25,990,147 bytes, `cde5153f…8b29`). | +| MLX model weights | `stabilityai/stable-audio-3-optimized` at commit `6736003cb57d06b7b1fdc36fad31b2a3709e4774` | The eight manifest size/hash pairs match the immutable revision's LFS objects. The unique model payload is 9,154,794,562 bytes. | +| TFLite model weights | The same immutable model revision | `sa3-tflite-pin.json` selects the official fp32 CPU set. Its eight unique LFS objects total 14,138,994,904 bytes. Shared SAME-S artifacts are installed once even though both small bundles reference them. | +| Python dependencies | The two `.in` files compiled by uv `0.11.7` for Python 3.11 | The MLX lock contains 19 exact packages and 282 hashes; the portable LiteRT lock contains 26 exact packages and 433 hashes. Installer invocation enforces `--require-hashes --only-binary :all:` against public PyPI with ambient index/config variables removed. | + +The model manager discloses the selected backend's exact unique model payload +before installation. Source archives, Python, uv, and dependency wheels add +download and on-disk overhead beyond that displayed model-byte figure. + +## Reproduce the audit + +From the repository root, with network access: + +```console +python3 scripts/audit-sa3-pins.py +``` + +This downloads and hashes the source and three host runtime pairs, checks both +sets of eight model objects against immutable Hugging Face LFS metadata, and +audits both dependency locks. For a release-bound pin bump, also perform the +full model-byte audit (roughly 23 GB): + +```console +python3 scripts/audit-sa3-pins.py --include-model-bytes +``` + +Regenerate each dependency lock with the same pinned uv release and compare the +result instead of editing it by hand: + +```console +uv pip compile --generate-hashes --python-version 3.11 \ + --output-file scripts/sa3-requirements.lock scripts/sa3-requirements.in +uv pip compile --generate-hashes --python-version 3.11 \ + --output-file scripts/sa3-tflite-requirements.lock scripts/sa3-tflite-requirements.in +git diff --exit-code -- scripts/sa3-requirements.lock scripts/sa3-tflite-requirements.lock +``` + +Reviewers should reject a pin update unless its immutable revision, exact size, +checksum, provenance source, and audit result are all present. The installer +independently rechecks sizes and hashes before extraction, execution, promotion, +recovery, and app-managed readiness. diff --git a/docs/spike-mrt2-pytorch.md b/docs/spike-mrt2-pytorch.md new file mode 100644 index 0000000..77187c6 --- /dev/null +++ b/docs/spike-mrt2-pytorch.md @@ -0,0 +1,134 @@ +# Issue #109 — PyTorch MRT2 portability spike + +Audit date: 2026-08-08. This note covers software/API validation. No Linux or +Windows NVIDIA machine was available in this workspace, so it contains no +fabricated performance results. + +## Outcome + +The port is consumable without an LSDJ fork: use the Transformers model snapshot +at an exact Hugging Face revision, keep the adapter thin, prefetch all assets, +and run with `local_files_only=True`. The repository is public and Apache-2.0. +The earlier Space named in discovery material is now authentication-gated, but +it is not needed by the proposed path. + +The production decision is **conditional no-go pending hardware qualification**. +Do not begin #110 until both target operating systems pass the two-deck, +ten-minute matrix and #108 resolves weight/processor licensing. This is a +maturity and evidence gate, not a rejection of the implementation. + +## Immutable dependency path + +| Component | Immutable reference | Finding | +| --- | --- | --- | +| PyTorch source/API | `multimodalart/magenta-realtime-torch@6d076baa3df3b10448876c400521a015a5137c59` | Public Apache-2.0 source; no PyTorch-specific release/tag | +| Base model + remote code | `magenta-community/magenta-realtime-2@92087988d05d0fe38b11f021f0b0d00a75afb86b` | Transformers custom model, ~2.46B reported parameters | +| Small model + remote code | `magenta-community/magenta-realtime-2-small@7037d99551c84ac5c6afb7f1a5e58c65e7233dbb` | Transformers custom model, ~282M reported parameters | +| MusicCoCa processor | `magenta-community/magenta-rt-musiccoca-torch@236c488e38aa98643805514996934d705668298b` | Text/audio encoder artifacts; exact revision must be supplied by the adapter | +| Original Google assets | `google/magenta-realtime-2@010aa0dcb0dfd27b24f0ad07b4dad63e8f9521cc` | Declared base model/weight provenance | + +The runnable fixture is `spike/mrt2_pytorch/harness.py`; the complete +machine-readable inventory is `spike/mrt2_pytorch/provenance.json`. + +The audited GitHub `pyproject.toml` is still the upstream JAX/MLX package: it +does not declare a PyTorch extra or pin `torch`/`transformers`. The Transformers +snapshot is therefore the cleaner dependency boundary. LSDJ must own a +target-specific lock for Python 3.12, PyTorch, Transformers, and CUDA wheels; +the spike's exact direct pins are candidates, not a production lock. + +## Control and state parity + +| LSDJ MLX behavior | PyTorch port | Disposition | +| --- | --- | --- | +| Weighted text prompt embeddings | `MusicCoCaProcessor.layer()` returns 12 style tokens | Thin adapter; cache tokens on changes, not per chunk | +| Text negative prompt | Not exposed by the MRT2 deck today | No mapping needed; upstream CFG negatives are masked conditioning, not negative text | +| Temperature and top-k per chunk | `generate(temperature=, top_k=)` | Direct | +| Prompt/note CFG matching `.mlxfn` | `generate(..., guidance=True)` | Direct but more expensive than upstream's default token-CFG path; benchmark parity mode | +| Drum adherence token | `cfg_drums` | Direct; true-CFG mode still treats drums through the learned token | +| Note states `-1/0/1/2/3` and drum `-1/0/1` | Raw `notes`/`drums` arrays | Direct; LSDJ retains onset-to-sustain decay | +| Small/base model selection | Separate pinned model repositories | Direct; switching requires a worker/model restart | +| Per-deck continuation | Returned state contains decoder, RNG, and codec state | One model can safely own two state objects; must be sustained-tested | +| Seed | Seed creates the RNG only when state is new | Adapter documents reset-to-reseed; current LSDJ UI has no MRT2 seed control | +| 25-frame and 5-frame chunks | Arbitrary `frames`; 40 ms/frame, 48 kHz stereo | Direct; output length/continuity must be measured on hardware | +| Audio style sampling | Processor source implements audio embedding/resampling | API match; golden/hardware parity is not present in upstream CI | +| Warm-up/reset | No stable high-level readiness API | Adapter performs a throwaway generate then clears state; readiness contract belongs in #110 | + +The upstream workflow at the audited revision runs macOS MLX tests only. It +does not run the PyTorch port on Linux, Windows, or CUDA, and checkpoint-heavy +parity tests are skipped. Claims in model cards are useful provenance, not a +substitute for LSDJ qualification. + +## Harness topology and ring model + +`shared-worker` loads one model in one process and keeps independent deck A/B +state, scheduling generation round-robin. `per-deck` starts two processes and +therefore two model instances. Both use the same control-change sequence and +the same 1.5-second prebuffer gate. + +The harness reports starvation duration/transitions as `underrun_proxy_*`. +LSDJ's Rust engine counts individual audio callback blocks after the ring is +primed, so the proxy is deliberately not named an engine underrun. Final +qualification must capture both the harness JSON and the app's native telemetry. + +Start with the shared-worker/two-state topology: it avoids duplicating a large +model and has the smallest support-floor risk. Its failure domain covers both +decks and its serialized inference load may miss real time. Move to per-deck +workers only if concurrent GPU execution materially improves the 5-frame +two-deck result on supported hardware and the measured VRAM floor is acceptable. + +## Packaging feasibility + +An installer can ship without user-installed Python, Git, a shell, or a CUDA +toolkit by bundling an embedded Python runtime, exact binary wheels, and +prefetched snapshots. Users still need a compatible NVIDIA driver. The official +PyTorch release matrix publishes the same CUDA wheels for Linux and Windows. + +Risks that must be closed before release: + +- `trust_remote_code=True` executes snapshot code; only the audited commit may + be acquired, hash-checked, and promoted atomically. +- `model.load_processor()` defaults to a mutable repository reference. The LSDJ + adapter must resolve the exact processor revision locally, as the harness does. +- `torch.compile` can require a compiler on Windows. Production must not trigger + an unbundled MSVC/toolchain install. AOTInductor artifacts are GPU-architecture + specific, so one artifact cannot establish a broad GPU support floor. +- Eager, `torch.compile`, CUDA graph, and AOT behavior have not been compared on + the target systems. Upstream exposes the fast CUDA graph path through a stream + surface rather than the simple resumable `generate` call, so a stable chunk API + may require an upstream contribution. +- Produce platform-specific, hash-locked wheels only after choosing the CUDA + runtime and minimum driver from the hardware results. + +## Licensing/provenance escalation + +The fork's code is Apache-2.0. The derived Transformers model cards say +Apache-2.0, while they declare `google/magenta-realtime-2` as their base and the +Google weights are CC-BY-4.0. The MusicCoCa artifact card is also CC-BY-4.0. +Issue #108 must decide the effective redistribution/notice obligations; this +spike makes no legal conclusion. + +## Decision gates for #110 + +Go only when all are true: + +1. Linux and Windows each sustain both decks for ten minutes, at 25 and 5 + frames, with zero native engine underruns on the proposed minimum GPU. +2. The parity-guidance path meets the budget, including a live prompt and note + onset/sustain change; token-CFG results cannot stand in for it. +3. Exact wheel and snapshot locks install offline in a clean bundled runtime. +4. Startup/readiness, one-deck crash behavior, whole-tree shutdown, and device + recovery are demonstrated on both platforms. +5. #108 approves the notices, acknowledgement, and redistribution path. + +Until those gates pass, #109's software deliverables are complete but the +production recommendation remains conditional no-go. + +## Primary evidence + +- Source/API: +- Source license: +- Base model: +- Small model: +- Google model card/weights: +- PyTorch binary matrix: +- Windows compiler caveat: diff --git a/docs/stable-audio-backends.md b/docs/stable-audio-backends.md new file mode 100644 index 0000000..c57e936 --- /dev/null +++ b/docs/stable-audio-backends.md @@ -0,0 +1,135 @@ +# Stable Audio 3 backend contract + +LSDJ selects one Stable Audio backend explicitly: + +- Apple Silicon macOS uses the existing MLX runtime. +- Linux and Windows use the official LiteRT/TFLite CPU runtime. +- Windows x64 has an optional PyTorch/CUDA candidate for Small Music and Small + SFX. It remains behind a fail-closed release gate; Auto therefore continues + to select TFLite until the issue #114 hardware and provenance matrix is done. +- Unsupported platforms fail with a diagnostic. They do not silently select a + different runtime. +- `LSDJ_SA3_BACKEND=mlx|tflite` is a diagnostic/developer override. The MLX + override remains restricted to Apple Silicon; TFLite remains restricted to + supported Linux/Windows x64 targets. +- `LSDJ_SA3_PREFERENCE=auto|gpu|cpu_tflite` is the backend-policy seam. An + explicit GPU request never silently falls back; while the release gate is + incomplete it fails before launch and asks the caller to confirm TFLite. + +Both adapters consume the same `GenerationRequest` contract and share one +argument translator. A populated control is either forwarded or rejected; it +is never silently discarded. + +## Feature matrix + +| Capability | MLX | TFLite | Windows CUDA candidate | Notes | +| --- | --- | --- | --- | --- | +| Music and SFX | Yes | Yes | Gated | Official Small Music/SFX models | +| Medium / 380 seconds | Yes | Yes | No | TFLite fallback; no unofficial FlashAttention build | +| Audio-to-audio | Yes | Yes | Gated | LSDJ normalizes input before every backend | +| Inpainting | Yes | Yes | Gated | Shared `inpaint_range` control | +| Continuation | Yes | Yes | Gated | Inpaint range from source duration to requested duration | +| Positive/negative prompt | Yes | Yes | Gated | Negative prompt requires CFG other than 1 | +| Seed, duration, steps, CFG, APG | Yes | Yes | Gated | CUDA maps directly to the pinned upstream Python API | +| Stacked LoRA with strength | Yes | Yes | Gated | Independent strength per adapter | +| Per-step LoRA gating | Yes upstream | No | No | Not exposed by LSDJ | +| Progress | Text stream | Text stream | Sampler callback | Normalized by LSDJ | +| Cancellation | Process stop | Process stop | Callback + process stop | CUDA also yields to realtime MRT2 | +| Partial audio preview | No | No | No | No pinned backend returns partial audio | + +The `/api/sa3/status` endpoint reports the preference choices, active backend, +readiness, capabilities, real limitations, current queued/running state, and on +Windows the CUDA release gate and qualification blockers. + +## Windows CUDA process and scheduling model + +The CUDA adapter calls the official pinned Python API; no upstream code is +copied into LSDJ. Each request runs in a disposable child and checks an exact +provenance stamp before heavyweight imports. It then verifies the shared +package versions, CUDA runtime, NVIDIA driver, device, reported memory, and the +measured reservation before loading Small Music or Small SFX. It never invokes +the Hub downloader and never falls back to PyTorch CPU. + +The managed launcher binds the private request to that child with an ephemeral +secret inherited through an allowlisted environment entry; the request contains +only its SHA-256. The worker verifies and removes the secret before importing +upstream code. Every structured event carries a bounded job ID, while secrets, +prompts, paths, and ambient credentials are excluded from diagnostics. + +The file-locked GPU broker is shared with MRT2 across processes. MRT2 leases +have realtime priority. SA3 is admitted only when no MRT2 lease or waiter is +present and its measured reservation fits the conservative budget. If MRT2 +arrives during sampling, the next upstream callback cancels the SA3 child. A +daemon watchdog provides the same hard process boundary while upstream model +loading or decoding offers no callback. Process exit releases the CUDA context. +OOM, driver reset, worker crash, and app exit are contained by the same process +boundary and the native process-tree supervisor. + +The candidate lock resolves the pinned SA3 requirements as PyTorch/torchaudio +2.7.1+cu126 and Hugging Face Hub 1.7.1. This differs from #110's MRT2 candidate, +so it is a shared-runtime hypothesis rather than a production upgrade. MRT2 +must pass its parity and dual-deck matrix on this exact lock. LSDJ will not ship +a second multi-gigabyte PyTorch environment if that qualification fails. + +## Audio boundary + +LSDJ accepts bounded, uncompressed integer PCM WAV input (8/16/24/32 bit, +8–384 kHz, mono through 32 channels). It converts internally to the official +runtime format: 44.1 kHz, stereo, PCM16. Mono is duplicated; multichannel input +uses its first two channels, matching the official TFLite path. No system +`ffmpeg`, shell, or media executable is invoked. + +Generation remains outside the audio callback and is serialized across both +backends. The TFLite adapter caps XNNPACK and common numeric runtimes at four +threads by default (configurable from 1–8) and launches at background priority. +Timeouts and output bounds stop a wedged or runaway request. Platform hardware +runs still need to establish practical RAM/CPU admission thresholds while both +MRT2 decks are active. + +Every generated file must be a non-empty 44.1 kHz stereo PCM16 WAV with exactly +`round(seconds * 44100)` frames before it can enter LSDJ's library/player. +Corrupt, truncated, oversized, or wrong-duration output fails the request. + +## Pinned upstream and storage + +The machine-readable trust handoff is +[`sa3-tflite-pin.json`](../sa3-tflite-pin.json): + +- code: `Stability-AI/stable-audio-3` at + `a0b57f5483c4588f827f3552b7d5c6ca2a9687be`; +- models: `stabilityai/stable-audio-3-optimized` at + `6736003cb57d06b7b1fdc36fad31b2a3709e4774`; +- eight fp32 model artifacts carry exact byte counts and SHA-256 digests; +- the official runtime dependency surface is resolved into the universal, + hash-locked `scripts/sa3-tflite-requirements.lock`. + +Measured download totals, including the shared T5Gemma encoder, are: + +- Small Music: 2,836,149,512 bytes; +- Small SFX: 2,836,149,512 bytes; +- both Small models together (shared files deduplicated): 4,674,908,056 bytes; +- Medium: 10,027,905,456 bytes; +- all three models (shared files deduplicated): 14,138,994,904 bytes. + +The app installer must download these pinned artifacts, verify them, and write +the warm/readiness stamp plus `.lsdj-provenance.json` before generation. That +stamp records the exact runtime and model repositories/revisions above; a +missing or mismatched stamp is a failed state. The runtime process receives +`HF_HUB_OFFLINE=1` and no Hugging Face token, so a missing file fails closed +instead of using upstream's mutable first-run downloader. Licensing, +attribution, acknowledgement, and credential UX remain owned by issue #108. + +## Evidence and remaining gates + +The repository suite exercises backend selection, argument parity, every LSDJ +control, PCM normalization, exact output validation, corrupt output, progress, +cancellation, timeouts, missing assets, and the 380-second command contract +without loading model weights. + +This does **not** claim a real model run. The hardened #107 installer now +consumes both manifests, verifies every byte, builds the isolated environment, +warms all three model pairs, and atomically promotes or rolls back the candidate. +Before release, Ubuntu plus Windows hardware runs must still verify Small/Medium +generation, LoRA, cancellation, storage, RAM/CPU use, and coexistence with both +active MRT2 decks. Partial preview and structured progress require a future +upstream API; they are reported as limitations today. diff --git a/docs/third-party-model-notices.md b/docs/third-party-model-notices.md new file mode 100644 index 0000000..76a9320 --- /dev/null +++ b/docs/third-party-model-notices.md @@ -0,0 +1,148 @@ +# Third-party model and runtime notices + +Audit date: 2026-08-08. Inventory version: `2026-08-08.4`. + +This document is a human-readable projection of +[`compliance/model-assets.json`](../compliance/model-assets.json). The JSON +manifest is authoritative for exact revisions and validation. This document +records upstream statements and open decisions; it is not legal advice and does +not assert that any project use or redistribution path has been approved. + +## Release blockers found by this audit + +1. **LSDJ has no project license file at the audited base revision.** Project + owners must select the LSDJ code license and notice before publishing a + licensed source or binary release. +2. **The PyTorch MRT2 converted weights have conflicting provenance metadata.** + Their cards say Apache-2.0 while also saying the weights are re-keyed, + numerically identical copies of Google's CC-BY-4.0 weights. The effective + redistribution, attribution, and acknowledgement path is an owner-review + gate. Re-keying must not be presented as relicensing. +3. **Current model downloaders are mutable.** `magenta-rt 2.0.2` downloads the + Google model repository without `revision=`; pinned Stable Audio code does the + same for `stabilityai/stable-audio-3-optimized`; the LoRA importer fetches + `resolve/main`. The inventory records audited snapshots, but releases must + make the runtime request those exact revisions and verify technical + provenance. +4. **The documented Maqam LoRA has no identified license grant.** Its card says + `license: other` and supplies no license file. Keep it a user-directed, + upstream reference. Do not mirror, bundle, or label it official until its + author and project owners confirm the path. +5. **Two optimized-model source revisions are not identified upstream.** The + pinned Stable Audio optimized snapshot names the T5Gemma and Stable Audio 3 + Medium source repositories, but does not identify the exact revisions used to + produce the derivative artifacts. The inventory records those revisions as + unresolved rather than substituting the repositories' audit-time heads. +6. **The optional Windows CUDA Small Music and Small SFX snapshots remain + gated and incomplete.** Their immutable revisions and root weight hashes are + pinned, but authenticated terms review, configuration and nested-component + hashes, measured VRAM reservations, and physical Windows/NVIDIA qualification + are incomplete. The installer and runtime fail closed; TFLite remains the + supported path. + +No reviewed third-party model weights belong in installers while their manifest +entry has `redistribution_confirmed: false`. + +## Inventory summary + +| Asset | Exact revision | Code license | Weight/asset terms | Acquisition | Gate | +| --- | --- | --- | --- | --- | --- | +| LSDJ code | `c9cd822ef6cbb86711e72d35f0f7e50a126d666f` | Unresolved: no project LICENSE/NOTICE found | n/a | Bundled app code | Owners select license and notice | +| Google MRT2 Python runtime (`magenta-rt 2.0.2`) | source `4bf995bdd9c29b818543574e1b3a6e67867c9a58`; wheel SHA-256 in manifest | Apache-2.0 | n/a | Hash-locked bundled sidecar dependency | Package notices not yet wired | +| Google MRT2 weights/resources | `010aa0dcb0dfd27b24f0ad07b4dad63e8f9521cc` | n/a | CC-BY-4.0 plus model-card usage statement | Download, not installer | Runtime pin + owner attribution decision | +| Apolinario/multimodalart PyTorch port | `6d076baa3df3b10448876c400521a015a5137c59` | Apache-2.0 | n/a | Credited implementation reference; not acquired or executed | #108 notice review gate | +| PyTorch MRT2 base | `92087988d05d0fe38b11f021f0b0d00a75afb86b` | Card declares Apache-2.0 remote code | Card says Apache-2.0; underlying Google weights say CC-BY-4.0 | Native exact download | License ambiguity must be resolved | +| PyTorch MRT2 small | `7037d99551c84ac5c6afb7f1a5e58c65e7233dbb` | Card declares Apache-2.0 remote code | Card says Apache-2.0; underlying Google weights say CC-BY-4.0 | Native exact download | License ambiguity must be resolved | +| PyTorch MusicCoCa processor | `236c488e38aa98643805514996934d705668298b` | Conversion-code treatment pending | CC-BY-4.0 | Native exact download | Confirm notice path | +| Stable Audio 3 runtime source | `a0b57f5483c4588f827f3552b7d5c6ca2a9687be` | MIT | n/a | Exact source archive download | Carry MIT notice | +| Stable Audio 3 optimized MLX/TFLite assets | `6736003cb57d06b7b1fdc36fad31b2a3709e4774` | n/a | Stability AI Community License plus Gemma Terms for T5Gemma components | Download, not installer | Runtime pin, owner path, acknowledgement | +| Stable Audio 3 Small Music PyTorch/CUDA | `0fef1392cd842149a2b6d445e181c97608faac06`; root weight SHA-256 in manifest | n/a | Unresolved pending authenticated gated-model review | Optional gated download, disabled | Config/nested hashes, #108 terms, VRAM, and Windows qualification | +| Stable Audio 3 Small SFX PyTorch/CUDA | `ae12755283df9d62ca39a9b050a39a0b607b8c20`; root weight SHA-256 in manifest | n/a | Unresolved pending authenticated gated-model review | Optional gated download, disabled | Config/nested hashes, #108 terms, VRAM, and Windows qualification | +| Google T5Gemma B-B UL2 source model | Exact conversion-source revision unresolved | n/a here | Gemma Terms of Use | Direct source is manually gated; LSDJ consumes Stability's optimized derivative | Identify source revision; owner derivative/notice decision | +| Stable Audio 3 Medium upstream source family | Exact conversion/training-source revision unresolved | n/a | Stability AI Community License plus Gemma Terms | Provenance reference only; direct source is gated | Identify source revision; base terms follow optimized model/LoRA review | +| Motif Maqam LoRA | `3e1d9aa6fcb72a619b4ced00a240c5039f76daf0` | n/a | Unresolved (`license: other` only); Stable Audio base terms also relevant | User-directed upstream download | No mirroring/bundling; runtime pin required | + +## Notice inputs + +### Google Magenta RealTime 2 + +- Runtime source: [Apache-2.0 at the locked source commit](https://github.com/magenta/magenta-realtime/blob/4bf995bdd9c29b818543574e1b3a6e67867c9a58/LICENSE). +- Weights and shared MusicCoCa/SpectroStream resources: + [CC-BY-4.0 model card at the audited snapshot](https://huggingface.co/google/magenta-realtime-2/blob/010aa0dcb0dfd27b24f0ad07b4dad63e8f9521cc/README.md). +- Attribution input: Magenta RealTime 2, authors Google DeepMind; copyright + 2026 Google LLC. Link the exact model card and CC-BY-4.0 legal code. +- The card asks users to act responsibly and not generate content that infringes + or violates others' rights. Include that link in the model disclosure rather + than paraphrasing it as a new LSDJ license term. + +### PyTorch MRT2 port and snapshots + +- Port source: [Apache-2.0 at `6d076…`](https://github.com/multimodalart/magenta-realtime-torch/blob/6d076baa3df3b10448876c400521a015a5137c59/LICENSE). +- The [base](https://huggingface.co/magenta-community/magenta-realtime-2/blob/92087988d05d0fe38b11f021f0b0d00a75afb86b/README.md) + and [small](https://huggingface.co/magenta-community/magenta-realtime-2-small/blob/7037d99551c84ac5c6afb7f1a5e58c65e7233dbb/README.md) + cards label the snapshots Apache-2.0 and state that their weights are re-keyed + and numerically identical to the Google checkpoint. +- Google's source weights are [declared CC-BY-4.0](https://huggingface.co/google/magenta-realtime-2/blob/010aa0dcb0dfd27b24f0ad07b4dad63e8f9521cc/README.md). + Show both statements until owners resolve the effective path. +- The [MusicCoCa processor snapshot](https://huggingface.co/magenta-community/magenta-rt-musiccoca-torch/tree/236c488e38aa98643805514996934d705668298b) + is declared CC-BY-4.0 and is also a converted Google artifact. + +### Stable Audio 3 and T5Gemma + +- Stable Audio source is [MIT at the pinned commit](https://github.com/Stability-AI/stable-audio-3/blob/a0b57f5483c4588f827f3552b7d5c6ca2a9687be/LICENSE), + copyright 2026 Stability AI. +- The exact optimized model snapshot carries the + [Stability AI Community License](https://huggingface.co/stabilityai/stable-audio-3-optimized/blob/6736003cb57d06b7b1fdc36fad31b2a3709e4774/LICENSE.md), + [Gemma Terms](https://huggingface.co/stabilityai/stable-audio-3-optimized/blob/6736003cb57d06b7b1fdc36fad31b2a3709e4774/LICENSE_GEMMA.md), + and a [NOTICE](https://huggingface.co/stabilityai/stable-audio-3-optimized/blob/6736003cb57d06b7b1fdc36fad31b2a3709e4774/NOTICE). +- Required notice inputs from those upstream files include the Stability AI + Community attribution, a “Powered by Stability AI” display, and the Gemma + terms notice. The application surface must also link Stability's + [acceptable-use policy](https://stability.ai/use-policy) and + [privacy policy](https://stability.ai/privacy-policy). +- The original [T5Gemma repository](https://huggingface.co/google/t5gemma-b-b-ul2) + is manually gated on Hugging Face. It requires an account and explicit Gemma + terms acceptance for direct access. The reviewed evidence does not identify + the exact revision used for Stability's optimized derivative. That derivative + is anonymously downloadable, but its pinned repository says T5Gemma is + redistributed under the Gemma Terms. +- The optional Windows CUDA path separately pins [Small Music at + `0fef139…`](https://huggingface.co/stabilityai/stable-audio-3-small-music/tree/0fef1392cd842149a2b6d445e181c97608faac06) + and [Small SFX at + `ae12755…`](https://huggingface.co/stabilityai/stable-audio-3-small-sfx/tree/ae12755283df9d62ca39a9b050a39a0b607b8c20). + Public metadata supplied each root weight hash, but the authenticated audit + has not completed the configuration/nested hashes or exact terms and notices. + These are optional fail-closed entries: they are neither installable nor + advertised while `releaseReady` or `gatedArtifactsComplete` is false. + +## LoRA inventory and user imports + +At the audited revision, `bundled_lora_ids` and `official_lora_ids` are empty. +LSDJ accepts arbitrary user-supplied safetensors and can download a repository +the user names; that generic capability is not an official catalog. + +The only adapter named in LSDJ documentation/tests is +[`motiftechnologies/stable-audio-3-maqam-lora@3e1d…`](https://huggingface.co/motiftechnologies/stable-audio-3-maqam-lora/tree/3e1d9aa6fcb72a619b4ced00a240c5039f76daf0). +Its card identifies Motif Technologies, a Stable Audio 3 Medium base, and +`license: other`, but contains no license text or redistribution permission. +Show that provenance and unresolved status. Do not mirror or bundle it. + +For every user-imported LoRA, show that LSDJ does not verify the user's rights, +that the adapter may carry independent terms, and that the base-model terms may +still apply. Never infer a license from `.safetensors` format or public access. + +## Distribution and credential rules for follow-up implementation + +- Keep all model weights out of installers until an exact manifest entry records + owner-confirmed redistribution permission. +- Downloads must use `revision=` (or an equivalent immutable URL), + verify expected provenance/hashes, and fail closed rather than falling back to + `main` or another mutable reference. +- Before a download whose terms require acknowledgement, show exact model, + revision, license/terms, notice, privacy, and acceptable-use links. Store the + acknowledgement against the inventory revision and asset revision. +- A gated upstream may require a user token after terms acceptance. Credentials + must be redacted from errors/logs and placed in the OS credential store or kept + intentionally ephemeral—never in plaintext application data. +- The LSDJ license must be presented separately and must say explicitly that it + does not relicense third-party model weights or adapters. diff --git a/docs/windows-release-checklist.md b/docs/windows-release-checklist.md new file mode 100644 index 0000000..3fe6095 --- /dev/null +++ b/docs/windows-release-checklist.md @@ -0,0 +1,79 @@ +# Issue #113 — Windows 11 x64 release qualification + +This checklist is intentionally unchecked where physical hardware, a selected +signing provider, or current security definitions are required. Hosted CI +evidence must not be substituted for these items. + +## Release identity and installer operations + +- [ ] Select an Authenticode provider/certificate and record the exact subject, + leaf thumbprint, timestamp service, legal owner, and expected Explorer + publisher text. +- [ ] Configure a protected `windows-release` Environment with separate approval, + no administrator bypass, and a least-privilege CI identity. +- [ ] Document provider credential/key storage, access review, rotation, expiry, + compromise, incident, and revocation procedures; perform one revocation drill. +- [ ] Confirm the protected provider exposes the reviewed one-file signing wrapper + contract used by `scripts/sign-windows.ps1`. +- [ ] On a clean Windows 11 x64 account, verify the final NSIS installer, installed + app, uninstaller, and every executable payload have the exact signer and a + trusted timestamp. +- [ ] Install, upgrade, attempt a downgrade, uninstall with preservation, and + uninstall with explicit data removal. Record screenshots of the disclosed + `%LOCALAPPDATA%\LSDJ` path and measured size. +- [ ] Repeat installation under a profile containing spaces and non-ASCII text, + with Windows long-path support disabled. +- [ ] Confirm Start menu behavior, window/titlebar, file/folder dialogs, + notifications, opener/trash behavior, packaged resources, update/restart, and + WebView2 present/missing/offline failure cases. + +## Hardware record + +Record exact Windows build, CPU, RAM, GPU, VRAM, NVIDIA driver, PyTorch/CUDA +runtime, audio device/driver, WASAPI rate/format/buffer, MIDI devices, FLX4 +firmware/driver, security state, LSDJ revision, and model/runtime revisions. + +- [ ] Establish and document the minimum NVIDIA GPU, VRAM, driver, CPU, RAM, and + free-disk floor from measured results. +- [ ] Run both MRT2 decks for at least ten minutes at 25 frames / approximately + one second. Require zero engine-reported underruns and capture p50/p95/p99 + generation latency, queue depth, temperature, and throttling. +- [ ] Run both armed decks for at least ten minutes at 5 frames / approximately + 200 ms with the same zero-underrun and telemetry gate. +- [ ] Validate default-device selection/change, WASAPI shared-mode 48 kHz and + non-48 kHz devices, stereo output, FLX4 four-channel master/cue, removal, + renegotiation, and sleep/resume. +- [ ] Validate FLX4 WinMM naming, transport, mixer, jog wheels, performance pads, + LEDs, required SysEx, hotplug/reconnect, and actionable device-contention errors. +- [ ] Run Stable Audio music, SFX, audio-to-audio, continuation, inpainting, + Small/Medium, LoRA, cancellation, and long-duration validation while both decks + remain active. Record CPU/RAM impact and deck telemetry. +- [ ] Confirm normal quit, forced host exit, worker crash, update, and uninstall + leave no Python, model, or GPU worker descendants. + +## Security and release response + +- [ ] Test the release candidate against current Microsoft Defender definitions; + record platform, engine, intelligence versions, detection result, and submission + ID/disposition for any false-positive report. +- [ ] Exercise the documented signing-key compromise and bad-signature release + stop path without publishing a release. +- [ ] Confirm all model services bind to `127.0.0.1`, the installer creates no + firewall exception, and no public listener appears during first run or playback. +- [ ] Confirm a clean machine installs verified MRT2 and Stable Audio + runtimes/models without system Python, Git, CUDA toolkit, WSL, compiler, or shell. +- [ ] Interrupt and corrupt each runtime/model download and promotion; the prior + verified version must remain usable and diagnostics must identify recovery. +- [ ] Confirm the single publisher refuses missing, unsigned, invalid, + untimestamped, duplicate, or unexpected Windows artifacts. + +## External blockers + +- Authenticode provider/certificate, exact publisher subject, timestamp service, + protected CI identity, and credential lifecycle decisions. +- Physical Windows 11 x64 + supported NVIDIA host with current drivers. +- Pioneer/AlphaTheta DDJ-FLX4 plus representative WASAPI devices. +- Current Defender/SmartScreen observation on the signed release candidate. +- #110 production runtime installation and NVIDIA qualification. +- #111 TFLite Stable Audio runtime installation and parity qualification. +- #108 final notices/acknowledgement and repository licensing decisions. diff --git a/docs/windows.md b/docs/windows.md new file mode 100644 index 0000000..e4d8403 --- /dev/null +++ b/docs/windows.md @@ -0,0 +1,205 @@ +# Windows 11 x64 packaging and operations + +Windows is a release candidate, not yet a supported LSDJ platform. The software +packaging and hosted-CI contracts in this document do not replace the unchecked +NVIDIA, WASAPI, MIDI, FLX4, sleep/resume, and Defender qualification in +[`windows-release-checklist.md`](windows-release-checklist.md). + +## Installer contract + +LSDJ produces one x64 NSIS `-setup.exe` for Windows 11. It installs for the +current user and does not request administrator privileges or add a firewall +rule. The installer creates an LSDJ Start menu entry, records calendar-version +metadata derived from the protected `vYYYY.MM.N` release tag, permits upgrades, +and refuses downgrades. MSI, Microsoft Store, portable ZIP, Windows 10, Windows +on ARM, and per-machine installation are outside this release. + +Application files and app-managed files use a deliberately shallow +`%LOCALAPPDATA%\LSDJ` tree so Python environments and model paths continue to +work when Windows long-path support is disabled: + +- `config` — LSDJ settings; +- `data` — generated songs, samples, and user registries; +- `cache` — reproducible cache data; +- `assets` — verified model weights and managed runtimes; +- `staging` — interrupted candidates on the same filesystem as `assets`. + +The packaged shell enables the `managed-runtime` feature. If the verified +MRT2 or Stable Audio service manifest is absent or invalid, that service remains +unavailable. The services are independently installed, resolved, and verified; +one service cannot redirect the other to its executable or dependency tree. +Managed builds never fall through to a system Python, `uv`, Git, a CUDA toolkit, +WSL, or a shell command. #110 and #111 remain responsible for their respective +PyTorch MRT2 and TFLite Stable Audio implementations. + +## Upgrade and uninstall + +An upgrade replaces application payloads in place and preserves the entire +app-managed tree. A normal uninstall removes the app binary, declared packaged +resources, registry entry, and shortcuts, while preserving downloaded models, +runtimes, settings, and user data. + +The graphical uninstaller offers an unchecked option to remove the preserved +data. If selected, it calculates the tree size and presents a second confirmation +showing `%LOCALAPPDATA%\LSDJ` and the measured KiB before deletion. Removal is +allowed only while the installer-owned `.lsdj-data-root` marker is a plain file +containing LSDJ's exact application identifier; a same-named, empty, linked, or +reparse-point entry is not sufficient. Marker creation uses exclusive Windows +file creation, and marker validation opens the reparse entry itself while +denying write/delete sharing, so neither operation follows or overwrites a link +raced into the marker path. + +Before installation, a hidden, always-selected NSIS section runs before Tauri's +path-creating `SetOutPath`. It canonicalizes the target, requires it to be +exactly `%LOCALAPPDATA%\LSDJ`, and records whether the root was absent or already +owned. The pre-install hook then accepts or creates a new root only when that +early probe saw it absent and the result is a plain empty directory. Creation in +the hook is required when `/D` puts application binaries somewhere other than +the fixed LocalAppData data root. A pre-existing empty root is rejected rather +than guessed to be LSDJ-owned. A pre-existing markerless root is adopted only +when its top level is exactly the five plain directories created by LSDJ's path +contract (`config`, `data`, `cache`, `assets`, and `staging`); foreign, partial, +file-bearing, and recursively reparse-bearing layouts are rejected. Root +junctions, symlinks, and other reparse points are always rejected. + +Explicit purge scans the tree without traversing reparse points before measuring +it, repeats the canonical-root, ownership-marker, and tree checks immediately +before deletion, and uses a custom recursive walk that never follows links or +uses a broad `RMDir /r`. A reparse point or marker replacement fails the purge +before ordinary uninstall payload deletion when detected during the initial +check, and always preserves the remaining tree. The equivalent explicit +automation switch is `/PURGE-LSDJ-DATA`; `/S` alone always preserves data. There +is no caller-supplied recursive target. + +NSIS installed uninstallers are self-copy launchers. A direct invocation such +as `uninstall.exe /S /PURGE-LSDJ-DATA` reports whether the temporary worker +started, not the worker's final script status. Automation that must distinguish +a completed uninstall (`0`) from a fail-closed refusal (`2`) must copy the +uninstaller to a unique temporary executable, run that copy while waiting with +final unquoted `_?=`, and then remove the copy. This is the +NSIS-documented worker form; the `_?=` argument must remain last so paths with +spaces are preserved. + +## WebView2 + +Windows 11 normally receives the evergreen WebView2 Runtime with Windows. LSDJ's +NSIS installer still uses Tauri's `downloadBootstrapper` mode when the runtime is +missing. This keeps the installer small and lets Microsoft's evergreen runtime +receive security updates independently. + +If WebView2 is already present, installation works offline. If it is absent, the +bootstrapper needs a network connection; download or installation failure aborts +with an actionable message instead of installing an app that cannot open. Users +may install Microsoft's WebView2 Evergreen Runtime separately and retry. Moving +to Tauri's roughly 127 MB `offlineInstaller` mode is a future reviewed release +policy change, not an automatic fallback. + +## Local services and firewall + +The Rust host allocates ephemeral ports and every model service binds only to +`127.0.0.1`. The installer opens no inbound port, adds no public-network binding, +and creates no Windows Firewall exception. Child processes live in a Windows Job +Object and are terminated as a tree on quit or host failure. Hosted CI exercises +the process contract without model hardware; abnormal exit with real CUDA work +remains a physical-machine gate. + +## Authenticode release gate + +Unsigned development installers are produced only in pull-request CI and are +labelled `windows-x64-unsigned-development`. They are not release inputs. CI runs +the release verifier against them and requires rejection. + +A protected `windows-release` Environment gates the release producer. The repo +defines a provider-neutral, non-shell interface: + +- `WINDOWS_SIGN_COMMAND_PATH` maps to + `LSDJ_WINDOWS_SIGN_COMMAND_PATH`, an absolute protected path to a reviewed + wrapper that accepts exactly one file path; +- `WINDOWS_EXPECTED_CERTIFICATE_SHA1` maps to the exact approved leaf + certificate thumbprint; and +- `WINDOWS_EXPECTED_SUBJECT` maps to the exact approved certificate subject and + expected Windows publisher identity. + +The selected provider must provision its credentialed wrapper after Environment +approval. The wrapper owns key access and timestamp-server configuration. The +repo never accepts a command string, PFX, password, or unverified subject. The +protected preflight also requires the Windows SDK `signtool.exe`. Every sign +operation immediately requires a valid Authenticode chain, exact leaf thumbprint +and subject, a timestamp certificate, and successful `signtool /pa` verification. +The installed app, uninstaller, executable payloads, and final NSIS installer are +verified again before the producer bundle is uploaded. + +No provider, certificate, publisher subject, protected CI identity, key storage, +rotation process, or revocation process has been selected yet. Consequently the +release job intentionally fails at signing preflight today and no output from +this branch is represented as signed. The owner decisions and operational drill +are explicit gates in the release checklist. + +The single publisher has no signing credentials. It requires the exact +`macos-arm64`, `linux-x64`, and `windows-x64` producer set, recomputes sizes and +SHA-256 hashes, and refuses to create a public release if Windows production, +signature verification, or artifact verification is missing. + +## Defender and SmartScreen response + +Authenticode establishes publisher and file integrity; it does not guarantee +SmartScreen reputation or that Microsoft Defender and third-party products will +never flag a new build. + +For a report: + +1. Do not advise bypassing or disabling protection. Quarantine the artifact and + record the LSDJ version, download URL, SHA-256, signature status, Windows + build, Defender platform/engine/security-intelligence versions, and detection + name. +2. Compare the file with the release index and verify the expected Authenticode + subject, thumbprint, and timestamp. Treat any mismatch as a security incident; + keep the release private or withdraw it and begin the provider's revocation + procedure. +3. If identity and hashes match, reproduce on a clean Windows 11 system with + current definitions and submit the exact artifact to Microsoft's malware + analysis portal as a suspected false positive. Preserve the submission ID in + the linked GitHub issue. +4. Publish the vendor disposition. Rebuild only from the protected tag workflow; + never re-sign or replace a published asset by hand. + +The final expected publisher text, support contact, and certificate incident +owner must be filled in after the provider decision and before Windows support is +announced. + +## Diagnostics and known limitations + +- Confirm the installer hash against `windows-x64-SHA256SUMS.txt` and + `release-index.json`. +- In Explorer, open **Properties → Digital Signatures** and require the publisher + documented for the release. Do not install if the signature is absent or + invalid. +- Model/runtime data and partial-download staging live under + `%LOCALAPPDATA%\LSDJ`; include sizes and runtime/model revisions in a report, + but never attach model weights or credentials. +- Local service logs are bounded and credential-redacted. There is no remote + service or firewall troubleshooting step because the supported binding is + loopback only. +- The minimum NVIDIA GPU, VRAM, driver, PyTorch/CUDA runtime, CPU, RAM, and free + disk are deliberately unspecified until measured qualification completes. +- WASAPI formats and device recovery, WinMM MIDI, FLX4 routing and LEDs, + sleep/resume, and model performance are not qualified by hosted CI. + +## Build and CI + +`scripts/build-windows-installer.ps1` accepts an exact release-shaped tag and +derives Windows/Tauri version metadata. `-UnsignedDevelopment` passes Tauri's +`--no-sign` and is the only pull-request build mode. `-Release` loads the +sign-command configuration and fails before packaging when protected identity +configuration is unavailable. + +Hosted `windows-2025` CI builds two unsigned versions, installs and upgrades +them, rejects a downgrade, verifies default preservation and explicit purge, +checks version metadata and the Start menu shortcut, and exercises a conservative +sub-`MAX_PATH` install location containing spaces and Unicode. It also attacks +ownership with pre-existing empty and foreign roots, install-time and purge-time +root junctions, marker junctions, a marker replacement between confirmation and +deletion, and a nested junction; every outside target must remain untouched. The +fresh-install success case plus empty-root rejection also guards the required +ordering of the early ownership probe relative to Tauri's `SetOutPath`. These +checks do not claim hardware or antivirus qualification. diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index bce9983..d3b7d36 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -38,6 +38,7 @@ import { BeatView } from './mixer/BeatView' import { MixerStrip, type ChannelControls } from './mixer/MixerStrip' import { RecordControl } from './mixer/RecordControl' import { AccentPicker } from './ui/AccentPicker' +import { AudioOutputHealth } from './ui/AudioOutputHealth' import { OutputDevicePicker } from './ui/OutputDevicePicker' import { BeatViewPicker } from './ui/BeatViewPicker' import { Switch } from './ui/Switch' @@ -883,6 +884,7 @@ function App() { value={cueDevice} mainDeviceName={mainDevice} /> + {/* Where master-bus recordings are saved. Empty = the OS Downloads diff --git a/frontend/src/audio/nativeEngine.test.ts b/frontend/src/audio/nativeEngine.test.ts index 188571d..e469cb7 100644 --- a/frontend/src/audio/nativeEngine.test.ts +++ b/frontend/src/audio/nativeEngine.test.ts @@ -3,6 +3,9 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { SAMPLE_RATE } from './types' import { createNativeEngine, + fetchGenerationApi, + getAudioOutputHealth, + reconnectAudioOutputs, styleAddTarget, styleMoveTarget, styleSetCursor, @@ -86,6 +89,147 @@ const SNAP = { } describe('createNativeEngine — control contract', () => { + it('authenticates generation requests with the in-memory launch capability', async () => { + const invoke = vi.fn((cmd: string) => + cmd === 'app_info' + ? Promise.resolve({ + generationPort: 4321, + generationCapability: 'b'.repeat(64), + }) + : Promise.resolve(undefined), + ) + vi.stubGlobal('__TAURI__', { core: { invoke } }) + const fetchMock = vi.fn(async (_url: string, _init: RequestInit) => { + void _url + void _init + return { ok: true } + }) + vi.stubGlobal('fetch', fetchMock) + + await fetchGenerationApi('/api/models') + + expect(fetchMock).toHaveBeenCalledTimes(1) + const [url, init] = fetchMock.mock.calls[0] + expect(url).toBe('http://127.0.0.1:4321/api/models') + expect((init.headers as Headers).get('x-lsdj-capability')).toBe('b'.repeat(64)) + }) + + it('routes Magenta requests to the distinct native gateway', async () => { + const invoke = vi.fn((cmd: string) => + cmd === 'app_info' + ? Promise.resolve({ + generationPort: 4321, + generationCapability: 's'.repeat(64), + magentaPort: 9876, + magentaCapability: 'm'.repeat(64), + }) + : Promise.resolve(undefined), + ) + vi.stubGlobal('__TAURI__', { core: { invoke } }) + const fetchMock = vi.fn(async (_url: string, _init: RequestInit) => { + void _url + void _init + return { ok: true } + }) + vi.stubGlobal('fetch', fetchMock) + + await fetchGenerationApi('/api/render', { method: 'POST' }) + await fetchGenerationApi('/api/generate', { method: 'POST' }) + + const [renderUrl, renderInit] = fetchMock.mock.calls[0] + expect(renderUrl).toBe('http://127.0.0.1:9876/api/render') + expect((renderInit.headers as Headers).get('x-lsdj-capability')).toBe('m'.repeat(64)) + const [generateUrl, generateInit] = fetchMock.mock.calls[1] + expect(generateUrl).toBe('http://127.0.0.1:4321/api/generate') + expect((generateInit.headers as Headers).get('x-lsdj-capability')).toBe('s'.repeat(64)) + }) + + it('fails closed when a managed Magenta gateway could not bind', async () => { + const invoke = vi.fn((cmd: string) => + cmd === 'app_info' + ? Promise.resolve({ + generationPort: 4321, + generationCapability: 's'.repeat(64), + magentaPort: null, + magentaCapability: null, + }) + : Promise.resolve(undefined), + ) + vi.stubGlobal('__TAURI__', { core: { invoke } }) + const fetchMock = vi.fn(async (_url: string, _init: RequestInit) => { + void _url + void _init + return { ok: true } + }) + vi.stubGlobal('fetch', fetchMock) + + await expect(fetchGenerationApi('/api/render', { method: 'POST' })).rejects.toThrow( + 'authentication is unavailable', + ) + expect(fetchMock).not.toHaveBeenCalled() + }) + + it('refreshes app_info when a first SA3 install starts the service', async () => { + let appInfoCalls = 0 + const invoke = vi.fn((cmd: string) => { + if (cmd !== 'app_info') return Promise.resolve(undefined) + appInfoCalls += 1 + return Promise.resolve( + appInfoCalls === 1 + ? { generationPort: null, generationCapability: null } + : { generationPort: 2468, generationCapability: 'n'.repeat(64) }, + ) + }) + vi.stubGlobal('__TAURI__', { core: { invoke } }) + const fetchMock = vi.fn(async (_url: string, _init: RequestInit) => { + void _url + void _init + return { ok: true } + }) + vi.stubGlobal('fetch', fetchMock) + + await fetchGenerationApi('/api/generate', { method: 'POST' }) + + expect(appInfoCalls).toBe(2) + const [url, init] = fetchMock.mock.calls[0] + expect(url).toBe('http://127.0.0.1:2468/api/generate') + expect((init.headers as Headers).get('x-lsdj-capability')).toBe('n'.repeat(64)) + }) + + it('uses the promoted SA3 connection on the next request without retrying either POST', async () => { + let appInfoCalls = 0 + const invoke = vi.fn((cmd: string) => { + if (cmd !== 'app_info') return Promise.resolve(undefined) + appInfoCalls += 1 + return Promise.resolve( + appInfoCalls === 1 + ? { generationPort: 1111, generationCapability: 'o'.repeat(64) } + : { generationPort: 2222, generationCapability: 'n'.repeat(64) }, + ) + }) + vi.stubGlobal('__TAURI__', { core: { invoke } }) + const fetchMock = vi.fn(async (_url: string, _init: RequestInit) => { + void _url + void _init + return { ok: true } + }) + vi.stubGlobal('fetch', fetchMock) + + await fetchGenerationApi('/api/generate', { method: 'POST' }) + await fetchGenerationApi('/api/generate', { method: 'POST' }) + + expect(appInfoCalls).toBe(2) + expect(fetchMock).toHaveBeenCalledTimes(2) + expect(fetchMock.mock.calls[0][0]).toBe('http://127.0.0.1:1111/api/generate') + expect((fetchMock.mock.calls[0][1]?.headers as Headers).get('x-lsdj-capability')).toBe( + 'o'.repeat(64), + ) + expect(fetchMock.mock.calls[1][0]).toBe('http://127.0.0.1:2222/api/generate') + expect((fetchMock.mock.calls[1][1]?.headers as Headers).get('x-lsdj-capability')).toBe( + 'n'.repeat(64), + ) + }) + it('createDeckChannel replays NO mixer config — the shell hydrates (phase C)', async () => { const engine = createNativeEngine() await engine.createDeckChannel( @@ -382,6 +526,29 @@ describe('createNativeEngine — output device', () => { const engine = createNativeEngine() await expect(engine.setMainDevice('FLX4')).rejects.toThrow('device busy') }) + + it('wires live output health and explicit reconnect commands', async () => { + const health = { + mainHealthy: false, + cueHealthy: false, + mainError: 'no output', + cueError: 'no output', + canReconnect: true, + } + const invoke = vi.fn((cmd: string, args?: unknown) => { + calls.push({ cmd, args }) + if (cmd === 'audio_output_health' || cmd === 'reconnect_audio_outputs') { + return Promise.resolve(health) + } + return Promise.resolve(undefined) + }) + vi.stubGlobal('__TAURI__', { core: { invoke } }) + + await expect(getAudioOutputHealth()).resolves.toEqual(health) + await expect(reconnectAudioOutputs()).resolves.toEqual(health) + expect(calls).toContainEqual({ cmd: 'audio_output_health', args: undefined }) + expect(calls).toContainEqual({ cmd: 'reconnect_audio_outputs', args: undefined }) + }) }) // The style intents (ADR-0020 phase B): continuous gestures (cursor, target diff --git a/frontend/src/audio/nativeEngine.ts b/frontend/src/audio/nativeEngine.ts index d554bff..3c1277e 100644 --- a/frontend/src/audio/nativeEngine.ts +++ b/frontend/src/audio/nativeEngine.ts @@ -60,20 +60,97 @@ export function isTauri(): boolean { return tauriGlobal() !== null } -let apiBaseUrlPromise: Promise | null = null +type ApiConnection = { baseUrl: string; capability: string | null } +type ApiConnections = { sa3: ApiConnection; magenta: ApiConnection } +let apiConnectionPromise: Promise | null = null +let apiConnectionOwner: TauriGlobal | null = null + +function loadApiConnections(): Promise { + const owner = tauriGlobal() + const unavailable = { baseUrl: '', capability: null } + if (!owner) return Promise.resolve({ sa3: unavailable, magenta: unavailable }) + // A webview has one bridge for its lifetime. Coupling the cache to that bridge + // also avoids carrying a stale launch capability across test/dev hot reloads. + if (apiConnectionOwner !== owner) { + apiConnectionOwner = owner + apiConnectionPromise = null + } + if (!apiConnectionPromise) { + apiConnectionPromise = invoke<{ + generationPort: number | null + generationCapability: string | null + magentaPort?: number | null + magentaCapability?: string | null + }>('app_info') + .then((info) => { + const sa3 = { + baseUrl: info.generationPort ? `http://127.0.0.1:${info.generationPort}` : '', + capability: info.generationCapability ?? null, + } + const hasDistinctMagentaGateway = + info.magentaPort !== undefined || info.magentaCapability !== undefined + return { + sa3, + // Bundled macOS reports no distinct gateway fields and deliberately + // retains the combined controller. Managed Linux/Windows supplies a + // Rust-owned Magenta endpoint here. Explicit nulls mean that gateway + // failed closed; they must never fall through to the SA3 controller. + magenta: hasDistinctMagentaGateway + ? { + baseUrl: info.magentaPort + ? `http://127.0.0.1:${info.magentaPort}` + : '', + capability: info.magentaCapability ?? null, + } + : sa3, + } + }) + .catch(() => ({ sa3: unavailable, magenta: unavailable })) + } + return apiConnectionPromise +} + +function isMagentaPath(path: string): boolean { + return path === '/api/render' || path === '/api/models' +} + +async function getApiConnection(path: string): Promise { + const magenta = isMagentaPath(path) + // SA3 is deliberately replaced during managed promotion, including both its + // port and capability. Resolve one fresh atomic app_info snapshot before each + // request; a POST is never retried against either the old or new process. + if (isTauri() && !magenta) apiConnectionPromise = null + let connections = await loadApiConnections() + let connection = magenta ? connections.magenta : connections.sa3 + // A fresh managed install legitimately starts without SA3. Do not pin that + // absence for the webview lifetime: the first request after promotion + // re-reads app_info and reaches the newly started generation server. + if (isTauri() && !connection.baseUrl) { + apiConnectionPromise = null + connections = await loadApiConnections() + connection = magenta ? connections.magenta : connections.sa3 + } + return connection +} /** Base URL for the backend `/api/*` generation endpoints (sa3/Magenta pad+track * render). FastAPI no longer serves the UI, so the Rust shell runs a generation * server on a loopback port it reports via `app_info`; the webview fetches - * `http://127.0.0.1:/api/...`. Resolved once and cached; falls back to '' - * (relative) if the port can't be resolved. */ + * `http://127.0.0.1:/api/...`. SA3 is resolved fresh because promotion + * replaces both the port and capability; missing connections fall back to ''. */ export function getApiBaseUrl(): Promise { - if (!apiBaseUrlPromise) { - apiBaseUrlPromise = invoke<{ generationPort: number | null }>('app_info') - .then((info) => (info.generationPort ? `http://127.0.0.1:${info.generationPort}` : '')) - .catch(() => '') + return getApiConnection('/api/generate').then((connection) => connection.baseUrl) +} + +/** Authenticated fetch to the app-owned loopback generation service. */ +export async function fetchGenerationApi(path: string, init: RequestInit = {}): Promise { + const connection = await getApiConnection(path) + if (isTauri() && !connection.capability) { + throw new Error('generation server authentication is unavailable') } - return apiBaseUrlPromise + const headers = new Headers(init.headers) + if (connection.capability) headers.set('x-lsdj-capability', connection.capability) + return fetch(`${connection.baseUrl}${path}`, { ...init, headers }) } /** The native MCP server endpoint + bearer token (ADR-0020 Phase 2), reported by @@ -101,6 +178,28 @@ export function setMcpPort(port: number): Promise { return invoke('set_mcp_port', { port }) } +/** Live health for the currently selected main/cue output topology. Combined cue + * follows the main stream; split cue follows its independent stream. */ +export type AudioOutputHealth = { + mainHealthy: boolean + cueHealthy: boolean + mainError: string | null + cueError: string | null + canReconnect: boolean +} + +/** Read current CPAL output health. Unlike the legacy `audioDeviceStarted` field, + * this reflects asynchronous stream failures after launch. */ +export function getAudioOutputHealth(): Promise { + return invoke('audio_output_health') +} + +/** Retry only unhealthy streams in the current topology. A failed split cue is + * rebuilt without interrupting a healthy main output. */ +export function reconnectAudioOutputs(): Promise { + return invoke('reconnect_audio_outputs') +} + /** Fire a command at the Rust engine (or a Tauri plugin, e.g. `plugin:dialog|open`). * Rejects (caught by callers that care) when the IPC bridge is absent — never * throws synchronously. Exported for the few non-engine native callers @@ -149,8 +248,8 @@ export type InstalledModel = { needsResources: boolean } -/** SA3's four readiness states (Rust `models`/`sa3.readiness`). */ -export type Sa3State = 'missing' | 'venv_missing' | 'not_warmed' | 'ready' +/** SA3 readiness states (Rust `models`/`sa3.readiness`). */ +export type Sa3State = 'missing' | 'venv_missing' | 'not_warmed' | 'ready' | 'failed' /** The source an SA3 checkout was installed from / is pinned to (Rust * `models::Sa3Source`): the `sa3-pin.json` repo + commit. */ @@ -183,7 +282,10 @@ export type ModelStatus = { } sa3: { state: Sa3State + backend: 'mlx' | 'tflite' | null sizeBytes: number + /** Exact model bytes from the selected backend's immutable manifest. */ + downloadBytes: number checkout: string | null /** What the installed checkout was fetched from (`null` when unstamped). */ installedSource: Sa3Source | null diff --git a/frontend/src/deck/DeckColumn.test.tsx b/frontend/src/deck/DeckColumn.test.tsx index 0c6248e..7c6d1cd 100644 --- a/frontend/src/deck/DeckColumn.test.tsx +++ b/frontend/src/deck/DeckColumn.test.tsx @@ -335,6 +335,18 @@ describe('DeckColumn', () => { expect(stat).toHaveClass('ui-stat--danger') }) + it('surfaces the upstream MRT2 negative-prompt limitation', () => { + renderPanel({ + connection: 'open', + runtimeDiagnostics: { + runtime: 'pytorch-cuda', + capabilities: { negative_prompt: false }, + }, + }) + const stat = screen.getByText('Negative prompt').parentElement! + expect(stat).toHaveTextContent('Unavailable upstream') + }) + it('disables transport while the worker is dead', () => { renderPanel({ connection: 'open', workerDied: true }) expect(screen.getByRole('button', { name: 'Play' })).toBeDisabled() diff --git a/frontend/src/deck/DeckColumn.tsx b/frontend/src/deck/DeckColumn.tsx index bf612da..b21c5c6 100644 --- a/frontend/src/deck/DeckColumn.tsx +++ b/frontend/src/deck/DeckColumn.tsx @@ -1041,6 +1041,12 @@ export function DeckColumn({ : 'default' } /> + {state.runtimeDiagnostics?.capabilities?.negative_prompt === false && ( + + )} )} diff --git a/frontend/src/deck/deckState.test.ts b/frontend/src/deck/deckState.test.ts index 938ad9f..caee7ef 100644 --- a/frontend/src/deck/deckState.test.ts +++ b/frontend/src/deck/deckState.test.ts @@ -48,14 +48,22 @@ describe('deckReducer', () => { expect(moved.bufferedSeconds).toBeCloseTo(1.9) }) - it('tracks generation speed from chunk events', () => { + it('tracks generation speed, latency, and queue depth from chunk events', () => { const state = reduce([ { type: 'server_event', - event: { event: 'chunk', index: 4, rtf: 1.86 }, + event: { + event: 'chunk', + index: 4, + rtf: 1.86, + generation_latency_ms: 107.5, + queue_depth: 2, + }, }, ]) expect(state.generationSpeed).toBe(1.86) + expect(state.generationLatencyMs).toBe(107.5) + expect(state.workerQueueDepth).toBe(2) }) it('surfaces worker errors and clears them when a style applies', () => { @@ -123,6 +131,49 @@ describe('deckReducer', () => { expect(state.model).toBe('mrt2_base') }) + it('retains runtime provenance from readiness diagnostics', () => { + const state = reduce([ + { + type: 'server_event', + event: { + event: 'ready', + deck: 'a', + model: 'mrt2_small', + runtime: { + runtime: 'pytorch-cuda', + accelerator: 'cuda', + hardware_qualified: false, + model_revision: 'model-sha', + cuda_device: 'NVIDIA Test', + }, + }, + }, + ]) + expect(state.runtimeDiagnostics).toMatchObject({ + runtime: 'pytorch-cuda', + hardware_qualified: false, + model_revision: 'model-sha', + cuda_device: 'NVIDIA Test', + }) + }) + + it('surfaces a fail-closed runtime startup error', () => { + const state = reduce([ + { + type: 'server_event', + event: { + event: 'startup_failed', + deck: 'a', + model: 'mrt2_small', + error: 'PyTorch reports no CUDA accelerator; MRT2 has no CPU fallback', + }, + }, + ]) + expect(state.workerDied).toBe(true) + expect(state.switchingModel).toBe(false) + expect(state.error).toContain('no CPU fallback') + }) + it('flags a dead worker; the transport drop arrives via the store projection', () => { const state = reduce([ { type: 'playing_changed', playing: true }, diff --git a/frontend/src/deck/deckState.ts b/frontend/src/deck/deckState.ts index 98dbf61..b22a2d1 100644 --- a/frontend/src/deck/deckState.ts +++ b/frontend/src/deck/deckState.ts @@ -2,9 +2,42 @@ * reducer, so the UI is a function of one state object and the stream's * health (buffer level, underruns) is always visible, never inferred. */ +export type Mrt2RuntimeDiagnostics = { + runtime: string + accelerator?: string + hardware_qualified?: boolean + experimental?: boolean + model_revision?: string + processor_revision?: string + upstream_source_revision?: string + torch_version?: string + torch_cuda_runtime?: string + nvidia_driver?: string | null + cuda_device?: string + cuda_capability?: number[] + cuda_total_memory_bytes?: number + capabilities?: { + weighted_prompts?: boolean + audio_style?: boolean + notes?: boolean + drums?: boolean + negative_prompt?: boolean + explicit_seed?: boolean + reset_to_reseed?: boolean + } +} + export type ServerEvent = - | { event: 'ready'; deck: string; model: string } - | { event: 'chunk'; index: number; rtf: number | null } + | { event: 'ready'; deck: string; model: string; runtime?: Mrt2RuntimeDiagnostics } + | { event: 'warming'; deck: string; model: string } + | { event: 'startup_failed'; deck: string; model: string; error: string } + | { + event: 'chunk' + index: number + rtf: number | null + generation_latency_ms?: number + queue_depth?: number | null + } | { event: 'style_applied' prompts: StylePrompt[] @@ -65,6 +98,9 @@ export type DeckState = { bufferedSeconds: number underruns: number generationSpeed: number | null + generationLatencyMs: number | null + workerQueueDepth: number | null + runtimeDiagnostics: Mrt2RuntimeDiagnostics | null error: string | null } @@ -88,6 +124,9 @@ export const initialDeckState: DeckState = { bufferedSeconds: 0, underruns: 0, generationSpeed: null, + generationLatencyMs: null, + workerQueueDepth: null, + runtimeDiagnostics: null, error: null, } @@ -132,8 +171,28 @@ function applyServerEvent(state: DeckState, event: ServerEvent): DeckState { model: event.model, switchingModel: false, workerDied: false, + runtimeDiagnostics: event.runtime ?? null, error: null, } + case 'warming': + return { + ...state, + model: event.model, + switchingModel: true, + workerDied: false, + error: null, + } + case 'startup_failed': + return { + ...state, + model: event.model, + switchingModel: false, + workerDied: true, + generationSpeed: null, + generationLatencyMs: null, + workerQueueDepth: null, + error: event.error, + } case 'model_loading': // The old worker (and its stream and prompt) is gone. Adopting the // target model now lets the RAM warning lead the load instead of @@ -148,9 +207,14 @@ function applyServerEvent(state: DeckState, event: ServerEvent): DeckState { generationSpeed: null, } case 'worker_died': - return { ...state, workerDied: true } + return { ...state, workerDied: true, generationLatencyMs: null, workerQueueDepth: null } case 'chunk': - return { ...state, generationSpeed: event.rtf } + return { + ...state, + generationSpeed: event.rtf, + generationLatencyMs: event.generation_latency_ms ?? null, + workerQueueDepth: event.queue_depth ?? null, + } case 'style_applied': return { ...state, diff --git a/frontend/src/deck/useDeck.test.tsx b/frontend/src/deck/useDeck.test.tsx index 426535f..e6ee97f 100644 --- a/frontend/src/deck/useDeck.test.tsx +++ b/frontend/src/deck/useDeck.test.tsx @@ -140,7 +140,7 @@ function installNativeTauri() { } // app_info feeds getApiBaseUrl(); null port → '' (relative fetches). return cmd === 'app_info' - ? Promise.resolve({ generationPort: null }) + ? Promise.resolve({ generationPort: null, generationCapability: "a".repeat(64) }) : Promise.resolve(undefined) }) class Channel { @@ -1957,7 +1957,7 @@ describe('useDeck realtime mirror + transport projection (ADR-0020)', () => { // Sever the harness echo: deck_play reaches Rust, no snapshot has landed yet. native.invoke.mockImplementation((cmd: string) => cmd === 'app_info' - ? Promise.resolve({ generationPort: null }) + ? Promise.resolve({ generationPort: null, generationCapability: "a".repeat(64) }) : Promise.resolve(undefined), ) @@ -2011,7 +2011,7 @@ describe('useDeck realtime mirror + transport projection (ADR-0020)', () => { // Sever the echo: the first tap's round-trip has not landed yet. native.invoke.mockImplementation((cmd: string) => cmd === 'app_info' - ? Promise.resolve({ generationPort: null }) + ? Promise.resolve({ generationPort: null, generationCapability: "a".repeat(64) }) : Promise.resolve(undefined), ) diff --git a/frontend/src/deck/useDeck.ts b/frontend/src/deck/useDeck.ts index d6a4f97..ba3eb6d 100644 --- a/frontend/src/deck/useDeck.ts +++ b/frontend/src/deck/useDeck.ts @@ -19,7 +19,7 @@ import { createLoudnessTracker, trimDbFor } from '../audio/master' import { STYLE_SAMPLE_SECONDS } from '../audio/styleSample' import { useAudioEngine } from '../audio/engineContext' import { - getApiBaseUrl, + fetchGenerationApi, setDeckCue, setDeckCuePoint, setDeckEq, @@ -663,7 +663,11 @@ export function useDeck(deckId: DeckId): DeckControls { // switch state. dispatch({ type: 'socket_open' }) const unsubscribeStatus = subscribeSidecarStatus(deckId, (status) => { - if (status.event === 'model_loading' || status.event === 'worker_died') { + if ( + status.event === 'model_loading' || + status.event === 'worker_died' || + status.event === 'startup_failed' + ) { channelRef.current?.reset() resetStreamMeasurements() } @@ -683,8 +687,7 @@ export function useDeck(deckId: DeckId): DeckControls { // manager installs or removes a Magenta model (`models://changed`, issue #43). let cancelled = false const fetchModels = () => { - void getApiBaseUrl() - .then((base) => fetch(`${base}/api/models`)) + void fetchGenerationApi('/api/models') .then((response) => (response.ok ? response.json() : null)) .then((info) => { if (cancelled || !info) return @@ -1467,11 +1470,10 @@ export function useDeck(deckId: DeckId): DeckControls { try { // The channel is created on demand: pads can fill before the // deck has ever played (prepping weapons before the set). - const apiBase = await getApiBaseUrl() const [channel, response] = await Promise.all([ ensureChannel(), - fetch( - `${apiBase}${engine === 'magenta' ? '/api/render' : '/api/generate'}`, + fetchGenerationApi( + engine === 'magenta' ? '/api/render' : '/api/generate', { method: 'POST', headers: { 'content-type': 'application/json' }, diff --git a/frontend/src/generation/client.test.ts b/frontend/src/generation/client.test.ts index 71615d1..373ae60 100644 --- a/frontend/src/generation/client.test.ts +++ b/frontend/src/generation/client.test.ts @@ -1,6 +1,6 @@ import { afterEach, describe, expect, it, vi } from 'vitest' -import { postMagentaRender, postSa3Generate } from './client' +import { postMagentaRender, postSa3Generate, startSa3Generate } from './client' afterEach(() => vi.unstubAllGlobals()) @@ -40,4 +40,56 @@ describe('generation client', () => { expect.objectContaining({ body: JSON.stringify({ prompt: 'piano', seconds: 60 }) }), ) }) + + it('isolates job status and sends authenticated cancellation before aborting', async () => { + let finishGenerate!: (response: Response) => void + const statuses: string[] = [] + const fetchMock = vi.fn((path: string, _init?: RequestInit) => { + void _init + if (path === '/api/generate') { + return new Promise((resolve) => { + finishGenerate = resolve + }) + } + if (path.endsWith('/cancel')) { + return Promise.resolve({ ok: true, status: 200 } as Response) + } + return Promise.resolve({ + ok: true, + json: async () => ({ + jobId: path.split('/').at(-1), + state: 'running', + progress: null, + detail: null, + }), + } as Response) + }) + vi.stubGlobal('fetch', fetchMock) + + const task = startSa3Generate( + { prompt: 'dub', seconds: 60, kind: 'track', seed: 4 }, + (status) => statuses.push(status.state), + ) + await vi.waitFor(() => { + expect(fetchMock.mock.calls.some(([path]) => path === '/api/generate')).toBe(true) + }) + const generateCall = fetchMock.mock.calls.find(([path]) => path === '/api/generate')! + expect((generateCall[1]?.headers as Headers).get('x-lsdj-job-id')).toBe(task.jobId) + + await task.cancel() + expect(task.wasCancelled()).toBe(true) + expect( + fetchMock.mock.calls.some( + ([path, init]) => path === `/api/jobs/${task.jobId}/cancel` && init?.method === 'POST', + ), + ).toBe(true) + expect((generateCall[1]?.signal as AbortSignal).aborted).toBe(true) + + finishGenerate({ + ok: true, + arrayBuffer: async () => new ArrayBuffer(4), + } as Response) + await task.result + expect(statuses).toContain('running') + }) }) diff --git a/frontend/src/generation/client.ts b/frontend/src/generation/client.ts index b3a32eb..66eec62 100644 --- a/frontend/src/generation/client.ts +++ b/frontend/src/generation/client.ts @@ -1,29 +1,126 @@ -import { getApiBaseUrl } from '../audio/nativeEngine' +import { fetchGenerationApi } from '../audio/nativeEngine' import type { MagentaGenerationRequest, TrackGenerationRequest, } from './songGeneration' -async function postJson(path: string, body: unknown): Promise { - const apiBase = await getApiBaseUrl() - const response = await fetch(`${apiBase}${path}`, { +export type GenerationJobStatus = { + jobId: string + state: string + progress: { + stage: string + current: number + total: number + message: string + } | null + detail: string | null +} + +export type Sa3GenerationTask = { + jobId: string + result: Promise + cancel: () => Promise + wasCancelled: () => boolean +} + +export type Sa3GenerationRequest = Omit & { + kind: 'sfx' | 'music' | 'track' +} + +function mintJobId(): string { + const cryptoApi = globalThis.crypto + if (!cryptoApi) throw new Error('secure generation job IDs are unavailable') + if (typeof cryptoApi.randomUUID === 'function') { + return cryptoApi.randomUUID() + } + const bytes = new Uint8Array(16) + cryptoApi.getRandomValues(bytes) + return Array.from(bytes, (byte) => byte.toString(16).padStart(2, '0')).join('') +} + +async function backendDetail(response: Response): Promise { + return response + .json() + .then((payload: { detail?: string }) => payload.detail ?? null) + .catch(() => null) +} + +async function postJson( + path: string, + body: unknown, + options: { signal?: AbortSignal; jobId?: string } = {}, +): Promise { + const headers: Record = { 'content-type': 'application/json' } + if (options.jobId) headers['x-lsdj-job-id'] = options.jobId + const response = await fetchGenerationApi(path, { method: 'POST', - headers: { 'content-type': 'application/json' }, + headers, body: JSON.stringify(body), + signal: options.signal, }) if (!response.ok) { - const detail = await response - .json() - .then((payload: { detail?: string }) => payload.detail) - .catch(() => null) + const detail = await backendDetail(response) throw new Error(detail || `generation failed (${response.status})`) } return response.arrayBuffer() } +export function startSa3Generate( + request: Sa3GenerationRequest, + onStatus?: (status: GenerationJobStatus) => void, +): Sa3GenerationTask { + const jobId = mintJobId() + const controller = new AbortController() + let stopped = false + let cancelled = false + let pollInFlight = false + + const poll = async () => { + if (stopped || pollInFlight || !onStatus) return + pollInFlight = true + try { + const response = await fetchGenerationApi(`/api/jobs/${encodeURIComponent(jobId)}`) + if (response.ok) onStatus((await response.json()) as GenerationJobStatus) + } catch { + // The POST owns user-visible errors. Poll failures are transient and must + // never become an unhandled rejection or create a second polling loop. + } finally { + pollInFlight = false + } + } + const pollTimer = onStatus ? globalThis.setInterval(() => void poll(), 400) : null + if (onStatus) void poll() + + const result = postJson('/api/generate', request, { + signal: controller.signal, + jobId, + }).finally(() => { + stopped = true + if (pollTimer !== null) globalThis.clearInterval(pollTimer) + }) + + const cancel = async () => { + if (cancelled || stopped) return + cancelled = true + // Registration and the user's click can race. Briefly retry 404 before + // aborting the response stream; the server's job event remains authoritative. + for (let attempt = 0; attempt < 8 && !stopped; attempt += 1) { + const response = await fetchGenerationApi( + `/api/jobs/${encodeURIComponent(jobId)}/cancel`, + { method: 'POST' }, + ).catch(() => null) + if (response && response.status !== 404) break + await new Promise((resolve) => globalThis.setTimeout(resolve, 25)) + } + controller.abort() + } + + return { jobId, result, cancel, wasCancelled: () => cancelled } +} + /** SA3-only seam: its type can carry text steering and LoRAs. */ -export function postSa3Generate(request: TrackGenerationRequest): Promise { - return postJson('/api/generate', request) +export function postSa3Generate(request: Sa3GenerationRequest): Promise { + return startSa3Generate(request).result } /** Magenta's separate seam cannot accept SA3-only options by construction. */ diff --git a/frontend/src/i18n/en.json b/frontend/src/i18n/en.json index 0a59884..8342030 100644 --- a/frontend/src/i18n/en.json +++ b/frontend/src/i18n/en.json @@ -121,6 +121,8 @@ "imported": "Imported", "inspect": "Show the full prompt for {{name}}", "pending": "{{title}} — composing…", + "cancel": "Cancel", + "cancelling": "cancelling…", "failed": "Track generation failed: {{message}}", "saveFailed": "Couldn't save {{title}} to the songs folder: {{message}}", "openFolderFailed": "Couldn't open the songs folder: {{message}}", @@ -177,6 +179,12 @@ "cueSameAsMain": "Phones on main (ch 3/4)", "cueSameAsMainNoCh": "Phones on main — needs a 4-ch main", "outputError": "Couldn't switch output: {{message}}", + "outputChecking": "Checking audio outputs…", + "outputHealthy": "Main and cue outputs are healthy.", + "outputMainUnavailable": "The main audio output is unavailable.", + "outputCueUnavailable": "The cue audio output is unavailable.", + "outputReconnect": "Reconnect audio", + "outputReconnecting": "Reconnecting…", "record": "Record", "recordLabel": "REC", "stopRecording": "Stop recording", @@ -331,6 +339,8 @@ "underruns": "Underruns", "generationSpeed": "Gen speed", "generationSpeedValue": "{{rtf}}×", + "negativePrompt": "Negative prompt", + "negativePromptUnavailable": "Unavailable upstream", "noData": "—", "position": "Position" }, @@ -400,6 +410,7 @@ "sa3": "Stable Audio 3", "none": "No models installed", "notInstalled": "Not installed", + "downloadSize": "{{size}} download", "needsResources": "Needs shared resources — repair to enable loading", "install": "Install", "repair": "Repair", @@ -413,7 +424,8 @@ "missing": "Not installed", "venv_missing": "Needs setup", "not_warmed": "Weights not downloaded", - "ready": "Ready" + "ready": "Ready", + "failed": "Verification failed — repair required" }, "stage": { "init": "Fetching shared resources…", diff --git a/frontend/src/media/MediaExplorer.test.tsx b/frontend/src/media/MediaExplorer.test.tsx index c488ac5..59861f1 100644 --- a/frontend/src/media/MediaExplorer.test.tsx +++ b/frontend/src/media/MediaExplorer.test.tsx @@ -302,9 +302,13 @@ describe('MediaExplorer', () => { it('mints and sends a fresh random seed for every Advanced take', async () => { let seed = 10 vi.stubGlobal('crypto', { - getRandomValues: (target: Uint32Array) => { - target[0] = seed - seed += 1 + getRandomValues: (target: Uint8Array | Uint32Array) => { + if (target instanceof Uint32Array) { + target[0] = seed + seed += 1 + } else { + target.fill(7) + } return target }, }) @@ -317,9 +321,9 @@ describe('MediaExplorer', () => { await composeTrack('take two') const calls = fetchMock.mock.calls as unknown as [string, RequestInit][] - const bodies = calls.map(([, init]) => - JSON.parse(init.body as string), - ) + const bodies = calls + .filter(([path]) => path === '/api/generate') + .map(([, init]) => JSON.parse(init.body as string)) expect(bodies.map((body) => body.seed)).toEqual([10, 11]) }) @@ -557,9 +561,11 @@ describe('MediaExplorer', () => { scrollIntoView.mockClear() scrolledRows = [] - act(() => bus.publish({ kind: 'browse_scroll', steps: 9 })) + await act(async () => bus.publish({ kind: 'browse_scroll', steps: 9 })) - expect(scrollIntoView).toHaveBeenCalledWith({ block: 'nearest' }) + await vi.waitFor(() => + expect(scrollIntoView).toHaveBeenCalledWith({ block: 'nearest' }), + ) expect(scrolledRows.at(-1)).toHaveTextContent('Track 3') }) @@ -596,6 +602,9 @@ describe('MediaExplorer', () => { const calls: { cmd: string; args: unknown }[] = [] const invoke = vi.fn(async (cmd: string, args?: unknown) => { calls.push({ cmd, args }) + if (cmd === 'app_info') { + return { generationPort: null, generationCapability: 'a'.repeat(64) } + } if (cmd === 'list_generated_samples') return [] if (cmd === 'save_generated_sample') { return { file: 'riff.wav', title: 'riff', prompt: 'riff', model: 'sfx', oneShot: true } @@ -692,6 +701,56 @@ describe('MediaExplorer', () => { expect(screen.getByText('#2')).toBeInTheDocument() }) + it('keeps the newer sample scan when overlapping refreshes finish out of order', async () => { + type ResolveSamples = (rows: { + file: string + title: string + prompt: string + model: string + oneShot: boolean + }[]) => void + const scans: ResolveSamples[] = [] + let onChange: ((e: { payload: unknown }) => void) | null = null + const invoke = vi.fn((cmd: string) => { + if (cmd === 'list_generated_samples') { + return new Promise((resolve: ResolveSamples) => scans.push(resolve)) + } + return Promise.resolve([]) + }) + const listen = vi.fn( + async (event: string, handler: (e: { payload: unknown }) => void) => { + if (event === 'library://changed') onChange = handler + return () => {} + }, + ) + vi.stubGlobal('__TAURI__', { core: { invoke }, event: { listen } }) + renderExplorer() + fireEvent.click(screen.getByRole('tab', { name: 'Samples' })) + expect(scans).toHaveLength(1) + + // A watcher scan starts while startup's scan is still pending. Finish the newer + // scan first, then the stale startup scan in the same batch: the old implementation + // would replace the two current rows with `one #3` because both completions read + // the same stale passive-effect ref and minted fresh ids. + act(() => onChange?.({ payload: { library: 'samples' } })) + expect(scans).toHaveLength(2) + await act(async () => { + scans[1]([ + { file: 'one.wav', title: 'one', prompt: 'one', model: 'sfx', oneShot: false }, + { file: 'two.wav', title: 'two', prompt: 'two', model: 'music', oneShot: false }, + ]) + scans[0]([ + { file: 'one.wav', title: 'one', prompt: 'one', model: 'sfx', oneShot: false }, + ]) + await Promise.resolve() + }) + + expect(screen.getByText('one', { selector: '.media__name-text' })).toBeInTheDocument() + expect(screen.getByText('two', { selector: '.media__name-text' })).toBeInTheDocument() + expect(screen.getByText('#1')).toBeInTheDocument() + expect(screen.getByText('#2')).toBeInTheDocument() + }) + it('restores samples, tagging a freeze and a hand-added file', async () => { const invoke = vi.fn(async (cmd: string) => { if (cmd === 'list_generated_samples') { @@ -754,6 +813,9 @@ describe('MediaExplorer', () => { const calls: { cmd: string; args: unknown }[] = [] const invoke = vi.fn(async (cmd: string, args?: unknown) => { calls.push({ cmd, args }) + if (cmd === 'app_info') { + return { generationPort: null, generationCapability: 'a'.repeat(64) } + } if (cmd === 'list_generated_songs') return [] if (cmd === 'save_generated_song') { return { file: 'keeper #1.wav', title: 'keeper #1', prompt: 'keeper', model: 'track' } @@ -795,6 +857,9 @@ describe('MediaExplorer', () => { const calls: { cmd: string; args: unknown }[] = [] const invoke = vi.fn(async (cmd: string, args?: unknown) => { calls.push({ cmd, args }) + if (cmd === 'app_info') { + return { generationPort: null, generationCapability: 'a'.repeat(64) } + } if (cmd === 'list_generated_songs') return [] if (cmd === 'save_generated_song') { return { file: 'guided.wav', title: 'guided', prompt: 'guided', model: 'track' } @@ -865,6 +930,9 @@ describe('MediaExplorer', () => { it('filters songs across title, prompt, model, and filename metadata', async () => { const invoke = vi.fn(async (cmd: string) => { + if (cmd === 'app_info') { + return { generationPort: null, generationCapability: 'a'.repeat(64) } + } if (cmd === 'list_generated_songs') { return [ { @@ -957,6 +1025,9 @@ describe('MediaExplorer', () => { it('promotes saved Basic settings to Advanced with the used seed fixed', async () => { const fetchMock = stubFetch() const invoke = vi.fn(async (cmd: string) => { + if (cmd === 'app_info') { + return { generationPort: null, generationCapability: 'a'.repeat(64) } + } if (cmd === 'list_generated_songs') { return [ { @@ -1022,6 +1093,9 @@ describe('MediaExplorer', () => { ]) const fetchMock = stubFetch() const invoke = vi.fn(async (cmd: string) => { + if (cmd === 'app_info') { + return { generationPort: null, generationCapability: 'a'.repeat(64) } + } if (cmd === 'list_generated_songs') { return [ { @@ -1264,6 +1338,9 @@ describe('MediaExplorer', () => { const calls: { cmd: string; args: unknown }[] = [] const invoke = vi.fn(async (cmd: string, args?: unknown) => { calls.push({ cmd, args }) + if (cmd === 'app_info') { + return { generationPort: null, generationCapability: 'a'.repeat(64) } + } if (cmd === 'list_generated_songs') return [] if (cmd === 'save_generated_song') { return { file: 'Porcelain Halo.wav', title: 'Porcelain Halo', prompt: '{"a":1}', model: 'track' } @@ -1309,6 +1386,9 @@ describe('MediaExplorer', () => { const calls: { cmd: string; args: unknown }[] = [] const invoke = vi.fn(async (cmd: string, args?: unknown) => { calls.push({ cmd, args }) + if (cmd === 'app_info') { + return { generationPort: null, generationCapability: 'a'.repeat(64) } + } if (cmd === 'list_generated_songs') return [] if (cmd === 'save_generated_song') { return { file: 'x.wav', title: 'x', prompt: 'x', model: 'track' } diff --git a/frontend/src/media/MediaExplorer.tsx b/frontend/src/media/MediaExplorer.tsx index 15aa0db..47001b1 100644 --- a/frontend/src/media/MediaExplorer.tsx +++ b/frontend/src/media/MediaExplorer.tsx @@ -9,7 +9,6 @@ import type { DeckId, TrackSource } from '../audio/types' import { LOOP_CROSSFADE_SECONDS } from '../audio/loops' import { encodeMetaFrame, - getApiBaseUrl, invoke, isTauri, subscribeLibraryChanged, @@ -18,7 +17,11 @@ import { import { useInterfaceStore } from '../audio/interfaceStore' import { useControlBus } from '../control/busContext' import { CrateBrowser } from '../crates/CrateBrowser' -import { postMagentaRender, postSa3Generate } from '../generation/client' +import { + postMagentaRender, + startSa3Generate, + type Sa3GenerationTask, +} from '../generation/client' import { adaptersForKind, stackForKind, @@ -90,6 +93,8 @@ type GeneratedTrack = prompt: string model: TrackEngine recipe: SongGenerationRecipeV1 + jobId?: string + progress?: string } | { id: number @@ -124,6 +129,8 @@ type GeneratedSample = prompt: string model: SampleEngine oneShot: boolean + jobId: string + progress?: string } | { id: number @@ -214,23 +221,33 @@ function hasVersionedRecipe(value: unknown): boolean { ) } +type LibraryRefreshState = { + issued: number + applied: number + idsByFile: Map +} + /** Re-list one library (songs or samples) from its on-disk registry, reconciled * against the folder by the Rust shell (hand-added files appear; deleted files drop * out). A row already held for a file keeps its id + in-memory wav (reuse by * filename), so a live re-list never churns; a row whose file vanished is dropped; an - * in-session take not yet on disk is kept. `ref` is read after the fetch resolves - * (freshest), and the id mint (`toRow`) runs OUTSIDE the state updater — StrictMode - * replays updaters, so they must be pure. A no-op outside Tauri. */ + * in-session take not yet on disk is kept. Overlapping scans are ordered by request, + * so an older startup scan cannot replace a newer watcher scan. New rows get a stable + * id per filename before the state updater; the updater itself reconciles against + * React's freshest `current` state and remains pure under StrictMode. A no-op outside + * Tauri. */ function reListLibrary< R extends { id: number; state: string; file?: string | null }, E extends { file: string }, >( command: string, - ref: { current: R[] }, + refresh: { current: LibraryRefreshState }, setRows: (next: (current: R[]) => R[]) => void, - toRow: (entry: E) => R, + mintId: () => number, + toRow: (entry: E, id: number) => R, ): void { if (!isTauri()) return + const request = ++refresh.current.issued void (async () => { let entries: E[] try { @@ -238,17 +255,34 @@ function reListLibrary< } catch { return // a failed scan just means no refresh; composing still works } - const byFile = new Map( - ref.current - .map((row) => [fileOf(row), row] as const) - .filter((pair): pair is readonly [string, R] => pair[0] != null), - ) - const restored = entries.map((entry) => byFile.get(entry.file) ?? toRow(entry)) + // A newer successful request already represents a later view of the registry. + // Failed newer requests do not advance `applied`, so an older successful scan + // may still provide the best available view. + if (request < refresh.current.applied) return + refresh.current.applied = request + const restored = entries.map((entry) => { + let id = refresh.current.idsByFile.get(entry.file) + if (id == null) { + id = mintId() + refresh.current.idsByFile.set(entry.file, id) + } + return [entry.file, toRow(entry, id)] as const + }) // Newest-first: in-session takes not yet on disk lead, above the restored // library reversed so the most recently composed file sits at the top (the // registry stores composition order, oldest first), sparing a scroll to the // take you just made. - setRows((current) => [...current.filter((row) => fileOf(row) == null), ...restored.reverse()]) + setRows((current) => { + const byFile = new Map( + current + .map((row) => [fileOf(row), row] as const) + .filter((pair): pair is readonly [string, R] => pair[0] != null), + ) + return [ + ...current.filter((row) => fileOf(row) == null), + ...restored.map(([file, row]) => byFile.get(file) ?? row).reverse(), + ] + }) })() } @@ -407,18 +441,26 @@ export function MediaExplorer({ // A ref, not state: two composes batched into one render (Enter + // click) must not mint the same id. const nextIdRef = useRef(1) - // The latest lists mirrored in refs (synced after commit). A live re-list (tab - // open, or the folder watcher firing) reads these from its effect/callback to reuse - // a row's id + in-memory wav by filename, so a refresh never churns ids or re-reads - // bytes — and the id mint stays OUTSIDE the state updater (StrictMode replays - // updaters, so they must be pure). At most one render stale, which is fine here. - const tracksRef = useRef([]) - const samplesRef = useRef([]) - useEffect(() => { - tracksRef.current = tracks - samplesRef.current = samples - }, [tracks, samples]) - + const trackRefreshRef = useRef({ + issued: 0, + applied: 0, + idsByFile: new Map(), + }) + const sampleRefreshRef = useRef({ + issued: 0, + applied: 0, + idsByFile: new Map(), + }) + const trackTasksRef = useRef(new Map()) + const sampleTasksRef = useRef(new Map()) + useEffect( + () => () => { + for (const task of [...trackTasksRef.current.values(), ...sampleTasksRef.current.values()]) { + void task.cancel() + } + }, + [], + ) const filteredTracks = tracks.filter((track) => matchesSearch( search, @@ -650,17 +692,18 @@ export function MediaExplorer({ } // The two libraries' re-list, each a thin {@link reListLibrary} call differing only - // in the command, the ref, the setter, and the registry-entry → row mapping (a + // in the command, the refresh state, the setter, and the registry-entry → row mapping (a // sample carries `oneShot`; a song's model runs through `asTrackEngine`). Used at // startup and by the folder watcher. const refreshSongs = useCallback( () => reListLibrary( 'list_generated_songs', - tracksRef, + trackRefreshRef, setTracks, - (entry) => ({ - id: nextIdRef.current++, + () => nextIdRef.current++, + (entry, id) => ({ + id, state: 'ready', title: entry.title, prompt: entry.prompt, @@ -675,10 +718,11 @@ export function MediaExplorer({ () => reListLibrary( 'list_generated_samples', - samplesRef, + sampleRefreshRef, setSamples, - (entry) => ({ - id: nextIdRef.current++, + () => nextIdRef.current++, + (entry, id) => ({ + id, state: 'ready', title: entry.title, prompt: entry.prompt, @@ -708,7 +752,11 @@ export function MediaExplorer({ ) const bus = useControlBus() - useEffect(() => + // Refresh the hardware handler in the same commit that exposes new rows. + // A passive effect leaves a frame where the DOM shows the new list but the + // bus still holds the previous render's empty/stale list closure, so a rotary + // tick in that window is silently ignored. + useLayoutEffect(() => bus.subscribe((intent) => { if (intent.kind === 'browse_tab') { // Rotary press: cycle the visible tab from the hardware. @@ -762,6 +810,23 @@ export function MediaExplorer({ const requestSeconds = oneShot ? sampleSeconds : sampleSeconds + LOOP_CROSSFADE_SECONDS setSampleError(null) setSampleSaveError(null) + const task = startSa3Generate( + { + prompt: trimmedPrompt, + seconds: requestSeconds, + kind: requestEngine, + ...(requestLoras.length > 0 ? { loras: requestLoras } : {}), + }, + (status) => + setSamples((current) => + current.map((sample) => + sample.id === id && sample.state === 'pending' + ? { ...sample, progress: status.progress?.message ?? status.state } + : sample, + ), + ), + ) + sampleTasksRef.current.set(id, task) setSamples((current) => [ { id, @@ -770,30 +835,13 @@ export function MediaExplorer({ prompt: trimmedPrompt, model: requestEngine, oneShot, + jobId: task.jobId, }, ...current, ]) void (async () => { try { - const apiBase = await getApiBaseUrl() - const response = await fetch(`${apiBase}/api/generate`, { - method: 'POST', - headers: { 'content-type': 'application/json' }, - body: JSON.stringify({ - prompt: trimmedPrompt, - seconds: requestSeconds, - kind: requestEngine, - ...(requestLoras.length > 0 ? { loras: requestLoras } : {}), - }), - }) - if (!response.ok) { - const detail = await response - .json() - .then((body: { detail?: string }) => body.detail) - .catch(() => null) - throw new Error(detail || `generation failed (${response.status})`) - } - const wav = await response.arrayBuffer() + const wav = await task.result setSamples((current) => current.map((sample) => sample.id === id @@ -834,11 +882,28 @@ export function MediaExplorer({ } } catch (error) { setSamples((current) => current.filter((sample) => sample.id !== id)) - setSampleError(error instanceof Error ? error.message : String(error)) + if (!task.wasCancelled()) { + setSampleError(error instanceof Error ? error.message : String(error)) + } + } finally { + sampleTasksRef.current.delete(id) } })() } + function cancelSampleGeneration(id: number) { + const task = sampleTasksRef.current.get(id) + if (!task) return + setSamples((current) => + current.map((sample) => + sample.id === id && sample.state === 'pending' + ? { ...sample, progress: t('media.generate.cancelling') } + : sample, + ), + ) + void task.cancel() + } + async function openSamplesFolder() { setSampleSaveError(null) try { @@ -883,6 +948,19 @@ export function MediaExplorer({ // blank title gets a random song title so a long/JSON prompt never becomes the // name. The row appends a session-unique #id to tell same-title siblings apart. const songTitle = title.trim() || randomSongTitle() + const task = + 'kind' in generation.request + ? startSa3Generate(generation.request, (status) => + setTracks((current) => + current.map((track) => + track.id === id && track.state === 'pending' + ? { ...track, progress: status.progress?.message ?? status.state } + : track, + ), + ), + ) + : null + if (task) trackTasksRef.current.set(id, task) setGenerateError(null) setSaveError(null) setRecipeNotice(null) @@ -894,14 +972,15 @@ export function MediaExplorer({ prompt: trimmedPrompt, model: requestEngine, recipe: generation.recipe, + ...(task ? { jobId: task.jobId } : {}), }, ...current, ]) void (async () => { try { const wav = - 'kind' in generation.request - ? await postSa3Generate(generation.request) + task + ? await task.result : await postMagentaRender(generation.request) setTracks((current) => current.map((track) => @@ -953,11 +1032,28 @@ export function MediaExplorer({ } } catch (error) { setTracks((current) => current.filter((track) => track.id !== id)) - setGenerateError(error instanceof Error ? error.message : String(error)) + if (!task?.wasCancelled()) { + setGenerateError(error instanceof Error ? error.message : String(error)) + } + } finally { + trackTasksRef.current.delete(id) } })() } + function cancelTrackGeneration(id: number) { + const task = trackTasksRef.current.get(id) + if (!task) return + setTracks((current) => + current.map((track) => + track.id === id && track.state === 'pending' + ? { ...track, progress: t('media.generate.cancelling') } + : track, + ), + ) + void task.cancel() + } + async function chooseFolder() { setFolderError(null) // The OS folder picker (dialog plugin) + a Rust dir listing — WKWebView has no @@ -1353,7 +1449,9 @@ export function MediaExplorer({ )} {track.state === 'pending' - ? t('media.generate.pending', { title: track.title }) + ? `${t('media.generate.pending', { title: track.title })}${ + track.progress ? ` · ${track.progress}` : '' + }` : track.title} {track.state === 'ready' && composed && ( @@ -1383,6 +1481,11 @@ export function MediaExplorer({ ? t('media.generate.imported') : t(`media.generate.engines.${track.model}`)} + {track.state === 'pending' && track.jobId && ( + + )} {track.state === 'ready' && hasVersionedRecipe(track.recipe) && ( )} + {sample.state === 'pending' && ( + + )} {`${sampleModelLabel(sample.model)} · ${t( sample.oneShot ? 'media.samples.oneShot' : 'media.samples.loop', diff --git a/frontend/src/mixer/mixer.css b/frontend/src/mixer/mixer.css index 93f2a5b..7e5888b 100644 --- a/frontend/src/mixer/mixer.css +++ b/frontend/src/mixer/mixer.css @@ -109,6 +109,33 @@ color: var(--color-led-danger); } +.audio-health { + margin: 0; + font: 500 var(--text-s) var(--font-ui); +} + +.audio-health--checking { + color: var(--color-text-muted); +} + +.audio-health--healthy { + color: var(--color-led-ok); +} + +.audio-health--error { + display: flex; + align-items: flex-start; + gap: var(--space-3); + color: var(--color-led-danger); +} + +.audio-health__messages { + display: flex; + flex: 1; + flex-direction: column; + gap: var(--space-1); +} + /* ---------- record control (top bar) ---------- */ /* A ● dot + REC at rest; the button outline and dot go danger-red while diff --git a/frontend/src/models/LoraProvider.test.tsx b/frontend/src/models/LoraProvider.test.tsx index 33a464a..b787304 100644 --- a/frontend/src/models/LoraProvider.test.tsx +++ b/frontend/src/models/LoraProvider.test.tsx @@ -42,7 +42,9 @@ function status(overrides: Partial = {}): ModelStatus { }, sa3: { state: 'ready', + backend: 'mlx', sizeBytes: 5_000_000_000, + downloadBytes: 9_154_794_562, checkout: '/sa3', installedSource: null, pinnedSource: { repo: 'https://github.com/Stability-AI/stable-audio-3', commit: 'pin' }, diff --git a/frontend/src/models/ModelManager.test.tsx b/frontend/src/models/ModelManager.test.tsx index 1631058..eaa4173 100644 --- a/frontend/src/models/ModelManager.test.tsx +++ b/frontend/src/models/ModelManager.test.tsx @@ -40,7 +40,9 @@ function status(overrides: Partial = {}): ModelStatus { }, sa3: { state: 'missing', + backend: 'tflite', sizeBytes: 0, + downloadBytes: 14_138_994_904, checkout: null, installedSource: null, pinnedSource: { repo: 'https://github.com/brxs/stable-audio-3', commit: 'pinned1' }, @@ -122,7 +124,9 @@ describe('ModelManager', () => { status({ sa3: { state: 'ready', + backend: 'mlx', sizeBytes: 5_000_000_000, + downloadBytes: 9_154_794_562, checkout: '/sa3', installedSource: { repo: 'https://github.com/brxs/stable-audio-3', commit: 'pinned1' }, pinnedSource: { repo: 'https://github.com/brxs/stable-audio-3', commit: 'pinned1' }, @@ -161,7 +165,9 @@ describe('ModelManager', () => { status({ sa3: { state: 'ready', + backend: 'mlx', sizeBytes: 5_000_000_000, + downloadBytes: 9_154_794_562, checkout: '/sa3', installedSource: { repo: 'https://github.com/brxs/stable-audio-3', commit: 'oldsha' }, pinnedSource: { repo: 'https://github.com/brxs/stable-audio-3', commit: 'newsha' }, @@ -180,7 +186,9 @@ describe('ModelManager', () => { status({ sa3: { state: 'ready', + backend: 'mlx', sizeBytes: 5_000_000_000, + downloadBytes: 9_154_794_562, checkout: '/sa3', installedSource: { repo: 'https://github.com/brxs/stable-audio-3', commit: 'pinned1' }, pinnedSource: { repo: 'https://github.com/brxs/stable-audio-3', commit: 'pinned1' }, @@ -203,4 +211,25 @@ describe('ModelManager', () => { expect(screen.getByRole('alert')).toHaveTextContent('no weights') }) + it('discloses the selected backend download and offers repair after verification failure', async () => { + modelStatus.mockResolvedValue( + status({ + sa3: { + state: 'failed', + backend: 'tflite', + sizeBytes: 0, + downloadBytes: 14_138_994_904, + checkout: '/sa3', + installedSource: null, + pinnedSource: { repo: 'https://github.com/Stability-AI/stable-audio-3', commit: 'pin' }, + updateAvailable: true, + }, + }), + ) + render() + await screen.findByText('Verification failed — repair required') + fireEvent.click(screen.getByText('Repair')) + expect(installModel).toHaveBeenCalledWith('sa3', undefined) + }) + }) diff --git a/frontend/src/models/ModelManager.tsx b/frontend/src/models/ModelManager.tsx index f759009..737c9a2 100644 --- a/frontend/src/models/ModelManager.tsx +++ b/frontend/src/models/ModelManager.tsx @@ -187,6 +187,9 @@ export function ModelManager() {
{t(`modelManager.sa3State.${sa3.state}`)} {sa3Present && sa3.sizeBytes > 0 ? ` · ${formatBytes(sa3.sizeBytes)}` : ''} + {!sa3Present && sa3.downloadBytes > 0 + ? ` · ${t('modelManager.downloadSize', { size: formatBytes(sa3.downloadBytes) })}` + : ''}
{sa3Ready && sa3.updateAvailable && (
{t('modelManager.updateAvailable')}
@@ -198,7 +201,7 @@ export function ModelManager() { sa3Label, !sa3Ready ? ( ) : sa3.updateAvailable ? ( + )} + + ) +} diff --git a/justfile b/justfile index 0ba2f6e..9426ceb 100644 --- a/justfile +++ b/justfile @@ -72,7 +72,12 @@ build: # default `uv run` sidecar/generation commands use the backend project dir; # override each command in dev, or point both at a freeze with LSDJ_BACKEND_BIN. tauri-dev: build - cd src-tauri && cargo tauri dev + cd src-tauri && cargo tauri dev --config tauri.macos.conf.json + +# Linux developer shell. Model services use source-tree overrides in dev; the +# distributable AppImage instead compiles the fail-closed managed-runtime seam. +tauri-linux-dev: build + cd src-tauri && cargo tauri dev --config tauri.linux.conf.json # Freeze the shared Python backend into an ONEDIR binary for bundling # (src-tauri/sidecar-dist/lsdj_backend). It serves deck inference, model tooling, @@ -84,13 +89,20 @@ freeze-backend: # Compatibility alias for the original packaging recipe name. freeze-sidecar: freeze-backend +# Resolve the separately bundled PyTorch/CUDA runtime without importing it into +# the macOS MLX backend environment. These are installation candidates until +# both clean-host installs and the NVIDIA qualification matrix pass. +lock-mrt2-pytorch: + uv pip compile backend/mrt2-pytorch-runtime.in --python-version 3.12 --python-platform x86_64-unknown-linux-gnu --index https://download.pytorch.org/whl/cu130 --default-index https://pypi.org/simple --index-strategy unsafe-best-match --only-binary :all: --generate-hashes --output-file backend/runtime-locks/mrt2-pytorch-linux-x86_64.txt + uv pip compile backend/mrt2-pytorch-runtime.in --python-version 3.12 --python-platform x86_64-pc-windows-msvc --index https://download.pytorch.org/whl/cu130 --default-index https://pypi.org/simple --index-strategy unsafe-best-match --only-binary :all: --generate-hashes --output-file backend/runtime-locks/mrt2-pytorch-windows-x86_64.txt + # Native shell developer bundle: build the app/DMG into # src-tauri/target/release/bundle/. The config applies an explicit ad-hoc # signature so the app bundle is structurally valid on Apple Silicon. This is # useful for local testing only; use `just tauri-release` for anything sent to # another Mac. tauri-build: build - cd src-tauri && cargo tauri build + cd src-tauri && cargo tauri build --config tauri.macos.conf.json # Distributable macOS build. Fails closed unless a Developer ID Application # identity and Apple notarization credentials are configured, then verifies the @@ -98,9 +110,17 @@ tauri-build: build tauri-release: ./scripts/build-macos-release.sh +# Linux x86_64 AppImage built on an Ubuntu 22.04-compatible host. Model +# runtimes remain app-managed and fail closed until their verified #110/#111 +# adapters supply explicit executables; no packaged path invokes developer +# Python/uv/Git/shell tooling. +tauri-linux-release: + ./scripts/build-linux-appimage.sh + # Create and push the next protected vYYYY.MM.N tag from a clean, current main. # Remote calendar-version tags are the ledger; no version file or bump is needed. -# The tag starts the macOS signing workflow and its Engineering approval gate. +# The tag starts the three-platform release workflow; macOS and Windows signing +# remain behind their protected Engineering approval gates. release: ./scripts/create-release.sh diff --git a/mrt2-pytorch-pin.json b/mrt2-pytorch-pin.json new file mode 100644 index 0000000..b5e138e --- /dev/null +++ b/mrt2-pytorch-pin.json @@ -0,0 +1,97 @@ +{ + "schemaVersion": 1, + "runtime": { + "python": [ + { + "target":"x86_64-unknown-linux-gnu", + "version":"3.12.13", + "url":"https://github.com/astral-sh/python-build-standalone/releases/download/20260807/cpython-3.12.13%2B20260807-x86_64-unknown-linux-gnu-install_only_stripped.tar.gz", + "size":34163738, + "sha256":"506191be3ee7bd190a8834dcdc1b3bc70aab50608deccc711935aa007239cabd", + "archiveRoot":"python", + "executable":"bin/python3", + "maxFiles":12000, + "maxExpandedBytes":536870912 + }, + { + "target":"x86_64-pc-windows-msvc", + "version":"3.12.13", + "url":"https://github.com/astral-sh/python-build-standalone/releases/download/20260807/cpython-3.12.13%2B20260807-x86_64-pc-windows-msvc-install_only_stripped.tar.gz", + "size":21962247, + "sha256":"18bcc65b17921806b72cdc88bcf000bf67a2c99a8fc381fe1629f2b9ba56858d", + "archiveRoot":"python", + "executable":"python.exe", + "maxFiles":12000, + "maxExpandedBytes":536870912 + } + ], + "uv": [ + { + "target":"x86_64-unknown-linux-gnu","version":"0.11.7", + "url":"https://github.com/astral-sh/uv/releases/download/0.11.7/uv-x86_64-unknown-linux-gnu.tar.gz", + "size":24249861,"sha256":"6681d691eb7f9c00ac6a3af54252f7ab29ae72f0c8f95bdc7f9d1401c23ea868", + "archiveRoot":"uv-x86_64-unknown-linux-gnu","executable":"uv","maxFiles":8,"maxExpandedBytes":67108864 + }, + { + "target":"x86_64-pc-windows-msvc","version":"0.11.7", + "url":"https://github.com/astral-sh/uv/releases/download/0.11.7/uv-x86_64-pc-windows-msvc.zip", + "size":23572531,"sha256":"fe0c7815acf4fc45f8a5eff58ed3cf7ae2e15c3cf1dceadbd10c816ec1690cc1", + "archiveRoot":"uv-x86_64-pc-windows-msvc","archiveFormat":"zip","executable":"uv.exe","maxFiles":8,"maxExpandedBytes":67108864 + } + ], + "locks": { + "x86_64-unknown-linux-gnu":"4749398e83c72359f04081f6c0090461cec823a67845766410b7b49ae55b1785", + "x86_64-pc-windows-msvc":"63c86af3d4ee0efe70318539272288cb193a38a7af4096ae7c12ec2c48e5b7dc" + }, + "wheelManifestSha256":"9dfd22a3f751831fcc1aaf457372d68838fd7432657dbdba2e43a8384f79b701" + }, + "models": { + "mrt2_small": { + "repository": "magenta-community/magenta-realtime-2-small", + "revision": "7037d99551c84ac5c6afb7f1a5e58c65e7233dbb", + "files": [ + {"path":"aoti.py","size":5953,"sha256":"52056e706de328df3a12637b359ce349bacdfb9793cc57ddcc756e62e4b66c86"}, + {"path":"codec_shapes.json","size":2824,"sha256":"5df62019bd7c0f7428b34005b7de2691e8396989d98b6a295948655f13d410f5"}, + {"path":"config.json","size":6151,"sha256":"88100af0e4a9bcb8462ce1145d902983cbdcd5ac8b9ade496aca1ad6d42109c8"}, + {"path":"configuration_magenta_rt2.py","size":4085,"sha256":"9fdbd0564174b1f7a7b71735ab310ecd64846a08d2d006becf77301b13a1190e"}, + {"path":"cudagraph.py","size":8624,"sha256":"68f53fe2b8578f37a6730e2c78622712b099d8d31014b946a08ab86fca15be3f"}, + {"path":"depthformer.py","size":17995,"sha256":"36e10c6206b66992ae75c94c0f4859007e6deb8a4f7a451dee20736e358b13a7"}, + {"path":"layers.py","size":14432,"sha256":"ea328a1f4c9a8ed2cbdabe801e5f976f2437ea8cf6c4978dcbcf09558b4644ef"}, + {"path":"model.safetensors","size":1128837704,"sha256":"da0146d5c442f16006fde3721f7ce6584e4c8f96024887a990f36bac46a625f5"}, + {"path":"modeling_magenta_rt2.py","size":26862,"sha256":"ea3bfbc9e998de706f36c956a8e59ea4f6313663279d5e814934e1e5668336fb"}, + {"path":"musiccoca.py","size":8064,"sha256":"b134f8db41ef0212929fc6377812b961dcf042a0cf4aa8344a9640796635625b"}, + {"path":"processing_musiccoca.py","size":3154,"sha256":"8293875ebac4221dcfcbfe4c795302fcf9c5360c43fb609729cb170e1199a1ef"}, + {"path":"spectrostream.py","size":14917,"sha256":"0cd124454bda9a1ee01c33077e6c506a65e827d87b4ef01c481b86776001dfbe"} + ] + }, + "mrt2_base": { + "repository": "magenta-community/magenta-realtime-2", + "revision": "92087988d05d0fe38b11f021f0b0d00a75afb86b", + "files": [ + {"path":"aoti.py","size":5953,"sha256":"52056e706de328df3a12637b359ce349bacdfb9793cc57ddcc756e62e4b66c86"}, + {"path":"codec_shapes.json","size":2824,"sha256":"5df62019bd7c0f7428b34005b7de2691e8396989d98b6a295948655f13d410f5"}, + {"path":"config.json","size":6153,"sha256":"8c89a1cfe10e9f4618aded3f29d64a49aefd51e0bafd9a1aac0b26a5628e7530"}, + {"path":"configuration_magenta_rt2.py","size":4085,"sha256":"9fdbd0564174b1f7a7b71735ab310ecd64846a08d2d006becf77301b13a1190e"}, + {"path":"cudagraph.py","size":8624,"sha256":"68f53fe2b8578f37a6730e2c78622712b099d8d31014b946a08ab86fca15be3f"}, + {"path":"depthformer.py","size":17995,"sha256":"36e10c6206b66992ae75c94c0f4859007e6deb8a4f7a451dee20736e358b13a7"}, + {"path":"layers.py","size":14432,"sha256":"ea328a1f4c9a8ed2cbdabe801e5f976f2437ea8cf6c4978dcbcf09558b4644ef"}, + {"path":"model.safetensors","size":9836752784,"sha256":"045ac86f5be520d37020c94ef13cad3a84e336f18d82166748fd6b36a305540e"}, + {"path":"modeling_magenta_rt2.py","size":26862,"sha256":"ea3bfbc9e998de706f36c956a8e59ea4f6313663279d5e814934e1e5668336fb"}, + {"path":"musiccoca.py","size":8064,"sha256":"b134f8db41ef0212929fc6377812b961dcf042a0cf4aa8344a9640796635625b"}, + {"path":"processing_musiccoca.py","size":3154,"sha256":"8293875ebac4221dcfcbfe4c795302fcf9c5360c43fb609729cb170e1199a1ef"}, + {"path":"spectrostream.py","size":14917,"sha256":"0cd124454bda9a1ee01c33077e6c506a65e827d87b4ef01c481b86776001dfbe"} + ] + } + }, + "processor": { + "repository": "magenta-community/magenta-rt-musiccoca-torch", + "revision": "236c488e38aa98643805514996934d705668298b", + "files": [ + {"path":"mel_params.npz","size":526388,"sha256":"8dbeb3747e1ad349c771f6bed5b1600f2be6fb361d82e996bc992d8d7e061d97"}, + {"path":"music_encoder.pt","size":371441867,"sha256":"97af5b160739e7699c67217049e5c35d9fa95a9abaa19debc0c52ea46c0ca8de"}, + {"path":"quantizer.pt","size":72678385,"sha256":"090e8ebab0a817d21833ca048a6ab9c3782c51890068d85d9fb14115aa1d79cf"}, + {"path":"spm.model","size":517448,"sha256":"ff325a99b61ba5726cf6437cde6eefbb633dbaa363a684f7a97ed99b55202cca"}, + {"path":"text_encoder.pt","size":419819353,"sha256":"d23093557e151d9cf9bdea5320434b5d5f441386d870577fc741ec5609ba5618"} + ] + } +} diff --git a/mrt2-pytorch-wheels.json b/mrt2-pytorch-wheels.json new file mode 100644 index 0000000..10e7ef9 --- /dev/null +++ b/mrt2-pytorch-wheels.json @@ -0,0 +1,762 @@ +{ + "schemaVersion": 1, + "python": "3.12", + "targets": { + "x86_64-unknown-linux-gnu": [ + { + "package": "annotated-doc", + "version": "0.0.5", + "filename": "annotated_doc-0.0.5-py3-none-any.whl", + "url": "https://files.pythonhosted.org/packages/3e/30/e900b21425a860e195f32e37657aa1f7c7f2b1bfb26f03ca209b90933c06/annotated_doc-0.0.5-py3-none-any.whl", + "size": 5302, + "sha256": "117bac03a25ede5df5440e855b32d556049ca169ead221505badf432fed4b101" + }, + { + "package": "anyio", + "version": "4.14.2", + "filename": "anyio-4.14.2-py3-none-any.whl", + "url": "https://files.pythonhosted.org/packages/da/35/f2287558c17e29fafc8ef3daf819bb9834061cfa43bff8014f7df7f63bdc/anyio-4.14.2-py3-none-any.whl", + "size": 125813, + "sha256": "9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494" + }, + { + "package": "certifi", + "version": "2026.7.22", + "filename": "certifi-2026.7.22-py3-none-any.whl", + "url": "https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl", + "size": 136983, + "sha256": "62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775" + }, + { + "package": "cuda-bindings", + "version": "13.3.1", + "filename": "cuda_bindings-13.3.1-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", + "url": "https://files.pythonhosted.org/packages/39/2a/6d2e9047d1fb243dbaa364b01e0297534b9ed7fd27dba1c9f361519cf69b/cuda_bindings-13.3.1-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", + "size": 6657965, + "sha256": "e32d08f71ebcdf00f0f41eab2eb37e8da94c8ed411cc9f7f7a019ce6b34abe3a" + }, + { + "package": "cuda-pathfinder", + "version": "1.6.0", + "filename": "cuda_pathfinder-1.6.0-py3-none-any.whl", + "url": "https://files.pythonhosted.org/packages/fc/b4/d088047afe39827556df21118cac9ffd20cc3f968c99a7681494d1eb333c/cuda_pathfinder-1.6.0-py3-none-any.whl", + "size": 54591, + "sha256": "1503af579d8379c24bdd65528379bc57039b0455be9f5f9686cf8e473a1fce51" + }, + { + "package": "cuda-toolkit", + "version": "13.0.2", + "filename": "cuda_toolkit-13.0.2-py2.py3-none-any.whl", + "url": "https://files.pythonhosted.org/packages/57/b2/453099f5f3b698d7d0eab38916aac44c7f76229f451709e2eb9db6615dcd/cuda_toolkit-13.0.2-py2.py3-none-any.whl", + "size": 2364, + "sha256": "b198824cf2f54003f50d64ada3a0f184b42ca0846c1c94192fa269ecd97a66eb" + }, + { + "package": "filelock", + "version": "3.32.2", + "filename": "filelock-3.32.2-py3-none-any.whl", + "url": "https://files.pythonhosted.org/packages/c1/e8/72f8cef9fdfeffe06213fe8508039396ee48daa0e3259457ed766173bfd6/filelock-3.32.2-py3-none-any.whl", + "size": 98830, + "sha256": "87dd94cf281e586d135fa51132b8e3d9a598b316e90377a288663c9321036c82" + }, + { + "package": "fsspec", + "version": "2026.7.0", + "filename": "fsspec-2026.7.0-py3-none-any.whl", + "url": "https://files.pythonhosted.org/packages/fd/3c/6a2bf344106328fd04963664a60b9bb6496fc25df8e962fcdc1367285fb9/fsspec-2026.7.0-py3-none-any.whl", + "size": 206583, + "sha256": "b57ddbafedfaef7018c1ecab32aa200a9d7ca26b77965f64e48b70061249d279" + }, + { + "package": "h11", + "version": "0.16.0", + "filename": "h11-0.16.0-py3-none-any.whl", + "url": "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", + "size": 37515, + "sha256": "63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86" + }, + { + "package": "hf-xet", + "version": "1.6.0", + "filename": "hf_xet-1.6.0-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", + "url": "https://files.pythonhosted.org/packages/67/4e/a28359bf1c1ecf11eba22123168c138698f7cb576ac678f5a2e16cd5da08/hf_xet-1.6.0-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", + "size": 4464663, + "sha256": "d62671bb130879cef0ee4c9ebe47a14af6c66ec53e6d84dc15936e5ffdfac82f" + }, + { + "package": "httpcore", + "version": "1.0.9", + "filename": "httpcore-1.0.9-py3-none-any.whl", + "url": "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", + "size": 78784, + "sha256": "2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55" + }, + { + "package": "httpx", + "version": "0.28.1", + "filename": "httpx-0.28.1-py3-none-any.whl", + "url": "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", + "size": 73517, + "sha256": "d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad" + }, + { + "package": "huggingface-hub", + "version": "1.5.0", + "filename": "huggingface_hub-1.5.0-py3-none-any.whl", + "url": "https://files.pythonhosted.org/packages/ec/74/2bc951622e2dbba1af9a460d93c51d15e458becd486e62c29cc0ccb08178/huggingface_hub-1.5.0-py3-none-any.whl", + "size": 596261, + "sha256": "c9c0b3ab95a777fc91666111f3b3ede71c0cdced3614c553a64e98920585c4ee" + }, + { + "package": "idna", + "version": "3.18", + "filename": "idna-3.18-py3-none-any.whl", + "url": "https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl", + "size": 65455, + "sha256": "7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2" + }, + { + "package": "jinja2", + "version": "3.1.6", + "filename": "jinja2-3.1.6-py3-none-any.whl", + "url": "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", + "size": 134899, + "sha256": "85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67" + }, + { + "package": "llvmlite", + "version": "0.48.0", + "filename": "llvmlite-0.48.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", + "url": "https://files.pythonhosted.org/packages/80/f2/72409351db66d0a317ec5087e076f31fb7b773a640db8a90ce6b5cac9edd/llvmlite-0.48.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", + "size": 59890118, + "sha256": "416fa4c2c66c2c6dc6d0a402648c19206e548efa0aa1eff01ad5cdad0af8217d" + }, + { + "package": "markdown-it-py", + "version": "4.2.0", + "filename": "markdown_it_py-4.2.0-py3-none-any.whl", + "url": "https://files.pythonhosted.org/packages/b3/81/4da04ced5a082363ecfa159c010d200ecbd959ae410c10c0264a38cac0f5/markdown_it_py-4.2.0-py3-none-any.whl", + "size": 91687, + "sha256": "9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a" + }, + { + "package": "markupsafe", + "version": "3.0.3", + "filename": "markupsafe-3.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", + "url": "https://files.pythonhosted.org/packages/3c/2e/8d0c2ab90a8c1d9a24f0399058ab8519a3279d1bd4289511d74e909f060e/markupsafe-3.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", + "size": 22947, + "sha256": "d6dd0be5b5b189d31db7cda48b91d7e0a9795f31430b7f271219ab30f1d3ac9d" + }, + { + "package": "mdurl", + "version": "0.1.2", + "filename": "mdurl-0.1.2-py3-none-any.whl", + "url": "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", + "size": 9979, + "sha256": "84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8" + }, + { + "package": "mpmath", + "version": "1.3.0", + "filename": "mpmath-1.3.0-py3-none-any.whl", + "url": "https://files.pythonhosted.org/packages/43/e3/7d92a15f894aa0c9c4b49b8ee9ac9850d6e63b03c9c32c0367a13ae62209/mpmath-1.3.0-py3-none-any.whl", + "size": 536198, + "sha256": "a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c" + }, + { + "package": "networkx", + "version": "3.6.1", + "filename": "networkx-3.6.1-py3-none-any.whl", + "url": "https://files.pythonhosted.org/packages/9e/c9/b2622292ea83fbb4ec318f5b9ab867d0a28ab43c5717bb85b0a5f6b3b0a4/networkx-3.6.1-py3-none-any.whl", + "size": 2068504, + "sha256": "d47fbf302e7d9cbbb9e2555a0d267983d2aa476bac30e90dfbe5669bd57f3762" + }, + { + "package": "numba", + "version": "0.66.0", + "filename": "numba-0.66.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", + "url": "https://files.pythonhosted.org/packages/2d/55/25c319845e9a4e08f16611ddbda56a192eb7b6ed13e1a2bff2da272ffb97/numba-0.66.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", + "size": 3866252, + "sha256": "0999e3ee1b18c48e1fb51d11af35ef59852c7f4f50569c9550c25faef0616ad1" + }, + { + "package": "numpy", + "version": "2.3.5", + "filename": "numpy-2.3.5-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", + "url": "https://files.pythonhosted.org/packages/b6/23/2a1b231b8ff672b4c450dac27164a8b2ca7d9b7144f9c02d2396518352eb/numpy-2.3.5-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", + "size": 16606086, + "sha256": "0d8163f43acde9a73c2a33605353a4f1bc4798745a8b1d73183b28e5b435ae28" + }, + { + "package": "nvidia-cublas", + "version": "13.1.1.3", + "filename": "nvidia_cublas-13.1.1.3-py3-none-manylinux_2_27_x86_64.whl", + "url": "https://files.pythonhosted.org/packages/3b/cd/154ca20c38269e05eff77c1464e6c1da89f50a6390b565e9d82e06bc11e1/nvidia_cublas-13.1.1.3-py3-none-manylinux_2_27_x86_64.whl", + "size": 423138758, + "sha256": "37936a16db8fe4ac1f065c2139360608a543a09275cb1a1af612e08cfa065436" + }, + { + "package": "nvidia-cuda-cupti", + "version": "13.0.85", + "filename": "nvidia_cuda_cupti-13.0.85-py3-none-manylinux_2_25_x86_64.whl", + "url": "https://files.pythonhosted.org/packages/33/6d/737d164b4837a9bbd202f5ae3078975f0525a55730fe871d8ed4e3b952b0/nvidia_cuda_cupti-13.0.85-py3-none-manylinux_2_25_x86_64.whl", + "size": 10715597, + "sha256": "4eb01c08e859bf924d222250d2e8f8b8ff6d3db4721288cf35d14252a4d933c8" + }, + { + "package": "nvidia-cuda-nvrtc", + "version": "13.0.88", + "filename": "nvidia_cuda_nvrtc-13.0.88-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", + "url": "https://files.pythonhosted.org/packages/c3/68/483a78f5e8f31b08fb1bb671559968c0ca3a065ac7acabfc7cee55214fd6/nvidia_cuda_nvrtc-13.0.88-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", + "size": 90215200, + "sha256": "ad9b6d2ead2435f11cbb6868809d2adeeee302e9bb94bcf0539c7a40d80e8575" + }, + { + "package": "nvidia-cuda-runtime", + "version": "13.0.96", + "filename": "nvidia_cuda_runtime-13.0.96-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", + "url": "https://files.pythonhosted.org/packages/2e/24/d1558f3b68b1d26e706813b1d10aa1d785e4698c425af8db8edc3dced472/nvidia_cuda_runtime-13.0.96-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", + "size": 2243632, + "sha256": "7f82250d7782aa23b6cfe765ecc7db554bd3c2870c43f3d1821f1d18aebf0548" + }, + { + "package": "nvidia-cudnn-cu13", + "version": "9.20.0.48", + "filename": "nvidia_cudnn_cu13-9.20.0.48-py3-none-manylinux_2_27_x86_64.whl", + "url": "https://files.pythonhosted.org/packages/6e/5e/edb9c0ae051602c3ccaffe424256463636d639e27d7f302dde9975ef9e7a/nvidia_cudnn_cu13-9.20.0.48-py3-none-manylinux_2_27_x86_64.whl", + "size": 366173588, + "sha256": "0c45dd8eeb50b603f07995b1b300c62ffe6a1980482b82b3bcf94a4ca9d49304" + }, + { + "package": "nvidia-cufft", + "version": "12.0.0.61", + "filename": "nvidia_cufft-12.0.0.61-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", + "url": "https://files.pythonhosted.org/packages/a8/2f/7b57e29836ea8714f81e9898409196f47d772d5ddedddf1592eadb8ab743/nvidia_cufft-12.0.0.61-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", + "size": 214085489, + "sha256": "6c44f692dce8fd5ffd3e3df134b6cdb9c2f72d99cf40b62c32dde45eea9ddad3" + }, + { + "package": "nvidia-cufile", + "version": "1.15.1.6", + "filename": "nvidia_cufile-1.15.1.6-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", + "url": "https://files.pythonhosted.org/packages/3f/70/4f193de89a48b71714e74602ee14d04e4019ad36a5a9f20c425776e72cd6/nvidia_cufile-1.15.1.6-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", + "size": 1223672, + "sha256": "08a3ecefae5a01c7f5117351c64f17c7c62efa5fffdbe24fc7d298da19cd0b44" + }, + { + "package": "nvidia-curand", + "version": "10.4.0.35", + "filename": "nvidia_curand-10.4.0.35-py3-none-manylinux_2_27_x86_64.whl", + "url": "https://files.pythonhosted.org/packages/a5/9f/be0a41ca4a4917abf5cb9ae0daff1a6060cc5de950aec0396de9f3b52bc5/nvidia_curand-10.4.0.35-py3-none-manylinux_2_27_x86_64.whl", + "size": 59544258, + "sha256": "1aee33a5da6e1db083fe2b90082def8915f30f3248d5896bcec36a579d941bfc" + }, + { + "package": "nvidia-cusolver", + "version": "12.0.4.66", + "filename": "nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_x86_64.whl", + "url": "https://files.pythonhosted.org/packages/5f/67/cba3777620cdacb99102da4042883709c41c709f4b6323c10781a9c3aa34/nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_x86_64.whl", + "size": 200941980, + "sha256": "0a759da5dea5c0ea10fd307de75cdeb59e7ea4fcb8add0924859b944babf1112" + }, + { + "package": "nvidia-cusparse", + "version": "12.6.3.3", + "filename": "nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", + "url": "https://files.pythonhosted.org/packages/fa/18/623c77619c31d62efd55302939756966f3ecc8d724a14dab2b75f1508850/nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", + "size": 145942937, + "sha256": "2b3c89c88d01ee0e477cb7f82ef60a11a4bcd57b6b87c33f789350b59759360b" + }, + { + "package": "nvidia-cusparselt-cu13", + "version": "0.8.1", + "filename": "nvidia_cusparselt_cu13-0.8.1-py3-none-manylinux2014_x86_64.whl", + "url": "https://files.pythonhosted.org/packages/34/7d/2661f2fb3ac4302f3a246f5fc030213ac60c1fe0bce84f9783dbd831dbb7/nvidia_cusparselt_cu13-0.8.1-py3-none-manylinux2014_x86_64.whl", + "size": 170148586, + "sha256": "786ce87568c303fadb5afcc7102d454cd3040d75f6f8626f5db460d1871f4dd0" + }, + { + "package": "nvidia-nccl-cu13", + "version": "2.29.7", + "filename": "nvidia_nccl_cu13-2.29.7-py3-none-manylinux_2_18_x86_64.whl", + "url": "https://files.pythonhosted.org/packages/67/f4/58e4e91b6919367c7aafb8e36fce9aad1a3047e536bf7e2fd560927d3a4c/nvidia_nccl_cu13-2.29.7-py3-none-manylinux_2_18_x86_64.whl", + "size": 205976000, + "sha256": "edd81538446786ec3b73972543e53bb43bcaf0bfc8ef76cb679fcc390ffe136d" + }, + { + "package": "nvidia-nvjitlink", + "version": "13.0.88", + "filename": "nvidia_nvjitlink-13.0.88-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", + "url": "https://files.pythonhosted.org/packages/56/7a/123e033aaff487c77107195fa5a2b8686795ca537935a24efae476c41f05/nvidia_nvjitlink-13.0.88-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", + "size": 40713933, + "sha256": "13a74f429e23b921c1109976abefacc69835f2f433ebd323d3946e11d804e47b" + }, + { + "package": "nvidia-nvshmem-cu13", + "version": "3.4.5", + "filename": "nvidia_nvshmem_cu13-3.4.5-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", + "url": "https://files.pythonhosted.org/packages/3c/35/a9bf80a609e74e3b000fef598933235c908fcefcef9026042b8e6dfde2a9/nvidia_nvshmem_cu13-3.4.5-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", + "size": 60412546, + "sha256": "290f0a2ee94c9f3687a02502f3b9299a9f9fe826e6d0287ee18482e78d495b80" + }, + { + "package": "nvidia-nvtx", + "version": "13.0.85", + "filename": "nvidia_nvtx-13.0.85-py3-none-manylinux1_x86_64.manylinux_2_5_x86_64.whl", + "url": "https://files.pythonhosted.org/packages/c2/f3/d86c845465a2723ad7e1e5c36dcd75ddb82898b3f53be47ebd429fb2fa5d/nvidia_nvtx-13.0.85-py3-none-manylinux1_x86_64.manylinux_2_5_x86_64.whl", + "size": 148047, + "sha256": "4936d1d6780fbe68db454f5e72a42ff64d1fd6397df9f363ae786930fd5c1cd4" + }, + { + "package": "packaging", + "version": "26.3", + "filename": "packaging-26.3-py3-none-any.whl", + "url": "https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl", + "size": 129956, + "sha256": "d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c" + }, + { + "package": "pygments", + "version": "2.20.0", + "filename": "pygments-2.20.0-py3-none-any.whl", + "url": "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", + "size": 1231151, + "sha256": "81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176" + }, + { + "package": "pyyaml", + "version": "6.0.3", + "filename": "pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", + "url": "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", + "size": 807870, + "sha256": "ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc" + }, + { + "package": "regex", + "version": "2026.7.19", + "filename": "regex-2026.7.19-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", + "url": "https://files.pythonhosted.org/packages/2a/8e/096d00c7c480ef2ff4265349b14e2261d4ab787ba1f74e2e80d1c58079c3/regex-2026.7.19-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", + "size": 801798, + "sha256": "9dce8ec9695f531a1b8a6f314fd4b393adcccf2ea861db480cdf97a301d01a68" + }, + { + "package": "resampy", + "version": "0.4.3", + "filename": "resampy-0.4.3-py3-none-any.whl", + "url": "https://files.pythonhosted.org/packages/4d/b9/3b00ac340a1aab3389ebcc52c779914a44aadf7b0cb7a3bf053195735607/resampy-0.4.3-py3-none-any.whl", + "size": 3076529, + "sha256": "ad2ed64516b140a122d96704e32bc0f92b23f45419e8b8f478e5a05f83edcebd" + }, + { + "package": "rich", + "version": "15.0.0", + "filename": "rich-15.0.0-py3-none-any.whl", + "url": "https://files.pythonhosted.org/packages/82/3b/64d4899d73f91ba49a8c18a8ff3f0ea8f1c1d75481760df8c68ef5235bf5/rich-15.0.0-py3-none-any.whl", + "size": 310654, + "sha256": "33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb" + }, + { + "package": "safetensors", + "version": "0.7.0", + "filename": "safetensors-0.7.0-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", + "url": "https://files.pythonhosted.org/packages/a0/60/429e9b1cb3fc651937727befe258ea24122d9663e4d5709a48c9cbfceecb/safetensors-0.7.0-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", + "size": 507152, + "sha256": "dac7252938f0696ddea46f5e855dd3138444e82236e3be475f54929f0c510d48" + }, + { + "package": "sentencepiece", + "version": "0.2.1", + "filename": "sentencepiece-0.2.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", + "url": "https://files.pythonhosted.org/packages/04/88/14f2f4a2b922d8b39be45bf63d79e6cd3a9b2f248b2fcb98a69b12af12f5/sentencepiece-0.2.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", + "size": 1387881, + "sha256": "0cdfecef430d985f1c2bcbfff3defd1d95dae876fbd0173376012d2d7d24044b" + }, + { + "package": "setuptools", + "version": "81.0.0", + "filename": "setuptools-81.0.0-py3-none-any.whl", + "url": "https://files.pythonhosted.org/packages/e1/e3/c164c88b2e5ce7b24d667b9bd83589cf4f3520d97cad01534cd3c4f55fdb/setuptools-81.0.0-py3-none-any.whl", + "size": 1062021, + "sha256": "fdd925d5c5d9f62e4b74b30d6dd7828ce236fd6ed998a08d81de62ce5a6310d6" + }, + { + "package": "shellingham", + "version": "1.5.4", + "filename": "shellingham-1.5.4-py2.py3-none-any.whl", + "url": "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", + "size": 9755, + "sha256": "7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686" + }, + { + "package": "sympy", + "version": "1.14.0", + "filename": "sympy-1.14.0-py3-none-any.whl", + "url": "https://files.pythonhosted.org/packages/a2/09/77d55d46fd61b4a135c444fc97158ef34a095e5681d0a6c10b75bf356191/sympy-1.14.0-py3-none-any.whl", + "size": 6299353, + "sha256": "e091cc3e99d2141a0ba2847328f5479b05d94a6635cb96148ccb3f34671bd8f5" + }, + { + "package": "tokenizers", + "version": "0.22.2", + "filename": "tokenizers-0.22.2-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", + "url": "https://files.pythonhosted.org/packages/2e/76/932be4b50ef6ccedf9d3c6639b056a967a86258c6d9200643f01269211ca/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", + "size": 3274982, + "sha256": "369cc9fc8cc10cb24143873a0d95438bb8ee257bb80c71989e3ee290e8d72c67" + }, + { + "package": "torch", + "version": "2.12.1+cu130", + "filename": "torch-2.12.1+cu130-cp312-cp312-manylinux_2_28_x86_64.whl", + "url": "https://download-r2.pytorch.org/whl/cu130/torch-2.12.1%2Bcu130-cp312-cp312-manylinux_2_28_x86_64.whl", + "size": 532716184, + "sha256": "4bafc356fbb622e2756179406825c3a56c17b401196435a1487c5b40c657706c" + }, + { + "package": "tqdm", + "version": "4.70.0", + "filename": "tqdm-4.70.0-py3-none-any.whl", + "url": "https://files.pythonhosted.org/packages/f9/1c/01bfd571a64e7f270e6bab5e33777debe0edc56759233ce84f27dec92d14/tqdm-4.70.0-py3-none-any.whl", + "size": 80184, + "sha256": "7f585706bfddbdebf89daac705b2dfcc16890130727d3197ca62c732b4310953" + }, + { + "package": "transformers", + "version": "5.8.0", + "filename": "transformers-5.8.0-py3-none-any.whl", + "url": "https://files.pythonhosted.org/packages/97/7b/5621d08b34ac35deb9fa14b58d27d124d21ef125ee1c64bc724ca47dfb63/transformers-5.8.0-py3-none-any.whl", + "size": 10630279, + "sha256": "e9d2cae6d195a7e1e05164c5ebf26142a7044e4dc4267274f4809204f92827e4" + }, + { + "package": "triton", + "version": "3.7.1", + "filename": "triton-3.7.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", + "url": "https://files.pythonhosted.org/packages/c4/6f/fb96d15db6f36d6eae4cafb998c2e0353bf59d7c4ea1662d7497f269134a/triton-3.7.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", + "size": 197719725, + "sha256": "7e40869937a68206ec70d7f25bb7ec6433cb083f9135e1f36dbd318dc449a728" + }, + { + "package": "typer", + "version": "0.27.1", + "filename": "typer-0.27.1-py3-none-any.whl", + "url": "https://files.pythonhosted.org/packages/43/89/9518bc0c3929bee36b3a4a8e3daddd6e03f92f9961c66d4983b837160543/typer-0.27.1-py3-none-any.whl", + "size": 122874, + "sha256": "53150287edd11baeb4e4722c8e394fcdf8181c0ae89485cba8d25c778d5edd56" + }, + { + "package": "typing-extensions", + "version": "4.16.0", + "filename": "typing_extensions-4.16.0-py3-none-any.whl", + "url": "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", + "size": 45571, + "sha256": "481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8" + } + ], + "x86_64-pc-windows-msvc": [ + { + "package": "annotated-doc", + "version": "0.0.5", + "filename": "annotated_doc-0.0.5-py3-none-any.whl", + "url": "https://files.pythonhosted.org/packages/3e/30/e900b21425a860e195f32e37657aa1f7c7f2b1bfb26f03ca209b90933c06/annotated_doc-0.0.5-py3-none-any.whl", + "size": 5302, + "sha256": "117bac03a25ede5df5440e855b32d556049ca169ead221505badf432fed4b101" + }, + { + "package": "anyio", + "version": "4.14.2", + "filename": "anyio-4.14.2-py3-none-any.whl", + "url": "https://files.pythonhosted.org/packages/da/35/f2287558c17e29fafc8ef3daf819bb9834061cfa43bff8014f7df7f63bdc/anyio-4.14.2-py3-none-any.whl", + "size": 125813, + "sha256": "9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494" + }, + { + "package": "certifi", + "version": "2026.7.22", + "filename": "certifi-2026.7.22-py3-none-any.whl", + "url": "https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl", + "size": 136983, + "sha256": "62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775" + }, + { + "package": "colorama", + "version": "0.4.6", + "filename": "colorama-0.4.6-py2.py3-none-any.whl", + "url": "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", + "size": 25335, + "sha256": "4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6" + }, + { + "package": "filelock", + "version": "3.32.2", + "filename": "filelock-3.32.2-py3-none-any.whl", + "url": "https://files.pythonhosted.org/packages/c1/e8/72f8cef9fdfeffe06213fe8508039396ee48daa0e3259457ed766173bfd6/filelock-3.32.2-py3-none-any.whl", + "size": 98830, + "sha256": "87dd94cf281e586d135fa51132b8e3d9a598b316e90377a288663c9321036c82" + }, + { + "package": "fsspec", + "version": "2026.7.0", + "filename": "fsspec-2026.7.0-py3-none-any.whl", + "url": "https://files.pythonhosted.org/packages/fd/3c/6a2bf344106328fd04963664a60b9bb6496fc25df8e962fcdc1367285fb9/fsspec-2026.7.0-py3-none-any.whl", + "size": 206583, + "sha256": "b57ddbafedfaef7018c1ecab32aa200a9d7ca26b77965f64e48b70061249d279" + }, + { + "package": "h11", + "version": "0.16.0", + "filename": "h11-0.16.0-py3-none-any.whl", + "url": "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", + "size": 37515, + "sha256": "63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86" + }, + { + "package": "hf-xet", + "version": "1.6.0", + "filename": "hf_xet-1.6.0-cp38-abi3-win_amd64.whl", + "url": "https://files.pythonhosted.org/packages/98/b7/8c59a66d15205024662f1d66968136f13893f96df1ddc5087e2e281fc95f/hf_xet-1.6.0-cp38-abi3-win_amd64.whl", + "size": 4033128, + "sha256": "fb4fadde1b2b70bf4c0c14a6dccbe7194b1c28947fefd5bbe3fed9d940676c3b" + }, + { + "package": "httpcore", + "version": "1.0.9", + "filename": "httpcore-1.0.9-py3-none-any.whl", + "url": "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", + "size": 78784, + "sha256": "2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55" + }, + { + "package": "httpx", + "version": "0.28.1", + "filename": "httpx-0.28.1-py3-none-any.whl", + "url": "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", + "size": 73517, + "sha256": "d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad" + }, + { + "package": "huggingface-hub", + "version": "1.5.0", + "filename": "huggingface_hub-1.5.0-py3-none-any.whl", + "url": "https://files.pythonhosted.org/packages/ec/74/2bc951622e2dbba1af9a460d93c51d15e458becd486e62c29cc0ccb08178/huggingface_hub-1.5.0-py3-none-any.whl", + "size": 596261, + "sha256": "c9c0b3ab95a777fc91666111f3b3ede71c0cdced3614c553a64e98920585c4ee" + }, + { + "package": "idna", + "version": "3.18", + "filename": "idna-3.18-py3-none-any.whl", + "url": "https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl", + "size": 65455, + "sha256": "7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2" + }, + { + "package": "jinja2", + "version": "3.1.6", + "filename": "jinja2-3.1.6-py3-none-any.whl", + "url": "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", + "size": 134899, + "sha256": "85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67" + }, + { + "package": "llvmlite", + "version": "0.48.0", + "filename": "llvmlite-0.48.0-cp312-cp312-win_amd64.whl", + "url": "https://files.pythonhosted.org/packages/16/78/d824ffff7521cd140dc2006e44ce2bc82e64b48d1b32e90e956308c85a74/llvmlite-0.48.0-cp312-cp312-win_amd64.whl", + "size": 41865022, + "sha256": "d45c7541a80934ec6d8ab0defe67439494ecd2193cbf852a44ba827808976ac1" + }, + { + "package": "markdown-it-py", + "version": "4.2.0", + "filename": "markdown_it_py-4.2.0-py3-none-any.whl", + "url": "https://files.pythonhosted.org/packages/b3/81/4da04ced5a082363ecfa159c010d200ecbd959ae410c10c0264a38cac0f5/markdown_it_py-4.2.0-py3-none-any.whl", + "size": 91687, + "sha256": "9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a" + }, + { + "package": "markupsafe", + "version": "3.0.3", + "filename": "markupsafe-3.0.3-cp312-cp312-win_amd64.whl", + "url": "https://files.pythonhosted.org/packages/aa/5b/bec5aa9bbbb2c946ca2733ef9c4ca91c91b6a24580193e891b5f7dbe8e1e/markupsafe-3.0.3-cp312-cp312-win_amd64.whl", + "size": 15105, + "sha256": "26a5784ded40c9e318cfc2bdb30fe164bdb8665ded9cd64d500a34fb42067b1c" + }, + { + "package": "mdurl", + "version": "0.1.2", + "filename": "mdurl-0.1.2-py3-none-any.whl", + "url": "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", + "size": 9979, + "sha256": "84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8" + }, + { + "package": "mpmath", + "version": "1.3.0", + "filename": "mpmath-1.3.0-py3-none-any.whl", + "url": "https://files.pythonhosted.org/packages/43/e3/7d92a15f894aa0c9c4b49b8ee9ac9850d6e63b03c9c32c0367a13ae62209/mpmath-1.3.0-py3-none-any.whl", + "size": 536198, + "sha256": "a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c" + }, + { + "package": "networkx", + "version": "3.6.1", + "filename": "networkx-3.6.1-py3-none-any.whl", + "url": "https://files.pythonhosted.org/packages/9e/c9/b2622292ea83fbb4ec318f5b9ab867d0a28ab43c5717bb85b0a5f6b3b0a4/networkx-3.6.1-py3-none-any.whl", + "size": 2068504, + "sha256": "d47fbf302e7d9cbbb9e2555a0d267983d2aa476bac30e90dfbe5669bd57f3762" + }, + { + "package": "numba", + "version": "0.66.0", + "filename": "numba-0.66.0-cp312-cp312-win_amd64.whl", + "url": "https://files.pythonhosted.org/packages/fc/eb/9e6171e378822ab191c7abcfd3d8cfc8644516f6c7834c22e210e4acc070/numba-0.66.0-cp312-cp312-win_amd64.whl", + "size": 2797403, + "sha256": "b075a4e7ebc43dc6294f223e2821659656209fd5e0ce53245877c23d66d6e1a9" + }, + { + "package": "numpy", + "version": "2.3.5", + "filename": "numpy-2.3.5-cp312-cp312-win_amd64.whl", + "url": "https://files.pythonhosted.org/packages/2d/57/8aeaf160312f7f489dea47ab61e430b5cb051f59a98ae68b7133ce8fa06a/numpy-2.3.5-cp312-cp312-win_amd64.whl", + "size": 12782922, + "sha256": "86945f2ee6d10cdfd67bcb4069c1662dd711f7e2a4343db5cecec06b87cf31aa" + }, + { + "package": "packaging", + "version": "26.3", + "filename": "packaging-26.3-py3-none-any.whl", + "url": "https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl", + "size": 129956, + "sha256": "d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c" + }, + { + "package": "pygments", + "version": "2.20.0", + "filename": "pygments-2.20.0-py3-none-any.whl", + "url": "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", + "size": 1231151, + "sha256": "81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176" + }, + { + "package": "pyyaml", + "version": "6.0.3", + "filename": "pyyaml-6.0.3-cp312-cp312-win_amd64.whl", + "url": "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", + "size": 154003, + "sha256": "5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b" + }, + { + "package": "regex", + "version": "2026.7.19", + "filename": "regex-2026.7.19-cp312-cp312-win_amd64.whl", + "url": "https://files.pythonhosted.org/packages/08/8e/c780c131f79b42ed22d1bd7da4096c2c35f813e835acd02ef0f018bd892c/regex-2026.7.19-cp312-cp312-win_amd64.whl", + "size": 277777, + "sha256": "e30d40268a28d54ce0437031750497004c22602b8e3ab891f759b795a003b312" + }, + { + "package": "resampy", + "version": "0.4.3", + "filename": "resampy-0.4.3-py3-none-any.whl", + "url": "https://files.pythonhosted.org/packages/4d/b9/3b00ac340a1aab3389ebcc52c779914a44aadf7b0cb7a3bf053195735607/resampy-0.4.3-py3-none-any.whl", + "size": 3076529, + "sha256": "ad2ed64516b140a122d96704e32bc0f92b23f45419e8b8f478e5a05f83edcebd" + }, + { + "package": "rich", + "version": "15.0.0", + "filename": "rich-15.0.0-py3-none-any.whl", + "url": "https://files.pythonhosted.org/packages/82/3b/64d4899d73f91ba49a8c18a8ff3f0ea8f1c1d75481760df8c68ef5235bf5/rich-15.0.0-py3-none-any.whl", + "size": 310654, + "sha256": "33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb" + }, + { + "package": "safetensors", + "version": "0.7.0", + "filename": "safetensors-0.7.0-cp38-abi3-win_amd64.whl", + "url": "https://files.pythonhosted.org/packages/5d/e6/ec8471c8072382cb91233ba7267fd931219753bb43814cbc71757bfd4dab/safetensors-0.7.0-cp38-abi3-win_amd64.whl", + "size": 341380, + "sha256": "d1239932053f56f3456f32eb9625590cc7582e905021f94636202a864d470755" + }, + { + "package": "sentencepiece", + "version": "0.2.1", + "filename": "sentencepiece-0.2.1-cp312-cp312-win_amd64.whl", + "url": "https://files.pythonhosted.org/packages/2d/81/92df5673c067148c2545b1bfe49adfd775bcc3a169a047f5a0e6575ddaca/sentencepiece-0.2.1-cp312-cp312-win_amd64.whl", + "size": 1054671, + "sha256": "4cdc7c36234fda305e85c32949c5211faaf8dd886096c7cea289ddc12a2d02de" + }, + { + "package": "setuptools", + "version": "81.0.0", + "filename": "setuptools-81.0.0-py3-none-any.whl", + "url": "https://files.pythonhosted.org/packages/e1/e3/c164c88b2e5ce7b24d667b9bd83589cf4f3520d97cad01534cd3c4f55fdb/setuptools-81.0.0-py3-none-any.whl", + "size": 1062021, + "sha256": "fdd925d5c5d9f62e4b74b30d6dd7828ce236fd6ed998a08d81de62ce5a6310d6" + }, + { + "package": "shellingham", + "version": "1.5.4", + "filename": "shellingham-1.5.4-py2.py3-none-any.whl", + "url": "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", + "size": 9755, + "sha256": "7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686" + }, + { + "package": "sympy", + "version": "1.14.0", + "filename": "sympy-1.14.0-py3-none-any.whl", + "url": "https://files.pythonhosted.org/packages/a2/09/77d55d46fd61b4a135c444fc97158ef34a095e5681d0a6c10b75bf356191/sympy-1.14.0-py3-none-any.whl", + "size": 6299353, + "sha256": "e091cc3e99d2141a0ba2847328f5479b05d94a6635cb96148ccb3f34671bd8f5" + }, + { + "package": "tokenizers", + "version": "0.22.2", + "filename": "tokenizers-0.22.2-cp39-abi3-win_amd64.whl", + "url": "https://files.pythonhosted.org/packages/65/71/0670843133a43d43070abeb1949abfdef12a86d490bea9cd9e18e37c5ff7/tokenizers-0.22.2-cp39-abi3-win_amd64.whl", + "size": 2747786, + "sha256": "c9ea31edff2968b44a88f97d784c2f16dc0729b8b143ed004699ebca91f05c48" + }, + { + "package": "torch", + "version": "2.12.1+cu130", + "filename": "torch-2.12.1+cu130-cp312-cp312-win_amd64.whl", + "url": "https://download-r2.pytorch.org/whl/cu130/torch-2.12.1%2Bcu130-cp312-cp312-win_amd64.whl", + "size": 1926431198, + "sha256": "52c5da6a0898d5d3473c02bd304b7a3bc0b72e351c6f3bfa0783e45ef9f4cd61" + }, + { + "package": "tqdm", + "version": "4.70.0", + "filename": "tqdm-4.70.0-py3-none-any.whl", + "url": "https://files.pythonhosted.org/packages/f9/1c/01bfd571a64e7f270e6bab5e33777debe0edc56759233ce84f27dec92d14/tqdm-4.70.0-py3-none-any.whl", + "size": 80184, + "sha256": "7f585706bfddbdebf89daac705b2dfcc16890130727d3197ca62c732b4310953" + }, + { + "package": "transformers", + "version": "5.8.0", + "filename": "transformers-5.8.0-py3-none-any.whl", + "url": "https://files.pythonhosted.org/packages/97/7b/5621d08b34ac35deb9fa14b58d27d124d21ef125ee1c64bc724ca47dfb63/transformers-5.8.0-py3-none-any.whl", + "size": 10630279, + "sha256": "e9d2cae6d195a7e1e05164c5ebf26142a7044e4dc4267274f4809204f92827e4" + }, + { + "package": "typer", + "version": "0.27.1", + "filename": "typer-0.27.1-py3-none-any.whl", + "url": "https://files.pythonhosted.org/packages/43/89/9518bc0c3929bee36b3a4a8e3daddd6e03f92f9961c66d4983b837160543/typer-0.27.1-py3-none-any.whl", + "size": 122874, + "sha256": "53150287edd11baeb4e4722c8e394fcdf8181c0ae89485cba8d25c778d5edd56" + }, + { + "package": "typing-extensions", + "version": "4.16.0", + "filename": "typing_extensions-4.16.0-py3-none-any.whl", + "url": "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", + "size": 45571, + "sha256": "481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8" + } + ] + } +} diff --git a/sa3-pin.json b/sa3-pin.json index 5bde79f..a93d15f 100644 --- a/sa3-pin.json +++ b/sa3-pin.json @@ -1,5 +1,133 @@ { "note": "Back on upstream: our MLX LoRA merge-at-load (ADR-0028) landed as PR #57 (2026-07-14), and #65 extended it with per-adapter strength= and step gating. Bump via ADR-0012's upgrade path.", "repo": "https://github.com/Stability-AI/stable-audio-3", - "commit": "0385302ea26522f00c80392c4b708df5ebf1adf5" + "commit": "a0b57f5483c4588f827f3552b7d5c6ca2a9687be", + "source": { + "url": "https://github.com/Stability-AI/stable-audio-3/archive/a0b57f5483c4588f827f3552b7d5c6ca2a9687be.tar.gz", + "sha256": "98e206e061a3b64a4e65f50b2802bdb6965910ac1fab65da919808dfb4497e9f", + "size": 50494239, + "archiveRoot": "stable-audio-3-a0b57f5483c4588f827f3552b7d5c6ca2a9687be", + "maxFiles": 2000, + "maxExpandedBytes": 134217728 + }, + "runtime": { + "requirements": "sa3-requirements.lock", + "python": [ + { + "target": "aarch64-apple-darwin", + "version": "3.11.13", + "url": "https://releases.astral.sh/github/python-build-standalone/releases/download/20251007/cpython-3.11.13%2B20251007-aarch64-apple-darwin-install_only_stripped.tar.gz", + "sha256": "78bc6defdc1dac5bf6765c8f938e6849383dbed831ea1e2d11576a4683fb1e8c", + "size": 18949778, + "archiveRoot": "python", + "executable": "bin/python3.11", + "maxFiles": 5000, + "maxExpandedBytes": 134217728 + }, + { + "target": "x86_64-unknown-linux-gnu", + "version": "3.11.13", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20251007/cpython-3.11.13%2B20251007-x86_64-unknown-linux-gnu-install_only_stripped.tar.gz", + "sha256": "43bfc42529843ecd1d9c08c4a239ede348f96ff0acaef2ec24b28dc059f4f0c3", + "size": 30157215, + "archiveRoot": "python", + "executable": "bin/python3.11", + "maxFiles": 5000, + "maxExpandedBytes": 536870912 + }, + { + "target": "x86_64-pc-windows-msvc", + "version": "3.11.13", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20251007/cpython-3.11.13%2B20251007-x86_64-pc-windows-msvc-install_only_stripped.tar.gz", + "sha256": "cde5153f59a67d9e108f2ed964526e9aed100eba180f54bee0496b4fd73a8b29", + "size": 25990147, + "archiveRoot": "python", + "executable": "python.exe", + "maxFiles": 5000, + "maxExpandedBytes": 536870912 + } + ], + "uv": [ + { + "target": "aarch64-apple-darwin", + "version": "0.11.7", + "url": "https://github.com/astral-sh/uv/releases/download/0.11.7/uv-aarch64-apple-darwin.tar.gz", + "sha256": "66e37d91f839e12481d7b932a1eccbfe732560f42c1cfb89faddfa2454534ba8", + "size": 20839135, + "archiveRoot": "uv-aarch64-apple-darwin", + "executable": "uv", + "maxFiles": 8, + "maxExpandedBytes": 67108864 + }, + { + "target": "x86_64-unknown-linux-gnu", + "version": "0.11.7", + "url": "https://github.com/astral-sh/uv/releases/download/0.11.7/uv-x86_64-unknown-linux-gnu.tar.gz", + "sha256": "6681d691eb7f9c00ac6a3af54252f7ab29ae72f0c8f95bdc7f9d1401c23ea868", + "size": 24249861, + "archiveRoot": "uv-x86_64-unknown-linux-gnu", + "executable": "uv", + "maxFiles": 8, + "maxExpandedBytes": 67108864 + }, + { + "target": "x86_64-pc-windows-msvc", + "version": "0.11.7", + "url": "https://github.com/astral-sh/uv/releases/download/0.11.7/uv-x86_64-pc-windows-msvc.zip", + "sha256": "fe0c7815acf4fc45f8a5eff58ed3cf7ae2e15c3cf1dceadbd10c816ec1690cc1", + "size": 23572531, + "archiveRoot": "uv-x86_64-pc-windows-msvc", + "archiveFormat": "zip", + "executable": "uv.exe", + "maxFiles": 8, + "maxExpandedBytes": 67108864 + } + ] + }, + "models": { + "repo": "stabilityai/stable-audio-3-optimized", + "revision": "6736003cb57d06b7b1fdc36fad31b2a3709e4774", + "artifacts": [ + { + "path": "MLX/dit_medium_f16.npz", + "sha256": "f9e5647ea3225818657d47d47ae4b34afa29c0568206ca89566c1a758944a38e", + "size": 2907300946 + }, + { + "path": "MLX/dit_sm-music_f16.npz", + "sha256": "8ed3f38e2597f361ee675051f1265d9aa2ae2fffce1c61acd2e9fe31e1db1cbc", + "size": 919193814 + }, + { + "path": "MLX/dit_sm-sfx_f16.npz", + "sha256": "7e702d2640699a57fe436ca975fda16832040ba568c1e092c2ae826987558118", + "size": 919193814 + }, + { + "path": "MLX/same_l_decoder_f32.npz", + "sha256": "84924be2122d3a20fce443f40b782d9cd88e8e73707476326003ac47659a2287", + "size": 1704311976 + }, + { + "path": "MLX/same_l_encoder_f32.npz", + "sha256": "c5caafc6bd29fc3d4cb7a08b0d3725041fa2100711f86a4f05d2638c532871d2", + "size": 1704313504 + }, + { + "path": "MLX/same_s_decoder_f32.npz", + "sha256": "909928a8e6937c1ebe6ac4b729f0462bd3773704a11ea18278e42671dc69bfe4", + "size": 218090820 + }, + { + "path": "MLX/same_s_encoder_f32.npz", + "sha256": "a48f80d81c30d74c45e2a3047082c4891f715e24c44645adb9c1f4f07afdaf0c", + "size": 214946620 + }, + { + "path": "MLX/t5gemma_f16.npz", + "sha256": "8deb20489f36d9aec539f26c9c67321f99bc5fe300d470435ed6e76be4f16bbd", + "size": 567443068 + } + ] + } } diff --git a/sa3-pytorch-cuda-pin.json b/sa3-pytorch-cuda-pin.json new file mode 100644 index 0000000..1c63ead --- /dev/null +++ b/sa3-pytorch-cuda-pin.json @@ -0,0 +1,93 @@ +{ + "schemaVersion": 1, + "releaseReady": false, + "backend": "pytorch_cuda", + "platform": "windows-x86_64", + "source": { + "repository": "https://github.com/Stability-AI/stable-audio-3", + "revision": "a0b57f5483c4588f827f3552b7d5c6ca2a9687be", + "archiveUrl": "https://github.com/Stability-AI/stable-audio-3/archive/a0b57f5483c4588f827f3552b7d5c6ca2a9687be.tar.gz", + "size": 50494239, + "sha256": "98e206e061a3b64a4e65f50b2802bdb6965910ac1fab65da919808dfb4497e9f", + "license": "MIT" + }, + "sharedRuntime": { + "python": "3.12", + "cuda": "12.6", + "requirementsInput": "backend/runtime-locks/windows-gpu-pytorch.in", + "requirementsLock": "backend/runtime-locks/windows-gpu-pytorch.txt", + "requirementsLockSize": 56238, + "requirementsLockSha256": "3c9bf7d79c3848ebe1da40fd14b26708b55d8157f008cb3a1944ddfb1cd597c4", + "packages": { + "torch": "2.7.1+cu126", + "torchaudio": "2.7.1+cu126", + "transformers": "5.8.0", + "huggingface-hub": "1.7.1", + "numpy": "2.3.5", + "safetensors": "0.7.0", + "sentencepiece": "0.2.1", + "resampy": "0.4.3" + }, + "mrt2RequalificationRequired": true + }, + "models": { + "small-music": { + "repository": "stabilityai/stable-audio-3-small-music", + "revision": "0fef1392cd842149a2b6d445e181c97608faac06", + "required": true, + "weight": { + "path": "model.safetensors", + "size": 2270384940, + "sha256": "da85866b11b01d0694d990785f6abbd79c8064df1b0e6f8aea52935e0ef84b64" + }, + "config": { + "path": "model_config.json", + "size": 10341, + "gitBlobSha1": "29ba617d6556c9e3c94bea19f07ea66d0c895e9e", + "sha256": null + } + }, + "small-sfx": { + "repository": "stabilityai/stable-audio-3-small-sfx", + "revision": "ae12755283df9d62ca39a9b050a39a0b607b8c20", + "required": true, + "weight": { + "path": "model.safetensors", + "size": 2270384940, + "sha256": "ed9cf1b6172f1a8c2921a9560c21109ff3239524563ced9dce6dcdef41e2f515" + }, + "config": { + "path": "model_config.json", + "size": 10454, + "gitBlobSha1": null, + "sha256": null + } + }, + "medium": { + "repository": "stabilityai/stable-audio-3-medium", + "revision": "27b5a21b791b1b033d193a9e1e3ce78493f102f9", + "required": false, + "enabled": false, + "blockedBy": "No official Windows FlashAttention 2 build has been qualified", + "weight": { + "path": "model.safetensors", + "size": 9222116660, + "sha256": "48d9c65e290e7bcd5194e0633bfc2424a59ee9683f5c2d58762d997b7d8ce0b5" + }, + "config": { + "path": "model_config.json", + "size": 10360, + "gitBlobSha1": null, + "sha256": null + } + } + }, + "gatedArtifactsComplete": false, + "releaseBlockers": [ + "Authenticated SHA-256 audit of all model configs and nested T5Gemma artifacts after the terms flow in issue #108", + "Small Music and Small SFX measured VRAM reservations with conservative headroom", + "MRT2 functional and real-time parity on the shared torch 2.7.1/CUDA 12.6 runtime", + "Windows NVIDIA OOM, cancellation, crash, exit, and VRAM-release evidence", + "Ten-minute dual-deck 25-frame and 5-frame runs with zero engine-reported underruns" + ] +} diff --git a/sa3-tflite-pin.json b/sa3-tflite-pin.json new file mode 100644 index 0000000..7a0db6d --- /dev/null +++ b/sa3-tflite-pin.json @@ -0,0 +1,86 @@ +{ + "schemaVersion": 1, + "runtime": { + "repo": "https://github.com/Stability-AI/stable-audio-3", + "revision": "a0b57f5483c4588f827f3552b7d5c6ca2a9687be", + "subdirectory": "optimized/tflite", + "entrypoint": "scripts/sa3_tflite.py", + "requirementsInput": "scripts/sa3-tflite-requirements.in", + "requirementsLock": "scripts/sa3-tflite-requirements.lock" + }, + "models": { + "repo": "stabilityai/stable-audio-3-optimized", + "revision": "6736003cb57d06b7b1fdc36fad31b2a3709e4774", + "precision": "fp32", + "shared": [ + { + "path": "tflite/t5gemma/encoder_fp16.tflite", + "installPath": "models/tflite/t5gemma/encoder_fp16.tflite", + "size": 563818608, + "sha256": "8530d0b3e6b9b9dcf1239145c2a853fb749708eaddbb472ff8f0802b50059372" + } + ], + "bundles": { + "sm-music": [ + { + "path": "tflite/sa3-sm-music/dit_fp32.tflite", + "installPath": "models/tflite/sa3-sm-music/dit_fp32.tflite", + "size": 1838758544, + "sha256": "d388700a2ca439c11e9a53506e964e93231386a2beb8173c6eec6d95f676ce09" + }, + { + "path": "tflite/same-s/enc_fp32.tflite", + "installPath": "models/tflite/same-s/enc_fp32.tflite", + "size": 215195204, + "sha256": "35ce38ea9f56e116036c683e37bf96c954d4fe0a435606ded0f62595b91f52a3" + }, + { + "path": "tflite/same-s/dec_fp32.tflite", + "installPath": "models/tflite/same-s/dec_fp32.tflite", + "size": 218377156, + "sha256": "cd87fa6686b24a56dc3497e05fbb26a34cf9604afe49c6631e829c9e70fccf21" + } + ], + "sm-sfx": [ + { + "path": "tflite/sa3-sm-sfx/dit_fp32.tflite", + "installPath": "models/tflite/sa3-sm-sfx/dit_fp32.tflite", + "size": 1838758544, + "sha256": "6060ecfeca34c4ab35bc1912a37e680e8cd7aab6c4bd9de1bc2655414891b8d8" + }, + { + "path": "tflite/same-s/enc_fp32.tflite", + "installPath": "models/tflite/same-s/enc_fp32.tflite", + "size": 215195204, + "sha256": "35ce38ea9f56e116036c683e37bf96c954d4fe0a435606ded0f62595b91f52a3" + }, + { + "path": "tflite/same-s/dec_fp32.tflite", + "installPath": "models/tflite/same-s/dec_fp32.tflite", + "size": 218377156, + "sha256": "cd87fa6686b24a56dc3497e05fbb26a34cf9604afe49c6631e829c9e70fccf21" + } + ], + "medium": [ + { + "path": "tflite/sa3-m/dit_fp32.tflite", + "installPath": "models/tflite/sa3-m/dit_fp32.tflite", + "size": 5816313104, + "sha256": "b811dc7d0135ca48afbc7a7bb7d19bdaaad13cbcb592418b8aa169e0c149daba" + }, + { + "path": "tflite/same-l/enc_fp32.tflite", + "installPath": "models/tflite/same-l/enc_fp32.tflite", + "size": 1823872896, + "sha256": "f8b5e95a7073e3b59e4a1c2b07836d86d514cc7eaaff05b3c7cbdd1620f141d5" + }, + { + "path": "tflite/same-l/dec_fp32.tflite", + "installPath": "models/tflite/same-l/dec_fp32.tflite", + "size": 1823900848, + "sha256": "3af34d35939ce6fc74d9f7b9d9bd6b99bc9568b614bcfe57da4a781bf40c8c6c" + } + ] + } + } +} diff --git a/sa3-tflite-wheels.json b/sa3-tflite-wheels.json new file mode 100644 index 0000000..3ea65b2 --- /dev/null +++ b/sa3-tflite-wheels.json @@ -0,0 +1,55 @@ +{ + "schemaVersion": 1, + "python": "3.11", + "common": [ + {"package":"annotated-doc","version":"0.0.5","filename":"annotated_doc-0.0.5-py3-none-any.whl","url":"https://files.pythonhosted.org/packages/3e/30/e900b21425a860e195f32e37657aa1f7c7f2b1bfb26f03ca209b90933c06/annotated_doc-0.0.5-py3-none-any.whl","size":5302,"sha256":"117bac03a25ede5df5440e855b32d556049ca169ead221505badf432fed4b101"}, + {"package":"annotated-types","version":"0.8.0","filename":"annotated_types-0.8.0-py3-none-any.whl","url":"https://files.pythonhosted.org/packages/99/91/8acff4f5e50511b911bbccb72b8628a49c68ce14148cd9f6431094859a90/annotated_types-0.8.0-py3-none-any.whl","size":13427,"sha256":"f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0"}, + {"package":"anyio","version":"4.14.2","filename":"anyio-4.14.2-py3-none-any.whl","url":"https://files.pythonhosted.org/packages/da/35/f2287558c17e29fafc8ef3daf819bb9834061cfa43bff8014f7df7f63bdc/anyio-4.14.2-py3-none-any.whl","size":125813,"sha256":"9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494"}, + {"package":"backports-strenum","version":"1.3.1","filename":"backports_strenum-1.3.1-py3-none-any.whl","url":"https://files.pythonhosted.org/packages/d6/50/56cf20e2ee5127b603b81d5a69580a1a325083e2b921aa8f067da83927c0/backports_strenum-1.3.1-py3-none-any.whl","size":8304,"sha256":"cdcfe36dc897e2615dc793b7d3097f54d359918fc448754a517e6f23044ccf83"}, + {"package":"certifi","version":"2026.7.22","filename":"certifi-2026.7.22-py3-none-any.whl","url":"https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl","size":136983,"sha256":"62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775"}, + {"package":"click","version":"8.4.2","filename":"click-8.4.2-py3-none-any.whl","url":"https://files.pythonhosted.org/packages/fb/e2/79c688af8b210d232694e31e59da9f6ec747bae31c3f5946e4e9b98860d5/click-8.4.2-py3-none-any.whl","size":119243,"sha256":"e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76"}, + {"package":"filelock","version":"3.32.2","filename":"filelock-3.32.2-py3-none-any.whl","url":"https://files.pythonhosted.org/packages/c1/e8/72f8cef9fdfeffe06213fe8508039396ee48daa0e3259457ed766173bfd6/filelock-3.32.2-py3-none-any.whl","size":98830,"sha256":"87dd94cf281e586d135fa51132b8e3d9a598b316e90377a288663c9321036c82"}, + {"package":"fastapi","version":"0.136.3","filename":"fastapi-0.136.3-py3-none-any.whl","url":"https://files.pythonhosted.org/packages/e0/82/45359b62a067409bd929ae8a56b8ed13e5a8c8a61194b3c236920999ab83/fastapi-0.136.3-py3-none-any.whl","size":117481,"sha256":"3d2a69bdf04b7e9f3afa292c3bc7a98816bbfafa10bc9b45f3f3700d2f761620"}, + {"package":"flatbuffers","version":"25.12.19","filename":"flatbuffers-25.12.19-py2.py3-none-any.whl","url":"https://files.pythonhosted.org/packages/e8/2d/d2a548598be01649e2d46231d151a6c56d10b964d94043a335ae56ea2d92/flatbuffers-25.12.19-py2.py3-none-any.whl","size":26661,"sha256":"7634f50c427838bb021c2d66a3d1168e9d199b0607e6329399f04846d42e20b4"}, + {"package":"fsspec","version":"2026.7.0","filename":"fsspec-2026.7.0-py3-none-any.whl","url":"https://files.pythonhosted.org/packages/fd/3c/6a2bf344106328fd04963664a60b9bb6496fc25df8e962fcdc1367285fb9/fsspec-2026.7.0-py3-none-any.whl","size":206583,"sha256":"b57ddbafedfaef7018c1ecab32aa200a9d7ca26b77965f64e48b70061249d279"}, + {"package":"h11","version":"0.16.0","filename":"h11-0.16.0-py3-none-any.whl","url":"https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl","size":37515,"sha256":"63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86"}, + {"package":"httpcore","version":"1.0.9","filename":"httpcore-1.0.9-py3-none-any.whl","url":"https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl","size":78784,"sha256":"2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55"}, + {"package":"httpx","version":"0.28.1","filename":"httpx-0.28.1-py3-none-any.whl","url":"https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl","size":73517,"sha256":"d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad"}, + {"package":"huggingface-hub","version":"1.27.0","filename":"huggingface_hub-1.27.0-py3-none-any.whl","url":"https://files.pythonhosted.org/packages/de/d8/95b735e183957c1f26d94c52977f09d466d55119cbbc1558ea4975e4c216/huggingface_hub-1.27.0-py3-none-any.whl","size":784926,"sha256":"7df6827c2f956c60fbaa64646e979e566db76f619dd0a9729dfb8c5a3eb4f68d"}, + {"package":"idna","version":"3.18","filename":"idna-3.18-py3-none-any.whl","url":"https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl","size":65455,"sha256":"7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2"}, + {"package":"packaging","version":"26.3","filename":"packaging-26.3-py3-none-any.whl","url":"https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl","size":129956,"sha256":"d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c"}, + {"package":"pydantic","version":"2.13.4","filename":"pydantic-2.13.4-py3-none-any.whl","url":"https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl","size":472262,"sha256":"45a282cde31d808236fd7ea9d919b128653c8b38b393d1c4ab335c62924d9aba"}, + {"package":"pycparser","version":"3.0","filename":"pycparser-3.0-py3-none-any.whl","url":"https://files.pythonhosted.org/packages/0c/c3/44f3fbbfa403ea2a7c779186dc20772604442dde72947e7d01069cbe98e3/pycparser-3.0-py3-none-any.whl","size":48172,"sha256":"b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992"}, + {"package":"python-multipart","version":"0.0.32","filename":"python_multipart-0.0.32-py3-none-any.whl","url":"https://files.pythonhosted.org/packages/e1/04/e8135ebd1ad02c56ec633277529b2602ff99ff634be76cdba5744cf554fd/python_multipart-0.0.32-py3-none-any.whl","size":30042,"sha256":"ff6d3f776f16878c894e52e107296ffc890e913c611b1a4ec6c44e2821fe2e23"}, + {"package":"starlette","version":"1.6.0","filename":"starlette-1.6.0-py3-none-any.whl","url":"https://files.pythonhosted.org/packages/c8/cb/6a6a47d5b464bd08695d254f3da6e7986cc70c9fa5d778eda57538edfe56/starlette-1.6.0-py3-none-any.whl","size":75969,"sha256":"a86dd39d14bb45f85a3d18525215a9ef0cfd1f192ac793220e72598c90335f0c"}, + {"package":"tqdm","version":"4.70.0","filename":"tqdm-4.70.0-py3-none-any.whl","url":"https://files.pythonhosted.org/packages/f9/1c/01bfd571a64e7f270e6bab5e33777debe0edc56759233ce84f27dec92d14/tqdm-4.70.0-py3-none-any.whl","size":80184,"sha256":"7f585706bfddbdebf89daac705b2dfcc16890130727d3197ca62c732b4310953"}, + {"package":"typing-inspection","version":"0.4.2","filename":"typing_inspection-0.4.2-py3-none-any.whl","url":"https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl","size":14611,"sha256":"4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7"}, + {"package":"typing-extensions","version":"4.16.0","filename":"typing_extensions-4.16.0-py3-none-any.whl","url":"https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl","size":45571,"sha256":"481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8"}, + {"package":"uvicorn","version":"0.49.0","filename":"uvicorn-0.49.0-py3-none-any.whl","url":"https://files.pythonhosted.org/packages/88/fa/e1388bbcf24ef3274f45c0c1c7b501fd14971037c1b6ee23610553307497/uvicorn-0.49.0-py3-none-any.whl","size":71376,"sha256":"ba3d14c3ee7e41c6c654c46c9eb489d33213cdd30aa1696eab1374337c13f68f"} + ], + "targets": { + "x86_64-unknown-linux-gnu": [ + {"package":"ai-edge-litert","version":"2.1.6","filename":"ai_edge_litert-2.1.6-cp311-cp311-manylinux_2_27_x86_64.whl","url":"https://files.pythonhosted.org/packages/b2/b9/c7153b2f6f37bc521876b8fa86125eb618bba9d84b20ba0bc5c05795eda7/ai_edge_litert-2.1.6-cp311-cp311-manylinux_2_27_x86_64.whl","size":17328951,"sha256":"af4f2ba681fa2c688746cbd7ddd71a2bbbdd9e6a51aa609d382bfad77d1c695e"}, + {"package":"cffi","version":"2.1.1","filename":"cffi-2.1.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl","url":"https://files.pythonhosted.org/packages/f7/a4/4399daaf8f7dfee9d7c3327fdb0426ee041cc63edc358b93911ceb2bfc7a/cffi-2.1.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl","size":217807,"sha256":"34e261f78cb6ceaaa36f42f2613f4380d94d9c759a9c73c769ee6e0247364632"}, + {"package":"hf-xet","version":"1.6.0","filename":"hf_xet-1.6.0-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl","url":"https://files.pythonhosted.org/packages/67/4e/a28359bf1c1ecf11eba22123168c138698f7cb576ac678f5a2e16cd5da08/hf_xet-1.6.0-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl","size":4464663,"sha256":"d62671bb130879cef0ee4c9ebe47a14af6c66ec53e6d84dc15936e5ffdfac82f"}, + {"package":"numpy","version":"2.4.6","filename":"numpy-2.4.6-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl","url":"https://files.pythonhosted.org/packages/02/03/74fe2a4cb3817d94d86402f2506554130a2f01414e299b5a843e5a8a957f/numpy-2.4.6-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl","size":16918164,"sha256":"89cd468399cfd2504718f0ba50e410dca55a170b61a02ad92bb18c8a65186e93"}, + {"package":"protobuf","version":"7.35.1","filename":"protobuf-7.35.1-cp310-abi3-manylinux2014_x86_64.whl","url":"https://files.pythonhosted.org/packages/e4/be/5b3cfe508bfab6761414ff944e3366eb13be4fd71efcd69450f89ba39f43/protobuf-7.35.1-cp310-abi3-manylinux2014_x86_64.whl","size":327130,"sha256":"74758715c53d7158fb76caf4f0cfdacc5329a4b1bb994f865d6cf302d413a1c4"}, + {"package":"pydantic-core","version":"2.46.4","filename":"pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl","url":"https://files.pythonhosted.org/packages/80/50/540cd3aeefc041beb111125c4bff779831a2111fc6b15a9138cda277d32c/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl","size":2089685,"sha256":"f9fa868638bf362d3d138ea55829cefb3d5f4b0d7f142234382a15e2485dbec4"}, + {"package":"pyyaml","version":"6.0.3","filename":"pyyaml-6.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl","url":"https://files.pythonhosted.org/packages/71/60/917329f640924b18ff085ab889a11c763e0b573da888e8404ff486657602/pyyaml-6.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl","size":806638,"sha256":"b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d"}, + {"package":"sentencepiece","version":"0.2.2","filename":"sentencepiece-0.2.2-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl","url":"https://files.pythonhosted.org/packages/06/5f/9117bf854aef817ad0d0ee9310eed0308a7e529e7eaf2e80ad9cd281ef82/sentencepiece-0.2.2-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl","size":1394242,"sha256":"1416b92f2f010333786fe6306ed2631121d5ea492219b0841e967b6765e64107"}, + {"package":"soundfile","version":"0.14.0","filename":"soundfile-0.14.0-py2.py3-none-manylinux_2_28_x86_64.whl","url":"https://files.pythonhosted.org/packages/7b/a2/70fd4432b924684c372df8b0a45708c36c057ef3596c9eb53e0a806b980b/soundfile-0.14.0-py2.py3-none-manylinux_2_28_x86_64.whl","size":1315963,"sha256":"1e38bac1853412871318e82a1ba69a8be677619b56025bbfcccdb41b6cafe82d"} + ], + "x86_64-pc-windows-msvc": [ + {"package":"ai-edge-litert","version":"2.1.6","filename":"ai_edge_litert-2.1.6-cp311-cp311-win_amd64.whl","url":"https://files.pythonhosted.org/packages/87/0f/031a56bc95a7109f4a71ec5510a34214d5f9d99ff01a926fb2129ec2288b/ai_edge_litert-2.1.6-cp311-cp311-win_amd64.whl","size":16587870,"sha256":"b20f4c8cdbbf6f64e3baa77853e55a1b29515a45d6dfbe6b2ced9b3a1efb5807"}, + {"package":"cffi","version":"2.1.1","filename":"cffi-2.1.1-cp311-cp311-win_amd64.whl","url":"https://files.pythonhosted.org/packages/73/c0/77ba02423c2f7d7091143c45cd49e0e6575c4c1967394bb542bd923a9b74/cffi-2.1.1-cp311-cp311-win_amd64.whl","size":185096,"sha256":"42f6930c31dc7f50732c9ae793c2786c7b6b044195967bbdde40bb9be81c4cc0"}, + {"package":"colorama","version":"0.4.6","filename":"colorama-0.4.6-py2.py3-none-any.whl","url":"https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl","size":25335,"sha256":"4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {"package":"hf-xet","version":"1.6.0","filename":"hf_xet-1.6.0-cp38-abi3-win_amd64.whl","url":"https://files.pythonhosted.org/packages/98/b7/8c59a66d15205024662f1d66968136f13893f96df1ddc5087e2e281fc95f/hf_xet-1.6.0-cp38-abi3-win_amd64.whl","size":4033128,"sha256":"fb4fadde1b2b70bf4c0c14a6dccbe7194b1c28947fefd5bbe3fed9d940676c3b"}, + {"package":"numpy","version":"2.4.6","filename":"numpy-2.4.6-cp311-cp311-win_amd64.whl","url":"https://files.pythonhosted.org/packages/c5/31/7fc6239c12bce7e931463251cca4426c465e1876ba3cc785402ef4dd8f4e/numpy-2.4.6-cp311-cp311-win_amd64.whl","size":12608406,"sha256":"1e254a00cdf42b1e4d5b3d68d33af63268d41340d8885df2ab6470f2e1500147"}, + {"package":"protobuf","version":"7.35.1","filename":"protobuf-7.35.1-cp310-abi3-win_amd64.whl","url":"https://files.pythonhosted.org/packages/0a/19/8d0cb6f20a1ef7b18f1c8986ad5783f22f84cce39c6ce9a6e645ea55192e/protobuf-7.35.1-cp310-abi3-win_amd64.whl","size":439996,"sha256":"230a75ddfc2de4806e56696ce9640c1cdfdb6543b7cfce98d42a4c0a0e7bdb87"}, + {"package":"pydantic-core","version":"2.46.4","filename":"pydantic_core-2.46.4-cp311-cp311-win_amd64.whl","url":"https://files.pythonhosted.org/packages/aa/e6/c505f83dfeda9a2e5c995cfd872949e4d05e12f7feb3dca72f633daefa94/pydantic_core-2.46.4-cp311-cp311-win_amd64.whl","size":2071114,"sha256":"6f2eeda33a839975441c86a4119e1383c50b47faf0cbb5176985565c6bb02c33"}, + {"package":"pyyaml","version":"6.0.3","filename":"pyyaml-6.0.3-cp311-cp311-win_amd64.whl","url":"https://files.pythonhosted.org/packages/da/e3/ea007450a105ae919a72393cb06f122f288ef60bba2dc64b26e2646fa315/pyyaml-6.0.3-cp311-cp311-win_amd64.whl","size":158763,"sha256":"9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf"}, + {"package":"sentencepiece","version":"0.2.2","filename":"sentencepiece-0.2.2-cp311-cp311-win_amd64.whl","url":"https://files.pythonhosted.org/packages/ab/62/9e2569867e3dcff7ad6d89642a9615b9801b5cd698abe7df3b490361f66e/sentencepiece-0.2.2-cp311-cp311-win_amd64.whl","size":1246268,"sha256":"70d4ca6f4d06df7f0ccab6fe4f49c8a712c8c8b6847b4f0af9a0e1dbb0e0337e"}, + {"package":"soundfile","version":"0.14.0","filename":"soundfile-0.14.0-py2.py3-none-win_amd64.whl","url":"https://files.pythonhosted.org/packages/ed/97/b39c18ac1df45e755ca22b8b00e872929da5d107998a207a5e4ac831bfda/soundfile-0.14.0-py2.py3-none-win_amd64.whl","size":1021480,"sha256":"299491d3499460fb1b74bb4bd78b57ffc2d243a5fafa7b6ec1b264875c78453e"} + ] + } +} diff --git a/scripts/assert-windows-release-rejects-unsigned.ps1 b/scripts/assert-windows-release-rejects-unsigned.ps1 new file mode 100644 index 0000000..ce7b1a7 --- /dev/null +++ b/scripts/assert-windows-release-rejects-unsigned.ps1 @@ -0,0 +1,31 @@ +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)] + [ValidateNotNullOrEmpty()] + [string] $Path +) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version Latest + +# Hosted pull-request CI deliberately creates unsigned development installers. +# Prove the release verifier refuses one with an otherwise syntactically valid +# expected identity. The exact NotSigned failure is required so a missing SDK +# tool or unrelated script error cannot masquerade as successful rejection. +$powerShell = (Get-Process -Id $PID).Path +$PSNativeCommandUseErrorActionPreference = $false +$output = & $powerShell -NoLogo -NoProfile -NonInteractive -File ` + "$PSScriptRoot/verify-windows-signatures.ps1" ` + -Path $Path ` + -ExpectedCertificateSha1 ('0' * 40) ` + -ExpectedSubject 'CN=Unsigned CI Sentinel' 2>&1 +$exitCode = $LASTEXITCODE +$rendered = $output | Out-String +if ($exitCode -eq 0) { + throw 'Release signature verification accepted an unsigned development installer.' +} +if ($rendered -notmatch 'Authenticode signature status is NotSigned') { + throw "Release verification failed for an unexpected reason instead of rejecting an unsigned artifact:`n$rendered" +} +Write-Host 'Release signature verification correctly rejected the unsigned development installer.' +exit 0 diff --git a/scripts/audit-sa3-cuda-pin.py b/scripts/audit-sa3-cuda-pin.py new file mode 100644 index 0000000..c21bf60 --- /dev/null +++ b/scripts/audit-sa3-cuda-pin.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +"""Audit the fail-closed Windows SA3/CUDA candidate manifest. + +This audit deliberately fails for a release while any gated artifact lacks an +application-controlled SHA-256. ``--allow-incomplete`` is only for reviewing +the public, immutable metadata before issue #108 supplies an authenticated +terms/download flow; it does not make the runtime installable. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import re +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent.parent +PIN_PATH = ROOT / "sa3-pytorch-cuda-pin.json" +REQUIREMENT = re.compile(r"(?m)^([A-Za-z0-9_.-]+)==([^ \\\n]+) \\") + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + while chunk := handle.read(1024 * 1024): + digest.update(chunk) + return digest.hexdigest() + + +def audit_lock(pin: dict) -> None: + runtime = pin["sharedRuntime"] + lock_path = ROOT / runtime["requirementsLock"] + if (lock_path.stat().st_size, sha256(lock_path)) != ( + runtime["requirementsLockSize"], + runtime["requirementsLockSha256"], + ): + raise RuntimeError("shared runtime lock size or SHA-256 does not match the pin") + text = lock_path.read_text(encoding="utf-8") + requirements = dict(REQUIREMENT.findall(text)) + if runtime["packages"].items() > requirements.items(): + raise RuntimeError("shared runtime direct package pins do not match the lock") + if any(value in text for value in ("git+", "http://", " @ ", "--editable")): + raise RuntimeError("shared runtime lock contains a mutable dependency") + blocks = re.split(r"(?m)(?=^[A-Za-z0-9_.-]+==)", text) + if any("==" in block and "--hash=sha256:" not in block for block in blocks): + raise RuntimeError("shared runtime lock contains an unhashed dependency") + + +def missing_artifact_hashes(pin: dict) -> list[str]: + missing = [] + for model_name, model in pin["models"].items(): + if not (model.get("required") or model.get("enabled")): + continue + for artifact_name in ("weight", "config"): + artifact = model[artifact_name] + if artifact.get("sha256") is None: + missing.append(f"{model_name}/{artifact['path']}") + return missing + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument( + "--allow-incomplete", + action="store_true", + help="review the non-release candidate even though gated hashes are missing", + ) + args = parser.parse_args() + pin = json.loads(PIN_PATH.read_text(encoding="utf-8")) + if pin.get("schemaVersion") != 1 or pin.get("backend") != "pytorch_cuda": + raise RuntimeError("unsupported Windows SA3/CUDA pin schema") + if pin.get("platform") != "windows-x86_64": + raise RuntimeError("the CUDA candidate must be Windows x64 only") + if pin["source"]["revision"] not in pin["source"]["archiveUrl"]: + raise RuntimeError("source archive URL is not tied to the immutable revision") + audit_lock(pin) + missing = missing_artifact_hashes(pin) + complete = not missing + if pin.get("gatedArtifactsComplete") is not complete: + raise RuntimeError("gatedArtifactsComplete disagrees with artifact hashes") + if pin.get("releaseReady") and (not complete or pin["releaseBlockers"]): + raise RuntimeError("releaseReady cannot be true while gates remain") + if missing and not args.allow_incomplete: + raise RuntimeError( + "gated artifact SHA-256 values are missing: " + ", ".join(missing) + ) + print( + "SA3/CUDA pin audit complete" + + (" (candidate remains release-blocked)" if missing else "") + ) + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except (OSError, RuntimeError, ValueError, json.JSONDecodeError) as error: + print(f"SA3/CUDA pin audit failed: {error}", file=sys.stderr) + raise SystemExit(1) diff --git a/scripts/audit-sa3-pins.py b/scripts/audit-sa3-pins.py new file mode 100644 index 0000000..362ca0f --- /dev/null +++ b/scripts/audit-sa3-pins.py @@ -0,0 +1,219 @@ +#!/usr/bin/env python3 +"""Reproduce the external provenance checks for both SA3 manifests. + +The default audit downloads and hashes the small source/runtime archives, uses +the immutable Hugging Face revision's LFS metadata for the eight multi-GB model +objects, and checks that both dependency locks remain fully hash-pinned. Pass +--include-model-bytes for the stronger (roughly 23 GB) end-to-end model audit. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import re +import sys +import urllib.parse +import urllib.request +from pathlib import Path +from typing import Any + + +ROOT = Path(__file__).resolve().parent.parent +PIN_PATH = ROOT / "sa3-pin.json" +TFLITE_PIN_PATH = ROOT / "sa3-tflite-pin.json" +LOCK_PATHS = ( + ROOT / "scripts" / "sa3-requirements.lock", + ROOT / "scripts" / "sa3-tflite-requirements.lock", +) +USER_AGENT = "LSDJ-SA3-pin-audit/1" + + +def fetch(url: str): + parsed = urllib.parse.urlparse(url) + if parsed.scheme != "https" or parsed.username or parsed.password: + raise RuntimeError(f"refusing non-HTTPS or credential-bearing URL: {url}") + request = urllib.request.Request(url, headers={"User-Agent": USER_AGENT}) + response = urllib.request.urlopen(request, timeout=60) + final = urllib.parse.urlparse(response.geturl()) + if final.scheme != "https": + response.close() + raise RuntimeError(f"redirect left HTTPS: {response.geturl()}") + return response + + +def read_json(url: str) -> dict[str, Any]: + with fetch(url) as response: + return json.load(response) + + +def read_text(url: str) -> str: + with fetch(url) as response: + return response.read().decode("utf-8") + + +def audit_publisher_checksum( + label: str, artifact: dict[str, Any], checksum_url: str +) -> None: + filename = Path(urllib.parse.unquote(urllib.parse.urlparse(artifact["url"]).path)).name + checksums = {} + for line in read_text(checksum_url).splitlines(): + fields = line.split(maxsplit=1) + if len(fields) == 2: + checksums[fields[1].lstrip("* ")] = fields[0].lower() + actual = checksums.get(filename) + if actual != artifact["sha256"].lower(): + raise RuntimeError(f"{label}: pin does not match publisher checksum metadata") + print(f"verified publisher checksum: {label} (sha256:{actual})") + + +def hash_url(label: str, artifact: dict[str, Any]) -> None: + digest = hashlib.sha256() + size = 0 + with fetch(artifact["url"]) as response: + while chunk := response.read(1024 * 1024): + digest.update(chunk) + size += len(chunk) + actual_hash = digest.hexdigest() + expected_hash = artifact["sha256"].lower() + expected_size = artifact["size"] + if (size, actual_hash) != (expected_size, expected_hash): + raise RuntimeError( + f"{label}: expected {expected_size} bytes/{expected_hash}, " + f"received {size} bytes/{actual_hash}" + ) + print(f"verified bytes: {label} ({size} bytes, sha256:{actual_hash})") + + +def unique_artifacts(artifacts: list[dict[str, Any]]) -> list[dict[str, Any]]: + unique: dict[str, dict[str, Any]] = {} + for artifact in artifacts: + path = artifact["path"] + prior = unique.get(path) + if prior is not None and ( + prior["size"], prior["sha256"] + ) != (artifact["size"], artifact["sha256"]): + raise RuntimeError(f"manifest disagrees about duplicate artifact {path}") + unique[path] = artifact + return list(unique.values()) + + +def audit_model_metadata( + label: str, models: dict[str, Any], artifacts: list[dict[str, Any]] +) -> None: + endpoint = ( + f"https://huggingface.co/api/models/{models['repo']}/revision/" + f"{models['revision']}?blobs=true" + ) + document = read_json(endpoint) + siblings = {item["rfilename"]: item for item in document.get("siblings", [])} + for artifact in unique_artifacts(artifacts): + path = artifact["path"] + sibling = siblings.get(path) + if sibling is None: + raise RuntimeError(f"model revision does not contain {path}") + lfs = sibling.get("lfs") or {} + actual_hash = lfs.get("sha256") or lfs.get("oid", "").removeprefix("sha256:") + actual_size = lfs.get("size", sibling.get("size")) + if (actual_size, actual_hash) != (artifact["size"], artifact["sha256"].lower()): + raise RuntimeError( + f"{path}: manifest does not match immutable revision LFS metadata" + ) + print( + f"verified LFS object ({label}): {path} " + f"({actual_size} bytes, sha256:{actual_hash})" + ) + + +def audit_model_bytes( + label: str, models: dict[str, Any], artifacts: list[dict[str, Any]] +) -> None: + for artifact in unique_artifacts(artifacts): + direct = { + **artifact, + "url": ( + f"https://huggingface.co/{models['repo']}/resolve/" + f"{models['revision']}/{artifact['path']}?download=true" + ), + } + hash_url(f"{label}: {artifact['path']}", direct) + + +def audit_lock(path: Path) -> None: + lock = path.read_text(encoding="utf-8") + packages = re.findall(r"(?m)^[A-Za-z0-9_.-]+==[^ \\\n]+", lock) + hashes = re.findall(r"--hash=sha256:[0-9a-f]{64}", lock) + if not packages or not hashes: + raise RuntimeError("dependency lock is missing packages or SHA-256 hashes") + blocks = re.split(r"(?m)(?=^[A-Za-z0-9_.-]+==)", lock) + unpinned = [ + block.split("==", 1)[0] + for block in blocks + if "==" in block and "--hash=" not in block + ] + if unpinned: + raise RuntimeError(f"dependency lock entries lack hashes: {', '.join(unpinned)}") + print( + f"verified lock structure: {path.name}: " + f"{len(packages)} packages, {len(hashes)} SHA-256 hashes" + ) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument( + "--include-model-bytes", + action="store_true", + help="download and hash both backends' model objects (roughly 23 GB)", + ) + args = parser.parse_args() + pin = json.loads(PIN_PATH.read_text(encoding="utf-8")) + tflite_pin = json.loads(TFLITE_PIN_PATH.read_text(encoding="utf-8")) + + hash_url("Stable Audio 3 source", pin["source"]) + for runtime in pin["runtime"]["uv"]: + audit_publisher_checksum( + f"uv {runtime['version']} ({runtime['target']})", + runtime, + f"{runtime['url']}.sha256", + ) + hash_url(f"uv {runtime['version']} ({runtime['target']})", runtime) + for runtime in pin["runtime"]["python"]: + release_match = re.search(r"/releases/download/([^/]+)/", runtime["url"]) + if release_match is None: + raise RuntimeError("Python runtime URL does not identify an immutable release") + release = release_match.group(1) + audit_publisher_checksum( + f"Python {runtime['version']} ({runtime['target']})", + runtime, + "https://github.com/astral-sh/python-build-standalone/" + f"releases/download/{release}/SHA256SUMS", + ) + hash_url(f"Python {runtime['version']} ({runtime['target']})", runtime) + mlx_artifacts = pin["models"]["artifacts"] + tflite_artifacts = [ + *tflite_pin["models"]["shared"], + *[ + artifact + for bundle in tflite_pin["models"]["bundles"].values() + for artifact in bundle + ], + ] + audit_model_metadata("MLX", pin["models"], mlx_artifacts) + audit_model_metadata("TFLite", tflite_pin["models"], tflite_artifacts) + if args.include_model_bytes: + audit_model_bytes("MLX", pin["models"], mlx_artifacts) + audit_model_bytes("TFLite", tflite_pin["models"], tflite_artifacts) + for lock_path in LOCK_PATHS: + audit_lock(lock_path) + print("SA3 pin audit complete") + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except (OSError, RuntimeError, ValueError, json.JSONDecodeError) as error: + print(f"SA3 pin audit failed: {error}", file=sys.stderr) + raise SystemExit(1) diff --git a/scripts/build-linux-appimage.sh b/scripts/build-linux-appimage.sh new file mode 100755 index 0000000..d99c9fb --- /dev/null +++ b/scripts/build-linux-appimage.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +# Build and verify LSDJ's Ubuntu 22.04-compatible x86_64 AppImage. Model +# runtimes/weights are external, verified, and app-managed; this shell exists +# only on the release builder and is not part of the installed runtime. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +LINUX_CONFIG="$REPO_ROOT/src-tauri/tauri.linux.conf.json" +RELEASE_VERSION="${LSDJ_RELEASE_VERSION:-}" +AUDIT_PATH="${LSDJ_LINUX_AUDIT_PATH:-$REPO_ROOT/src-tauri/target/release/bundle/appimage/linux-package-audit.json}" + +fail() { + echo "Linux release: $*" >&2 + exit 1 +} + +[ "$(uname -s)" = "Linux" ] || fail "must be built on Linux" +[ "$(uname -m)" = "x86_64" ] || fail "must be built for x86_64" + +GLIBC_DESCRIPTION="$(getconf GNU_LIBC_VERSION 2>/dev/null || true)" +[[ "$GLIBC_DESCRIPTION" =~ ^glibc\ ([0-9]+)\.([0-9]+)$ ]] || fail \ + "an auditable glibc build host is required" +GLIBC_MAJOR="${BASH_REMATCH[1]}" +GLIBC_MINOR="${BASH_REMATCH[2]}" +if (( GLIBC_MAJOR > 2 || (GLIBC_MAJOR == 2 && GLIBC_MINOR > 35) )); then + fail "build host glibc $GLIBC_MAJOR.$GLIBC_MINOR is newer than Ubuntu 22.04's 2.35 floor" +fi + +VERSION_ARGS=() +if [ -n "$RELEASE_VERSION" ]; then + [[ "$RELEASE_VERSION" =~ ^v?([0-9]{4})\.(0[1-9]|1[0-2])\.([1-9][0-9]*)$ ]] || fail \ + "LSDJ_RELEASE_VERSION must look like vYYYY.MM.N with a positive release number" + RELEASE_YEAR=$((10#${BASH_REMATCH[1]})) + RELEASE_MONTH=$((10#${BASH_REMATCH[2]})) + RELEASE_NUMBER=$((10#${BASH_REMATCH[3]})) + RELEASE_VERSION="${RELEASE_YEAR}.${RELEASE_MONTH}.${RELEASE_NUMBER}" + VERSION_ARGS=(--config "{\"version\":\"$RELEASE_VERSION\"}") +fi + +echo "Linux release: building frontend" +npm run build --prefix "$REPO_ROOT/frontend" + +echo "Linux release: building AppImage on glibc $GLIBC_MAJOR.$GLIBC_MINOR" +( + cd "$REPO_ROOT/src-tauri" + cargo tauri build --ci \ + --features managed-runtime \ + --config "$LINUX_CONFIG" \ + "${VERSION_ARGS[@]}" +) + +shopt -s nullglob +APPIMAGES=("$REPO_ROOT"/src-tauri/target/release/bundle/appimage/*.AppImage) +[[ "${#APPIMAGES[@]}" -eq 1 ]] || fail \ + "expected exactly one AppImage, found ${#APPIMAGES[@]}" + +python3 "$REPO_ROOT/scripts/verify_linux_appimage.py" \ + "${APPIMAGES[0]}" \ + --output "$AUDIT_PATH" + +echo "Linux release: verified ${APPIMAGES[0]}" +echo "Linux release: native dependency audit $AUDIT_PATH" diff --git a/scripts/build-macos-release.sh b/scripts/build-macos-release.sh index 668889b..d528195 100755 --- a/scripts/build-macos-release.sh +++ b/scripts/build-macos-release.sh @@ -11,6 +11,7 @@ DMG_DIR="$REPO_ROOT/src-tauri/target/release/bundle/dmg" BACKEND_DIR="$REPO_ROOT/src-tauri/sidecar-dist/lsdj_backend" BACKEND_BIN="$BACKEND_DIR/lsdj_backend" RELEASE_CONFIG="$REPO_ROOT/src-tauri/tauri.release.conf.json" +MACOS_CONFIG="$REPO_ROOT/src-tauri/tauri.macos.conf.json" ENTITLEMENTS="$REPO_ROOT/src-tauri/entitlements.plist" RELEASE_VERSION="${LSDJ_RELEASE_VERSION:-}" EXPECTED_BUNDLE_ID="works.protocol.lsdj" @@ -136,11 +137,13 @@ echo "macOS release: building, signing, notarizing, and stapling" if [ -n "$RELEASE_VERSION" ]; then cargo tauri build --ci \ --features bundled-backend \ + --config "$MACOS_CONFIG" \ --config "$RELEASE_CONFIG" \ --config "{\"version\":\"$RELEASE_VERSION\"}" else cargo tauri build --ci \ --features bundled-backend \ + --config "$MACOS_CONFIG" \ --config "$RELEASE_CONFIG" fi ) diff --git a/scripts/build-windows-installer.ps1 b/scripts/build-windows-installer.ps1 new file mode 100644 index 0000000..91afd60 --- /dev/null +++ b/scripts/build-windows-installer.ps1 @@ -0,0 +1,111 @@ +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)] + [ValidatePattern('^v[0-9]{4}\.(0[1-9]|1[0-2])\.[1-9][0-9]*$')] + [string] $ReleaseTag, + + [switch] $Release, + + [switch] $UnsignedDevelopment +) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version Latest + +if (-not $IsWindows -or -not [Environment]::Is64BitOperatingSystem -or -not [Environment]::Is64BitProcess) { + throw 'LSDJ Windows installers must be built by a 64-bit process on Windows x64.' +} +if ($Release -eq $UnsignedDevelopment) { + throw 'Choose exactly one of -Release or -UnsignedDevelopment.' +} + +$match = [regex]::Match($ReleaseTag, '^v(?[0-9]{4})\.(?[0-9]{2})\.(?[1-9][0-9]*)$') +$version = '{0}.{1}.{2}' -f ` + [int] $match.Groups['year'].Value, ` + [int] $match.Groups['month'].Value, ` + [int] $match.Groups['build'].Value + +$repoRoot = Split-Path -Parent $PSScriptRoot +$tauriRoot = Join-Path $repoRoot 'src-tauri' +$frontendDist = Join-Path $repoRoot 'frontend/dist' +if (-not (Test-Path -LiteralPath $frontendDist -PathType Container)) { + throw 'frontend/dist is missing; build the frontend before packaging.' +} + +$tempRoot = if ([string]::IsNullOrWhiteSpace($env:RUNNER_TEMP)) { + [System.IO.Path]::GetTempPath() +} else { + $env:RUNNER_TEMP +} +$versionConfig = Join-Path $tempRoot "lsdj-windows-version-$([guid]::NewGuid().ToString('N')).json" +$ciInstallerHooks = $null +$versionConfiguration = @{ version = $version } +if ($UnsignedDevelopment) { + # Hosted installer tests need one synchronization point after purge + # confirmation and before the destructive revalidation. Compile that test + # branch only into explicitly unsigned development installers; the release + # config always uses the reviewed production hook directly. + $ciInstallerHooks = Join-Path $tempRoot "lsdj-windows-hooks-$([guid]::NewGuid().ToString('N')).nsh" + $productionHooks = Join-Path $tauriRoot 'windows/installer-hooks.nsh' + $ciHookText = "!define LSDJ_CI_ADVERSARIAL_TESTS`r`n" + + [System.IO.File]::ReadAllText($productionHooks) + [System.IO.File]::WriteAllText( + $ciInstallerHooks, + $ciHookText, + [System.Text.UTF8Encoding]::new($false) + ) + $versionConfiguration['bundle'] = @{ + windows = @{ + nsis = @{ installerHooks = $ciInstallerHooks } + } + } +} +[System.IO.File]::WriteAllText( + $versionConfig, + ($versionConfiguration | ConvertTo-Json -Depth 8 -Compress), + [System.Text.UTF8Encoding]::new($false) +) + +try { + $arguments = @( + 'tauri', 'build', '--ci', '--bundles', 'nsis', + '--features', 'managed-runtime', '--config', $versionConfig + ) + if ($UnsignedDevelopment) { + $arguments += '--no-sign' + } else { + & "$PSScriptRoot/sign-windows.ps1" -Preflight + $arguments += @('--config', 'tauri.windows.release.conf.json') + } + + Push-Location $tauriRoot + try { + & cargo @arguments + if ($LASTEXITCODE -ne 0) { + throw "Tauri Windows packaging failed with exit code $LASTEXITCODE." + } + } finally { + Pop-Location + } +} finally { + Remove-Item -LiteralPath $versionConfig -Force -ErrorAction SilentlyContinue + if ($null -ne $ciInstallerHooks) { + Remove-Item -LiteralPath $ciInstallerHooks -Force -ErrorAction SilentlyContinue + } +} + +$bundleRoot = Join-Path $tauriRoot 'target/release/bundle/nsis' +$matchingInstallers = @( + Get-ChildItem -LiteralPath $bundleRoot -Filter '*-setup.exe' -File | + Where-Object { $_.VersionInfo.ProductVersion -eq $version } +) +if ($matchingInstallers.Count -ne 1) { + throw "Expected one Windows NSIS installer for version $version; found $($matchingInstallers.Count)." +} + +$installer = $matchingInstallers[0].FullName +Write-Host "Built Windows installer: $installer" +if (-not [string]::IsNullOrWhiteSpace($env:GITHUB_OUTPUT)) { + Add-Content -LiteralPath $env:GITHUB_OUTPUT -Value "installer=$installer" + Add-Content -LiteralPath $env:GITHUB_OUTPUT -Value "version=$version" +} diff --git a/scripts/release_artifact.py b/scripts/release_artifact.py new file mode 100644 index 0000000..d78c718 --- /dev/null +++ b/scripts/release_artifact.py @@ -0,0 +1,609 @@ +#!/usr/bin/env python3 +"""Create and verify fail-closed release producer bundles. + +Release producers never hand a bare installer to the publisher. Each producer +uploads a directory containing its assets, a checksum file, and metadata that +binds those assets to the release tag and source revision. The publisher uses +this module to verify every required producer before it creates a draft GitHub +Release, then verifies the uploaded draft before making it public. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import re +import shutil +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import NoReturn + +SCHEMA_VERSION = 1 +METADATA_NAME = "release-metadata.json" +CHECKSUMS_NAME = "SHA256SUMS.txt" +TAG_PATTERN = re.compile(r"^v[0-9]{4}\.(0[1-9]|1[0-2])\.[1-9][0-9]*$") +REVISION_PATTERN = re.compile(r"^[0-9a-f]{40}$") +PRODUCER_PATTERN = re.compile(r"^[a-z0-9]+(?:-[a-z0-9]+)*$") +SHA256_PATTERN = re.compile(r"^[0-9a-f]{64}$") +PORTABLE_FILENAME_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$") +WINDOWS_RESERVED_STEMS = { + "aux", + "clock$", + "con", + "nul", + "prn", + *(f"com{number}" for number in range(1, 10)), + *(f"lpt{number}" for number in range(1, 10)), +} + + +class ArtifactError(RuntimeError): + """The producer bundle or draft release violated the release contract.""" + + +@dataclass(frozen=True) +class ProducerPolicy: + platform: str + architecture: str + asset_suffix: str + asset_count: int + + +# Every policy entry is mandatory. Keep this set identical to the publisher's +# --required-producer arguments; omission, duplication, or an unexpected bundle +# fails before a GitHub Release is created. +PRODUCER_POLICIES = { + "macos-arm64": ProducerPolicy( + platform="macos", + architecture="arm64", + asset_suffix=".dmg", + asset_count=1, + ), + "linux-x64": ProducerPolicy( + platform="linux", + architecture="x86_64", + asset_suffix=".appimage", + asset_count=1, + ), + "windows-x64": ProducerPolicy( + platform="windows", + architecture="x86_64", + asset_suffix=".exe", + asset_count=1, + ), +} + + +def fail(message: str) -> NoReturn: + raise ArtifactError(message) + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def require_release_identity(release_tag: str, revision: str) -> None: + if not TAG_PATTERN.fullmatch(release_tag): + fail(f"invalid release tag: {release_tag!r}") + if not REVISION_PATTERN.fullmatch(revision): + fail(f"invalid source revision: {revision!r}") + + +def require_plain_file(path: Path, label: str) -> None: + if path.is_symlink() or not path.is_file(): + fail(f"{label} must be a regular non-symlink file: {path}") + + +def portable_filename_key(filename: str, label: str = "asset filename") -> str: + """Validate one name on Linux, macOS, and Windows and return its alias key.""" + + if ( + not PORTABLE_FILENAME_PATTERN.fullmatch(filename) + or filename.endswith(".") + or filename.split(".", 1)[0].casefold() in WINDOWS_RESERVED_STEMS + ): + fail(f"unsafe or non-portable {label}: {filename!r}") + return filename.casefold() + + +def require_unique_portable_names(names: list[str], label: str) -> None: + seen: set[str] = set() + for name in names: + key = portable_filename_key(name, label) + if key in seen: + fail(f"duplicate or case-colliding {label}: {name!r}") + seen.add(key) + + +def require_empty_output(output_dir: Path) -> None: + if output_dir.exists(): + if output_dir.is_symlink() or not output_dir.is_dir(): + fail(f"output path is not a plain directory: {output_dir}") + if any(output_dir.iterdir()): + fail(f"output directory must be empty: {output_dir}") + else: + output_dir.mkdir(parents=True) + + +def canonical_json(data: object) -> str: + return json.dumps(data, indent=2, sort_keys=True) + "\n" + + +def write_text_lf(path: Path, content: str) -> None: + with path.open("w", encoding="utf-8", newline="\n") as destination: + destination.write(content) + + +def create_bundle( + *, + producer: str, + release_tag: str, + revision: str, + assets: list[Path], + output_dir: Path, +) -> None: + require_release_identity(release_tag, revision) + if not PRODUCER_PATTERN.fullmatch(producer): + fail(f"invalid producer name: {producer!r}") + policy = PRODUCER_POLICIES.get(producer) + if policy is None: + fail(f"producer is not in the release policy: {producer}") + if len(assets) != policy.asset_count: + fail( + f"{producer} must emit exactly {policy.asset_count} asset(s); " + f"received {len(assets)}" + ) + + reserved_names = { + portable_filename_key(METADATA_NAME), + portable_filename_key(CHECKSUMS_NAME), + } + asset_names: set[str] = set() + for asset in assets: + require_plain_file(asset, "release asset") + name_key = portable_filename_key(asset.name) + if asset.suffix.lower() != policy.asset_suffix: + fail(f"{producer} asset must end in {policy.asset_suffix}: {asset.name}") + if asset.stat().st_size <= 0: + fail(f"{producer} asset must not be empty: {asset.name}") + if name_key in reserved_names or name_key in asset_names: + fail(f"duplicate or reserved asset name: {asset.name}") + asset_names.add(name_key) + + require_empty_output(output_dir) + manifest_assets = [] + checksum_lines = [] + for asset in sorted(assets, key=lambda candidate: candidate.name): + destination = output_dir / asset.name + shutil.copyfile(asset, destination) + digest = sha256(destination) + size = destination.stat().st_size + manifest_assets.append( + {"filename": destination.name, "sha256": digest, "size": size} + ) + checksum_lines.append(f"{digest} {destination.name}\n") + + metadata = { + "architecture": policy.architecture, + "assets": manifest_assets, + "platform": policy.platform, + "producer": producer, + "release_tag": release_tag, + "revision": revision, + "schema_version": SCHEMA_VERSION, + } + write_text_lf(output_dir / METADATA_NAME, canonical_json(metadata)) + write_text_lf(output_dir / CHECKSUMS_NAME, "".join(checksum_lines)) + + +def load_json(path: Path, label: str) -> dict: + require_plain_file(path, label) + + def unique_object(pairs: list[tuple[str, object]]) -> dict: + value = {} + for key, item in pairs: + if key in value: + fail(f"{label} contains duplicate JSON key: {key!r}") + value[key] = item + return value + + try: + value = json.loads( + path.read_text(encoding="utf-8"), object_pairs_hook=unique_object + ) + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: + fail(f"could not parse {label} {path}: {exc}") + if not isinstance(value, dict): + fail(f"{label} must contain a JSON object: {path}") + return value + + +def load_checksums(path: Path) -> dict[str, str]: + require_plain_file(path, "checksum file") + try: + lines = path.read_text(encoding="utf-8").splitlines() + except (OSError, UnicodeDecodeError) as exc: + fail(f"could not read checksum file {path}: {exc}") + checksums: dict[str, str] = {} + checksum_keys: set[str] = set() + for line in lines: + parts = line.split(" ", 1) + if len(parts) != 2 or not SHA256_PATTERN.fullmatch(parts[0]): + fail(f"malformed checksum line in {path}: {line!r}") + filename = parts[1] + filename_key = portable_filename_key(filename, "checksum filename") + if filename_key in checksum_keys: + fail(f"unsafe or duplicate checksum filename in {path}: {filename!r}") + checksum_keys.add(filename_key) + checksums[filename] = parts[0] + return checksums + + +def verify_bundle( + *, + bundle_dir: Path, + producer: str, + release_tag: str, + revision: str, +) -> list[Path]: + if bundle_dir.is_symlink() or not bundle_dir.is_dir(): + fail(f"missing producer bundle for {producer}: {bundle_dir}") + policy = PRODUCER_POLICIES.get(producer) + if policy is None: + fail(f"required producer is not in the release policy: {producer}") + + entries = list(bundle_dir.iterdir()) + if any(entry.is_symlink() or not entry.is_file() for entry in entries): + fail(f"producer bundle contains a symlink or nested path: {bundle_dir}") + require_unique_portable_names( + [entry.name for entry in entries], "producer bundle filename" + ) + + metadata = load_json(bundle_dir / METADATA_NAME, "release metadata") + metadata_fields = { + "architecture", + "assets", + "platform", + "producer", + "release_tag", + "revision", + "schema_version", + } + if set(metadata) != metadata_fields: + fail(f"{producer} metadata fields do not exactly match schema version 1") + expected_identity = { + "architecture": policy.architecture, + "platform": policy.platform, + "producer": producer, + "release_tag": release_tag, + "revision": revision, + "schema_version": SCHEMA_VERSION, + } + for key, expected in expected_identity.items(): + if metadata.get(key) != expected: + fail( + f"{producer} metadata {key!r} is {metadata.get(key)!r}; " + f"expected {expected!r}" + ) + + manifest_assets = metadata.get("assets") + if ( + not isinstance(manifest_assets, list) + or len(manifest_assets) != policy.asset_count + ): + fail(f"{producer} metadata has the wrong number of assets") + checksums = load_checksums(bundle_dir / CHECKSUMS_NAME) + expected_files = {METADATA_NAME, CHECKSUMS_NAME} + verified_assets = [] + manifest_names: set[str] = set() + manifest_keys: set[str] = set() + for item in manifest_assets: + if not isinstance(item, dict): + fail(f"{producer} metadata contains a non-object asset entry") + if set(item) != {"filename", "sha256", "size"}: + fail(f"{producer} asset metadata fields do not exactly match the schema") + filename = item.get("filename") + digest = item.get("sha256") + size = item.get("size") + if not isinstance(filename, str): + fail(f"{producer} metadata contains an unsafe asset filename: {filename!r}") + filename_key = portable_filename_key(filename) + if filename_key in manifest_keys: + fail(f"{producer} metadata contains duplicate asset {filename}") + if Path(filename).suffix.lower() != policy.asset_suffix: + fail(f"{producer} asset has the wrong suffix: {filename}") + if not isinstance(digest, str) or not SHA256_PATTERN.fullmatch(digest): + fail(f"{producer} metadata has an invalid SHA-256 for {filename}") + if not isinstance(size, int) or isinstance(size, bool) or size <= 0: + fail(f"{producer} metadata has an invalid size for {filename}") + + asset = bundle_dir / filename + require_plain_file(asset, "release asset") + if asset.stat().st_size != size: + fail(f"{producer} asset size does not match metadata: {filename}") + if sha256(asset) != digest: + fail(f"{producer} asset checksum does not match metadata: {filename}") + if checksums.get(filename) != digest: + fail( + f"{producer} asset checksum does not match {CHECKSUMS_NAME}: {filename}" + ) + manifest_names.add(filename) + manifest_keys.add(filename_key) + expected_files.add(filename) + verified_assets.append(asset) + + if set(checksums) != manifest_names: + fail(f"{producer} checksum file does not exactly match its metadata assets") + if {entry.name for entry in entries} != expected_files: + fail(f"{producer} bundle contains missing or unexpected files") + return verified_assets + + +def verify_bundles( + *, + input_root: Path, + required_producers: list[str], + release_tag: str, + revision: str, + output_dir: Path, +) -> None: + require_release_identity(release_tag, revision) + if not required_producers or len(set(required_producers)) != len( + required_producers + ): + fail("required producers must be a non-empty unique list") + policy_producers = set(PRODUCER_POLICIES) + if set(required_producers) != policy_producers: + fail( + "required producers do not exactly match the release policy: " + f"received {sorted(required_producers)!r}, " + f"expected {sorted(policy_producers)!r}" + ) + if input_root.is_symlink() or not input_root.is_dir(): + fail(f"release input root is missing or unsafe: {input_root}") + + producer_dirs = list(input_root.iterdir()) + if any(path.is_symlink() or not path.is_dir() for path in producer_dirs): + fail(f"release input contains a non-directory producer entry: {input_root}") + if {path.name for path in producer_dirs} != set(required_producers): + fail("downloaded producer set does not exactly match the required producer set") + + require_empty_output(output_dir) + release_assets: list[dict] = [] + output_names = {portable_filename_key("release-index.json")} + for producer in sorted(required_producers): + bundle = input_root / producer + verified_assets = verify_bundle( + bundle_dir=bundle, + producer=producer, + release_tag=release_tag, + revision=revision, + ) + publish_files = [ + *verified_assets, + bundle / METADATA_NAME, + bundle / CHECKSUMS_NAME, + ] + for source in publish_files: + if source.name == METADATA_NAME: + destination_name = f"{producer}-{METADATA_NAME}" + elif source.name == CHECKSUMS_NAME: + destination_name = f"{producer}-{CHECKSUMS_NAME}" + else: + destination_name = source.name + destination_key = portable_filename_key( + destination_name, "published release filename" + ) + if destination_key in output_names: + fail( + f"release producers collide on published filename: {destination_name}" + ) + output_names.add(destination_key) + destination = output_dir / destination_name + shutil.copyfile(source, destination) + release_assets.append( + { + "filename": destination_name, + "sha256": sha256(destination), + "size": destination.stat().st_size, + } + ) + + release_index = { + "assets": sorted(release_assets, key=lambda item: item["filename"]), + "producers": sorted(required_producers), + "release_tag": release_tag, + "revision": revision, + "schema_version": SCHEMA_VERSION, + } + index_path = output_dir / "release-index.json" + write_text_lf(index_path, canonical_json(release_index)) + + +def require_draft_release_identity( + *, + data: dict, + release_tag: str, + revision: str, + expected_release_id: int | None = None, +) -> int: + """Bind one draft release to its immutable ID, tag, and source revision.""" + + require_release_identity(release_tag, revision) + if data.get("tag_name") != release_tag: + fail("draft GitHub Release is attached to the wrong tag") + if data.get("draft") is not True: + fail("GitHub Release must remain a draft until its assets are verified") + release_id = data.get("id") + if ( + not isinstance(release_id, int) + or isinstance(release_id, bool) + or release_id <= 0 + ): + fail("draft GitHub Release has an invalid immutable release ID") + if expected_release_id is not None and release_id != expected_release_id: + fail("draft GitHub Release ID does not match the release created by this run") + if data.get("target_commitish") != revision: + fail("draft GitHub Release is attached to the wrong source revision") + source_marker = f"Source revision: {revision}" + body = data.get("body") + if not isinstance(body, str) or not ( + body == source_marker or body.startswith(source_marker + "\n") + ): + fail("draft GitHub Release is missing its source revision marker") + return release_id + + +def verify_github_release( + *, + release_json: Path, + verified_dir: Path, + release_tag: str, + revision: str, + expected_release_id: int, +) -> None: + data = load_json(release_json, "GitHub release response") + require_draft_release_identity( + data=data, + release_tag=release_tag, + revision=revision, + expected_release_id=expected_release_id, + ) + + local_files = {} + local_keys: set[str] = set() + if verified_dir.is_symlink() or not verified_dir.is_dir(): + fail(f"verified release directory is missing or unsafe: {verified_dir}") + for path in verified_dir.iterdir(): + require_plain_file(path, "verified release file") + filename_key = portable_filename_key(path.name, "verified release filename") + if filename_key in local_keys: + fail(f"verified release contains case-colliding filename: {path.name}") + local_keys.add(filename_key) + size = path.stat().st_size + if size <= 0: + fail(f"verified release file must not be empty: {path.name}") + local_files[path.name] = {"sha256": sha256(path), "size": size} + + remote_files = {} + remote_keys: set[str] = set() + assets = data.get("assets") + if not isinstance(assets, list): + fail("GitHub release response has no asset list") + for asset in assets: + if not isinstance(asset, dict): + fail("GitHub release response contains a non-object asset") + name = asset.get("name") + size = asset.get("size") + state = asset.get("state") + if not isinstance(name, str) or not isinstance(size, int): + fail("GitHub release response contains invalid asset metadata") + filename_key = portable_filename_key(name, "GitHub release filename") + if state != "uploaded": + fail(f"GitHub release asset did not finish uploading: {name!r}") + if filename_key in remote_keys: + fail(f"GitHub release contains a duplicate asset: {name}") + remote_keys.add(filename_key) + local = local_files.get(name) + if local is None or size != local["size"]: + fail( + "draft GitHub Release assets do not exactly match the verified local files" + ) + digest = asset.get("digest") + if digest != f"sha256:{local['sha256']}": + fail(f"GitHub release asset digest does not match: {name}") + remote_files[name] = {"sha256": local["sha256"], "size": size} + if remote_files != local_files: + fail( + "draft GitHub Release assets do not exactly match the verified local files" + ) + + +def parser() -> argparse.ArgumentParser: + root = argparse.ArgumentParser(description=__doc__) + commands = root.add_subparsers(dest="command", required=True) + + create = commands.add_parser("create", help="create one producer bundle") + create.add_argument("--producer", required=True) + create.add_argument("--release-tag", required=True) + create.add_argument("--revision", required=True) + create.add_argument("--asset", action="append", required=True, type=Path) + create.add_argument("--output-dir", required=True, type=Path) + + verify = commands.add_parser("verify", help="verify all required producer bundles") + verify.add_argument("--input-root", required=True, type=Path) + verify.add_argument("--required-producer", action="append", required=True) + verify.add_argument("--release-tag", required=True) + verify.add_argument("--revision", required=True) + verify.add_argument("--output-dir", required=True, type=Path) + + verify_release = commands.add_parser( + "verify-github-release", help="verify an uploaded draft before publication" + ) + verify_release.add_argument("--release-json", required=True, type=Path) + verify_release.add_argument("--verified-dir", required=True, type=Path) + verify_release.add_argument("--release-tag", required=True) + verify_release.add_argument("--revision", required=True) + verify_release.add_argument("--expected-release-id", required=True, type=int) + + verify_identity = commands.add_parser( + "verify-draft-identity", + help="verify a draft release identity and print its immutable ID", + ) + verify_identity.add_argument("--release-json", required=True, type=Path) + verify_identity.add_argument("--release-tag", required=True) + verify_identity.add_argument("--revision", required=True) + verify_identity.add_argument("--expected-release-id", type=int) + return root + + +def main(argv: list[str] | None = None) -> int: + args = parser().parse_args(argv) + try: + if args.command == "create": + create_bundle( + producer=args.producer, + release_tag=args.release_tag, + revision=args.revision, + assets=args.asset, + output_dir=args.output_dir, + ) + elif args.command == "verify": + verify_bundles( + input_root=args.input_root, + required_producers=args.required_producer, + release_tag=args.release_tag, + revision=args.revision, + output_dir=args.output_dir, + ) + elif args.command == "verify-github-release": + verify_github_release( + release_json=args.release_json, + verified_dir=args.verified_dir, + release_tag=args.release_tag, + revision=args.revision, + expected_release_id=args.expected_release_id, + ) + else: + data = load_json(args.release_json, "GitHub release response") + release_id = require_draft_release_identity( + data=data, + release_tag=args.release_tag, + revision=args.revision, + expected_release_id=args.expected_release_id, + ) + print(release_id) + except ArtifactError as exc: + print(f"release artifact: {exc}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/sa3-install.sh b/scripts/sa3-install.sh deleted file mode 100755 index 1f0e75b..0000000 --- a/scripts/sa3-install.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env bash -# Post-acquire Stable Audio 3 install (ADR-0012/0013): build the MLX venv and -# warm the three DiTs so the ~8 GB of weights download here and never inside a -# request. Idempotent — the `.lsdj-warmed` stamp, written ONLY here, gates -# re-warming (rm it to re-warm). -# -# Run by the in-app model manager (the Rust shell, after a tarball extract). Pass -# the checkout ROOT (the dir that contains `optimized/mlx`). -# -# Usage: scripts/sa3-install.sh -set -euo pipefail - -root="${1:?usage: sa3-install.sh }" -mlx="$root/optimized/mlx" -[ -d "$mlx" ] || { echo "no optimized/mlx under $root" >&2; exit 1; } - -if [ ! -x "$mlx/.venv/bin/python" ]; then - # -y skips install.sh's interactive [Y/n] uv-bootstrap prompt (no controlling - # tty when the shell spawns us); --python 3.11 has uv provision a standalone - # interpreter, so no system Python 3.11 is required. - (cd "$mlx" && ./install.sh -y --python 3.11) -fi - -stamp="$mlx/.lsdj-warmed" -if [ -f "$stamp" ]; then - echo "sa3 weights already warmed ($stamp)" - exit 0 -fi - -tmp="$(mktemp -d)" -trap 'rm -rf "$tmp"' EXIT -for spec in "sm-sfx same-s" "sm-music same-s" "medium same-l"; do - set -- $spec - echo "warming $1/$2…" - (cd "$mlx" && .venv/bin/python scripts/sa3_mlx.py --prompt "setup warm-up" \ - --dit "$1" --decoder "$2" --seconds 1 --steps 1 --out "$tmp/warm.wav") -done -touch "$stamp" diff --git a/scripts/sa3-requirements.in b/scripts/sa3-requirements.in new file mode 100644 index 0000000..0692f49 --- /dev/null +++ b/scripts/sa3-requirements.in @@ -0,0 +1,7 @@ +# Direct dependencies of the pinned Stable Audio 3 MLX source. The native +# installer consumes the generated, hash-locked sa3-requirements.lock file; +# update both files when the source pin changes. +mlx>=0.30 +numpy>=1.24 +sentencepiece>=0.2 +huggingface_hub>=0.20 diff --git a/scripts/sa3-requirements.lock b/scripts/sa3-requirements.lock new file mode 100644 index 0000000..7d7dfe8 --- /dev/null +++ b/scripts/sa3-requirements.lock @@ -0,0 +1,328 @@ +# This file was autogenerated by uv via the following command: +# uv pip compile --generate-hashes --python-version 3.11 --output-file scripts/sa3-requirements.lock scripts/sa3-requirements.in +anyio==4.14.2 \ + --hash=sha256:9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494 \ + --hash=sha256:cfa139f3ed1a23ee8f88a145ddb5ac7605b8bbfd8592baacd7ce3d8bb4313c7f + # via httpx +certifi==2026.7.22 \ + --hash=sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775 \ + --hash=sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55 + # via + # httpcore + # httpx +click==8.4.2 \ + --hash=sha256:9a6cea6e60b17ebe0a44c5cc636d94f09bd66142c1cd7d8b4cd731c4917a15f6 \ + --hash=sha256:e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76 + # via huggingface-hub +filelock==3.32.2 \ + --hash=sha256:87dd94cf281e586d135fa51132b8e3d9a598b316e90377a288663c9321036c82 \ + --hash=sha256:c33351e1f49cae33414acbc6d56784e6ecee82514ec90795da1161fc4836b5b8 + # via huggingface-hub +fsspec==2026.7.0 \ + --hash=sha256:b57ddbafedfaef7018c1ecab32aa200a9d7ca26b77965f64e48b70061249d279 \ + --hash=sha256:c803c40f4cf860b49dea58ee3e1c33cb9c790520e233537e1340049f89b82a88 + # via huggingface-hub +h11==0.16.0 \ + --hash=sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1 \ + --hash=sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86 + # via httpcore +hf-xet==1.6.0 \ + --hash=sha256:0e6e21fa3cdfcdcd76748564bf593870a5e013f47d97cf10aed63aa222cff5b7 \ + --hash=sha256:23379c2f9ec8696d952b16414a2bae72cad86a52df869b050698ba60f538c675 \ + --hash=sha256:2e58454a340b3556dfa4972d5451aff4fba8dd42a236600ba1a1d2b1514f0fef \ + --hash=sha256:35cec30d75c6f9eb9c16a77cef68e85a103b72e24d4b473714ec9ff06428bab9 \ + --hash=sha256:3dc3e35441ba395006af5aaacc40ef2e603c51ef46c3530b9156185f00935ea3 \ + --hash=sha256:4fc74352a17015bd0ee90038bc9efe38db894cde45f268b6712b04fce8cd0acb \ + --hash=sha256:5153e6bb103ad49d6ea9f1b2e230db5a2ea32551ad09a706d2f61d7c7c80d80e \ + --hash=sha256:5789835d7c6bc9436962853192082374297fb72d7eff7e7762ec25ceb7e25338 \ + --hash=sha256:633dc0cd71d32da58ab8c03ad38e2fac452c15c2b0a2866ebf6ededfe0a5061d \ + --hash=sha256:70cbb9c896901600128cb9b6f06e132954fbede1db30f31f7c6c63f84cb7c31d \ + --hash=sha256:75765820ce4700db3750c94acc8fe27c5fae4c9ec000a0dbac3ca082acf97765 \ + --hash=sha256:8fb4f71cba6129110c3374a33f919001ff130488fc23553698e34cc1c2a1198c \ + --hash=sha256:948f15d3a9545cfe5932f6bd8b440f6ae630aee108f14b7bd6c561f7c2dcc522 \ + --hash=sha256:d62671bb130879cef0ee4c9ebe47a14af6c66ec53e6d84dc15936e5ffdfac82f \ + --hash=sha256:f0906082d9932ae0c0057fa194041c22b4e2cdb46b2592ef3b91f020d62a081a \ + --hash=sha256:f2f7278c05c22fd60cb436cda1269649b3e81db65ecdc8496e5e164aa4143e7b \ + --hash=sha256:fb4fadde1b2b70bf4c0c14a6dccbe7194b1c28947fefd5bbe3fed9d940676c3b + # via huggingface-hub +httpcore==1.0.9 \ + --hash=sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55 \ + --hash=sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8 + # via httpx +httpx==0.28.1 \ + --hash=sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc \ + --hash=sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad + # via huggingface-hub +huggingface-hub==1.27.0 \ + --hash=sha256:7df6827c2f956c60fbaa64646e979e566db76f619dd0a9729dfb8c5a3eb4f68d \ + --hash=sha256:c1fed40ea82a6b41b477f5243546549b792ae0a93abcea608cff66089bf8f8df + # via -r scripts/sa3-requirements.in +idna==3.18 \ + --hash=sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2 \ + --hash=sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848 + # via + # anyio + # httpx +mlx==0.32.0 \ + --hash=sha256:0a0e38a409b9cae29647ec9e75ce9747b224ce7e5d91adbfad7eac37b6118ffd \ + --hash=sha256:102043c6455fe0939509c1e96caec4678f51e241981238da97c83beeed5653f6 \ + --hash=sha256:13ac6469479cda4bfd6954e0b574b92930b87073f7c79be121a68b31a3a6c596 \ + --hash=sha256:13f793c354ea9dc589bbd113f4b7d299900fb440199bbb403546845852b2499b \ + --hash=sha256:23e83c8e74a23156696e9f9905d16a17b7d27b5a596c1bc0f720a98df1c5aadf \ + --hash=sha256:253c5d20c573b277fc64a3eec491984e7ad4c64f9b246c1b3fee903b7d1db824 \ + --hash=sha256:2a180cd39ac68b397b85cc4658d0b8e0ab58166c2549fc9ab2ca5f99d15ef0c3 \ + --hash=sha256:2ee79b1f8c2c2a329afc95ece7dce0be798d43f3de771a6370d2b9f9702bbd9a \ + --hash=sha256:2f41445eb4b5c5bfe44f6635d0be3564485bd983de405772f7e4f17fbd6e3a9b \ + --hash=sha256:316106a764da928f057b40838315a64713040600a4596a806a9cbb5f5a1b8e26 \ + --hash=sha256:4192a2d02014a13a6a1030bf13dfb4e4fe05ec3ffa47678ee37da29111e25cb1 \ + --hash=sha256:4c8925d9d22d57b26885cb0858d2d4463d7526b17363c166820db5ea949731ad \ + --hash=sha256:4edffbdb1f7c185e35dc4e48611966d012b1dda9225a0dabd0fbd31651374a71 \ + --hash=sha256:50bc29bfaf31dff5138a472b56963bd6ece0fa67800c6c382745aaa41126d01f \ + --hash=sha256:5d5041205173e44f176d00b8119e7db7802c298a0f845486f0281c45122646ed \ + --hash=sha256:72c605368d145c756877057d7e3c54f169c9899fe1f83232bfb3a6342561e234 \ + --hash=sha256:73303259f2bda7fb4a0c782a7299e0e28a2890b9b6fbfc4b635fb8032208ec7e \ + --hash=sha256:78804098c9f64978b6048ffdfd78689b9e06efa2a530c541d0bd73ce44d2f589 \ + --hash=sha256:7c8d3a7b506ab45b3f7976495126c16830d988ec53289134b2bb64dae1efb835 \ + --hash=sha256:8637003c6eb089443d149fdb483f5d7a7846d6cc43d112148d7aa07cf1356c04 \ + --hash=sha256:8dfb577faa4dc413cfd0d6eb78f230d3b3b6169df4473e84408abdeb21346e9d \ + --hash=sha256:9fea39d8ecf1d08e5c3d5d70936d5a1ca6b890353c1b0b96c4e6232349d48e36 \ + --hash=sha256:abb786ee1e9638759be82583222fc7d09c5650ef90ad2b7c5da7d1931a8676dc \ + --hash=sha256:b0fdec519890dd3aa295920940356295012dea3a0390f229cd08d72890888427 \ + --hash=sha256:c6feb17e32160b70c7634aab925cf3f8c5c7bebbf99f227c48450478e1008af2 \ + --hash=sha256:deb284f3a5cd0c3e87bed80c2bee9dcbf946bdad44d75592f6fb784da878c1c0 \ + --hash=sha256:df6fa6785fb7a6f8d8e3e91c41074c885aa253b09c2b69e3c4d4f905e3c457e3 \ + --hash=sha256:e0db558267bb2d13fac4f85674456adbe0f085c570b9219e03d4e95fdc11c4d0 \ + --hash=sha256:e51e0a000e35998e2a1ea69b3ff5f68cd9a2ff9f58d60bef73bc29b5e3af4a55 \ + --hash=sha256:e5cdb9bf7c1a9320827a65f7ed63e3742d5b9280d31affc0c277e77982d465ae \ + --hash=sha256:e5f778001562ccce26cf6e5be1050d2afc78e2902bad206201ab9f5a6d0f886a \ + --hash=sha256:ea5a594355c89c0095eaba413fd39d4caa8642fa13432dfb0c9354d141046467 \ + --hash=sha256:f67557bd9ce31cbb519b39e9455b19cca698eb153ab6f4deef5b4d5509d94df3 \ + --hash=sha256:fea003b4e471976f55b40b7bb7943c7b054e1edb011d5b15b1b964905ab7785a + # via -r scripts/sa3-requirements.in +mlx-metal==0.32.0 \ + --hash=sha256:1bd94a1ce5b03a0c898771a3e759f0124300c6ab5155127906a1d50b1f3fcf19 \ + --hash=sha256:3af76a498d84804f66119800499f9d143d7dffb0878a0dd0d7c2846e58565fd7 \ + --hash=sha256:5b64b20ac24b0c401f489de01e8209edc4d372125201f19314e6f39e385322aa + # via mlx +numpy==2.4.6 \ + --hash=sha256:001fbb8e08d942dd57599e781f2472269ee7f2755fae407b4f67b2f0b17da3f1 \ + --hash=sha256:0280e0356c0829a18d9de1cb7eee50ec22ca639878d7240307ca0943d73cd2c4 \ + --hash=sha256:043191bfa8eab18c776647b62723ac9dddece59743b13f49b2016094129c2b3f \ + --hash=sha256:06ca2f61ec4385a07a6977c55ba998a4466c123642b4a32694d3128fce18c079 \ + --hash=sha256:0a041d3d761dc3c35cc56ce0351506a02bcbc25f7b169f652435141a17db9096 \ + --hash=sha256:0ab0a9c4ffb1a6d95ef519fe4247dba8eb6b18ad93999f76b7f657039acabd47 \ + --hash=sha256:0c9136e14ed34a9e343a31c533d78a9813a69a3148332bce5e9821cb2f996e66 \ + --hash=sha256:110f8b71aacb688ec69062bb7f6938a0f8acb01b7c1c4beb453c65b6d234584d \ + --hash=sha256:112b06a867b235ef466ed3508ddf0238050df9c727cafb5301ac385b899189a1 \ + --hash=sha256:17f9ade344e7d9b464a084d69bcf18fc691cb1db67c62ed80820bf4926d78f0e \ + --hash=sha256:1e254a00cdf42b1e4d5b3d68d33af63268d41340d8885df2ab6470f2e1500147 \ + --hash=sha256:1e978ec1e8bd0e0e4de6bb75de9d30cbb74db6b6a2bb727618613703ca0167dd \ + --hash=sha256:25c692919ac5a01f170a3bfcd62d745b24fd095c353d50812637d6fcab442e75 \ + --hash=sha256:260a5d70215b61ab4fadf5c7baacd64821842975eea312125ed3c39a6391b063 \ + --hash=sha256:2803abfebfc990042cd494d8ce2d5f82e9d847af6d35ec486923aa19dbad5e73 \ + --hash=sha256:29a287e0cf63ff528da061de6b9f64a4618da591ca1046aafc54062e40ca7eab \ + --hash=sha256:29cb7f67d10b479ff07c17d33e39f78c07f71c40ef30d63c153d340e96cd3fb4 \ + --hash=sha256:3213d622a0283a39a93d188f3cf72b26862df52fbb4ca3697f51705016523d41 \ + --hash=sha256:33111801a01c12a8a1e3721f0a9232f8cfc8ae2c6b7098167e6f623c6073f402 \ + --hash=sha256:357cc07a6d7b0b182ff02249616a03742827ebb1277546b5c7cd7f7620a45698 \ + --hash=sha256:38efbc8de75c7a0fc1ac190162d892787f3f47b57cc291231aafee36b80982b7 \ + --hash=sha256:4081eb135ac24158bd51cdfbef16f1c64df7063b1143f24731387137c092bec8 \ + --hash=sha256:40fdc1ae7125e518ea98e53e69a4ebc27e1fd50510c47b7ea130cf21e5e1d42b \ + --hash=sha256:4cfe66903cc32a9921a6733d96b19bb6abf310397581bbad89c228f5abaf0ee8 \ + --hash=sha256:511dbaf848decaaaf4b4ca48032619fb3138710c4bf7da7617765edad1ef96b0 \ + --hash=sha256:55cced7c52e981362f708ad635198e97a752dfba412cc03c23bbf3bd8d5cd662 \ + --hash=sha256:56b39e5e0622a09a25bf5baf62f4bcf0cb8a41ae6e2819cf49bbc5a74c083f91 \ + --hash=sha256:5dbbdb29840ca3d91ee0fece42fc29278886d908280bfec0a5846c6f901a3eb0 \ + --hash=sha256:5f9fb9157b4ce2971008323afe46053787b526ef624fea915b261468a8421a0f \ + --hash=sha256:6180d8b35af935aed8ece3a85e0a43f87393ae0ac87c8d2c8bd2c993f7270ef3 \ + --hash=sha256:68a5124b13fa6cc2086764a20005d30bc0548146f7f5322f02fce212ca14317f \ + --hash=sha256:68bb27509ac1b9a3443094260f6326150663b06abe40b73a2f81160623da5b67 \ + --hash=sha256:6f41ae150c4e32db4f3310cdaf64b1593a03dbabe29eec77fc9b50fe64061df6 \ + --hash=sha256:7265a2f3d436e54ef9f2b52b5c937e6be778781bd97a590319d7348f1c1ca997 \ + --hash=sha256:72fbe16c6fac95aedf5937fa873445cec2110be35d8a4e9433d7501fd98dae6b \ + --hash=sha256:7d92c3819208a60205a12a245c91ad70cb0a85336659b19b834205573ac8456e \ + --hash=sha256:8155154c7c691289fe18f510b5d4657c68c67989f293f0535a91360392ff6538 \ + --hash=sha256:81a1cca95ed5bb92aa8b10dd2cdc9a0d3853a50fad926c28b5d7e8ea54389627 \ + --hash=sha256:89cd468399cfd2504718f0ba50e410dca55a170b61a02ad92bb18c8a65186e93 \ + --hash=sha256:8ad03c0965fb3c692200e74d458ca28c1dbb4ce96f9a479a8aa041ad5fabca02 \ + --hash=sha256:90f9849678c75fe7afa2d348ac842c168b0a4d3d61919687216dfc547976d853 \ + --hash=sha256:948424b06129ce883307e8cff868c31396d8dc7630a59c61d70d98dbe70f222c \ + --hash=sha256:9cd5ffd25db4e7ba6a375693b3fc0fc1791ec636c17db3720da19bde7180ec43 \ + --hash=sha256:a0df0043bdb289bde1f62da130d20df23d58b45429f752bc7a8fc5325a225ecd \ + --hash=sha256:a2c306dea656c12c68f51f4cea133cbe78ca7435eb28c735eac1d3ebe73be6e8 \ + --hash=sha256:a7830bab239b79cda9c08c2da014761cafb48da6150e1da17ac06283f43b6089 \ + --hash=sha256:a7c711e21628b52034bb5ab8d1bce291f752fcc5e92accc615778acee1ff4778 \ + --hash=sha256:aaf159caa35993cb1f56fb9b8e4610d35758e7ca005412eb1daa856a78c9c4b1 \ + --hash=sha256:ae506e6902902557576a26ff33eda8695e7ecb3cb36c3b573a0765dee114ebdb \ + --hash=sha256:b507f5c4c1d508876d1819b6bf9a49d365b96320b5d4993426b33a23ca4b8261 \ + --hash=sha256:bf162abab1c1a736333192707cef898e735a5ca00f38f27eeedf44b39d9e85eb \ + --hash=sha256:c1a2af6c6ef86344a6b0db6b97834208bf598db514f2b155042439b62605601a \ + --hash=sha256:c2d37ab77531417474168eb79d6d80b14f821a966818505d03013d0833edb7a8 \ + --hash=sha256:c4fc99836233ea196540b17ab0983aff60ed07941751930f5f4d05bc3b3b7359 \ + --hash=sha256:d581b735e177fdcdce6fed8e7e8880a3fb6ee4e3653a3ac6af01c6f4c03effc5 \ + --hash=sha256:d6da64deb6b8ed903e7560180a92f2d804ee1ba5eeb849ac2748b8c1aba1f6d7 \ + --hash=sha256:d8e8286dd7cea7895157318d1b91cdacac64c479f3cbc8dce548331728484751 \ + --hash=sha256:ddea102b48f9e339f3948bf22040944184627a30fdf7f858667673b9c5f033c8 \ + --hash=sha256:dfa20cc6ca228e6b155b11da03825975ce66aea520985dbbddf0f2a5a495c605 \ + --hash=sha256:e3e5193ef5a3dc73bceee50f7fdc2c90dbb76c42df8d8fae3d1067a583df579e \ + --hash=sha256:e3eeb0aabd6bd5ce64faae67e9935203a6991b4bc2a485a767fbafb2c5125f45 \ + --hash=sha256:e5805d5a22fd19c8ccff10a9561f9df94436b0545619ea579db2d3c35294bce2 \ + --hash=sha256:e85b752a1e912b70eaad4fafbd4d1238007ab221de2009b9a2f5ae7461239895 \ + --hash=sha256:eaf7fa2de5c0be8ae6ff8e9bea2ccd725e980541244521d8d4b5f3354a27babe \ + --hash=sha256:ebfb099f8dcf083deef3ac1ca4c1503f387cf76296fcb3816b66f5ecb5f54fdb \ + --hash=sha256:ece3d2cfe132e7d51f44a832b303895e6f2d499c5e74dfbdb06ee246147a304a \ + --hash=sha256:ed9749eef4cbd126da3dc1d6bcb3a57f5eb7ac6a6484146bdbf743f552dfc577 \ + --hash=sha256:ede83e07a75dd06bc501566c1eca2afc0d61677c1472ac9ad93fdee6e638a48d \ + --hash=sha256:ef4aea96ce4d3b074422cb4f2f64e216bf9e213004bb58ecfdf50ea02ea8eb9a \ + --hash=sha256:f3a3570c4a2a16746ac2c31a7c7c7b0c186b95ce902e33db6f28094ed7387dda \ + --hash=sha256:f407cb6b8e9d6d8c626bc73c945db1706035af8fd632295547bf1c9e46d092d6 \ + --hash=sha256:f74a575920ab21fe304421a3fc28793d82e299cae9eccb37084e9fc7f3617c20 + # via -r scripts/sa3-requirements.in +packaging==26.3 \ + --hash=sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79 \ + --hash=sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c + # via huggingface-hub +pyyaml==6.0.3 \ + --hash=sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c \ + --hash=sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a \ + --hash=sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3 \ + --hash=sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956 \ + --hash=sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6 \ + --hash=sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c \ + --hash=sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65 \ + --hash=sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a \ + --hash=sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0 \ + --hash=sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b \ + --hash=sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1 \ + --hash=sha256:22ba7cfcad58ef3ecddc7ed1db3409af68d023b7f940da23c6c2a1890976eda6 \ + --hash=sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7 \ + --hash=sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e \ + --hash=sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007 \ + --hash=sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310 \ + --hash=sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4 \ + --hash=sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9 \ + --hash=sha256:3ff07ec89bae51176c0549bc4c63aa6202991da2d9a6129d7aef7f1407d3f295 \ + --hash=sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea \ + --hash=sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0 \ + --hash=sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e \ + --hash=sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac \ + --hash=sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9 \ + --hash=sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7 \ + --hash=sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35 \ + --hash=sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb \ + --hash=sha256:5cf4e27da7e3fbed4d6c3d8e797387aaad68102272f8f9752883bc32d61cb87b \ + --hash=sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69 \ + --hash=sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5 \ + --hash=sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b \ + --hash=sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c \ + --hash=sha256:6344df0d5755a2c9a276d4473ae6b90647e216ab4757f8426893b5dd2ac3f369 \ + --hash=sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd \ + --hash=sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824 \ + --hash=sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198 \ + --hash=sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065 \ + --hash=sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c \ + --hash=sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c \ + --hash=sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764 \ + --hash=sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196 \ + --hash=sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b \ + --hash=sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00 \ + --hash=sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac \ + --hash=sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8 \ + --hash=sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e \ + --hash=sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28 \ + --hash=sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3 \ + --hash=sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5 \ + --hash=sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4 \ + --hash=sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b \ + --hash=sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf \ + --hash=sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5 \ + --hash=sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702 \ + --hash=sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8 \ + --hash=sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788 \ + --hash=sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da \ + --hash=sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d \ + --hash=sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc \ + --hash=sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c \ + --hash=sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba \ + --hash=sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f \ + --hash=sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917 \ + --hash=sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5 \ + --hash=sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26 \ + --hash=sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f \ + --hash=sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b \ + --hash=sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be \ + --hash=sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c \ + --hash=sha256:efd7b85f94a6f21e4932043973a7ba2613b059c4a000551892ac9f1d11f5baf3 \ + --hash=sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6 \ + --hash=sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926 \ + --hash=sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0 + # via huggingface-hub +sentencepiece==0.2.2 \ + --hash=sha256:046b15ea22d8042e2e173561d464ec3b64a9c2081324df70ebce7bf7ebb3e497 \ + --hash=sha256:0e2aae42960392d6dcb9a72d8e1e65a97294c965071b43c7b3429a42f350250e \ + --hash=sha256:1120e0791540615e650b2e9bea835bf38a7362455d8ab62dee7968219c2d79a0 \ + --hash=sha256:1402d8ee36f0d851cea8eee4dbb85fea14643b7503cf4d00d102eec0fe3ca719 \ + --hash=sha256:1416b92f2f010333786fe6306ed2631121d5ea492219b0841e967b6765e64107 \ + --hash=sha256:16c84ddef8d3084a8af37208acd365b08092ca089080f1a71fbfdd911adda9b3 \ + --hash=sha256:1edb10e520e4bddf74d85b0f5ae74cc2d60c2b448885080bfb618bc2b3a49f6b \ + --hash=sha256:201a8e0f55501a76e08dbf2c54bc45f4642b379271e89c667d517bfbc2191f2a \ + --hash=sha256:252908153eeec06c3ca3a32077e64a49d572e3d89881475b4e0f02d99d9fcc7c \ + --hash=sha256:38111ed1f79268f399c505028023d5eaaf0ab4e5eafceb709468b0d3323e7838 \ + --hash=sha256:3ab3f1ae98970b5590e2209341522718900ba19bcc2c207ffaa6bd417ad960c5 \ + --hash=sha256:3d2b5e824b5622038dc7b490897efe05ebbbb9e7350fc142f3ecc8789ef9bdf6 \ + --hash=sha256:3ec27c152a1f1b24bc9168b55a5880f3c16e2334e697da6f55a1046a22405a3d \ + --hash=sha256:3f5851441ab1ef8634963a5100b733a8bbeefe623e0c5c005b1f1f3880e574cf \ + --hash=sha256:3fd9ce2ab4460c713cfdeb4aca693ca6732a11538e05fb332d5af42e3d7fde25 \ + --hash=sha256:44284adc6fbe9d5bdd480541431a3d93f674fa44736714d3ad4bcee8283ace7d \ + --hash=sha256:443ac618c7a2a1377cf5c82581fbb849591d14e656d5e5a3e4682d4e36a34e4e \ + --hash=sha256:46ba07b543add034de0ff47ac5f907e9a06682f91d85121a972764628933be6b \ + --hash=sha256:4f0603267cd15b92b68c2c0e852a441507614b70dc7773659baa6b8c214a91fd \ + --hash=sha256:524e2a85c028a0d2f9935191fa751e5ef9d9bcc39616f70ab14b28d0369c9936 \ + --hash=sha256:54a83df9260a89c1734256e620fe1f1a6bfedd7547139d4dc1384efac11a3a85 \ + --hash=sha256:59d6588712101ccfcae9b03692be3aaae1514c2078666d7b05f15ba3a702e41b \ + --hash=sha256:63250cfab8b80a1ef82a614eb2b3cadfec2c405f870cedc139d08e2f063eb708 \ + --hash=sha256:64b656f025355cf8c51abe9fbe3848540756c6d7ca5e6791b1afa664bc24c7cb \ + --hash=sha256:65d84ec36888de4a848eee5f910e67fbc79b064685ef1e10a502e14520ead9c9 \ + --hash=sha256:69e9dc8078e128286ed3b975e37c837ba96e215a50c3ef9f3f8b7ab9e5a832a0 \ + --hash=sha256:6dd76f3e5c8b2eb8a3a3efee787bbf5b9a66e52a048fe09cab85eca33fec6790 \ + --hash=sha256:70d4ca6f4d06df7f0ccab6fe4f49c8a712c8c8b6847b4f0af9a0e1dbb0e0337e \ + --hash=sha256:72b7825b331b1b7e7c45be2e674b3e3c65af608fa376bad2d851b20aaf0cdc78 \ + --hash=sha256:741b4b367140e9b5c36b5a14c72179f2c946d991ea9a7c031a2a1ee6ad097b99 \ + --hash=sha256:74f0ee601047c0c12a783088b51be4e6214a62ecd9e02278c477433cd16e0ed9 \ + --hash=sha256:76ff5814db72e7462dece042d7593cdf102b8ec82c2b1cc201a2add34ee3050d \ + --hash=sha256:77c3ce990b23441e5ecfa5bce181fd6f408b564aeb6d7e1d1e7de9c5612501c8 \ + --hash=sha256:79bac5a251f23a7341e28fda9ce0d5319edf45328239ce037c0682936f137906 \ + --hash=sha256:7c6e7bf684dc12145bfa685d3060beaea55139134ba848289bee514ed42e7383 \ + --hash=sha256:7fc14c1585139fa6b68775e616a6b90cf622ebf219f9558c0aeaf5d253ee6c9b \ + --hash=sha256:89625fb43765cccaa1443b9adb61f283e5fe4cb1536728205d06bada730caa53 \ + --hash=sha256:8b2db2056c97224e122054fd794543cde5d24b7cae28424f6e3eb79bbe08e42b \ + --hash=sha256:8d44b20234905ff022b7d535f79d1f823ad7670c9851cc4f03cdc34787cdb3ab \ + --hash=sha256:8eed98514bffe5ecac37f493f91869c351fbb05629328bfdbc08502c6c094dc0 \ + --hash=sha256:8f1f61592e7cabd45d49ce8cc0ef42ca655c091e037153754fb3fa59725b5914 \ + --hash=sha256:b23fe17779834d3c27aaf2edac9486d04cca1a7deb8f5facda35150ac6263a91 \ + --hash=sha256:bc7b0b1da20f856bfac5f84b2673fe534b167e41980b27442ca8f78c2b7eb77e \ + --hash=sha256:c62bd361cec1f5b556eb8210264ecfff37486cd990c3386cc00310f26c54090a \ + --hash=sha256:c76c9b3324efd79029eeb0fd2ced1964bdbeca7d45e030b46fa3ef3cf74f8032 \ + --hash=sha256:c798f0b327bac10dc95cdac77b9a197ab2bd7dd1e60ebd7586a12d918d4be711 \ + --hash=sha256:c8a168b040bc61681293f79a949b5d911c8e25086f4260285b8d97ab5f1195da \ + --hash=sha256:caad9566e2ef0e5640d36032c69b0edc7ac6028277b93d93815898804fac450c \ + --hash=sha256:cbce24284f51f71d10a42b7b9c964dcb9048b28f1c8e5db40bcbcb6f428cba6a \ + --hash=sha256:cd810878180a52950e5a61f25ada5248a453bbdbafe474f89514135fbc1f633d \ + --hash=sha256:d254c98ca6387655400b3959c33c83efd807f5edeb608e3aca45800ceaa77151 \ + --hash=sha256:d795c4ac689a57f9d4ba2288126ec7901d389ad5827d2f8b8533c883974fe563 \ + --hash=sha256:df88b0c34f2fa909d322f7b06b1398e1e81af4b2f42a7b8e3556f928b25d1811 \ + --hash=sha256:eb8da9d9a9b418422c21a07fd19b9d9228692b7a7468a45eec6b11642d3c808b \ + --hash=sha256:f7c06c751c19d923435a54bff4f7e66e728fad160e8da28254f133abc9725820 \ + --hash=sha256:fa9f5ef0e2a82233dd0b8b32ea3f5710e0c44afbc07ed3620219f32601e56090 \ + --hash=sha256:fd523c4992041faa5c2b3cde62253d11a96c30d73a34afe48a486e8e2254cd1c + # via -r scripts/sa3-requirements.in +tqdm==4.70.0 \ + --hash=sha256:55b0b0dbd97462d06ebee91e4dac24ed4d4702be82b24f07e6c1d27e08cea220 \ + --hash=sha256:7f585706bfddbdebf89daac705b2dfcc16890130727d3197ca62c732b4310953 + # via huggingface-hub +typing-extensions==4.16.0 \ + --hash=sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8 \ + --hash=sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5 + # via + # anyio + # huggingface-hub diff --git a/scripts/sa3-tflite-requirements.in b/scripts/sa3-tflite-requirements.in new file mode 100644 index 0000000..6834b1c --- /dev/null +++ b/scripts/sa3-tflite-requirements.in @@ -0,0 +1,14 @@ +# Official optimized/tflite/requirements.txt dependency surface at +# Stability-AI/stable-audio-3@a0b57f5483c4588f827f3552b7d5c6ca2a9687be. +# The generated lock is consumed by issue #107's standalone runtime installer. +ai-edge-litert>=1.0 +numpy>=1.24 +sentencepiece>=0.2 +soundfile>=0.12 +huggingface-hub>=0.20 +# LSDJ's generation-only loopback service runs in this same verified portable +# environment. Keep its minimal server closure explicit; no `uvicorn[standard]` +# extras (watchers, dotenv, alternate event loops) are needed in a packaged app. +fastapi==0.136.3 +python-multipart==0.0.32 +uvicorn==0.49.0 diff --git a/scripts/sa3-tflite-requirements.lock b/scripts/sa3-tflite-requirements.lock new file mode 100644 index 0000000..2321552 --- /dev/null +++ b/scripts/sa3-tflite-requirements.lock @@ -0,0 +1,672 @@ +# This file was autogenerated by uv via the following command: +# uv pip compile --universal --python-version 3.11 --generate-hashes scripts/sa3-tflite-requirements.in --output-file scripts/sa3-tflite-requirements.lock +ai-edge-litert==2.1.6 \ + --hash=sha256:0156d9051c9a2500cd4c4ecedb77676eff0b45bd8efb1001e8e730a8cd0db4ee \ + --hash=sha256:1a278957648d0aff7b38c03e6f92ca4ad8c1414bb085d505dc97b59b8b768cfa \ + --hash=sha256:2057f8996666e8e2a52ed45fe16d8cc237434a02ed7845d2febedceb1811f66d \ + --hash=sha256:282f3521e31d9d1cb4f60143f6b64fa07a8ac8488b098c37c809b574fa3dc264 \ + --hash=sha256:2e8a3f92fa407690189533bea8b64d49bb1b8a9e96f707ba27e1a64a9c3cc8cf \ + --hash=sha256:303cb32ca33e8d12a9360dd8a130c2a7c94b7e75727da34be2eacab65f0da8c6 \ + --hash=sha256:3349289e114cd1f8396632c93d77f58a94212d2567e64dcdb1e2eca6220d67e6 \ + --hash=sha256:5819dcafe62005483744636dfe429fcd429a29ba382187eeb156dda2ada2f82c \ + --hash=sha256:5adf0c9afde6151dc7f2989d039c800f3060d98d40bb5dfc95e426ad4eb3680b \ + --hash=sha256:8475d18c73698d9244380ad72f67caf540e5c72d9a80c66d17c4b81ca81a21c8 \ + --hash=sha256:af4f2ba681fa2c688746cbd7ddd71a2bbbdd9e6a51aa609d382bfad77d1c695e \ + --hash=sha256:b20f4c8cdbbf6f64e3baa77853e55a1b29515a45d6dfbe6b2ced9b3a1efb5807 \ + --hash=sha256:c7c93fb0dc2a1d45750443731d0193974737f9f1ffb36eb6c1889bdfb61bf091 \ + --hash=sha256:e037e41a15c3285302da8ebf32ff285b085c5616e929ed93cd2f563531d7d998 \ + --hash=sha256:e8aa6393e1293fac837f764b5a9ba3ef81bb16068c94a2068d9dfb109a24e3b9 \ + --hash=sha256:edf598814004e594b40c888f52cae59e950dbeffd821e83ba45d28db0a0aa3f5 \ + --hash=sha256:f5be19734dce243c141106c3699395b62d770a61fedd097b3a10af82006bf57f \ + --hash=sha256:f722298b070343e24634ab51a55b7185d32d8f84d4df8659f9d2989e76d7d63f \ + --hash=sha256:fa58e1ddf39d8d6c190db808bf8289f22987fbf412ed0107a12617108c51bc94 \ + --hash=sha256:fc361114c68c194ce9ee9e6b2748fa40d23aea9e12173225ef0affe823353099 + # via -r scripts/sa3-tflite-requirements.in +annotated-doc==0.0.5 \ + --hash=sha256:117bac03a25ede5df5440e855b32d556049ca169ead221505badf432fed4b101 \ + --hash=sha256:c7e58ce09192557605d8bbd92836d7e1d520ac9580096042c0bfd197efacf1bb + # via fastapi +annotated-types==0.8.0 \ + --hash=sha256:13b2beaad985e05e2d6407ee4c4f35590b11f8d693a258a561055cac8f64cab7 \ + --hash=sha256:f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0 + # via pydantic +anyio==4.14.2 \ + --hash=sha256:9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494 \ + --hash=sha256:cfa139f3ed1a23ee8f88a145ddb5ac7605b8bbfd8592baacd7ce3d8bb4313c7f + # via + # httpx + # starlette +backports-strenum==1.3.1 \ + --hash=sha256:77c52407342898497714f0596e86188bb7084f89063226f4ba66863482f42414 \ + --hash=sha256:cdcfe36dc897e2615dc793b7d3097f54d359918fc448754a517e6f23044ccf83 + # via ai-edge-litert +certifi==2026.7.22 \ + --hash=sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775 \ + --hash=sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55 + # via + # httpcore + # httpx +cffi==2.1.1 \ + --hash=sha256:046bfc24911b37851ee1b51aab8bffe713d89c68c6a057b09484ce9fd5f69b4e \ + --hash=sha256:06c72bb76605a4b0cd0aad6930b69d4baf7dd5d806cfc409b824191099700e66 \ + --hash=sha256:0beceaabe56af686895136a2de78db54ecd8e4046b236b8fd6d6cb61389e9bf2 \ + --hash=sha256:154852545011f779917b11c78db2358d095da62a9a172b78ad0a583ee5adc0d0 \ + --hash=sha256:194cffa889098ced9976c3fc6340305e43f6303657d298da55366907c05c22d6 \ + --hash=sha256:19ee6127ee34de7d83ce3d371ebc5ed91addbdcc39f9ab15ce4eb35a4e534971 \ + --hash=sha256:1a18a57b58cfb21fc28d72e876acf10eaed67a1ed96226f92af4df681d571c4c \ + --hash=sha256:1aa5645c30469b09530c4ebca77ebf8f17618293c58f8549cb1a543a50236e7d \ + --hash=sha256:1dea0e4d7d4f11f619fe8c1d76caf49e24405b4b5743c0e3be16a500ecd930c9 \ + --hash=sha256:208f941bb9d18e768138677f0a6d2ce01f590df56043dda1df1535ac57c88517 \ + --hash=sha256:210019b6c7cf07f081b4c54635c8cf744377001350e29cc0f81c4377b4797735 \ + --hash=sha256:246fa40ce8645a614ff682e0b70f37134e460eaf93a775e0cbe3cca585a67a80 \ + --hash=sha256:25792eac27877609e7bb06d42ff88278a6624fff2ba9bbb523c09616b117e80f \ + --hash=sha256:27350daa11d4f10c540e6e89dada4c54feb7256ad03e9a4dc075ebad7ba360d1 \ + --hash=sha256:28907ab9bfb6aa13184cfc17c6b8e1023c5ab6fd7076d8c20a35e59fe04f8f29 \ + --hash=sha256:2ae64be792b8966f2c69538199728b290e34726562896df1e5dc8ffd8d8188e8 \ + --hash=sha256:31348097ff5bbe827ccc41795d4dd099d9f0625e7def00ee653c137a490c2a6c \ + --hash=sha256:3143d81e29e1e20a9ce10901ec369012947876596f75a222235965f2b7ae832e \ + --hash=sha256:3222ba5d678f80a030e6afbcc33dc1ae5cb45facabb61cee2c7016b8432fde48 \ + --hash=sha256:3311ed60d36f83378794e1009ac6258bafbf81f7888b4caa7b35a521e3f95813 \ + --hash=sha256:334644fbac4eff73d985a17a91226df55d0f394160c4cfb880e084c8f7161cac \ + --hash=sha256:34e261f78cb6ceaaa36f42f2613f4380d94d9c759a9c73c769ee6e0247364632 \ + --hash=sha256:363e05fa78e15116c3c32c210ee36884fd6b9afa6d440e47112c3bd511d64cb6 \ + --hash=sha256:398aff33cee2767e3e781d2554c54bd0dff386bb437581e0d8011fde1a942ec1 \ + --hash=sha256:3d22a20b1fb1632cc72c22f95f7b0d2961c3e1c235f245ba4c606c4771035659 \ + --hash=sha256:42a494cee34437f05546455144f2b5d9ac09b1face62bcfce597d2e521066688 \ + --hash=sha256:42e2f76b9455f5a9a844f770bf3e200ed3da0e15f5df3db9c31fe80b04b3d004 \ + --hash=sha256:42f6930c31dc7f50732c9ae793c2786c7b6b044195967bbdde40bb9be81c4cc0 \ + --hash=sha256:456a61fa52d579ebf9df2e9552ead5129855dbaff6c1e5a9b1bc408809bdc062 \ + --hash=sha256:471cee653ae88de62096552e6d24ccb4a5adb8c8c9f10b5054d0122c15bf2779 \ + --hash=sha256:49cbc70e6542d4ccccb936558d1064a8012541e78f821f955cff24e357776c94 \ + --hash=sha256:4a7c934f7360e8cd64fe9efadcbd10c7c6364f531e432b9a4bf5ccbc9e0e8b50 \ + --hash=sha256:4be96343e422f2dfcd12ab5c9f5aebe03f82f737c6bffeca6830b3875cb44aab \ + --hash=sha256:4f42141fc14250de6dde5ee7ea4432be017252d91f19c5ad043c084cea629cac \ + --hash=sha256:507a24c282e0f42f8ed737cf048572cbf580468da5555764a8331735e9c736b6 \ + --hash=sha256:51b31d1c98274844cfd7838ce00bfc27c7423a4dc00fc0772fc3331c2cc90676 \ + --hash=sha256:58acb8ab8e295e6c5ea12f888cbb13cf21511ef2a3303a23f4325c29d17fe5c1 \ + --hash=sha256:5a59cc1c4442bc3d5c703bf720b51138d0bfc173618807c9ee2490a7541dd3d9 \ + --hash=sha256:5bb4e7ea95dcd6a014a6fef62e62467d67d8e582326443f3d68e71d6320a9fcf \ + --hash=sha256:5c58fe613dc5e5336357eff555824a314d8e43282600435c8d1cb6a7a2fedd13 \ + --hash=sha256:5e7cecbaadb83884793e05828cee59b210b24583b9c7425d0ba6a754fe22eb4e \ + --hash=sha256:616f097f2fe415bc92a247f02e11f634e1f9e9a83d327e3c915c15089c87869e \ + --hash=sha256:63bbfd5ded17c4840ac07cd8f1c21ba9d9708141f840b324f422f41b207e3973 \ + --hash=sha256:64faea20f4e2613363a1a9b9c7dd73058f3ecd00133a511e72ad7c511658f527 \ + --hash=sha256:661c298b4821edebead0c91edd2b00374d67ad7c5a1f7a91d4442633b79d6a72 \ + --hash=sha256:68e62fe11f30d5ca8289242866f0a5291402d8529ca2178ab8afc5c9694ae890 \ + --hash=sha256:6a8dddef476fab96d066d578fc88526767b836ab5ab21754e1d5bf3879c31c7c \ + --hash=sha256:6e192623c49c94421616a5778fba35cf0d5a8d000650c1967ef4448ee5cdd990 \ + --hash=sha256:7225e4514edb64eb6740324353e0da0711954fd8d7da4576755b1c6e09b697cd \ + --hash=sha256:75f80557d1389eddbd0de2681f6a390a0c5338c31ddaa821381c203fc3fd50d9 \ + --hash=sha256:770de9db11e84213beec501cfcaa013b019820ca881e03344dea5844f7876d94 \ + --hash=sha256:7750c6449dff7864bb9bb27ddfb0267756189201a3afc911d82b3caacd70dfc3 \ + --hash=sha256:7bde5e4cc5c10140859842b9d383af292b22639a4dffb725314baf45968cef80 \ + --hash=sha256:7ce713ace7c0e4520535b42b77eaa742c16dab813978064913e5a3cf82973b41 \ + --hash=sha256:7da0c5eff80f0197f3b3d1232ec5a682a9325f4ae9016a78f5f5ca35f9ced1f5 \ + --hash=sha256:7dbb61fe3a7699468030f71bbe5f8a0e326a151daa91beb11a6fc1f980c55e1c \ + --hash=sha256:811bd1e21d32de12efca32393a0ab3f5133b54fce9bd44b8bd77ab07da14bf6a \ + --hash=sha256:8ef53b2de9bcb9197d31854256575d59dbac0cba72ac627bb291ef5eceb74be4 \ + --hash=sha256:937c0052c05a31ca1daf18de3158eed4dbfcb9cc107adbea227728d647be701e \ + --hash=sha256:9d2055050ea716bd38b7f7f1579c275386646b4894c155a3e2f3cd62ed41b7c6 \ + --hash=sha256:9f8d177621de5cb38ee3e731eda45d421db093ec0739f46a5594babda7987a98 \ + --hash=sha256:a2d7755bef5a12ed488f4ef1f1b69ee9191d7396083b755a5d2295f6edb4768b \ + --hash=sha256:a48d62ab9d6f4f98c983223a547af44be6ca3691074c31cecced6facd3ba2dc1 \ + --hash=sha256:a4f00aa42f75d6e4595e8866e748cc1705adc0cddfeb2ca86d0d03993d63ba03 \ + --hash=sha256:a6e721d4b0e45d5b65e87534470e67b18dcd092c83f68fba09f152b9cbc061af \ + --hash=sha256:a730a083190634c65cca36ba5f489531576ebd79bcd5c8e172130f6453127231 \ + --hash=sha256:a931079504ecc49efed7744c476a5c343a92fabf66dec2db95edb1b2fdc770e2 \ + --hash=sha256:aa9511c62d14da7aacc9b4bf51f3f697a621e83b2d6919008243c3aad168eea3 \ + --hash=sha256:ab36d55f9ed2d067327667c2fea18dda018eb628dd6347aa01dda6cf1f5d3836 \ + --hash=sha256:ad2c86c495b899d862ea0f4b42891b8713a3bd45dd4105c7fd51c2a72f39f3a5 \ + --hash=sha256:aeae0e330c9f6acd681f647d46cefd30c29f93e3392882e792e82080c9691399 \ + --hash=sha256:b0431303acaea1089ad4b3e9ce4e6518193def1118d4073ca848635ee4ea2e96 \ + --hash=sha256:b5bdfd1c873d4e093aabc0ca84c4ca6dbc4f752afb5c86f146d9742580c9da2e \ + --hash=sha256:baed1e86cc735622097354b9d1281406caf42ff42a886d29faa8e8d1630333be \ + --hash=sha256:c1453022f490d2459a11819d83ad1d586e9ff65a12ac3e705ffebd46d3685dcf \ + --hash=sha256:c26608d2222fb1e94487e4a387d85f13eb55d5ed725cb25a0c589ac4ee60e7bc \ + --hash=sha256:c7659f22557c5a0bc4855cd635f55edec690cc008a40768527762cb9fb263455 \ + --hash=sha256:c8c69575568085ba0b1b10c0249d779a214aea6f6522e949a0fc9fb0fcb449d0 \ + --hash=sha256:c8d2c9fd1f2d16f780d15127abb050d13d1a76c03a4bd87d7e4980e45e511e12 \ + --hash=sha256:ca82be1a1d406ecfe1d25dc16cb33488e5a16bf4438c9fb590484ea29d92478b \ + --hash=sha256:cc572dace3f60ef98d7b12ff411d20f5362feb31a0439eab0085bbfd349982d7 \ + --hash=sha256:d18e5ac0f2f03f4f518d3e23db0f0cad7faa1da8620e9c09461d443bbf6e6692 \ + --hash=sha256:d28630f5854ab07ab1fd4aba756de52326c82e6be15d414b12793f1975048b54 \ + --hash=sha256:d9c275eaacd24aa73f94ffd6de08fc3f932424d8b6c376f4bed7cde376fe7bc3 \ + --hash=sha256:da0e573f9f97159390c89d9f1a9e41908b66d408cc5b58d08cf3847d844c531b \ + --hash=sha256:dd31f52ea1086513bb9df30f8fcee9b8918323ae067a3d5b78bc826a000712be \ + --hash=sha256:dddad92b554513a31f272570678ba307fb9f618f05e3d4a5eacafff9eae03e1d \ + --hash=sha256:df423d40ee8654634421812bc3b196da3f9bd7d32929da813f8394c4348a5358 \ + --hash=sha256:df913725b79db7bcf03448f36b7bf8815363417d5b58deecf9305e3e30f0f21a \ + --hash=sha256:e0bcb7e0f677f543555d2adff3bf19c05f66cdb4796e5ff602442ab2fe3c4ef7 \ + --hash=sha256:e2d65b31f36619cda3999b78b2aa9632e76b78448e7a56fc4240824200e7c4fc \ + --hash=sha256:e6e8cff14d6fb0be70a09c0bdc58096f501952d04624ebf867e0e56da2df8960 \ + --hash=sha256:f16c709686a78c727bbbf059f92b0bf41c6fc60deec706d2dc19f529175a6125 \ + --hash=sha256:f24fb43132a4c6b4cb4eb029492919b2db645be6808d738f244fd146c03c32cb \ + --hash=sha256:f53e442b08449d42821fa4a4fba000095af9f62742a500f978a9f557ec44339a \ + --hash=sha256:f5cfbc5fe74540d335175b656c725d74d90e3730c626d92575eea35029d9afaa \ + --hash=sha256:f81b3b8f3d4e343550fa4baa0e479bba9f2d29ce9c2e9b51d1ce1718d7442fcf \ + --hash=sha256:f8ec5e643a9a937f64e1999eb9f75d072263751912dc5cd06d3c85f8f44be7c3 \ + --hash=sha256:fb92203a88b3d3053034db775110081c49d28be6551923805e039924093761e4 \ + --hash=sha256:fcd22650c908d7b7da162bbfaab594a1227a15d1643a98c68b122ac642fa2264 + # via soundfile +click==8.4.2 \ + --hash=sha256:9a6cea6e60b17ebe0a44c5cc636d94f09bd66142c1cd7d8b4cd731c4917a15f6 \ + --hash=sha256:e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76 + # via + # huggingface-hub + # uvicorn +colorama==0.4.6 ; sys_platform == 'win32' \ + --hash=sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44 \ + --hash=sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6 + # via + # click + # tqdm +fastapi==0.136.3 \ + --hash=sha256:3d2a69bdf04b7e9f3afa292c3bc7a98816bbfafa10bc9b45f3f3700d2f761620 \ + --hash=sha256:e487fae93ad408e6f47641ee4dfe389864fd7bec92e547ea8498fc13f43e83ab + # via -r scripts/sa3-tflite-requirements.in +filelock==3.32.2 \ + --hash=sha256:87dd94cf281e586d135fa51132b8e3d9a598b316e90377a288663c9321036c82 \ + --hash=sha256:c33351e1f49cae33414acbc6d56784e6ecee82514ec90795da1161fc4836b5b8 + # via huggingface-hub +flatbuffers==25.12.19 \ + --hash=sha256:7634f50c427838bb021c2d66a3d1168e9d199b0607e6329399f04846d42e20b4 + # via ai-edge-litert +fsspec==2026.7.0 \ + --hash=sha256:b57ddbafedfaef7018c1ecab32aa200a9d7ca26b77965f64e48b70061249d279 \ + --hash=sha256:c803c40f4cf860b49dea58ee3e1c33cb9c790520e233537e1340049f89b82a88 + # via huggingface-hub +h11==0.16.0 \ + --hash=sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1 \ + --hash=sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86 + # via + # httpcore + # uvicorn +hf-xet==1.6.0 ; platform_machine == 'AMD64' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64' \ + --hash=sha256:0e6e21fa3cdfcdcd76748564bf593870a5e013f47d97cf10aed63aa222cff5b7 \ + --hash=sha256:23379c2f9ec8696d952b16414a2bae72cad86a52df869b050698ba60f538c675 \ + --hash=sha256:2e58454a340b3556dfa4972d5451aff4fba8dd42a236600ba1a1d2b1514f0fef \ + --hash=sha256:35cec30d75c6f9eb9c16a77cef68e85a103b72e24d4b473714ec9ff06428bab9 \ + --hash=sha256:3dc3e35441ba395006af5aaacc40ef2e603c51ef46c3530b9156185f00935ea3 \ + --hash=sha256:4fc74352a17015bd0ee90038bc9efe38db894cde45f268b6712b04fce8cd0acb \ + --hash=sha256:5153e6bb103ad49d6ea9f1b2e230db5a2ea32551ad09a706d2f61d7c7c80d80e \ + --hash=sha256:5789835d7c6bc9436962853192082374297fb72d7eff7e7762ec25ceb7e25338 \ + --hash=sha256:633dc0cd71d32da58ab8c03ad38e2fac452c15c2b0a2866ebf6ededfe0a5061d \ + --hash=sha256:70cbb9c896901600128cb9b6f06e132954fbede1db30f31f7c6c63f84cb7c31d \ + --hash=sha256:75765820ce4700db3750c94acc8fe27c5fae4c9ec000a0dbac3ca082acf97765 \ + --hash=sha256:8fb4f71cba6129110c3374a33f919001ff130488fc23553698e34cc1c2a1198c \ + --hash=sha256:948f15d3a9545cfe5932f6bd8b440f6ae630aee108f14b7bd6c561f7c2dcc522 \ + --hash=sha256:d62671bb130879cef0ee4c9ebe47a14af6c66ec53e6d84dc15936e5ffdfac82f \ + --hash=sha256:f0906082d9932ae0c0057fa194041c22b4e2cdb46b2592ef3b91f020d62a081a \ + --hash=sha256:f2f7278c05c22fd60cb436cda1269649b3e81db65ecdc8496e5e164aa4143e7b \ + --hash=sha256:fb4fadde1b2b70bf4c0c14a6dccbe7194b1c28947fefd5bbe3fed9d940676c3b + # via huggingface-hub +httpcore==1.0.9 \ + --hash=sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55 \ + --hash=sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8 + # via httpx +httpx==0.28.1 \ + --hash=sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc \ + --hash=sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad + # via huggingface-hub +huggingface-hub==1.27.0 \ + --hash=sha256:7df6827c2f956c60fbaa64646e979e566db76f619dd0a9729dfb8c5a3eb4f68d \ + --hash=sha256:c1fed40ea82a6b41b477f5243546549b792ae0a93abcea608cff66089bf8f8df + # via -r scripts/sa3-tflite-requirements.in +idna==3.18 \ + --hash=sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2 \ + --hash=sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848 + # via + # anyio + # httpx +numpy==2.4.6 ; python_full_version < '3.12' \ + --hash=sha256:001fbb8e08d942dd57599e781f2472269ee7f2755fae407b4f67b2f0b17da3f1 \ + --hash=sha256:0280e0356c0829a18d9de1cb7eee50ec22ca639878d7240307ca0943d73cd2c4 \ + --hash=sha256:043191bfa8eab18c776647b62723ac9dddece59743b13f49b2016094129c2b3f \ + --hash=sha256:06ca2f61ec4385a07a6977c55ba998a4466c123642b4a32694d3128fce18c079 \ + --hash=sha256:0a041d3d761dc3c35cc56ce0351506a02bcbc25f7b169f652435141a17db9096 \ + --hash=sha256:0ab0a9c4ffb1a6d95ef519fe4247dba8eb6b18ad93999f76b7f657039acabd47 \ + --hash=sha256:0c9136e14ed34a9e343a31c533d78a9813a69a3148332bce5e9821cb2f996e66 \ + --hash=sha256:110f8b71aacb688ec69062bb7f6938a0f8acb01b7c1c4beb453c65b6d234584d \ + --hash=sha256:112b06a867b235ef466ed3508ddf0238050df9c727cafb5301ac385b899189a1 \ + --hash=sha256:17f9ade344e7d9b464a084d69bcf18fc691cb1db67c62ed80820bf4926d78f0e \ + --hash=sha256:1e254a00cdf42b1e4d5b3d68d33af63268d41340d8885df2ab6470f2e1500147 \ + --hash=sha256:1e978ec1e8bd0e0e4de6bb75de9d30cbb74db6b6a2bb727618613703ca0167dd \ + --hash=sha256:25c692919ac5a01f170a3bfcd62d745b24fd095c353d50812637d6fcab442e75 \ + --hash=sha256:260a5d70215b61ab4fadf5c7baacd64821842975eea312125ed3c39a6391b063 \ + --hash=sha256:2803abfebfc990042cd494d8ce2d5f82e9d847af6d35ec486923aa19dbad5e73 \ + --hash=sha256:29a287e0cf63ff528da061de6b9f64a4618da591ca1046aafc54062e40ca7eab \ + --hash=sha256:29cb7f67d10b479ff07c17d33e39f78c07f71c40ef30d63c153d340e96cd3fb4 \ + --hash=sha256:3213d622a0283a39a93d188f3cf72b26862df52fbb4ca3697f51705016523d41 \ + --hash=sha256:33111801a01c12a8a1e3721f0a9232f8cfc8ae2c6b7098167e6f623c6073f402 \ + --hash=sha256:357cc07a6d7b0b182ff02249616a03742827ebb1277546b5c7cd7f7620a45698 \ + --hash=sha256:38efbc8de75c7a0fc1ac190162d892787f3f47b57cc291231aafee36b80982b7 \ + --hash=sha256:4081eb135ac24158bd51cdfbef16f1c64df7063b1143f24731387137c092bec8 \ + --hash=sha256:40fdc1ae7125e518ea98e53e69a4ebc27e1fd50510c47b7ea130cf21e5e1d42b \ + --hash=sha256:4cfe66903cc32a9921a6733d96b19bb6abf310397581bbad89c228f5abaf0ee8 \ + --hash=sha256:511dbaf848decaaaf4b4ca48032619fb3138710c4bf7da7617765edad1ef96b0 \ + --hash=sha256:55cced7c52e981362f708ad635198e97a752dfba412cc03c23bbf3bd8d5cd662 \ + --hash=sha256:56b39e5e0622a09a25bf5baf62f4bcf0cb8a41ae6e2819cf49bbc5a74c083f91 \ + --hash=sha256:5dbbdb29840ca3d91ee0fece42fc29278886d908280bfec0a5846c6f901a3eb0 \ + --hash=sha256:5f9fb9157b4ce2971008323afe46053787b526ef624fea915b261468a8421a0f \ + --hash=sha256:6180d8b35af935aed8ece3a85e0a43f87393ae0ac87c8d2c8bd2c993f7270ef3 \ + --hash=sha256:68a5124b13fa6cc2086764a20005d30bc0548146f7f5322f02fce212ca14317f \ + --hash=sha256:68bb27509ac1b9a3443094260f6326150663b06abe40b73a2f81160623da5b67 \ + --hash=sha256:6f41ae150c4e32db4f3310cdaf64b1593a03dbabe29eec77fc9b50fe64061df6 \ + --hash=sha256:7265a2f3d436e54ef9f2b52b5c937e6be778781bd97a590319d7348f1c1ca997 \ + --hash=sha256:72fbe16c6fac95aedf5937fa873445cec2110be35d8a4e9433d7501fd98dae6b \ + --hash=sha256:7d92c3819208a60205a12a245c91ad70cb0a85336659b19b834205573ac8456e \ + --hash=sha256:8155154c7c691289fe18f510b5d4657c68c67989f293f0535a91360392ff6538 \ + --hash=sha256:81a1cca95ed5bb92aa8b10dd2cdc9a0d3853a50fad926c28b5d7e8ea54389627 \ + --hash=sha256:89cd468399cfd2504718f0ba50e410dca55a170b61a02ad92bb18c8a65186e93 \ + --hash=sha256:8ad03c0965fb3c692200e74d458ca28c1dbb4ce96f9a479a8aa041ad5fabca02 \ + --hash=sha256:90f9849678c75fe7afa2d348ac842c168b0a4d3d61919687216dfc547976d853 \ + --hash=sha256:948424b06129ce883307e8cff868c31396d8dc7630a59c61d70d98dbe70f222c \ + --hash=sha256:9cd5ffd25db4e7ba6a375693b3fc0fc1791ec636c17db3720da19bde7180ec43 \ + --hash=sha256:a0df0043bdb289bde1f62da130d20df23d58b45429f752bc7a8fc5325a225ecd \ + --hash=sha256:a2c306dea656c12c68f51f4cea133cbe78ca7435eb28c735eac1d3ebe73be6e8 \ + --hash=sha256:a7830bab239b79cda9c08c2da014761cafb48da6150e1da17ac06283f43b6089 \ + --hash=sha256:a7c711e21628b52034bb5ab8d1bce291f752fcc5e92accc615778acee1ff4778 \ + --hash=sha256:aaf159caa35993cb1f56fb9b8e4610d35758e7ca005412eb1daa856a78c9c4b1 \ + --hash=sha256:ae506e6902902557576a26ff33eda8695e7ecb3cb36c3b573a0765dee114ebdb \ + --hash=sha256:b507f5c4c1d508876d1819b6bf9a49d365b96320b5d4993426b33a23ca4b8261 \ + --hash=sha256:bf162abab1c1a736333192707cef898e735a5ca00f38f27eeedf44b39d9e85eb \ + --hash=sha256:c1a2af6c6ef86344a6b0db6b97834208bf598db514f2b155042439b62605601a \ + --hash=sha256:c2d37ab77531417474168eb79d6d80b14f821a966818505d03013d0833edb7a8 \ + --hash=sha256:c4fc99836233ea196540b17ab0983aff60ed07941751930f5f4d05bc3b3b7359 \ + --hash=sha256:d581b735e177fdcdce6fed8e7e8880a3fb6ee4e3653a3ac6af01c6f4c03effc5 \ + --hash=sha256:d6da64deb6b8ed903e7560180a92f2d804ee1ba5eeb849ac2748b8c1aba1f6d7 \ + --hash=sha256:d8e8286dd7cea7895157318d1b91cdacac64c479f3cbc8dce548331728484751 \ + --hash=sha256:ddea102b48f9e339f3948bf22040944184627a30fdf7f858667673b9c5f033c8 \ + --hash=sha256:dfa20cc6ca228e6b155b11da03825975ce66aea520985dbbddf0f2a5a495c605 \ + --hash=sha256:e3e5193ef5a3dc73bceee50f7fdc2c90dbb76c42df8d8fae3d1067a583df579e \ + --hash=sha256:e3eeb0aabd6bd5ce64faae67e9935203a6991b4bc2a485a767fbafb2c5125f45 \ + --hash=sha256:e5805d5a22fd19c8ccff10a9561f9df94436b0545619ea579db2d3c35294bce2 \ + --hash=sha256:e85b752a1e912b70eaad4fafbd4d1238007ab221de2009b9a2f5ae7461239895 \ + --hash=sha256:eaf7fa2de5c0be8ae6ff8e9bea2ccd725e980541244521d8d4b5f3354a27babe \ + --hash=sha256:ebfb099f8dcf083deef3ac1ca4c1503f387cf76296fcb3816b66f5ecb5f54fdb \ + --hash=sha256:ece3d2cfe132e7d51f44a832b303895e6f2d499c5e74dfbdb06ee246147a304a \ + --hash=sha256:ed9749eef4cbd126da3dc1d6bcb3a57f5eb7ac6a6484146bdbf743f552dfc577 \ + --hash=sha256:ede83e07a75dd06bc501566c1eca2afc0d61677c1472ac9ad93fdee6e638a48d \ + --hash=sha256:ef4aea96ce4d3b074422cb4f2f64e216bf9e213004bb58ecfdf50ea02ea8eb9a \ + --hash=sha256:f3a3570c4a2a16746ac2c31a7c7c7b0c186b95ce902e33db6f28094ed7387dda \ + --hash=sha256:f407cb6b8e9d6d8c626bc73c945db1706035af8fd632295547bf1c9e46d092d6 \ + --hash=sha256:f74a575920ab21fe304421a3fc28793d82e299cae9eccb37084e9fc7f3617c20 + # via + # -r scripts/sa3-tflite-requirements.in + # ai-edge-litert + # soundfile +numpy==2.5.1 ; python_full_version >= '3.12' \ + --hash=sha256:08d60c810432eb83360958dea0999ac4cfb94531ea8efcbf0b7f277c2068aeb2 \ + --hash=sha256:09e9bfd8d2cf479c7d174804fb3811c53a8e9f20a37444008606b57d6b7a826d \ + --hash=sha256:0bfebd8695f9863592fe744be833a258120b14a9f39da255e8aa8fade2c0ddd1 \ + --hash=sha256:17a25e09640602e10bc8de0e6fa2b3fd68eedd84ba6d7842dc8f32f9ab87bd0b \ + --hash=sha256:1c6759f538fb912fc46de0a6b1758ccf7b57bc7c7ebebc23974fdac3de8db0cd \ + --hash=sha256:224ca51130ef7da85bea2191625181cb4f337f9cb64b471f10c1a12aa8b60077 \ + --hash=sha256:24d0eb82c0541d3415a33425db64ae439dffccd7b4dbcb30e7c35120205c506a \ + --hash=sha256:2ae0ca40bcb22d6ba59c1dfd5446f49940b0f2d821fde133f10dda11f816b84e \ + --hash=sha256:2c889b56fe48b1018f764b0eec8df59ab654e9148aa91faa12596043500de277 \ + --hash=sha256:30b44a6b53a7ae63c54c089a8726e5563ed302716c5b7ccc85afade40b0e7ff6 \ + --hash=sha256:32985c896d897419ef8da6917872d80b78ad0ea26d85b23245c7366ffde76d75 \ + --hash=sha256:3935f3b419b244a02732676fa5317a9193cc596a4c0646db07e5b421229ac9f7 \ + --hash=sha256:4939237038ada79308dda3204ac6462df056b5672b2e25db1149cf873668b3e1 \ + --hash=sha256:4b4ff1608417eb7a59da7b967bbb798cacfe071d2caf526a24281cd562072ed9 \ + --hash=sha256:54ad769f17bc2d833b620851989f62054fb9ab93c969d9e1dc3c8e3d56beea21 \ + --hash=sha256:59fda5e192b570217ec2580c96f00e9a7e12ef6866a900eb089b62c1a32545ca \ + --hash=sha256:5a4c988b38d261deeeaad9954e3deb091ad905c94e8bb6708654ef1d97f286b0 \ + --hash=sha256:5a6db61f9aaa57e369905c67d852045d3c4f7126405b29d09b19dec118e9c9cb \ + --hash=sha256:6165343f81b56ef8f514f396989e529b61d9dc709b99421b07e9f3e698e2287d \ + --hash=sha256:61ac47e772e6b8ea489e1d2f441a34c5c3ac17327e7ce294cbdf535795ad4e75 \ + --hash=sha256:6c3fe51bc6a16453d452997053454f309e8e0ed7b42d6b361ce4ac8c32913d74 \ + --hash=sha256:6eab239876581b2b3c5a242281b6007bbdbcd1c7085d7709bb57c5929b11e6bf \ + --hash=sha256:78798bd5b9ad744056af8efa90e3b9ddaa53272a0848a483084a1cc0a13b2dc0 \ + --hash=sha256:7c786fe9a5bbe360022e584c5a34cf6b54265c71bd7ec8ac3d8fec38968071f8 \ + --hash=sha256:83ce9c80d5b521b0d77ddcbe5447c218d247929b6cc056ca5351342accfff0af \ + --hash=sha256:9726558e8db4a5bf7929a70ae50f63abda4daf0efe810e3bfbab95976f75fc1a \ + --hash=sha256:99d5095fa265a0c4152e7bb12759e14381ef5496152f1ce58f44bdf55c44beb4 \ + --hash=sha256:a33276be12fa045805f477f22482088b66bb758ffbe89a9d21457de863a32e22 \ + --hash=sha256:a48a113e6afea91f5608793bafa7ef2ad481fefbda87ec5069f483de61cb9fa3 \ + --hash=sha256:ab451b59c5643c570974c43aef780703ef1d3b4965d2be07afd530615a9358d1 \ + --hash=sha256:ab84dc6b074fa881cae55bea94cc4f68e285181ba7f32497bf7dee6b1496165b \ + --hash=sha256:ab87a91b3cc3382b8956095bd8f95e00cf679bb81554339be1a2ba404a1473c1 \ + --hash=sha256:c12afb53450fa976d4c681c50a7423729a4c51c0465ed9f32b8a9cabbc472373 \ + --hash=sha256:caf3e317d33d60c37986b452613f4ab51246d0691350c03d0cb4a898627f4a95 \ + --hash=sha256:dc932a65ded7ce9013d120845a2514dcccb1a67bfc8deb8d37633762951904a6 \ + --hash=sha256:e68d8dd1e7eba712948f2053a29ec86917bc70ba1358df869d9f06649ef9cf09 \ + --hash=sha256:e824c2acf8862052246be5a44c15da1777940c60d010dd2aab897824d9c430f9 \ + --hash=sha256:e8c11c405efc5ff6816d5983c96cdfa215bab3428961243af3ff59b228490438 \ + --hash=sha256:efd736408cc97c79b9e6917338dfc8f06013b2274f992e96b1d9a81a71e2a2c2 \ + --hash=sha256:f089d7b00756190aacf1f5d34bdf38c3c430ac82b4f868f8cede73380460fce7 \ + --hash=sha256:f2479a47f8d5932d1718168a681ad6e536a9df484c83cfcf9de365e164537ace \ + --hash=sha256:f7119ebff1a9829e9f431a4f9d28e703023bb6b9fe7c8f724467dbfc27c94ab3 \ + --hash=sha256:f7d60026c0bdb1380e83bfa7a0419c4577ee4b9a08880afcb6dadeb74c649fa2 \ + --hash=sha256:f7feb014281029e628ba2d5a007407443b06e418b6fe451d1e2adcbc8eba0107 + # via + # -r scripts/sa3-tflite-requirements.in + # ai-edge-litert + # soundfile +packaging==26.3 \ + --hash=sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79 \ + --hash=sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c + # via huggingface-hub +protobuf==7.35.1 \ + --hash=sha256:11d6b0ec246892d85215b0a13ca6e0233cf5284b68f0ac02646427f4ff88a799 \ + --hash=sha256:230a75ddfc2de4806e56696ce9640c1cdfdb6543b7cfce98d42a4c0a0e7bdb87 \ + --hash=sha256:24f857477359a85c0c235261b8ba905fd51b2562f4a64ca1df5473f29850cbf6 \ + --hash=sha256:353652e4efd0bca5b5fc2656abf8307ef351f0cf938c9eba09f0e09c20a25c30 \ + --hash=sha256:4bc97768d8fe4ad6743c8a19403e314511ed9f6d13205b687e52421c023ac1b9 \ + --hash=sha256:74758715c53d7158fb76caf4f0cfdacc5329a4b1bb994f865d6cf302d413a1c4 \ + --hash=sha256:b73f9489a4b8b1c9cb1f8ed951c736392592edb24b9d6819f36d2e10b171d5b4 \ + --hash=sha256:ce115a26fe0c39a2c29973d914d327e516a6455464489fe3cd1e51a1b354f81a + # via ai-edge-litert +pycparser==3.0 ; implementation_name != 'PyPy' \ + --hash=sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29 \ + --hash=sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992 + # via cffi +pydantic==2.13.4 \ + --hash=sha256:45a282cde31d808236fd7ea9d919b128653c8b38b393d1c4ab335c62924d9aba \ + --hash=sha256:c40756b57adaa8b1efeeced5c196f3f3b7c435f90e84ea7f443901bec8099ef6 + # via fastapi +pydantic-core==2.46.4 \ + --hash=sha256:00c603d540afdd6b80eb39f078f33ebd46211f02f33e34a32d9f053bba711de0 \ + --hash=sha256:0186750b482eefa11d7f435892b09c5c606193ef3375bcf94aa00ae6bfb66262 \ + --hash=sha256:041bde0a48fd37cf71cab1c9d56d3e8625a3793fef1f7dd232b3ff37e978ecda \ + --hash=sha256:0c563b08bca408dc7f65f700633d8442fffb2421fc47b8101377e9fd65051ff0 \ + --hash=sha256:0cbe8b01f948de4286c74cdd6c667aceb38f5c1e26f0693b3983d9d74887c65e \ + --hash=sha256:0ce40cd7b21210e99342afafbd4d0f76d784eb5b1d60f3bdc566be4983c6c73b \ + --hash=sha256:0e96592440881c74a213e5ad528e2b24d3d4f940de2766bed9010ab1d9e51594 \ + --hash=sha256:10e17cbb10a330363733efc4d7c4d0dd827ac0909b8f6a6542298fed1ea62f29 \ + --hash=sha256:133878133d271ade3d41d1bfb2a45ec38dbdbda40bc065921c6b04e4630127e2 \ + --hash=sha256:14d4edf427bdcf950a8a02d7cb44a08614388dd6e1bdcbf4f67504fa7887da9c \ + --hash=sha256:14f4c5d6db102bd796a627bbb3a17b4cf4574b9ae861d8b7c9a9661c6dd3362d \ + --hash=sha256:17299feefe090f2caa5b8e37222bb5f663e4935a8bfa6931d4102e5df1a9f398 \ + --hash=sha256:184c081504d17f1c1066e430e117142b2c77d9448a97f7b65c6ac9fd9aee238d \ + --hash=sha256:18e5ceec2ab67e6d5f1a9085e5a24c9c4e2ac4545730bfe668680bca05e555f3 \ + --hash=sha256:19e51f073cd3df251856a8a4189fbdf1de4012c3ebacfb1884f94f1eb406079f \ + --hash=sha256:1a7dd0b3ee80d90150e3495a3a13ac34dbcbfd4f012996a6a1d8900e91b5c0fb \ + --hash=sha256:1d8ba486450b14f3b1d63bc521d410ec7565e52f887b9fb671791886436a42f7 \ + --hash=sha256:2108ba5c1c1eca18030634489dc544844144ee36357f2f9f780b93e7ddbb44b5 \ + --hash=sha256:228ee9bae8bef5b1e97ec58302f80357c37199e0d0a99174e138d28e6957b9d9 \ + --hash=sha256:23ace664830ee0bfe014a0c7bc248b1f7f25ed7ad103852c317624a1083af462 \ + --hash=sha256:2412e734dcb48da14d4e4006b82b46b74f2518b8a26ee7e58c6844a6cd6d03c4 \ + --hash=sha256:29c61fc04a3d840155ff08e475a04809278972fe6aef51e2720554e96367e34b \ + --hash=sha256:2f84c03c8607173d16b5a854ec68a2f9079ae03237a54fb506d13af47e1d018d \ + --hash=sha256:3009f12e4e90b7f88b4f9adb1b0c4a3d58fe7820f3238c190047209d148026df \ + --hash=sha256:3245406455a5d98187ec35530fd772b1d799b26667980872c8d4614991e2c4a2 \ + --hash=sha256:3447661d99f75a3683a4cf5c87da72f2161964611864dbbeac7fbb118bb4bfc0 \ + --hash=sha256:372429a130e469c9cd698925ce5fc50940b7a1336b0d82038e63d5bbc4edc519 \ + --hash=sha256:395aebd9183f9d112f569aeb5b2214d1a10a33bec8456447f7fbdfa51d38d4cd \ + --hash=sha256:3a233125ac121aa3ffba9a2b59edfc4a985a76092dc8279586ab4b71390875e7 \ + --hash=sha256:3be77f45df024d789a672ae34f8b06fb346c4f9f46ea714956660ea4862e89ac \ + --hash=sha256:3bf92c5d0e00fefaab325a4d27828fe6b6e2a21848686b5b60d2d9eeb09d76c6 \ + --hash=sha256:3ecbc122d18468d06ca279dc26a8c2e2d5acb10943bb35e36ae92096dc3b5565 \ + --hash=sha256:3fb702cd90b0446a3a1c5e470bfa0dd23c0233b676a9099ddcc964fa6ca13898 \ + --hash=sha256:428e04521a40150c85216fc8b85e8d39fece235a9cf5e383761238c7fa9b96fb \ + --hash=sha256:432c179df7874eeb73307aad2df0755e1ae0efa61ff0ea89b93e194411ae3928 \ + --hash=sha256:4a05d69cba51d852c5c3e92758653245a50c0b646ced0cf05bd793ed592839d6 \ + --hash=sha256:4c63ebc82684aa89d9a3bcbd13d515b3be44250dc68dd3bd81526c1cb31286c3 \ + --hash=sha256:4fc73cb559bdb54b1134a706a2802a4cddd27a0633f5abb7e53056268751ac6a \ + --hash=sha256:4fcbe087dbc2068af7eda3aa87634eba216dbda64d1ae73c8684b621d33f6596 \ + --hash=sha256:56cb4851bcaf3d117eddcef4fe66afd750a50274b0da8e22be256d10e5611987 \ + --hash=sha256:5855698a4856556d86e8e6cd8434bc3ac0314ee8e12089ae0e143f64c6256e4e \ + --hash=sha256:5a4330cdbc57162e4b3aa303f588ba752257694c9c9be3e7ebb11b4aca659b5d \ + --hash=sha256:5b712b53160b79a5850310b912a5ef8e57e56947c8ad690c227f5c9d7e561712 \ + --hash=sha256:5d5902252db0d3cedf8d4a1bc68f70eeb430f7e4c7104c8c476753519b423008 \ + --hash=sha256:617d7e2ca7dcb8c5cf6bcb8c59b8832c94b36196bbf1cbd1bfb56ed341905edd \ + --hash=sha256:62f875393d7f270851f20523dd2e29f082bcc82292d66db2b64ea71f64b6e1c1 \ + --hash=sha256:633147d34cf4550417f12e2b1a0383973bdf5cdfde212cb09e9a581cf10820be \ + --hash=sha256:66ce7632c22d837c95301830e111ad0128a32b8207533b60896a96c4915192ea \ + --hash=sha256:6b3ace8194b0e5204818c92802dcdca7fc6d88aabbb799d7c795540d9cd6d292 \ + --hash=sha256:6f2eeda33a839975441c86a4119e1383c50b47faf0cbb5176985565c6bb02c33 \ + --hash=sha256:7027560ee92211647d0d34e3f7cd6f50da56399d26a9c8ad0da286d3869a53f3 \ + --hash=sha256:7283d57845ecf5a163403eb0702dfc220cc4fbdd18919cb5ccea4f95ee1cdab4 \ + --hash=sha256:7a5f930472650a82629163023e630d160863fce524c616f4e5186e5de9d9a49b \ + --hash=sha256:7bfb192b3f4b9e8a89b6277b6ce787564f62cfd272055f6e685726b111dc7826 \ + --hash=sha256:811ff8e9c313ab425368bcbb36e5c4ebd7108c2bbf4e4089cfbb0b01eff63fac \ + --hash=sha256:8233f2947cf85404441fd7e0085f53b10c93e0ee78611099b5c7237e36aacbf7 \ + --hash=sha256:82cf5301172168103724d49a1444d3378cb20cdee30b116a1bd6031236298a5d \ + --hash=sha256:8358a950c8909158e3df31538a7e4edc2d7265a7c54b47f0864d9e5bae9dcebf \ + --hash=sha256:85bb3611ff1802f3ee7fdd7dbff26b56f343fb432d57a4728fdd49b6ef35e2f4 \ + --hash=sha256:86e1a4418c6cd97d60c95c71164158eaf7324fae7b0923264016baa993eba6fc \ + --hash=sha256:8b9bab013d1c7a79d3501ff86d0bc9c31bf587db4551677b96bec07df78c6b15 \ + --hash=sha256:8c5dac79fa1614d1e06ca695109c6105923bd9c7d1d6c918d4e637b7e6b32fd3 \ + --hash=sha256:8d0820e8192167f80d88d64038e609c31452eeca865b4e1d9950a27a4609b00b \ + --hash=sha256:8daafc69c93ee8a0204506a3b6b30f586ef54028f52aeeeb5c4cfc5184fd5914 \ + --hash=sha256:9037063db01f09b09e237c282b6792bd4da634b5402c4e7f0c61effed7701a04 \ + --hash=sha256:905a0ed8ea6f2d61c1738835f99b699348d7857379083e5fc497fa0c967a407c \ + --hash=sha256:90884113d8b48f760e9587002789ddd741e76ab9f89518cd1e43b1f1a52ec44b \ + --hash=sha256:91a06d2e259ecfbd8c901d70c3c507900458498142b3026a296b7de4d1322cc9 \ + --hash=sha256:926c9541b14b12b1681dca8a0b75feb510b06c6341b70a8e500c2fdcff837cce \ + --hash=sha256:9401557acd873c3a7f3eb9383edef8ac4968f9510e340f4808d427e75667e7b4 \ + --hash=sha256:9551187363ffc0de2a00b2e47c25aeaeb1020b69b668762966df15fc5659dd5a \ + --hash=sha256:962ccbab7b642487b1d8b7df90ef677e03134cf1fd8880bf698649b22a69371f \ + --hash=sha256:97e7cf2be5c77b7d1a9713a05605d49460d02c6078d38d8bef3cbe323c548424 \ + --hash=sha256:9aa768456404a8bf48a4406685ac2bec8e72b62c69313734fa3b73cf33b3a894 \ + --hash=sha256:9bc519fbf2b7578398853d815009ae5e4d4603d12f4e3f91da8c06852d3da3e9 \ + --hash=sha256:9d56801be94b86a9da183e5f3766e6310752b99ff647e38b09a9500d88e46e76 \ + --hash=sha256:9f444c499b3eefd3a92e348059471ea0c3a6e303d9c1cec09fa748fd9f895201 \ + --hash=sha256:9fa8ae11da9e2b3126c6426f147e0fba88d96d65921799bb30c6abd1cb2c97fb \ + --hash=sha256:a0f62d0a58f4e7da165457e995725421e0064f2255d8eccebc49f41bbc23b109 \ + --hash=sha256:a396dcc17e5a0b164dbe026896245a4fa9ff402edca1dff0be3d53a517f74de4 \ + --hash=sha256:aaa2a54443eff1950ba5ddc6b6ccda0d9c84a364276a62f969bdf2a390650848 \ + --hash=sha256:ad785e92e6dc634c21555edc8bd6b64957ab844541bcb96a1366c202951ae526 \ + --hash=sha256:af8244b2bef6aaad6d92cda81372de7f8c8d36c9f0c3ea36e827c60e7d9467a0 \ + --hash=sha256:b078afbc25f3a1436c7a1d2cd3e322497ee99615ba97c563566fdf46aff1ee01 \ + --hash=sha256:b2f69dec1725e79a012d920df1707de5caf7ed5e08f3be4435e25803efc47458 \ + --hash=sha256:b8458003118a712e66286df6a707db01c52c0f52f7db8e4a38f0da1d3b94fc4e \ + --hash=sha256:bb63e0198ca18aad131c089b9204c23079c3afa95487e561f4c522d519e55aba \ + --hash=sha256:bfec22eab3c8cc2ceec0248aec886624116dc079afa027ecc8ad4a7e62010f8a \ + --hash=sha256:c1747f85cee84c26985853c6f3d9bd3e75da5212912443fa111c113b9c246f39 \ + --hash=sha256:c1b3f518abeca3aa13c712fd202306e145abf59a18b094a6bafb2d2bbf59192c \ + --hash=sha256:c50f2528cf200c5eed56faf3f4e22fcd5f38c157a8b78576e6ba3168ec35f000 \ + --hash=sha256:c68fcd102d71ea85c5b2dfac3f4f8476eff42a9e078fd5faefff6d145063536b \ + --hash=sha256:c7a7bd4e39e8e4c12c39cd480356842b6a8a06e41b23a55a5e3e191718838ddf \ + --hash=sha256:c94f0688e7b8d0a67abf40e57a7eaaecd17cc9586706a31b76c031f63df052b4 \ + --hash=sha256:cbaf13819775b7f769bf4a1f066cb6df7a28d4480081a589828ef190226881cd \ + --hash=sha256:cd2213145bcc2ba85884d0ac63d222fece9209678f77b9b4d76f054c561adb28 \ + --hash=sha256:ce5c1d2a8b27468f433ca974829c44060b8097eedc39933e3c206a90ee49c4a9 \ + --hash=sha256:d396ec2b979760aaf3218e76c24e65bd0aca24983298653b3a9d7a45f9e47b30 \ + --hash=sha256:d51026d73fcfd93610abc7b27789c26b313920fcfb20e27462d74a7f8b06e983 \ + --hash=sha256:d80ee3d731373b24cebbc10d689ca4ee1875caf0d5703a245db18efd4dd37fc1 \ + --hash=sha256:d995260fdf4e1db774581b4900e0f832abe3c7c84996726bbc161b19c8f29e76 \ + --hash=sha256:da4b951fe36dc7c3a1ccb4e3cd1747c3542b8c9ceede8fc86cae054e764485f5 \ + --hash=sha256:daa27d92c36f24388fe3ad306b174781c747627f134452e4f128ea00ce1fe8c4 \ + --hash=sha256:db06ffe51636ffe9ca531fe9023dd64bdd794be8754cb5df57c5498ae5b518a7 \ + --hash=sha256:e0d65b8c354be7fb5f720c3caa8bc940bc2d20ce749c8e06135f07f8ed95dd7c \ + --hash=sha256:e68b7a074f65a2fd746c52a7ce6142ab7006074ac269ace0c25cd8ba171f8066 \ + --hash=sha256:e739fee756ba1010f8bcccb534252e85a35fe45ae92c295a06059ce58b74ccd3 \ + --hash=sha256:e846ae7835bf0703ae43f534ab79a867146dadd59dc9ca5c8b53d5c8f7c9ef02 \ + --hash=sha256:e9c26f834c65f5752f3f06cb08cb86a913ceb7274d0db6e267808a708b46bc89 \ + --hash=sha256:ea793e075b70290d89d8142074262885d3f7da19634845135751bd6344f73b50 \ + --hash=sha256:f027324c56cd5406ca49c124b0db10e56c69064fec039acc571c29020cc87c76 \ + --hash=sha256:f13a646d65d09fbf1bc6b3a9635d30095c8e7e5cc419ff35ecc563c5fd04cd49 \ + --hash=sha256:f47286a97f0bc9b8859519809077b91b2cefe4ae47fcbf5e466a009c1c5d742b \ + --hash=sha256:f747929cf940cddb5b3668a390056ddd5ba2e5010615ea2dcf4f9c4f3ab8791d \ + --hash=sha256:f99626688942fb746e545232e7726926f3be91b5975f8b55327665fafda991c7 \ + --hash=sha256:f9fa868638bf362d3d138ea55829cefb3d5f4b0d7f142234382a15e2485dbec4 \ + --hash=sha256:fbdb89b3e1c94a30cc5edfce477c6e6a5dc4d8f84665b455c27582f211a1c72c \ + --hash=sha256:fc010ab034c8c7452522748bf937df58020d256ccae0874463d1f4d01758af8e \ + --hash=sha256:fc3e9034a63de20e15e8ade85358bc6efc614008cab72898b4b4952bea0509ff \ + --hash=sha256:fd8b3d9fd264be37976686c7f65cd52a83f5e84f4bfd2adf9c1d469676bbb6ae + # via pydantic +python-multipart==0.0.32 \ + --hash=sha256:be54b7f3fa167bb83e4fcd936b887b708f4e57fe75911c02aebf53efaf8d938e \ + --hash=sha256:ff6d3f776f16878c894e52e107296ffc890e913c611b1a4ec6c44e2821fe2e23 + # via -r scripts/sa3-tflite-requirements.in +pyyaml==6.0.3 \ + --hash=sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c \ + --hash=sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a \ + --hash=sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3 \ + --hash=sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956 \ + --hash=sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6 \ + --hash=sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c \ + --hash=sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65 \ + --hash=sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a \ + --hash=sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0 \ + --hash=sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b \ + --hash=sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1 \ + --hash=sha256:22ba7cfcad58ef3ecddc7ed1db3409af68d023b7f940da23c6c2a1890976eda6 \ + --hash=sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7 \ + --hash=sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e \ + --hash=sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007 \ + --hash=sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310 \ + --hash=sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4 \ + --hash=sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9 \ + --hash=sha256:3ff07ec89bae51176c0549bc4c63aa6202991da2d9a6129d7aef7f1407d3f295 \ + --hash=sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea \ + --hash=sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0 \ + --hash=sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e \ + --hash=sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac \ + --hash=sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9 \ + --hash=sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7 \ + --hash=sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35 \ + --hash=sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb \ + --hash=sha256:5cf4e27da7e3fbed4d6c3d8e797387aaad68102272f8f9752883bc32d61cb87b \ + --hash=sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69 \ + --hash=sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5 \ + --hash=sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b \ + --hash=sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c \ + --hash=sha256:6344df0d5755a2c9a276d4473ae6b90647e216ab4757f8426893b5dd2ac3f369 \ + --hash=sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd \ + --hash=sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824 \ + --hash=sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198 \ + --hash=sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065 \ + --hash=sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c \ + --hash=sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c \ + --hash=sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764 \ + --hash=sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196 \ + --hash=sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b \ + --hash=sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00 \ + --hash=sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac \ + --hash=sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8 \ + --hash=sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e \ + --hash=sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28 \ + --hash=sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3 \ + --hash=sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5 \ + --hash=sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4 \ + --hash=sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b \ + --hash=sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf \ + --hash=sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5 \ + --hash=sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702 \ + --hash=sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8 \ + --hash=sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788 \ + --hash=sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da \ + --hash=sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d \ + --hash=sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc \ + --hash=sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c \ + --hash=sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba \ + --hash=sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f \ + --hash=sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917 \ + --hash=sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5 \ + --hash=sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26 \ + --hash=sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f \ + --hash=sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b \ + --hash=sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be \ + --hash=sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c \ + --hash=sha256:efd7b85f94a6f21e4932043973a7ba2613b059c4a000551892ac9f1d11f5baf3 \ + --hash=sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6 \ + --hash=sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926 \ + --hash=sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0 + # via huggingface-hub +sentencepiece==0.2.2 \ + --hash=sha256:046b15ea22d8042e2e173561d464ec3b64a9c2081324df70ebce7bf7ebb3e497 \ + --hash=sha256:0e2aae42960392d6dcb9a72d8e1e65a97294c965071b43c7b3429a42f350250e \ + --hash=sha256:1120e0791540615e650b2e9bea835bf38a7362455d8ab62dee7968219c2d79a0 \ + --hash=sha256:1402d8ee36f0d851cea8eee4dbb85fea14643b7503cf4d00d102eec0fe3ca719 \ + --hash=sha256:1416b92f2f010333786fe6306ed2631121d5ea492219b0841e967b6765e64107 \ + --hash=sha256:16c84ddef8d3084a8af37208acd365b08092ca089080f1a71fbfdd911adda9b3 \ + --hash=sha256:1edb10e520e4bddf74d85b0f5ae74cc2d60c2b448885080bfb618bc2b3a49f6b \ + --hash=sha256:201a8e0f55501a76e08dbf2c54bc45f4642b379271e89c667d517bfbc2191f2a \ + --hash=sha256:252908153eeec06c3ca3a32077e64a49d572e3d89881475b4e0f02d99d9fcc7c \ + --hash=sha256:38111ed1f79268f399c505028023d5eaaf0ab4e5eafceb709468b0d3323e7838 \ + --hash=sha256:3ab3f1ae98970b5590e2209341522718900ba19bcc2c207ffaa6bd417ad960c5 \ + --hash=sha256:3d2b5e824b5622038dc7b490897efe05ebbbb9e7350fc142f3ecc8789ef9bdf6 \ + --hash=sha256:3ec27c152a1f1b24bc9168b55a5880f3c16e2334e697da6f55a1046a22405a3d \ + --hash=sha256:3f5851441ab1ef8634963a5100b733a8bbeefe623e0c5c005b1f1f3880e574cf \ + --hash=sha256:3fd9ce2ab4460c713cfdeb4aca693ca6732a11538e05fb332d5af42e3d7fde25 \ + --hash=sha256:44284adc6fbe9d5bdd480541431a3d93f674fa44736714d3ad4bcee8283ace7d \ + --hash=sha256:443ac618c7a2a1377cf5c82581fbb849591d14e656d5e5a3e4682d4e36a34e4e \ + --hash=sha256:46ba07b543add034de0ff47ac5f907e9a06682f91d85121a972764628933be6b \ + --hash=sha256:4f0603267cd15b92b68c2c0e852a441507614b70dc7773659baa6b8c214a91fd \ + --hash=sha256:524e2a85c028a0d2f9935191fa751e5ef9d9bcc39616f70ab14b28d0369c9936 \ + --hash=sha256:54a83df9260a89c1734256e620fe1f1a6bfedd7547139d4dc1384efac11a3a85 \ + --hash=sha256:59d6588712101ccfcae9b03692be3aaae1514c2078666d7b05f15ba3a702e41b \ + --hash=sha256:63250cfab8b80a1ef82a614eb2b3cadfec2c405f870cedc139d08e2f063eb708 \ + --hash=sha256:64b656f025355cf8c51abe9fbe3848540756c6d7ca5e6791b1afa664bc24c7cb \ + --hash=sha256:65d84ec36888de4a848eee5f910e67fbc79b064685ef1e10a502e14520ead9c9 \ + --hash=sha256:69e9dc8078e128286ed3b975e37c837ba96e215a50c3ef9f3f8b7ab9e5a832a0 \ + --hash=sha256:6dd76f3e5c8b2eb8a3a3efee787bbf5b9a66e52a048fe09cab85eca33fec6790 \ + --hash=sha256:70d4ca6f4d06df7f0ccab6fe4f49c8a712c8c8b6847b4f0af9a0e1dbb0e0337e \ + --hash=sha256:72b7825b331b1b7e7c45be2e674b3e3c65af608fa376bad2d851b20aaf0cdc78 \ + --hash=sha256:741b4b367140e9b5c36b5a14c72179f2c946d991ea9a7c031a2a1ee6ad097b99 \ + --hash=sha256:74f0ee601047c0c12a783088b51be4e6214a62ecd9e02278c477433cd16e0ed9 \ + --hash=sha256:76ff5814db72e7462dece042d7593cdf102b8ec82c2b1cc201a2add34ee3050d \ + --hash=sha256:77c3ce990b23441e5ecfa5bce181fd6f408b564aeb6d7e1d1e7de9c5612501c8 \ + --hash=sha256:79bac5a251f23a7341e28fda9ce0d5319edf45328239ce037c0682936f137906 \ + --hash=sha256:7c6e7bf684dc12145bfa685d3060beaea55139134ba848289bee514ed42e7383 \ + --hash=sha256:7fc14c1585139fa6b68775e616a6b90cf622ebf219f9558c0aeaf5d253ee6c9b \ + --hash=sha256:89625fb43765cccaa1443b9adb61f283e5fe4cb1536728205d06bada730caa53 \ + --hash=sha256:8b2db2056c97224e122054fd794543cde5d24b7cae28424f6e3eb79bbe08e42b \ + --hash=sha256:8d44b20234905ff022b7d535f79d1f823ad7670c9851cc4f03cdc34787cdb3ab \ + --hash=sha256:8eed98514bffe5ecac37f493f91869c351fbb05629328bfdbc08502c6c094dc0 \ + --hash=sha256:8f1f61592e7cabd45d49ce8cc0ef42ca655c091e037153754fb3fa59725b5914 \ + --hash=sha256:b23fe17779834d3c27aaf2edac9486d04cca1a7deb8f5facda35150ac6263a91 \ + --hash=sha256:bc7b0b1da20f856bfac5f84b2673fe534b167e41980b27442ca8f78c2b7eb77e \ + --hash=sha256:c62bd361cec1f5b556eb8210264ecfff37486cd990c3386cc00310f26c54090a \ + --hash=sha256:c76c9b3324efd79029eeb0fd2ced1964bdbeca7d45e030b46fa3ef3cf74f8032 \ + --hash=sha256:c798f0b327bac10dc95cdac77b9a197ab2bd7dd1e60ebd7586a12d918d4be711 \ + --hash=sha256:c8a168b040bc61681293f79a949b5d911c8e25086f4260285b8d97ab5f1195da \ + --hash=sha256:caad9566e2ef0e5640d36032c69b0edc7ac6028277b93d93815898804fac450c \ + --hash=sha256:cbce24284f51f71d10a42b7b9c964dcb9048b28f1c8e5db40bcbcb6f428cba6a \ + --hash=sha256:cd810878180a52950e5a61f25ada5248a453bbdbafe474f89514135fbc1f633d \ + --hash=sha256:d254c98ca6387655400b3959c33c83efd807f5edeb608e3aca45800ceaa77151 \ + --hash=sha256:d795c4ac689a57f9d4ba2288126ec7901d389ad5827d2f8b8533c883974fe563 \ + --hash=sha256:df88b0c34f2fa909d322f7b06b1398e1e81af4b2f42a7b8e3556f928b25d1811 \ + --hash=sha256:eb8da9d9a9b418422c21a07fd19b9d9228692b7a7468a45eec6b11642d3c808b \ + --hash=sha256:f7c06c751c19d923435a54bff4f7e66e728fad160e8da28254f133abc9725820 \ + --hash=sha256:fa9f5ef0e2a82233dd0b8b32ea3f5710e0c44afbc07ed3620219f32601e56090 \ + --hash=sha256:fd523c4992041faa5c2b3cde62253d11a96c30d73a34afe48a486e8e2254cd1c + # via -r scripts/sa3-tflite-requirements.in +soundfile==0.14.0 \ + --hash=sha256:0a6ae43c50c71b4e020cc55382925cb89451c1ed1a0c3d0f5d802da269226849 \ + --hash=sha256:19be05428da76ed61a4cad29b8e4bcf43a3e5c100089d2ec81dc961eed1b0dd4 \ + --hash=sha256:1e38bac1853412871318e82a1ba69a8be677619b56025bbfcccdb41b6cafe82d \ + --hash=sha256:299491d3499460fb1b74bb4bd78b57ffc2d243a5fafa7b6ec1b264875c78453e \ + --hash=sha256:8ba81ae3a89fd5ab3bef8a8eb481fbbe794e806309675a89b4df48b8d31908a8 \ + --hash=sha256:ba1c1a2d618bca5c406647c83b89f07cc8810fa506a50622a6993ba130c1de11 \ + --hash=sha256:d828d35a059626da52f1415b5faee610aeab393319cb3fc4a9aef47b619fc14c \ + --hash=sha256:e090704718e124e7c844695236f1fce8d18a5e761eaf7c82dfcd124620805f98 \ + --hash=sha256:e85724a90bc99a6e8062c0b4ddf725f53b2a3b70afd4da875e9d2cfc4e92f377 + # via -r scripts/sa3-tflite-requirements.in +starlette==1.6.0 \ + --hash=sha256:a86dd39d14bb45f85a3d18525215a9ef0cfd1f192ac793220e72598c90335f0c \ + --hash=sha256:d4e3ac5e546444960c710297a3c9fc3f7ebae1b7e963f3d36173b49da535be9b + # via fastapi +tqdm==4.70.0 \ + --hash=sha256:55b0b0dbd97462d06ebee91e4dac24ed4d4702be82b24f07e6c1d27e08cea220 \ + --hash=sha256:7f585706bfddbdebf89daac705b2dfcc16890130727d3197ca62c732b4310953 + # via + # ai-edge-litert + # huggingface-hub +typing-extensions==4.16.0 \ + --hash=sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8 \ + --hash=sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5 + # via + # ai-edge-litert + # anyio + # fastapi + # huggingface-hub + # pydantic + # pydantic-core + # soundfile + # starlette + # typing-inspection +typing-inspection==0.4.2 \ + --hash=sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7 \ + --hash=sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464 + # via + # fastapi + # pydantic +uvicorn==0.49.0 \ + --hash=sha256:ba3d14c3ee7e41c6c654c46c9eb489d33213cdd30aa1696eab1374337c13f68f \ + --hash=sha256:ebf4271aa580d9de97f93192d4595176df6e91f9aae919ca73e4fc07df1e66a3 + # via -r scripts/sa3-tflite-requirements.in diff --git a/scripts/sign-windows.ps1 b/scripts/sign-windows.ps1 new file mode 100644 index 0000000..fea127c --- /dev/null +++ b/scripts/sign-windows.ps1 @@ -0,0 +1,73 @@ +[CmdletBinding()] +param( + [string] $Path, + + [switch] $Preflight +) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version Latest + +function Require-ProtectedValue { + param( + [Parameter(Mandatory = $true)] + [string] $Name + ) + + $value = [Environment]::GetEnvironmentVariable($Name) + if ([string]::IsNullOrWhiteSpace($value)) { + throw "Protected Windows release configuration '$Name' is missing." + } + return $value.Trim() +} + +$providerCommand = Require-ProtectedValue 'LSDJ_WINDOWS_SIGN_COMMAND_PATH' +$expectedThumbprint = (Require-ProtectedValue 'LSDJ_WINDOWS_EXPECTED_CERTIFICATE_SHA1') -replace '\s', '' +$expectedSubject = Require-ProtectedValue 'LSDJ_WINDOWS_EXPECTED_SUBJECT' + +if (-not [System.IO.Path]::IsPathFullyQualified($providerCommand)) { + throw 'LSDJ_WINDOWS_SIGN_COMMAND_PATH must be an absolute executable path.' +} +if ($expectedThumbprint -notmatch '^[0-9A-Fa-f]{40}$') { + throw 'LSDJ_WINDOWS_EXPECTED_CERTIFICATE_SHA1 must contain exactly 40 hexadecimal characters.' +} +$provider = Get-Item -LiteralPath $providerCommand -ErrorAction Stop +if (($provider.Attributes -band [IO.FileAttributes]::ReparsePoint) -ne 0 -or $provider.PSIsContainer) { + throw 'The Windows signing provider command must be a plain executable file, not a link or directory.' +} +(Get-Command 'signtool.exe' -ErrorAction Stop) | Out-Null + +if ($Preflight) { + Write-Host "Windows signing interface is configured for subject '$expectedSubject' and certificate $($expectedThumbprint.ToUpperInvariant())." + exit 0 +} + +if ([string]::IsNullOrWhiteSpace($Path)) { + throw 'Path is required when signing an executable payload.' +} + +$target = Get-Item -LiteralPath $Path -ErrorAction Stop +if ($target.PSIsContainer -or ($target.Attributes -band [IO.FileAttributes]::ReparsePoint) -ne 0) { + throw "Signing target must be a plain file: $Path" +} +if ($target.Extension -notin @('.exe', '.dll')) { + throw "Refusing to sign a non-executable payload: $($target.FullName)" +} + +# The selected provider owns key access and timestamp configuration. Its wrapper +# receives exactly one literal path, never a shell command string. This keeps the +# repo compatible with certificate-store, HSM, or managed/keyless providers +# without pretending one has been selected. +$global:LASTEXITCODE = 0 +& $provider.FullName $target.FullName +if ($LASTEXITCODE -ne 0) { + throw "Windows signing provider failed with exit code $LASTEXITCODE for $($target.FullName)." +} + +& "$PSScriptRoot/verify-windows-signatures.ps1" ` + -Path $target.FullName ` + -ExpectedCertificateSha1 $expectedThumbprint ` + -ExpectedSubject $expectedSubject +if ($LASTEXITCODE -ne 0) { + throw "Signature verification failed for $($target.FullName)." +} diff --git a/scripts/test-windows-installer.ps1 b/scripts/test-windows-installer.ps1 new file mode 100644 index 0000000..f9f265c --- /dev/null +++ b/scripts/test-windows-installer.ps1 @@ -0,0 +1,946 @@ +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)] + [ValidateNotNullOrEmpty()] + [string] $OlderInstaller, + + [Parameter(Mandatory = $true)] + [ValidateNotNullOrEmpty()] + [string] $NewerInstaller, + + [Parameter(Mandatory = $true)] + [ValidatePattern('^[0-9]+\.[0-9]+\.[0-9]+$')] + [string] $OlderVersion, + + [Parameter(Mandatory = $true)] + [ValidatePattern('^[0-9]+\.[0-9]+\.[0-9]+$')] + [string] $NewerVersion, + + [switch] $RequireSigned +) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version Latest + +if (-not $IsWindows -or $env:GITHUB_ACTIONS -ne 'true') { + throw 'The destructive installer lifecycle smoke test may run only on an isolated GitHub Actions Windows runner.' +} +if (-not [Environment]::Is64BitOperatingSystem -or -not [Environment]::Is64BitProcess) { + throw 'The Windows shipping smoke test requires an x64 OS and process.' +} + +$older = Get-Item -LiteralPath $OlderInstaller -ErrorAction Stop +$newer = Get-Item -LiteralPath $NewerInstaller -ErrorAction Stop +foreach ($installer in @($older, $newer)) { + if ($installer.PSIsContainer -or ($installer.Attributes -band [IO.FileAttributes]::ReparsePoint) -ne 0) { + throw "Installer must be a plain file: $($installer.FullName)" + } + if ($installer.Extension -ne '.exe' -or $installer.VersionInfo.ProductName -ne 'LSDJ') { + throw "Installer has unexpected Windows version metadata: $($installer.FullName)" + } +} +if ($older.VersionInfo.ProductVersion -ne $OlderVersion) { + throw "Older installer metadata is $($older.VersionInfo.ProductVersion), expected $OlderVersion." +} +if ($newer.VersionInfo.ProductVersion -ne $NewerVersion) { + throw "Newer installer metadata is $($newer.VersionInfo.ProductVersion), expected $NewerVersion." +} + +$dataRoot = Join-Path $env:LOCALAPPDATA 'LSDJ' +$expectedRoot = [System.IO.Path]::GetFullPath((Join-Path $env:LOCALAPPDATA 'LSDJ')) +if ([System.IO.Path]::GetFullPath($dataRoot) -cne $expectedRoot -or (Split-Path -Leaf $dataRoot) -cne 'LSDJ') { + throw "Refusing to test an unexpected data root: $dataRoot" +} +if (Test-Path -LiteralPath $dataRoot) { + throw "The isolated runner is not clean; refusing to overwrite existing LSDJ data at $dataRoot." +} + +$app = Join-Path $dataRoot 'lsdj-app.exe' +$uninstaller = Join-Path $dataRoot 'uninstall.exe' +$marker = Join-Path $dataRoot '.lsdj-data-root' +$startMenuShortcut = Join-Path $env:APPDATA 'Microsoft\Windows\Start Menu\Programs\LSDJ\LSDJ.lnk' +$registryKey = 'HKCU:\Software\Microsoft\Windows\CurrentVersion\Uninstall\LSDJ' +$ciPurgeReady = Join-Path $env:TEMP 'lsdj-ci-before-purge.ready' +$ciInstallerTrace = Join-Path $env:TEMP 'lsdj-ci-installer.trace' +$ownerMarkerWithNul = [byte[]]::new(20) +[Text.Encoding]::ASCII.GetBytes('works.protocol.lsdj').CopyTo($ownerMarkerWithNul, 0) +$ownerMarkerWithNulHex = [Convert]::ToHexString($ownerMarkerWithNul) + +function Get-CiInstallerTrace { + if (Test-Path -LiteralPath $ciInstallerTrace -PathType Leaf) { + return [System.IO.File]::ReadAllText($ciInstallerTrace) + } + return '' +} + +function Write-CiInstallerTrace { + Write-Host 'CI installer trace:' + if (Test-Path -LiteralPath $ciInstallerTrace -PathType Leaf) { + Get-Content -LiteralPath $ciInstallerTrace | ForEach-Object { + Write-Host " $_" + } + } else { + Write-Host ' ' + } +} + +function Assert-CiInstallerTraceContract { + param( + [string[]] $Required = @(), + + [string[]] $Forbidden = @() + ) + + # Signed release installers compile every trace macro to no instructions. + if ($RequireSigned) { + return + } + + $trace = Get-CiInstallerTrace + $lastRequiredIndex = -1 + foreach ($needle in $Required) { + $requiredIndex = $trace.IndexOf( + $needle, + $lastRequiredIndex + 1, + [StringComparison]::Ordinal + ) + if ($requiredIndex -lt 0) { + Write-CiInstallerTrace + throw "CI installer trace is missing or misorders required checkpoint: $needle" + } + $lastRequiredIndex = $requiredIndex + } + foreach ($needle in $Forbidden) { + if ($trace.IndexOf($needle, [StringComparison]::Ordinal) -ge 0) { + Write-CiInstallerTrace + throw "CI installer trace reached forbidden checkpoint: $needle" + } + } +} + +function Invoke-CheckedProcess { + param( + [Parameter(Mandatory = $true)] + [string] $FilePath, + + [string[]] $ArgumentList = @(), + + [int[]] $ExpectedExitCodes = @(0) + ) + + Remove-Item -LiteralPath $ciInstallerTrace -Force -ErrorAction SilentlyContinue + $process = Start-Process -FilePath $FilePath -ArgumentList $ArgumentList -Wait -PassThru + if ($process.ExitCode -notin $ExpectedExitCodes) { + $trace = Get-CiInstallerTrace + Write-CiInstallerTrace + throw "Process exited $($process.ExitCode), expected $($ExpectedExitCodes -join ', '): $FilePath $($ArgumentList -join ' ')`nCI installer trace:`n$trace" + } + return $process.ExitCode +} + +function Invoke-ExpectedFailure { + param( + [Parameter(Mandatory = $true)] + [string] $FilePath, + + [string[]] $ArgumentList = @() + ) + + Remove-Item -LiteralPath $ciInstallerTrace -Force -ErrorAction SilentlyContinue + $process = Start-Process -FilePath $FilePath -ArgumentList $ArgumentList -Wait -PassThru + if ($process.ExitCode -eq 0) { + $trace = Get-CiInstallerTrace + Write-CiInstallerTrace + throw "Process unexpectedly succeeded: $FilePath $($ArgumentList -join ' ')`nCI installer trace:`n$trace" + } + return $process.ExitCode +} + +function New-UninstallerWorkerCopy { + param( + [Parameter(Mandatory = $true)] + [string] $FilePath + ) + + $workerName = "lsdj-uninstall-worker-$([Guid]::NewGuid().ToString('N')).exe" + $workerPath = Join-Path $env:RUNNER_TEMP $workerName + try { + Copy-Item -LiteralPath $FilePath -Destination $workerPath + return $workerPath + } catch { + Remove-Item -LiteralPath $workerPath -Force -ErrorAction SilentlyContinue + throw + } +} + +function Stop-UninstallerWorker { + param( + [System.Diagnostics.Process] $Process, + + [int] $TimeoutMilliseconds = 10000 + ) + + if ($null -eq $Process -or $Process.HasExited) { + return + } + + $Process.Kill($true) + if (-not $Process.WaitForExit($TimeoutMilliseconds)) { + throw "Timed out terminating uninstaller worker process $($Process.Id)." + } +} + +function Invoke-ExpectedUninstallFailure { + param( + [Parameter(Mandatory = $true)] + [string] $FilePath, + + [Parameter(Mandatory = $true)] + [string] $InstallDirectory, + + [string[]] $ArgumentList = @() + ) + + # NSIS's installed uninstaller is only a self-copy launcher: its exit code + # reports whether the temporary worker started, not the worker's result. + # Exercise the documented worker form so CI observes the script exit code. + # https://nsis.sourceforge.io/Docs/AppendixD.html + # `_?=` must remain the final, unquoted command-line argument. + $workerPath = $null + $worker = $null + try { + $workerPath = New-UninstallerWorkerCopy -FilePath $FilePath + Remove-Item -LiteralPath $ciInstallerTrace -Force -ErrorAction SilentlyContinue + $worker = Start-Process ` + -FilePath $workerPath ` + -ArgumentList (@($ArgumentList) + "_?=$InstallDirectory") ` + -PassThru + if (-not $worker.WaitForExit(30000)) { + throw "Timed out waiting for uninstaller worker: $workerPath" + } + if ($worker.ExitCode -ne 2) { + $trace = Get-CiInstallerTrace + Write-CiInstallerTrace + throw "Uninstaller worker exited $($worker.ExitCode), expected 2: $workerPath$([Environment]::NewLine)CI installer trace:$([Environment]::NewLine)$trace" + } + return $worker.ExitCode + } finally { + try { + Stop-UninstallerWorker -Process $worker + } finally { + if ($null -ne $workerPath) { + Remove-Item -LiteralPath $workerPath -Force -ErrorAction SilentlyContinue + } + } + } +} + +function Require-InstalledVersion { + param([string] $Version) + + if (-not (Test-Path -LiteralPath $app -PathType Leaf)) { + throw "Installed app is missing: $app" + } + $actual = (Get-Item -LiteralPath $app).VersionInfo.ProductVersion + if ($actual -ne $Version) { + throw "Installed app version is $actual, expected $Version." + } + $registered = (Get-ItemProperty -LiteralPath $registryKey -Name DisplayVersion).DisplayVersion + if ($registered -ne $Version) { + throw "Registered app version is $registered, expected $Version." + } +} + +function Set-DisplayVersionEvidence { + param( + [Parameter(Mandatory = $true)] + [AllowEmptyString()] + [string] $Value + ) + + New-ItemProperty -LiteralPath $registryKey -Name DisplayVersion -Value $Value ` + -PropertyType String -Force | Out-Null + $actual = (Get-ItemProperty -LiteralPath $registryKey -Name DisplayVersion).DisplayVersion + $rawKey = Get-Item -LiteralPath $registryKey -ErrorAction Stop + $rawKind = $rawKey.GetValueKind('DisplayVersion') + $rawValue = $rawKey.GetValue( + 'DisplayVersion', + $null, + [Microsoft.Win32.RegistryValueOptions]::DoNotExpandEnvironmentNames + ) + if ($actual -cne $Value -or + $rawKind -ne [Microsoft.Win32.RegistryValueKind]::String -or + $rawValue -cne $Value) { + throw "Could not establish exact DisplayVersion test evidence: $Value" + } +} + +function Require-No-Workers { + # Runtime programs are independently declared by the MRT2 and SA3 service + # manifests and do not have one stable process name. The installer lifecycle + # never launches the app, so observing the shell is the authoritative leak. + $remaining = @(Get-Process -Name 'lsdj-app' -ErrorAction SilentlyContinue) + if ($remaining.Count -ne 0) { + throw "Installer lifecycle left LSDJ processes running: $($remaining.Name -join ', ')" + } +} + +function Get-UninstallRegistrySnapshot { + $key = Get-Item -LiteralPath $registryKey -ErrorAction Stop + $values = foreach ($name in @($key.GetValueNames() | Sort-Object)) { + $value = $key.GetValue( + $name, + $null, + [Microsoft.Win32.RegistryValueOptions]::DoNotExpandEnvironmentNames + ) + if ($value -is [byte[]]) { + $value = [Convert]::ToBase64String($value) + } elseif ($value -is [string[]]) { + $value = $value -join "`0" + } else { + $value = [string] $value + } + [ordered]@{ + Name = $name + Kind = [string] ($key.GetValueKind($name)) + Value = $value + } + } + return (ConvertTo-Json -InputObject @($values) -Compress) +} + +function Get-InstalledStateSnapshot { + foreach ($path in @($app, $uninstaller, $marker, $settingsSentinel, $modelSentinel)) { + if (-not (Test-Path -LiteralPath $path -PathType Leaf)) { + throw "Cannot snapshot missing installed state: $path" + } + } + + $markerEntry = Get-Item -LiteralPath $marker -Force -ErrorAction Stop + if (($markerEntry.Attributes -band [IO.FileAttributes]::ReparsePoint) -ne 0) { + throw "Cannot snapshot a reparse-point ownership marker: $marker" + } + $markerLinkTypeProperty = $markerEntry.PSObject.Properties['LinkType'] + $markerLinkType = if ($null -eq $markerLinkTypeProperty) { + '' + } else { + [string] $markerLinkTypeProperty.Value + } + $markerTargetProperty = $markerEntry.PSObject.Properties['Target'] + $markerTarget = if ($null -eq $markerTargetProperty -or $null -eq $markerTargetProperty.Value) { + '' + } elseif ($markerTargetProperty.Value -is [string[]]) { + $markerTargetProperty.Value -join "`0" + } else { + [string] $markerTargetProperty.Value + } + + return [ordered]@{ + App = (Get-FileHash -LiteralPath $app -Algorithm SHA256).Hash + Uninstaller = (Get-FileHash -LiteralPath $uninstaller -Algorithm SHA256).Hash + Marker = (Get-FileHash -LiteralPath $marker -Algorithm SHA256).Hash + MarkerAttributes = [int64] $markerEntry.Attributes + MarkerLinkType = $markerLinkType + MarkerTarget = $markerTarget + Settings = (Get-FileHash -LiteralPath $settingsSentinel -Algorithm SHA256).Hash + Model = (Get-FileHash -LiteralPath $modelSentinel -Algorithm SHA256).Hash + Registry = (Get-UninstallRegistrySnapshot) + } +} + +function Assert-InstalledStateSnapshotUnchanged { + param( + [Parameter(Mandatory = $true)] + [System.Collections.IDictionary] $Before + ) + + $after = Get-InstalledStateSnapshot + foreach ($name in $Before.Keys) { + if ($after[$name] -cne $Before[$name]) { + throw "Rejected installer changed installed state: $name" + } + } +} + +function Remove-ReparseDirectoryEntry { + param( + [Parameter(Mandatory = $true)] + [string] $Path + ) + + $entry = Get-Item -LiteralPath $Path -Force -ErrorAction Stop + if (($entry.Attributes -band [IO.FileAttributes]::ReparsePoint) -eq 0 -or -not $entry.PSIsContainer) { + throw "Refusing to unlink an entry that is not a directory reparse point: $Path" + } + [System.IO.Directory]::Delete($entry.FullName) +} + +function New-RecognizedLsdjLayout { + New-Item -ItemType Directory -Path $dataRoot -Force | Out-Null + foreach ($name in @('config', 'data', 'cache', 'assets', 'staging')) { + New-Item -ItemType Directory -Path (Join-Path $dataRoot $name) -Force | Out-Null + } +} + +function Start-LifecycleScenario { + param( + [Parameter(Mandatory = $true)] + [string] $Name + ) + + Write-Host "[Windows installer lifecycle] $Name" +} + +# An empty foreign root is still not evidence of ownership. This also proves +# the hook's hidden root probe runs before Tauri's path-creating SetOutPath: +# otherwise fresh and pre-existing empty roots would be indistinguishable. +Start-LifecycleScenario 'reject pre-existing empty data root' +New-Item -ItemType Directory -Path $dataRoot -Force | Out-Null +Invoke-ExpectedFailure $older.FullName @('/S') | Out-Null +if ((Test-Path -LiteralPath $marker) -or (Test-Path -LiteralPath $app)) { + throw 'Installer claimed a pre-existing empty LocalAppData root.' +} +Remove-Item -LiteralPath $dataRoot -Recurse -Force + +# A foreign pre-existing directory must never be claimed just because it has the +# expected basename. The failed installer must not add a marker or payload. +Start-LifecycleScenario 'reject foreign pre-existing data root' +New-Item -ItemType Directory -Path $dataRoot -Force | Out-Null +$foreignSentinel = Join-Path $dataRoot 'foreign-owner.txt' +[System.IO.File]::WriteAllText($foreignSentinel, 'not LSDJ') +Invoke-ExpectedFailure $older.FullName @('/S') | Out-Null +if (-not (Test-Path -LiteralPath $foreignSentinel -PathType Leaf) -or + (Test-Path -LiteralPath $marker) -or + (Test-Path -LiteralPath $app)) { + throw 'Installer claimed or modified a pre-existing foreign LocalAppData root.' +} +Remove-Item -LiteralPath $dataRoot -Recurse -Force + +# A root junction must be rejected before either its target or an ownership +# marker is touched. +Start-LifecycleScenario 'reject install-time data-root junction' +$rootJunctionTarget = Join-Path $env:RUNNER_TEMP 'lsdj-root-junction-target' +New-Item -ItemType Directory -Path $rootJunctionTarget -Force | Out-Null +$rootJunctionSentinel = Join-Path $rootJunctionTarget 'outside.txt' +[System.IO.File]::WriteAllText($rootJunctionSentinel, 'outside root') +New-Item -ItemType Junction -Path $dataRoot -Target $rootJunctionTarget | Out-Null +Invoke-ExpectedFailure $older.FullName @('/S') | Out-Null +if (-not (Test-Path -LiteralPath $rootJunctionSentinel -PathType Leaf) -or + (Test-Path -LiteralPath (Join-Path $rootJunctionTarget '.lsdj-data-root'))) { + throw 'Installer followed or marked a LocalAppData root junction.' +} +Remove-ReparseDirectoryEntry $dataRoot +Remove-Item -LiteralPath $rootJunctionTarget -Recurse -Force + +# Even an otherwise recognizable legacy layout is unsafe when its marker entry +# is a junction/reparse point. +Start-LifecycleScenario 'reject install-time ownership-marker junction' +New-RecognizedLsdjLayout +$installMarkerTarget = Join-Path $env:RUNNER_TEMP 'lsdj-install-marker-target' +New-Item -ItemType Directory -Path $installMarkerTarget -Force | Out-Null +$installMarkerSentinel = Join-Path $installMarkerTarget 'outside.txt' +[System.IO.File]::WriteAllText($installMarkerSentinel, 'outside marker') +New-Item -ItemType Junction -Path $marker -Target $installMarkerTarget | Out-Null +Invoke-ExpectedFailure $older.FullName @('/S') | Out-Null +if (-not (Test-Path -LiteralPath $installMarkerSentinel -PathType Leaf) -or + (Test-Path -LiteralPath $app)) { + throw 'Installer followed or accepted a marker reparse point.' +} +Remove-ReparseDirectoryEntry $marker +Remove-Item -LiteralPath $dataRoot -Recurse -Force +Remove-Item -LiteralPath $installMarkerTarget -Recurse -Force + +# A recognizable five-root shell is not safe to adopt if anything nested below +# it is a junction. Installation must not mark or write through the link. +Start-LifecycleScenario 'reject nested junction during legacy adoption' +New-RecognizedLsdjLayout +$installNestedTarget = Join-Path $env:RUNNER_TEMP 'lsdj-install-nested-target' +New-Item -ItemType Directory -Path $installNestedTarget -Force | Out-Null +$installNestedSentinel = Join-Path $installNestedTarget 'outside.txt' +[System.IO.File]::WriteAllText($installNestedSentinel, 'outside nested install') +$installNestedJunction = Join-Path $dataRoot 'assets\linked-outside' +New-Item -ItemType Junction -Path $installNestedJunction -Target $installNestedTarget | Out-Null +Invoke-ExpectedFailure $older.FullName @('/S') | Out-Null +if (-not (Test-Path -LiteralPath $installNestedSentinel -PathType Leaf) -or + (Test-Path -LiteralPath $marker) -or + (Test-Path -LiteralPath $app)) { + throw 'Installer adopted or followed a nested directory reparse point.' +} +Remove-ReparseDirectoryEntry $installNestedJunction +Remove-Item -LiteralPath $dataRoot -Recurse -Force +Remove-Item -LiteralPath $installNestedTarget -Recurse -Force + +# The native validator reads one byte beyond the exact owner identifier. This +# rejects an embedded trailing NUL even though string conversion alone could +# make that 20-byte file compare equal to the expected 19-character text. +Start-LifecycleScenario 'reject install with NUL-extended ownership marker' +New-RecognizedLsdjLayout +$nulInstallSentinel = Join-Path $dataRoot 'data\nul-marker-install.txt' +[System.IO.File]::WriteAllText($nulInstallSentinel, 'preserve NUL marker root') +[System.IO.File]::WriteAllBytes($marker, $ownerMarkerWithNul) +Invoke-ExpectedFailure $older.FullName @('/S') | Out-Null +$actualNulMarkerHex = [Convert]::ToHexString([System.IO.File]::ReadAllBytes($marker)) +if ($actualNulMarkerHex -cne $ownerMarkerWithNulHex -or + -not (Test-Path -LiteralPath $nulInstallSentinel -PathType Leaf) -or + (Test-Path -LiteralPath $app)) { + throw 'Installer accepted or modified a NUL-extended ownership marker root.' +} +Remove-Item -LiteralPath $dataRoot -Recurse -Force + +# The one markerless migration case is the complete five-root layout created by +# platform_paths.rs. It may be adopted, upgraded, and preserved normally. +Start-LifecycleScenario 'adopt recognized markerless legacy layout' +New-RecognizedLsdjLayout +$legacySentinel = Join-Path $dataRoot 'data\recognized-layout.txt' +[System.IO.File]::WriteAllText($legacySentinel, 'recognized LSDJ layout') +Invoke-CheckedProcess $older.FullName @('/S') +if (-not (Test-Path -LiteralPath $marker -PathType Leaf)) { + throw 'Installer did not establish ownership for a recognized LSDJ layout.' +} +Invoke-CheckedProcess $uninstaller @('/S') +if (-not (Test-Path -LiteralPath $legacySentinel -PathType Leaf)) { + throw 'Default uninstall did not preserve an adopted LSDJ layout.' +} +Remove-Item -LiteralPath $dataRoot -Recurse -Force + +# Initial per-user install: version metadata, Start menu integration, and the +# marker that scopes the optional destructive uninstall. +Start-LifecycleScenario 'fresh per-user install' +Invoke-CheckedProcess $older.FullName @('/S') +Require-InstalledVersion $OlderVersion +if (-not (Test-Path -LiteralPath $startMenuShortcut -PathType Leaf)) { + throw "Start menu shortcut is missing: $startMenuShortcut" +} +if (-not (Test-Path -LiteralPath $marker -PathType Leaf)) { + throw 'Installer did not create the data-root ownership marker.' +} + +$settingsSentinel = Join-Path $dataRoot 'data\用户 settings\preserve.txt' +$modelSentinel = Join-Path $dataRoot 'assets\models with spaces\模型.bin' +New-Item -ItemType Directory -Path (Split-Path -Parent $settingsSentinel) -Force | Out-Null +New-Item -ItemType Directory -Path (Split-Path -Parent $modelSentinel) -Force | Out-Null +[System.IO.File]::WriteAllText($settingsSentinel, 'preserve settings') +[System.IO.File]::WriteAllText($modelSentinel, 'preserve model') + +# Upgrade in place preserves app-managed data; the old signed/unsigned binary is +# replaced and registry metadata follows the newer calendar version. +Start-LifecycleScenario 'upgrade in place and preserve app-managed data' +Invoke-CheckedProcess $newer.FullName @('/S', '/UPDATE') +Assert-CiInstallerTraceContract -Required @( + "preinstall: installed version=$OlderVersion validity=1", + 'preinstall: version compare=1', + 'preinstall: ready', + 'postinstall: begin' +) -Forbidden @('abort:') +Require-InstalledVersion $NewerVersion +foreach ($sentinel in @($settingsSentinel, $modelSentinel)) { + if (-not (Test-Path -LiteralPath $sentinel -PathType Leaf)) { + throw "Upgrade removed preserved data: $sentinel" + } +} +Require-No-Workers + +$registrySentinelName = 'LsdjCiPreserve' +$registrySentinelValue = 'preserve registry metadata' +New-ItemProperty -LiteralPath $registryKey -Name $registrySentinelName ` + -Value $registrySentinelValue -PropertyType String -Force | Out-Null +$registryBaseline = Get-ItemProperty -LiteralPath $registryKey + +function Require-RejectedInstallPreservedState { + param( + [AllowEmptyString()] + [string] $ExpectedDisplayVersion = $NewerVersion, + + [switch] $DisplayVersionMissing + ) + + if (-not (Test-Path -LiteralPath $app -PathType Leaf)) { + throw "Rejected installer removed the installed app: $app" + } + $actualAppVersion = (Get-Item -LiteralPath $app).VersionInfo.ProductVersion + if ($actualAppVersion -ne $NewerVersion) { + throw "Rejected installer changed app version to $actualAppVersion." + } + + $registered = Get-ItemProperty -LiteralPath $registryKey -ErrorAction Stop + $displayVersionProperty = $registered.PSObject.Properties['DisplayVersion'] + if ($DisplayVersionMissing) { + if ($null -ne $displayVersionProperty) { + throw 'Rejected installer recreated missing DisplayVersion metadata.' + } + } elseif ($null -eq $displayVersionProperty -or + $displayVersionProperty.Value -cne $ExpectedDisplayVersion) { + throw "Rejected installer changed DisplayVersion metadata." + } + + foreach ($name in @('DisplayName', 'InstallLocation', 'UninstallString', $registrySentinelName)) { + if ($registered.$name -cne $registryBaseline.$name) { + throw "Rejected installer changed registry metadata: $name" + } + } + foreach ($sentinel in @($settingsSentinel, $modelSentinel)) { + if (-not (Test-Path -LiteralPath $sentinel -PathType Leaf)) { + throw "Rejected installer modified app-managed data: $sentinel" + } + } + Require-No-Workers +} + +$forbiddenPostVersionGuardTrace = @( + 'preinstall: begin', + 'preinstall: canonical', + 'preinstall: ready', + 'postinstall:' +) + +# A same-version unattended reinstall remains supported. Besides exercising the +# success path, its trace directly proves the pinned plugin reports valid=>1. +Start-LifecycleScenario 'same-version silent reinstall' +Invoke-CheckedProcess $newer.FullName @('/S') +Assert-CiInstallerTraceContract -Required @( + "preinstall: installed version=$NewerVersion validity=1", + 'preinstall: version compare=0', + 'preinstall: ready', + 'postinstall: begin' +) -Forbidden @('abort:') +Require-InstalledVersion $NewerVersion +foreach ($sentinel in @($settingsSentinel, $modelSentinel)) { + if (-not (Test-Path -LiteralPath $sentinel -PathType Leaf)) { + throw "Same-version reinstall modified app-managed data: $sentinel" + } +} +Require-No-Workers + +# allowDowngrades=false must reject unattended rollback and leave the newer app. +Start-LifecycleScenario 'reject unattended downgrade' +$rejectedStateBefore = Get-InstalledStateSnapshot +Invoke-CheckedProcess -FilePath $older.FullName -ArgumentList @('/S') -ExpectedExitCodes @(2) | Out-Null +Assert-CiInstallerTraceContract -Required @( + "preinstall: installed version=$NewerVersion validity=1", + 'preinstall: version compare=-1', + 'abort: unattended downgrade' +) -Forbidden $forbiddenPostVersionGuardTrace +Require-RejectedInstallPreservedState +Assert-InstalledStateSnapshotUnchanged -Before $rejectedStateBefore + +# Explicit update mode must not bypass the unattended downgrade guard. +Start-LifecycleScenario 'reject unattended downgrade in update mode' +$rejectedStateBefore = Get-InstalledStateSnapshot +Invoke-CheckedProcess -FilePath $older.FullName -ArgumentList @('/S', '/UPDATE') ` + -ExpectedExitCodes @(2) | Out-Null +Assert-CiInstallerTraceContract -Required @( + "preinstall: installed version=$NewerVersion validity=1", + 'preinstall: version compare=-1', + 'abort: unattended downgrade' +) -Forbidden $forbiddenPostVersionGuardTrace +Require-RejectedInstallPreservedState +Assert-InstalledStateSnapshotUnchanged -Before $rejectedStateBefore + +# Passive mode is unattended too, despite not setting NSIS's silent flag. +Start-LifecycleScenario 'reject passive downgrade' +$passiveStateBefore = Get-InstalledStateSnapshot +Invoke-CheckedProcess -FilePath $older.FullName -ArgumentList @('/P') -ExpectedExitCodes @(2) | Out-Null +Assert-CiInstallerTraceContract -Required @( + 'abort: passive install unsupported' +) -Forbidden @('probe:', 'preinstall:', 'postinstall:') +Require-RejectedInstallPreservedState +Assert-InstalledStateSnapshotUnchanged -Before $passiveStateBefore + +# NSIS silent mode suppresses GUI initialization, so a combined /S /P reaches +# the repeated PREINSTALL policy check only after the read-only ownership probe. +Start-LifecycleScenario 'reject combined silent and passive mode' +$combinedPassiveStateBefore = Get-InstalledStateSnapshot +Invoke-CheckedProcess -FilePath $older.FullName -ArgumentList @('/S', '/P') ` + -ExpectedExitCodes @(2) | Out-Null +Assert-CiInstallerTraceContract -Required @( + 'abort: passive install unsupported' +) -Forbidden @('preinstall: begin', 'preinstall: ready', 'postinstall:') +Require-RejectedInstallPreservedState +Assert-InstalledStateSnapshotUnchanged -Before $combinedPassiveStateBefore + +# Existing install evidence with empty, malformed, or missing version metadata +# is unsafe. Each refusal must preserve the exact damaged evidence for repair. +Start-LifecycleScenario 'reject existing install with empty version metadata' +Set-DisplayVersionEvidence -Value '' +$rejectedStateBefore = Get-InstalledStateSnapshot +Invoke-CheckedProcess -FilePath $older.FullName -ArgumentList @('/S') -ExpectedExitCodes @(2) | Out-Null +Assert-CiInstallerTraceContract -Required @( + 'abort: installed version missing' +) -Forbidden $forbiddenPostVersionGuardTrace +Require-RejectedInstallPreservedState -ExpectedDisplayVersion '' +Assert-InstalledStateSnapshotUnchanged -Before $rejectedStateBefore + +Start-LifecycleScenario 'reject existing install with corrupt version metadata' +$corruptDisplayVersion = 'not-a-semver' +Set-DisplayVersionEvidence -Value $corruptDisplayVersion +$rejectedStateBefore = Get-InstalledStateSnapshot +Invoke-CheckedProcess -FilePath $older.FullName -ArgumentList @('/S') -ExpectedExitCodes @(2) | Out-Null +Assert-CiInstallerTraceContract -Required @( + "preinstall: version evidence installed=$corruptDisplayVersion present=1 existing=1", + "preinstall: installed version=$corruptDisplayVersion validity=0", + 'abort: installed version invalid' +) -Forbidden @( + 'preinstall: version compare=', + 'preinstall: begin', + 'preinstall: canonical', + 'preinstall: ready', + 'postinstall:' +) +Require-RejectedInstallPreservedState -ExpectedDisplayVersion $corruptDisplayVersion +Assert-InstalledStateSnapshotUnchanged -Before $rejectedStateBefore + +Start-LifecycleScenario 'reject existing install with missing version metadata' +Remove-ItemProperty -LiteralPath $registryKey -Name DisplayVersion +$rejectedStateBefore = Get-InstalledStateSnapshot +Invoke-CheckedProcess -FilePath $older.FullName -ArgumentList @('/S') -ExpectedExitCodes @(2) | Out-Null +Assert-CiInstallerTraceContract -Required @( + 'abort: installed version missing' +) -Forbidden $forbiddenPostVersionGuardTrace +Require-RejectedInstallPreservedState -DisplayVersionMissing +Assert-InstalledStateSnapshotUnchanged -Before $rejectedStateBefore +Set-DisplayVersionEvidence -Value $NewerVersion +Require-InstalledVersion $NewerVersion + +if ($RequireSigned) { + $installedPayloads = @( + Get-ChildItem -LiteralPath $dataRoot -Recurse -File | + Where-Object { $_.Extension -in @('.exe', '.dll') } | + ForEach-Object FullName + ) + if ($installedPayloads.Count -eq 0) { + throw 'The installed release contains no executable payloads to verify.' + } + & "$PSScriptRoot/verify-windows-signatures.ps1" -Path $installedPayloads +} + +# The default uninstall removes application binaries and shortcuts but preserves +# every app-owned runtime, model, setting, and user-data file. +Start-LifecycleScenario 'default uninstall preserves app-managed data' +Invoke-CheckedProcess $uninstaller @('/S') +Start-Sleep -Milliseconds 500 +Require-No-Workers +if (Test-Path -LiteralPath $app) { + throw 'Default uninstall left the application binary behind.' +} +if (Test-Path -LiteralPath $startMenuShortcut) { + throw 'Default uninstall left the Start menu shortcut behind.' +} +foreach ($sentinel in @($settingsSentinel, $modelSentinel)) { + if (-not (Test-Path -LiteralPath $sentinel -PathType Leaf)) { + throw "Default uninstall removed user data: $sentinel" + } +} + +# An invalid marker must make explicit automation fail closed while preserving +# the exact root. Restore the installer-owned marker only after proving refusal. +Start-LifecycleScenario 'reject purge with invalid ownership marker' +Invoke-CheckedProcess $newer.FullName @('/S') +[System.IO.File]::WriteAllText($marker, 'foreign-owner') +Invoke-ExpectedUninstallFailure ` + -FilePath $uninstaller ` + -InstallDirectory $dataRoot ` + -ArgumentList @('/S', '/PURGE-LSDJ-DATA') | Out-Null +if (-not (Test-Path -LiteralPath $dataRoot -PathType Container)) { + throw 'Invalid ownership marker allowed explicit data removal.' +} +[System.IO.File]::WriteAllText($marker, 'works.protocol.lsdj') + +# A trailing raw NUL must also invalidate destructive ownership validation. +# The failed purge must stop before ordinary app removal and preserve the exact +# marker bytes and data root. +Start-LifecycleScenario 'reject purge with NUL-extended ownership marker' +Invoke-CheckedProcess $newer.FullName @('/S') +[System.IO.File]::WriteAllBytes($marker, $ownerMarkerWithNul) +Invoke-ExpectedUninstallFailure ` + -FilePath $uninstaller ` + -InstallDirectory $dataRoot ` + -ArgumentList @('/S', '/PURGE-LSDJ-DATA') | Out-Null +$actualNulMarkerHex = [Convert]::ToHexString([System.IO.File]::ReadAllBytes($marker)) +if ($actualNulMarkerHex -cne $ownerMarkerWithNulHex -or + -not (Test-Path -LiteralPath $dataRoot -PathType Container) -or + -not (Test-Path -LiteralPath $app -PathType Leaf)) { + throw 'NUL-extended ownership marker allowed uninstall or data removal.' +} +[System.IO.File]::WriteAllText($marker, 'works.protocol.lsdj') + +# Replace the entire owned root with a junction immediately before explicit +# purge. The purge-time root junction check must stop before even Tauri's narrow +# payload deletion, and +# the outside target must remain byte-for-byte untouched. +Start-LifecycleScenario 'reject purge after data-root junction replacement' +Invoke-CheckedProcess $newer.FullName @('/S') +$parkedRoot = Join-Path $env:RUNNER_TEMP 'lsdj-owned-root-parked' +Move-Item -LiteralPath $dataRoot -Destination $parkedRoot +$purgeRootTarget = Join-Path $env:RUNNER_TEMP 'lsdj-purge-root-target' +New-Item -ItemType Directory -Path $purgeRootTarget -Force | Out-Null +$purgeRootMarker = Join-Path $purgeRootTarget '.lsdj-data-root' +$purgeRootSentinel = Join-Path $purgeRootTarget 'lsdj-app.exe' +[System.IO.File]::WriteAllText($purgeRootMarker, 'works.protocol.lsdj') +[System.IO.File]::WriteAllText($purgeRootSentinel, 'outside root payload') +New-Item -ItemType Junction -Path $dataRoot -Target $purgeRootTarget | Out-Null +$parkedUninstaller = Join-Path $parkedRoot 'uninstall.exe' +Invoke-ExpectedUninstallFailure ` + -FilePath $parkedUninstaller ` + -InstallDirectory $parkedRoot ` + -ArgumentList @('/S', '/PURGE-LSDJ-DATA') | Out-Null +if (([System.IO.File]::ReadAllText($purgeRootSentinel)) -ne 'outside root payload' -or + -not (Test-Path -LiteralPath $parkedUninstaller -PathType Leaf)) { + throw 'Purge-time root junction was followed or ordinary uninstall continued after refusal.' +} +Remove-ReparseDirectoryEntry $dataRoot +Remove-Item -LiteralPath $purgeRootTarget -Recurse -Force +Move-Item -LiteralPath $parkedRoot -Destination $dataRoot + +# A marker junction is rejected both as ownership evidence and as a tree entry; +# its outside target must remain untouched. +Start-LifecycleScenario 'reject purge with ownership-marker junction' +Invoke-CheckedProcess $newer.FullName @('/S') +[System.IO.File]::Delete($marker) +$purgeMarkerTarget = Join-Path $env:RUNNER_TEMP 'lsdj-purge-marker-target' +New-Item -ItemType Directory -Path $purgeMarkerTarget -Force | Out-Null +$purgeMarkerSentinel = Join-Path $purgeMarkerTarget 'outside.txt' +[System.IO.File]::WriteAllText($purgeMarkerSentinel, 'outside purge marker') +New-Item -ItemType Junction -Path $marker -Target $purgeMarkerTarget | Out-Null +Invoke-ExpectedUninstallFailure ` + -FilePath $uninstaller ` + -InstallDirectory $dataRoot ` + -ArgumentList @('/S', '/PURGE-LSDJ-DATA') | Out-Null +if (-not (Test-Path -LiteralPath $purgeMarkerSentinel -PathType Leaf) -or + -not (Test-Path -LiteralPath $dataRoot -PathType Container)) { + throw 'Explicit purge followed or removed a marker reparse point.' +} +Remove-ReparseDirectoryEntry $marker +[System.IO.File]::WriteAllText($marker, 'works.protocol.lsdj') +Remove-Item -LiteralPath $purgeMarkerTarget -Recurse -Force + +# Unsigned CI installers pause after the initial ownership/size decision and +# core binary removal. Replace the marker during that window; the immediate +# destructive revalidation must detect the change and preserve the root. +Start-LifecycleScenario 'reject ownership-marker replacement after purge confirmation' +Invoke-CheckedProcess $newer.FullName @('/S') +Remove-Item -LiteralPath $ciPurgeReady -Force -ErrorAction SilentlyContinue +$racedPurgeWorker = $null +$racedPurge = $null +try { + $racedPurgeWorker = New-UninstallerWorkerCopy -FilePath $uninstaller + $racedPurge = Start-Process -FilePath $racedPurgeWorker ` + -ArgumentList @( + '/S', + '/PURGE-LSDJ-DATA', + '/LSDJ-CI-PAUSE-BEFORE-PURGE', + "_?=$dataRoot" + ) ` + -PassThru + $raceDeadline = [DateTime]::UtcNow.AddSeconds(20) + while (-not (Test-Path -LiteralPath $ciPurgeReady -PathType Leaf) -and + -not $racedPurge.HasExited) { + if ([DateTime]::UtcNow -ge $raceDeadline) { + throw 'Timed out waiting for the CI purge synchronization point.' + } + Start-Sleep -Milliseconds 100 + } + if ($racedPurge.HasExited) { + throw "Purge exited before the marker-replacement test (exit $($racedPurge.ExitCode))." + } + [System.IO.File]::WriteAllText($marker, 'replaced-after-confirmation') + if (-not $racedPurge.WaitForExit(30000)) { + throw 'Timed out waiting for the refused marker-replacement purge to exit.' + } + if ($racedPurge.ExitCode -ne 2 -or + -not (Test-Path -LiteralPath $dataRoot -PathType Container)) { + throw 'Marker replacement after confirmation did not fail closed.' + } +} finally { + try { + Stop-UninstallerWorker -Process $racedPurge + } finally { + try { + if ($null -ne $racedPurgeWorker) { + Remove-Item -LiteralPath $racedPurgeWorker -Force -ErrorAction SilentlyContinue + } + } finally { + Remove-Item -LiteralPath $ciPurgeReady -Force -ErrorAction SilentlyContinue + } + } +} +[System.IO.File]::WriteAllText($marker, 'works.protocol.lsdj') + +# Nested junctions are never traversed for size or removal. Purge refuses the +# tree and leaves both the root and outside target intact. +Start-LifecycleScenario 'reject purge with nested junction' +Invoke-CheckedProcess $newer.FullName @('/S') +$nestedTarget = Join-Path $env:RUNNER_TEMP 'lsdj-nested-junction-target' +New-Item -ItemType Directory -Path $nestedTarget -Force | Out-Null +$nestedSentinel = Join-Path $nestedTarget 'outside.txt' +[System.IO.File]::WriteAllText($nestedSentinel, 'outside nested junction') +$nestedJunction = Join-Path $dataRoot 'data\linked-outside' +New-Item -ItemType Junction -Path $nestedJunction -Target $nestedTarget | Out-Null +Invoke-ExpectedUninstallFailure ` + -FilePath $uninstaller ` + -InstallDirectory $dataRoot ` + -ArgumentList @('/S', '/PURGE-LSDJ-DATA') | Out-Null +if (-not (Test-Path -LiteralPath $nestedSentinel -PathType Leaf) -or + -not (Test-Path -LiteralPath $dataRoot -PathType Container)) { + throw 'Explicit purge traversed a nested directory reparse point.' +} +Remove-ReparseDirectoryEntry $nestedJunction +Remove-Item -LiteralPath $nestedTarget -Recurse -Force + +# Explicit automation opt-in mirrors the GUI checkbox + path/size confirmation. +Start-LifecycleScenario 'explicit purge removes owned data root' +Invoke-CheckedProcess $newer.FullName @('/S') +Invoke-CheckedProcess $uninstaller @('/S', '/PURGE-LSDJ-DATA') +Start-Sleep -Milliseconds 500 +if (Test-Path -LiteralPath $dataRoot) { + throw "Explicit data removal did not remove $dataRoot." +} +Require-No-Workers + +# A non-default install location with spaces, Unicode, and a long (but pre-MAX_PATH) +# directory proves package resources do not depend on Windows long-path support. +Start-LifecycleScenario 'custom install path with spaces Unicode and long leaf' +$longLeaf = ('path segment ' * 10).Trim() +$unicodeInstall = Join-Path $env:RUNNER_TEMP "LSDJ installer 路径 $longLeaf" +if ($unicodeInstall.Length -ge 240) { + throw "The long-path-disabled smoke target exceeded its conservative budget: $($unicodeInstall.Length)" +} +Invoke-CheckedProcess $newer.FullName @('/S', "/D=$unicodeInstall") +$unicodeApp = Join-Path $unicodeInstall 'lsdj-app.exe' +$unicodeUninstaller = Join-Path $unicodeInstall 'uninstall.exe' +if (-not (Test-Path -LiteralPath $unicodeApp -PathType Leaf)) { + throw "Unicode/space install did not produce the app at $unicodeApp." +} + +# Exercise NSIS's documented worker form with a final, unquoted `_?=` value +# whose remainder contains spaces and Unicode. The unsafe purge must expose +# exact worker exit 2 and preserve both the custom app and owned data root. +Start-LifecycleScenario 'reject purge worker with spaces and Unicode install path' +[System.IO.File]::WriteAllText($marker, 'foreign-owner') +Invoke-ExpectedUninstallFailure ` + -FilePath $unicodeUninstaller ` + -InstallDirectory $unicodeInstall ` + -ArgumentList @('/S', '/PURGE-LSDJ-DATA') | Out-Null +if (-not (Test-Path -LiteralPath $unicodeApp -PathType Leaf) -or + -not (Test-Path -LiteralPath $dataRoot -PathType Container)) { + throw 'Unicode/space worker invocation allowed uninstall or data removal.' +} +[System.IO.File]::WriteAllText($marker, 'works.protocol.lsdj') + +Invoke-CheckedProcess $unicodeUninstaller @('/S') +Start-Sleep -Milliseconds 500 +if (Test-Path -LiteralPath $unicodeApp) { + throw 'Unicode/space uninstall left the app binary behind.' +} + +# The custom-location uninstall intentionally retains its remembered location. +# Override it explicitly so the final purge cleans the isolated runner's normal +# application/data root as well as the remembered-location registry state. +Start-LifecycleScenario 'final explicit cleanup after custom install location' +Invoke-CheckedProcess $newer.FullName @('/S', "/D=$dataRoot") +Invoke-CheckedProcess $uninstaller @('/S', '/PURGE-LSDJ-DATA') +Start-Sleep -Milliseconds 500 +if (Test-Path -LiteralPath $dataRoot) { + throw 'Final explicit cleanup did not remove the LSDJ data root.' +} +Require-No-Workers +Write-Host 'Windows NSIS install/upgrade/downgrade/uninstall lifecycle passed.' diff --git a/scripts/tests/test_release_artifact.py b/scripts/tests/test_release_artifact.py new file mode 100644 index 0000000..fbd1d7f --- /dev/null +++ b/scripts/tests/test_release_artifact.py @@ -0,0 +1,431 @@ +import importlib.util +import json +import re +import sys +import tempfile +import unittest +from pathlib import Path + + +MODULE_PATH = Path(__file__).parents[1] / "release_artifact.py" +REPO_ROOT = Path(__file__).parents[2] +SPEC = importlib.util.spec_from_file_location("release_artifact", MODULE_PATH) +assert SPEC is not None and SPEC.loader is not None +release_artifact = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = release_artifact +SPEC.loader.exec_module(release_artifact) + +REVISION = "a" * 40 +TAG = "v2026.08.7" +REQUIRED_PRODUCERS = ["macos-arm64", "linux-x64", "windows-x64"] + + +class ReleaseArtifactTest(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.root = Path(self.temporary.name) + self.macos_asset = self.root / "LSDJ_2026.08.7_aarch64.dmg" + self.macos_asset.write_bytes(b"verified dmg bytes") + self.linux_asset = self.root / "LSDJ_2026.08.7_amd64.AppImage" + self.linux_asset.write_bytes(b"verified appimage bytes") + self.windows_asset = self.root / "LSDJ_2026.08.7_x64-setup.exe" + self.windows_asset.write_bytes(b"verified signed nsis bytes") + + def tearDown(self): + self.temporary.cleanup() + + def create_bundle(self, producer="macos-arm64"): + asset = { + "macos-arm64": self.macos_asset, + "linux-x64": self.linux_asset, + "windows-x64": self.windows_asset, + }[producer] + bundle = self.root / "incoming" / producer + release_artifact.create_bundle( + producer=producer, + release_tag=TAG, + revision=REVISION, + assets=[asset], + output_dir=bundle, + ) + return bundle + + def create_all_bundles(self): + self.create_bundle("macos-arm64") + self.create_bundle("linux-x64") + self.create_bundle("windows-x64") + return self.root / "incoming" + + def draft_release(self, assets, **updates): + data = { + "id": 12345, + "tag_name": TAG, + "target_commitish": REVISION, + "draft": True, + "body": f"Source revision: {REVISION}\n\nGenerated notes", + "assets": assets, + } + data.update(updates) + return data + + def test_create_and_verify_bundle(self): + incoming = self.create_all_bundles() + output = self.root / "verified" + + release_artifact.verify_bundles( + input_root=incoming, + required_producers=REQUIRED_PRODUCERS, + release_tag=TAG, + revision=REVISION, + output_dir=output, + ) + + self.assertEqual( + {path.name for path in output.iterdir()}, + { + self.macos_asset.name, + self.linux_asset.name, + self.windows_asset.name, + "macos-arm64-release-metadata.json", + "macos-arm64-SHA256SUMS.txt", + "linux-x64-release-metadata.json", + "linux-x64-SHA256SUMS.txt", + "windows-x64-release-metadata.json", + "windows-x64-SHA256SUMS.txt", + "release-index.json", + }, + ) + index = json.loads((output / "release-index.json").read_text()) + self.assertEqual(index["release_tag"], TAG) + self.assertEqual(index["revision"], REVISION) + self.assertEqual( + index["producers"], ["linux-x64", "macos-arm64", "windows-x64"] + ) + + def test_tampered_asset_fails_closed(self): + incoming = self.create_all_bundles() + (incoming / "macos-arm64" / self.macos_asset.name).write_bytes(b"tampered") + + with self.assertRaisesRegex(release_artifact.ArtifactError, "size|checksum"): + release_artifact.verify_bundles( + input_root=incoming, + required_producers=REQUIRED_PRODUCERS, + release_tag=TAG, + revision=REVISION, + output_dir=self.root / "verified", + ) + + def test_empty_installer_fails_closed(self): + self.macos_asset.write_bytes(b"") + + with self.assertRaisesRegex( + release_artifact.ArtifactError, "must not be empty" + ): + self.create_bundle() + + def test_host_specific_or_ambiguous_asset_names_fail_closed(self): + for filename in ("LSDJ\\setup.dmg", "LSDJ\nsetup.dmg"): + with self.subTest(filename=filename): + with self.assertRaisesRegex( + release_artifact.ArtifactError, "non-portable" + ): + release_artifact.portable_filename_key(filename) + + def test_missing_required_producer_fails_closed(self): + incoming = self.root / "incoming" + incoming.mkdir() + + with self.assertRaisesRegex(release_artifact.ArtifactError, "producer set"): + release_artifact.verify_bundles( + input_root=incoming, + required_producers=REQUIRED_PRODUCERS, + release_tag=TAG, + revision=REVISION, + output_dir=self.root / "verified", + ) + + def test_unexpected_bundle_file_fails_closed(self): + incoming = self.create_all_bundles() + (incoming / "macos-arm64" / "surprise.txt").write_text("not declared") + + with self.assertRaisesRegex(release_artifact.ArtifactError, "unexpected"): + release_artifact.verify_bundles( + input_root=incoming, + required_producers=REQUIRED_PRODUCERS, + release_tag=TAG, + revision=REVISION, + output_dir=self.root / "verified", + ) + + def test_wrong_release_identity_fails_closed(self): + incoming = self.create_all_bundles() + + with self.assertRaisesRegex(release_artifact.ArtifactError, "release_tag"): + release_artifact.verify_bundles( + input_root=incoming, + required_producers=REQUIRED_PRODUCERS, + release_tag="v2026.08.8", + revision=REVISION, + output_dir=self.root / "verified", + ) + + def test_required_producer_arguments_must_exactly_match_policy(self): + incoming = self.create_all_bundles() + with self.assertRaisesRegex(release_artifact.ArtifactError, "release policy"): + release_artifact.verify_bundles( + input_root=incoming, + required_producers=["macos-arm64"], + release_tag=TAG, + revision=REVISION, + output_dir=self.root / "verified", + ) + + def test_policy_requires_exact_three_platform_producer_set(self): + self.assertEqual( + set(release_artifact.PRODUCER_POLICIES), set(REQUIRED_PRODUCERS) + ) + + def test_draft_release_assets_must_match_exactly(self): + verified = self.root / "verified" + verified.mkdir() + (verified / "asset.dmg").write_bytes(b"one") + response = self.root / "release.json" + response.write_text( + json.dumps( + self.draft_release( + [ + { + "name": "asset.dmg", + "size": 3, + "state": "uploaded", + "digest": "sha256:" + + release_artifact.sha256(verified / "asset.dmg"), + } + ] + ) + ) + ) + + release_artifact.verify_github_release( + release_json=response, + verified_dir=verified, + release_tag=TAG, + revision=REVISION, + expected_release_id=12345, + ) + + data = json.loads(response.read_text()) + data["assets"][0]["size"] = 4 + response.write_text(json.dumps(data)) + with self.assertRaisesRegex(release_artifact.ArtifactError, "exactly match"): + release_artifact.verify_github_release( + release_json=response, + verified_dir=verified, + release_tag=TAG, + revision=REVISION, + expected_release_id=12345, + ) + + def test_github_digest_is_required(self): + verified = self.root / "verified" + verified.mkdir() + asset = verified / "asset.dmg" + asset.write_bytes(b"one") + response = self.root / "release.json" + + for digest_entry in ({}, {"digest": None}): + with self.subTest(digest_entry=digest_entry): + response.write_text( + json.dumps( + self.draft_release( + [ + { + "name": asset.name, + "size": asset.stat().st_size, + "state": "uploaded", + **digest_entry, + } + ] + ) + ) + ) + with self.assertRaisesRegex(release_artifact.ArtifactError, "digest"): + release_artifact.verify_github_release( + release_json=response, + verified_dir=verified, + release_tag=TAG, + revision=REVISION, + expected_release_id=12345, + ) + + def test_case_colliding_release_names_fail_closed(self): + with self.assertRaisesRegex(release_artifact.ArtifactError, "case-colliding"): + release_artifact.require_unique_portable_names( + ["LSDJ.dmg", "lsdj.DMG"], "release filename" + ) + + def test_github_digest_is_verified_when_present(self): + verified = self.root / "verified" + verified.mkdir() + asset = verified / "asset.dmg" + asset.write_bytes(b"one") + response = self.root / "release.json" + response.write_text( + json.dumps( + self.draft_release( + [ + { + "name": asset.name, + "size": asset.stat().st_size, + "state": "uploaded", + "digest": "sha256:" + "0" * 64, + } + ] + ) + ) + ) + + with self.assertRaisesRegex(release_artifact.ArtifactError, "digest"): + release_artifact.verify_github_release( + release_json=response, + verified_dir=verified, + release_tag=TAG, + revision=REVISION, + expected_release_id=12345, + ) + + def test_draft_identity_cannot_be_redirected_to_a_replacement(self): + replacement = self.draft_release([], id=67890) + + with self.assertRaisesRegex(release_artifact.ArtifactError, "Release ID"): + release_artifact.require_draft_release_identity( + data=replacement, + release_tag=TAG, + revision=REVISION, + expected_release_id=12345, + ) + + def test_draft_identity_requires_the_exact_source_revision(self): + for updates in ( + {"target_commitish": "b" * 40}, + {"body": "Source revision: " + "b" * 40}, + ): + with self.subTest(updates=updates): + with self.assertRaisesRegex( + release_artifact.ArtifactError, "source revision" + ): + release_artifact.require_draft_release_identity( + data=self.draft_release([], **updates), + release_tag=TAG, + revision=REVISION, + expected_release_id=12345, + ) + + def test_public_release_is_never_accepted_for_pre_publish_verification(self): + verified = self.root / "verified" + verified.mkdir() + response = self.root / "release.json" + response.write_text(json.dumps({"tag_name": TAG, "draft": False, "assets": []})) + + with self.assertRaisesRegex(release_artifact.ArtifactError, "remain a draft"): + release_artifact.verify_github_release( + release_json=response, + verified_dir=verified, + release_tag=TAG, + revision=REVISION, + expected_release_id=12345, + ) + + +class WorkflowContractTest(unittest.TestCase): + def test_release_workflow_keeps_one_least_privilege_publisher(self): + workflow = (REPO_ROOT / ".github/workflows/macos-release.yml").read_text() + + self.assertEqual(workflow.count("contents: write"), 1) + self.assertEqual(len(re.findall(r"^ publish:$", workflow, re.MULTILINE)), 1) + self.assertIn("needs.produce_macos.result == 'success'", workflow) + self.assertIn("needs.produce_linux.result == 'success'", workflow) + self.assertIn("needs.produce_windows.result == 'success'", workflow) + self.assertIn("--required-producer macos-arm64", workflow) + self.assertIn("--required-producer linux-x64", workflow) + self.assertIn("--required-producer windows-x64", workflow) + self.assertIn("runs-on: ubuntu-22.04", workflow) + self.assertEqual(workflow.count("environment:\n name: windows-release"), 1) + self.assertRegex(workflow, r"(?m)^on:\n push:\n tags:$") + self.assertNotIn("pull_request:", workflow) + self.assertNotIn("workflow_dispatch:", workflow) + + def test_exact_three_producers_feed_the_publisher(self): + workflow = (REPO_ROOT / ".github/workflows/macos-release.yml").read_text() + expected_jobs = {"produce_macos", "produce_linux", "produce_windows"} + expected_artifacts = { + "release-macos-arm64", + "release-linux-x64", + "release-windows-x64", + } + expected_producers = {"macos-arm64", "linux-x64", "windows-x64"} + + producer_jobs = set(re.findall(r"(?m)^ (produce_[a-z]+):$", workflow)) + self.assertEqual(producer_jobs, expected_jobs) + + publisher = workflow[workflow.index(" publish:") :] + publisher_needs = set(re.findall(r"(?m)^ - (produce_[a-z]+)$", publisher)) + required_results = set( + re.findall(r"needs\.(produce_[a-z]+)\.result == 'success'", publisher) + ) + downloaded_artifacts = set( + re.findall(r"(?m)^ name: (release-[a-z0-9-]+)$", publisher) + ) + required_producers = set( + re.findall(r"--required-producer ([a-z0-9-]+)", publisher) + ) + + self.assertEqual(publisher_needs, expected_jobs) + self.assertEqual(required_results, expected_jobs) + self.assertEqual(downloaded_artifacts, expected_artifacts) + self.assertEqual(required_producers, expected_producers) + + def test_release_is_verified_before_the_draft_becomes_public(self): + workflow = (REPO_ROOT / ".github/workflows/macos-release.yml").read_text() + + create = workflow.index("CREATE_RESPONSE=") + verify = workflow.index("verify-github-release") + publish = workflow.index("--method PATCH") + self.assertLess(create, verify) + self.assertLess(verify, publish) + + def test_failed_draft_cleanup_is_bound_to_the_created_release_id(self): + workflow = (REPO_ROOT / ".github/workflows/macos-release.yml").read_text() + cleanup = workflow[ + workflow.index("cleanup_draft()") : workflow.index("trap cleanup_draft") + ] + + self.assertIn("releases/$DRAFT_RELEASE_ID", cleanup) + self.assertIn("--expected-release-id", cleanup) + self.assertIn("verify-draft-identity", cleanup) + self.assertIn("--method DELETE", cleanup) + self.assertNotIn("releases/tags/", cleanup) + self.assertNotIn("gh release delete", cleanup) + + def test_official_actions_are_immutably_pinned(self): + for relative in ( + ".github/workflows/ci.yml", + ".github/workflows/macos-release.yml", + ): + workflow = (REPO_ROOT / relative).read_text() + uses = re.findall(r"^\s+uses: ([^\s#]+)", workflow, re.MULTILINE) + self.assertTrue(uses) + for action in uses: + with self.subTest(workflow=relative, action=action): + self.assertRegex(action, r"^[^@]+@[0-9a-f]{40}$") + + def test_windows_ci_has_no_forced_bash_steps(self): + workflow = (REPO_ROOT / ".github/workflows/ci.yml").read_text() + + shared = workflow[: workflow.index(" linux_appimage:")] + self.assertNotIn("shell: bash", shared) + self.assertIn("if: runner.os == 'Linux'", shared) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/tests/test_verify_linux_appimage.py b/scripts/tests/test_verify_linux_appimage.py new file mode 100644 index 0000000..75fda63 --- /dev/null +++ b/scripts/tests/test_verify_linux_appimage.py @@ -0,0 +1,186 @@ +import importlib.util +import stat +import sys +import tempfile +import unittest +from pathlib import Path + + +MODULE_PATH = Path(__file__).parents[1] / "verify_linux_appimage.py" +SPEC = importlib.util.spec_from_file_location("verify_linux_appimage", MODULE_PATH) +assert SPEC is not None and SPEC.loader is not None +verify_linux_appimage = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = verify_linux_appimage +SPEC.loader.exec_module(verify_linux_appimage) + + +class AppImageLayoutTest(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.root = Path(self.temporary.name) + app_run = self.root / "AppRun" + app_run.write_text("entry") + app_run.chmod(app_run.stat().st_mode | stat.S_IXUSR) + (self.root / "lsdj-app.png").write_bytes(b"png") + (self.root / "lsdj-app.desktop").write_text( + "[Desktop Entry]\n" + "Type=Application\n" + "Name=LSDJ\n" + "Exec=lsdj-app\n" + "Icon=lsdj-app\n" + "Categories=AudioVideo;Audio;\n" + ) + binary = self.root / "usr/bin/lsdj-app" + binary.parent.mkdir(parents=True) + binary.write_bytes(b"elf") + binary.chmod(binary.stat().st_mode | stat.S_IXUSR) + + def tearDown(self): + self.temporary.cleanup() + + def test_desktop_and_binary_layout_produce_deterministic_audit(self): + audit = verify_linux_appimage.verify_extracted( + self.root, ["libasound.so.2", "libc.so.6"] + ) + + self.assertEqual(audit["architecture"], "x86_64") + self.assertEqual(audit["desktop"]["name"], "LSDJ") + self.assertEqual(audit["elfNeeded"], ["libasound.so.2", "libc.so.6"]) + + def test_missing_audio_category_fails(self): + (self.root / "lsdj-app.desktop").write_text( + "[Desktop Entry]\nType=Application\nName=LSDJ\nExec=lsdj-app\n" + ) + + with self.assertRaisesRegex( + verify_linux_appimage.AppImageError, "audio/music category" + ): + verify_linux_appimage.verify_extracted(self.root, ["libc.so.6"]) + + def test_canonical_exec_value_is_accepted(self): + verify_linux_appimage.verify_extracted(self.root, ["libc.so.6"]) + + def test_noncanonical_exec_values_fail_closed_without_normalization(self): + desktop = self.root / "lsdj-app.desktop" + for value in ( + "'lsdj-app'", + "l'sdj-'app", + r"lsdj\-app", + 'lsdj-app "%U"', + "malicious-lsdj-app", + "sh -c lsdj-app", + "lsdj-app --unsafe", + "lsdj-app %U", + "lsdj-app %U extra", + '"lsdj-app', + " lsdj-app", + "lsdj-app ", + "lsdj-app\t", + "", + ): + with self.subTest(value=value): + desktop.write_text( + "[Desktop Entry]\n" + "Type=Application\nName=LSDJ\n" + f"Exec={value}\nCategories=Audio;\n" + ) + with self.assertRaisesRegex( + verify_linux_appimage.AppImageError, + "desktop Exec must be the canonical value", + ): + verify_linux_appimage.verify_extracted(self.root, ["libc.so.6"]) + + def test_duplicate_desktop_keys_fail_closed(self): + (self.root / "lsdj-app.desktop").write_text( + "[Desktop Entry]\n" + "Type=Application\nName=LSDJ\nName=Other\n" + "Exec=lsdj-app\nCategories=Audio;\n" + ) + + with self.assertRaisesRegex( + verify_linux_appimage.AppImageError, "duplicate desktop key" + ): + verify_linux_appimage.verify_extracted(self.root, ["libc.so.6"]) + + def test_internal_desktop_symlink_is_accepted(self): + desktop = self.root / "lsdj-app.desktop" + packaged = self.root / "usr/share/applications/lsdj-app.desktop" + packaged.parent.mkdir(parents=True) + desktop.replace(packaged) + try: + desktop.symlink_to(Path("usr/share/applications/lsdj-app.desktop")) + except OSError as error: + self.skipTest(f"symlinks are unavailable: {error}") + + audit = verify_linux_appimage.verify_extracted(self.root, ["libc.so.6"]) + + self.assertEqual(audit["desktop"]["file"], "lsdj-app.desktop") + + def test_desktop_symlink_cannot_escape_package_root(self): + desktop = self.root / "lsdj-app.desktop" + outside = self.root.parent / f"{self.root.name}-outside.desktop" + desktop.replace(outside) + self.addCleanup(outside.unlink, missing_ok=True) + try: + desktop.symlink_to(outside) + except OSError as error: + self.skipTest(f"symlinks are unavailable: {error}") + + with self.assertRaisesRegex( + verify_linux_appimage.AppImageError, "unsafe desktop entry" + ): + verify_linux_appimage.verify_extracted(self.root, ["libc.so.6"]) + + def test_absolute_desktop_symlink_is_rejected(self): + desktop = self.root / "lsdj-app.desktop" + packaged = self.root / "usr/share/applications/lsdj-app.desktop" + packaged.parent.mkdir(parents=True) + desktop.replace(packaged) + try: + desktop.symlink_to(packaged.resolve()) + except OSError as error: + self.skipTest(f"symlinks are unavailable: {error}") + + with self.assertRaisesRegex( + verify_linux_appimage.AppImageError, "absolute symlink" + ): + verify_linux_appimage.verify_extracted(self.root, ["libc.so.6"]) + + def test_internal_apprun_symlink_is_accepted(self): + app_run = self.root / "AppRun" + app_run.unlink() + try: + app_run.symlink_to(Path("usr/bin/lsdj-app")) + except OSError as error: + self.skipTest(f"symlinks are unavailable: {error}") + + verify_linux_appimage.verify_extracted(self.root, ["libc.so.6"]) + + def test_apprun_symlink_cannot_escape_package_root(self): + app_run = self.root / "AppRun" + outside = self.root.parent / f"{self.root.name}-outside-AppRun" + app_run.replace(outside) + self.addCleanup(outside.unlink, missing_ok=True) + try: + app_run.symlink_to(Path("..") / outside.name) + except OSError as error: + self.skipTest(f"symlinks are unavailable: {error}") + + with self.assertRaisesRegex( + verify_linux_appimage.AppImageError, "unsafe AppRun entry point" + ): + verify_linux_appimage.verify_extracted(self.root, ["libc.so.6"]) + + def test_apprun_must_resolve_to_a_regular_file(self): + app_run = self.root / "AppRun" + app_run.unlink() + app_run.mkdir() + + with self.assertRaisesRegex( + verify_linux_appimage.AppImageError, "unsafe AppRun entry point" + ): + verify_linux_appimage.verify_extracted(self.root, ["libc.so.6"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/tests/test_windows_packaging.py b/scripts/tests/test_windows_packaging.py new file mode 100644 index 0000000..fc908be --- /dev/null +++ b/scripts/tests/test_windows_packaging.py @@ -0,0 +1,409 @@ +import json +import unittest +from pathlib import Path + +REPO_ROOT = Path(__file__).parents[2] +TAURI_ROOT = REPO_ROOT / "src-tauri" + + +class WindowsPackagingContractTest(unittest.TestCase): + def test_windows_release_security_files_require_engineering_review(self): + owner = "@protocol-works/engineering" + entries = { + line.strip() + for line in (REPO_ROOT / ".github/CODEOWNERS").read_text().splitlines() + if line.strip() and not line.lstrip().startswith("#") + } + windows_scripts = sorted((REPO_ROOT / "scripts").glob("*windows*.ps1")) + self.assertGreaterEqual(len(windows_scripts), 6) + protected = [ + *(path.relative_to(REPO_ROOT) for path in windows_scripts), + Path("src-tauri/windows/English.nsh"), + Path("src-tauri/windows/installer-hooks.nsh"), + ] + for path in protected: + with self.subTest(path=path): + self.assertIn(f"/{path.as_posix()} {owner}", entries) + + def test_nsis_is_current_user_and_blocks_downgrades(self): + config = json.loads((TAURI_ROOT / "tauri.windows.conf.json").read_text()) + bundle = config["bundle"] + windows = bundle["windows"] + nsis = windows["nsis"] + + self.assertEqual(bundle["targets"], ["nsis"]) + self.assertEqual(nsis["installMode"], "currentUser") + self.assertEqual(nsis["startMenuFolder"], "LSDJ") + self.assertFalse(windows["allowDowngrades"]) + self.assertEqual( + windows["webviewInstallMode"], + {"type": "downloadBootstrapper", "silent": True}, + ) + + def test_uninstall_data_removal_is_explicit_disclosed_and_scoped(self): + hooks = (TAURI_ROOT / "windows/installer-hooks.nsh").read_text() + + self.assertIn('!define LSDJ_DATA_ROOT "$LOCALAPPDATA\\LSDJ"', hooks) + self.assertIn(".lsdj-data-root", hooks) + self.assertIn('!define LSDJ_OWNER_ID "works.protocol.lsdj"', hooks) + self.assertIn("NSIS_HOOK_PREINSTALL", hooks) + self.assertIn("GetFullPathNameW", hooks) + self.assertIn("CreateFileW", hooks) + self.assertIn("LSDJ_FILE_FLAG_OPEN_REPARSE_POINT", hooks) + self.assertIn("!define LSDJ_OWNER_ID_BYTES 19", hooks) + self.assertIn("!define LSDJ_OWNER_ID_READ_BYTES 20", hooks) + self.assertEqual(len("works.protocol.lsdj".encode("ascii")), 19) + self.assertIn("System::Alloc 52", hooks) + self.assertIn( + "GetFileInformationByHandle(p R6, p R7)", + hooks, + ) + self.assertIn("System::Call '*$R7(&i4 .R8)'", hooks) + self.assertIn("System::Free $R7", hooks) + self.assertIn( + "ReadFile(p R6, m .R8, i ${LSDJ_OWNER_ID_READ_BYTES}, *i .R7, p 0)", + hooks, + ) + self.assertNotIn("GetFileInformationByHandle(p R6, *(", hooks) + self.assertNotIn('FileOpen $R5 "${LSDJ_DATA_MARKER}" r', hooks) + marker_validator_start = hooks.index("!macro LSDJ_DEFINE_MARKER_VALIDATOR") + marker_validator_end = hooks.index("!macroend", marker_validator_start) + marker_validator = hooks[marker_validator_start:marker_validator_end] + self.assertIn( + "${OrIf} $R7 != ${LSDJ_OWNER_ID_BYTES}", + marker_validator, + ) + self.assertIn("LSDJ_FILE_ATTRIBUTE_REPARSE_POINT", hooks) + self.assertIn("LsdjExistingLayoutIsRecognized", hooks) + self.assertIn("Section -LsdjProbeDataRootBeforeTauri", hooks) + self.assertIn("StrCpy $LsdjInstallRootState 1", hooks) + self.assertIn("LsdjDataRootIsEmpty", hooks) + self.assertIn("LsdjInstallTreeIsLinkFree", hooks) + self.assertIn('CreateDirectory "${LSDJ_DATA_ROOT}"', hooks) + self.assertIn("LsdjOwnedDataRootIsSafe", hooks) + self.assertIn("LsdjTreeIsLinkFree", hooks) + self.assertIn("LsdjDeleteTreeWithoutLinks", hooks) + self.assertIn("IfErrors lsdj_install_tree_empty_candidate", hooks) + self.assertIn("IfErrors lsdj_empty_recheck", hooks) + self.assertIn("IfErrors lsdj_tree_empty_candidate", hooks) + self.assertIn("IfErrors lsdj_delete_empty_candidate", hooks) + for empty_recheck in ( + "lsdj_install_tree_empty_candidate:", + "lsdj_empty_recheck:", + "lsdj_tree_empty_candidate:", + "lsdj_delete_empty_candidate:", + ): + recheck_start = hooks.index(empty_recheck) + recheck = hooks[recheck_start : recheck_start + 700] + self.assertIn("GetFileAttributesW", recheck) + self.assertIn("LSDJ_FILE_ATTRIBUTE_REPARSE_POINT", recheck) + self.assertIn("LSDJ_FILE_ATTRIBUTE_DIRECTORY", recheck) + self.assertIn("/PURGE-LSDJ-DATA", hooks) + self.assertIn("${GetSize}", hooks) + self.assertIn("Location: ${LSDJ_DATA_ROOT}", hooks) + self.assertIn("Size: $R8 KiB", hooks) + self.assertIn("StrCpy $LsdjDeleteData $DeleteAppDataCheckboxState", hooks) + self.assertIn("StrCpy $DeleteAppDataCheckboxState 0", hooks) + self.assertIn("${If} $LsdjDeleteData = 1", hooks) + preuninstall_start = hooks.index("!macro NSIS_HOOK_PREUNINSTALL") + preuninstall_end = hooks.index("!macroend", preuninstall_start) + preuninstall = hooks[preuninstall_start:preuninstall_end] + self.assertIn( + "preuninstall: owned root safe=$LsdjOwnedRootSafe tree safe=$LsdjTreeSafe", + preuninstall, + ) + self.assertIn("abort: unsafe data removal", preuninstall) + self.assertIn("SetErrorLevel 2\n Quit", preuninstall) + self.assertNotIn('Abort "Refusing unsafe LSDJ data removal."', preuninstall) + self.assertIn('DeleteRegKey SHCTX "${MANUPRODUCTKEY}"', hooks) + self.assertNotIn("RMDir /r", hooks) + self.assertNotIn('RMDir /r "$LOCALAPPDATA"', hooks) + + probe_start = hooks.index("Section -LsdjProbeDataRootBeforeTauri") + probe_end = hooks.index("SectionEnd", probe_start) + probe = hooks[probe_start:probe_end] + preinstall_start = hooks.index("!macro NSIS_HOOK_PREINSTALL") + create_start = hooks.index('CreateDirectory "${LSDJ_DATA_ROOT}"') + self.assertLess(probe_start, probe_end) + self.assertLess(probe_end, preinstall_start) + self.assertLess(preinstall_start, create_start) + self.assertNotIn("CreateDirectory", probe) + preinstall_end = hooks.index("!macroend", preinstall_start) + preinstall = hooks[preinstall_start:preinstall_end] + self.assertIn("!define MUI_CUSTOMFUNCTION_GUIINIT LsdjRejectPassiveMode", hooks) + passive_start = hooks.index("Function LsdjRejectPassiveMode") + passive_end = hooks.index("FunctionEnd", passive_start) + passive_callback = hooks[passive_start:passive_end] + self.assertIn("!insertmacro LSDJ_REJECT_PASSIVE_MODE", passive_callback) + passive_macro_start = hooks.index("!macro LSDJ_REJECT_PASSIVE_MODE") + passive_macro_end = hooks.index("!macroend", passive_macro_start) + passive_macro = hooks[passive_macro_start:passive_macro_end] + self.assertIn( + '${GetOptions} $CMDLINE "/P" $LsdjPassiveRequested', passive_macro + ) + self.assertIn("SetErrorLevel 2", passive_macro) + self.assertIn("Quit", passive_macro) + self.assertLess(passive_end, preinstall_start) + self.assertTrue( + preinstall.lstrip().startswith( + "!macro NSIS_HOOK_PREINSTALL\n !insertmacro LSDJ_REJECT_PASSIVE_MODE" + ) + ) + self.assertIn("${If} ${Silent}", preinstall) + self.assertNotIn("$PassiveMode", preinstall) + self.assertNotIn("${AndIf} $UpdateMode != 1", preinstall) + self.assertIn( + 'ReadRegStr $LsdjInstalledVersion SHCTX "${UNINSTKEY}" "DisplayVersion"', + preinstall, + ) + self.assertIn( + 'ReadRegStr $LsdjRegistryEvidence SHCTX "${UNINSTKEY}" "UninstallString"', + preinstall, + ) + self.assertIn( + '${FileExists} "$INSTDIR\\${MAINBINARYNAME}.exe"', + preinstall, + ) + self.assertIn( + 'nsis_tauri_utils::SemverCompare "$LsdjInstalledVersion" "lsdj-invalid-semver"', + preinstall, + ) + self.assertIn('${If} "$LsdjInstalledVersion" == ""', preinstall) + self.assertNotIn('${If} $LsdjInstalledVersion = ""', preinstall) + self.assertIn( + 'nsis_tauri_utils::SemverCompare "${VERSION}" "$LsdjInstalledVersion"', + preinstall, + ) + self.assertIn("${If} $LsdjVersionCompare = -1", preinstall) + self.assertIn("${ElseIf} $LsdjVersionCompare != 0", preinstall) + self.assertIn("${AndIf} $LsdjVersionCompare != 1", preinstall) + self.assertIn("abort: invalid version comparison", preinstall) + version_guard = preinstall[ + : preinstall.index("Call LsdjCanonicalDataRootIsValid") + ] + self.assertNotIn("$R6", version_guard) + self.assertNotIn("$R7", version_guard) + self.assertIn("SetErrorLevel 2", preinstall) + self.assertLess( + preinstall.index("SetErrorLevel 2"), + preinstall.index("Call LsdjCanonicalDataRootIsValid"), + ) + + language = (TAURI_ROOT / "windows/English.nsh").read_text() + self.assertIn("path and size will be confirmed", language) + + def test_release_signing_uses_only_the_protected_provider_interface(self): + release = json.loads( + (TAURI_ROOT / "tauri.windows.release.conf.json").read_text() + ) + sign = release["bundle"]["windows"]["signCommand"] + self.assertEqual(sign["command"], "pwsh.exe") + self.assertIn("../scripts/sign-windows.ps1", sign["args"]) + self.assertIn("%1", sign["args"]) + + signer = (REPO_ROOT / "scripts/sign-windows.ps1").read_text() + verifier = (REPO_ROOT / "scripts/verify-windows-signatures.ps1").read_text() + for name in ( + "LSDJ_WINDOWS_SIGN_COMMAND_PATH", + "LSDJ_WINDOWS_EXPECTED_CERTIFICATE_SHA1", + "LSDJ_WINDOWS_EXPECTED_SUBJECT", + ): + self.assertIn(name, signer) + self.assertIn("TimeStamperCertificate", verifier) + self.assertIn("signtool.exe", verifier) + self.assertIn("Get-Command 'signtool.exe'", signer) + self.assertNotRegex( + signer, r"(?i)certificate_base64|pfx_password|azure|digicert" + ) + + def test_hosted_ci_builds_unsigned_but_exercises_release_rejection(self): + workflow = (REPO_ROOT / ".github/workflows/ci.yml").read_text() + build = (REPO_ROOT / "scripts/build-windows-installer.ps1").read_text() + lifecycle = (REPO_ROOT / "scripts/test-windows-installer.ps1").read_text() + windows_doc = (REPO_ROOT / "docs/windows.md").read_text() + hooks = (TAURI_ROOT / "windows/installer-hooks.nsh").read_text() + + self.assertIn("-UnsignedDevelopment", workflow) + self.assertIn("assert-windows-release-rejects-unsigned.ps1", workflow) + self.assertIn("test-windows-installer.ps1", workflow) + self.assertIn("windows-x64-unsigned-development", workflow) + self.assertIn("cargo install tauri-cli --version '=2.11.2' --locked", workflow) + self.assertIn("LSDJ_CI_ADVERSARIAL_TESTS", build) + self.assertIn("if ($UnsignedDevelopment)", build) + self.assertIn('FileOpen $R5 "$TEMP\\lsdj-ci-installer.trace" a', hooks) + self.assertIn("FileSeek $R5 0 END", hooks) + self.assertIn("Var LsdjCiTraceHadErrors", hooks) + self.assertIn("Var LsdjCiTraceMessage", hooks) + self.assertIn('StrCpy $LsdjCiTraceMessage "${MESSAGE}"', hooks) + trace_else = hooks.index("!else", hooks.index("!macro LSDJ_CI_TRACE MESSAGE")) + trace_end = hooks.index("!endif", trace_else) + self.assertEqual( + hooks[trace_else:trace_end].count("FileOpen"), + 0, + "Production trace macro must expand to no file operations.", + ) + self.assertIn("Get-CiInstallerTrace", lifecycle) + self.assertIn("Write-CiInstallerTrace", lifecycle) + self.assertIn("CI installer trace:", lifecycle) + self.assertIn("function Get-InstalledStateSnapshot", lifecycle) + self.assertIn("function Get-UninstallRegistrySnapshot", lifecycle) + self.assertIn("function Assert-CiInstallerTraceContract", lifecycle) + self.assertIn("function Set-DisplayVersionEvidence", lifecycle) + self.assertIn("function New-UninstallerWorkerCopy", lifecycle) + self.assertIn("function Stop-UninstallerWorker", lifecycle) + self.assertIn("function Invoke-ExpectedUninstallFailure", lifecycle) + self.assertNotIn("Invoke-ExpectedFailure $uninstaller", lifecycle) + copy_helper_start = lifecycle.index("function New-UninstallerWorkerCopy") + stop_helper_start = lifecycle.index( + "function Stop-UninstallerWorker", copy_helper_start + ) + worker_helper_start = lifecycle.index( + "function Invoke-ExpectedUninstallFailure" + ) + copy_helper = lifecycle[copy_helper_start:stop_helper_start] + stop_helper = lifecycle[stop_helper_start:worker_helper_start] + worker_helper_end = lifecycle.index( + "function Require-InstalledVersion", worker_helper_start + ) + worker_helper = lifecycle[worker_helper_start:worker_helper_end] + self.assertIn("} catch {", copy_helper) + self.assertIn("Remove-Item -LiteralPath $workerPath", copy_helper) + self.assertIn("$Process.Kill($true)", stop_helper) + self.assertIn("$Process.WaitForExit($TimeoutMilliseconds)", stop_helper) + self.assertIn( + '-ArgumentList (@($ArgumentList) + "_?=$InstallDirectory")', + worker_helper, + ) + self.assertIn("$worker.WaitForExit(30000)", worker_helper) + self.assertIn("$worker.ExitCode -ne 2", worker_helper) + self.assertIn("} finally {", worker_helper) + self.assertIn("Stop-UninstallerWorker -Process $worker", worker_helper) + self.assertIn( + "Remove-Item -LiteralPath $workerPath", + worker_helper, + ) + race_start = lifecycle.index( + "Start-LifecycleScenario 'reject ownership-marker replacement" + ) + race_end = lifecycle.index( + "Start-LifecycleScenario 'reject purge with nested junction'", + race_start, + ) + race = lifecycle[race_start:race_end] + self.assertIn('"_?=$dataRoot"\n )', race) + self.assertIn("$racedPurge.ExitCode -ne 2", race) + self.assertIn("$racedPurge.WaitForExit(30000)", race) + self.assertIn("} finally {", race) + self.assertIn("Stop-UninstallerWorker -Process $racedPurge", race) + self.assertGreaterEqual(race.count("} finally {"), 2) + self.assertIn( + "Remove-Item -LiteralPath $racedPurgeWorker", + race, + ) + unicode_worker_start = lifecycle.index( + "Start-LifecycleScenario 'reject purge worker with spaces and Unicode" + ) + unicode_worker_end = lifecycle.index( + "Invoke-CheckedProcess $unicodeUninstaller @('/S')", unicode_worker_start + ) + unicode_worker = lifecycle[unicode_worker_start:unicode_worker_end] + self.assertIn("-FilePath $unicodeUninstaller", unicode_worker) + self.assertIn("-InstallDirectory $unicodeInstall", unicode_worker) + self.assertIn("Invoke-ExpectedUninstallFailure", unicode_worker) + self.assertNotIn(".WaitForExit()", race) + self.assertIn( + "NSIS installed uninstallers are self-copy launchers", windows_doc + ) + self.assertIn("final unquoted `_?=`", windows_doc) + self.assertIn("fail-closed refusal (`2`)", windows_doc) + self.assertIn("$lastRequiredIndex = -1", lifecycle) + self.assertIn("[StringComparison]::Ordinal", lifecycle) + self.assertIn( + "Get-FileHash -LiteralPath $uninstaller -Algorithm SHA256", lifecycle + ) + self.assertIn("Assert-InstalledStateSnapshotUnchanged", lifecycle) + self.assertIn("MarkerAttributes", lifecycle) + self.assertIn("MarkerLinkType", lifecycle) + self.assertIn("MarkerTarget", lifecycle) + self.assertIn("[IO.FileAttributes]::ReparsePoint", lifecycle) + self.assertIn("-ExpectedExitCodes @(2)", lifecycle) + for contract in ( + "pre-existing empty LocalAppData root", + "foreign LocalAppData root", + "root junction", + "purge-time root junction", + "marker reparse point", + "NUL-extended ownership marker", + "marker-replacement test", + "nested directory reparse point", + "reject unattended downgrade in update mode", + "reject passive downgrade", + "reject combined silent and passive mode", + "same-version silent reinstall", + "reject existing install with empty version metadata", + "reject existing install with corrupt version metadata", + "reject existing install with missing version metadata", + ): + self.assertIn(contract, lifecycle) + self.assertIn("function Start-LifecycleScenario", lifecycle) + self.assertIn("preinstall: version compare=-1", lifecycle) + self.assertIn("preinstall: version compare=0", lifecycle) + self.assertIn("preinstall: version compare=1", lifecycle) + self.assertIn("validity=1", lifecycle) + self.assertIn("validity=0", lifecycle) + self.assertIn("abort: passive install unsupported", lifecycle) + self.assertIn("preinstall: version evidence installed=", hooks) + self.assertIn("Require-No-Workers", lifecycle) + self.assertIn( + "adopt recognized markerless legacy layout", + lifecycle, + ) + + rejection = ( + REPO_ROOT / "scripts/assert-windows-release-rejects-unsigned.ps1" + ).read_text() + self.assertIn("Authenticode signature status is NotSigned", rejection) + success_exit = rejection.rindex("exit 0") + self.assertGreater(success_exit, rejection.index("if ($exitCode -eq 0)")) + self.assertGreater(success_exit, rejection.index("if ($rendered -notmatch")) + + def test_release_producer_is_required_and_has_no_publish_permission(self): + workflow = (REPO_ROOT / ".github/workflows/macos-release.yml").read_text() + producer = workflow[ + workflow.index(" produce_windows:") : workflow.index(" publish:") + ] + + self.assertIn("environment:\n name: windows-release", producer) + self.assertIn("verify-windows-release-install.ps1", producer) + self.assertIn("--producer windows-x64", producer) + self.assertNotIn("contents: write", producer) + self.assertEqual(workflow.count("contents: write"), 1) + self.assertRegex(workflow, r"(?m)^ - produce_windows$") + self.assertIn("--required-producer windows-x64", workflow) + + def test_managed_runtime_feature_forbids_system_python_fallback(self): + cargo = (TAURI_ROOT / "Cargo.toml").read_text() + lib = (TAURI_ROOT / "src/lib.rs").read_text() + sidecar = (TAURI_ROOT / "src/sidecar.rs").read_text() + generation = (TAURI_ROOT / "src/generation.rs").read_text() + diagnostics = (TAURI_ROOT / "src/platform_diagnostics.rs").read_text() + + self.assertRegex(cargo, r"(?m)^managed-runtime = \[\]$") + self.assertIn("crate::managed_runtime::Service::Mrt2,", sidecar) + self.assertIn("crate::managed_runtime::Service::Sa3,", generation) + self.assertIn("crate::managed_runtime::resolve", sidecar) + self.assertIn("crate::managed_runtime::resolve", generation) + self.assertIn('mode: "managed"', diagnostics) + self.assertIn("developer_fallback_allowed: false", diagnostics) + + combined = "\n".join((lib, sidecar, generation, diagnostics)) + for stale_single_launcher_symbol in ( + "LSDJ_MANAGED_BACKEND_REQUIRED", + "managed_backend_path", + "runtime_launch", + ): + self.assertNotIn(stale_single_launcher_symbol, combined) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/verify-windows-release-install.ps1 b/scripts/verify-windows-release-install.ps1 new file mode 100644 index 0000000..98d0aa7 --- /dev/null +++ b/scripts/verify-windows-release-install.ps1 @@ -0,0 +1,74 @@ +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)] + [ValidateNotNullOrEmpty()] + [string] $Installer, + + [Parameter(Mandatory = $true)] + [ValidatePattern('^[0-9]+\.[0-9]+\.[0-9]+$')] + [string] $ExpectedVersion +) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version Latest + +if (-not $IsWindows -or $env:GITHUB_ACTIONS -ne 'true') { + throw 'Release installer verification may run only on an isolated GitHub Actions Windows runner.' +} + +$setup = Get-Item -LiteralPath $Installer -ErrorAction Stop +& "$PSScriptRoot/verify-windows-signatures.ps1" -Path $setup.FullName + +$dataRoot = Join-Path $env:LOCALAPPDATA 'LSDJ' +if (Test-Path -LiteralPath $dataRoot) { + throw "The release verification runner is not clean: $dataRoot already exists." +} +$app = Join-Path $dataRoot 'lsdj-app.exe' +$uninstaller = Join-Path $dataRoot 'uninstall.exe' +$sentinel = Join-Path $dataRoot 'data\release-preservation.txt' + +function Invoke-ReleaseProcess { + param([string] $FilePath, [string[]] $ArgumentList) + + $process = Start-Process -FilePath $FilePath -ArgumentList $ArgumentList -Wait -PassThru + if ($process.ExitCode -ne 0) { + throw "Release lifecycle command exited $($process.ExitCode): $FilePath" + } +} + +Invoke-ReleaseProcess $setup.FullName @('/S') +if ((Get-Item -LiteralPath $app).VersionInfo.ProductVersion -ne $ExpectedVersion) { + throw "Installed release does not report version $ExpectedVersion." +} +$payloads = @( + Get-ChildItem -LiteralPath $dataRoot -Recurse -File | + Where-Object { $_.Extension -in @('.exe', '.dll') } | + ForEach-Object FullName +) +if ($payloads.Count -lt 2) { + throw 'Expected at least the signed app and uninstaller payloads.' +} +& "$PSScriptRoot/verify-windows-signatures.ps1" -Path $payloads + +New-Item -ItemType Directory -Path (Split-Path -Parent $sentinel) -Force | Out-Null +[System.IO.File]::WriteAllText($sentinel, 'preserve') +Invoke-ReleaseProcess $uninstaller @('/S') +Start-Sleep -Milliseconds 500 +if (-not (Test-Path -LiteralPath $sentinel -PathType Leaf)) { + throw 'Default release uninstall did not preserve app-owned data.' +} +if (Test-Path -LiteralPath $app) { + throw 'Default release uninstall left the app binary behind.' +} + +Invoke-ReleaseProcess $setup.FullName @('/S') +Invoke-ReleaseProcess $uninstaller @('/S', '/PURGE-LSDJ-DATA') +Start-Sleep -Milliseconds 500 +if (Test-Path -LiteralPath $dataRoot) { + throw 'Explicit release data removal did not remove the app-owned data root.' +} +$workers = @(Get-Process -Name 'lsdj-app' -ErrorAction SilentlyContinue) +if ($workers.Count -ne 0) { + throw "Release install/uninstall left worker processes running: $($workers.Name -join ', ')" +} +Write-Host 'Signed Windows release installer and installed payloads verified.' diff --git a/scripts/verify-windows-signatures.ps1 b/scripts/verify-windows-signatures.ps1 new file mode 100644 index 0000000..c722676 --- /dev/null +++ b/scripts/verify-windows-signatures.ps1 @@ -0,0 +1,58 @@ +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)] + [ValidateNotNullOrEmpty()] + [string[]] $Path, + + [string] $ExpectedCertificateSha1 = $env:LSDJ_WINDOWS_EXPECTED_CERTIFICATE_SHA1, + + [string] $ExpectedSubject = $env:LSDJ_WINDOWS_EXPECTED_SUBJECT +) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version Latest + +$thumbprint = $ExpectedCertificateSha1 -replace '\s', '' +if ($thumbprint -notmatch '^[0-9A-Fa-f]{40}$') { + throw 'ExpectedCertificateSha1 must contain exactly 40 hexadecimal characters.' +} +if ([string]::IsNullOrWhiteSpace($ExpectedSubject)) { + throw 'ExpectedSubject is required and must exactly match the approved Authenticode subject.' +} + +$signTool = $null +foreach ($entry in $Path) { + $target = Get-Item -LiteralPath $entry -ErrorAction Stop + if ($target.PSIsContainer -or ($target.Attributes -band [IO.FileAttributes]::ReparsePoint) -ne 0) { + throw "Signature target must be a plain file: $entry" + } + if ($target.Extension -notin @('.exe', '.dll')) { + throw "Signature target must be an executable payload: $($target.FullName)" + } + + $signature = Get-AuthenticodeSignature -LiteralPath $target.FullName + if ($signature.Status -ne [System.Management.Automation.SignatureStatus]::Valid) { + throw "Authenticode signature status is $($signature.Status) for $($target.FullName): $($signature.StatusMessage)" + } + if ($null -eq $signature.SignerCertificate) { + throw "Authenticode signer certificate is missing for $($target.FullName)." + } + if ($signature.SignerCertificate.Thumbprint -ne $thumbprint.ToUpperInvariant()) { + throw "Unexpected Authenticode certificate for $($target.FullName)." + } + if ($signature.SignerCertificate.Subject -cne $ExpectedSubject) { + throw "Unexpected Authenticode subject for $($target.FullName): $($signature.SignerCertificate.Subject)" + } + if ($null -eq $signature.TimeStamperCertificate) { + throw "Authenticode timestamp is missing for $($target.FullName)." + } + + if ($null -eq $signTool) { + $signTool = (Get-Command 'signtool.exe' -ErrorAction Stop).Source + } + & $signTool verify /pa /all /v $target.FullName + if ($LASTEXITCODE -ne 0) { + throw "signtool trust verification failed with exit code $LASTEXITCODE for $($target.FullName)." + } + Write-Host "Verified Authenticode signer and timestamp: $($target.FullName)" +} diff --git a/scripts/verify_linux_appimage.py b/scripts/verify_linux_appimage.py new file mode 100755 index 0000000..f8779e9 --- /dev/null +++ b/scripts/verify_linux_appimage.py @@ -0,0 +1,201 @@ +#!/usr/bin/env python3 +"""Verify an LSDJ x86_64 AppImage and emit deterministic package audit data.""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import shutil +import stat +import subprocess +import tempfile +from pathlib import Path + + +class AppImageError(RuntimeError): + """The produced AppImage violated the Linux package contract.""" + + +def require(condition: bool, message: str) -> None: + if not condition: + raise AppImageError(message) + + +def safe_packaged_file(root: Path, path: Path, kind: str) -> Path: + """Resolve a regular file without allowing a package-root escape.""" + root = root.resolve(strict=True) + if path.is_symlink(): + target = path.readlink() + require(not target.is_absolute(), f"unsafe {kind}: absolute symlink") + try: + resolved = path.resolve(strict=True) + resolved.relative_to(root) + except (OSError, ValueError): + raise AppImageError(f"unsafe {kind}: {path}") from None + require(resolved.is_file(), f"unsafe {kind}: {path}") + return resolved + + +def desktop_entries(root: Path, path: Path) -> dict[str, str]: + path = safe_packaged_file(root, path, "desktop entry") + entries: dict[str, str] = {} + section = "" + for raw_line in path.read_text(encoding="utf-8").splitlines(): + line = raw_line.strip() + if not line or line.startswith("#"): + continue + if line.startswith("[") and line.endswith("]"): + section = line[1:-1] + continue + if section != "Desktop Entry" or "=" not in raw_line: + continue + # Preserve the raw value. In particular, do not normalize quotes, + # escapes, or surrounding whitespace in the security-sensitive Exec + # field before the canonical policy below sees it. + key, value = raw_line.split("=", 1) + if key in entries: + raise AppImageError(f"duplicate desktop key: {key}") + entries[key] = value + return entries + + +def desktop_exec(value: str) -> str: + """Require the exact launcher emitted by this Tauri bundle configuration.""" + require( + value == "lsdj-app", + "desktop Exec must be the canonical value lsdj-app", + ) + return value + + +def needed_libraries(binary: Path) -> list[str]: + readelf = shutil.which("readelf") + require(readelf is not None, "readelf is required for the build-time ELF audit") + result = subprocess.run( + [readelf, "--dynamic", str(binary)], + check=True, + capture_output=True, + text=True, + ) + needed = sorted(set(re.findall(r"\(NEEDED\).*?\[(.+?)\]", result.stdout))) + require(needed, "packaged executable has no ELF NEEDED entries") + lowered = {library.casefold() for library in needed} + forbidden = { + library + for library in lowered + if "python" in library + or library.startswith("libcuda.") + or library.startswith("libcudart.") + } + require( + not forbidden, + "AppImage shell must not link system Python/CUDA libraries: " + + ", ".join(sorted(forbidden)), + ) + return needed + + +def verify_extracted(root: Path, libraries: list[str]) -> dict: + require(root.is_dir() and not root.is_symlink(), "missing extracted AppImage root") + app_run = safe_packaged_file(root, root / "AppRun", "AppRun entry point") + if os.name != "nt": + require( + app_run.stat().st_mode & stat.S_IXUSR != 0, + "AppRun entry point is not executable", + ) + + desktop_files = sorted(root.glob("*.desktop")) + require(len(desktop_files) == 1, "AppImage must contain exactly one desktop entry") + desktop = desktop_entries(root, desktop_files[0]) + require(desktop.get("Type") == "Application", "desktop Type must be Application") + require(desktop.get("Name") == "LSDJ", "desktop Name must be LSDJ") + desktop_exec(desktop.get("Exec", "")) + categories = {item for item in desktop.get("Categories", "").split(";") if item} + require( + bool(categories & {"Audio", "AudioVideo", "Music"}), + "desktop entry must advertise an audio/music category", + ) + + binary = root / "usr/bin/lsdj-app" + require( + binary.is_file() and not binary.is_symlink(), + "AppImage has no safe lsdj-app binary", + ) + # Windows does not preserve POSIX mode bits in the shared pure-layout unit + # test. Production verification runs on Linux, where this remains required. + if os.name != "nt": + require( + binary.stat().st_mode & stat.S_IXUSR != 0, + "packaged lsdj-app binary is not executable", + ) + require(any(root.glob("*.png")), "AppImage root has no desktop icon") + + return { + "architecture": "x86_64", + "desktop": { + "categories": sorted(categories), + "exec": desktop["Exec"], + "file": desktop_files[0].name, + "icon": desktop.get("Icon"), + "name": desktop["Name"], + }, + "elfNeeded": sorted(libraries), + "platform": "linux", + "schemaVersion": 1, + } + + +def verify_appimage(appimage: Path) -> dict: + require( + appimage.is_file() and not appimage.is_symlink(), + f"AppImage must be a regular non-symlink file: {appimage}", + ) + require(appimage.suffix == ".AppImage", "artifact must end in .AppImage") + require(appimage.stat().st_size > 0, "AppImage must not be empty") + require( + appimage.stat().st_mode & stat.S_IXUSR != 0, + "AppImage must have its executable bit set", + ) + + with tempfile.TemporaryDirectory(prefix="lsdj-appimage-") as temporary: + extraction_dir = Path(temporary) + result = subprocess.run( + [str(appimage.resolve()), "--appimage-extract"], + cwd=extraction_dir, + check=False, + capture_output=True, + text=True, + ) + require( + result.returncode == 0, + f"AppImage extraction failed ({result.returncode}): {result.stderr[-2000:]}", + ) + root = extraction_dir / "squashfs-root" + binary = root / "usr/bin/lsdj-app" + libraries = needed_libraries(binary) + audit = verify_extracted(root, libraries) + audit["artifact"] = appimage.name + return audit + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("appimage", type=Path) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + audit = verify_appimage(args.appimage) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text( + json.dumps(audit, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + newline="\n", + ) + + +if __name__ == "__main__": + try: + main() + except (AppImageError, OSError, subprocess.SubprocessError) as error: + raise SystemExit(f"Linux package verification failed: {error}") from error diff --git a/spike/mrt2_pytorch/README.md b/spike/mrt2_pytorch/README.md new file mode 100644 index 0000000..29dc0ee --- /dev/null +++ b/spike/mrt2_pytorch/README.md @@ -0,0 +1,65 @@ +# PyTorch MRT2 two-deck spike harness + +This directory is an isolated issue #109 fixture. It does not import or change +the production backend. It benchmarks the immutable upstream Transformers +snapshots in two process topologies, at 25 frames (~1 second) and 5 frames +(~200 ms), with a 1.5 second playback-prebuffer simulation. + +The harness records cold start, warm-up, per-deck p50/p95/p99 generation +latency, output duration, an underrun proxy, RSS, PyTorch CUDA allocation, and +`nvidia-smi` VRAM/driver/temperature rows. JSON marks dry runs as `synthetic`; +they are never qualification evidence. + +## CI/dry run + +From the repository root: + +```sh +python3 -m unittest discover -s spike/mrt2_pytorch/tests -v +python3 -m spike.mrt2_pytorch.harness \ + --backend dry-run --duration-seconds 2 \ + --output /tmp/mrt2-dry-run.json +``` + +## NVIDIA run + +Use Python 3.12 and install the direct candidate pins: + +```sh +python -m venv .venv-mrt2-spike +.venv-mrt2-spike/bin/python -m pip install -r spike/mrt2_pytorch/requirements-candidate.txt +``` + +On Windows, use `.venv-mrt2-spike\Scripts\python.exe` for the same commands. +Prefetch the immutable snapshots while online: + +```sh +hf download magenta-community/magenta-realtime-2-small \ + --revision 7037d99551c84ac5c6afb7f1a5e58c65e7233dbb +hf download magenta-community/magenta-rt-musiccoca-torch \ + --revision 236c488e38aa98643805514996934d705668298b +``` + +Then disconnect or set `HF_HUB_OFFLINE=1` and run the complete matrix: + +```sh +python -m spike.mrt2_pytorch.harness \ + --backend upstream \ + --model mrt2_small \ + --topologies shared-worker,per-deck \ + --frames 25,5 \ + --duration-seconds 600 \ + --prompt-change-seconds 60 \ + --output mrt2-small-eager.json +``` + +Repeat with `--acceleration torch-compile`; do not treat a runtime compiler as +a distributable solution until Windows packaging proves it needs no developer +toolchain. The default enables classifier-free guidance because that matches +LSDJ's `.mlxfn` path. `--token-cfg` intentionally measures upstream's cheaper, +non-parity conditioning-token path. + +The model adapter always uses exact revisions and `local_files_only=True`. +Any cache miss therefore fails clearly instead of silently downloading a +different asset. See `provenance.json` and +`docs/issue-109-hardware-checklist.md` before running qualification. diff --git a/spike/mrt2_pytorch/__init__.py b/spike/mrt2_pytorch/__init__.py new file mode 100644 index 0000000..8c581c9 --- /dev/null +++ b/spike/mrt2_pytorch/__init__.py @@ -0,0 +1 @@ +"""Issue #109's isolated PyTorch MRT2 validation harness.""" diff --git a/spike/mrt2_pytorch/harness.py b/spike/mrt2_pytorch/harness.py new file mode 100644 index 0000000..962562d --- /dev/null +++ b/spike/mrt2_pytorch/harness.py @@ -0,0 +1,863 @@ +"""Two-deck MRT2 benchmark harness for issue #109. + +This module deliberately has no LSDJ production imports. The real adapter loads +the pinned Hugging Face snapshot with ``local_files_only=True``; the dry adapter +lets CI exercise scheduling, topology, ring accounting, and result schemas with +no model, GPU, or network. +""" + +from __future__ import annotations + +import argparse +import dataclasses +import json +import math +import multiprocessing +import os +import pathlib +import platform +import queue +import statistics +import subprocess +import sys +import time +import traceback +from collections.abc import Iterable +from typing import Any + +SAMPLE_RATE = 48_000 +CHANNELS = 2 +FRAME_SECONDS = 0.04 +DECKS = (0, 1) +DEFAULT_PREBUFFER_SECONDS = 1.5 + +SOURCE_REPOSITORY = "https://github.com/multimodalart/magenta-realtime-torch.git" +SOURCE_REVISION = "6d076baa3df3b10448876c400521a015a5137c59" +MODEL_REVISIONS = { + "mrt2_base": ( + "magenta-community/magenta-realtime-2", + "92087988d05d0fe38b11f021f0b0d00a75afb86b", + ), + "mrt2_small": ( + "magenta-community/magenta-realtime-2-small", + "7037d99551c84ac5c6afb7f1a5e58c65e7233dbb", + ), +} +PROCESSOR_REPOSITORY = "magenta-community/magenta-rt-musiccoca-torch" +PROCESSOR_REVISION = "236c488e38aa98643805514996934d705668298b" + + +@dataclasses.dataclass +class RingBudget: + """Event-level proxy for LSDJ's per-deck 1.5 second prebuffer gate. + + The production engine counts callback blocks that find a primed ring short. + The harness cannot observe the device callback, so it records starvation + intervals and transitions. Both are labelled proxies in the JSON output. + """ + + prebuffer_seconds: float + fill_seconds: float = 0.0 + primed: bool = False + last_time: float | None = None + playback_started_at: float | None = None + underrun_events: int = 0 + underrun_seconds: float = 0.0 + fill_min_seconds: float | None = None + fill_max_seconds: float = 0.0 + + def advance(self, now: float) -> None: + if self.last_time is None: + self.last_time = now + return + elapsed = max(0.0, now - self.last_time) + self.last_time = now + if not self.primed: + return + before = self.fill_seconds + self.fill_seconds = max(0.0, before - elapsed) + self.fill_min_seconds = ( + self.fill_seconds + if self.fill_min_seconds is None + else min(self.fill_min_seconds, self.fill_seconds) + ) + if elapsed > before: + self.underrun_seconds += elapsed - before + if before > 0.0 or self.underrun_events == 0: + self.underrun_events += 1 + + def push(self, audio_seconds: float, now: float) -> None: + self.advance(now) + self.fill_seconds += max(0.0, audio_seconds) + self.fill_max_seconds = max(self.fill_max_seconds, self.fill_seconds) + if not self.primed and self.fill_seconds >= self.prebuffer_seconds: + self.primed = True + self.playback_started_at = now + self.fill_min_seconds = self.fill_seconds + + def ready_for_generation(self, target_seconds: float, chunk_seconds: float) -> bool: + if not self.primed: + return True + return self.fill_seconds <= max(0.0, target_seconds - chunk_seconds) + + def result(self, origin: float) -> dict[str, Any]: + return { + "prebuffer_seconds": self.prebuffer_seconds, + "primed": self.primed, + "time_to_prime_seconds": ( + None + if self.playback_started_at is None + else round(self.playback_started_at - origin, 6) + ), + "fill_final_seconds": round(self.fill_seconds, 6), + "fill_min_seconds": ( + None + if self.fill_min_seconds is None + else round(self.fill_min_seconds, 6) + ), + "fill_max_seconds": round(self.fill_max_seconds, 6), + "underrun_proxy_events": self.underrun_events, + "underrun_proxy_seconds": round(self.underrun_seconds, 6), + } + + +def percentile(values: Iterable[float], percent: float) -> float | None: + ordered = sorted(values) + if not ordered: + return None + if len(ordered) == 1: + return ordered[0] + rank = (len(ordered) - 1) * percent / 100.0 + low = math.floor(rank) + high = math.ceil(rank) + if low == high: + return ordered[low] + return ordered[low] + (ordered[high] - ordered[low]) * (rank - low) + + +def latency_summary(values: list[float]) -> dict[str, Any]: + return { + "count": len(values), + "mean_ms": None if not values else round(statistics.fmean(values) * 1_000, 3), + "p50_ms": _milliseconds(percentile(values, 50)), + "p95_ms": _milliseconds(percentile(values, 95)), + "p99_ms": _milliseconds(percentile(values, 99)), + "max_ms": _milliseconds(max(values) if values else None), + } + + +def _milliseconds(value: float | None) -> float | None: + return None if value is None else round(value * 1_000, 3) + + +def _rss_bytes() -> int | None: + """Current worker RSS without adding a benchmark-only dependency.""" + + if sys.platform.startswith("linux"): + try: + for line in pathlib.Path("/proc/self/status").read_text().splitlines(): + if line.startswith("VmRSS:"): + return int(line.split()[1]) * 1_024 + except (OSError, ValueError, IndexError): + return None + if sys.platform == "win32": + try: + import ctypes + from ctypes import wintypes + + class ProcessMemoryCounters(ctypes.Structure): + _fields_ = [ + ("cb", wintypes.DWORD), + ("PageFaultCount", wintypes.DWORD), + ("PeakWorkingSetSize", ctypes.c_size_t), + ("WorkingSetSize", ctypes.c_size_t), + ("QuotaPeakPagedPoolUsage", ctypes.c_size_t), + ("QuotaPagedPoolUsage", ctypes.c_size_t), + ("QuotaPeakNonPagedPoolUsage", ctypes.c_size_t), + ("QuotaNonPagedPoolUsage", ctypes.c_size_t), + ("PagefileUsage", ctypes.c_size_t), + ("PeakPagefileUsage", ctypes.c_size_t), + ] + + counters = ProcessMemoryCounters() + counters.cb = ctypes.sizeof(counters) + ok = ctypes.windll.psapi.GetProcessMemoryInfo( + ctypes.windll.kernel32.GetCurrentProcess(), + ctypes.byref(counters), + counters.cb, + ) + return int(counters.WorkingSetSize) if ok else None + except (AttributeError, OSError): + return None + try: + import resource + + value = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + return int(value if sys.platform == "darwin" else value * 1_024) + except (ImportError, OSError): + return None + + +def _gpu_snapshot(worker_pids: set[int]) -> dict[str, Any] | None: + """Best-effort NVIDIA metadata. Raw rows survive driver schema differences.""" + + try: + gpu = subprocess.run( + [ + "nvidia-smi", + "--query-gpu=index,name,uuid,driver_version,memory.total,memory.used,temperature.gpu,pstate,power.draw", + "--format=csv,noheader,nounits", + ], + check=True, + capture_output=True, + text=True, + timeout=10, + ) + apps = subprocess.run( + [ + "nvidia-smi", + "--query-compute-apps=pid,used_memory", + "--format=csv,noheader,nounits", + ], + check=True, + capture_output=True, + text=True, + timeout=10, + ) + except (FileNotFoundError, subprocess.SubprocessError): + return None + worker_memory_mib = 0.0 + matching_rows: list[str] = [] + for row in apps.stdout.splitlines(): + cells = [cell.strip() for cell in row.split(",")] + try: + pid = int(cells[0]) + used = float(cells[1]) + except (ValueError, IndexError): + continue + if pid in worker_pids: + matching_rows.append(row) + worker_memory_mib += used + return { + "gpu_rows": gpu.stdout.splitlines(), + "matching_compute_rows": matching_rows, + "worker_vram_mib": worker_memory_mib, + } + + +@dataclasses.dataclass(frozen=True) +class RunConfig: + backend: str + topology: str + frames: int + duration_seconds: float + prebuffer_seconds: float + target_ahead_seconds: float + model: str + acceleration: str + guidance: bool + dry_latency_ms: float + startup_timeout_seconds: float + worker_timeout_seconds: float + seed: int + prompt_change_seconds: float + + @property + def chunk_seconds(self) -> float: + return self.frames * FRAME_SECONDS + + +class DryAdapter: + def __init__(self, config: RunConfig): + self.delay = config.dry_latency_ms / 1_000.0 + self.states: dict[int, int] = {} + + def metadata(self) -> dict[str, Any]: + return {"adapter": "dry-run", "accelerator": "simulated"} + + def reset(self) -> None: + self.states.clear() + + def generate(self, deck: int, controls: dict[str, Any], frames: int) -> int: + time.sleep(self.delay) + self.states[deck] = self.states.get(deck, 0) + frames + return frames * round(SAMPLE_RATE * FRAME_SECONDS) + + def device_memory(self) -> dict[str, int | None]: + return { + "cuda_allocated_bytes": None, + "cuda_reserved_bytes": None, + "cuda_peak_allocated_bytes": None, + } + + +class UpstreamAdapter: + """Thin, snapshot-pinned adapter around upstream's Transformers API.""" + + def __init__(self, config: RunConfig): + import torch + from huggingface_hub import snapshot_download + from transformers import AutoModel + + if not torch.cuda.is_available(): + raise RuntimeError("PyTorch reports no CUDA accelerator") + model_repo, model_revision = MODEL_REVISIONS[config.model] + # Offline-only is intentional: installer/acquisition is a separate concern. + model_path = snapshot_download( + repo_id=model_repo, + revision=model_revision, + local_files_only=True, + ) + processor_path = snapshot_download( + repo_id=PROCESSOR_REPOSITORY, + revision=PROCESSOR_REVISION, + local_files_only=True, + ) + self.torch = torch + self.model = ( + AutoModel.from_pretrained( + model_path, + trust_remote_code=True, + dtype=torch.bfloat16, + local_files_only=True, + ) + .to("cuda") + .eval() + ) + self.model.load_processor(processor_path, device="cuda") + if config.acceleration == "torch-compile": + self.model.compile_steps() + elif config.acceleration != "eager": + raise ValueError(f"unsupported acceleration mode {config.acceleration!r}") + self.guidance = config.guidance + self.states: dict[int, Any] = {} + self.style_tokens: dict[ + tuple[tuple[str, ...], tuple[float, ...]], list[int] + ] = {} + self.torch.cuda.reset_peak_memory_stats() + + def metadata(self) -> dict[str, Any]: + torch = self.torch + props = torch.cuda.get_device_properties(torch.cuda.current_device()) + return { + "adapter": "transformers-remote-code", + "torch_version": torch.__version__, + "torch_cuda_runtime": torch.version.cuda, + "cudnn_version": torch.backends.cudnn.version(), + "cuda_device": props.name, + "cuda_capability": list(torch.cuda.get_device_capability()), + "cuda_total_memory_bytes": props.total_memory, + } + + def reset(self) -> None: + self.states.clear() + + def _tokens(self, controls: dict[str, Any]) -> list[int]: + prompts = tuple(controls["prompts"]) + weights = tuple(float(value) for value in controls["weights"]) + key = prompts, weights + if key not in self.style_tokens: + self.style_tokens[key] = self.model.processor.layer(prompts, weights) + return self.style_tokens[key] + + def generate(self, deck: int, controls: dict[str, Any], frames: int) -> int: + torch = self.torch + torch.cuda.synchronize() + audio, state = self.model.generate( + style=self._tokens(controls), + notes=controls["notes"], + drums=controls["drums"], + cfg_musiccoca=controls["cfg_musiccoca"], + cfg_notes=controls["cfg_notes"], + cfg_drums=controls["cfg_drums"], + temperature=controls["temperature"], + top_k=controls["top_k"], + frames=frames, + seed=controls["seed"], + state=self.states.get(deck), + guidance=self.guidance, + ) + torch.cuda.synchronize() + if getattr(audio, "ndim", None) != 2 or audio.shape[1] != CHANNELS: + raise RuntimeError(f"upstream returned invalid audio shape {audio.shape!r}") + self.states[deck] = state + return int(audio.shape[0]) + + def device_memory(self) -> dict[str, int]: + torch = self.torch + return { + "cuda_allocated_bytes": torch.cuda.memory_allocated(), + "cuda_reserved_bytes": torch.cuda.memory_reserved(), + "cuda_peak_allocated_bytes": torch.cuda.max_memory_allocated(), + } + + +def _controls(deck: int, changed: bool, onset: bool, seed: int) -> dict[str, Any]: + notes = [-1] * 128 + if changed: + notes[60 + deck * 7] = 2 if onset else 1 + return { + "prompts": ( + ["warm disco funk", "analog synth bass"] + if not changed + else ["broken beat percussion", "ambient pads"] + ), + "weights": [0.7, 0.3] if not changed else [0.55, 0.45], + "temperature": 1.1 if not changed else 0.95, + "top_k": 50 if not changed else 64, + "cfg_musiccoca": 1.6 if not changed else 2.0, + "cfg_notes": 2.4, + "cfg_drums": 4.0, + "notes": notes, + "drums": [-1] if deck == 0 else [0], + "seed": seed + deck, + } + + +def _worker_main( + worker_id: int, + request_queue: Any, + result_queue: Any, + config: RunConfig, +) -> None: + started = time.perf_counter() + try: + adapter = ( + DryAdapter(config) + if config.backend == "dry-run" + else UpstreamAdapter(config) + ) + result_queue.put( + { + "type": "ready", + "worker": worker_id, + "pid": os.getpid(), + "startup_seconds": time.perf_counter() - started, + "rss_bytes": _rss_bytes(), + "metadata": adapter.metadata(), + "device_memory": adapter.device_memory(), + } + ) + while True: + request = request_queue.get() + action = request["action"] + if action == "shutdown": + result_queue.put( + { + "type": "stopped", + "worker": worker_id, + "rss_bytes": _rss_bytes(), + "device_memory": adapter.device_memory(), + } + ) + return + if action == "reset": + adapter.reset() + result_queue.put({"type": "reset", "worker": worker_id}) + continue + if action != "generate": + raise ValueError(f"unknown worker action {action!r}") + generated_at = time.perf_counter() + sample_frames = adapter.generate( + request["deck"], request["controls"], request["frames"] + ) + result_queue.put( + { + "type": "chunk", + "worker": worker_id, + "deck": request["deck"], + "sequence": request["sequence"], + "control_change": request["control_change"], + "latency_seconds": time.perf_counter() - generated_at, + "sample_frames": sample_frames, + "rss_bytes": _rss_bytes(), + "device_memory": adapter.device_memory(), + } + ) + except BaseException as error: + result_queue.put( + { + "type": "error", + "worker": worker_id, + "pid": os.getpid(), + "error": f"{type(error).__name__}: {error}", + "traceback": traceback.format_exc(), + } + ) + + +def _wait_for_messages( + result_queue: Any, + expected_type: str, + count: int, + timeout: float, +) -> list[dict[str, Any]]: + deadline = time.monotonic() + timeout + messages = [] + while len(messages) < count: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise TimeoutError( + f"timed out waiting for {count} {expected_type!r} messages" + ) + message = result_queue.get(timeout=remaining) + if message["type"] == "error": + raise RuntimeError( + f"worker failed: {message['error']}\n{message['traceback']}" + ) + if message["type"] != expected_type: + raise RuntimeError( + f"expected worker message {expected_type!r}, got {message['type']!r}" + ) + messages.append(message) + return messages + + +def run_benchmark(config: RunConfig) -> dict[str, Any]: + if config.topology not in {"shared-worker", "per-deck"}: + raise ValueError(f"unknown topology {config.topology!r}") + if config.frames not in {5, 25}: + raise ValueError("issue #109 requires 5-frame or 25-frame runs") + if config.target_ahead_seconds < config.prebuffer_seconds: + raise ValueError("target ahead must be at least the prebuffer threshold") + + context = multiprocessing.get_context("spawn") + result_queue = context.Queue() + worker_ids = [0] if config.topology == "shared-worker" else [0, 1] + request_queues = {worker: context.Queue() for worker in worker_ids} + processes = { + worker: context.Process( + target=_worker_main, + args=(worker, request_queues[worker], result_queue, config), + name=f"mrt2-bench-{worker}", + ) + for worker in worker_ids + } + launch_started = time.perf_counter() + for process in processes.values(): + process.start() + + worker_for_deck = ( + {0: 0, 1: 0} if config.topology == "shared-worker" else {0: 0, 1: 1} + ) + startup: list[dict[str, Any]] = [] + shutdown: list[dict[str, Any]] = [] + try: + startup = _wait_for_messages( + result_queue, + "ready", + len(worker_ids), + config.startup_timeout_seconds, + ) + ready_completed = time.perf_counter() + + # Warm each model process, then clear continuation state before measurement. + for worker in worker_ids: + deck = worker if config.topology == "per-deck" else 0 + request_queues[worker].put( + { + "action": "generate", + "deck": deck, + "sequence": -1, + "frames": config.frames, + "controls": _controls(deck, False, False, config.seed), + "control_change": False, + } + ) + warmup = _wait_for_messages( + result_queue, + "chunk", + len(worker_ids), + config.worker_timeout_seconds, + ) + for worker in worker_ids: + request_queues[worker].put({"action": "reset"}) + _wait_for_messages( + result_queue, + "reset", + len(worker_ids), + config.worker_timeout_seconds, + ) + + origin = time.perf_counter() + deadline = origin + config.duration_seconds + rings = { + deck: RingBudget(config.prebuffer_seconds, last_time=origin) + for deck in DECKS + } + latencies = {deck: [] for deck in DECKS} + change_latencies = {deck: [] for deck in DECKS} + sample_frames = {deck: 0 for deck in DECKS} + sequences = {deck: 0 for deck in DECKS} + changed = {deck: False for deck in DECKS} + onset_pending = {deck: False for deck in DECKS} + inflight_workers: set[int] = set() + rss_peak = {message["worker"]: message.get("rss_bytes") for message in startup} + cuda_peaks = {worker: 0 for worker in worker_ids} + gpu_samples: list[dict[str, Any]] = [] + next_gpu_sample = origin + round_robin = 0 + + while True: + now = time.perf_counter() + bounded_now = min(now, deadline) + for ring in rings.values(): + ring.advance(bounded_now) + + elapsed = bounded_now - origin + for deck in DECKS: + if not changed[deck] and elapsed >= config.prompt_change_seconds: + changed[deck] = True + onset_pending[deck] = True + + if now < deadline: + candidates = list(DECKS) + if config.topology == "shared-worker": + candidates = [round_robin, 1 - round_robin] + for deck in candidates: + worker = worker_for_deck[deck] + if worker in inflight_workers: + continue + if not rings[deck].ready_for_generation( + config.target_ahead_seconds, config.chunk_seconds + ): + continue + request_queues[worker].put( + { + "action": "generate", + "deck": deck, + "sequence": sequences[deck], + "frames": config.frames, + "controls": _controls( + deck, + changed[deck], + onset_pending[deck], + config.seed, + ), + "control_change": onset_pending[deck], + } + ) + sequences[deck] += 1 + inflight_workers.add(worker) + if config.topology == "shared-worker": + round_robin = 1 - deck + + worker_pids = {process.pid for process in processes.values() if process.pid} + if now >= next_gpu_sample: + sample = _gpu_snapshot(worker_pids) + if sample is not None: + sample["elapsed_seconds"] = round(now - origin, 3) + gpu_samples.append(sample) + next_gpu_sample = now + 1.0 + + if now >= deadline and not inflight_workers: + break + try: + message = result_queue.get(timeout=0.05) + except queue.Empty: + continue + if message["type"] == "error": + raise RuntimeError( + f"worker failed: {message['error']}\n{message['traceback']}" + ) + if message["type"] != "chunk": + raise RuntimeError(f"unexpected worker message {message['type']!r}") + worker = message["worker"] + deck = message["deck"] + inflight_workers.discard(worker) + completed = time.perf_counter() + if completed <= deadline: + audio_seconds = message["sample_frames"] / SAMPLE_RATE + rings[deck].push(audio_seconds, completed) + sample_frames[deck] += message["sample_frames"] + latency = message["latency_seconds"] + latencies[deck].append(latency) + if message["control_change"]: + change_latencies[deck].append(latency) + onset_pending[deck] = False + rss = message.get("rss_bytes") + if rss is not None: + rss_peak[worker] = max(rss_peak[worker] or 0, rss) + allocated = message.get("device_memory", {}).get( + "cuda_peak_allocated_bytes" + ) + if allocated is not None: + cuda_peaks[worker] = max(cuda_peaks[worker], allocated) + + ended = time.perf_counter() + for ring in rings.values(): + ring.advance(deadline) + + stop_started = time.perf_counter() + for worker in worker_ids: + request_queues[worker].put({"action": "shutdown"}) + shutdown = _wait_for_messages( + result_queue, + "stopped", + len(worker_ids), + config.worker_timeout_seconds, + ) + for process in processes.values(): + process.join(timeout=config.worker_timeout_seconds) + shutdown_seconds = time.perf_counter() - stop_started + + vram_samples = [sample["worker_vram_mib"] for sample in gpu_samples] + return { + "schema_version": 1, + "qualification": "synthetic" if config.backend == "dry-run" else "hardware", + "config": dataclasses.asdict(config), + "pins": { + "source_repository": SOURCE_REPOSITORY, + "source_revision": SOURCE_REVISION, + "model": { + "repository": MODEL_REVISIONS[config.model][0], + "revision": MODEL_REVISIONS[config.model][1], + }, + "processor": { + "repository": PROCESSOR_REPOSITORY, + "revision": PROCESSOR_REVISION, + }, + }, + "host": { + "platform": platform.platform(), + "system": platform.system(), + "release": platform.release(), + "machine": platform.machine(), + "python": platform.python_version(), + }, + "workers": startup, + "cold_start_wall_seconds": round(ready_completed - launch_started, 6), + "warmup": warmup, + "measurement_wall_seconds": round(ended - origin, 6), + "shutdown_seconds": round(shutdown_seconds, 6), + "shutdown": shutdown, + "failure_domain": ( + "both decks share one process" + if config.topology == "shared-worker" + else "one process failure is isolated to one deck" + ), + "decks": { + str(deck): { + "latency": latency_summary(latencies[deck]), + "control_change_latency": latency_summary(change_latencies[deck]), + "generated_audio_seconds": round( + sample_frames[deck] / SAMPLE_RATE, 6 + ), + "generated_audio_to_wall_ratio": round( + sample_frames[deck] / SAMPLE_RATE / config.duration_seconds, 6 + ), + "ring": rings[deck].result(origin), + } + for deck in DECKS + }, + "memory": { + "worker_rss_peak_bytes": rss_peak, + "worker_cuda_peak_allocated_bytes": cuda_peaks, + "nvidia_worker_vram_peak_mib": max(vram_samples) + if vram_samples + else None, + "nvidia_samples": gpu_samples, + }, + "notes": [ + "underrun_proxy_* is event-level 1.5 s ring simulation, not Rust engine telemetry", + "a hardware qualification must also record the app's engine-reported underrun counter", + ], + } + finally: + for worker, process in processes.items(): + if process.is_alive(): + try: + request_queues[worker].put({"action": "shutdown"}) + process.join(timeout=2) + except (OSError, ValueError): + pass + if process.is_alive(): + process.terminate() + process.join(timeout=2) + for request_queue in request_queues.values(): + request_queue.close() + result_queue.close() + + +def _parse_csv_ints(value: str) -> list[int]: + return [int(item.strip()) for item in value.split(",") if item.strip()] + + +def _parse_csv_strings(value: str) -> list[str]: + return [item.strip() for item in value.split(",") if item.strip()] + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--backend", choices=("dry-run", "upstream"), default="dry-run") + parser.add_argument( + "--topologies", default="shared-worker,per-deck", help="comma-separated matrix" + ) + parser.add_argument("--frames", default="25,5", help="comma-separated matrix") + parser.add_argument("--duration-seconds", type=float, default=600.0) + parser.add_argument( + "--prebuffer-seconds", type=float, default=DEFAULT_PREBUFFER_SECONDS + ) + parser.add_argument( + "--target-ahead-seconds", type=float, default=DEFAULT_PREBUFFER_SECONDS + ) + parser.add_argument("--model", choices=tuple(MODEL_REVISIONS), default="mrt2_small") + parser.add_argument( + "--acceleration", choices=("eager", "torch-compile"), default="eager" + ) + parser.add_argument( + "--token-cfg", + action="store_true", + help="use upstream token CFG instead of MLX-parity classifier-free guidance", + ) + parser.add_argument("--dry-latency-ms", type=float, default=10.0) + parser.add_argument("--startup-timeout-seconds", type=float, default=900.0) + parser.add_argument("--worker-timeout-seconds", type=float, default=300.0) + parser.add_argument("--seed", type=int, default=109) + parser.add_argument("--prompt-change-seconds", type=float, default=30.0) + parser.add_argument("--output", type=pathlib.Path) + return parser.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(argv) + topologies = _parse_csv_strings(args.topologies) + frames = _parse_csv_ints(args.frames) + results = [] + for topology in topologies: + for frame_count in frames: + config = RunConfig( + backend=args.backend, + topology=topology, + frames=frame_count, + duration_seconds=args.duration_seconds, + prebuffer_seconds=args.prebuffer_seconds, + target_ahead_seconds=args.target_ahead_seconds, + model=args.model, + acceleration=args.acceleration, + guidance=not args.token_cfg, + dry_latency_ms=args.dry_latency_ms, + startup_timeout_seconds=args.startup_timeout_seconds, + worker_timeout_seconds=args.worker_timeout_seconds, + seed=args.seed, + prompt_change_seconds=min( + args.prompt_change_seconds, args.duration_seconds / 2 + ), + ) + results.append(run_benchmark(config)) + document = { + "schema_version": 1, + "created_at_utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "results": results, + } + rendered = json.dumps(document, indent=2, sort_keys=True) + if args.output: + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(rendered + "\n", encoding="utf-8") + else: + print(rendered) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/spike/mrt2_pytorch/provenance.json b/spike/mrt2_pytorch/provenance.json new file mode 100644 index 0000000..8a47c12 --- /dev/null +++ b/spike/mrt2_pytorch/provenance.json @@ -0,0 +1,49 @@ +{ + "schema_version": 1, + "audited_at": "2026-08-08", + "code": { + "repository": "https://github.com/multimodalart/magenta-realtime-torch.git", + "revision": "6d076baa3df3b10448876c400521a015a5137c59", + "revision_date": "2026-06-23T23:34:02Z", + "license_file": "https://github.com/multimodalart/magenta-realtime-torch/blob/6d076baa3df3b10448876c400521a015a5137c59/LICENSE", + "license_spdx": "Apache-2.0", + "release_status": "No PyTorch-specific tag or package extra at the audited revision; pin the commit/model snapshot." + }, + "models": { + "mrt2_base": { + "repository": "magenta-community/magenta-realtime-2", + "revision": "92087988d05d0fe38b11f021f0b0d00a75afb86b", + "reported_parameters": 2459164696, + "card_license": "Apache-2.0" + }, + "mrt2_small": { + "repository": "magenta-community/magenta-realtime-2-small", + "revision": "7037d99551c84ac5c6afb7f1a5e58c65e7233dbb", + "reported_parameters": 282195480, + "card_license": "Apache-2.0" + }, + "musiccoca_processor": { + "repository": "magenta-community/magenta-rt-musiccoca-torch", + "revision": "236c488e38aa98643805514996934d705668298b", + "card_license": "CC-BY-4.0" + }, + "original_google_weights": { + "repository": "google/magenta-realtime-2", + "revision": "010aa0dcb0dfd27b24f0ad07b4dad63e8f9521cc", + "card_license": "CC-BY-4.0" + } + }, + "runtime_candidate": { + "python": "3.12", + "torch": "2.12.1", + "transformers": "5.8.0", + "cuda_wheel_candidate": "cu130", + "status": "Registry-verified direct pins; not yet qualified on target hardware." + }, + "known_mutable_defaults_avoided_by_harness": [ + "AutoModel.from_pretrained without revision", + "model.load_processor() default repository without revision", + "hf_hub_download without revision" + ], + "license_review_required": "The derived Transformers cards say Apache-2.0 while the declared base Google weights and MusicCoCa artifacts say CC-BY-4.0. Issue #108 must determine redistribution and attribution obligations." +} diff --git a/spike/mrt2_pytorch/requirements-candidate.txt b/spike/mrt2_pytorch/requirements-candidate.txt new file mode 100644 index 0000000..223ec2a --- /dev/null +++ b/spike/mrt2_pytorch/requirements-candidate.txt @@ -0,0 +1,10 @@ +# Issue #109 benchmark candidate, registry-verified 2026-08-08. +# Direct requirements are exact; produce target-specific hash locks only after +# the Linux/Windows CUDA candidate is selected from real-hardware results. +torch==2.12.1 +transformers==5.8.0 +huggingface-hub==1.1.5 +numpy==2.3.5 +safetensors==0.7.0 +sentencepiece==0.2.1 +resampy==0.4.3 diff --git a/spike/mrt2_pytorch/tests/test_harness.py b/spike/mrt2_pytorch/tests/test_harness.py new file mode 100644 index 0000000..bc1b261 --- /dev/null +++ b/spike/mrt2_pytorch/tests/test_harness.py @@ -0,0 +1,87 @@ +from __future__ import annotations + +import unittest + +from spike.mrt2_pytorch.harness import ( + PROCESSOR_REVISION, + SOURCE_REVISION, + RingBudget, + RunConfig, + latency_summary, + percentile, + run_benchmark, +) + + +class RingBudgetTests(unittest.TestCase): + def test_prebuffer_shortfall_is_not_an_underrun(self) -> None: + ring = RingBudget(1.5, last_time=0.0) + + ring.push(1.0, 0.5) + ring.advance(4.0) + + self.assertFalse(ring.primed) + self.assertEqual(ring.underrun_events, 0) + self.assertEqual(ring.underrun_seconds, 0.0) + + def test_primed_ring_records_starvation_once_until_refilled(self) -> None: + ring = RingBudget(1.5, last_time=0.0) + ring.push(1.5, 0.0) + + ring.advance(2.0) + ring.advance(3.0) + + self.assertTrue(ring.primed) + self.assertEqual(ring.underrun_events, 1) + self.assertAlmostEqual(ring.underrun_seconds, 1.5) + + +class SummaryTests(unittest.TestCase): + def test_percentiles_interpolate_deterministically(self) -> None: + self.assertEqual(percentile([1.0, 2.0, 3.0, 4.0], 50), 2.5) + self.assertEqual(percentile([], 99), None) + + def test_latency_schema_reports_milliseconds(self) -> None: + summary = latency_summary([0.001, 0.003]) + + self.assertEqual(summary["count"], 2) + self.assertEqual(summary["p50_ms"], 2.0) + + +class DryRunTests(unittest.TestCase): + def _config(self, topology: str, frames: int) -> RunConfig: + return RunConfig( + backend="dry-run", + topology=topology, + frames=frames, + duration_seconds=0.12, + prebuffer_seconds=0.04, + target_ahead_seconds=max(0.04, frames * 0.04), + model="mrt2_small", + acceleration="eager", + guidance=True, + dry_latency_ms=1.0, + startup_timeout_seconds=10.0, + worker_timeout_seconds=10.0, + seed=109, + prompt_change_seconds=0.02, + ) + + def test_shared_worker_exercises_two_independent_deck_states(self) -> None: + result = run_benchmark(self._config("shared-worker", 5)) + + self.assertEqual(len(result["workers"]), 1) + self.assertGreater(result["decks"]["0"]["latency"]["count"], 0) + self.assertGreater(result["decks"]["1"]["latency"]["count"], 0) + self.assertTrue(result["decks"]["0"]["ring"]["primed"]) + + def test_per_deck_topology_reports_two_processes_and_exact_pins(self) -> None: + result = run_benchmark(self._config("per-deck", 25)) + + self.assertEqual(len(result["workers"]), 2) + self.assertEqual(result["pins"]["source_revision"], SOURCE_REVISION) + self.assertEqual(result["pins"]["processor"]["revision"], PROCESSOR_REVISION) + + +if __name__ == "__main__": + unittest.main() diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock index a9a2648..d820c16 100644 --- a/src-tauri/Cargo.lock +++ b/src-tauri/Cargo.lock @@ -96,6 +96,15 @@ version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" +dependencies = [ + "derive_arbitrary", +] + [[package]] name = "arrayvec" version = "0.7.6" @@ -603,6 +612,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + [[package]] name = "chacha20" version = "0.10.1" @@ -911,6 +926,17 @@ dependencies = [ "serde_core", ] +[[package]] +name = "derive_arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "derive_more" version = "2.1.1" @@ -1245,6 +1271,7 @@ checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" dependencies = [ "crc32fast", "miniz_oxide", + "zlib-rs", ] [[package]] @@ -1571,8 +1598,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" dependencies = [ "cfg-if", + "js-sys", "libc", "wasi", + "wasm-bindgen", ] [[package]] @@ -1594,11 +1623,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi 6.0.0", "rand_core", "wasip2", "wasip3", + "wasm-bindgen", ] [[package]] @@ -1906,6 +1937,22 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-rustls" +version = "0.27.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "tokio", + "tokio-rustls", + "tower-service", + "webpki-roots", +] + [[package]] name = "hyper-util" version = "0.1.20" @@ -2448,11 +2495,19 @@ version = "0.4.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "953f07c43838f8e6f9758cab68bf5bed85465e7587ebe0b823f1bcd81978ad3a" +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + [[package]] name = "lsdj-app" version = "0.1.0" dependencies = [ "axum", + "flate2", + "hex", "hound", "libc", "lsdj-engine", @@ -2466,7 +2521,9 @@ dependencies = [ "schemars 1.2.1", "serde", "serde_json", + "sha2", "symphonia", + "tar", "tauri", "tauri-build", "tauri-plugin-dialog", @@ -2474,6 +2531,8 @@ dependencies = [ "tokio", "tokio-util", "trash", + "windows-sys 0.61.2", + "zip", ] [[package]] @@ -3372,6 +3431,62 @@ dependencies = [ "memchr", ] +[[package]] +name = "quinn" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror 2.0.18", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560" +dependencies = [ + "bytes", + "getrandom 0.4.2", + "lru-slab", + "rand", + "rand_pcg", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror 2.0.18", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35a133f956daabe89a61a685c2649f13d82d5aa4bd5d12d1277e1072a21c0694" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.61.2", +] + [[package]] name = "quote" version = "1.0.45" @@ -3410,6 +3525,15 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core", +] + [[package]] name = "raw-window-handle" version = "0.6.2" @@ -3502,21 +3626,28 @@ checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" dependencies = [ "base64 0.22.1", "bytes", + "futures-channel", "futures-core", + "futures-util", "http", "http-body", "http-body-util", "hyper", + "hyper-rustls", "hyper-util", "js-sys", "log", "percent-encoding", "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", "sync_wrapper", "tokio", + "tokio-rustls", "tower", "tower-http", "tower-service", @@ -3524,6 +3655,7 @@ dependencies = [ "wasm-bindgen", "wasm-bindgen-futures", "web-sys", + "webpki-roots", ] [[package]] @@ -3593,6 +3725,20 @@ dependencies = [ "windows-sys 0.60.2", ] +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + [[package]] name = "rmcp" version = "1.8.0" @@ -3730,6 +3876,41 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "rustls" +version = "0.23.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0283386ce02abc0151e1761d08802dfe86c173b0b494af5cbc086574e453da06" +dependencies = [ + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + [[package]] name = "rustversion" version = "1.0.22" @@ -4227,6 +4408,12 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + [[package]] name = "swift-rs" version = "1.0.7" @@ -4538,6 +4725,17 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "tar" +version = "0.4.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f6221d9a6003c78398e3b239969f352578258df48c8eb051caadae0015bc840" +dependencies = [ + "filetime", + "libc", + "xattr", +] + [[package]] name = "target-lexicon" version = "0.12.16" @@ -4990,6 +5188,16 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + [[package]] name = "tokio-stream" version = "0.1.18" @@ -5350,6 +5558,12 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + [[package]] name = "url" version = "2.5.8" @@ -5603,6 +5817,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "web_atoms" version = "0.2.4" @@ -5659,6 +5883,15 @@ dependencies = [ "system-deps", ] +[[package]] +name = "webpki-roots" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "webview2-com" version = "0.38.2" @@ -6517,6 +6750,16 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "xattr" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" +dependencies = [ + "libc", + "rustix", +] + [[package]] name = "yoke" version = "0.8.3" @@ -6642,6 +6885,12 @@ dependencies = [ "synstructure", ] +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" + [[package]] name = "zerotrie" version = "0.2.4" @@ -6675,12 +6924,44 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "zip" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa8cd6af31c3b31c6631b8f483848b91589021b28fffe50adada48d4f4d2ed1" +dependencies = [ + "arbitrary", + "crc32fast", + "flate2", + "indexmap 2.14.0", + "memchr", + "zopfli", +] + +[[package]] +name = "zlib-rs" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34b31d188d9d685a4f9c7b46d6e36631b07058d2cfe190267adce54dc230bf12" + [[package]] name = "zmij" version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" +[[package]] +name = "zopfli" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249" +dependencies = [ + "bumpalo", + "crc32fast", + "log", + "simd-adler32", +] + [[package]] name = "zvariant" version = "5.12.0" diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index 09efc56..cfb3142 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -20,6 +20,12 @@ path = "src/main.rs" # runtime and startup must resolve it from Contents/Resources. Developer builds # omit the feature and retain the uv-based source-tree commands. bundled-backend = [] +# Linux and Windows packages never consult developer tooling. They resolve a +# hash-verified, atomically promoted service manifest under app-owned storage. +managed-runtime = [] +# Compatibility name used by the Windows packaging slice. Keep one policy in +# Rust even while the stacked shipping PRs converge on the canonical name. +managed-backend = ["managed-runtime"] # The cargo WORKSPACE. The app is the root package; the audio engine is a member # library crate so it stays headless-testable (`cargo test -p lsdj-engine`) @@ -72,15 +78,23 @@ serde_json = "1" rmcp = { version = "1.8", default-features = false, features = ["server", "transport-streamable-http-server", "macros", "schemars"] } axum = "0.8" tokio-util = "0.7.18" -tokio = { version = "1.52.3", features = ["rt-multi-thread", "net"] } +tokio = { version = "1.52.3", features = ["rt-multi-thread", "net", "time"] } schemars = "1" rand = "0.10" -# The MCP `generate_sample` tool proxies the loopback generation server's -# `/api/generate` from the Rust shell (the webview's HTTP client isn't reachable -# from a tool). reqwest is the de-facto async Rust HTTP client; loopback is plain -# HTTP, so default TLS is dropped to keep the dependency lean. Shares axum's -# hyper/tokio tree. -reqwest = { version = "0.12", default-features = false, features = ["json"] } +# The MCP `generate_sample` proxies the loopback generation server, while the +# model installer uses the blocking client on its dedicated worker thread. +# Rustls keeps authenticated HTTPS portable without depending on OpenSSL/curl. +reqwest = { version = "0.12", default-features = false, features = ["blocking", "json", "rustls-tls"] } +# Native, bounded `.tar.gz` extraction and application-controlled SHA-256 checks +# for downloaded runtime/model artifacts (issue #107). Exact pins make changes +# to the installer trust boundary explicit in review. +flate2 = "=1.1.9" +tar = "=0.4.46" +# Official uv Windows releases are ZIP archives. Keep extraction native and +# apply the same path/count/expanded-size policy as the tar.gz installer path. +zip = { version = "=4.6.1", default-features = false, features = ["deflate"] } +sha2 = "=0.10.9" +hex = "=0.4.3" # Shell-side track decode (ADR-0030): pure-Rust demux + decode covering the # folder browser's extension allowlist (wav/aiff via riff, flac/ogg-vorbis in # the defaults, mp3, and m4a via isomp4+aac+alac). The de-facto Rust audio @@ -96,6 +110,20 @@ rubato = { version = "=3.0.0", default-features = false, features = ["fft_resamp # instead of relying on rubato/realfft's transitive copy. rustfft = "=6.4.1" +# Windows process-tree lifetime: Job Objects keep wrappers and every descendant +# tied to the Rust host, including when it exits without running destructors. +# The child is created suspended and resumed only after assignment, closing the +# spawn/assign race. Target-scoped so Unix builds carry no Windows surface. +[target.'cfg(windows)'.dependencies] +windows-sys = { version = "0.61", features = [ + "Win32_Foundation", + "Win32_Security", + "Win32_System_Diagnostics_ToolHelp", + "Win32_System_JobObjects", + "Win32_System_SystemInformation", + "Win32_System_Threading", +] } + [dev-dependencies] # The ADR-0025 corpus regression replays the spike WAVs (16-bit PCM) through # the Rust estimator; hound is the de-facto WAV codec — reputable, tiny, and diff --git a/src-tauri/engine/src/device.rs b/src-tauri/engine/src/device.rs index 1e29f04..80a238a 100644 --- a/src-tauri/engine/src/device.rs +++ b/src-tauri/engine/src/device.rs @@ -1,6 +1,6 @@ //! The cpal device wrapper: a thin host around the device-free [`Engine`] core. //! -//! Opens a stereo / f32 output stream with `BufferSize::Fixed(256)` and, in its +//! Opens an output stream with `BufferSize::Fixed(256)` and, in its //! callback, sets FTZ/DAZ once and drains the engine's output ring(s) wrapped in //! `assert_no_alloc`. The callback is the ONLY real-time path; it allocates //! nothing, takes no lock, makes no syscall, and logs nothing. Ported from the @@ -8,13 +8,14 @@ //! now built on the library so the device path stays exercisable. //! //! The engine renders at exactly [`SAMPLE_RATE`] (48000). A device that offers a -//! 48000/f32 config is opened there and drained bit-exact (the fast path). A -//! device with NO 48000/f32 config — e.g. a 44100 Bluetooth speaker — is opened -//! at its own f32 rate and the 48 kHz stream is resampled to it on the callback -//! via [`OutputResampler`] (ADR-0029). All device-rate knowledge lives here; the -//! host's output ring stays a clean 48 kHz interleaved-stereo contract. +//! 48000 config in a supported sample format (`f32`, `i16`, or `u16`) is opened +//! there. A device with no 48000 config — e.g. a 44100 Bluetooth speaker — is +//! opened at its own rate and the 48 kHz stream is resampled to it on the callback +//! via [`OutputResampler`] (ADR-0029). Format conversion, clipping, and device +//! channel mapping happen only at the final callback boundary; the host's output +//! ring stays a clean 48 kHz interleaved-stereo `f32` contract. //! -//! Graceful no-device exit: if no output device or no usable f32 config is +//! Graceful no-device exit: if no output device or no usable config is //! available (likely in a sandbox / headless CI), [`run_stream`] returns //! [`DeviceError::Unavailable`] rather than hanging or panicking. @@ -22,6 +23,8 @@ use cpal::traits::{DeviceTrait, HostTrait, StreamTrait}; use cpal::{BufferSize, StreamConfig}; use rubato::audioadapter_buffers::direct::InterleavedSlice; use rubato::{Fft, FixedSync, Resampler}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; use crate::host::OutputConsumer; use crate::{Engine, CHANNELS, SAMPLE_RATE}; @@ -34,7 +37,7 @@ const REQUESTED_BUFFER: u32 = 256; /// case — callers treat it as "no device, exit cleanly", not a failure. #[derive(Debug)] pub enum DeviceError { - /// No output device, or no usable f32 config at all (e.g. a sandbox). A + /// No output device, or no usable config at all (e.g. a sandbox). A /// non-48000 device is NOT this case anymore — it is opened and resampled /// (ADR-0029). Not a bug — exit cleanly. Unavailable(String), @@ -59,27 +62,59 @@ pub struct StreamInfo { pub device_name: String, pub device_channels: u16, pub sample_rate: u32, + pub sample_format: cpal::SampleFormat, pub buffer_frames: BufferSize, } +/// A cloneable, allocation-free health signal shared with CPAL's error callback. +/// The signal is deliberately one-way: a failed stream is replaced, never reset. +#[derive(Clone)] +struct StreamHealth(Arc); + +impl StreamHealth { + fn healthy() -> Self { + Self(Arc::new(AtomicBool::new(true))) + } + + fn mark_failed(&self) { + self.0.store(false, Ordering::Release); + } + + fn is_healthy(&self) -> bool { + self.0.load(Ordering::Acquire) + } +} + /// A running output stream driving an [`Engine`]. The cpal stream stops when this /// is dropped; the `Engine` lives inside the callback for the stream's lifetime. pub struct AudioStream { _stream: cpal::Stream, info: StreamInfo, + /// CPAL invokes its error callback on a backend-owned thread. Keep that path + /// bounded and non-blocking: it flips this preallocated atomic and does no + /// logging, allocation, locking, or application IPC. The shell polls the + /// value off the real-time path and owns user-facing diagnostics/recovery. + healthy: StreamHealth, } impl AudioStream { pub fn info(&self) -> &StreamInfo { &self.info } + + /// Whether CPAL has reported an asynchronous error since this stream was + /// started. A newly opened stream is healthy; recovery replaces the stream + /// (and therefore this signal) rather than trying to reset it in place. + pub fn is_healthy(&self) -> bool { + self.healthy.is_healthy() + } } /// One output device the engine can open, for the picker UI. pub struct OutputDeviceInfo { pub name: String, - /// Channels of its chosen usable (f32, ≥ stereo) config — the widest 48000/f32 - /// config, or the fallback config when the device cannot do 48000. + /// Channels of its chosen usable (`f32`, `i16`, or `u16`) config — the widest + /// preferred 48000 config, or the fallback when the device cannot do 48000. pub channels: u16, /// Whether it can carry the headphone cue: a ≥4-channel device lands master /// on 1/2 and the cue on 3/4 (the FLX4 phones jack). @@ -94,15 +129,28 @@ fn device_name(device: &cpal::Device) -> String { .unwrap_or_else(|_| "".into()) } +/// The device-boundary sample formats the engine supports. The engine and output +/// rings remain `f32`; integer support is deliberately confined to the final, +/// allocation-free callback conversion. +fn sample_format_rank(format: cpal::SampleFormat) -> Option { + match format { + cpal::SampleFormat::F32 => Some(0), + cpal::SampleFormat::I16 => Some(1), + cpal::SampleFormat::U16 => Some(2), + _ => None, + } +} + /// Choose a device's output config for the engine. Preference order: /// -/// 1. An exact 48000/f32 config (≥ stereo), WIDEST channel count — the bit-exact -/// fast path. A ≥4-channel device (the FLX4) lands master on 1/2, cue on 3/4. -/// 2. Otherwise the device's own default config, if it is f32 / ≥ stereo — its -/// nominal rate (e.g. 44100 for a Bluetooth speaker), which the OS will not -/// itself resample, so we resample 48000 → it directly (ADR-0029). -/// 3. Otherwise any f32 / ≥ stereo config, at the supported rate NEAREST 48000 -/// (widest channels as a tie-break). +/// 1. An exact 48000 config, preferring the widest channel layout and then `f32`, +/// `i16`, and `u16` within that layout. A ≥4-channel device (the FLX4) lands +/// master on 1/2 and cue on 3/4; a mono device receives a balanced downmix. +/// 2. Otherwise the device's own default config, when its sample format is +/// supported — its nominal rate (e.g. 44100 for a Bluetooth speaker), which +/// the OS will not itself resample, so we resample 48000 → it directly. +/// 3. Otherwise any supported config, at the supported rate NEAREST 48000 +/// (widest channels, then `f32`/`i16`/`u16`, as tie-breaks). /// /// The returned config's `sample_rate()` is the rate the stream opens at; the /// caller resamples when it is not [`SAMPLE_RATE`]. @@ -110,37 +158,43 @@ fn pick_config(device: &cpal::Device) -> Option { let exact = device.supported_output_configs().ok().and_then(|configs| { configs .filter(|cfg| { - cfg.channels() >= CHANNELS - && cfg.sample_format() == cpal::SampleFormat::F32 + cfg.channels() > 0 + && sample_format_rank(cfg.sample_format()).is_some() && cfg.min_sample_rate() <= SAMPLE_RATE && cfg.max_sample_rate() >= SAMPLE_RATE }) - .max_by_key(|cfg| cfg.channels()) + .min_by_key(|cfg| { + ( + u16::MAX - cfg.channels(), + sample_format_rank(cfg.sample_format()).unwrap_or(u8::MAX), + ) + }) .map(|cfg| cfg.with_sample_rate(SAMPLE_RATE)) }); if exact.is_some() { return exact; } - // No 48000/f32 config: fall back to a resampled rate. Prefer the device's own + // No exact 48000 config: fall back to a resampled rate. Prefer the device's own // default (its nominal rate, so the OS does not double-resample under us). if let Ok(default) = device.default_output_config() { - if default.sample_format() == cpal::SampleFormat::F32 && default.channels() >= CHANNELS { + if default.channels() > 0 && sample_format_rank(default.sample_format()).is_some() { return Some(default); } } - // Last resort: the f32 / ≥ stereo config whose supported range lands a rate - // closest to 48000 (widest channels breaks ties). `clamp` gives the nearest - // in-range rate — for a 44100-only device that is 44100. + // Last resort: the supported config whose range lands nearest 48000. `clamp` + // gives the nearest in-range rate — for a 44100-only device that is 44100. device.supported_output_configs().ok().and_then(|configs| { configs - .filter(|cfg| { - cfg.channels() >= CHANNELS && cfg.sample_format() == cpal::SampleFormat::F32 - }) + .filter(|cfg| cfg.channels() > 0 && sample_format_rank(cfg.sample_format()).is_some()) .min_by_key(|cfg| { let rate = SAMPLE_RATE.clamp(cfg.min_sample_rate(), cfg.max_sample_rate()); - (rate.abs_diff(SAMPLE_RATE), u16::MAX - cfg.channels()) + ( + rate.abs_diff(SAMPLE_RATE), + u16::MAX - cfg.channels(), + sample_format_rank(cfg.sample_format()).unwrap_or(u8::MAX), + ) }) .map(|cfg| { let rate = SAMPLE_RATE.clamp(cfg.min_sample_rate(), cfg.max_sample_rate()); @@ -149,10 +203,10 @@ fn pick_config(device: &cpal::Device) -> Option { }) } -/// Enumerate the output devices the engine can open (any f32, ≥ stereo config — -/// exact 48000 or a resampled fallback) with their chosen channel count, for the -/// picker. Off the RT path — called from a command when the picker opens. Empty -/// on a headless host. +/// Enumerate the output devices the engine can open (any `f32`, `i16`, or `u16` +/// config — exact 48000 or a resampled fallback) with their chosen channel count, +/// for the picker. Off the RT path — called from a command when the picker opens. +/// Empty on a headless host. pub fn list_output_devices() -> Vec { let host = cpal::default_host(); let Ok(devices) = host.output_devices() else { @@ -185,9 +239,9 @@ fn find_output_device(host: &cpal::Host, name: &str) -> Result, ) -> Result<(cpal::Device, StreamConfig, StreamInfo), DeviceError> { @@ -203,13 +257,14 @@ fn open_output( let supported = pick_config(&device).ok_or_else(|| { DeviceError::Unavailable(format!( - "device '{device_name}' has no usable f32 output config \ - (no f32 ≥ stereo config at any sample rate)" + "device '{device_name}' has no usable output config \ + (supported sample formats: f32, i16, u16)" )) })?; let device_channels = supported.channels(); let device_rate = supported.sample_rate(); + let sample_format = supported.sample_format(); let buffer_size = match supported.buffer_size() { cpal::SupportedBufferSize::Range { min, max } => { BufferSize::Fixed(REQUESTED_BUFFER.clamp(*min, *max)) @@ -227,30 +282,98 @@ fn open_output( device_name, device_channels, sample_rate: device_rate, + sample_format, buffer_frames: buffer_size, }; Ok((device, config, info)) } -/// Zero a wider interleaved device buffer, then lay each `(channel_offset, src)` -/// interleaved-stereo block onto channels `[offset, offset+1]`. One primitive for -/// every routing: master on 1/2 is `&[(0, master)]`; the FLX4 combined path is -/// `&[(0, master), (2, cue)]`; a split cue on the FLX4 phones is `&[(2, cue)]`. -/// A frame past a block's length (or an offset past the device) is left silent. -/// `placements` is a small fixed stack slice, so this stays pure and alloc-free — -/// RT-safe. -fn spread(data: &mut [f32], dev_ch: usize, placements: &[(usize, &[f32])]) { +/// A sample type accepted at the cpal device boundary. Conversions explicitly +/// clamp before quantization: `dasp_sample`'s raw `f32 → i16` conversion wraps at +/// exactly `1.0`, which is not acceptable when a limiter or resampler reaches the +/// positive endpoint. Implementations are branch/arithmetic only and RT-safe. +trait DeviceSample: cpal::SizedSample + Copy + Send + 'static { + fn from_f32_clipped(sample: f32) -> Self; +} + +/// Clamp finite and infinite values to the cpal PCM domain. A NaN is silence: +/// propagating it to a float device can poison downstream host processing, while +/// integer casts happen to turn it into zero and would otherwise disagree. +#[inline] +fn clip_sample(sample: f32) -> f32 { + if sample.is_nan() { + 0.0 + } else { + sample.clamp(-1.0, 1.0) + } +} + +impl DeviceSample for f32 { + #[inline] + fn from_f32_clipped(sample: f32) -> Self { + clip_sample(sample) + } +} + +impl DeviceSample for i16 { + #[inline] + fn from_f32_clipped(sample: f32) -> Self { + let sample = clip_sample(sample); + if sample <= -1.0 { + i16::MIN + } else if sample >= 1.0 { + i16::MAX + } else { + (sample * 32_768.0).round() as i16 + } + } +} + +impl DeviceSample for u16 { + #[inline] + fn from_f32_clipped(sample: f32) -> Self { + let sample = clip_sample(sample); + if sample <= -1.0 { + u16::MIN + } else if sample >= 1.0 { + u16::MAX + } else { + ((sample + 1.0) * 32_767.5).round() as u16 + } + } +} + +/// Silence an interleaved device buffer, then map each `(channel_offset, src)` +/// interleaved-stereo `f32` block into it while clipping and converting to the +/// device sample type. Master on 1/2 is `&[(0, master)]`; the FLX4 combined path +/// is `&[(0, master), (2, cue)]`; a split FLX4 cue is `&[(2, cue)]`. +/// +/// A mono device gets `(left + right) / 2`, which preserves headroom and avoids +/// selecting one side of stereo content. A frame past a source's length (or an +/// offset past the device) stays silent. `placements` is a fixed stack slice, so +/// this is allocation/lock/syscall/log free and safe in the realtime callback. +fn write_mapped(data: &mut [T], dev_ch: usize, placements: &[(usize, &[f32])]) { + let silence = T::from_f32_clipped(0.0); + data.fill(silence); + if dev_ch == 0 { + return; + } + let frames = data.len() / dev_ch; for f in 0..frames { - let base = f * dev_ch; - for c in 0..dev_ch { - data[base + c] = 0.0; - } for &(offset, src) in placements { - if offset + 1 < dev_ch && 2 * f + 1 < src.len() { - data[base + offset] = src[2 * f]; - data[base + offset + 1] = src[2 * f + 1]; + let src_base = 2 * f; + if src_base + 1 >= src.len() { + continue; + } + if dev_ch == 1 && offset == 0 { + let mono = (src[src_base] + src[src_base + 1]) * 0.5; + data[f] = T::from_f32_clipped(mono); + } else if offset + 1 < dev_ch { + let dst_base = f * dev_ch + offset; + data[dst_base] = T::from_f32_clipped(src[src_base]); + data[dst_base + 1] = T::from_f32_clipped(src[src_base + 1]); } } } @@ -262,7 +385,7 @@ fn spread(data: &mut [f32], dev_ch: usize, placements: &[(usize, &[f32])]) { const RESAMPLER_BLOCKS: usize = 2; /// Resamples the engine's 48 kHz interleaved-stereo feed to a device with no -/// 48000/f32 config (e.g. a 44100 Bluetooth speaker), via rubato's synchronous +/// usable 48000 config (e.g. a 44100 Bluetooth speaker), via rubato's synchronous /// FFT resampler at the fixed [`SAMPLE_RATE`] → device-rate ratio (ADR-0029). /// /// rubato works in fixed `chunk_frames` blocks; the cpal callback's block size is @@ -313,7 +436,14 @@ impl OutputResampler { let input = vec![0.0; resampler.input_frames_max() * CHANNELS as usize]; let chunk = vec![0.0; chunk_frames * CHANNELS as usize]; let carry = vec![0.0; chunk_frames * CHANNELS as usize]; - Some(OutputResampler { resampler, input, chunk, carry, carry_len: 0, chunk_frames }) + Some(OutputResampler { + resampler, + input, + chunk, + carry, + carry_len: 0, + chunk_frames, + }) } /// **RT path.** Fill `out` (interleaved-stereo at the device rate) entirely, @@ -382,7 +512,123 @@ impl OutputResampler { } } -/// Open `selected` (a 48000/f32 config, or a resampled fallback rate), build a +/// The complete mutable state captured by a production cpal output callback. +/// Constructed before stream start; its vectors never resize after capture. +struct OutputWriter { + device_channels: usize, + primary_offset: usize, + primary: OutputConsumer, + secondary: Option, + primary_resampler: Option, + secondary_resampler: Option, + scratch: Vec, + secondary_scratch: Vec, +} + +impl OutputWriter { + /// **RT path.** Drain/resample and convert an entire cpal callback in bounded, + /// frame-aligned tiles. The scratch buffers are reusable working space, not a + /// limit on callback length: hosts may deliver a block larger than the granted + /// size, and every frame still consumes its matching ring input. Iteration is + /// arithmetic over preallocated slices only. + fn write(&mut self, data: &mut [T]) { + let silence = T::from_f32_clipped(0.0); + let dev_ch = self.device_channels; + if dev_ch == 0 { + data.fill(silence); + return; + } + + let primary_scratch_frames = self.scratch.len() / CHANNELS as usize; + let tile_frames = if self.secondary.is_some() { + primary_scratch_frames.min(self.secondary_scratch.len() / CHANNELS as usize) + } else { + primary_scratch_frames + }; + if tile_frames == 0 { + data.fill(silence); + return; + } + + let total_frames = data.len() / dev_ch; + let mut frame_start = 0; + while frame_start < total_frames { + let frames = (total_frames - frame_start).min(tile_frames); + let device_start = frame_start * dev_ch; + let device_end = device_start + frames * dev_ch; + let stereo_samples = frames * CHANNELS as usize; + let primary_tile = &mut self.scratch[..stereo_samples]; + + if let Some(resampler) = self.primary_resampler.as_mut() { + resampler.fill(&mut self.primary, primary_tile); + } else { + self.primary.drain_into(primary_tile); + } + + if let Some(secondary) = self.secondary.as_mut() { + let secondary_tile = &mut self.secondary_scratch[..stereo_samples]; + if let Some(resampler) = self.secondary_resampler.as_mut() { + resampler.fill(secondary, secondary_tile); + } else { + secondary.drain_into(secondary_tile); + } + write_mapped( + &mut data[device_start..device_end], + dev_ch, + &[(0, primary_tile), (2, secondary_tile)], + ); + } else { + write_mapped( + &mut data[device_start..device_end], + dev_ch, + &[(self.primary_offset, primary_tile)], + ); + } + frame_start += frames; + } + + // cpal supplies whole frames, but make a malformed trailing partial frame + // deterministic and silent without reading another engine frame. + data[total_frames * dev_ch..].fill(silence); + } +} + +/// **RT path.** The legacy engine-in-callback exerciser uses the same bounded +/// tiling rule as [`OutputWriter::write`], rendering every callback frame even +/// when cpal hands it a block larger than the preallocated scratch. +fn render_engine_chunks( + data: &mut [T], + dev_ch: usize, + engine: &mut Engine, + scratch: &mut [f32], +) { + let silence = T::from_f32_clipped(0.0); + if dev_ch == 0 { + data.fill(silence); + return; + } + let tile_frames = scratch.len() / CHANNELS as usize; + if tile_frames == 0 { + data.fill(silence); + return; + } + + let total_frames = data.len() / dev_ch; + let mut frame_start = 0; + while frame_start < total_frames { + let frames = (total_frames - frame_start).min(tile_frames); + let device_start = frame_start * dev_ch; + let device_end = device_start + frames * dev_ch; + let stereo_samples = frames * CHANNELS as usize; + let tile = &mut scratch[..stereo_samples]; + engine.render(tile, frames); + write_mapped(&mut data[device_start..device_end], dev_ch, &[(0, tile)]); + frame_start += frames; + } + data[total_frames * dev_ch..].fill(silence); +} + +/// Open `selected` (a supported 48000 config, or a resampled fallback rate), build a /// stream that drains `primary` onto channels 1/2 — and, when `secondary` is /// `Some` AND the device has ≥4 channels, /// also drains it onto channels 3/4 (the FLX4 combined master+cue path). On a @@ -408,28 +654,63 @@ impl OutputResampler { /// the rings; with no device nothing drains them, which is fine). fn open_spread_stream( selected: Option<&str>, - mut primary: OutputConsumer, + primary: OutputConsumer, secondary: Option, primary_on_phones: bool, ) -> Result { let (device, config, info) = open_output(selected)?; + match info.sample_format { + cpal::SampleFormat::F32 => { + build_spread_stream::(device, config, info, primary, secondary, primary_on_phones) + } + cpal::SampleFormat::I16 => { + build_spread_stream::(device, config, info, primary, secondary, primary_on_phones) + } + cpal::SampleFormat::U16 => { + build_spread_stream::(device, config, info, primary, secondary, primary_on_phones) + } + format => Err(DeviceError::Unavailable(format!( + "selected unsupported output sample format {format}" + ))), + } +} + +/// Typed half of [`open_spread_stream`]. The sample-format dispatch happens once, +/// before cpal starts the stream; every callback then drains/resamples into fixed +/// `f32` scratch and performs only channel mapping plus scalar conversion. +fn build_spread_stream( + device: cpal::Device, + config: StreamConfig, + info: StreamInfo, + primary: OutputConsumer, + secondary: Option, + primary_on_phones: bool, +) -> Result { let device_channels = info.device_channels as usize; // The secondary (cue) feed needs channels 3/4 — only a ≥4-channel device (the // FLX4) can carry it alongside the primary. Drop it on a narrower device. - let mut secondary = if device_channels >= 4 { secondary } else { None }; + let secondary = if device_channels >= 4 { + secondary + } else { + None + }; let secondary_routed = secondary.is_some(); // Where the primary lands: a standalone cue stream on a ≥4-channel device (the // FLX4 chosen as a SEPARATE cue device) belongs on the phones channels 3/4 // (offset 2), not 1/2 (its MASTER RCA). Master, and a cue on a stereo device // (laptop jack, Bluetooth), land on 1/2 (offset 0). - let primary_offset = if primary_on_phones && device_channels >= 4 { 2 } else { 0 }; + let primary_offset = if primary_on_phones && device_channels >= 4 { + 2 + } else { + 0 + }; // When the device opened at a rate other than the engine's 48 kHz, build a // resampler per feed (off the RT path; the callback only `fill`s them). A // failure here is fatal — playing 48 kHz audio straight into a 44.1 kHz buffer // would be pitched wrong — so it surfaces as a stream error. The resampler's - // chunk granularity is the requested buffer; its FIFO decouples that from the + // chunk granularity is the granted buffer; its FIFO decouples that from the // actual callback block size, so a varying block is served exactly. let device_rate = info.sample_rate; let chunk_frames = match info.buffer_frames { @@ -443,101 +724,56 @@ fn open_spread_stream( )) }) }; - let mut primary_resampler = if device_rate != SAMPLE_RATE { + let primary_resampler = if device_rate != SAMPLE_RATE { Some(build_resampler("master")?) } else { None }; - let mut secondary_resampler = if device_rate != SAMPLE_RATE && secondary_routed { + let secondary_resampler = if device_rate != SAMPLE_RATE && secondary_routed { Some(build_resampler("cue")?) } else { None }; let mut first_call = true; - // Per-callback scratch for wide (>2ch) devices: the rings (and the resamplers) - // produce interleaved stereo, so on a wider device we gather stereo into these - // scratches and spread into the device buffer — same path whether the stereo - // came from a straight ring drain or a resample. Sized ONCE here, off the RT - // path, for a generous worst-case block; the callback never resizes them. + // Per-callback f32 scratch: rings and resamplers remain interleaved stereo, + // regardless of the device's sample format or channel layout. Sized ONCE here, + // off the RT path, for a generous worst-case block; the callback never resizes. let mut scratch: Vec = Vec::new(); let mut secondary_scratch: Vec = Vec::new(); - if device_channels != CHANNELS as usize { - scratch_reserve(&mut scratch, REQUESTED_BUFFER as usize * 4); - if secondary_routed { - scratch_reserve(&mut secondary_scratch, REQUESTED_BUFFER as usize * 4); - } + scratch_reserve(&mut scratch, chunk_frames.saturating_mul(4)); + if secondary_routed { + scratch_reserve(&mut secondary_scratch, chunk_frames.saturating_mul(4)); } + let mut output = OutputWriter { + device_channels, + primary_offset, + primary, + secondary, + primary_resampler, + secondary_resampler, + scratch, + secondary_scratch, + }; - let err_fn = |e| eprintln!("lsdj-engine: stream error: {e}"); + let healthy = StreamHealth::healthy(); + let error_health = healthy.clone(); + let err_fn = move |_error| { + // This may run on an audio-backend thread. Never format/log/lock/send + // from here: one atomic store is sufficient for the shell's live poll. + error_health.mark_failed(); + }; let stream = device .build_output_stream( config, - move |data: &mut [f32], _info: &cpal::OutputCallbackInfo| { + move |data: &mut [T], _info: &cpal::OutputCallbackInfo| { no_alloc(|| { if first_call { set_ftz_daz(); first_call = false; } - let dev_ch = device_channels; - match primary_resampler.as_mut() { - // Bit-exact: device runs at 48 kHz, drain straight through. - None => { - if dev_ch == CHANNELS as usize { - // Stereo fast path: drain into the device buffer. - primary.drain_into(data); - } else { - // Wider device: drain stereo into scratch, spread. - let want = (data.len() / dev_ch) * CHANNELS as usize; - let usable = scratch.len().min(want); - primary.drain_into(&mut scratch[..usable]); - if let Some(secondary) = secondary.as_mut() { - // Combined: primary on 1/2, secondary (cue) 3/4. - let su = secondary_scratch.len().min(want); - secondary.drain_into(&mut secondary_scratch[..su]); - spread( - data, - dev_ch, - &[(0, &scratch[..usable]), (2, &secondary_scratch[..su])], - ); - } else { - // Lone feed (master, or a split cue). - spread(data, dev_ch, &[(primary_offset, &scratch[..usable])]); - } - } - } - // Device runs at another rate: resample 48 kHz → device rate. - // `fill` serves exactly the bytes asked for (its FIFO absorbs - // the chunk-vs-block-size difference). - Some(pr) => { - if dev_ch == CHANNELS as usize { - // Stereo device: resample into the device buffer. - pr.fill(&mut primary, data); - } else { - // Wider device: resample into scratch, then spread. - let want = (data.len() / dev_ch) * CHANNELS as usize; - let usable = scratch.len().min(want); - if let (Some(sr), Some(secondary)) = - (secondary_resampler.as_mut(), secondary.as_mut()) - { - // Combined: both feeds resampled, master 1/2, cue 3/4. - let su = secondary_scratch.len().min(want); - pr.fill(&mut primary, &mut scratch[..usable]); - sr.fill(secondary, &mut secondary_scratch[..su]); - spread( - data, - dev_ch, - &[(0, &scratch[..usable]), (2, &secondary_scratch[..su])], - ); - } else { - // Lone feed (master, or a split cue). - pr.fill(&mut primary, &mut scratch[..usable]); - spread(data, dev_ch, &[(primary_offset, &scratch[..usable])]); - } - } - } - } + output.write(data); }); }, err_fn, @@ -552,6 +788,7 @@ fn open_spread_stream( Ok(AudioStream { _stream: stream, info, + healthy, }) } @@ -583,9 +820,9 @@ pub fn open_cue_stream( open_spread_stream(cue_dev, cue, None, true) } -/// Open the default output device at exactly 48000/stereo/f32, build the stream -/// that renders `engine` in its callback, start it, and return the running -/// stream. The `engine` is MOVED into the audio callback. +/// Open the default output device at exactly 48000 in a supported sample format, +/// build the stream that renders `engine` in its callback, start it, and return +/// the running stream. The `engine` is MOVED into the audio callback. /// /// This is the original engine-in-callback path (Phase 1 / `device_run`). The /// Tauri app now drives audio through [`open_main_stream`] / [`open_cue_stream`] + @@ -597,7 +834,7 @@ pub fn open_cue_stream( /// On any sandbox/headless condition (no device, no 48000 config) this returns /// [`DeviceError::Unavailable`] without hanging — the caller decides whether that /// is fatal. -pub fn run_stream(mut engine: Engine) -> Result { +pub fn run_stream(engine: Engine) -> Result { let (device, config, info) = open_output(None)?; if info.sample_rate != SAMPLE_RATE { return Err(DeviceError::Unavailable(format!( @@ -605,26 +842,49 @@ pub fn run_stream(mut engine: Engine) -> Result { info.sample_rate ))); } - let device_channels = info.device_channels; + match info.sample_format { + cpal::SampleFormat::F32 => build_engine_stream::(device, config, info, engine), + cpal::SampleFormat::I16 => build_engine_stream::(device, config, info, engine), + cpal::SampleFormat::U16 => build_engine_stream::(device, config, info, engine), + format => Err(DeviceError::Unavailable(format!( + "selected unsupported output sample format {format}" + ))), + } +} - // Per-callback scratch for wide (>2ch) devices: the engine renders exactly - // stereo, so on a wider device we render into this stereo scratch and spread - // it into the device buffer (extra channels zeroed). On the common stereo - // device the scratch stays empty and the fast path renders straight into - // `data`. Sized ONCE here, off the RT path, for a generous worst-case block - // (4× the requested buffer); the callback never resizes it. +/// Typed implementation of the legacy engine-in-callback hardware spike. The +/// production app uses [`build_spread_stream`], but keeping this path typed makes +/// the standalone device exerciser work on integer-only hosts too. +fn build_engine_stream( + device: cpal::Device, + config: StreamConfig, + info: StreamInfo, + mut engine: Engine, +) -> Result { + let device_channels = info.device_channels as usize; + + // The engine renders internal stereo f32 into a pre-sized buffer; typed PCM + // conversion and channel mapping happen in the same final-boundary primitive + // as the production ring-drain path. Sized ONCE here, off the RT path. let mut first_call = true; let mut scratch: Vec = Vec::new(); - if device_channels as usize != CHANNELS as usize { - scratch_reserve(&mut scratch, REQUESTED_BUFFER as usize * 4); - } + let granted_frames = match info.buffer_frames { + BufferSize::Fixed(n) => n as usize, + BufferSize::Default => REQUESTED_BUFFER as usize, + }; + scratch_reserve(&mut scratch, granted_frames.saturating_mul(4)); - let err_fn = |e| eprintln!("lsdj-engine: stream error: {e}"); + let healthy = StreamHealth::healthy(); + let error_health = healthy.clone(); + let err_fn = move |_error| { + // Same non-blocking contract as the production spread-stream path. + error_health.mark_failed(); + }; let stream = device .build_output_stream( config, - move |data: &mut [f32], _info: &cpal::OutputCallbackInfo| { + move |data: &mut [T], _info: &cpal::OutputCallbackInfo| { // Everything below MUST be alloc/lock/syscall/log free. The guard // proves it (warns in release if violated). crate::device::no_alloc(|| { @@ -632,35 +892,7 @@ pub fn run_stream(mut engine: Engine) -> Result { crate::device::set_ftz_daz(); first_call = false; } - let dev_ch = device_channels as usize; - let frames = data.len() / dev_ch; - - if dev_ch == CHANNELS as usize { - // Stereo fast path: render straight into the device buffer. - engine.render(data, frames); - } else { - // Wider device: render stereo into scratch, then spread. - // `scratch` was pre-sized below on the first wide call; - // if cpal ever hands a bigger block than expected we skip - // the overflow rather than alloc on the RT thread. - let want = frames * CHANNELS as usize; - let usable = scratch.len().min(want); - let frames_usable = usable / CHANNELS as usize; - engine.render(&mut scratch[..usable], frames_usable); - for f in 0..frames { - let base = f * dev_ch; - if f < frames_usable { - data[base] = scratch[2 * f]; - data[base + 1] = scratch[2 * f + 1]; - } else { - data[base] = 0.0; - data[base + 1] = 0.0; - } - for c in 2..dev_ch { - data[base + c] = 0.0; - } - } - } + render_engine_chunks(data, device_channels, &mut engine, &mut scratch); }); }, err_fn, @@ -675,6 +907,7 @@ pub fn run_stream(mut engine: Engine) -> Result { Ok(AudioStream { _stream: stream, info, + healthy, }) } @@ -682,7 +915,7 @@ pub fn run_stream(mut engine: Engine) -> Result { /// callback. Pulled out so the intent — allocate the worst-case block ONCE, /// never on the RT thread — is explicit. fn scratch_reserve(scratch: &mut Vec, frames: usize) { - scratch.resize(frames * CHANNELS as usize, 0.0); + scratch.resize(frames.saturating_mul(CHANNELS as usize), 0.0); } /// `assert_no_alloc` wrapper, isolated here so `lib.rs`/tests don't depend on the @@ -694,21 +927,20 @@ pub(crate) fn no_alloc(f: impl FnOnce() -> T) -> T { } /// Enable flush-to-zero / denormals-are-zero on the calling (audio) thread so a -/// decaying denormal tail never trips the CPU's slow denormal path. Ported -/// verbatim from the spike. +/// decaying denormal tail never trips the CPU's slow denormal path. Derived from +/// the spike, with direct MXCSR access for cross-toolchain compatibility. #[inline] pub(crate) fn set_ftz_daz() { - #[cfg(all(target_arch = "x86_64", target_feature = "sse"))] + #[cfg(target_arch = "x86_64")] unsafe { - use std::arch::x86_64::{ - _MM_FLUSH_ZERO_ON, _MM_GET_FLUSH_ZERO_MODE, _MM_SET_FLUSH_ZERO_MODE, - }; - let _ = _MM_GET_FLUSH_ZERO_MODE(); - _MM_SET_FLUSH_ZERO_MODE(_MM_FLUSH_ZERO_ON); - // DAZ via the MXCSR DAZ bit (bit 6). - let mut mxcsr: u32; + // x86_64 guarantees SSE2. Manipulate MXCSR directly instead of the + // deprecated `_MM_*FLUSH_ZERO*` intrinsics so this also compiles cleanly + // under MSVC. Initialized storage is required because the inline assembly + // writes through a pointer, which Rust's definite-initialization analysis + // deliberately does not infer. + let mut mxcsr = 0u32; std::arch::asm!("stmxcsr [{}]", in(reg) &mut mxcsr, options(nostack)); - mxcsr |= 1 << 6; + mxcsr |= (1 << 15) | (1 << 6); // FTZ | DAZ std::arch::asm!("ldmxcsr [{}]", in(reg) &mxcsr, options(nostack, readonly)); } #[cfg(target_arch = "aarch64")] @@ -723,9 +955,203 @@ pub(crate) fn set_ftz_daz() { #[cfg(test)] mod tests { - use super::{spread, OutputConsumer, OutputResampler, CHANNELS, SAMPLE_RATE}; + use super::{ + write_mapped, DeviceSample, OutputConsumer, OutputResampler, OutputWriter, StreamHealth, + CHANNELS, SAMPLE_RATE, + }; use rubato::Resampler; + #[test] + fn stream_health_is_one_way_and_shared_with_the_error_callback() { + let health = StreamHealth::healthy(); + let callback_view = health.clone(); + assert!(health.is_healthy()); + + callback_view.mark_failed(); + + assert!(!health.is_healthy()); + assert!(!callback_view.is_healthy()); + } + + /// The three supported output types agree at silence and both PCM endpoints; + /// out-of-domain values clip instead of wrapping, and NaN becomes silence. + #[test] + fn device_sample_conversion_clips_extrema() { + let source = [ + f32::NEG_INFINITY, + -1.5, + -1.0, + -0.5, + 0.0, + 0.5, + 1.0, + 1.5, + f32::INFINITY, + f32::NAN, + ]; + let f32_out = source.map(::from_f32_clipped); + assert_eq!( + f32_out, + [-1.0, -1.0, -1.0, -0.5, 0.0, 0.5, 1.0, 1.0, 1.0, 0.0] + ); + + let i16_out = source.map(::from_f32_clipped); + assert_eq!( + i16_out, + [ + i16::MIN, + i16::MIN, + i16::MIN, + -16_384, + 0, + 16_384, + i16::MAX, + i16::MAX, + i16::MAX, + 0, + ] + ); + + let u16_out = source.map(::from_f32_clipped); + assert_eq!( + u16_out, + [ + 0, + 0, + 0, + 16_384, + 32_768, + 49_151, + u16::MAX, + u16::MAX, + u16::MAX, + 32_768 + ] + ); + } + + /// A mono host receives a headroom-preserving arithmetic downmix rather than + /// silently losing either side; the final result is clipped to the PCM range. + #[test] + fn write_mapped_downmixes_stereo_to_mono() { + let src = [1.0, -1.0, 0.8, 0.4, 2.0, 2.0]; + let mut data = [9.0f32; 3]; + write_mapped(&mut data, 1, &[(0, &src)]); + assert_eq!(data, [0.0, 0.6, 1.0]); + } + + /// Stereo is the identity layout apart from the required final-boundary clip. + #[test] + fn write_mapped_preserves_stereo_and_clips() { + let src = [-2.0, 0.25, 0.5, 2.0]; + let mut data = [9.0f32; 4]; + write_mapped(&mut data, 2, &[(0, &src)]); + assert_eq!(data, [-1.0, 0.25, 0.5, 1.0]); + } + + /// Integer callbacks use the same layout primitive, including unsigned PCM's + /// non-zero equilibrium value. + #[test] + fn write_mapped_converts_stereo_to_integer_pcm() { + let src = [-1.0, 0.0, 0.5, 1.0]; + let mut i16_data = [9i16; 4]; + write_mapped(&mut i16_data, 2, &[(0, &src)]); + assert_eq!(i16_data, [i16::MIN, 0, 16_384, i16::MAX]); + + let mut u16_data = [9u16; 4]; + write_mapped(&mut u16_data, 2, &[(0, &src)]); + assert_eq!(u16_data, [u16::MIN, 32_768, 49_151, u16::MAX]); + } + + /// A callback larger than the fixed scratch is processed tile-by-tile. Every + /// output frame consumes its matching ring frame; no tail is zeroed or left + /// poisoned after the first tile. + #[test] + fn long_callback_drains_primary_past_scratch_capacity() { + const FRAMES: usize = 19; + let source: Vec = (0..FRAMES) + .flat_map(|frame| { + let sample = frame as f32 / FRAMES as f32; + [sample, -sample] + }) + .collect(); + let (mut producer, primary) = OutputConsumer::new_test_pair(FRAMES + 1); + for &sample in &source { + assert!(producer.push(sample).is_ok()); + } + + let mut output = OutputWriter { + device_channels: CHANNELS as usize, + primary_offset: 0, + primary, + secondary: None, + primary_resampler: None, + secondary_resampler: None, + scratch: vec![0.0; 3 * CHANNELS as usize], + secondary_scratch: Vec::new(), + }; + let mut data = vec![9.0f32; FRAMES * CHANNELS as usize]; + output.write(&mut data); + + assert_eq!(data, source, "all 19 frames survive a 3-frame scratch tile"); + } + + /// Combined master/cue routing remains aligned across many scratch boundaries, + /// including when the secondary scratch is the smaller tiling constraint. + #[test] + fn long_callback_drains_primary_and_secondary_in_lockstep() { + const FRAMES: usize = 17; + const DEVICE_CHANNELS: usize = 6; + let master: Vec = (0..FRAMES) + .flat_map(|frame| { + let sample = frame as f32 * 0.01; + [sample, -sample] + }) + .collect(); + let cue: Vec = (0..FRAMES) + .flat_map(|frame| { + let sample = 0.2 + frame as f32 * 0.01; + [sample, -sample] + }) + .collect(); + let (mut master_producer, primary) = OutputConsumer::new_test_pair(FRAMES + 1); + let (mut cue_producer, cue_consumer) = OutputConsumer::new_test_pair(FRAMES + 1); + for &sample in &master { + assert!(master_producer.push(sample).is_ok()); + } + for &sample in &cue { + assert!(cue_producer.push(sample).is_ok()); + } + + let mut output = OutputWriter { + device_channels: DEVICE_CHANNELS, + primary_offset: 0, + primary, + secondary: Some(cue_consumer), + primary_resampler: None, + secondary_resampler: None, + scratch: vec![0.0; 4 * CHANNELS as usize], + secondary_scratch: vec![0.0; 2 * CHANNELS as usize], + }; + let mut data = vec![9.0f32; FRAMES * DEVICE_CHANNELS]; + output.write(&mut data); + + for (frame, output) in data.chunks_exact(DEVICE_CHANNELS).enumerate() { + assert_eq!( + output, + &[ + master[2 * frame], + master[2 * frame + 1], + cue[2 * frame], + cue[2 * frame + 1], + 0.0, + 0.0, + ], + "frame {frame} stays aligned after repeated two-frame tiles", + ); + } + } + /// A lone block at offset 0 lands on channels 1/2 and zeroes the rest of each /// frame (master, or a split cue on a stereo/wide non-FLX4 device). #[test] @@ -733,7 +1159,7 @@ mod tests { let src = [0.1, 0.2, 0.3, 0.4]; // two stereo frames let dev_ch = 4; let mut data = vec![9.0f32; 2 * dev_ch]; // pre-fill to prove zeroing - spread(&mut data, dev_ch, &[(0, &src)]); + write_mapped(&mut data, dev_ch, &[(0, &src)]); assert_eq!(data, vec![0.1, 0.2, 0.0, 0.0, 0.3, 0.4, 0.0, 0.0]); } @@ -744,7 +1170,7 @@ mod tests { let cue = [0.7, 0.8]; // one stereo frame let dev_ch = 4; let mut data = vec![9.0f32; dev_ch]; - spread(&mut data, dev_ch, &[(2, &cue)]); + write_mapped(&mut data, dev_ch, &[(2, &cue)]); assert_eq!(data, vec![0.0, 0.0, 0.7, 0.8]); } @@ -755,7 +1181,7 @@ mod tests { let src = [0.5, 0.6]; // one stereo frame only let dev_ch = 2; let mut data = vec![9.0f32; 2 * dev_ch]; // two frames - spread(&mut data, dev_ch, &[(0, &src)]); + write_mapped(&mut data, dev_ch, &[(0, &src)]); assert_eq!(data, vec![0.5, 0.6, 0.0, 0.0]); } @@ -767,7 +1193,7 @@ mod tests { let cue = [0.7, 0.8]; let dev_ch = 6; let mut data = vec![9.0f32; dev_ch]; - spread(&mut data, dev_ch, &[(0, &master), (2, &cue)]); + write_mapped(&mut data, dev_ch, &[(0, &master), (2, &cue)]); assert_eq!(data, vec![0.1, 0.2, 0.7, 0.8, 0.0, 0.0]); } @@ -779,7 +1205,7 @@ mod tests { let cue = [0.7, 0.8]; // one frame — second frame's cue runs dry let dev_ch = 4; let mut data = vec![9.0f32; 2 * dev_ch]; - spread(&mut data, dev_ch, &[(0, &master), (2, &cue)]); + write_mapped(&mut data, dev_ch, &[(0, &master), (2, &cue)]); assert_eq!( data, vec![0.1, 0.2, 0.7, 0.8, 0.3, 0.4, 0.0, 0.0], @@ -801,6 +1227,48 @@ mod tests { /// Resampler chunk size (frames) used throughout. const CHUNK_FRAMES: usize = 256; + /// Oversized callbacks also tile correctly after the non-48k resampler. Once + /// startup latency clears, a DC signal must fill the entire long callback, + /// including its final frame beyond many scratch boundaries. + #[test] + fn long_resampled_callback_has_no_silent_tail() { + const LEFT: f32 = 0.3; + const RIGHT: f32 = -0.3; + const TILE_FRAMES: usize = 3; + const CALLBACK_FRAMES: usize = 29; + let (mut producer, primary) = OutputConsumer::new_test_pair(1 << 16); + while producer.slots() >= CHANNELS as usize { + assert!(producer.push(LEFT).is_ok()); + assert!(producer.push(RIGHT).is_ok()); + } + + let mut output = OutputWriter { + device_channels: CHANNELS as usize, + primary_offset: 0, + primary, + secondary: None, + primary_resampler: Some( + OutputResampler::new(DEVICE_RATE, 8).expect("small 44.1k resampler builds"), + ), + secondary_resampler: None, + scratch: vec![0.0; TILE_FRAMES * CHANNELS as usize], + secondary_scratch: Vec::new(), + }; + let mut warmup = vec![0.0f32; 8 * CHANNELS as usize]; + for _ in 0..32 { + output.write(&mut warmup); + } + + let mut data = vec![-9.0f32; CALLBACK_FRAMES * CHANNELS as usize]; + output.write(&mut data); + assert!( + data.chunks_exact(CHANNELS as usize) + .all(|frame| { (frame[0] - LEFT).abs() < 0.02 && (frame[1] - RIGHT).abs() < 0.02 }), + "all {CALLBACK_FRAMES} frames are resampled through {TILE_FRAMES}-frame tiles: {:?}", + &data[data.len() - 8..], + ); + } + /// Fill `n_in` interleaved-stereo frames of `input` with a 48 kHz-domain sine at /// `freq`, continuing from `phase0`; returns the phase to resume from so /// successive blocks stay continuous. @@ -824,7 +1292,10 @@ mod tests { OutputResampler::new(DEVICE_RATE, CHUNK_FRAMES).expect("44.1k resampler builds"); assert_eq!(r.chunk.len(), CHUNK_FRAMES * CHANNELS as usize); let n_in = r.resampler.input_frames_next(); - assert!(n_in >= CHUNK_FRAMES, "downsample pulls ≥ output frames, got {n_in}"); + assert!( + n_in >= CHUNK_FRAMES, + "downsample pulls ≥ output frames, got {n_in}" + ); assert!( r.input.len() >= n_in * CHANNELS as usize, "input scratch ({}) fits the demand ({n_in})", @@ -832,7 +1303,10 @@ mod tests { ); fill_sine(&mut r.input, n_in, 1_000.0, 0.0); assert!(r.resample_chunk(n_in), "resample succeeds"); - assert!(r.chunk.iter().all(|s| s.is_finite()), "output is finite (no NaN/inf)"); + assert!( + r.chunk.iter().all(|s| s.is_finite()), + "output is finite (no NaN/inf)" + ); } /// Over many blocks the resampler consumes input at exactly the 48000/44100 @@ -885,7 +1359,10 @@ mod tests { let out_rms = (sum_sq / n as f64).sqrt(); let in_rms = 0.5 / std::f64::consts::SQRT_2; // amplitude-0.5 sine let db = 20.0 * (out_rms / in_rms).log10(); - assert!(db.abs() < 1.0, "sine energy preserved within 1 dB, got {db:.2} dB (rms {out_rms:.4})"); + assert!( + db.abs() < 1.0, + "sine energy preserved within 1 dB, got {db:.2} dB (rms {out_rms:.4})" + ); } /// `fill` serves any block size — including ones that differ from the resampler @@ -929,9 +1406,9 @@ mod tests { out[..len].fill(-9.0); r.fill(&mut consumer, &mut out[..len]); top_up(&mut producer); - let ok = out[..len].chunks_exact(CHANNELS as usize).all(|frame| { - (frame[0] - LEFT).abs() < 0.02 && (frame[1] - RIGHT).abs() < 0.02 - }); + let ok = out[..len] + .chunks_exact(CHANNELS as usize) + .all(|frame| (frame[0] - LEFT).abs() < 0.02 && (frame[1] - RIGHT).abs() < 0.02); assert!( ok, "every {bf}-frame block keeps left≈{LEFT}/right≈{RIGHT} (continuous, \ diff --git a/src-tauri/engine/src/host.rs b/src-tauri/engine/src/host.rs index 6961dc3..e6494a1 100644 --- a/src-tauri/engine/src/host.rs +++ b/src-tauri/engine/src/host.rs @@ -389,6 +389,13 @@ enum Command { SwapMasterRing(Producer), /// Switch the CUE output device: the cue counterpart of [`SwapMasterRing`]. SwapCueRing(Producer), + /// Switch the MASTER and CUE producers as one queue operation. Combined + /// output streams consume both rings, so accepting only one half would leave + /// the other bus connected to a stream that is about to be dropped. + SwapOutputRings { + master: Producer, + cue: Producer, + }, } /// A point-in-time copy of the per-deck state the IPC thread reads back: track @@ -505,9 +512,9 @@ impl OutputConsumer { /// A freshly-built output ring producer (master OR cue) for an output-device /// switch. Built off the render thread ([`Host::new_output_ring`]) and handed /// back to it via [`Host::install_master_ring`] -/// / [`Host::install_cue_ring`] once the new device stream is open on the matching -/// consumer. Opaque: the producer only travels from the host into the render -/// thread. +/// / [`Host::install_cue_ring`] (or paired by [`Host::install_output_rings`]) +/// once the new device stream is open on the matching consumer. Opaque: the +/// producer only travels from the host into the render thread. pub struct OutputRing(Producer); /// Owns the [`Engine`] on a dedicated render thread and exposes thread-safe @@ -532,6 +539,23 @@ pub struct Host { render_thread: Option>, } +/// Push one complete control operation onto the single-writer command ring. +/// Kept separate from `Host` so saturation behavior can be exercised without a +/// render thread racing the test's queue setup. +fn enqueue_command(commands: &Mutex>, command: Command) -> bool { + let mut producer = match commands.lock() { + Ok(producer) => producer, + Err(poisoned) => poisoned.into_inner(), + }; + match producer.push(command) { + Ok(()) => true, + Err(_) => { + eprintln!("lsdj-host: command queue full — dropping a control command"); + false + } + } +} + impl Host { /// Build the engine, create its [`DECK_COUNT`] decks, KEEP the engine on a /// newly spawned render thread, and return the [`Host`], the device-side @@ -607,7 +631,9 @@ impl Host { /// (carried in [`OutputRing`]) and its device-side [`OutputConsumer`] for the /// new cpal stream. The ring is symmetric — master vs cue is decided by which /// of [`install_master_ring`](Self::install_master_ring) / - /// [`install_cue_ring`](Self::install_cue_ring) the caller installs it with. + /// [`install_cue_ring`](Self::install_cue_ring) the caller installs it with, + /// or by pairing both rings in + /// [`install_output_rings`](Self::install_output_rings). /// The render thread keeps filling the CURRENT ring until that install swaps /// it, so the caller opens the new stream on this consumer FIRST and only /// installs on success, leaving audio undisturbed if the device fails to open. @@ -638,23 +664,23 @@ impl Host { self.send(Command::SwapCueRing(ring.0)) } + /// Hand a combined stream's MASTER and CUE rings to the render thread in one + /// command. The queue accepts both producers or neither; callers may only + /// retire the previous split cue after this returns true. + pub fn install_output_rings(&self, master: OutputRing, cue: OutputRing) -> bool { + self.send(Command::SwapOutputRings { + master: master.0, + cue: cue.0, + }) + } + /// Enqueue a command for the render thread. Drops the command (logged) if the /// queue is momentarily full — a non-blocking control surface never stalls the /// caller (the UI/IPC thread). Returns whether the command was enqueued. fn send(&self, command: Command) -> bool { // The Mutex only serialises IPC callers against each other (the producer // half is single-writer); it is never touched by the cpal callback. - let mut producer = match self.commands.lock() { - Ok(p) => p, - Err(poisoned) => poisoned.into_inner(), - }; - match producer.push(command) { - Ok(()) => true, - Err(_) => { - eprintln!("lsdj-host: command queue full — dropping a control command"); - false - } - } + enqueue_command(&self.commands, command) } // --- Control surface (one method per Engine control op) --- @@ -1078,6 +1104,13 @@ impl RenderLoop { // so a cue-device change never disturbs the master stream. self.cue_output = cue; } + Command::SwapOutputRings { master, cue } => { + // A combined device drains both rings. Apply the pair from one + // command so queue pressure can never expose a half-swapped + // topology to the render loop or its device stream. + self.output = master; + self.cue_output = cue; + } } } @@ -1659,4 +1692,67 @@ mod tests { "after the swap the render loop fills the new cue ring" ); } + + #[test] + fn swap_output_rings_repoints_master_and_cue_together() { + let mut host = TestHost::new(); + let (master_tx, master_rx) = + RingBuffer::::new(OUTPUT_RING_FRAMES * CHANNELS as usize); + let (cue_tx, cue_rx) = RingBuffer::::new(OUTPUT_RING_FRAMES * CHANNELS as usize); + + host.send(Command::SwapOutputRings { + master: master_tx, + cue: cue_tx, + }); + host.loop_state.step(); + + assert!(master_rx.slots() > 0, "the new master ring is filled"); + assert!(cue_rx.slots() > 0, "the new cue ring is filled"); + } + + #[test] + fn combined_ring_install_uses_one_queue_slot() { + let (mut command_tx, mut command_rx) = + RingBuffer::::new(COMMAND_QUEUE_DEPTH); + for _ in 0..COMMAND_QUEUE_DEPTH - 1 { + assert!(command_tx.push(Command::SetCrossfade(0.5)).is_ok()); + } + let (master, _master_rx) = RingBuffer::::new(1); + let (cue, _cue_rx) = RingBuffer::::new(1); + let commands = Mutex::new(command_tx); + + assert!(enqueue_command( + &commands, + Command::SwapOutputRings { master, cue } + )); + for _ in 0..COMMAND_QUEUE_DEPTH - 1 { + assert!(matches!(command_rx.pop(), Ok(Command::SetCrossfade(_)))); + } + assert!(matches!( + command_rx.pop(), + Ok(Command::SwapOutputRings { .. }) + )); + assert!(command_rx.pop().is_err()); + } + + #[test] + fn full_queue_rejects_the_combined_ring_pair_without_partial_commands() { + let (mut command_tx, mut command_rx) = + RingBuffer::::new(COMMAND_QUEUE_DEPTH); + for _ in 0..COMMAND_QUEUE_DEPTH { + assert!(command_tx.push(Command::SetCrossfade(0.5)).is_ok()); + } + let (master, _master_rx) = RingBuffer::::new(1); + let (cue, _cue_rx) = RingBuffer::::new(1); + let commands = Mutex::new(command_tx); + + assert!(!enqueue_command( + &commands, + Command::SwapOutputRings { master, cue } + )); + for _ in 0..COMMAND_QUEUE_DEPTH { + assert!(matches!(command_rx.pop(), Ok(Command::SetCrossfade(_)))); + } + assert!(command_rx.pop().is_err()); + } } diff --git a/src-tauri/src/child_process.rs b/src-tauri/src/child_process.rs index 3eb1e3b..0116f7e 100644 --- a/src-tauri/src/child_process.rs +++ b/src-tauri/src/child_process.rs @@ -1,54 +1,1282 @@ -//! Process-tree supervision for Python services launched through wrappers such as -//! `uv run`. Killing only the immediate [`Child`] would orphan the real Python -//! process, so every supervised command gets its own process group and teardown -//! signals the whole group before reaping its leader. - -use std::io; -use std::process::{Child, Command}; - -/// Spawn `command` as the leader of a fresh process group. Descendants inherit -/// that group, which lets [`kill_group`] take down wrappers and their workers in -/// one operation. -pub(crate) fn spawn_grouped(command: &mut Command) -> io::Result { - #[cfg(unix)] - { - use std::os::unix::process::CommandExt; - command.process_group(0); +//! Shared process-tree supervision for every long-lived backend command. +//! +//! Development commands often have a wrapper topology (`uv` -> Python -> model +//! workers), so owning only [`Child`] is not enough. [`SupervisedChild`] keeps +//! the platform tree-lifetime primitive alive for exactly as long as the +//! service: +//! +//! - Unix children lead a fresh process group. A small, syscall-only watchdog +//! inherited during spawn observes a close-on-exec pipe, and kills the group +//! if the Rust host exits without running destructors. +//! - Windows children are created suspended, assigned to a kill-on-close Job +//! Object, and only then resumed. Suspending closes the otherwise unavoidable +//! race where the child could create an untracked descendant before job +//! assignment. +//! +//! The handle also centralises bounded readiness polling, graceful/forced +//! shutdown, startup-failure cleanup, and diagnostic redaction. + +use std::collections::VecDeque; +use std::io::{self, Read}; +use std::process::{Child, ChildStderr, ChildStdout, Command, ExitStatus}; +use std::time::{Duration, Instant}; + +const POLL_INTERVAL: Duration = Duration::from_millis(20); +const FORCE_WAIT: Duration = Duration::from_secs(2); +#[cfg(unix)] +const TREE_REAP_SWEEPS: usize = 100; +const DIAGNOSTIC_BYTES: usize = 16 * 1024; +const DIAGNOSTIC_LINES: usize = 128; +const DIAGNOSTIC_LINE_BYTES: usize = 2048; +const CHILD_LINE_BYTES: usize = 64 * 1024; + +/// Result of a bounded readiness wait. +#[derive(Debug)] +pub(crate) enum Readiness { + Ready, + Exited(ExitStatus), + TimedOut, +} + +/// What happened while stopping a supervised service. +#[derive(Debug)] +pub(crate) struct ShutdownReport { + pub(crate) status: Option, + pub(crate) forced: bool, +} + +/// Emit one bounded lifecycle diagnostic when graceful teardown needed force or +/// supervision itself failed. Normal graceful exits stay quiet. +pub(crate) fn log_shutdown(label: &str, result: io::Result) { + let message = match result { + Ok(report) if report.forced => { + let status = report + .status + .map(|status| status.to_string()) + .unwrap_or_else(|| "status unavailable".to_string()); + Some(format!( + "lsdj-app: {label}: forced process-tree shutdown ({status})" + )) + } + Err(error) => Some(format!( + "lsdj-app: {label}: process-tree shutdown failed: {error}" + )), + Ok(_) => None, + }; + if let Some(message) = message { + eprintln!("{}", sanitize_diagnostic(&message)); } - command.spawn() } -/// Kill a supervised child's entire process group and reap the immediate child. +/// A child plus the platform primitive that owns all of its descendants. +pub(crate) struct SupervisedChild { + child: Child, + exit_status: Option, + tree_cleaned: bool, + #[cfg(unix)] + parent_guard: Option, + #[cfg(windows)] + job: Option, +} + +/// Spawn `command` under process-tree supervision. /// -/// The bounded re-sweep closes the same mid-fork race guarded by the model -/// installer's teardown: a descendant forked during the first signal remains in -/// the group and is caught by a subsequent pass. -pub(crate) fn kill_group(child: &mut Child) { +/// The input remains a [`Command`], rather than a shell string, so executable, +/// arguments, environment, CWD, and stdio stay structured and paths containing +/// spaces or Unicode pass to the OS unchanged. +pub(crate) fn spawn_grouped(command: &mut Command) -> io::Result { + scrub_child_environment(command); #[cfg(unix)] { - let group = -(child.id() as libc::pid_t); - // SAFETY: `child` was created by [`spawn_grouped`], so its live pid is - // also the process-group id. A negative pid targets that group. - unsafe { - libc::kill(group, libc::SIGKILL); + spawn_unix(command) + } + + #[cfg(windows)] + { + spawn_windows(command) + } + + #[cfg(not(any(unix, windows)))] + { + let child = command.spawn()?; + Ok(SupervisedChild { + child, + exit_status: None, + tree_cleaned: false, + }) + } +} + +/// Remove credentials and interpreter/package-manager injection knobs from all +/// supervised children. Application-owned values such as `UV_CACHE_DIR` and +/// `HF_HUB_OFFLINE` remain explicit on the command; ambient user/session values +/// may not alter executable discovery, dependency resolution, or diagnostics. +fn scrub_child_environment(command: &mut Command) { + const KEYS: &[&str] = &[ + "HF_TOKEN", + "HUGGING_FACE_HUB_TOKEN", + "PYTHONPATH", + "PYTHONHOME", + "PYTHONSTARTUP", + "PYTHONINSPECT", + "PYTHONBREAKPOINT", + "PYTHONUSERBASE", + "PYTHONWARNINGS", + "PYTHONPYCACHEPREFIX", + "UV_CONFIG_FILE", + "UV_PROJECT", + "UV_WORKING_DIR", + "UV_CONSTRAINT", + "UV_BUILD_CONSTRAINT", + "UV_OVERRIDE", + "UV_EXCLUDE", + "UV_FIND_LINKS", + "UV_INDEX", + "UV_INDEX_URL", + "UV_EXTRA_INDEX_URL", + "UV_DEFAULT_INDEX", + "UV_INDEX_STRATEGY", + "UV_INSECURE_HOST", + "UV_KEYRING_PROVIDER", + "UV_NO_VERIFY_HASHES", + "UV_SYSTEM_PYTHON", + "UV_PYTHON", + "UV_PYTHON_DOWNLOADS", + "PIP_CONFIG_FILE", + "PIP_CONSTRAINT", + "PIP_REQUIRE_VIRTUALENV", + "PIP_INDEX_URL", + "PIP_EXTRA_INDEX_URL", + "PIP_TRUSTED_HOST", + "PIP_FIND_LINKS", + ]; + for key in KEYS { + command.env_remove(key); + } +} + +impl SupervisedChild { + #[cfg(unix)] + pub(crate) fn id(&self) -> u32 { + self.child.id() + } + + pub(crate) fn take_stdout(&mut self) -> Option { + self.child.stdout.take() + } + + pub(crate) fn take_stderr(&mut self) -> Option { + self.child.stderr.take() + } + + /// Poll the leader. If it exited, also remove any descendants it left + /// behind before returning the status. + pub(crate) fn try_wait(&mut self) -> io::Result> { + if let Some(status) = self.exit_status { + return Ok(Some(status)); } - let _ = child.wait(); - for _ in 0..100 { - // SAFETY: signal 0 probes group liveness without signalling it. - if unsafe { libc::kill(group, 0) } == -1 { - break; + let status = self.child.try_wait()?; + if let Some(status) = status { + self.exit_status = Some(status); + self.cleanup_remaining_tree(); + } + Ok(status) + } + + /// Wait for the leader and clean up descendants left by a wrapper that + /// returned before its worker. + pub(crate) fn wait(&mut self) -> io::Result { + if let Some(status) = self.exit_status { + return Ok(status); + } + let status = self.child.wait()?; + self.exit_status = Some(status); + self.cleanup_remaining_tree(); + Ok(status) + } + + /// Poll a service-specific readiness probe until it succeeds, the child + /// exits, or the timeout expires. Readiness transport stays with the owner + /// (TCP for generation/sidecars); lifecycle and timeout semantics live here. + pub(crate) fn wait_for_readiness( + &mut self, + timeout: Duration, + mut probe: impl FnMut() -> io::Result, + ) -> io::Result { + let deadline = Instant::now() + timeout; + loop { + if probe()? { + return Ok(Readiness::Ready); + } + if let Some(status) = self.try_wait()? { + return Ok(Readiness::Exited(status)); } - // SAFETY: as above, target only the supervised process group. + if Instant::now() >= deadline { + return Ok(Readiness::TimedOut); + } + std::thread::sleep( + POLL_INTERVAL.min(deadline.saturating_duration_since(Instant::now())), + ); + } + } + + /// Give a service a bounded grace period, then force its entire tree down. + /// + /// Unix sends `SIGTERM` to the group first. Windows has no generally safe + /// graceful signal for GUI/background processes, so its grace period lets a + /// protocol shutdown (socket EOF, for example) complete before the Job is + /// terminated. + pub(crate) fn shutdown(&mut self, grace: Duration) -> io::Result { + self.shutdown_with_hook(grace, || {}) + } + + fn shutdown_with_hook( + &mut self, + grace: Duration, + after_graceful_signal: impl FnOnce(), + ) -> io::Result { + if let Some(status) = self.try_wait()? { + return Ok(ShutdownReport { + status: Some(status), + forced: false, + }); + } + + #[cfg(unix)] + self.signal_unix_group(libc::SIGTERM); + after_graceful_signal(); + + if let Some(status) = self.wait_direct_timeout(grace)? { + self.cleanup_remaining_tree(); + return Ok(ShutdownReport { + status: Some(status), + forced: false, + }); + } + + let status = self.force_kill()?; + Ok(ShutdownReport { + status, + forced: true, + }) + } + + /// Immediately terminate the complete tree and wait a bounded amount of + /// time for the leader to become reapable. + pub(crate) fn force_kill(&mut self) -> io::Result> { + if let Some(status) = self.exit_status { + self.cleanup_remaining_tree(); + return Ok(Some(status)); + } + self.force_tree(); + let status = self.wait_direct_timeout(FORCE_WAIT)?; + self.cleanup_remaining_tree(); + if self.exit_status.is_none() { + return Err(io::Error::new( + io::ErrorKind::TimedOut, + "process leader did not exit after tree termination", + )); + } + Ok(status) + } + + fn wait_direct_timeout(&mut self, timeout: Duration) -> io::Result> { + if let Some(status) = self.exit_status { + return Ok(Some(status)); + } + let deadline = Instant::now() + timeout; + loop { + if let Some(status) = self.child.try_wait()? { + self.exit_status = Some(status); + return Ok(Some(status)); + } + if Instant::now() >= deadline { + return Ok(None); + } + std::thread::sleep( + POLL_INTERVAL.min(deadline.saturating_duration_since(Instant::now())), + ); + } + } + + fn force_tree(&mut self) { + #[cfg(unix)] + self.signal_unix_group(libc::SIGKILL); + + #[cfg(windows)] + if let Some(job) = self.job.as_ref() { + use std::os::windows::io::AsRawHandle; + use windows_sys::Win32::System::JobObjects::TerminateJobObject; + // SAFETY: the handle is a live Job Object owned by `self`. unsafe { - libc::kill(group, libc::SIGKILL); + TerminateJobObject(job.as_raw_handle() as _, 1); + } + } + + #[cfg(not(any(unix, windows)))] + { + let _ = self.child.kill(); + } + } + + fn cleanup_remaining_tree(&mut self) { + if self.tree_cleaned { + return; + } + self.force_tree(); + + #[cfg(unix)] + { + // Closing the pipe also wakes the abnormal-exit watchdog if it won a + // race with the group signal. + self.parent_guard.take(); + let group = -(self.id() as libc::pid_t); + for _ in 0..TREE_REAP_SWEEPS { + // SAFETY: signal 0 probes the supervised group without sending a + // signal. The group id was fixed at spawn. + if unsafe { libc::kill(group, 0) } == -1 { + break; + } + // SAFETY: target only the supervised process group. + unsafe { + libc::kill(group, libc::SIGKILL); + } + std::thread::sleep(Duration::from_millis(10)); } - std::thread::sleep(std::time::Duration::from_millis(10)); } + + #[cfg(windows)] + { + // KILL_ON_JOB_CLOSE is the abnormal-exit guarantee; explicit job + // termination above makes normal cleanup deterministic. + self.job.take(); + } + self.tree_cleaned = true; } - #[cfg(not(unix))] + #[cfg(unix)] + fn signal_unix_group(&self, signal: libc::c_int) { + let group = -(self.id() as libc::pid_t); + // SAFETY: this pid is the process-group leader created in `spawn_unix`. + unsafe { + libc::kill(group, signal); + } + } +} + +impl Drop for SupervisedChild { + fn drop(&mut self) { + if self.exit_status.is_none() { + let _ = self.force_kill(); + } else { + self.cleanup_remaining_tree(); + } + } +} + +#[cfg(unix)] +fn spawn_unix(command: &mut Command) -> io::Result { + use std::os::fd::{FromRawFd, OwnedFd}; + use std::os::unix::process::CommandExt; + + let mut pipe_fds = [0; 2]; + // SAFETY: valid storage for the two returned descriptors. + if unsafe { libc::pipe(pipe_fds.as_mut_ptr()) } == -1 { + return Err(io::Error::last_os_error()); + } + // SAFETY: the successful `pipe` call returned ownership of both fds. + let read_guard = unsafe { OwnedFd::from_raw_fd(pipe_fds[0]) }; + let write_guard = unsafe { OwnedFd::from_raw_fd(pipe_fds[1]) }; + + for fd in pipe_fds { + // SAFETY: both descriptors are live. CLOEXEC prevents either end leaking + // into unrelated commands spawned by the host/service. + let flags = unsafe { libc::fcntl(fd, libc::F_GETFD) }; + if flags == -1 || unsafe { libc::fcntl(fd, libc::F_SETFD, flags | libc::FD_CLOEXEC) } == -1 + { + return Err(io::Error::last_os_error()); + } + } + + let max_fd = unix_close_bound(); + let read_fd = pipe_fds[0]; + let write_fd = pipe_fds[1]; + // SAFETY: this closure uses only async-signal-safe syscalls between fork and + // exec. The watchdog child never returns into Rust or allocates. + unsafe { + command.pre_exec(move || { + if libc::setpgid(0, 0) == -1 { + return Err(io::Error::last_os_error()); + } + let service_pid = libc::getpid(); + let watchdog = libc::fork(); + if watchdog == -1 { + return Err(io::Error::last_os_error()); + } + if watchdog == 0 { + // The watchdog shares the service process group, so the host's + // graceful group signal reaches it too. It must survive that + // signal in order to observe a host crash during the grace + // period; the later group SIGKILL remains unignorable. + libc::signal(libc::SIGTERM, libc::SIG_IGN); + libc::close(write_fd); + // Close the command's stdio and Rust's private exec-error pipe so + // this watcher cannot keep either alive. Its only input is the + // dedicated host-lifetime pipe. + let mut fd = 3; + while fd < max_fd { + if fd != read_fd { + libc::close(fd); + } + fd += 1; + } + unix_watch_parent(read_fd, service_pid); + } + libc::close(read_fd); + Ok(()) + }); + } + + let child = command.spawn()?; + drop(read_guard); + Ok(SupervisedChild { + child, + exit_status: None, + tree_cleaned: false, + parent_guard: Some(write_guard), + }) +} + +#[cfg(unix)] +fn unix_close_bound() -> libc::c_int { + let mut limit = std::mem::MaybeUninit::::uninit(); + // SAFETY: `limit` points to writable storage for `getrlimit`. + let result = unsafe { libc::getrlimit(libc::RLIMIT_NOFILE, limit.as_mut_ptr()) }; + if result == 0 { + // SAFETY: initialized on success. + let current = unsafe { limit.assume_init() }.rlim_cur; + current.min(65_536) as libc::c_int + } else { + 1024 + } +} + +/// Watch the Rust host and the command leader without invoking any Rust runtime +/// after `fork`. This function never returns. +#[cfg(unix)] +unsafe fn unix_watch_parent(read_fd: libc::c_int, service_pid: libc::pid_t) -> ! { + loop { + let mut poll_fd = libc::pollfd { + fd: read_fd, + events: libc::POLLIN | libc::POLLHUP | libc::POLLERR, + revents: 0, + }; + // SAFETY: `poll_fd` is valid for one element. A short timeout also lets + // us notice the service leader's exit and clean its leftovers. + let polled = unsafe { libc::poll(&mut poll_fd, 1, 100) }; + if polled > 0 && poll_fd.revents != 0 { + let mut byte = 0u8; + // SAFETY: valid one-byte destination; the host never writes, so EOF + // is the expected event. + if unsafe { libc::read(read_fd, (&mut byte as *mut u8).cast(), 1) } == 0 { + // SAFETY: the watchdog shares the service's process group. + unsafe { libc::kill(-service_pid, libc::SIGKILL) }; + unsafe { libc::_exit(0) }; + } + } + // Children are reparented as soon as their parent exits, even while the + // leader is still a zombie waiting for the Rust host to reap it. + if unsafe { libc::getppid() } != service_pid { + unsafe { libc::kill(-service_pid, libc::SIGKILL) }; + unsafe { libc::_exit(0) }; + } + } +} + +#[cfg(windows)] +fn spawn_windows(command: &mut Command) -> io::Result { + use std::os::windows::io::{AsRawHandle, FromRawHandle, OwnedHandle}; + use std::os::windows::process::CommandExt; + use windows_sys::Win32::System::JobObjects::{ + AssignProcessToJobObject, CreateJobObjectW, JobObjectExtendedLimitInformation, + SetInformationJobObject, JOBOBJECT_EXTENDED_LIMIT_INFORMATION, + JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE, + }; + use windows_sys::Win32::System::Threading::CREATE_SUSPENDED; + + command.creation_flags(CREATE_SUSPENDED); + let mut child = command.spawn()?; + + // SAFETY: null security/name pointers request an unnamed Job with defaults. + let raw_job = unsafe { CreateJobObjectW(std::ptr::null(), std::ptr::null()) }; + if raw_job.is_null() { + let error = io::Error::last_os_error(); + let _ = child.kill(); + let _ = child.wait(); + return Err(error); + } + // SAFETY: ownership of the newly-created handle transfers to `OwnedHandle`. + let job = unsafe { OwnedHandle::from_raw_handle(raw_job as _) }; + let mut limits = JOBOBJECT_EXTENDED_LIMIT_INFORMATION::default(); + limits.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE; + // SAFETY: live Job handle and correctly-sized information structure. + if unsafe { + SetInformationJobObject( + job.as_raw_handle() as _, + JobObjectExtendedLimitInformation, + (&limits as *const JOBOBJECT_EXTENDED_LIMIT_INFORMATION).cast(), + std::mem::size_of_val(&limits) as u32, + ) + } == 0 + { + let error = io::Error::last_os_error(); + let _ = child.kill(); + let _ = child.wait(); + return Err(error); + } + // SAFETY: both handles are live and owned for the remainder of this scope. + if unsafe { AssignProcessToJobObject(job.as_raw_handle() as _, child.as_raw_handle() as _) } + == 0 { + let error = io::Error::last_os_error(); let _ = child.kill(); let _ = child.wait(); + return Err(error); + } + + if let Err(error) = resume_windows_process(child.id()) { + // Closing a kill-on-close Job takes down the still-suspended child. + drop(job); + let _ = child.wait(); + return Err(error); + } + + Ok(SupervisedChild { + child, + exit_status: None, + tree_cleaned: false, + job: Some(job), + }) +} + +#[cfg(windows)] +fn resume_windows_process(process_id: u32) -> io::Result<()> { + use std::os::windows::io::{AsRawHandle, FromRawHandle, OwnedHandle}; + use windows_sys::Win32::Foundation::INVALID_HANDLE_VALUE; + use windows_sys::Win32::System::Diagnostics::ToolHelp::{ + CreateToolhelp32Snapshot, Thread32First, Thread32Next, TH32CS_SNAPTHREAD, THREADENTRY32, + }; + use windows_sys::Win32::System::Threading::{OpenThread, ResumeThread, THREAD_SUSPEND_RESUME}; + + // A newly-created suspended process has exactly one thread. Enumerating it + // is necessary because `std::process::Child` exposes the process handle but + // not CreateProcessW's primary-thread handle. + // SAFETY: system snapshot request with no process filter. + let raw_snapshot = unsafe { CreateToolhelp32Snapshot(TH32CS_SNAPTHREAD, 0) }; + if raw_snapshot == INVALID_HANDLE_VALUE { + return Err(io::Error::last_os_error()); + } + // SAFETY: ownership of the snapshot handle transfers here. + let snapshot = unsafe { OwnedHandle::from_raw_handle(raw_snapshot as _) }; + let mut entry = THREADENTRY32 { + dwSize: std::mem::size_of::() as u32, + ..Default::default() + }; + // SAFETY: snapshot and entry pointers are valid. + let mut has_entry = unsafe { Thread32First(snapshot.as_raw_handle() as _, &mut entry) } != 0; + while has_entry { + if entry.th32OwnerProcessID == process_id { + // SAFETY: request the minimal right for the enumerated thread. + let raw_thread = unsafe { OpenThread(THREAD_SUSPEND_RESUME, 0, entry.th32ThreadID) }; + if !raw_thread.is_null() { + // SAFETY: ownership of the opened thread handle transfers here. + let thread = unsafe { OwnedHandle::from_raw_handle(raw_thread as _) }; + // SAFETY: the process was created with CREATE_SUSPENDED, so its + // primary thread has a positive suspend count. + if unsafe { ResumeThread(thread.as_raw_handle() as _) } != u32::MAX { + return Ok(()); + } + return Err(io::Error::last_os_error()); + } + } + // SAFETY: same live snapshot and initialized entry. + has_entry = unsafe { Thread32Next(snapshot.as_raw_handle() as _, &mut entry) } != 0; + } + Err(io::Error::new( + io::ErrorKind::NotFound, + "cannot find suspended child thread", + )) +} + +/// A bounded, already-redacted tail suitable for UI-facing crash diagnostics. +#[derive(Debug, Default)] +pub(crate) struct DiagnosticTail { + lines: VecDeque, + bytes: usize, + omitted: usize, +} + +impl DiagnosticTail { + pub(crate) fn push(&mut self, line: &str) { + let line = sanitize_diagnostic(line); + let bytes = line.len(); + while !self.lines.is_empty() + && (self.lines.len() >= DIAGNOSTIC_LINES || self.bytes + bytes > DIAGNOSTIC_BYTES) + { + if let Some(removed) = self.lines.pop_front() { + self.bytes = self.bytes.saturating_sub(removed.len()); + self.omitted += 1; + } + } + self.bytes += bytes; + self.lines.push_back(line); + } + + pub(crate) fn is_empty(&self) -> bool { + self.lines.is_empty() + } + + pub(crate) fn render(&self) -> String { + let mut rendered = self.lines.iter().cloned().collect::>().join("\n"); + if self.omitted > 0 { + let prefix = format!("[{} earlier diagnostic lines omitted]\n", self.omitted); + rendered.insert_str(0, &prefix); + } + rendered + } +} + +/// Read newline-delimited child output without ever allocating in proportion to +/// a child-controlled line. Once the cap is reached, the remainder of that line +/// is discarded through the next newline; the bounded prefix is still delivered +/// so callers can retain useful diagnostics or attempt structured parsing. +pub(crate) fn read_bounded_lines( + mut reader: impl Read, + mut on_line: impl FnMut(&str), +) -> io::Result<()> { + let mut chunk = [0u8; 8192]; + let mut line = Vec::with_capacity(CHILD_LINE_BYTES.min(8192)); + let mut discarding = false; + + loop { + let read = match reader.read(&mut chunk) { + Err(error) if error.kind() == io::ErrorKind::Interrupted => continue, + result => result?, + }; + if read == 0 { + break; + } + for &byte in &chunk[..read] { + if byte == b'\n' { + if line.last() == Some(&b'\r') { + line.pop(); + } + let decoded = String::from_utf8_lossy(&line); + on_line(&decoded); + line.clear(); + discarding = false; + } else if !discarding { + if line.len() < CHILD_LINE_BYTES { + line.push(byte); + } else { + discarding = true; + } + } + } + } + + if !line.is_empty() || discarding { + let decoded = String::from_utf8_lossy(&line); + on_line(&decoded); + } + Ok(()) +} + +/// Redact provider credentials and bound a single child-origin diagnostic for +/// logs, error returns, and UI emission. Keeping this as the only diagnostic +/// boundary helper prevents one structured-output path from bypassing the same +/// rules used by stderr tails. +pub(crate) fn sanitize_diagnostic(line: &str) -> String { + let mut line = truncate_utf8(line, DIAGNOSTIC_LINE_BYTES).to_string(); + redact_url_credentials(&mut line); + + // Longest/specific provider spellings go first. Identifier-boundary checks + // keep the generic `token`/`secret` forms from matching their suffixes. + const KEYS: &[(&str, bool)] = &[ + ("hugging_face_hub_token", false), + ("hugging-face-hub-token", false), + ("authorization", true), + ("refresh_token", false), + ("refresh-token", false), + ("access_token", false), + ("access-token", false), + ("client_secret", false), + ("client-secret", false), + ("clientsecret", false), + ("accesskey", false), + ("api_key", false), + ("api-key", false), + ("apikey", false), + ("hf_token", false), + ("hf-token", false), + ("password", false), + ("passwd", false), + ("credential", false), + ("token", false), + ("secret", false), + ]; + for &(key, consume_remainder) in KEYS { + redact_key_values(&mut line, key, consume_remainder); + } + redact_bearer_values(&mut line); + truncate_utf8(&line, DIAGNOSTIC_LINE_BYTES).to_string() +} + +fn identifier_byte(byte: u8) -> bool { + byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'-') +} + +fn redact_key_values(line: &mut String, key: &str, consume_remainder: bool) { + let mut search_from = 0; + loop { + let lower = line[search_from..].to_ascii_lowercase(); + let Some(relative) = lower.find(key) else { + break; + }; + let key_start = search_from + relative; + let key_end = key_start + key.len(); + let bytes = line.as_bytes(); + if key_start > 0 && identifier_byte(bytes[key_start - 1]) { + search_from = key_end; + continue; + } + if key_end < bytes.len() && identifier_byte(bytes[key_end]) { + search_from = key_end; + continue; + } + + let mut cursor = key_end; + if matches!(line.as_bytes().get(cursor), Some(b'\'' | b'"')) { + cursor += 1; + } + while matches!(line.as_bytes().get(cursor), Some(b' ' | b'\t')) { + cursor += 1; + } + if !matches!(line.as_bytes().get(cursor), Some(b'=' | b':')) { + search_from = key_end; + continue; + } + cursor += 1; + while matches!(line.as_bytes().get(cursor), Some(b' ' | b'\t')) { + cursor += 1; + } + + let quote = line + .as_bytes() + .get(cursor) + .copied() + .filter(|b| matches!(b, b'\'' | b'"')); + if quote.is_some() { + cursor += 1; + } + let value_start = cursor; + let value_end = if let Some(quote) = quote { + line.as_bytes()[value_start..] + .iter() + .position(|&byte| byte == quote) + .map_or(line.len(), |offset| value_start + offset) + } else if consume_remainder { + line.len() + } else { + line.as_bytes()[value_start..] + .iter() + .position(|byte| matches!(byte, b' ' | b'\t' | b',' | b';' | b'&' | b'}' | b']')) + .map_or(line.len(), |offset| value_start + offset) + }; + if value_end > value_start { + line.replace_range(value_start..value_end, "[REDACTED]"); + search_from = value_start + "[REDACTED]".len(); + } else { + search_from = key_end; + } + } +} + +fn redact_bearer_values(line: &mut String) { + let mut search_from = 0; + loop { + let lower = line[search_from..].to_ascii_lowercase(); + let Some(relative) = lower.find("bearer") else { + break; + }; + let start = search_from + relative; + let end = start + "bearer".len(); + let bytes = line.as_bytes(); + if (start > 0 && identifier_byte(bytes[start - 1])) + || (end < bytes.len() && identifier_byte(bytes[end])) + { + search_from = end; + continue; + } + let mut value_start = end; + while matches!(line.as_bytes().get(value_start), Some(b' ' | b'\t')) { + value_start += 1; + } + let value_end = line.as_bytes()[value_start..] + .iter() + .position(|byte| { + matches!( + byte, + b' ' | b'\t' | b',' | b';' | b'"' | b'\'' | b'}' | b']' + ) + }) + .map_or(line.len(), |offset| value_start + offset); + if value_end > value_start { + line.replace_range(value_start..value_end, "[REDACTED]"); + search_from = value_start + "[REDACTED]".len(); + } else { + search_from = end; + } + } +} + +fn redact_url_credentials(line: &mut String) { + let mut search_from = 0; + while let Some(scheme_offset) = line[search_from..].find("://") { + let credentials_start = search_from + scheme_offset + 3; + let remainder = &line[credentials_start..]; + let authority_end = remainder.find(['/', ' ', '\t']).unwrap_or(remainder.len()); + let Some(at) = remainder[..authority_end].rfind('@') else { + search_from = credentials_start + authority_end; + continue; + }; + line.replace_range(credentials_start..credentials_start + at, "[REDACTED]"); + search_from = credentials_start + "[REDACTED]@".len(); + } +} + +fn truncate_utf8(value: &str, max_bytes: usize) -> &str { + if value.len() <= max_bytes { + return value; + } + let mut end = max_bytes; + while !value.is_char_boundary(end) { + end -= 1; + } + &value[..end] +} + +#[cfg(test)] +mod tests { + use super::*; + use std::path::{Path, PathBuf}; + + const HELPER_ROLE: &str = "LSDJ_PROCESS_HELPER_ROLE"; + const HELPER_PID_FILE: &str = "LSDJ_PROCESS_HELPER_PID_FILE"; + + fn helper_command(role: &str, pid_file: &Path) -> Command { + let mut command = Command::new(std::env::current_exe().expect("current test executable")); + command + .args([ + "--ignored", + "--exact", + "child_process::tests::process_helper", + "--nocapture", + ]) + .env(HELPER_ROLE, role) + .env(HELPER_PID_FILE, pid_file); + command + } + + fn test_dir(label: &str) -> PathBuf { + let path = std::env::temp_dir().join(format!( + "lsdj-supervisor-{label}-{}-{:?}", + std::process::id(), + std::thread::current().id() + )); + let _ = std::fs::remove_dir_all(&path); + std::fs::create_dir_all(&path).unwrap(); + path + } + + fn wait_for_pids(path: &Path) -> (u32, u32) { + let deadline = Instant::now() + Duration::from_secs(10); + loop { + if let Ok(contents) = std::fs::read_to_string(path) { + let pids = contents + .split_whitespace() + .filter_map(|value| value.parse::().ok()) + .collect::>(); + if pids.len() == 2 { + return (pids[0], pids[1]); + } + } + assert!( + Instant::now() < deadline, + "helper did not report its process tree" + ); + std::thread::sleep(Duration::from_millis(20)); + } + } + + fn wait_until_gone(pid: u32) -> bool { + let deadline = Instant::now() + Duration::from_secs(10); + while Instant::now() < deadline { + if !process_is_alive(pid) { + return true; + } + std::thread::sleep(Duration::from_millis(20)); + } + false + } + + #[cfg(unix)] + fn process_is_alive(pid: u32) -> bool { + // SAFETY: signal 0 probes without signalling. + unsafe { libc::kill(pid as libc::pid_t, 0) == 0 } + } + + #[cfg(windows)] + fn process_is_alive(pid: u32) -> bool { + use windows_sys::Win32::Foundation::{CloseHandle, WAIT_TIMEOUT}; + use windows_sys::Win32::System::Threading::{ + OpenProcess, WaitForSingleObject, PROCESS_QUERY_LIMITED_INFORMATION, + }; + const SYNCHRONIZE_ACCESS: u32 = 0x0010_0000; + // SAFETY: read-only liveness handle for a test-owned pid. + let process = unsafe { + OpenProcess( + SYNCHRONIZE_ACCESS | PROCESS_QUERY_LIMITED_INFORMATION, + 0, + pid, + ) + }; + if process.is_null() { + return false; + } + // SAFETY: live process handle and zero timeout. + let result = unsafe { WaitForSingleObject(process, 0) }; + // SAFETY: close the handle opened above. + unsafe { CloseHandle(process) }; + result == WAIT_TIMEOUT + } + + #[cfg(not(any(unix, windows)))] + fn process_is_alive(_pid: u32) -> bool { + false + } + + /// Process-tree stand-in built from the test executable itself, so lifecycle + /// coverage is portable and needs no Python, shell, or downloaded fixture. + #[test] + #[ignore] + #[allow(clippy::zombie_processes)] // the supervisor tests intentionally own/reap the tree + fn process_helper() { + let role = std::env::var(HELPER_ROLE).expect("helper role"); + let pid_file = PathBuf::from(std::env::var_os(HELPER_PID_FILE).expect("pid file")); + match role.as_str() { + "host" => { + let mut command = helper_command("child", &pid_file); + let _child = spawn_grouped(&mut command).expect("host spawns supervised child"); + let _ = wait_for_pids(&pid_file); + // Deliberately bypass destructors, matching a panic/abort-style + // host exit. OS/job/watchdog lifetime must still remove the tree. + std::process::exit(0); + } + "host-shutdown" => { + let mut command = helper_command("child", &pid_file); + let mut child = spawn_grouped(&mut command).expect("host spawns supervised child"); + let _ = wait_for_pids(&pid_file); + let marker = pid_file.with_extension("shutdown"); + let _ = child.shutdown_with_hook(Duration::from_secs(30), || { + std::fs::write(marker, b"signalled").expect("write shutdown marker"); + }); + } + "child" | "startup-failure" => { + #[cfg(unix)] + // SAFETY: make graceful shutdown exhaust its deadline so the + // explicit teardown test covers the forced path. + unsafe { + libc::signal(libc::SIGTERM, libc::SIG_IGN); + } + let grandchild = helper_command("grandchild", &pid_file) + .spawn() + .expect("spawn grandchild"); + std::fs::write( + &pid_file, + format!("{} {}", std::process::id(), grandchild.id()), + ) + .expect("write pid file"); + if role == "startup-failure" { + std::process::exit(23); + } + loop { + std::thread::sleep(Duration::from_secs(60)); + } + } + "grandchild" => { + #[cfg(unix)] + // SAFETY: see the child role above. + unsafe { + libc::signal(libc::SIGTERM, libc::SIG_IGN); + } + loop { + std::thread::sleep(Duration::from_secs(60)); + } + } + other => panic!("unknown helper role {other}"), + } + } + + #[test] + fn explicit_shutdown_removes_child_and_grandchild() { + let dir = test_dir("shutdown"); + let pid_file = dir.join("pids"); + let mut child = spawn_grouped(&mut helper_command("child", &pid_file)).unwrap(); + let (child_pid, grandchild_pid) = wait_for_pids(&pid_file); + + let report = child.shutdown(Duration::from_millis(100)).unwrap(); + assert!(report.forced, "helpers ignore graceful shutdown"); + assert!(report.status.is_some(), "leader should be reaped"); + assert!( + wait_until_gone(child_pid), + "child survived explicit shutdown" + ); + assert!( + wait_until_gone(grandchild_pid), + "grandchild survived explicit shutdown" + ); + let _ = std::fs::remove_dir_all(dir); + } + + #[test] + fn startup_failure_removes_grandchild() { + let dir = test_dir("startup-failure"); + let pid_file = dir.join("pids"); + let mut child = spawn_grouped(&mut helper_command("startup-failure", &pid_file)).unwrap(); + let (_child_pid, grandchild_pid) = wait_for_pids(&pid_file); + + let status = child.wait().unwrap(); + assert!(!status.success(), "startup stand-in must fail"); + assert!( + wait_until_gone(grandchild_pid), + "startup failure left its grandchild running" + ); + let _ = std::fs::remove_dir_all(dir); + } + + #[test] + fn terminal_status_is_stable_across_repeated_poll_shutdown_and_wait() { + let dir = test_dir("cached-status"); + let pid_file = dir.join("pids"); + let mut child = spawn_grouped(&mut helper_command("startup-failure", &pid_file)).unwrap(); + let _ = wait_for_pids(&pid_file); + + let deadline = Instant::now() + Duration::from_secs(10); + let first = loop { + if let Some(status) = child.try_wait().unwrap() { + break status; + } + assert!(Instant::now() < deadline, "helper did not exit"); + std::thread::sleep(Duration::from_millis(20)); + }; + assert!(!first.success()); + assert_eq!(child.try_wait().unwrap(), Some(first)); + + // Once terminal, shutdown must return the cached status without sending + // a signal to the old pid/process-group, and wait must remain idempotent. + let report = child.shutdown(Duration::ZERO).unwrap(); + assert_eq!(report.status, Some(first)); + assert!(!report.forced); + assert_eq!(child.wait().unwrap(), first); + let _ = std::fs::remove_dir_all(dir); + } + + #[test] + fn abnormal_host_exit_removes_child_and_grandchild() { + let dir = test_dir("abnormal-host"); + let pid_file = dir.join("pids"); + let status = helper_command("host", &pid_file).status().unwrap(); + assert!(status.success(), "host helper failed: {status}"); + let (child_pid, grandchild_pid) = wait_for_pids(&pid_file); + + assert!( + wait_until_gone(child_pid), + "child survived abnormal host exit" + ); + assert!( + wait_until_gone(grandchild_pid), + "grandchild survived abnormal host exit" + ); + let _ = std::fs::remove_dir_all(dir); + } + + #[cfg(unix)] + #[test] + fn host_exit_during_grace_still_removes_sigterm_resistant_tree() { + let dir = test_dir("host-exit-during-grace"); + let pid_file = dir.join("pids"); + let marker = pid_file.with_extension("shutdown"); + let mut host = helper_command("host-shutdown", &pid_file) + .spawn() + .expect("spawn host helper"); + let (child_pid, grandchild_pid) = wait_for_pids(&pid_file); + + let deadline = Instant::now() + Duration::from_secs(10); + while !marker.exists() { + assert!( + Instant::now() < deadline, + "host never entered shutdown grace" + ); + std::thread::sleep(Duration::from_millis(20)); + } + // `Child::kill` is SIGKILL on Unix: destructors cannot close the parent + // guard. The watchdog must survive the preceding group SIGTERM, observe + // the kernel closing that guard, and kill the resistant descendants. + host.kill().expect("kill host during grace"); + let _ = host.wait(); + + assert!( + wait_until_gone(child_pid), + "child survived host death during grace" + ); + assert!( + wait_until_gone(grandchild_pid), + "grandchild survived host death during grace" + ); + let _ = std::fs::remove_dir_all(dir); + } + + #[test] + fn readiness_reports_ready_exit_and_timeout() { + let dir = test_dir("readiness"); + let pid_file = dir.join("pids"); + let mut running = spawn_grouped(&mut helper_command("child", &pid_file)).unwrap(); + let _ = wait_for_pids(&pid_file); + let mut polls = 0; + let ready = running + .wait_for_readiness(Duration::from_secs(1), || { + polls += 1; + Ok(polls == 2) + }) + .unwrap(); + assert!(matches!(ready, Readiness::Ready)); + let timed_out = running + .wait_for_readiness(Duration::from_millis(30), || Ok(false)) + .unwrap(); + assert!(matches!(timed_out, Readiness::TimedOut)); + let _ = running.force_kill(); + + let failure_file = dir.join("failure-pids"); + let mut failed = + spawn_grouped(&mut helper_command("startup-failure", &failure_file)).unwrap(); + let _ = wait_for_pids(&failure_file); + let exited = failed + .wait_for_readiness(Duration::from_secs(5), || Ok(false)) + .unwrap(); + assert!(matches!(exited, Readiness::Exited(status) if !status.success())); + let _ = std::fs::remove_dir_all(dir); + } + + #[test] + fn diagnostics_are_bounded_and_redacted() { + let mut diagnostics = DiagnosticTail::default(); + diagnostics.push( + "download https://url-user:url-pass@example.com HF_TOKEN=hf-secret \ + HUGGING_FACE_HUB_TOKEN: hub-secret client_secret=oauth-secret \ + access-token=access-secret api_key=api-secret \ + Authorization: Bearer auth-secret", + ); + diagnostics.push( + r#"{"HF_TOKEN":"json-hf-secret","client_secret":"json-oauth-secret",\ + "Authorization":"Bearer json-auth-secret"}"#, + ); + let redacted = diagnostics.render(); + for secret in [ + "url-user", + "url-pass", + "hf-secret", + "hub-secret", + "oauth-secret", + "access-secret", + "api-secret", + "auth-secret", + "json-hf-secret", + "json-oauth-secret", + "json-auth-secret", + ] { + assert!(!redacted.contains(secret), "leaked {secret}: {redacted}"); + } + assert!(redacted.contains("[REDACTED]")); + for index in 0..1000 { + diagnostics.push(&format!("line-{index} {}", "x".repeat(200))); + } + let rendered = diagnostics.render(); + assert!(rendered.contains("omitted")); + assert!(rendered.len() <= DIAGNOSTIC_BYTES + 100); + } + + #[test] + fn bounded_line_reader_discards_multi_megabyte_line_without_losing_next_line() { + let mut input = vec![b'x'; 4 * 1024 * 1024]; + input.extend_from_slice(b"\nnext\r\nfinal-without-newline"); + let mut lines = Vec::new(); + read_bounded_lines(std::io::Cursor::new(input), |line| { + lines.push(line.to_string()) + }) + .unwrap(); + + assert_eq!(lines.len(), 3); + assert_eq!(lines[0].len(), CHILD_LINE_BYTES); + assert!(lines[0].bytes().all(|byte| byte == b'x')); + assert_eq!(lines[1], "next"); + assert_eq!(lines[2], "final-without-newline"); + } + + #[test] + fn supervised_children_scrub_credentials_and_injection_environment() { + let mut command = Command::new("unused"); + for key in [ + "HF_TOKEN", + "HUGGING_FACE_HUB_TOKEN", + "PYTHONPATH", + "PYTHONHOME", + "UV_CONFIG_FILE", + "UV_OVERRIDE", + "UV_INDEX_URL", + "PIP_CONFIG_FILE", + "PIP_INDEX_URL", + ] { + command.env(key, "attacker-controlled"); + } + command.env("UV_CACHE_DIR", "app-controlled-cache"); + command.env("HF_HUB_OFFLINE", "1"); + + scrub_child_environment(&mut command); + let environment = command.get_envs().collect::>(); + for key in [ + "HF_TOKEN", + "HUGGING_FACE_HUB_TOKEN", + "PYTHONPATH", + "PYTHONHOME", + "UV_CONFIG_FILE", + "UV_OVERRIDE", + "UV_INDEX_URL", + "PIP_CONFIG_FILE", + "PIP_INDEX_URL", + ] { + assert!( + environment + .iter() + .any(|(name, value)| *name == key && value.is_none()), + "{key} was not explicitly removed" + ); + } + assert!(environment.iter().any(|(name, value)| { + *name == "UV_CACHE_DIR" && value == &Some(std::ffi::OsStr::new("app-controlled-cache")) + })); + assert!(environment.iter().any(|(name, value)| { + *name == "HF_HUB_OFFLINE" && value == &Some(std::ffi::OsStr::new("1")) + })); } } diff --git a/src-tauri/src/generation.rs b/src-tauri/src/generation.rs index 0f1fd6e..ad44bca 100644 --- a/src-tauri/src/generation.rs +++ b/src-tauri/src/generation.rs @@ -2,10 +2,11 @@ //! //! The native shell hosts the realtime decks (the inference sidecars, [`crate::sidecar`]) //! and serves the frontend from the Tauri asset host, so FastAPI no longer serves -//! the UI. But the Stable Audio 3 / Magenta pad+track GENERATION still lives behind -//! HTTP (`/api/render`, `/api/generate`). This module spawns the FastAPI generation -//! server on a loopback port — the controller is generation-only: no deck workers, no -//! static mount — and the webview fetches it via `getApiBaseUrl()`. +//! the UI. This module supervises the Stable Audio 3 generation service on a +//! loopback port. In managed Linux/Windows builds Magenta rendering belongs to +//! the Rust gateway (`crate::magenta_gateway`), so this child receives no MRT2 +//! paths or dependencies. The bundled macOS backend retains its existing +//! combined `/api/render` + `/api/generate` behavior. //! //! Mirrors the sidecar's spawn/supervise/Drop-kill pattern. Started with the app; a //! failed spawn just leaves generation unreachable (the UI already surfaces those as @@ -13,17 +14,72 @@ use std::io; use std::net::{TcpListener, TcpStream}; +#[cfg(not(feature = "managed-runtime"))] use std::path::Path; -use std::process::{Child, Command}; +use std::process::{Command, ExitStatus}; use std::sync::Mutex; -use std::time::{Duration, Instant}; +use std::time::Duration; + +use crate::child_process::{Readiness, SupervisedChild}; /// The supervised generation server: its chosen loopback port (exposed to the /// webview via `app_info`) and the child process. Held in Tauri managed state; /// dropping it kills the child. pub struct GenerationServer { - port: Option, - child: Mutex>, + state: Mutex, +} + +trait GenerationProcess: Send { + fn try_wait(&mut self) -> io::Result>; + fn shutdown(&mut self) -> io::Result<()>; +} + +impl GenerationProcess for SupervisedChild { + fn try_wait(&mut self) -> io::Result> { + SupervisedChild::try_wait(self) + } + + fn shutdown(&mut self) -> io::Result<()> { + let report = SupervisedChild::shutdown(self, Duration::from_millis(500))?; + crate::child_process::log_shutdown("generation server", Ok(report)); + Ok(()) + } +} + +enum GenerationProcessState { + Stopped, + Running { + port: u16, + capability: String, + process: Box, + }, + /// Shutdown was requested, but the supervisor could not prove that the + /// complete process tree was reaped. The handle stays owned here so every + /// later quiesce/resume can retry; an installer must not rename through it. + Uncertain { + process: Box, + was_running: bool, + }, +} + +struct GenerationSpawnFailure { + error: io::Error, + uncertain_process: Option>, +} + +impl GenerationSpawnFailure { + fn reaped(error: io::Error) -> Self { + Self { + error, + uncertain_process: None, + } + } +} + +impl From for GenerationSpawnFailure { + fn from(error: io::Error) -> Self { + Self::reaped(error) + } } impl GenerationServer { @@ -31,32 +87,27 @@ impl GenerationServer { /// failed spawn yields `port() == None` and generation is simply unreachable (the /// webview surfaces that as fetch errors). pub fn start() -> GenerationServer { - match Self::spawn() { - Ok((port, child)) => { - println!("lsdj-app: generation server on 127.0.0.1:{port}"); - GenerationServer { - port: Some(port), - child: Mutex::new(Some(child)), - } - } - Err(e) => { - eprintln!("lsdj-app: generation server spawn failed: {e}"); - GenerationServer { - port: None, - child: Mutex::new(None), - } - } + let server = GenerationServer { + state: Mutex::new(GenerationProcessState::Stopped), + }; + if let Err(error) = server.resume() { + // A fresh managed install intentionally has no runtime yet. The + // model manager calls `resume` immediately after first promotion. + eprintln!("lsdj-app: generation server unavailable: {error}"); } + server } - fn spawn() -> io::Result<(u16, Child)> { + fn spawn( + capability: &str, + ) -> Result<(u16, Box), GenerationSpawnFailure> { // Pick a free loopback port, then hand it to the child (uvicorn binds it). // The brief drop→rebind window on loopback is benign. let port = { let listener = TcpListener::bind("127.0.0.1:0")?; listener.local_addr()?.port() }; - let mut command = generation_command(port)?; + let mut command = generation_command(port, capability)?; let mut child = crate::child_process::spawn_grouped(&mut command)?; // Confirm the child actually came up before advertising the port — a @@ -64,27 +115,132 @@ impl GenerationServer { // otherwise leave the app pointing the webview at a dead port. Bounded so // a slow-but-working server is reported optimistically rather than // blocking the window; a child that EXITS is reported as a failure. - let deadline = Instant::now() + Duration::from_millis(1500); let addr = ("127.0.0.1", port); - loop { - if TcpStream::connect(addr).is_ok() { - return Ok((port, child)); - } - if matches!(child.try_wait(), Ok(Some(_))) { - let _ = child.wait(); - return Err(io::Error::other("generation server exited before binding")); + let readiness = child.wait_for_readiness(Duration::from_millis(1500), || { + Ok(TcpStream::connect(addr).is_ok()) + }); + finish_generation_startup(port, Box::new(child), readiness) + } + + fn resume_with_spawn( + &self, + spawn: impl FnOnce( + &str, + ) -> Result<(u16, Box), GenerationSpawnFailure>, + ) -> io::Result<()> { + let mut state = self + .state + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + let previous = std::mem::replace(&mut *state, GenerationProcessState::Stopped); + match previous { + GenerationProcessState::Stopped => {} + GenerationProcessState::Running { + port, + capability, + mut process, + } => match process.try_wait() { + Ok(None) => { + *state = GenerationProcessState::Running { + port, + capability, + process, + }; + return Ok(()); + } + Ok(Some(_)) => {} + Err(error) => { + *state = GenerationProcessState::Uncertain { + process, + was_running: true, + }; + return Err(error); + } + }, + GenerationProcessState::Uncertain { + mut process, + was_running, + } => { + if let Err(error) = process.shutdown() { + *state = GenerationProcessState::Uncertain { + process, + was_running, + }; + return Err(error); + } } - if Instant::now() >= deadline { - return Ok((port, child)); // still launching; advertise optimistically + } + let capability = crate::local_auth::generate_capability(); + let (port, process) = match spawn(&capability) { + Ok(spawned) => spawned, + Err(spawn) => { + if let Some(process) = spawn.uncertain_process { + *state = GenerationProcessState::Uncertain { + process, + was_running: false, + }; + } + return Err(spawn.error); } - std::thread::sleep(Duration::from_millis(50)); + }; + println!("lsdj-app: generation server on 127.0.0.1:{port}"); + *state = GenerationProcessState::Running { + port, + capability, + process, + }; + Ok(()) + } + + /// Return the port and capability from one lock acquisition. Neither half is + /// ever observable without the other across promotion/resume transitions. + pub fn connection(&self) -> Option<(u16, String)> { + match &*self + .state + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + { + GenerationProcessState::Running { + port, capability, .. + } => Some((*port, capability.clone())), + GenerationProcessState::Stopped | GenerationProcessState::Uncertain { .. } => None, } } - /// The loopback port the generation server bound, or `None` if disabled / not - /// running. The webview reads this through `app_info` to build the API base URL. - pub fn port(&self) -> Option { - self.port + /// Start (or recover) the service from the currently promoted verified + /// generation. A running healthy child is left untouched. This is called on + /// startup and after every managed SA3 promotion/rollback. + pub fn resume(&self) -> io::Result<()> { + self.resume_with_spawn(Self::spawn) + } + + /// Stop and reap the service before its managed generation is renamed. + /// Returns whether a live child was present so tests/lifecycle diagnostics + /// can distinguish first install from an update. + pub fn quiesce(&self) -> io::Result { + let mut state = self + .state + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + let previous = std::mem::replace(&mut *state, GenerationProcessState::Stopped); + let (mut process, was_running) = match previous { + GenerationProcessState::Stopped => return Ok(false), + GenerationProcessState::Running { process, .. } => (process, true), + GenerationProcessState::Uncertain { + process, + was_running, + } => (process, was_running), + }; + match process.shutdown() { + Ok(()) => Ok(was_running), + Err(error) => { + *state = GenerationProcessState::Uncertain { + process, + was_running, + }; + Err(error) + } + } } /// Kill the generation server child. Called explicitly from the app's @@ -92,8 +248,37 @@ impl GenerationServer { /// macOS quit (`process::exit` skips destructors), so [`Drop`] alone would /// leak the process. pub fn shutdown(&self) { - if let Some(mut child) = self.child.lock().unwrap_or_else(|p| p.into_inner()).take() { - crate::child_process::kill_group(&mut child); + if let Err(error) = self.quiesce() { + crate::child_process::log_shutdown("generation server", Err(error)); + } + } +} + +fn finish_generation_startup( + port: u16, + mut process: Box, + readiness: io::Result, +) -> Result<(u16, Box), GenerationSpawnFailure> { + match readiness { + Ok(Readiness::Ready | Readiness::TimedOut) => { + // Preserve the existing macOS contract: a slow-but-running + // service is advertised optimistically after the bounded wait. + Ok((port, process)) + } + Ok(Readiness::Exited(status)) => Err(GenerationSpawnFailure::reaped(io::Error::other( + format!("generation server exited before binding ({status})"), + ))), + Err(readiness_error) => match process.shutdown() { + Ok(()) => Err(GenerationSpawnFailure::reaped(readiness_error)), + Err(cleanup_error) => Err(GenerationSpawnFailure { + error: io::Error::new( + readiness_error.kind(), + format!( + "{readiness_error}; generation startup cleanup also failed: {cleanup_error}" + ), + ), + uncertain_process: Some(process), + }), } } } @@ -108,36 +293,68 @@ impl Drop for GenerationServer { /// `LSDJ_BACKEND_BIN --generation-server`; dev is overridable via /// `LSDJ_GENERATION_CMD` and defaults to `uv run python -m lsdj.controller`. /// `--port` is always appended. -pub fn generation_command(port: u16) -> io::Result { +pub fn generation_command(port: u16, capability: &str) -> io::Result { + #[cfg(feature = "managed-runtime")] + { + use std::ffi::OsString; + + let paths = crate::platform_paths::get(); + let ephemeral = paths + .backend_env() + .into_iter() + // The managed SA3 interpreter has its own dependency closure and + // receives no MRT2 location. This makes accidental `/api/render` + // use fail closed instead of coupling the services again. + .filter(|(name, _)| name.to_str() != Some("MAGENTA_HOME")) + .chain(std::iter::once(( + OsString::from("LSDJ_API_CAPABILITY"), + OsString::from(capability), + ))); + crate::managed_runtime::resolve( + paths.assets(), + crate::managed_runtime::Service::Sa3, + ) + .and_then(|resolved| { + resolved.into_command(["--port".into(), port.to_string().into()], ephemeral) + }) + .map_err(io::Error::other) + } + // The release bundle shares one frozen dependency tree with the deck // sidecars. Its dispatcher needs an explicit mode because both CLIs accept // `--port`; the exact OsString also preserves paths containing spaces. + #[cfg(not(feature = "managed-runtime"))] if let Some(program) = std::env::var_os("LSDJ_BACKEND_BIN") { let mut cmd = Command::new(program); + cmd.env("LSDJ_API_CAPABILITY", capability); cmd.args(["--generation-server", "--port", &port.to_string()]); return Ok(cmd); } - let overridden = std::env::var("LSDJ_GENERATION_CMD"); - let spec = overridden - .clone() - .unwrap_or_else(|_| "uv run python -m lsdj.controller".to_string()); - let mut parts = spec.split_whitespace(); - let program = parts.next().ok_or_else(|| { - io::Error::new(io::ErrorKind::InvalidInput, "empty LSDJ_GENERATION_CMD") - })?; - let mut cmd = Command::new(program); - cmd.args(parts); - cmd.args(["--port", &port.to_string()]); - if overridden.is_err() { - // The default `uv run` needs the backend project dir as its CWD. A packaged - // build returned through LSDJ_BACKEND_BIN above and never reaches this path. - cmd.current_dir(Path::new(env!("CARGO_MANIFEST_DIR")).join("../backend")); - } - Ok(cmd) + #[cfg(not(feature = "managed-runtime"))] + { + let overridden = std::env::var("LSDJ_GENERATION_CMD"); + let spec = overridden + .clone() + .unwrap_or_else(|_| "uv run python -m lsdj.controller".to_string()); + let mut parts = spec.split_whitespace(); + let program = parts.next().ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidInput, "empty LSDJ_GENERATION_CMD") + })?; + let mut cmd = Command::new(program); + cmd.env("LSDJ_API_CAPABILITY", capability); + cmd.args(parts); + cmd.args(["--port", &port.to_string()]); + if overridden.is_err() { + // The default `uv run` needs the backend project dir as its CWD. A packaged + // build returned through LSDJ_BACKEND_BIN above and never reaches this path. + cmd.current_dir(Path::new(env!("CARGO_MANIFEST_DIR")).join("../backend")); + } + Ok(cmd) + } } -#[cfg(test)] +#[cfg(all(test, not(feature = "managed-runtime")))] mod tests { use super::*; @@ -148,16 +365,123 @@ mod tests { std::env::set_var("LSDJ_GENERATION_CMD", "echo hi"); // The override is split into program + args with `--port` always appended. - let cmd = generation_command(5123).unwrap(); - let argv: Vec<_> = cmd.get_args().map(|a| a.to_string_lossy().into_owned()).collect(); + let cmd = generation_command(5123, "test-capability-0123456789abcdef").unwrap(); + let argv: Vec<_> = cmd + .get_args() + .map(|a| a.to_string_lossy().into_owned()) + .collect(); assert_eq!(cmd.get_program().to_string_lossy(), "echo"); assert_eq!(argv, ["hi", "--port", "5123"]); + assert!(cmd.get_envs().any(|(key, value)| { + key == "LSDJ_API_CAPABILITY" + && value.is_some_and(|value| value == "test-capability-0123456789abcdef") + })); // Now-always-on `start()` never fails the app: a command that exits without // binding the port (echo) degrades to no advertised port. let server = GenerationServer::start(); - assert_eq!(server.port(), None); + assert_eq!(server.connection(), None); std::env::remove_var("LSDJ_GENERATION_CMD"); } } + +#[cfg(test)] +mod process_tests { + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::Arc; + + use super::*; + + struct RetryProcess { + shutdowns: Arc, + } + + impl GenerationProcess for RetryProcess { + fn try_wait(&mut self) -> io::Result> { + Ok(None) + } + + fn shutdown(&mut self) -> io::Result<()> { + if self.shutdowns.fetch_add(1, Ordering::AcqRel) == 0 { + Err(io::Error::other("first reap is uncertain")) + } else { + Ok(()) + } + } + } + + #[test] + fn failed_sa3_reap_retains_ownership_until_a_positive_retry() { + let shutdowns = Arc::new(AtomicUsize::new(0)); + let server = GenerationServer { + state: Mutex::new(GenerationProcessState::Running { + port: 4321, + capability: "capability".to_string(), + process: Box::new(RetryProcess { + shutdowns: shutdowns.clone(), + }), + }), + }; + + assert!(server.quiesce().is_err()); + assert_eq!(server.connection(), None); + assert!(matches!( + &*server.state.lock().unwrap(), + GenerationProcessState::Uncertain { .. } + )); + assert_eq!(shutdowns.load(Ordering::Acquire), 1); + + assert!(matches!(server.quiesce(), Ok(true))); + assert!(matches!( + &*server.state.lock().unwrap(), + GenerationProcessState::Stopped + )); + assert_eq!(shutdowns.load(Ordering::Acquire), 2); + } + + #[test] + fn readiness_error_with_failed_cleanup_retains_ownership_until_second_shutdown() { + let shutdowns = Arc::new(AtomicUsize::new(0)); + let startup = finish_generation_startup( + 4321, + Box::new(RetryProcess { + shutdowns: shutdowns.clone(), + }), + Err(io::Error::other("readiness OS error")), + ); + let failure = match startup { + Err(failure) => failure, + Ok(_) => panic!("readiness error must fail startup"), + }; + assert!(failure.error.to_string().contains("readiness OS error")); + assert!(failure + .error + .to_string() + .contains("startup cleanup also failed")); + assert!(failure.uncertain_process.is_some()); + assert_eq!(shutdowns.load(Ordering::Acquire), 1); + + let server = GenerationServer { + state: Mutex::new(GenerationProcessState::Stopped), + }; + let error = server + .resume_with_spawn(move |_| Err(failure)) + .unwrap_err(); + assert!(error.to_string().contains("readiness OS error")); + assert_eq!(server.connection(), None); + assert!(matches!( + &*server.state.lock().unwrap(), + GenerationProcessState::Uncertain { .. } + )); + + // The startup cleanup was the first shutdown attempt. Quiesce owns the + // second attempt and cannot expose Stopped until that positive reap. + assert!(matches!(server.quiesce(), Ok(false))); + assert!(matches!( + &*server.state.lock().unwrap(), + GenerationProcessState::Stopped + )); + assert_eq!(shutdowns.load(Ordering::Acquire), 2); + } +} diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index dbbf6e4..3cc4765 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -44,10 +44,18 @@ mod commands; mod decode; mod generation; mod library; +mod local_auth; mod loras; +#[cfg(feature = "managed-runtime")] +mod magenta_gateway; +#[cfg_attr(not(feature = "managed-runtime"), allow(dead_code))] +mod managed_runtime; mod mcp; mod midi; mod models; +mod platform_diagnostics; +mod platform_paths; +mod runtime_installer; mod samples; mod settings; mod sidecar; @@ -57,6 +65,9 @@ mod style; mod style_send; mod watcher; +#[cfg(all(feature = "bundled-backend", feature = "managed-runtime"))] +compile_error!("bundled-backend and managed-runtime are mutually exclusive"); + /// The default per-deck model the sidecars load (mirrors `controller.py` /// `DEFAULT_MODEL`). const DEFAULT_MODEL: &str = "mrt2_small"; @@ -86,14 +97,25 @@ fn configure_bundled_backend(app: &tauri::App) -> Result<(), Box Result<(), Box> { + // Managed releases resolve MRT2 and SA3 independently from verified + // manifests. Clear all developer launch seams before either service starts; + // the managed command builders are compile-time isolated from PATH fallback. + std::env::remove_var("LSDJ_BACKEND_BIN"); + std::env::remove_var("LSDJ_SIDECAR_CMD"); + std::env::remove_var("LSDJ_GENERATION_CMD"); + Ok(()) +} + +#[cfg(not(any(feature = "bundled-backend", feature = "managed-runtime")))] fn configure_bundled_backend(_app: &tauri::App) -> Result<(), Box> { Ok(()) } /// Tauri-managed audio state held ALONGSIDE the [`Host`]: the running output -/// streams (kept alive so their Drop does not stop audio), the current device -/// choices, and whether the main device actually started. +/// streams (kept alive so their Drop does not stop audio), their live health, +/// and the current device choices. /// /// The `Host` is managed separately so the commands can take it as /// `tauri::State<'_, Host>` directly. This struct holds the things the commands @@ -107,6 +129,10 @@ fn configure_bundled_backend(_app: &tauri::App) -> Result<(), Box, /// The MAIN output stream — master → ch 1/2, and cue → ch 3/4 in combined /// mode. Kept alive; replaced when the main device (or the combined/split /// mode) changes. `None` in the sandbox/headless case. @@ -114,8 +140,11 @@ struct AudioState { /// The CUE output stream in SPLIT mode (a separate device); `None` in combined /// mode (the cue rides the main stream's 3/4). cue_stream: Mutex>, - /// Whether the main device came up at startup (the `app_info` flag). - device_started: bool, + /// Most recent synchronous open failure for each route. Asynchronous CPAL + /// failures live in each [`AudioStream`]'s atomic signal and are summarized + /// dynamically; these strings only preserve useful startup/reconnect detail. + main_error: Mutex>, + cue_error: Mutex>, /// The current main device name (empty = system default), so a cue-only switch /// can recompute the combined/split topology. main_name: Mutex, @@ -123,6 +152,147 @@ struct AudioState { cue_name: Mutex, } +#[derive(Debug, Clone, Copy)] +struct StreamHealthSnapshot { + healthy: bool, + channels: u16, +} + +/// Live output health projected to `app_info`, diagnostics, and the Settings UI. +/// `cueHealthy` follows the applicable route: the main stream in combined mode, +/// or the dedicated cue stream in split mode. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)] +#[serde(rename_all = "camelCase")] +struct AudioOutputHealth { + main_healthy: bool, + cue_healthy: bool, + main_error: Option, + cue_error: Option, + /// False for a healthy topology and for a healthy stereo main whose only cue + /// issue is that it lacks channels 3/4; choosing another route is required. + can_reconnect: bool, +} + +fn summarize_audio_health( + main: Option, + cue: Option, + combined: bool, + main_open_error: Option, + cue_open_error: Option, +) -> AudioOutputHealth { + // A synchronous failure to open the selected route is authoritative even + // when startup kept a healthy default/fallback stream alive. The fallback + // preserves sound, but it is not the persisted route the UI displays and + // the reconnect command must keep targeting that selection. + let main_healthy = main_open_error.is_none() && main.is_some_and(|stream| stream.healthy); + let main_error = if main_healthy { + None + } else { + main_open_error.or_else(|| { + Some(if main.is_some() { + "The main audio stream stopped after it started.".into() + } else { + "No main audio output is running.".into() + }) + }) + }; + + let (cue_healthy, cue_error, cue_can_reconnect) = if combined && !main_healthy { + ( + false, + Some("The cue is unavailable until the selected main output reconnects.".into()), + true, + ) + } else if combined { + match main { + Some(stream) if stream.channels < 4 => ( + false, + Some("Phones on main require an output with channels 3/4.".into()), + false, + ), + Some(_) => (true, None, false), + None => unreachable!("a combined cue cannot be healthy without a main stream"), + } + } else { + let healthy = cue.is_some_and(|stream| stream.healthy); + let error = if healthy { + None + } else if cue.is_some() { + Some("The cue audio stream stopped after it started.".into()) + } else { + cue_open_error.or_else(|| Some("No cue audio output is running.".into())) + }; + (healthy, error, !healthy) + }; + + AudioOutputHealth { + main_healthy, + cue_healthy, + main_error, + cue_error, + can_reconnect: !main_healthy || cue_can_reconnect, + } +} + +impl AudioState { + fn retain_requested_routes(&self, main_name: String, cue_name: String) { + *self.main_name.lock().unwrap_or_else(|p| p.into_inner()) = main_name; + *self.cue_name.lock().unwrap_or_else(|p| p.into_inner()) = cue_name; + } + + fn output_health(&self) -> AudioOutputHealth { + let main = self + .main_stream + .lock() + .unwrap_or_else(|p| p.into_inner()) + .as_ref() + .map(|stream| StreamHealthSnapshot { + healthy: stream.is_healthy(), + channels: stream.info().device_channels, + }); + let cue = self + .cue_stream + .lock() + .unwrap_or_else(|p| p.into_inner()) + .as_ref() + .map(|stream| StreamHealthSnapshot { + healthy: stream.is_healthy(), + channels: stream.info().device_channels, + }); + let main_name = self + .main_name + .lock() + .unwrap_or_else(|p| p.into_inner()) + .clone(); + let cue_name = self + .cue_name + .lock() + .unwrap_or_else(|p| p.into_inner()) + .clone(); + summarize_audio_health( + main, + cue, + is_combined(&main_name, &cue_name), + self.main_error + .lock() + .unwrap_or_else(|p| p.into_inner()) + .clone(), + self.cue_error + .lock() + .unwrap_or_else(|p| p.into_inner()) + .clone(), + ) + } + + fn set_main_error(&self, error: Option) { + *self.main_error.lock().unwrap_or_else(|p| p.into_inner()) = error; + } + + fn set_cue_error(&self, error: Option) { + *self.cue_error.lock().unwrap_or_else(|p| p.into_inner()) = error; + } +} + /// Deck producer handles NOT owned by a sidecar (sidecars disabled, or a spawn /// failed) — held in managed state only to keep their input rings open (dropping /// a producer closes its ring). Empty when every deck has a live sidecar. @@ -141,7 +311,7 @@ struct SidecarStatus { /// drains the host's output ring, and return the [`Host`], the [`AudioState`] /// holding the stream, and the two deck producer handles (for the sidecar feed). /// The device-start path is graceful: a missing device leaves the host running -/// headlessly with `device_started = false`. +/// headlessly with a live, reconnectable unhealthy status. fn start_audio() -> (Host, AudioState, [DeckHandle; lsdj_engine::DECK_COUNT]) { let (host, master, cue, deck_handles) = Host::new(); @@ -149,7 +319,7 @@ fn start_audio() -> (Host, AudioState, [DeckHandle; lsdj_engine::DECK_COUNT]) { // device is ≥4-channel (the FLX4), exactly as before. A separate cue device is // opted into later via `set_cue_device`. These are the ORIGINAL ring consumers // matching the render thread's producers, so no ring install is needed yet. - let (main_stream, device_started) = + let (main_stream, main_error) = match engine_device::open_main_stream(None, master, Some(cue)) { Ok(stream) => { let info = stream.info(); @@ -158,36 +328,77 @@ fn start_audio() -> (Host, AudioState, [DeckHandle; lsdj_engine::DECK_COUNT]) { "lsdj-app: audio device started — device='{}' channels={} rate={} buffer={:?}", info.device_name, info.device_channels, info.sample_rate, info.buffer_frames ); - (Some(stream), true) + (Some(stream), None) } Err(DeviceError::Unavailable(msg)) => { // Expected in a sandbox / headless CI: no exact-48000/f32 device. // Log and continue with no stream — the host renders into the ring, // the window opens, control/read-back work. eprintln!("lsdj-app: audio device unavailable ({msg}) — continuing without audio"); - (None, false) + (None, Some(DeviceError::Unavailable(msg).to_string())) } Err(DeviceError::Stream(msg)) => { eprintln!("lsdj-app: audio stream error ({msg}) — continuing without audio"); - (None, false) + (None, Some(DeviceError::Stream(msg).to_string())) } }; let state = AudioState { + transition: Mutex::new(()), main_stream: Mutex::new(main_stream), cue_stream: Mutex::new(None), - device_started, + main_error: Mutex::new(main_error), + cue_error: Mutex::new(None), main_name: Mutex::new(String::new()), cue_name: Mutex::new(String::new()), }; (host, state, deck_handles) } -/// Spawn one inference sidecar per deck, each fed by its [`DeckHandle`] and -/// reporting status as a `sidecar://status` Tauri event. Started with the app (the -/// native cutover default — no flag): a deck whose sidecar fails to spawn closes its -/// ring and stays silent, like the no-audio-device path, without failing the app. The -/// returned idle-handle vec is now always empty (kept for the call signature). +fn sidecar_status_sink( + app: tauri::AppHandle, + idx: usize, + status_feed: analysis::live::AnalysisFeed, +) -> Box { + Box::new(move |json| { + use tauri::{Emitter, Manager}; + if let Some(event) = sidecar::status_event(&json) { + if let Some(store) = app.try_state::() { + match event.as_str() { + "worker_died" | "startup_failed" => store.set_worker_health(idx, true, false), + "model_loading" | "warming" => store.set_worker_health(idx, false, true), + "ready" => store.set_worker_health(idx, false, false), + _ => {} + } + } + if event == "ready" { + if let Some(sender) = app.try_state::() { + sender.resend(idx); + } + if let Some(notes) = app.try_state::() { + notes.reassert_generation(idx); + } + } + } + if sidecar::transport_ended(&json) { + if let Some(store) = app.try_state::() { + store.set_playing(idx, false); + } + let origin = app + .try_state::() + .map_or(0.0, |host| host.health().context_frames as f64); + status_feed.reset(idx, origin); + if let Some(notes) = app.try_state::() { + notes.reset(idx); + } + } + let _ = app.emit("sidecar://status", SidecarStatus { deck: idx, json }); + }) +} + +/// Spawn two MLX processes on macOS, or the #109 one-process/two-state PyTorch +/// worker on Linux and Windows. A failed launch leaves the corresponding rings +/// silent without failing the rest of the app. fn start_sidecars( app: &tauri::AppHandle, handles: [DeckHandle; lsdj_engine::DECK_COUNT], @@ -195,86 +406,50 @@ fn start_sidecars( feed: &analysis::live::AnalysisFeed, ) -> (sidecar::Sidecars, Vec) { const DECK_IDS: [&str; lsdj_engine::DECK_COUNT] = ["a", "b"]; + if matches!( + sidecar::mrt2_runtime_for_platform().as_deref(), + Ok("pytorch-cuda") + ) { + let models = std::array::from_fn(|_| DEFAULT_MODEL.to_string()); + let sinks = [ + sidecar_status_sink(app.clone(), 0, feed.clone()), + sidecar_status_sink(app.clone(), 1, feed.clone()), + ]; + let mut shared = sidecar::SharedSidecar::parked( + models, + handles, + sinks, + taps.clone(), + feed.clone(), + ); + if let Err(error) = shared.activate() { + // A fresh managed install has no runtime yet. Keep both permanent + // DeckHandles parked inside the supervisor; the first MRT2 install + // activates this same topology without restarting the app. + eprintln!("lsdj-app: shared MRT2 sidecar unavailable: {error}"); + } + return (sidecar::Sidecars::new_shared(shared), Vec::new()); + } + let mut decks = Vec::new(); for (idx, handle) in handles.into_iter().enumerate() { - let app = app.clone(); let deck_id = DECK_IDS[idx]; - let status_feed = feed.clone(); match sidecar::Sidecar::spawn( deck_id, idx, DEFAULT_MODEL, handle, - move |json| { - use tauri::{Emitter, Manager}; - // Worker health lives in the store too (ADR-0020 phase A): the - // same events the webview reducer derives its operability from - // write the shell-side truth, so an agent sees a dead or - // switching worker without a webview round-trip. - if let Some(event) = sidecar::status_event(&json) { - if let Some(store) = app.try_state::() { - match event.as_str() { - "worker_died" => store.set_worker_health(idx, true, false), - "model_loading" => store.set_worker_health(idx, false, true), - "ready" => store.set_worker_health(idx, false, false), - _ => {} - } - } - // A fresh worker has no conditioning: push the deck's - // current style blend again (ADR-0020 phase B — the - // shell sender owns the resend the webview used to do), and - // re-send the authored generation params (issue #84) — the - // worker starts at the reference baseline, so this is the - // moment the deck's persisted tuning (re)takes effect, for a - // render on a stopped deck as much as the live stream. - if event == "ready" { - if let Some(sender) = app.try_state::() { - sender.resend(idx); - } - if let Some(notes) = app.try_state::() { - notes.reassert_generation(idx); - } - } - } - // The transport derivation lives in Rust (ADR-0020: the store owns - // `playing`): a dying or model-switching worker stops generating, so - // the store drops the deck's transport before the event is relayed. - // `try_state`: the reader threads start before `setup` manages the - // store, and pre-boot status can't concern a playing deck anyway. - if sidecar::transport_ended(&json) { - if let Some(store) = app.try_state::() { - store.set_playing(idx, false); - } - // The stream is discontinuous: reset the deck's beat analysis - // shell-side (ADR-0025 — estimates never span streams), with - // the engine-frame origin captured now. No webview round-trip. - let origin = app - .try_state::() - .map_or(0.0, |host| host.health().context_frames as f64); - status_feed.reset(idx, origin); - // Held note steering dies with the stream too (ADR-0023): - // the worker dropped its conditioning, so the shell service - // must drop the matching held state. - if let Some(notes) = app.try_state::() { - notes.reset(idx); - } - } - let _ = app.emit("sidecar://status", SidecarStatus { deck: idx, json }); - }, + sidecar_status_sink(app.clone(), idx, feed.clone()), taps.clone(), feed.clone(), ) { Ok(sidecar) => decks.push(Some(sidecar)), - Err(e) => { - // A failed spawn drops that deck's handle (ring closes); the deck - // stays silent, like the no-audio-device path. - eprintln!("lsdj-app: deck {deck_id} sidecar spawn failed: {e}"); + Err(error) => { + eprintln!("lsdj-app: deck {deck_id} sidecar spawn failed: {error}"); decks.push(None); } } } - // Every handle was moved into a sidecar (or dropped on a failed spawn), so no - // idle handles remain. (sidecar::Sidecars::new(decks), Vec::new()) } @@ -285,10 +460,21 @@ fn start_sidecars( #[serde(rename_all = "camelCase")] struct AppInfo { version: String, + /// Backward-compatible main-output flag, now computed from the live stream + /// signal rather than frozen at startup. audio_device_started: bool, + audio_output: AudioOutputHealth, /// The loopback port the generation server bound (`None` if disabled / not /// running). The webview builds the `/api/*` base URL from it (gap 2). generation_port: Option, + /// Per-launch bearer capability for the generation service. It exists only in + /// Rust state/the webview process and is never written to settings or logs. + generation_capability: Option, + /// Managed Linux/Windows route Magenta through a separate Rust-owned + /// gateway. On bundled macOS these mirror the combined generation service, + /// preserving its established behavior. + magenta_port: Option, + magenta_capability: Option, /// The loopback port the MCP server bound (`None` only if the loopback bind /// failed — the server is otherwise always on), and the bearer token a client must /// present (ADR-0020 Phase 2). Surfaced so the client config can point at @@ -299,19 +485,44 @@ struct AppInfo { #[tauri::command] fn app_info( + app: tauri::AppHandle, state: tauri::State<'_, AudioState>, generation: tauri::State<'_, generation::GenerationServer>, mcp: tauri::State<'_, mcp::McpServer>, ) -> AppInfo { + let audio_output = state.output_health(); + let generation_connection = generation.connection(); + #[cfg(feature = "managed-runtime")] + let (magenta_port, magenta_capability) = { + let gateway = app.state::(); + (gateway.port(), gateway.capability()) + }; + #[cfg(not(feature = "managed-runtime"))] + let (magenta_port, magenta_capability) = generation_connection + .clone() + .map_or((None, None), |(port, capability)| (Some(port), Some(capability))); + #[cfg(not(feature = "managed-runtime"))] + let _ = app; AppInfo { version: env!("CARGO_PKG_VERSION").to_string(), - audio_device_started: state.device_started, - generation_port: generation.port(), + audio_device_started: audio_output.main_healthy, + audio_output, + generation_port: generation_connection.as_ref().map(|(port, _)| *port), + generation_capability: generation_connection.map(|(_, capability)| capability), + magenta_port, + magenta_capability, mcp_port: mcp.port(), mcp_token: mcp.token(), } } +/// Return current output health without the unrelated app/server diagnostics. +/// Settings polls this at a low fixed rate while its drawer is mounted. +#[tauri::command] +fn audio_output_health(audio: tauri::State<'_, AudioState>) -> AudioOutputHealth { + audio.output_health() +} + /// Mint a new MCP bearer token, persist it, and swap it in live (the Settings /// "Rotate token" button). Returns the new token; errors if the server isn't running. #[tauri::command] @@ -368,6 +579,23 @@ fn selector(name: &str) -> Option<&str> { (!name.is_empty()).then_some(name) } +fn cue_reselect_is_noop( + current_name: &str, + requested_name: &str, + applicable_stream_healthy: bool, +) -> bool { + current_name == requested_name && applicable_stream_healthy +} + +/// The explicit reconnect command rebuilds only failed streams. This avoids a +/// gratuitous master gap when only a split cue device disappeared, and avoids +/// retrying a healthy stereo main when combined cue merely lacks channels 3/4. +fn recovery_targets(health: &AudioOutputHealth, combined: bool) -> (bool, bool) { + let main = !health.main_healthy; + let cue = !combined && !health.cue_healthy; + (main, cue) +} + /// (Re)open the MAIN stream for the given device choices. In combined mode the cue /// rides the main device's channels 3/4 (and any split cue stream is dropped); in /// split mode the main stream is master-only and the existing cue stream is left @@ -391,15 +619,18 @@ fn reopen_main( (None, None) }; let stream = engine_device::open_main_stream(selector(main_name), master_consumer, cue_consumer) - .map_err(|e| e.to_string())?; - if !host.install_master_ring(master_ring) { - return Err(ENGINE_BUSY.into()); - } + .map_err(|error| error.to_string())?; if let Some(cue_ring) = cue_ring { - // Combined: the cue now rides the main stream's 3/4 — install its ring - // (best-effort; the cue is secondary) and drop any split cue stream. - host.install_cue_ring(cue_ring); + // A combined stream consumes both buses. Install the pair with one + // queue command; a full queue must leave both old rings and any split + // cue stream intact rather than accepting only the master half. + if !host.install_output_rings(master_ring, cue_ring) { + return Err(ENGINE_BUSY.to_string()); + } *audio.cue_stream.lock().unwrap_or_else(|p| p.into_inner()) = None; + audio.set_cue_error(None); + } else if !host.install_master_ring(master_ring) { + return Err(ENGINE_BUSY.to_string()); } let info = stream.info(); println!( @@ -407,6 +638,7 @@ fn reopen_main( info.device_name, info.device_channels ); *audio.main_stream.lock().unwrap_or_else(|p| p.into_inner()) = Some(stream); + audio.set_main_error(None); Ok(()) } @@ -415,10 +647,17 @@ fn reopen_main( /// switch never interrupt the audience's master. fn reopen_cue_split(host: &Host, audio: &AudioState, cue_name: &str) -> Result<(), String> { let (cue_ring, cue_consumer) = host.new_output_ring(); - let stream = - engine_device::open_cue_stream(selector(cue_name), cue_consumer).map_err(|e| e.to_string())?; + let stream = engine_device::open_cue_stream(selector(cue_name), cue_consumer).map_err( + |error| { + let error = error.to_string(); + audio.set_cue_error(Some(error.clone())); + error + }, + )?; if !host.install_cue_ring(cue_ring) { - return Err(ENGINE_BUSY.into()); + let error = ENGINE_BUSY.to_string(); + audio.set_cue_error(Some(error.clone())); + return Err(error); } let info = stream.info(); println!( @@ -426,6 +665,7 @@ fn reopen_cue_split(host: &Host, audio: &AudioState, cue_name: &str) -> Result<( info.device_name, info.device_channels ); *audio.cue_stream.lock().unwrap_or_else(|p| p.into_inner()) = Some(stream); + audio.set_cue_error(None); Ok(()) } @@ -440,6 +680,10 @@ fn set_main_device( app: tauri::AppHandle, name: String, ) -> Result<(), String> { + let _transition = audio + .transition + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); let cue_name = audio.cue_name.lock().unwrap_or_else(|p| p.into_inner()).clone(); reopen_main(&host, &audio, &name, &cue_name)?; *audio.main_name.lock().unwrap_or_else(|p| p.into_inner()) = name.clone(); @@ -463,16 +707,25 @@ fn set_cue_device( app: tauri::AppHandle, name: String, ) -> Result<(), String> { + let _transition = audio + .transition + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); let main_name = audio.main_name.lock().unwrap_or_else(|p| p.into_inner()).clone(); - let was_combined = { - let cue_name = audio.cue_name.lock().unwrap_or_else(|p| p.into_inner()); - // Re-selecting the already-active cue device would tear down and rebuild - // the cue stream for no change (a needless cue glitch) — short-circuit. - if name == *cue_name { - return Ok(()); - } - is_combined(&main_name, &cue_name) + let current_cue = audio.cue_name.lock().unwrap_or_else(|p| p.into_inner()).clone(); + let was_combined = is_combined(&main_name, ¤t_cue); + let health = audio.output_health(); + let applicable_stream_healthy = if was_combined { + health.main_healthy + } else { + health.cue_healthy }; + // Re-selecting the already-active cue device would tear down and rebuild the + // cue stream for no change (a needless cue glitch) — short-circuit only while + // its applicable stream is healthy. Re-selecting a failed stream retries. + if cue_reselect_is_noop(¤t_cue, &name, applicable_stream_healthy) { + return Ok(()); + } if is_combined(&main_name, &name) { // Cue rides the main device's 3/4: reopen main with cue duty (it also drops // any split cue stream). @@ -498,12 +751,66 @@ fn set_cue_device( } } *audio.cue_name.lock().unwrap_or_else(|p| p.into_inner()) = name.clone(); - let main_name = audio.main_name.lock().unwrap_or_else(|p| p.into_inner()).clone(); + let main_name = audio + .main_name + .lock() + .unwrap_or_else(|p| p.into_inner()) + .clone(); settings::update(&app, |s| s.cue_device = name.clone()); store.set_output_devices(main_name, name); Ok(()) } +/// Retry failed output streams using the currently selected topology. Transitions +/// are serialized with device switches, and only unhealthy streams are rebuilt: +/// a failed split cue never interrupts a healthy audience-facing main output. +fn reconnect_audio_outputs_with( + audio: &AudioState, + mut reopen_main_route: impl FnMut(&str, &str) -> Result<(), String>, + mut reopen_cue_route: impl FnMut(&str) -> Result<(), String>, +) -> Result { + let main_name = audio.main_name.lock().unwrap_or_else(|p| p.into_inner()).clone(); + let cue_name = audio.cue_name.lock().unwrap_or_else(|p| p.into_inner()).clone(); + let combined = is_combined(&main_name, &cue_name); + let (retry_main, retry_cue) = recovery_targets(&audio.output_health(), combined); + let mut failures = Vec::new(); + + if retry_main { + if let Err(error) = reopen_main_route(&main_name, &cue_name) { + audio.set_main_error(Some(error.clone())); + failures.push(format!("main: {error}")); + } + } + if retry_cue { + if let Err(error) = reopen_cue_route(&cue_name) { + audio.set_cue_error(Some(error.clone())); + failures.push(format!("cue: {error}")); + } + } + + if failures.is_empty() { + Ok(audio.output_health()) + } else { + Err(format!("Couldn't reconnect audio outputs ({})", failures.join("; "))) + } +} + +#[tauri::command] +fn reconnect_audio_outputs( + host: tauri::State<'_, Host>, + audio: tauri::State<'_, AudioState>, +) -> Result { + let _transition = audio + .transition + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + reconnect_audio_outputs_with( + &audio, + |main_name, cue_name| reopen_main(&host, &audio, main_name, cue_name), + |cue_name| reopen_cue_split(&host, &audio, cue_name), + ) +} + /// Set (and persist) the recordings folder — "" = Downloads. The picker's /// native dialog supplies real paths; the recorder recreates or falls back /// at start, so no validation beyond ownership is needed here. @@ -528,11 +835,10 @@ pub fn run() { .plugin(tauri_plugin_opener::init()) .setup(|app| { configure_bundled_backend(app)?; - // Relocate the Magenta model weights out of ~/Documents (which users - // may sync to iCloud) into the app-owned data dir, migrating a prior - // install. MUST run before any backend process spawns so they — and - // magenta_rt.paths, read at import — inherit MAGENTA_HOME (issue #43). - models::ensure_magenta_home(); + // Resolve every filesystem root once and pass the contract to Python + // through inherited environment variables. This also performs the + // restart-safe macOS model migration. MUST precede every service. + platform_paths::configure(app)?; // Start the audio host (engine + render thread + device), then spawn // the per-deck inference sidecars fed by the deck handles. Everything // is held in managed state for the app's lifetime. @@ -546,30 +852,29 @@ pub fn run() { if !shell_settings.main_device.is_empty() || !shell_settings.cue_device.is_empty() { let main = &shell_settings.main_device; let cue = &shell_settings.cue_device; + // The persisted route is the requested topology even when the + // device is currently unplugged. Retain it before attempting the + // open so health and explicit reconnect never fall back to the + // startup default as their target. + audio_state.retain_requested_routes(main.clone(), cue.clone()); match reopen_main(&host, &audio_state, main, cue) { Ok(()) => { - *audio_state.main_name.lock().unwrap_or_else(|p| p.into_inner()) = - main.clone(); + // The cue selection was retained above; if its split + // stream is unplugged, its own open error remains live and + // reconnect keeps targeting the persisted name. if !is_combined(main, cue) { match reopen_cue_split(&host, &audio_state, cue) { - Ok(()) => { - *audio_state - .cue_name - .lock() - .unwrap_or_else(|p| p.into_inner()) = cue.clone(); - } + Ok(()) => {} Err(e) => eprintln!( "lsdj-app: persisted cue device '{cue}' not applied: {e}" ), } - } else { - *audio_state.cue_name.lock().unwrap_or_else(|p| p.into_inner()) = - cue.clone(); } } - Err(e) => eprintln!( - "lsdj-app: persisted main device '{main}' not applied: {e}" - ), + Err(e) => { + audio_state.set_main_error(Some(e.clone())); + eprintln!("lsdj-app: persisted main device '{main}' not applied: {e}"); + } } } // The per-deck analysis PCM taps (gap 1): the sidecars tee model PCM @@ -585,36 +890,36 @@ pub fn run() { // The sa3/Magenta generation server (gap 2): the gen-only FastAPI on a // loopback port the webview fetches; started with the app. let generation_server = generation::GenerationServer::start(); - // The generated-songs library: a fixed folder under the user's Documents - // (override never reaches it from the webview) plus a JSON registry the - // take list restores from. Auto-save / list / load / delete all go - // through it. Fall back to a relative path only if Documents can't be - // resolved (effectively never on macOS) so the app still runs. - let songs_dir = app - .path() - .document_dir() - .map(|d| { - models::migrate_legacy_dir( - &d.join("LSDJ").join("generated_songs"), - &d.join("LSDJai").join("generated_songs"), + #[cfg(feature = "managed-runtime")] + let magenta_gateway = magenta_gateway::MagentaGateway::start(); + // The generated-songs library: the durable-data root from the platform + // contract plus a JSON registry the take list restores from. On macOS + // this remains Documents/LSDJ. Auto-save / list / load / delete all go + // through it. + let paths = platform_paths::get(); + let songs_dir = paths.legacy_data().map_or_else( + || paths.data().join("generated_songs"), + |legacy| { + platform_paths::migrate_legacy_dir( + &paths.data().join("generated_songs"), + &legacy.join("generated_songs"), ) - }) - .unwrap_or_else(|_| std::path::PathBuf::from("LSDJ/generated_songs")); + }, + ); app.manage(songs::SongLibrary::new(songs_dir.clone())); // The generated-samples library: the short-loop counterpart of the songs // folder (ADR-0022), the home for deck freezes / generated pads / composed // SFX-Music that used to die at session end. Same fixed-folder + registry // discipline. - let samples_dir = app - .path() - .document_dir() - .map(|d| { - models::migrate_legacy_dir( - &d.join("LSDJ").join("generated_samples"), - &d.join("LSDJai").join("generated_samples"), + let samples_dir = paths.legacy_data().map_or_else( + || paths.data().join("generated_samples"), + |legacy| { + platform_paths::migrate_legacy_dir( + &paths.data().join("generated_samples"), + &legacy.join("generated_samples"), ) - }) - .unwrap_or_else(|_| std::path::PathBuf::from("LSDJ/generated_samples")); + }, + ); app.manage(samples::SampleLibrary::new(samples_dir.clone())); // Watch both library folders so the Media Explorer tabs live-reload on a // change (a deck auto-saving a sample, a hand-drop/-delete); Rust owns the @@ -764,6 +1069,8 @@ pub fn run() { app.manage(analysis_feed); app.manage(analysis::track::TrackAnalysis::new(lsdj_engine::DECK_COUNT)); app.manage(generation_server); + #[cfg(feature = "managed-runtime")] + app.manage(magenta_gateway); // The native MCP server (ADR-0020 Phase 2): an external agent as a // co-DJ. Always on, loopback-only, token-guarded; its tools mutate the // same managed state the IPC commands do. Reaches that state through the @@ -774,11 +1081,14 @@ pub fn run() { }) .invoke_handler(tauri::generate_handler![ app_info, + audio_output_health, + platform_diagnostics::platform_diagnostics, rotate_mcp_token, set_mcp_port, list_output_devices, set_main_device, set_cue_device, + reconnect_audio_outputs, set_recordings_folder, commands::set_crossfade, commands::set_eq, @@ -887,6 +1197,8 @@ pub fn run() { if let tauri::RunEvent::Exit = event { use tauri::Manager; app.state::().shutdown(); + #[cfg(feature = "managed-runtime")] + app.state::().shutdown(); app.state::().shutdown(); app.state::().shutdown(); app.state::().shutdown(); @@ -896,12 +1208,29 @@ pub fn run() { #[cfg(test)] mod tests { - use super::{bundled_backend_path, is_combined}; + use super::{ + bundled_backend_path, cue_reselect_is_noop, is_combined, recovery_targets, + reconnect_audio_outputs_with, summarize_audio_health, AudioState, StreamHealthSnapshot, + }; + + fn headless_audio_state() -> AudioState { + AudioState { + transition: std::sync::Mutex::new(()), + main_stream: std::sync::Mutex::new(None), + cue_stream: std::sync::Mutex::new(None), + main_error: std::sync::Mutex::new(None), + cue_error: std::sync::Mutex::new(None), + main_name: std::sync::Mutex::new(String::new()), + cue_name: std::sync::Mutex::new(String::new()), + } + } #[test] fn bundled_backend_lives_under_the_tauri_resource_dir() { assert_eq!( - bundled_backend_path(std::path::Path::new("/Applications/LSDJ.app/Contents/Resources")), + bundled_backend_path(std::path::Path::new( + "/Applications/LSDJ.app/Contents/Resources" + )), std::path::Path::new( "/Applications/LSDJ.app/Contents/Resources/lsdj_backend/lsdj_backend" ) @@ -931,4 +1260,167 @@ mod tests { assert!(!is_combined("", "DDJ-FLX4")); // default main, a named cue device assert!(!is_combined("DDJ-FLX4", "Built-in Output")); } + + #[test] + fn startup_without_a_device_is_live_unhealthy_and_reconnectable() { + let health = summarize_audio_health( + None, + None, + true, + Some("audio device unavailable: no default output device".into()), + None, + ); + assert!(!health.main_healthy); + assert!(!health.cue_healthy); + assert!(health.can_reconnect); + assert_eq!( + health.main_error.as_deref(), + Some("audio device unavailable: no default output device") + ); + } + + #[test] + fn persisted_main_open_error_overrides_a_healthy_fallback_stream() { + let health = summarize_audio_health( + Some(StreamHealthSnapshot { + healthy: true, + channels: 4, + }), + None, + true, + Some("persisted USB output is unplugged".into()), + None, + ); + + assert!(!health.main_healthy); + assert!(!health.cue_healthy); + assert!(health.can_reconnect); + assert_eq!( + health.main_error.as_deref(), + Some("persisted USB output is unplugged") + ); + } + + #[test] + fn unplugged_persisted_route_is_retained_and_reconnect_retries_that_target() { + let audio = headless_audio_state(); + audio.retain_requested_routes("Persisted USB".into(), "".into()); + audio.set_main_error(Some("device unplugged".into())); + let attempted = std::cell::RefCell::new(None); + + let result = reconnect_audio_outputs_with( + &audio, + |main_name, cue_name| { + attempted.replace(Some((main_name.to_string(), cue_name.to_string()))); + Ok(()) + }, + |_| panic!("combined topology must not open a split cue"), + ); + + assert!(result.is_ok(), "simulated replug should reopen the route"); + assert_eq!( + attempted.into_inner(), + Some(("Persisted USB".into(), "".into())) + ); + assert_eq!( + *audio.main_name.lock().unwrap(), + "Persisted USB", + "failed boot hydration must not fall back to the default target" + ); + } + + #[test] + fn combined_install_backpressure_remains_visible_and_retryable() { + let audio = headless_audio_state(); + audio.retain_requested_routes("Four-channel USB".into(), "".into()); + audio.set_main_error(Some("stream stopped".into())); + + let result = reconnect_audio_outputs_with( + &audio, + |main_name, cue_name| { + assert_eq!(main_name, "Four-channel USB"); + assert_eq!(cue_name, ""); + Err(super::ENGINE_BUSY.into()) + }, + |_| panic!("combined topology must not attempt an independent cue swap"), + ); + + assert_eq!( + result, + Err(format!( + "Couldn't reconnect audio outputs (main: {})", + super::ENGINE_BUSY + )), + "queue pressure must be reported to the caller" + ); + let health = audio.output_health(); + assert_eq!(health.main_error.as_deref(), Some(super::ENGINE_BUSY)); + assert!(!health.main_healthy); + assert!(!health.cue_healthy); + assert!(health.can_reconnect); + assert_eq!(recovery_targets(&health, true), (true, false)); + } + + #[test] + fn asynchronous_main_failure_invalidates_main_and_combined_cue() { + let health = summarize_audio_health( + Some(StreamHealthSnapshot { + healthy: false, + channels: 4, + }), + None, + true, + None, + None, + ); + assert!(!health.main_healthy); + assert!(!health.cue_healthy); + assert!(health.can_reconnect); + assert!(health.main_error.unwrap().contains("stopped after it started")); + } + + #[test] + fn failed_split_cue_can_recover_without_reopening_healthy_main() { + let health = summarize_audio_health( + Some(StreamHealthSnapshot { + healthy: true, + channels: 2, + }), + Some(StreamHealthSnapshot { + healthy: false, + channels: 4, + }), + false, + None, + None, + ); + assert!(health.main_healthy); + assert!(!health.cue_healthy); + assert_eq!(recovery_targets(&health, false), (false, true)); + } + + #[test] + fn combined_stereo_cue_requires_a_route_change_not_a_retry_loop() { + let health = summarize_audio_health( + Some(StreamHealthSnapshot { + healthy: true, + channels: 2, + }), + None, + true, + None, + None, + ); + assert!(health.main_healthy); + assert!(!health.cue_healthy); + assert!(!health.can_reconnect); + assert_eq!(recovery_targets(&health, true), (false, false)); + } + + #[test] + fn same_cue_selection_is_a_noop_only_while_the_route_is_healthy() { + assert!(cue_reselect_is_noop("DDJ-FLX4", "DDJ-FLX4", true)); + assert!(!cue_reselect_is_noop("DDJ-FLX4", "DDJ-FLX4", false)); + assert!(!cue_reselect_is_noop("DDJ-FLX4", "Built-in", true)); + } } diff --git a/src-tauri/src/local_auth.rs b/src-tauri/src/local_auth.rs new file mode 100644 index 0000000..f7f47c6 --- /dev/null +++ b/src-tauri/src/local_auth.rs @@ -0,0 +1,41 @@ +//! Ephemeral capabilities for loopback-only child services. + +/// Mint a 256-bit capability without touching process-global state or disk. +pub fn generate_capability() -> String { + let bytes: [u8; 32] = rand::random(); + hex::encode(bytes) +} + +/// Length-oblivious constant-time comparison for short secret byte strings. +pub fn constant_time_eq(left: &[u8], right: &[u8]) -> bool { + let max_len = left.len().max(right.len()); + let mut difference = left.len() ^ right.len(); + for index in 0..max_len { + difference |= usize::from( + left.get(index).copied().unwrap_or(0) + ^ right.get(index).copied().unwrap_or(0), + ); + } + difference == 0 +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn capabilities_are_full_width_and_not_reused() { + let first = generate_capability(); + let second = generate_capability(); + assert_eq!(first.len(), 64); + assert!(first.bytes().all(|byte| byte.is_ascii_hexdigit())); + assert_ne!(first, second); + } + + #[test] + fn secret_comparison_rejects_wrong_values_and_lengths() { + assert!(constant_time_eq(b"same", b"same")); + assert!(!constant_time_eq(b"same", b"samf")); + assert!(!constant_time_eq(b"same", b"same-longer")); + } +} diff --git a/src-tauri/src/loras.rs b/src-tauri/src/loras.rs index 2b445c9..179d0b3 100644 --- a/src-tauri/src/loras.rs +++ b/src-tauri/src/loras.rs @@ -6,7 +6,7 @@ //! models (no central index file): //! //! ```text -//! ~/Library/Application Support/LSDJ/sa3-loras/// +//! /sa3-loras/// //! adapter_model.safetensors (the adapter — any single *.safetensors) //! adapter_config.json (PEFT convention only) //! lora.json (import manifest: source / type / rank) @@ -25,11 +25,14 @@ use std::collections::BTreeMap; use std::io::Read; use std::path::{Path, PathBuf}; -use std::process::Command; use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; -use crate::models::{cancelled, dir_size, stream_child, InstallShared, Progress}; +use crate::models::{cancelled, dir_size, is_cancelled, InstallShared, Progress}; +use crate::runtime_installer::download::{ + client as installer_client, download_verified, fetch_bytes_bounded, PinnedArtifact, +}; /// The two DiT families an adapter can ride (`loras.BASES` in Python). pub const BASES: &[&str] = &["small", "medium"]; @@ -47,18 +50,16 @@ const PICKLE_EXTS: &[&str] = &["ckpt", "pt", "pth", "bin"]; // A safetensors JSON header beyond this is not a plausible adapter — bail // before allocating attacker-controlled sizes. const MAX_HEADER_BYTES: u64 = 64 * 1024 * 1024; +const MAX_HF_INFO_BYTES: u64 = 8 * 1024 * 1024; +const MAX_ADAPTER_BYTES: u64 = 4 * 1024 * 1024 * 1024; +const MAX_CONFIG_BYTES: u64 = 1024 * 1024; const MANIFEST: &str = "lora.json"; -/// The registry root. `$SA3_LORAS_HOME` wins (dev/test override); otherwise the -/// app-owned data dir, beside the SA3 checkout. Mirrors `loras.loras_dir`. +/// The registry root resolved by the Rust host. An explicit dev/user override is +/// captured into this contract during startup; callers never guess an OS path. pub fn loras_dir() -> PathBuf { - if let Some(override_home) = std::env::var_os("SA3_LORAS_HOME") { - if !override_home.is_empty() { - return PathBuf::from(override_home); - } - } - crate::models::app_support_base().join("sa3-loras") + crate::platform_paths::get().loras_home().to_path_buf() } // --- Registry discovery ---------------------------------------------------- @@ -83,20 +84,43 @@ pub struct LoraInfo { #[serde(rename_all = "camelCase")] struct LoraManifest { source: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + revision: Option, convention: String, adapter_type: String, rank: Option, + #[serde(default)] + files: Vec, +} + +#[derive(Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct LoraArtifactRecord { + filename: String, + size: u64, + sha256: String, } /// A well-formed adapter directory: exactly one `*.safetensors` inside (the /// same rule as `loras._adapter_file` in Python and the runtime's resolver). fn adapter_file(dir: &Path) -> Option { + let directory = std::fs::symlink_metadata(dir).ok()?; + if directory.file_type().is_symlink() || !directory.is_dir() { + return None; + } + let canonical_dir = std::fs::canonicalize(dir).ok()?; let entries = std::fs::read_dir(dir).ok()?; let mut hits: Vec = entries .flatten() .map(|entry| entry.path()) .filter(|path| { - path.is_file() && path.extension().is_some_and(|ext| ext == "safetensors") + std::fs::symlink_metadata(path).is_ok_and(|metadata| { + !metadata.file_type().is_symlink() + && metadata.is_file() + && path.extension().is_some_and(|ext| ext == "safetensors") + && std::fs::canonicalize(path) + .is_ok_and(|canonical| canonical.starts_with(&canonical_dir)) + }) }) .collect(); if hits.len() == 1 { @@ -131,19 +155,49 @@ fn parse_name(name: &str) -> Result<(&str, &str), String> { /// model discovery: unreadable entries and malformed directories are skipped. pub fn discover(root: &Path) -> Vec { let mut adapters = Vec::new(); + let Ok(root_metadata) = std::fs::symlink_metadata(root) else { + return adapters; + }; + if root_metadata.file_type().is_symlink() || !root_metadata.is_dir() { + return adapters; + } + let Ok(canonical_root) = std::fs::canonicalize(root) else { + return adapters; + }; for base in BASES { - let Ok(entries) = std::fs::read_dir(root.join(base)) else { + let base_dir = root.join(base); + let Ok(base_metadata) = std::fs::symlink_metadata(&base_dir) else { + continue; + }; + if base_metadata.file_type().is_symlink() + || !base_metadata.is_dir() + || !std::fs::canonicalize(&base_dir).is_ok_and(|path| path.starts_with(&canonical_root)) + { + continue; + } + let Ok(entries) = std::fs::read_dir(&base_dir) else { continue; }; for entry in entries.flatten() { let dir = entry.path(); let slug = entry.file_name().to_string_lossy().into_owned(); - if !dir.is_dir() || !valid_slug(&slug) || adapter_file(&dir).is_none() { + let trusted_dir = std::fs::symlink_metadata(&dir).is_ok_and(|metadata| { + !metadata.file_type().is_symlink() + && metadata.is_dir() + && std::fs::canonicalize(&dir) + .is_ok_and(|path| path.starts_with(&canonical_root)) + }); + if !trusted_dir || !valid_slug(&slug) || adapter_file(&dir).is_none() { continue; } let manifest: Option = std::fs::read_to_string(dir.join(MANIFEST)) .ok() .and_then(|data| serde_json::from_str(&data).ok()); + if manifest.as_ref().is_some_and(|item| { + !item.files.is_empty() && validate_registry_generation(&dir).is_err() + }) { + continue; + } adapters.push(LoraInfo { name: format!("{base}/{slug}"), base: (*base).to_string(), @@ -220,8 +274,8 @@ fn read_safetensors_header(path: &Path) -> Result { if path.extension().is_none_or(|ext| ext != "safetensors") { return Err(format!("'{file_name}' is not a .safetensors adapter")); } - let mut file = std::fs::File::open(path) - .map_err(|e| format!("cannot open '{file_name}': {e}"))?; + let mut file = + std::fs::File::open(path).map_err(|e| format!("cannot open '{file_name}': {e}"))?; let mut len_bytes = [0u8; 8]; file.read_exact(&mut len_bytes) .map_err(|_| format!("'{file_name}' is not a safetensors file"))?; @@ -319,9 +373,8 @@ fn base_from_model_name(model_name: &str) -> Option<&'static str> { /// `rank × fan_in`, `M_xs` is `rank × rank`. fn rank_from_tensors(tensors: &BTreeMap>) -> Option { for (key, shape) in tensors { - let is_rank_first = key.ends_with(".lora_A.weight") - || key.ends_with(".lora_A") - || key.ends_with(".M_xs"); + let is_rank_first = + key.ends_with(".lora_A.weight") || key.ends_with(".lora_A") || key.ends_with(".M_xs"); if is_rank_first { if let Some(&rank) = shape.first() { return u32::try_from(rank).ok(); @@ -360,9 +413,7 @@ pub fn validate_adapter(path: &Path) -> Result { if tensors.keys().any(|key| key.ends_with(".lora_A.weight")) { let config_path = path.with_file_name("adapter_config.json"); if !config_path.is_file() { - return Err( - "the PEFT adapter is missing its adapter_config.json sibling".into(), - ); + return Err("the PEFT adapter is missing its adapter_config.json sibling".into()); } let config: PeftConfig = std::fs::read_to_string(&config_path) .ok() @@ -508,9 +559,9 @@ fn choose_hf_files(filenames: &[String]) -> Result, String> { match safetensors.as_slice() { [single] => (*single).clone(), [] => { - let has_pickle = filenames.iter().any(|name| { - is_pickle(Path::new(name.as_str())) - }); + let has_pickle = filenames + .iter() + .any(|name| is_pickle(Path::new(name.as_str()))); return Err(if has_pickle { "the repo only ships pickle-format weights (.ckpt/.pt/.bin), \ which are refused — only .safetensors adapters are accepted" @@ -519,7 +570,11 @@ fn choose_hf_files(filenames: &[String]) -> Result, String> { "no .safetensors adapter in the repo".into() }); } - _ => return Err("the repo holds more than one .safetensors — not a single adapter".into()), + _ => { + return Err( + "the repo holds more than one .safetensors — not a single adapter".into(), + ) + } } }; let mut files = vec![adapter]; @@ -532,12 +587,23 @@ fn choose_hf_files(filenames: &[String]) -> Result, String> { /// The HF model-info response — only the file list is read. #[derive(Deserialize)] struct HfModelInfo { + sha: String, siblings: Vec, } #[derive(Deserialize)] struct HfSibling { rfilename: String, + #[serde(default)] + size: Option, + #[serde(default)] + lfs: Option, +} + +#[derive(Deserialize)] +struct HfLfs { + sha256: String, + size: u64, } /// Run one adapter import to completion (on the install thread): fetch or copy @@ -568,7 +634,7 @@ pub(crate) fn install( // The HF staging dir is cleaned up on every exit; a local import's source // files are the user's and stay put. if let Some(temp) = &staged.temp { - let _ = std::fs::remove_dir_all(temp); + drop(StagingCleanup(Some(temp.clone()))); } result } @@ -580,6 +646,9 @@ struct StagedAdapter { adapter: PathBuf, config: Option, slug_seed: String, + source: String, + revision: Option, + files: Vec, /// The temp dir to clean up after placing (None for a local import, whose /// source files are the user's and must stay put). temp: Option, @@ -588,10 +657,7 @@ struct StagedAdapter { /// Reconcile the inferred base with an explicit choice. An explicit base wins /// only when the shapes are silent; a contradiction is refused with the /// reasoning (the issue's "incompatible adapters refused with clear reasoning"). -fn resolve_base( - facts: &AdapterFacts, - explicit: Option<&str>, -) -> Result<&'static str, String> { +fn resolve_base(facts: &AdapterFacts, explicit: Option<&str>) -> Result<&'static str, String> { match (facts.inferred_base, explicit) { (Some(inferred), Some(chosen)) if inferred != chosen => Err(format!( "the adapter's layer widths identify the {inferred} DiT — it cannot ride \ @@ -611,9 +677,81 @@ fn resolve_base( } } -/// Fetch an adapter from HuggingFace into a temp dir: the model-info file list -/// first, then each chosen file via the resolve endpoint. Uses `curl` like the -/// SA3 checkout fetch (no HTTP stack in the shell). +fn create_private_staging(parent: &Path, prefix: &str) -> Result { + std::fs::create_dir_all(parent) + .map_err(|error| format!("cannot create adapter staging root: {error}"))?; + for _ in 0..32 { + let directory = parent.join(format!("{prefix}-{:032x}", rand::random::())); + match std::fs::create_dir(&directory) { + Ok(()) => { + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&directory, std::fs::Permissions::from_mode(0o700)) + .map_err(|error| format!("cannot protect adapter staging: {error}"))?; + } + return Ok(directory); + } + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => continue, + Err(error) => return Err(format!("cannot create adapter staging: {error}")), + } + } + Err("cannot allocate a unique adapter staging directory".into()) +} + +struct StagingCleanup(Option); + +impl Drop for StagingCleanup { + fn drop(&mut self) { + let Some(path) = self.0.take() else { return }; + match std::fs::symlink_metadata(&path) { + Ok(metadata) if metadata.file_type().is_symlink() => { + let _ = std::fs::remove_file(path); + } + Ok(metadata) if metadata.is_dir() => { + let _ = std::fs::remove_dir_all(path); + } + _ => {} + } + } +} + +fn hash_record(path: &Path) -> Result { + let metadata = std::fs::symlink_metadata(path) + .map_err(|error| format!("cannot inspect staged adapter artifact: {error}"))?; + if metadata.file_type().is_symlink() || !metadata.is_file() { + return Err("staged adapter artifact is not a regular file".into()); + } + let filename = path + .file_name() + .and_then(|name| name.to_str()) + .filter(|name| valid_slug(name)) + .ok_or("staged adapter filename is unsafe")? + .to_string(); + let bytes = std::fs::read(path) + .map_err(|error| format!("cannot hash staged adapter artifact: {error}"))?; + Ok(LoraArtifactRecord { + filename, + size: metadata.len(), + sha256: hex::encode(Sha256::digest(bytes)), + }) +} + +fn write_private_file(path: &Path, bytes: &[u8]) -> Result<(), String> { + use std::io::Write; + let mut file = std::fs::OpenOptions::new() + .create_new(true) + .write(true) + .open(path) + .map_err(|error| format!("cannot create staged adapter artifact: {error}"))?; + file.write_all(bytes) + .and_then(|_| file.sync_all()) + .map_err(|error| format!("cannot sync staged adapter artifact: {error}")) +} + +/// Fetch an adapter from an immutable Hugging Face commit. Metadata and files +/// use the app's bounded/cancellable HTTPS client; LFS artifacts are verified +/// against the repository-reported SHA-256 before they become import inputs. fn fetch_hf_adapter( progress: &Progress, shared: &InstallShared, @@ -622,54 +760,103 @@ fn fetch_hf_adapter( if !valid_hf_repo(repo) { return Err(format!("'{repo}' is not a HuggingFace repo id")); } - let temp = std::env::temp_dir().join(format!("lsdj-lora-{}", std::process::id())); - let _ = std::fs::remove_dir_all(&temp); - std::fs::create_dir_all(&temp).map_err(|e| format!("cannot create temp dir: {e}"))?; + let temp = create_private_staging( + &crate::platform_paths::get().staging().join("loras"), + "download", + )?; + let mut cleanup = StagingCleanup(Some(temp.clone())); progress("fetch", None, None); - let info_path = temp.join("model-info.json"); - let mut curl = Command::new("curl"); - curl.args(["-fLsS", "-o"]) - .arg(&info_path) - .arg(format!("https://huggingface.co/api/models/{repo}")); - stream_child(shared, "hf-info", curl, |_| {}) - .map_err(|e| format!("cannot reach the HuggingFace repo '{repo}': {e}"))?; - cancelled(shared)?; - let info: HfModelInfo = std::fs::read_to_string(&info_path) + let client = installer_client()?; + let token = std::env::var("HF_TOKEN") .ok() - .and_then(|data| serde_json::from_str(&data).ok()) - .ok_or_else(|| format!("unexpected HuggingFace response for '{repo}'"))?; + .or_else(|| std::env::var("HUGGING_FACE_HUB_TOKEN").ok()); + let info_bytes = fetch_bytes_bounded( + &client, + &format!("https://huggingface.co/api/models/{repo}?blobs=true"), + token.as_deref(), + MAX_HF_INFO_BYTES, + || is_cancelled(shared), + ) + .map_err(|error| format!("cannot reach the HuggingFace repo '{repo}': {error}"))?; + cancelled(shared)?; + let info: HfModelInfo = serde_json::from_slice(&info_bytes) + .map_err(|_| format!("unexpected HuggingFace response for '{repo}'"))?; + if info.sha.len() != 40 || !info.sha.bytes().all(|byte| byte.is_ascii_hexdigit()) { + return Err("HuggingFace did not return a full immutable repository revision".into()); + } let filenames: Vec = info .siblings - .into_iter() - .map(|sibling| sibling.rfilename) + .iter() + .map(|sibling| sibling.rfilename.clone()) .collect(); let files = choose_hf_files(&filenames)?; let mut adapter = None; let mut config = None; + let mut records = Vec::new(); for file in &files { cancelled(shared)?; progress("download", None, Some(file.clone())); let dest = temp.join(file); - let mut curl = Command::new("curl"); - curl.args(["-fLsS", "-o"]) - .arg(&dest) - .arg(format!("https://huggingface.co/{repo}/resolve/main/{file}")); - stream_child(shared, "hf-download", curl, |_| {}) - .map_err(|e| format!("download of '{file}' failed: {e}"))?; + let sibling = info + .siblings + .iter() + .find(|sibling| sibling.rfilename == *file) + .ok_or("chosen adapter file disappeared from repository metadata")?; + let url = format!( + "https://huggingface.co/{repo}/resolve/{}/{file}?download=true", + info.sha + ); + if let Some(lfs) = &sibling.lfs { + let bound = if file.ends_with(".safetensors") { + MAX_ADAPTER_BYTES + } else { + MAX_CONFIG_BYTES + }; + if lfs.size == 0 + || lfs.size > bound + || sibling.size.is_some_and(|size| size != lfs.size) + { + return Err(format!( + "repository metadata has an invalid size for '{file}'" + )); + } + let artifact = PinnedArtifact { + url, + sha256: lfs.sha256.clone(), + size: lfs.size, + }; + download_verified(&client, &artifact, &dest, token.as_deref(), || { + is_cancelled(shared) + })?; + } else { + if file.ends_with(".safetensors") { + return Err("HuggingFace adapter weights lack SHA-256 LFS provenance".into()); + } + let bytes = + fetch_bytes_bounded(&client, &url, token.as_deref(), MAX_CONFIG_BYTES, || { + is_cancelled(shared) + })?; + write_private_file(&dest, &bytes)?; + } + records.push(hash_record(&dest)?); if file.ends_with(".safetensors") { adapter = Some(dest); } else { config = Some(dest); } } + let retained_temp = cleanup.0.take().ok_or("adapter staging disappeared")?; Ok(StagedAdapter { adapter: adapter.ok_or("the repo download produced no adapter")?, config, // The repo's own name seeds the slug (`owner/name` → `name`). slug_seed: repo.split('/').next_back().unwrap_or(repo).to_string(), - temp: Some(temp), + source: format!("https://huggingface.co/{repo}"), + revision: Some(info.sha), + files: records, + temp: Some(retained_temp), }) } @@ -684,13 +871,33 @@ fn stage_local_adapter(path: &Path) -> Result { execute arbitrary code)" )); } - let adapter = if path.is_dir() { - adapter_file(path).ok_or_else(|| { - format!("expected one .safetensors adapter in '{file_name}'") - })? + let source_metadata = std::fs::symlink_metadata(path) + .map_err(|error| format!("cannot inspect local adapter source: {error}"))?; + if source_metadata.file_type().is_symlink() { + return Err("local adapter source must not be a symbolic link or reparse point".into()); + } + let adapter = if source_metadata.is_dir() { + adapter_file(path) + .ok_or_else(|| format!("expected one .safetensors adapter in '{file_name}'"))? + } else if source_metadata.is_file() { + path.to_path_buf() } else { + return Err("local adapter source is not a regular file or directory".into()); + }; + let source_root = if source_metadata.is_dir() { path.to_path_buf() + } else { + path.parent() + .ok_or("local adapter file has no parent directory")? + .to_path_buf() }; + let canonical_root = std::fs::canonicalize(&source_root) + .map_err(|error| format!("cannot canonicalize local adapter root: {error}"))?; + let canonical_adapter = std::fs::canonicalize(&adapter) + .map_err(|error| format!("cannot canonicalize local adapter: {error}"))?; + if !canonical_adapter.starts_with(&canonical_root) { + return Err("local adapter escapes its selected source directory".into()); + } let config_path = adapter.with_file_name("adapter_config.json"); let slug_seed = adapter .file_stem() @@ -708,10 +915,30 @@ fn stage_local_adapter(path: &Path) -> Result { } else { slug_seed }; + let config = match std::fs::symlink_metadata(&config_path) { + Ok(metadata) if !metadata.file_type().is_symlink() && metadata.is_file() => { + let canonical = std::fs::canonicalize(&config_path) + .map_err(|error| format!("cannot canonicalize adapter config: {error}"))?; + if !canonical.starts_with(&canonical_root) { + return Err("adapter config escapes its selected source directory".into()); + } + Some(config_path) + } + Ok(_) => return Err("adapter config must not be a symbolic link or reparse point".into()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => None, + Err(error) => return Err(format!("cannot inspect adapter config: {error}")), + }; + let mut files = vec![hash_record(&adapter)?]; + if let Some(config) = &config { + files.push(hash_record(config)?); + } Ok(StagedAdapter { adapter, - config: config_path.is_file().then_some(config_path), + config, slug_seed, + source: canonical_root.to_string_lossy().into_owned(), + revision: None, + files, temp: None, }) } @@ -727,17 +954,36 @@ fn place_adapter( facts: &AdapterFacts, ) -> Result<(), String> { let dest = root.join(base).join(slug); - if dest.exists() { - return Err(format!( - "an adapter named '{base}/{slug}' is already installed — delete it first" - )); + match std::fs::symlink_metadata(&dest) { + Ok(_) => { + return Err(format!( + "an adapter named '{base}/{slug}' is already installed — delete it first" + )) + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(format!("cannot inspect adapter destination: {error}")), } std::fs::create_dir_all(root.join(base)) .map_err(|e| format!("cannot create the adapter registry: {e}"))?; - let staging = root.join(base).join(format!(".{slug}.importing")); - let _ = std::fs::remove_dir_all(&staging); - std::fs::create_dir_all(&staging) - .map_err(|e| format!("cannot stage the adapter: {e}"))?; + let base_dir = root.join(base); + let root_metadata = std::fs::symlink_metadata(root) + .map_err(|error| format!("cannot inspect adapter registry: {error}"))?; + let base_metadata = std::fs::symlink_metadata(&base_dir) + .map_err(|error| format!("cannot inspect adapter base directory: {error}"))?; + let canonical_root = std::fs::canonicalize(root) + .map_err(|error| format!("cannot canonicalize adapter registry: {error}"))?; + let canonical_base = std::fs::canonicalize(&base_dir) + .map_err(|error| format!("cannot canonicalize adapter base directory: {error}"))?; + if root_metadata.file_type().is_symlink() + || base_metadata.file_type().is_symlink() + || !root_metadata.is_dir() + || !base_metadata.is_dir() + || !canonical_base.starts_with(&canonical_root) + { + return Err("adapter registry path is not a trusted contained directory".into()); + } + recover_import_staging(&base_dir)?; + let staging = create_private_staging(&base_dir, ".lsdj-import")?; let place = (|| -> Result<(), String> { let adapter_name = staged @@ -751,17 +997,18 @@ fn place_adapter( .map_err(|e| format!("cannot copy adapter_config.json: {e}"))?; } let manifest = LoraManifest { - source: staged.slug_seed.clone(), + source: staged.source.clone(), + revision: staged.revision.clone(), convention: facts.convention.as_str().to_string(), adapter_type: facts.adapter_type.clone(), rank: facts.rank, + files: staged.files.clone(), }; let json = serde_json::to_string_pretty(&manifest) .map_err(|e| format!("cannot write the manifest: {e}"))?; - std::fs::write(staging.join(MANIFEST), json) - .map_err(|e| format!("cannot write the manifest: {e}"))?; - std::fs::rename(&staging, &dest) - .map_err(|e| format!("cannot place the adapter: {e}")) + write_private_file(&staging.join(MANIFEST), json.as_bytes())?; + validate_registry_generation(&staging)?; + std::fs::rename(&staging, &dest).map_err(|e| format!("cannot place the adapter: {e}")) })(); if place.is_err() { let _ = std::fs::remove_dir_all(&staging); @@ -769,6 +1016,75 @@ fn place_adapter( place } +fn recover_import_staging(base_dir: &Path) -> Result<(), String> { + let entries = match std::fs::read_dir(base_dir) { + Ok(entries) => entries, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(error) => return Err(format!("cannot inspect adapter staging: {error}")), + }; + for entry in entries { + let entry = entry.map_err(|error| format!("cannot inspect adapter staging: {error}"))?; + let name = entry.file_name(); + if !name.to_string_lossy().starts_with(".lsdj-import-") { + continue; + } + let metadata = std::fs::symlink_metadata(entry.path()) + .map_err(|error| format!("cannot inspect interrupted adapter staging: {error}"))?; + if metadata.file_type().is_symlink() || !metadata.is_dir() { + return Err("interrupted adapter staging is not a trusted directory".into()); + } + std::fs::remove_dir_all(entry.path()) + .map_err(|error| format!("cannot recover interrupted adapter staging: {error}"))?; + } + Ok(()) +} + +fn validate_registry_generation(directory: &Path) -> Result<(), String> { + let adapter = + adapter_file(directory).ok_or("adapter generation has no unique regular weights")?; + let manifest_path = directory.join(MANIFEST); + let metadata = std::fs::symlink_metadata(&manifest_path) + .map_err(|error| format!("cannot inspect adapter provenance: {error}"))?; + if metadata.file_type().is_symlink() || !metadata.is_file() { + return Err("adapter provenance is not a regular file".into()); + } + let manifest: LoraManifest = serde_json::from_slice( + &std::fs::read(&manifest_path) + .map_err(|error| format!("cannot read adapter provenance: {error}"))?, + ) + .map_err(|error| format!("adapter provenance is invalid: {error}"))?; + if manifest.files.is_empty() { + return Err("adapter provenance has no artifact inventory".into()); + } + let expected = manifest + .files + .iter() + .map(|record| (record.filename.clone(), record)) + .collect::>(); + let adapter_name = adapter + .file_name() + .and_then(|name| name.to_str()) + .ok_or("adapter filename is unsafe")?; + if !expected.contains_key(adapter_name) { + return Err("adapter provenance does not inventory its weights".into()); + } + for (filename, record) in expected { + if !valid_slug(&filename) + || record.size == 0 + || hex::decode(&record.sha256).map_or(true, |digest| digest.len() != 32) + { + return Err("adapter provenance contains an invalid artifact".into()); + } + let actual = hash_record(&directory.join(&filename))?; + if actual.size != record.size || actual.sha256 != record.sha256.to_ascii_lowercase() { + return Err(format!( + "adapter artifact '{filename}' failed provenance validation" + )); + } + } + Ok(()) +} + // --- Tauri commands -------------------------------------------------------- /// Import an adapter from a HuggingFace repo id or a local path (issue #66). @@ -791,8 +1107,21 @@ pub fn install_lora( pub fn delete_lora(app: tauri::AppHandle, name: String) -> Result<(), String> { use tauri::Emitter; let (base, slug) = parse_name(&name)?; - let dir = loras_dir().join(base).join(slug); - if adapter_file(&dir).is_none() { + let root = loras_dir(); + let dir = root.join(base).join(slug); + let root_metadata = std::fs::symlink_metadata(&root) + .map_err(|error| format!("cannot inspect adapter registry: {error}"))?; + let dir_metadata = + std::fs::symlink_metadata(&dir).map_err(|_| format!("unknown adapter '{name}'"))?; + let contained = !root_metadata.file_type().is_symlink() + && root_metadata.is_dir() + && !dir_metadata.file_type().is_symlink() + && dir_metadata.is_dir() + && std::fs::canonicalize(&root).is_ok_and(|canonical_root| { + std::fs::canonicalize(&dir) + .is_ok_and(|canonical_dir| canonical_dir.starts_with(canonical_root)) + }); + if !contained || adapter_file(&dir).is_none() { return Err(format!("unknown adapter '{name}'")); } std::fs::remove_dir_all(&dir).map_err(|e| format!("cannot delete '{name}': {e}"))?; @@ -891,11 +1220,17 @@ mod tests { let path = tmp.join("adapter_model.safetensors"); write_safetensors( &path, - &[("x.lora_A.weight", &[8, 1024]), ("x.lora_B.weight", &[1024, 8])], + &[ + ("x.lora_A.weight", &[8, 1024]), + ("x.lora_B.weight", &[1024, 8]), + ], &[], ); let error = validate_adapter(&path).unwrap_err(); - assert!(error.contains("adapter_config.json"), "unexpected error: {error}"); + assert!( + error.contains("adapter_config.json"), + "unexpected error: {error}" + ); let _ = std::fs::remove_dir_all(&tmp); } @@ -915,7 +1250,10 @@ mod tests { &[3072, 16], ), ], - &[("lora_config", r#"{"adapter_type": "dora", "rank": 16, "alpha": 32}"#)], + &[( + "lora_config", + r#"{"adapter_type": "dora", "rank": 16, "alpha": 32}"#, + )], ); let facts = validate_adapter(&path).unwrap(); assert_eq!(facts.convention, Convention::Native); @@ -960,7 +1298,10 @@ mod tests { let path = tmp.join("weights.safetensors"); write_safetensors(&path, &[("model.embed.weight", &[512, 1024])], &[]); let error = validate_adapter(&path).unwrap_err(); - assert!(error.contains("not a recognised"), "unexpected error: {error}"); + assert!( + error.contains("not a recognised"), + "unexpected error: {error}" + ); let _ = std::fs::remove_dir_all(&tmp); } @@ -1072,12 +1413,7 @@ mod tests { ); let registry = root.join("registry"); - let staged = StagedAdapter { - adapter: source.clone(), - config: None, - slug_seed: "maqam".into(), - temp: None, - }; + let staged = stage_local_adapter(&source).unwrap(); let facts = validate_adapter(&source).unwrap(); let base = resolve_base(&facts, None).unwrap(); place_adapter(®istry, base, "maqam", &staged, &facts).unwrap(); @@ -1092,7 +1428,10 @@ mod tests { // A second import under the same name is refused, not overwritten. let error = place_adapter(®istry, base, "maqam", &staged, &facts).unwrap_err(); - assert!(error.contains("already installed"), "unexpected error: {error}"); + assert!( + error.contains("already installed"), + "unexpected error: {error}" + ); let _ = std::fs::remove_dir_all(&root); } @@ -1102,7 +1441,11 @@ mod tests { // Well-formed. let good = root.join("small").join("crackle"); std::fs::create_dir_all(&good).unwrap(); - write_safetensors(&good.join("crackle.safetensors"), &[("x.lora_A", &[4, 1024])], &[]); + write_safetensors( + &good.join("crackle.safetensors"), + &[("x.lora_A", &[4, 1024])], + &[], + ); // No safetensors. std::fs::create_dir_all(root.join("small").join("empty")).unwrap(); // Two safetensors — ambiguous, skipped (matches the Python resolver). @@ -1113,16 +1456,66 @@ mod tests { // A dot-dir never becomes a name. let hidden = root.join("medium").join(".importing"); std::fs::create_dir_all(&hidden).unwrap(); - write_safetensors(&hidden.join("x.safetensors"), &[("x.lora_A", &[4, 1536])], &[]); + write_safetensors( + &hidden.join("x.safetensors"), + &[("x.lora_A", &[4, 1536])], + &[], + ); let names: Vec = discover(&root).into_iter().map(|info| info.name).collect(); assert_eq!(names, vec!["small/crackle".to_string()]); let _ = std::fs::remove_dir_all(&root); } + #[cfg(unix)] + #[test] + fn discovery_and_local_import_reject_symlink_escape_paths() { + use std::os::unix::fs::symlink; + + let root = temp_root("symlink-escape"); + let outside = root.join("outside"); + std::fs::create_dir_all(&outside).unwrap(); + let weights = outside.join("escape.safetensors"); + write_safetensors(&weights, &[("x.lora_A", &[4, 1024])], &[]); + + let registry = root.join("registry"); + std::fs::create_dir_all(registry.join("small")).unwrap(); + symlink(&outside, registry.join("small").join("escaped-dir")).unwrap(); + assert!(discover(®istry).is_empty()); + + let linked_file = root.join("linked.safetensors"); + symlink(&weights, &linked_file).unwrap(); + let error = match stage_local_adapter(&linked_file) { + Ok(_) => panic!("symlink import unexpectedly succeeded"), + Err(error) => error, + }; + assert!(error.contains("symbolic link")); + let _ = std::fs::remove_dir_all(&root); + } + + #[test] + fn interrupted_random_staging_is_recovered_without_touching_other_entries() { + let root = temp_root("recover-staging"); + let base = root.join("small"); + std::fs::create_dir_all(&base).unwrap(); + let interrupted = base.join(".lsdj-import-0123456789abcdef"); + std::fs::create_dir_all(&interrupted).unwrap(); + std::fs::write(interrupted.join("partial"), b"partial").unwrap(); + let unrelated = base.join("leave-me"); + std::fs::create_dir_all(&unrelated).unwrap(); + + recover_import_staging(&base).unwrap(); + assert!(!interrupted.exists()); + assert!(unrelated.is_dir()); + let _ = std::fs::remove_dir_all(&root); + } + #[test] fn slugs_are_derived_and_sanitised() { - assert_eq!(slugify("stable-audio-3-maqam-lora").unwrap(), "stable-audio-3-maqam-lora"); + assert_eq!( + slugify("stable-audio-3-maqam-lora").unwrap(), + "stable-audio-3-maqam-lora" + ); assert_eq!(slugify("My Adapter (v2)").unwrap(), "My-Adapter--v2-"); assert_eq!(slugify("..sneaky").unwrap(), "sneaky"); assert!(slugify("...").is_err()); diff --git a/src-tauri/src/magenta_gateway.rs b/src-tauri/src/magenta_gateway.rs new file mode 100644 index 0000000..92cd093 --- /dev/null +++ b/src-tauri/src/magenta_gateway.rs @@ -0,0 +1,1861 @@ +//! Native, authenticated Magenta render gateway for managed Linux/Windows. +//! +//! The public loopback HTTP service is deliberately separate from Stable Audio +//! 3. It owns one lazy, warm, disposable MRT2 render worker and translates the +//! user-facing `{prompt, seconds}` request into the reviewed binary protocol's +//! authoritative integer frame count and monotonic sequence. Every response is +//! bounded, sequence-bound, byte-counted, and SHA-256 checked before it becomes +//! a WAV. A cancellation, deadline, dropped HTTP request, or protocol mismatch +//! tears down and reaps the complete worker process tree; the next request starts +//! from a freshly revalidated managed generation. + +use std::collections::BTreeMap; +use std::io::{self, Read, Write}; +use std::net::{Shutdown, TcpListener, TcpStream}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +use axum::body::Bytes; +use axum::extract::{DefaultBodyLimit, Request, State}; +use axum::http::{header, HeaderValue, Method, StatusCode}; +use axum::middleware::Next; +use axum::response::{IntoResponse, Response}; +use axum::routing::{get, post}; +use axum::Router; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use tokio_util::sync::CancellationToken; + +use crate::child_process::SupervisedChild; + +const FRAME_STATUS: u8 = 2; +const FRAME_AUTH: u8 = 5; +const FRAME_RENDER_REQUEST: u8 = 6; +const FRAME_RENDER_BEGIN: u8 = 7; +const FRAME_RENDER_CHUNK: u8 = 8; +const FRAME_RENDER_END: u8 = 9; +const FRAME_RENDER_CANCEL: u8 = 10; +const FRAME_RENDER_ERROR: u8 = 11; + +const RENDER_SCHEMA_VERSION: u32 = 1; +const RENDER_SAMPLE_RATE: u64 = 48_000; +const RENDER_CHANNELS: u64 = 2; +const RENDER_BYTES_PER_FRAME: u64 = RENDER_CHANNELS * 4; +const MIN_RENDER_FRAMES: u64 = 24_000; +const MAX_RENDER_FRAMES: u64 = 8_640_000; +const MAX_RENDER_PCM_BYTES: usize = (MAX_RENDER_FRAMES * RENDER_BYTES_PER_FRAME) as usize; +const MAX_RENDER_REQUEST_BYTES: usize = 64 * 1024; +const MAX_RENDER_PROMPT_CHARS: usize = 32_000; +const MAX_RENDER_CONTROL_BYTES: usize = 1024; +const MAX_RENDER_METADATA_BYTES: usize = 8 * 1024; +const MAX_RENDER_CHUNK_BYTES: usize = 1024 * 1024; +const ACCEPT_TIMEOUT: Duration = Duration::from_secs(30); +const READY_TIMEOUT: Duration = Duration::from_secs(180); +const IO_POLL: Duration = Duration::from_millis(50); +const WRITE_TIMEOUT: Duration = Duration::from_secs(5); +const SAFE_ORIGINS: &[&str] = &[ + "tauri://localhost", + "http://tauri.localhost", + "https://tauri.localhost", +]; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum FailureKind { + Unavailable, + Protocol, + Deadline, + Cancelled, +} + +#[derive(Debug)] +struct RenderFailure { + kind: FailureKind, + detail: &'static str, +} + +impl RenderFailure { + fn unavailable() -> Self { + Self { + kind: FailureKind::Unavailable, + detail: "Magenta runtime is not installed or failed verification", + } + } + + fn protocol(detail: &'static str) -> Self { + Self { + kind: FailureKind::Protocol, + detail, + } + } + + fn deadline() -> Self { + Self { + kind: FailureKind::Deadline, + detail: "Magenta render timed out", + } + } + + fn cancelled() -> Self { + Self { + kind: FailureKind::Cancelled, + detail: "Magenta render was cancelled", + } + } +} + +impl From for RenderFailure { + fn from(_: io::Error) -> Self { + Self::protocol("Magenta render worker connection failed") + } +} + +#[derive(Clone)] +struct RequestCancellation { + request: Arc, + lifecycle: Arc, +} + +impl RequestCancellation { + fn cancelled(&self) -> bool { + self.request.load(Ordering::Acquire) || self.lifecycle.load(Ordering::Acquire) + } +} + +struct CancelOnDrop { + flag: Arc, + armed: bool, +} + +impl CancelOnDrop { + fn new(flag: Arc) -> Self { + Self { flag, armed: true } + } + + fn disarm(&mut self) { + self.armed = false; + } +} + +impl Drop for CancelOnDrop { + fn drop(&mut self) { + if self.armed { + self.flag.store(true, Ordering::Release); + } + } +} + +trait ProcessTree: Send { + fn shutdown(&mut self) -> io::Result<()>; +} + +struct ManagedProcess { + child: SupervisedChild, +} + +impl ProcessTree for ManagedProcess { + fn shutdown(&mut self) -> io::Result<()> { + let report = self.child.shutdown(Duration::from_millis(500))?; + crate::child_process::log_shutdown("MRT2 render worker", Ok(report)); + Ok(()) + } +} + +struct ManagedRenderWorker { + stream: TcpStream, + process: Box, + next_sequence: u64, +} + +impl ManagedRenderWorker { + fn render( + &mut self, + request: &WorkerRenderRequest, + cancellation: &RequestCancellation, + ) -> Result, RenderFailure> { + let payload = serde_json::to_vec(request) + .map_err(|_| RenderFailure::protocol("Magenta render request is invalid"))?; + if payload.len() > MAX_RENDER_REQUEST_BYTES { + return Err(RenderFailure::protocol( + "Magenta render request is too large", + )); + } + self.stream.set_write_timeout(Some(WRITE_TIMEOUT))?; + write_frame(&mut self.stream, FRAME_RENDER_REQUEST, &payload)?; + let duration = request.frames as f64 / RENDER_SAMPLE_RATE as f64; + let deadline = Instant::now() + Duration::from_secs_f64((duration * 2.0).max(90.0)); + match read_render_response(&mut self.stream, request, cancellation, deadline) { + Err(error) if error.kind == FailureKind::Cancelled => { + let cancel = WorkerRenderCancel { + schema_version: RENDER_SCHEMA_VERSION, + job_id: request.job_id.clone(), + sequence: request.sequence, + }; + if let Ok(payload) = serde_json::to_vec(&cancel) { + if payload.len() <= MAX_RENDER_CONTROL_BYTES { + let _ = write_frame(&mut self.stream, FRAME_RENDER_CANCEL, &payload); + } + } + Err(error) + } + result => result, + } + } +} + +trait WorkerFactory: Send + Sync { + fn spawn( + &self, + cancellation: &RequestCancellation, + ) -> Result; +} + +struct WorkerSpawnFailure { + failure: RenderFailure, + uncertain_process: Option>, +} + +impl WorkerSpawnFailure { + fn reaped(failure: RenderFailure) -> Self { + Self { + failure, + uncertain_process: None, + } + } +} + +impl From for WorkerSpawnFailure { + fn from(failure: RenderFailure) -> Self { + Self::reaped(failure) + } +} + +struct ManagedWorkerFactory; + +impl WorkerFactory for ManagedWorkerFactory { + fn spawn( + &self, + cancellation: &RequestCancellation, + ) -> Result { + spawn_managed_worker(cancellation) + } +} + +enum WorkerState { + Stopped, + Running(ManagedRenderWorker), + /// A failed teardown left process-tree ownership uncertain. No new worker + /// may launch, and promotion may not rename, until shutdown later succeeds. + Uncertain { + process: Box, + was_warm: bool, + }, +} + +struct GatewayCore { + worker: Mutex, + factory: Arc, + lifecycle: Mutex>, + quiescing: AtomicBool, +} + +impl GatewayCore { + fn new(factory: Arc) -> Self { + Self { + worker: Mutex::new(WorkerState::Stopped), + factory, + lifecycle: Mutex::new(Arc::new(AtomicBool::new(false))), + quiescing: AtomicBool::new(false), + } + } + + fn cancellation(&self, request: Arc) -> RequestCancellation { + RequestCancellation { + request, + lifecycle: self + .lifecycle + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .clone(), + } + } + + fn render( + &self, + prompt: String, + frames: u64, + request_cancel: Arc, + ) -> Result, RenderFailure> { + if self.quiescing.load(Ordering::Acquire) { + return Err(RenderFailure::unavailable()); + } + let cancellation = self.cancellation(request_cancel); + if cancellation.cancelled() { + return Err(RenderFailure::cancelled()); + } + let mut worker = self + .worker + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + if cancellation.cancelled() || self.quiescing.load(Ordering::Acquire) { + return Err(RenderFailure::cancelled()); + } + if matches!(&*worker, WorkerState::Stopped) { + match self.factory.spawn(&cancellation) { + Ok(spawned) => *worker = WorkerState::Running(spawned), + Err(spawn) => { + if let Some(process) = spawn.uncertain_process { + *worker = WorkerState::Uncertain { + process, + was_warm: false, + }; + } + return Err(spawn.failure); + } + } + } + let resident = match &mut *worker { + WorkerState::Running(resident) => resident, + WorkerState::Uncertain { .. } => { + return Err(RenderFailure::protocol( + "Magenta render worker could not be reaped", + )) + } + WorkerState::Stopped => unreachable!("worker spawn installed a running state"), + }; + let sequence = resident.next_sequence; + let request = WorkerRenderRequest { + schema_version: RENDER_SCHEMA_VERSION, + job_id: format!("render-{:032x}", rand::random::()), + sequence, + prompt, + frames, + }; + let result = resident.render(&request, &cancellation); + if result.is_ok() && sequence < u64::MAX { + resident.next_sequence = sequence + 1; + return result; + } + let WorkerState::Running(mut finished) = + std::mem::replace(&mut *worker, WorkerState::Stopped) + else { + unreachable!("render state stayed running while its lock was held") + }; + let _ = finished.stream.shutdown(Shutdown::Both); + if finished.process.shutdown().is_err() { + *worker = WorkerState::Uncertain { + process: finished.process, + was_warm: true, + }; + return Err(RenderFailure::protocol( + "Magenta render worker could not be reaped", + )); + } + result + } + + /// Cancel in-flight/queued renders, then kill and reap the warm worker. + /// Returns whether a worker was resident before the quiesce. + fn quiesce(&self) -> Result { + self.quiescing.store(true, Ordering::Release); + self.lifecycle + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .store(true, Ordering::Release); + let mut worker = self + .worker + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + let previous = std::mem::replace(&mut *worker, WorkerState::Stopped); + let (mut process, was_warm) = match previous { + WorkerState::Stopped => return Ok(false), + WorkerState::Running(resident) => { + let _ = resident.stream.shutdown(Shutdown::Both); + (resident.process, true) + } + WorkerState::Uncertain { process, was_warm } => (process, was_warm), + }; + match process.shutdown() { + Ok(()) => Ok(was_warm), + Err(_) => { + *worker = WorkerState::Uncertain { process, was_warm }; + Err("Magenta render worker could not be reaped".to_string()) + } + } + } + + /// Open a fresh request generation. If an update displaced a previously warm + /// renderer, eagerly restore it from the now-current verified generation. + fn resume(&self, restore_warm_worker: bool) -> Result<(), String> { + *self + .lifecycle + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) = Arc::new(AtomicBool::new(false)); + self.quiescing.store(false, Ordering::Release); + if !restore_warm_worker { + return Ok(()); + } + let cancellation = self.cancellation(Arc::new(AtomicBool::new(false))); + let mut worker = self + .worker + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + match &*worker { + WorkerState::Running(_) => return Ok(()), + WorkerState::Uncertain { .. } => { + return Err("Magenta render worker could not be reaped".to_string()) + } + WorkerState::Stopped => {} + } + match self.factory.spawn(&cancellation) { + Ok(spawned) => *worker = WorkerState::Running(spawned), + Err(spawn) => { + if let Some(process) = spawn.uncertain_process { + *worker = WorkerState::Uncertain { + process, + was_warm: false, + }; + } + return Err(spawn.failure.detail.to_string()); + } + } + Ok(()) + } +} + +#[derive(Clone)] +struct HttpState { + core: Arc, +} + +#[derive(Clone)] +struct AuthState { + capability: Arc, +} + +/// The always-available public HTTP gateway. Absence of an MRT2 runtime affects +/// only render requests; it never prevents the window, model manager, or model +/// status endpoint from starting. +pub struct MagentaGateway { + port: Option, + capability: String, + cancel: CancellationToken, + core: Arc, +} + +impl MagentaGateway { + pub fn start() -> Self { + let capability = crate::local_auth::generate_capability(); + let core = Arc::new(GatewayCore::new(Arc::new(ManagedWorkerFactory))); + match bind_loopback() { + Ok((listener, port)) => { + let cancel = serve(listener, port, &capability, core.clone()); + Self { + port: Some(port), + capability, + cancel, + core, + } + } + Err(error) => { + eprintln!("lsdj-app: Magenta gateway bind failed: {error}"); + Self { + port: None, + capability, + cancel: CancellationToken::new(), + core, + } + } + } + } + + pub fn port(&self) -> Option { + self.port + } + + pub fn capability(&self) -> Option { + self.port.map(|_| self.capability.clone()) + } + + pub fn quiesce(&self) -> Result { + self.core.quiesce() + } + + pub fn resume(&self, restore_warm_worker: bool) -> Result<(), String> { + self.core.resume(restore_warm_worker) + } + + pub fn shutdown(&self) { + self.cancel.cancel(); + if let Err(error) = self.core.quiesce() { + eprintln!("lsdj-app: Magenta gateway shutdown failed: {error}"); + } + } +} + +impl Drop for MagentaGateway { + fn drop(&mut self) { + self.shutdown(); + } +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct HttpRenderRequest { + prompt: String, + seconds: f64, +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct WorkerRenderRequest { + schema_version: u32, + job_id: String, + sequence: u64, + prompt: String, + frames: u64, +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct WorkerRenderCancel { + schema_version: u32, + job_id: String, + sequence: u64, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct RenderReady { + schema_version: u32, + event: String, + model: String, + runtime: String, + next_sequence: u64, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct RenderBegin { + schema_version: u32, + job_id: String, + sequence: u64, + sample_rate: u64, + channels: u64, + sample_format: String, + frames: u64, + pcm_bytes: u64, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct RenderEnd { + schema_version: u32, + job_id: String, + sequence: u64, + frames: u64, + pcm_bytes: u64, + sha256: String, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct RenderError { + schema_version: u32, + job_id: Option, + sequence: u64, + code: String, + message: String, +} + +async fn render_clip(State(state): State, body: Bytes) -> Response { + if body.len() > MAX_RENDER_REQUEST_BYTES { + return json_error(StatusCode::PAYLOAD_TOO_LARGE, "request body is too large"); + } + let parsed: HttpRenderRequest = match serde_json::from_slice(&body) { + Ok(parsed) => parsed, + Err(_) => return json_error(StatusCode::UNPROCESSABLE_ENTITY, "body must be JSON"), + }; + let prompt = parsed.prompt.trim().to_string(); + if prompt.is_empty() { + return json_error( + StatusCode::UNPROCESSABLE_ENTITY, + "'prompt' must be a non-empty string", + ); + } + if prompt.chars().count() > MAX_RENDER_PROMPT_CHARS { + return json_error( + StatusCode::UNPROCESSABLE_ENTITY, + "'prompt' must be at most 32000 characters", + ); + } + let frames = match frames_for_seconds(parsed.seconds) { + Some(frames) => frames, + None => { + return json_error( + StatusCode::UNPROCESSABLE_ENTITY, + "'seconds' must be 0.5-180", + ) + } + }; + + let request_cancel = Arc::new(AtomicBool::new(false)); + let mut drop_guard = CancelOnDrop::new(request_cancel.clone()); + let core = state.core.clone(); + let result = + tauri::async_runtime::spawn_blocking(move || core.render(prompt, frames, request_cancel)) + .await; + drop_guard.disarm(); + match result { + Ok(Ok(pcm)) => match float32_wav(&pcm) { + Ok(wav) => (StatusCode::OK, [(header::CONTENT_TYPE, "audio/wav")], wav).into_response(), + Err(_) => json_error(StatusCode::BAD_GATEWAY, "Magenta returned invalid audio"), + }, + Ok(Err(error)) => failure_response(error), + Err(_) => json_error(StatusCode::BAD_GATEWAY, "Magenta render task failed"), + } +} + +async fn model_info() -> Response { + let mut estimates = BTreeMap::new(); + estimates.insert("mrt2_small", 2.0); + estimates.insert("mrt2_base", 6.0); + axum::Json(serde_json::json!({ + "models": crate::models::magenta_models_for_gateway(), + "sample_rate": RENDER_SAMPLE_RATE, + "channels": RENDER_CHANNELS, + "chunk_seconds": 1.0, + "total_ram_gb": total_ram_gb(), + "model_ram_estimate_gb": estimates, + })) + .into_response() +} + +fn failure_response(error: RenderFailure) -> Response { + let status = match error.kind { + FailureKind::Unavailable => StatusCode::SERVICE_UNAVAILABLE, + FailureKind::Protocol => StatusCode::BAD_GATEWAY, + FailureKind::Deadline => StatusCode::GATEWAY_TIMEOUT, + FailureKind::Cancelled => StatusCode::from_u16(499).unwrap_or(StatusCode::BAD_GATEWAY), + }; + json_error(status, error.detail) +} + +fn json_error(status: StatusCode, detail: &str) -> Response { + (status, axum::Json(serde_json::json!({ "detail": detail }))).into_response() +} + +fn frames_for_seconds(seconds: f64) -> Option { + if !seconds.is_finite() || !(0.5..=180.0).contains(&seconds) { + return None; + } + let frames = (seconds * RENDER_SAMPLE_RATE as f64 + 0.5).floor() as u64; + (MIN_RENDER_FRAMES..=MAX_RENDER_FRAMES) + .contains(&frames) + .then_some(frames) +} + +fn float32_wav(pcm: &[u8]) -> Result, ()> { + if pcm.len() > MAX_RENDER_PCM_BYTES + || !pcm.len().is_multiple_of(RENDER_BYTES_PER_FRAME as usize) + { + return Err(()); + } + let data_len = u32::try_from(pcm.len()).map_err(|_| ())?; + let riff_len = 36u32.checked_add(data_len).ok_or(())?; + let mut wav = Vec::with_capacity(44 + pcm.len()); + wav.extend_from_slice(b"RIFF"); + wav.extend_from_slice(&riff_len.to_le_bytes()); + wav.extend_from_slice(b"WAVEfmt "); + wav.extend_from_slice(&16u32.to_le_bytes()); + wav.extend_from_slice(&3u16.to_le_bytes()); + wav.extend_from_slice(&(RENDER_CHANNELS as u16).to_le_bytes()); + wav.extend_from_slice(&(RENDER_SAMPLE_RATE as u32).to_le_bytes()); + wav.extend_from_slice(&((RENDER_SAMPLE_RATE * RENDER_BYTES_PER_FRAME) as u32).to_le_bytes()); + wav.extend_from_slice(&(RENDER_BYTES_PER_FRAME as u16).to_le_bytes()); + wav.extend_from_slice(&32u16.to_le_bytes()); + wav.extend_from_slice(b"data"); + wav.extend_from_slice(&data_len.to_le_bytes()); + wav.extend_from_slice(pcm); + Ok(wav) +} + +fn spawn_managed_worker( + cancellation: &RequestCancellation, +) -> Result { + let listener = TcpListener::bind("127.0.0.1:0") + .map_err(|_| WorkerSpawnFailure::reaped(RenderFailure::unavailable()))?; + listener + .set_nonblocking(true) + .map_err(|_| WorkerSpawnFailure::reaped(RenderFailure::unavailable()))?; + let port = listener + .local_addr() + .map_err(|_| WorkerSpawnFailure::reaped(RenderFailure::unavailable()))? + .port(); + let token = crate::local_auth::generate_capability(); + let mut command = + crate::sidecar::authenticated_render_worker_command(crate::DEFAULT_MODEL, port, &token) + .map_err(|_| WorkerSpawnFailure::reaped(RenderFailure::unavailable()))?; + let mut child = crate::child_process::spawn_grouped(&mut command) + .map_err(|_| WorkerSpawnFailure::reaped(RenderFailure::unavailable()))?; + + let result = + accept_worker(&listener, &mut child, &token, cancellation).and_then(|mut stream| { + stream.set_nodelay(true).ok(); + let next_sequence = read_worker_ready( + &mut stream, + crate::DEFAULT_MODEL, + "pytorch-cuda", + cancellation, + Instant::now() + READY_TIMEOUT, + )?; + Ok((stream, next_sequence)) + }); + match result { + Ok((stream, next_sequence)) => Ok(ManagedRenderWorker { + stream, + process: Box::new(ManagedProcess { child }), + next_sequence, + }), + Err(error) => { + let mut process: Box = Box::new(ManagedProcess { child }); + match process.shutdown() { + Ok(()) => Err(WorkerSpawnFailure::reaped(error)), + Err(_) => Err(WorkerSpawnFailure { + failure: RenderFailure::protocol( + "Magenta render worker startup failed and could not be reaped", + ), + uncertain_process: Some(process), + }), + } + } + } +} + +fn read_worker_ready( + stream: &mut TcpStream, + model: &str, + runtime: &str, + cancellation: &RequestCancellation, + deadline: Instant, +) -> Result { + let (frame_type, payload) = read_bounded_frame( + stream, + &[FRAME_STATUS, FRAME_RENDER_ERROR], + MAX_RENDER_METADATA_BYTES, + cancellation, + deadline, + )?; + if frame_type == FRAME_RENDER_ERROR { + validate_startup_error(&payload)?; + return Err(RenderFailure::unavailable()); + } + let ready: RenderReady = serde_json::from_slice(&payload) + .map_err(|_| RenderFailure::protocol("Magenta worker readiness is invalid"))?; + if ready.schema_version != RENDER_SCHEMA_VERSION + || ready.event != "render_ready" + || ready.model != model + || ready.runtime != runtime + || ready.next_sequence != 1 + { + return Err(RenderFailure::protocol( + "Magenta worker readiness is invalid", + )); + } + Ok(ready.next_sequence) +} + +fn accept_worker( + listener: &TcpListener, + child: &mut SupervisedChild, + token: &str, + cancellation: &RequestCancellation, +) -> Result { + let deadline = Instant::now() + ACCEPT_TIMEOUT; + loop { + if cancellation.cancelled() { + return Err(RenderFailure::cancelled()); + } + if Instant::now() >= deadline { + return Err(RenderFailure::deadline()); + } + if child + .try_wait() + .map_err(|_| RenderFailure::unavailable())? + .is_some() + { + return Err(RenderFailure::unavailable()); + } + match listener.accept() { + Ok((mut stream, _)) => { + // The launch token is single-use: the first connection attempt + // consumes it, even when authentication fails. + stream + .set_nonblocking(false) + .map_err(|_| RenderFailure::protocol("Magenta worker connection failed"))?; + stream + .set_read_timeout(Some(IO_POLL)) + .map_err(|_| RenderFailure::protocol("Magenta worker connection failed"))?; + let (frame_type, payload) = read_bounded_frame( + &mut stream, + &[FRAME_AUTH], + 256, + cancellation, + deadline.min(Instant::now() + Duration::from_secs(1)), + )?; + if frame_type != FRAME_AUTH + || !(32..=256).contains(&payload.len()) + || !crate::local_auth::constant_time_eq(&payload, token.as_bytes()) + { + return Err(RenderFailure::protocol( + "Magenta worker authentication failed", + )); + } + return Ok(stream); + } + Err(error) if error.kind() == io::ErrorKind::WouldBlock => { + std::thread::sleep(IO_POLL); + } + Err(_) => return Err(RenderFailure::unavailable()), + } + } +} + +fn validate_startup_error(payload: &[u8]) -> Result<(), RenderFailure> { + let error: RenderError = serde_json::from_slice(payload) + .map_err(|_| RenderFailure::protocol("Magenta worker error is invalid"))?; + if error.schema_version != RENDER_SCHEMA_VERSION + || error.job_id.is_some() + || error.sequence != 0 + || error.code.is_empty() + || error.code.len() > 64 + || error.message.len() > 512 + { + return Err(RenderFailure::protocol("Magenta worker error is invalid")); + } + Ok(()) +} + +fn validate_render_error( + payload: &[u8], + request: &WorkerRenderRequest, +) -> Result<(), RenderFailure> { + let error: RenderError = serde_json::from_slice(payload) + .map_err(|_| RenderFailure::protocol("Magenta worker error is invalid"))?; + if error.schema_version != RENDER_SCHEMA_VERSION + || error.job_id.as_deref() != Some(&request.job_id) + || error.sequence != request.sequence + || error.code.is_empty() + || error.code.len() > 64 + || error.message.len() > 512 + { + return Err(RenderFailure::protocol("Magenta worker error is invalid")); + } + Err(RenderFailure::protocol("Magenta render worker failed")) +} + +fn read_render_response( + stream: &mut TcpStream, + request: &WorkerRenderRequest, + cancellation: &RequestCancellation, + deadline: Instant, +) -> Result, RenderFailure> { + let (frame_type, payload) = read_bounded_frame( + stream, + &[FRAME_RENDER_BEGIN, FRAME_RENDER_ERROR], + MAX_RENDER_METADATA_BYTES, + cancellation, + deadline, + )?; + if frame_type == FRAME_RENDER_ERROR { + validate_render_error(&payload, request)?; + unreachable!("a valid worker error is returned as a render failure"); + } + let begin: RenderBegin = serde_json::from_slice(&payload) + .map_err(|_| RenderFailure::protocol("Magenta render begin is invalid"))?; + let expected_bytes = request + .frames + .checked_mul(RENDER_BYTES_PER_FRAME) + .ok_or_else(|| RenderFailure::protocol("Magenta render size overflow"))?; + if begin.schema_version != RENDER_SCHEMA_VERSION + || begin.job_id != request.job_id + || begin.sequence != request.sequence + || begin.sample_rate != RENDER_SAMPLE_RATE + || begin.channels != RENDER_CHANNELS + || begin.sample_format != "f32le" + || begin.frames != request.frames + || begin.pcm_bytes != expected_bytes + || begin.pcm_bytes as usize > MAX_RENDER_PCM_BYTES + { + return Err(RenderFailure::protocol("Magenta render begin is invalid")); + } + + let mut pcm = Vec::with_capacity(expected_bytes as usize); + let mut digest = Sha256::new(); + loop { + let (frame_type, payload) = read_bounded_frame( + stream, + &[FRAME_RENDER_CHUNK, FRAME_RENDER_END, FRAME_RENDER_ERROR], + MAX_RENDER_CHUNK_BYTES, + cancellation, + deadline, + )?; + match frame_type { + FRAME_RENDER_CHUNK => { + if payload.is_empty() + || !payload + .len() + .is_multiple_of(RENDER_BYTES_PER_FRAME as usize) + || pcm.len().saturating_add(payload.len()) > expected_bytes as usize + { + return Err(RenderFailure::protocol("Magenta PCM chunk is invalid")); + } + digest.update(&payload); + pcm.extend_from_slice(&payload); + } + FRAME_RENDER_ERROR => { + if payload.len() > MAX_RENDER_METADATA_BYTES { + return Err(RenderFailure::protocol("Magenta worker error is too large")); + } + validate_render_error(&payload, request)?; + unreachable!("a valid worker error is returned as a render failure"); + } + FRAME_RENDER_END => { + if payload.len() > MAX_RENDER_METADATA_BYTES { + return Err(RenderFailure::protocol("Magenta render end is too large")); + } + let end: RenderEnd = serde_json::from_slice(&payload) + .map_err(|_| RenderFailure::protocol("Magenta render end is invalid"))?; + let actual_hash = hex::encode(digest.finalize()); + if end.schema_version != RENDER_SCHEMA_VERSION + || end.job_id != request.job_id + || end.sequence != request.sequence + || end.frames != request.frames + || end.pcm_bytes != expected_bytes + || pcm.len() != expected_bytes as usize + || end.sha256.len() != 64 + || !end.sha256.bytes().all(|byte| byte.is_ascii_hexdigit()) + || end.sha256 != actual_hash + { + return Err(RenderFailure::protocol("Magenta render end is invalid")); + } + return Ok(pcm); + } + _ => unreachable!("frame type was checked"), + } + } +} + +fn write_frame(writer: &mut impl Write, frame_type: u8, payload: &[u8]) -> io::Result<()> { + let length = u32::try_from(payload.len()) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "frame is too large"))?; + writer.write_all(&[frame_type])?; + writer.write_all(&length.to_le_bytes())?; + writer.write_all(payload)?; + writer.flush() +} + +fn read_bounded_frame( + reader: &mut impl Read, + allowed_types: &[u8], + maximum: usize, + cancellation: &RequestCancellation, + deadline: Instant, +) -> Result<(u8, Vec), RenderFailure> { + let mut header = [0u8; 5]; + read_exact_cancellable(reader, &mut header, cancellation, deadline)?; + let frame_type = header[0]; + if !allowed_types.contains(&frame_type) { + return Err(RenderFailure::protocol("Magenta frame is out of order")); + } + let length = u32::from_le_bytes(header[1..5].try_into().expect("four bytes")) as usize; + // Metadata and PCM share this helper. A caller that accepts chunks passes + // the chunk cap, but control metadata remains capped before allocation or + // reading so an END/ERROR frame cannot consume a chunk-sized buffer. + let maximum = if frame_type == FRAME_RENDER_CHUNK { + maximum + } else { + maximum.min(MAX_RENDER_METADATA_BYTES) + }; + if length > maximum { + return Err(RenderFailure::protocol( + "Magenta frame exceeds its size cap", + )); + } + let mut payload = vec![0u8; length]; + read_exact_cancellable(reader, &mut payload, cancellation, deadline)?; + Ok((frame_type, payload)) +} + +fn read_exact_cancellable( + reader: &mut impl Read, + mut output: &mut [u8], + cancellation: &RequestCancellation, + deadline: Instant, +) -> Result<(), RenderFailure> { + while !output.is_empty() { + if cancellation.cancelled() { + return Err(RenderFailure::cancelled()); + } + if Instant::now() >= deadline { + return Err(RenderFailure::deadline()); + } + match reader.read(output) { + Ok(0) => { + return Err(RenderFailure::protocol( + "Magenta render response was truncated", + )) + } + Ok(read) => output = &mut output[read..], + Err(error) + if matches!( + error.kind(), + io::ErrorKind::WouldBlock + | io::ErrorKind::TimedOut + | io::ErrorKind::Interrupted + ) => {} + Err(_) => { + return Err(RenderFailure::protocol( + "Magenta render worker connection failed", + )) + } + } + } + Ok(()) +} + +fn bind_loopback() -> io::Result<(TcpListener, u16)> { + let listener = TcpListener::bind("127.0.0.1:0")?; + let port = listener.local_addr()?.port(); + listener.set_nonblocking(true)?; + Ok((listener, port)) +} + +fn serve( + listener: TcpListener, + port: u16, + capability: &str, + core: Arc, +) -> CancellationToken { + let router = gateway_router(capability, core); + let cancel = CancellationToken::new(); + let serve_cancel = cancel.clone(); + tauri::async_runtime::spawn(async move { + let listener = match tokio::net::TcpListener::from_std(listener) { + Ok(listener) => listener, + Err(error) => { + eprintln!("lsdj-app: Magenta gateway listener failed: {error}"); + return; + } + }; + println!("lsdj-app: Magenta gateway on http://127.0.0.1:{port}"); + if let Err(error) = axum::serve(listener, router) + .with_graceful_shutdown(async move { serve_cancel.cancelled().await }) + .await + { + eprintln!("lsdj-app: Magenta gateway stopped: {error}"); + } + }); + cancel +} + +fn gateway_router(capability: &str, core: Arc) -> Router { + let auth = AuthState { + capability: Arc::from(capability), + }; + Router::new() + .route("/api/render", post(render_clip).options(preflight)) + .route("/api/models", get(model_info).options(preflight)) + .layer(DefaultBodyLimit::max(MAX_RENDER_REQUEST_BYTES)) + .layer(axum::middleware::from_fn_with_state(auth, authenticate)) + .with_state(HttpState { core }) +} + +async fn preflight() -> StatusCode { + StatusCode::NO_CONTENT +} + +async fn authenticate(State(auth): State, request: Request, next: Next) -> Response { + let origin = request + .headers() + .get(header::ORIGIN) + .and_then(|value| value.to_str().ok()) + .map(str::to_owned); + if origin + .as_deref() + .is_some_and(|origin| !SAFE_ORIGINS.contains(&origin)) + { + return json_error(StatusCode::FORBIDDEN, "origin is not allowed"); + } + if request.method() == Method::OPTIONS { + let requested_method = request + .headers() + .get(header::ACCESS_CONTROL_REQUEST_METHOD) + .and_then(|value| value.to_str().ok()); + let requested_headers = request + .headers() + .get(header::ACCESS_CONTROL_REQUEST_HEADERS) + .and_then(|value| value.to_str().ok()) + .unwrap_or_default() + .split(',') + .map(|value| value.trim().to_ascii_lowercase()) + .filter(|value| !value.is_empty()) + .collect::>(); + if origin.is_none() + || !matches!(requested_method, Some("GET" | "POST")) + || requested_headers + .iter() + .any(|value| !matches!(value.as_str(), "content-type" | "x-lsdj-capability")) + { + return json_error(StatusCode::FORBIDDEN, "preflight rejected"); + } + let mut response = StatusCode::NO_CONTENT.into_response(); + add_cors(&mut response, origin.as_deref().expect("origin checked")); + response.headers_mut().insert( + header::ACCESS_CONTROL_ALLOW_METHODS, + HeaderValue::from_static("GET, POST"), + ); + response.headers_mut().insert( + header::ACCESS_CONTROL_ALLOW_HEADERS, + HeaderValue::from_static("content-type, x-lsdj-capability"), + ); + response.headers_mut().insert( + header::ACCESS_CONTROL_MAX_AGE, + HeaderValue::from_static("600"), + ); + return response; + } + let supplied = request + .headers() + .get("x-lsdj-capability") + .map(|value| value.as_bytes()) + .unwrap_or_default(); + if !crate::local_auth::constant_time_eq(supplied, auth.capability.as_bytes()) { + return json_error(StatusCode::UNAUTHORIZED, "authentication required"); + } + let mut response = next.run(request).await; + if let Some(origin) = origin.as_deref() { + add_cors(&mut response, origin); + } + response +} + +fn add_cors(response: &mut Response, origin: &str) { + if let Ok(origin) = HeaderValue::from_str(origin) { + response + .headers_mut() + .insert(header::ACCESS_CONTROL_ALLOW_ORIGIN, origin); + response + .headers_mut() + .insert(header::VARY, HeaderValue::from_static("Origin")); + } +} + +#[cfg(target_os = "linux")] +fn total_ram_gb() -> Option { + std::fs::read_to_string("/proc/meminfo") + .ok()? + .lines() + .find_map(|line| line.strip_prefix("MemTotal:"))? + .split_whitespace() + .next()? + .parse::() + .ok() + .map(|kilobytes| kilobytes / 1024.0 / 1024.0) +} + +#[cfg(target_os = "windows")] +fn total_ram_gb() -> Option { + use windows_sys::Win32::System::SystemInformation::{GlobalMemoryStatusEx, MEMORYSTATUSEX}; + + let mut status: MEMORYSTATUSEX = unsafe { std::mem::zeroed() }; + status.dwLength = std::mem::size_of::() as u32; + // SAFETY: `status` is writable and advertises its exact structure size. + (unsafe { GlobalMemoryStatusEx(&mut status) } != 0) + .then_some(status.ullTotalPhys as f64 / 1024.0 / 1024.0 / 1024.0) +} + +#[cfg(not(any(target_os = "linux", target_os = "windows")))] +fn total_ram_gb() -> Option { + None +} + +#[cfg(test)] +mod tests { + use std::collections::VecDeque; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::thread; + + use super::*; + + #[derive(Clone, Copy)] + enum Scenario { + Valid, + WrongSequence, + WrongFrames, + WrongTotals, + WrongHash, + OutOfOrder, + OversizeEnd, + MisalignedChunk, + Stall, + LongStall, + } + + struct FakeProcess { + shutdowns: Arc, + shutdown_failures: Arc, + } + + impl ProcessTree for FakeProcess { + fn shutdown(&mut self) -> io::Result<()> { + self.shutdowns.fetch_add(1, Ordering::AcqRel); + if self + .shutdown_failures + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |remaining| { + remaining.checked_sub(1) + }) + .is_ok() + { + return Err(io::Error::other("fake process is not reaped")); + } + Ok(()) + } + } + + struct FakeFactory { + scenarios: Mutex>, + spawns: Arc, + shutdowns: Arc, + shutdown_failures: Arc, + sequences: Arc>>, + } + + impl FakeFactory { + fn new(scenarios: impl IntoIterator) -> Arc { + Arc::new(Self { + scenarios: Mutex::new(scenarios.into_iter().collect()), + spawns: Arc::new(AtomicUsize::new(0)), + shutdowns: Arc::new(AtomicUsize::new(0)), + shutdown_failures: Arc::new(AtomicUsize::new(0)), + sequences: Arc::new(Mutex::new(Vec::new())), + }) + } + + fn fail_shutdowns(&self, count: usize) { + self.shutdown_failures.store(count, Ordering::Release); + } + } + + impl WorkerFactory for FakeFactory { + fn spawn( + &self, + _cancellation: &RequestCancellation, + ) -> Result { + let scenario = self + .scenarios + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .pop_front() + .ok_or_else(RenderFailure::unavailable)?; + self.spawns.fetch_add(1, Ordering::AcqRel); + let listener = + TcpListener::bind("127.0.0.1:0").map_err(|_| RenderFailure::unavailable())?; + let address = listener + .local_addr() + .map_err(|_| RenderFailure::unavailable())?; + let client = TcpStream::connect(address).map_err(RenderFailure::from)?; + client + .set_read_timeout(Some(IO_POLL)) + .map_err(RenderFailure::from)?; + let (server, _) = listener.accept().map_err(RenderFailure::from)?; + let sequences = self.sequences.clone(); + thread::spawn(move || serve_scenario(server, scenario, sequences)); + Ok(ManagedRenderWorker { + stream: client, + process: Box::new(FakeProcess { + shutdowns: self.shutdowns.clone(), + shutdown_failures: self.shutdown_failures.clone(), + }), + next_sequence: 1, + }) + } + } + + fn read_test_frame(stream: &mut TcpStream) -> (u8, Vec) { + let mut header = [0u8; 5]; + stream.read_exact(&mut header).expect("request header"); + let length = u32::from_le_bytes(header[1..].try_into().unwrap()) as usize; + let mut payload = vec![0u8; length]; + stream.read_exact(&mut payload).expect("request payload"); + (header[0], payload) + } + + fn serve_scenario(mut stream: TcpStream, scenario: Scenario, sequences: Arc>>) { + let (frame_type, payload) = read_test_frame(&mut stream); + assert_eq!(frame_type, FRAME_RENDER_REQUEST); + let request: serde_json::Value = serde_json::from_slice(&payload).unwrap(); + let job_id = request["jobId"].as_str().unwrap(); + let sequence = request["sequence"].as_u64().unwrap(); + sequences + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .push(sequence); + let frames = request["frames"].as_u64().unwrap(); + if matches!(scenario, Scenario::Stall | Scenario::LongStall) { + let delay = if matches!(scenario, Scenario::LongStall) { + Duration::from_secs(2) + } else { + Duration::from_millis(250) + }; + thread::sleep(delay); + return; + } + if matches!(scenario, Scenario::OutOfOrder) { + write_frame(&mut stream, FRAME_RENDER_END, b"{}").ok(); + return; + } + + let pcm = vec![0x3fu8; frames as usize * RENDER_BYTES_PER_FRAME as usize]; + let begin_sequence = if matches!(scenario, Scenario::WrongSequence) { + sequence + 1 + } else { + sequence + }; + let begin_frames = if matches!(scenario, Scenario::WrongFrames) { + frames + 1 + } else { + frames + }; + let begin = serde_json::json!({ + "schemaVersion": RENDER_SCHEMA_VERSION, + "jobId": job_id, + "sequence": begin_sequence, + "sampleRate": RENDER_SAMPLE_RATE, + "channels": RENDER_CHANNELS, + "sampleFormat": "f32le", + "frames": begin_frames, + "pcmBytes": pcm.len(), + }); + if write_frame( + &mut stream, + FRAME_RENDER_BEGIN, + &serde_json::to_vec(&begin).unwrap(), + ) + .is_err() + { + return; + } + if matches!(scenario, Scenario::OversizeEnd) { + let _ = stream.write_all(&[FRAME_RENDER_END]); + let _ = stream.write_all(&((MAX_RENDER_METADATA_BYTES + 1) as u32).to_le_bytes()); + return; + } + let chunk = if matches!(scenario, Scenario::MisalignedChunk) { + &pcm[..3] + } else { + &pcm + }; + if write_frame(&mut stream, FRAME_RENDER_CHUNK, chunk).is_err() { + return; + } + let reported_bytes = if matches!(scenario, Scenario::WrongTotals) { + pcm.len() as u64 + RENDER_BYTES_PER_FRAME + } else { + pcm.len() as u64 + }; + let hash = if matches!(scenario, Scenario::WrongHash) { + "0".repeat(64) + } else { + hex::encode(Sha256::digest(&pcm)) + }; + let end = serde_json::json!({ + "schemaVersion": RENDER_SCHEMA_VERSION, + "jobId": job_id, + "sequence": sequence, + "frames": frames, + "pcmBytes": reported_bytes, + "sha256": hash, + }); + write_frame( + &mut stream, + FRAME_RENDER_END, + &serde_json::to_vec(&end).unwrap(), + ) + .ok(); + } + + fn render_with( + core: &GatewayCore, + cancellation: Arc, + ) -> Result, RenderFailure> { + core.render("test prompt".to_string(), 2, cancellation) + } + + #[test] + fn seconds_are_converted_to_authoritative_integer_frames() { + assert_eq!(frames_for_seconds(0.5), Some(24_000)); + assert_eq!( + frames_for_seconds((24_000.5) / RENDER_SAMPLE_RATE as f64), + Some(24_001) + ); + assert_eq!(frames_for_seconds(180.0), Some(MAX_RENDER_FRAMES)); + assert_eq!(frames_for_seconds(0.499), None); + assert_eq!(frames_for_seconds(f64::NAN), None); + } + + #[test] + fn valid_fake_worker_response_is_accepted_exactly() { + let factory = FakeFactory::new([Scenario::Valid]); + let core = GatewayCore::new(factory.clone()); + let pcm = render_with(&core, Arc::new(AtomicBool::new(false))).unwrap(); + assert_eq!(pcm, vec![0x3f; 2 * RENDER_BYTES_PER_FRAME as usize]); + assert_eq!(factory.spawns.load(Ordering::Acquire), 1); + assert_eq!(core.quiesce(), Ok(true)); + assert_eq!(factory.shutdowns.load(Ordering::Acquire), 1); + } + + #[test] + fn every_protocol_violation_discards_and_reaps_the_worker() { + for scenario in [ + Scenario::WrongSequence, + Scenario::WrongFrames, + Scenario::WrongTotals, + Scenario::WrongHash, + Scenario::OutOfOrder, + Scenario::OversizeEnd, + Scenario::MisalignedChunk, + ] { + let factory = FakeFactory::new([scenario]); + let core = GatewayCore::new(factory.clone()); + let error = render_with(&core, Arc::new(AtomicBool::new(false))).unwrap_err(); + assert_eq!(error.kind, FailureKind::Protocol); + assert_eq!(factory.shutdowns.load(Ordering::Acquire), 1); + assert!(matches!( + &*core.worker.lock().unwrap(), + WorkerState::Stopped + )); + } + } + + #[test] + fn next_request_recovers_with_a_fresh_worker_after_failure() { + let factory = FakeFactory::new([Scenario::WrongHash, Scenario::Valid]); + let core = GatewayCore::new(factory.clone()); + assert!(render_with(&core, Arc::new(AtomicBool::new(false))).is_err()); + assert!(render_with(&core, Arc::new(AtomicBool::new(false))).is_ok()); + assert_eq!(factory.spawns.load(Ordering::Acquire), 2); + assert_eq!(*factory.sequences.lock().unwrap(), [1, 1]); + assert_eq!(factory.shutdowns.load(Ordering::Acquire), 1); + assert_eq!(core.quiesce(), Ok(true)); + assert_eq!(factory.shutdowns.load(Ordering::Acquire), 2); + } + + #[test] + fn uncertain_reap_state_is_retained_and_blocks_promotion() { + let factory = FakeFactory::new([Scenario::Valid]); + let core = GatewayCore::new(factory.clone()); + assert!(render_with(&core, Arc::new(AtomicBool::new(false))).is_ok()); + factory.fail_shutdowns(1); + + assert!(core.quiesce().is_err()); + assert!(matches!( + &*core.worker.lock().unwrap(), + WorkerState::Uncertain { .. } + )); + assert_eq!(factory.shutdowns.load(Ordering::Acquire), 1); + + assert_eq!(core.quiesce(), Ok(true)); + assert!(matches!( + &*core.worker.lock().unwrap(), + WorkerState::Stopped + )); + assert_eq!(factory.shutdowns.load(Ordering::Acquire), 2); + } + + struct FailedStartupFactory { + shutdowns: Arc, + } + + impl WorkerFactory for FailedStartupFactory { + fn spawn( + &self, + _cancellation: &RequestCancellation, + ) -> Result { + // The production factory reaches this shape only after failed + // accept/readiness cleanup. Count that first failed reap here; the + // retained process succeeds when quiesce retries it. + self.shutdowns.store(1, Ordering::Release); + Err(WorkerSpawnFailure { + failure: RenderFailure::protocol( + "Magenta render worker startup failed and could not be reaped", + ), + uncertain_process: Some(Box::new(FakeProcess { + shutdowns: self.shutdowns.clone(), + shutdown_failures: Arc::new(AtomicUsize::new(0)), + })), + }) + } + } + + #[test] + fn failed_startup_cleanup_retains_process_until_positive_reap() { + let shutdowns = Arc::new(AtomicUsize::new(0)); + let core = GatewayCore::new(Arc::new(FailedStartupFactory { + shutdowns: shutdowns.clone(), + })); + + assert!(render_with(&core, Arc::new(AtomicBool::new(false))).is_err()); + assert!(matches!( + &*core.worker.lock().unwrap(), + WorkerState::Uncertain { .. } + )); + assert_eq!(shutdowns.load(Ordering::Acquire), 1); + + assert_eq!(core.quiesce(), Ok(false)); + assert!(matches!( + &*core.worker.lock().unwrap(), + WorkerState::Stopped + )); + assert_eq!(shutdowns.load(Ordering::Acquire), 2); + } + + async fn host_test_router( + core: Arc, + capability: &str, + ) -> (std::net::SocketAddr, CancellationToken) { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let cancel = CancellationToken::new(); + let serve_cancel = cancel.clone(); + let router = gateway_router(capability, core); + tokio::spawn(async move { + axum::serve(listener, router) + .with_graceful_shutdown(async move { serve_cancel.cancelled().await }) + .await + .unwrap(); + }); + (address, cancel) + } + + #[tokio::test(flavor = "multi_thread")] + async fn router_enforces_auth_cors_body_and_request_bounds() { + let capability = "c".repeat(64); + let core = Arc::new(GatewayCore::new(FakeFactory::new([]))); + let (address, cancel) = host_test_router(core, &capability).await; + let client = reqwest::Client::new(); + let render = format!("http://{address}/api/render"); + + assert_eq!( + client.get(&render).send().await.unwrap().status(), + StatusCode::UNAUTHORIZED + ); + assert_eq!( + client + .get(&render) + .header("x-lsdj-capability", "wrong") + .send() + .await + .unwrap() + .status(), + StatusCode::UNAUTHORIZED + ); + assert_eq!( + client + .get(&render) + .header("x-lsdj-capability", &capability) + .header(header::ORIGIN, "https://hostile.example") + .send() + .await + .unwrap() + .status(), + StatusCode::FORBIDDEN + ); + + let allowed = client + .get(&render) + .header("x-lsdj-capability", &capability) + .header(header::ORIGIN, SAFE_ORIGINS[0]) + .send() + .await + .unwrap(); + assert_eq!(allowed.status(), StatusCode::METHOD_NOT_ALLOWED); + assert_eq!( + allowed.headers().get(header::ACCESS_CONTROL_ALLOW_ORIGIN), + Some(&HeaderValue::from_static("tauri://localhost")) + ); + + let preflight = client + .request(Method::OPTIONS, &render) + .header(header::ORIGIN, SAFE_ORIGINS[0]) + .header(header::ACCESS_CONTROL_REQUEST_METHOD, "POST") + .header( + header::ACCESS_CONTROL_REQUEST_HEADERS, + "content-type, x-lsdj-capability", + ) + .send() + .await + .unwrap(); + assert_eq!(preflight.status(), StatusCode::NO_CONTENT); + assert_eq!( + preflight.headers().get(header::ACCESS_CONTROL_ALLOW_ORIGIN), + Some(&HeaderValue::from_static("tauri://localhost")) + ); + + let oversized = client + .post(&render) + .header("x-lsdj-capability", &capability) + .header(header::CONTENT_TYPE, "application/json") + .body(vec![b'x'; MAX_RENDER_REQUEST_BYTES + 1]) + .send() + .await + .unwrap(); + assert_eq!(oversized.status(), StatusCode::PAYLOAD_TOO_LARGE); + + for invalid in [ + serde_json::json!({"prompt": "ok", "seconds": 2.0, "extra": true}), + serde_json::json!({"prompt": " ", "seconds": 2.0}), + serde_json::json!({"prompt": "x".repeat(MAX_RENDER_PROMPT_CHARS + 1), "seconds": 2.0}), + serde_json::json!({"prompt": "ok", "seconds": 0.49}), + serde_json::json!({"prompt": "ok", "seconds": 180.01}), + ] { + let response = client + .post(&render) + .header("x-lsdj-capability", &capability) + .json(&invalid) + .send() + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::UNPROCESSABLE_ENTITY); + } + cancel.cancel(); + } + + #[tokio::test(flavor = "multi_thread")] + async fn real_http_disconnect_cancels_kills_and_reaps_worker() { + let capability = "d".repeat(64); + let factory = FakeFactory::new([Scenario::LongStall]); + let core = Arc::new(GatewayCore::new(factory.clone())); + let (address, cancel) = host_test_router(core.clone(), &capability).await; + let body = br#"{"prompt":"disconnect me","seconds":2.0}"#; + let mut stream = TcpStream::connect(address).unwrap(); + write!( + stream, + "POST /api/render HTTP/1.1\r\nHost: {address}\r\nContent-Type: application/json\r\nx-lsdj-capability: {capability}\r\nContent-Length: {}\r\n\r\n", + body.len() + ) + .unwrap(); + stream.write_all(body).unwrap(); + stream.flush().unwrap(); + + let spawn_deadline = Instant::now() + Duration::from_secs(1); + while factory.spawns.load(Ordering::Acquire) == 0 && Instant::now() < spawn_deadline { + tokio::time::sleep(Duration::from_millis(10)).await; + } + assert_eq!(factory.spawns.load(Ordering::Acquire), 1); + drop(stream); + + let reap_deadline = Instant::now() + Duration::from_secs(1); + while factory.shutdowns.load(Ordering::Acquire) == 0 && Instant::now() < reap_deadline { + tokio::time::sleep(Duration::from_millis(10)).await; + } + assert_eq!(factory.shutdowns.load(Ordering::Acquire), 1); + assert!(matches!( + &*core.worker.lock().unwrap(), + WorkerState::Stopped + )); + cancel.cancel(); + } + + #[test] + fn cancellation_interrupts_a_stalled_worker_and_reaps_it() { + let factory = FakeFactory::new([Scenario::Stall]); + let core = Arc::new(GatewayCore::new(factory.clone())); + let cancellation = Arc::new(AtomicBool::new(false)); + let flag = cancellation.clone(); + thread::spawn(move || { + thread::sleep(Duration::from_millis(10)); + flag.store(true, Ordering::Release); + }); + let error = render_with(&core, cancellation).unwrap_err(); + assert_eq!(error.kind, FailureKind::Cancelled); + assert_eq!(factory.shutdowns.load(Ordering::Acquire), 1); + assert!(matches!( + &*core.worker.lock().unwrap(), + WorkerState::Stopped + )); + } + + #[test] + fn deadline_and_drop_cancellation_are_observed_while_reading() { + let request = Arc::new(AtomicBool::new(false)); + { + let _guard = CancelOnDrop::new(request.clone()); + } + assert!(request.load(Ordering::Acquire)); + + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let mut client = TcpStream::connect(address).unwrap(); + client.set_read_timeout(Some(IO_POLL)).unwrap(); + let (_server, _) = listener.accept().unwrap(); + let cancellation = RequestCancellation { + request: Arc::new(AtomicBool::new(false)), + lifecycle: Arc::new(AtomicBool::new(false)), + }; + let error = read_bounded_frame( + &mut client, + &[FRAME_RENDER_BEGIN], + MAX_RENDER_METADATA_BYTES, + &cancellation, + Instant::now() + Duration::from_millis(20), + ) + .unwrap_err(); + assert_eq!(error.kind, FailureKind::Deadline); + } + + fn protocol_test_python() -> Option { + let mut candidates = Vec::new(); + if let Some(configured) = std::env::var_os("LSDJ_TEST_PYTHON") { + candidates.push(configured.into()); + } + candidates.push("/opt/homebrew/bin/python3".into()); + candidates.push("python3".into()); + candidates.push("python".into()); + candidates.into_iter().find(|candidate| { + let Ok(output) = std::process::Command::new(candidate) + .arg("--version") + .output() + else { + return false; + }; + let version = format!( + "{}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + let Some(version) = version.split_whitespace().nth(1) else { + return false; + }; + let mut parts = version + .split('.') + .filter_map(|part| part.parse::().ok()); + matches!( + (parts.next(), parts.next()), + (Some(major), Some(minor)) if major > 3 || (major == 3 && minor >= 11) + ) + }) + } + + #[test] + fn rust_gateway_round_trips_two_requests_with_the_real_python_protocol() { + let Some(python) = protocol_test_python() else { + eprintln!("skipping Python protocol compatibility test: Python 3.11+ unavailable"); + return; + }; + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + listener.set_nonblocking(true).unwrap(); + let port = listener.local_addr().unwrap().port(); + let token = crate::local_auth::generate_capability(); + let sidecar = + std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../backend/lsdj/sidecar.py"); + let harness = r#" +import importlib.util +import socket +import sys +import types + +package = types.ModuleType("lsdj") +package.__path__ = [] +sys.modules["lsdj"] = package +mrt2 = types.ModuleType("lsdj.mrt2") +mrt2.AUTO_RUNTIME = "auto" +mrt2.PYTORCH_CUDA_RUNTIME = "pytorch-cuda" +mrt2.RUNTIME_CHOICES = ("auto", "mlx", "pytorch-cuda") +mrt2.create_engine = lambda **kwargs: None +mrt2.public_startup_error = lambda error: str(error) +mrt2.runtime_manifest = lambda: {} +sys.modules["lsdj.mrt2"] = mrt2 +worker = types.ModuleType("lsdj.worker") +worker.run_deck_worker = lambda *args, **kwargs: None +sys.modules["lsdj.worker"] = worker +spec = importlib.util.spec_from_file_location("lsdj.sidecar", sys.argv[3]) +sidecar = importlib.util.module_from_spec(spec) +sys.modules["lsdj.sidecar"] = sidecar +spec.loader.exec_module(sidecar) + +class Engine: + def warm_up(self): + pass + def render_clip(self, prompt, seconds): + frames = int(seconds * sidecar.RENDER_SAMPLE_RATE + 0.5) + return b"\0" * (frames * sidecar.RENDER_BYTES_PER_FRAME) + +sock = socket.create_connection(("127.0.0.1", int(sys.argv[1]))) +sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) +sidecar.write_frame(sock, sidecar.FRAME_AUTH, sys.argv[2].encode()) +sidecar.run_render_worker( + sock, + "mrt2_small", + runtime="pytorch-cuda", + engine_factory=lambda model: Engine(), +) +"#; + let mut command = std::process::Command::new(python); + command + .arg("-c") + .arg(harness) + .arg(port.to_string()) + .arg(&token) + .arg(&sidecar); + let mut child = crate::child_process::spawn_grouped(&mut command).unwrap(); + let cancellation = RequestCancellation { + request: Arc::new(AtomicBool::new(false)), + lifecycle: Arc::new(AtomicBool::new(false)), + }; + let mut stream = accept_worker(&listener, &mut child, &token, &cancellation).unwrap(); + let next_sequence = read_worker_ready( + &mut stream, + "mrt2_small", + "pytorch-cuda", + &cancellation, + Instant::now() + Duration::from_secs(5), + ) + .unwrap(); + assert_eq!(next_sequence, 1); + + let core = GatewayCore::new(FakeFactory::new(std::iter::empty())); + *core.worker.lock().unwrap() = WorkerState::Running(ManagedRenderWorker { + stream, + process: Box::new(ManagedProcess { child }), + next_sequence, + }); + let first = core + .render( + "first compatibility render".to_string(), + MIN_RENDER_FRAMES, + Arc::new(AtomicBool::new(false)), + ) + .unwrap(); + let second = core + .render( + "second compatibility render".to_string(), + MIN_RENDER_FRAMES, + Arc::new(AtomicBool::new(false)), + ) + .unwrap(); + assert_eq!( + first.len(), + MIN_RENDER_FRAMES as usize * RENDER_BYTES_PER_FRAME as usize + ); + assert_eq!(second.len(), first.len()); + assert!(matches!( + &*core.worker.lock().unwrap(), + WorkerState::Running(ManagedRenderWorker { + next_sequence: 3, + .. + }) + )); + assert_eq!(core.quiesce(), Ok(true)); + } +} diff --git a/src-tauri/src/managed_runtime.rs b/src-tauri/src/managed_runtime.rs new file mode 100644 index 0000000..2970017 --- /dev/null +++ b/src-tauri/src/managed_runtime.rs @@ -0,0 +1,1182 @@ +//! Verified launch contract for app-managed Python services. +//! +//! Linux and Windows releases never execute a path merely because it exists. +//! Every service lives in an atomically promoted generation and is described by +//! a structured manifest. Resolution revalidates the target, generation +//! identity, provenance, complete file inventory, sizes, and SHA-256 digests +//! before returning an absolute program plus fixed argv. No shell, `PATH` +//! lookup, system Python, Git, or `uv` participates in a production launch. + +use std::collections::{BTreeMap, BTreeSet}; +use std::ffi::OsString; +use std::fs::{self, File}; +use std::io::{Read, Write}; +use std::path::{Component, Path, PathBuf}; +use std::process::Command; + +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +pub(crate) const MANIFEST_NAME: &str = ".lsdj-launch-manifest.json"; +const GENERATION_NAME: &str = ".lsdj-generation"; +const SCHEMA_VERSION: u32 = 1; +const MAX_MANIFEST_BYTES: u64 = 32 * 1024 * 1024; + +const STATIC_ENV_KEYS: &[&str] = &[ + "DO_NOT_TRACK", + "HF_HUB_DISABLE_TELEMETRY", + "HF_HUB_OFFLINE", + "NO_COLOR", + "PYTHONDONTWRITEBYTECODE", + "PYTHONNOUSERSITE", + "PYTHONUTF8", +]; + +const EPHEMERAL_ENV_KEYS: &[&str] = &[ + "LSDJ_API_CAPABILITY", + "LSDJ_ASSETS_HOME", + "LSDJ_CACHE_HOME", + "LSDJ_CONFIG_HOME", + "LSDJ_DATA_HOME", + "LSDJ_STAGING_HOME", + "LSDJ_WORKER_LAUNCH_TOKEN", + "MAGENTA_HOME", + "SA3_HOME", + "SA3_LORAS_HOME", + "SA3_MLX_HOME", + "SYSTEMROOT", + "TEMP", + "TMP", + "WINDIR", +]; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Service { + Mrt2, + Sa3, + #[allow(dead_code)] + Sa3Cuda, +} + +impl Service { + pub(crate) const fn wire_name(self) -> &'static str { + match self { + Self::Mrt2 => "mrt2", + Self::Sa3 => "sa3-tflite", + Self::Sa3Cuda => "sa3-pytorch-cuda", + } + } +} + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(crate) struct CommandSpec { + pub(crate) program: String, + #[serde(default)] + pub(crate) argv: Vec, + pub(crate) cwd: String, + #[serde(default)] + pub(crate) environment: BTreeMap, + #[serde(default)] + pub(crate) ephemeral_environment: Vec, +} + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct FileSeal { + path: String, + size: u64, + sha256: String, + executable: bool, +} + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct LaunchManifest { + schema_version: u32, + target: String, + generation: String, + provenance: BTreeMap, + services: BTreeMap, + files: Vec, +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct GenerationIdentity<'a> { + schema_version: u32, + target: &'a str, + provenance: &'a BTreeMap, + services: &'a BTreeMap, +} + +/// A fully verified command description. Dynamic arguments are appended as +/// individual argv items and ephemeral secrets are accepted only through the +/// manifest's explicit allowlist. +#[derive(Clone, Debug)] +pub(crate) struct VerifiedCommand { + root: PathBuf, + program: PathBuf, + cwd: PathBuf, + argv: Vec, + environment: BTreeMap, + ephemeral_environment: BTreeSet, + host_environment: Vec<(OsString, OsString)>, + generation: String, + target: String, +} + +impl VerifiedCommand { + #[allow(dead_code)] + pub(crate) fn generation(&self) -> &str { + &self.generation + } + + #[allow(dead_code)] + pub(crate) fn program(&self) -> &Path { + &self.program + } + + pub(crate) fn into_command( + self, + extra_args: impl IntoIterator, + ephemeral: impl IntoIterator, + ) -> Result { + // Revalidate immediately before spawn. A model-manager mutation after + // an earlier status probe never inherits authority to execute. + let service = service_for_program(&self.root, &self.program, &self.generation)?; + let refreshed = resolve_at(&self.root, &service, &self.target)?; + if refreshed.generation != self.generation || refreshed.program != self.program { + return Err("managed runtime changed while preparing launch".into()); + } + + let mut command = Command::new(&self.program); + command.env_clear().current_dir(&self.cwd).args(&self.argv); + for (name, value) in &self.environment { + command.env(name, value); + } + let mut seen = BTreeSet::new(); + for (name, value) in self.host_environment.into_iter().chain(ephemeral) { + let Some(name) = name.to_str() else { + return Err("managed runtime environment name is not UTF-8".into()); + }; + if !self.ephemeral_environment.contains(name) + || !EPHEMERAL_ENV_KEYS.contains(&name) + || self.environment.contains_key(name) + || !seen.insert(name.to_string()) + { + return Err(format!( + "managed runtime rejected undeclared environment key {name:?}" + )); + } + command.env(name, value); + } + command.args(extra_args); + Ok(command) + } +} + +fn service_for_program(root: &Path, program: &Path, generation: &str) -> Result { + let manifest = read_manifest(root)?; + if manifest.generation != generation { + return Err("managed runtime generation is stale".into()); + } + manifest + .services + .iter() + .find_map(|(name, spec)| { + checked_relative(&spec.program) + .ok() + .map(|relative| (name, root.join(relative))) + .filter(|(_, candidate)| candidate == program) + .map(|(name, _)| name.clone()) + }) + .ok_or_else(|| "managed runtime service disappeared".into()) +} + +pub(crate) fn host_target() -> String { + match (std::env::consts::ARCH, std::env::consts::OS) { + ("x86_64", "linux") => "x86_64-unknown-linux-gnu".into(), + ("x86_64", "windows") => "x86_64-pc-windows-msvc".into(), + ("aarch64", "macos") => "aarch64-apple-darwin".into(), + (arch, os) => format!("{arch}-{os}"), + } +} + +/// Stable service layout consumed by both shipping branches. +pub(crate) fn service_root(assets: &Path, service: Service) -> PathBuf { + assets + .join("backend") + .join("services") + .join(service.wire_name()) + .join("current") +} + +pub(crate) fn resolve(assets: &Path, service: Service) -> Result { + let mut verified = resolve_at( + &service_root(assets, service), + service.wire_name(), + &host_target(), + )?; + verified.host_environment = crate::platform_paths::get().managed_child_env()?; + Ok(verified) +} + +fn resolve_at(root: &Path, service: &str, target: &str) -> Result { + let root_metadata = fs::symlink_metadata(root) + .map_err(|error| format!("managed runtime is unavailable: {error}"))?; + if root_metadata.file_type().is_symlink() || !root_metadata.is_dir() { + return Err("managed runtime root is not a trusted directory".into()); + } + let manifest = read_manifest(root)?; + validate_manifest(root, &manifest, target)?; + let spec = manifest + .services + .get(service) + .ok_or_else(|| format!("managed runtime does not provide {service}"))?; + validate_command_spec(spec)?; + let program = root.join(checked_relative(&spec.program)?); + let cwd = root.join(checked_relative(&spec.cwd)?); + let canonical_root = fs::canonicalize(root) + .map_err(|error| format!("cannot canonicalize managed runtime root: {error}"))?; + for (kind, path) in [("program", &program), ("working directory", &cwd)] { + let canonical = fs::canonicalize(path) + .map_err(|error| format!("cannot canonicalize managed runtime {kind}: {error}"))?; + if !canonical.starts_with(&canonical_root) { + return Err(format!("managed runtime {kind} escapes its generation")); + } + } + let program_metadata = fs::symlink_metadata(&program) + .map_err(|error| format!("cannot inspect managed runtime program: {error}"))?; + if program_metadata.file_type().is_symlink() || !program_metadata.is_file() { + return Err("managed runtime program is not a regular file".into()); + } + let cwd_metadata = fs::symlink_metadata(&cwd) + .map_err(|error| format!("cannot inspect managed runtime working directory: {error}"))?; + if cwd_metadata.file_type().is_symlink() || !cwd_metadata.is_dir() { + return Err("managed runtime working directory is not a directory".into()); + } + + Ok(VerifiedCommand { + root: root.to_path_buf(), + program, + cwd, + argv: spec.argv.iter().map(OsString::from).collect(), + environment: spec.environment.clone(), + ephemeral_environment: spec.ephemeral_environment.iter().cloned().collect(), + host_environment: Vec::new(), + generation: manifest.generation, + target: manifest.target, + }) +} + +fn read_manifest(root: &Path) -> Result { + let path = root.join(MANIFEST_NAME); + let metadata = fs::symlink_metadata(&path) + .map_err(|error| format!("managed runtime manifest is unavailable: {error}"))?; + if metadata.file_type().is_symlink() || !metadata.is_file() { + return Err("managed runtime manifest is not a regular file".into()); + } + if metadata.len() == 0 || metadata.len() > MAX_MANIFEST_BYTES { + return Err("managed runtime manifest has an invalid size".into()); + } + let bytes = fs::read(path).map_err(|error| format!("cannot read runtime manifest: {error}"))?; + serde_json::from_slice(&bytes) + .map_err(|error| format!("managed runtime manifest is invalid: {error}")) +} + +fn validate_manifest(root: &Path, manifest: &LaunchManifest, target: &str) -> Result<(), String> { + if manifest.schema_version != SCHEMA_VERSION { + return Err("managed runtime manifest schema is unsupported".into()); + } + if manifest.target != target { + return Err(format!( + "managed runtime target mismatch: expected {target}, found {}", + manifest.target + )); + } + let expected_generation = + generation_id(&manifest.target, &manifest.provenance, &manifest.services)?; + if manifest.generation != expected_generation { + return Err("managed runtime manifest generation identity is invalid".into()); + } + let generation_path = root.join(GENERATION_NAME); + let generation = read_regular_bounded(&generation_path, 128)?; + if generation != format!("{}\n", manifest.generation).as_bytes() { + return Err("managed runtime generation stamp is stale".into()); + } + if manifest.services.is_empty() || manifest.files.is_empty() { + return Err("managed runtime manifest is incomplete".into()); + } + for spec in manifest.services.values() { + validate_command_spec(spec)?; + } + + let mut declared = BTreeSet::new(); + for seal in &manifest.files { + let relative = checked_relative(&seal.path)?; + if seal.path == MANIFEST_NAME || !declared.insert(seal.path.clone()) { + return Err("managed runtime file inventory is invalid".into()); + } + let path = root.join(relative); + let metadata = fs::symlink_metadata(&path) + .map_err(|error| format!("managed runtime file is missing: {error}"))?; + if metadata.file_type().is_symlink() || !metadata.is_file() { + return Err(format!( + "managed runtime file is not regular: {}", + seal.path + )); + } + if metadata.len() != seal.size { + return Err(format!("managed runtime file size changed: {}", seal.path)); + } + if hash_file(&path)? != seal.sha256 { + return Err(format!( + "managed runtime file digest changed: {}", + seal.path + )); + } + if executable(&metadata) != seal.executable { + return Err(format!("managed runtime file mode changed: {}", seal.path)); + } + } + + let actual = inventory_paths(root)?; + if actual != declared { + return Err("managed runtime contains missing or unexpected files".into()); + } + Ok(()) +} + +fn validate_command_spec(spec: &CommandSpec) -> Result<(), String> { + checked_relative(&spec.program)?; + checked_relative(&spec.cwd)?; + if spec.argv.iter().any(|value| value.contains('\0')) { + return Err("managed runtime argv contains NUL".into()); + } + let mut ephemeral = BTreeSet::new(); + for name in &spec.ephemeral_environment { + if !EPHEMERAL_ENV_KEYS.contains(&name.as_str()) || !ephemeral.insert(name) { + return Err(format!( + "managed runtime ephemeral environment is invalid: {name}" + )); + } + } + for (name, value) in &spec.environment { + if !STATIC_ENV_KEYS.contains(&name.as_str()) + || value.contains('\0') + || spec.ephemeral_environment.contains(name) + { + return Err(format!( + "managed runtime static environment is invalid: {name}" + )); + } + } + Ok(()) +} + +fn checked_relative(value: &str) -> Result { + if value.is_empty() || value.contains('\0') || value.contains('\\') { + return Err("managed runtime path is invalid".into()); + } + let path = PathBuf::from(value); + if path.is_absolute() + || path + .components() + .any(|part| !matches!(part, Component::Normal(_))) + { + return Err("managed runtime path must be a contained relative path".into()); + } + Ok(path) +} + +fn inventory_paths(root: &Path) -> Result, String> { + let mut paths = BTreeSet::new(); + walk_files(root, root, &mut |relative, _path, metadata| { + if metadata.file_type().is_symlink() { + return Err(format!( + "managed runtime contains a symbolic link: {}", + relative.display() + )); + } + if metadata.is_file() && relative != Path::new(MANIFEST_NAME) { + paths.insert(path_wire(relative)?); + } + Ok(()) + })?; + Ok(paths) +} + +fn inventory(root: &Path) -> Result, String> { + let mut files = Vec::new(); + walk_files(root, root, &mut |relative, path, metadata| { + if metadata.file_type().is_symlink() { + return Err(format!( + "managed runtime candidate contains a symbolic link: {}", + relative.display() + )); + } + if metadata.is_file() && relative != Path::new(MANIFEST_NAME) { + files.push(FileSeal { + path: path_wire(relative)?, + size: metadata.len(), + sha256: hash_file(path)?, + executable: executable(metadata), + }); + } + Ok(()) + })?; + files.sort_by(|a, b| a.path.cmp(&b.path)); + Ok(files) +} + +fn walk_files( + root: &Path, + directory: &Path, + visit: &mut impl FnMut(&Path, &Path, &fs::Metadata) -> Result<(), String>, +) -> Result<(), String> { + let mut entries = fs::read_dir(directory) + .map_err(|error| format!("cannot read managed runtime directory: {error}"))? + .collect::, _>>() + .map_err(|error| format!("cannot enumerate managed runtime directory: {error}"))?; + entries.sort_by_key(|entry| entry.file_name()); + for entry in entries { + let path = entry.path(); + let metadata = fs::symlink_metadata(&path) + .map_err(|error| format!("cannot inspect managed runtime entry: {error}"))?; + let relative = path + .strip_prefix(root) + .map_err(|_| "managed runtime entry escaped its root")?; + visit(relative, &path, &metadata)?; + if metadata.is_dir() && !metadata.file_type().is_symlink() { + walk_files(root, &path, visit)?; + } else if !metadata.is_file() && !metadata.file_type().is_symlink() { + return Err("managed runtime contains an unsupported filesystem entry".into()); + } + } + Ok(()) +} + +fn path_wire(path: &Path) -> Result { + let mut pieces = Vec::new(); + for component in path.components() { + match component { + Component::Normal(value) => pieces.push( + value + .to_str() + .ok_or("managed runtime path is not valid UTF-8")?, + ), + _ => return Err("managed runtime path is not relative".into()), + } + } + Ok(pieces.join("/")) +} + +fn hash_file(path: &Path) -> Result { + let mut file = File::open(path) + .map_err(|error| format!("cannot open managed runtime file for hashing: {error}"))?; + let mut hash = Sha256::new(); + let mut buffer = [0u8; 128 * 1024]; + loop { + let count = file + .read(&mut buffer) + .map_err(|error| format!("cannot hash managed runtime file: {error}"))?; + if count == 0 { + break; + } + hash.update(&buffer[..count]); + } + Ok(hex::encode(hash.finalize())) +} + +fn generation_id( + target: &str, + provenance: &BTreeMap, + services: &BTreeMap, +) -> Result { + let bytes = serde_json::to_vec(&GenerationIdentity { + schema_version: SCHEMA_VERSION, + target, + provenance, + services, + }) + .map_err(|error| format!("cannot serialize runtime generation identity: {error}"))?; + Ok(hex::encode(Sha256::digest(bytes))) +} + +fn read_regular_bounded(path: &Path, max: u64) -> Result, String> { + let metadata = fs::symlink_metadata(path) + .map_err(|error| format!("cannot inspect managed runtime stamp: {error}"))?; + if metadata.file_type().is_symlink() || !metadata.is_file() || metadata.len() > max { + return Err("managed runtime stamp is not a bounded regular file".into()); + } + fs::read(path).map_err(|error| format!("cannot read managed runtime stamp: {error}")) +} + +#[cfg(unix)] +fn executable(metadata: &fs::Metadata) -> bool { + use std::os::unix::fs::PermissionsExt; + metadata.permissions().mode() & 0o111 != 0 +} + +#[cfg(not(unix))] +fn executable(_metadata: &fs::Metadata) -> bool { + false +} + +/// Seal a fully built candidate. Callers must have already downloaded every +/// external artifact through the native pinned downloader and completed all +/// offline setup/warm-up checks. The promotion validator calls [`resolve_at`] +/// before and after the rename. +pub(crate) fn seal_candidate( + root: &Path, + target: &str, + provenance: BTreeMap, + services: BTreeMap, +) -> Result { + if root.join(MANIFEST_NAME).exists() { + fs::remove_file(root.join(MANIFEST_NAME)) + .map_err(|error| format!("cannot replace runtime manifest: {error}"))?; + } + for spec in services.values() { + validate_command_spec(spec)?; + } + let generation = generation_id(target, &provenance, &services)?; + write_synced( + &root.join(GENERATION_NAME), + format!("{generation}\n").as_bytes(), + )?; + let manifest = LaunchManifest { + schema_version: SCHEMA_VERSION, + target: target.into(), + generation: generation.clone(), + provenance, + services, + files: inventory(root)?, + }; + let bytes = serde_json::to_vec_pretty(&manifest) + .map_err(|error| format!("cannot serialize runtime manifest: {error}"))?; + write_synced(&root.join(MANIFEST_NAME), &bytes)?; + for service in manifest.services.keys() { + resolve_at(root, service, target)?; + } + Ok(generation) +} + +pub(crate) fn validate_candidate(root: &Path, service: Service) -> Result<(), String> { + resolve_at(root, service.wire_name(), &host_target()).map(|_| ()) +} + +fn write_synced(path: &Path, bytes: &[u8]) -> Result<(), String> { + let mut file = File::create(path) + .map_err(|error| format!("cannot create managed runtime stamp: {error}"))?; + file.write_all(bytes) + .and_then(|_| file.flush()) + .and_then(|_| file.sync_all()) + .map_err(|error| format!("cannot sync managed runtime stamp: {error}")) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::{AtomicU64, Ordering}; + + static NEXT: AtomicU64 = AtomicU64::new(0); + + fn root(label: &str) -> PathBuf { + std::env::temp_dir().join(format!( + "lsdj-managed-{label}-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )) + } + + fn command(program: &str) -> CommandSpec { + CommandSpec { + program: program.into(), + argv: vec!["--model-free".into()], + cwd: "runtime".into(), + environment: BTreeMap::from([ + ("HF_HUB_OFFLINE".into(), "1".into()), + ("PYTHONNOUSERSITE".into(), "1".into()), + ]), + ephemeral_environment: vec!["LSDJ_ASSETS_HOME".into(), "LSDJ_API_CAPABILITY".into()], + } + } + + fn install(root: &Path, services: &[&str]) { + fs::create_dir_all(root.join("runtime/bin")).unwrap(); + fs::write(root.join("runtime/bin/python"), b"verified interpreter").unwrap(); + fs::write(root.join("runtime/backend.py"), b"verified adapter").unwrap(); + let services = services + .iter() + .map(|name| ((*name).into(), command("runtime/bin/python"))) + .collect(); + seal_candidate( + root, + "x86_64-pc-windows-msvc", + BTreeMap::from([ + ("requirementsSha256".into(), "a".repeat(64)), + ("sourceRevision".into(), "b".repeat(40)), + ]), + services, + ) + .unwrap(); + } + + #[cfg(unix)] + fn install_spawnable(root: &Path) { + use std::os::unix::fs::PermissionsExt; + + let program = root.join("runtime/bin/backend"); + fs::create_dir_all(program.parent().unwrap()).unwrap(); + fs::write( + &program, + b"#!/bin/sh\nprintf '%s|%s|%s|%s|%s' \"$1\" \"$2\" \"${LSDJ_API_CAPABILITY-unset}\" \"${LSDJ_WORKER_LAUNCH_TOKEN-unset}\" \"${HOME-unset}\"\n", + ) + .unwrap(); + let mut permissions = fs::metadata(&program).unwrap().permissions(); + permissions.set_mode(0o700); + fs::set_permissions(&program, permissions).unwrap(); + fs::write(root.join("runtime/backend.py"), b"verified adapter").unwrap(); + let spec = CommandSpec { + program: "runtime/bin/backend".into(), + argv: vec!["fixed".into()], + cwd: "runtime".into(), + environment: BTreeMap::from([("HF_HUB_OFFLINE".into(), "1".into())]), + ephemeral_environment: vec![ + "LSDJ_API_CAPABILITY".into(), + "LSDJ_WORKER_LAUNCH_TOKEN".into(), + "SYSTEMROOT".into(), + "WINDIR".into(), + "TEMP".into(), + "TMP".into(), + ], + }; + seal_candidate( + root, + &host_target(), + BTreeMap::from([("sourceRevision".into(), "b".repeat(40))]), + BTreeMap::from([("mrt2".into(), spec)]), + ) + .unwrap(); + } + + #[cfg(unix)] + fn promote_spawnable(label: &str) -> (PathBuf, PathBuf) { + let root = root(label); + let candidate = root.join("candidate"); + let home = root.join("home"); + let backup = root.join("backup"); + install_spawnable(&candidate); + crate::runtime_installer::promotion::promote(&candidate, &home, &backup, |path| { + resolve_at(path, "mrt2", &host_target()).map(|_| ()) + }) + .unwrap(); + (root, home) + } + + #[cfg(windows)] + const WINDOWS_HELPER_ROLE: &str = "LSDJ_API_CAPABILITY"; + #[cfg(windows)] + const WINDOWS_HELPER_PID_FILE: &str = "LSDJ_STAGING_HOME"; + + #[cfg(windows)] + fn windows_helper_command(role: &str, pid_file: &Path) -> Command { + let mut command = Command::new(std::env::current_exe().expect("current test executable")); + command + .args([ + "--ignored", + "--exact", + "managed_runtime::tests::windows_managed_runtime_process_helper", + "--nocapture", + ]) + .env(WINDOWS_HELPER_ROLE, role) + .env(WINDOWS_HELPER_PID_FILE, pid_file); + command + } + + #[cfg(windows)] + fn install_windows_spawnable(root: &Path, revision: &str) { + let program = root.join("runtime/bin/managed helper.exe"); + fs::create_dir_all(program.parent().unwrap()).unwrap(); + fs::copy(std::env::current_exe().unwrap(), &program).unwrap(); + fs::write( + root.join("runtime").join(format!("{revision}.marker")), + revision, + ) + .unwrap(); + let spec = CommandSpec { + program: "runtime/bin/managed helper.exe".into(), + argv: vec![ + "--ignored".into(), + "--exact".into(), + "managed_runtime::tests::windows_managed_runtime_process_helper".into(), + "--nocapture".into(), + ], + cwd: "runtime".into(), + environment: BTreeMap::new(), + ephemeral_environment: [ + WINDOWS_HELPER_ROLE, + WINDOWS_HELPER_PID_FILE, + "SYSTEMROOT", + "WINDIR", + "TEMP", + "TMP", + ] + .into_iter() + .map(str::to_string) + .collect(), + }; + seal_candidate( + root, + &host_target(), + BTreeMap::from([("sourceRevision".into(), revision.into())]), + BTreeMap::from([("mrt2".into(), spec)]), + ) + .unwrap(); + } + + #[cfg(windows)] + fn wait_for_windows_pids(path: &Path) -> (u32, u32) { + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + loop { + if let Ok(contents) = fs::read_to_string(path) { + let pids = contents + .split_whitespace() + .filter_map(|value| value.parse::().ok()) + .collect::>(); + if pids.len() == 2 { + return (pids[0], pids[1]); + } + } + assert!( + std::time::Instant::now() < deadline, + "managed runtime helper did not report its process tree" + ); + std::thread::sleep(std::time::Duration::from_millis(20)); + } + } + + #[cfg(windows)] + fn windows_process_is_alive(pid: u32) -> bool { + use windows_sys::Win32::Foundation::{CloseHandle, WAIT_TIMEOUT}; + use windows_sys::Win32::System::Threading::{ + OpenProcess, WaitForSingleObject, PROCESS_QUERY_LIMITED_INFORMATION, + }; + const SYNCHRONIZE_ACCESS: u32 = 0x0010_0000; + // SAFETY: this opens a read-only liveness handle for a test-owned pid. + let process = unsafe { + OpenProcess( + SYNCHRONIZE_ACCESS | PROCESS_QUERY_LIMITED_INFORMATION, + 0, + pid, + ) + }; + if process.is_null() { + return false; + } + // SAFETY: `process` is a live handle and the zero timeout cannot block. + let result = unsafe { WaitForSingleObject(process, 0) }; + // SAFETY: close exactly the handle opened above. + unsafe { CloseHandle(process) }; + result == WAIT_TIMEOUT + } + + #[cfg(windows)] + fn wait_until_windows_process_is_gone(pid: u32) -> bool { + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + while std::time::Instant::now() < deadline { + if !windows_process_is_alive(pid) { + return true; + } + std::thread::sleep(std::time::Duration::from_millis(20)); + } + false + } + + /// Process-tree stand-in copied into a sealed generation. The service role + /// launches a descendant from the same locked executable so promotion is + /// qualified against the real Windows Job Object and filesystem semantics. + #[cfg(windows)] + #[test] + #[ignore] + #[allow(clippy::zombie_processes)] + fn windows_managed_runtime_process_helper() { + let role = std::env::var(WINDOWS_HELPER_ROLE).expect("helper role"); + let pid_file = + PathBuf::from(std::env::var_os(WINDOWS_HELPER_PID_FILE).expect("helper pid file")); + match role.as_str() { + "service" => { + let grandchild = windows_helper_command("grandchild", &pid_file) + .spawn() + .expect("spawn managed runtime grandchild"); + fs::write( + &pid_file, + format!("{} {}", std::process::id(), grandchild.id()), + ) + .expect("write managed runtime pid file"); + loop { + std::thread::sleep(std::time::Duration::from_secs(60)); + } + } + "grandchild" => loop { + std::thread::sleep(std::time::Duration::from_secs(60)); + }, + other => panic!("unknown managed runtime helper role {other}"), + } + } + + #[cfg(windows)] + #[test] + fn windows_reaps_active_managed_tree_before_promoting_and_removing_old_generation() { + let root = root("windows locked promotion 资产"); + let first_candidate = root.join("first candidate"); + let next_candidate = root.join("next candidate"); + let home = root.join("active generation"); + let backup = root.join("old generation"); + let pid_file = root.join("managed process ids"); + + install_windows_spawnable(&first_candidate, "old-revision"); + crate::runtime_installer::promotion::promote(&first_candidate, &home, &backup, |path| { + resolve_at(path, "mrt2", &host_target()).map(|_| ()) + }) + .unwrap(); + let old_generation = resolve_at(&home, "mrt2", &host_target()) + .unwrap() + .generation() + .to_string(); + install_windows_spawnable(&next_candidate, "new-revision"); + + let mut ephemeral = vec![ + ( + OsString::from(WINDOWS_HELPER_ROLE), + OsString::from("service"), + ), + ( + OsString::from(WINDOWS_HELPER_PID_FILE), + pid_file.as_os_str().to_owned(), + ), + ]; + ephemeral.extend( + ["SYSTEMROOT", "WINDIR", "TEMP", "TMP"] + .into_iter() + .filter_map(|name| { + std::env::var_os(name).map(|value| (OsString::from(name), value)) + }), + ); + let mut command = resolve_at(&home, "mrt2", &host_target()) + .unwrap() + .into_command([], ephemeral) + .unwrap(); + let mut process = crate::child_process::spawn_grouped(&mut command).unwrap(); + let (service_pid, grandchild_pid) = wait_for_windows_pids(&pid_file); + assert!(windows_process_is_alive(service_pid)); + assert!(windows_process_is_alive(grandchild_pid)); + + let report = process + .shutdown(std::time::Duration::from_millis(100)) + .unwrap(); + assert!( + report.forced, + "live managed tree should require Job teardown" + ); + assert!(report.status.is_some(), "service leader was not reaped"); + assert!( + wait_until_windows_process_is_gone(service_pid), + "managed service survived quiesce" + ); + assert!( + wait_until_windows_process_is_gone(grandchild_pid), + "managed descendant survived quiesce" + ); + + crate::runtime_installer::promotion::promote(&next_candidate, &home, &backup, |path| { + resolve_at(path, "mrt2", &host_target()).map(|_| ()) + }) + .unwrap(); + let promoted = resolve_at(&home, "mrt2", &host_target()).unwrap(); + assert_ne!(promoted.generation(), old_generation); + assert!(home.join("runtime/new-revision.marker").is_file()); + assert!(!home.join("runtime/old-revision.marker").exists()); + assert!(!next_candidate.exists(), "candidate should be promoted"); + assert!(!backup.exists(), "old generation should be removed"); + fs::remove_dir_all(root).unwrap(); + } + + #[test] + fn clean_host_fails_closed_and_install_produces_structured_commands() { + let root = root("clean host with spaces 资产"); + assert!(resolve_at(&root, "mrt2", "x86_64-pc-windows-msvc").is_err()); + install(&root, &["mrt2", "sa3-tflite"]); + for service in ["mrt2", "sa3-tflite"] { + let resolved = resolve_at(&root, service, "x86_64-pc-windows-msvc").unwrap(); + assert!(resolved.program().is_absolute()); + let command = resolved + .into_command( + [OsString::from("--port"), OsString::from("1234")], + [( + OsString::from("LSDJ_ASSETS_HOME"), + root.as_os_str().to_owned(), + )], + ) + .unwrap(); + assert_eq!(command.get_program(), root.join("runtime/bin/python")); + assert_eq!( + command + .get_args() + .map(|value| value.to_string_lossy().into_owned()) + .collect::>(), + ["--model-free", "--port", "1234"] + ); + } + let _ = fs::remove_dir_all(root); + } + + #[test] + fn tamper_stale_target_unknown_schema_and_unexpected_files_fail_closed() { + let root = root("tamper"); + install(&root, &["mrt2"]); + fs::write(root.join("runtime/backend.py"), b"tampered adapter").unwrap(); + assert!(resolve_at(&root, "mrt2", "x86_64-pc-windows-msvc") + .unwrap_err() + .contains("digest changed")); + + fs::remove_dir_all(&root).unwrap(); + install(&root, &["mrt2"]); + fs::write(root.join(GENERATION_NAME), b"stale\n").unwrap(); + assert!(resolve_at(&root, "mrt2", "x86_64-pc-windows-msvc") + .unwrap_err() + .contains("generation stamp")); + + fs::remove_dir_all(&root).unwrap(); + install(&root, &["mrt2"]); + assert!(resolve_at(&root, "mrt2", "x86_64-unknown-linux-gnu") + .unwrap_err() + .contains("target mismatch")); + + let manifest_path = root.join(MANIFEST_NAME); + let mut manifest: serde_json::Value = + serde_json::from_slice(&fs::read(&manifest_path).unwrap()).unwrap(); + manifest["schemaVersion"] = 999.into(); + fs::write(&manifest_path, serde_json::to_vec(&manifest).unwrap()).unwrap(); + assert!(resolve_at(&root, "mrt2", "x86_64-pc-windows-msvc") + .unwrap_err() + .contains("schema")); + + fs::remove_dir_all(&root).unwrap(); + install(&root, &["mrt2"]); + fs::write(root.join("runtime/injected.py"), b"untrusted").unwrap(); + assert!(resolve_at(&root, "mrt2", "x86_64-pc-windows-msvc") + .unwrap_err() + .contains("unexpected")); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn traversal_symlink_and_undeclared_secret_environment_are_rejected() { + let root = root("escape"); + fs::create_dir_all(root.join("runtime/bin")).unwrap(); + fs::write(root.join("runtime/bin/python"), b"python").unwrap(); + let mut services = BTreeMap::new(); + services.insert("mrt2".into(), command("../system-python")); + assert!( + seal_candidate(&root, "x86_64-pc-windows-msvc", BTreeMap::new(), services) + .unwrap_err() + .contains("contained relative") + ); + + fs::remove_dir_all(&root).unwrap(); + install(&root, &["mrt2"]); + for name in [ + "HF_TOKEN", + "HUGGING_FACE_HUB_TOKEN", + "PATH", + "HOME", + "PYTHONPATH", + "LSDJ_GENERATION_CMD", + "LSDJ_SIDECAR_CMD", + "LSDJ_ALLOW_UNVERIFIED_MRT2_CUDA", + "LSDJ_ALLOW_UNVERIFIED_SA3_CUDA", + ] { + let resolved = resolve_at(&root, "mrt2", "x86_64-pc-windows-msvc").unwrap(); + assert!(resolved + .into_command([], [(OsString::from(name), OsString::from("secret"))]) + .unwrap_err() + .contains("undeclared")); + } + + #[cfg(unix)] + { + use std::os::unix::fs::symlink; + fs::remove_dir_all(&root).unwrap(); + install(&root, &["mrt2"]); + fs::remove_file(root.join("runtime/backend.py")).unwrap(); + symlink("/etc/hosts", root.join("runtime/backend.py")).unwrap(); + assert!(resolve_at(&root, "mrt2", "x86_64-pc-windows-msvc").is_err()); + } + let _ = fs::remove_dir_all(root); + } + + #[test] + fn promotion_recovery_preserves_last_known_good_generation() { + let root = root("promotion"); + let home = root.join("home"); + let backup = root.join("backup"); + let candidate = root.join("candidate"); + fs::create_dir_all(&root).unwrap(); + install(&home, &["mrt2", "sa3-tflite"]); + let old_generation = resolve_at(&home, "mrt2", "x86_64-pc-windows-msvc") + .unwrap() + .generation() + .to_string(); + install(&candidate, &["mrt2", "sa3-tflite"]); + fs::write(candidate.join("runtime/backend.py"), b"candidate crashed").unwrap(); + let validate = |path: &Path| { + resolve_at(path, "mrt2", "x86_64-pc-windows-msvc") + .and_then(|_| resolve_at(path, "sa3-tflite", "x86_64-pc-windows-msvc")) + .map(|_| ()) + }; + assert!( + crate::runtime_installer::promotion::promote(&candidate, &home, &backup, validate) + .is_err() + ); + assert_eq!( + resolve_at(&home, "mrt2", "x86_64-pc-windows-msvc") + .unwrap() + .generation(), + old_generation + ); + let _ = fs::remove_dir_all(root); + } + + #[cfg(unix)] + #[test] + fn promoted_runtime_revalidates_at_the_real_spawn_boundary() { + let (root, home) = promote_spawnable("spawn unicode 资产"); + let output = resolve_at(&home, "mrt2", &host_target()) + .unwrap() + .into_command( + [OsString::from("dynamic")], + [ + ( + OsString::from("LSDJ_API_CAPABILITY"), + OsString::from("capability-secret"), + ), + ( + OsString::from("LSDJ_WORKER_LAUNCH_TOKEN"), + OsString::from("worker-secret"), + ), + ], + ) + .unwrap() + .output() + .unwrap(); + assert!(output.status.success()); + assert_eq!( + String::from_utf8(output.stdout).unwrap(), + "fixed|dynamic|capability-secret|worker-secret|unset" + ); + let _ = fs::remove_dir_all(root); + + let assert_race_rejected = |label: &str, mutate: &dyn Fn(&Path)| { + let (root, home) = promote_spawnable(label); + let verified = resolve_at(&home, "mrt2", &host_target()).unwrap(); + mutate(&home); + assert!(verified + .into_command([], std::iter::empty::<(OsString, OsString)>()) + .is_err()); + let _ = fs::remove_dir_all(root); + }; + + assert_race_rejected("spawn missing", &|home| { + fs::remove_file(home.join("runtime/bin/backend")).unwrap(); + }); + assert_race_rejected("spawn tampered", &|home| { + fs::write(home.join("runtime/backend.py"), b"tampered adapter").unwrap(); + }); + assert_race_rejected("spawn unexpected", &|home| { + fs::write(home.join("runtime/injected.py"), b"untrusted").unwrap(); + }); + assert_race_rejected("spawn stale target", &|home| { + let path = home.join(MANIFEST_NAME); + let mut manifest: serde_json::Value = + serde_json::from_slice(&fs::read(&path).unwrap()).unwrap(); + manifest["target"] = "stale-unknown-target".into(); + fs::write(path, serde_json::to_vec(&manifest).unwrap()).unwrap(); + }); + assert_race_rejected("spawn symlink", &|home| { + use std::os::unix::fs::symlink; + let program = home.join("runtime/bin/backend"); + fs::remove_file(&program).unwrap(); + symlink("/bin/true", program).unwrap(); + }); + } + + #[test] + fn windows_host_essentials_are_explicit_and_other_host_values_stay_cleared() { + let root = root("windows host environment"); + install(&root, &["mrt2"]); + let manifest_path = root.join(MANIFEST_NAME); + let mut manifest: LaunchManifest = + serde_json::from_slice(&fs::read(&manifest_path).unwrap()).unwrap(); + let spec = manifest.services.get_mut("mrt2").unwrap(); + spec.ephemeral_environment.extend( + ["SYSTEMROOT", "WINDIR", "TEMP", "TMP"] + .into_iter() + .map(str::to_string), + ); + fs::remove_file(&manifest_path).unwrap(); + seal_candidate( + &root, + "x86_64-pc-windows-msvc", + manifest.provenance, + manifest.services, + ) + .unwrap(); + + let mut verified = resolve_at(&root, "mrt2", "x86_64-pc-windows-msvc").unwrap(); + verified.host_environment = vec![ + (OsString::from("SYSTEMROOT"), OsString::from(r"C:\Windows")), + (OsString::from("WINDIR"), OsString::from(r"C:\Windows")), + (OsString::from("TEMP"), OsString::from(r"C:\LSDJ\tmp")), + (OsString::from("TMP"), OsString::from(r"C:\LSDJ\tmp")), + ]; + let command = verified + .into_command( + [], + [(OsString::from("LSDJ_API_CAPABILITY"), OsString::from("cap"))], + ) + .unwrap(); + let environment: BTreeMap<_, _> = command + .get_envs() + .filter_map(|(name, value)| { + value.map(|value| { + ( + name.to_string_lossy().into_owned(), + value.to_string_lossy().into_owned(), + ) + }) + }) + .collect(); + assert_eq!(environment.get("SYSTEMROOT"), Some(&r"C:\Windows".into())); + assert_eq!(environment.get("WINDIR"), Some(&r"C:\Windows".into())); + assert_eq!(environment.get("TEMP"), Some(&r"C:\LSDJ\tmp".into())); + assert_eq!(environment.get("TMP"), Some(&r"C:\LSDJ\tmp".into())); + assert_eq!(environment.get("LSDJ_API_CAPABILITY"), Some(&"cap".into())); + for forbidden in [ + "PATH", + "HOME", + "HF_TOKEN", + "HUGGING_FACE_HUB_TOKEN", + "PYTHONPATH", + "LSDJ_GENERATION_CMD", + "LSDJ_SIDECAR_CMD", + "LSDJ_ALLOW_UNVERIFIED_MRT2_CUDA", + "LSDJ_ALLOW_UNVERIFIED_SA3_CUDA", + ] { + assert!(!environment.contains_key(forbidden)); + } + let _ = fs::remove_dir_all(root); + } +} diff --git a/src-tauri/src/mcp.rs b/src-tauri/src/mcp.rs index d3ff6d6..99e2b08 100644 --- a/src-tauri/src/mcp.rs +++ b/src-tauri/src/mcp.rs @@ -36,6 +36,8 @@ use tokio_util::sync::CancellationToken; use crate::commands::{valid_deck, DrumModeArg, EqBandArg, FxKindArg}; use crate::generation::GenerationServer; +#[cfg(feature = "managed-runtime")] +use crate::magenta_gateway::MagentaGateway; use crate::samples::{NewSample, SampleLibrary}; use crate::sidecar::Sidecars; use crate::songs::{NewSong, SongLibrary}; @@ -915,11 +917,26 @@ impl McpHandler { /// validation. `magenta` routes to the Magenta renderer (`/api/render`, body /// `{prompt, seconds}`); the rest are Stable Audio 3 (`/api/generate`). async fn generate_clip(&self, prompt: &str, seconds: f32, kind: &str) -> Result, String> { - let port = self - .app - .state::() - .port() - .ok_or("the generation server is not running")?; + let (port, capability) = if kind == "magenta" { + #[cfg(feature = "managed-runtime")] + { + let gateway = self.app.state::(); + ( + gateway.port().ok_or("the Magenta gateway is not running")?, + gateway + .capability() + .ok_or("the Magenta gateway authentication capability is unavailable")?, + ) + } + #[cfg(not(feature = "managed-runtime"))] + { + let generation = self.app.state::(); + generation.connection().ok_or("the generation server is not running")? + } + } else { + let generation = self.app.state::(); + generation.connection().ok_or("the generation server is not running")? + }; // sa3 generation is serialised; a full track (medium model) can take minutes, // so allow generous headroom but never wait forever for a wedged worker. let client = reqwest::Client::builder() @@ -933,6 +950,8 @@ impl McpHandler { }; let response = client .post(format!("http://127.0.0.1:{port}{path}")) + .header("x-lsdj-capability", capability) + .header("x-lsdj-job-id", crate::local_auth::generate_capability()) .json(&body) .send() .await @@ -1071,12 +1090,11 @@ impl ServerHandler for McpHandler { pub struct McpServer { app: AppHandle, token: Arc>, - /// Where the token is persisted (under the app data dir); `None` if the dir can't - /// be resolved (then the token is in-memory only). - token_path: Option, + /// Where the token is persisted under the resolved configuration root. + token_path: PathBuf, /// Where the chosen port is persisted, so it's stable across launches and the - /// config snippet doesn't churn; `None` if the dir can't be resolved. - port_path: Option, + /// config snippet doesn't churn. + port_path: PathBuf, running: Mutex, } @@ -1095,19 +1113,16 @@ impl McpServer { /// carry the bearer token (also persisted). pub fn start(app: AppHandle) -> McpServer { let token_path = token_file(&app); - let token_string = match &token_path { - Some(path) => load_or_generate_token(path), - None => generate_token(), - }; + let token_string = load_or_generate_token(&token_path); let token = Arc::new(RwLock::new(token_string)); let port_path = port_file(&app); - let desired = port_path.as_deref().and_then(load_port); + let desired = load_port(&port_path); let running = spawn_server(&app, &token, desired); // Remember the actually-bound port so an ephemeral assignment is reused. - if let (Some(port), Some(path)) = (running.port, &port_path) { - save_port(path, port); + if let Some(port) = running.port { + save_port(&port_path, port); } McpServer { @@ -1133,9 +1148,7 @@ impl McpServer { /// at once (a leaked token is invalidated without restarting). Returns the new token. pub fn rotate(&self) -> Option { let next = generate_token(); - if let Some(path) = &self.token_path { - save_token(path, &next); - } + save_token(&self.token_path, &next); *write_lock(&self.token) = next.clone(); Some(next) } @@ -1164,9 +1177,7 @@ impl McpServer { ) }; previous.cancel.cancel(); - if let Some(path) = &self.port_path { - save_port(path, port); - } + save_port(&self.port_path, port); Ok(port) } @@ -1312,20 +1323,14 @@ fn write_lock(lock: &RwLock) -> std::sync::RwLockWriteGuard<'_, String> lock.write().unwrap_or_else(|p| p.into_inner()) } -/// The token file under the app data dir (`None` if it can't be resolved). -fn token_file(app: &AppHandle) -> Option { - app.path() - .app_data_dir() - .ok() - .map(|dir| dir.join("mcp-token")) +/// The token file under the host-resolved configuration root. +fn token_file(_app: &AppHandle) -> PathBuf { + crate::platform_paths::get().config().join("mcp-token") } -/// The port file under the app data dir (`None` if it can't be resolved). -fn port_file(app: &AppHandle) -> Option { - app.path() - .app_data_dir() - .ok() - .map(|dir| dir.join("mcp-port")) +/// The port file under the host-resolved configuration root. +fn port_file(_app: &AppHandle) -> PathBuf { + crate::platform_paths::get().config().join("mcp-port") } /// Read the persisted port — a plain decimal `u16` ≥ 1024 (privileged ports are diff --git a/src-tauri/src/midi/drivers.rs b/src-tauri/src/midi/drivers.rs index fafa9de..4581f80 100644 --- a/src-tauri/src/midi/drivers.rs +++ b/src-tauri/src/midi/drivers.rs @@ -45,11 +45,25 @@ pub const DRIVERS: [Driver; 2] = [ }, ]; -/// The first registry driver whose fragment the port name contains, or `None` -/// for a non-controller port (which the service attaches as a keyboard-note -/// source instead). +/// Normalize the display punctuation ALSA/CoreMIDI/WinMM may add around a USB +/// product name while retaining only identity-bearing alphanumerics. The raw +/// name is still used to open and persist the exact port. +fn identity(name: &str) -> String { + name.chars() + .filter(|character| character.is_alphanumeric()) + .flat_map(char::to_uppercase) + .collect() +} + +/// The first registry driver whose normalized fragment occurs in the normalized +/// port name, or `None` for a non-controller port. ALSA commonly reports names +/// such as `DDJ-FLX4:DDJ-FLX4 MIDI 1 24:0`; punctuation/case must not make that +/// class-compliant controller disappear. pub fn driver_for_name(name: &str) -> Option<&'static Driver> { - DRIVERS.iter().find(|d| name.contains(d.name_fragment)) + let name = identity(name); + DRIVERS + .iter() + .find(|driver| name.contains(&identity(driver.name_fragment))) } #[cfg(test)] @@ -60,6 +74,14 @@ mod tests { fn matches_ports_by_fragment_in_registry_order() { assert_eq!(driver_for_name("DDJ-FLX4").map(|d| d.id), Some("flx4")); assert_eq!(driver_for_name("Pioneer DDJ-FLX4 MIDI 1").map(|d| d.id), Some("flx4")); + assert_eq!( + driver_for_name("ddj-flx4:ddj-flx4 midi 1 24:0").map(|d| d.id), + Some("flx4") + ); + assert_eq!( + driver_for_name("AlphaTheta DDJ FLX4 MIDI 1").map(|d| d.id), + Some("flx4") + ); assert_eq!(driver_for_name("DDJ-400").map(|d| d.id), Some("ddj400")); assert_eq!(driver_for_name("IAC Driver Bus 1").map(|d| d.id), None); assert_eq!(driver_for_name("KeyLab 61").map(|d| d.id), None); diff --git a/src-tauri/src/midi/mod.rs b/src-tauri/src/midi/mod.rs index 89b7eb8..f05e46e 100644 --- a/src-tauri/src/midi/mod.rs +++ b/src-tauri/src/midi/mod.rs @@ -19,7 +19,7 @@ //! proceeds, intent kinds migrate from the forward list to native //! application without touching the transport or the translator. //! -//! Input callbacks run on CoreMIDI threads: they translate, route, and +//! Input callbacks run on the platform MIDI backend's threads: they translate, route, and //! return — the heavy lifting (LED frames, beat math) lives on the painter //! and scheduler threads. Nothing here goes near the cpal callback. @@ -234,6 +234,7 @@ impl MidiService { // Dropping the wrapper is fine: coremidi 0.9 never disposes clients // (its `Drop` is deliberately disabled upstream), so the underlying // client — and the delivery it anchors — lives as long as the app. + #[cfg(target_os = "macos")] if let Err(e) = MidiInput::new("LSDJ hot-plug anchor") { eprintln!("lsdj-app: midi hot-plug anchor failed: {e}"); } @@ -475,10 +476,13 @@ fn bind_output(shared: &Arc, driver: &Driver, name: &str) { return; } }; - let port = output - .ports() - .into_iter() - .find(|p| output.port_name(p).is_ok_and(|n| n.contains(driver.name_fragment))); + let port = output.ports().into_iter().find(|port| { + output + .port_name(port) + .ok() + .and_then(|name| driver_for_name(&name)) + .is_some_and(|candidate| candidate.id == driver.id) + }); let Some(port) = port else { eprintln!("lsdj-app: no midi output port for '{name}'"); return; diff --git a/src-tauri/src/models.rs b/src-tauri/src/models.rs index 1d39ad2..654c54f 100644 --- a/src-tauri/src/models.rs +++ b/src-tauri/src/models.rs @@ -1,31 +1,46 @@ //! Model manager (issue #43): status, install, and delete for the two model //! families — Magenta deck models and the Stable Audio 3 stack — surfaced in the //! settings-drawer panel. Rust owns the lifecycle (mirrors ADR-0022); the actual -//! downloads are delegated to Python/shell tooling Rust orchestrates: +//! downloads are orchestrated by Rust: //! //! - **Magenta** → the frozen sidecar's `--init-resources` / `--download-model` //! modes (`backend/lsdj/sidecar.py`), which reuse `magenta_rt.cli` verbatim and //! stream a JSON progress contract on stdout. Resources are fetched first so a //! freshly downloaded model is actually loadable (a model's two files are not //! enough without `resources/musiccoca` + `resources/spectrostream`). -//! - **Stable Audio 3** → `curl` the pinned source tarball (`sa3-pin.json`), -//! `tar`-extract it, and run `scripts/sa3-install.sh` (build+warm steps; no -//! git, no tty, no system Python 3.11). +//! - **Stable Audio 3** → download the app-bundled immutable manifest's source, +//! runtime, dependency, and model artifacts over native HTTPS; verify every +//! SHA-256; extract with bounded native archive APIs; then build, warm, and +//! atomically promote the candidate without a shell or external tools. //! //! Status facts mirror the stable conventions in `backend/lsdj/paths.py` and //! `backend/lsdj/sa3.py` (the two-file model layout, the SA3 candidate list, the //! four readiness states). The webview never gets filesystem access — the same //! trust boundary as the rest of the library surface. -use std::io::{BufRead, BufReader}; +use std::collections::{BTreeMap, BTreeSet}; use std::path::{Path, PathBuf}; -use std::process::{Child, Command, Stdio}; +use std::process::{Command, Stdio}; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; use serde::{Deserialize, Serialize}; +#[cfg(feature = "managed-runtime")] +use tauri::Manager; use tauri::{AppHandle, Emitter}; +use crate::child_process::{ + read_bounded_lines, sanitize_diagnostic, DiagnosticTail, SupervisedChild, +}; +use crate::runtime_installer::archive::{ + extract_tar_gz_cancellable, extract_zip_cancellable, ArchiveLimits, +}; +use crate::runtime_installer::download::{ + client as installer_client, download_verified, link_or_copy_verified, verify_file_cancellable, + PinnedArtifact, +}; +use crate::runtime_installer::promotion; + /// The official models the manager offers to download (mirrors /// `engine.KNOWN_MODELS`). This is the installable catalog, NOT a discovery gate: /// `discover_installed` finds any model folder, but only these can be downloaded. @@ -36,6 +51,7 @@ const SA3_MISSING: &str = "missing"; const SA3_VENV_MISSING: &str = "venv_missing"; const SA3_NOT_WARMED: &str = "not_warmed"; const SA3_READY: &str = "ready"; +const SA3_FAILED: &str = "failed"; const WARMED_STAMP: &str = ".lsdj-warmed"; @@ -44,178 +60,267 @@ const WARMED_STAMP: &str = ".lsdj-warmed"; // pin and offer an in-app update. Written by Rust after a fetch (the shell // installer doesn't know the commit). Lives beside `.lsdj-warmed` in optimized/mlx. const SOURCE_STAMP: &str = ".lsdj-source.json"; +const INSTALL_MANIFEST_STAMP: &str = ".lsdj-install-manifest.json"; +const MLX_REQUIREMENTS_LOCK: &str = include_str!("../../scripts/sa3-requirements.lock"); +const TFLITE_REQUIREMENTS_LOCK: &str = include_str!("../../scripts/sa3-tflite-requirements.lock"); +const TFLITE_WHEEL_PIN_JSON: &str = include_str!("../../sa3-tflite-wheels.json"); +#[cfg(any(feature = "managed-runtime", test))] +const MRT2_PIN_JSON: &str = include_str!("../../mrt2-pytorch-pin.json"); +#[cfg(any(feature = "managed-runtime", test))] +const MRT2_WHEEL_PIN_JSON: &str = include_str!("../../mrt2-pytorch-wheels.json"); +#[cfg(any(feature = "managed-runtime", test))] +const MRT2_LINUX_LOCK: &str = + include_str!("../../backend/runtime-locks/mrt2-pytorch-linux-x86_64.txt"); +#[cfg(any(feature = "managed-runtime", test))] +const MRT2_WINDOWS_LOCK: &str = + include_str!("../../backend/runtime-locks/mrt2-pytorch-windows-x86_64.txt"); +const TFLITE_PROVENANCE_STAMP: &str = ".lsdj-provenance.json"; +#[cfg(feature = "managed-runtime")] +const MRT2_IDENTITY_STAMP: &str = ".lsdj-mrt2-install"; + +const BACKEND_SOURCES: &[(&str, &[u8])] = &[ + ( + "__init__.py", + include_bytes!("../../backend/lsdj/__init__.py"), + ), + ( + "controller.py", + include_bytes!("../../backend/lsdj/controller.py"), + ), + ("engine.py", include_bytes!("../../backend/lsdj/engine.py")), + ("frozen.py", include_bytes!("../../backend/lsdj/frozen.py")), + ( + "gpu_broker.py", + include_bytes!("../../backend/lsdj/gpu_broker.py"), + ), + ("loras.py", include_bytes!("../../backend/lsdj/loras.py")), + ("mrt2.py", include_bytes!("../../backend/lsdj/mrt2.py")), + ( + "mrt2_pytorch.py", + include_bytes!("../../backend/lsdj/mrt2_pytorch.py"), + ), + ( + "runtime_paths.py", + include_bytes!("../../backend/lsdj/runtime_paths.py"), + ), + ("sa3.py", include_bytes!("../../backend/lsdj/sa3.py")), + ( + "sa3_cuda.py", + include_bytes!("../../backend/lsdj/sa3_cuda.py"), + ), + ( + "sa3_audio.py", + include_bytes!("../../backend/lsdj/sa3_audio.py"), + ), + ( + "sa3_contract.py", + include_bytes!("../../backend/lsdj/sa3_contract.py"), + ), + ( + "sidecar.py", + include_bytes!("../../backend/lsdj/sidecar.py"), + ), + ("worker.py", include_bytes!("../../backend/lsdj/worker.py")), +]; -// --- Path resolution (mirrors backend/lsdj/paths.py + sa3.py) -------------- +const BACKEND_PATH_ENVIRONMENT: &[&str] = &[ + "LSDJ_ASSETS_HOME", + "LSDJ_CACHE_HOME", + "LSDJ_CONFIG_HOME", + "LSDJ_DATA_HOME", + "LSDJ_STAGING_HOME", +]; +const MRT2_PATH_ENVIRONMENT: &[&str] = &["MAGENTA_HOME"]; +const SA3_PATH_ENVIRONMENT: &[&str] = &["SA3_HOME", "SA3_LORAS_HOME", "SA3_MLX_HOME"]; -fn home_dir() -> PathBuf { - std::env::var_os("HOME") - .map(PathBuf::from) - .unwrap_or_else(|| PathBuf::from("/")) +const WINDOWS_CHILD_ENVIRONMENT: &[&str] = &["SYSTEMROOT", "WINDIR", "TEMP", "TMP"]; + +fn service_ephemeral_environment( + service: crate::managed_runtime::Service, + secret: &str, +) -> Vec { + let service_paths = match service { + crate::managed_runtime::Service::Mrt2 => MRT2_PATH_ENVIRONMENT, + crate::managed_runtime::Service::Sa3 | crate::managed_runtime::Service::Sa3Cuda => { + SA3_PATH_ENVIRONMENT + } + }; + std::iter::once(secret) + .chain(BACKEND_PATH_ENVIRONMENT.iter().copied()) + .chain(service_paths.iter().copied()) + .chain(WINDOWS_CHILD_ENVIRONMENT.iter().copied()) + .map(str::to_string) + .collect() } -/// The `magenta-rt-v2` data root: `(MAGENTA_HOME or ~/Documents/Magenta)/ -/// magenta-rt-v2`. The `magenta-rt-v2` segment is ALWAYS appended, even when -/// `MAGENTA_HOME` is set — matching `paths.magenta_home()`. -fn magenta_home() -> PathBuf { - let base = std::env::var_os("MAGENTA_HOME") - .map(PathBuf::from) - .unwrap_or_else(|| home_dir().join("Documents").join("Magenta")); - base.join("magenta-rt-v2") +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Sa3Backend { + Mlx, + Tflite, } -/// The Magenta models dir (`paths.models_dir()`). -pub fn magenta_models_dir() -> PathBuf { - magenta_home().join("models") +impl Sa3Backend { + fn wire_name(self) -> &'static str { + match self { + Self::Mlx => "mlx", + Self::Tflite => "tflite", + } + } + + fn runtime_dir(self, checkout: &Path) -> PathBuf { + checkout.join("optimized").join(match self { + Self::Mlx => "mlx", + Self::Tflite => "tflite", + }) + } + + fn script(self) -> &'static str { + match self { + Self::Mlx => "sa3_mlx.py", + Self::Tflite => "sa3_tflite.py", + } + } + + fn requirements(self) -> (&'static str, &'static str) { + match self { + Self::Mlx => ("sa3-requirements.lock", MLX_REQUIREMENTS_LOCK), + Self::Tflite => ("sa3-tflite-requirements.lock", TFLITE_REQUIREMENTS_LOCK), + } + } } -/// The app-owned data root for model weights — `~/Library/Application Support/ -/// LSDJ`. Kept out of `~/Documents` (which users may sync to iCloud, where -/// multi-GB weights don't belong and Finder ops on offloaded files fail, -8013). -pub(crate) fn app_support_base() -> PathBuf { - home_dir() - .join("Library") - .join("Application Support") - .join("LSDJ") +fn sa3_backend_for(os: &str, arch: &str) -> Result { + match (os, arch) { + ("macos", "aarch64") => Ok(Sa3Backend::Mlx), + ("linux", "x86_64") | ("windows", "x86_64") => Ok(Sa3Backend::Tflite), + _ => Err(format!("Stable Audio 3 is unsupported on {os}/{arch}")), + } } -/// The app-owned home for the Stable Audio 3 checkout — where in-app installs go -/// and the first place the resolver looks (kept out of `~/Documents`, like the -/// Magenta weights). -fn sa3_app_home() -> PathBuf { - app_support_base().join("stable-audio-3") +fn host_sa3_backend() -> Result { + sa3_backend_for(std::env::consts::OS, std::env::consts::ARCH) } -/// Decide the one-time migration: `Some((from, to))` when a prior install lives -/// at `old_rtv2` and `new_rtv2` doesn't exist yet, else `None`. Pure (no I/O -/// beyond the existence checks) so the policy is unit-testable. -fn migration_move(new_rtv2: &Path, old_rtv2: &Path) -> Option<(PathBuf, PathBuf)> { - (!new_rtv2.exists() && old_rtv2.is_dir()) - .then(|| (old_rtv2.to_path_buf(), new_rtv2.to_path_buf())) +// --- Host-resolved paths (mirrors the explicit Python environment) -------- + +/// The `magenta-rt-v2` data root. `MAGENTA_HOME`'s compatibility semantics +/// append this segment; the host owns the resolved base. +fn magenta_home() -> PathBuf { + crate::platform_paths::get() + .magenta_base() + .join("magenta-rt-v2") } -/// Move one directory from the previous brand root when its LSDJ destination -/// does not exist yet. If the move fails, keep using the legacy directory so a -/// branding change can never make installed models or adapters disappear. -pub(crate) fn migrate_legacy_dir(new_dir: &Path, old_dir: &Path) -> PathBuf { - let Some((from, to)) = migration_move(new_dir, old_dir) else { - return new_dir.to_path_buf(); - }; - if let Some(parent) = to.parent() { - if let Err(error) = std::fs::create_dir_all(parent) { - eprintln!( - "lsdj-app: could not prepare branded data directory {}: {error}", - parent.display() - ); - return from; - } - } - match std::fs::rename(&from, &to) { - Ok(()) => { - eprintln!("lsdj-app: migrated branded data → {}", to.display()); - to - } - Err(error) => { - eprintln!( - "lsdj-app: could not migrate {} to {}: {error}", - from.display(), - to.display() - ); - from - } - } -} - -/// Point `MAGENTA_HOME` at the app-owned data dir and migrate a prior -/// `~/Documents/Magenta` install into it (a same-volume rename — instant, no -/// multi-GB copy). A pre-set `MAGENTA_HOME` (a dev/user override) wins. Must run -/// once at startup BEFORE any backend process is spawned, so the children — and -/// `magenta_rt.paths`, which reads the env at import — inherit the new location. -pub fn ensure_magenta_home() { - let base = app_support_base(); - let legacy_base = home_dir() - .join("Library") - .join("Application Support") - .join("LSDJai"); - let branded_magenta = migrate_legacy_dir( - &base.join("magenta-rt-v2"), - &legacy_base.join("magenta-rt-v2"), - ); - let branded_sa3 = migrate_legacy_dir( - &base.join("stable-audio-3"), - &legacy_base.join("stable-audio-3"), - ); - let branded_loras = migrate_legacy_dir(&base.join("sa3-loras"), &legacy_base.join("sa3-loras")); - if branded_sa3.starts_with(&legacy_base) && std::env::var_os("SA3_MLX_HOME").is_none() { - std::env::set_var("SA3_MLX_HOME", branded_sa3); - } - if branded_loras.starts_with(&legacy_base) && std::env::var_os("SA3_LORAS_HOME").is_none() { - std::env::set_var("SA3_LORAS_HOME", branded_loras); - } - if std::env::var_os("MAGENTA_HOME").is_some() { - return; // respect an explicit override (dev, or a custom location) - } - if branded_magenta.starts_with(&legacy_base) { - std::env::set_var("MAGENTA_HOME", legacy_base); - return; - } - let old_base = home_dir().join("Documents").join("Magenta"); - if let Some((from, to)) = migration_move(&base.join("magenta-rt-v2"), &old_base.join("magenta-rt-v2")) { - let _ = std::fs::create_dir_all(&base); - if std::fs::rename(&from, &to).is_err() { - // Cross-volume / perms: keep the existing install rather than strand it. - std::env::set_var("MAGENTA_HOME", &old_base); - return; - } - eprintln!("lsdj-app: migrated model weights → {}", to.display()); +/// The Magenta models dir (`paths.models_dir()`). +pub fn magenta_models_dir() -> PathBuf { + #[cfg(feature = "managed-runtime")] + if managed_mrt2_host() { + return crate::managed_runtime::service_root( + crate::platform_paths::get().assets(), + crate::managed_runtime::Service::Mrt2, + ) + .join("models"); } - std::env::set_var("MAGENTA_HOME", &base); + magenta_home().join("models") +} + +/// The host-owned home for the Stable Audio 3 checkout. +fn sa3_app_home() -> PathBuf { + crate::platform_paths::get().sa3_home().to_path_buf() } /// Whether the shared resources a model load needs are present — without these /// (`mrt models init` fetches them) a model's two files cannot load. fn resources_present() -> bool { + #[cfg(feature = "managed-runtime")] + if managed_mrt2_host() { + let root = crate::managed_runtime::service_root( + crate::platform_paths::get().assets(), + crate::managed_runtime::Service::Mrt2, + ); + let pin = mrt2_pin(); + return validate_mrt2_identity(&root, &pin).is_ok() + && mrt2_snapshot_present(&root, "musiccoca", &pin.processor); + } let resources = magenta_home().join("resources"); resources.join("musiccoca").is_dir() && resources.join("spectrostream").is_dir() } +#[cfg(feature = "managed-runtime")] +fn managed_mrt2_host() -> bool { + matches!( + crate::managed_runtime::host_target().as_str(), + "x86_64-unknown-linux-gnu" | "x86_64-pc-windows-msvc" + ) +} + +#[cfg(feature = "managed-runtime")] +fn mrt2_snapshot_present(root: &Path, install_name: &str, snapshot: &SnapshotPin) -> bool { + snapshot.files.iter().all(|file| { + let Ok(metadata) = + std::fs::symlink_metadata(root.join("models").join(install_name).join(&file.path)) + else { + return false; + }; + metadata.is_file() && !metadata.file_type().is_symlink() && metadata.len() == file.size + }) +} + /// SA3 checkout roots to probe, in order (mirrors `sa3._checkout_candidates`). fn sa3_candidates() -> Vec { - let mut candidates = Vec::new(); - if let Some(override_home) = std::env::var_os("SA3_MLX_HOME") { - if !override_home.is_empty() { - candidates.push(PathBuf::from(override_home)); - } - } - candidates.push(sa3_app_home()); // the app-owned home — where in-app installs go - candidates + vec![sa3_app_home()] } -/// The SA3 install state + the resolved checkout root (mirrors `sa3.readiness`): -/// the first candidate with an `optimized/mlx` dir, classified `missing` / -/// `venv_missing` / `not_warmed` / `ready`. +/// The SA3 install state + resolved checkout for this platform's backend. fn sa3_status() -> (&'static str, Option) { - let mut first_with_mlx: Option = None; + let Ok(backend) = host_sa3_backend() else { + return (SA3_FAILED, None); + }; + let mut first_with_runtime: Option = None; for checkout in sa3_candidates() { - let mlx = checkout.join("optimized").join("mlx"); - if !mlx.is_dir() { + let runtime = backend.runtime_dir(&checkout); + if !runtime.is_dir() { continue; } - if first_with_mlx.is_none() { - first_with_mlx = Some(checkout.clone()); + if first_with_runtime.is_none() { + first_with_runtime = Some(checkout.clone()); + } + + // A manifest marks an app-managed install. Never let its legacy stamps + // bypass current trust policy: readiness means the manifest, runtime, + // provenance, warm-up, and all eight model hashes validate. Only an + // older hand-installed checkout with no app manifest uses the historical + // interpreter/script/stamp heuristic below. + let manifest = checkout.join(INSTALL_MANIFEST_STAMP); + if !matches!( + std::fs::symlink_metadata(&manifest), + Err(error) if error.kind() == std::io::ErrorKind::NotFound + ) { + let state = if validate_sa3_install(&checkout, &sa3_pin(), backend).is_ok() { + SA3_READY + } else { + SA3_FAILED + }; + return (state, Some(checkout)); + } + // Preserve the historical hand-installed MLX probe on macOS. Portable + // installs require the authenticated app manifest and provenance stamp. + if backend == Sa3Backend::Tflite { + return (SA3_FAILED, Some(checkout)); } - let python = mlx.join(".venv").join("bin").join("python"); - let script = mlx.join("scripts").join("sa3_mlx.py"); + let python = crate::platform_paths::venv_python(&runtime.join(".venv")); + let script = runtime.join("scripts").join(backend.script()); if !(python.is_file() && script.is_file()) { continue; } - let state = if mlx.join(WARMED_STAMP).is_file() { + let state = if runtime.join(WARMED_STAMP).is_file() { SA3_READY } else { SA3_NOT_WARMED }; return (state, Some(checkout)); } - match first_with_mlx { + match first_with_runtime { Some(checkout) => (SA3_VENV_MISSING, Some(checkout)), None => (SA3_MISSING, None), } @@ -233,7 +338,9 @@ pub struct Sa3Source { } fn source_stamp_path(checkout: &Path) -> PathBuf { - checkout.join("optimized").join("mlx").join(SOURCE_STAMP) + host_sa3_backend() + .map(|backend| backend.runtime_dir(checkout).join(SOURCE_STAMP)) + .unwrap_or_else(|_| checkout.join(SOURCE_STAMP)) } /// The source recorded in a checkout's stamp, or `None` when absent (a checkout @@ -243,12 +350,12 @@ fn read_source_stamp(checkout: &Path) -> Option { serde_json::from_str(&data).ok() } -/// Record what was fetched into the checkout. Best-effort: a failed write just -/// means the next `model_status` treats the checkout as updatable. -fn write_source_stamp(checkout: &Path, source: &Sa3Source) { - if let Ok(json) = serde_json::to_string_pretty(source) { - let _ = std::fs::write(source_stamp_path(checkout), json); - } +/// Record what was fetched into the checkout. A secure candidate is never +/// promoted without this provenance marker. +fn write_source_stamp(checkout: &Path, source: &Sa3Source) -> Result<(), String> { + let json = serde_json::to_string_pretty(source) + .map_err(|error| format!("cannot serialize SA3 source stamp: {error}"))?; + write_synced(&source_stamp_path(checkout), json.as_bytes()) } /// The currently pinned source (`sa3-pin.json`). @@ -319,9 +426,12 @@ pub(crate) fn dir_size(path: &Path) -> u64 { /// one is summed once. fn sa3_checkout_size(checkout: &Path) -> u64 { static CACHE: Mutex> = Mutex::new(None); - let stamp_mtime = std::fs::metadata(checkout.join("optimized").join("mlx").join(WARMED_STAMP)) - .and_then(|m| m.modified()) - .ok(); + let stamp_mtime = host_sa3_backend() + .ok() + .and_then(|backend| { + std::fs::metadata(backend.runtime_dir(checkout).join(WARMED_STAMP)).ok() + }) + .and_then(|metadata| metadata.modified().ok()); let mut cache = CACHE.lock().unwrap_or_else(|p| p.into_inner()); if let (Some(mtime), Some((path, cached_mtime, size))) = (stamp_mtime, cache.as_ref()) { if path == checkout && *cached_mtime == mtime { @@ -384,7 +494,11 @@ pub struct MagentaStatus { #[serde(rename_all = "camelCase")] pub struct Sa3Status { state: &'static str, + backend: Option<&'static str>, size_bytes: u64, + /// Exact unique model bytes in the selected backend's pinned manifest. The + /// UI shows this before install; source/runtime/wheel overhead is additional. + download_bytes: u64, checkout: Option, /// The source the installed checkout was fetched from (`None` when the /// checkout predates stamping or was placed by hand). @@ -420,19 +534,51 @@ pub struct ModelStatus { fn status(active: Option<(Family, String)>) -> ModelStatus { let models_dir = magenta_models_dir(); let resources = resources_present(); - let installed = discover_installed(&models_dir) - .into_iter() - .map(|name| { - let size_bytes = dir_size(&models_dir.join(&name)); - InstalledModel { + #[cfg(feature = "managed-runtime")] + let installed = if managed_mrt2_host() { + let root = models_dir.parent().unwrap_or(&models_dir); + let pin = mrt2_pin(); + if validate_mrt2_identity(root, &pin).is_ok() { + pin.models + .iter() + .filter(|(name, snapshot)| mrt2_snapshot_present(root, name, snapshot)) + .map(|(name, _)| InstalledModel { + name: name.clone(), + size_bytes: dir_size(&models_dir.join(name)), + needs_resources: !resources, + }) + .collect() + } else { + Vec::new() + } + } else { + discover_installed(&models_dir) + .into_iter() + .map(|name| InstalledModel { + size_bytes: dir_size(&models_dir.join(&name)), name, - size_bytes, needs_resources: !resources, - } + }) + .collect() + }; + #[cfg(not(feature = "managed-runtime"))] + let installed = discover_installed(&models_dir) + .into_iter() + .map(|name| InstalledModel { + size_bytes: dir_size(&models_dir.join(&name)), + name, + needs_resources: !resources, }) .collect(); let (sa3_state, sa3_checkout) = sa3_status(); - let sa3_size = sa3_checkout.as_ref().map(|c| sa3_checkout_size(c)).unwrap_or(0); + let backend = host_sa3_backend().ok(); + let download_bytes = backend + .and_then(|backend| model_download_bytes(&sa3_pin(), backend).ok()) + .unwrap_or(0); + let sa3_size = sa3_checkout + .as_ref() + .map(|c| sa3_checkout_size(c)) + .unwrap_or(0); let pinned = pinned_source(); let installed_source = sa3_checkout.as_ref().and_then(|c| read_source_stamp(c)); let update_available = @@ -446,20 +592,38 @@ fn status(active: Option<(Family, String)>) -> ModelStatus { }, sa3: Sa3Status { state: sa3_state, + backend: backend.map(Sa3Backend::wire_name), size_bytes: sa3_size, + download_bytes, checkout: sa3_checkout.map(|c| c.to_string_lossy().into_owned()), installed_source, pinned_source: pinned, update_available, }, loras: crate::loras::discover(&crate::loras::loras_dir()), - installing: active.map(|(family, name)| ActiveInstall { - family, - name, - }), + installing: active.map(|(family, name)| ActiveInstall { family, name }), } } +/// Installed MRT2 models for the Rust-owned managed render gateway's lightweight +/// `/api/models` compatibility endpoint. Only the authenticated, generation- +/// bound install identity is trusted; a candidate or hand-placed directory never +/// becomes loadable merely because files exist. +#[cfg(feature = "managed-runtime")] +pub(crate) fn magenta_models_for_gateway() -> Vec { + let models_dir = magenta_models_dir(); + let root = models_dir.parent().unwrap_or(&models_dir); + let pin = mrt2_pin(); + if !managed_mrt2_host() || validate_mrt2_identity(root, &pin).is_err() { + return Vec::new(); + } + pin.models + .iter() + .filter(|(name, snapshot)| mrt2_snapshot_present(root, name, snapshot)) + .map(|(name, _)| name.clone()) + .collect() +} + // --- Install / delete ------------------------------------------------------ /// Which family a command targets. `lowercase` serde is the single source of the @@ -485,37 +649,383 @@ struct ModelProgress { file: Option, } -fn emit(app: &AppHandle, family: Family, name: &str, stage: &str, message: Option, file: Option) { +fn emit( + app: &AppHandle, + family: Family, + name: &str, + stage: &str, + message: Option, + file: Option, +) { let _ = app.emit( "model://progress", ModelProgress { family, name: name.to_string(), - stage: stage.to_string(), - message, - file, + stage: sanitize_diagnostic(stage), + message: message.as_deref().map(sanitize_diagnostic), + file: file.as_deref().map(sanitize_diagnostic), }, ); } /// The pinned SA3 source (`sa3-pin.json`, the single bump point). Compiled in so /// a released binary carries the pin it was built with. -#[derive(Deserialize)] +#[derive(Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +struct ArchivePin { + #[serde(flatten)] + artifact: PinnedArtifact, + archive_root: String, + max_files: u64, + max_expanded_bytes: u64, + #[serde(default = "default_archive_format")] + archive_format: String, +} + +fn default_archive_format() -> String { + "tar.gz".into() +} + +impl ArchivePin { + fn limits(&self) -> ArchiveLimits { + ArchiveLimits { + max_files: self.max_files, + max_expanded_bytes: self.max_expanded_bytes, + materialize_safe_links: false, + } + } + + fn extract( + &self, + source: std::fs::File, + destination: &Path, + materialize_safe_links: bool, + is_cancelled: &dyn Fn() -> bool, + ) -> Result<(), String> { + let limits = ArchiveLimits { + materialize_safe_links, + ..self.limits() + }; + match self.archive_format.as_str() { + "tar.gz" => extract_tar_gz_cancellable( + source, + destination, + &self.archive_root, + limits, + is_cancelled, + ), + "zip" if !materialize_safe_links => extract_zip_cancellable( + source, + destination, + &self.archive_root, + limits, + is_cancelled, + ), + "zip" => Err("ZIP runtime archives may not contain links".into()), + _ => Err("runtime archive format is unsupported".into()), + } + } +} + +#[derive(Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +struct UvPin { + target: String, + version: String, + #[serde(flatten)] + archive: ArchivePin, + executable: String, +} + +#[derive(Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +struct PythonPin { + target: String, + version: String, + #[serde(flatten)] + archive: ArchivePin, + executable: String, +} + +#[derive(Clone, Deserialize)] +struct RuntimePin { + requirements: String, + python: Vec, + uv: Vec, +} + +#[derive(Clone, Deserialize)] +struct ModelArtifactPin { + path: String, + sha256: String, + size: u64, +} + +impl ModelArtifactPin { + fn filename(&self) -> Result<&str, String> { + let filename = self + .path + .strip_prefix("MLX/") + .ok_or("SA3 model path must be below MLX/")?; + if filename.is_empty() + || filename.contains('/') + || filename.contains('\\') + || filename == "." + || filename == ".." + { + return Err("SA3 model path is unsafe".into()); + } + Ok(filename) + } + + fn artifact(&self, models: &ModelsPin) -> Result { + let filename = self.filename()?; + Ok(PinnedArtifact { + url: format!( + "https://huggingface.co/{}/resolve/{}/MLX/{filename}?download=true", + models.repo, models.revision + ), + sha256: self.sha256.clone(), + size: self.size, + }) + } +} + +#[derive(Clone, Deserialize)] +struct ModelsPin { + repo: String, + revision: String, + artifacts: Vec, +} + +#[derive(Clone, Deserialize)] struct Sa3Pin { repo: String, commit: String, + source: ArchivePin, + runtime: RuntimePin, + models: ModelsPin, } +const SA3_PIN_JSON: &str = include_str!("../../sa3-pin.json"); +const TFLITE_PIN_JSON: &str = include_str!("../../sa3-tflite-pin.json"); +#[allow(dead_code)] +const SA3_CUDA_PIN_JSON: &str = include_str!("../../sa3-pytorch-cuda-pin.json"); +#[allow(dead_code)] +const SA3_CUDA_LOCK: &str = include_str!("../../backend/runtime-locks/windows-gpu-pytorch.txt"); + fn sa3_pin() -> Sa3Pin { - const PIN: &str = include_str!("../../sa3-pin.json"); - serde_json::from_str(PIN).expect("sa3-pin.json is valid JSON") + serde_json::from_str(SA3_PIN_JSON).expect("sa3-pin.json is valid JSON") +} + +#[derive(Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +struct TfliteRuntimePin { + repo: String, + revision: String, + subdirectory: String, + entrypoint: String, + requirements_lock: String, +} + +#[derive(Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +struct TfliteArtifactPin { + path: String, + install_path: String, + sha256: String, + size: u64, +} + +impl TfliteArtifactPin { + fn artifact(&self, models: &TfliteModelsPin) -> PinnedArtifact { + PinnedArtifact { + url: format!( + "https://huggingface.co/{}/resolve/{}/{}?download=true", + models.repo, models.revision, self.path + ), + sha256: self.sha256.clone(), + size: self.size, + } + } +} + +#[derive(Clone, Deserialize)] +struct TfliteModelsPin { + repo: String, + revision: String, + precision: String, + shared: Vec, + bundles: BTreeMap>, +} + +#[derive(Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +struct TflitePin { + schema_version: u32, + runtime: TfliteRuntimePin, + models: TfliteModelsPin, +} + +fn tflite_pin() -> TflitePin { + serde_json::from_str(TFLITE_PIN_JSON).expect("sa3-tflite-pin.json is valid JSON") +} + +#[derive(Clone, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct WheelPin { + package: String, + version: String, + filename: String, + #[serde(flatten)] + artifact: PinnedArtifact, +} + +#[derive(Clone, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct WheelManifest { + schema_version: u32, + python: String, + common: Vec, + targets: BTreeMap>, +} + +fn wheel_manifest() -> WheelManifest { + serde_json::from_str(TFLITE_WHEEL_PIN_JSON).expect("sa3-tflite-wheels.json is valid JSON") +} + +#[cfg(any(feature = "managed-runtime", test))] +#[derive(Clone, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct SnapshotFilePin { + path: String, + size: u64, + sha256: String, +} + +#[cfg(any(feature = "managed-runtime", test))] +#[derive(Clone, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct SnapshotPin { + repository: String, + revision: String, + files: Vec, +} + +#[cfg(any(feature = "managed-runtime", test))] +#[derive(Clone, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct Mrt2RuntimePin { + python: Vec, + uv: Vec, + locks: BTreeMap, + wheel_manifest_sha256: String, } -fn sa3_install_script() -> PathBuf { - if let Some(path) = std::env::var_os("LSDJ_SA3_INSTALL_SH") { - return PathBuf::from(path); +#[cfg(any(feature = "managed-runtime", test))] +#[derive(Clone, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct Mrt2Pin { + schema_version: u32, + runtime: Mrt2RuntimePin, + models: BTreeMap, + processor: SnapshotPin, +} + +#[cfg(any(feature = "managed-runtime", test))] +#[derive(Clone, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct Mrt2WheelManifest { + schema_version: u32, + python: String, + targets: BTreeMap>, +} + +#[cfg(any(feature = "managed-runtime", test))] +fn mrt2_pin() -> Mrt2Pin { + serde_json::from_str(MRT2_PIN_JSON).expect("mrt2-pytorch-pin.json is valid JSON") +} + +#[cfg(any(feature = "managed-runtime", test))] +fn mrt2_wheel_manifest() -> Mrt2WheelManifest { + serde_json::from_str(MRT2_WHEEL_PIN_JSON).expect("mrt2-pytorch-wheels.json is valid JSON") +} + +/// Release/install gate for the optional Windows CUDA bundle. The public +/// model metadata is intentionally compiled into the app for review, but the +/// installer must never expose a download until issue #108 has supplied every +/// required gated hash and the physical qualification blockers are cleared. +#[allow(dead_code)] +fn validate_sa3_cuda_install_gate(json: &str, lock: &[u8]) -> Result<(), String> { + use sha2::{Digest, Sha256}; + + let pin: serde_json::Value = serde_json::from_str(json) + .map_err(|error| format!("SA3 CUDA pin is invalid JSON: {error}"))?; + if pin.get("schemaVersion").and_then(|value| value.as_u64()) != Some(1) + || pin.get("backend").and_then(|value| value.as_str()) != Some("pytorch_cuda") + || pin.get("platform").and_then(|value| value.as_str()) != Some("windows-x86_64") + { + return Err("SA3 CUDA pin has an unsupported schema or target".into()); + } + if pin.get("releaseReady").and_then(|value| value.as_bool()) != Some(true) + || pin + .get("gatedArtifactsComplete") + .and_then(|value| value.as_bool()) + != Some(true) + || pin + .get("releaseBlockers") + .and_then(|value| value.as_array()) + .is_none_or(|blockers| !blockers.is_empty()) + { + return Err("SA3 CUDA candidate is not release-ready; TFLite remains active".into()); + } + let runtime = pin + .get("sharedRuntime") + .ok_or("SA3 CUDA shared runtime pin is missing")?; + let expected_size = runtime + .get("requirementsLockSize") + .and_then(|value| value.as_u64()) + .ok_or("SA3 CUDA lock size is missing")?; + let expected_hash = runtime + .get("requirementsLockSha256") + .and_then(|value| value.as_str()) + .ok_or("SA3 CUDA lock hash is missing")?; + if lock.len() as u64 != expected_size || hex::encode(Sha256::digest(lock)) != expected_hash { + return Err("SA3 CUDA shared runtime lock does not match its pin".into()); + } + let models = pin + .get("models") + .and_then(|value| value.as_object()) + .ok_or("SA3 CUDA model pins are missing")?; + for (name, model) in models { + let required = model + .get("required") + .and_then(|value| value.as_bool()) + .unwrap_or(false) + || model + .get("enabled") + .and_then(|value| value.as_bool()) + .unwrap_or(false); + if !required { + continue; + } + for artifact_name in ["weight", "config"] { + let artifact = model + .get(artifact_name) + .ok_or_else(|| format!("SA3 CUDA {name} {artifact_name} pin is missing"))?; + let hash = artifact + .get("sha256") + .and_then(|value| value.as_str()) + .unwrap_or_default(); + if hash.len() != 64 || !hash.bytes().all(|byte| byte.is_ascii_hexdigit()) { + return Err(format!( + "SA3 CUDA {name} {artifact_name} SHA-256 is incomplete" + )); + } + } } - Path::new(env!("CARGO_MANIFEST_DIR")).join("../scripts/sa3-install.sh") + Ok(()) } /// Shared install state: at most one install runs at a time; the running stage's @@ -526,7 +1036,7 @@ fn sa3_install_script() -> PathBuf { pub(crate) struct InstallShared { busy: AtomicBool, cancelled: AtomicBool, - current_child: Mutex>, + current_child: Mutex>, active: Mutex>, } @@ -552,7 +1062,11 @@ impl InstallManager { /// The in-flight install `(family, name)`, for `model_status`. `name` is the /// model for Magenta, `""` for SA3. pub fn active_install(&self) -> Option<(Family, String)> { - self.shared.active.lock().unwrap_or_else(|p| p.into_inner()).clone() + self.shared + .active + .lock() + .unwrap_or_else(|p| p.into_inner()) + .clone() } /// Start an install on a background thread; progress arrives as @@ -573,13 +1087,13 @@ impl InstallManager { return Err(format!("unknown model '{name}'")); } let model = name.clone(); - self.start(app, family, name, move |progress, shared| { - install_magenta(progress, shared, &model) + self.start(app, family, name, move |app, progress, shared| { + install_magenta(app, progress, shared, &model) }) } // `model://progress` carries the model name for Magenta, "" for SA3. - Family::Sa3 => self.start(app, family, String::new(), move |progress, shared| { - install_sa3(progress, shared, update) + Family::Sa3 => self.start(app, family, String::new(), move |app, progress, shared| { + install_sa3(app, progress, shared, update) }), Family::Lora => Err("adapters are installed via install_lora".into()), } @@ -594,7 +1108,7 @@ impl InstallManager { spec: crate::loras::ImportSpec, ) -> Result<(), String> { let name = spec.display_name()?; - self.start(app, Family::Lora, name, move |progress, shared| { + self.start(app, Family::Lora, name, move |_app, progress, shared| { crate::loras::install(progress, shared, &spec) }) } @@ -607,7 +1121,7 @@ impl InstallManager { app: AppHandle, family: Family, name: String, - job: impl FnOnce(&Progress, &InstallShared) -> Result<(), String> + Send + 'static, + job: impl FnOnce(&AppHandle, &Progress, &InstallShared) -> Result<(), String> + Send + 'static, ) -> Result<(), String> { if self.shared.busy.swap(true, Ordering::AcqRel) { return Err("an install is already running".into()); @@ -623,8 +1137,11 @@ impl InstallManager { let progress = move |stage: &str, message: Option, file: Option| { emit(&progress_app, family, &name, stage, message, file); }; - let result = job(&progress, &shared); - *shared.current_child.lock().unwrap_or_else(|p| p.into_inner()) = None; + let result = job(&app, &progress, &shared); + *shared + .current_child + .lock() + .unwrap_or_else(|p| p.into_inner()) = None; match result { Ok(()) => emit(&app, family, "", "done", None, None), // A user cancel is a clean stop, not a failure — the UI must @@ -653,8 +1170,14 @@ impl InstallManager { /// Cancel an in-flight install: flag it and kill the running stage's child. pub fn cancel(&self) { self.shared.cancelled.store(true, Ordering::Release); - if let Some(mut child) = self.shared.current_child.lock().unwrap_or_else(|p| p.into_inner()).take() { - kill_group(&mut child); + if let Some(mut child) = self + .shared + .current_child + .lock() + .unwrap_or_else(|p| p.into_inner()) + .take() + { + let _ = child.force_kill(); } } @@ -677,36 +1200,6 @@ impl Drop for InstallManager { } } -/// Kill the child's whole process group (it was spawned as a group leader, so its -/// pgid equals its pid), then reap the leader. This takes down `uv run`'s python -/// grandchild and any shell descendants — killing only the leader would orphan -/// them and leave the download running. The grandchildren are reparented to -/// launchd, which reaps them. -fn kill_group(child: &mut Child) { - let group = -(child.id() as libc::pid_t); - // SAFETY: `kill(2)` with a negative pid signals the process group; the pid is a - // live child we own. A failure (already-exited group) is ignored. - unsafe { - libc::kill(group, libc::SIGKILL); - } - let _ = child.wait(); - // A descendant that was mid-fork during the sweep can miss the signal — but - // it is still IN the group (fork inherits the pgid), so re-sweep until the - // group has no members (signal 0 probes without signalling). One pass - // suffices in practice; the bound keeps a stray unkillable member from - // spinning this thread forever. - for _ in 0..100 { - // SAFETY: as above; signal 0 sends nothing. - if unsafe { libc::kill(group, 0) } == -1 { - break; - } - unsafe { - libc::kill(group, libc::SIGKILL); - } - std::thread::sleep(std::time::Duration::from_millis(10)); - } -} - pub(crate) fn cancelled(shared: &InstallShared) -> Result<(), String> { if shared.cancelled.load(Ordering::Acquire) { Err("cancelled".into()) @@ -715,6 +1208,30 @@ pub(crate) fn cancelled(shared: &InstallShared) -> Result<(), String> { } } +pub(crate) fn is_cancelled(shared: &InstallShared) -> bool { + shared.cancelled.load(Ordering::Acquire) +} + +/// Publish a newly spawned child to cancellation and close the spawn/park race. +/// +/// `cancel()` stores the flag before taking `current_child`. If cancellation +/// lands after the process is spawned but before this lock is acquired, the +/// second flag check below takes responsibility for terminating the child. +fn park_child(shared: &InstallShared, child: SupervisedChild) -> Result<(), String> { + let mut current = shared + .current_child + .lock() + .unwrap_or_else(|p| p.into_inner()); + *current = Some(child); + if shared.cancelled.load(Ordering::Acquire) { + let mut child = current.take().expect("newly parked child is present"); + drop(current); + let _ = child.force_kill(); + return Err("cancelled".into()); + } + Ok(()) +} + /// Run `cmd` to completion, feeding each stdout line to `on_line` and draining /// stderr to the app log (so the pipe cannot fill and deadlock). Parks the child /// in `shared` so cancel/shutdown can kill it. Returns an error on a non-zero @@ -725,45 +1242,64 @@ pub(crate) fn stream_child( mut cmd: Command, mut on_line: impl FnMut(&str), ) -> Result<(), String> { + cancelled(shared)?; cmd.stdout(Stdio::piped()).stderr(Stdio::piped()); - // Run the child as its own process-group leader so a cancel can kill the whole - // tree: the install runs `uv run python …` (and sa3-install.sh shells further), - // and killing only the immediate child would orphan the real worker — which - // keeps the stdout pipe open and wedges the reader below. See `kill_group`. - { - use std::os::unix::process::CommandExt; - cmd.process_group(0); - } - let mut child = cmd.spawn().map_err(|e| format!("{label}: cannot spawn ({e})"))?; - let stdout = child.stdout.take().expect("piped stdout"); - let stderr = child.stderr.take().expect("piped stderr"); - *shared.current_child.lock().unwrap_or_else(|p| p.into_inner()) = Some(child); + let mut child = crate::child_process::spawn_grouped(&mut cmd) + .map_err(|error| sanitize_diagnostic(&format!("{label}: cannot spawn ({error})")))?; + let stdout = child.take_stdout().expect("piped stdout"); + let stderr = child.take_stderr().expect("piped stderr"); + park_child(shared, child)?; let drain_label = label.to_string(); let stderr_drain = std::thread::spawn(move || { - for line in BufReader::new(stderr).lines().map_while(Result::ok) { - eprintln!("lsdj-app: {drain_label}: {line}"); + let mut diagnostics = DiagnosticTail::default(); + if let Err(error) = read_bounded_lines(stderr, |line| { + diagnostics.push(line); + }) { + diagnostics.push(&format!("stderr read failed: {error}")); } + (drain_label, diagnostics) }); - for line in BufReader::new(stdout).lines().map_while(Result::ok) { - on_line(&line); - } - let _ = stderr_drain.join(); + let stdout_result = read_bounded_lines(stdout, |line| { + on_line(line); + }); + let (drain_label, diagnostics) = stderr_drain + .join() + .unwrap_or_else(|_| (label.to_string(), DiagnosticTail::default())); // Reclaim the child to read its exit status; cancel() may have taken it. - let Some(mut child) = shared.current_child.lock().unwrap_or_else(|p| p.into_inner()).take() else { + let Some(mut child) = shared + .current_child + .lock() + .unwrap_or_else(|p| p.into_inner()) + .take() + else { return Err("cancelled".into()); }; - let status = child.wait().map_err(|e| format!("{label}: wait failed ({e})"))?; + let status = child + .wait() + .map_err(|error| sanitize_diagnostic(&format!("{label}: wait failed ({error})")))?; cancelled(shared)?; + stdout_result + .map_err(|error| sanitize_diagnostic(&format!("{label}: stdout read failed ({error})")))?; if !status.success() { - return Err(format!("{label}: exited with {status}")); + let detail = diagnostics.render(); + let message = if detail.is_empty() { + format!("{label}: exited with {status}") + } else { + format!("{label}: exited with {status}; diagnostics:\n{detail}") + }; + return Err(sanitize_diagnostic(&message)); + } + if !diagnostics.is_empty() { + eprintln!("lsdj-app: {drain_label}: {}", diagnostics.render()); } Ok(()) } /// One parsed line of the sidecar's JSON progress contract. #[derive(Deserialize)] +#[cfg(any(not(feature = "managed-runtime"), all(test, unix)))] struct SidecarLine { event: String, file: Option, @@ -776,148 +1312,1976 @@ struct SidecarLine { /// emit; tests record the events while the install actually runs. pub(crate) type Progress = dyn Fn(&str, Option, Option); -fn install_magenta(progress: &Progress, shared: &InstallShared, name: &str) -> Result<(), String> { - progress("download", None, None); - let mut cmd = crate::sidecar::sidecar_base_command().map_err(|e| e.to_string())?; - if !resources_present() { - // Fetch the shared resources first, in the same child — without them the - // downloaded model cannot load. - cmd.arg("--init-resources"); +#[cfg(feature = "managed-runtime")] +#[derive(Clone, Copy)] +struct Mrt2Lifecycle { + render_was_warm: bool, +} + +#[cfg(feature = "managed-runtime")] +fn quiesce_mrt2_services(app: &AppHandle) -> Result { + let gateway = app.state::(); + let render_was_warm = gateway + .quiesce() + .map_err(|error| format!("cannot quiesce Magenta renderer before promotion: {error}"))?; + if let Err(error) = app.state::().quiesce_shared() { + // No rename has happened. Restore the still-current verified render + // generation before returning the deck teardown error. + return Err(mrt2_quiesce_recovery_error( + error, + gateway.resume(render_was_warm), + )); } - cmd.args(["--download-model", name]); - run_download(progress, shared, cmd) + Ok(Mrt2Lifecycle { render_was_warm }) } -/// Spawn the download tooling and map its JSON progress contract onto the sink. -/// Takes the fully-built command so the spawn+parse path is testable against a -/// stub without mutating the process environment. -fn run_download(progress: &Progress, shared: &InstallShared, cmd: Command) -> Result<(), String> { - let mut last_error: Option = None; - let result = stream_child(shared, "download-model", cmd, |line| { - let Ok(parsed) = serde_json::from_str::(line) else { - return; - }; - match parsed.event.as_str() { - // The keyed stage label is the user-facing wording; only the file path - // (data) rides along. Upstream `message`/`done` lines are not shown. - "stage" => progress(parsed.stage.as_deref().unwrap_or("download"), None, None), - "file" => progress("download", None, parsed.file), - "error" => last_error = parsed.message.or(Some("download failed".into())), - _ => {} - } - }); - // Prefer the tooling's own error message over the generic non-zero exit. - result.map_err(|exit_err| last_error.unwrap_or(exit_err)) -} - -fn install_sa3(progress: &Progress, shared: &InstallShared, update: bool) -> Result<(), String> { - let (_state, existing) = sa3_status(); - let (checkout, fetched) = match existing { - // Resume an existing checkout (venv_missing / not_warmed / ready) — the - // installer is idempotent (the `.lsdj-warmed` stamp gates re-warming). An - // explicit update instead re-fetches the pinned source, swapping it in - // (fetch_sa3_checkout backs up and restores on failure) and re-building. - Some(checkout) if !update => (checkout, false), - _ => (fetch_sa3_checkout(progress, shared)?, true), - }; - cancelled(shared)?; - run_sa3_installer(progress, shared, &checkout, &sa3_install_script())?; - if fetched { - // Stamp the fetched source so a later pin bump shows as "update available". - write_source_stamp(&checkout, &pinned_source()); +#[cfg(feature = "managed-runtime")] +fn mrt2_quiesce_recovery_error(shared_error: String, gateway_resume: Result<(), String>) -> String { + let primary = format!( + "cannot quiesce realtime MRT2 decks before promotion: {shared_error}" + ); + match gateway_resume { + Ok(()) => primary, + Err(resume_error) => format!( + "{primary}; the Magenta renderer also could not resume: {resume_error}" + ), } - Ok(()) } -/// Run the SA3 build+warm script (`install.sh -y --python 3.11` + warm). Takes -/// the checkout + script paths so the run is testable against a stub without -/// touching the process environment. -fn run_sa3_installer( - progress: &Progress, - shared: &InstallShared, - checkout: &Path, - script: &Path, +#[cfg(feature = "managed-runtime")] +fn resume_mrt2_services(app: &AppHandle, lifecycle: Mrt2Lifecycle) -> Result<(), String> { + // Restore a previously warm renderer completely before launching the deck + // process, avoiding concurrent cold model allocations during promotion. + app.state::() + .resume(lifecycle.render_was_warm) + .map_err(|error| format!("cannot resume Magenta renderer: {error}"))?; + app.state::() + .resume_shared() + .map_err(|error| format!("cannot resume realtime MRT2 decks: {error}")) +} + +/// Promotion owns the commit/rollback result, but service restart is part of +/// operational success. A failed promotion still attempts to restore the prior +/// verified generation, and reports both causes if that recovery also fails. +#[cfg(feature = "managed-runtime")] +fn finish_promotion( + label: &str, + promoted: Result<(), String>, + resumed: Result<(), String>, ) -> Result<(), String> { - progress("install", None, None); - let mut cmd = Command::new("bash"); - cmd.arg(script).arg(checkout); - // The script's stdout (warming N/3 …) is English and un-keyed; the "install" - // stage label carries the wording. stderr is drained to the app log already. - stream_child(shared, "sa3-install", cmd, |_line| {}) + match (promoted, resumed) { + (Ok(()), Ok(())) => Ok(()), + (Ok(()), Err(resume)) => Err(format!( + "{label} was promoted but its service could not resume: {resume}" + )), + (Err(promote), Ok(())) => Err(promote), + (Err(promote), Err(resume)) => Err(format!( + "{promote}; the previous {label} service also could not resume: {resume}" + )), + } } -/// Fetch + extract the pinned SA3 source into the conventional home, returning -/// the checkout root. Extracts to a temp dir and renames the single archive top -/// dir into place, so a partial fetch never leaves a broken checkout. -fn fetch_sa3_checkout(progress: &Progress, shared: &InstallShared) -> Result { - let pin = sa3_pin(); - let home = sa3_app_home(); - let url = format!("{}/archive/{}.tar.gz", pin.repo.trim_end_matches('/'), pin.commit); - let tmp = std::env::temp_dir().join(format!("lsdj-sa3-{}", pin.commit)); - let _ = std::fs::remove_dir_all(&tmp); - std::fs::create_dir_all(&tmp).map_err(|e| format!("cannot create temp dir: {e}"))?; - let tarball = tmp.join("sa3.tar.gz"); - - progress("fetch", None, None); - let mut curl = Command::new("curl"); - curl.args(["-fLsS", "-o"]).arg(&tarball).arg(&url); - stream_child(shared, "curl", curl, |_| {})?; - cancelled(shared)?; +/// Execute the rename window in its only safe order. Keeping the sequence in +/// one small, injectable helper makes the Windows "dead before rename" rule and +/// rollback restart behavior testable without Python, CUDA, or model files. +#[cfg(feature = "managed-runtime")] +fn run_promotion_lifecycle( + label: &str, + quiesce: impl FnOnce() -> Result, + promote: impl FnOnce() -> Result<(), String>, + resume: impl FnOnce(L) -> Result<(), String>, +) -> Result<(), String> { + let lifecycle = quiesce()?; + let promoted = promote(); + let resumed = resume(lifecycle); + finish_promotion(label, promoted, resumed) +} - progress("extract", None, None); - let extract = tmp.join("extract"); - std::fs::create_dir_all(&extract).map_err(|e| format!("cannot create extract dir: {e}"))?; - let mut tar = Command::new("tar"); - tar.arg("-xzf").arg(&tarball).arg("-C").arg(&extract); - stream_child(shared, "tar", tar, |_| {})?; +fn install_magenta( + app: &AppHandle, + progress: &Progress, + shared: &InstallShared, + name: &str, +) -> Result<(), String> { + #[cfg(feature = "managed-runtime")] + { + install_mrt2_managed(app, progress, shared, name) + } - let top = single_subdir(&extract).ok_or("unexpected SA3 archive layout")?; - if let Some(parent) = home.parent() { - std::fs::create_dir_all(parent).map_err(|e| format!("cannot create SA3 home: {e}"))?; - } - // Swap the new checkout in non-destructively: if anything is already at `home` - // (a stale partial — a valid one wouldn't reach here), move it aside first and - // restore it if the rename fails, so a failure never destroys it irrecoverably. - if home.exists() { - let backup = home.with_extension("old"); - let _ = std::fs::remove_dir_all(&backup); - std::fs::rename(&home, &backup).map_err(|e| format!("cannot stage SA3 home: {e}"))?; - if let Err(e) = std::fs::rename(&top, &home) { - let _ = std::fs::rename(&backup, &home); - return Err(format!("cannot place SA3 checkout: {e}")); - } - let _ = std::fs::remove_dir_all(&backup); - } else { - std::fs::rename(&top, &home).map_err(|e| format!("cannot place SA3 checkout: {e}"))?; + #[cfg(not(feature = "managed-runtime"))] + { + let _ = app; + progress("download", None, None); + let mut cmd = crate::sidecar::sidecar_base_command().map_err(|e| e.to_string())?; + if !resources_present() { + // Fetch the shared resources first, in the same child — without them the + // downloaded model cannot load. + cmd.arg("--init-resources"); + } + cmd.args(["--download-model", name]); + run_download(progress, shared, cmd) } - let _ = std::fs::remove_dir_all(&tmp); - Ok(home) } -/// The single immediate subdirectory of `dir`, or `None` if there is not exactly -/// one (a GitHub source archive extracts to one `-/` dir). -fn single_subdir(dir: &Path) -> Option { - let mut found: Option = None; - for entry in std::fs::read_dir(dir).ok()?.flatten() { - if entry.file_type().ok()?.is_dir() { - if found.is_some() { - return None; +#[cfg(feature = "managed-runtime")] +fn install_mrt2_managed( + app: &AppHandle, + progress: &Progress, + shared: &InstallShared, + name: &str, +) -> Result<(), String> { + let pin = mrt2_pin(); + validate_mrt2_pin(&pin)?; + let target = host_installer_target()?; + if !matches!( + target, + "x86_64-unknown-linux-gnu" | "x86_64-pc-windows-msvc" + ) { + return Err("the managed PyTorch MRT2 runtime is Linux/Windows x86-64 only".into()); + } + let snapshot = pin.models.get(name).ok_or("MRT2 model pin is missing")?; + let python = pin + .runtime + .python + .iter() + .find(|item| item.target == target) + .ok_or("MRT2 Python pin is missing")?; + let uv = pin + .runtime + .uv + .iter() + .find(|item| item.target == target) + .ok_or("MRT2 uv pin is missing")?; + let paths = crate::platform_paths::get(); + let home = + crate::managed_runtime::service_root(paths.assets(), crate::managed_runtime::Service::Mrt2); + let work = paths.staging().join("mrt2").join(target); + let candidate = work.join("candidate"); + let backup = paths.staging().join("mrt2-previous"); + std::fs::create_dir_all(&work) + .map_err(|error| format!("cannot create MRT2 staging: {error}"))?; + if let Some(parent) = home.parent() { + std::fs::create_dir_all(parent) + .map_err(|error| format!("cannot create MRT2 service root: {error}"))?; + } + let cancelled_now = || is_cancelled(shared); + promotion::recover(&home, &backup, |root| { + validate_mrt2_candidate(root, &pin, name, &cancelled_now) + })?; + if candidate.exists() { + std::fs::remove_dir_all(&candidate) + .map_err(|error| format!("cannot clear interrupted MRT2 candidate: {error}"))?; + } + + let reusable = validate_mrt2_identity(&home, &pin).is_ok() + && crate::managed_runtime::validate_candidate(&home, crate::managed_runtime::Service::Mrt2) + .is_ok(); + if reusable { + copy_regular_tree(&home, &candidate)?; + // `copy_regular_tree` prefers hard links. Unlink every stamp that will + // be rewritten so refreshing the candidate can never truncate the + // still-active generation through a shared inode. + for metadata in [ + crate::managed_runtime::MANIFEST_NAME, + ".lsdj-generation", + MRT2_IDENTITY_STAMP, + ] { + let path = candidate.join(metadata); + if path.exists() { + std::fs::remove_file(path) + .map_err(|error| format!("cannot refresh MRT2 generation metadata: {error}"))?; } - found = Some(entry.path()); } + let backend = candidate.join("lsdj_backend"); + if backend.exists() { + std::fs::remove_dir_all(&backend) + .map_err(|error| format!("cannot refresh MRT2 backend sources: {error}"))?; + } + } else { + std::fs::create_dir_all(&candidate) + .map_err(|error| format!("cannot create MRT2 candidate: {error}"))?; + install_mrt2_runtime(progress, shared, &work, &candidate, target, python, uv)?; } - found + install_backend_sources(&candidate)?; + install_mrt2_snapshot(progress, shared, &work, &candidate, name, snapshot)?; + install_mrt2_snapshot( + progress, + shared, + &work, + &candidate, + "musiccoca", + &pin.processor, + )?; + write_mrt2_identity(&candidate, &pin)?; + materialize_contained_file_links(&candidate)?; + smoke_test_materialized_backend( + shared, + &candidate, + &crate::platform_paths::venv_python(&candidate.join("runtime").join(".venv")), + &["lsdj.gpu_broker", "lsdj.mrt2_pytorch"], + )?; + seal_mrt2_candidate(&candidate, &pin, python)?; + validate_mrt2_candidate(&candidate, &pin, name, &cancelled_now)?; + progress("promote", None, None); + run_promotion_lifecycle( + "MRT2", + || quiesce_mrt2_services(app), + || { + promotion::promote(&candidate, &home, &backup, |root| { + validate_mrt2_candidate(root, &pin, name, &cancelled_now) + }) + }, + |lifecycle| resume_mrt2_services(app, lifecycle), + )?; + let _ = std::fs::remove_dir_all(&work); + Ok(()) } -// --- Tauri commands -------------------------------------------------------- +#[cfg(feature = "managed-runtime")] +fn mrt2_identity(pin: &Mrt2Pin) -> String { + use sha2::{Digest, Sha256}; + let mut digest = Sha256::new(); + digest.update(MRT2_PIN_JSON.as_bytes()); + digest.update(MRT2_WHEEL_PIN_JSON.as_bytes()); + digest.update( + mrt2_lock_for(&crate::managed_runtime::host_target()) + .unwrap_or_default() + .as_bytes(), + ); + for (name, bytes) in BACKEND_SOURCES { + digest.update(name.as_bytes()); + digest.update(bytes); + } + let _ = pin; + format!("{}\n", hex::encode(digest.finalize())) +} -#[tauri::command] -pub fn model_status(installer: tauri::State<'_, InstallManager>) -> ModelStatus { - status(installer.active_install()) +#[cfg(feature = "managed-runtime")] +fn write_mrt2_identity(root: &Path, pin: &Mrt2Pin) -> Result<(), String> { + write_synced( + &root.join(MRT2_IDENTITY_STAMP), + mrt2_identity(pin).as_bytes(), + ) } -#[tauri::command] -pub fn install_model( +#[cfg(feature = "managed-runtime")] +fn validate_mrt2_identity(root: &Path, pin: &Mrt2Pin) -> Result<(), String> { + let path = root.join(MRT2_IDENTITY_STAMP); + let metadata = std::fs::symlink_metadata(&path) + .map_err(|error| format!("MRT2 install identity is missing: {error}"))?; + if metadata.file_type().is_symlink() || !metadata.is_file() { + return Err("MRT2 install identity is not a regular file".into()); + } + if std::fs::read_to_string(path) + .map_err(|error| format!("cannot read MRT2 identity: {error}"))? + != mrt2_identity(pin) + { + return Err("MRT2 install identity is stale".into()); + } + Ok(()) +} + +#[cfg(feature = "managed-runtime")] +fn copy_regular_tree(source: &Path, destination: &Path) -> Result<(), String> { + let metadata = std::fs::symlink_metadata(source) + .map_err(|error| format!("cannot inspect reusable MRT2 runtime: {error}"))?; + if metadata.file_type().is_symlink() || !metadata.is_dir() { + return Err("reusable MRT2 runtime is not a trusted directory".into()); + } + std::fs::create_dir_all(destination) + .map_err(|error| format!("cannot create MRT2 candidate directory: {error}"))?; + for entry in std::fs::read_dir(source) + .map_err(|error| format!("cannot enumerate reusable MRT2 runtime: {error}"))? + { + let entry = + entry.map_err(|error| format!("cannot enumerate reusable MRT2 runtime: {error}"))?; + let from = entry.path(); + let to = destination.join(entry.file_name()); + let metadata = std::fs::symlink_metadata(&from) + .map_err(|error| format!("cannot inspect reusable MRT2 artifact: {error}"))?; + if metadata.file_type().is_symlink() { + return Err("reusable MRT2 runtime contains a symbolic link".into()); + } + if metadata.is_dir() { + copy_regular_tree(&from, &to)?; + } else if metadata.is_file() { + if std::fs::hard_link(&from, &to).is_err() { + std::fs::copy(&from, &to) + .map_err(|error| format!("cannot copy reusable MRT2 artifact: {error}"))?; + } + } else { + return Err("reusable MRT2 runtime contains an unsupported entry".into()); + } + } + Ok(()) +} + +#[cfg(feature = "managed-runtime")] +fn download_mrt2_wheelhouse( + progress: &Progress, + shared: &InstallShared, + work: &Path, + target: &str, +) -> Result<(PathBuf, Vec), String> { + let pins = mrt2_wheel_pins_for(target)?; + let directory = work.join("wheelhouse"); + if directory.exists() { + std::fs::remove_dir_all(&directory) + .map_err(|error| format!("cannot clear interrupted MRT2 wheelhouse: {error}"))?; + } + std::fs::create_dir_all(&directory) + .map_err(|error| format!("cannot create MRT2 wheelhouse: {error}"))?; + let client = installer_client()?; + for pin in &pins { + cancelled(shared)?; + progress("fetch", None, Some(pin.filename.clone())); + download_verified( + &client, + &pin.artifact, + &directory.join(&pin.filename), + None, + || is_cancelled(shared), + )?; + } + verify_wheelhouse(&directory, &pins, &|| is_cancelled(shared))?; + Ok((directory, pins)) +} + +#[cfg(feature = "managed-runtime")] +fn install_mrt2_runtime( + progress: &Progress, + shared: &InstallShared, + work: &Path, + candidate: &Path, + target: &str, + python: &PythonPin, + uv: &UvPin, +) -> Result<(), String> { + let client = installer_client()?; + let blobs = work.join("blobs"); + std::fs::create_dir_all(&blobs) + .map_err(|error| format!("cannot create MRT2 blob staging: {error}"))?; + progress("fetch", None, Some(format!("Python {}", python.version))); + let python_archive = blobs.join("python.tar.gz"); + download_verified( + &client, + &python.archive.artifact, + &python_archive, + None, + || is_cancelled(shared), + )?; + let python_root = candidate.join("runtime").join(".python"); + python.archive.extract( + std::fs::File::open(&python_archive) + .map_err(|error| format!("cannot open MRT2 Python: {error}"))?, + &python_root, + true, + &|| is_cancelled(shared), + )?; + let runtime_python = python_root.join(&python.executable); + verify_python_version(shared, &runtime_python, &python.version)?; + + progress("fetch", None, Some(format!("uv {}", uv.version))); + let uv_archive = blobs.join(if uv.archive.archive_format == "zip" { + "uv.zip" + } else { + "uv.tar.gz" + }); + download_verified(&client, &uv.archive.artifact, &uv_archive, None, || { + is_cancelled(shared) + })?; + let uv_root = work.join("uv"); + if uv_root.exists() { + std::fs::remove_dir_all(&uv_root) + .map_err(|error| format!("cannot clear MRT2 uv staging: {error}"))?; + } + uv.archive.extract( + std::fs::File::open(&uv_archive) + .map_err(|error| format!("cannot open MRT2 uv: {error}"))?, + &uv_root, + false, + &|| is_cancelled(shared), + )?; + let uv_executable = uv_root.join(&uv.executable); + verify_uv_version(shared, &uv_executable, &uv.version)?; + let (wheelhouse, wheels) = download_mrt2_wheelhouse(progress, shared, work, target)?; + + let runtime = candidate.join("runtime"); + let venv = runtime.join(".venv"); + let cache = work.join("uv-cache"); + let mut create = uv_command(&uv_executable, &runtime, &cache); + create + .args([ + "venv", + "--relocatable", + "--no-managed-python", + "--no-python-downloads", + "--no-config", + "--link-mode", + "copy", + "--python", + ]) + .arg(&runtime_python) + .arg(&venv); + stream_child(shared, "mrt2-venv", create, |_| {})?; + let venv_python = crate::platform_paths::venv_python(&venv); + let mut install = uv_command(&uv_executable, &runtime, &cache); + install + .args(["pip", "install", "--python"]) + .arg(&venv_python); + configure_portable_install(&mut install, &wheelhouse, &wheels, &|| is_cancelled(shared))?; + stream_child(shared, "mrt2-dependencies", install, |_| {})?; + let mut check = Command::new(&venv_python); + check + .current_dir(&runtime) + .env("HF_HUB_OFFLINE", "1") + .args([ + "-c", + "import torch, transformers, safetensors, sentencepiece, resampy", + ]); + stream_child(shared, "mrt2-runtime-check", check, |_| {}) +} + +#[cfg(feature = "managed-runtime")] +fn install_mrt2_snapshot( + progress: &Progress, + shared: &InstallShared, + work: &Path, + candidate: &Path, + install_name: &str, + snapshot: &SnapshotPin, +) -> Result<(), String> { + let client = installer_client()?; + let token = std::env::var("HF_TOKEN") + .ok() + .or_else(|| std::env::var("HUGGING_FACE_HUB_TOKEN").ok()); + for file in &snapshot.files { + cancelled(shared)?; + let artifact = snapshot_artifact(snapshot, file)?; + progress("fetch", None, Some(format!("{install_name}/{}", file.path))); + let staged = work + .join("blobs") + .join("snapshots") + .join(install_name) + .join(&file.path); + download_verified(&client, &artifact, &staged, token.as_deref(), || { + is_cancelled(shared) + })?; + let destination = candidate.join("models").join(install_name).join(&file.path); + link_or_copy_verified(&staged, &destination, &artifact, &|| is_cancelled(shared))?; + } + Ok(()) +} + +#[cfg(feature = "managed-runtime")] +fn verify_mrt2_snapshot( + root: &Path, + install_name: &str, + snapshot: &SnapshotPin, + is_cancelled: &dyn Fn() -> bool, +) -> Result<(), String> { + for file in &snapshot.files { + verify_file_cancellable( + &root.join("models").join(install_name).join(&file.path), + &snapshot_artifact(snapshot, file)?, + is_cancelled, + ) + .map_err(|error| { + format!( + "MRT2 {install_name}/{} failed integrity: {error}", + file.path + ) + })?; + } + Ok(()) +} + +#[cfg(feature = "managed-runtime")] +fn validate_mrt2_candidate( + root: &Path, + pin: &Mrt2Pin, + model: &str, + is_cancelled: &dyn Fn() -> bool, +) -> Result<(), String> { + validate_mrt2_identity(root, pin)?; + verify_mrt2_snapshot( + root, + model, + pin.models.get(model).ok_or("MRT2 model pin is missing")?, + is_cancelled, + )?; + verify_mrt2_snapshot(root, "musiccoca", &pin.processor, is_cancelled)?; + crate::managed_runtime::validate_candidate(root, crate::managed_runtime::Service::Mrt2) +} + +/// Spawn the download tooling and map its JSON progress contract onto the sink. +/// Takes the fully-built command so the spawn+parse path is testable against a +/// stub without mutating the process environment. +#[cfg(any(not(feature = "managed-runtime"), all(test, unix)))] +fn run_download(progress: &Progress, shared: &InstallShared, cmd: Command) -> Result<(), String> { + let mut last_error: Option = None; + let result = stream_child(shared, "download-model", cmd, |line| { + let Ok(parsed) = serde_json::from_str::(line) else { + return; + }; + match parsed.event.as_str() { + // The keyed stage label is the user-facing wording; only the file path + // (data) rides along. Upstream `message`/`done` lines are not shown. + "stage" => progress(parsed.stage.as_deref().unwrap_or("download"), None, None), + "file" => progress("download", None, parsed.file), + "error" => { + last_error = Some(sanitize_diagnostic( + parsed.message.as_deref().unwrap_or("download failed"), + )); + } + _ => {} + } + }); + // Prefer the tooling's own error message over the generic non-zero exit. + result.map_err(|exit_err| sanitize_diagnostic(&last_error.unwrap_or(exit_err))) +} + +fn install_sa3( + app: &AppHandle, + progress: &Progress, + shared: &InstallShared, + _update: bool, +) -> Result<(), String> { + let pin = sa3_pin(); + validate_sa3_pin(&pin)?; + let backend = host_sa3_backend()?; + if backend == Sa3Backend::Tflite { + validate_tflite_pin(&tflite_pin(), &pin)?; + } + let uv = host_uv_pin(&pin)?; + let python = host_python_pin(&pin)?; + let staging = crate::platform_paths::get().staging().join("sa3"); + let work = staging.join(&pin.commit); + let candidate = work.join("candidate"); + let backup = staging.join("previous"); + let home = sa3_app_home(); + std::fs::create_dir_all(&work) + .map_err(|error| format!("cannot create SA3 staging root: {error}"))?; + if let Some(parent) = home.parent() { + std::fs::create_dir_all(parent) + .map_err(|error| format!("cannot create SA3 asset root: {error}"))?; + } + + // Finish (or roll back) a prior process that stopped between promotion + // renames before doing any network work. + let install_cancelled = || shared.cancelled.load(Ordering::Acquire); + promotion::recover(&home, &backup, |path| { + validate_sa3_install_cancellable(path, &pin, backend, &install_cancelled) + })?; + cancelled(shared)?; + build_sa3_candidate( + progress, + shared, + &pin, + backend, + (uv, python), + &work, + &candidate, + )?; + cancelled(shared)?; + progress("promote", None, None); + #[cfg(feature = "managed-runtime")] + let service_was_running = app + .state::() + .quiesce() + .map_err(|error| format!("cannot quiesce SA3 before promotion: {error}"))?; + let promoted = promotion::promote(&candidate, &home, &backup, |path| { + validate_sa3_install_cancellable(path, &pin, backend, &install_cancelled) + }); + #[cfg(feature = "managed-runtime")] + let resumed = app + .state::() + .resume() + .map_err(|error| format!("cannot resume SA3 after promotion: {error}")); + #[cfg(feature = "managed-runtime")] + let resumed = if promoted.is_err() && !service_was_running { + // First install had no prior service to restore; keep the promotion + // cause authoritative instead of appending the expected "not installed" + // resume failure. + Ok(()) + } else { + resumed + }; + #[cfg(feature = "managed-runtime")] + finish_promotion("SA3", promoted, resumed)?; + #[cfg(not(feature = "managed-runtime"))] + promoted?; + #[cfg(not(feature = "managed-runtime"))] + let _ = app; + // Verified blobs are hard-linked into the promoted tree. Removing retry + // state here reclaims only the staging directory entries, not model bytes. + let _ = std::fs::remove_dir_all(&work); + Ok(()) +} + +fn build_sa3_candidate( + progress: &Progress, + shared: &InstallShared, + pin: &Sa3Pin, + backend: Sa3Backend, + runtime_pins: (&UvPin, &PythonPin), + work: &Path, + candidate: &Path, +) -> Result<(), String> { + let (uv, python) = runtime_pins; + let blobs = work.join("blobs"); + std::fs::create_dir_all(&blobs) + .map_err(|error| format!("cannot create SA3 blob staging: {error}"))?; + if candidate.exists() { + std::fs::remove_dir_all(candidate) + .map_err(|error| format!("cannot clear interrupted SA3 candidate: {error}"))?; + } + + let client = installer_client()?; + progress("fetch", None, Some("Stable Audio 3 source".into())); + let source_archive = blobs.join("stable-audio-3.tar.gz"); + download_verified(&client, &pin.source.artifact, &source_archive, None, || { + shared.cancelled.load(Ordering::Acquire) + })?; + cancelled(shared)?; + + progress("extract", None, None); + let source = std::fs::File::open(&source_archive) + .map_err(|error| format!("cannot open verified SA3 source: {error}"))?; + pin.source.extract(source, candidate, false, &|| { + shared.cancelled.load(Ordering::Acquire) + })?; + validate_source_layout(candidate, backend)?; + cancelled(shared)?; + + progress("fetch", None, Some(format!("uv {}", uv.version))); + let uv_archive = blobs.join("uv.tar.gz"); + download_verified(&client, &uv.archive.artifact, &uv_archive, None, || { + shared.cancelled.load(Ordering::Acquire) + })?; + let uv_dir = work.join("uv"); + if uv_dir.exists() { + std::fs::remove_dir_all(&uv_dir) + .map_err(|error| format!("cannot clear interrupted uv runtime: {error}"))?; + } + let source = std::fs::File::open(&uv_archive) + .map_err(|error| format!("cannot open verified uv archive: {error}"))?; + uv.archive.extract(source, &uv_dir, false, &|| { + shared.cancelled.load(Ordering::Acquire) + })?; + let uv_executable = uv_dir.join(&uv.executable); + if !uv_executable.is_file() { + return Err("verified uv archive did not contain the pinned executable".into()); + } + verify_uv_version(shared, &uv_executable, &uv.version)?; + + progress("fetch", None, Some(format!("Python {}", python.version))); + let python_archive = blobs.join("python.tar.gz"); + download_verified( + &client, + &python.archive.artifact, + &python_archive, + None, + || shared.cancelled.load(Ordering::Acquire), + )?; + let runtime = backend.runtime_dir(candidate); + let python_dir = runtime.join(".python"); + let source = std::fs::File::open(&python_archive) + .map_err(|error| format!("cannot open verified Python archive: {error}"))?; + python.archive.extract(source, &python_dir, true, &|| { + shared.cancelled.load(Ordering::Acquire) + })?; + let python_executable = python_dir.join(&python.executable); + if !python_executable.is_file() { + return Err("verified Python archive did not contain the pinned executable".into()); + } + verify_python_version(shared, &python_executable, &python.version)?; + + let hf_token = std::env::var("HF_TOKEN") + .ok() + .or_else(|| std::env::var("HUGGING_FACE_HUB_TOKEN").ok()); + install_sa3_models( + progress, + shared, + pin, + backend, + &blobs.join("models"), + candidate, + hf_token.as_deref(), + )?; + + let portable_wheels = if backend == Sa3Backend::Tflite { + Some(download_wheelhouse(progress, shared, work)?) + } else { + None + }; + + progress("install", None, None); + let (requirements_name, requirements_lock) = backend.requirements(); + let requirements = runtime.join(requirements_name); + write_synced(&requirements, requirements_lock.as_bytes())?; + run_sa3_setup( + shared, + &uv_executable, + &python_executable, + &runtime, + work, + requirements_name, + portable_wheels + .as_ref() + .map(|(directory, pins)| (directory.as_path(), pins.as_slice())), + )?; + warm_sa3(shared, &runtime, work, backend)?; + if backend == Sa3Backend::Tflite { + install_backend_sources(candidate)?; + } + write_source_stamp(candidate, &pinned_source())?; + if backend == Sa3Backend::Tflite { + write_tflite_provenance(&runtime, &tflite_pin())?; + } + write_synced( + &candidate.join(INSTALL_MANIFEST_STAMP), + install_manifest(backend).as_bytes(), + )?; + if backend == Sa3Backend::Tflite { + materialize_contained_file_links(candidate)?; + smoke_test_materialized_backend( + shared, + candidate, + &crate::platform_paths::venv_python(&runtime.join(".venv")), + &["lsdj.sa3_cuda", "lsdj.sa3"], + )?; + seal_sa3_candidate(candidate, pin, python)?; + } + validate_sa3_install_cancellable(candidate, pin, backend, &|| { + shared.cancelled.load(Ordering::Acquire) + }) +} + +fn install_sa3_models( + progress: &Progress, + shared: &InstallShared, + pin: &Sa3Pin, + backend: Sa3Backend, + model_blobs: &Path, + candidate: &Path, + hf_token: Option<&str>, +) -> Result<(), String> { + let client = installer_client()?; + let artifacts = model_artifacts(pin, backend)?; + for (install_path, artifact) in artifacts { + cancelled(shared)?; + progress( + "fetch", + None, + Some(install_path.to_string_lossy().into_owned()), + ); + let staged = model_blobs.join(&install_path); + download_verified(&client, &artifact, &staged, hf_token, || { + shared.cancelled.load(Ordering::Acquire) + })?; + link_or_copy_verified(&staged, &candidate.join(&install_path), &artifact, &|| { + shared.cancelled.load(Ordering::Acquire) + })?; + } + Ok(()) +} + +fn model_artifacts( + pin: &Sa3Pin, + backend: Sa3Backend, +) -> Result, String> { + match backend { + Sa3Backend::Mlx => { + let mut artifacts = BTreeMap::new(); + for model in &pin.models.artifacts { + let filename = model.filename()?; + let install_path = PathBuf::from("optimized/mlx/models/mlx").join(filename); + artifacts.insert(install_path, model.artifact(&pin.models)?); + } + Ok(artifacts) + } + Sa3Backend::Tflite => tflite_artifacts(&tflite_pin()), + } +} + +fn model_download_bytes(pin: &Sa3Pin, backend: Sa3Backend) -> Result { + model_artifacts(pin, backend)? + .values() + .try_fold(0u64, |total, artifact| { + total + .checked_add(artifact.size) + .ok_or_else(|| "SA3 model download size overflow".to_string()) + }) +} + +fn tflite_artifacts(pin: &TflitePin) -> Result, String> { + let mut artifacts = BTreeMap::new(); + for model in pin + .models + .shared + .iter() + .chain(pin.models.bundles.values().flatten()) + { + let install_path = checked_install_path(&model.install_path)?; + let artifact = model.artifact(&pin.models); + artifact.validate()?; + if let Some(existing) = artifacts.insert(install_path.clone(), artifact.clone()) { + if existing.url != artifact.url + || existing.sha256 != artifact.sha256 + || existing.size != artifact.size + { + return Err(format!( + "TFLite model manifest disagrees about {}", + install_path.display() + )); + } + } + } + Ok(artifacts) +} + +fn checked_install_path(value: &str) -> Result { + let path = PathBuf::from(value); + if value.contains('\\') + || path.is_absolute() + || path + .components() + .any(|component| !matches!(component, std::path::Component::Normal(_))) + || !value.starts_with("models/tflite/") + { + return Err("TFLite model install path is unsafe".into()); + } + Ok(PathBuf::from("optimized/tflite").join(path)) +} + +fn normalized_package(value: &str) -> String { + value + .chars() + .map(|character| match character { + '_' | '.' => '-', + other => other.to_ascii_lowercase(), + }) + .collect() +} + +fn wheel_pins_for(target: &str) -> Result, String> { + let manifest = wheel_manifest(); + if manifest.schema_version != 1 || manifest.python != "3.11" { + return Err("TFLite wheel manifest schema/Python version is unsupported".into()); + } + let platform = manifest + .targets + .get(target) + .ok_or_else(|| format!("no pinned TFLite wheel set exists for {target}"))? + .clone(); + let expected_count = match target { + "x86_64-unknown-linux-gnu" => 33, + "x86_64-pc-windows-msvc" => 34, + _ => return Err(format!("TFLite wheels are unsupported for {target}")), + }; + let mut pins = manifest.common; + pins.extend(platform); + if pins.len() != expected_count { + return Err(format!( + "TFLite wheel manifest has {} artifacts; expected {expected_count}", + pins.len() + )); + } + + let lock = TFLITE_REQUIREMENTS_LOCK.to_ascii_lowercase(); + let mut packages = BTreeSet::new(); + let mut filenames = BTreeSet::new(); + for pin in &pins { + pin.artifact.validate()?; + let package = normalized_package(&pin.package); + if package.is_empty() + || pin.version.is_empty() + || !packages.insert(package.clone()) + || !filenames.insert(pin.filename.clone()) + || pin.filename.contains('/') + || pin.filename.contains('\\') + || pin.filename == "." + || pin.filename == ".." + || !pin.filename.ends_with(".whl") + || !pin + .artifact + .url + .starts_with("https://files.pythonhosted.org/") + || !pin.artifact.url.ends_with(&pin.filename) + || !lock.contains(&format!( + "{}=={}", + package, + pin.version.to_ascii_lowercase() + )) + || !lock.contains(&format!("--hash=sha256:{}", pin.artifact.sha256)) + { + return Err(format!( + "TFLite wheel pin is unsafe or disagrees with the lock: {}", + pin.filename + )); + } + } + pins.sort_by(|left, right| left.filename.cmp(&right.filename)); + Ok(pins) +} + +#[cfg(any(feature = "managed-runtime", test))] +fn snapshot_artifact( + snapshot: &SnapshotPin, + file: &SnapshotFilePin, +) -> Result { + if file.path.is_empty() + || file.path.contains('/') + || file.path.contains('\\') + || file.path == "." + || file.path == ".." + { + return Err("MRT2 snapshot artifact path is unsafe".into()); + } + let artifact = PinnedArtifact { + url: format!( + "https://huggingface.co/{}/resolve/{}/{}?download=true", + snapshot.repository, snapshot.revision, file.path + ), + sha256: file.sha256.clone(), + size: file.size, + }; + artifact.validate()?; + Ok(artifact) +} + +#[cfg(any(feature = "managed-runtime", test))] +fn mrt2_lock_for(target: &str) -> Result<&'static str, String> { + match target { + "x86_64-unknown-linux-gnu" => Ok(MRT2_LINUX_LOCK), + "x86_64-pc-windows-msvc" => Ok(MRT2_WINDOWS_LOCK), + _ => Err(format!("MRT2 is unsupported for {target}")), + } +} + +#[cfg(any(feature = "managed-runtime", test))] +fn mrt2_wheel_pins_for(target: &str) -> Result, String> { + let manifest = mrt2_wheel_manifest(); + if manifest.schema_version != 1 || manifest.python != "3.12" { + return Err("MRT2 wheel manifest schema/Python version is unsupported".into()); + } + let pins = manifest + .targets + .get(target) + .ok_or_else(|| format!("no pinned MRT2 wheel set exists for {target}"))? + .clone(); + let expected_count = match target { + "x86_64-unknown-linux-gnu" => 56, + "x86_64-pc-windows-msvc" => 38, + _ => return Err(format!("MRT2 wheels are unsupported for {target}")), + }; + if pins.len() != expected_count { + return Err(format!( + "MRT2 wheel manifest must contain {expected_count} artifacts" + )); + } + let lock = mrt2_lock_for(target)?.to_ascii_lowercase(); + let mut packages = BTreeSet::new(); + let mut filenames = BTreeSet::new(); + for pin in &pins { + pin.artifact.validate()?; + let package = normalized_package(&pin.package); + let trusted_url = pin + .artifact + .url + .starts_with("https://files.pythonhosted.org/") + || pin + .artifact + .url + .starts_with("https://download-r2.pytorch.org/"); + if !trusted_url + || !packages.insert(package.clone()) + || !filenames.insert(pin.filename.clone()) + || pin.filename.contains('/') + || pin.filename.contains('\\') + || !pin.filename.ends_with(".whl") + || !lock.contains(&format!( + "{}=={}", + package, + pin.version.to_ascii_lowercase() + )) + || !lock.contains(&format!("--hash=sha256:{}", pin.artifact.sha256)) + { + return Err(format!( + "MRT2 wheel pin disagrees with its lock: {}", + pin.filename + )); + } + } + Ok(pins) +} + +#[cfg(any(feature = "managed-runtime", test))] +fn validate_snapshot(snapshot: &SnapshotPin, expected: &BTreeSet<&str>) -> Result<(), String> { + if snapshot.revision.len() != 40 + || !snapshot + .revision + .bytes() + .all(|byte| byte.is_ascii_hexdigit()) + || snapshot.repository.split('/').count() != 2 + { + return Err("MRT2 snapshot repository/revision is invalid".into()); + } + let mut actual = BTreeSet::new(); + for file in &snapshot.files { + snapshot_artifact(snapshot, file)?; + if !actual.insert(file.path.as_str()) { + return Err("MRT2 snapshot contains a duplicate artifact".into()); + } + } + if &actual != expected { + return Err("MRT2 snapshot artifact inventory is incomplete".into()); + } + Ok(()) +} + +#[cfg(any(feature = "managed-runtime", test))] +fn validate_mrt2_pin(pin: &Mrt2Pin) -> Result<(), String> { + if pin.schema_version != 1 + || content_digest(MRT2_WHEEL_PIN_JSON.as_bytes()) != pin.runtime.wheel_manifest_sha256 + { + return Err("MRT2 runtime manifest identity is invalid".into()); + } + for target in ["x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"] { + let lock = mrt2_lock_for(target)?; + if pin.runtime.locks.get(target) != Some(&content_digest(lock.as_bytes())) { + return Err("MRT2 dependency lock digest is stale".into()); + } + mrt2_wheel_pins_for(target)?; + let python = pin + .runtime + .python + .iter() + .find(|item| item.target == target) + .ok_or("MRT2 Python runtime pin is missing")?; + let uv = pin + .runtime + .uv + .iter() + .find(|item| item.target == target) + .ok_or("MRT2 uv runtime pin is missing")?; + python.archive.artifact.validate()?; + uv.archive.artifact.validate()?; + if !python.version.starts_with("3.12.") || uv.version != "0.11.7" { + return Err("MRT2 runtime tool version is inconsistent".into()); + } + } + let model_files = [ + "aoti.py", + "codec_shapes.json", + "config.json", + "configuration_magenta_rt2.py", + "cudagraph.py", + "depthformer.py", + "layers.py", + "model.safetensors", + "modeling_magenta_rt2.py", + "musiccoca.py", + "processing_musiccoca.py", + "spectrostream.py", + ] + .into_iter() + .collect(); + if pin.models.keys().cloned().collect::>() + != ["mrt2_base".to_string(), "mrt2_small".to_string()] + .into_iter() + .collect() + { + return Err("MRT2 model catalog is incomplete".into()); + } + for snapshot in pin.models.values() { + validate_snapshot(snapshot, &model_files)?; + } + let processor_files = [ + "mel_params.npz", + "music_encoder.pt", + "quantizer.pt", + "spm.model", + "text_encoder.pt", + ] + .into_iter() + .collect(); + validate_snapshot(&pin.processor, &processor_files) +} + +fn verify_wheelhouse( + directory: &Path, + pins: &[WheelPin], + is_cancelled: &dyn Fn() -> bool, +) -> Result<(), String> { + let expected = pins + .iter() + .map(|pin| pin.filename.clone()) + .collect::>(); + let entries = std::fs::read_dir(directory) + .map_err(|error| format!("cannot enumerate TFLite wheelhouse: {error}"))?; + let mut actual = BTreeSet::new(); + for entry in entries { + let entry = + entry.map_err(|error| format!("cannot enumerate TFLite wheelhouse: {error}"))?; + let metadata = std::fs::symlink_metadata(entry.path()) + .map_err(|error| format!("cannot inspect TFLite wheel: {error}"))?; + let Some(filename) = entry.file_name().to_str().map(str::to_owned) else { + return Err("TFLite wheel filename is not UTF-8".into()); + }; + if metadata.file_type().is_symlink() || !metadata.is_file() || !actual.insert(filename) { + return Err("TFLite wheelhouse contains a non-regular or duplicate entry".into()); + } + } + if actual != expected { + return Err("TFLite wheelhouse contains missing or unexpected artifacts".into()); + } + for pin in pins { + verify_file_cancellable(&directory.join(&pin.filename), &pin.artifact, is_cancelled) + .map_err(|error| { + format!("TFLite wheel {} failed verification: {error}", pin.filename) + })?; + } + Ok(()) +} + +fn download_wheelhouse( + progress: &Progress, + shared: &InstallShared, + work: &Path, +) -> Result<(PathBuf, Vec), String> { + let pins = wheel_pins_for(host_installer_target()?)?; + let directory = work.join("wheelhouse"); + if directory.exists() { + std::fs::remove_dir_all(&directory) + .map_err(|error| format!("cannot clear interrupted TFLite wheelhouse: {error}"))?; + } + std::fs::create_dir_all(&directory) + .map_err(|error| format!("cannot create TFLite wheelhouse: {error}"))?; + let client = installer_client()?; + for pin in &pins { + cancelled(shared)?; + progress("fetch", None, Some(pin.filename.clone())); + download_verified( + &client, + &pin.artifact, + &directory.join(&pin.filename), + None, + || shared.cancelled.load(Ordering::Acquire), + )?; + } + verify_wheelhouse(&directory, &pins, &|| { + shared.cancelled.load(Ordering::Acquire) + })?; + Ok((directory, pins)) +} + +fn verify_uv_version( + shared: &InstallShared, + executable: &Path, + expected: &str, +) -> Result<(), String> { + let mut command = Command::new(executable); + command.arg("--version"); + let mut output = String::new(); + stream_child(shared, "verify-uv", command, |line| { + if output.is_empty() { + output.push_str(line); + } + })?; + let actual = output.split_whitespace().nth(1).unwrap_or_default(); + if actual != expected { + return Err(format!( + "verified uv executable reported version {actual:?}, expected {expected}" + )); + } + Ok(()) +} + +fn verify_python_version( + shared: &InstallShared, + executable: &Path, + expected: &str, +) -> Result<(), String> { + let mut command = Command::new(executable); + command.args(["-c", "import platform; print(platform.python_version())"]); + let mut output = String::new(); + stream_child(shared, "verify-python", command, |line| { + if output.is_empty() { + output.push_str(line); + } + })?; + if output != expected { + return Err(format!( + "verified Python executable reported version {output:?}, expected {expected}" + )); + } + Ok(()) +} + +fn run_sa3_setup( + shared: &InstallShared, + uv: &Path, + runtime_python: &Path, + runtime_dir: &Path, + work: &Path, + requirements_name: &str, + portable_wheels: Option<(&Path, &[WheelPin])>, +) -> Result<(), String> { + let venv = runtime_dir.join(".venv"); + let cache = work.join("uv-cache"); + + let mut create_venv = uv_command(uv, runtime_dir, &cache); + create_venv.args([ + "venv", + "--relocatable", + "--no-managed-python", + "--no-python-downloads", + "--no-config", + "--link-mode", + "copy", + "--python", + ]); + create_venv.arg(runtime_python); + create_venv.arg(&venv); + stream_child(shared, "sa3-venv", create_venv, |_| {})?; + + let python = crate::platform_paths::venv_python(&venv); + if !python.is_file() { + return Err("uv did not create the platform virtual-environment interpreter".into()); + } + let mut install_dependencies = uv_command(uv, runtime_dir, &cache); + install_dependencies + .args(["pip", "install", "--python"]) + .arg(&python); + if let Some((wheelhouse, pins)) = portable_wheels { + configure_portable_install(&mut install_dependencies, wheelhouse, pins, &|| { + shared.cancelled.load(Ordering::Acquire) + })?; + } else { + let requirements = runtime_dir.join(requirements_name); + install_dependencies + .args([ + "--require-hashes", + "--only-binary", + ":all:", + "--link-mode", + "copy", + "--default-index", + "https://pypi.org/simple", + "--no-config", + "-r", + ]) + .arg(&requirements); + } + stream_child(shared, "sa3-dependencies", install_dependencies, |_| {}) +} + +fn configure_portable_install( + command: &mut Command, + wheelhouse: &Path, + pins: &[WheelPin], + is_cancelled: &dyn Fn() -> bool, +) -> Result<(), String> { + verify_wheelhouse(wheelhouse, pins, is_cancelled)?; + command + .args([ + "--offline", + "--no-index", + "--no-deps", + "--only-binary", + ":all:", + "--link-mode", + "copy", + "--no-config", + ]) + .env("UV_OFFLINE", "1") + .env("UV_NO_INDEX", "1") + .env_remove("ALL_PROXY") + .env_remove("HTTPS_PROXY") + .env_remove("HTTP_PROXY"); + for pin in pins { + command.arg(wheelhouse.join(&pin.filename)); + } + Ok(()) +} + +fn uv_command(uv: &Path, cwd: &Path, cache: &Path) -> Command { + let mut command = Command::new(uv); + command + .current_dir(cwd) + .env("UV_CACHE_DIR", cache) + .env_remove("UV_INSECURE_HOST") + .env_remove("UV_INDEX") + .env_remove("UV_INDEX_URL") + .env_remove("UV_EXTRA_INDEX_URL") + .env_remove("UV_NO_VERIFY_HASHES") + .env_remove("PIP_INDEX_URL") + .env_remove("PIP_EXTRA_INDEX_URL") + .env_remove("PIP_TRUSTED_HOST"); + command +} + +fn install_backend_sources(candidate: &Path) -> Result<(), String> { + let package = candidate.join("lsdj_backend").join("lsdj"); + for (filename, bytes) in BACKEND_SOURCES { + write_synced(&package.join(filename), bytes)?; + } + write_synced( + &candidate.join("lsdj_backend").join("launch.py"), + b"from lsdj.frozen import main\nmain()\n", + ) +} + +fn source_closure_digest(sources: &[(&str, &[u8])]) -> String { + use sha2::{Digest, Sha256}; + + let mut digest = Sha256::new(); + for (name, bytes) in sources { + digest.update((name.len() as u64).to_le_bytes()); + digest.update(name.as_bytes()); + digest.update((bytes.len() as u64).to_le_bytes()); + digest.update(bytes); + } + hex::encode(digest.finalize()) +} + +fn backend_sources_digest() -> String { + source_closure_digest(BACKEND_SOURCES) +} + +const ISOLATED_IMPORT_SCRIPT: &str = r#" +import importlib +import pathlib +import sys + +package_root = pathlib.Path(sys.argv[1]).resolve(strict=True) +sys.path.insert(0, str(package_root)) +for name in sys.argv[2:]: + module = importlib.import_module(name) + pathlib.Path(module.__file__).resolve(strict=True).relative_to(package_root) +"#; + +fn isolated_backend_import_command( + python: &Path, + candidate: &Path, + modules: &[&str], + host_environment: impl IntoIterator, +) -> Result { + if modules.is_empty() + || modules.iter().any(|name| { + name.is_empty() + || !name + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'.')) + }) + { + return Err("managed backend import smoke module list is invalid".into()); + } + let package_root = candidate.join("lsdj_backend"); + if !package_root.join("lsdj").is_dir() { + return Err("managed backend source closure is missing".into()); + } + let mut command = Command::new(python); + command + .env_clear() + .env("PYTHONDONTWRITEBYTECODE", "1") + .env("PYTHONNOUSERSITE", "1") + .env("PYTHONUTF8", "1") + .current_dir(candidate) + .args(["-I", "-c", ISOLATED_IMPORT_SCRIPT]) + .arg(package_root) + .args(modules); + for (name, value) in host_environment { + command.env(name, value); + } + Ok(command) +} + +fn smoke_test_materialized_backend( + shared: &InstallShared, + candidate: &Path, + python: &Path, + modules: &[&str], +) -> Result<(), String> { + let host_environment = crate::platform_paths::get().managed_child_env()?; + let command = isolated_backend_import_command(python, candidate, modules, host_environment)?; + stream_child(shared, "managed-backend-import", command, |_| {}) +} + +fn relative_wire(root: &Path, path: &Path) -> Result { + let relative = path + .strip_prefix(root) + .map_err(|_| "managed SA3 launcher escaped its candidate")?; + let mut parts = Vec::new(); + for component in relative.components() { + match component { + std::path::Component::Normal(value) => parts.push( + value + .to_str() + .ok_or("managed SA3 launcher path is not UTF-8")?, + ), + _ => return Err("managed SA3 launcher path is unsafe".into()), + } + } + if parts.is_empty() { + return Err("managed SA3 launcher path is empty".into()); + } + Ok(parts.join("/")) +} + +fn content_digest(bytes: &[u8]) -> String { + use sha2::{Digest, Sha256}; + hex::encode(Sha256::digest(bytes)) +} + +fn seal_sa3_candidate( + candidate: &Path, + pin: &Sa3Pin, + python_pin: &PythonPin, +) -> Result<(), String> { + let runtime = Sa3Backend::Tflite.runtime_dir(candidate); + let program = crate::platform_paths::venv_python(&runtime.join(".venv")); + let mut provenance = BTreeMap::new(); + provenance.insert("source.repository".into(), pin.repo.clone()); + provenance.insert("source.revision".into(), pin.commit.clone()); + provenance.insert("source.sha256".into(), pin.source.artifact.sha256.clone()); + provenance.insert("backend.sources.sha256".into(), backend_sources_digest()); + provenance.insert("python.version".into(), python_pin.version.clone()); + provenance.insert( + "python.sha256".into(), + python_pin.archive.artifact.sha256.clone(), + ); + provenance.insert( + "requirements.sha256".into(), + content_digest(TFLITE_REQUIREMENTS_LOCK.as_bytes()), + ); + provenance.insert( + "wheels.sha256".into(), + content_digest(TFLITE_WHEEL_PIN_JSON.as_bytes()), + ); + let portable = tflite_pin(); + provenance.insert("models.repository".into(), portable.models.repo); + provenance.insert("models.revision".into(), portable.models.revision); + + let environment = [ + ("DO_NOT_TRACK", "1"), + ("HF_HUB_DISABLE_TELEMETRY", "1"), + ("HF_HUB_OFFLINE", "1"), + ("NO_COLOR", "1"), + ("PYTHONDONTWRITEBYTECODE", "1"), + ("PYTHONNOUSERSITE", "1"), + ("PYTHONUTF8", "1"), + ] + .into_iter() + .map(|(key, value)| (key.to_string(), value.to_string())) + .collect(); + let ephemeral_environment = service_ephemeral_environment( + crate::managed_runtime::Service::Sa3, + "LSDJ_API_CAPABILITY", + ); + let spec = crate::managed_runtime::CommandSpec { + program: relative_wire(candidate, &program)?, + argv: vec!["launch.py".into(), "--generation-server".into()], + cwd: "lsdj_backend".into(), + environment, + ephemeral_environment, + }; + crate::managed_runtime::seal_candidate( + candidate, + &crate::managed_runtime::host_target(), + provenance, + [( + crate::managed_runtime::Service::Sa3.wire_name().into(), + spec, + )] + .into_iter() + .collect(), + )?; + Ok(()) +} + +fn materialize_contained_file_links(root: &Path) -> Result<(), String> { + fn collect(directory: &Path, links: &mut Vec) -> Result<(), String> { + for entry in std::fs::read_dir(directory) + .map_err(|error| format!("cannot scan managed runtime links: {error}"))? + { + let entry = + entry.map_err(|error| format!("cannot scan managed runtime links: {error}"))?; + let path = entry.path(); + let metadata = std::fs::symlink_metadata(&path) + .map_err(|error| format!("cannot inspect managed runtime link: {error}"))?; + if metadata.file_type().is_symlink() { + links.push(path); + } else if metadata.is_dir() { + collect(&path, links)?; + } else if !metadata.is_file() { + return Err("managed runtime contains an unsupported filesystem entry".into()); + } + } + Ok(()) + } + + let canonical_root = std::fs::canonicalize(root) + .map_err(|error| format!("cannot canonicalize managed runtime candidate: {error}"))?; + let mut links = Vec::new(); + collect(root, &mut links)?; + for link in links { + let target = std::fs::canonicalize(&link) + .map_err(|error| format!("cannot resolve managed runtime link: {error}"))?; + let target_metadata = std::fs::metadata(&target) + .map_err(|error| format!("cannot inspect managed runtime link target: {error}"))?; + if !target.starts_with(&canonical_root) || !target_metadata.is_file() { + return Err("managed runtime link is not a contained regular file".into()); + } + let replacement = + link.with_file_name(format!(".lsdj-materialize-{:032x}", rand::random::())); + std::fs::copy(&target, &replacement) + .map_err(|error| format!("cannot materialize managed runtime link: {error}"))?; + std::fs::remove_file(&link) + .map_err(|error| format!("cannot replace managed runtime link: {error}"))?; + std::fs::rename(&replacement, &link) + .map_err(|error| format!("cannot commit materialized runtime file: {error}"))?; + } + Ok(()) +} + +#[cfg(feature = "managed-runtime")] +fn seal_mrt2_candidate(candidate: &Path, pin: &Mrt2Pin, python: &PythonPin) -> Result<(), String> { + let program = crate::platform_paths::venv_python(&candidate.join("runtime").join(".venv")); + let mut provenance = BTreeMap::new(); + provenance.insert( + "runtime.pin.sha256".into(), + content_digest(MRT2_PIN_JSON.as_bytes()), + ); + provenance.insert("backend.sources.sha256".into(), backend_sources_digest()); + provenance.insert( + "runtime.wheels.sha256".into(), + content_digest(MRT2_WHEEL_PIN_JSON.as_bytes()), + ); + provenance.insert("python.version".into(), python.version.clone()); + provenance.insert( + "python.sha256".into(), + python.archive.artifact.sha256.clone(), + ); + provenance.insert( + "processor.repository".into(), + pin.processor.repository.clone(), + ); + provenance.insert("processor.revision".into(), pin.processor.revision.clone()); + for (name, snapshot) in &pin.models { + provenance.insert( + format!("model.{name}.repository"), + snapshot.repository.clone(), + ); + provenance.insert(format!("model.{name}.revision"), snapshot.revision.clone()); + } + let environment = [ + ("DO_NOT_TRACK", "1"), + ("HF_HUB_DISABLE_TELEMETRY", "1"), + ("HF_HUB_OFFLINE", "1"), + ("NO_COLOR", "1"), + ("PYTHONDONTWRITEBYTECODE", "1"), + ("PYTHONNOUSERSITE", "1"), + ("PYTHONUTF8", "1"), + ] + .into_iter() + .map(|(key, value)| (key.to_string(), value.to_string())) + .collect(); + let ephemeral_environment = service_ephemeral_environment( + crate::managed_runtime::Service::Mrt2, + "LSDJ_WORKER_LAUNCH_TOKEN", + ); + let spec = crate::managed_runtime::CommandSpec { + program: relative_wire(candidate, &program)?, + argv: vec!["launch.py".into()], + cwd: "lsdj_backend".into(), + environment, + ephemeral_environment, + }; + crate::managed_runtime::seal_candidate( + candidate, + &crate::managed_runtime::host_target(), + provenance, + [( + crate::managed_runtime::Service::Mrt2.wire_name().into(), + spec, + )] + .into_iter() + .collect(), + )?; + Ok(()) +} + +fn warm_sa3( + shared: &InstallShared, + runtime_dir: &Path, + work: &Path, + backend: Sa3Backend, +) -> Result<(), String> { + let python = crate::platform_paths::venv_python(&runtime_dir.join(".venv")); + let script = runtime_dir.join("scripts").join(backend.script()); + let warm_dir = work.join("warm"); + if warm_dir.exists() { + std::fs::remove_dir_all(&warm_dir) + .map_err(|error| format!("cannot clear interrupted SA3 warm-up: {error}"))?; + } + std::fs::create_dir_all(&warm_dir) + .map_err(|error| format!("cannot create SA3 warm-up directory: {error}"))?; + for (dit, decoder) in [ + ("sm-sfx", "same-s"), + ("sm-music", "same-s"), + ("medium", "same-l"), + ] { + cancelled(shared)?; + let output = warm_dir.join(format!("{dit}.wav")); + let mut command = Command::new(&python); + command + .current_dir(runtime_dir) + // All required weights were downloaded and verified by Rust. Force + // the upstream helper offline so it cannot silently fetch a mutable + // replacement during candidate validation. + .env("HF_HUB_OFFLINE", "1") + .arg(&script) + .args([ + "--prompt", + "setup warm-up", + "--dit", + dit, + "--decoder", + decoder, + "--seconds", + "1", + "--steps", + "1", + "--out", + ]) + .arg(&output); + if backend == Sa3Backend::Tflite { + command.args(["--precision", "fp32", "--threads", "4"]); + } + stream_child(shared, "sa3-warm", command, |_| {})?; + if !output.is_file() { + return Err(format!("SA3 warm-up did not produce {dit} output")); + } + } + write_synced(&runtime_dir.join(WARMED_STAMP), b"")?; + let _ = std::fs::remove_dir_all(warm_dir); + Ok(()) +} + +fn validate_source_layout(checkout: &Path, backend: Sa3Backend) -> Result<(), String> { + let runtime = backend.runtime_dir(checkout); + if !runtime.is_dir() || !runtime.join("scripts").join(backend.script()).is_file() { + return Err("verified source archive has an unexpected SA3 layout".into()); + } + Ok(()) +} + +fn validate_sa3_install(checkout: &Path, pin: &Sa3Pin, backend: Sa3Backend) -> Result<(), String> { + validate_sa3_install_cancellable(checkout, pin, backend, &|| false) +} + +fn validate_sa3_install_cancellable( + checkout: &Path, + pin: &Sa3Pin, + backend: Sa3Backend, + is_cancelled: &dyn Fn() -> bool, +) -> Result<(), String> { + if is_cancelled() { + return Err("cancelled".into()); + } + validate_source_layout(checkout, backend)?; + let runtime = backend.runtime_dir(checkout); + let python = crate::platform_paths::venv_python(&runtime.join(".venv")); + if !python.is_file() { + return Err("SA3 virtual-environment interpreter is missing".into()); + } + if !runtime.join(WARMED_STAMP).is_file() { + return Err("SA3 warm-up stamp is missing".into()); + } + let runtime_python = host_python_pin(pin)?; + if !runtime + .join(".python") + .join(&runtime_python.executable) + .is_file() + { + return Err("pinned SA3 Python runtime is missing".into()); + } + if read_source_stamp(checkout).as_ref() != Some(&pinned_source()) { + return Err("SA3 source provenance does not match the pin".into()); + } + let manifest_path = checkout.join(INSTALL_MANIFEST_STAMP); + let manifest_metadata = std::fs::symlink_metadata(&manifest_path) + .map_err(|error| format!("cannot inspect SA3 install manifest: {error}"))?; + if manifest_metadata.file_type().is_symlink() || !manifest_metadata.is_file() { + return Err("SA3 install manifest is not a regular file".into()); + } + let installed_manifest = std::fs::read_to_string(&manifest_path) + .map_err(|error| format!("cannot read SA3 install manifest: {error}"))?; + if installed_manifest != install_manifest(backend) { + return Err("SA3 install manifest does not match this application".into()); + } + if backend == Sa3Backend::Tflite { + validate_tflite_provenance(&runtime, &tflite_pin())?; + } + validate_sa3_model_artifacts(checkout, pin, backend, is_cancelled)?; + if backend == Sa3Backend::Tflite { + crate::managed_runtime::validate_candidate(checkout, crate::managed_runtime::Service::Sa3)?; + } + Ok(()) +} + +fn validate_sa3_model_artifacts( + checkout: &Path, + pin: &Sa3Pin, + backend: Sa3Backend, + is_cancelled: &dyn Fn() -> bool, +) -> Result<(), String> { + for (install_path, artifact) in model_artifacts(pin, backend)? { + if let Err(error) = + verify_file_cancellable(&checkout.join(&install_path), &artifact, is_cancelled) + { + if error == "cancelled" { + return Err(error); + } + return Err(format!( + "SA3 model {} failed integrity validation: {error}", + install_path.display() + )); + } + } + Ok(()) +} + +fn install_manifest(backend: Sa3Backend) -> &'static str { + match backend { + Sa3Backend::Mlx => SA3_PIN_JSON, + Sa3Backend::Tflite => TFLITE_PIN_JSON, + } +} + +fn tflite_provenance(pin: &TflitePin) -> serde_json::Value { + serde_json::json!({ + "runtime": { + "repo": pin.runtime.repo, + "revision": pin.runtime.revision, + }, + "models": { + "repo": pin.models.repo, + "revision": pin.models.revision, + }, + }) +} + +fn write_tflite_provenance(runtime: &Path, pin: &TflitePin) -> Result<(), String> { + let bytes = serde_json::to_vec_pretty(&tflite_provenance(pin)) + .map_err(|error| format!("cannot serialize TFLite provenance: {error}"))?; + write_synced(&runtime.join(TFLITE_PROVENANCE_STAMP), &bytes) +} + +fn validate_tflite_provenance(runtime: &Path, pin: &TflitePin) -> Result<(), String> { + let bytes = std::fs::read(runtime.join(TFLITE_PROVENANCE_STAMP)) + .map_err(|error| format!("cannot read TFLite provenance: {error}"))?; + let actual: serde_json::Value = serde_json::from_slice(&bytes) + .map_err(|error| format!("TFLite provenance is invalid: {error}"))?; + if actual != tflite_provenance(pin) { + return Err("TFLite runtime/model provenance does not match the application pin".into()); + } + Ok(()) +} + +fn validate_sa3_pin(pin: &Sa3Pin) -> Result<(), String> { + if pin.commit.len() != 40 || !pin.commit.bytes().all(|byte| byte.is_ascii_hexdigit()) { + return Err("SA3 source revision must be a full immutable commit".into()); + } + pin.source.artifact.validate()?; + if !pin.source.artifact.url.contains(&pin.commit) + || !pin.source.archive_root.ends_with(&pin.commit) + || pin.source.max_files == 0 + || pin.source.max_expanded_bytes == 0 + { + return Err("SA3 source archive metadata does not match its revision".into()); + } + if pin.runtime.requirements != "sa3-requirements.lock" { + return Err("SA3 runtime pin is incomplete".into()); + } + for uv in &pin.runtime.uv { + uv.archive.artifact.validate()?; + if uv.target.is_empty() + || uv.version.is_empty() + || !uv.archive.artifact.url.contains(&uv.version) + || uv.executable.is_empty() + || uv.executable.contains('/') + || uv.executable.contains('\\') + || !matches!(uv.archive.archive_format.as_str(), "tar.gz" | "zip") + || (uv.archive.archive_format == "zip" && !uv.archive.artifact.url.ends_with(".zip")) + { + return Err("uv runtime pin is inconsistent".into()); + } + } + for python in &pin.runtime.python { + python.archive.artifact.validate()?; + if python.target.is_empty() + || python.version.split('.').count() != 3 + || !python.archive.artifact.url.contains(&python.version) + || python.executable.is_empty() + || python.executable.starts_with('/') + || python.executable.contains("..") + { + return Err("Python runtime pin is inconsistent".into()); + } + } + if pin.models.revision.len() != 40 + || !pin + .models + .revision + .bytes() + .all(|byte| byte.is_ascii_hexdigit()) + || pin.models.repo.split('/').count() != 2 + || pin.models.repo.contains("..") + { + return Err("SA3 model revision/repository pin is invalid".into()); + } + let expected: std::collections::BTreeSet<_> = [ + "dit_medium_f16.npz", + "dit_sm-music_f16.npz", + "dit_sm-sfx_f16.npz", + "same_l_decoder_f32.npz", + "same_l_encoder_f32.npz", + "same_s_decoder_f32.npz", + "same_s_encoder_f32.npz", + "t5gemma_f16.npz", + ] + .into_iter() + .collect(); + let mut actual = std::collections::BTreeSet::new(); + for model in &pin.models.artifacts { + let filename = model.filename()?; + model.artifact(&pin.models)?.validate()?; + if !actual.insert(filename) { + return Err("SA3 model manifest contains a duplicate artifact".into()); + } + } + if actual != expected { + return Err("SA3 model manifest does not cover every inference artifact".into()); + } + Ok(()) +} + +fn validate_tflite_pin(pin: &TflitePin, source: &Sa3Pin) -> Result<(), String> { + if pin.schema_version != 1 + || pin.runtime.repo != source.repo + || pin.runtime.revision != source.commit + || pin.runtime.subdirectory != "optimized/tflite" + || pin.runtime.entrypoint != "scripts/sa3_tflite.py" + || pin.runtime.requirements_lock != "scripts/sa3-tflite-requirements.lock" + || pin.models.revision.len() != 40 + || !pin + .models + .revision + .bytes() + .all(|byte| byte.is_ascii_hexdigit()) + || pin.models.repo.split('/').count() != 2 + || pin.models.precision != "fp32" + { + return Err("TFLite runtime/model pin is inconsistent".into()); + } + let artifacts = tflite_artifacts(pin)?; + if artifacts.len() != 8 { + return Err("TFLite manifest must cover exactly eight unique model artifacts".into()); + } + let expected: std::collections::BTreeSet<_> = [ + "optimized/tflite/models/tflite/t5gemma/encoder_fp16.tflite", + "optimized/tflite/models/tflite/sa3-sm-music/dit_fp32.tflite", + "optimized/tflite/models/tflite/sa3-sm-sfx/dit_fp32.tflite", + "optimized/tflite/models/tflite/sa3-m/dit_fp32.tflite", + "optimized/tflite/models/tflite/same-s/enc_fp32.tflite", + "optimized/tflite/models/tflite/same-s/dec_fp32.tflite", + "optimized/tflite/models/tflite/same-l/enc_fp32.tflite", + "optimized/tflite/models/tflite/same-l/dec_fp32.tflite", + ] + .into_iter() + .map(PathBuf::from) + .collect(); + if artifacts + .keys() + .cloned() + .collect::>() + != expected + { + return Err("TFLite manifest does not cover every inference artifact".into()); + } + wheel_pins_for("x86_64-unknown-linux-gnu")?; + wheel_pins_for("x86_64-pc-windows-msvc")?; + Ok(()) +} + +fn host_uv_pin(pin: &Sa3Pin) -> Result<&UvPin, String> { + let target = host_installer_target()?; + let uv = pin + .runtime + .uv + .iter() + .find(|artifact| artifact.target == target) + .ok_or("no pinned uv runtime exists for this platform")?; + uv.archive.artifact.validate()?; + if uv.version.is_empty() + || !uv.archive.artifact.url.contains(&uv.version) + || uv.executable.contains('/') + || uv.executable.contains('\\') + || uv.executable.is_empty() + { + return Err("uv runtime pin is inconsistent".into()); + } + Ok(uv) +} + +fn host_python_pin(pin: &Sa3Pin) -> Result<&PythonPin, String> { + let target = host_installer_target()?; + pin.runtime + .python + .iter() + .find(|artifact| artifact.target == target) + .ok_or_else(|| "no pinned Python runtime exists for this platform".into()) +} + +fn host_installer_target() -> Result<&'static str, String> { + installer_target_for(std::env::consts::OS, std::env::consts::ARCH) +} + +fn installer_target_for(os: &str, arch: &str) -> Result<&'static str, String> { + match (os, arch) { + ("macos", "aarch64") => Ok("aarch64-apple-darwin"), + ("linux", "x86_64") => Ok("x86_64-unknown-linux-gnu"), + ("windows", "x86_64") => Ok("x86_64-pc-windows-msvc"), + (os, arch) => Err(format!("no pinned SA3 runtime exists for {os}/{arch}")), + } +} + +fn write_synced(path: &Path, bytes: &[u8]) -> Result<(), String> { + use std::io::Write; + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent) + .map_err(|error| format!("cannot create install metadata directory: {error}"))?; + } + let mut file = std::fs::File::create(path) + .map_err(|error| format!("cannot create install metadata: {error}"))?; + file.write_all(bytes) + .and_then(|_| file.sync_all()) + .map_err(|error| format!("cannot sync install metadata: {error}")) +} + +// --- Tauri commands -------------------------------------------------------- + +#[tauri::command] +pub fn model_status(installer: tauri::State<'_, InstallManager>) -> ModelStatus { + status(installer.active_install()) +} + +#[tauri::command] +pub fn install_model( installer: tauri::State<'_, InstallManager>, app: AppHandle, family: Family, @@ -975,6 +3339,29 @@ pub fn open_model_folder(app: AppHandle, family: Family) -> Result<(), String> { mod tests { use super::*; + fn import_smoke_python() -> PathBuf { + let backend = Path::new(env!("CARGO_MANIFEST_DIR")).join("../backend"); + let managed = crate::platform_paths::venv_python(&backend.join(".venv")); + if managed.is_file() { + return managed; + } + let candidates: &[&str] = if cfg!(target_os = "windows") { + &["python.exe", "python3.exe"] + } else { + &["python3", "python"] + }; + let search = std::env::var_os("PATH").expect("Python is available on CI PATH"); + for directory in std::env::split_paths(&search) { + for name in candidates { + let candidate = directory.join(name); + if candidate.is_file() { + return candidate; + } + } + } + panic!("Python executable is unavailable for managed import smoke test"); + } + fn touch(path: &Path) { if let Some(parent) = path.parent() { std::fs::create_dir_all(parent).unwrap(); @@ -995,7 +3382,10 @@ mod tests { touch(&tmp.join("custom_x").join("custom_x.mlxfn")); touch(&tmp.join("custom_x").join("custom_x_state.safetensors")); - assert_eq!(discover_installed(&tmp), vec!["custom_x".to_string(), "mrt2_small".to_string()]); + assert_eq!( + discover_installed(&tmp), + vec!["custom_x".to_string(), "mrt2_small".to_string()] + ); let _ = std::fs::remove_dir_all(&tmp); } @@ -1014,7 +3404,569 @@ mod tests { fn sa3_pin_parses() { let pin = sa3_pin(); assert!(pin.repo.starts_with("https://")); - assert!(!pin.commit.is_empty()); + validate_sa3_pin(&pin).unwrap(); + assert_eq!(pin.commit.len(), 40); + assert_eq!(pin.models.artifacts.len(), 8); + assert!(pin.source.artifact.url.starts_with("https://")); + + let portable = tflite_pin(); + validate_tflite_pin(&portable, &pin).unwrap(); + assert_eq!(tflite_artifacts(&portable).unwrap().len(), 8); + assert_eq!( + model_download_bytes(&pin, Sa3Backend::Tflite).unwrap(), + 14_138_994_904 + ); + } + + #[test] + fn mrt2_runtime_models_remote_code_and_wheels_are_fully_pinned() { + let pin = mrt2_pin(); + validate_mrt2_pin(&pin).unwrap(); + assert_eq!( + mrt2_wheel_pins_for("x86_64-unknown-linux-gnu") + .unwrap() + .len(), + 56 + ); + assert_eq!( + mrt2_wheel_pins_for("x86_64-pc-windows-msvc").unwrap().len(), + 38 + ); + for (name, snapshot) in &pin.models { + assert_eq!(snapshot.revision.len(), 40); + assert!(snapshot + .files + .iter() + .any(|file| file.path == "model.safetensors")); + for required_code in [ + "configuration_magenta_rt2.py", + "modeling_magenta_rt2.py", + "depthformer.py", + "layers.py", + "musiccoca.py", + "processing_musiccoca.py", + "spectrostream.py", + "cudagraph.py", + "aoti.py", + ] { + assert!( + snapshot.files.iter().any(|file| file.path == required_code), + "{name} omits executable remote-code artifact {required_code}" + ); + } + for file in &snapshot.files { + let artifact = snapshot_artifact(snapshot, file).unwrap(); + assert!(artifact.url.contains(&snapshot.revision)); + assert!(!artifact.url.contains("/resolve/main/")); + } + } + assert_eq!(pin.processor.files.len(), 5); + } + + #[test] + fn backend_source_closure_carries_gpu_modules_and_service_exclusive_secrets() { + let sources: BTreeMap<_, _> = BACKEND_SOURCES.iter().copied().collect(); + assert_eq!( + sources.get("gpu_broker.py").copied(), + Some(include_bytes!("../../backend/lsdj/gpu_broker.py").as_slice()) + ); + assert_eq!( + sources.get("sa3_cuda.py").copied(), + Some(include_bytes!("../../backend/lsdj/sa3_cuda.py").as_slice()) + ); + assert_eq!(backend_sources_digest().len(), 64); + assert_ne!( + source_closure_digest(&[("gpu_broker.py", b"changed")]), + backend_sources_digest() + ); + + let sa3: BTreeSet<_> = service_ephemeral_environment( + crate::managed_runtime::Service::Sa3, + "LSDJ_API_CAPABILITY", + ) + .into_iter() + .collect(); + let mrt2: BTreeSet<_> = service_ephemeral_environment( + crate::managed_runtime::Service::Mrt2, + "LSDJ_WORKER_LAUNCH_TOKEN", + ) + .into_iter() + .collect(); + assert!(sa3.contains("LSDJ_API_CAPABILITY")); + assert!(!sa3.contains("LSDJ_WORKER_LAUNCH_TOKEN")); + assert!(mrt2.contains("LSDJ_WORKER_LAUNCH_TOKEN")); + assert!(!mrt2.contains("LSDJ_API_CAPABILITY")); + for name in BACKEND_PATH_ENVIRONMENT + .iter() + .chain(WINDOWS_CHILD_ENVIRONMENT) + { + assert!(sa3.contains(*name)); + assert!(mrt2.contains(*name)); + } + assert!(sa3.contains("SA3_HOME")); + assert!(!sa3.contains("MAGENTA_HOME")); + assert!(mrt2.contains("MAGENTA_HOME")); + assert!(!mrt2.contains("SA3_HOME")); + } + + #[cfg(feature = "managed-runtime")] + #[test] + fn managed_promotion_quiesces_before_rename_and_resumes_after_rollback() { + use std::cell::RefCell; + + let events = RefCell::new(Vec::new()); + let result = run_promotion_lifecycle( + "fake runtime", + || { + events.borrow_mut().push("quiesce"); + Ok("prior generation") + }, + || { + events.borrow_mut().push("promote"); + Err("fake promotion failed".to_string()) + }, + |generation| { + assert_eq!(generation, "prior generation"); + events.borrow_mut().push("resume"); + Ok(()) + }, + ); + assert_eq!(result.unwrap_err(), "fake promotion failed"); + assert_eq!(*events.borrow(), ["quiesce", "promote", "resume"]); + } + + #[cfg(feature = "managed-runtime")] + #[test] + fn failed_quiesce_never_enters_the_rename_window() { + use std::cell::RefCell; + + let events = RefCell::new(Vec::new()); + let result = run_promotion_lifecycle( + "fake runtime", + || { + events.borrow_mut().push("quiesce"); + Err::<(), _>("worker could not be reaped".to_string()) + }, + || { + events.borrow_mut().push("promote"); + Ok(()) + }, + |_| { + events.borrow_mut().push("resume"); + Ok(()) + }, + ); + assert_eq!(result.unwrap_err(), "worker could not be reaped"); + assert_eq!(*events.borrow(), ["quiesce"]); + } + + #[cfg(all(feature = "managed-runtime", windows))] + #[test] + fn windows_uncertain_reap_leaves_active_and_candidate_generations_unrenamed() { + let root = std::env::temp_dir().join(format!( + "lsdj-windows-uncertain-reap-{}-{:?}", + std::process::id(), + std::thread::current().id() + )); + let _ = std::fs::remove_dir_all(&root); + let home = root.join("active generation"); + let candidate = root.join("candidate generation"); + let backup = root.join("old generation"); + std::fs::create_dir_all(&home).unwrap(); + std::fs::create_dir_all(&candidate).unwrap(); + std::fs::write(home.join("active.marker"), b"active").unwrap(); + std::fs::write(candidate.join("candidate.marker"), b"candidate").unwrap(); + + let result = run_promotion_lifecycle( + "Windows managed runtime", + || Err::<(), _>("process-tree reap is uncertain".to_string()), + || crate::runtime_installer::promotion::promote(&candidate, &home, &backup, |_| Ok(())), + |_| Ok(()), + ); + + assert_eq!(result.unwrap_err(), "process-tree reap is uncertain"); + assert!(home.join("active.marker").is_file()); + assert!(candidate.join("candidate.marker").is_file()); + assert!(!backup.exists(), "rename window must remain unopened"); + std::fs::remove_dir_all(root).unwrap(); + } + + #[cfg(feature = "managed-runtime")] + #[test] + fn promotion_waits_for_a_second_positive_reap_before_rename() { + use std::cell::{Cell, RefCell}; + + let attempts = Cell::new(0usize); + let events = RefCell::new(Vec::new()); + let run = || { + run_promotion_lifecycle( + "fake runtime", + || { + events.borrow_mut().push("quiesce"); + let attempt = attempts.get(); + attempts.set(attempt + 1); + if attempt == 0 { + Err("process reap is uncertain".to_string()) + } else { + Ok(()) + } + }, + || { + events.borrow_mut().push("promote"); + Ok(()) + }, + |_| { + events.borrow_mut().push("resume"); + Ok(()) + }, + ) + }; + + assert!(run().is_err()); + assert_eq!(*events.borrow(), ["quiesce"]); + assert!(run().is_ok()); + assert_eq!(*events.borrow(), ["quiesce", "quiesce", "promote", "resume"]); + } + + #[cfg(feature = "managed-runtime")] + #[test] + fn shared_quiesce_and_gateway_recovery_errors_are_both_reported() { + let error = mrt2_quiesce_recovery_error( + "shared process is not reaped".to_string(), + Err("gateway spawn failed".to_string()), + ); + assert!(error.contains("shared process is not reaped")); + assert!(error.contains("gateway spawn failed")); + } + + #[test] + fn materialized_backend_imports_are_isolated_and_missing_modules_fail_closed() { + let root = std::env::temp_dir().join(format!( + "lsdj-backend-import-{}-{:?}", + std::process::id(), + std::thread::current().id() + )); + let _ = std::fs::remove_dir_all(&root); + let candidate = root.join("materialized candidate"); + let unavailable_checkout = root.join("source checkout unavailable"); + std::fs::create_dir_all(&candidate).unwrap(); + std::fs::create_dir_all(&unavailable_checkout).unwrap(); + install_backend_sources(&candidate).unwrap(); + let python = import_smoke_python(); + let host_environment = crate::platform_paths::managed_child_env_for_current_host( + &root.join("safe managed temp"), + ) + .unwrap(); + + let mut command = isolated_backend_import_command( + &python, + &candidate, + &["lsdj.gpu_broker", "lsdj.sa3_cuda"], + host_environment.clone(), + ) + .unwrap(); + command.current_dir(&unavailable_checkout); + let declared: BTreeSet<_> = command + .get_envs() + .filter_map(|(name, value)| value.map(|_| name.to_string_lossy().into_owned())) + .collect(); + for forbidden in [ + "PATH", + "HOME", + "HF_TOKEN", + "HUGGING_FACE_HUB_TOKEN", + "PYTHONPATH", + "LSDJ_GENERATION_CMD", + "LSDJ_SIDECAR_CMD", + "LSDJ_ALLOW_UNVERIFIED_MRT2_CUDA", + "LSDJ_ALLOW_UNVERIFIED_SA3_CUDA", + ] { + assert!(!declared.contains(forbidden)); + } + let output = command.output().unwrap(); + assert!( + output.status.success(), + "isolated imports failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + + for module in ["gpu_broker", "sa3_cuda"] { + let path = candidate + .join("lsdj_backend") + .join("lsdj") + .join(format!("{module}.py")); + let bytes = std::fs::read(&path).unwrap(); + std::fs::remove_file(&path).unwrap(); + let mut missing = isolated_backend_import_command( + &python, + &candidate, + &[&format!("lsdj.{module}")], + host_environment.clone(), + ) + .unwrap(); + missing.current_dir(&unavailable_checkout); + assert!(!missing.output().unwrap().status.success()); + std::fs::write(path, bytes).unwrap(); + } + let _ = std::fs::remove_dir_all(root); + } + + #[cfg(unix)] + #[test] + fn managed_runtime_materializes_contained_file_links_and_rejects_escapes() { + use std::os::unix::fs::symlink; + + let root = std::env::temp_dir().join(format!( + "lsdj-materialize-{}-{:?}", + std::process::id(), + std::thread::current().id() + )); + let _ = std::fs::remove_dir_all(&root); + std::fs::create_dir_all(root.join("runtime")).unwrap(); + std::fs::write(root.join("runtime").join("python"), b"runtime").unwrap(); + symlink("python", root.join("runtime").join("python3")).unwrap(); + materialize_contained_file_links(&root).unwrap(); + let materialized = root.join("runtime").join("python3"); + assert!(!std::fs::symlink_metadata(&materialized) + .unwrap() + .file_type() + .is_symlink()); + assert_eq!(std::fs::read(&materialized).unwrap(), b"runtime"); + + let outside = root.with_extension("outside"); + std::fs::write(&outside, b"outside").unwrap(); + symlink(&outside, root.join("escape")).unwrap(); + assert!(materialize_contained_file_links(&root) + .unwrap_err() + .contains("contained regular file")); + let _ = std::fs::remove_dir_all(&root); + let _ = std::fs::remove_file(outside); + } + + #[test] + fn sa3_cuda_pin_is_compiled_in_but_fails_closed_until_every_gate_is_complete() { + let error = validate_sa3_cuda_install_gate(SA3_CUDA_PIN_JSON, SA3_CUDA_LOCK.as_bytes()) + .unwrap_err(); + assert!(error.contains("not release-ready")); + + let mut pin: serde_json::Value = serde_json::from_str(SA3_CUDA_PIN_JSON).unwrap(); + pin["releaseReady"] = serde_json::json!(true); + pin["gatedArtifactsComplete"] = serde_json::json!(true); + pin["releaseBlockers"] = serde_json::json!([]); + let error = validate_sa3_cuda_install_gate( + &serde_json::to_string(&pin).unwrap(), + SA3_CUDA_LOCK.as_bytes(), + ) + .unwrap_err(); + assert!(error.contains("config SHA-256 is incomplete")); + + for model in ["small-music", "small-sfx"] { + pin["models"][model]["config"]["sha256"] = serde_json::json!("0".repeat(64)); + } + validate_sa3_cuda_install_gate( + &serde_json::to_string(&pin).unwrap(), + SA3_CUDA_LOCK.as_bytes(), + ) + .unwrap(); + } + + #[test] + fn sa3_backend_mapping_is_explicit_and_fail_closed() { + assert_eq!( + sa3_backend_for("macos", "aarch64").unwrap(), + Sa3Backend::Mlx + ); + for os in ["linux", "windows"] { + assert_eq!(sa3_backend_for(os, "x86_64").unwrap(), Sa3Backend::Tflite); + } + assert_eq!( + installer_target_for("macos", "aarch64").unwrap(), + "aarch64-apple-darwin" + ); + assert_eq!( + installer_target_for("linux", "x86_64").unwrap(), + "x86_64-unknown-linux-gnu" + ); + assert_eq!( + installer_target_for("windows", "x86_64").unwrap(), + "x86_64-pc-windows-msvc" + ); + for target in [ + ("macos", "x86_64"), + ("linux", "aarch64"), + ("freebsd", "x86_64"), + ] { + assert!(sa3_backend_for(target.0, target.1).is_err()); + } + } + + #[test] + fn tflite_provenance_round_trips_and_rejects_drift() { + let root = std::env::temp_dir().join(format!( + "lsdj-tflite-provenance-{}-{:?}", + std::process::id(), + std::thread::current().id() + )); + let _ = std::fs::remove_dir_all(&root); + std::fs::create_dir_all(&root).unwrap(); + let pin = tflite_pin(); + write_tflite_provenance(&root, &pin).unwrap(); + validate_tflite_provenance(&root, &pin).unwrap(); + std::fs::write(root.join(TFLITE_PROVENANCE_STAMP), b"{}").unwrap(); + assert!(validate_tflite_provenance(&root, &pin) + .unwrap_err() + .contains("does not match")); + let _ = std::fs::remove_dir_all(root); + } + + #[test] + fn tflite_install_paths_cannot_escape_the_runtime() { + assert_eq!( + checked_install_path("models/tflite/same-s/dec_fp32.tflite").unwrap(), + PathBuf::from("optimized/tflite/models/tflite/same-s/dec_fp32.tflite") + ); + for unsafe_path in [ + "../outside", + "/absolute", + "models/tflite/../../outside", + "models\\tflite\\outside", + ] { + assert!(checked_install_path(unsafe_path).is_err()); + } + } + + #[test] + fn portable_wheel_sets_are_complete_and_platform_specific() { + let linux = wheel_pins_for("x86_64-unknown-linux-gnu").unwrap(); + let windows = wheel_pins_for("x86_64-pc-windows-msvc").unwrap(); + assert_eq!(linux.len(), 33); + assert_eq!(windows.len(), 34); + assert!(linux.iter().any(|pin| pin.package == "pydantic-core")); + assert!(windows.iter().any(|pin| pin.package == "colorama")); + assert!(wheel_pins_for("aarch64-unknown-linux-gnu").is_err()); + } + + #[test] + fn portable_wheelhouse_rejects_missing_unexpected_and_tampered_files() { + use sha2::{Digest, Sha256}; + + let root = std::env::temp_dir().join(format!( + "lsdj-wheelhouse-{}-{:?}", + std::process::id(), + std::thread::current().id() + )); + let _ = std::fs::remove_dir_all(&root); + std::fs::create_dir_all(&root).unwrap(); + let mut pins = wheel_pins_for("x86_64-unknown-linux-gnu").unwrap(); + for pin in &mut pins { + let bytes = pin.filename.as_bytes(); + pin.artifact.size = bytes.len() as u64; + pin.artifact.sha256 = hex::encode(Sha256::digest(bytes)); + std::fs::write(root.join(&pin.filename), bytes).unwrap(); + } + verify_wheelhouse(&root, &pins, &|| false).unwrap(); + + let missing = root.join(&pins[0].filename); + std::fs::remove_file(&missing).unwrap(); + assert!(verify_wheelhouse(&root, &pins, &|| false) + .unwrap_err() + .contains("missing or unexpected")); + std::fs::write(&missing, pins[0].filename.as_bytes()).unwrap(); + + std::fs::write(root.join("surprise.whl"), b"surprise").unwrap(); + assert!(verify_wheelhouse(&root, &pins, &|| false) + .unwrap_err() + .contains("missing or unexpected")); + std::fs::remove_file(root.join("surprise.whl")).unwrap(); + + std::fs::write(&missing, vec![b'x'; pins[0].artifact.size as usize]).unwrap(); + assert!(verify_wheelhouse(&root, &pins, &|| false) + .unwrap_err() + .contains("verification")); + let _ = std::fs::remove_dir_all(root); + } + + #[test] + fn portable_dependency_install_is_strictly_offline_and_exact() { + use sha2::{Digest, Sha256}; + + let root = std::env::temp_dir().join(format!( + "lsdj-offline-wheels-{}-{:?}", + std::process::id(), + std::thread::current().id() + )); + let _ = std::fs::remove_dir_all(&root); + std::fs::create_dir_all(&root).unwrap(); + let mut pins = wheel_pins_for("x86_64-unknown-linux-gnu").unwrap(); + for pin in &mut pins { + let bytes = pin.filename.as_bytes(); + pin.artifact.size = bytes.len() as u64; + pin.artifact.sha256 = hex::encode(Sha256::digest(bytes)); + std::fs::write(root.join(&pin.filename), bytes).unwrap(); + } + let mut command = Command::new("uv"); + configure_portable_install(&mut command, &root, &pins, &|| false).unwrap(); + let args = command + .get_args() + .map(|value| value.to_string_lossy().into_owned()) + .collect::>(); + for required in ["--offline", "--no-index", "--no-deps", "--no-config"] { + assert!(args.iter().any(|arg| arg == required)); + } + assert!(!args.iter().any(|arg| arg.starts_with("http"))); + assert_eq!( + args.iter().filter(|arg| arg.ends_with(".whl")).count(), + pins.len() + ); + let env = command + .get_envs() + .map(|(name, value)| { + ( + name.to_string_lossy().into_owned(), + value.map(|item| item.to_string_lossy().into_owned()), + ) + }) + .collect::>(); + assert_eq!(env.get("UV_OFFLINE"), Some(&Some("1".into()))); + assert_eq!(env.get("UV_NO_INDEX"), Some(&Some("1".into()))); + for proxy in ["ALL_PROXY", "HTTPS_PROXY", "HTTP_PROXY"] { + assert_eq!(env.get(proxy), Some(&None)); + } + let _ = std::fs::remove_dir_all(root); + } + + #[test] + fn app_managed_model_validation_hashes_all_eight_artifacts() { + use sha2::{Digest, Sha256}; + + let root = std::env::temp_dir().join(format!( + "lsdj-model-integrity-{}-{:?}", + std::process::id(), + std::thread::current().id() + )); + let _ = std::fs::remove_dir_all(&root); + let model_dir = root + .join("optimized") + .join("mlx") + .join("models") + .join("mlx"); + std::fs::create_dir_all(&model_dir).unwrap(); + let mut pin = sa3_pin(); + for (index, model) in pin.models.artifacts.iter_mut().enumerate() { + let bytes = format!("fixture-{index}").into_bytes(); + model.size = bytes.len() as u64; + model.sha256 = hex::encode(Sha256::digest(&bytes)); + std::fs::write(model_dir.join(model.filename().unwrap()), bytes).unwrap(); + } + + validate_sa3_model_artifacts(&root, &pin, Sa3Backend::Mlx, &|| false).unwrap(); + let tampered = pin.models.artifacts[3].filename().unwrap(); + let original_size = pin.models.artifacts[3].size as usize; + std::fs::write(model_dir.join(tampered), vec![b'x'; original_size]).unwrap(); + let error = + validate_sa3_model_artifacts(&root, &pin, Sa3Backend::Mlx, &|| false).unwrap_err(); + assert!(error.contains("SHA-256"), "unexpected error: {error}"); + let _ = std::fs::remove_dir_all(root); } #[test] @@ -1028,7 +3980,7 @@ mod tests { repo: "https://github.com/brxs/stable-audio-3".into(), commit: "abc123def456".into(), }; - write_source_stamp(&tmp, &src); + write_source_stamp(&tmp, &src).unwrap(); assert_eq!(read_source_stamp(&tmp), Some(src)); let _ = std::fs::remove_dir_all(&tmp); } @@ -1046,11 +3998,17 @@ mod tests { // Exact match. assert!(!sa3_update_available(Some(&pin.clone()), &pin, true)); // Short-SHA stamp vs full-SHA pin (prefix) counts as a match. - let short = Sa3Source { repo: pin.repo.clone(), commit: "36ef977".into() }; + let short = Sa3Source { + repo: pin.repo.clone(), + commit: "36ef977".into(), + }; assert!(!sa3_update_available(Some(&short), &pin, true)); // A different commit, or a different repo (e.g. after reverting to // upstream), is updatable. - let other_commit = Sa3Source { repo: pin.repo.clone(), commit: "deadbeef".into() }; + let other_commit = Sa3Source { + repo: pin.repo.clone(), + commit: "deadbeef".into(), + }; assert!(sa3_update_available(Some(&other_commit), &pin, true)); let other_repo = Sa3Source { repo: "https://github.com/Stability-AI/stable-audio-3".into(), @@ -1058,55 +4016,20 @@ mod tests { }; assert!(sa3_update_available(Some(&other_repo), &pin, true)); // A trailing slash on the repo is ignored. - let slash = Sa3Source { repo: format!("{}/", pin.repo), commit: pin.commit.clone() }; + let slash = Sa3Source { + repo: format!("{}/", pin.repo), + commit: pin.commit.clone(), + }; assert!(!sa3_update_available(Some(&slash), &pin, true)); } - #[test] - fn migration_moves_a_prior_install_only_when_the_new_dir_is_absent() { - let tmp = std::env::temp_dir().join(format!("lsdj-migrate-test-{}", std::process::id())); - let _ = std::fs::remove_dir_all(&tmp); - let new_rtv2 = tmp.join("new").join("magenta-rt-v2"); - let old_rtv2 = tmp.join("old").join("magenta-rt-v2"); - - // No prior install → nothing to move. - assert_eq!(migration_move(&new_rtv2, &old_rtv2), None); - - // Prior install present, new dir absent → move it. - std::fs::create_dir_all(&old_rtv2).unwrap(); - assert_eq!( - migration_move(&new_rtv2, &old_rtv2), - Some((old_rtv2.clone(), new_rtv2.clone())), - ); - - // New dir already exists (already migrated) → leave the old one be. - std::fs::create_dir_all(&new_rtv2).unwrap(); - assert_eq!(migration_move(&new_rtv2, &old_rtv2), None); - let _ = std::fs::remove_dir_all(&tmp); - } - - #[test] - fn legacy_brand_directory_is_moved_without_losing_its_contents() { - let tmp = - std::env::temp_dir().join(format!("lsdj-brand-migrate-test-{}", std::process::id())); - let _ = std::fs::remove_dir_all(&tmp); - let old_dir = tmp.join("LSDJai").join("stable-audio-3"); - let new_dir = tmp.join("LSDJ").join("stable-audio-3"); - std::fs::create_dir_all(&old_dir).unwrap(); - std::fs::write(old_dir.join("model.bin"), b"model").unwrap(); - - assert_eq!(migrate_legacy_dir(&new_dir, &old_dir), new_dir); - assert_eq!(std::fs::read(new_dir.join("model.bin")).unwrap(), b"model"); - assert!(!old_dir.exists()); - let _ = std::fs::remove_dir_all(&tmp); - } - // --- End-to-end install: actually run the pipeline against a stub backend. // These spawn real processes and exercise the full spawn → stream-parse → // progress → on-disk-result path (no weights, no GUI, no network). Env is set // ONLY on the child Command (never process-global), so they can't race the // sidecar tests that share this binary's environment. + #[cfg(unix)] fn shared() -> InstallShared { InstallShared { busy: AtomicBool::new(false), @@ -1116,6 +4039,36 @@ mod tests { } } + #[cfg(unix)] + #[test] + fn cancellation_between_spawn_and_park_terminates_the_child() { + let shared = shared(); + let mut command = Command::new("sh"); + command.arg("-c").arg("sleep 30"); + let child = crate::child_process::spawn_grouped(&mut command).expect("spawn child"); + let pid = child.id() as libc::pid_t; + + // Model the precise race: cancel() observed an empty slot after the OS + // spawn completed but before stream_child published the handle. + shared.cancelled.store(true, Ordering::Release); + assert_eq!(park_child(&shared, child), Err("cancelled".into())); + assert!( + shared + .current_child + .lock() + .unwrap_or_else(|p| p.into_inner()) + .is_none(), + "cancelled child must not remain parked" + ); + // SAFETY: signal 0 only probes whether the already-recorded pid exists. + assert_eq!( + unsafe { libc::kill(pid, 0) }, + -1, + "child survived cancellation" + ); + } + + #[cfg(unix)] fn write_exec(path: &Path, body: &str) { use std::os::unix::fs::PermissionsExt; std::fs::write(path, body).unwrap(); @@ -1124,6 +4077,7 @@ mod tests { // A stand-in for the frozen sidecar's `--download-model` mode: writes the two // model files into $MAGENTA_HOME and emits the JSON progress contract. + #[cfg(unix)] const STUB_SIDECAR: &str = r#"#!/bin/sh name="" while [ $# -gt 0 ]; do @@ -1141,6 +4095,7 @@ printf '{"event":"file","file":"models/%s/%s_state.safetensors"}\n' "$name" "$na printf '{"event":"done"}\n' "#; + #[cfg(unix)] #[test] fn install_magenta_runs_the_tooling_and_the_model_appears() { let tmp = std::env::temp_dir().join(format!("lsdj-install-test-{}", std::process::id())); @@ -1174,13 +4129,17 @@ printf '{"event":"done"}\n' let _ = std::fs::remove_dir_all(&tmp); } + #[cfg(unix)] #[test] fn run_download_reports_a_tooling_error() { let tmp = std::env::temp_dir().join(format!("lsdj-install-err-{}", std::process::id())); let _ = std::fs::remove_dir_all(&tmp); std::fs::create_dir_all(&tmp).unwrap(); let stub = tmp.join("fail.sh"); - write_exec(&stub, "#!/bin/sh\nprintf '{\"event\":\"error\",\"message\":\"no weights\"}\\n'\nexit 1\n"); + write_exec( + &stub, + "#!/bin/sh\nprintf '{\"event\":\"error\",\"message\":\"no weights\"}\\n'\nexit 1\n", + ); let mut cmd = Command::new("sh"); cmd.arg(&stub); @@ -1190,31 +4149,67 @@ printf '{"event":"done"}\n' let _ = std::fs::remove_dir_all(&tmp); } + #[cfg(unix)] #[test] - fn run_sa3_installer_runs_the_script_and_reports_the_stage() { - let tmp = std::env::temp_dir().join(format!("lsdj-sa3install-test-{}", std::process::id())); + fn run_download_sanitizes_and_bounds_structured_tooling_errors() { + let tmp = std::env::temp_dir().join(format!("lsdj-install-secret-{}", std::process::id())); let _ = std::fs::remove_dir_all(&tmp); - let checkout = tmp.join("co"); - std::fs::create_dir_all(&checkout).unwrap(); - let marker = tmp.join("ran"); - let stub = tmp.join("install.sh"); - // The script's own stdout is intentionally NOT streamed to the UI (it is - // English and un-keyed); the touched marker proves it actually ran. - write_exec(&stub, &format!("#!/bin/sh\n: > \"{}\"\nexit 0\n", marker.display())); - - let events = Arc::new(Mutex::new(Vec::new())); - let sink = events.clone(); - let progress = move |stage: &str, _message: Option, _file: Option| { - sink.lock().unwrap().push(stage.to_string()); - }; - let result = run_sa3_installer(&progress, &shared(), &checkout, &stub); + std::fs::create_dir_all(&tmp).unwrap(); + let stub = tmp.join("fail-secret.sh"); + let message = format!( + "HF_TOKEN=hf-secret HUGGING_FACE_HUB_TOKEN=hub-secret \ + client_secret=oauth-secret access_token=access-secret \ + api_key=api-secret Authorization: Bearer auth-secret {}", + "x".repeat(5000) + ); + let json = serde_json::json!({"event": "error", "message": message}); + write_exec( + &stub, + &format!("#!/bin/sh\nprintf '%s\\n' '{}'\nexit 1\n", json), + ); + let mut cmd = Command::new("sh"); + cmd.arg(&stub); - assert!(result.is_ok(), "sa3 install failed: {result:?}"); - assert!(marker.exists(), "the installer script did not run"); - assert!(events.lock().unwrap().iter().any(|stage| stage == "install")); + let noop = |_: &str, _: Option, _: Option| {}; + let error = run_download(&noop, &shared(), cmd).expect_err("stub must fail"); + assert!( + error.len() <= 2048, + "unbounded error length: {}", + error.len() + ); + for secret in [ + "hf-secret", + "hub-secret", + "oauth-secret", + "access-secret", + "api-secret", + "auth-secret", + ] { + assert!(!error.contains(secret), "leaked {secret}: {error}"); + } + assert!(error.contains("[REDACTED]")); let _ = std::fs::remove_dir_all(&tmp); } + #[cfg(unix)] + #[test] + fn uv_command_keeps_unicode_paths_structured() { + let root = Path::new("/tmp/LSDJ profile ü with spaces"); + let executable = root.join("runtime tools").join("uv"); + let cwd = root.join("Stable Audio 3"); + let cache = root.join("staging cache"); + let venv = cwd.join(".venv"); + let mut command = uv_command(&executable, &cwd, &cache); + command.arg("venv").arg(&venv); + + assert_eq!(command.get_program(), executable.as_os_str()); + assert_eq!(command.get_current_dir(), Some(cwd.as_path())); + let args: Vec<_> = command.get_args().collect(); + assert_eq!(args[0], std::ffi::OsStr::new("venv")); + assert_eq!(args[1], venv.as_os_str()); + } + + #[cfg(unix)] #[test] fn cancel_kills_the_whole_process_group() { use std::time::Duration; @@ -1263,7 +4258,7 @@ printf '{"event":"done"}\n' .unwrap_or_else(|p| p.into_inner()) .take() .expect("child parked before its stdout flowed"); - kill_group(&mut child); + let _ = child.force_kill(); }); // The group kill signals the grandchild atomically; its reaping (by diff --git a/src-tauri/src/platform_diagnostics.rs b/src-tauri/src/platform_diagnostics.rs new file mode 100644 index 0000000..25d01ab --- /dev/null +++ b/src-tauri/src/platform_diagnostics.rs @@ -0,0 +1,462 @@ +//! Structured desktop/runtime diagnostics for platform support. +//! +//! The command returns facts and stable advisory codes, not prose. The webview +//! can localise those codes, support bundles can retain the evidence, and a +//! headless test can validate classification without pretending that CI has a +//! real PipeWire/ALSA/MIDI desktop. + +use std::path::Path; + +#[cfg(any(target_os = "linux", test))] +use std::collections::HashMap; + +use serde::Serialize; + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct PlatformDiagnostics { + platform: &'static str, + architecture: &'static str, + runtime_mode: &'static str, + developer_fallback_allowed: bool, + roots: RootDiagnostics, + linux: Option, +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +struct RootDiagnostics { + config: String, + data: String, + cache: String, + assets: String, + staging: String, +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +struct LinuxDiagnostics { + distribution_id: Option, + distribution_version: Option, + distribution_support: &'static str, + session_type: &'static str, + audio_backend: &'static str, + pipewire_socket_detected: bool, + pulse_socket_detected: bool, + alsa_devices_detected: bool, + midi_sequencer_access: &'static str, + advisories: Vec<&'static str>, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +struct RuntimePolicy { + mode: &'static str, + developer_fallback_allowed: bool, +} + +/// Pure release-policy table. The feature pair is kept explicit so tests cover +/// every supported build shape without pretending the current host is macOS, +/// Windows, or Linux. The mutually-exclusive feature guard in `lib.rs` rejects +/// the otherwise ambiguous `(true, true)` build before this value is observed. +const fn runtime_policy(bundled_backend: bool, managed_runtime: bool) -> RuntimePolicy { + if bundled_backend { + RuntimePolicy { + mode: "bundled", + developer_fallback_allowed: false, + } + } else if managed_runtime { + RuntimePolicy { + mode: "managed", + developer_fallback_allowed: false, + } + } else { + RuntimePolicy { + mode: "developer", + developer_fallback_allowed: true, + } + } +} + +const fn compiled_runtime_policy() -> RuntimePolicy { + runtime_policy( + cfg!(feature = "bundled-backend"), + cfg!(feature = "managed-runtime"), + ) +} + +#[cfg(any(target_os = "linux", test))] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct LinuxEvidence { + pipewire_socket: bool, + pulse_socket: bool, + alsa_devices: bool, + midi_sequencer: MidiSequencerAccess, +} + +#[cfg(any(target_os = "linux", test))] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum MidiSequencerAccess { + Available, + PermissionDenied, + Missing, +} + +#[cfg(any(target_os = "linux", test))] +impl MidiSequencerAccess { + const fn as_str(self) -> &'static str { + match self { + Self::Available => "available", + Self::PermissionDenied => "permissionDenied", + Self::Missing => "missing", + } + } +} + +/// Return the host facts that explain Linux path, desktop-session, audio, MIDI, +/// and runtime-launch behavior. No probe opens an audio or MIDI stream. +#[tauri::command] +pub fn platform_diagnostics() -> PlatformDiagnostics { + let paths = crate::platform_paths::get(); + let runtime = compiled_runtime_policy(); + let roots = RootDiagnostics { + config: display(paths.config()), + data: display(paths.data()), + cache: display(paths.cache()), + assets: display(paths.assets()), + staging: display(paths.staging()), + }; + PlatformDiagnostics { + platform: std::env::consts::OS, + architecture: std::env::consts::ARCH, + runtime_mode: runtime.mode, + developer_fallback_allowed: runtime.developer_fallback_allowed, + roots, + linux: collect_linux(), + } +} + +fn display(path: &Path) -> String { + path.to_string_lossy().into_owned() +} + +#[cfg(target_os = "linux")] +fn collect_linux() -> Option { + let os_release = std::fs::read_to_string("/etc/os-release").unwrap_or_default(); + let distribution = parse_os_release(&os_release); + let distribution_id = distribution.get("ID").cloned(); + let distribution_version = distribution.get("VERSION_ID").cloned(); + let distribution_support = + distribution_support(distribution_id.as_deref(), distribution_version.as_deref()); + let session_type = session_type(|name| std::env::var(name).ok()); + + let runtime_dir = std::env::var_os("XDG_RUNTIME_DIR") + .map(std::path::PathBuf::from) + .filter(|path| path.is_absolute()); + let evidence = LinuxEvidence { + pipewire_socket: runtime_dir + .as_ref() + .is_some_and(|dir| dir.join("pipewire-0").exists()), + pulse_socket: runtime_dir + .as_ref() + .is_some_and(|dir| dir.join("pulse/native").exists()), + alsa_devices: Path::new("/dev/snd").is_dir(), + midi_sequencer: midi_sequencer_access(Path::new("/dev/snd/seq")), + }; + Some(linux_diagnostics( + distribution_id, + distribution_version, + distribution_support, + session_type, + evidence, + )) +} + +#[cfg(not(target_os = "linux"))] +fn collect_linux() -> Option { + None +} + +#[cfg(any(target_os = "linux", test))] +fn linux_diagnostics( + distribution_id: Option, + distribution_version: Option, + distribution_support: &'static str, + session_type: &'static str, + evidence: LinuxEvidence, +) -> LinuxDiagnostics { + let audio_backend = if evidence.pipewire_socket { + "pipewireAlsa" + } else if evidence.pulse_socket { + "pulseAlsa" + } else { + "alsa" + }; + LinuxDiagnostics { + distribution_id, + distribution_version, + distribution_support, + session_type, + audio_backend, + pipewire_socket_detected: evidence.pipewire_socket, + pulse_socket_detected: evidence.pulse_socket, + alsa_devices_detected: evidence.alsa_devices, + midi_sequencer_access: evidence.midi_sequencer.as_str(), + advisories: advisory_codes(distribution_support, session_type, evidence), + } +} + +#[cfg(any(target_os = "linux", test))] +fn parse_os_release(content: &str) -> HashMap { + content + .lines() + .filter_map(|line| { + let line = line.trim(); + if line.is_empty() || line.starts_with('#') { + return None; + } + let (key, value) = line.split_once('=')?; + if key.is_empty() + || !key + .bytes() + .all(|byte| byte.is_ascii_uppercase() || byte == b'_') + { + return None; + } + let value = value.trim(); + let value = if value.len() >= 2 + && ((value.starts_with('"') && value.ends_with('"')) + || (value.starts_with('\'') && value.ends_with('\''))) + { + &value[1..value.len() - 1] + } else { + value + }; + Some((key.to_string(), value.chars().take(128).collect())) + }) + .collect() +} + +#[cfg(any(target_os = "linux", test))] +fn distribution_support(id: Option<&str>, version: Option<&str>) -> &'static str { + if !id.is_some_and(|id| id.eq_ignore_ascii_case("ubuntu")) { + return if id.is_some() { "community" } else { "unknown" }; + } + match version.and_then(version_pair) { + Some((major, minor)) if (major, minor) >= (22, 4) => "supported", + Some(_) => "unsupportedVersion", + None => "unknown", + } +} + +#[cfg(any(target_os = "linux", test))] +fn version_pair(version: &str) -> Option<(u32, u32)> { + let mut pieces = version.split('.'); + let major = pieces.next()?.parse().ok()?; + let minor = pieces.next().unwrap_or("0").parse().ok()?; + Some((major, minor)) +} + +#[cfg(any(target_os = "linux", test))] +fn session_type(get: impl Fn(&str) -> Option) -> &'static str { + match get("XDG_SESSION_TYPE") + .as_deref() + .map(str::to_ascii_lowercase) + .as_deref() + { + Some("wayland") => "wayland", + Some("x11") => "x11", + _ if get("WAYLAND_DISPLAY").is_some() => "wayland", + _ if get("DISPLAY").is_some() => "x11", + _ => "unknown", + } +} + +#[cfg(any(target_os = "linux", test))] +fn advisory_codes( + distribution_support: &str, + session_type: &str, + evidence: LinuxEvidence, +) -> Vec<&'static str> { + let mut codes = Vec::new(); + if distribution_support != "supported" { + codes.push("linux.distribution.notSupported"); + } + if session_type == "unknown" { + codes.push("linux.session.notDetected"); + } + if !evidence.alsa_devices { + codes.push("linux.audio.alsaDevicesMissing"); + } + match evidence.midi_sequencer { + MidiSequencerAccess::Available => {} + MidiSequencerAccess::PermissionDenied => { + codes.push("linux.midi.sequencerPermissionDenied"); + } + MidiSequencerAccess::Missing => codes.push("linux.midi.sequencerMissing"), + } + codes +} + +#[cfg(target_os = "linux")] +fn midi_sequencer_access(path: &Path) -> MidiSequencerAccess { + use std::ffi::CString; + use std::os::unix::ffi::OsStrExt; + + if !path.exists() { + return MidiSequencerAccess::Missing; + } + let Ok(path) = CString::new(path.as_os_str().as_bytes()) else { + return MidiSequencerAccess::PermissionDenied; + }; + // `access` checks the real user's read/write permission without opening an + // ALSA sequencer client or changing device state. + if unsafe { libc::access(path.as_ptr(), libc::R_OK | libc::W_OK) } == 0 { + MidiSequencerAccess::Available + } else { + MidiSequencerAccess::PermissionDenied + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn ubuntu_2204_and_newer_are_the_only_supported_distribution_contract() { + assert_eq!( + distribution_support(Some("ubuntu"), Some("22.04")), + "supported" + ); + assert_eq!( + distribution_support(Some("Ubuntu"), Some("24.04")), + "supported" + ); + assert_eq!( + distribution_support(Some("ubuntu"), Some("20.04")), + "unsupportedVersion" + ); + assert_eq!( + distribution_support(Some("fedora"), Some("42")), + "community" + ); + assert_eq!(distribution_support(None, None), "unknown"); + } + + #[test] + fn os_release_parser_does_not_evaluate_shell_syntax() { + let parsed = parse_os_release( + "ID=ubuntu\nVERSION_ID=\"22.04\"\nNAME='Ubuntu Linux'\nBAD-KEY=value\n", + ); + assert_eq!(parsed.get("ID").map(String::as_str), Some("ubuntu")); + assert_eq!(parsed.get("VERSION_ID").map(String::as_str), Some("22.04")); + assert_eq!(parsed.get("NAME").map(String::as_str), Some("Ubuntu Linux")); + assert!(!parsed.contains_key("BAD-KEY")); + } + + #[test] + fn desktop_session_uses_xdg_then_safe_display_fallbacks() { + assert_eq!( + session_type(|name| (name == "XDG_SESSION_TYPE").then(|| "wayland".into())), + "wayland" + ); + assert_eq!( + session_type(|name| (name == "DISPLAY").then(|| ":99".into())), + "x11" + ); + assert_eq!(session_type(|_| None), "unknown"); + } + + #[test] + fn diagnostics_report_transport_evidence_without_claiming_hardware() { + assert_eq!(MidiSequencerAccess::Available.as_str(), "available"); + assert_eq!(MidiSequencerAccess::Missing.as_str(), "missing"); + let evidence = LinuxEvidence { + pipewire_socket: true, + pulse_socket: true, + alsa_devices: false, + midi_sequencer: MidiSequencerAccess::PermissionDenied, + }; + let diagnostics = linux_diagnostics( + Some("ubuntu".into()), + Some("22.04".into()), + "supported", + "wayland", + evidence, + ); + assert_eq!(diagnostics.audio_backend, "pipewireAlsa"); + assert_eq!( + diagnostics.advisories, + [ + "linux.audio.alsaDevicesMissing", + "linux.midi.sequencerPermissionDenied" + ] + ); + } + + #[test] + fn runtime_policy_table_allows_fallback_only_for_featureless_development() { + assert_eq!( + runtime_policy(true, false), + RuntimePolicy { + mode: "bundled", + developer_fallback_allowed: false, + } + ); + assert_eq!( + runtime_policy(false, true), + RuntimePolicy { + mode: "managed", + developer_fallback_allowed: false, + } + ); + assert_eq!( + runtime_policy(false, false), + RuntimePolicy { + mode: "developer", + developer_fallback_allowed: true, + } + ); + } + + #[test] + fn compiled_runtime_policy_matches_the_selected_release_feature() { + let expected = runtime_policy( + cfg!(feature = "bundled-backend"), + cfg!(feature = "managed-runtime"), + ); + assert_eq!(compiled_runtime_policy(), expected); + if cfg!(feature = "bundled-backend") || cfg!(feature = "managed-runtime") { + assert!(!expected.developer_fallback_allowed); + } else { + assert_eq!(expected.mode, "developer"); + assert!(expected.developer_fallback_allowed); + } + } + + #[test] + fn diagnostics_serialization_exposes_the_compiled_runtime_policy() { + let runtime = compiled_runtime_policy(); + let value = serde_json::to_value(PlatformDiagnostics { + platform: "test", + architecture: "test-arch", + runtime_mode: runtime.mode, + developer_fallback_allowed: runtime.developer_fallback_allowed, + roots: RootDiagnostics { + config: "/config".into(), + data: "/data".into(), + cache: "/cache".into(), + assets: "/assets".into(), + staging: "/staging".into(), + }, + linux: None, + }) + .unwrap(); + assert_eq!(value["runtimeMode"], runtime.mode); + assert_eq!( + value["developerFallbackAllowed"], + runtime.developer_fallback_allowed + ); + assert!(value.get("runtime_mode").is_none()); + } +} diff --git a/src-tauri/src/platform_paths.rs b/src-tauri/src/platform_paths.rs new file mode 100644 index 0000000..0b8a5a2 --- /dev/null +++ b/src-tauri/src/platform_paths.rs @@ -0,0 +1,735 @@ +//! Host-owned cross-platform filesystem contract (issue #107). +//! +//! Rust resolves every application root once, before any Python process starts. +//! Children inherit the explicit `LSDJ_*_HOME` variables installed by +//! [`configure`]; Python must consume those values rather than reconstructing an +//! operating-system path from `$HOME`. +//! +//! The roots have deliberately different ownership: +//! - `config`: small durable settings and credentials. +//! - `data`: user-created songs, samples, and registries. +//! - `cache`: disposable, reproducible files. +//! - `assets`: downloaded model weights and runtimes. +//! - `staging`: incomplete downloads/installs. It is on the same filesystem as +//! `assets`, so a validated future installer can promote atomically. + +use std::ffi::OsString; +use std::io; +use std::path::{Path, PathBuf}; +use std::sync::OnceLock; + +use tauri::Manager; + +const APP_ID: &str = "works.protocol.lsdj"; +const APP_NAME: &str = "LSDJ"; +const APP_SLUG: &str = "lsdj"; + +/// The operating-system families whose path policy differs. Kept independent +/// of `cfg!` so every mapping is unit-tested on every CI host. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[allow(dead_code)] // non-host variants are exercised by portable unit tests +enum Platform { + MacOs, + Windows, + Linux, +} + +/// Platform-native unscoped directories. Production gets these from Tauri's +/// directory resolver (known-folder APIs on Windows, XDG on Linux); tests feed +/// synthetic roots so spaces and non-ASCII profiles are covered everywhere. +#[derive(Clone, Debug)] +struct NativeDirs { + home: PathBuf, + config: PathBuf, + data: PathBuf, + local_data: PathBuf, + cache: PathBuf, + documents: PathBuf, +} + +/// The canonical application roots plus the actual backend asset locations. +/// The latter can temporarily point at a legacy macOS directory if an old +/// install cannot be renamed (for example because of permissions), ensuring a +/// migration failure never makes existing models disappear. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct AppPaths { + config: PathBuf, + data: PathBuf, + cache: PathBuf, + assets: PathBuf, + staging: PathBuf, + magenta_base: PathBuf, + sa3_home: PathBuf, + loras_home: PathBuf, + legacy_data: Option, +} + +impl AppPaths { + pub fn config(&self) -> &Path { + &self.config + } + + pub fn data(&self) -> &Path { + &self.data + } + + pub fn cache(&self) -> &Path { + &self.cache + } + + pub fn assets(&self) -> &Path { + &self.assets + } + + pub fn staging(&self) -> &Path { + &self.staging + } + + pub fn magenta_base(&self) -> &Path { + &self.magenta_base + } + + pub fn sa3_home(&self) -> &Path { + &self.sa3_home + } + + pub fn loras_home(&self) -> &Path { + &self.loras_home + } + + fn managed_runtime_temp(&self) -> PathBuf { + self.cache.join("managed-runtime-tmp") + } + + /// The old macOS Documents brand root, used only to migrate generated + /// libraries independently when a destination already contains other data. + pub fn legacy_data(&self) -> Option<&Path> { + self.legacy_data.as_deref() + } + + pub(crate) fn backend_env(&self) -> [(OsString, OsString); 9] { + [ + pair("LSDJ_CONFIG_HOME", &self.config), + pair("LSDJ_DATA_HOME", &self.data), + pair("LSDJ_CACHE_HOME", &self.cache), + pair("LSDJ_ASSETS_HOME", &self.assets), + pair("LSDJ_STAGING_HOME", &self.staging), + // Compatibility variables consumed by upstream magenta-rt-v2 and + // the current SA3 entry points. Rust remains their source of truth. + pair("MAGENTA_HOME", &self.magenta_base), + pair("SA3_HOME", &self.sa3_home), + pair("SA3_MLX_HOME", &self.sa3_home), + pair("SA3_LORAS_HOME", &self.loras_home), + ] + } + + /// Environment that an absolute-path managed child still needs after + /// `env_clear`. Unix needs nothing. Windows gets its canonical system root + /// from the host directory API for runtime/DLL discovery and uses a checked, + /// app-owned temp directory; never inherit `PATH`, a profile home, + /// credentials, or developer overrides. + pub(crate) fn managed_child_env(&self) -> Result, String> { + managed_child_env_for_current_host(&self.managed_runtime_temp()) + } +} + +pub(crate) fn managed_child_env_for_current_host( + safe_temp: &Path, +) -> Result, String> { + managed_child_env_for(platform(), safe_temp, windows_system_root) +} + +fn managed_child_env_for( + platform: Platform, + safe_temp: &Path, + system_root: impl FnOnce() -> Result, +) -> Result, String> { + if platform != Platform::Windows { + return Ok(Vec::new()); + } + let system_root = std::fs::canonicalize(PathBuf::from(system_root()?)) + .map_err(|error| format!("cannot resolve Windows system root: {error}"))?; + require_real_directory(&system_root, "Windows system root")?; + let safe_parent = safe_temp + .parent() + .ok_or("managed runtime temp directory has no app-owned parent")?; + std::fs::create_dir_all(safe_parent) + .map_err(|error| format!("cannot create managed runtime temp parent: {error}"))?; + require_real_directory(safe_parent, "managed runtime temp parent")?; + match std::fs::create_dir(safe_temp) { + Ok(()) => {} + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => {} + Err(error) => { + return Err(format!( + "cannot create managed runtime temp directory: {error}" + )) + } + } + require_real_directory(safe_temp, "managed runtime temp directory")?; + let safe_parent = std::fs::canonicalize(safe_parent) + .map_err(|error| format!("cannot resolve managed runtime temp parent: {error}"))?; + let safe_temp = std::fs::canonicalize(safe_temp) + .map_err(|error| format!("cannot resolve managed runtime temp directory: {error}"))?; + if safe_temp.parent() != Some(safe_parent.as_path()) { + return Err("managed runtime temp directory escapes its app-owned parent".into()); + } + Ok(vec![ + pair("SYSTEMROOT", &system_root), + pair("WINDIR", &system_root), + pair("TEMP", &safe_temp), + pair("TMP", &safe_temp), + ]) +} + +fn require_real_directory(path: &Path, label: &str) -> Result<(), String> { + let metadata = std::fs::symlink_metadata(path) + .map_err(|error| format!("cannot inspect {label}: {error}"))?; + if !metadata.is_dir() || metadata.file_type().is_symlink() || is_reparse_point(&metadata) { + return Err(format!("{label} is not a real directory")); + } + Ok(()) +} + +#[cfg(windows)] +fn is_reparse_point(metadata: &std::fs::Metadata) -> bool { + use std::os::windows::fs::MetadataExt; + + const FILE_ATTRIBUTE_REPARSE_POINT: u32 = 0x400; + metadata.file_attributes() & FILE_ATTRIBUTE_REPARSE_POINT != 0 +} + +#[cfg(not(windows))] +fn is_reparse_point(_: &std::fs::Metadata) -> bool { + false +} + +#[cfg(windows)] +fn windows_system_root() -> Result { + use std::os::windows::ffi::OsStringExt; + use windows_sys::Win32::System::SystemInformation::GetWindowsDirectoryW; + + let mut buffer = vec![0_u16; 260]; + loop { + // SAFETY: `buffer` is writable for the advertised length. The API + // returns either the number of UTF-16 code units written (excluding + // NUL), the required capacity, or zero with a Win32 error. + let length = unsafe { GetWindowsDirectoryW(buffer.as_mut_ptr(), buffer.len() as u32) }; + if length == 0 { + return Err(format!( + "Windows directory API failed: {}", + io::Error::last_os_error() + )); + } + let length = length as usize; + if length < buffer.len() { + return Ok(OsString::from_wide(&buffer[..length])); + } + if length >= 32_768 { + return Err("Windows directory API returned an invalid path length".into()); + } + buffer.resize(length + 1, 0); + } +} + +#[cfg(not(windows))] +fn windows_system_root() -> Result { + Err("Windows directory API is unavailable on this host".into()) +} + +fn pair(name: &str, value: &Path) -> (OsString, OsString) { + (OsString::from(name), value.as_os_str().to_owned()) +} + +fn platform() -> Platform { + #[cfg(target_os = "macos")] + return Platform::MacOs; + #[cfg(target_os = "windows")] + return Platform::Windows; + #[cfg(all(not(target_os = "macos"), not(target_os = "windows")))] + return Platform::Linux; +} + +fn resolve(platform: Platform, native: NativeDirs) -> AppPaths { + let (config, data, cache, assets, staging, legacy_data) = match platform { + Platform::MacOs => { + // Preserve all current user-visible locations: generated media stays + // in Documents/LSDJ, model assets stay in Application Support/LSDJ, + // and shell settings remain under Tauri's bundle identifier. + let assets = native.data.join(APP_NAME); + ( + native.config.join(APP_ID), + native.documents.join(APP_NAME), + native.cache.join(APP_ID), + assets.clone(), + assets.join(".staging"), + Some(native.documents.join("LSDJai")), + ) + } + Platform::Windows => { + // Everything is non-roaming and deliberately shallow. `LSDJ` + // replaces the reverse-DNS identifier to preserve path budget for + // Python environments and model filenames when long paths are off. + let base = native.local_data.join(APP_NAME); + ( + base.join("config"), + base.join("data"), + base.join("cache"), + base.join("assets"), + base.join("staging"), + None, + ) + } + Platform::Linux => { + // Tauri's native bases follow XDG_CONFIG_HOME, XDG_DATA_HOME, and + // XDG_CACHE_HOME, falling back respectively to ~/.config, + // ~/.local/share, and ~/.cache when variables are absent/invalid. + let data = native.data.join(APP_SLUG); + ( + native.config.join(APP_SLUG), + data.clone(), + native.cache.join(APP_SLUG), + data.join("assets"), + data.join("staging"), + None, + ) + } + }; + let sa3_home = match platform { + Platform::MacOs => assets.join("stable-audio-3"), + Platform::Windows | Platform::Linux => { + crate::managed_runtime::service_root(&assets, crate::managed_runtime::Service::Sa3) + } + }; + AppPaths { + magenta_base: assets.clone(), + sa3_home, + loras_home: assets.join("sa3-loras"), + config, + data, + cache, + assets, + staging, + legacy_data, + } +} + +fn native_dirs( + app: &tauri::App, + platform: Platform, +) -> Result> { + let paths = app.path(); + match platform { + Platform::MacOs => { + let data = paths.data_dir()?; + Ok(NativeDirs { + home: paths.home_dir().unwrap_or_else(|_| data.clone()), + config: paths.config_dir()?, + local_data: data.clone(), + cache: paths.cache_dir()?, + documents: paths.document_dir().unwrap_or_else(|_| data.clone()), + data, + }) + } + Platform::Windows => { + let local_data = paths.local_data_dir()?; + Ok(NativeDirs { + home: local_data.clone(), + config: local_data.clone(), + data: local_data.clone(), + cache: local_data.clone(), + documents: local_data.clone(), + local_data, + }) + } + Platform::Linux => { + let data = paths.data_dir()?; + Ok(NativeDirs { + home: data.clone(), + config: paths.config_dir()?, + local_data: data.clone(), + cache: paths.cache_dir()?, + documents: data.clone(), + data, + }) + } + } +} + +static CONFIGURED: OnceLock = OnceLock::new(); + +/// Resolve, prepare, and publish the path contract. This must be called in +/// Tauri setup before starting any sidecar, server, watcher, or installer. +pub fn configure(app: &tauri::App) -> Result<&'static AppPaths, Box> { + if let Some(paths) = CONFIGURED.get() { + return Ok(paths); + } + let platform = platform(); + let native = native_dirs(app, platform)?; + let mut paths = resolve(platform, native.clone()); + prepare_roots(&paths)?; + if platform == Platform::MacOs { + migrate_macos(&mut paths, &native); + } + + // Explicit dev/user overrides remain supported, but the resolved value is + // captured into the host contract and then passed to every child. + if let Some(value) = nonempty_env("MAGENTA_HOME") { + paths.magenta_base = PathBuf::from(value); + } + if let Some(value) = nonempty_env("SA3_MLX_HOME") { + paths.sa3_home = PathBuf::from(value); + } + if let Some(value) = nonempty_env("SA3_LORAS_HOME") { + paths.loras_home = PathBuf::from(value); + } + + for (name, value) in paths.backend_env() { + std::env::set_var(name, value); + } + CONFIGURED + .set(paths) + .map_err(|_| io::Error::new(io::ErrorKind::AlreadyExists, "paths already configured"))?; + Ok(CONFIGURED.get().expect("paths were just configured")) +} + +fn nonempty_env(name: &str) -> Option { + std::env::var_os(name).filter(|value| !value.is_empty()) +} + +fn prepare_roots(paths: &AppPaths) -> io::Result<()> { + for dir in [ + paths.config(), + paths.data(), + paths.cache(), + paths.assets(), + paths.staging(), + ] { + std::fs::create_dir_all(dir)?; + } + Ok(()) +} + +/// Return the configured contract. Calling this before [`configure`] is a +/// programming error: doing so would reintroduce independent path guessing. +pub fn get() -> &'static AppPaths { + CONFIGURED + .get() + .expect("platform paths must be configured before services start") +} + +/// Resolve the interpreter in a virtual environment without assuming Unix's +/// `bin/python` layout on Windows. +pub fn venv_python(venv: &Path) -> PathBuf { + venv_python_for(platform(), venv) +} + +fn venv_python_for(platform: Platform, venv: &Path) -> PathBuf { + match platform { + Platform::Windows => venv.join("Scripts").join("python.exe"), + Platform::MacOs | Platform::Linux => venv.join("bin").join("python"), + } +} + +/// Move one legacy directory only when its destination is absent. `rename` is +/// atomic on the same filesystem; after success, a restart observes the +/// destination and does nothing. On failure the caller receives the old path, +/// keeping existing user data visible. +pub fn migrate_legacy_dir(new_dir: &Path, old_dir: &Path) -> PathBuf { + if new_dir.exists() || !old_dir.is_dir() { + return new_dir.to_path_buf(); + } + if let Some(parent) = new_dir.parent() { + if let Err(error) = std::fs::create_dir_all(parent) { + eprintln!( + "lsdj-app: could not prepare data directory {}: {error}", + parent.display() + ); + return old_dir.to_path_buf(); + } + } + match std::fs::rename(old_dir, new_dir) { + Ok(()) => { + eprintln!("lsdj-app: migrated data → {}", new_dir.display()); + new_dir.to_path_buf() + } + Err(error) => { + eprintln!( + "lsdj-app: could not migrate {} to {}: {error}", + old_dir.display(), + new_dir.display() + ); + old_dir.to_path_buf() + } + } +} + +fn migrate_macos(paths: &mut AppPaths, native: &NativeDirs) { + let old_brand = native.data.join("LSDJai"); + let new_magenta = paths.assets.join("magenta-rt-v2"); + let branded_magenta = migrate_legacy_dir(&new_magenta, &old_brand.join("magenta-rt-v2")); + let branded_sa3 = migrate_legacy_dir( + &paths.assets.join("stable-audio-3"), + &old_brand.join("stable-audio-3"), + ); + let branded_loras = migrate_legacy_dir( + &paths.assets.join("sa3-loras"), + &old_brand.join("sa3-loras"), + ); + + paths.sa3_home = branded_sa3; + paths.loras_home = branded_loras; + if branded_magenta.starts_with(&old_brand) { + paths.magenta_base = old_brand; + return; + } + + // Pre-model-manager installs lived under Documents/Magenta. Preserve the + // old location if a same-volume rename cannot complete. + let old_magenta_base = native.home.join("Documents").join("Magenta"); + let migrated = migrate_legacy_dir(&new_magenta, &old_magenta_base.join("magenta-rt-v2")); + paths.magenta_base = migrated + .parent() + .map(Path::to_path_buf) + .unwrap_or_else(|| paths.assets.clone()); +} + +#[cfg(test)] +mod tests { + use super::*; + + fn native(profile: &str) -> NativeDirs { + let home = PathBuf::from(profile); + NativeDirs { + config: home.join("config base"), + data: home.join("data base"), + local_data: home.join("local data"), + cache: home.join("cache base"), + documents: home.join("Documents"), + home, + } + } + + #[test] + fn macos_preserves_existing_user_visible_locations() { + let roots = resolve(Platform::MacOs, native("/Users/DJ Name")); + assert_eq!(roots.data, Path::new("/Users/DJ Name/Documents/LSDJ")); + assert_eq!(roots.assets, Path::new("/Users/DJ Name/data base/LSDJ")); + assert_eq!( + roots.config, + Path::new("/Users/DJ Name/config base/works.protocol.lsdj") + ); + } + + #[test] + fn windows_roots_are_non_roaming_shallow_and_unicode_safe() { + let roots = resolve(Platform::Windows, native(r"C:\Users\Zoë 王")); + let base = Path::new(r"C:\Users\Zoë 王") + .join("local data") + .join("LSDJ"); + assert_eq!(roots.config, base.join("config")); + assert_eq!(roots.data, base.join("data")); + assert_eq!(roots.cache, base.join("cache")); + assert_eq!(roots.assets, base.join("assets")); + assert_eq!(roots.staging, base.join("staging")); + assert!(!roots.config.to_string_lossy().contains(APP_ID)); + } + + #[test] + fn linux_roots_follow_xdg_native_bases_and_keep_staging_with_assets() { + let roots = resolve(Platform::Linux, native("/home/DJ 名")); + assert_eq!(roots.config, Path::new("/home/DJ 名/config base/lsdj")); + assert_eq!(roots.data, Path::new("/home/DJ 名/data base/lsdj")); + assert_eq!(roots.cache, Path::new("/home/DJ 名/cache base/lsdj")); + assert_eq!(roots.assets, roots.data.join("assets")); + assert_eq!(roots.staging, roots.data.join("staging")); + } + + #[test] + fn linux_standard_xdg_fallbacks_are_scoped_to_lsdj() { + let home = PathBuf::from("/home/DJ Name"); + let roots = resolve( + Platform::Linux, + NativeDirs { + config: home.join(".config"), + data: home.join(".local/share"), + local_data: home.join(".local/share"), + cache: home.join(".cache"), + documents: home.clone(), + home: home.clone(), + }, + ); + assert_eq!(roots.config, home.join(".config/lsdj")); + assert_eq!(roots.data, home.join(".local/share/lsdj")); + assert_eq!(roots.cache, home.join(".cache/lsdj")); + } + + #[test] + fn venv_python_handles_both_layouts_without_parsing_a_command_string() { + let root = Path::new("/profiles/DJ Name/模型/.venv"); + assert_eq!( + venv_python_for(Platform::Windows, root), + root.join("Scripts").join("python.exe") + ); + assert_eq!( + venv_python_for(Platform::Linux, root), + root.join("bin").join("python") + ); + } + + #[test] + fn backend_environment_preserves_spaces_and_non_ascii() { + let roots = resolve(Platform::Linux, native("/home/DJ Name/音楽")); + let values: std::collections::HashMap<_, _> = roots.backend_env().into_iter().collect(); + assert_eq!( + values + .get(std::ffi::OsStr::new("LSDJ_ASSETS_HOME")) + .map(OsString::as_os_str), + Some(roots.assets.as_os_str()), + ); + assert_eq!( + values + .get(std::ffi::OsStr::new("SA3_HOME")) + .map(OsString::as_os_str), + Some(roots.sa3_home.as_os_str()), + ); + assert_eq!( + values + .get(std::ffi::OsStr::new("SA3_MLX_HOME")) + .map(OsString::as_os_str), + Some(roots.sa3_home.as_os_str()), + ); + } + + #[test] + fn managed_windows_children_receive_only_system_root_and_app_owned_temp() { + let root = std::env::temp_dir().join(format!( + "lsdj-managed-child-env-{}-{:?}", + std::process::id(), + std::thread::current().id() + )); + let _ = std::fs::remove_dir_all(&root); + let system_root = root.join("Windows Root"); + let safe_temp = root.join("cache").join("managed tmp"); + std::fs::create_dir_all(&system_root).unwrap(); + + let environment = managed_child_env_for(Platform::Windows, &safe_temp, || { + Ok(system_root.as_os_str().to_owned()) + }) + .unwrap(); + let values: std::collections::HashMap<_, _> = environment.into_iter().collect(); + let canonical_root = std::fs::canonicalize(&system_root).unwrap(); + let canonical_temp = std::fs::canonicalize(&safe_temp).unwrap(); + assert_eq!(values.len(), 4); + for name in ["SYSTEMROOT", "WINDIR"] { + assert_eq!( + values.get(std::ffi::OsStr::new(name)), + Some(&canonical_root.as_os_str().to_owned()) + ); + } + for name in ["TEMP", "TMP"] { + assert_eq!( + values.get(std::ffi::OsStr::new(name)), + Some(&canonical_temp.as_os_str().to_owned()) + ); + } + for forbidden in [ + "PATH", + "HOME", + "HF_TOKEN", + "HUGGING_FACE_HUB_TOKEN", + "PYTHONPATH", + "LSDJ_GENERATION_CMD", + "LSDJ_SIDECAR_CMD", + ] { + assert!(!values.contains_key(std::ffi::OsStr::new(forbidden))); + } + let _ = std::fs::remove_dir_all(root); + } + + #[test] + fn non_windows_managed_children_inherit_no_host_environment() { + let safe_temp = Path::new("/unused-managed-temp"); + for platform in [Platform::MacOs, Platform::Linux] { + assert!(managed_child_env_for(platform, safe_temp, || { + panic!("non-Windows launch must not query the Windows directory API") + }) + .unwrap() + .is_empty()); + } + } + + #[cfg(unix)] + #[test] + fn managed_windows_temp_rejects_a_link_escape() { + use std::os::unix::fs::symlink; + + let root = std::env::temp_dir().join(format!( + "lsdj-managed-temp-link-{}-{:?}", + std::process::id(), + std::thread::current().id() + )); + let _ = std::fs::remove_dir_all(&root); + let system_root = root.join("Windows Root"); + let safe_parent = root.join("cache"); + let outside = root.join("outside"); + let safe_temp = safe_parent.join("managed tmp"); + std::fs::create_dir_all(&system_root).unwrap(); + std::fs::create_dir_all(&safe_parent).unwrap(); + std::fs::create_dir_all(&outside).unwrap(); + symlink(&outside, &safe_temp).unwrap(); + + let error = managed_child_env_for(Platform::Windows, &safe_temp, || { + Ok(system_root.as_os_str().to_owned()) + }) + .unwrap_err(); + assert!(error.contains("not a real directory")); + let _ = std::fs::remove_dir_all(root); + } + + #[cfg(windows)] + #[test] + fn windows_system_root_comes_from_the_host_directory_api() { + let root = PathBuf::from(windows_system_root().unwrap()); + assert!(root.is_absolute()); + assert!(root.is_dir()); + } + + #[test] + fn migration_is_atomic_and_restart_safe() { + let root = std::env::temp_dir().join(format!( + "lsdj-path-migrate-{}-{}", + std::process::id(), + "音楽 folder" + )); + let old = root.join("old brand").join("models"); + let new = root.join("new brand").join("models"); + let _ = std::fs::remove_dir_all(&root); + std::fs::create_dir_all(&old).unwrap(); + std::fs::write(old.join("weight.bin"), b"model").unwrap(); + + assert_eq!(migrate_legacy_dir(&new, &old), new); + assert_eq!(std::fs::read(new.join("weight.bin")).unwrap(), b"model"); + // A restart is a no-op and leaves the promoted destination intact. + assert_eq!(migrate_legacy_dir(&new, &old), new); + assert!(!old.exists()); + let _ = std::fs::remove_dir_all(&root); + } + + #[test] + fn failed_migration_keeps_the_legacy_directory_active() { + let root = + std::env::temp_dir().join(format!("lsdj-path-migrate-failure-{}", std::process::id())); + let old = root.join("legacy").join("models"); + let blocked_parent = root.join("blocked"); + let new = blocked_parent.join("models"); + let _ = std::fs::remove_dir_all(&root); + std::fs::create_dir_all(&old).unwrap(); + std::fs::write(old.join("weight.bin"), b"model").unwrap(); + std::fs::write(&blocked_parent, b"not a directory").unwrap(); + + assert_eq!(migrate_legacy_dir(&new, &old), old); + assert_eq!(std::fs::read(old.join("weight.bin")).unwrap(), b"model"); + assert!(!new.exists()); + let _ = std::fs::remove_dir_all(&root); + } +} diff --git a/src-tauri/src/runtime_installer/archive.rs b/src-tauri/src/runtime_installer/archive.rs new file mode 100644 index 0000000..d81bb27 --- /dev/null +++ b/src-tauri/src/runtime_installer/archive.rs @@ -0,0 +1,1026 @@ +//! Strict native `.tar.gz` extraction for authenticated installer artifacts. + +use std::collections::HashSet; +use std::fs::{self, File, OpenOptions}; +use std::io::{self, Read, Seek, Write}; +use std::path::{Path, PathBuf}; + +use flate2::read::GzDecoder; + +#[derive(Clone, Copy, Debug)] +pub(crate) struct ArchiveLimits { + pub(crate) max_files: u64, + pub(crate) max_expanded_bytes: u64, + /// Some signed runtime distributions contain convenience symlinks. When + /// enabled, safe in-root file links are materialized as regular copies; + /// no archive-controlled link is ever created on disk. + pub(crate) materialize_safe_links: bool, +} + +/// Extract a gzip-compressed tar whose every entry must live below one exact +/// top-level directory. Links and non-file/directory entries are never created. +/// The caller supplies limits from the authenticated app manifest. +#[cfg(test)] +pub(crate) fn extract_tar_gz( + source: impl Read, + destination: &Path, + expected_root: &str, + limits: ArchiveLimits, +) -> Result<(), String> { + extract_tar_gz_cancellable(source, destination, expected_root, limits, &|| false) +} + +/// Cancellation-aware production entry point. The cancellation reader sits +/// below gzip so it also interrupts decoder reads of archive metadata, while +/// file/link copies check between bounded chunks. +pub(crate) fn extract_tar_gz_cancellable( + source: impl Read, + destination: &Path, + expected_root: &str, + limits: ArchiveLimits, + is_cancelled: &dyn Fn() -> bool, +) -> Result<(), String> { + validate_root(expected_root)?; + if limits.max_files == 0 || limits.max_expanded_bytes == 0 { + return Err("archive limits must be non-zero".into()); + } + prepare_empty_destination(destination)?; + + let metadata_allowance = limits + .max_files + .checked_mul(4096) + .and_then(|bytes| bytes.checked_add(1024 * 1024)) + .ok_or("archive decompression limit overflow")?; + let decompressed_limit = limits + .max_expanded_bytes + .checked_add(metadata_allowance) + .ok_or("archive decompression limit overflow")?; + let source = CancellationReader { + inner: source, + is_cancelled, + }; + let decoder = GzDecoder::new(source); + // This counts every decompressed tar byte, including GNU long-name and + // local PAX records that the tar crate consumes before yielding an entry. + let decoder = BudgetReader::new(decoder, decompressed_limit); + let mut archive = tar::Archive::new(decoder); + let entries = archive + .entries() + .map_err(|error| format!("cannot read archive: {error}"))?; + let mut count = 0u64; + let mut materialized_count = 0u64; + let mut expanded = 0u64; + let mut seen = HashSet::new(); + let mut materialized = HashSet::new(); + let mut deferred_links = Vec::new(); + + for entry in entries { + if is_cancelled() { + return Err("cancelled".into()); + } + let mut entry = entry.map_err(|error| format!("cannot read archive entry: {error}"))?; + count = count.checked_add(1).ok_or("archive file count overflow")?; + if count > limits.max_files { + return Err(format!( + "archive contains more than {} entries", + limits.max_files + )); + } + + let entry_type = entry.header().entry_type(); + if entry_type.is_pax_global_extensions() { + // `tar` applies local PAX/GNU path extensions to the following + // entry before yielding it. Global headers are yielded separately; + // they contain metadata only, never filesystem content. Count and + // bound them, then discard them without trusting path-like keys. + let size = entry + .header() + .size() + .map_err(|error| format!("archive metadata has invalid size: {error}"))?; + expanded = expanded + .checked_add(size) + .ok_or("archive expanded size overflow")?; + if expanded > limits.max_expanded_bytes { + return Err(format!( + "archive expands beyond {} bytes", + limits.max_expanded_bytes + )); + } + copy_cancellable(&mut entry, &mut io::sink(), is_cancelled) + .map_err(|error| format!("cannot read archive metadata: {error}"))?; + continue; + } + let is_link = entry_type.is_symlink() || entry_type.is_hard_link(); + if is_link && !limits.materialize_safe_links { + return Err("archive links are not permitted".into()); + } + if !(entry_type.is_file() || entry_type.is_dir() || is_link) { + return Err(format!( + "archive contains unsupported special entry type {:?}", + entry_type + )); + } + + let relative = checked_relative_path(&entry, expected_root)?; + // Windows/macOS default filesystems are case-insensitive. Reject an + // archive that is only unambiguous on a case-sensitive extraction host. + let key = relative.to_string_lossy().to_lowercase(); + if !seen.insert(key) { + return Err("archive contains a duplicate path".into()); + } + for ancestor in relative + .ancestors() + .filter(|path| !path.as_os_str().is_empty()) + { + let key = ancestor.to_string_lossy().to_lowercase(); + if materialized.insert(key) { + materialized_count = materialized_count + .checked_add(1) + .ok_or("archive materialized file count overflow")?; + if materialized_count > limits.max_files { + return Err(format!( + "archive materializes more than {} filesystem entries", + limits.max_files + )); + } + } + } + let output = destination.join(&relative); + + if is_link { + let target = + checked_link_target(&entry, &relative, expected_root, entry_type.is_hard_link())?; + deferred_links.push((output, destination.join(target))); + continue; + } + + if entry_type.is_dir() { + fs::create_dir_all(&output) + .map_err(|error| format!("cannot create archive directory: {error}"))?; + set_safe_permissions(&output, true, false)?; + continue; + } + if relative.as_os_str().is_empty() { + return Err("archive root must be a directory".into()); + } + + let size = entry + .header() + .size() + .map_err(|error| format!("archive entry has invalid size: {error}"))?; + expanded = expanded + .checked_add(size) + .ok_or("archive expanded size overflow")?; + if expanded > limits.max_expanded_bytes { + return Err(format!( + "archive expands beyond {} bytes", + limits.max_expanded_bytes + )); + } + if let Some(parent) = output.parent() { + fs::create_dir_all(parent) + .map_err(|error| format!("cannot create archive parent: {error}"))?; + } + let mut file = OpenOptions::new() + .create_new(true) + .write(true) + .open(&output) + .map_err(|error| format!("cannot create archive file: {error}"))?; + let copied = copy_cancellable(&mut entry, &mut file, is_cancelled) + .map_err(|error| format!("cannot extract archive file: {error}"))?; + if copied != size { + return Err("archive entry ended before its declared size".into()); + } + let executable = entry.header().mode().unwrap_or(0) & 0o111 != 0; + set_safe_permissions(&output, false, executable)?; + } + + if count == 0 { + return Err("archive is empty".into()); + } + for (output, target) in deferred_links { + if is_cancelled() { + return Err("cancelled".into()); + } + let metadata = fs::metadata(&target) + .map_err(|_| "archive link target is missing or not a regular file".to_string())?; + if !metadata.is_file() { + return Err("archive link target is not a regular file".into()); + } + expanded = expanded + .checked_add(metadata.len()) + .ok_or("archive expanded size overflow")?; + if expanded > limits.max_expanded_bytes { + return Err(format!( + "archive expands beyond {} bytes", + limits.max_expanded_bytes + )); + } + if let Some(parent) = output.parent() { + fs::create_dir_all(parent) + .map_err(|error| format!("cannot create archive link parent: {error}"))?; + } + let mut source = File::open(&target) + .map_err(|error| format!("cannot open safe archive link target: {error}"))?; + let mut destination = OpenOptions::new() + .create_new(true) + .write(true) + .open(&output) + .map_err(|error| format!("cannot create materialized archive link: {error}"))?; + copy_cancellable(&mut source, &mut destination, is_cancelled) + .map_err(|error| format!("cannot materialize safe archive link: {error}"))?; + set_safe_permissions(&output, false, is_executable(&target))?; + } + Ok(()) +} + +/// Extract a ZIP archive under the same portable path, count, and expanded +/// byte policy as [`extract_tar_gz_cancellable`]. Official uv Windows builds +/// use ZIP; archive-provided links and special files remain forbidden. +pub(crate) fn extract_zip_cancellable( + source: impl Read + Seek, + destination: &Path, + expected_root: &str, + limits: ArchiveLimits, + is_cancelled: &dyn Fn() -> bool, +) -> Result<(), String> { + validate_root(expected_root)?; + if limits.max_files == 0 || limits.max_expanded_bytes == 0 { + return Err("archive limits must be non-zero".into()); + } + prepare_empty_destination(destination)?; + let mut archive = zip::ZipArchive::new(source) + .map_err(|error| format!("cannot read ZIP archive: {error}"))?; + if archive.is_empty() { + return Err("archive is empty".into()); + } + if archive.len() as u64 > limits.max_files { + return Err(format!( + "archive contains more than {} entries", + limits.max_files + )); + } + + let mut expanded = 0u64; + let mut seen = HashSet::new(); + let mut materialized = HashSet::new(); + let mut materialized_count = 0u64; + for index in 0..archive.len() { + if is_cancelled() { + return Err("cancelled".into()); + } + let mut entry = archive + .by_index(index) + .map_err(|error| format!("cannot read ZIP entry: {error}"))?; + let raw = entry.name_raw(); + if raw.is_empty() + || raw[0] == b'/' + || raw[0] == b'\\' + || raw.iter().any(|byte| *byte == b'\\' || *byte == 0) + { + return Err("archive contains an absolute or platform-ambiguous path".into()); + } + let text = + std::str::from_utf8(raw).map_err(|_| "archive path is not valid UTF-8".to_string())?; + let relative = checked_text_relative_path(text, expected_root)?; + let key = relative.to_string_lossy().to_lowercase(); + if !seen.insert(key) { + return Err("archive contains a duplicate path".into()); + } + for ancestor in relative + .ancestors() + .filter(|path| !path.as_os_str().is_empty()) + { + let key = ancestor.to_string_lossy().to_lowercase(); + if materialized.insert(key) { + materialized_count = materialized_count + .checked_add(1) + .ok_or("archive materialized file count overflow")?; + if materialized_count > limits.max_files { + return Err(format!( + "archive materializes more than {} filesystem entries", + limits.max_files + )); + } + } + } + + let unix_kind = entry.unix_mode().unwrap_or(0) & 0o170000; + if unix_kind == 0o120000 || !(entry.is_file() || entry.is_dir()) { + return Err("archive contains a link or unsupported special entry".into()); + } + let output = destination.join(&relative); + if entry.is_dir() { + fs::create_dir_all(&output) + .map_err(|error| format!("cannot create archive directory: {error}"))?; + set_safe_permissions(&output, true, false)?; + continue; + } + if relative.as_os_str().is_empty() { + return Err("archive root must be a directory".into()); + } + expanded = expanded + .checked_add(entry.size()) + .ok_or("archive expanded size overflow")?; + if expanded > limits.max_expanded_bytes { + return Err(format!( + "archive expands beyond {} bytes", + limits.max_expanded_bytes + )); + } + if let Some(parent) = output.parent() { + fs::create_dir_all(parent) + .map_err(|error| format!("cannot create archive parent: {error}"))?; + } + let mut file = OpenOptions::new() + .create_new(true) + .write(true) + .open(&output) + .map_err(|error| format!("cannot create archive file: {error}"))?; + let copied = copy_cancellable(&mut entry, &mut file, is_cancelled) + .map_err(|error| format!("cannot extract archive file: {error}"))?; + if copied != entry.size() { + return Err("archive entry ended before its declared size".into()); + } + let executable = entry.unix_mode().unwrap_or(0) & 0o111 != 0; + set_safe_permissions(&output, false, executable)?; + } + Ok(()) +} + +struct CancellationReader<'a, R> { + inner: R, + is_cancelled: &'a dyn Fn() -> bool, +} + +impl Read for CancellationReader<'_, R> { + fn read(&mut self, buffer: &mut [u8]) -> io::Result { + if (self.is_cancelled)() { + return Err(io::Error::new(io::ErrorKind::Interrupted, "cancelled")); + } + self.inner.read(buffer) + } +} + +struct BudgetReader { + inner: R, + remaining: u64, +} + +impl BudgetReader { + fn new(inner: R, limit: u64) -> Self { + Self { + inner, + remaining: limit, + } + } +} + +impl Read for BudgetReader { + fn read(&mut self, buffer: &mut [u8]) -> io::Result { + if buffer.is_empty() { + return Ok(0); + } + if self.remaining == 0 { + let mut probe = [0u8; 1]; + return match self.inner.read(&mut probe)? { + 0 => Ok(0), + _ => Err(io::Error::new( + io::ErrorKind::InvalidData, + "archive decompressed data exceeds its budget", + )), + }; + } + let allowed = usize::try_from(self.remaining.min(buffer.len() as u64)) + .expect("bounded by the buffer length"); + let count = self.inner.read(&mut buffer[..allowed])?; + self.remaining = self.remaining.saturating_sub(count as u64); + Ok(count) + } +} + +fn copy_cancellable( + reader: &mut impl Read, + writer: &mut impl Write, + is_cancelled: &dyn Fn() -> bool, +) -> io::Result { + let mut total = 0u64; + let mut buffer = [0u8; 128 * 1024]; + loop { + if is_cancelled() { + return Err(io::Error::new(io::ErrorKind::Interrupted, "cancelled")); + } + let count = reader.read(&mut buffer)?; + if count == 0 { + return Ok(total); + } + writer.write_all(&buffer[..count])?; + total = total + .checked_add(count as u64) + .ok_or_else(|| io::Error::other("archive copy byte count overflow"))?; + } +} + +fn validate_root(root: &str) -> Result<(), String> { + if root.is_empty() + || root == "." + || root == ".." + || root.contains('/') + || root.contains('\\') + || root.contains(':') + { + return Err("archive root is unsafe".into()); + } + Ok(()) +} + +fn checked_relative_path( + entry: &tar::Entry<'_, R>, + expected_root: &str, +) -> Result { + let raw = entry.path_bytes(); + if raw.is_empty() + || raw[0] == b'/' + || raw[0] == b'\\' + || raw.iter().any(|byte| *byte == b'\\' || *byte == 0) + { + return Err("archive contains an absolute or platform-ambiguous path".into()); + } + let text = + std::str::from_utf8(&raw).map_err(|_| "archive path is not valid UTF-8".to_string())?; + checked_text_relative_path(text, expected_root) +} + +fn checked_text_relative_path(text: &str, expected_root: &str) -> Result { + if text.len() > 4096 { + return Err("archive path exceeds the portable length limit".into()); + } + let mut segments: Vec<&str> = text.split('/').collect(); + if segments.last() == Some(&"") { + segments.pop(); + } + if segments.is_empty() || segments[0] != expected_root { + return Err("archive path escapes or does not match its pinned root".into()); + } + if segments.len() > 129 { + return Err("archive path exceeds the nesting limit".into()); + } + for segment in &segments { + validate_portable_segment(segment)?; + } + let mut relative = PathBuf::new(); + for segment in &segments[1..] { + relative.push(segment); + } + if relative.is_absolute() + || relative + .components() + .any(|component| !matches!(component, std::path::Component::Normal(_))) + { + return Err("archive path is not a safe relative path".into()); + } + Ok(relative) +} + +fn validate_portable_segment(segment: &str) -> Result<(), String> { + if segment.is_empty() + || segment == "." + || segment == ".." + || segment.ends_with(' ') + || segment.ends_with('.') + || segment.chars().any(|character| { + character <= '\u{1f}' || matches!(character, '<' | '>' | ':' | '"' | '|' | '?' | '*') + }) + { + return Err("archive path is not portable to Windows".into()); + } + let stem = segment + .split('.') + .next() + .unwrap_or_default() + .trim_end_matches([' ', '.']) + .to_ascii_uppercase(); + let reserved = matches!(stem.as_str(), "CON" | "PRN" | "AUX" | "NUL") + || stem + .strip_prefix("COM") + .or_else(|| stem.strip_prefix("LPT")) + .is_some_and(|number| number.len() == 1 && matches!(number.as_bytes()[0], b'1'..=b'9')); + if reserved { + return Err("archive path uses a reserved Windows device name".into()); + } + Ok(()) +} + +fn checked_link_target( + entry: &tar::Entry<'_, R>, + link_path: &Path, + expected_root: &str, + hard_link: bool, +) -> Result { + let target = entry + .link_name() + .map_err(|error| format!("archive link target is invalid: {error}"))? + .ok_or("archive link has no target")?; + let target = target + .to_str() + .ok_or("archive link target is not valid UTF-8")?; + if target.is_empty() + || target.starts_with('/') + || target.starts_with('\\') + || target.contains('\\') + { + return Err("archive link target is absolute or platform-ambiguous".into()); + } + + let mut resolved = if hard_link { + PathBuf::new() + } else { + link_path + .parent() + .unwrap_or_else(|| Path::new("")) + .to_path_buf() + }; + let mut segments = target.split('/').peekable(); + if hard_link && segments.peek() == Some(&expected_root) { + segments.next(); + } + for segment in segments { + match segment { + "" | "." => {} + ".." => { + if !resolved.pop() { + return Err("archive link target escapes its pinned root".into()); + } + } + normal => { + validate_portable_segment(normal)?; + resolved.push(normal); + } + } + } + if resolved.as_os_str().is_empty() { + return Err("archive link target does not name a file".into()); + } + Ok(resolved) +} + +#[cfg(unix)] +fn is_executable(path: &Path) -> bool { + use std::os::unix::fs::PermissionsExt; + fs::metadata(path) + .map(|metadata| metadata.permissions().mode() & 0o111 != 0) + .unwrap_or(false) +} + +#[cfg(not(unix))] +fn is_executable(_path: &Path) -> bool { + false +} + +fn prepare_empty_destination(destination: &Path) -> Result<(), String> { + match fs::symlink_metadata(destination) { + Ok(metadata) => { + if metadata.file_type().is_symlink() || !metadata.is_dir() { + return Err("archive destination is not a real directory".into()); + } + let mut entries = fs::read_dir(destination) + .map_err(|error| format!("cannot inspect archive destination: {error}"))?; + if entries.next().is_some() { + return Err("archive destination must be empty".into()); + } + } + Err(error) if error.kind() == io::ErrorKind::NotFound => { + fs::create_dir_all(destination) + .map_err(|error| format!("cannot create archive destination: {error}"))?; + } + Err(error) => { + return Err(format!("cannot inspect archive destination: {error}")); + } + } + Ok(()) +} + +#[cfg(unix)] +fn set_safe_permissions(path: &Path, directory: bool, executable: bool) -> Result<(), String> { + use std::os::unix::fs::PermissionsExt; + let mode = if directory || executable { + 0o755 + } else { + 0o644 + }; + fs::set_permissions(path, fs::Permissions::from_mode(mode)) + .map_err(|error| format!("cannot set archive entry permissions: {error}")) +} + +#[cfg(not(unix))] +fn set_safe_permissions(_path: &Path, _directory: bool, _executable: bool) -> Result<(), String> { + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use flate2::write::GzEncoder; + use flate2::Compression; + use std::io::Cursor; + + fn fixture(entries: &[(&str, tar::EntryType, &[u8], Option<&str>)]) -> Vec { + let encoder = GzEncoder::new(Vec::new(), Compression::fast()); + let mut builder = tar::Builder::new(encoder); + for (name, kind, body, link) in entries { + let mut header = tar::Header::new_gnu(); + header.set_entry_type(*kind); + header.set_mode(if kind.is_file() { 0o644 } else { 0o755 }); + header.set_size(body.len() as u64); + set_raw_name(&mut header, name); + if let Some(link) = link { + header.set_link_name(link).unwrap(); + } + header.set_cksum(); + builder.append(&header, *body).unwrap(); + } + let encoder = builder.into_inner().unwrap(); + encoder.finish().unwrap() + } + + fn zip_fixture(entries: &[(&str, &[u8])]) -> Vec { + let cursor = Cursor::new(Vec::new()); + let mut writer = zip::ZipWriter::new(cursor); + let options = zip::write::SimpleFileOptions::default() + .compression_method(zip::CompressionMethod::Deflated) + .unix_permissions(0o755); + for (name, body) in entries { + writer.start_file(*name, options).unwrap(); + writer.write_all(body).unwrap(); + } + writer.finish().unwrap().into_inner() + } + + fn set_raw_name(header: &mut tar::Header, name: &str) { + assert!(name.len() < 100); + header.as_mut_bytes()[..100].fill(0); + header.as_mut_bytes()[..name.len()].copy_from_slice(name.as_bytes()); + } + + fn temp(label: &str) -> PathBuf { + std::env::temp_dir().join(format!("lsdj-archive-{label}-{}", std::process::id())) + } + + fn limits() -> ArchiveLimits { + ArchiveLimits { + max_files: 10, + max_expanded_bytes: 1024, + materialize_safe_links: false, + } + } + + #[test] + fn extracts_regular_files_below_the_pinned_root() { + let bytes = fixture(&[ + ("root/", tar::EntryType::Directory, b"", None), + ("root/nested/", tar::EntryType::Directory, b"", None), + ( + "root/nested/file.txt", + tar::EntryType::Regular, + b"safe", + None, + ), + ]); + let out = temp("valid"); + let _ = fs::remove_dir_all(&out); + extract_tar_gz(&bytes[..], &out, "root", limits()).unwrap(); + assert_eq!(fs::read(out.join("nested/file.txt")).unwrap(), b"safe"); + let _ = fs::remove_dir_all(out); + } + + #[test] + fn rejects_parent_absolute_and_windows_ambiguous_paths() { + for (label, name) in [ + ("parent", "root/../../escape"), + ("absolute", "/root/escape"), + ("backslash", "root\\..\\escape"), + ("drive", "root/C:/escape"), + ] { + let bytes = fixture(&[(name, tar::EntryType::Regular, b"bad", None)]); + let out = temp(label); + let escaped = out.parent().unwrap().join("escape"); + let _ = fs::remove_dir_all(&out); + let _ = fs::remove_file(&escaped); + assert!(extract_tar_gz(&bytes[..], &out, "root", limits()).is_err()); + assert!(!escaped.exists()); + let _ = fs::remove_dir_all(out); + } + } + + #[test] + fn rejects_windows_reserved_invalid_and_case_colliding_paths() { + for (label, name) in [ + ("con", "root/CON"), + ("device-extension", "root/com1.txt"), + ("lpt", "root/LPT9.log"), + ("trailing-dot", "root/name."), + ("trailing-space", "root/name "), + ("invalid-char", "root/na= 2, + ); + assert!( + result.unwrap_err().contains("cancelled"), + "cancellation should remain identifiable" + ); + let _ = fs::remove_dir_all(out); + } + + #[test] + fn bounds_hidden_gnu_long_name_metadata_before_tar_materializes_it() { + let encoder = GzEncoder::new(Vec::new(), Compression::fast()); + let mut builder = tar::Builder::new(encoder); + let mut header = tar::Header::new_gnu(); + header.set_entry_type(tar::EntryType::Regular); + header.set_mode(0o644); + header.set_size(1); + header.set_cksum(); + let long_name = format!("root/{}", "a".repeat(2 * 1024 * 1024)); + builder + .append_data(&mut header, long_name, &b"x"[..]) + .unwrap(); + let encoder = builder.into_inner().unwrap(); + let bytes = encoder.finish().unwrap(); + + let out = temp("hidden-metadata-budget"); + let _ = fs::remove_dir_all(&out); + let error = extract_tar_gz( + &bytes[..], + &out, + "root", + ArchiveLimits { + max_files: 1, + max_expanded_bytes: 1, + materialize_safe_links: false, + }, + ) + .unwrap_err(); + assert!( + error.contains("decompressed data exceeds"), + "unexpected error: {error}" + ); + let _ = fs::remove_dir_all(out); + } + + #[cfg(unix)] + #[test] + fn refuses_a_symbolic_link_destination() { + use std::os::unix::fs::symlink; + + let bytes = fixture(&[("root/file", tar::EntryType::Regular, b"safe", None)]); + let root = temp("destination-link-root"); + let target = temp("destination-link-target"); + let out = root.join("out"); + let _ = fs::remove_dir_all(&root); + let _ = fs::remove_dir_all(&target); + fs::create_dir_all(&root).unwrap(); + fs::create_dir_all(&target).unwrap(); + symlink(&target, &out).unwrap(); + + assert!(extract_tar_gz(&bytes[..], &out, "root", limits()) + .unwrap_err() + .contains("real directory")); + assert!(!target.join("file").exists()); + let _ = fs::remove_dir_all(root); + let _ = fs::remove_dir_all(target); + } + + #[test] + fn zip_extracts_one_pinned_root_with_portable_paths() { + let bytes = zip_fixture(&[("uv-x86_64-pc-windows-msvc/uv.exe", b"fixture")]); + let out = temp("zip-safe"); + let _ = fs::remove_dir_all(&out); + extract_zip_cancellable( + Cursor::new(bytes), + &out, + "uv-x86_64-pc-windows-msvc", + limits(), + &|| false, + ) + .unwrap(); + assert_eq!(fs::read(out.join("uv.exe")).unwrap(), b"fixture"); + let _ = fs::remove_dir_all(out); + } + + #[test] + fn zip_rejects_traversal_wrong_roots_and_expansion_over_budget() { + for (label, name, expected) in [ + ("zip-traversal", "root/../escape", "portable"), + ("zip-wrong-root", "other/file", "pinned root"), + ] { + let bytes = zip_fixture(&[(name, b"fixture")]); + let out = temp(label); + let _ = fs::remove_dir_all(&out); + let error = + extract_zip_cancellable(Cursor::new(bytes), &out, "root", limits(), &|| false) + .unwrap_err(); + assert!(error.contains(expected), "unexpected error: {error}"); + let _ = fs::remove_dir_all(out); + } + + let bytes = zip_fixture(&[("root/file", b"too large")]); + let out = temp("zip-budget"); + let _ = fs::remove_dir_all(&out); + let error = extract_zip_cancellable( + Cursor::new(bytes), + &out, + "root", + ArchiveLimits { + max_files: 10, + max_expanded_bytes: 1, + materialize_safe_links: false, + }, + &|| false, + ) + .unwrap_err(); + assert!(error.contains("expands beyond")); + let _ = fs::remove_dir_all(out); + } +} diff --git a/src-tauri/src/runtime_installer/download.rs b/src-tauri/src/runtime_installer/download.rs new file mode 100644 index 0000000..44750d6 --- /dev/null +++ b/src-tauri/src/runtime_installer/download.rs @@ -0,0 +1,688 @@ +//! Authenticated-transport download plus application-controlled SHA-256. + +use std::ffi::OsString; +use std::fs::{self, File, OpenOptions}; +use std::io::{self, Read, Write}; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use reqwest::redirect::Policy; +use reqwest::Client; +use serde::Deserialize; +use sha2::{Digest, Sha256}; + +// A transfer may legitimately take hours, but a single body read must not. The +// bounded idle timeout is what lets the install worker observe cancellation +// when a peer stops sending bytes without closing the connection. +const DOWNLOAD_READ_IDLE_TIMEOUT: Duration = Duration::from_secs(30); +const DOWNLOAD_CANCEL_POLL_INTERVAL: Duration = Duration::from_millis(100); +const DOWNLOAD_RESPONSE_TIMEOUT: Duration = Duration::from_secs(30); + +/// One immutable artifact in the app-bundled trust manifest. +#[derive(Clone, Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct PinnedArtifact { + pub(crate) url: String, + pub(crate) sha256: String, + pub(crate) size: u64, +} + +impl PinnedArtifact { + pub(crate) fn validate(&self) -> Result<(), String> { + let url = reqwest::Url::parse(&self.url) + .map_err(|error| format!("artifact URL is invalid: {error}"))?; + if url.scheme() != "https" { + return Err("artifact URL must use HTTPS".into()); + } + if !url.username().is_empty() || url.password().is_some() { + return Err("artifact URL must not contain credentials".into()); + } + let digest = hex::decode(&self.sha256) + .map_err(|_| "artifact SHA-256 is not hexadecimal".to_string())?; + if digest.len() != 32 { + return Err("artifact SHA-256 must contain exactly 32 bytes".into()); + } + if self.size == 0 { + return Err("artifact size must be non-zero".into()); + } + Ok(()) + } +} + +/// HTTP client used only from the installer's dedicated blocking worker. +/// Redirects may not downgrade authenticated transport. +pub(crate) fn client() -> Result { + Client::builder() + .user_agent(concat!("LSDJ/", env!("CARGO_PKG_VERSION"))) + .connect_timeout(Duration::from_secs(30)) + .redirect(Policy::custom(|attempt| { + if attempt.url().scheme() != "https" { + attempt.error("artifact redirect attempted to leave HTTPS") + } else if attempt.previous().len() >= 10 { + attempt.error("artifact redirect limit exceeded") + } else { + attempt.follow() + } + })) + .build() + .map_err(|error| format!("cannot create HTTPS client: {error}")) +} + +/// Fetch a small HTTPS resource (metadata or an immutable text/config file) +/// with the same redirect, cancellation, response, and body-stall policy as +/// artifact downloads. The caller supplies a strict maximum; the response is +/// never written or parsed after that bound is crossed. +pub(crate) fn fetch_bytes_bounded bool>( + client: &Client, + url: &str, + bearer_token: Option<&str>, + max_bytes: u64, + is_cancelled: F, +) -> Result, String> { + let parsed = + reqwest::Url::parse(url).map_err(|error| format!("resource URL is invalid: {error}"))?; + if parsed.scheme() != "https" || !parsed.username().is_empty() || parsed.password().is_some() { + return Err("resource URL must be credential-free HTTPS".into()); + } + if max_bytes == 0 { + return Err("resource byte bound must be non-zero".into()); + } + if is_cancelled() { + return Err("cancelled".into()); + } + let mut request = client.get(parsed); + if let Some(token) = bearer_token.filter(|token| !token.is_empty()) { + request = request.bearer_auth(token); + } + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|error| format!("cannot start resource transfer runtime: {error}"))?; + runtime.block_on(async { + let mut response = wait_for_progress( + request.send(), + &is_cancelled, + tokio::time::Instant::now() + DOWNLOAD_RESPONSE_TIMEOUT, + "resource response headers timed out", + ) + .await? + .and_then(reqwest::Response::error_for_status) + .map_err(|error| format!("resource download failed: {error}"))?; + if response.url().scheme() != "https" { + return Err("resource response did not use HTTPS".into()); + } + if response + .content_length() + .is_some_and(|length| length > max_bytes) + { + return Err("resource response exceeds its byte bound".into()); + } + let mut bytes = Vec::new(); + let mut idle_deadline = tokio::time::Instant::now() + DOWNLOAD_READ_IDLE_TIMEOUT; + loop { + let chunk = wait_for_progress( + response.chunk(), + &is_cancelled, + idle_deadline, + "resource response body stalled", + ) + .await? + .map_err(|error| format!("cannot read resource response: {error}"))?; + let Some(chunk) = chunk else { break }; + if chunk.is_empty() { + continue; + } + idle_deadline = tokio::time::Instant::now() + DOWNLOAD_READ_IDLE_TIMEOUT; + let total = bytes + .len() + .checked_add(chunk.len()) + .ok_or("resource byte count overflow")?; + if total as u64 > max_bytes { + return Err("resource response exceeds its byte bound".into()); + } + bytes.extend_from_slice(&chunk); + } + if is_cancelled() { + return Err("cancelled".into()); + } + Ok(bytes) + }) +} + +/// Download to a sibling `.part`, checking the expected byte count and digest +/// while streaming. A previously verified destination is reused, which makes a +/// retry after interruption deterministic without trusting a partial file. +pub(crate) fn download_verified bool>( + client: &Client, + artifact: &PinnedArtifact, + destination: &Path, + bearer_token: Option<&str>, + is_cancelled: F, +) -> Result<(), String> { + artifact.validate()?; + match fs::symlink_metadata(destination) { + Ok(metadata) if metadata.file_type().is_symlink() => { + return Err("artifact destination must not be a symbolic link".into()); + } + Ok(metadata) if metadata.is_file() => { + if verify_file_cancellable(destination, artifact, &is_cancelled).is_ok() { + return Ok(()); + } + if is_cancelled() { + return Err("cancelled".into()); + } + fs::remove_file(destination) + .map_err(|error| format!("cannot replace invalid cached artifact: {error}"))?; + } + Ok(_) => return Err("artifact destination exists but is not a file".into()), + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => return Err(format!("cannot inspect artifact destination: {error}")), + } + + let parent = destination + .parent() + .ok_or("artifact destination has no parent")?; + fs::create_dir_all(parent) + .map_err(|error| format!("cannot create artifact staging directory: {error}"))?; + let partial = partial_path(destination)?; + if partial.exists() { + fs::remove_file(&partial) + .map_err(|error| format!("cannot discard interrupted artifact: {error}"))?; + } + + let result = (|| { + if is_cancelled() { + return Err("cancelled".into()); + } + let mut request = client.get(&artifact.url); + if let Some(token) = bearer_token.filter(|token| !token.is_empty()) { + request = request.bearer_auth(token); + } + let file = OpenOptions::new() + .create_new(true) + .write(true) + .open(&partial) + .map_err(|error| format!("cannot create partial artifact: {error}"))?; + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|error| format!("cannot start artifact transfer runtime: {error}"))?; + runtime.block_on(async { + let response = wait_for_progress( + request.send(), + &is_cancelled, + tokio::time::Instant::now() + DOWNLOAD_RESPONSE_TIMEOUT, + "artifact response headers timed out", + ) + .await? + .and_then(reqwest::Response::error_for_status) + .map_err(|error| format!("artifact download failed: {error}"))?; + if response.url().scheme() != "https" { + return Err("artifact response did not use HTTPS".into()); + } + if let Some(length) = response.content_length() { + if length > artifact.size { + return Err(format!( + "artifact response exceeds pinned size ({} > {})", + length, artifact.size + )); + } + } + stream_response_to_file( + response, + file, + artifact, + &is_cancelled, + DOWNLOAD_READ_IDLE_TIMEOUT, + ) + .await + })?; + fs::rename(&partial, destination) + .map_err(|error| format!("cannot commit verified artifact: {error}"))?; + sync_parent(parent); + Ok(()) + })(); + + if result.is_err() { + let _ = fs::remove_file(&partial); + } + result +} + +async fn stream_response_to_file( + mut response: reqwest::Response, + mut file: File, + artifact: &PinnedArtifact, + is_cancelled: &dyn Fn() -> bool, + idle_timeout: Duration, +) -> Result<(), String> { + let mut hasher = Sha256::new(); + let mut total = 0u64; + let mut idle_deadline = tokio::time::Instant::now() + idle_timeout; + loop { + if is_cancelled() { + return Err("cancelled".into()); + } + let next = response.chunk(); + let chunk = wait_for_progress( + next, + is_cancelled, + idle_deadline, + "artifact response body stalled", + ) + .await? + .map_err(|error| format!("cannot read artifact response: {error}"))?; + let Some(chunk) = chunk else { + break; + }; + if chunk.is_empty() { + continue; + } + idle_deadline = tokio::time::Instant::now() + idle_timeout; + total = total + .checked_add(chunk.len() as u64) + .ok_or("artifact byte count overflow")?; + if total > artifact.size { + return Err(format!( + "artifact exceeds pinned size (more than {})", + artifact.size + )); + } + hasher.update(&chunk); + file.write_all(&chunk) + .map_err(|error| format!("cannot write partial artifact: {error}"))?; + } + finish_verified_file(file, hasher, total, artifact) +} + +async fn wait_for_progress( + future: impl std::future::Future>, + is_cancelled: &dyn Fn() -> bool, + deadline: tokio::time::Instant, + timeout_error: &'static str, +) -> Result, String> { + tokio::pin!(future); + loop { + let now = tokio::time::Instant::now(); + if now >= deadline { + return Err(timeout_error.into()); + } + let wait = DOWNLOAD_CANCEL_POLL_INTERVAL.min(deadline.saturating_duration_since(now)); + match tokio::time::timeout(wait, &mut future).await { + Ok(result) => return Ok(result), + Err(_) if is_cancelled() => return Err("cancelled".into()), + Err(_) => continue, + } + } +} + +#[cfg(test)] +fn copy_and_verify( + reader: &mut impl Read, + mut file: File, + artifact: &PinnedArtifact, + is_cancelled: &dyn Fn() -> bool, +) -> Result<(), String> { + let mut hasher = Sha256::new(); + let mut total = 0u64; + let mut buffer = [0u8; 128 * 1024]; + loop { + if is_cancelled() { + return Err("cancelled".into()); + } + let count = match reader.read(&mut buffer) { + Ok(count) => count, + Err(error) + if matches!( + error.kind(), + io::ErrorKind::TimedOut | io::ErrorKind::WouldBlock + ) && is_cancelled() => + { + return Err("cancelled".into()); + } + Err(error) => return Err(format!("cannot read artifact response: {error}")), + }; + if count == 0 { + break; + } + total = total + .checked_add(count as u64) + .ok_or("artifact byte count overflow")?; + if total > artifact.size { + return Err(format!( + "artifact exceeds pinned size (more than {})", + artifact.size + )); + } + hasher.update(&buffer[..count]); + file.write_all(&buffer[..count]) + .map_err(|error| format!("cannot write partial artifact: {error}"))?; + } + finish_verified_file(file, hasher, total, artifact) +} + +fn finish_verified_file( + mut file: File, + hasher: Sha256, + total: u64, + artifact: &PinnedArtifact, +) -> Result<(), String> { + if total != artifact.size { + return Err(format!( + "artifact size mismatch (expected {}, received {total})", + artifact.size + )); + } + let actual = hex::encode(hasher.finalize()); + if actual != artifact.sha256.to_ascii_lowercase() { + return Err(format!( + "artifact SHA-256 mismatch (expected {}, received {actual})", + artifact.sha256 + )); + } + file.flush() + .and_then(|_| file.sync_all()) + .map_err(|error| format!("cannot sync verified artifact: {error}"))?; + Ok(()) +} + +#[cfg(test)] +pub(crate) fn verify_file(path: &Path, artifact: &PinnedArtifact) -> Result<(), String> { + verify_file_cancellable(path, artifact, &|| false) +} + +pub(crate) fn verify_file_cancellable( + path: &Path, + artifact: &PinnedArtifact, + is_cancelled: &dyn Fn() -> bool, +) -> Result<(), String> { + artifact.validate()?; + let metadata = + fs::symlink_metadata(path).map_err(|error| format!("cannot inspect artifact: {error}"))?; + if metadata.file_type().is_symlink() { + return Err("artifact must not be a symbolic link".into()); + } + if !metadata.is_file() || metadata.len() != artifact.size { + return Err("artifact size does not match the manifest".into()); + } + let mut file = File::open(path).map_err(|error| format!("cannot open artifact: {error}"))?; + let mut hasher = Sha256::new(); + let mut buffer = [0u8; 128 * 1024]; + loop { + if is_cancelled() { + return Err("cancelled".into()); + } + let count = file + .read(&mut buffer) + .map_err(|error| format!("cannot hash artifact: {error}"))?; + if count == 0 { + break; + } + hasher.update(&buffer[..count]); + } + let actual = hex::encode(hasher.finalize()); + if actual != artifact.sha256.to_ascii_lowercase() { + return Err("artifact SHA-256 does not match the manifest".into()); + } + Ok(()) +} + +/// Place a verified staged blob into a candidate install without copying when +/// the filesystem supports hard links. The link is created by the application +/// only after the source hash is verified; archive-provided links are rejected. +pub(crate) fn link_or_copy_verified( + staged: &Path, + destination: &Path, + artifact: &PinnedArtifact, + is_cancelled: &dyn Fn() -> bool, +) -> Result<(), String> { + verify_file_cancellable(staged, artifact, is_cancelled)?; + if let Some(parent) = destination.parent() { + fs::create_dir_all(parent) + .map_err(|error| format!("cannot create model directory: {error}"))?; + } + if destination.exists() { + fs::remove_file(destination) + .map_err(|error| format!("cannot replace staged model: {error}"))?; + } + if fs::hard_link(staged, destination).is_err() { + let result = copy_file_cancellable(staged, destination, is_cancelled) + .and_then(|()| verify_file_cancellable(destination, artifact, is_cancelled)); + if result.is_err() { + let _ = fs::remove_file(destination); + } + result?; + } + Ok(()) +} + +fn copy_file_cancellable( + source: &Path, + destination: &Path, + is_cancelled: &dyn Fn() -> bool, +) -> Result<(), String> { + let mut source = File::open(source) + .map_err(|error| format!("cannot open staged model for copying: {error}"))?; + let mut destination = OpenOptions::new() + .create_new(true) + .write(true) + .open(destination) + .map_err(|error| format!("cannot create staged model copy: {error}"))?; + let mut buffer = [0u8; 128 * 1024]; + loop { + if is_cancelled() { + return Err("cancelled".into()); + } + let count = source + .read(&mut buffer) + .map_err(|error| format!("cannot read staged model: {error}"))?; + if count == 0 { + break; + } + destination + .write_all(&buffer[..count]) + .map_err(|error| format!("cannot copy staged model: {error}"))?; + } + destination + .flush() + .and_then(|_| destination.sync_all()) + .map_err(|error| format!("cannot sync staged model copy: {error}")) +} + +fn partial_path(destination: &Path) -> Result { + let name = destination + .file_name() + .ok_or("artifact destination has no file name")?; + let mut partial_name = OsString::from(name); + partial_name.push(".part"); + Ok(destination.with_file_name(partial_name)) +} + +fn sync_parent(parent: &Path) { + if let Ok(dir) = File::open(parent) { + let _ = dir.sync_all(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn artifact(bytes: &[u8]) -> PinnedArtifact { + PinnedArtifact { + url: "https://example.test/artifact".into(), + sha256: hex::encode(Sha256::digest(bytes)), + size: bytes.len() as u64, + } + } + + #[test] + fn rejects_non_https_credentials_and_malformed_hashes() { + let mut pin = artifact(b"ok"); + pin.url = "http://example.test/file".into(); + assert!(pin.validate().unwrap_err().contains("HTTPS")); + pin.url = "https://user:secret@example.test/file".into(); + assert!(pin.validate().unwrap_err().contains("credentials")); + pin.url = "https://example.test/file".into(); + pin.sha256 = "00".into(); + assert!(pin.validate().unwrap_err().contains("32 bytes")); + } + + #[test] + fn copy_is_bounded_and_hash_verified() { + let tmp = std::env::temp_dir().join(format!( + "lsdj-download-copy-{}-{:?}", + std::process::id(), + std::thread::current().id() + )); + let _ = fs::remove_dir_all(&tmp); + fs::create_dir_all(&tmp).unwrap(); + + let pin = artifact(b"verified bytes"); + let out = tmp.join("artifact"); + let file = File::create(&out).unwrap(); + copy_and_verify(&mut &b"verified bytes"[..], file, &pin, &|| false).unwrap(); + verify_file(&out, &pin).unwrap(); + + let too_long = PinnedArtifact { + size: 3, + ..artifact(b"bad") + }; + let file = File::create(tmp.join("too-long")).unwrap(); + assert!( + copy_and_verify(&mut &b"four"[..], file, &too_long, &|| false) + .unwrap_err() + .contains("exceeds") + ); + + let bad_hash = PinnedArtifact { + sha256: "00".repeat(32), + ..artifact(b"same size") + }; + let file = File::create(tmp.join("bad-hash")).unwrap(); + assert!( + copy_and_verify(&mut &b"same size"[..], file, &bad_hash, &|| false) + .unwrap_err() + .contains("SHA-256 mismatch") + ); + let _ = fs::remove_dir_all(&tmp); + } + + #[test] + fn cancellation_stops_before_more_bytes_are_written() { + let pin = artifact(b"will not be copied"); + let tmp = std::env::temp_dir().join(format!("lsdj-download-cancel-{}", std::process::id())); + let file = File::create(&tmp).unwrap(); + assert_eq!( + copy_and_verify(&mut &b"will not be copied"[..], file, &pin, &|| true), + Err("cancelled".into()) + ); + assert_eq!(fs::metadata(&tmp).unwrap().len(), 0); + let _ = fs::remove_file(tmp); + } + + #[test] + fn cancellation_interrupts_a_stalled_body_while_the_read_is_pending() { + use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::Arc; + + let cancelled = Arc::new(AtomicBool::new(false)); + let cancel = Arc::clone(&cancelled); + let canceller = std::thread::spawn(move || { + std::thread::sleep(Duration::from_millis(25)); + cancel.store(true, Ordering::Release); + }); + let started = std::time::Instant::now(); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let result = runtime.block_on(async { + wait_for_progress( + std::future::pending::>, io::Error>>(), + &|| cancelled.load(Ordering::Acquire), + tokio::time::Instant::now() + Duration::from_millis(250), + "artifact response body stalled", + ) + .await + }); + assert!(matches!(result, Err(error) if error == "cancelled")); + assert!( + started.elapsed() < Duration::from_secs(2), + "stalled read did not honor its deadline" + ); + canceller.join().unwrap(); + } + + #[test] + fn stalled_body_hits_the_no_progress_deadline() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let result = runtime.block_on(async { + wait_for_progress( + std::future::pending::>, io::Error>>(), + &|| false, + tokio::time::Instant::now() + Duration::from_millis(25), + "artifact response body stalled", + ) + .await + }); + assert!(matches!(result, Err(error) if error == "artifact response body stalled")); + } + + #[test] + fn cached_verification_and_copy_observe_cancellation() { + use std::sync::atomic::{AtomicUsize, Ordering}; + + let root = std::env::temp_dir().join(format!( + "lsdj-download-local-cancel-{}-{:?}", + std::process::id(), + std::thread::current().id() + )); + let _ = fs::remove_dir_all(&root); + fs::create_dir_all(&root).unwrap(); + let bytes = vec![7u8; 512 * 1024]; + let source = root.join("source"); + fs::write(&source, &bytes).unwrap(); + let pin = artifact(&bytes); + + let checks = AtomicUsize::new(0); + assert_eq!( + verify_file_cancellable(&source, &pin, &|| { + checks.fetch_add(1, Ordering::AcqRel) >= 1 + }), + Err("cancelled".into()) + ); + + let destination = root.join("copy"); + let checks = AtomicUsize::new(0); + assert_eq!( + copy_file_cancellable(&source, &destination, &|| { + checks.fetch_add(1, Ordering::AcqRel) >= 1 + }), + Err("cancelled".into()) + ); + let _ = fs::remove_dir_all(root); + } + + #[cfg(unix)] + #[test] + fn cached_artifacts_must_not_be_symbolic_links() { + use std::os::unix::fs::symlink; + + let root = std::env::temp_dir().join(format!("lsdj-download-link-{}", std::process::id())); + let _ = fs::remove_dir_all(&root); + fs::create_dir_all(&root).unwrap(); + let target = root.join("target"); + let cached = root.join("cached"); + fs::write(&target, b"verified bytes").unwrap(); + symlink(&target, &cached).unwrap(); + + assert!(verify_file(&cached, &artifact(b"verified bytes")) + .unwrap_err() + .contains("symbolic link")); + let _ = fs::remove_dir_all(root); + } +} diff --git a/src-tauri/src/runtime_installer/mod.rs b/src-tauri/src/runtime_installer/mod.rs new file mode 100644 index 0000000..13e2c6a --- /dev/null +++ b/src-tauri/src/runtime_installer/mod.rs @@ -0,0 +1,10 @@ +//! Native runtime/model installation primitives (issue #107). +//! +//! These modules deliberately know nothing about Tauri or SA3. The model +//! manager supplies the authenticated, application-bundled manifest and owns +//! progress/cancellation; these helpers enforce the filesystem/network trust +//! boundary and make the final swap recoverable. + +pub(crate) mod archive; +pub(crate) mod download; +pub(crate) mod promotion; diff --git a/src-tauri/src/runtime_installer/promotion.rs b/src-tauri/src/runtime_installer/promotion.rs new file mode 100644 index 0000000..fd2f1b1 --- /dev/null +++ b/src-tauri/src/runtime_installer/promotion.rs @@ -0,0 +1,293 @@ +//! Recoverable same-filesystem promotion for validated installs. + +use std::fs::{self, File}; +use std::path::Path; + +/// Resolve a promotion interrupted between renames. A ready current install +/// wins; an absent or invalid current install is replaced by the preserved one. +pub(crate) fn recover( + home: &Path, + backup: &Path, + validate: impl Fn(&Path) -> Result<(), String>, +) -> Result<(), String> { + let rejected = backup.with_extension("rejected"); + if rejected.exists() { + fs::remove_dir_all(&rejected) + .map_err(|error| format!("cannot clear rejected install: {error}"))?; + } + if !backup.exists() { + return Ok(()); + } + if !backup.is_dir() { + return Err("installer backup exists but is not a directory".into()); + } + let backup_parent = backup + .parent() + .ok_or("installer backup has no parent directory")?; + let home_parent = home + .parent() + .ok_or("install home has no parent directory")?; + ensure_same_filesystem(backup_parent, home_parent)?; + if !home.exists() { + validate(backup) + .map_err(|error| format!("preserved install is not ready for recovery: {error}"))?; + fs::rename(backup, home) + .map_err(|error| format!("cannot restore interrupted install: {error}"))?; + sync_parent(home); + return Ok(()); + } + match validate(home) { + Ok(()) => { + // The new install was promoted and the process stopped before cleanup. + // It is ready, so the old backup is no longer part of rollback state. + fs::remove_dir_all(backup) + .map_err(|error| format!("cannot clear completed install backup: {error}"))?; + return Ok(()); + } + Err(error) if error == "cancelled" => return Err(error), + Err(_) => {} + } + + validate(backup) + .map_err(|error| format!("preserved install is not ready for recovery: {error}"))?; + + fs::rename(home, &rejected) + .map_err(|error| format!("cannot quarantine interrupted install: {error}"))?; + if let Err(error) = fs::rename(backup, home) { + let _ = fs::rename(&rejected, home); + return Err(format!("cannot restore previous install: {error}")); + } + fs::remove_dir_all(rejected) + .map_err(|error| format!("cannot clear rejected install after rollback: {error}"))?; + sync_parent(home); + Ok(()) +} + +/// Validate the candidate, preserve the current tree, rename the candidate into +/// place, and validate it once more before discarding the previous tree. +pub(crate) fn promote( + candidate: &Path, + home: &Path, + backup: &Path, + validate: impl Fn(&Path) -> Result<(), String> + Copy, +) -> Result<(), String> { + if candidate == home || candidate == backup || home == backup { + return Err("installer promotion paths must be distinct".into()); + } + validate(candidate).map_err(|error| format!("candidate is not ready: {error}"))?; + let candidate_parent = candidate + .parent() + .ok_or("candidate install has no parent directory")?; + let home_parent = home + .parent() + .ok_or("install home has no parent directory")?; + let backup_parent = backup + .parent() + .ok_or("installer backup has no parent directory")?; + ensure_same_filesystem(candidate_parent, home_parent)?; + ensure_same_filesystem(backup_parent, home_parent)?; + recover(home, backup, validate)?; + + if home.exists() { + fs::rename(home, backup) + .map_err(|error| format!("cannot preserve previous install: {error}"))?; + sync_parent(home); + } + if let Err(error) = fs::rename(candidate, home) { + if backup.exists() && !home.exists() { + let _ = fs::rename(backup, home); + } + return Err(format!("cannot promote candidate install: {error}")); + } + sync_parent(home); + + if let Err(error) = validate(home) { + let _ = fs::rename(home, candidate); + if backup.exists() { + fs::rename(backup, home) + .map_err(|restore| format!( + "promoted install failed validation ({error}); previous install could not be restored: {restore}" + ))?; + } + sync_parent(home); + return Err(format!("promoted install failed validation: {error}")); + } + + // A crash before this cleanup is harmless: `recover` observes a ready home + // on the next attempt and removes the stale backup. + if backup.exists() { + fs::remove_dir_all(backup) + .map_err(|error| format!("cannot clear previous install after promotion: {error}"))?; + } + Ok(()) +} + +/// Promotion uses rename for the commit point, so all three trees must live on +/// one volume. The host path contract intentionally places staging beside the +/// asset root; this check also rejects a misconfigured cross-volume override. +pub(crate) fn ensure_same_filesystem(left: &Path, right: &Path) -> Result<(), String> { + same_filesystem(left, right).and_then(|same| { + if same { + Ok(()) + } else { + Err("installer staging and asset roots are on different filesystems".into()) + } + }) +} + +#[cfg(unix)] +fn same_filesystem(left: &Path, right: &Path) -> Result { + use std::os::unix::fs::MetadataExt; + let left = fs::metadata(left) + .map_err(|error| format!("cannot inspect installer staging filesystem: {error}"))?; + let right = fs::metadata(right) + .map_err(|error| format!("cannot inspect installer asset filesystem: {error}"))?; + Ok(left.dev() == right.dev()) +} + +#[cfg(windows)] +fn same_filesystem(left: &Path, right: &Path) -> Result { + use std::path::Component; + let left = fs::canonicalize(left) + .map_err(|error| format!("cannot resolve installer staging volume: {error}"))?; + let right = fs::canonicalize(right) + .map_err(|error| format!("cannot resolve installer asset volume: {error}"))?; + let volume = |path: &Path| match path.components().next() { + Some(Component::Prefix(prefix)) => { + Some(prefix.as_os_str().to_string_lossy().to_lowercase()) + } + _ => None, + }; + Ok(volume(&left).is_some() && volume(&left) == volume(&right)) +} + +#[cfg(not(any(unix, windows)))] +fn same_filesystem(_left: &Path, _right: &Path) -> Result { + Err("atomic installer promotion is unsupported on this platform".into()) +} + +fn sync_parent(path: &Path) { + if let Some(parent) = path.parent() { + if let Ok(dir) = File::open(parent) { + let _ = dir.sync_all(); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn root(label: &str) -> std::path::PathBuf { + std::env::temp_dir().join(format!("lsdj-promote-{label}-{}", std::process::id())) + } + + fn install(path: &Path, value: &str, ready: bool) { + fs::create_dir_all(path).unwrap(); + fs::write(path.join("value"), value).unwrap(); + if ready { + fs::write(path.join("ready"), b"").unwrap(); + } + } + + fn validate(path: &Path) -> Result<(), String> { + if path.join("ready").is_file() { + Ok(()) + } else { + Err("missing readiness marker".into()) + } + } + + #[test] + fn successful_promotion_replaces_ready_install_and_cleans_backup() { + let root = root("success"); + let _ = fs::remove_dir_all(&root); + let home = root.join("home"); + let candidate = root.join("candidate"); + let backup = root.join("backup"); + install(&home, "old", true); + install(&candidate, "new", true); + promote(&candidate, &home, &backup, validate).unwrap(); + assert_eq!(fs::read_to_string(home.join("value")).unwrap(), "new"); + assert!(!candidate.exists()); + assert!(!backup.exists()); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn staging_and_assets_must_share_a_filesystem() { + let root = root("filesystem"); + let _ = fs::remove_dir_all(&root); + let left = root.join("left"); + let right = root.join("right"); + fs::create_dir_all(&left).unwrap(); + fs::create_dir_all(&right).unwrap(); + ensure_same_filesystem(&left, &right).unwrap(); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn post_rename_validation_failure_restores_previous_install() { + let root = root("rollback"); + let _ = fs::remove_dir_all(&root); + let home = root.join("home"); + let candidate = root.join("candidate"); + let backup = root.join("backup"); + install(&home, "old", true); + install(&candidate, "new", true); + let validate_after_rename = |path: &Path| { + if path == home { + Err("simulated final validation failure".into()) + } else { + validate(path) + } + }; + let error = promote(&candidate, &home, &backup, validate_after_rename).unwrap_err(); + assert!(error.contains("failed validation")); + assert_eq!(fs::read_to_string(home.join("value")).unwrap(), "old"); + assert_eq!(fs::read_to_string(candidate.join("value")).unwrap(), "new"); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn retry_recovers_each_interrupted_promotion_state() { + let root = root("recover"); + let _ = fs::remove_dir_all(&root); + let home = root.join("home"); + let backup = root.join("backup"); + + // Interrupted after moving the previous home aside. + install(&backup, "old", true); + recover(&home, &backup, validate).unwrap(); + assert_eq!(fs::read_to_string(home.join("value")).unwrap(), "old"); + + // Interrupted after moving an invalid candidate into home. + fs::rename(&home, &backup).unwrap(); + install(&home, "broken", false); + recover(&home, &backup, validate).unwrap(); + assert_eq!(fs::read_to_string(home.join("value")).unwrap(), "old"); + + // Interrupted after a ready candidate was promoted. + fs::rename(&home, &backup).unwrap(); + install(&home, "new", true); + recover(&home, &backup, validate).unwrap(); + assert_eq!(fs::read_to_string(home.join("value")).unwrap(), "new"); + assert!(!backup.exists()); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn recovery_never_restores_an_unvalidated_backup() { + let root = root("invalid-backup"); + let _ = fs::remove_dir_all(&root); + let home = root.join("home"); + let backup = root.join("backup"); + install(&backup, "broken", false); + + let error = recover(&home, &backup, validate).unwrap_err(); + assert!(error.contains("not ready for recovery")); + assert!(!home.exists()); + assert!(backup.exists()); + let _ = fs::remove_dir_all(root); + } +} diff --git a/src-tauri/src/settings.rs b/src-tauri/src/settings.rs index 8c41c14..e1fd6eb 100644 --- a/src-tauri/src/settings.rs +++ b/src-tauri/src/settings.rs @@ -3,7 +3,7 @@ //! Settings the SHELL consumes — the output devices and the recordings //! folder — used to live in webview localStorage, replayed into the engine //! at boot by App.tsx. Persistence follows ownership: they now persist here -//! (a JSON file under the app data dir, beside the MCP token), hydrate into +//! (a JSON file under the host-resolved config dir, beside the MCP token), hydrate into //! the engine and the interface store during `setup` — before the webview //! exists — and mutate through the same commands every controller uses. The //! webview's pickers become projections. Presentation-only preferences @@ -15,7 +15,7 @@ use std::sync::{mpsc, Mutex}; use std::time::Duration; use serde::{Deserialize, Serialize}; -use tauri::{AppHandle, Manager}; +use tauri::AppHandle; use crate::store::{GenerationSnap, InterfaceState, InterfaceStore}; @@ -101,8 +101,8 @@ impl Default for DeckMixerSetting { } } -fn settings_file(app: &AppHandle) -> Option { - app.path().app_data_dir().ok().map(|dir| dir.join("settings.json")) +fn settings_file() -> PathBuf { + crate::platform_paths::get().config().join("settings.json") } /// Load from a concrete path (the testable core): a missing or unreadable @@ -141,8 +141,8 @@ pub fn save_to(path: &Path, settings: &ShellSettings) { } } -pub fn load(app: &AppHandle) -> ShellSettings { - settings_file(app).map(|p| load_from(&p)).unwrap_or_default() +pub fn load(_app: &AppHandle) -> ShellSettings { + load_from(&settings_file()) } /// One writer at a time: `update` runs from the main thread (the device / @@ -163,17 +163,8 @@ pub fn update_at(path: &Path, mutate: impl FnOnce(&mut ShellSettings)) -> ShellS } /// Read-modify-write one field; the single mutation path the commands use. -pub fn update(app: &AppHandle, mutate: impl FnOnce(&mut ShellSettings)) -> ShellSettings { - match settings_file(app) { - Some(path) => update_at(&path, mutate), - // No data dir (boot-path edge): mutate the defaults so the caller - // still gets the value it wrote, exactly as before — just unsaved. - None => { - let mut settings = ShellSettings::default(); - mutate(&mut settings); - settings - } - } +pub fn update(_app: &AppHandle, mutate: impl FnOnce(&mut ShellSettings)) -> ShellSettings { + update_at(&settings_file(), mutate) } /// The settings-write debounce: a fader ride or a pad drag settles before it diff --git a/src-tauri/src/sidecar.rs b/src-tauri/src/sidecar.rs index 1235715..899138f 100644 --- a/src-tauri/src/sidecar.rs +++ b/src-tauri/src/sidecar.rs @@ -34,7 +34,7 @@ use std::io::{self, Read, Write}; use std::net::{TcpListener, TcpStream}; -use std::process::{Child, Command}; +use std::process::Command; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; use std::thread::{self, JoinHandle}; @@ -44,6 +44,7 @@ use lsdj_engine::DeckHandle; use tauri::ipc::{Channel, InvokeResponseBody}; use crate::analysis::live::AnalysisFeed; +use crate::child_process::SupervisedChild; /// Per-deck analysis taps: a webview [`Channel`] each deck's realtime PCM is teed /// to (gap 1). The TS beat/loudness/band analysis (ADR-0017: stays in TypeScript) @@ -88,7 +89,10 @@ impl PcmTaps { // so the restart hung holding the deck-slot mutex. let channel = slot.lock().unwrap_or_else(|p| p.into_inner()).clone(); if let Some(channel) = channel { - if channel.send(InvokeResponseBody::Raw(bytes.to_vec())).is_err() { + if channel + .send(InvokeResponseBody::Raw(bytes.to_vec())) + .is_err() + { *slot.lock().unwrap_or_else(|p| p.into_inner()) = None; } } @@ -104,6 +108,8 @@ pub const FRAME_CONTROL: u8 = 3; /// Engine → sidecar: a style-sample embed (M15). Binary, not JSON, because it /// carries raw PCM: `[u32 LE id length][id utf-8][interleaved f32 LE PCM]`. pub const FRAME_EMBED: u8 = 4; +/// Sidecar -> engine: the per-launch capability. This must be the first frame. +pub const FRAME_AUTH: u8 = 5; /// Cap on a single frame's payload — a guard against a desynced/hostile stream /// allocating unbounded memory. A 1 s PCM chunk is 384 000 bytes; 16 MiB is far @@ -118,8 +124,17 @@ const ACCEPT_TIMEOUT: Duration = Duration::from_secs(30); /// Write one framed message: a type byte, a little-endian `u32` length, then the /// payload. Flushes so the consumer sees it promptly (the socket is `nodelay`). pub fn write_frame(w: &mut impl Write, frame_type: u8, payload: &[u8]) -> io::Result<()> { + let len = u32::try_from(payload.len()).map_err(|_| { + io::Error::new(io::ErrorKind::InvalidInput, "sidecar frame payload is too large") + })?; + if len > MAX_FRAME_BYTES { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("sidecar frame length {len} exceeds the cap"), + )); + } w.write_all(&[frame_type])?; - w.write_all(&(payload.len() as u32).to_le_bytes())?; + w.write_all(&len.to_le_bytes())?; w.write_all(payload)?; w.flush() } @@ -157,6 +172,12 @@ fn pcm_from_le_bytes(bytes: &[u8]) -> Vec { .collect() } +type StatusSink = Box; +type PcmSink = Box; +type DeckStatusSinks = [StatusSink; lsdj_engine::DECK_COUNT]; +type DeckPcmSinks = [PcmSink; lsdj_engine::DECK_COUNT]; +type SharedStatusSinks = [Arc>; lsdj_engine::DECK_COUNT]; + /// The read loop: drain frames from the sidecar until EOF/error. PCM frames are /// posted to the deck's ring (the non-RT producer side) and then TEED to `on_pcm` /// (gap 1: the analysis feed to the webview); status frames go to `on_status` (the @@ -197,6 +218,41 @@ pub fn run_reader( deck_handle } +/// Shared-worker variant of [`run_reader`]. Every PCM/status payload starts +/// with a deck byte (`0` or `1`); the remaining bytes are exactly the existing +/// per-deck payload. Invalid deck indices are ignored without disturbing the +/// other stream. +pub fn run_shared_reader( + mut stream: impl Read, + mut deck_handles: [DeckHandle; lsdj_engine::DECK_COUNT], + on_status: &mut DeckStatusSinks, + on_pcm: &mut DeckPcmSinks, +) -> [DeckHandle; lsdj_engine::DECK_COUNT] { + while let Ok(Some((frame_type, payload))) = read_frame(&mut stream) { + let Some((&deck, body)) = payload.split_first() else { + continue; + }; + let deck = deck as usize; + if deck >= lsdj_engine::DECK_COUNT { + continue; + } + match frame_type { + FRAME_PCM => { + let samples = pcm_from_le_bytes(body); + deck_handles[deck].post_pcm(&samples); + on_pcm[deck](body); + } + FRAME_STATUS => { + if let Ok(text) = String::from_utf8(body.to_vec()) { + on_status[deck](text); + } + } + _ => {} + } + } + deck_handles +} + /// What a reader thread hands back when its sidecar connection ends: the deck /// ring producer ([`DeckHandle`]) and the status sink. The engine's input ring is /// PERMANENT across a sidecar exit (the consumer lives inside the engine), so the @@ -204,18 +260,49 @@ pub fn run_reader( /// a model switch. [`Sidecar::restart`] joins the reader to take these back. struct ReaderExit { handle: DeckHandle, - on_status: Box, + on_status: StatusSink, } /// The freshly-built control writer, child handle, stop flag, and reader thread — /// the pieces a (re)spawn produces and a [`Sidecar`] installs. struct ReaderParts { control: Arc>>, - child: Arc>>, + child: Arc>>, stop: Arc, reader: JoinHandle, } +struct SharedReaderExit { + handles: [DeckHandle; lsdj_engine::DECK_COUNT], +} + +struct SharedReaderParts { + control: Arc>>, + process: Arc>, + stop: Arc, + reader: JoinHandle, +} + +trait SharedProcess: Send { + fn shutdown(&mut self) -> io::Result<()>; +} + +impl SharedProcess for SupervisedChild { + fn shutdown(&mut self) -> io::Result<()> { + let report = SupervisedChild::shutdown(self, Duration::from_millis(500))?; + crate::child_process::log_shutdown("shared sidecar", Ok(report)); + Ok(()) + } +} + +enum SharedProcessState { + Stopped, + Running(Box), + /// Teardown did not positively reap the process tree. Ownership is retained + /// and promotion remains blocked until a later retry reaches `Stopped`. + Uncertain(Box), +} + /// One supervised deck sidecar: the spawned Python process, the control writer /// (engine → sidecar), and the reader thread (sidecar → engine). Dropping it /// stops the reader, closes the socket, and kills the child. @@ -231,7 +318,7 @@ pub struct Sidecar { /// The control-writer half of the socket; `None` until the sidecar connects, /// and after a teardown. Behind a `Mutex` so IPC callers serialise writes. control: Arc>>, - child: Arc>>, + child: Arc>>, stop: Arc, /// The accept+read thread; its result carries the reclaimable [`ReaderExit`]. reader: Option>, @@ -241,13 +328,17 @@ pub struct Sidecar { /// FALLIBLE prefix, done BEFORE any [`DeckHandle`] is committed, so a bad launch /// (or a bind failure) never costs the deck its ring producer. [`Sidecar::restart`] /// runs this first and leaves the running sidecar untouched if it fails. -fn bind_and_launch(deck_id: &str, model: &str) -> io::Result<(TcpListener, Child)> { +fn bind_and_launch( + deck_id: &str, + model: &str, +) -> io::Result<(TcpListener, SupervisedChild, String)> { let listener = TcpListener::bind("127.0.0.1:0")?; listener.set_nonblocking(false).ok(); let port = listener.local_addr()?.port(); - let mut command = sidecar_command(deck_id, model, port)?; + let token = crate::local_auth::generate_capability(); + let mut command = authenticated_sidecar_command(deck_id, model, port, &token)?; let child = crate::child_process::spawn_grouped(&mut command)?; - Ok((listener, child)) + Ok((listener, child, token)) } /// The PCM tee closure handed to a reader thread: forward each deck PCM frame @@ -282,9 +373,10 @@ fn pcm_tee( fn start_reader( listener: TcpListener, deck_id: &str, - child: Child, + child: SupervisedChild, + token: String, handle: DeckHandle, - mut on_status: Box, + mut on_status: StatusSink, mut on_pcm: impl FnMut(&[u8]) + Send + 'static, ) -> ReaderParts { let control: Arc>> = Arc::new(Mutex::new(None)); @@ -300,7 +392,12 @@ fn start_reader( // is set — a teardown / restart wakes a never-connected accept promptly // instead of waiting out ACCEPT_TIMEOUT (which would freeze the deck's // control while the supervisor joins this thread). - let stream = match accept_with_timeout(&listener, &stop_for_reader, ACCEPT_TIMEOUT) { + let stream = match accept_authenticated_with_timeout( + &listener, + &stop_for_reader, + ACCEPT_TIMEOUT, + token.as_bytes(), + ) { Some(s) => s, None => { eprintln!("lsdj-sidecar-{deck_label}: sidecar never connected"); @@ -322,7 +419,9 @@ fn start_reader( // we asked it to stop (a clean shutdown / model switch). *control_for_reader.lock().unwrap_or_else(|p| p.into_inner()) = None; if !stop_for_reader.load(Ordering::Acquire) { - on_status(format!("{{\"event\":\"worker_died\",\"deck\":\"{deck_label}\"}}")); + on_status(format!( + "{{\"event\":\"worker_died\",\"deck\":\"{deck_label}\"}}" + )); } ReaderExit { handle, on_status } }) @@ -335,11 +434,91 @@ fn start_reader( } } +fn bind_and_launch_shared( + models: &[String; lsdj_engine::DECK_COUNT], +) -> io::Result<(TcpListener, SupervisedChild, String)> { + let listener = TcpListener::bind("127.0.0.1:0")?; + listener.set_nonblocking(false).ok(); + let port = listener.local_addr()?.port(); + let token = crate::local_auth::generate_capability(); + let mut command = authenticated_shared_sidecar_command(models, port, &token)?; + let child = crate::child_process::spawn_grouped(&mut command)?; + Ok((listener, child, token)) +} + +fn start_shared_reader( + listener: TcpListener, + child: SupervisedChild, + token: String, + handles: [DeckHandle; lsdj_engine::DECK_COUNT], + on_status: SharedStatusSinks, + mut on_pcm: DeckPcmSinks, +) -> SharedReaderParts { + let control: Arc>> = Arc::new(Mutex::new(None)); + let stop = Arc::new(AtomicBool::new(false)); + let control_for_reader = control.clone(); + let stop_for_reader = stop.clone(); + let reader = thread::Builder::new() + .name("lsdj-sidecar-shared".to_string()) + .spawn(move || { + let mut reader_status: DeckStatusSinks = std::array::from_fn(|deck| { + let sink = on_status[deck].clone(); + let status_stop = stop_for_reader.clone(); + Box::new(move |message| { + if !status_stop.load(Ordering::Acquire) { + (sink.lock().unwrap_or_else(|poisoned| poisoned.into_inner()))(message) + } + }) as StatusSink + }); + let stream = match accept_authenticated_with_timeout( + &listener, + &stop_for_reader, + ACCEPT_TIMEOUT, + token.as_bytes(), + ) { + Some(stream) => stream, + None => { + eprintln!("lsdj-sidecar-shared: sidecar never connected"); + return SharedReaderExit { handles }; + } + }; + stream.set_nodelay(true).ok(); + match stream.try_clone() { + Ok(writer) => { + *control_for_reader.lock().unwrap_or_else(|p| p.into_inner()) = Some(writer) + } + Err(error) => { + eprintln!("lsdj-sidecar-shared: cannot split socket: {error}"); + return SharedReaderExit { handles }; + } + } + let handles = run_shared_reader(stream, handles, &mut reader_status, &mut on_pcm); + *control_for_reader.lock().unwrap_or_else(|p| p.into_inner()) = None; + if !stop_for_reader.load(Ordering::Acquire) { + for (deck, sink) in on_status.iter().enumerate() { + (sink.lock().unwrap_or_else(|poisoned| poisoned.into_inner()))(format!( + "{{\"event\":\"worker_died\",\"deck\":\"{}\"}}", + ["a", "b"][deck] + )); + } + } + SharedReaderExit { handles } + }) + .expect("failed to spawn shared LSDJ sidecar reader thread"); + SharedReaderParts { + control, + process: Arc::new(Mutex::new(SharedProcessState::Running(Box::new(child)))), + stop, + reader, + } +} + impl Sidecar { /// Spawn and supervise the sidecar for `deck_id`, feeding `deck_handle` and /// reporting status through `on_status`. Binds a loopback listener, launches /// the Python sidecar pointed at the bound port, accepts its connection, and - /// starts the reader thread. The spawn command is [`sidecar_command`] + /// starts the reader thread. The spawn command is built from + /// [`sidecar_base_command`] /// (`LSDJ_BACKEND_BIN` in a release; overridable via `LSDJ_SIDECAR_CMD` in dev). /// /// Errors if the listener cannot bind or the process cannot launch — the @@ -354,11 +533,12 @@ impl Sidecar { taps: PcmTaps, feed: AnalysisFeed, ) -> io::Result { - let (listener, child) = bind_and_launch(deck_id, model)?; + let (listener, child, token) = bind_and_launch(deck_id, model)?; let parts = start_reader( listener, deck_id, child, + token, deck_handle, Box::new(on_status), pcm_tee(taps.clone(), feed.clone(), deck_idx), @@ -395,7 +575,7 @@ impl Sidecar { // — and its ring producer — are completely untouched; only after this // succeeds do we reclaim the handle, so it is never at risk on a recoverable // error. - let (listener, child) = bind_and_launch(&self.deck_id, new_model)?; + let (listener, child, token) = bind_and_launch(&self.deck_id, new_model)?; // `stop` suppresses the old reader's `worker_died` across the deliberate // switch (and wakes a never-connected accept). @@ -405,11 +585,19 @@ impl Sidecar { // the socket open and the reader blocked in `read_frame`; `shutdown` tears // down the SHARED socket so the reader's read returns EOF at once (and // signals the old sidecar to exit). The child kill then terminates it. - if let Some(writer) = self.control.lock().unwrap_or_else(|p| p.into_inner()).take() { + if let Some(writer) = self + .control + .lock() + .unwrap_or_else(|p| p.into_inner()) + .take() + { let _ = writer.shutdown(std::net::Shutdown::Both); } if let Some(mut old) = self.child.lock().unwrap_or_else(|p| p.into_inner()).take() { - crate::child_process::kill_group(&mut old); + crate::child_process::log_shutdown( + &format!("sidecar {} restart", self.deck_id), + old.shutdown(Duration::from_millis(500)), + ); } let exit = self .reader @@ -428,6 +616,7 @@ impl Sidecar { listener, &self.deck_id, child, + token, exit.handle, on_status, pcm_tee(self.taps.clone(), self.feed.clone(), self.deck_idx), @@ -470,25 +659,342 @@ impl Sidecar { } } -/// All per-deck sidecars, held in Tauri managed state. The deck-control commands +/// One supervised PyTorch process owning both deck states. Different model +/// selections remain legal, but load two model instances inside this one process; +/// equal selections share one upstream model and keep independent continuation. +pub struct SharedSidecar { + models: [String; lsdj_engine::DECK_COUNT], + taps: PcmTaps, + feed: AnalysisFeed, + on_status: SharedStatusSinks, + control: Arc>>, + process: Arc>, + stop: Arc, + reader: Option>, + /// Reclaimed ring producers parked after a replacement launch failure. A + /// later selection can recover without reconstructing the native engine. + parked: Option, +} + +impl SharedSidecar { + /// Construct the shared CUDA topology without launching Python. The native + /// engine's permanent ring producers remain parked here until a verified + /// managed MRT2 generation is installed (or a later retry succeeds). + pub fn parked( + models: [String; lsdj_engine::DECK_COUNT], + handles: [DeckHandle; lsdj_engine::DECK_COUNT], + on_status: DeckStatusSinks, + taps: PcmTaps, + feed: AnalysisFeed, + ) -> Self { + Self { + models, + taps, + feed, + on_status: on_status.map(|sink| Arc::new(Mutex::new(sink))), + control: Arc::new(Mutex::new(None)), + process: Arc::new(Mutex::new(SharedProcessState::Stopped)), + stop: Arc::new(AtomicBool::new(true)), + reader: None, + parked: Some(SharedReaderExit { handles }), + } + } + + #[cfg(all(test, unix, not(feature = "managed-runtime")))] + pub fn spawn( + models: [String; lsdj_engine::DECK_COUNT], + handles: [DeckHandle; lsdj_engine::DECK_COUNT], + on_status: DeckStatusSinks, + taps: PcmTaps, + feed: AnalysisFeed, + ) -> Result { + let mut sidecar = Self::parked(models, handles, on_status, taps, feed); + if let Err(error) = sidecar.activate() { + let handles = sidecar + .parked + .take() + .expect("failed shared activation preserves deck handles") + .handles; + return Err((error, handles)); + } + Ok(sidecar) + } + + /// Start a worker from parked handles. Resolution and manifest validation + /// occur inside `bind_and_launch_shared` immediately before spawn, so an + /// install that completed after app startup becomes usable without restart. + pub fn activate(&mut self) -> io::Result<()> { + match &*self.process.lock().unwrap_or_else(|p| p.into_inner()) { + SharedProcessState::Running(_) => return Ok(()), + SharedProcessState::Uncertain(_) => { + return Err(io::Error::other( + "shared sidecar process reap is still uncertain", + )) + } + SharedProcessState::Stopped => {} + } + if self.parked.is_none() { + return Err(io::Error::other( + "shared sidecar has no parked deck handles", + )); + } + let (listener, child, token) = bind_and_launch_shared(&self.models)?; + let exit = self + .parked + .take() + .ok_or_else(|| io::Error::other("shared sidecar has no parked deck handles"))?; + let on_pcm: DeckPcmSinks = [ + Box::new(pcm_tee(self.taps.clone(), self.feed.clone(), 0)), + Box::new(pcm_tee(self.taps.clone(), self.feed.clone(), 1)), + ]; + let parts = start_shared_reader( + listener, + child, + token, + exit.handles, + self.on_status.clone(), + on_pcm, + ); + self.control = parts.control; + self.process = parts.process; + self.stop = parts.stop; + self.reader = Some(parts.reader); + Ok(()) + } + + /// Stop and fully reap the worker while retaining the permanent deck ring + /// producers. Promotion may rename the managed generation only after this + /// succeeds (notably on Windows, where a live Python process holds DLLs). + #[cfg(feature = "managed-runtime")] + pub fn quiesce(&mut self) -> io::Result<()> { + let exit = self.stop_and_reclaim()?; + self.parked = Some(exit); + Ok(()) + } + + fn send_control(&self, deck: usize, json: &str) { + if deck >= lsdj_engine::DECK_COUNT { + return; + } + let mut payload = Vec::with_capacity(1 + json.len()); + payload.push(deck as u8); + payload.extend_from_slice(json.as_bytes()); + let mut guard = self.control.lock().unwrap_or_else(|p| p.into_inner()); + if let Some(stream) = guard.as_mut() { + if let Err(error) = write_frame(stream, FRAME_CONTROL, &payload) { + eprintln!("lsdj-sidecar-shared: control write failed: {error}"); + *guard = None; + } + } + } + + fn send_embed(&self, deck: usize, id: &str, pcm: &[u8]) { + if deck >= lsdj_engine::DECK_COUNT { + return; + } + let mut payload = Vec::with_capacity(1 + 4 + id.len() + pcm.len()); + payload.push(deck as u8); + payload.extend_from_slice(&(id.len() as u32).to_le_bytes()); + payload.extend_from_slice(id.as_bytes()); + payload.extend_from_slice(pcm); + let mut guard = self.control.lock().unwrap_or_else(|p| p.into_inner()); + if let Some(stream) = guard.as_mut() { + if let Err(error) = write_frame(stream, FRAME_EMBED, &payload) { + eprintln!("lsdj-sidecar-shared: embed write failed: {error}"); + *guard = None; + } + } + } + + fn restart(&mut self, deck: usize, model: &str) -> io::Result<()> { + if deck >= lsdj_engine::DECK_COUNT { + return Err(io::Error::new(io::ErrorKind::InvalidInput, "invalid deck")); + } + let mut models = self.models.clone(); + models[deck] = model.to_string(); + + // A single shared CUDA worker owns both deck states, so both become + // unavailable together. Publish that fact before stopping the old + // generation and before any replacement model allocation can begin. + // Gate the old reader first so a final stale `ready` cannot race after + // these loading events; the process itself is stopped below. + self.stop.store(true, Ordering::Release); + for (index, sink) in self.on_status.iter().enumerate() { + let deck_label = ["a", "b"][index]; + (sink.lock().unwrap_or_else(|poisoned| poisoned.into_inner()))( + serde_json::json!({ + "event": "model_loading", + "deck": deck_label, + "model": &models[index], + }) + .to_string(), + ); + } + + let exit = self.stop_and_reclaim()?; + // Stop-and-reap is intentional for shared CUDA. Launch-first remains the + // per-deck policy above, but would temporarily require two resident model + // generations here and can OOM a minimum-VRAM host. + let (listener, child, token) = match bind_and_launch_shared(&models) { + Ok(launch) => launch, + Err(error) => { + self.parked = Some(exit); + return Err(io::Error::new( + error.kind(), + format!( + "shared CUDA replacement failed after the old worker was stopped; reselect a model to retry: {error}" + ), + )); + } + }; + + let on_pcm: DeckPcmSinks = [ + Box::new(pcm_tee(self.taps.clone(), self.feed.clone(), 0)), + Box::new(pcm_tee(self.taps.clone(), self.feed.clone(), 1)), + ]; + let parts = start_shared_reader( + listener, + child, + token, + exit.handles, + self.on_status.clone(), + on_pcm, + ); + self.models = models; + self.control = parts.control; + self.process = parts.process; + self.stop = parts.stop; + self.reader = Some(parts.reader); + Ok(()) + } + + fn stop_and_reclaim(&mut self) -> io::Result { + self.stop.store(true, Ordering::Release); + if let Some(writer) = self + .control + .lock() + .unwrap_or_else(|p| p.into_inner()) + .take() + { + let _ = writer.shutdown(std::net::Shutdown::Both); + } + let shutdown_result = + stop_shared_process(&mut self.process.lock().unwrap_or_else(|p| p.into_inner())); + let exit = if let Some(reader) = self.reader.take() { + reader + .join() + .map_err(|_| io::Error::other("shared sidecar reader thread panicked"))? + } else { + self.parked + .take() + .ok_or_else(|| io::Error::other("shared sidecar has no deck handles to reclaim"))? + }; + if let Err(error) = shutdown_result { + self.parked = Some(exit); + return Err(error); + } + Ok(exit) + } +} + +fn stop_shared_process(state: &mut SharedProcessState) -> io::Result<()> { + let previous = std::mem::replace(state, SharedProcessState::Stopped); + let mut process = match previous { + SharedProcessState::Stopped => return Ok(()), + SharedProcessState::Running(process) | SharedProcessState::Uncertain(process) => process, + }; + match process.shutdown() { + Ok(()) => Ok(()), + Err(error) => { + *state = SharedProcessState::Uncertain(process); + Err(error) + } + } +} + +impl Drop for SharedSidecar { + fn drop(&mut self) { + self.stop.store(true, Ordering::Release); + if let Some(writer) = self + .control + .lock() + .unwrap_or_else(|p| p.into_inner()) + .take() + { + let _ = writer.shutdown(std::net::Shutdown::Both); + } + if let Err(error) = + stop_shared_process(&mut self.process.lock().unwrap_or_else(|p| p.into_inner())) + { + crate::child_process::log_shutdown("shared sidecar", Err(error)); + } + if let Some(reader) = self.reader.take() { + let _ = reader.join(); + } + } +} + +/// All model sidecars, held in Tauri managed state. The deck-control commands /// forward validated JSON to the matching sidecar; a deck with no sidecar (spawn /// failed, or sidecars disabled) silently drops the command. Each slot is a /// `Mutex` so `deck_set_model` can mutate one sidecar (a model switch) through the /// shared `tauri::State` without a supervisor thread. pub struct Sidecars { decks: Vec>>, + shared: Mutex>, } impl Sidecars { pub fn new(decks: Vec>) -> Self { Sidecars { decks: decks.into_iter().map(Mutex::new).collect(), + shared: Mutex::new(None), } } + pub fn new_shared(shared: SharedSidecar) -> Self { + Sidecars { + decks: (0..lsdj_engine::DECK_COUNT) + .map(|_| Mutex::new(None)) + .collect(), + shared: Mutex::new(Some(shared)), + } + } + + /// Quiesce the managed shared worker before its verified generation is + /// renamed. A no-op for the macOS per-deck topology. + #[cfg(feature = "managed-runtime")] + pub fn quiesce_shared(&self) -> Result<(), String> { + let mut shared = self.shared.lock().unwrap_or_else(|p| p.into_inner()); + if let Some(shared) = shared.as_mut() { + shared.quiesce().map_err(|error| error.to_string())?; + } + Ok(()) + } + + /// Revalidate and activate a parked shared worker. This covers both first + /// install and post-promotion restart without reconstructing the audio host. + #[cfg(feature = "managed-runtime")] + pub fn resume_shared(&self) -> Result<(), String> { + let mut shared = self.shared.lock().unwrap_or_else(|p| p.into_inner()); + let shared = shared + .as_mut() + .ok_or_else(|| "shared MRT2 decks are unavailable on this platform".to_string())?; + shared.activate().map_err(|error| error.to_string()) + } + /// Forward a JSON deck command to the sidecar for `deck` (a no-op for a deck /// without a live sidecar). `deck` is validated by the IPC layer. pub fn send(&self, deck: usize, json: &str) { + if let Some(shared) = self + .shared + .lock() + .unwrap_or_else(|p| p.into_inner()) + .as_ref() + { + shared.send_control(deck, json); + return; + } if let Some(slot) = self.decks.get(deck) { if let Some(sidecar) = slot.lock().unwrap_or_else(|p| p.into_inner()).as_ref() { sidecar.send_control(json); @@ -499,6 +1005,15 @@ impl Sidecars { /// Route a style-sample embed (M15) to a deck's sidecar (a no-op for a deck /// without a live sidecar). `deck` is validated by the IPC layer. pub fn embed(&self, deck: usize, id: &str, pcm: &[u8]) { + if let Some(shared) = self + .shared + .lock() + .unwrap_or_else(|p| p.into_inner()) + .as_ref() + { + shared.send_embed(deck, id, pcm); + return; + } if let Some(slot) = self.decks.get(deck) { if let Some(sidecar) = slot.lock().unwrap_or_else(|p| p.into_inner()).as_ref() { sidecar.send_embed(id, pcm); @@ -511,6 +1026,13 @@ impl Sidecars { /// (in which case the running sidecar is left untouched). `deck` is validated /// by the IPC layer. pub fn restart(&self, deck: usize, model: &str) -> Result<(), String> { + let mut shared = self.shared.lock().unwrap_or_else(|p| p.into_inner()); + if let Some(shared) = shared.as_mut() { + return shared + .restart(deck, model) + .map_err(|error| error.to_string()); + } + drop(shared); let slot = self.decks.get(deck).ok_or("invalid deck")?; let mut guard = slot.lock().unwrap_or_else(|p| p.into_inner()); match guard.as_mut() { @@ -525,6 +1047,7 @@ impl Sidecars { /// the Python sidecars also self-terminate on the socket EOF, but this makes /// the teardown deterministic. pub fn shutdown(&self) { + self.shared.lock().unwrap_or_else(|p| p.into_inner()).take(); for slot in &self.decks { slot.lock().unwrap_or_else(|p| p.into_inner()).take(); } @@ -539,11 +1062,19 @@ impl Drop for Sidecar { // first; this also tells a healthy Python worker to stop cleanly. Then // kill the whole process group so a `uv run` wrapper cannot leave that // worker alive holding the peer socket open. - if let Some(writer) = self.control.lock().unwrap_or_else(|p| p.into_inner()).take() { + if let Some(writer) = self + .control + .lock() + .unwrap_or_else(|p| p.into_inner()) + .take() + { let _ = writer.shutdown(std::net::Shutdown::Both); } if let Some(mut child) = self.child.lock().unwrap_or_else(|p| p.into_inner()).take() { - crate::child_process::kill_group(&mut child); + crate::child_process::log_shutdown( + &format!("sidecar {}", self.deck_id), + child.shutdown(Duration::from_millis(500)), + ); } if let Some(reader) = self.reader.take() { let _ = reader.join(); @@ -556,18 +1087,34 @@ impl Drop for Sidecar { /// without a dedicated timer thread, and checks `stop` each iteration so a /// teardown (`Drop`) or a model switch (`restart`) unblocks a never-connected /// accept promptly rather than waiting out the whole `timeout`. -fn accept_with_timeout( +fn accept_authenticated_with_timeout( listener: &TcpListener, stop: &AtomicBool, timeout: Duration, + expected_token: &[u8], ) -> Option { let deadline = std::time::Instant::now() + timeout; listener.set_nonblocking(true).ok(); loop { match listener.accept() { - Ok((stream, _)) => { + Ok((mut stream, _)) => { stream.set_nonblocking(false).ok(); - return Some(stream); + let remaining = deadline.saturating_duration_since(std::time::Instant::now()); + let handshake_timeout = remaining.min(Duration::from_secs(1)); + let _ = stream.set_read_timeout(Some(handshake_timeout)); + let authenticated = matches!( + read_frame(&mut stream), + Ok(Some((FRAME_AUTH, ref supplied))) + if crate::local_auth::constant_time_eq(supplied, expected_token) + ); + let _ = stream.set_read_timeout(None); + if authenticated { + return Some(stream); + } + let _ = stream.shutdown(std::net::Shutdown::Both); + if stop.load(Ordering::Acquire) || std::time::Instant::now() >= deadline { + return None; + } } Err(e) if e.kind() == io::ErrorKind::WouldBlock => { if stop.load(Ordering::Acquire) || std::time::Instant::now() >= deadline { @@ -583,34 +1130,122 @@ fn accept_with_timeout( /// The base sidecar launch command — program + base args + CWD — with NO mode /// flags. A packaged build uses the exact `LSDJ_BACKEND_BIN` path; otherwise /// `LSDJ_SIDECAR_CMD` is overridable (whitespace-split) and dev defaults to -/// `uv run python -m lsdj.sidecar`. The deck path ([`sidecar_command`]) and the model +/// `uv run python -m lsdj.sidecar`. The authenticated deck path and the model /// manager's installer (issue #43: `--init-resources` / `--download-model`) both /// build on this, so the resolution lives in one place — a download is NOT a /// deck, so it must not inherit `--deck`/`--model`/`--port`. +#[cfg_attr(feature = "managed-runtime", allow(dead_code))] pub fn sidecar_base_command() -> io::Result { + #[cfg(feature = "managed-runtime")] + { + let paths = crate::platform_paths::get(); + crate::managed_runtime::resolve( + paths.assets(), + crate::managed_runtime::Service::Mrt2, + ) + .and_then(|resolved| resolved.into_command([], paths.backend_env())) + .map_err(io::Error::other) + } + // A distributable app sets this to the exact bundled executable during // Tauri setup. Keep it as an OsString and pass it directly to Command so an // app copied into a path containing spaces still works. + #[cfg(not(feature = "managed-runtime"))] if let Some(program) = std::env::var_os("LSDJ_BACKEND_BIN") { return Ok(Command::new(program)); } - let overridden = std::env::var("LSDJ_SIDECAR_CMD"); - let spec = overridden - .clone() - .unwrap_or_else(|_| "uv run python -m lsdj.sidecar".to_string()); - let mut parts = spec.split_whitespace(); - let program = parts - .next() - .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "empty LSDJ_SIDECAR_CMD"))?; - let mut cmd = Command::new(program); - cmd.args(parts); - if overridden.is_err() { - // The default `uv run` needs the backend project dir as its CWD. A packaged - // build returned through LSDJ_BACKEND_BIN above and never reaches this path. - cmd.current_dir(std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../backend")); + #[cfg(not(feature = "managed-runtime"))] + { + let overridden = std::env::var("LSDJ_SIDECAR_CMD"); + let spec = overridden + .clone() + .unwrap_or_else(|_| "uv run python -m lsdj.sidecar".to_string()); + let mut parts = spec.split_whitespace(); + let program = parts + .next() + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "empty LSDJ_SIDECAR_CMD"))?; + let mut cmd = Command::new(program); + cmd.args(parts); + if overridden.is_err() { + // The default `uv run` needs the backend project dir as its CWD. A packaged + // build returned through LSDJ_BACKEND_BIN above and never reaches this path. + cmd.current_dir(std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../backend")); + } + Ok(cmd) + } +} + +#[cfg(feature = "managed-runtime")] +fn authenticated_sidecar_base_command(token: &str) -> io::Result { + use std::ffi::OsString; + + let paths = crate::platform_paths::get(); + let ephemeral = paths + .backend_env() + .into_iter() + .filter(|(name, _)| { + !matches!( + name.to_str(), + Some("SA3_HOME" | "SA3_MLX_HOME" | "SA3_LORAS_HOME") + ) + }) + .chain(std::iter::once(( + OsString::from("LSDJ_WORKER_LAUNCH_TOKEN"), + OsString::from(token), + ))); + crate::managed_runtime::resolve( + paths.assets(), + crate::managed_runtime::Service::Mrt2, + ) + .and_then(|resolved| resolved.into_command([], ephemeral)) + .map_err(io::Error::other) +} + +/// Build the dedicated managed MRT2 renderer command. The render worker is a +/// separate, disposable process from both realtime decks and the SA3 server; +/// every launch re-resolves and revalidates the promoted MRT2 manifest and gets +/// a fresh one-use loopback capability. +#[cfg(feature = "managed-runtime")] +pub(crate) fn authenticated_render_worker_command( + model: &str, + port: u16, + token: &str, +) -> io::Result { + let mut command = authenticated_sidecar_base_command(token)?; + command.args([ + "--render-worker", + "--model", + model, + "--runtime", + "pytorch-cuda", + "--port", + &port.to_string(), + ]); + Ok(command) +} + +/// Runtime selected by the native platform. The value is always sent over the +/// process boundary: Python never guesses and never falls back from CUDA to CPU. +/// The override exists for model-free contract tests and qualification hosts; +/// the Python policy layer still rejects an impossible platform/runtime pair. +pub fn mrt2_runtime_for_platform() -> io::Result { + let runtime = std::env::var("LSDJ_MRT2_RUNTIME").unwrap_or_else(|_| { + if cfg!(target_os = "macos") { + "mlx".to_string() + } else if cfg!(any(target_os = "linux", target_os = "windows")) { + "pytorch-cuda".to_string() + } else { + "unsupported".to_string() + } + }); + match runtime.as_str() { + "mlx" | "pytorch-cuda" => Ok(runtime), + _ => Err(io::Error::new( + io::ErrorKind::Unsupported, + format!("unsupported MRT2 runtime {runtime:?} for this platform"), + )), } - Ok(cmd) } /// Whether a status event ends the deck's transport: the worker stopped @@ -626,7 +1261,7 @@ pub fn sidecar_base_command() -> io::Result { pub fn transport_ended(status_json: &str) -> bool { matches!( status_event(status_json).as_deref(), - Some("worker_died") | Some("model_loading") | Some("stopped") + Some("worker_died") | Some("startup_failed") | Some("model_loading") | Some("stopped") ) } @@ -640,15 +1275,62 @@ pub fn status_event(status_json: &str) -> Option { .map(str::to_owned) } -/// Build the command that launches the Python sidecar for a deck, pointed at the -/// loopback `port` — the base command plus the deck-mode flags. -pub fn sidecar_command(deck_id: &str, model: &str, port: u16) -> io::Result { - let mut cmd = sidecar_base_command()?; +fn authenticated_sidecar_command( + deck_id: &str, + model: &str, + port: u16, + token: &str, +) -> io::Result { + #[cfg(feature = "managed-runtime")] + let mut cmd = authenticated_sidecar_base_command(token)?; + #[cfg(not(feature = "managed-runtime"))] + let mut cmd = { + let mut cmd = sidecar_base_command()?; + cmd.env("LSDJ_WORKER_LAUNCH_TOKEN", token); + cmd + }; + let runtime = mrt2_runtime_for_platform()?; cmd.args([ "--deck", deck_id, "--model", model, + "--runtime", + &runtime, + "--port", + &port.to_string(), + ]); + Ok(cmd) +} + +fn authenticated_shared_sidecar_command( + models: &[String; lsdj_engine::DECK_COUNT], + port: u16, + token: &str, +) -> io::Result { + #[cfg(feature = "managed-runtime")] + let mut cmd = authenticated_sidecar_base_command(token)?; + #[cfg(not(feature = "managed-runtime"))] + let mut cmd = { + let mut cmd = sidecar_base_command()?; + cmd.env("LSDJ_WORKER_LAUNCH_TOKEN", token); + cmd + }; + let runtime = mrt2_runtime_for_platform()?; + if runtime != "pytorch-cuda" { + return Err(io::Error::new( + io::ErrorKind::Unsupported, + format!("shared MRT2 worker requires pytorch-cuda, got {runtime}"), + )); + } + cmd.args([ + "--shared", + "--model-a", + &models[0], + "--model-b", + &models[1], + "--runtime", + &runtime, "--port", &port.to_string(), ]); @@ -660,20 +1342,110 @@ mod tests { use super::*; use lsdj_engine::Engine; use std::net::TcpStream; + #[cfg(all(unix, not(feature = "managed-runtime")))] use std::os::unix::fs::PermissionsExt; + use std::sync::atomic::AtomicUsize; + + #[cfg(all(unix, not(feature = "managed-runtime")))] + static SIDECAR_ENV_LOCK: Mutex<()> = Mutex::new(()); + + #[test] + fn native_platform_selects_one_explicit_mrt2_runtime() { + let runtime = mrt2_runtime_for_platform().expect("supported build target"); + if cfg!(target_os = "macos") { + assert_eq!(runtime, "mlx"); + } else { + assert_eq!(runtime, "pytorch-cuda"); + } + } + + #[test] + fn shared_deck_handles_can_park_until_the_first_managed_install() { + let mut engine = Engine::new(); + let handles = [engine.create_deck(0), engine.create_deck(1)]; + let sinks: DeckStatusSinks = std::array::from_fn(|_| { + Box::new(|_message| {}) as StatusSink + }); + let mut shared = SharedSidecar::parked( + ["mrt2_small".into(), "mrt2_small".into()], + handles, + sinks, + PcmTaps::new(lsdj_engine::DECK_COUNT), + AnalysisFeed::disconnected(lsdj_engine::DECK_COUNT), + ); + + assert!(shared.reader.is_none()); + assert!(shared.parked.is_some()); + assert!(matches!( + &*shared + .process + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()), + SharedProcessState::Stopped + )); + assert!(shared.stop.load(Ordering::Acquire)); + + let shutdowns = Arc::new(AtomicUsize::new(1)); + *shared.process.lock().unwrap() = + SharedProcessState::Uncertain(Box::new(RetrySharedProcess { + shutdowns: shutdowns.clone(), + })); + assert!(shared.reader.is_none()); + assert!(shared.activate().is_err()); + assert_eq!(shutdowns.load(Ordering::Acquire), 1); + } + + struct RetrySharedProcess { + shutdowns: Arc, + } + + impl SharedProcess for RetrySharedProcess { + fn shutdown(&mut self) -> io::Result<()> { + if self.shutdowns.fetch_add(1, Ordering::AcqRel) == 0 { + Err(io::Error::other("first reap is uncertain")) + } else { + Ok(()) + } + } + } + + #[test] + fn failed_shared_reap_retains_supervisor_until_a_positive_retry() { + let shutdowns = Arc::new(AtomicUsize::new(0)); + let mut state = SharedProcessState::Running(Box::new(RetrySharedProcess { + shutdowns: shutdowns.clone(), + })); + + assert!(stop_shared_process(&mut state).is_err()); + assert!(matches!(state, SharedProcessState::Uncertain(_))); + assert_eq!(shutdowns.load(Ordering::Acquire), 1); + + assert!(stop_shared_process(&mut state).is_ok()); + assert!(matches!(state, SharedProcessState::Stopped)); + assert_eq!(shutdowns.load(Ordering::Acquire), 2); + } #[test] fn transport_ended_matches_only_worker_end_events() { // The three events after which the worker is no longer generating. assert!(transport_ended(r#"{"event":"worker_died","deck":"a"}"#)); - assert!(transport_ended(r#"{"event":"model_loading","deck":"a","model":"mrt2_base"}"#)); + assert!(transport_ended( + r#"{"event":"startup_failed","deck":"a","error":"CUDA unavailable"}"# + )); + assert!(transport_ended( + r#"{"event":"model_loading","deck":"a","model":"mrt2_base"}"# + )); // The worker halting itself (a generation failure) ends the transport // too — missing it wedged the play button behind a stale store. - assert!(transport_ended(r#"{"event":"stopped","reason":"generation failed"}"#)); + assert!(transport_ended( + r#"{"event":"stopped","reason":"generation failed"}"# + )); // Everything else — including the events of a healthy stream — is not a // transport signal, and neither is garbage. A plain error is NOT a // stop: the worker survives bad payloads without ending the stream. - assert!(!transport_ended(r#"{"event":"ready","deck":"a","model":"mrt2_small"}"#)); + assert!(!transport_ended( + r#"{"event":"ready","deck":"a","model":"mrt2_small"}"# + )); assert!(!transport_ended(r#"{"event":"chunk","index":3,"rtf":1.2}"#)); assert!(!transport_ended(r#"{"event":"error","error":"boom"}"#)); assert!(!transport_ended("not json")); @@ -746,6 +1518,53 @@ mod tests { assert_eq!(statuses, vec!["{\"event\":\"chunk\"}".to_string()]); } + #[test] + fn shared_reader_routes_prefixed_frames_to_independent_decks() { + let mut engine = Engine::new(); + let handles = [engine.create_deck(0), engine.create_deck(1)]; + let free_before = [handles[0].free_samples(), handles[1].free_samples()]; + let pcm = [0.25f32, -0.25f32] + .into_iter() + .flat_map(f32::to_le_bytes) + .collect::>(); + let mut wire = Vec::new(); + let mut deck_one_pcm = vec![1u8]; + deck_one_pcm.extend_from_slice(&pcm); + write_frame(&mut wire, FRAME_PCM, &deck_one_pcm).unwrap(); + let mut deck_zero_status = vec![0u8]; + deck_zero_status.extend_from_slice(br#"{"event":"ready"}"#); + write_frame(&mut wire, FRAME_STATUS, &deck_zero_status).unwrap(); + + let statuses = Arc::new(Mutex::new(Vec::<(usize, String)>::new())); + let teed = Arc::new(Mutex::new(Vec::<(usize, Vec)>::new())); + let statuses_zero = statuses.clone(); + let statuses_one = statuses.clone(); + let teed_zero = teed.clone(); + let teed_one = teed.clone(); + let mut status_sinks: DeckStatusSinks = [ + Box::new(move |value| statuses_zero.lock().unwrap().push((0, value))), + Box::new(move |value| statuses_one.lock().unwrap().push((1, value))), + ]; + let mut pcm_sinks: DeckPcmSinks = [ + Box::new(move |value| teed_zero.lock().unwrap().push((0, value.to_vec()))), + Box::new(move |value| teed_one.lock().unwrap().push((1, value.to_vec()))), + ]; + let handles = run_shared_reader( + std::io::Cursor::new(wire), + handles, + &mut status_sinks, + &mut pcm_sinks, + ); + + assert_eq!(handles[0].free_samples(), free_before[0]); + assert_eq!(free_before[1] - handles[1].free_samples(), 2); + assert_eq!( + *statuses.lock().unwrap(), + vec![(0, "{\"event\":\"ready\"}".to_string())] + ); + assert_eq!(*teed.lock().unwrap(), vec![(1, pcm)]); + } + /// A status frame arriving over a real loopback socket reaches the sink — the /// transport itself (accept/connect/nodelay), end to end without Python. #[test] @@ -768,33 +1587,182 @@ mod tests { assert_eq!(statuses, vec!["{\"event\":\"ready\"}".to_string()]); } + #[test] + fn wrong_first_client_cannot_capture_the_sidecar_listener() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let token = b"correct-sidecar-token-0123456789abcdef".to_vec(); + let expected = token.clone(); + let stop = Arc::new(AtomicBool::new(false)); + let stop_for_accept = stop.clone(); + let accept = thread::spawn(move || { + let mut stream = accept_authenticated_with_timeout( + &listener, + &stop_for_accept, + Duration::from_secs(2), + &expected, + ) + .expect("legitimate sidecar should be accepted after rejecting the racer"); + read_frame(&mut stream).unwrap() + }); + + let mut attacker = TcpStream::connect(addr).unwrap(); + write_frame(&mut attacker, FRAME_AUTH, b"wrong-sidecar-token-0123456789abcdef") + .unwrap(); + drop(attacker); + + let mut legitimate = TcpStream::connect(addr).unwrap(); + write_frame(&mut legitimate, FRAME_AUTH, &token).unwrap(); + write_frame(&mut legitimate, FRAME_STATUS, br#"{"event":"ready"}"#).unwrap(); + drop(legitimate); + + assert_eq!( + accept.join().unwrap(), + Some((FRAME_STATUS, br#"{"event":"ready"}"#.to_vec())) + ); + } + + #[cfg(all(windows, feature = "managed-runtime"))] + fn windows_python() -> std::path::PathBuf { + let search = std::env::var_os("PATH").expect("Python is available on CI PATH"); + for directory in std::env::split_paths(&search) { + for name in ["python.exe", "python3.exe"] { + let candidate = directory.join(name); + if candidate.is_file() { + return candidate; + } + } + } + panic!("Python executable is unavailable for Windows protocol qualification"); + } + + /// Native Windows qualification for the real Rust/Python wire boundary. + /// The stand-in is stdlib-only and model-free, but the socket, authenticated + /// handshake, bidirectional framing, structured argv, and supervised Python + /// process are the same primitives used by the packaged sidecar. + #[cfg(all(windows, feature = "managed-runtime"))] + #[test] + fn windows_python_round_trips_authenticated_control_status_and_pcm() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let port = listener.local_addr().unwrap().port(); + let token = "windows-protocol-token-0123456789abcdef"; + let root = std::env::temp_dir().join(format!( + "lsdj-windows-python-protocol-{}-{port}-资产", + std::process::id() + )); + let _ = std::fs::remove_dir_all(&root); + std::fs::create_dir_all(&root).unwrap(); + let script = root.join("sidecar stand-in.py"); + std::fs::write( + &script, + r#"import json +import os +import socket +import struct +import sys + +def receive_exact(sock, length): + payload = b"" + while len(payload) < length: + chunk = sock.recv(length - len(payload)) + if not chunk: + raise RuntimeError("truncated frame") + payload += chunk + return payload + +def receive_frame(sock): + header = receive_exact(sock, 5) + frame_type, length = struct.unpack(">(); + assert_eq!(read_frame(&mut stream).unwrap(), Some((FRAME_PCM, pcm))); + drop(stream); + let status = child.wait().unwrap(); + assert!( + status.success(), + "Python protocol stand-in failed: {status}" + ); + std::fs::remove_dir_all(root).unwrap(); + } + /// In-process model switch: `restart` respawns the sidecar with a new model, /// reusing the deck's permanent ring producer, and suppresses a false /// `worker_died` across the deliberate switch. Wires a minimal stdlib-only /// wrapper + Python stand-in (no models) via `LSDJ_SIDECAR_CMD`, matching the /// `uv run` parent/grandchild topology used in development. + #[cfg(all(unix, not(feature = "managed-runtime")))] #[test] fn restart_switches_model_without_a_worker_died() { + let _env_guard = SIDECAR_ENV_LOCK.lock().unwrap(); // A stand-in sidecar: connect to --port, announce ready with --model, then // deliberately ignore socket EOF. Teardown must kill it as the wrapper's // process-group child; killing only the wrapper leaves this process and // the Rust reader alive forever. No backend deps. - let script = r#"import socket, struct, json, argparse, time + let script = r#"import socket, struct, json, argparse, time, os p = argparse.ArgumentParser() p.add_argument('--port', type=int) p.add_argument('--model') p.add_argument('--deck') a, _ = p.parse_known_args() s = socket.create_connection(('127.0.0.1', a.port)) +token = os.environ['LSDJ_WORKER_LAUNCH_TOKEN'].encode() +s.sendall(struct.pack('> \"{}\"\nwait \"$child\"\n", + python.display(), + pidfile.display() + ), + ) + .unwrap(); + let mut permissions = std::fs::metadata(&wrapper).unwrap().permissions(); + permissions.set_mode(0o755); + std::fs::set_permissions(&wrapper, permissions).unwrap(); + std::env::set_var("LSDJ_SIDECAR_CMD", wrapper.as_os_str()); + std::env::set_var("LSDJ_MRT2_RUNTIME", "pytorch-cuda"); + std::env::set_var("LSDJ_TEST_PIDFILE", pidfile.as_os_str()); + std::env::set_var("LSDJ_TEST_OVERLAP", overlap.as_os_str()); + + let mut engine = Engine::new(); + let handles = [engine.create_deck(0), engine.create_deck(1)]; + let statuses = Arc::new(Mutex::new(Vec::::new())); + let sinks: DeckStatusSinks = std::array::from_fn(|_| { + let statuses = statuses.clone(); + Box::new(move |message| statuses.lock().unwrap().push(message)) as StatusSink + }); + let taps = PcmTaps::new(2); + let feed = AnalysisFeed::disconnected(2); + let mut shared = SharedSidecar::spawn( + ["model_a".into(), "model_b".into()], + handles, + sinks, + taps, + feed, + ) + .map_err(|(error, _)| error) + .expect("spawn shared stand-in"); + + let saw_ready = |model: &str| { + let deadline = std::time::Instant::now() + Duration::from_secs(10); + while std::time::Instant::now() < deadline { + if statuses + .lock() + .unwrap() + .iter() + .any(|status| status.contains("ready") && status.contains(model)) + { + return true; + } + thread::sleep(Duration::from_millis(20)); + } + false + }; + assert!(saw_ready("model_a")); + assert!(saw_ready("model_b")); + + shared.restart(0, "model_c").expect("serialized switch"); + assert!(saw_ready("model_c")); + assert!(!overlap.exists(), "old and replacement workers overlapped"); + + std::env::set_var("LSDJ_SIDECAR_CMD", tmp.join("missing-sidecar")); + let error = shared.restart(1, "model_x").unwrap_err(); + assert!(error.to_string().contains("reselect a model to retry")); + std::env::set_var("LSDJ_SIDECAR_CMD", wrapper.as_os_str()); + shared + .restart(0, "model_d") + .expect("retry from parked handles"); + assert!(saw_ready("model_d")); + assert!(!overlap.exists(), "recovery overlapped CUDA generations"); + + assert!( + !statuses + .lock() + .unwrap() + .iter() + .any(|status| status.contains("worker_died")), + "deliberate switches and launch failure must suppress worker_died" + ); + shared + .restart(1, "load_fail") + .expect("replacement process launched before model-load failure"); + let deadline = std::time::Instant::now() + Duration::from_secs(10); + while std::time::Instant::now() < deadline + && !statuses + .lock() + .unwrap() + .iter() + .any(|status| status.contains("startup_failed")) + { + thread::sleep(Duration::from_millis(20)); + } + assert!( + statuses + .lock() + .unwrap() + .iter() + .any(|status| status.contains("startup_failed")) + ); + shared + .restart(1, "model_e") + .expect("recover after replacement model-load failure"); + assert!(saw_ready("model_e")); + assert!(!overlap.exists(), "load recovery overlapped CUDA generations"); + + let log = statuses.lock().unwrap(); + for model in [ + "model_c", + "model_b", + "model_x", + "model_d", + "load_fail", + "model_e", + ] { + assert!( + log.iter() + .any(|status| status.contains("model_loading") && status.contains(model)), + "missing both-deck loading state for {model}" + ); + } + assert!(log.iter().any(|status| status.contains("worker_died"))); + drop(log); + + drop(shared); + let pids: Vec = std::fs::read_to_string(&pidfile) + .unwrap() + .lines() + .map(|line| line.parse().unwrap()) + .collect(); + assert_eq!(pids.len(), 5, "failed launch must not create a child"); + for pid in pids { + let mut gone = false; + for _ in 0..1000 { + if unsafe { libc::kill(pid, 0) } == -1 { + gone = true; + break; + } + thread::sleep(Duration::from_millis(10)); + } + assert!(gone, "shared worker {pid} survived serialized transition"); + } + + for name in [ + "LSDJ_SIDECAR_CMD", + "LSDJ_MRT2_RUNTIME", + "LSDJ_TEST_PIDFILE", + "LSDJ_TEST_OVERLAP", + ] { + std::env::remove_var(name); + } + let _ = std::fs::remove_dir_all(&tmp); + } } diff --git a/src-tauri/src/songs.rs b/src-tauri/src/songs.rs index c8f5aac..7b6ce40 100644 --- a/src-tauri/src/songs.rs +++ b/src-tauri/src/songs.rs @@ -205,6 +205,27 @@ fn reconcile(existing: Vec, disk: &[String]) -> Vec { #[cfg(test)] mod tests { use super::*; + use std::path::PathBuf; + use std::sync::atomic::{AtomicU64, Ordering}; + + static NEXT_TEMP_DIR: AtomicU64 = AtomicU64::new(0); + + fn test_temp_dir(label: &str) -> PathBuf { + std::env::temp_dir().join(format!( + "{label}-{}-{}", + std::process::id(), + NEXT_TEMP_DIR.fetch_add(1, Ordering::Relaxed) + )) + } + + #[test] + fn test_temp_directory_names_are_windows_safe() { + let path = test_temp_dir("lsdj-song-test"); + let name = path.file_name().unwrap().to_string_lossy(); + assert!(!name + .chars() + .any(|character| r#"<>:"/\|?*"#.contains(character))); + } fn entry(file: &str, model: Option<&str>) -> SongEntry { SongEntry { @@ -307,11 +328,7 @@ mod tests { #[test] fn unknown_future_recipe_shape_survives_library_reconciliation() { - let dir = std::env::temp_dir().join(format!( - "lsdj-future-song-recipe-test-{}-{}", - std::process::id(), - std::thread::current().name().unwrap_or("thread") - )); + let dir = test_temp_dir("lsdj-future-song-recipe-test"); std::fs::remove_dir_all(&dir).ok(); std::fs::create_dir_all(&dir).unwrap(); std::fs::write(dir.join("future.wav"), b"RIFF future bytes").unwrap(); @@ -352,11 +369,7 @@ mod tests { #[test] fn a_fresh_library_instance_restores_the_recorded_recipe() { - let dir = std::env::temp_dir().join(format!( - "lsdj-song-recipe-test-{}-{}", - std::process::id(), - std::thread::current().name().unwrap_or("thread") - )); + let dir = test_temp_dir("lsdj-song-recipe-test"); std::fs::remove_dir_all(&dir).ok(); let recipe = GenerationRecipe { version: 1, diff --git a/src-tauri/tauri.conf.json b/src-tauri/tauri.conf.json index bd7edb4..2146b7b 100644 --- a/src-tauri/tauri.conf.json +++ b/src-tauri/tauri.conf.json @@ -14,8 +14,6 @@ "title": "LSDJ", "width": 1280, "height": 800, - "titleBarStyle": "Overlay", - "hiddenTitle": true, "backgroundColor": "#050507" } ], @@ -25,7 +23,6 @@ }, "bundle": { "active": true, - "targets": ["app", "dmg"], "category": "Music", "icon": [ "icons/32x32.png", @@ -33,11 +30,6 @@ "icons/128x128@2x.png", "icons/icon.icns", "icons/icon.ico" - ], - "macOS": { - "minimumSystemVersion": "11.0", - "signingIdentity": "-", - "entitlements": "entitlements.plist" - } + ] } } diff --git a/src-tauri/tauri.linux.conf.json b/src-tauri/tauri.linux.conf.json new file mode 100644 index 0000000..2eca149 --- /dev/null +++ b/src-tauri/tauri.linux.conf.json @@ -0,0 +1,28 @@ +{ + "$schema": "https://schema.tauri.app/config/2", + "app": { + "windows": [ + { + "label": "main", + "title": "LSDJ", + "width": 1280, + "height": 800, + "decorations": true, + "backgroundColor": "#050507" + } + ] + }, + "bundle": { + "targets": ["appimage"], + "icon": [ + "icons/32x32.png", + "icons/128x128.png", + "icons/128x128@2x.png" + ], + "linux": { + "appimage": { + "bundleMediaFramework": false + } + } + } +} diff --git a/src-tauri/tauri.macos.conf.json b/src-tauri/tauri.macos.conf.json new file mode 100644 index 0000000..159b5cb --- /dev/null +++ b/src-tauri/tauri.macos.conf.json @@ -0,0 +1,24 @@ +{ + "$schema": "https://schema.tauri.app/config/2", + "app": { + "windows": [ + { + "label": "main", + "title": "LSDJ", + "width": 1280, + "height": 800, + "titleBarStyle": "Overlay", + "hiddenTitle": true, + "backgroundColor": "#050507" + } + ] + }, + "bundle": { + "targets": ["app", "dmg"], + "macOS": { + "minimumSystemVersion": "11.0", + "signingIdentity": "-", + "entitlements": "entitlements.plist" + } + } +} diff --git a/src-tauri/tauri.windows.conf.json b/src-tauri/tauri.windows.conf.json new file mode 100644 index 0000000..6de184a --- /dev/null +++ b/src-tauri/tauri.windows.conf.json @@ -0,0 +1,22 @@ +{ + "$schema": "https://schema.tauri.app/config/2", + "bundle": { + "targets": ["nsis"], + "windows": { + "allowDowngrades": false, + "webviewInstallMode": { + "type": "downloadBootstrapper", + "silent": true + }, + "nsis": { + "installMode": "currentUser", + "startMenuFolder": "LSDJ", + "installerHooks": "./windows/installer-hooks.nsh", + "languages": ["English"], + "customLanguageFiles": { + "English": "./windows/English.nsh" + } + } + } + } +} diff --git a/src-tauri/tauri.windows.release.conf.json b/src-tauri/tauri.windows.release.conf.json new file mode 100644 index 0000000..c0ca4fa --- /dev/null +++ b/src-tauri/tauri.windows.release.conf.json @@ -0,0 +1,21 @@ +{ + "$schema": "https://schema.tauri.app/config/2", + "bundle": { + "windows": { + "signCommand": { + "command": "pwsh.exe", + "args": [ + "-NoLogo", + "-NoProfile", + "-NonInteractive", + "-ExecutionPolicy", + "Bypass", + "-File", + "../scripts/sign-windows.ps1", + "-Path", + "%1" + ] + } + } + } +} diff --git a/src-tauri/windows/English.nsh b/src-tauri/windows/English.nsh new file mode 100644 index 0000000..f50a0a0 --- /dev/null +++ b/src-tauri/windows/English.nsh @@ -0,0 +1,27 @@ +LangString addOrReinstall ${LANG_ENGLISH} "Add/Reinstall components" +LangString alreadyInstalled ${LANG_ENGLISH} "Already Installed" +LangString alreadyInstalledLong ${LANG_ENGLISH} "${PRODUCTNAME} ${VERSION} is already installed. Select the operation you want to perform and click Next to continue." +LangString appRunning ${LANG_ENGLISH} "${PRODUCTNAME} is running. Close it, then try again." +LangString appRunningOkKill ${LANG_ENGLISH} "${PRODUCTNAME} is running.$\nClick OK to close it." +LangString chooseMaintenanceOption ${LANG_ENGLISH} "Choose the maintenance option to perform." +LangString choowHowToInstall ${LANG_ENGLISH} "Choose how you want to install ${PRODUCTNAME}." +LangString createDesktop ${LANG_ENGLISH} "Create desktop shortcut" +LangString dontUninstall ${LANG_ENGLISH} "Do not uninstall" +LangString dontUninstallDowngrade ${LANG_ENGLISH} "Do not uninstall (downgrading without uninstall is disabled)" +LangString failedToKillApp ${LANG_ENGLISH} "Failed to close ${PRODUCTNAME}. Close it, then try again." +LangString installingWebview2 ${LANG_ENGLISH} "Installing WebView2..." +LangString newerVersionInstalled ${LANG_ENGLISH} "A newer version of ${PRODUCTNAME} is installed. This installer cannot downgrade it." +LangString older ${LANG_ENGLISH} "older" +LangString olderOrUnknownVersionInstalled ${LANG_ENGLISH} "An $R4 version of ${PRODUCTNAME} is installed. Select how to continue." +LangString silentDowngrades ${LANG_ENGLISH} "Downgrades are disabled for this installer.$\n" +LangString unableToUninstall ${LANG_ENGLISH} "Unable to uninstall ${PRODUCTNAME}." +LangString uninstallApp ${LANG_ENGLISH} "Uninstall ${PRODUCTNAME}" +LangString uninstallBeforeInstalling ${LANG_ENGLISH} "Uninstall before installing" +LangString unknown ${LANG_ENGLISH} "unknown" +LangString webview2AbortError ${LANG_ENGLISH} "WebView2 could not be installed. LSDJ cannot run without it. Check the network connection, or install the Microsoft WebView2 Evergreen Runtime and retry." +LangString webview2DownloadError ${LANG_ENGLISH} "WebView2 download failed: $0" +LangString webview2DownloadSuccess ${LANG_ENGLISH} "WebView2 bootstrapper downloaded successfully" +LangString webview2Downloading ${LANG_ENGLISH} "Downloading the Microsoft WebView2 bootstrapper..." +LangString webview2InstallError ${LANG_ENGLISH} "WebView2 installation failed with exit code $1" +LangString webview2InstallSuccess ${LANG_ENGLISH} "WebView2 installed successfully" +LangString deleteAppData ${LANG_ENGLISH} "Also remove downloaded models, runtimes, settings, and user data (the path and size will be confirmed)" diff --git a/src-tauri/windows/installer-hooks.nsh b/src-tauri/windows/installer-hooks.nsh new file mode 100644 index 0000000..abd2e6b --- /dev/null +++ b/src-tauri/windows/installer-hooks.nsh @@ -0,0 +1,1098 @@ +; LSDJ's app-managed assets share Tauri's current-user install root at +; $LOCALAPPDATA\LSDJ. Ownership must be established before Tauri copies any +; payload there. Uninstall preserves the tree by default and recursively +; removes it only after explicit opt-in, exact-path checks, and reparse-safe +; validation. /PURGE-LSDJ-DATA is the equivalent explicit automation choice. + +!define LSDJ_DATA_ROOT "$LOCALAPPDATA\LSDJ" +!define LSDJ_DATA_MARKER "${LSDJ_DATA_ROOT}\.lsdj-data-root" +!define LSDJ_DATA_MARKER_NEW "${LSDJ_DATA_ROOT}\.lsdj-data-root.new" +!define LSDJ_OWNER_ID "works.protocol.lsdj" +!define LSDJ_OWNER_ID_BYTES 19 +!define LSDJ_OWNER_ID_READ_BYTES 20 +!define LSDJ_FILE_ATTRIBUTE_DIRECTORY 0x10 +!define LSDJ_FILE_ATTRIBUTE_REPARSE_POINT 0x400 +!define LSDJ_FILE_ATTRIBUTE_NORMAL 0x80 +!define LSDJ_FILE_FLAG_OPEN_REPARSE_POINT 0x200000 +!define LSDJ_FILE_SHARE_READ 0x1 +!define LSDJ_GENERIC_READ 0x80000000 +!define LSDJ_GENERIC_WRITE 0x40000000 +!define LSDJ_OPEN_EXISTING 3 +!define LSDJ_CREATE_NEW 1 +!define LSDJ_INVALID_FILE_ATTRIBUTES -1 +!define LSDJ_INVALID_HANDLE_VALUE -1 + +Var LsdjCanonicalRootSafe +Var LsdjDeleteData +Var LsdjDeleteFailure +Var LsdjDataRemovalFailed +Var LsdjInstallRootState +Var LsdjInstalledVersion +Var LsdjInstalledVersionPresent +Var LsdjInstalledVersionValidity +Var LsdjExistingInstall +Var LsdjMarkerSafe +Var LsdjOwnedRootSafe +Var LsdjPassiveRequested +Var LsdjRegistryEvidence +Var LsdjRootEmpty +Var LsdjSafeLayout +Var LsdjTreeSafe +Var LsdjVersionCompare + +; Unsigned hosted-test installers can leave a narrow control-flow trace when a +; silent fail-closed branch returns only NSIS's generic exit code. Production +; builds do not define LSDJ_CI_ADVERSARIAL_TESTS, so every trace call expands +; to no instructions. Preserve both the scratch register and the caller's NSIS +; error flag so diagnostics cannot change installer decisions. +!ifdef LSDJ_CI_ADVERSARIAL_TESTS + Var LsdjCiTraceHadErrors + Var LsdjCiTraceMessage + !macro LSDJ_CI_TRACE MESSAGE + ${If} ${Errors} + StrCpy $LsdjCiTraceHadErrors 1 + ${Else} + StrCpy $LsdjCiTraceHadErrors 0 + ${EndIf} + ; Capture interpolated register values before using R5 for the trace file + ; handle, otherwise messages containing $R5 log that temporary handle. + StrCpy $LsdjCiTraceMessage "${MESSAGE}" + Push $R5 + ClearErrors + FileOpen $R5 "$TEMP\lsdj-ci-installer.trace" a + ${IfNot} ${Errors} + ; NSIS preserves existing contents for mode `a` but still positions the + ; file pointer at byte zero. Seek explicitly so checkpoints cannot + ; overwrite one another. + FileSeek $R5 0 END + FileWrite $R5 "$LsdjCiTraceMessage$\r$\n" + FileClose $R5 + ${EndIf} + Pop $R5 + ${If} $LsdjCiTraceHadErrors = 1 + SetErrors + ${Else} + ClearErrors + ${EndIf} + !macroend +!else + !macro LSDJ_CI_TRACE MESSAGE + !macroend +!endif + +; Passive mode reaches Tauri's maintenance page before any installer section, +; and that page may uninstall an existing NSIS or legacy WiX installation. +; Reject /P at GUI initialization, before PageReinstall can run. Repeat the +; same owned command-line check in PREINSTALL for the combined /S /P case, +; where NSIS does not initialize the GUI. +!macro LSDJ_REJECT_PASSIVE_MODE + StrCpy $LsdjPassiveRequested 0 + ClearErrors + ${GetOptions} $CMDLINE "/P" $LsdjPassiveRequested + ${IfNot} ${Errors} + !insertmacro LSDJ_CI_TRACE "abort: passive install unsupported" + SetErrorLevel 2 + Quit + ${EndIf} + ClearErrors +!macroend + +!define MUI_CUSTOMFUNCTION_GUIINIT LsdjRejectPassiveMode +Function LsdjRejectPassiveMode + !insertmacro LSDJ_REJECT_PASSIVE_MODE +FunctionEnd + +; GetFullPathNameW is lexical and does not traverse the candidate. The exact +; canonical target must equal canonical LOCALAPPDATA + \LSDJ; callers then +; separately reject a root reparse point before reading or changing it. +!macro LSDJ_DEFINE_CANONICAL_ROOT_VALIDATOR FUNCTION_NAME +Function ${FUNCTION_NAME} + Push $R3 + Push $R4 + Push $R5 + Push $R6 + Push $R7 + Push $R8 + StrCpy $LsdjCanonicalRootSafe 0 + + System::Call 'kernel32::GetFullPathNameW(w "$LOCALAPPDATA", i 1024, w .R7, p 0) i .R8' + ${If} $R8 = 0 + ${OrIf} $R8 >= 1024 + Goto lsdj_canonical_done + ${EndIf} + StrCpy $R6 "$R7\LSDJ" + System::Call 'kernel32::GetFullPathNameW(w R6, i 1024, w .R4, p 0) i .R3' + ${If} $R3 = 0 + ${OrIf} $R3 >= 1024 + Goto lsdj_canonical_done + ${EndIf} + System::Call 'kernel32::GetFullPathNameW(w "${LSDJ_DATA_ROOT}", i 1024, w .R5, p 0) i .R8' + ${If} $R8 = 0 + ${OrIf} $R8 >= 1024 + Goto lsdj_canonical_done + ${EndIf} + System::Call 'kernel32::lstrcmpiW(w R5, w R4) i .R8' + ${If} $R8 = 0 + StrCpy $LsdjCanonicalRootSafe 1 + ${EndIf} + + lsdj_canonical_done: + Pop $R8 + Pop $R7 + Pop $R6 + Pop $R5 + Pop $R4 + Pop $R3 +FunctionEnd +!macroend + +!insertmacro LSDJ_DEFINE_CANONICAL_ROOT_VALIDATOR LsdjCanonicalDataRootIsValid +!insertmacro LSDJ_DEFINE_CANONICAL_ROOT_VALIDATOR un.LsdjCanonicalDataRootIsValid + +; Return LsdjMarkerSafe=1 only for a plain, non-reparse marker whose complete +; contents are the exact LSDJ application identifier. CreateFile opens the +; reparse entry itself and denies write/delete sharing. Both metadata and +; content are then read from that same native handle, so validation never +; reopens the marker by path. +!macro LSDJ_DEFINE_MARKER_VALIDATOR FUNCTION_NAME +Function ${FUNCTION_NAME} + Push $R5 + Push $R6 + Push $R7 + Push $R8 + !ifdef LSDJ_CI_ADVERSARIAL_TESTS + Push $R9 + !endif + StrCpy $LsdjMarkerSafe 0 + !ifdef LSDJ_CI_ADVERSARIAL_TESTS + System::Call 'kernel32::CreateFileW(w "${LSDJ_DATA_MARKER}", i ${LSDJ_GENERIC_READ}, i ${LSDJ_FILE_SHARE_READ}, p 0, i ${LSDJ_OPEN_EXISTING}, i ${LSDJ_FILE_FLAG_OPEN_REPARSE_POINT}, p 0) p .R6 ?e' + Pop $R9 + !insertmacro LSDJ_CI_TRACE "marker-validate: create handle=$R6 error=$R9" + !else + System::Call 'kernel32::CreateFileW(w "${LSDJ_DATA_MARKER}", i ${LSDJ_GENERIC_READ}, i ${LSDJ_FILE_SHARE_READ}, p 0, i ${LSDJ_OPEN_EXISTING}, i ${LSDJ_FILE_FLAG_OPEN_REPARSE_POINT}, p 0) p .R6' + !endif + StrCmp $R6 ${LSDJ_INVALID_HANDLE_VALUE} lsdj_marker_done + StrCpy $R7 0 + System::Alloc 52 + Pop $R7 + !insertmacro LSDJ_CI_TRACE "marker-validate: info buffer=$R7" + StrCmp $R7 0 lsdj_marker_close + !ifdef LSDJ_CI_ADVERSARIAL_TESTS + System::Call 'kernel32::GetFileInformationByHandle(p R6, p R7) i .R8 ?e' + Pop $R9 + !insertmacro LSDJ_CI_TRACE "marker-validate: info result=$R8 error=$R9" + !else + System::Call 'kernel32::GetFileInformationByHandle(p R6, p R7) i .R8' + !endif + ${If} $R8 = 0 + Goto lsdj_marker_info_failed + ${EndIf} + System::Call '*$R7(&i4 .R8)' + System::Free $R7 + StrCpy $R7 0 + !insertmacro LSDJ_CI_TRACE "marker-validate: attrs=$R8" + IntOp $R5 $R8 & ${LSDJ_FILE_ATTRIBUTE_REPARSE_POINT} + ${If} $R5 <> 0 + Goto lsdj_marker_close + ${EndIf} + IntOp $R5 $R8 & ${LSDJ_FILE_ATTRIBUTE_DIRECTORY} + ${If} $R5 <> 0 + Goto lsdj_marker_close + ${EndIf} + !ifdef LSDJ_CI_ADVERSARIAL_TESTS + System::Call 'kernel32::ReadFile(p R6, m .R8, i ${LSDJ_OWNER_ID_READ_BYTES}, *i .R7, p 0) i .R5 ?e' + Pop $R9 + !insertmacro LSDJ_CI_TRACE "marker-validate: read result=$R5 bytes=$R7 error=$R9" + !else + System::Call 'kernel32::ReadFile(p R6, m .R8, i ${LSDJ_OWNER_ID_READ_BYTES}, *i .R7, p 0) i .R5' + !endif + ${If} $R5 = 0 + ${OrIf} $R7 != ${LSDJ_OWNER_ID_BYTES} + Goto lsdj_marker_close + ${EndIf} + StrCmp $R8 "${LSDJ_OWNER_ID}" 0 lsdj_marker_close + StrCpy $LsdjMarkerSafe 1 + Goto lsdj_marker_close + + lsdj_marker_info_failed: + System::Free $R7 + StrCpy $R7 0 + + lsdj_marker_close: + System::Call 'kernel32::CloseHandle(p R6)' + lsdj_marker_done: + !ifdef LSDJ_CI_ADVERSARIAL_TESTS + Pop $R9 + !endif + Pop $R8 + Pop $R7 + Pop $R6 + Pop $R5 +FunctionEnd +!macroend + +!insertmacro LSDJ_DEFINE_MARKER_VALIDATOR LsdjDataMarkerIsValid +!insertmacro LSDJ_DEFINE_MARKER_VALIDATOR un.LsdjDataMarkerIsValid + +; Create the marker without following or overwriting an entry raced into the +; temporary path. The fixed byte count is asserted by the packaging contracts. +Function LsdjCreateDataMarker + Push $R6 + Push $R7 + Push $R8 + !ifdef LSDJ_CI_ADVERSARIAL_TESTS + Push $R9 + !endif + StrCpy $LsdjMarkerSafe 0 + StrCpy $R7 0 + !insertmacro LSDJ_CI_TRACE "marker-create: begin" + !ifdef LSDJ_CI_ADVERSARIAL_TESTS + System::Call 'kernel32::CreateFileW(w "${LSDJ_DATA_MARKER_NEW}", i ${LSDJ_GENERIC_WRITE}, i 0, p 0, i ${LSDJ_CREATE_NEW}, i ${LSDJ_FILE_ATTRIBUTE_NORMAL}|${LSDJ_FILE_FLAG_OPEN_REPARSE_POINT}, p 0) p .R6 ?e' + Pop $R9 + !insertmacro LSDJ_CI_TRACE "marker-create: create handle=$R6 error=$R9" + !else + System::Call 'kernel32::CreateFileW(w "${LSDJ_DATA_MARKER_NEW}", i ${LSDJ_GENERIC_WRITE}, i 0, p 0, i ${LSDJ_CREATE_NEW}, i ${LSDJ_FILE_ATTRIBUTE_NORMAL}|${LSDJ_FILE_FLAG_OPEN_REPARSE_POINT}, p 0) p .R6' + !endif + StrCmp $R6 ${LSDJ_INVALID_HANDLE_VALUE} lsdj_marker_create_done + !ifdef LSDJ_CI_ADVERSARIAL_TESTS + System::Call 'kernel32::WriteFile(p R6, m "${LSDJ_OWNER_ID}", i ${LSDJ_OWNER_ID_BYTES}, *i .R8, p 0) i .R7 ?e' + Pop $R9 + !insertmacro LSDJ_CI_TRACE "marker-create: write result=$R7 bytes=$R8 error=$R9" + !else + System::Call 'kernel32::WriteFile(p R6, m "${LSDJ_OWNER_ID}", i ${LSDJ_OWNER_ID_BYTES}, *i .R8, p 0) i .R7' + !endif + System::Call 'kernel32::CloseHandle(p R6)' + ${If} $R7 = 0 + ${OrIf} $R8 != ${LSDJ_OWNER_ID_BYTES} + Goto lsdj_marker_done + ${EndIf} + StrCpy $R7 0 + ClearErrors + Rename "${LSDJ_DATA_MARKER_NEW}" "${LSDJ_DATA_MARKER}" + !ifdef LSDJ_CI_ADVERSARIAL_TESTS + ${If} ${Errors} + StrCpy $R8 1 + ${Else} + StrCpy $R8 0 + ${EndIf} + System::Call 'kernel32::GetLastError() i .R9' + !insertmacro LSDJ_CI_TRACE "marker-create: rename error-flag=$R8 native-error=$R9" + ${If} $R8 = 1 + Goto lsdj_marker_create_done + ${EndIf} + !else + IfErrors lsdj_marker_create_done + !endif + !insertmacro LSDJ_CI_TRACE "marker-create: rename succeeded" + StrCpy $R7 1 + Call LsdjDataMarkerIsValid + !insertmacro LSDJ_CI_TRACE "marker-create: validate safe=$LsdjMarkerSafe" + ${If} $LsdjMarkerSafe = 1 + Goto lsdj_marker_create_done + ${EndIf} + StrCpy $LsdjMarkerSafe 0 + + lsdj_marker_done: + Delete "${LSDJ_DATA_MARKER_NEW}" + lsdj_marker_create_done: + ${If} $LsdjMarkerSafe != 1 + Delete "${LSDJ_DATA_MARKER_NEW}" + ${If} $R7 = 1 + Delete "${LSDJ_DATA_MARKER}" + ${EndIf} + ${EndIf} + !ifdef LSDJ_CI_ADVERSARIAL_TESTS + Pop $R9 + !endif + Pop $R8 + Pop $R7 + Pop $R6 +FunctionEnd + +; A legacy app-created layout without an ownership marker is recognized only +; when its top level is exactly the five roots created by platform_paths.rs. +; Empty, partial, foreign, file-bearing, or reparse-bearing roots are rejected. +Function LsdjExistingLayoutIsRecognized + Push $0 + Push $1 + Push $2 + Push $3 + Push $4 + Push $5 + Push $6 + Push $7 + StrCpy $LsdjSafeLayout 0 + StrCpy $2 0 + StrCpy $3 0 + StrCpy $4 0 + StrCpy $5 0 + StrCpy $6 0 + + ClearErrors + FindFirst $0 $1 "${LSDJ_DATA_ROOT}\*" + IfErrors lsdj_layout_done + lsdj_layout_next: + StrCmp $1 "." lsdj_layout_advance + StrCmp $1 ".." lsdj_layout_advance + StrCmp $1 "config" lsdj_layout_config + StrCmp $1 "data" lsdj_layout_data + StrCmp $1 "cache" lsdj_layout_cache + StrCmp $1 "assets" lsdj_layout_assets + StrCmp $1 "staging" lsdj_layout_staging + Goto lsdj_layout_close + + lsdj_layout_config: + StrCpy $2 1 + Goto lsdj_layout_validate_directory + lsdj_layout_data: + StrCpy $3 1 + Goto lsdj_layout_validate_directory + lsdj_layout_cache: + StrCpy $4 1 + Goto lsdj_layout_validate_directory + lsdj_layout_assets: + StrCpy $5 1 + Goto lsdj_layout_validate_directory + lsdj_layout_staging: + StrCpy $6 1 + + lsdj_layout_validate_directory: + System::Call 'kernel32::GetFileAttributesW(w "${LSDJ_DATA_ROOT}\$1") i .r7' + ${If} $7 = ${LSDJ_INVALID_FILE_ATTRIBUTES} + Goto lsdj_layout_close + ${EndIf} + IntOp $7 $7 & ${LSDJ_FILE_ATTRIBUTE_REPARSE_POINT} + ${If} $7 <> 0 + Goto lsdj_layout_close + ${EndIf} + System::Call 'kernel32::GetFileAttributesW(w "${LSDJ_DATA_ROOT}\$1") i .r7' + IntOp $7 $7 & ${LSDJ_FILE_ATTRIBUTE_DIRECTORY} + ${If} $7 = 0 + Goto lsdj_layout_close + ${EndIf} + + lsdj_layout_advance: + ClearErrors + FindNext $0 $1 + IfErrors lsdj_layout_complete + Goto lsdj_layout_next + + lsdj_layout_complete: + ${If} $2 = 1 + ${AndIf} $3 = 1 + ${AndIf} $4 = 1 + ${AndIf} $5 = 1 + ${AndIf} $6 = 1 + StrCpy $LsdjSafeLayout 1 + ${EndIf} + + lsdj_layout_close: + FindClose $0 + lsdj_layout_done: + Pop $7 + Pop $6 + Pop $5 + Pop $4 + Pop $3 + Pop $2 + Pop $1 + Pop $0 +FunctionEnd + +Function LsdjInstallTreeIsLinkFree + Exch $0 + Push $1 + Push $2 + Push $3 + ${If} $LsdjTreeSafe = 0 + Goto lsdj_install_tree_done + ${EndIf} + + System::Call 'kernel32::GetFileAttributesW(w r0) i .r1' + ${If} $1 = ${LSDJ_INVALID_FILE_ATTRIBUTES} + Goto lsdj_install_tree_unsafe + ${EndIf} + IntOp $3 $1 & ${LSDJ_FILE_ATTRIBUTE_REPARSE_POINT} + ${If} $3 <> 0 + Goto lsdj_install_tree_unsafe + ${EndIf} + IntOp $3 $1 & ${LSDJ_FILE_ATTRIBUTE_DIRECTORY} + ${If} $3 = 0 + Goto lsdj_install_tree_unsafe + ${EndIf} + + ClearErrors + FindFirst $1 $2 "$0\*" + IfErrors lsdj_install_tree_empty_candidate + lsdj_install_tree_next: + StrCmp $2 "." lsdj_install_tree_advance + StrCmp $2 ".." lsdj_install_tree_advance + System::Call 'kernel32::GetFileAttributesW(w "$0\$2") i .r3' + ${If} $3 = ${LSDJ_INVALID_FILE_ATTRIBUTES} + Goto lsdj_install_tree_unsafe_close + ${EndIf} + IntOp $3 $3 & ${LSDJ_FILE_ATTRIBUTE_REPARSE_POINT} + ${If} $3 <> 0 + Goto lsdj_install_tree_unsafe_close + ${EndIf} + System::Call 'kernel32::GetFileAttributesW(w "$0\$2") i .r3' + IntOp $3 $3 & ${LSDJ_FILE_ATTRIBUTE_DIRECTORY} + ${If} $3 <> 0 + Push "$0\$2" + Call LsdjInstallTreeIsLinkFree + ${If} $LsdjTreeSafe = 0 + Goto lsdj_install_tree_close + ${EndIf} + ${EndIf} + lsdj_install_tree_advance: + ClearErrors + FindNext $1 $2 + IfErrors lsdj_install_tree_close + Goto lsdj_install_tree_next + + lsdj_install_tree_unsafe_close: + StrCpy $LsdjTreeSafe 0 + lsdj_install_tree_close: + FindClose $1 + Goto lsdj_install_tree_done + ; FindFirst reports an error for a plain empty directory. Re-read the entry + ; itself before accepting that error as an empty leaf, so disappearance, + ; replacement, and reparse-point races still fail closed. + lsdj_install_tree_empty_candidate: + System::Call 'kernel32::GetFileAttributesW(w r0) i .r1' + ${If} $1 = ${LSDJ_INVALID_FILE_ATTRIBUTES} + Goto lsdj_install_tree_unsafe + ${EndIf} + IntOp $3 $1 & ${LSDJ_FILE_ATTRIBUTE_REPARSE_POINT} + ${If} $3 <> 0 + Goto lsdj_install_tree_unsafe + ${EndIf} + IntOp $3 $1 & ${LSDJ_FILE_ATTRIBUTE_DIRECTORY} + ${If} $3 = 0 + Goto lsdj_install_tree_unsafe + ${EndIf} + Goto lsdj_install_tree_done + lsdj_install_tree_unsafe: + StrCpy $LsdjTreeSafe 0 + lsdj_install_tree_done: + Pop $3 + Pop $2 + Pop $1 + Pop $0 +FunctionEnd + +Function LsdjDataRootIsEmpty + Push $0 + Push $1 + StrCpy $LsdjRootEmpty 0 + ClearErrors + FindFirst $0 $1 "${LSDJ_DATA_ROOT}\*" + IfErrors lsdj_empty_recheck + lsdj_empty_next: + StrCmp $1 "." lsdj_empty_advance + StrCmp $1 ".." lsdj_empty_advance + Goto lsdj_empty_close + lsdj_empty_advance: + ClearErrors + FindNext $0 $1 + IfErrors lsdj_empty_confirmed + Goto lsdj_empty_next + lsdj_empty_confirmed: + StrCpy $LsdjRootEmpty 1 + lsdj_empty_close: + FindClose $0 + Goto lsdj_empty_done + ; Tauri's SetOutPath creates a genuinely empty root on first install. Accept + ; the failed enumeration only after the root is still the same plain + ; directory shape required by the ownership checks. + lsdj_empty_recheck: + System::Call 'kernel32::GetFileAttributesW(w "${LSDJ_DATA_ROOT}") i .r0' + ${If} $0 = ${LSDJ_INVALID_FILE_ATTRIBUTES} + Goto lsdj_empty_done + ${EndIf} + IntOp $1 $0 & ${LSDJ_FILE_ATTRIBUTE_REPARSE_POINT} + ${If} $1 <> 0 + Goto lsdj_empty_done + ${EndIf} + IntOp $1 $0 & ${LSDJ_FILE_ATTRIBUTE_DIRECTORY} + ${If} $1 = 0 + Goto lsdj_empty_done + ${EndIf} + StrCpy $LsdjRootEmpty 1 + lsdj_empty_done: + Pop $1 + Pop $0 +FunctionEnd + +; installerHooks is included before Tauri declares any of its sections. NSIS +; executes sections in declaration order, so this hidden, always-selected probe +; observes the root before Tauri's Install section executes SetOutPath. It does +; not create or mark anything: PREINSTALL uses this captured state after +; SetOutPath and immediately revalidates before establishing ownership. +Section -LsdjProbeDataRootBeforeTauri + StrCpy $LsdjInstallRootState 0 + !insertmacro LSDJ_CI_TRACE "probe: begin root=${LSDJ_DATA_ROOT}" + Call LsdjCanonicalDataRootIsValid + !insertmacro LSDJ_CI_TRACE "probe: canonical=$LsdjCanonicalRootSafe" + ${If} $LsdjCanonicalRootSafe != 1 + !insertmacro LSDJ_CI_TRACE "abort: probe canonical root" + Abort "Refusing to install: the LSDJ data root is not the exact LocalAppData target." + ${EndIf} + + System::Call 'kernel32::GetFileAttributesW(w "${LSDJ_DATA_ROOT}") i .R8' + !insertmacro LSDJ_CI_TRACE "probe: root attributes=$R8" + ${If} $R8 = ${LSDJ_INVALID_FILE_ATTRIBUTES} + Goto lsdj_probe_absent + ${EndIf} + IntOp $R7 $R8 & ${LSDJ_FILE_ATTRIBUTE_REPARSE_POINT} + ${If} $R7 <> 0 + !insertmacro LSDJ_CI_TRACE "abort: probe root reparse" + Abort "Refusing to install into a junction, symbolic link, or reparse point at ${LSDJ_DATA_ROOT}." + ${EndIf} + IntOp $R7 $R8 & ${LSDJ_FILE_ATTRIBUTE_DIRECTORY} + ${If} $R7 = 0 + !insertmacro LSDJ_CI_TRACE "abort: probe root not directory" + Abort "Refusing to install: ${LSDJ_DATA_ROOT} exists but is not a directory." + ${EndIf} + + ClearErrors + FindFirst $R7 $R8 "${LSDJ_DATA_MARKER}" + IfErrors lsdj_probe_legacy + FindClose $R7 + !insertmacro LSDJ_CI_TRACE "probe: marker entry present" + Call LsdjDataMarkerIsValid + !insertmacro LSDJ_CI_TRACE "probe: marker safe=$LsdjMarkerSafe" + ${If} $LsdjMarkerSafe != 1 + !insertmacro LSDJ_CI_TRACE "abort: probe marker invalid" + Abort "Refusing to install: the LSDJ ownership marker is invalid or is a reparse point." + ${EndIf} + StrCpy $LsdjInstallRootState 3 + Goto lsdj_probe_done + + lsdj_probe_legacy: + !insertmacro LSDJ_CI_TRACE "probe: marker absent, checking legacy layout" + Call LsdjExistingLayoutIsRecognized + !insertmacro LSDJ_CI_TRACE "probe: legacy layout=$LsdjSafeLayout" + StrCpy $LsdjTreeSafe 1 + ${If} $LsdjSafeLayout = 1 + Push "${LSDJ_DATA_ROOT}" + Call LsdjInstallTreeIsLinkFree + ${EndIf} + !insertmacro LSDJ_CI_TRACE "probe: legacy tree=$LsdjTreeSafe" + ${If} $LsdjSafeLayout != 1 + ${OrIf} $LsdjTreeSafe != 1 + !insertmacro LSDJ_CI_TRACE "abort: probe legacy ownership" + Abort "Refusing to claim a pre-existing foreign or unrecognized directory at ${LSDJ_DATA_ROOT}." + ${EndIf} + StrCpy $LsdjInstallRootState 2 + !insertmacro LSDJ_CI_TRACE "probe: classified legacy state=2" + Goto lsdj_probe_done + + lsdj_probe_absent: + StrCpy $LsdjInstallRootState 1 + !insertmacro LSDJ_CI_TRACE "probe: classified absent state=1" + lsdj_probe_done: +SectionEnd + +; Validate the exact root and marker together. GetFileAttributesW reports the +; root entry itself, so directory junctions and symbolic links are rejected. +Function un.LsdjOwnedDataRootIsSafe + Push $R7 + Push $R8 + StrCpy $LsdjOwnedRootSafe 0 + Call un.LsdjCanonicalDataRootIsValid + ${If} $LsdjCanonicalRootSafe != 1 + Goto lsdj_owned_root_done + ${EndIf} + System::Call 'kernel32::GetFileAttributesW(w "${LSDJ_DATA_ROOT}") i .R8' + ${If} $R8 = ${LSDJ_INVALID_FILE_ATTRIBUTES} + Goto lsdj_owned_root_done + ${EndIf} + IntOp $R7 $R8 & ${LSDJ_FILE_ATTRIBUTE_REPARSE_POINT} + ${If} $R7 <> 0 + Goto lsdj_owned_root_done + ${EndIf} + IntOp $R7 $R8 & ${LSDJ_FILE_ATTRIBUTE_DIRECTORY} + ${If} $R7 = 0 + Goto lsdj_owned_root_done + ${EndIf} + Call un.LsdjDataMarkerIsValid + ${If} $LsdjMarkerSafe = 1 + StrCpy $LsdjOwnedRootSafe 1 + ${EndIf} + + lsdj_owned_root_done: + Pop $R8 + Pop $R7 +FunctionEnd + +; Walk the tree without traversing a reparse point. Purge is refused before +; GetSize if any link is present, so the disclosed size covers only the tree +; that the safe deleter is allowed to remove. +Function un.LsdjTreeIsLinkFree + Exch $0 + Push $1 + Push $2 + Push $3 + ${If} $LsdjTreeSafe = 0 + Goto lsdj_tree_done + ${EndIf} + + System::Call 'kernel32::GetFileAttributesW(w r0) i .r1' + ${If} $1 = ${LSDJ_INVALID_FILE_ATTRIBUTES} + Goto lsdj_tree_unsafe + ${EndIf} + IntOp $3 $1 & ${LSDJ_FILE_ATTRIBUTE_REPARSE_POINT} + ${If} $3 <> 0 + Goto lsdj_tree_unsafe + ${EndIf} + IntOp $3 $1 & ${LSDJ_FILE_ATTRIBUTE_DIRECTORY} + ${If} $3 = 0 + Goto lsdj_tree_unsafe + ${EndIf} + + ClearErrors + FindFirst $1 $2 "$0\*" + IfErrors lsdj_tree_empty_candidate + lsdj_tree_next: + StrCmp $2 "." lsdj_tree_advance + StrCmp $2 ".." lsdj_tree_advance + System::Call 'kernel32::GetFileAttributesW(w "$0\$2") i .r3' + ${If} $3 = ${LSDJ_INVALID_FILE_ATTRIBUTES} + Goto lsdj_tree_unsafe_close + ${EndIf} + IntOp $3 $3 & ${LSDJ_FILE_ATTRIBUTE_REPARSE_POINT} + ${If} $3 <> 0 + Goto lsdj_tree_unsafe_close + ${EndIf} + System::Call 'kernel32::GetFileAttributesW(w "$0\$2") i .r3' + IntOp $3 $3 & ${LSDJ_FILE_ATTRIBUTE_DIRECTORY} + ${If} $3 <> 0 + Push "$0\$2" + Call un.LsdjTreeIsLinkFree + ${If} $LsdjTreeSafe = 0 + Goto lsdj_tree_close + ${EndIf} + ${EndIf} + lsdj_tree_advance: + ClearErrors + FindNext $1 $2 + IfErrors lsdj_tree_close + Goto lsdj_tree_next + + lsdj_tree_unsafe_close: + StrCpy $LsdjTreeSafe 0 + lsdj_tree_close: + FindClose $1 + Goto lsdj_tree_done + lsdj_tree_empty_candidate: + System::Call 'kernel32::GetFileAttributesW(w r0) i .r1' + ${If} $1 = ${LSDJ_INVALID_FILE_ATTRIBUTES} + Goto lsdj_tree_unsafe + ${EndIf} + IntOp $3 $1 & ${LSDJ_FILE_ATTRIBUTE_REPARSE_POINT} + ${If} $3 <> 0 + Goto lsdj_tree_unsafe + ${EndIf} + IntOp $3 $1 & ${LSDJ_FILE_ATTRIBUTE_DIRECTORY} + ${If} $3 = 0 + Goto lsdj_tree_unsafe + ${EndIf} + Goto lsdj_tree_done + lsdj_tree_unsafe: + StrCpy $LsdjTreeSafe 0 + lsdj_tree_done: + Pop $3 + Pop $2 + Pop $1 + Pop $0 +FunctionEnd + +; Recursive deletion mirrors the validation walk and refuses any reparse entry +; observed during deletion. It never invokes NSIS's broad recursive-directory +; removal and never descends through a junction or symbolic link, including one +; introduced after confirmation. +Function un.LsdjDeleteTreeWithoutLinks + Exch $0 + Push $1 + Push $2 + Push $3 + ${If} $LsdjDeleteFailure = 1 + Goto lsdj_delete_done + ${EndIf} + + System::Call 'kernel32::GetFileAttributesW(w r0) i .r1' + ${If} $1 = ${LSDJ_INVALID_FILE_ATTRIBUTES} + Goto lsdj_delete_failed + ${EndIf} + IntOp $3 $1 & ${LSDJ_FILE_ATTRIBUTE_REPARSE_POINT} + ${If} $3 <> 0 + Goto lsdj_delete_failed + ${EndIf} + IntOp $3 $1 & ${LSDJ_FILE_ATTRIBUTE_DIRECTORY} + ${If} $3 = 0 + Goto lsdj_delete_failed + ${EndIf} + + ClearErrors + FindFirst $1 $2 "$0\*" + IfErrors lsdj_delete_empty_candidate + lsdj_delete_next: + StrCmp $2 "." lsdj_delete_advance + StrCmp $2 ".." lsdj_delete_advance + System::Call 'kernel32::GetFileAttributesW(w "$0\$2") i .r3' + ${If} $3 = ${LSDJ_INVALID_FILE_ATTRIBUTES} + Goto lsdj_delete_failed_close + ${EndIf} + IntOp $3 $3 & ${LSDJ_FILE_ATTRIBUTE_REPARSE_POINT} + ${If} $3 <> 0 + Goto lsdj_delete_failed_close + ${EndIf} + System::Call 'kernel32::GetFileAttributesW(w "$0\$2") i .r3' + IntOp $3 $3 & ${LSDJ_FILE_ATTRIBUTE_DIRECTORY} + ${If} $3 <> 0 + Push "$0\$2" + Call un.LsdjDeleteTreeWithoutLinks + ${If} $LsdjDeleteFailure = 1 + Goto lsdj_delete_close + ${EndIf} + ${Else} + ClearErrors + Delete "$0\$2" + IfErrors lsdj_delete_failed_close + ${EndIf} + lsdj_delete_advance: + ClearErrors + FindNext $1 $2 + IfErrors lsdj_delete_close + Goto lsdj_delete_next + + lsdj_delete_failed_close: + StrCpy $LsdjDeleteFailure 1 + lsdj_delete_close: + FindClose $1 + ${If} $LsdjDeleteFailure = 0 + ClearErrors + RMDir "$0" + IfErrors lsdj_delete_failed + ${EndIf} + Goto lsdj_delete_done + lsdj_delete_empty_candidate: + System::Call 'kernel32::GetFileAttributesW(w r0) i .r1' + ${If} $1 = ${LSDJ_INVALID_FILE_ATTRIBUTES} + Goto lsdj_delete_failed + ${EndIf} + IntOp $3 $1 & ${LSDJ_FILE_ATTRIBUTE_REPARSE_POINT} + ${If} $3 <> 0 + Goto lsdj_delete_failed + ${EndIf} + IntOp $3 $1 & ${LSDJ_FILE_ATTRIBUTE_DIRECTORY} + ${If} $3 = 0 + Goto lsdj_delete_failed + ${EndIf} + ClearErrors + RMDir "$0" + IfErrors lsdj_delete_failed + Goto lsdj_delete_done + lsdj_delete_failed: + StrCpy $LsdjDeleteFailure 1 + lsdj_delete_done: + Pop $3 + Pop $2 + Pop $1 + Pop $0 +FunctionEnd + +!macro NSIS_HOOK_PREINSTALL + !insertmacro LSDJ_REJECT_PASSIVE_MODE + + ; Tauri's native silent downgrade check aborts with exit code 0 and leaves its + ; comparison in a volatile register. Re-establish install evidence and + ; validate DisplayVersion independently before root checks or payload writes. + ${If} ${Silent} + StrCpy $LsdjExistingInstall 0 + StrCpy $LsdjInstalledVersion "" + StrCpy $LsdjInstalledVersionPresent 0 + + ClearErrors + ReadRegStr $LsdjInstalledVersion SHCTX "${UNINSTKEY}" "DisplayVersion" + ${IfNot} ${Errors} + StrCpy $LsdjExistingInstall 1 + StrCpy $LsdjInstalledVersionPresent 1 + ${EndIf} + ClearErrors + ReadRegStr $LsdjRegistryEvidence SHCTX "${UNINSTKEY}" "UninstallString" + ${IfNot} ${Errors} + StrCpy $LsdjExistingInstall 1 + ${EndIf} + ClearErrors + ReadRegStr $LsdjRegistryEvidence SHCTX "${UNINSTKEY}" "DisplayName" + ${IfNot} ${Errors} + StrCpy $LsdjExistingInstall 1 + ${EndIf} + ${If} ${FileExists} "$INSTDIR\${MAINBINARYNAME}.exe" + StrCpy $LsdjExistingInstall 1 + ${EndIf} + ClearErrors + !insertmacro LSDJ_CI_TRACE "preinstall: version evidence installed=$LsdjInstalledVersion present=$LsdjInstalledVersionPresent existing=$LsdjExistingInstall" + + ${If} $LsdjExistingInstall = 1 + ${If} "$LsdjInstalledVersion" == "" + !insertmacro LSDJ_CI_TRACE "abort: installed version missing" + SetErrorLevel 2 + Abort "Unable to verify the existing LSDJ version." + ${EndIf} + + ; nsis-tauri-utils orders any valid SemVer above an invalid one. Comparing + ; against a deliberately invalid sentinel distinguishes invalid metadata + ; from a legitimate upgrade, which SemverCompare alone otherwise cannot. + nsis_tauri_utils::SemverCompare "$LsdjInstalledVersion" "lsdj-invalid-semver" + Pop $LsdjInstalledVersionValidity + !insertmacro LSDJ_CI_TRACE "preinstall: installed version=$LsdjInstalledVersion validity=$LsdjInstalledVersionValidity" + ${If} $LsdjInstalledVersionValidity != 1 + !insertmacro LSDJ_CI_TRACE "abort: installed version invalid" + SetErrorLevel 2 + Abort "Unable to verify the existing LSDJ version." + ${EndIf} + + nsis_tauri_utils::SemverCompare "${VERSION}" "$LsdjInstalledVersion" + Pop $LsdjVersionCompare + !insertmacro LSDJ_CI_TRACE "preinstall: version compare=$LsdjVersionCompare" + ${If} $LsdjVersionCompare = -1 + !insertmacro LSDJ_CI_TRACE "abort: unattended downgrade" + SetErrorLevel 2 + Abort "Refusing to downgrade LSDJ from a newer installed version." + ${ElseIf} $LsdjVersionCompare != 0 + ${AndIf} $LsdjVersionCompare != 1 + !insertmacro LSDJ_CI_TRACE "abort: invalid version comparison" + SetErrorLevel 2 + Abort "Unable to compare the installed LSDJ version safely." + ${EndIf} + ${EndIf} + ${EndIf} + + ; SetOutPath has now created a root that the early section proved absent, or + ; selected an existing root whose ownership/layout the early section proved. + ; Revalidate that captured state before writing anything into the directory. + !insertmacro LSDJ_CI_TRACE "preinstall: begin state=$LsdjInstallRootState" + Call LsdjCanonicalDataRootIsValid + !insertmacro LSDJ_CI_TRACE "preinstall: canonical=$LsdjCanonicalRootSafe" + ${If} $LsdjCanonicalRootSafe != 1 + !insertmacro LSDJ_CI_TRACE "abort: preinstall canonical root" + Abort "Refusing to install: the LSDJ data root is not the exact LocalAppData target." + ${EndIf} + + System::Call 'kernel32::GetFileAttributesW(w "${LSDJ_DATA_ROOT}") i .R8' + !insertmacro LSDJ_CI_TRACE "preinstall: root attributes=$R8" + ${If} $R8 = ${LSDJ_INVALID_FILE_ATTRIBUTES} + ${If} $LsdjInstallRootState != 1 + Goto lsdj_install_root_missing + ${EndIf} + ; A custom /D install location means Tauri's SetOutPath did not create the + ; separate LocalAppData root. The early section proved it absent; create it + ; now, fail if another entry won the race, and validate the new entry below. + ClearErrors + CreateDirectory "${LSDJ_DATA_ROOT}" + IfErrors lsdj_install_root_create_failed + System::Call 'kernel32::GetFileAttributesW(w "${LSDJ_DATA_ROOT}") i .R8' + ${If} $R8 = ${LSDJ_INVALID_FILE_ATTRIBUTES} + Goto lsdj_install_root_create_failed + ${EndIf} + ${EndIf} + IntOp $R7 $R8 & ${LSDJ_FILE_ATTRIBUTE_REPARSE_POINT} + ${If} $R7 <> 0 + !insertmacro LSDJ_CI_TRACE "abort: preinstall root reparse" + Abort "Refusing to install into a junction, symbolic link, or reparse point at ${LSDJ_DATA_ROOT}." + ${EndIf} + IntOp $R7 $R8 & ${LSDJ_FILE_ATTRIBUTE_DIRECTORY} + ${If} $R7 = 0 + !insertmacro LSDJ_CI_TRACE "abort: preinstall root not directory" + Abort "Refusing to install: ${LSDJ_DATA_ROOT} exists but is not a directory." + ${EndIf} + + ${If} $LsdjInstallRootState = 1 + Call LsdjDataRootIsEmpty + !insertmacro LSDJ_CI_TRACE "preinstall: fresh root empty=$LsdjRootEmpty" + ${If} $LsdjRootEmpty != 1 + !insertmacro LSDJ_CI_TRACE "abort: preinstall fresh root changed" + Abort "Refusing to install: the newly created LSDJ root changed before ownership was established." + ${EndIf} + Goto lsdj_write_data_marker + ${ElseIf} $LsdjInstallRootState = 2 + Call LsdjExistingLayoutIsRecognized + !insertmacro LSDJ_CI_TRACE "preinstall: legacy layout=$LsdjSafeLayout" + StrCpy $LsdjTreeSafe 1 + ${If} $LsdjSafeLayout = 1 + Push "${LSDJ_DATA_ROOT}" + Call LsdjInstallTreeIsLinkFree + ${EndIf} + !insertmacro LSDJ_CI_TRACE "preinstall: legacy tree=$LsdjTreeSafe" + ${If} $LsdjSafeLayout != 1 + ${OrIf} $LsdjTreeSafe != 1 + !insertmacro LSDJ_CI_TRACE "abort: preinstall legacy ownership changed" + Abort "Refusing to install: the recognized LSDJ layout changed before ownership was established." + ${EndIf} + Goto lsdj_write_data_marker + ${ElseIf} $LsdjInstallRootState = 3 + Call LsdjDataMarkerIsValid + !insertmacro LSDJ_CI_TRACE "preinstall: owned marker safe=$LsdjMarkerSafe" + ${If} $LsdjMarkerSafe != 1 + Abort "Refusing to install: LSDJ ownership changed after the early root probe." + ${EndIf} + Goto lsdj_install_root_ready + ${Else} + !insertmacro LSDJ_CI_TRACE "abort: preinstall unknown root state" + Abort "Refusing to install: the LSDJ data root was not safely classified before SetOutPath." + ${EndIf} + + lsdj_write_data_marker: + Call LsdjCreateDataMarker + !insertmacro LSDJ_CI_TRACE "preinstall: marker result=$LsdjMarkerSafe" + ${If} $LsdjMarkerSafe != 1 + Goto lsdj_marker_create_failed + ${EndIf} + Goto lsdj_install_root_ready + + lsdj_marker_create_failed: + Delete "${LSDJ_DATA_MARKER_NEW}" + Abort "Refusing to install: a plain LSDJ ownership marker could not be established safely." + lsdj_install_root_missing: + Abort "Refusing to install: Tauri did not create the LSDJ root classified by the early ownership probe." + lsdj_install_root_create_failed: + Abort "Refusing to install: the separately located LSDJ data root could not be created safely." + lsdj_install_root_ready: + !insertmacro LSDJ_CI_TRACE "preinstall: ready" +!macroend + +!macro NSIS_HOOK_POSTINSTALL + ; A second check ensures copying never silently replaced the owned root or + ; marker. Do not rewrite or repair either safety boundary here. + !insertmacro LSDJ_CI_TRACE "postinstall: begin" + Call LsdjCanonicalDataRootIsValid + !insertmacro LSDJ_CI_TRACE "postinstall: canonical=$LsdjCanonicalRootSafe" + ${If} $LsdjCanonicalRootSafe != 1 + Abort "LSDJ installation did not retain the exact LocalAppData root." + ${EndIf} + System::Call 'kernel32::GetFileAttributesW(w "${LSDJ_DATA_ROOT}") i .R8' + !insertmacro LSDJ_CI_TRACE "postinstall: root attributes=$R8" + ${If} $R8 = ${LSDJ_INVALID_FILE_ATTRIBUTES} + Abort "LSDJ installation lost its LocalAppData root." + ${EndIf} + IntOp $R7 $R8 & ${LSDJ_FILE_ATTRIBUTE_REPARSE_POINT} + ${If} $R7 <> 0 + Abort "LSDJ installation encountered an unsafe data-root reparse point." + ${EndIf} + Call LsdjDataMarkerIsValid + !insertmacro LSDJ_CI_TRACE "postinstall: marker safe=$LsdjMarkerSafe" + ${If} $LsdjMarkerSafe != 1 + Abort "LSDJ installation did not retain its plain ownership marker." + ${EndIf} +!macroend + +!macro NSIS_HOOK_PREUNINSTALL + ; The normal checkbox is deliberately unchecked by default. Silent removal + ; must name the destructive option; /S alone always preserves user assets. + StrCpy $LsdjDeleteData 0 + StrCpy $LsdjDataRemovalFailed 0 + ClearErrors + ${GetOptions} $CMDLINE "/PURGE-LSDJ-DATA" $R8 + ${IfNot} ${Errors} + StrCpy $DeleteAppDataCheckboxState 1 + ${EndIf} + + ${If} $DeleteAppDataCheckboxState = 1 + Call un.LsdjOwnedDataRootIsSafe + StrCpy $LsdjTreeSafe 1 + ${If} $LsdjOwnedRootSafe = 1 + Push "${LSDJ_DATA_ROOT}" + Call un.LsdjTreeIsLinkFree + ${EndIf} + !insertmacro LSDJ_CI_TRACE "preuninstall: owned root safe=$LsdjOwnedRootSafe tree safe=$LsdjTreeSafe" + ${If} $LsdjOwnedRootSafe != 1 + ${OrIf} $LsdjTreeSafe != 1 + StrCpy $LsdjDataRemovalFailed 1 + DetailPrint "Refusing to remove LSDJ data: exact ownership or reparse-safety validation failed at ${LSDJ_DATA_ROOT}" + StrCpy $DeleteAppDataCheckboxState 0 + ${IfNot} ${Silent} + MessageBox MB_ICONSTOP|MB_OK "LSDJ will preserve the data at:$\n${LSDJ_DATA_ROOT}$\n$\nThe root or ownership marker is invalid, or the tree contains a reparse point, so automatic removal is unsafe." + ${EndIf} + ; Stop before Tauri's ordinary payload deletion too: the application and + ; data share a root in the default layout, so continuing after a root + ; ownership failure could make even narrow file deletions unsafe. + !insertmacro LSDJ_CI_TRACE "abort: unsafe data removal" + SetErrorLevel 2 + Quit + ${EndIf} + + ; GetSize reports KiB. The link-free walk above prevents it from traversing + ; junctions while measuring the exact tree that may be removed. + ${GetSize} "${LSDJ_DATA_ROOT}" "/S=0K" $R8 $R9 $R7 + DetailPrint "Selected LSDJ data removal: ${LSDJ_DATA_ROOT} ($R8 KiB)" + ${IfNot} ${Silent} + MessageBox MB_ICONEXCLAMATION|MB_YESNO|MB_DEFBUTTON2 "Permanently remove downloaded models, runtimes, settings, and user data?$\n$\nLocation: ${LSDJ_DATA_ROOT}$\nSize: $R8 KiB$\n$\nThis cannot be undone." IDYES lsdj_confirm_data_removal IDNO lsdj_keep_data + lsdj_keep_data: + StrCpy $DeleteAppDataCheckboxState 0 + Goto lsdj_data_decision_done + lsdj_confirm_data_removal: + ${EndIf} + ${EndIf} + lsdj_data_decision_done: + + ; Tauri's generic checkbox handling recursively removes undisclosed bundle-ID + ; APPDATA roots. Preserve the choice separately and suppress that deletion. + StrCpy $LsdjDeleteData $DeleteAppDataCheckboxState + StrCpy $DeleteAppDataCheckboxState 0 +!macroend + +!macro NSIS_HOOK_POSTUNINSTALL + ${If} $LsdjDeleteData = 1 + ${AndIf} $UpdateMode <> 1 + !ifdef LSDJ_CI_ADVERSARIAL_TESTS + ; Unsigned hosted-CI installers can pause after confirmation/core removal + ; so the test can deterministically replace the marker before this second + ; validation. This branch is absent from release installers. + ClearErrors + ${GetOptions} $CMDLINE "/LSDJ-CI-PAUSE-BEFORE-PURGE" $R8 + ${IfNot} ${Errors} + FileOpen $R7 "$TEMP\lsdj-ci-before-purge.ready" w + FileWrite $R7 "ready" + FileClose $R7 + Sleep 5000 + Delete "$TEMP\lsdj-ci-before-purge.ready" + ${EndIf} + !endif + + ; Revalidate canonical path, root, marker, and every tree entry immediately + ; before deletion. The deletion walk performs the same checks again. + Call un.LsdjOwnedDataRootIsSafe + StrCpy $LsdjTreeSafe 1 + ${If} $LsdjOwnedRootSafe = 1 + Push "${LSDJ_DATA_ROOT}" + Call un.LsdjTreeIsLinkFree + ${EndIf} + ${If} $LsdjOwnedRootSafe = 1 + ${AndIf} $LsdjTreeSafe = 1 + StrCpy $LsdjDeleteFailure 0 + Push "${LSDJ_DATA_ROOT}" + Call un.LsdjDeleteTreeWithoutLinks + ${If} $LsdjDeleteFailure = 0 + ; Match Tauri's explicit-data-removal registry cleanup without invoking + ; its generic recursive directory deletion. + DeleteRegKey SHCTX "${MANUPRODUCTKEY}" + DeleteRegKey /ifempty SHCTX "${MANUKEY}" + DeleteRegValue HKCU "${MANUPRODUCTKEY}" "Installer Language" + DeleteRegKey /ifempty HKCU "${MANUPRODUCTKEY}" + DeleteRegKey /ifempty HKCU "${MANUKEY}" + ${Else} + DetailPrint "LSDJ data removal stopped because a reparse point or filesystem error was observed" + StrCpy $LsdjDataRemovalFailed 1 + ${EndIf} + ${Else} + DetailPrint "LSDJ data was preserved because exact ownership or reparse-safety validation changed" + StrCpy $LsdjDataRemovalFailed 1 + ${EndIf} + ${EndIf} + ${If} $LsdjDataRemovalFailed = 1 + SetErrorLevel 2 + ${EndIf} +!macroend