From 6e937a34f9036d92e909db3ce8848a5c39dc8e3b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 9 Aug 2026 21:48:57 +0900 Subject: [PATCH 01/34] test(analysis): add governed YouTube known-stem benchmark --- AGENTS.md | 13 + ARCHITECTURE.md | 46 +- CHANGELOG.md | 14 + CLAUDE.md | 8 + README.md | 4 + docs/PRD.md | 85 +++ docs/README.md | 40 ++ docs/TRD.md | 146 +++++ ...e-separation-runtime-and-model-delivery.md | 80 +++ .../0002-known-stem-youtube-quality-gate.md | 81 +++ ...0003-ephemeral-benchmark-evidence-model.md | 56 ++ docs/adr/README.md | 13 + docs/architecture/diagrams.md | 160 ++++++ docs/architecture/overview.md | 14 + .../real-audio-accuracy-acceptance.md | 130 +++++ docs/documentation-coverage-matrix.md | 74 +++ docs/engineering/acceptance-criteria.md | 18 + docs/engineering/harness-engineering.md | 1 + .../youtube-known-stem-validation.md | 194 +++++++ docs/operations/deploy-runbook.md | 38 ++ .../plans/2026-03-28-ml-engine-integration.md | 19 +- docs/release/release-policy.md | 16 + docs/security/dependency-policy.md | 7 +- scripts/checks/run_root_tests.mjs | 2 + scripts/checks/verify_docs.py | 56 +- scripts/checks/verify_supply_chain.py | 326 +++++++++-- services/analysis-engine/pyproject.toml | 3 + .../separation/audio_separator.py | 11 +- .../model_weights/bandsplit-v1.json | 7 - .../src/bandscope_analysis/youtube.py | 22 +- .../tests/known_stem_benchmark.py | 524 ++++++++++++++++++ .../tests/test_documentation_policy.py | 26 + .../analysis-engine/tests/test_separation.py | 3 + .../tests/test_supply_chain_policy.py | 57 ++ .../analysis-engine/tests/test_youtube.py | 3 + .../tests/test_youtube_stem_e2e.py | 422 ++++++++++++++ .../supplemental-component-inventory.json | 44 +- 37 files changed, 2654 insertions(+), 109 deletions(-) create mode 100644 docs/PRD.md create mode 100644 docs/README.md create mode 100644 docs/TRD.md create mode 100644 docs/adr/0001-source-separation-runtime-and-model-delivery.md create mode 100644 docs/adr/0002-known-stem-youtube-quality-gate.md create mode 100644 docs/adr/0003-ephemeral-benchmark-evidence-model.md create mode 100644 docs/adr/README.md create mode 100644 docs/architecture/diagrams.md create mode 100644 docs/doctoring/real-audio-accuracy-acceptance.md create mode 100644 docs/documentation-coverage-matrix.md create mode 100644 docs/engineering/youtube-known-stem-validation.md delete mode 100644 services/analysis-engine/src/bandscope_analysis/separation/model_weights/bandsplit-v1.json create mode 100644 services/analysis-engine/tests/known_stem_benchmark.py create mode 100644 services/analysis-engine/tests/test_documentation_policy.py create mode 100644 services/analysis-engine/tests/test_youtube_stem_e2e.py diff --git a/AGENTS.md b/AGENTS.md index fca448ce9..81070181f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -3,6 +3,9 @@ ## Project overview - BandScope is a local-first desktop app for rehearsal prep: a practical song view with likely harmony by section and by instrument or vocal role, form and groove cues, stems, playable ranges, simplification guidance, transposition or setup cues, part-overlap cues, visible confidence, and rehearsal priorities. - Authoritative delivery rules live in `ARCHITECTURE.md`, `docs/plans/`, and the root verification scripts. +- Canonical product/technical requirements, ADRs, diagrams, and documentation sufficiency live in + `docs/PRD.md`, `docs/TRD.md`, `docs/adr/README.md`, `docs/architecture/diagrams.md`, and + `docs/documentation-coverage-matrix.md`. - Brand, tone, UX copy, and prioritization rules live in `docs/brand-story.md` and must be applied to PRDs, TRDs, UI copy, onboarding, empty states, and error messages. - App security rules live in `docs/security/app-security.md` and must be applied to file handling, URL intake, subprocesses, IPC, WebView usage, model loading, updates, logging, cache handling, and export behavior. - Dependency, SBOM, and supply-chain rules live in `docs/security/dependency-policy.md` and must be applied to dependency additions, GitHub Actions, releases, bundled binaries, and model artifacts. @@ -59,9 +62,19 @@ This section applies to any agent (Claude, Codex, Cursor, opencode, ...) working - Frontend tests: `npm run test --workspaces --if-present` - Python tests: `uv run --project services/analysis-engine pytest --cov=src/bandscope_analysis --cov-report=term-missing --cov-fail-under=100` - Typecheck: `npm run typecheck --workspaces --if-present && uv run --project services/analysis-engine mypy src` +- Known-stem offline contract: `uv run --project services/analysis-engine pytest services/analysis-engine/tests/test_youtube_stem_e2e.py -m 'not youtube_stem_e2e' -vv` +- Known-stem live lane is explicit opt-in only; follow + `docs/engineering/youtube-known-stem-validation.md` and never claim a skipped or provider-failed + invocation passed. ## Architecture references - `ARCHITECTURE.md` +- `docs/README.md` +- `docs/PRD.md` +- `docs/TRD.md` +- `docs/adr/README.md` +- `docs/architecture/diagrams.md` +- `docs/documentation-coverage-matrix.md` - `docs/engineering/acceptance-criteria.md` - `docs/engineering/harness-engineering.md` - `docs/workflow/one-day-delivery-plan.md` diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 3302a6fc3..673bf10e5 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -1,6 +1,16 @@ # ARCHITECTURE.md -Last updated: 2026-03-11 +Last updated: 2026-08-09 + +## Documentation authority + +- Product requirements live in `docs/PRD.md`. +- Technical requirements live in `docs/TRD.md`. +- Decision status and supersession live in `docs/adr/README.md`. +- Component, UML, deployment, and logical artifact views live in + `docs/architecture/diagrams.md`. +- Sufficiency and requirement-to-evidence traceability live in + `docs/documentation-coverage-matrix.md`. ## Brand source @@ -76,6 +86,40 @@ Last updated: 2026-03-11 - Typical roles include bass, guitar, keyboard players, keyboard left hand, keyboard right hand, lead vocal, backing vocal, horns, strings, and other arrangement-carrying parts. - Shared contracts should be able to carry different harmonic guidance for simultaneous roles in the same section. +## Source separation and model delivery + +- Production separation uses Demucs 4.0.1 `htdemucs` and returns exactly vocals, bass, drums, and + other for downstream local analysis. The retired `bandsplit-v1` profile is not a production + model. +- Demucs random temporal shifts are disabled (`shifts=0`) so the same bytes and model produce + reproducible local analysis and benchmark evidence. +- Model inference is local after provisioning, but the current first load may fetch the exact + official weight artifact into a user runtime cache. It is not bundled with the repository or + release artifacts. +- The exact signature, source URL, full SHA-256, byte size, distribution status, and model-rights + uncertainty are tracked in `supply-chain/supplemental-component-inventory.json` and ADR-0001. +- Full-SHA verification before torch deserialization and a model-rights decision remain release + blockers. Demucs' filename-prefix check alone is not promoted to full release evidence. +- Current dependency markers exclude Demucs on macOS Intel; unsupported platforms must surface the + existing safe fallback rather than pretending to separate stems. + +## Known-stem validation boundary + +- The active known-stem branch crosses the production YouTube downloader and production separator, + while its reference loader, alignment, and metric utilities remain test-only. +- It pins a creator-published vocal source and a separate finished master by exact hosts, byte + counts, full SHA-256 values, member, and member size; downloads and waveforms stay in test-owned + ephemeral storage. +- The finished master proves candidate identity. YouTube-to-master and master-to-vocal global lags + are composed once before separation; predicted stems are never realigned. Quality requires + duration/identity checks, zero-mean SI-SDR improvement over the downloaded mixture, and correct + vocal-stem assignment margin. +- Deterministic metric/integrity/security contracts run in ordinary CI. Live network/model execution + is explicit opt-in and cannot be scheduled or made release-blocking until authorization and + calibration requirements in ADR-0002 are met. +- The capability has no relational persistence. ADR-0003 and the logical artifact model in + `docs/architecture/diagrams.md` are authoritative instead of a physical ERD. + ## Rehearsal outputs - Core rehearsal artifacts should include: diff --git a/CHANGELOG.md b/CHANGELOG.md index eea696893..7a3f9e37b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,9 +4,23 @@ ### Added +- Added an opt-in real-YouTube/Demucs benchmark that verifies vocal separation against a + creator-published, SHA-256-pinned known stem without adding media files to the repository. +- Added an independently pinned creator master for YouTube asset identity, full extracted-member + hashing, composed global offsets, calibrated provisional sentinels, and deterministic Demucs + inference (`shifts=0`). +- Added canonical PRD, TRD, ADR, architecture/UML/logical-artifact diagrams, traceability, and + machine-checked documentation coverage for the known-stem quality boundary. +- Replaced the retired FFT-era bandsplit model inventory with the exact htdemucs runtime artifact, + full SHA-256, byte size, delivery status, ffmpeg prerequisite, and release blockers. - Display the analyzed song tempo (BPM) as a badge in the rehearsal workspace. - 각 합주 역할(Role)별 개인 연습 진행도를 0~100% 범위로 기록 및 시각화할 수 있는 연습 진척도(`practiceProgress`) 트래커 기능 추가. UI 컨트롤(슬라이더 및 +/- 버튼)과 한/영 다국어 지원 포함. +### Fixed + +- Kept YouTube TLS verification enabled while honoring OS-managed CA roots used by managed + desktop environments. + ## [0.1.3] - 2026-04-29 ### Fixed diff --git a/CLAUDE.md b/CLAUDE.md index 82c2c704a..9ad62939d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -33,6 +33,10 @@ npm run test # JS workspace vitest suites + pytest with 100% coverage gat npm run build # vite builds per workspace ``` +The canonical documentation graph starts at `docs/README.md`; product requirements, technical +requirements, decision records, diagrams, and sufficiency are not replaceable by a PR body or old +plan. + Per-workspace and single-test: ```bash @@ -54,6 +58,10 @@ Three layers, decoupled through shared contracts: - `apps/desktop` — Tauri 2 + Vite + React 19 shell (Tailwind 4, Base UI, Storybook). Feature screens live in `src/features/` (home, workspace, chords, ranges, player, settings). `src/lib/analysis.ts` and `src/lib/job_runner.ts` call typed Tauri IPC commands, with a browser fallback that serves demo data when not running inside Tauri. - `apps/desktop/src-tauri/src/main.rs` — the Rust orchestration boundary. Tauri commands (`start_analysis_job`, `get_analysis_job_status`, `select_local_audio_source`, `import_youtube_url`) validate untrusted input (project IDs, file paths, URLs) and spawn the Python engine as a subprocess. There is no loopback HTTP listener and no network path for local analysis. - `services/analysis-engine` — Python package `bandscope_analysis` (librosa/numpy). Entry point `cli.py` reads a JSON job request on stdin and prints a structured job-status JSON envelope on stdout (`--progress-jsonl` streams progress lines). `api.py` orchestrates the pipeline across the `separation`, `sections`, `roles`, `chords`, `ranges`, `temporal`, `transcription`, and `youtube` modules. +- Production source separation uses `htdemucs` on supported platforms. The exact runtime model + artifact is inventoried but not bundled; current first use may retrieve it, and full-hash pre-load + enforcement remains a documented release blocker. The active known-stem test crosses the + production YouTube and separator boundaries; see `docs/TRD.md` and the operator guide. Data flow: React UI → Tauri IPC command → Rust validation + Python subprocess over stdin/stdout → job status and progress events emitted back to the UI. diff --git a/README.md b/README.md index 74312e3e4..bd25a7689 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,10 @@ App security source of truth: `docs/security/app-security.md` Dependency and SBOM source of truth: `docs/security/dependency-policy.md` Cross-platform build policy source of truth: `docs/security/cross-platform-build-policy.md` GitHub bootstrap execution source of truth: `docs/workflow/github-bootstrap-execution-policy.md` +Documentation authority index: `docs/README.md` +Product requirements: `docs/PRD.md` +Technical requirements: `docs/TRD.md` +Architecture decisions and diagrams: `docs/adr/README.md`, `docs/architecture/diagrams.md` ## Public repository baseline diff --git a/docs/PRD.md b/docs/PRD.md new file mode 100644 index 000000000..0d305e630 --- /dev/null +++ b/docs/PRD.md @@ -0,0 +1,85 @@ +# BandScope Product Requirements Document + +Status: Active authority +Last updated: 2026-08-09 + +## Product outcome + +BandScope turns a legally accessible song into a local-first, editable rehearsal view: form, +role-specific harmony and range, groove and entry cues, separated stem previews, confidence, and +rehearsal priority. It serves band leaders and players who need actionable preparation without a +DAW or notation-grade transcription workflow. + +The current conversation adds one essential proof obligation: BandScope must demonstrate that its +production YouTube intake and production separator improve a real, known source rather than merely +returning plausible-looking arrays or synthetic demo output. + +This is a bounded source-separation slice of GitHub issue #770, not completion of its broader +harmony, beat/tempo, structure, range, cue, confidence, public-corpus, private-corpus, manifest, +CPU/GPU, and report requirements. + +## Users and jobs + +- A band leader imports an authorized public YouTube track and needs trustworthy separated material + for assigning and checking rehearsal parts. +- A player needs a local stem preview and an honest confidence/failure state, not a silent fallback + to the original mixture. +- A maintainer needs reproducible evidence that model, downloader, fixture, and quality thresholds + still work together after dependency, model, or platform changes. +- A release owner needs evidence that external-media rights, model provenance, security boundaries, + and failure recovery are controlled. + +## Product requirements + +| ID | Requirement | Acceptance evidence | Status | +|---|---|---|---| +| PRD-KS-001 | Exercise the production YouTube download boundary with a real public mix whose creator-published source contains a known stem. | Live test calls `download_youtube_audio()` and validates the exact video ID. | `active_branch` | +| PRD-KS-002 | Exercise the real production source separator, not a mock, FFT profile, or generated-only mixture. | Live test calls `AudioStemSeparator.separate()` and receives canonical vocals/bass/drums/other arrays. | `active_branch` | +| PRD-KS-003 | Measure improvement against ground truth with an independently defined metric. | Zero-mean SI-SDR improvement is at least the provisional +0.5 dB sentinel over the downloaded mix; an authorized YouTube baseline is still required before release blocking. | `active_branch` | +| PRD-KS-004 | Verify semantic stem assignment. | Vocal SI-SDR exceeds the best incorrectly named stem by at least 3.0 dB. | `active_branch` | +| PRD-KS-005 | Detect fixture drift instead of blaming the model. | The downloaded mix is aligned to a separately pinned creator master with duration drift ≤ 1.0 s and correlation ≥ 0.90; that lag is composed once with the master-to-vocal lag before inference. | `active_branch` | +| PRD-KS-006 | Keep normal CI deterministic while preserving a real integration proof. | Metric, alignment, integrity, redirect, path, cleanup, and failure tests run offline; live access is explicit opt-in and fail-closed. | `active_branch` | +| PRD-KS-007 | Respect content and platform restrictions. | No cookies, account login, paywall, DRM, geo, or anti-bot bypass; operator records authorization before live use. | `active_branch` | +| PRD-KS-008 | Keep downloaded media ephemeral and private. | Test-owned directory is removed on success and failure; raw audio, full paths, URLs, tokens, and cookies are not logged or retained. | `active_branch` | +| PRD-KS-009 | Make release quality evidence reviewable. | Exact commit, model identity, fixture hashes, platform, command, outcome, and numeric scores are retained as a bounded CI/operator artifact. | `planned` | +| PRD-KS-010 | Fail safely when the live ecosystem is unavailable. | Download/model/integrity/drift failures are distinct, do not become passes, and do not block unrelated development work. | `active_branch` | + +## Scope and non-goals + +The first fixture makes a quantitative claim only for the vocal stem of Brad Sucks' *Making Me +Nervous*. The reference is a dry, loop-oriented vocal stem, so a separately pinned finished master +establishes YouTube asset identity. It does not prove four-stem quality, all genres, all YouTube transcodes, perceptual quality, +or notation accuracy. The benchmark is a quality sentinel, not a general downloader, media archive, +model-training dataset, or legal opinion. + +BandScope must not retain user media in hosted telemetry or introduce a relational benchmark +database merely to satisfy documentation conventions. Results remain ephemeral until a separate +audited evidence-retention requirement is accepted. + +## Failure experience + +The user-facing product must explain whether import, model availability, decode, or separation +failed and offer local-file fallback without exposing raw provider errors or sensitive paths. The +benchmark itself must retain stable diagnostic codes and numeric scores; it must never silently skip +after explicit opt-in. + +## Release acceptance + +The known-stem lane becomes blocking for a release only after all of the following exist: + +1. documented authorization for the chosen live access mode; +2. full-hash pre-load verification of the exact model artifact; +3. at least one recorded passing supported-platform run on the exact release candidate; +4. thresholds calibrated on an authorized YouTube candidate and a drift/flake triage owner; +5. ordinary CI, security, coverage, packaging, SBOM, review, and provenance gates pass. + +Until then, the deterministic offline contract is required and live evidence is advisory but must +fail closed when deliberately invoked. + +## Ownership and rollout + +The analysis-engine owner owns metrics, fixture integrity, alignment, separator integration, and +failure taxonomy. Release engineering owns model/tool inventory and retained evidence. Repository +governance owns rights/platform authorization and the decision to make live execution scheduled or +blocking. Rollout proceeds from local opt-in, to controlled release-candidate evidence, to a +blocking lane only through a superseding ADR. diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 000000000..4e932d538 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,40 @@ +# BandScope Documentation Authority + +## Canonical set + +BandScope decisions must be reconstructable from this repository without chat history. Use these +documents as the non-duplicative authority graph: + +- Product intent and acceptance outcomes: `docs/PRD.md` +- Technical contracts and quality gates: `docs/TRD.md` +- System boundaries and ownership: `ARCHITECTURE.md` +- Architecture, UML, deployment, and logical artifact diagrams: + `docs/architecture/diagrams.md` +- Decision status and supersession: `docs/adr/README.md` +- Current documentation sufficiency and known gaps: `docs/documentation-coverage-matrix.md` +- Real-audio MIR accuracy definitions, claim boundaries, and Issue #770 roadmap: + `docs/doctoring/real-audio-accuracy-acceptance.md` +- Live known-stem operator procedure and evidence: + `docs/engineering/youtube-known-stem-validation.md` +- Security source: `docs/security/app-security.md` +- Release and rollback controls: `docs/release/release-policy.md` and + `docs/operations/deploy-runbook.md` + +## Status vocabulary + +- `implemented_on_develop`: present on the protected default branch. +- `active_branch`: implemented on an unmerged branch; not shipped. +- `planned`: approved or proposed work without an implementation. +- `research_only`: evidence or experiment with no product commitment. +- `out_of_scope`: an explicit non-goal. + +Documents must use these labels when current and future behavior could otherwise be confused. A PR +body, chat transcript, or old implementation plan is evidence, not a replacement for the canonical +set. + +## Change rule + +Every material product, model, API, workflow, persistence, security, or release change must update +the affected canonical document or state why no documentation change is required. ADRs supersede +earlier decisions; they are not silently rewritten. `scripts/checks/verify_docs.py` enforces the +presence and cross-links of this authority graph. diff --git a/docs/TRD.md b/docs/TRD.md new file mode 100644 index 000000000..6b45c04c2 --- /dev/null +++ b/docs/TRD.md @@ -0,0 +1,146 @@ +# BandScope Technical Requirements Document + +Status: Active authority +Last updated: 2026-08-09 + +## System contract + +BandScope is a local-first React/Tauri desktop application with a Rust validation/orchestration +boundary and a Python analysis subprocess. The stable product hierarchy is `song -> section -> +role`. Shared TypeScript contracts carry rehearsal results; raw media and separated arrays stay +inside the local analysis boundary. + +This TRD defines the real known-stem validation slice. Detailed commands and fixture provenance are +in `docs/engineering/youtube-known-stem-validation.md`; decisions are in `docs/adr/README.md`; UML +and data-flow views are in `docs/architecture/diagrams.md`. + +## Technical requirements + +| ID | Requirement | Implementation or proof | +|---|---|---| +| TRD-KS-001 | Reuse the production downloader with strict HTTPS YouTube URL validation, playlist disabled, public access only, duration ≤ 900 seconds, and completed file ≤ 50 MiB. | `bandscope_analysis.youtube.download_youtube_audio` and unit tests. | +| TRD-KS-002 | Pin the source archive, extracted WAV, and creator master by exact HTTPS host, byte size, and full SHA-256. | `KnownStemFixture`, `download_verified_reference_stem`, and `download_verified_creator_master`. | +| TRD-KS-003 | Stream bounded downloads and one member only; never call `extractall()`. | Test-only fixture loader and hostile archive tests. | +| TRD-KS-004 | Decode YouTube mix, creator master, and vocal reference to mono 44.1 kHz; estimate YouTube-to-master and master-to-vocal global lags, compose them once, select one 12-second active window, and never align predicted stems separately. | `align_active_reference_window` and `align_known_stem_through_master`. | +| TRD-KS-005 | Produce finite, equal-length `vocals`, `bass`, `drums`, and `other` arrays through `AudioStemSeparator`. | Live assertion at production separator boundary. | +| TRD-KS-006 | Calculate zero-mean SI-SDR from hand-defined projection/residual arithmetic; reject non-finite, short, or silent input. | `zero_mean_si_sdr` offline tests. | +| TRD-KS-007 | Gate vocal SI-SDR improvement ≥ a provisional +0.5 dB and vocal assignment margin ≥ 3.0 dB. | Live assertions; creator-master calibration supports the sentinel, but an authorized YouTube baseline is required before promotion. | +| TRD-KS-008 | Reject candidate drift when YouTube/master duration differs by > 1.0 s or aligned identity correlation is < 0.90. | Pre-inference live assertions against the pinned finished master. | +| TRD-KS-009 | Run deterministic contract/security tests by default and require `BANDSCOPE_RUN_YOUTUBE_STEM_E2E=1` for live network/model execution. | Pytest marker and environment guard. | +| TRD-KS-010 | Clean every downloaded/scored artifact on success and failure. | Nested `TemporaryDirectory` plus postcondition. | +| TRD-KS-011 | Bind model identity to inventory and full SHA-256 before release. | Inventory records htdemucs signature `955717e8`, 84,141,911 bytes, and SHA-256 `8726e21a…a8b4`; full-hash pre-load enforcement is an open release blocker. | + +## Data and class contracts + +| Type | Required fields | Lifetime | +|---|---|---| +| `KnownStemFixture` | YouTube URL/video ID; archive/member/master URLs, hosts, full SHA-256 values, byte sizes; decoded master duration; target stem | Version-controlled test metadata | +| `AlignedStemWindow` | mixture/reference arrays; single lag; reference start; correlation | Process memory only | +| `KnownStemBenchmarkWindow` | YouTube/master lag; master/vocal lag; composed mixture/reference window; identity correlation | Process memory only | +| Separation result | canonical stem arrays; sample rate; duration; role types; notes | Process memory and downstream local analysis | +| Benchmark evidence | commit, fixture IDs/hashes, model signature/hash, platform, timestamps, scores, outcome code | `planned`; bounded artifact, never raw audio | + +No relational database exists for this capability. The logical artifact model in +`docs/architecture/diagrams.md` is authoritative; a database ERD would falsely imply persistence. + +## Metric contract + +For zero-mean estimate $\hat{s}$ and reference $s$: + +$$ +s_{target}=\frac{\langle \hat{s},s\rangle}{\|s\|^2}s,\qquad +\mathrm{SI\text{-}SDR}=10\log_{10}\frac{\|s_{target}\|^2}{\|\hat{s}-s_{target}\|^2}. +$$ + +Improvement subtracts the downloaded mixture's SI-SDR from the separated vocal's SI-SDR. The +assignment margin subtracts the best non-vocal stem score from the named vocal score. Expectations +are literal thresholds, not values recomputed by production helpers. + +## Platform and resource matrix + +| Platform | Dependency state | Live lane status | +|---|---|---| +| Linux x86_64 | Demucs/torch resolved; CPU inference supported | Supported for controlled evidence | +| Windows amd64/arm64 | Demucs dependency marker permits installation; release build must prove wheel/tool compatibility | Unproven | +| macOS arm64 | Demucs dependency marker permits installation | Unproven | +| macOS Intel | Demucs dependency marker excludes installation | Explicitly unavailable; product must surface safe fallback | + +The scored excerpt is 12 seconds, mono PCM at 44.1 kHz, with a 13-second separator duration bound +and 10 MiB scored-file bound. No release latency ceiling is yet accepted; record wall time and peak +memory during calibration rather than inventing a target. + +Production separation passes `shifts=0` to Demucs. This removes its random temporal augmentation so +the same audio, model, platform, and precision produce repeatable benchmark inputs and avoids a +global random-seed side effect in the test harness. + +## Model delivery and supply chain + +`AudioStemSeparator` currently asks Demucs 4.0.1 for `htdemucs`. Demucs maps that name to signature +`955717e8` and retrieves `955717e8-8726e21a.th` into a user runtime cache on first use. Demucs checks +the eight-hex filename hash prefix before deserializing. BandScope independently records the full +SHA-256 and exact byte size, but current production code does not yet enforce the full digest before +load. Therefore the model is not bundled and offline inference is guaranteed only after a trusted +cache is provisioned. ADR-0001 makes full-hash pre-load verification and an explicit redistribution +license decision release blockers. + +`ffmpeg` is an operator-provided executable resolved from `PATH`; yt-dlp is a locked Python package. +Release evidence must record their resolved versions and may not describe either as bundled unless +packaging and licensing change. + +## Failure taxonomy + +- `unsupported_url`, `restricted_content`, `duration_exceeded`, `size_exceeded`: production intake + policy failures. +- `download_failed`, `download_error`, `file_not_found`: live media/provider/tool failures. +- Reference byte/hash/member/redirect error: fixture integrity or SSRF boundary failure. +- YouTube/master duration drift above 1.0 s or identity correlation below 0.90: wrong or drifted + candidate/transcode. +- Model import/retrieval/load error: platform or supply-chain failure. +- Non-finite/shape/threshold error: separator correctness failure. + +Explicit live invocation converts all of these to a failing test. A failure blocks only the evidence +lane; it does not authorize a bypass or stop unrelated repository work. + +## Verification and evidence + +Default verification runs the 16 deterministic known-stem contract tests and explicitly excludes +the live marker. A live run uses the exact +command in the operator guide. Evidence must include exact commit and dependency lock, model full +hash, fixture archive full hash, public video ID, OS/architecture, result code, correlation, baseline +SI-SDR, vocal SI-SDR, improvement, assignment margin, duration, and cleanup result. Raw audio, +archive contents, local paths, provider response bodies, cookies, and credentials are forbidden. + +On 2026-08-09, commit `5a3648a11d9097b8da48bb4a3ccbd97986aec25b` passed all 13 offline +contract tests. Its explicit live attempt successfully validated the pinned reference archive but +failed at the production YouTube download boundary with HTTP 502 and produced no model score. This +is failure evidence, not a passing live benchmark. + +A separate creator-master calibration on the same environment measured deterministic `shifts=0` +SI-SDR improvement of +1.752 dB and vocal assignment margin of +7.631 dB. The old dry-vocal/mix +correlation was only 0.016856, proving it was not a valid identity gate. These values justify only +the provisional +0.5/+3.0 sentinels and the separate master identity design; they are not an +authorized YouTube pass. + +## Traceability + +`docs/documentation-coverage-matrix.md` maps product requirements and ADRs to modules, tests, and +release controls. Any threshold, fixture, model, persistence, or automation-policy change must +update that matrix and the applicable ADR before merge. + +Issue #770's complete real-audio acceptance program is tracked separately in +`docs/doctoring/real-audio-accuracy-acceptance.md`. This TRD implements only its known-vocal-stem +production-path slice. + +## References + +- Brad Sucks. (2004, May 3). *Making Me Nervous source*. + https://www.bradsucks.net/news/archives/2004/05/03/making-me-nervous-source +- Le Roux, J., Wisdom, S., Erdogan, H., & Hershey, J. R. (2019). SDR—Half-baked or well + done? In *ICASSP 2019—2019 IEEE International Conference on Acoustics, Speech and Signal + Processing* (pp. 626–630). IEEE. https://doi.org/10.1109/ICASSP.2019.8683855 +- National Institute of Standards and Technology. (2023). *Artificial intelligence risk + management framework (AI RMF 1.0)* (NIST AI 100-1). https://doi.org/10.6028/NIST.AI.100-1 +- Rouard, S., Stoller, D., & Défossez, A. (2023). Hybrid transformers for music source + separation. In *ICASSP 2023—2023 IEEE International Conference on Acoustics, Speech and + Signal Processing*. IEEE. https://arxiv.org/abs/2211.08553 +- YouTube. (n.d.). *Terms of Service*. https://www.youtube.com/static?template=terms diff --git a/docs/adr/0001-source-separation-runtime-and-model-delivery.md b/docs/adr/0001-source-separation-runtime-and-model-delivery.md new file mode 100644 index 000000000..d9b51d69f --- /dev/null +++ b/docs/adr/0001-source-separation-runtime-and-model-delivery.md @@ -0,0 +1,80 @@ +# ADR-0001: Source Separation Runtime and Model Delivery + +Status: Accepted with release blockers +Date: 2026-08-09 + +## Context and drivers + +The retired band-splitting profile was an FFT-era approximation and did not perform real source +separation. BandScope now uses Demucs 4.0.1 `htdemucs` to return vocals, bass, drums, and other for +local rehearsal analysis. The production boundary must remain local-first after model provisioning, +bounded on CPU, platform-honest, and traceable to an exact model artifact. + +The current Demucs loader downloads weights on first use and verifies only the eight-hex hash prefix +embedded in `955717e8-8726e21a.th`. The exact artifact is 84,141,911 bytes with SHA-256 +`8726e21a993978c7ba086d3872e7608d7d5bfca646ca4aca459ffda844faa8b4`. The Demucs code is MIT +licensed, but no separate commercial redistribution grant for the official weights was identified; +the upstream licensing discussion characterizes the weights as scientific-use material. + +## Decision + +1. `htdemucs` is the only production four-source model name until a superseding ADR. +2. The old `bandsplit-v1-profile` asset and inventory record are retired and must not reappear. +3. The exact official source URL, signature, full SHA-256, byte size, license uncertainty, cache + location, and release usage remain in `supply-chain/supplemental-component-inventory.json`. +4. Runtime retrieval is not equivalent to bundling. Documentation and SBOM evidence must preserve + that distinction. +5. A release claiming source-separation readiness must verify the full SHA-256 before any torch + deserialization and must have a recorded legal decision for its chosen download or distribution + path. +6. Until those blockers are implemented, first-load network access is explicit, offline inference is + guaranteed only with a trusted pre-provisioned cache, and source separation is unavailable on + macOS Intel under the current dependency markers. + +## Alternatives considered + +- Keep the FFT profile: rejected because it produces structurally plausible but invalid stems. +- Bundle official htdemucs weights immediately: rejected because repository/release size and model + redistribution rights are unresolved. +- Rely on Demucs' eight-hex prefix only: retained temporarily as current behavior, rejected as the + release target because it is weaker than the repository's full-integrity policy. +- Replace with ONNX or another commercially licensed model: viable future work, but it requires + parity, quality, platform, performance, and licensing evidence. + +## Consequences + +BandScope obtains real separation quality but inherits torch/Demucs resource cost, platform gaps, +runtime model retrieval, and an upstream model-rights decision. Release evidence cannot describe the +model as bundled or fully offline today. The supplemental inventory check now fails if the runtime +model is missing, incompletely pinned, or replaced by the retired profile. + +## Security and governance implications + +Model bytes are untrusted until verified. Full-hash verification must precede pickle/torch checkpoint +deserialization; a post-load hash is insufficient. Cache paths must be user-scoped, non-symlinked, +bounded, and cleaned or quarantined on mismatch. No user-supplied checkpoint is accepted. Model +downloads and errors must not expose tokens, usernames, or full paths. + +## Acceptance, recovery, and rollback + +- Inventory/model-name consistency check passes. +- A corrupt or substituted model fails before deserialization. +- Supported platform tests prove canonical finite stems and known-stem quality. +- Unsupported platforms return a stable fallback error. +- Rollback disables source separation or restores the previous exact approved model artifact; it + never restores the FFT profile as a production separator. + +## Supersession triggers + +Supersede this ADR when BandScope adopts a differently licensed model, bundles weights, implements an +ONNX/Rust inference path, changes the four-source contract, or makes GPU execution part of the +release baseline. + +## References + +- Défossez, A., Usunier, N., Bottou, L., & Bach, F. (2019). Music source separation in the + waveform domain. *arXiv*. https://arxiv.org/abs/1911.13254 +- Meta Research. (n.d.). *Demucs* [Source code]. GitHub. + https://github.com/facebookresearch/demucs +- Rouard, S., Stoller, D., & Défossez, A. (2023). Hybrid transformers for music source + separation. In *ICASSP 2023*. IEEE. https://arxiv.org/abs/2211.08553 diff --git a/docs/adr/0002-known-stem-youtube-quality-gate.md b/docs/adr/0002-known-stem-youtube-quality-gate.md new file mode 100644 index 000000000..c055f8c6a --- /dev/null +++ b/docs/adr/0002-known-stem-youtube-quality-gate.md @@ -0,0 +1,81 @@ +# ADR-0002: Known-Stem YouTube Quality Gate + +Status: Proposed on active branch +Date: 2026-08-09 + +## Context and drivers + +Unit tests with generated mixtures prove arithmetic but not the production downloader, transcoding, +alignment, decoder, model, and stem naming together. A real-world sentinel is required. The fixture +must have creator-published source material, stable integrity metadata, a matching public YouTube +mix, and bounded execution. + +## Decision + +Use Brad Sucks' *Making Me Nervous* as the first vocal sentinel. Fetch the real YouTube mix through +`download_youtube_audio()`, authenticate both the source archive/exact `vocals.wav` and a separately +pinned creator-hosted finished master, compose the YouTube-to-master and master-to-vocal global +offsets once, score one strongest 12-second vocal window, and run the production +`AudioStemSeparator` with deterministic Demucs `shifts=0`. + +The provisional sentinel rejects YouTube/master duration drift above 1.0 seconds or identity +correlation below 0.90, and requires vocal SI-SDR improvement ≥ +0.5 dB plus vocal assignment margin +≥ 3.0 dB. A creator-master calibration measured +1.752 dB and +7.631 dB respectively with +`shifts=0`; it also showed that dry-vocal/mix correlation (0.016856) is not a valid identity check. +Offline metric, alignment, integrity, SSRF/path, cleanup, and failure tests run normally. The live +lane is explicit opt-in, never silently skips after opt-in, and is not scheduled or release-blocking +until rights/platform authorization and an authorized YouTube calibration are recorded. + +## Alternatives considered + +- Synthetic mixtures only: rejected as insufficient production-boundary evidence. +- Redistribute YouTube/reference audio in git: rejected for rights, repository size, and data + retention reasons. +- Run live on every PR: rejected until platform authorization, provider stability, model caching, + cost, and false-failure policy are established. +- Use correlation alone: rejected because correlation cannot prove separation improvement or correct + semantic stem assignment. +- Independently time-shift every predicted stem: rejected because it can hide separator latency or + phase defects and inflate scores. + +## Consequences + +The live lane can fail for provider availability independently of model correctness. That failure is +retained honestly and blocks only that evidence lane. A creator-master probe is calibration +evidence, not proof that the YouTube candidate passes. A single vocal fixture does not establish +four-source or genre-wide validity. Additional fixtures require separate provenance and calibrated +threshold review, not threshold weakening. + +## Security, privacy, and legal implications + +The test crosses public network, archive, decoder, ffmpeg, model, filesystem, and subprocess trust +boundaries. It uses strict HTTPS/host/size/hash/member allowlists, test-owned temporary storage, +bounded media, sanitized diagnostics, and cleanup. It adds no cookies, credentials, account login, +paywall, DRM, geo, or anti-bot bypass. Creator permission for source files does not itself authorize +automated YouTube access; the operator must verify the intended access against current terms and +rights. + +## Acceptance, recovery, and rollback + +- Sixteen deterministic contract tests pass in ordinary CI; the root runner explicitly excludes the + live marker. +- A controlled live run on the exact candidate records all required scores and cleanup evidence. +- Fixture drift causes a distinct pre-model failure. +- Provider/model unavailability remains a failure after explicit opt-in. +- Rollback removes the live gate without removing deterministic metric/security tests or weakening + production intake controls. + +## Supersession triggers + +Supersede this ADR when the fixture changes, a four-stem/multi-genre suite is adopted, live execution +becomes scheduled/blocking, the production downloader changes, or a perceptual metric becomes a +release requirement. + +## References + +- Brad Sucks. (2004, May 3). *Making Me Nervous source*. + https://www.bradsucks.net/news/archives/2004/05/03/making-me-nervous-source +- Le Roux, J., Wisdom, S., Erdogan, H., & Hershey, J. R. (2019). SDR—Half-baked or well + done? In *ICASSP 2019* (pp. 626–630). IEEE. + https://doi.org/10.1109/ICASSP.2019.8683855 +- YouTube. (n.d.). *Terms of Service*. https://www.youtube.com/static?template=terms diff --git a/docs/adr/0003-ephemeral-benchmark-evidence-model.md b/docs/adr/0003-ephemeral-benchmark-evidence-model.md new file mode 100644 index 000000000..89b476c25 --- /dev/null +++ b/docs/adr/0003-ephemeral-benchmark-evidence-model.md @@ -0,0 +1,56 @@ +# ADR-0003: Ephemeral Benchmark Evidence Model + +Status: Proposed on active branch +Date: 2026-08-09 + +## Context and drivers + +The known-stem benchmark handles copyrighted media, decoded waveforms, separated arrays, local +paths, model artifacts, and numeric evidence. Adding a database merely to retain benchmark state +would expand privacy, authorization, migration, backup, and deletion obligations without a current +product need. + +## Decision + +Downloaded audio, extracted references, scored windows, and separated stems are ephemeral and live +only inside a test-owned temporary directory or process memory. Cleanup occurs on success and +failure. The repository stores fixture metadata and thresholds. When controlled evidence retention +is authorized, retain only exact commit/lock/model/fixture identities, platform, timestamps, numeric +scores, duration, outcome code, and cleanup result as a bounded Actions/operator artifact. + +No relational database is introduced. `docs/architecture/diagrams.md` contains the authoritative +logical artifact relationship model; it is intentionally not a physical ERD. + +## Alternatives considered + +- Persist every run and media asset: rejected for rights, privacy, cost, and operational scope. +- Persist only numeric results in an application database: deferred because there is no current + query, tenant, retention, or product workflow requiring it. +- Retain no evidence: rejected because release and regression decisions need traceable results. + +## Consequences + +Trend analysis is initially manual or artifact-based. Evidence retention must have an explicit TTL +and access policy. Reproduction depends on external fixture availability, so exact identity and +failure codes are essential. A future hosted evidence service is a separate bounded context and may +not access user audio or BandScope's local project files directly. + +## Security and governance implications + +Evidence excludes raw audio, source archive content, full URLs, local paths, usernames, cookies, +credentials, and provider response bodies. Actions artifacts must be access-controlled, checksum +bound to the candidate, and expire under repository policy. PII masking is not needed because PII is +not collected; purpose limitation and non-collection are the control. + +## Acceptance, recovery, and rollback + +- Temporary root is empty after the live test exits. +- Logs contain stable public fixture IDs and numeric results only. +- Evidence schema rejects raw media/path fields. +- Rollback deletes retained numeric artifacts according to TTL without affecting local projects. + +## Supersession triggers + +Supersede this ADR if recurring trend queries, audited release history, multi-tenant evidence, or a +hosted benchmark service is approved. That ADR must supply a physical ERD, authorization model, +retention/deletion policy, migrations, backup/restore, and rollback. diff --git a/docs/adr/README.md b/docs/adr/README.md new file mode 100644 index 000000000..3270ac4ed --- /dev/null +++ b/docs/adr/README.md @@ -0,0 +1,13 @@ +# Architecture Decision Records + +ADRs are immutable decision records. Amend factual links or typographical errors in place; change a +decision through a new ADR that names the superseded record. + +| ADR | Status | Decision | +|---|---|---| +| `0001-source-separation-runtime-and-model-delivery.md` | Accepted with release blockers | Use real four-source htdemucs locally and inventory the exact runtime artifact; require full-hash pre-load verification and a model-rights decision before release readiness. | +| `0002-known-stem-youtube-quality-gate.md` | Proposed on active branch | Validate the production YouTube-to-separator path with a creator-published known vocal stem, single alignment, SI-SDR improvement, and assignment margin. | +| `0003-ephemeral-benchmark-evidence-model.md` | Proposed on active branch | Keep media and signal arrays ephemeral; retain only bounded evidence when authorized, so a relational ERD is not currently authoritative. | + +Status meanings are `Proposed`, `Accepted`, `Deprecated`, and `Superseded`. An accepted decision may +still carry explicit release blockers; acceptance does not assert that every follow-up is shipped. diff --git a/docs/architecture/diagrams.md b/docs/architecture/diagrams.md new file mode 100644 index 000000000..6188719f8 --- /dev/null +++ b/docs/architecture/diagrams.md @@ -0,0 +1,160 @@ +# BandScope Architecture and UML Diagrams + +These diagrams describe current `develop` behavior plus the explicitly labeled active known-stem +branch. They do not imply that unmerged work is shipped. + +## Component view + +```mermaid +flowchart TD + UI["React rehearsal UI"] -->|"typed IPC"| Rust["Tauri Rust boundary"] + Rust -->|"stdin/stdout JSON"| Engine["Python analysis engine"] + Engine --> Separator["htdemucs separator"] + Engine --> Analysis["section / role / harmony analysis"] + Contracts["shared TypeScript contracts"] --- UI + Contracts --- Rust +``` + +## Known-stem identity and alignment sequence (active branch) + +```mermaid +sequenceDiagram + actor Operator + participant Test as Pytest benchmark + participant Intake as Production YouTube intake + participant Ref as Pinned reference loader + participant Align as Global aligner + Operator->>Test: Explicit opt-in + par Untrusted downloads + Test->>Intake: Public YouTube URL + Intake-->>Test: Bounded decoded mix + and + Test->>Ref: Archive + master metadata + Ref-->>Test: Authenticated master + vocals.wav + end + Test->>Align: Mix + master + vocal + Align-->>Test: Identity proof + composed 12 s window +``` + +## Known-stem inference and scoring sequence (active branch) + +```mermaid +sequenceDiagram + participant Test as Pytest benchmark + participant Sep as Production htdemucs + participant Score as SI-SDR scorer + Test->>Sep: Scored mix window + Sep-->>Test: vocals / bass / drums / other + Test->>Score: Stems + mix + reference + Score-->>Test: Identity, SI-SDRi, assignment margin +``` + +## Benchmark state model + +```mermaid +stateDiagram-v2 + [*] --> Disabled + Disabled --> Preflight: explicit opt-in + Preflight --> Fetching: authorization and tools present + Preflight --> Failed: policy or tool failure + Fetching --> Aligning: exact IDs and hashes pass + Fetching --> Failed: download or integrity failure + Aligning --> Separating: duration and identity pass + Aligning --> Failed: fixture drift + Separating --> Scoring: finite canonical stems + Separating --> Failed: model or shape failure + Scoring --> Passed: thresholds pass + Scoring --> Failed: threshold failure + Passed --> Cleaned + Failed --> Cleaned + Cleaned --> [*] +``` + +## UML class view + +```mermaid +classDiagram + class KnownStemFixture { + +youtube_url: str + +video_id: str + +reference_archive_sha256: str + +reference_archive_bytes: int + +reference_member: str + +reference_member_sha256: str + +creator_master_sha256: str + +creator_master_bytes: int + +target_stem: str + } + class AlignedStemWindow { + +mixture: ndarray + +reference: ndarray + +lag_samples: int + +correlation: float + } + class AudioStemSeparator { + +separate(audio_path) AudioSeparationResult + } + class KnownStemBenchmarkWindow { + +mixture: ndarray + +reference: ndarray + +youtube_to_master_lag_samples: int + +master_to_reference_lag_samples: int + +identity_correlation: float + } + class BenchmarkScore { + +baseline_si_sdr: float + +vocal_si_sdr: float + +improvement_db: float + +assignment_margin_db: float + } + KnownStemFixture --> AlignedStemWindow: authenticates assets + AlignedStemWindow --> KnownStemBenchmarkWindow: composes two lags + KnownStemBenchmarkWindow --> AudioStemSeparator: supplies one mix window + AudioStemSeparator --> BenchmarkScore: supplies named stems +``` + +`BenchmarkScore` is a logical contract planned for retained evidence; current test assertions compute +these values without instantiating a production class. + +## Deployment and trust boundaries + +```mermaid +flowchart TB + subgraph Desktop["User desktop"] + App["BandScope app"] + Cache["User-scoped model cache"] + Temp["Ephemeral media root"] + App --> Cache + App --> Temp + end + YouTube["YouTube media boundary"] --> App + Source["Pinned creator archive"] --> App + Master["Pinned creator master"] --> App + Model["Official model host"] --> Cache + App --> Evidence["Bounded numeric evidence"] +``` + +The model cache is persistent; media temp is not. The public hosts, cache contents, media, decoders, +and model bytes are untrusted until their respective policy and integrity checks pass. + +## Logical artifact relationship model (not a physical ERD) + +```mermaid +erDiagram + KNOWN_STEM_FIXTURE ||--|| REFERENCE_ARCHIVE : pins + KNOWN_STEM_FIXTURE ||--|| CREATOR_MASTER : pins + KNOWN_STEM_FIXTURE ||--|| YOUTUBE_MIX : identifies + REFERENCE_ARCHIVE ||--|| REFERENCE_STEM : contains + YOUTUBE_MIX ||--|| CREATOR_MASTER : identity-checks + CREATOR_MASTER ||--|| ALIGNED_WINDOW : anchors + YOUTUBE_MIX ||--|| ALIGNED_WINDOW : yields + REFERENCE_STEM ||--|| ALIGNED_WINDOW : aligns + ALIGNED_WINDOW ||--|{ SEPARATED_STEM : produces + ALIGNED_WINDOW ||--|| BENCHMARK_EVIDENCE : scores + SEPARATED_STEM }|--|| BENCHMARK_EVIDENCE : contributes +``` + +Only `KNOWN_STEM_FIXTURE` metadata is version-controlled. `YOUTUBE_MIX`, `CREATOR_MASTER`, +`REFERENCE_STEM`, `ALIGNED_WINDOW`, and `SEPARATED_STEM` bytes are ephemeral. +`BENCHMARK_EVIDENCE` is planned as a bounded artifact, not a database row. ADR-0003 requires a new +physical ERD only if persistence is introduced. diff --git a/docs/architecture/overview.md b/docs/architecture/overview.md index 3cf5261b9..8b8bb00c4 100644 --- a/docs/architecture/overview.md +++ b/docs/architecture/overview.md @@ -39,6 +39,20 @@ GitHub is the source of truth for repository governance, PR review, CI/CD, Code - bootstrap local audio projects by validating the selected file in Rust, then passing only typed source metadata through the orchestration boundary - keep project and temp/cache bootstrap roots under Tauri-resolved app-owned directories rather than the shared OS temp namespace +## Source-separation runtime + +- The Python engine uses the real four-source `htdemucs` model on supported platforms; the old FFT + profile is retired. +- Inference is local after a trusted cache is provisioned. Current first use may retrieve the exact + inventoried model artifact; full-hash pre-load enforcement remains a release blocker. +- The known-stem validation branch proves the real YouTube intake → creator-master identity → + composed master/vocal alignment → deterministic separator → SI-SDR scoring path. Test-only + reference handling never becomes a general runtime downloader. +- Media and stem arrays are ephemeral. Only bounded numeric/provenance evidence may be retained after + authorization, so no physical benchmark database or ERD exists. + +See `docs/TRD.md`, `docs/adr/README.md`, and `docs/architecture/diagrams.md`. + ## CI/CD and release flow - PRs into `develop` and `main` run CI, dependency review, security audit, secret-scan gate, SBOM generation, and CodeQL diff --git a/docs/doctoring/real-audio-accuracy-acceptance.md b/docs/doctoring/real-audio-accuracy-acceptance.md new file mode 100644 index 000000000..79a675a62 --- /dev/null +++ b/docs/doctoring/real-audio-accuracy-acceptance.md @@ -0,0 +1,130 @@ +# Real-Audio Accuracy Acceptance Doctoring + +Status: Partial implementation on active branch +Tracks: GitHub issue #770 +Last updated: 2026-08-09 + +## Purpose and claim boundary + +BandScope needs decoded-audio acceptance evidence that distinguishes “the pipeline ran” from “the +rehearsal output was measurably accurate.” Passing any registered benchmark supports only the exact +versioned fixtures, annotations, model/backend, metrics, and tolerances in its manifest. It does not +establish universal musical correctness, genre/culture invariance, perceptual superiority, or safe +replacement of human rehearsal judgment. + +The current active branch implements one vocal source-separation sentinel. It does not complete +issue #770's complete harmony, beat/tempo, structure, range, rehearsal cue, overlap, confidence, +multi-corpus, CPU/GPU, manifest, JSON, or accessible HTML program. + +## Evidence tiers + +1. Deterministic redistributable PCM: versioned, license-clean generated or checked-in waveforms with + immutable manifests. They must exercise actual decode, not direct feature arrays. +2. Redistributable public corpus slice: exact audio/annotation license, source/DOI, file hash, split, + provenance, and transformations. +3. Separately licensed private benchmark: fail closed when credentials/manifest are absent and retain + only aggregate metrics, bounded error exemplars, configuration hashes, and provenance-safe + artifacts. + +The known-stem YouTube sentinel is an authorization-gated external integration sentinel, not a +substitute for tier 1 or proof that tier 2 redistribution rights exist. + +## Metric registry + +| Domain | Required metrics | Interpretation boundary | Current status | +|---|---|---|---| +| Source separation | Per-stem SI-SDR/SDR equivalent, improvement over mixture, semantic assignment, mixture consistency, finite output | Energy-ratio metrics do not establish perceptual quality; human listening protocol required for such claims. | Vocal SI-SDRi and assignment implemented on active branch; no passing live score | +| Harmony | Segment chord symbol recall, duration-weighted WCSR, root/major-minor/seventh mappings, no-chord, boundary error | Vocabulary and time alignment must be reported; one opaque aggregate is insufficient. | Planned | +| Beat/tempo | Beat precision/recall/F, continuity-aware metrics, tempo Acc1 and Acc2 | Half/double tempo must remain visible; confidence needs calibration. | Planned | +| Structure | Boundary P/R/F at strict/relaxed windows, segment-label agreement, order/repetition/pickup preservation | A correct label with materially wrong boundary remains an error. | Planned | +| Range | Note/semitone endpoint error and exact out-of-range classification | Stem/role identity and octave policy must be registered. | Planned | +| Rehearsal cues | Entry/dropout/stop/pickup event P/R and timing error | Event tolerance must reflect rehearsal use, not be widened after failure. | Planned | +| Role overlap | Activity interval IoU or registered equivalent | Aggregate overlap must not hide severe role-specific misses. | Planned | +| Confidence | Reliability/calibration curve and Brier-style score where probabilistic | Confidence text without probabilistic semantics is not scored as calibrated. | Planned | + +## Regression and uncertainty policy + +The first protected baseline is descriptive; thresholds must not be invented as “industry +standard.” Later gates use preregistered practical/statistical tolerances by metric and fixture +family. Dataset and track-level values are retained so aggregates cannot hide severe regressions. +Nondeterministic stages report repeated-run or bootstrap uncertainty. A regression waiver must name +the exact metric/fixture, evidence, owner, expiry, and rollback; silent threshold reduction is +forbidden. + +The provisional known-stem thresholds are deliberately limited to the first vocal sentinel. They +must be recalibrated across repeated supported-platform runs before release blocking. + +## Manifest and report contract + +The planned accuracy manifest records fixture ID/hash/license/provenance, annotations, transforms, +engine/model/backend/version/hash, CPU/GPU device and precision, thread count, elapsed time, peak +RSS/VRAM, metric definitions/version, track- and aggregate-level exact values, registered tolerance, +uncertainty, outcome, limitations, commit/base, and cleanup result. It rejects unknown fields, +malformed manifests, checksum drift, missing configured GPU evidence, and synthetic fallback +presented as corpus success. + +Machine-readable JSON and accessible HTML render the same exact values. Neither format contains raw +private audio, copyrighted excerpts, absolute paths, credentials, cookies, or provider response +bodies. + +## Rights, security, and privacy + +Audio, annotations, metadata, manifests, decoders, model artifacts, and benchmark storage are +untrusted. Enforce bounds for duration, channels, sample rate, decoded bytes, file count, and output +size. Use fixed argument arrays, no shell interpolation, verified manifests/hashes, least privilege, +and explicit storage roots. Ordinary local analysis must not gain a new network dependency because +an acceptance workflow uses authorized external storage. + +PII masking is not the control: benchmarks should avoid collecting identity data. Purpose-bound +authorization, non-collection, isolated credentials, bounded evidence, access control, retention, +deletion, and tamper-evident provenance preserve utility without exposing media or identities. + +## Operations and rollback + +One documented command must eventually run the complete registered acceptance suite and produce +deterministic JSON/HTML. A provider or corpus outage blocks only its tier, never becomes a pass, and +does not stop unrelated engineering. Rollback restores the previous exact manifest/model/backend and +removes unsupported accuracy claims; it does not delete failing evidence, weaken metrics, or replace +real audio with mocks. + +## Current source-separation slice + +The active branch: + +- crosses the production YouTube downloader and htdemucs separator; +- authenticates a creator-published vocal stem archive, exact extracted member, and separate + creator-hosted finished master; +- composes YouTube-to-master and master-to-vocal global lags once and scores a 12-second active + window without aligning predictions independently; +- provisionally requires duration drift ≤ 1.0 s, master identity correlation ≥ 0.90, vocal SI-SDR + improvement ≥ +0.5 dB, and vocal assignment margin ≥ 3.0 dB; +- passes `shifts=0` to Demucs for deterministic inference; +- runs 16 metric/alignment/integrity/security/cleanup cases offline and explicitly excludes the live + marker from required CI; +- keeps live access explicit opt-in and fail-closed. + +On 2026-08-09, the offline contract passed at `5a3648a11d9097b8da48bb4a3ccbd97986aec25b`. +The live attempt failed at YouTube HTTP 502 before model execution, so no passing score exists. +Creator-master calibration produced deterministic +1.752 dB SI-SDR improvement and +7.631 dB +assignment margin, while dry-vocal/mix correlation was only 0.016856. Those results support the +provisional sentinel and separate identity check, not a YouTube pass or release-blocking threshold. + +## References + +- Le Roux, J., Wisdom, S., Erdogan, H., & Hershey, J. R. (2019). SDR—Half-baked or well + done? In *ICASSP 2019* (pp. 626–630). IEEE. + https://doi.org/10.1109/ICASSP.2019.8683855 +- National Institute of Standards and Technology. (2023). *Artificial intelligence risk + management framework (AI RMF 1.0)* (NIST AI 100-1). https://doi.org/10.6028/NIST.AI.100-1 +- Odekerken, D., Koops, H. V., & Volk, A. (2021). Improving audio chord estimation by alignment + and integration of crowd-sourced symbolic music. *Transactions of the International Society for + Music Information Retrieval, 4*(1), 141–155. https://doi.org/10.5334/tismir.81 +- Raffel, C., McFee, B., Humphrey, E. J., Salamon, J., Nieto, O., Liang, D., & Ellis, D. P. W. + (2014). MIR_EVAL: A transparent implementation of common MIR metrics. In *Proceedings of the + 15th International Society for Music Information Retrieval Conference* (pp. 367–372). +- Schreiber, H., & Müller, M. (2020). Music tempo estimation: Are we done yet? + *Transactions of the International Society for Music Information Retrieval, 3*(1), 111–125. + https://doi.org/10.5334/tismir.43 +- Stöter, F.-R., Liutkus, A., & Ito, N. (2018). The 2018 signal separation evaluation campaign. + In *Latent Variable Analysis and Signal Separation*. Springer. + https://doi.org/10.1007/978-3-319-93764-9_35 diff --git a/docs/documentation-coverage-matrix.md b/docs/documentation-coverage-matrix.md new file mode 100644 index 000000000..45e654611 --- /dev/null +++ b/docs/documentation-coverage-matrix.md @@ -0,0 +1,74 @@ +# Documentation Coverage and Traceability Matrix + +Last evaluated: 2026-08-09 +Evaluation scope: real known-stem YouTube source-separation validation and the affected BandScope +runtime/release boundaries. + +## Sufficiency verdict + +The pre-change repository had a strong benchmark operator note but was insufficient: it lacked a +canonical PRD, TRD, ADRs, UML, logical data model, traceability, model inventory consistency, and +release/operations criteria. This branch adds those authorities and mechanical presence checks. + +The documentation graph is now structurally sufficient and explicitly code-current, but the product +is not yet release-ready for source separation. A passing live run, full-hash pre-load model +verification, model-rights decision, threshold calibration, supported-platform evidence, and +bounded evidence artifact remain open. + +Issue #770 remains open. This branch must not be described as completing the full real-audio MIR +acceptance layer. + +## Artifact coverage + +| Family | Canonical authority | Assessment | Remaining gap | +|---|---|---|---| +| PRD | `docs/PRD.md` | Adequate for product outcome, scope, users, acceptance, legal boundary, rollout, and non-goals. | Broader multi-fixture/four-stem requirements await evidence. | +| TRD | `docs/TRD.md` | Adequate for interfaces, metrics, schema, platform matrix, failure taxonomy, model delivery, evidence, and traceability. | Performance budget and calibrated thresholds are not yet accepted. | +| Architecture | `ARCHITECTURE.md`, `docs/architecture/overview.md` | Updated for htdemucs and known-stem boundaries. | Full-hash model resolver remains implementation work. | +| ADR | `docs/adr/README.md`, ADR-0001..0003 | Captures model, live quality gate, and persistence/ERD decisions with alternatives and supersession. | ADR-0002/0003 remain Proposed until branch merge. | +| UML | `docs/architecture/diagrams.md` | Component, sequence, state, class, and deployment views included. | No additional UML is needed for the bounded slice. | +| ERD/data | `docs/architecture/diagrams.md`, ADR-0003 | Logical artifact relationships and persistence status are explicit. | Physical ERD is intentionally not applicable until persistence exists. | +| Security/privacy | `docs/engineering/youtube-known-stem-validation.md`, `docs/security/app-security.md`, ADRs | Threats, trust boundaries, non-collection, integrity, cleanup, and legal limits covered. | Full model hash must be enforced before load. | +| Test strategy | `docs/TRD.md`, operator guide, acceptance criteria | Offline/live split and metric/failure contracts covered. | No successful live score has been recorded. | +| MIR doctoring | `docs/doctoring/real-audio-accuracy-acceptance.md` | Issue #770 metrics, claim boundaries, tiers, and roadmap are separated from the bounded vocal slice. | Accuracy manifest, reports, other MIR families, and corpus tiers remain open. | +| Operations/release | runbook and release policy | Preflight, evidence, triage, rollback, and blocking conditions covered. | Platform matrix and live pass are incomplete. | +| Supply chain | supplemental inventory and dependency policy | Retired model removed; exact runtime artifact, ffmpeg status, and hash recorded. | Weight redistribution rights and pre-load enforcement unresolved. | +| Automation | active CWL autonomous loop and `docs/workflow/pr-review-merge-scheduler.md` | BandScope continuity and no-status-only termination are covered without creating a competing writer. | Dedicated BandScope loop remains paused due writer topology/active-task capacity. | + +## Requirement-to-evidence traceability + +| Requirement | Decision/research | Module or artifact | Test/evidence | Release control | +|---|---|---|---|---| +| PRD-KS-001, KS-007 | ADR-0002; YouTube Terms | `bandscope_analysis.youtube` | `test_youtube.py`; opted-in live test | Authorization preflight | +| PRD-KS-002, KS-004 | ADR-0001/0002; Rouard et al. (2023) | `separation/audio_separator.py` | `test_youtube_stem_e2e.py` live case | Exact model identity and supported platform | +| PRD-KS-003 | Le Roux et al. (2019) | `tests/known_stem_benchmark.py` | SI-SDR unit tests and live threshold | Calibration plus exact-candidate score | +| PRD-KS-005 | ADR-0002 | master identity plus composed global alignment helpers | delayed/composed-window tests; live duration/correlation | Authorized YouTube calibration and drift triage | +| PRD-KS-006, KS-010 | ADR-0002 | pytest marker and failure taxonomy | 16 default offline tests; explicit live failure | Advisory until promotion ADR | +| PRD-KS-008 | ADR-0003 | temporary directory and sanitized errors | cleanup postcondition and archive failure tests | Evidence excludes raw media/paths | +| PRD-KS-009 | ADR-0003; NIST AI RMF TEVV | planned bounded evidence schema | No retained score yet | Required before blocking release gate | +| TRD-KS-011 | ADR-0001 | supplemental inventory | inventory consistency tests | Full-hash pre-load blocker | + +## Live evidence snapshot + +| Date | Commit under test | Offline contract | Live result | Classification | +|---|---|---|---|---| +| 2026-08-09 | `5a3648a11d9097b8da48bb4a3ccbd97986aec25b` | 13 passed | Reference archive verified; YouTube download failed with HTTP 502 before separation; no score | Exact failure evidence, not a pass | + +Separate creator-master calibration on that environment measured `shifts=0` vocal SI-SDR +improvement +1.752 dB and assignment margin +7.631 dB. Dry-vocal/mix correlation was 0.016856, so +the branch now uses a separately pinned finished master for identity. This probe did not download +YouTube and is not a live pass. + +## Machine-checkable contract + +`scripts/checks/verify_docs.py` requires the canonical index, PRD, TRD, ADR index and records, +diagram authority, and this matrix, and checks cross-links from architecture and the index. +`scripts/checks/verify_supply_chain.py` derives the configured separator model name and rejects an +inventory that lacks it, uses the retired bandsplit profile, omits required fields, lacks a full +SHA-256/positive byte size, or uses a non-HTTPS model source. + +## Re-evaluation triggers + +Re-run this matrix whenever the model/signature, fixture, threshold, downloader, separator output +contract, supported platform, persistence policy, evidence retention, workflow scheduling, or +release-blocking status changes. diff --git a/docs/engineering/acceptance-criteria.md b/docs/engineering/acceptance-criteria.md index 6bce19771..841d37878 100644 --- a/docs/engineering/acceptance-criteria.md +++ b/docs/engineering/acceptance-criteria.md @@ -42,6 +42,24 @@ Changes touching files, URLs, subprocesses, IPC, WebView, updates, model downloa For protected branches, intended checks are documented in `docs/security/github-required-checks.md`. Work should not reduce or bypass these checks. +## Source-separation quality gates + +- Every separator or downloader change must keep the 16 deterministic known-stem metric, alignment, + archive-integrity, redirect/path, cleanup, and failure-contract cases passing. +- A live evidence claim must cross `download_youtube_audio()` and `AudioStemSeparator.separate()` on + the same exact candidate, authenticate the separately pinned creator master, compose the two + global offsets once, and record duration drift, master identity correlation, baseline/vocal + SI-SDR, improvement, assignment margin, model identity, platform, and cleanup. +- The provisional live thresholds are YouTube/master duration drift ≤ 1.0 s, identity correlation ≥ + 0.90, vocal SI-SDR improvement ≥ +0.5 dB, and vocal assignment margin ≥ 3.0 dB. The quality + thresholds are supported by creator-master calibration only; an authorized YouTube baseline is + required before promotion. Changing a threshold requires calibration evidence and an ADR; a + provider or model failure does not justify weakening it. +- Skipped, disabled, HTTP/provider-failed, model-unavailable, integrity-failed, drifted, non-finite, + predecessor-head, or stale-base execution is not passing evidence. +- Before the lane can block a release, ADR-0001/0002 blockers—authorization, full-hash pre-load + verification, exact-candidate pass, calibration, and supported-platform evidence—must be closed. + ## Evidence policy Completion claims must be backed by command output and/or GitHub run evidence from the current change set. diff --git a/docs/engineering/harness-engineering.md b/docs/engineering/harness-engineering.md index 2a8bcce32..13c0ab37f 100644 --- a/docs/engineering/harness-engineering.md +++ b/docs/engineering/harness-engineering.md @@ -27,6 +27,7 @@ Quickcheck aggregates lint/type/test/build and repository policy checks intended - Dependency sync: `uv sync --project services/analysis-engine --group dev` - Tests: `uv run --project services/analysis-engine pytest --cov=src/bandscope_analysis --cov-report=term-missing --cov-fail-under=100` +- Real YouTube known-stem validation: `docs/engineering/youtube-known-stem-validation.md` ## CI parity expectation diff --git a/docs/engineering/youtube-known-stem-validation.md b/docs/engineering/youtube-known-stem-validation.md new file mode 100644 index 000000000..b387f2d46 --- /dev/null +++ b/docs/engineering/youtube-known-stem-validation.md @@ -0,0 +1,194 @@ +# YouTube Known-Stem Validation + +## Purpose + +BandScope has an opt-in benchmark that downloads a real YouTube mix through the production +`download_youtube_audio()` boundary, separates a 12-second active excerpt with the real CPU +`htdemucs` model, and compares the resulting vocal stem with a known vocal source. + +The benchmark lives in `services/analysis-engine/tests/test_youtube_stem_e2e.py`. Product and +technical requirements are canonical in `docs/PRD.md` and `docs/TRD.md`; ADR-0001 through ADR-0003 +record model, live-gate, and persistence decisions. Its signal, +alignment, archive-integrity, and failure-path tests run offline in the normal Python test suite. +The network/model case is marked `youtube_stem_e2e` and skipped unless explicitly enabled. It is +not a required pull-request or default CI check. + +## Fixture provenance and scope + +- Composition: *Making Me Nervous* by Brad Sucks. +- YouTube fixture: `https://www.youtube.com/watch?v=e4pIpWVbMKs` (video ID + `e4pIpWVbMKs`). +- Creator source page: `https://www.bradsucks.net/news/archives/2004/05/03/making-me-nervous-source`. +- HTTPS source archive: + `https://bradmedia.com/media/source/making_me_nervous-120bpm.zip`. +- Archive size: `31,055,394` bytes. +- Archive SHA-256: + `473578daa0bcf022448a144c5df9111ddf11e5a90e77f3649254e7813ba4981d`. +- Exact reference member: `vocals.wav`, `25,603,092` uncompressed bytes, SHA-256 + `4c7bb41c3f8bda1471dfd214b84f1d3457af344feeba33f0b31982ed0d808afc`. +- Creator-hosted finished master: `01 Brad Sucks - Making Me Nervous.mp3` on the exact + `static1.squarespace.com` HTTPS host, `4,941,627` bytes, SHA-256 + `fc7f7c2a0387e46885e5c133cbd6d14d7de4d48908b68f1135354df0a336cf1d`, decoded mono duration + `155.945238` seconds at 44.1 kHz. +- Permission evidence: the creator-published archive readme grants broad reuse permission for the + supplied source material. No source audio is redistributed in this repository. + +This fixture provides a dry, loop-oriented full-length vocal source plus instrument loops, not four +rendered full-length canonical stems. Dry-vocal correlation cannot establish recording identity, so +the separately pinned finished master is used only for the YouTube identity check. The benchmark +therefore makes a quantitative claim only about vocal isolation. It separately checks that Demucs +still returns finite, equal-length +`vocals`/`bass`/`drums`/`other` arrays and that `vocals` is the best named match for the reference. + +## Evaluation contract + +1. Download the YouTube audio through the same Python downloader used by BandScope. +2. Fetch the source archive and finished master over verified HTTPS into pytest's private + `tmp_path`. +3. Require exact hosts, byte counts, and full SHA-256 values for the archive, extracted vocal WAV, + and finished master before accepting the references. +4. Load the YouTube mix, creator master, and vocal reference at mono 44.1 kHz. +5. Reject fixture drift when YouTube/master decoded duration differs by more than `1.0 s`. +6. Estimate a global YouTube-to-master lag and require aligned identity correlation ≥ `0.90`. +7. Estimate a separate global master-to-vocal lag, compose the two offsets once, and select the + strongest 12-second vocal window. Predicted stems are never aligned independently. +8. Run real `htdemucs` separation with deterministic `shifts=0` on the selected mixture excerpt. +9. Provisionally require vocal SI-SDR improvement over the unseparated mixture of at least + `+0.5 dB`. +10. Require the named vocal output to beat the best wrong stem by at least `3.0 dB` SI-SDR. + +The metric is zero-mean scale-invariant signal-to-distortion ratio (SI-SDR). The improvement score +is `SI-SDR(separated vocal, reference) - SI-SDR(downloaded mix, reference)`, so the gate measures +whether separation improves over returning the transcoded YouTube mixture unchanged. Silent and +non-finite inputs fail instead of receiving an artificial finite score. + +The +0.5/+3.0 dB values are provisional sentinels, not industry standards. On the pinned creator +master, deterministic `shifts=0` produced +1.752 dB SI-SDR improvement and +7.631 dB assignment +margin. The previous dry-vocal/mix correlation was only 0.016856, which is why it is no longer an +identity gate. An authorized YouTube run must calibrate the final release threshold; this offline +creator-master probe is not a live pass. + +## Running the benchmark + +Install the analysis-engine development dependencies and ensure `ffmpeg` is on `PATH`. The first +Demucs run may obtain model weights through Demucs unless they are already present in its cache. +Prefer a pre-provisioned, integrity-verified model cache for repeatable runs. + +The exact current model artifact is Demucs 4.0.1 htdemucs signature `955717e8`, file +`955717e8-8726e21a.th`, 84,141,911 bytes, full SHA-256 +`8726e21a993978c7ba086d3872e7608d7d5bfca646ca4aca459ffda844faa8b4`. It is runtime-fetched and +not bundled. Demucs currently enforces only the filename's eight-hex hash prefix before load; +ADR-0001 therefore treats BandScope-owned full-hash pre-load enforcement and a model-rights decision +as release blockers. + +Before enabling the test, the operator must confirm that the intended use is permitted by the +content rightsholder and the applicable YouTube terms. The creator's permission for the reference +source does not by itself grant permission for automated access to YouTube. + +```bash +UV_CACHE_DIR=/tmp/bandscope-uv-cache \ +BANDSCOPE_RUN_YOUTUBE_STEM_E2E=1 \ +uv run --project services/analysis-engine \ + pytest services/analysis-engine/tests/test_youtube_stem_e2e.py \ + -m youtube_stem_e2e -vv +``` + +If YouTube access, either fixed reference asset, `ffmpeg`, or model weights are unavailable, the +opted-in test fails. It must not silently turn an unavailable or changed fixture into a passing +result. + +## Platform and evidence status + +- Linux x86_64: controlled CPU evidence supported. +- Windows amd64/arm64 and macOS arm64: dependency markers permit Demucs, but this benchmark has not + recorded exact-platform passing evidence. +- macOS Intel: current dependency markers exclude Demucs; separation must fail safely and offer the + product fallback. + +On 2026-08-09, exact commit `5a3648a11d9097b8da48bb4a3ccbd97986aec25b` passed all 13 then-current default +offline cases. An explicit live attempt authenticated and extracted the pinned reference archive, +then failed in the production YouTube downloader with HTTP 502 before separation. It produced no +correlation or SI-SDR score and is recorded as failure evidence, not a live pass. See +`docs/documentation-coverage-matrix.md`. + +The corrected branch now has 16 offline known-stem contract cases, including exact extracted-member +hash, creator-master authentication, composed-offset recovery, and explicit required-CI exclusion of +the live marker. A creator-master-only calibration produced the provisional scores above without +calling YouTube; it is calibration evidence, not exact-candidate success. + +## Security Notes + +### Attack surface + +The opt-in test crosses three public HTTPS download boundaries, decodes untrusted audio/ZIP data, +writes temporary files, invokes the existing `ffmpeg` yt-dlp postprocessor, and loads the existing +Demucs model. + +### Trust boundary + +YouTube media, yt-dlp metadata, the public source archive, finished master, ZIP metadata, audio +decoder input, and model weights are outside the repository trust boundary. The pytest `tmp_path` is +the only permitted storage root for downloaded media and extracted references. + +### Realistic threats + +- Fixture replacement, redirect, truncation, or a ZIP bomb could substitute malicious or misleading + decoder input. +- A changed YouTube transcode or different recording could make an unrelated signal look like a + separator regression. +- Login cookies, geo/DRM bypasses, or automated CI execution could expand legal, privacy, and account + risk. +- Decoder/model vulnerabilities and first-run model downloads remain upstream supply-chain risks. + +### Mitigations + +- The live case requires the distinct `BANDSCOPE_RUN_YOUTUBE_STEM_E2E=1` opt-in and is excluded from + default CI. +- The initial reference URL and every redirect target must use HTTPS on the exact allowlisted host; + redirect targets are validated before their follow-up request. TLS verification is never + disabled. +- The source archive is bounded by exact size and SHA-256; extraction accepts one exact target + member, rejects a missing/duplicate/encrypted target, enforces its exact uncompressed size and + full SHA-256, and ignores every non-target entry. It never calls `extractall()`. The finished + master is independently pinned by exact host, byte count, and full SHA-256. +- The production YouTube downloader keeps its standard-URL allowlist, duration/size bounds, + `noplaylist`, and no-geo-bypass policy. This test adds no cookies, credentials, login, paywall, + DRM, or bot-evasion behavior. TLS validation stays enabled while yt-dlp uses the operating + system's managed CA trust store rather than a separate certifi-only bundle. +- Alignment is global and bounded. Duration and creator-master identity correlation distinguish + fixture drift from model quality failure; the two lags are composed once and model outputs are not + optimized after separation. Demucs random shift augmentation is disabled with `shifts=0`. +- Raw audio and full paths are not logged or committed. A nested temporary directory explicitly + deletes the reference, YouTube media, and scored WAV on both success and failure. Numeric scores + and stable public fixture IDs are sufficient diagnostics. + +### Test points + +Offline tests cover SI-SDR behavior, invalid/silent signals, delayed/composed-window recovery, +archive/member/master authentication, ignored non-target/path-traversal entries, hash mismatch, +pre-request redirect rejection, member-size drift, deterministic Demucs invocation, and required-CI +marker exclusion. The live case covers the production downloader, real decoding, real Demucs output +shape/finiteness, SI-SDR improvement, and fixed-name assignment. + +### Remaining risk + +YouTube availability and transcoding are mutable, the informal source-pack permission is not legal +advice, and the test does not establish platform authorization. Upstream media decoders and Demucs +weights remain separate trust decisions. The fixture has only one full-length known canonical stem, +so the test cannot claim quantitative four-stem accuracy. + +The model-weight redistribution license is not established, current Demucs verification uses only a +hash prefix, and no successful exact-candidate live score or supported-platform matrix has yet been +retained. These remain explicit release blockers rather than undocumented assumptions. + +## References + +- Brad Sucks. (2004, May 3). *Making Me Nervous source*. + https://www.bradsucks.net/news/archives/2004/05/03/making-me-nervous-source +- Le Roux, J., Wisdom, S., Erdogan, H., & Hershey, J. R. (2019). SDR—Half-baked or well done? In + *ICASSP 2019—2019 IEEE International Conference on Acoustics, Speech and Signal Processing* + (pp. 626–630). IEEE. https://doi.org/10.1109/ICASSP.2019.8683855 +- YouTube. (n.d.). *Terms of Service*. https://www.youtube.com/static?template=terms +- Rouard, S., Stoller, D., & Défossez, A. (2023). Hybrid transformers for music source + separation. In *ICASSP 2023—2023 IEEE International Conference on Acoustics, Speech and + Signal Processing*. IEEE. https://arxiv.org/abs/2211.08553 diff --git a/docs/operations/deploy-runbook.md b/docs/operations/deploy-runbook.md index b9dd2ca99..150ef531a 100644 --- a/docs/operations/deploy-runbook.md +++ b/docs/operations/deploy-runbook.md @@ -24,6 +24,44 @@ When runtime behavior is touched, verify: 2. no new high vulnerabilities are introduced (`npm audit --workspaces --audit-level=high`) 3. policy checks for supply chain/security gates pass +## Source-separation preflight and evidence + +For a release candidate that claims YouTube source separation: + +1. record exact commit, live base tip, lockfiles, OS, architecture, Python, Demucs, torch, yt-dlp, + and `ffmpeg -version`; +2. confirm content/platform authorization and do not provide cookies, credentials, login, paywall, + DRM, geo, or anti-bot bypasses; +3. verify the htdemucs model's exact source, 84,141,911-byte size, and full SHA-256 from the + supplemental inventory before load; fail closed on cache symlink, mismatch, or missing artifact; +4. authenticate the archive, extracted vocal member, and finished master by exact host, byte count, + and full SHA-256; record the master duration and require deterministic Demucs `shifts=0`; +5. run the offline known-stem contract, then the explicit live command from + `docs/engineering/youtube-known-stem-validation.md` on the unchanged candidate; +6. retain bounded numeric/provenance evidence only: duration drift, identity correlation, composed + lags, baseline/vocal SI-SDR, improvement, assignment margin, outcome code, and cleanup result; +7. verify the temporary media root is empty and no raw audio, archive content, full path, URL, + cookie, credential, or provider response was retained. + +The live lane needs a 20-minute operator timeout until calibration establishes a tighter limit. A +provider 5xx may receive one clean rerun only when current evidence supports transience. Otherwise +classify the first failing boundary and continue unrelated repository work; never convert failure to +skip/pass. + +### Triage and rollback + +- Integrity/member mismatch: quarantine/delete the cache or temp artifact and investigate source + drift before another load. +- Correlation failure: treat as YouTube/reference fixture drift before diagnosing the model. +- Finite/shape/threshold failure: treat as separator correctness or model-version regression. +- Platform import failure: surface the supported local-file/fallback state; do not install an + unreviewed wheel or model. +- Rollback removes release-blocking/live scheduling and restores the previous exact approved model; + it does not restore the retired FFT profile or weaken intake/security tests. + +Evidence artifacts expire after 30 days unless release governance approves a different TTL. Raw +media is never an evidence artifact. + ## Incident handling note If required workflows fail due to repository-controlled code/configuration, treat as `FAILED` and remediate in code. Use `BLOCKED` only for external permission/platform limitations. diff --git a/docs/plans/2026-03-28-ml-engine-integration.md b/docs/plans/2026-03-28-ml-engine-integration.md index c9d34b6f8..bc49b15c4 100644 --- a/docs/plans/2026-03-28-ml-engine-integration.md +++ b/docs/plans/2026-03-28-ml-engine-integration.md @@ -12,10 +12,15 @@ This document outlines the MECE execution strategy to incrementally substitute m - **Tech**: Add `librosa` or `soundfile` for robust decoding. - **Output**: Real file ingestion and tempo/beat arrays. -### Track 2: Spectral & Stem Separation (#106) +### Track 2: Spectral & Stem Separation (#106) (IMPLEMENTED; RELEASE EVIDENCE OPEN) - **Goal**: Deconstruct the mixed audio into isolated stems. -- **Tech**: Integrate `demucs` (or a smaller alternative) running locally. -- **Output**: 4 or 6 discrete stems (vocals, bass, drums, other). +- **Tech**: Demucs 4.0.1 `htdemucs` running locally on CPU after model provisioning. +- **Output**: 4 discrete stems (vocals, bass, drums, other). +- **Validity**: The active known-stem branch adds production-path vocal SI-SDR improvement and stem + assignment checks; see `docs/PRD.md`, `docs/TRD.md`, and ADR-0002. +- **Open release blockers**: full-SHA model verification before deserialization, model-rights + decision, successful exact-candidate live evidence, threshold calibration, and supported-platform + proof. ### Track 3: Harmonic & Pitch Pipelines (#107) (COMPLETED) @@ -42,12 +47,16 @@ The integration of ML libraries like `librosa`, `torch`, and `demucs` exposes th The primary trust boundary is between the user's filesystem (audio files) and the Python local analysis engine. All input audio is untrusted. ### Mitigations -We will restrict audio ingestion through `librosa`/`soundfile` using strict format constraints. We will execute ML tasks locally, without reaching out to external networks, and run them under low privileges where possible. +We restrict audio ingestion through `librosa`/`soundfile` using strict format constraints. Model +inference runs locally and under low privilege where possible. First use is not currently fully +offline: Demucs may retrieve the exact inventoried model into a user cache. Offline execution is +guaranteed only after trusted provisioning. The release target requires full-SHA verification before +deserialization; see ADR-0001. ### Test Points - Loading truncated or corrupted WAV/MP3 files. - Providing extremely large audio files to test OOM behavior. -- Validating that no external network calls occur during offline ML processing. +- Validating that no external network calls occur after trusted model-cache provisioning. ### Realistic Threats - OOM (Out Of Memory) crashing the user's host OS during `demucs` execution. diff --git a/docs/release/release-policy.md b/docs/release/release-policy.md index 924f8f364..18d170a40 100644 --- a/docs/release/release-policy.md +++ b/docs/release/release-policy.md @@ -25,3 +25,19 @@ BandScope distributes release artifacts through GitHub Releases. - release workflows must not attach assets after a GitHub Release is already published - release artifacts must remain traceable to the GitHub Release record - missing SBOM or missing supplemental inventory means the release baseline is incomplete + +## Source-separation release evidence + +- The deterministic known-stem contract is required for every change that touches YouTube intake, + decode, separation, alignment, metrics, model delivery, or fixture metadata. +- Live known-stem evidence is advisory while ADR-0002 is Proposed. It becomes blocking only through + a superseding/accepted ADR after authorization, full-hash pre-load model verification, calibrated + thresholds, supported-platform evidence, and a stable bounded evidence artifact exist. +- A release must not advertise verified source-separation quality unless the exact integrated + release candidate records a passing live production-path run. A skipped, provider-failed, stale, + or predecessor-head result does not transfer. +- Release artifacts must identify the exact htdemucs signature/hash and whether weights are bundled, + pre-provisioned, or runtime-fetched. Current policy permits runtime cache retrieval only; it does + not authorize model-weight redistribution. +- Release rollback must preserve deterministic metric/security coverage and remove any invalid + quality claim, scheduled live access, or unverified model artifact. diff --git a/docs/security/dependency-policy.md b/docs/security/dependency-policy.md index f7271e68d..662981d3d 100644 --- a/docs/security/dependency-policy.md +++ b/docs/security/dependency-policy.md @@ -109,7 +109,12 @@ Current controlled exceptions: Retired third-party deprecation and advisory signal: - `proc-macro-hack v0.5.20+deprecated`, `RUSTSEC-2025-0057` for `fxhash`, and `RUSTSEC-2026-0097` for legacy `rand 0.7.3` were removed by a compatible Tauri lockfile refresh that moved `tauri` to `2.11.0` and `tauri-utils` to `2.9.0`, dropping the `kuchikiki`/`selectors`/`phf 0.8` owner chain. Do not reintroduce this chain or restore the `RUSTSEC-2026-0097` Cargo audit exception; `scripts/checks/verify_supply_chain.py` rejects any future `rand 0.7.x` lockfile entry. -- `GHSA-53q9-r3pm-6pq6` (`torch.load` RCE, fixed in torch 2.6) is allowed only for `torch 2.2.2` in `services/analysis-engine`: torch 2.2.2 is the last release publishing macOS Intel (x86_64) wheels, and the cross-platform build policy mandates macOS Intel + arm64. The vulnerable API only ever loads demucs's pinned model weights (bundled/checksum-tracked per this policy); user-supplied audio never reaches `torch.load`. The exception is encoded in `.github/workflows/dependency-review.yml` (`allow-ghsas`) and `services/analysis-engine/osv-scanner.toml`, and must be removed when the engine migrates off torch (e.g. ONNX runtime) or the Intel-mac mandate changes. +- The former `GHSA-53q9-r3pm-6pq6` exception for torch 2.2.2 is retired. The current lock resolves + torch 2.12.1 on supported Linux and the Demucs dependency marker excludes macOS Intel rather than + retaining the vulnerable torch build. No repo-local dependency-review allowlist or analysis-engine + OSV exception for that advisory is active. Do not restore either stale exception. Separately, + ADR-0001 requires full-SHA verification of the exact htdemucs artifact before any torch checkpoint + deserialization can qualify as release-ready. - Yanked `fastrand 2.4.0` was transiently inherited through target-specific `wry`/`dom_query` HTML parsing dependencies and must stay updated to `2.4.1` or newer in `apps/desktop/src-tauri/Cargo.lock`; `scripts/checks/verify_supply_chain.py` guards against reintroducing the yanked version. ## Required checks intent diff --git a/scripts/checks/run_root_tests.mjs b/scripts/checks/run_root_tests.mjs index 156de8379..440a40907 100644 --- a/scripts/checks/run_root_tests.mjs +++ b/scripts/checks/run_root_tests.mjs @@ -66,6 +66,8 @@ runPython([ "scripts/checks/run_analysis_command.py", "pytest", "tests", + "-m", + "not youtube_stem_e2e", "--cov=src/bandscope_analysis", "--cov-report=term-missing", "--cov-fail-under=100", diff --git a/scripts/checks/verify_docs.py b/scripts/checks/verify_docs.py index 850921591..2ecfea178 100644 --- a/scripts/checks/verify_docs.py +++ b/scripts/checks/verify_docs.py @@ -15,6 +15,16 @@ Path("docs/repository/bootstrap-plan.md"), Path("docs/repository/gitflow.md"), Path("docs/architecture/overview.md"), + Path("docs/architecture/diagrams.md"), + Path("docs/README.md"), + Path("docs/PRD.md"), + Path("docs/TRD.md"), + Path("docs/adr/README.md"), + Path("docs/adr/0001-source-separation-runtime-and-model-delivery.md"), + Path("docs/adr/0002-known-stem-youtube-quality-gate.md"), + Path("docs/adr/0003-ephemeral-benchmark-evidence-model.md"), + Path("docs/documentation-coverage-matrix.md"), + Path("docs/doctoring/real-audio-accuracy-acceptance.md"), Path("docs/i18n/i18n-policy.md"), Path("docs/release/release-policy.md"), Path(".github/CODEOWNERS"), @@ -57,29 +67,45 @@ "docs/security/dependency-policy.md", "docs/security/cross-platform-build-policy.md", "docs/workflow/github-bootstrap-execution-policy.md", + "docs/PRD.md", + "docs/TRD.md", + "docs/adr/README.md", + "docs/architecture/diagrams.md", + ], + Path("docs/README.md"): [ + "docs/PRD.md", + "docs/TRD.md", + "docs/adr/README.md", + "docs/architecture/diagrams.md", + "docs/documentation-coverage-matrix.md", ], } -def main() -> int: - """Return a failing exit code when required docs or references are missing.""" - missing = [str(path) for path in REQUIRED_PATHS if not path.exists()] - if missing: - print("Missing required docs:") - for path in missing: - print(f"- {path}") - return 1 - broken_refs: list[str] = [] +def documentation_violations(root: Path = Path(".")) -> list[str]: + """Return missing canonical files and broken authority-reference violations.""" + violations = [ + f"missing file: {path}" for path in REQUIRED_PATHS if not (root / path).exists() + ] for path, required_texts in REQUIRED_REFERENCES.items(): - content = path.read_text(encoding="utf-8") + absolute_path = root / path + if not absolute_path.exists(): + continue + content = absolute_path.read_text(encoding="utf-8") for required_text in required_texts: if required_text not in content: - broken_refs.append(f"{path} missing reference: {required_text}") + violations.append(f"{path} missing reference: {required_text}") + return violations + + +def main() -> int: + """Return a failing exit code when required docs or references are missing.""" + violations = documentation_violations() - if broken_refs: - print("Missing required doc references:") - for item in broken_refs: - print(f"- {item}") + if violations: + print("Documentation check failed:") + for violation in violations: + print(f"- {violation}") return 1 print("Documentation check passed") diff --git a/scripts/checks/verify_supply_chain.py b/scripts/checks/verify_supply_chain.py index 1cd561e5c..509396aac 100644 --- a/scripts/checks/verify_supply_chain.py +++ b/scripts/checks/verify_supply_chain.py @@ -1,6 +1,7 @@ """Verify that repository-controlled supply-chain controls stay in place.""" import functools +import json import re import shlex from datetime import date @@ -37,6 +38,92 @@ Path("supply-chain/supplemental-component-inventory.json"), ] +SUPPLEMENTAL_INVENTORY_PATH = Path("supply-chain/supplemental-component-inventory.json") +SEPARATOR_IMPLEMENTATION_PATH = Path( + "services/analysis-engine/src/bandscope_analysis/separation/audio_separator.py" +) +FULL_SHA256_PATTERN = re.compile(r"^sha256:[0-9a-f]{64}$") +RUNTIME_MODEL_PATTERN = re.compile(r'model_name:\s*str\s*=\s*"([^"]+)"') +REQUIRED_MODEL_ARTIFACT_FIELDS = { + "name", + "runtimeModelName", + "version", + "sourceUrl", + "license", + "checksum", + "sizeBytes", + "storagePath", + "distribution", + "releaseUsage", + "verification", +} + + +def supplemental_inventory_violations( + inventory_path: Path = SUPPLEMENTAL_INVENTORY_PATH, + separator_path: Path = SEPARATOR_IMPLEMENTATION_PATH, +) -> list[str]: + """Return stale, incomplete, or runtime-mismatched model inventory violations.""" + violations: list[str] = [] + try: + inventory = json.loads(inventory_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + return [f"supplemental inventory is unreadable: {error.__class__.__name__}"] + try: + separator_source = separator_path.read_text(encoding="utf-8") + except OSError as error: + return [f"separator implementation is unreadable: {error.__class__.__name__}"] + + runtime_match = RUNTIME_MODEL_PATTERN.search(separator_source) + if runtime_match is None: + return ["separator implementation does not declare a runtime model"] + runtime_model = runtime_match.group(1) + artifacts = inventory.get("modelArtifacts") + if not isinstance(artifacts, list): + return ["supplemental inventory modelArtifacts must be a list"] + + matching_runtime_artifacts: list[dict[str, object]] = [] + for artifact in artifacts: + if not isinstance(artifact, dict): + violations.append("supplemental inventory model artifact must be an object") + continue + name = str(artifact.get("name", "")) + if name.startswith("bandsplit-"): + violations.append(f"supplemental inventory contains retired model: {name}") + if artifact.get("runtimeModelName") == runtime_model: + matching_runtime_artifacts.append(artifact) + + if not matching_runtime_artifacts: + violations.append( + f"supplemental inventory missing runtime model: {runtime_model}" + ) + return violations + + for artifact in matching_runtime_artifacts: + missing_fields = sorted(REQUIRED_MODEL_ARTIFACT_FIELDS - artifact.keys()) + if missing_fields: + violations.append( + f"supplemental inventory runtime model {runtime_model} missing fields: " + + ", ".join(missing_fields) + ) + checksum = artifact.get("checksum") + if not isinstance(checksum, str) or not FULL_SHA256_PATTERN.fullmatch(checksum): + violations.append( + f"supplemental inventory runtime model {runtime_model} requires full SHA-256" + ) + source_url = artifact.get("sourceUrl") + if not isinstance(source_url, str) or not source_url.startswith("https://"): + violations.append( + f"supplemental inventory runtime model {runtime_model} requires HTTPS source" + ) + size_bytes = artifact.get("sizeBytes") + if not isinstance(size_bytes, int) or size_bytes <= 0: + violations.append( + f"supplemental inventory runtime model {runtime_model} requires positive sizeBytes" + ) + return violations + + PINNED_ACTION = re.compile(r"^\s*-?\s*uses:\s+[^@\s]+@[0-9a-f]{40}(\s+#.*)?$") USES_ACTION = re.compile(r"^\s*-?\s*uses:\s+") LOCAL_ACTION = re.compile(r"^\s*-?\s*uses:\s+\./") @@ -95,8 +182,12 @@ f"{TRUSTED_SCORECARD_SCRIPTS_DIR}/{OSSF_SARIF_NORMALIZER}", } RELEASE_ARTIFACT_GLOB = re.compile(r"(?:^|\s)artifacts/\*") -RELEASE_ASSET_VALIDATOR = "scripts/release/select_release_assets.py --output release-assets.txt" -RELEASE_ASSET_REVALIDATOR = "scripts/release/select_release_assets.py --input release-assets.txt" +RELEASE_ASSET_VALIDATOR = ( + "scripts/release/select_release_assets.py --output release-assets.txt" +) +RELEASE_ASSET_REVALIDATOR = ( + "scripts/release/select_release_assets.py --input release-assets.txt" +) RELEASE_ASSET_MAPFILE = "mapfile -t release_assets < release-assets.txt" WORKSPACE_EXEC_PATTERN = re.compile(r"\bnpm\s+exec\s+--workspace\b") RUST_RAND_ADVISORY_ID = "GHSA-cq8v-f236-94qc" @@ -198,18 +289,18 @@ def workflow_job_content_for_step(lines: list[str], line_index: int) -> str: for reverse_index in range(line_index, -1, -1): candidate = lines[reverse_index] candidate_without_comment = candidate.strip().partition("#")[0].strip() - if len(candidate) - len(candidate.lstrip(" ")) == 2 and candidate_without_comment.endswith( - ":" - ): + if len(candidate) - len( + candidate.lstrip(" ") + ) == 2 and candidate_without_comment.endswith(":"): job_start = reverse_index break job_end = len(lines) for forward_index in range(job_start + 1, len(lines)): candidate = lines[forward_index] candidate_without_comment = candidate.strip().partition("#")[0].strip() - if len(candidate) - len(candidate.lstrip(" ")) == 2 and candidate_without_comment.endswith( - ":" - ): + if len(candidate) - len( + candidate.lstrip(" ") + ) == 2 and candidate_without_comment.endswith(":"): job_end = forward_index break return "\n".join(lines[job_start:job_end]) @@ -231,7 +322,9 @@ def step_run_command_from_block(step_lines: list[str], step_indent: int) -> str: if stripped.startswith("run:") and (indent > step_indent or is_step_start): run_indent = indent run_value = stripped.partition(":")[2].strip() - command_lines.append("" if run_value in {"|", "|-", ">", ">-"} else run_value) + command_lines.append( + "" if run_value in {"|", "|-", ">", ">-"} else run_value + ) continue stripped = "" if raw_stripped.startswith("#") else raw_stripped if stripped and indent <= run_indent: @@ -260,7 +353,9 @@ def workflow_run_steps(content: str) -> list[WorkflowRunStep]: return run_steps -def step_with_value_from_block(step_lines: list[str], step_indent: int, key: str) -> str | None: +def step_with_value_from_block( + step_lines: list[str], step_indent: int, key: str +) -> str | None: """Return a workflow step ``with`` value for ``key`` when scoped under with.""" with_indent: int | None = None key_pattern = re.compile(rf"^\s*{re.escape(key)}\s*:\s*(?P.*?)\s*$") @@ -303,7 +398,9 @@ def step_env_from_block(step_lines: list[str], step_indent: int) -> dict[str, st return env -def step_scalar_value_from_block(step_lines: list[str], step_indent: int, key: str) -> str | None: +def step_scalar_value_from_block( + step_lines: list[str], step_indent: int, key: str +) -> str | None: """Return a simple top-level scalar value from a workflow step block.""" for step_line in step_lines: stripped = step_line.partition("#")[0].strip() @@ -319,7 +416,9 @@ def step_scalar_value_from_block(step_lines: list[str], step_indent: int, key: s def step_is_blocking(step_lines: list[str], step_indent: int) -> bool: """Return whether a workflow step should block when its command fails.""" - continue_on_error = step_scalar_value_from_block(step_lines, step_indent, "continue-on-error") + continue_on_error = step_scalar_value_from_block( + step_lines, step_indent, "continue-on-error" + ) if continue_on_error is None: return True normalized = re.sub(r"\s+", "", continue_on_error.casefold()) @@ -385,7 +484,9 @@ def nested_shell_commands(tokens: list[str]) -> list[str]: for option_index in range(index + 1, len(tokens)): option = tokens[option_index] if option == "-c" or ( - option.startswith("-") and not option.startswith("--") and "c" in option[1:] + option.startswith("-") + and not option.startswith("--") + and "c" in option[1:] ): if option_index + 1 < len(tokens): nested_commands.append(tokens[option_index + 1]) @@ -527,7 +628,9 @@ def command_contains_token_sequence( return False -def executed_command_token_lists(tokens: list[str], *, recursion_depth: int = 0) -> list[list[str]]: +def executed_command_token_lists( + tokens: list[str], *, recursion_depth: int = 0 +) -> list[list[str]]: """Return tokenized commands after unwrapping allowed command wrappers.""" tokens = strip_shell_assignment_prefix(tokens) if not tokens: @@ -706,10 +809,16 @@ def verify_pinned_actions() -> list[str]: Path(".github/workflows").glob("*.yaml") ) for path in workflow_paths: - for idx, line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1): + for idx, line in enumerate( + path.read_text(encoding="utf-8").splitlines(), start=1 + ): if USES_ACTION.match(line) is None: continue - if PINNED_ACTION.match(line) or LOCAL_ACTION.match(line) or DOCKER_ACTION.match(line): + if ( + PINNED_ACTION.match(line) + or LOCAL_ACTION.match(line) + or DOCKER_ACTION.match(line) + ): continue violations.append( f"{repo_display_path(path)}:{idx} -> workflow action must be pinned by SHA" @@ -730,7 +839,9 @@ def workflow_top_level_env(content: str) -> dict[str, str]: env_line_without_comment = env_line.partition("#")[0].rstrip() if not env_line_without_comment.strip(): continue - indent = len(env_line_without_comment) - len(env_line_without_comment.lstrip(" ")) + indent = len(env_line_without_comment) - len( + env_line_without_comment.lstrip(" ") + ) if indent == 0: break if child_indent is None: @@ -767,13 +878,20 @@ def workflow_top_level_key_lines(content: str, keys: set[str]) -> list[tuple[int def workflow_publishes_scorecard_results(content: str) -> bool: """Return whether a workflow publishes OSSF Scorecard results.""" workflow_body = "\n".join(line.partition("#")[0] for line in content.splitlines()) - return "ossf/scorecard-action" in workflow_body and "publish_results:" in workflow_body + return ( + "ossf/scorecard-action" in workflow_body and "publish_results:" in workflow_body + ) -def checkout_step_has_default_branch_guard(step_lines: list[str], step_indent: int) -> bool: +def checkout_step_has_default_branch_guard( + step_lines: list[str], step_indent: int +) -> bool: """Return whether a checkout step carries the Git default branch env guard.""" env = step_env_from_block(step_lines, step_indent) - return all(env.get(key) == value for key, value in CHECKOUT_DEFAULT_BRANCH_GUARD_ENV.items()) + return all( + env.get(key) == value + for key, value in CHECKOUT_DEFAULT_BRANCH_GUARD_ENV.items() + ) def verify_checkout_default_branch_guard() -> list[str]: @@ -786,14 +904,17 @@ def verify_checkout_default_branch_guard() -> list[str]: for path in workflow_paths: content = path.read_text(encoding="utf-8") has_checkout = any( - checkout_uses_pattern.search(line.partition("#")[0]) for line in content.splitlines() + checkout_uses_pattern.search(line.partition("#")[0]) + for line in content.splitlines() ) if not has_checkout: continue if workflow_publishes_scorecard_results(content): checkout_steps = [ (step_indent, step_lines) - for _, step_indent, step_lines in workflow_step_blocks(content.splitlines()) + for _, step_indent, step_lines in workflow_step_blocks( + content.splitlines() + ) if any( checkout_uses_pattern.search(step_line.partition("#")[0]) for step_line in step_lines @@ -809,9 +930,14 @@ def verify_checkout_default_branch_guard() -> list[str]: ) continue env = workflow_top_level_env(content) - if all(env.get(key) == value for key, value in CHECKOUT_DEFAULT_BRANCH_GUARD_ENV.items()): + if all( + env.get(key) == value + for key, value in CHECKOUT_DEFAULT_BRANCH_GUARD_ENV.items() + ): continue - violations.append(f"{repo_display_path(path)}: {CHECKOUT_DEFAULT_BRANCH_GUARD_VIOLATION}") + violations.append( + f"{repo_display_path(path)}: {CHECKOUT_DEFAULT_BRANCH_GUARD_VIOLATION}" + ) return violations @@ -877,7 +1003,9 @@ def evaluate_job(job_lines: list[str], start_line: int) -> None: ) if workflow_publishes_scorecard_results(content): - for line_number, _ in workflow_top_level_key_lines(content, {"env", "defaults"}): + for line_number, _ in workflow_top_level_key_lines( + content, {"env", "defaults"} + ): if path is None: violations.append(OSSF_PUBLISH_GLOBAL_CONFIG_VIOLATION) else: @@ -1025,7 +1153,9 @@ def workflow_job_step_blocks(line_index: int) -> list[tuple[int, int, list[str]] def scorecard_artifact_download_decompression_violations(content: str) -> list[str]: """Return Scorecard downloads that rely on action-owned ZIP decompression.""" - content_without_comments = "\n".join(line.partition("#")[0] for line in content.splitlines()) + content_without_comments = "\n".join( + line.partition("#")[0] for line in content.splitlines() + ) if "actions/download-artifact" not in content_without_comments: return [] if "ossf-scorecard-results" not in content_without_comments: @@ -1057,7 +1187,10 @@ def invokes_scorecard_extractor(command: str) -> bool: continue if "ossf-scorecard-results" not in step_content: continue - if step_with_value_from_block(step_lines, block_indent, "skip-decompress") != "true": + if ( + step_with_value_from_block(step_lines, block_indent, "skip-decompress") + != "true" + ): violations.append(OSSF_DOWNLOAD_DECOMPRESSION_VIOLATION) continue @@ -1086,7 +1219,10 @@ def invokes_scorecard_extractor(command: str) -> bool: ( position for position, (block_indent, block_lines) in enumerate(later_steps) - if (OSSF_SARIF_NORMALIZER in step_run_command_from_block(block_lines, block_indent)) + if ( + OSSF_SARIF_NORMALIZER + in step_run_command_from_block(block_lines, block_indent) + ) ), None, ) @@ -1106,7 +1242,9 @@ def invokes_scorecard_extractor(command: str) -> bool: def release_artifact_download_decompression_violations(content: str) -> list[str]: """Return release downloads that rely on action-owned ZIP decompression.""" - content_without_comments = "\n".join(line.partition("#")[0] for line in content.splitlines()) + content_without_comments = "\n".join( + line.partition("#")[0] for line in content.splitlines() + ) if "actions/download-artifact" not in content_without_comments: return [] if "bandscope-*-${{ github.sha }}" not in content_without_comments: @@ -1141,7 +1279,10 @@ def is_blocking_required_step(block_lines: list[str], block_indent: int) -> bool continue if "bandscope-*-${{ github.sha }}" not in step_content: continue - if step_with_value_from_block(step_lines, block_indent, "skip-decompress") != "true": + if ( + step_with_value_from_block(step_lines, block_indent, "skip-decompress") + != "true" + ): violations.append(RELEASE_DOWNLOAD_DECOMPRESSION_VIOLATION) continue @@ -1160,7 +1301,9 @@ def is_blocking_required_step(block_lines: list[str], block_indent: int) -> bool ( position for position, (block_indent, block_lines) in enumerate(later_steps) - if invokes_release_extractor(step_run_command_from_block(block_lines, block_indent)) + if invokes_release_extractor( + step_run_command_from_block(block_lines, block_indent) + ) and is_blocking_required_step(block_lines, block_indent) ), None, @@ -1217,7 +1360,9 @@ def _verify_dependency_review_coverage(missing: list[str]) -> None: def _verify_security_audit_coverage(missing: list[str]) -> None: - audit = read_workflow(Path(".github/workflows/security-audit.yml"), "security audit", missing) + audit = read_workflow( + Path(".github/workflows/security-audit.yml"), "security audit", missing + ) for token in ["develop", "main", "pull_request", "push"]: if audit and token not in audit: missing.append(f"security audit workflow missing trigger token: {token}") @@ -1235,9 +1380,12 @@ def _verify_security_audit_coverage(missing: list[str]) -> None: "cargo +stable audit", ]: if audit and not any( - command_contains_token_sequence(command, token) for command in audit_run_commands + command_contains_token_sequence(command, token) + for command in audit_run_commands ): - missing.append(f"security audit workflow missing vulnerability audit token: {token}") + missing.append( + f"security audit workflow missing vulnerability audit token: {token}" + ) def _verify_codeql_coverage(missing: list[str]) -> None: @@ -1273,7 +1421,9 @@ def _verify_secret_scan_coverage(missing: list[str]) -> None: def _verify_build_coverage(missing: list[str]) -> None: - build = read_workflow(Path(".github/workflows/build-baseline.yml"), "build baseline", missing) + build = read_workflow( + Path(".github/workflows/build-baseline.yml"), "build baseline", missing + ) for token in [ "develop", "main", @@ -1301,9 +1451,13 @@ def _verify_build_coverage(missing: list[str]) -> None: if build and token not in build: missing.append(f"build workflow missing token: {token}") if build and "windows-latest" in build: - missing.append("build workflow should not rely on windows-latest for architecture coverage") + missing.append( + "build workflow should not rely on windows-latest for architecture coverage" + ) if build and "macos-latest" in build: - missing.append("build workflow should not rely on macos-latest for architecture coverage") + missing.append( + "build workflow should not rely on macos-latest for architecture coverage" + ) def _verify_scorecard_coverage(missing: list[str], workflow_paths: list[Path]) -> None: @@ -1340,10 +1494,16 @@ def _verify_scorecard_coverage(missing: list[str], workflow_paths: list[Path]) - ) for workflow_path in workflow_paths: workflow_content = workflow_path.read_text(encoding="utf-8") - missing.extend(scorecard_sarif_upload_normalization_violations(workflow_content)) - missing.extend(scorecard_artifact_download_decompression_violations(workflow_content)) missing.extend( - ossf_scorecard_publish_restriction_violations(workflow_content, workflow_path) + scorecard_sarif_upload_normalization_violations(workflow_content) + ) + missing.extend( + scorecard_artifact_download_decompression_violations(workflow_content) + ) + missing.extend( + ossf_scorecard_publish_restriction_violations( + workflow_content, workflow_path + ) ) @@ -1363,7 +1523,9 @@ def verify_workflow_coverage() -> list[str]: ) for workflow_path in workflow_paths: workflow_content = workflow_path.read_text(encoding="utf-8") - missing.extend(release_artifact_download_decompression_violations(workflow_content)) + missing.extend( + release_artifact_download_decompression_violations(workflow_content) + ) _verify_scorecard_coverage(missing, workflow_paths) @@ -1463,11 +1625,20 @@ def record_step_violation( if not stripped: continue - if workflow_defaults_run_indent is not None and indent <= workflow_defaults_run_indent: + if ( + workflow_defaults_run_indent is not None + and indent <= workflow_defaults_run_indent + ): workflow_defaults_run_indent = None - if workflow_defaults_indent is not None and indent <= workflow_defaults_indent: + if ( + workflow_defaults_indent is not None + and indent <= workflow_defaults_indent + ): workflow_defaults_indent = None - if job_defaults_run_indent is not None and indent <= job_defaults_run_indent: + if ( + job_defaults_run_indent is not None + and indent <= job_defaults_run_indent + ): job_defaults_run_indent = None if job_defaults_indent is not None and indent <= job_defaults_indent: job_defaults_indent = None @@ -1488,7 +1659,12 @@ def record_step_violation( if indent == 0 and stripped == "jobs:": in_jobs = True continue - if in_jobs and indent == 2 and stripped.endswith(":") and not stripped.startswith("-"): + if ( + in_jobs + and indent == 2 + and stripped.endswith(":") + and not stripped.startswith("-") + ): record_step_violation( step_working_directory, current_job_default_directory, @@ -1514,7 +1690,9 @@ def record_step_violation( if job_defaults_indent is not None and stripped == "run:": job_defaults_run_indent = indent continue - if job_defaults_run_indent is not None and stripped.startswith("working-directory:"): + if job_defaults_run_indent is not None and stripped.startswith( + "working-directory:" + ): current_job_default_directory = yaml_scalar_value(stripped) continue @@ -1531,7 +1709,10 @@ def record_step_violation( if stripped.startswith("working-directory:"): step_working_directory = yaml_scalar_value(stripped) - if WORKSPACE_EXEC_PATTERN.search(stripped) or line_number in workspace_exec_lines: + if ( + WORKSPACE_EXEC_PATTERN.search(stripped) + or line_number in workspace_exec_lines + ): step_uses_workspace_exec = True return violations @@ -1561,7 +1742,9 @@ def verify_release_asset_allowlist_policy() -> list[str]: and command_contains_token_sequence(command, RELEASE_ASSET_VALIDATOR) for index, job_content, command, is_blocking in run_steps ) - release_command_lines = [line.strip() for line in shell_logical_lines(release_command)] + release_command_lines = [ + line.strip() for line in shell_logical_lines(release_command) + ] revalidator_indexes = [ line_index for line_index, line in enumerate(release_command_lines) @@ -1614,7 +1797,9 @@ def verify_release_asset_allowlist_policy() -> list[str]: for line in shell_logical_lines(command): if not command_contains_token_sequence(line, "gh release create"): continue - if RELEASE_ARTIFACT_GLOB.search(line) or release_create_explicit_asset_tokens(line): + if RELEASE_ARTIFACT_GLOB.search( + line + ) or release_create_explicit_asset_tokens(line): add_release_asset_allowlist_violation(violations, path) break else: @@ -1644,7 +1829,9 @@ def rust_dependency_advisory_violations( current_name = str(package.get("name", "")) version = str(package.get("version", "")) if current_name == "fastrand" and version == RUST_FASTRAND_YANKED_VERSION: - violations.append(f"{lockfile}: fastrand {version} is yanked and must stay updated") + violations.append( + f"{lockfile}: fastrand {version} is yanked and must stay updated" + ) continue if current_name != "rand": if current_name == "glib": @@ -1806,7 +1993,9 @@ def rust_osv_exception_violations( ) for advisory_id, reason in sorted(osv_ignores.items()): if not reason.strip(): - violations.append(f"{osv_config}: OSV ignore for {advisory_id} needs a reason") + violations.append( + f"{osv_config}: OSV ignore for {advisory_id} needs a reason" + ) return violations @@ -1941,22 +2130,30 @@ def glib_legacy_exception_owners_are_allowed( """Return whether every glib ancestor matches the documented GTK/WebKit stack.""" if not legacy_glib_ancestors: return False - ancestor_names = {ancestor.rsplit(" ", maxsplit=1)[0] for ancestor in legacy_glib_ancestors} - direct_owner_names = {owner.rsplit(" ", maxsplit=1)[0] for owner in legacy_glib_direct_owners} + ancestor_names = { + ancestor.rsplit(" ", maxsplit=1)[0] for ancestor in legacy_glib_ancestors + } + direct_owner_names = { + owner.rsplit(" ", maxsplit=1)[0] for owner in legacy_glib_direct_owners + } if not direct_owner_names <= RUST_GLIB_LEGACY_DIRECT_OWNER_NAMES: return False off_chain_ancestors = legacy_glib_ancestors - glib_exception_owned_packages allowed_app_roots = { ancestor for ancestor in off_chain_ancestors - if ancestor.rsplit(" ", maxsplit=1)[0] in RUST_GLIB_LEGACY_ALLOWED_APP_ROOT_NAMES + if ancestor.rsplit(" ", maxsplit=1)[0] + in RUST_GLIB_LEGACY_ALLOWED_APP_ROOT_NAMES } if off_chain_ancestors != allowed_app_roots: return False - if not glib_allowed_app_roots_reach_glib_through_tauri(package_dependencies, allowed_app_roots): + if not glib_allowed_app_roots_reach_glib_through_tauri( + package_dependencies, allowed_app_roots + ): return False return ancestor_names <= ( - RUST_GLIB_LEGACY_ALLOWED_ANCESTOR_NAMES | RUST_GLIB_LEGACY_ALLOWED_APP_ROOT_NAMES + RUST_GLIB_LEGACY_ALLOWED_ANCESTOR_NAMES + | RUST_GLIB_LEGACY_ALLOWED_APP_ROOT_NAMES ) @@ -1972,7 +2169,8 @@ def glib_allowed_app_roots_reach_glib_through_tauri( in cargo_lock_reachable_package_keys(package_dependencies, dependency) } glib_reaching_dependency_names = { - dependency.rsplit(" ", maxsplit=1)[0] for dependency in glib_reaching_dependencies + dependency.rsplit(" ", maxsplit=1)[0] + for dependency in glib_reaching_dependencies } if glib_reaching_dependency_names != {RUST_GLIB_LEGACY_ROOT_NAME}: return False @@ -1996,7 +2194,10 @@ def cargo_lock_has_named_dependency_path( continue current_name = current.rsplit(" ", maxsplit=1)[0] next_matched_count = matched_count - if matched_count < len(package_names) and current_name == package_names[matched_count]: + if ( + matched_count < len(package_names) + and current_name == package_names[matched_count] + ): next_matched_count += 1 if next_matched_count == len(package_names): return True @@ -2099,7 +2300,9 @@ def store_current_package() -> None: in_dependencies = True dependency_tokens = [] continue - current_package["dependencies"] = parse_cargo_lock_string_list(normalized_value) + current_package["dependencies"] = parse_cargo_lock_string_list( + normalized_value + ) continue if normalized_key in {"name", "version"}: current_package[normalized_key] = parse_cargo_lock_scalar(normalized_value) @@ -2210,7 +2413,9 @@ def cargo_lock_reachable_package_keys_by_name( for package_key in package_dependencies: package_name = package_key.rsplit(" ", maxsplit=1)[0] if package_name == root_package_name: - reachable.update(cargo_lock_reachable_package_keys(package_dependencies, package_key)) + reachable.update( + cargo_lock_reachable_package_keys(package_dependencies, package_key) + ) return reachable @@ -2247,6 +2452,7 @@ def main() -> int: violations.extend(rust_osv_exception_violations()) violations.extend(rust_trivy_exception_violations()) violations.extend(rust_dependency_advisory_violations()) + violations.extend(supplemental_inventory_violations()) if violations: print("Supply-chain verification failed:") diff --git a/services/analysis-engine/pyproject.toml b/services/analysis-engine/pyproject.toml index 092372dd2..c830f63fe 100644 --- a/services/analysis-engine/pyproject.toml +++ b/services/analysis-engine/pyproject.toml @@ -32,6 +32,9 @@ packages = ["src/bandscope_analysis"] [tool.pytest.ini_options] testpaths = ["tests"] pythonpath = ["src"] +markers = [ + "youtube_stem_e2e: opt-in network and real-model validation against a known public stem", +] filterwarnings = [ "ignore::DeprecationWarning", ] diff --git a/services/analysis-engine/src/bandscope_analysis/separation/audio_separator.py b/services/analysis-engine/src/bandscope_analysis/separation/audio_separator.py index c36e0f1fc..e7ebe80b5 100644 --- a/services/analysis-engine/src/bandscope_analysis/separation/audio_separator.py +++ b/services/analysis-engine/src/bandscope_analysis/separation/audio_separator.py @@ -9,9 +9,10 @@ Security Notes: - Treats the selected audio file as untrusted input: the path is normalized and verified to be a file, and a maximum byte size is enforced before decode. -- Inference runs locally on CPU with no network access. The model weights are - loaded from the local Demucs cache or a configured bundled path; offline - weight bundling is tracked in the supplemental component inventory. +- Inference runs locally on CPU after model provisioning. The first Demucs + model load may fetch the inventoried weight into its user cache; BandScope + does not bundle that artifact, and offline operation requires a trusted + pre-provisioned cache. - Does not log or persist raw audio, separated stems, or full source paths. - Fails with bounded, filename-scoped errors so callers can surface a safe failure without leaking local directory structure. @@ -66,6 +67,9 @@ class AudioSeparationConfig: max_duration_seconds: float = float(MAX_ANALYSIS_DURATION_SECONDS) model_name: str = "htdemucs" device: str = "cpu" + # Disable Demucs' random time-shift augmentation so repeated analysis of + # the same bytes is deterministic and benchmark evidence is reproducible. + shifts: int = 0 # Demucs splits long audio into overlapping segments internally, bounding # memory so long tracks do not OOM the host on CPU. overlap: float = 0.25 @@ -167,6 +171,7 @@ def _apply_model(self, model: Any, audio: AudioStemArray) -> dict[str, np.ndarra model, normalized[None], device=self.config.device, + shifts=self.config.shifts, split=True, overlap=self.config.overlap, progress=False, diff --git a/services/analysis-engine/src/bandscope_analysis/separation/model_weights/bandsplit-v1.json b/services/analysis-engine/src/bandscope_analysis/separation/model_weights/bandsplit-v1.json deleted file mode 100644 index 15698992c..000000000 --- a/services/analysis-engine/src/bandscope_analysis/separation/model_weights/bandsplit-v1.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "modelId": "bandsplit-v1", - "bassCutoffHz": 250.0, - "vocalLowHz": 300.0, - "vocalHighHz": 3400.0, - "drumLowHz": 3400.0 -} diff --git a/services/analysis-engine/src/bandscope_analysis/youtube.py b/services/analysis-engine/src/bandscope_analysis/youtube.py index c98f4e513..b6b609f3c 100644 --- a/services/analysis-engine/src/bandscope_analysis/youtube.py +++ b/services/analysis-engine/src/bandscope_analysis/youtube.py @@ -1,7 +1,14 @@ -""" -YouTube import capabilities for BandScope. +"""YouTube import capabilities for BandScope. This module provides a safe wrapper around yt-dlp to download audio from YouTube. + +Security Notes: +- Accepts only bounded, standard HTTPS YouTube watch URLs and disables playlists, + geographic bypass, credentials, and interactive authentication. +- Keeps certificate verification enabled and uses the operating system trust + store so managed desktop CA policy is honored. +- Rejects metadata over 15 minutes and completed files over 50 MiB, returns + sanitized public errors, and never logs the requested URL or downloaded audio. """ import argparse @@ -41,7 +48,13 @@ def validate_url(url: str) -> bool: parsed = urllib.parse.urlparse(url) if parsed.scheme != "https": return False - host = parsed.netloc.lower().split(":")[0] + if ( + parsed.username is not None + or parsed.password is not None + or parsed.port not in (None, 443) + ): + return False + host = parsed.hostname if host == "youtu.be": path = parsed.path.strip("/") @@ -130,6 +143,9 @@ def download_youtube_audio(url: str, out_dir: str) -> Dict[str, Any]: "noplaylist": True, "postprocessors": [{"key": "FFmpegExtractAudio"}], "geo_bypass": False, + # Keep TLS verification enabled while honoring OS-managed CA roots + # (including enterprise desktop trust stores) instead of certifi only. + "compat_opts": {"no-certifi"}, } try: diff --git a/services/analysis-engine/tests/known_stem_benchmark.py b/services/analysis-engine/tests/known_stem_benchmark.py new file mode 100644 index 000000000..6442ed900 --- /dev/null +++ b/services/analysis-engine/tests/known_stem_benchmark.py @@ -0,0 +1,524 @@ +"""Utilities for the opt-in real-YouTube known-stem benchmark. + +The helpers live under ``tests`` deliberately: they fetch only the fixed public +benchmark assets below and are not part of BandScope's production download API. +""" + +from __future__ import annotations + +import hashlib +import math +import re +import zipfile +from dataclasses import dataclass +from pathlib import Path +from typing import Any +from urllib.parse import urlsplit +from urllib.request import HTTPRedirectHandler, Request, build_opener + +import numpy as np + +_DOWNLOAD_CHUNK_BYTES = 64 * 1024 +_ENERGY_EPSILON = 1e-12 +_MAX_REFERENCE_BYTES = 64 * 1024 * 1024 +_CANONICAL_STEMS = {"vocals", "bass", "drums", "other"} + +# Provisional sentinel thresholds derived from the pinned creator master on the +# documented Linux CPU baseline. They remain advisory until an authorized +# YouTube candidate is measured and ADR-0002 is accepted. +MIN_MASTER_IDENTITY_CORRELATION = 0.90 +MIN_VOCAL_SI_SDR_IMPROVEMENT_DB = 0.5 +MIN_VOCAL_ASSIGNMENT_MARGIN_DB = 3.0 +MAX_MASTER_DURATION_DRIFT_SECONDS = 1.0 + + +class _AllowlistedRedirectHandler(HTTPRedirectHandler): + """Allow reference redirects only when HTTPS and the exact host are preserved.""" + + def __init__(self, expected_host: str) -> None: + """Store the only host that a redirect may target.""" + super().__init__() + self._expected_host = expected_host + + def redirect_request( + self, + request: Request, + file_pointer: Any, + code: int, + message: str, + headers: Any, + new_url: str, + ) -> Request | None: + """Validate a redirect target before urllib creates or sends its request.""" + _validate_fixture_url(new_url, self._expected_host) + return super().redirect_request( + request, + file_pointer, + code, + message, + headers, + new_url, + ) + + +@dataclass(frozen=True) +class KnownStemFixture: + """Describe one immutable reference archive and its matching YouTube mix.""" + + youtube_url: str + video_id: str + reference_archive_url: str + reference_archive_host: str + reference_archive_sha256: str + reference_archive_bytes: int + reference_member: str + reference_member_sha256: str + reference_member_bytes: int + creator_master_url: str + creator_master_host: str + creator_master_sha256: str + creator_master_bytes: int + creator_master_duration_seconds: float + target_stem: str + + +@dataclass(frozen=True) +class AlignedStemWindow: + """Hold one globally aligned active reference and mixture window.""" + + mixture: np.ndarray + reference: np.ndarray + lag_samples: int + reference_start: int + correlation: float + + +@dataclass(frozen=True) +class KnownStemBenchmarkWindow: + """Hold identity evidence plus one globally composed vocal scoring window.""" + + mixture: np.ndarray + reference: np.ndarray + youtube_to_master_lag_samples: int + master_to_reference_lag_samples: int + reference_start: int + identity_correlation: float + + +BRAD_SUCKS_FIXTURE = KnownStemFixture( + youtube_url="https://www.youtube.com/watch?v=e4pIpWVbMKs", + video_id="e4pIpWVbMKs", + reference_archive_url=("https://bradmedia.com/media/source/making_me_nervous-120bpm.zip"), + reference_archive_host="bradmedia.com", + reference_archive_sha256=("473578daa0bcf022448a144c5df9111ddf11e5a90e77f3649254e7813ba4981d"), + reference_archive_bytes=31_055_394, + reference_member="vocals.wav", + reference_member_sha256=("4c7bb41c3f8bda1471dfd214b84f1d3457af344feeba33f0b31982ed0d808afc"), + reference_member_bytes=25_603_092, + creator_master_url=( + "https://static1.squarespace.com/static/5bf9a31c96d4550b42f456f2/" + "5c002e7503ce649ee6716b51/5c00331d6d2a731d3dfa9896/1543517055733/" + "01%2BBrad%2BSucks%2B-%2BMaking%2BMe%2BNervous.mp3" + ), + creator_master_host="static1.squarespace.com", + creator_master_sha256=("fc7f7c2a0387e46885e5c133cbd6d14d7de4d48908b68f1135354df0a336cf1d"), + creator_master_bytes=4_941_627, + creator_master_duration_seconds=155.945238, + target_stem="vocals", +) + + +def _as_finite_signal(values: np.ndarray, name: str) -> np.ndarray: + """Return a finite one-dimensional float64 signal.""" + signal = np.ravel(np.asarray(values, dtype=np.float64)) + if signal.size < 2: + raise ValueError(f"{name} signal must contain at least two samples") + if not np.isfinite(signal).all(): + raise ValueError(f"{name} signal must contain only finite samples") + return signal + + +def zero_mean_si_sdr(estimate: np.ndarray, reference: np.ndarray) -> float: + """Return zero-mean scale-invariant signal-to-distortion ratio in decibels.""" + estimated_signal = _as_finite_signal(estimate, "estimate") + reference_signal = _as_finite_signal(reference, "reference") + if estimated_signal.shape != reference_signal.shape: + raise ValueError("estimate and reference signals must have equal lengths") + + estimated_signal = estimated_signal - float(np.mean(estimated_signal)) + reference_signal = reference_signal - float(np.mean(reference_signal)) + reference_energy = float(np.dot(reference_signal, reference_signal)) + estimate_energy = float(np.dot(estimated_signal, estimated_signal)) + if reference_energy <= _ENERGY_EPSILON: + raise ValueError("reference signal has insufficient audio energy") + if estimate_energy <= _ENERGY_EPSILON: + raise ValueError("estimate signal has insufficient audio energy") + + scale = float(np.dot(estimated_signal, reference_signal) / reference_energy) + projection = scale * reference_signal + projection_energy = float(np.dot(projection, projection)) + residual = estimated_signal - projection + residual_energy = float(np.dot(residual, residual)) + if projection_energy <= _ENERGY_EPSILON: + return float("-inf") + if residual_energy <= _ENERGY_EPSILON: + return float("inf") + return float(10.0 * math.log10(projection_energy / residual_energy)) + + +def si_sdr_improvement(estimate: np.ndarray, mixture: np.ndarray, reference: np.ndarray) -> float: + """Return SI-SDR improvement over using the downloaded mixture as the estimate.""" + separation_score = zero_mean_si_sdr(estimate, reference) + mixture_score = zero_mean_si_sdr(mixture, reference) + improvement = separation_score - mixture_score + if math.isnan(improvement): + raise ValueError("SI-SDR improvement is undefined for these signals") + return float(improvement) + + +def _fft_cross_correlation(observation: np.ndarray, reference: np.ndarray) -> np.ndarray: + """Match ``numpy.correlate(observation, reference, 'full')`` using an FFT.""" + result_size = observation.size + reference.size - 1 + fft_size = 1 << (result_size - 1).bit_length() + spectrum = np.fft.rfft(observation, fft_size) * np.fft.rfft(reference[::-1], fft_size) + return np.fft.irfft(spectrum, fft_size)[:result_size] + + +def _rms_envelope(signal: np.ndarray, hop_samples: int) -> np.ndarray: + """Return a log-RMS envelope with one value per non-overlapping hop.""" + frame_count = math.ceil(signal.size / hop_samples) + padded = np.zeros(frame_count * hop_samples, dtype=np.float64) + padded[: signal.size] = signal + frames = padded.reshape(frame_count, hop_samples) + rms = np.sqrt(np.mean(np.square(frames), axis=1)) + envelope = np.log1p(10.0 * rms) + return envelope - float(np.mean(envelope)) + + +def _strongest_window_start(signal: np.ndarray, window_samples: int) -> int: + """Return the sample index of the maximum-energy fixed-width window.""" + energy = np.square(signal) + cumulative = np.concatenate((np.zeros(1, dtype=np.float64), np.cumsum(energy))) + window_energy = cumulative[window_samples:] - cumulative[:-window_samples] + return int(np.argmax(window_energy)) + + +def _normalized_correlation(left: np.ndarray, right: np.ndarray) -> float: + """Return absolute zero-mean Pearson correlation for two equal windows.""" + left_centered = left - float(np.mean(left)) + right_centered = right - float(np.mean(right)) + denominator = math.sqrt( + float(np.dot(left_centered, left_centered)) * float(np.dot(right_centered, right_centered)) + ) + if denominator <= _ENERGY_EPSILON: + raise ValueError("aligned benchmark window has insufficient audio energy") + return float(abs(np.dot(left_centered, right_centered)) / denominator) + + +def align_active_reference_window( + mixture: np.ndarray, + reference: np.ndarray, + *, + sample_rate: int, + window_seconds: float, + max_lag_seconds: float, + envelope_hop_seconds: float = 0.01, + refinement_seconds: float = 0.25, +) -> AlignedStemWindow: + """Align once globally, then return the strongest known-stem scoring window. + + The global lag is estimated from low-rate RMS envelopes. A bounded waveform + refinement is then performed around that lag for the chosen active window. + The resulting single offset is applied to both the mixture and reference; + stems are never aligned independently. + """ + mixture_signal = _as_finite_signal(mixture, "mixture") + reference_signal = _as_finite_signal(reference, "reference") + if sample_rate <= 0: + raise ValueError("sample_rate must be positive") + if window_seconds <= 0.0 or max_lag_seconds < 0.0: + raise ValueError("alignment durations are invalid") + if envelope_hop_seconds <= 0.0 or refinement_seconds < 0.0: + raise ValueError("alignment resolution is invalid") + + window_samples = int(round(window_seconds * sample_rate)) + if window_samples < 2 or window_samples > reference_signal.size: + raise ValueError("reference is shorter than the requested scoring window") + hop_samples = max(1, int(round(envelope_hop_seconds * sample_rate))) + mixture_envelope = _rms_envelope(mixture_signal, hop_samples) + reference_envelope = _rms_envelope(reference_signal, hop_samples) + coarse_correlation = _fft_cross_correlation(mixture_envelope, reference_envelope) + coarse_lags = np.arange( + -reference_envelope.size + 1, + mixture_envelope.size, + dtype=np.int64, + ) + max_lag_frames = int(math.ceil(max_lag_seconds * sample_rate / hop_samples)) + valid_coarse = np.flatnonzero(np.abs(coarse_lags) <= max_lag_frames) + if valid_coarse.size == 0: + raise ValueError("reference fixture has no permitted alignment lag") + best_coarse_index = int(valid_coarse[np.argmax(coarse_correlation[valid_coarse])]) + coarse_lag_samples = int(coarse_lags[best_coarse_index]) * hop_samples + + reference_start = _strongest_window_start(reference_signal, window_samples) + reference_window = reference_signal[reference_start : reference_start + window_samples] + expected_mixture_start = reference_start + coarse_lag_samples + refinement_samples = int(round(refinement_seconds * sample_rate)) + search_start = max(0, expected_mixture_start - refinement_samples) + search_end = min( + mixture_signal.size, + expected_mixture_start + window_samples + refinement_samples, + ) + mixture_search = mixture_signal[search_start:search_end] + if mixture_search.size < window_samples: + raise ValueError("reference fixture does not overlap the downloaded mixture") + + refined_correlation = _fft_cross_correlation(mixture_search, reference_window) + refined_lags = np.arange( + -reference_window.size + 1, + mixture_search.size, + dtype=np.int64, + ) + valid_refined = np.flatnonzero( + (refined_lags >= 0) & (refined_lags + window_samples <= mixture_search.size) + ) + if valid_refined.size == 0: + raise ValueError("reference fixture cannot produce a full scoring window") + best_refined_index = int(valid_refined[np.argmax(np.abs(refined_correlation[valid_refined]))]) + mixture_start = search_start + int(refined_lags[best_refined_index]) + mixture_window = mixture_signal[mixture_start : mixture_start + window_samples] + correlation = _normalized_correlation(mixture_window, reference_window) + return AlignedStemWindow( + mixture=mixture_window, + reference=reference_window, + lag_samples=mixture_start - reference_start, + reference_start=reference_start, + correlation=correlation, + ) + + +def align_known_stem_through_master( + youtube_mix: np.ndarray, + creator_master: np.ndarray, + reference_stem: np.ndarray, + *, + sample_rate: int, + window_seconds: float, + max_lag_seconds: float, +) -> KnownStemBenchmarkWindow: + """Compose YouTube-to-master and master-to-stem offsets once. + + The creator master establishes that the downloaded candidate is the pinned + recording. A separate global offset maps the dry vocal into that master. + The two offsets are composed before inference; predicted stems are never + shifted independently to improve their scores. + """ + youtube_signal = _as_finite_signal(youtube_mix, "YouTube mixture") + master_signal = _as_finite_signal(creator_master, "creator master") + reference_signal = _as_finite_signal(reference_stem, "reference") + identity = align_active_reference_window( + youtube_signal, + master_signal, + sample_rate=sample_rate, + window_seconds=window_seconds, + max_lag_seconds=max_lag_seconds, + ) + master_to_reference = align_active_reference_window( + master_signal, + reference_signal, + sample_rate=sample_rate, + window_seconds=window_seconds, + max_lag_seconds=max_lag_seconds, + ) + window_samples = int(round(window_seconds * sample_rate)) + youtube_start = ( + master_to_reference.reference_start + master_to_reference.lag_samples + identity.lag_samples + ) + youtube_end = youtube_start + window_samples + if youtube_start < 0 or youtube_end > youtube_signal.size: + raise ValueError("reference fixture does not overlap the downloaded mixture") + mixture_window = youtube_signal[youtube_start:youtube_end] + return KnownStemBenchmarkWindow( + mixture=mixture_window, + reference=master_to_reference.reference, + youtube_to_master_lag_samples=identity.lag_samples, + master_to_reference_lag_samples=master_to_reference.lag_samples, + reference_start=master_to_reference.reference_start, + identity_correlation=identity.correlation, + ) + + +def _validate_fixture_url(url: str, expected_host: str) -> None: + """Require an HTTPS URL on the fixture's exact allowlisted host.""" + parsed = urlsplit(url) + if ( + parsed.scheme != "https" + or parsed.hostname != expected_host + or parsed.username is not None + or parsed.password is not None + or parsed.port not in (None, 443) + ): + raise ValueError("Untrusted reference fixture URL") + + +def _validate_fixture_definition(fixture: KnownStemFixture) -> None: + """Reject path-like fields, malformed hashes, and excessive resource bounds.""" + if fixture.target_stem not in _CANONICAL_STEMS: + raise ValueError("Untrusted reference fixture target stem") + if ( + not fixture.reference_member.endswith(".wav") + or "/" in fixture.reference_member + or "\\" in fixture.reference_member + or "\x00" in fixture.reference_member + ): + raise ValueError("Untrusted reference fixture member") + hashes = ( + fixture.reference_archive_sha256, + fixture.reference_member_sha256, + fixture.creator_master_sha256, + ) + if any(not re.fullmatch(r"[0-9a-f]{64}", digest) for digest in hashes): + raise ValueError("Untrusted reference fixture SHA-256") + if not 0 < fixture.reference_archive_bytes <= _MAX_REFERENCE_BYTES: + raise ValueError("Untrusted reference fixture archive size") + if not 0 < fixture.reference_member_bytes <= _MAX_REFERENCE_BYTES: + raise ValueError("Untrusted reference fixture member size") + if not 0 < fixture.creator_master_bytes <= _MAX_REFERENCE_BYTES: + raise ValueError("Untrusted reference fixture master size") + if not math.isfinite(fixture.creator_master_duration_seconds): + raise ValueError("Untrusted reference fixture master duration") + if fixture.creator_master_duration_seconds <= 0.0: + raise ValueError("Untrusted reference fixture master duration") + _validate_fixture_url(fixture.creator_master_url, fixture.creator_master_host) + + +def _open_fixture_url(request: Request, expected_host: str) -> Any: + """Open a fixture URL with pre-request validation for every redirect target.""" + opener = build_opener(_AllowlistedRedirectHandler(expected_host)) + return opener.open(request, timeout=30.0) + + +def _validated_fixture_root(directory: Path) -> Path: + """Return a real caller-owned directory for bounded fixture outputs.""" + root_input = Path(directory) + if root_input.is_symlink() or not root_input.is_dir(): + raise ValueError("Untrusted reference fixture directory") + return root_input.resolve(strict=True) + + +def _download_verified_file( + *, + url: str, + expected_host: str, + expected_sha256: str, + expected_bytes: int, + destination: Path, +) -> Path: + """Download one exact HTTPS file with host, size, and SHA-256 checks.""" + _validate_fixture_url(url, expected_host) + if destination.exists(): + raise ValueError("Untrusted reference fixture destination") + request = Request(url, headers={"User-Agent": "BandScope-known-stem-benchmark/1.0"}) + try: + with ( + _open_fixture_url(request, expected_host) as response, + destination.open("xb") as output, + ): + _validate_fixture_url(response.geturl(), expected_host) + content_length = response.headers.get("Content-Length") + if content_length is not None: + try: + declared_bytes = int(content_length) + except ValueError as error: + raise ValueError("Untrusted reference fixture byte count") from error + if declared_bytes != expected_bytes: + raise ValueError("Untrusted reference fixture byte count") + + digest = hashlib.sha256() + downloaded_bytes = 0 + while chunk := response.read(_DOWNLOAD_CHUNK_BYTES): + downloaded_bytes += len(chunk) + if downloaded_bytes > expected_bytes: + raise ValueError("Untrusted reference fixture byte count") + digest.update(chunk) + output.write(chunk) + if downloaded_bytes != expected_bytes: + raise ValueError("Untrusted reference fixture byte count") + if digest.hexdigest() != expected_sha256: + raise ValueError("Untrusted reference fixture SHA-256") + except Exception: + destination.unlink(missing_ok=True) + raise + return destination + + +def download_verified_creator_master(fixture: KnownStemFixture, directory: Path) -> Path: + """Download and authenticate the exact creator-hosted finished master.""" + _validate_fixture_definition(fixture) + root = _validated_fixture_root(directory) + return _download_verified_file( + url=fixture.creator_master_url, + expected_host=fixture.creator_master_host, + expected_sha256=fixture.creator_master_sha256, + expected_bytes=fixture.creator_master_bytes, + destination=root / "known-reference-master.mp3", + ) + + +def download_verified_reference_stem(fixture: KnownStemFixture, directory: Path) -> Path: + """Download, authenticate, and safely extract one exact reference stem. + + TLS verification remains enabled. The initial and final URL hosts are + allowlisted, the compressed byte count and SHA-256 are exact, and only the + named ZIP member with its expected uncompressed size is streamed out. + """ + _validate_fixture_definition(fixture) + root = _validated_fixture_root(directory) + archive_path = root / "known-reference-source.zip" + destination = root / f"known-reference-{fixture.target_stem}.wav" + if archive_path.exists() or destination.exists(): + raise ValueError("Untrusted reference fixture destination") + try: + _download_verified_file( + url=fixture.reference_archive_url, + expected_host=fixture.reference_archive_host, + expected_sha256=fixture.reference_archive_sha256, + expected_bytes=fixture.reference_archive_bytes, + destination=archive_path, + ) + + with zipfile.ZipFile(archive_path) as source_archive: + members = [ + member + for member in source_archive.infolist() + if member.filename == fixture.reference_member + ] + if len(members) != 1: + raise ValueError("Untrusted reference fixture member") + member = members[0] + if ( + member.is_dir() + or member.flag_bits & 0x1 + or member.file_size != fixture.reference_member_bytes + ): + raise ValueError("Untrusted reference fixture member size") + + extracted_digest = hashlib.sha256() + extracted_bytes = 0 + with source_archive.open(member, "r") as source, destination.open("xb") as output: + while chunk := source.read(_DOWNLOAD_CHUNK_BYTES): + extracted_bytes += len(chunk) + if extracted_bytes > fixture.reference_member_bytes: + raise ValueError("Untrusted reference fixture member size") + extracted_digest.update(chunk) + output.write(chunk) + if extracted_bytes != fixture.reference_member_bytes: + raise ValueError("Untrusted reference fixture member size") + if extracted_digest.hexdigest() != fixture.reference_member_sha256: + raise ValueError("Untrusted reference fixture SHA-256") + except Exception: + destination.unlink(missing_ok=True) + raise + finally: + archive_path.unlink(missing_ok=True) + return destination diff --git a/services/analysis-engine/tests/test_documentation_policy.py b/services/analysis-engine/tests/test_documentation_policy.py new file mode 100644 index 000000000..ecbd8116a --- /dev/null +++ b/services/analysis-engine/tests/test_documentation_policy.py @@ -0,0 +1,26 @@ +"""Tests for the canonical repository documentation contract.""" + +from pathlib import Path + +from conftest import load_module + + +def test_documentation_contract_reports_missing_canonical_authorities(tmp_path: Path) -> None: + """Reject a repository that omits the PRD, TRD, ADR index, or diagram authority.""" + documentation = load_module("scripts/checks/verify_docs.py", "verify_docs_contract_missing") + + violations = documentation.documentation_violations(tmp_path) + + assert "missing file: docs/PRD.md" in violations + assert "missing file: docs/TRD.md" in violations + assert "missing file: docs/adr/README.md" in violations + assert "missing file: docs/architecture/diagrams.md" in violations + assert "missing file: docs/documentation-coverage-matrix.md" in violations + + +def test_documentation_contract_accepts_checked_in_authorities() -> None: + """Accept the checked-in documentation graph when every canonical authority is present.""" + documentation = load_module("scripts/checks/verify_docs.py", "verify_docs_contract_repo") + repo_root = Path(__file__).resolve().parents[3] + + assert documentation.documentation_violations(repo_root) == [] diff --git a/services/analysis-engine/tests/test_separation.py b/services/analysis-engine/tests/test_separation.py index f8e098521..a7cdfb7dc 100644 --- a/services/analysis-engine/tests/test_separation.py +++ b/services/analysis-engine/tests/test_separation.py @@ -357,6 +357,7 @@ def fake_apply_model( batch: _FakeTensor, *, device: str, + shifts: int, split: bool, overlap: float, progress: bool, @@ -365,6 +366,7 @@ def fake_apply_model( { "batch_shape": batch.array.shape, "device": device, + "shifts": shifts, "split": split, "overlap": overlap, "progress": progress, @@ -392,6 +394,7 @@ def fake_apply_model( assert calls == { "batch_shape": (1, 2, samples), "device": "cpu", + "shifts": 0, "split": True, "overlap": 0.375, "progress": False, diff --git a/services/analysis-engine/tests/test_supply_chain_policy.py b/services/analysis-engine/tests/test_supply_chain_policy.py index ab43df89f..bdc353c4f 100644 --- a/services/analysis-engine/tests/test_supply_chain_policy.py +++ b/services/analysis-engine/tests/test_supply_chain_policy.py @@ -13,6 +13,63 @@ from conftest import load_module, make_symlink_or_skip +def test_supplemental_inventory_rejects_obsolete_or_missing_runtime_model( + tmp_path: Path, +) -> None: + """Require the runtime separator model, not the retired FFT profile, in inventory.""" + supply_chain = load_module( + "scripts/checks/verify_supply_chain.py", + "verify_supply_chain_model_inventory_missing", + ) + inventory_path = tmp_path / "supply-chain" / "supplemental-component-inventory.json" + inventory_path.parent.mkdir(parents=True) + inventory_path.write_text( + json.dumps( + { + "modelArtifacts": [ + { + "name": "bandsplit-v1-profile", + "runtimeModelName": "bandsplit-v1", + } + ] + } + ), + encoding="utf-8", + ) + separator_path = tmp_path / "audio_separator.py" + separator_path.write_text('model_name: str = "htdemucs"\n', encoding="utf-8") + + violations = supply_chain.supplemental_inventory_violations( + inventory_path, + separator_path, + ) + + assert "supplemental inventory contains retired model: bandsplit-v1-profile" in violations + assert "supplemental inventory missing runtime model: htdemucs" in violations + + +def test_supplemental_inventory_accepts_pinned_htdemucs_runtime_model() -> None: + """Accept the checked-in full-hash htdemucs runtime artifact record.""" + supply_chain = load_module( + "scripts/checks/verify_supply_chain.py", + "verify_supply_chain_model_inventory_repo", + ) + repo_root = Path(__file__).resolve().parents[3] + + violations = supply_chain.supplemental_inventory_violations( + repo_root / "supply-chain" / "supplemental-component-inventory.json", + repo_root + / "services" + / "analysis-engine" + / "src" + / "bandscope_analysis" + / "separation" + / "audio_separator.py", + ) + + assert violations == [] + + def central_required_workflow_policy_text() -> str: """Return the repository policy text that delegates review automation centrally.""" repo_root = Path(__file__).resolve().parents[3] diff --git a/services/analysis-engine/tests/test_youtube.py b/services/analysis-engine/tests/test_youtube.py index 5531ac9d5..4d1ea9c91 100644 --- a/services/analysis-engine/tests/test_youtube.py +++ b/services/analysis-engine/tests/test_youtube.py @@ -60,6 +60,8 @@ def test_validate_url_edge_cases() -> None: assert validate_url("https://evil.com/youtube.com/watch?v=123") is False assert validate_url("https://evil.com?youtube.com/watch?v=123") is False assert validate_url("https://evil.com#youtube.com/watch?v=123") is False + assert validate_url("https://youtube.com:443@evil.example/watch?v=abc123DEF45") is False + assert validate_url("https://youtube.com:444/watch?v=abc123DEF45") is False # Allowlist behavior and explicit default ports assert validate_url("https://kr.youtube.com/watch?v=abc123DEF45") is False @@ -113,6 +115,7 @@ def test_download_youtube_audio_success( assert called_opts["noprogress"] is True assert called_opts["noplaylist"] is True assert called_opts["geo_bypass"] is False + assert called_opts["compat_opts"] == {"no-certifi"} assert called_opts["postprocessors"] == [{"key": "FFmpegExtractAudio"}] assert "%(id)s.%(ext)s" in called_opts["outtmpl"] diff --git a/services/analysis-engine/tests/test_youtube_stem_e2e.py b/services/analysis-engine/tests/test_youtube_stem_e2e.py new file mode 100644 index 000000000..b1513ab61 --- /dev/null +++ b/services/analysis-engine/tests/test_youtube_stem_e2e.py @@ -0,0 +1,422 @@ +"""Tests for the opt-in YouTube known-stem separation benchmark.""" + +from __future__ import annotations + +import hashlib +import io +import os +import tempfile +import zipfile +from dataclasses import replace +from pathlib import Path +from urllib.request import Request + +import numpy as np +import pytest +import soundfile as sf +from known_stem_benchmark import ( + BRAD_SUCKS_FIXTURE, + MAX_MASTER_DURATION_DRIFT_SECONDS, + MIN_MASTER_IDENTITY_CORRELATION, + MIN_VOCAL_ASSIGNMENT_MARGIN_DB, + MIN_VOCAL_SI_SDR_IMPROVEMENT_DB, + KnownStemFixture, + _AllowlistedRedirectHandler, + align_active_reference_window, + align_known_stem_through_master, + download_verified_creator_master, + download_verified_reference_stem, + si_sdr_improvement, + zero_mean_si_sdr, +) + +from bandscope_analysis.separation.audio_separator import ( + AudioSeparationConfig, + AudioStemSeparator, +) +from bandscope_analysis.youtube import download_youtube_audio + + +class _FakeResponse(io.BytesIO): + """Provide the small subset of an HTTPS response used by the fixture loader.""" + + def __init__(self, payload: bytes, final_url: str) -> None: + """Initialize a response with stable headers and a final URL.""" + super().__init__(payload) + self.headers = {"Content-Length": str(len(payload))} + self._final_url = final_url + + def geturl(self) -> str: + """Return the URL after redirects.""" + return self._final_url + + +def _archive_payload( + member_name: str, + member_payload: bytes, + *, + extra_members: dict[str, bytes] | None = None, +) -> bytes: + """Build a small in-memory ZIP archive for reference-integrity tests.""" + payload = io.BytesIO() + with zipfile.ZipFile(payload, "w", compression=zipfile.ZIP_DEFLATED) as archive: + for extra_name, extra_payload in (extra_members or {}).items(): + archive.writestr(extra_name, extra_payload) + archive.writestr(member_name, member_payload) + return payload.getvalue() + + +def _fixture_for_archive(payload: bytes, *, member_payload: bytes) -> KnownStemFixture: + """Return a fixture definition whose integrity values match a test archive.""" + return KnownStemFixture( + youtube_url="https://www.youtube.com/watch?v=e4pIpWVbMKs", + video_id="e4pIpWVbMKs", + reference_archive_url="https://fixtures.example/reference.zip", + reference_archive_host="fixtures.example", + reference_archive_sha256=hashlib.sha256(payload).hexdigest(), + reference_archive_bytes=len(payload), + reference_member="vocals.wav", + reference_member_sha256=hashlib.sha256(member_payload).hexdigest(), + reference_member_bytes=len(member_payload), + creator_master_url="https://fixtures.example/master.mp3", + creator_master_host="fixtures.example", + creator_master_sha256=hashlib.sha256(b"creator master").hexdigest(), + creator_master_bytes=len(b"creator master"), + creator_master_duration_seconds=4.0, + target_stem="vocals", + ) + + +def test_zero_mean_si_sdr_improvement_rewards_a_cleaner_estimate() -> None: + """Measure separation improvement relative to returning the mixture unchanged.""" + sample_rate = 8_000 + time = np.arange(sample_rate * 2, dtype=np.float64) / sample_rate + reference = np.sin(2 * np.pi * 223.0 * time) + interference = 0.9 * np.sin(2 * np.pi * 997.0 * time + 0.3) + mixture = reference + interference + estimate = reference + 0.05 * interference + + improvement = si_sdr_improvement(estimate, mixture, reference) + + assert improvement > 20.0 + assert zero_mean_si_sdr(estimate, reference) > zero_mean_si_sdr(mixture, reference) + + +@pytest.mark.parametrize( + ("estimate", "reference", "message"), + [ + (np.array([0.0, np.nan, 1.0]), np.ones(3), "finite"), + (np.zeros(8), np.arange(8, dtype=np.float64), "estimate.*energy"), + (np.ones(8), np.ones(8), "reference.*energy"), + ], +) +def test_zero_mean_si_sdr_rejects_invalid_signals( + estimate: np.ndarray, reference: np.ndarray, message: str +) -> None: + """Reject non-finite and effectively silent benchmark inputs.""" + with pytest.raises(ValueError, match=message): + zero_mean_si_sdr(estimate, reference) + + +def test_align_active_reference_window_recovers_delay_and_loud_section() -> None: + """Use one global offset to align a known stem with a delayed mixture.""" + rng = np.random.default_rng(20260809) + sample_rate = 1_000 + reference = np.zeros(4_000, dtype=np.float64) + reference[700:1_700] = 0.25 * rng.standard_normal(1_000) + reference[2_200:3_200] = rng.standard_normal(1_000) + lag_samples = 137 + mixture = 0.01 * rng.standard_normal(reference.size + 300) + mixture[lag_samples : lag_samples + reference.size] += reference + + aligned = align_active_reference_window( + mixture, + reference, + sample_rate=sample_rate, + window_seconds=0.8, + max_lag_seconds=0.5, + envelope_hop_seconds=0.02, + refinement_seconds=0.08, + ) + + assert aligned.lag_samples == lag_samples + assert aligned.reference_start >= 2_100 + assert aligned.reference_start <= 2_400 + assert aligned.mixture.shape == aligned.reference.shape == (800,) + assert aligned.correlation > 0.99 + + +def test_download_verified_reference_stem_extracts_only_the_pinned_member( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Accept an exact HTTPS archive and extract only its expected stem member.""" + member_payload = b"known vocal stem" + unexpected_name = "bandscope-zip-slip-must-not-exist" + archive_payload = _archive_payload( + "vocals.wav", + member_payload, + extra_members={f"../{unexpected_name}": b"untrusted extra member"}, + ) + fixture = _fixture_for_archive(archive_payload, member_payload=member_payload) + + def fake_open_fixture_url(request: object, expected_host: str) -> _FakeResponse: + """Return the pinned archive without using the network.""" + assert expected_host == fixture.reference_archive_host + return _FakeResponse(archive_payload, fixture.reference_archive_url) + + monkeypatch.setattr("known_stem_benchmark._open_fixture_url", fake_open_fixture_url) + + extracted = download_verified_reference_stem(fixture, tmp_path) + + assert extracted == tmp_path / "known-reference-vocals.wav" + assert extracted.read_bytes() == member_payload + assert not (tmp_path / "known-reference-source.zip").exists() + assert not (tmp_path.parent / unexpected_name).exists() + + +def test_reference_redirect_handler_rejects_off_host_before_following() -> None: + """Reject an off-host HTTPS redirect before creating its follow-up request.""" + handler = _AllowlistedRedirectHandler("fixtures.example") + original = Request("https://fixtures.example/reference.zip") + + with pytest.raises(ValueError, match="reference fixture URL"): + handler.redirect_request( + original, + None, + 302, + "Found", + {}, + "https://169.254.169.254/latest/meta-data", + ) + + +@pytest.mark.parametrize("failure", ["hash", "redirect", "member-hash", "member-size"]) +def test_download_verified_reference_stem_rejects_untrusted_archive_data( + failure: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Fail closed for changed bytes, insecure redirects, and ZIP size drift.""" + member_payload = b"known vocal stem" + archive_payload = _archive_payload("vocals.wav", member_payload) + fixture = _fixture_for_archive(archive_payload, member_payload=member_payload) + if failure == "hash": + fixture = replace(fixture, reference_archive_sha256="0" * 64) + if failure == "member-hash": + fixture = replace(fixture, reference_member_sha256="0" * 64) + if failure == "member-size": + fixture = replace(fixture, reference_member_bytes=len(member_payload) + 1) + final_url = ( + "http://fixtures.example/reference.zip" + if failure == "redirect" + else fixture.reference_archive_url + ) + + def fake_open_fixture_url(request: object, expected_host: str) -> _FakeResponse: + """Return controlled archive bytes for a negative integrity test.""" + return _FakeResponse(archive_payload, final_url) + + monkeypatch.setattr("known_stem_benchmark._open_fixture_url", fake_open_fixture_url) + + with pytest.raises(ValueError, match="reference fixture"): + download_verified_reference_stem(fixture, tmp_path) + + assert not (tmp_path / "known-reference-source.zip").exists() + assert not (tmp_path / "known-reference-vocals.wav").exists() + + +@pytest.mark.parametrize( + ("field", "value"), + [ + ("target_stem", "../outside"), + ("reference_member", "../vocals.wav"), + ("reference_archive_bytes", 65 * 1024 * 1024), + ], +) +def test_download_verified_reference_stem_rejects_unsafe_fixture_definition( + field: str, + value: str | int, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Reject unsafe path fields and resource bounds before opening a URL.""" + member_payload = b"known vocal stem" + archive_payload = _archive_payload("vocals.wav", member_payload) + fixture = replace( + _fixture_for_archive(archive_payload, member_payload=member_payload), + **{field: value}, + ) + network_opened = False + + def fake_open_fixture_url(request: object, expected_host: str) -> _FakeResponse: + """Record an unexpected network request from invalid fixture data.""" + nonlocal network_opened + network_opened = True + return _FakeResponse(archive_payload, fixture.reference_archive_url) + + monkeypatch.setattr("known_stem_benchmark._open_fixture_url", fake_open_fixture_url) + + with pytest.raises(ValueError, match="reference fixture"): + download_verified_reference_stem(fixture, tmp_path) + + assert network_opened is False + + +def test_download_verified_creator_master_authenticates_exact_file( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Authenticate the creator master independently from the dry vocal archive.""" + member_payload = b"known vocal stem" + archive_payload = _archive_payload("vocals.wav", member_payload) + master_payload = b"exact creator master" + fixture = replace( + _fixture_for_archive(archive_payload, member_payload=member_payload), + creator_master_sha256=hashlib.sha256(master_payload).hexdigest(), + creator_master_bytes=len(master_payload), + ) + + def fake_open_fixture_url(request: object, expected_host: str) -> _FakeResponse: + """Return the pinned creator master without using the network.""" + assert expected_host == fixture.creator_master_host + return _FakeResponse(master_payload, fixture.creator_master_url) + + monkeypatch.setattr("known_stem_benchmark._open_fixture_url", fake_open_fixture_url) + + master_path = download_verified_creator_master(fixture, tmp_path) + + assert master_path == tmp_path / "known-reference-master.mp3" + assert master_path.read_bytes() == master_payload + + +def test_align_known_stem_through_master_composes_two_global_offsets() -> None: + """Use creator-master identity and vocal alignment without shifting model outputs.""" + rng = np.random.default_rng(20260809) + sample_rate = 1_000 + reference = np.zeros(4_000, dtype=np.float64) + reference[2_000:3_000] = rng.standard_normal(1_000) + master_lag = 123 + master = 0.001 * rng.standard_normal(4_500) + master[master_lag : master_lag + reference.size] += reference + youtube_lag = 211 + youtube = 0.001 * rng.standard_normal(5_000) + youtube[youtube_lag : youtube_lag + master.size] += master + + aligned = align_known_stem_through_master( + youtube, + master, + reference, + sample_rate=sample_rate, + window_seconds=0.8, + max_lag_seconds=0.5, + ) + + assert aligned.youtube_to_master_lag_samples == youtube_lag + assert aligned.master_to_reference_lag_samples == master_lag + assert aligned.identity_correlation > 0.99 + assert aligned.mixture.shape == aligned.reference.shape == (800,) + expected_start = aligned.reference_start + master_lag + youtube_lag + np.testing.assert_allclose(aligned.mixture, youtube[expected_start : expected_start + 800]) + + +def test_required_root_suite_explicitly_excludes_live_youtube_marker() -> None: + """Keep external YouTube access out of required CI while retaining offline tests.""" + repo_root = Path(__file__).resolve().parents[3] + runner = (repo_root / "scripts/checks/run_root_tests.mjs").read_text(encoding="utf-8") + + assert '"-m",\n "not youtube_stem_e2e"' in runner + + +def _assert_real_youtube_known_stem_separation(root: Path) -> None: + """Run the live benchmark inside an ephemeral, caller-owned media directory.""" + fixture = BRAD_SUCKS_FIXTURE + reference_path = download_verified_reference_stem(fixture, root) + master_path = download_verified_creator_master(fixture, root) + youtube_dir = root / "youtube" + youtube_dir.mkdir() + + download = download_youtube_audio(fixture.youtube_url, str(youtube_dir)) + assert download["ok"], f"YouTube fixture failed: {download.get('error', {}).get('code')}" + metadata = download["metadata"] + assert metadata["id"] == fixture.video_id + mixture_path = Path(metadata["filepath"]).resolve(strict=True) + assert mixture_path.is_relative_to(youtube_dir.resolve()) + + import librosa + + mixture, sample_rate = librosa.load(mixture_path, sr=44_100, mono=True) + creator_master, master_sample_rate = librosa.load(master_path, sr=44_100, mono=True) + reference, reference_sample_rate = librosa.load(reference_path, sr=44_100, mono=True) + assert sample_rate == master_sample_rate == reference_sample_rate == 44_100 + decoded_master_duration = creator_master.size / sample_rate + assert abs(decoded_master_duration - fixture.creator_master_duration_seconds) <= 0.05, ( + "Pinned creator-master decode duration drifted" + ) + duration_drift = abs((mixture.size - creator_master.size) / sample_rate) + assert duration_drift <= MAX_MASTER_DURATION_DRIFT_SECONDS, ( + f"YouTube/master duration drift was {duration_drift:.3f} s" + ) + aligned = align_known_stem_through_master( + mixture, + creator_master, + reference, + sample_rate=sample_rate, + window_seconds=12.0, + max_lag_seconds=10.0, + ) + assert aligned.identity_correlation >= MIN_MASTER_IDENTITY_CORRELATION, ( + f"YouTube/master identity correlation was only {aligned.identity_correlation:.4f}" + ) + + scored_mix_path = root / "youtube-known-stem-window.wav" + sf.write(scored_mix_path, aligned.mixture, sample_rate, subtype="PCM_24") + separator = AudioStemSeparator( + AudioSeparationConfig( + target_sample_rate=sample_rate, + max_file_bytes=10 * 1024 * 1024, + max_duration_seconds=13.0, + shifts=0, + ) + ) + separation = separator.separate(scored_mix_path) + stems = separation["stems"] + + assert set(stems) == {"vocals", "bass", "drums", "other"} + assert all(stem.shape == aligned.reference.shape for stem in stems.values()) + assert all(np.isfinite(stem).all() for stem in stems.values()) + + scores = {name: zero_mean_si_sdr(stem, aligned.reference) for name, stem in stems.items()} + assert np.isfinite(np.asarray(list(scores.values()))).all(), "Stem SI-SDR was non-finite" + vocal_score = scores["vocals"] + best_wrong_score = max(score for name, score in scores.items() if name != "vocals") + improvement = si_sdr_improvement(stems["vocals"], aligned.mixture, aligned.reference) + assignment_margin = vocal_score - best_wrong_score + evidence = ( + f"video={fixture.video_id}; model=htdemucs/955717e8-8726e21a; " + f"identity_correlation={aligned.identity_correlation:.4f}; " + f"youtube_master_lag={aligned.youtube_to_master_lag_samples}; " + f"master_vocal_lag={aligned.master_to_reference_lag_samples}; " + f"si_sdri={improvement:.3f}dB; assignment_margin={assignment_margin:.3f}dB" + ) + + assert np.isfinite(improvement), "Vocal SI-SDR improvement was non-finite" + assert np.isfinite(assignment_margin), "Vocal stem assignment margin was non-finite" + assert improvement >= MIN_VOCAL_SI_SDR_IMPROVEMENT_DB, ( + f"Vocal SI-SDR improvement missed the provisional threshold; {evidence}" + ) + assert assignment_margin >= MIN_VOCAL_ASSIGNMENT_MARGIN_DB, ( + f"Vocal stem assignment margin missed the provisional threshold; {evidence}" + ) + + +@pytest.mark.youtube_stem_e2e +@pytest.mark.skipif( + os.environ.get("BANDSCOPE_RUN_YOUTUBE_STEM_E2E") != "1", + reason=( + "live YouTube, the pinned public stem archive, ffmpeg, and Demucs weights are required; " + "set BANDSCOPE_RUN_YOUTUBE_STEM_E2E=1" + ), +) +def test_real_youtube_audio_separates_the_known_vocal_stem(tmp_path: Path) -> None: + """Download a real YouTube mix and verify Demucs against its known vocal stem.""" + with tempfile.TemporaryDirectory(prefix="known-stem-media-", dir=tmp_path) as media_dir: + _assert_real_youtube_known_stem_separation(Path(media_dir)) + + assert not any(tmp_path.iterdir()) diff --git a/supply-chain/supplemental-component-inventory.json b/supply-chain/supplemental-component-inventory.json index 784d90d57..8f4b03e05 100644 --- a/supply-chain/supplemental-component-inventory.json +++ b/supply-chain/supplemental-component-inventory.json @@ -1,30 +1,46 @@ { - "version": 1, + "version": 2, "generatedBy": "repo-maintained inventory", - "bundledBinaries": [ + "packageManagedTools": [ { "name": "yt-dlp", - "version": ">=2026.3.17", + "version": ">=2026.7.4", "sourceUrl": "https://pypi.org/project/yt-dlp/", "license": "Unlicense", "storagePath": "services/analysis-engine/uv.lock", - "releaseUsage": "Used by analysis-engine to extract audio from YouTube URLs." + "distribution": "python-package", + "releaseUsage": "Used by the analysis engine to extract public YouTube audio after strict URL validation." + } + ], + "operatorProvidedTools": [ + { + "name": "ffmpeg", + "version": "operator-managed supported release", + "sourceUrl": "https://ffmpeg.org/download.html", + "license": "LGPL-2.1-or-later or GPL-2.0-or-later, depending on build configuration", + "storagePath": "system PATH; not bundled by BandScope", + "distribution": "operator-provided", + "releaseUsage": "Required by yt-dlp audio extraction and media decoding; release preflight records the resolved version." } ], "modelArtifacts": [ { - "name": "bandsplit-v1-profile", - "version": "1.0.0", - "sourceUrl": "local-repo://services/analysis-engine/src/bandscope_analysis/separation/model_weights/bandsplit-v1.json", - "license": "Proprietary", - "checksum": "sha256:ced4ae5c9077aace1694b6fafee1877e46e836e293545dcb6ea06cb579984254", - "storagePath": "services/analysis-engine/src/bandscope_analysis/separation/model_weights/bandsplit-v1.json", - "releaseUsage": "Local-first lightweight profile used by analysis-engine stem separation.", - "verification": "SHA256 verified in bandscope_analysis.separation.audio_separator.AudioStemSeparator._load_model_profile" + "name": "Hybrid Transformer Demucs four-source weights", + "runtimeModelName": "htdemucs", + "version": "demucs-4.0.1-signature-955717e8", + "sourceUrl": "https://dl.fbaipublicfiles.com/demucs/hybrid_transformer/955717e8-8726e21a.th", + "license": "No separate model-weight redistribution grant identified; runtime retrieval only, not bundled", + "checksum": "sha256:8726e21a993978c7ba086d3872e7608d7d5bfca646ca4aca459ffda844faa8b4", + "sizeBytes": 84141911, + "storagePath": "user runtime cache managed by torch.hub/demucs; not committed or bundled", + "distribution": "runtime-cache", + "releaseUsage": "Loaded locally on supported platforms to separate vocals, bass, drums, and other stems.", + "verification": "Demucs 4.0.1 checks the filename hash prefix 8726e21a before deserialization; the full SHA-256 and byte size were independently recorded on 2026-08-09. Full-hash pre-load enforcement remains a release blocker tracked by ADR-0001." } ], "notes": [ - "Add ffmpeg, yt-dlp, model weights, or sidecar assets here before they ship.", - "Track source URL, version, checksum, license, storage path, and release usage for every item." + "The retired bandsplit-v1 FFT profile is not a production separator artifact and must not reappear.", + "Track source URL, version, full checksum, byte size, license, distribution, storage path, and release usage for every model artifact.", + "Runtime retrieval does not authorize redistribution; release packaging must fail if it attempts to bundle an artifact without an explicit license decision." ] } From 1458f707b9c87e30ce44c027ccac9cda1bd7d315 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 9 Aug 2026 21:51:38 +0900 Subject: [PATCH 02/34] docs(quality): record exact-head live failure --- docs/TRD.md | 6 ++++++ docs/doctoring/real-audio-accuracy-acceptance.md | 3 +++ docs/documentation-coverage-matrix.md | 1 + docs/engineering/youtube-known-stem-validation.md | 7 +++++++ 4 files changed, 17 insertions(+) diff --git a/docs/TRD.md b/docs/TRD.md index 6b45c04c2..52be4134b 100644 --- a/docs/TRD.md +++ b/docs/TRD.md @@ -121,6 +121,12 @@ correlation was only 0.016856, proving it was not a valid identity gate. These v the provisional +0.5/+3.0 sentinels and the separate master identity design; they are not an authorized YouTube pass. +After the correction, the byte-identical implementation tree published on GitHub as exact commit +`6e937a34f9036d92e909db3ce8848a5c39dc8e3b` passed the full quickcheck. Its live retry +authenticated all three reference artifacts and the pre-provisioned model full hash, then failed +closed at production YouTube intake with HTTP 502 after 65.49 seconds. No identity or separation +score was emitted. + ## Traceability `docs/documentation-coverage-matrix.md` maps product requirements and ADRs to modules, tests, and diff --git a/docs/doctoring/real-audio-accuracy-acceptance.md b/docs/doctoring/real-audio-accuracy-acceptance.md index 79a675a62..c6d96ed80 100644 --- a/docs/doctoring/real-audio-accuracy-acceptance.md +++ b/docs/doctoring/real-audio-accuracy-acceptance.md @@ -108,6 +108,9 @@ The live attempt failed at YouTube HTTP 502 before model execution, so no passin Creator-master calibration produced deterministic +1.752 dB SI-SDR improvement and +7.631 dB assignment margin, while dry-vocal/mix correlation was only 0.016856. Those results support the provisional sentinel and separate identity check, not a YouTube pass or release-blocking threshold. +The corrected byte-identical implementation tree published on GitHub as exact commit +`6e937a34f9036d92e909db3ce8848a5c39dc8e3b` later passed full quickcheck, but its clean live +retry again failed at YouTube HTTP 502 before separation; live success therefore remains absent. ## References diff --git a/docs/documentation-coverage-matrix.md b/docs/documentation-coverage-matrix.md index 45e654611..65d2c1955 100644 --- a/docs/documentation-coverage-matrix.md +++ b/docs/documentation-coverage-matrix.md @@ -53,6 +53,7 @@ acceptance layer. | Date | Commit under test | Offline contract | Live result | Classification | |---|---|---|---|---| | 2026-08-09 | `5a3648a11d9097b8da48bb4a3ccbd97986aec25b` | 13 passed | Reference archive verified; YouTube download failed with HTTP 502 before separation; no score | Exact failure evidence, not a pass | +| 2026-08-09 | `6e937a34f9036d92e909db3ce8848a5c39dc8e3b` (published byte-identical implementation tree) | Full quickcheck: 680 Python passed, 24 skipped, live marker deselected; 100% source coverage | Archive, extracted vocal, creator master, and pre-provisioned model hash verified; production YouTube download failed with HTTP 502 after 65.49 s; no score | Exact implementation-head failure evidence, not a pass | Separate creator-master calibration on that environment measured `shifts=0` vocal SI-SDR improvement +1.752 dB and assignment margin +7.631 dB. Dry-vocal/mix correlation was 0.016856, so diff --git a/docs/engineering/youtube-known-stem-validation.md b/docs/engineering/youtube-known-stem-validation.md index b387f2d46..31908379a 100644 --- a/docs/engineering/youtube-known-stem-validation.md +++ b/docs/engineering/youtube-known-stem-validation.md @@ -116,6 +116,13 @@ hash, creator-master authentication, composed-offset recovery, and explicit requ the live marker. A creator-master-only calibration produced the provisional scores above without calling YouTube; it is calibration evidence, not exact-candidate success. +The byte-identical implementation tree published on GitHub as exact commit +`6e937a34f9036d92e909db3ce8848a5c39dc8e3b` passed the full quickcheck. A clean live retry +authenticated the archive, extracted vocal, creator master, and pre-provisioned htdemucs full +SHA-256. Production YouTube intake again failed closed with `download_failed` after HTTP 502 in +65.49 seconds, before separation. It produced no identity correlation or SI-SDR score and remains +exact implementation-head failure evidence, not a live pass. + ## Security Notes ### Attack surface From bab2bb9924223aec8d2cce61b72ce25310a5efdd Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 9 Aug 2026 22:16:14 +0900 Subject: [PATCH 03/34] docs(governance): canonicalize review evidence --- CHANGELOG.md | 2 ++ CLAUDE.md | 2 +- CONTRIBUTING.md | 4 +++- docs/documentation-coverage-matrix.md | 5 ++++- docs/repository/bootstrap-plan.md | 10 +++++++--- docs/repository/gitflow.md | 7 +++++-- docs/repository/governance.md | 9 +++++++-- docs/workflow/github-bootstrap-execution-policy.md | 4 +++- scripts/checks/verify_docs.py | 9 +++++++++ 9 files changed, 41 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a3f9e37b..df5b36a6e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,8 @@ - Kept YouTube TLS verification enabled while honoring OS-managed CA roots used by managed desktop environments. +- Reconciled stale CodeRabbit-gate wording with the canonical stable-check and review-equivalent + policy; a rate-limited or status-only success is not treated as a completed review. ## [0.1.3] - 2026-04-29 diff --git a/CLAUDE.md b/CLAUDE.md index 9ad62939d..fe41df8f6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -74,7 +74,7 @@ Supporting packages: ## Key conventions - Coverage is a hard gate: the Python engine requires 100% test coverage and 100% docstring coverage (Ruff `D100`–`D107` across `src`, `tests`, and repo scripts). Exported TypeScript declarations in `packages/shared-types` and `apps/desktop/src` require JSDoc with a description; `no-console` is an error. -- Gitflow: `develop` is the default branch; `feature/*` targets `develop`, `main` is the protected release branch. Direct pushes to protected branches are not allowed, and every merge needs the required checks plus a passing CodeRabbit review (see `CONTRIBUTING.md` and `docs/repository/gitflow.md`). +- Gitflow: `develop` is the default branch; `feature/*` targets `develop`, `main` is the protected release branch. Direct pushes to protected branches are not allowed, and every merge needs the stable checks and review-equivalent policy in `docs/security/github-required-checks.md`. CodeRabbit is requested by default and actionable findings must be addressed, but a stale or rate-limited hosted status is not review evidence. - The PR template (`.github/PULL_REQUEST_TEMPLATE.md`) requires a quickcheck confirmation, `Security Notes` (attack surface, trust boundary, mitigations, test points), a dependency/supply-chain checklist, and i18n impact. - i18n: the UI ships Korean and English locales (`apps/desktop/src/locales/ko`, `en`). Any user-visible string change must update both. - Documents under `docs/plans/` must include `Security Notes`; `scripts/checks/verify_security_notes.py` enforces this mechanically. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fdf6bd787..59dc9c079 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -16,7 +16,9 @@ Read `docs/repository/gitflow.md` before opening a PR. ## Pull requests are mandatory - direct push to `main` or `develop` is not allowed -- every protected-branch merge requires a passing `CodeRabbit` check +- every protected-branch merge requires the stable checks and review-equivalent policy in + `docs/security/github-required-checks.md`; request CodeRabbit and address its current actionable + findings, but do not treat a stale or rate-limited hosted status as review evidence - all review conversations must be resolved before merge - required checks must stay green; do not bypass them diff --git a/docs/documentation-coverage-matrix.md b/docs/documentation-coverage-matrix.md index 65d2c1955..4cd4b6eb7 100644 --- a/docs/documentation-coverage-matrix.md +++ b/docs/documentation-coverage-matrix.md @@ -34,6 +34,7 @@ acceptance layer. | Operations/release | runbook and release policy | Preflight, evidence, triage, rollback, and blocking conditions covered. | Platform matrix and live pass are incomplete. | | Supply chain | supplemental inventory and dependency policy | Retired model removed; exact runtime artifact, ffmpeg status, and hash recorded. | Weight redistribution rights and pre-load enforcement unresolved. | | Automation | active CWL autonomous loop and `docs/workflow/pr-review-merge-scheduler.md` | BandScope continuity and no-status-only termination are covered without creating a competing writer. | Dedicated BandScope loop remains paused due writer topology/active-task capacity. | +| Review governance | `docs/security/github-required-checks.md`, governance, gitflow, contributing, bootstrap policy | Stable required checks and review-equivalent policy are canonical; CodeRabbit is requested and actionable findings are addressed, but stale/rate-limited status-only success is not a completed review. | A provider rate limit can still defer review, blocking only merge. | ## Requirement-to-evidence traceability @@ -63,7 +64,9 @@ YouTube and is not a live pass. ## Machine-checkable contract `scripts/checks/verify_docs.py` requires the canonical index, PRD, TRD, ADR index and records, -diagram authority, and this matrix, and checks cross-links from architecture and the index. +diagram authority, and this matrix; checks cross-links from architecture and the index; and requires +contributing, governance, gitflow, bootstrap, and GitHub bootstrap policy to link the canonical +required-check authority so review policy cannot silently fork. `scripts/checks/verify_supply_chain.py` derives the configured separator model name and rejects an inventory that lacks it, uses the retired bandsplit profile, omits required fields, lacks a full SHA-256/positive byte size, or uses a non-HTTPS model source. diff --git a/docs/repository/bootstrap-plan.md b/docs/repository/bootstrap-plan.md index b16f458a1..e295e6449 100644 --- a/docs/repository/bootstrap-plan.md +++ b/docs/repository/bootstrap-plan.md @@ -17,7 +17,8 @@ - direct push blocked - PR required -- passing `CodeRabbit` gate required +- review-equivalent policy required; use the current authority in + `docs/security/github-required-checks.md` - conversation resolution required - force push blocked - branch deletion blocked @@ -28,7 +29,6 @@ After workflows exist, require these stable checks on `main` and `develop`: -- `CodeRabbit` - `ci / build-and-test` - `dependency-review` - `security-audit` @@ -48,7 +48,11 @@ After bootstrap creates `develop`, the repository default branch is `develop`. ` ## Review substitution rule -For this harness baseline, a passing `CodeRabbit` check replaces GitHub's built-in approving-review gate. Protected branches still require PRs, conversation resolution, and all required checks. +The original bootstrap assumed a hosted `CodeRabbit` status could replace GitHub's approving-review +gate. Current policy supersedes that assumption: request CodeRabbit, address current actionable +findings, and use the stable checks plus review-equivalent policy in +`docs/security/github-required-checks.md`. A stale, rate-limited, or status-only context is not a +completed review. Protected branches still require PRs and conversation resolution. ## Path note diff --git a/docs/repository/gitflow.md b/docs/repository/gitflow.md index 266e3ac47..aa1a88bfa 100644 --- a/docs/repository/gitflow.md +++ b/docs/repository/gitflow.md @@ -3,7 +3,7 @@ ## Branch roles - `develop`: repository default branch after bootstrap and the protected integration branch -- `main`: release branch, protected, `CodeRabbit` gate required +- `main`: release branch, protected by the canonical stable checks and review-equivalent policy - `feature/*`: short-lived work branches targeting `develop` - `release/*`: release preparation branches targeting `main` - `hotfix/*`: urgent fixes targeting `main`, with follow-up sync back into `develop` @@ -18,5 +18,8 @@ ## Rules - protected branches do not accept direct pushes -- every protected-branch merge requires the `CodeRabbit` gate and the required checks +- every protected-branch merge requires the stable checks, conversation resolution, and + review-equivalent policy in `docs/security/github-required-checks.md` +- request CodeRabbit and address current actionable findings, but do not treat a stale, + rate-limited, or status-only context as a completed review - release and hotfix paths do not bypass dependency, security, SBOM, or release-preflight gates diff --git a/docs/repository/governance.md b/docs/repository/governance.md index 495da5cf7..463e66a62 100644 --- a/docs/repository/governance.md +++ b/docs/repository/governance.md @@ -9,13 +9,18 @@ BandScope is a public GitHub repository. GitHub is the source of truth for code, - `develop` is the repository default branch after bootstrap - `main` is the protected release branch - `develop` is the protected integration branch -- both branches require PR-based merges, a passing `CodeRabbit` gate, conversation resolution, force-push prohibition, branch-deletion prohibition, and admin enforcement +- both branches require PR-based merges, the stable checks and review-equivalent policy in + `docs/security/github-required-checks.md`, conversation resolution, force-push prohibition, + branch-deletion prohibition, and admin enforcement ## Review policy - every merge into `main` or `develop` goes through a PR - CODEOWNERS routes review to the right owners -- a passing `CodeRabbit` check substitutes for GitHub's built-in approving-review gate in this harness baseline +- CodeRabbit is the default requested AI review and its current actionable findings must be + addressed; its hosted status context is not itself a stable required check because it can be + stale or rate-limited +- a status-only success without a completed review is not review-equivalent evidence - self-approval, direct push, and arbitrary rule weakening are out of policy ## No direct push policy diff --git a/docs/workflow/github-bootstrap-execution-policy.md b/docs/workflow/github-bootstrap-execution-policy.md index 736b695aa..69823249c 100644 --- a/docs/workflow/github-bootstrap-execution-policy.md +++ b/docs/workflow/github-bootstrap-execution-policy.md @@ -87,7 +87,9 @@ Do not treat these as TODOs, later hardening, or optional recommendations. ### Phase 5. Initial protection baseline - apply PR-only merge -- require `CodeRabbit` as the review-equivalent gate +- request CodeRabbit and require the current stable-check/review-equivalent policy in + `docs/security/github-required-checks.md`; do not equate a stale or rate-limited status context + with a completed review - disable force push - restrict deletion - checks can be tightened later after workflows exist diff --git a/scripts/checks/verify_docs.py b/scripts/checks/verify_docs.py index 2ecfea178..8df901ed7 100644 --- a/scripts/checks/verify_docs.py +++ b/scripts/checks/verify_docs.py @@ -48,6 +48,7 @@ ] REQUIRED_REFERENCES = { + Path("CONTRIBUTING.md"): ["docs/security/github-required-checks.md"], Path("README.md"): [ "docs/security/app-security.md", "docs/security/dependency-policy.md", @@ -79,6 +80,14 @@ "docs/architecture/diagrams.md", "docs/documentation-coverage-matrix.md", ], + Path("docs/repository/bootstrap-plan.md"): [ + "docs/security/github-required-checks.md" + ], + Path("docs/repository/gitflow.md"): ["docs/security/github-required-checks.md"], + Path("docs/repository/governance.md"): ["docs/security/github-required-checks.md"], + Path("docs/workflow/github-bootstrap-execution-policy.md"): [ + "docs/security/github-required-checks.md" + ], } From 92f0da6d2005dbbc5d99b840b10b8bc64eff6812 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 9 Aug 2026 22:33:20 +0900 Subject: [PATCH 04/34] docs(governance): define exact-head review evidence --- CHANGELOG.md | 3 ++- docs/documentation-coverage-matrix.md | 2 +- docs/security/github-required-checks.md | 23 +++++++++++++++++++++++ scripts/checks/verify_docs.py | 15 +++++++++------ 4 files changed, 35 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index df5b36a6e..729ce684c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,7 +21,8 @@ - Kept YouTube TLS verification enabled while honoring OS-managed CA roots used by managed desktop environments. - Reconciled stale CodeRabbit-gate wording with the canonical stable-check and review-equivalent - policy; a rate-limited or status-only success is not treated as a completed review. + policy; qualifying evidence is now defined against the exact current head, and a rate-limited, + status-only, author, or predecessor review is not treated as completed review evidence. ## [0.1.3] - 2026-04-29 diff --git a/docs/documentation-coverage-matrix.md b/docs/documentation-coverage-matrix.md index 4cd4b6eb7..409087b2b 100644 --- a/docs/documentation-coverage-matrix.md +++ b/docs/documentation-coverage-matrix.md @@ -34,7 +34,7 @@ acceptance layer. | Operations/release | runbook and release policy | Preflight, evidence, triage, rollback, and blocking conditions covered. | Platform matrix and live pass are incomplete. | | Supply chain | supplemental inventory and dependency policy | Retired model removed; exact runtime artifact, ffmpeg status, and hash recorded. | Weight redistribution rights and pre-load enforcement unresolved. | | Automation | active CWL autonomous loop and `docs/workflow/pr-review-merge-scheduler.md` | BandScope continuity and no-status-only termination are covered without creating a competing writer. | Dedicated BandScope loop remains paused due writer topology/active-task capacity. | -| Review governance | `docs/security/github-required-checks.md`, governance, gitflow, contributing, bootstrap policy | Stable required checks and review-equivalent policy are canonical; CodeRabbit is requested and actionable findings are addressed, but stale/rate-limited status-only success is not a completed review. | A provider rate limit can still defer review, blocking only merge. | +| Review governance | `docs/security/github-required-checks.md`, governance, gitflow, contributing, bootstrap policy | Stable checks and review are cumulative; qualifying evidence is an exact-head completed CodeRabbit artifact or exact-head independent non-author `APPROVED` review. Status-only, rate-limited, author, or predecessor evidence is excluded. | A provider rate limit can still defer review, blocking only merge. | ## Requirement-to-evidence traceability diff --git a/docs/security/github-required-checks.md b/docs/security/github-required-checks.md index eb62fdae3..1a0a80930 100644 --- a/docs/security/github-required-checks.md +++ b/docs/security/github-required-checks.md @@ -74,5 +74,28 @@ BandScope still requests CodeRabbit on PRs and treats it as the default AI revie However, the hosted `CodeRabbit` status context has shown repeated stale `PENDING` and stale `CHANGES_REQUESTED` states after all actionable review was cleared. Because of that operational behavior, protected branches require the stable repository-owned checks above rather than the external `CodeRabbit` status context itself. +## Review-equivalent evidence + +Review evidence is evaluated separately from required checks and conversation resolution. Before a +protected-branch merge, the exact current PR head SHA must have at least one of these durable review +artifacts: + +- a completed CodeRabbit review whose artifact identifies the exact current PR head SHA or its + exact base-to-head range, is not rate-limited or failed, and has no valid actionable finding or + unresolved review thread; or +- an `APPROVED` GitHub review from an eligible independent non-author reviewer, recorded against + the exact current PR head SHA, with no valid unresolved review thread. + +Any new commit makes predecessor-head review evidence stale. The current head must be reviewed +again unless repository policy provides an explicit, durable equivalent bound to that same head. +Status contexts, check runs, reactions, issue comments that only request, acknowledge, queue, +rate-limit, or fail a review, author/self reviews, and summaries without an exact-head binding are +not review-equivalent evidence. A completed review does not replace any stable required check, and +green checks do not replace a completed review. + +If neither qualifying route is currently available, defer that merge, keep the PR open, and +continue other safe repository work. Do not weaken protection, invent a reviewer, self-approve, or +reinterpret a provider status as review evidence. + Missing repository state should trigger GitHub bootstrap per `docs/workflow/github-bootstrap-execution-policy.md`. Only missing admin permissions or platform capability should be reported as `BLOCKED`. diff --git a/scripts/checks/verify_docs.py b/scripts/checks/verify_docs.py index 8df901ed7..47678a37d 100644 --- a/scripts/checks/verify_docs.py +++ b/scripts/checks/verify_docs.py @@ -80,11 +80,16 @@ "docs/architecture/diagrams.md", "docs/documentation-coverage-matrix.md", ], - Path("docs/repository/bootstrap-plan.md"): [ - "docs/security/github-required-checks.md" - ], + Path("docs/repository/bootstrap-plan.md"): ["docs/security/github-required-checks.md"], Path("docs/repository/gitflow.md"): ["docs/security/github-required-checks.md"], Path("docs/repository/governance.md"): ["docs/security/github-required-checks.md"], + Path("docs/security/github-required-checks.md"): [ + "## Review-equivalent evidence", + "exact current PR head SHA", + "independent non-author", + "Status contexts, check runs, reactions, issue comments", + "defer that merge", + ], Path("docs/workflow/github-bootstrap-execution-policy.md"): [ "docs/security/github-required-checks.md" ], @@ -93,9 +98,7 @@ def documentation_violations(root: Path = Path(".")) -> list[str]: """Return missing canonical files and broken authority-reference violations.""" - violations = [ - f"missing file: {path}" for path in REQUIRED_PATHS if not (root / path).exists() - ] + violations = [f"missing file: {path}" for path in REQUIRED_PATHS if not (root / path).exists()] for path, required_texts in REQUIRED_REFERENCES.items(): absolute_path = root / path if not absolute_path.exists(): From 3947f636de8fa0d2b35122f903ee027a67996451 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 9 Aug 2026 23:30:45 +0900 Subject: [PATCH 05/34] fix(security): verify benchmark runtime artifacts --- ARCHITECTURE.md | 10 +- CHANGELOG.md | 10 +- CLAUDE.md | 4 +- apps/desktop/package.json | 2 +- docs/PRD.md | 6 +- docs/TRD.md | 25 ++- ...e-separation-runtime-and-model-delivery.md | 19 +-- docs/adr/README.md | 2 +- docs/architecture/diagrams.md | 19 ++- docs/architecture/overview.md | 9 +- docs/documentation-coverage-matrix.md | 16 +- docs/engineering/acceptance-criteria.md | 8 +- .../youtube-known-stem-validation.md | 33 ++-- docs/operations/deploy-runbook.md | 6 +- .../plans/2026-03-28-ml-engine-integration.md | 8 +- docs/repository/bootstrap-plan.md | 1 + .../github-bootstrap-execution-policy.md | 2 +- package-lock.json | 20 +-- scripts/checks/verify_docs.py | 15 ++ scripts/checks/verify_supply_chain.py | 22 ++- .../src/bandscope_analysis/api.py | 9 ++ .../separation/audio_separator.py | 69 ++++++-- .../src/bandscope_analysis/youtube.py | 45 +++++- .../tests/known_stem_benchmark.py | 6 +- services/analysis-engine/tests/test_api.py | 9 ++ .../tests/test_documentation_policy.py | 14 ++ .../analysis-engine/tests/test_separation.py | 151 +++++++++++++++++- .../tests/test_supply_chain_policy.py | 88 ++++++++++ .../analysis-engine/tests/test_youtube.py | 116 +++++++++++++- .../tests/test_youtube_stem_e2e.py | 61 ++++++- .../supplemental-component-inventory.json | 6 +- 31 files changed, 688 insertions(+), 123 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 673bf10e5..db2a6e76c 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -93,13 +93,13 @@ Last updated: 2026-08-09 model. - Demucs random temporal shifts are disabled (`shifts=0`) so the same bytes and model produce reproducible local analysis and benchmark evidence. -- Model inference is local after provisioning, but the current first load may fetch the exact - official weight artifact into a user runtime cache. It is not bundled with the repository or - release artifacts. +- Model inference is local after operator provisioning. The runtime requires the exact official + weight artifact in a local cache and never downloads it; missing, symlinked, wrong-sized, or + full-SHA-mismatched bytes fail before Demucs/torch deserialization. - The exact signature, source URL, full SHA-256, byte size, distribution status, and model-rights uncertainty are tracked in `supply-chain/supplemental-component-inventory.json` and ADR-0001. -- Full-SHA verification before torch deserialization and a model-rights decision remain release - blockers. Demucs' filename-prefix check alone is not promoted to full release evidence. +- Full-SHA verification before torch deserialization is implemented. A recorded model-rights and + permitted-delivery decision remains a release blocker; the official weights are not bundled. - Current dependency markers exclude Demucs on macOS Intel; unsupported platforms must surface the existing safe fallback rather than pretending to separate stems. diff --git a/CHANGELOG.md b/CHANGELOG.md index 729ce684c..a7f8aa588 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,8 +18,14 @@ ### Fixed -- Kept YouTube TLS verification enabled while honoring OS-managed CA roots used by managed - desktop environments. +- Kept YouTube TLS verification enabled while retaining yt-dlp's maintained CA-bundle fallback for + minimal containers without a configured system trust store. +- Raised `pdfjs-dist`, `nanoid`, and `undici` to patched versions for the current high-severity + advisories and added a mutation-sensitive lockfile floor contract. +- Made htdemucs inference fail closed before deserialization unless the exact local artifact passes + filename, non-symlink, byte-size, and full-SHA-256 checks; the runtime no longer downloads weights. +- Bound release/live benchmark ffmpeg execution to an operator-provided absolute path and full + SHA-256 while preserving fail-closed provider and model evidence. - Reconciled stale CodeRabbit-gate wording with the canonical stable-check and review-equivalent policy; qualifying evidence is now defined against the exact current head, and a rate-limited, status-only, author, or predecessor review is not treated as completed review evidence. diff --git a/CLAUDE.md b/CLAUDE.md index fe41df8f6..b6ee31220 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -59,8 +59,8 @@ Three layers, decoupled through shared contracts: - `apps/desktop/src-tauri/src/main.rs` — the Rust orchestration boundary. Tauri commands (`start_analysis_job`, `get_analysis_job_status`, `select_local_audio_source`, `import_youtube_url`) validate untrusted input (project IDs, file paths, URLs) and spawn the Python engine as a subprocess. There is no loopback HTTP listener and no network path for local analysis. - `services/analysis-engine` — Python package `bandscope_analysis` (librosa/numpy). Entry point `cli.py` reads a JSON job request on stdin and prints a structured job-status JSON envelope on stdout (`--progress-jsonl` streams progress lines). `api.py` orchestrates the pipeline across the `separation`, `sections`, `roles`, `chords`, `ranges`, `temporal`, `transcription`, and `youtube` modules. - Production source separation uses `htdemucs` on supported platforms. The exact runtime model - artifact is inventoried but not bundled; current first use may retrieve it, and full-hash pre-load - enforcement remains a documented release blocker. The active known-stem test crosses the + artifact is inventoried but not bundled; operators must provision it locally, and production + verifies its byte size and full SHA-256 before local-only Demucs loading. The active known-stem test crosses the production YouTube and separator boundaries; see `docs/TRD.md` and the operator guide. Data flow: React UI → Tauri IPC command → Rust validation + Python subprocess over stdin/stdout → job status and progress events emitted back to the UI. diff --git a/apps/desktop/package.json b/apps/desktop/package.json index e7685d6f0..e09719b22 100644 --- a/apps/desktop/package.json +++ b/apps/desktop/package.json @@ -20,7 +20,7 @@ "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", "lucide-react": "^1.24.0", - "pdfjs-dist": "6.1.200", + "pdfjs-dist": "6.2.108", "react": "^19.2.4", "react-dom": "^19.2.7", "sonner": "^2.0.7", diff --git a/docs/PRD.md b/docs/PRD.md index 0d305e630..36430d272 100644 --- a/docs/PRD.md +++ b/docs/PRD.md @@ -10,9 +10,9 @@ role-specific harmony and range, groove and entry cues, separated stem previews, rehearsal priority. It serves band leaders and players who need actionable preparation without a DAW or notation-grade transcription workflow. -The current conversation adds one essential proof obligation: BandScope must demonstrate that its -production YouTube intake and production separator improve a real, known source rather than merely -returning plausible-looking arrays or synthetic demo output. +Issue #770 and ADR-0002 establish one essential proof obligation: BandScope must demonstrate that +its production YouTube intake and production separator improve a real, known source rather than +merely returning plausible-looking arrays or synthetic demo output. This is a bounded source-separation slice of GitHub issue #770, not completion of its broader harmony, beat/tempo, structure, range, cue, confidence, public-corpus, private-corpus, manifest, diff --git a/docs/TRD.md b/docs/TRD.md index 52be4134b..5cb917bc4 100644 --- a/docs/TRD.md +++ b/docs/TRD.md @@ -28,7 +28,7 @@ and data-flow views are in `docs/architecture/diagrams.md`. | TRD-KS-008 | Reject candidate drift when YouTube/master duration differs by > 1.0 s or aligned identity correlation is < 0.90. | Pre-inference live assertions against the pinned finished master. | | TRD-KS-009 | Run deterministic contract/security tests by default and require `BANDSCOPE_RUN_YOUTUBE_STEM_E2E=1` for live network/model execution. | Pytest marker and environment guard. | | TRD-KS-010 | Clean every downloaded/scored artifact on success and failure. | Nested `TemporaryDirectory` plus postcondition. | -| TRD-KS-011 | Bind model identity to inventory and full SHA-256 before release. | Inventory records htdemucs signature `955717e8`, 84,141,911 bytes, and SHA-256 `8726e21a…a8b4`; full-hash pre-load enforcement is an open release blocker. | +| TRD-KS-011 | Bind model identity to inventory and full SHA-256 before release. | `AudioStemSeparator._verified_model_artifact_path` requires the exact local filename, 84,141,911 bytes, and SHA-256 `8726e21a…a8b4` before local-only Demucs loading. | ## Data and class contracts @@ -75,17 +75,16 @@ global random-seed side effect in the test harness. ## Model delivery and supply chain -`AudioStemSeparator` currently asks Demucs 4.0.1 for `htdemucs`. Demucs maps that name to signature -`955717e8` and retrieves `955717e8-8726e21a.th` into a user runtime cache on first use. Demucs checks -the eight-hex filename hash prefix before deserializing. BandScope independently records the full -SHA-256 and exact byte size, but current production code does not yet enforce the full digest before -load. Therefore the model is not bundled and offline inference is guaranteed only after a trusted -cache is provisioned. ADR-0001 makes full-hash pre-load verification and an explicit redistribution -license decision release blockers. +`AudioStemSeparator` maps `htdemucs` to signature `955717e8` and requires the exact local +`955717e8-8726e21a.th` artifact. It rejects a missing/symlinked file, wrong byte count, or full +SHA-256 mismatch before passing a local repository to Demucs, so this runtime has no model-download +path. The model remains unbundled; ADR-0001 still requires an explicit rights and permitted-delivery +decision before release readiness. -`ffmpeg` is an operator-provided executable resolved from `PATH`; yt-dlp is a locked Python package. -Release evidence must record their resolved versions and may not describe either as bundled unless -packaging and licensing change. +`ffmpeg` is operator-provided. Ordinary local imports may use yt-dlp's normal discovery, but release +evidence and the live benchmark require `BANDSCOPE_FFMPEG_PATH` to name an absolute executable and +`BANDSCOPE_FFMPEG_SHA256` to bind its full digest; the production downloader revalidates those bytes +before provider access. yt-dlp is locked at the exact version in `uv.lock`. ## Failure taxonomy @@ -95,7 +94,7 @@ packaging and licensing change. - Reference byte/hash/member/redirect error: fixture integrity or SSRF boundary failure. - YouTube/master duration drift above 1.0 s or identity correlation below 0.90: wrong or drifted candidate/transcode. -- Model import/retrieval/load error: platform or supply-chain failure. +- Model import/provisioning/identity/load error: platform or supply-chain failure. - Non-finite/shape/threshold error: separator correctness failure. Explicit live invocation converts all of these to a failing test. A failure blocks only the evidence @@ -103,7 +102,7 @@ lane; it does not authorize a bypass or stop unrelated repository work. ## Verification and evidence -Default verification runs the 16 deterministic known-stem contract tests and explicitly excludes +Default verification runs the 25 deterministic known-stem contract tests and explicitly excludes the live marker. A live run uses the exact command in the operator guide. Evidence must include exact commit and dependency lock, model full hash, fixture archive full hash, public video ID, OS/architecture, result code, correlation, baseline diff --git a/docs/adr/0001-source-separation-runtime-and-model-delivery.md b/docs/adr/0001-source-separation-runtime-and-model-delivery.md index d9b51d69f..b161341f6 100644 --- a/docs/adr/0001-source-separation-runtime-and-model-delivery.md +++ b/docs/adr/0001-source-separation-runtime-and-model-delivery.md @@ -10,8 +10,9 @@ separation. BandScope now uses Demucs 4.0.1 `htdemucs` to return vocals, bass, d local rehearsal analysis. The production boundary must remain local-first after model provisioning, bounded on CPU, platform-honest, and traceable to an exact model artifact. -The current Demucs loader downloads weights on first use and verifies only the eight-hex hash prefix -embedded in `955717e8-8726e21a.th`. The exact artifact is 84,141,911 bytes with SHA-256 +The upstream Demucs remote loader downloads weights on first use and verifies only the eight-hex +hash prefix embedded in `955717e8-8726e21a.th`. BandScope does not use that remote loader: its +production boundary requires a pre-provisioned local artifact and verifies 84,141,911 bytes plus SHA-256 `8726e21a993978c7ba086d3872e7608d7d5bfca646ca4aca459ffda844faa8b4`. The Demucs code is MIT licensed, but no separate commercial redistribution grant for the official weights was identified; the upstream licensing discussion characterizes the weights as scientific-use material. @@ -27,25 +28,25 @@ the upstream licensing discussion characterizes the weights as scientific-use ma 5. A release claiming source-separation readiness must verify the full SHA-256 before any torch deserialization and must have a recorded legal decision for its chosen download or distribution path. -6. Until those blockers are implemented, first-load network access is explicit, offline inference is - guaranteed only with a trusted pre-provisioned cache, and source separation is unavailable on - macOS Intel under the current dependency markers. +6. Runtime model download is prohibited. Missing, symlinked, wrong-sized, or full-hash-mismatched + local bytes fail before deserialization. Source separation remains unavailable on macOS Intel + under the current dependency markers. ## Alternatives considered - Keep the FFT profile: rejected because it produces structurally plausible but invalid stems. - Bundle official htdemucs weights immediately: rejected because repository/release size and model redistribution rights are unresolved. -- Rely on Demucs' eight-hex prefix only: retained temporarily as current behavior, rejected as the - release target because it is weaker than the repository's full-integrity policy. +- Rely on Demucs' eight-hex prefix only: rejected because it is weaker than the repository's + full-integrity policy. - Replace with ONNX or another commercially licensed model: viable future work, but it requires parity, quality, platform, performance, and licensing evidence. ## Consequences BandScope obtains real separation quality but inherits torch/Demucs resource cost, platform gaps, -runtime model retrieval, and an upstream model-rights decision. Release evidence cannot describe the -model as bundled or fully offline today. The supplemental inventory check now fails if the runtime +operator provisioning, and an upstream model-rights decision. Release evidence cannot describe the +model as bundled. The supplemental inventory check now fails if the runtime model is missing, incompletely pinned, or replaced by the retired profile. ## Security and governance implications diff --git a/docs/adr/README.md b/docs/adr/README.md index 3270ac4ed..8c24568b3 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -5,7 +5,7 @@ decision through a new ADR that names the superseded record. | ADR | Status | Decision | |---|---|---| -| `0001-source-separation-runtime-and-model-delivery.md` | Accepted with release blockers | Use real four-source htdemucs locally and inventory the exact runtime artifact; require full-hash pre-load verification and a model-rights decision before release readiness. | +| `0001-source-separation-runtime-and-model-delivery.md` | Accepted with release blockers | Use real four-source htdemucs locally, require exact full-hash pre-load verification, and retain the model-rights/permitted-delivery decision as a release blocker. | | `0002-known-stem-youtube-quality-gate.md` | Proposed on active branch | Validate the production YouTube-to-separator path with a creator-published known vocal stem, single alignment, SI-SDR improvement, and assignment margin. | | `0003-ephemeral-benchmark-evidence-model.md` | Proposed on active branch | Keep media and signal arrays ephemeral; retain only bounded evidence when authorized, so a relational ERD is not currently authoritative. | diff --git a/docs/architecture/diagrams.md b/docs/architecture/diagrams.md index 6188719f8..97da3b5e0 100644 --- a/docs/architecture/diagrams.md +++ b/docs/architecture/diagrams.md @@ -122,16 +122,19 @@ these values without instantiating a production class. flowchart TB subgraph Desktop["User desktop"] App["BandScope app"] + Benchmark["KnownStemBenchmark
explicit opt-in validation"] Cache["User-scoped model cache"] Temp["Ephemeral media root"] - App --> Cache - App --> Temp + Benchmark --> App + Benchmark --> Cache + Benchmark --> Temp end - YouTube["YouTube media boundary"] --> App - Source["Pinned creator archive"] --> App - Master["Pinned creator master"] --> App - Model["Official model host"] --> Cache - App --> Evidence["Bounded numeric evidence"] + YouTube["YouTube media boundary"] --> Benchmark + Source["Pinned creator archive"] --> Benchmark + Master["Pinned creator master"] --> Benchmark + Provisioner["Operator-controlled model provisioning"] --> Cache + Model["Official model host"] --> Provisioner + Benchmark --> Evidence["Bounded numeric evidence"] ``` The model cache is persistent; media temp is not. The public hosts, cache contents, media, decoders, @@ -144,7 +147,7 @@ erDiagram KNOWN_STEM_FIXTURE ||--|| REFERENCE_ARCHIVE : pins KNOWN_STEM_FIXTURE ||--|| CREATOR_MASTER : pins KNOWN_STEM_FIXTURE ||--|| YOUTUBE_MIX : identifies - REFERENCE_ARCHIVE ||--|| REFERENCE_STEM : contains + REFERENCE_ARCHIVE ||--|{ REFERENCE_STEM : contains YOUTUBE_MIX ||--|| CREATOR_MASTER : identity-checks CREATOR_MASTER ||--|| ALIGNED_WINDOW : anchors YOUTUBE_MIX ||--|| ALIGNED_WINDOW : yields diff --git a/docs/architecture/overview.md b/docs/architecture/overview.md index 8b8bb00c4..f4475e749 100644 --- a/docs/architecture/overview.md +++ b/docs/architecture/overview.md @@ -43,11 +43,12 @@ GitHub is the source of truth for repository governance, PR review, CI/CD, Code - The Python engine uses the real four-source `htdemucs` model on supported platforms; the old FFT profile is retired. -- Inference is local after a trusted cache is provisioned. Current first use may retrieve the exact - inventoried model artifact; full-hash pre-load enforcement remains a release blocker. -- The known-stem validation branch proves the real YouTube intake → creator-master identity → +- Inference is local after a trusted cache is provisioned. The runtime never retrieves weights and + verifies the exact inventoried byte size and full SHA-256 before local deserialization. +- The known-stem validation branch defines and exercises the real YouTube intake → creator-master identity → composed master/vocal alignment → deterministic separator → SI-SDR scoring path. Test-only - reference handling never becomes a general runtime downloader. + reference handling never becomes a general runtime downloader. No passing live run currently + validates that complete path; the recorded YouTube attempts failed before model scoring. - Media and stem arrays are ephemeral. Only bounded numeric/provenance evidence may be retained after authorization, so no physical benchmark database or ERD exists. diff --git a/docs/documentation-coverage-matrix.md b/docs/documentation-coverage-matrix.md index 409087b2b..ca6646b90 100644 --- a/docs/documentation-coverage-matrix.md +++ b/docs/documentation-coverage-matrix.md @@ -11,8 +11,8 @@ canonical PRD, TRD, ADRs, UML, logical data model, traceability, model inventory release/operations criteria. This branch adds those authorities and mechanical presence checks. The documentation graph is now structurally sufficient and explicitly code-current, but the product -is not yet release-ready for source separation. A passing live run, full-hash pre-load model -verification, model-rights decision, threshold calibration, supported-platform evidence, and +is not yet release-ready for source separation. A passing live run, model-rights decision, +threshold calibration, supported-platform evidence, and bounded evidence artifact remain open. Issue #770 remains open. This branch must not be described as completing the full real-audio MIR @@ -24,15 +24,15 @@ acceptance layer. |---|---|---|---| | PRD | `docs/PRD.md` | Adequate for product outcome, scope, users, acceptance, legal boundary, rollout, and non-goals. | Broader multi-fixture/four-stem requirements await evidence. | | TRD | `docs/TRD.md` | Adequate for interfaces, metrics, schema, platform matrix, failure taxonomy, model delivery, evidence, and traceability. | Performance budget and calibrated thresholds are not yet accepted. | -| Architecture | `ARCHITECTURE.md`, `docs/architecture/overview.md` | Updated for htdemucs and known-stem boundaries. | Full-hash model resolver remains implementation work. | +| Architecture | `ARCHITECTURE.md`, `docs/architecture/overview.md` | Updated for htdemucs, local-only full-hash model loading, and known-stem boundaries. | Model-rights and permitted-delivery decision remains open. | | ADR | `docs/adr/README.md`, ADR-0001..0003 | Captures model, live quality gate, and persistence/ERD decisions with alternatives and supersession. | ADR-0002/0003 remain Proposed until branch merge. | | UML | `docs/architecture/diagrams.md` | Component, sequence, state, class, and deployment views included. | No additional UML is needed for the bounded slice. | | ERD/data | `docs/architecture/diagrams.md`, ADR-0003 | Logical artifact relationships and persistence status are explicit. | Physical ERD is intentionally not applicable until persistence exists. | -| Security/privacy | `docs/engineering/youtube-known-stem-validation.md`, `docs/security/app-security.md`, ADRs | Threats, trust boundaries, non-collection, integrity, cleanup, and legal limits covered. | Full model hash must be enforced before load. | +| Security/privacy | `docs/engineering/youtube-known-stem-validation.md`, `docs/security/app-security.md`, ADRs | Threats, trust boundaries, non-collection, pre-load full-hash verification, cleanup, and legal limits covered. | Model-rights/legal delivery decision remains open. | | Test strategy | `docs/TRD.md`, operator guide, acceptance criteria | Offline/live split and metric/failure contracts covered. | No successful live score has been recorded. | | MIR doctoring | `docs/doctoring/real-audio-accuracy-acceptance.md` | Issue #770 metrics, claim boundaries, tiers, and roadmap are separated from the bounded vocal slice. | Accuracy manifest, reports, other MIR families, and corpus tiers remain open. | | Operations/release | runbook and release policy | Preflight, evidence, triage, rollback, and blocking conditions covered. | Platform matrix and live pass are incomplete. | -| Supply chain | supplemental inventory and dependency policy | Retired model removed; exact runtime artifact, ffmpeg status, and hash recorded. | Weight redistribution rights and pre-load enforcement unresolved. | +| Supply chain | supplemental inventory and dependency policy | Retired model removed; exact runtime artifact is verified before load; release ffmpeg is bound to an absolute path/full digest. | Weight retrieval/distribution rights remain unresolved. | | Automation | active CWL autonomous loop and `docs/workflow/pr-review-merge-scheduler.md` | BandScope continuity and no-status-only termination are covered without creating a competing writer. | Dedicated BandScope loop remains paused due writer topology/active-task capacity. | | Review governance | `docs/security/github-required-checks.md`, governance, gitflow, contributing, bootstrap policy | Stable checks and review are cumulative; qualifying evidence is an exact-head completed CodeRabbit artifact or exact-head independent non-author `APPROVED` review. Status-only, rate-limited, author, or predecessor evidence is excluded. | A provider rate limit can still defer review, blocking only merge. | @@ -44,16 +44,16 @@ acceptance layer. | PRD-KS-002, KS-004 | ADR-0001/0002; Rouard et al. (2023) | `separation/audio_separator.py` | `test_youtube_stem_e2e.py` live case | Exact model identity and supported platform | | PRD-KS-003 | Le Roux et al. (2019) | `tests/known_stem_benchmark.py` | SI-SDR unit tests and live threshold | Calibration plus exact-candidate score | | PRD-KS-005 | ADR-0002 | master identity plus composed global alignment helpers | delayed/composed-window tests; live duration/correlation | Authorized YouTube calibration and drift triage | -| PRD-KS-006, KS-010 | ADR-0002 | pytest marker and failure taxonomy | 16 default offline tests; explicit live failure | Advisory until promotion ADR | +| PRD-KS-006, KS-010 | ADR-0002 | pytest marker and failure taxonomy | 25 default offline tests; explicit live failure | Advisory until promotion ADR | | PRD-KS-008 | ADR-0003 | temporary directory and sanitized errors | cleanup postcondition and archive failure tests | Evidence excludes raw media/paths | | PRD-KS-009 | ADR-0003; NIST AI RMF TEVV | planned bounded evidence schema | No retained score yet | Required before blocking release gate | -| TRD-KS-011 | ADR-0001 | supplemental inventory | inventory consistency tests | Full-hash pre-load blocker | +| TRD-KS-011 | ADR-0001 | supplemental inventory and local loader | missing/size/hash/local-repository tests | Implemented on active PR; exact-head and protected-main proof pending | ## Live evidence snapshot | Date | Commit under test | Offline contract | Live result | Classification | |---|---|---|---|---| -| 2026-08-09 | `5a3648a11d9097b8da48bb4a3ccbd97986aec25b` | 13 passed | Reference archive verified; YouTube download failed with HTTP 502 before separation; no score | Exact failure evidence, not a pass | +| 2026-08-09 | `5a3648a11d9097b8da48bb4a3ccbd97986aec25b` | Historical partial suite: 13 passed via `pytest -m "not youtube_stem_e2e" services/analysis-engine/tests/test_youtube_stem_e2e.py`; it predated separate creator-master authentication, composed two-lag recovery, and deterministic production-separator assertions now present in the 25-case manifest | Reference archive verified; YouTube download failed with HTTP 502 before separation; no score | Exact failure evidence, not a pass | | 2026-08-09 | `6e937a34f9036d92e909db3ce8848a5c39dc8e3b` (published byte-identical implementation tree) | Full quickcheck: 680 Python passed, 24 skipped, live marker deselected; 100% source coverage | Archive, extracted vocal, creator master, and pre-provisioned model hash verified; production YouTube download failed with HTTP 502 after 65.49 s; no score | Exact implementation-head failure evidence, not a pass | Separate creator-master calibration on that environment measured `shifts=0` vocal SI-SDR diff --git a/docs/engineering/acceptance-criteria.md b/docs/engineering/acceptance-criteria.md index 841d37878..7deee7fd7 100644 --- a/docs/engineering/acceptance-criteria.md +++ b/docs/engineering/acceptance-criteria.md @@ -44,7 +44,7 @@ For protected branches, intended checks are documented in `docs/security/github- ## Source-separation quality gates -- Every separator or downloader change must keep the 16 deterministic known-stem metric, alignment, +- Every separator or downloader change must keep the 25 deterministic known-stem metric, alignment, archive-integrity, redirect/path, cleanup, and failure-contract cases passing. - A live evidence claim must cross `download_youtube_audio()` and `AudioStemSeparator.separate()` on the same exact candidate, authenticate the separately pinned creator master, compose the two @@ -57,8 +57,10 @@ For protected branches, intended checks are documented in `docs/security/github- provider or model failure does not justify weakening it. - Skipped, disabled, HTTP/provider-failed, model-unavailable, integrity-failed, drifted, non-finite, predecessor-head, or stale-base execution is not passing evidence. -- Before the lane can block a release, ADR-0001/0002 blockers—authorization, full-hash pre-load - verification, exact-candidate pass, calibration, and supported-platform evidence—must be closed. +- Before the lane can block a release, ADR-0001/0002 blockers—authorization, a recorded + model-rights/legal delivery decision, exact-candidate pass, calibration, and supported-platform + evidence—must be closed. Full-hash pre-load verification is already implemented and must remain + green. ## Evidence policy diff --git a/docs/engineering/youtube-known-stem-validation.md b/docs/engineering/youtube-known-stem-validation.md index 31908379a..00fc9be83 100644 --- a/docs/engineering/youtube-known-stem-validation.md +++ b/docs/engineering/youtube-known-stem-validation.md @@ -70,16 +70,16 @@ creator-master probe is not a live pass. ## Running the benchmark -Install the analysis-engine development dependencies and ensure `ffmpeg` is on `PATH`. The first -Demucs run may obtain model weights through Demucs unless they are already present in its cache. -Prefer a pre-provisioned, integrity-verified model cache for repeatable runs. +Install the analysis-engine development dependencies. Provision an absolute `ffmpeg` executable and +the exact htdemucs artifact locally; the live lane requires their path/digest identities and never +downloads model weights. The exact current model artifact is Demucs 4.0.1 htdemucs signature `955717e8`, file `955717e8-8726e21a.th`, 84,141,911 bytes, full SHA-256 -`8726e21a993978c7ba086d3872e7608d7d5bfca646ca4aca459ffda844faa8b4`. It is runtime-fetched and -not bundled. Demucs currently enforces only the filename's eight-hex hash prefix before load; -ADR-0001 therefore treats BandScope-owned full-hash pre-load enforcement and a model-rights decision -as release blockers. +`8726e21a993978c7ba086d3872e7608d7d5bfca646ca4aca459ffda844faa8b4`. It is not bundled. +BandScope requires the exact filename, byte count, and full digest before passing a local repository +to Demucs; missing or changed bytes fail before deserialization. ADR-0001 retains the model-rights +and permitted-delivery decision as a release blocker. Before enabling the test, the operator must confirm that the intended use is permitted by the content rightsholder and the applicable YouTube terms. The creator's permission for the reference @@ -87,6 +87,9 @@ source does not by itself grant permission for automated access to YouTube. ```bash UV_CACHE_DIR=/tmp/bandscope-uv-cache \ +BANDSCOPE_FFMPEG_PATH=/absolute/path/to/ffmpeg \ +BANDSCOPE_FFMPEG_SHA256=<64-lowercase-hex-digest> \ +BANDSCOPE_HTDEMUCS_MODEL_PATH=/absolute/path/to/955717e8-8726e21a.th \ BANDSCOPE_RUN_YOUTUBE_STEM_E2E=1 \ uv run --project services/analysis-engine \ pytest services/analysis-engine/tests/test_youtube_stem_e2e.py \ @@ -111,9 +114,9 @@ then failed in the production YouTube downloader with HTTP 502 before separation correlation or SI-SDR score and is recorded as failure evidence, not a live pass. See `docs/documentation-coverage-matrix.md`. -The corrected branch now has 16 offline known-stem contract cases, including exact extracted-member -hash, creator-master authentication, composed-offset recovery, and explicit required-CI exclusion of -the live marker. A creator-master-only calibration produced the provisional scores above without +The corrected branch now has 25 offline known-stem contract cases, including exact extracted-member +hash, creator-master authentication, composed-offset recovery, signed identity correlation, +model/ffmpeg pre-load identity failures, and explicit required-CI exclusion of the live marker. A creator-master-only calibration produced the provisional scores above without calling YouTube; it is calibration evidence, not exact-candidate success. The byte-identical implementation tree published on GitHub as exact commit @@ -145,7 +148,7 @@ the only permitted storage root for downloaded media and extracted references. separator regression. - Login cookies, geo/DRM bypasses, or automated CI execution could expand legal, privacy, and account risk. -- Decoder/model vulnerabilities and first-run model downloads remain upstream supply-chain risks. +- Decoder/model vulnerabilities and operator-provisioned model provenance remain upstream supply-chain risks. ### Mitigations @@ -160,8 +163,8 @@ the only permitted storage root for downloaded media and extracted references. master is independently pinned by exact host, byte count, and full SHA-256. - The production YouTube downloader keeps its standard-URL allowlist, duration/size bounds, `noplaylist`, and no-geo-bypass policy. This test adds no cookies, credentials, login, paywall, - DRM, or bot-evasion behavior. TLS validation stays enabled while yt-dlp uses the operating - system's managed CA trust store rather than a separate certifi-only bundle. + DRM, or bot-evasion behavior. TLS validation stays enabled and yt-dlp retains its maintained CA + bundle fallback, so a minimal container does not silently depend on an absent system trust store. - Alignment is global and bounded. Duration and creator-master identity correlation distinguish fixture drift from model quality failure; the two lags are composed once and model outputs are not optimized after separation. Demucs random shift augmentation is disabled with `shifts=0`. @@ -184,8 +187,8 @@ advice, and the test does not establish platform authorization. Upstream media d weights remain separate trust decisions. The fixture has only one full-length known canonical stem, so the test cannot claim quantitative four-stem accuracy. -The model-weight redistribution license is not established, current Demucs verification uses only a -hash prefix, and no successful exact-candidate live score or supported-platform matrix has yet been +The model-weight redistribution license is not established. BandScope now verifies the exact byte +count and full SHA-256 before local-only loading, and no successful exact-candidate live score or supported-platform matrix has yet been retained. These remain explicit release blockers rather than undocumented assumptions. ## References diff --git a/docs/operations/deploy-runbook.md b/docs/operations/deploy-runbook.md index 150ef531a..21491d445 100644 --- a/docs/operations/deploy-runbook.md +++ b/docs/operations/deploy-runbook.md @@ -28,8 +28,10 @@ When runtime behavior is touched, verify: For a release candidate that claims YouTube source separation: -1. record exact commit, live base tip, lockfiles, OS, architecture, Python, Demucs, torch, yt-dlp, - and `ffmpeg -version`; +1. record exact commit, live base tip, lockfiles, OS, architecture, Python, Demucs, torch, and yt-dlp; + set `BANDSCOPE_FFMPEG_PATH` to an absolute executable and + `BANDSCOPE_FFMPEG_SHA256` to its verified full digest, then record that resolved path, digest, + trusted package/source provenance, and the executable's `-version` output; 2. confirm content/platform authorization and do not provide cookies, credentials, login, paywall, DRM, geo, or anti-bot bypasses; 3. verify the htdemucs model's exact source, 84,141,911-byte size, and full SHA-256 from the diff --git a/docs/plans/2026-03-28-ml-engine-integration.md b/docs/plans/2026-03-28-ml-engine-integration.md index bc49b15c4..ff92adc4d 100644 --- a/docs/plans/2026-03-28-ml-engine-integration.md +++ b/docs/plans/2026-03-28-ml-engine-integration.md @@ -48,10 +48,10 @@ The primary trust boundary is between the user's filesystem (audio files) and th ### Mitigations We restrict audio ingestion through `librosa`/`soundfile` using strict format constraints. Model -inference runs locally and under low privilege where possible. First use is not currently fully -offline: Demucs may retrieve the exact inventoried model into a user cache. Offline execution is -guaranteed only after trusted provisioning. The release target requires full-SHA verification before -deserialization; see ADR-0001. +inference runs locally and under low privilege where possible. The implemented loader never +retrieves model bytes: it requires trusted local provisioning and verifies the exact filename, byte +count, and full SHA-256 before deserialization. Model rights and permitted delivery remain release +decisions; see ADR-0001. ### Test Points - Loading truncated or corrupted WAV/MP3 files. diff --git a/docs/repository/bootstrap-plan.md b/docs/repository/bootstrap-plan.md index e295e6449..69b586099 100644 --- a/docs/repository/bootstrap-plan.md +++ b/docs/repository/bootstrap-plan.md @@ -32,6 +32,7 @@ After workflows exist, require these stable checks on `main` and `develop`: - `ci / build-and-test` - `dependency-review` - `security-audit` +- `trivy-fs-scan` - `CodeQL` - `sbom` - `release-preflight` diff --git a/docs/workflow/github-bootstrap-execution-policy.md b/docs/workflow/github-bootstrap-execution-policy.md index 69823249c..8ff085026 100644 --- a/docs/workflow/github-bootstrap-execution-policy.md +++ b/docs/workflow/github-bootstrap-execution-policy.md @@ -97,7 +97,7 @@ Do not treat these as TODOs, later hardening, or optional recommendations. ### Phase 6. Bootstrap PR - create `bootstrap/setup` or equivalent from `develop` -- add workflows, security docs, CODEOWNERS, dependency review, SBOM, builds, and required evidence docs +- add workflows, security docs, CODEOWNERS, dependency review, `trivy-fs-scan`, SBOM, builds, and required evidence docs - add or confirm lockfiles, dependency review, audit, SBOM, and supplemental inventory for bundled binaries and model artifacts - merge through PR review, not direct push diff --git a/package-lock.json b/package-lock.json index cf1c991c1..209617988 100644 --- a/package-lock.json +++ b/package-lock.json @@ -32,7 +32,7 @@ "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", "lucide-react": "^1.24.0", - "pdfjs-dist": "6.1.200", + "pdfjs-dist": "6.2.108", "react": "^19.2.4", "react-dom": "^19.2.7", "sonner": "^2.0.7", @@ -6075,9 +6075,9 @@ "license": "MIT" }, "node_modules/nanoid": { - "version": "3.3.16", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.16.tgz", - "integrity": "sha512-bzlKTyNJ7+LdGIIwy8ijFpIqEQIvafahV7eYykJ8Cvh42EdJeODoJ6gUJXpQJvej1BddH8OqTXZNE/KfbWAu8Q==", + "version": "3.3.18", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.18.tgz", + "integrity": "sha512-DTg4MJbGMWkfi6VZFdNt2/caMbQy4Ou+Op/hJQvGEWcnVfoA1QA+xzRKAzw9jD6+GVOOeYr/mIcuDSdug6F6+w==", "dev": true, "funding": [ { @@ -6368,9 +6368,9 @@ } }, "node_modules/pdfjs-dist": { - "version": "6.1.200", - "resolved": "https://registry.npmjs.org/pdfjs-dist/-/pdfjs-dist-6.1.200.tgz", - "integrity": "sha512-o8MolyzirkkLrcdsae/HEOiIcXWI7DS5zGpvqW8xTC2YUsW30rltFw2bDGvw/fskUdEMrQm2br68jzDS5BH2vw==", + "version": "6.2.108", + "resolved": "https://registry.npmjs.org/pdfjs-dist/-/pdfjs-dist-6.2.108.tgz", + "integrity": "sha512-YxFb+SQcodN2rnX9Tn3dHYlqfb7NjlzzfONPpJd+AKoKtUjEdevTfbC07d5TcczzOK6261auRkP/M8OBHs9vFQ==", "license": "Apache-2.0", "engines": { "node": ">=22.13.0 || >=24" @@ -7179,9 +7179,9 @@ } }, "node_modules/undici": { - "version": "7.28.0", - "resolved": "https://registry.npmjs.org/undici/-/undici-7.28.0.tgz", - "integrity": "sha512-cRZYrTDwWznlnRiPjggAGxZXanty6M8RV1ff8Wm4LWXBp7/IG8v5DnOm74DtUBp9OONpK75YlPnIjQqX0dBDtA==", + "version": "7.29.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-7.29.0.tgz", + "integrity": "sha512-IDxfleLmmbSskfWSUATiN1nfn2rDuvnMOqb5CWR92iIfojA0Ud+ulOAAEQ57LPr9rWmsreUyf5lwyao+7GNNVw==", "dev": true, "license": "MIT", "engines": { diff --git a/scripts/checks/verify_docs.py b/scripts/checks/verify_docs.py index 47678a37d..928e86e2e 100644 --- a/scripts/checks/verify_docs.py +++ b/scripts/checks/verify_docs.py @@ -1,5 +1,6 @@ """Verify that required repository documentation files and references exist.""" +import re from pathlib import Path REQUIRED_PATHS = [ @@ -107,6 +108,20 @@ def documentation_violations(root: Path = Path(".")) -> list[str]: for required_text in required_texts: if required_text not in content: violations.append(f"{path} missing reference: {required_text}") + plans_root = root / "docs" / "plans" + if plans_root.exists(): + security_heading = re.compile( + r"^(?:#{1,6}\s+Security Notes\s*$|" + r"\*\*Security Notes(?::)?\*\*)(?:\s|$)", + re.MULTILINE, + ) + for absolute_path in sorted(plans_root.rglob("*.md")): + content = absolute_path.read_text(encoding="utf-8") + if security_heading.search(content) is None: + relative_path = absolute_path.relative_to(root) + violations.append( + f"{relative_path} missing section: Security Notes" + ) return violations diff --git a/scripts/checks/verify_supply_chain.py b/scripts/checks/verify_supply_chain.py index 509396aac..6a55115a6 100644 --- a/scripts/checks/verify_supply_chain.py +++ b/scripts/checks/verify_supply_chain.py @@ -57,6 +57,7 @@ "releaseUsage", "verification", } +MODEL_ARTIFACT_STRING_FIELDS = REQUIRED_MODEL_ARTIFACT_FIELDS - {"sizeBytes"} def supplemental_inventory_violations( @@ -69,6 +70,8 @@ def supplemental_inventory_violations( inventory = json.loads(inventory_path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError) as error: return [f"supplemental inventory is unreadable: {error.__class__.__name__}"] + if not isinstance(inventory, dict): + return ["supplemental inventory root must be an object"] try: separator_source = separator_path.read_text(encoding="utf-8") except OSError as error: @@ -87,7 +90,8 @@ def supplemental_inventory_violations( if not isinstance(artifact, dict): violations.append("supplemental inventory model artifact must be an object") continue - name = str(artifact.get("name", "")) + name_value = artifact.get("name") + name = name_value if isinstance(name_value, str) else "" if name.startswith("bandsplit-"): violations.append(f"supplemental inventory contains retired model: {name}") if artifact.get("runtimeModelName") == runtime_model: @@ -106,6 +110,13 @@ def supplemental_inventory_violations( f"supplemental inventory runtime model {runtime_model} missing fields: " + ", ".join(missing_fields) ) + for field in sorted(MODEL_ARTIFACT_STRING_FIELDS & artifact.keys()): + value = artifact[field] + if not isinstance(value, str) or not value.strip(): + violations.append( + f"supplemental inventory runtime model {runtime_model} " + f"field {field} must be a non-empty string" + ) checksum = artifact.get("checksum") if not isinstance(checksum, str) or not FULL_SHA256_PATTERN.fullmatch(checksum): violations.append( @@ -117,9 +128,14 @@ def supplemental_inventory_violations( f"supplemental inventory runtime model {runtime_model} requires HTTPS source" ) size_bytes = artifact.get("sizeBytes") - if not isinstance(size_bytes, int) or size_bytes <= 0: + if ( + not isinstance(size_bytes, int) + or isinstance(size_bytes, bool) + or size_bytes <= 0 + ): violations.append( - f"supplemental inventory runtime model {runtime_model} requires positive sizeBytes" + f"supplemental inventory runtime model {runtime_model} " + "requires positive integer sizeBytes" ) return violations diff --git a/services/analysis-engine/src/bandscope_analysis/api.py b/services/analysis-engine/src/bandscope_analysis/api.py index b376de293..892a00954 100644 --- a/services/analysis-engine/src/bandscope_analysis/api.py +++ b/services/analysis-engine/src/bandscope_analysis/api.py @@ -899,6 +899,15 @@ def _stem_separation_failure( "Stem separation failed because the source file was missing.", ) if isinstance(error, ValueError): + if "htdemucs model artifact" in error_message.lower(): + return ( + "runtime_error", + "Stem separation model is unavailable.", + ( + "Stem separation unavailable because the verified model artifact " + "is missing or invalid." + ), + ) if "not available on this platform" in error_message or "demucs/torch" in error_message: return ( "runtime_error", diff --git a/services/analysis-engine/src/bandscope_analysis/separation/audio_separator.py b/services/analysis-engine/src/bandscope_analysis/separation/audio_separator.py index e7ebe80b5..e13bb9010 100644 --- a/services/analysis-engine/src/bandscope_analysis/separation/audio_separator.py +++ b/services/analysis-engine/src/bandscope_analysis/separation/audio_separator.py @@ -1,4 +1,4 @@ -"""Local audio source separation using a bundled Demucs model. +"""Local audio source separation using a verified Demucs model. Replaces the previous FFT band-masking heuristic — which scored around -39 dB SI-SDR on a realistic mix (i.e. not real separation) — with Demucs (htdemucs), a @@ -9,10 +9,9 @@ Security Notes: - Treats the selected audio file as untrusted input: the path is normalized and verified to be a file, and a maximum byte size is enforced before decode. -- Inference runs locally on CPU after model provisioning. The first Demucs - model load may fetch the inventoried weight into its user cache; BandScope - does not bundle that artifact, and offline operation requires a trusted - pre-provisioned cache. +- Inference runs locally on CPU only after an operator provisions the exact + inventoried model artifact. Missing or changed bytes fail closed before + Demucs can deserialize them; this runtime never downloads model weights. - Does not log or persist raw audio, separated stems, or full source paths. - Fails with bounded, filename-scoped errors so callers can surface a safe failure without leaking local directory structure. @@ -20,10 +19,9 @@ from __future__ import annotations -import contextlib +import hashlib import logging import os -import sys import warnings from dataclasses import dataclass from pathlib import Path @@ -46,6 +44,11 @@ # Demucs htdemucs emits these four sources; this is the canonical stem set. _STEM_ORDER: tuple[AudioStemName, ...] = ("vocals", "bass", "drums", "other") _EMPTY_RANGE_EPS = 1e-9 +_HTDEMUCS_MODEL_SIGNATURE = "955717e8" +_HTDEMUCS_MODEL_FILENAME = "955717e8-8726e21a.th" +_HTDEMUCS_MODEL_SHA256 = "8726e21a993978c7ba086d3872e7608d7d5bfca646ca4aca459ffda844faa8b4" +_HTDEMUCS_MODEL_BYTES = 84_141_911 +_MODEL_PATH_ENV = "BANDSCOPE_HTDEMUCS_MODEL_PATH" def _contains_parent_path_segment(path: Path) -> bool: @@ -66,6 +69,7 @@ class AudioSeparationConfig: max_file_bytes: int = MAX_AUDIO_FILE_BYTES max_duration_seconds: float = float(MAX_ANALYSIS_DURATION_SECONDS) model_name: str = "htdemucs" + model_artifact_path: Path | None = None device: str = "cpu" # Disable Demucs' random time-shift augmentation so repeated analysis of # the same bytes is deterministic and benchmark evidence is reproducible. @@ -137,9 +141,9 @@ def _load_model(self) -> Any: wheels (see pyproject platform markers); elsewhere separation fails with a clear error the pipeline already surfaces safely. - The first load fetches model weights, whose download progress torch may - print to stdout — that would corrupt the CLI's JSON stdout protocol, so - stdout is redirected to stderr while the model is obtained. + The runtime passes a local repository to Demucs, disabling its remote + model path. Full byte size and SHA-256 are checked before Demucs or + torch can deserialize the artifact. """ if self._model is None: try: @@ -151,12 +155,53 @@ def _load_model(self) -> Any: "Stem separation is not available on this platform (demucs/torch not installed)" ) from error - with contextlib.redirect_stdout(sys.stderr): - model = get_model(self.config.model_name) + artifact_path = self._verified_model_artifact_path() + model = get_model(_HTDEMUCS_MODEL_SIGNATURE, repo=artifact_path.parent) model.eval() self._model = model return self._model + def _verified_model_artifact_path(self) -> Path: + """Return the exact local htdemucs artifact after full identity checks.""" + configured = self.config.model_artifact_path + if configured is None: + configured_text = os.environ.get(_MODEL_PATH_ENV) + if configured_text: + configured = Path(configured_text) + else: + torch_home = Path( + os.environ.get( + "TORCH_HOME", + Path(os.environ.get("XDG_CACHE_HOME", Path.home() / ".cache")) / "torch", + ) + ) + configured = torch_home / "hub" / "checkpoints" / _HTDEMUCS_MODEL_FILENAME + + if configured.is_symlink(): + raise ValueError("The htdemucs model artifact path must not be a symlink") + try: + artifact_path = configured.expanduser().resolve(strict=True) + except (FileNotFoundError, OSError) as error: + raise ValueError( + "The verified htdemucs model artifact is unavailable; provision the " + f"inventoried file and set {_MODEL_PATH_ENV}" + ) from error + if not artifact_path.is_file() or artifact_path.name != _HTDEMUCS_MODEL_FILENAME: + raise ValueError( + "The verified htdemucs model artifact is unavailable; the exact " + f"{_HTDEMUCS_MODEL_FILENAME} file is required" + ) + if artifact_path.stat().st_size != _HTDEMUCS_MODEL_BYTES: + raise ValueError("The htdemucs model artifact failed byte-size verification") + + digest = hashlib.sha256() + with artifact_path.open("rb") as model_file: + for chunk in iter(lambda: model_file.read(1024 * 1024), b""): + digest.update(chunk) + if digest.hexdigest() != _HTDEMUCS_MODEL_SHA256: + raise ValueError("The htdemucs model artifact failed full SHA-256 verification") + return artifact_path + def _apply_model(self, model: Any, audio: AudioStemArray) -> dict[str, np.ndarray[Any, Any]]: """Apply Demucs to a mono signal, returning demucs-source-name -> mono array.""" import torch diff --git a/services/analysis-engine/src/bandscope_analysis/youtube.py b/services/analysis-engine/src/bandscope_analysis/youtube.py index b6b609f3c..7db0a2c5b 100644 --- a/services/analysis-engine/src/bandscope_analysis/youtube.py +++ b/services/analysis-engine/src/bandscope_analysis/youtube.py @@ -5,18 +5,20 @@ Security Notes: - Accepts only bounded, standard HTTPS YouTube watch URLs and disables playlists, geographic bypass, credentials, and interactive authentication. -- Keeps certificate verification enabled and uses the operating system trust - store so managed desktop CA policy is honored. +- Keeps certificate verification enabled and retains yt-dlp's maintained CA + bundle fallback so minimal containers do not depend on an absent system store. - Rejects metadata over 15 minutes and completed files over 50 MiB, returns sanitized public errors, and never logs the requested URL or downloaded audio. """ import argparse +import hashlib import json import os import re import sys import urllib.parse +from pathlib import Path from typing import Any, Dict, Optional import yt_dlp # type: ignore @@ -28,6 +30,9 @@ "Failed to download audio from YouTube. Please use a local audio file instead." ) YOUTUBE_IMPORT_FAILED_MESSAGE = "YouTube import failed. Please use a local audio file instead." +FFMPEG_PATH_ENV = "BANDSCOPE_FFMPEG_PATH" +FFMPEG_SHA256_ENV = "BANDSCOPE_FFMPEG_SHA256" +FULL_SHA256_PATTERN = re.compile(r"^[0-9a-f]{64}$") def validate_url(url: str) -> bool: @@ -114,6 +119,36 @@ def _handle_download_error(e: yt_dlp.utils.DownloadError) -> Dict[str, Any]: } +def _verified_ffmpeg_location() -> str | None: + """Return an exact operator-provisioned ffmpeg path when identity is configured.""" + configured_path = os.environ.get(FFMPEG_PATH_ENV) + configured_digest = os.environ.get(FFMPEG_SHA256_ENV) + if configured_path is None and configured_digest is None: + return None + if configured_path is None or configured_digest is None: + raise ValueError("ffmpeg release identity requires both path and SHA-256") + if not FULL_SHA256_PATTERN.fullmatch(configured_digest): + raise ValueError("ffmpeg release identity requires a full lowercase SHA-256") + + raw_path = Path(configured_path) + if not raw_path.is_absolute(): + raise ValueError("ffmpeg release identity requires an absolute executable path") + try: + executable_path = raw_path.resolve(strict=True) + except (FileNotFoundError, OSError) as error: + raise ValueError("ffmpeg release executable is unavailable") from error + if not executable_path.is_file() or not os.access(executable_path, os.X_OK): + raise ValueError("ffmpeg release executable is unavailable") + + digest = hashlib.sha256() + with executable_path.open("rb") as executable_file: + for chunk in iter(lambda: executable_file.read(1024 * 1024), b""): + digest.update(chunk) + if digest.hexdigest() != configured_digest: + raise ValueError("ffmpeg release executable failed SHA-256 verification") + return str(executable_path) + + def download_youtube_audio(url: str, out_dir: str) -> Dict[str, Any]: """ Download audio from a YouTube URL to the specified directory. @@ -143,12 +178,12 @@ def download_youtube_audio(url: str, out_dir: str) -> Dict[str, Any]: "noplaylist": True, "postprocessors": [{"key": "FFmpegExtractAudio"}], "geo_bypass": False, - # Keep TLS verification enabled while honoring OS-managed CA roots - # (including enterprise desktop trust stores) instead of certifi only. - "compat_opts": {"no-certifi"}, } try: + ffmpeg_location = _verified_ffmpeg_location() + if ffmpeg_location is not None: + ydl_opts["ffmpeg_location"] = ffmpeg_location with yt_dlp.YoutubeDL(ydl_opts) as ydl: info = ydl.extract_info(url, download=False) if info is None: diff --git a/services/analysis-engine/tests/known_stem_benchmark.py b/services/analysis-engine/tests/known_stem_benchmark.py index 6442ed900..9dbf2c9c1 100644 --- a/services/analysis-engine/tests/known_stem_benchmark.py +++ b/services/analysis-engine/tests/known_stem_benchmark.py @@ -204,7 +204,7 @@ def _strongest_window_start(signal: np.ndarray, window_samples: int) -> int: def _normalized_correlation(left: np.ndarray, right: np.ndarray) -> float: - """Return absolute zero-mean Pearson correlation for two equal windows.""" + """Return signed zero-mean Pearson correlation for two equal windows.""" left_centered = left - float(np.mean(left)) right_centered = right - float(np.mean(right)) denominator = math.sqrt( @@ -212,7 +212,7 @@ def _normalized_correlation(left: np.ndarray, right: np.ndarray) -> float: ) if denominator <= _ENERGY_EPSILON: raise ValueError("aligned benchmark window has insufficient audio energy") - return float(abs(np.dot(left_centered, right_centered)) / denominator) + return float(np.dot(left_centered, right_centered) / denominator) def align_active_reference_window( @@ -284,7 +284,7 @@ def align_active_reference_window( ) if valid_refined.size == 0: raise ValueError("reference fixture cannot produce a full scoring window") - best_refined_index = int(valid_refined[np.argmax(np.abs(refined_correlation[valid_refined]))]) + best_refined_index = int(valid_refined[np.argmax(refined_correlation[valid_refined])]) mixture_start = search_start + int(refined_lags[best_refined_index]) mixture_window = mixture_signal[mixture_start : mixture_start + window_samples] correlation = _normalized_correlation(mixture_window, reference_window) diff --git a/services/analysis-engine/tests/test_api.py b/services/analysis-engine/tests/test_api.py index 18273791d..90f27f6f5 100644 --- a/services/analysis-engine/tests/test_api.py +++ b/services/analysis-engine/tests/test_api.py @@ -1007,6 +1007,15 @@ def put(self, item: tuple[str, object]) -> None: "Stem separation is unavailable on this platform.", "Stem separation unavailable because Demucs or torch is not installed.", ), + ( + ValueError("The verified htdemucs model artifact is unavailable"), + "runtime_error", + "Stem separation model is unavailable.", + ( + "Stem separation unavailable because the verified model artifact " + "is missing or invalid." + ), + ), ( RuntimeError("oom /secret/audio.wav"), "runtime_error", diff --git a/services/analysis-engine/tests/test_documentation_policy.py b/services/analysis-engine/tests/test_documentation_policy.py index ecbd8116a..3382670d4 100644 --- a/services/analysis-engine/tests/test_documentation_policy.py +++ b/services/analysis-engine/tests/test_documentation_policy.py @@ -24,3 +24,17 @@ def test_documentation_contract_accepts_checked_in_authorities() -> None: repo_root = Path(__file__).resolve().parents[3] assert documentation.documentation_violations(repo_root) == [] + + +def test_documentation_contract_checks_every_nested_plan_security_section( + tmp_path: Path, +) -> None: + """Reject newly added plan documents that omit their security boundary.""" + documentation = load_module("scripts/checks/verify_docs.py", "verify_docs_contract_nested_plan") + plan = tmp_path / "docs" / "plans" / "future" / "unsafe-plan.md" + plan.parent.mkdir(parents=True) + plan.write_text("# Plan\n\nNo trust-boundary analysis yet.\n", encoding="utf-8") + + violations = documentation.documentation_violations(tmp_path) + + assert "docs/plans/future/unsafe-plan.md missing section: Security Notes" in violations diff --git a/services/analysis-engine/tests/test_separation.py b/services/analysis-engine/tests/test_separation.py index a7cdfb7dc..53a28366a 100644 --- a/services/analysis-engine/tests/test_separation.py +++ b/services/analysis-engine/tests/test_separation.py @@ -4,12 +4,14 @@ import os import sys +from pathlib import Path from types import ModuleType import numpy as np import pytest import soundfile as sf +from bandscope_analysis.separation import audio_separator as audio_separator_module from bandscope_analysis.separation.audio_separator import ( AudioSeparationConfig, AudioStemSeparator, @@ -232,7 +234,7 @@ def _patch_demucs(monkeypatch: pytest.MonkeyPatch, per_source: dict | None = Non return for that stem; unspecified sources return silence. """ - def fake_get_model(name: str) -> _FakeModel: + def fake_get_model(name: str, *, repo: Path) -> _FakeModel: return _FakeModel() def fake_apply_model( @@ -249,9 +251,149 @@ def fake_apply_model( return out _install_fake_demucs(monkeypatch, fake_get_model) + monkeypatch.setattr( + AudioStemSeparator, + "_verified_model_artifact_path", + lambda self: Path("/verified/955717e8-8726e21a.th"), + ) monkeypatch.setattr(AudioStemSeparator, "_apply_model", fake_apply_model) +def test_audio_stem_separator_rejects_missing_model_before_demucs_load( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Fail closed before Demucs can fetch or deserialize a missing model.""" + calls = {"n": 0} + + def fake_get_model(name: str, *, repo: Path) -> _FakeModel: + calls["n"] += 1 + return _FakeModel() + + _install_fake_demucs(monkeypatch, fake_get_model) + separator = AudioStemSeparator( + AudioSeparationConfig(model_artifact_path=tmp_path / "missing-model.th") + ) + + with pytest.raises(ValueError, match="verified htdemucs model artifact is unavailable"): + separator._load_model() + + assert calls["n"] == 0 + + +def test_audio_stem_separator_verifies_full_model_identity_before_local_load( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Bind Demucs loading to the exact inventoried bytes in a local repository.""" + model_bytes = b"verified local model" + model_path = tmp_path / "955717e8-8726e21a.th" + model_path.write_bytes(model_bytes) + calls: list[tuple[str, Path]] = [] + + def fake_get_model(name: str, *, repo: Path) -> _FakeModel: + calls.append((name, repo)) + return _FakeModel() + + _install_fake_demucs(monkeypatch, fake_get_model) + monkeypatch.setattr(audio_separator_module, "_HTDEMUCS_MODEL_BYTES", len(model_bytes)) + monkeypatch.setattr( + audio_separator_module, + "_HTDEMUCS_MODEL_SHA256", + __import__("hashlib").sha256(model_bytes).hexdigest(), + ) + separator = AudioStemSeparator(AudioSeparationConfig(model_artifact_path=model_path)) + + assert separator._load_model() is separator._load_model() + assert calls == [("955717e8", tmp_path)] + + +def test_audio_stem_separator_rejects_wrong_model_digest_before_local_load( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Reject same-sized model bytes that fail the full SHA-256 identity check.""" + model_path = tmp_path / "955717e8-8726e21a.th" + model_path.write_bytes(b"untrusted") + called = False + + def fake_get_model(name: str, *, repo: Path) -> _FakeModel: + nonlocal called + called = True + return _FakeModel() + + _install_fake_demucs(monkeypatch, fake_get_model) + monkeypatch.setattr(audio_separator_module, "_HTDEMUCS_MODEL_BYTES", len(b"untrusted")) + monkeypatch.setattr(audio_separator_module, "_HTDEMUCS_MODEL_SHA256", "0" * 64) + separator = AudioStemSeparator(AudioSeparationConfig(model_artifact_path=model_path)) + + with pytest.raises(ValueError, match="failed full SHA-256 verification"): + separator._load_model() + + assert called is False + + +def test_audio_stem_separator_resolves_verified_model_from_environment( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Allow explicit environment provisioning without opening a remote model path.""" + model_bytes = b"environment model" + model_path = tmp_path / "955717e8-8726e21a.th" + model_path.write_bytes(model_bytes) + monkeypatch.setenv("BANDSCOPE_HTDEMUCS_MODEL_PATH", str(model_path)) + monkeypatch.setattr(audio_separator_module, "_HTDEMUCS_MODEL_BYTES", len(model_bytes)) + monkeypatch.setattr( + audio_separator_module, + "_HTDEMUCS_MODEL_SHA256", + __import__("hashlib").sha256(model_bytes).hexdigest(), + ) + + separator = AudioStemSeparator() + + assert separator._verified_model_artifact_path() == model_path + + +def test_audio_stem_separator_resolves_verified_model_from_torch_home( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Use the conventional local torch cache when no explicit path is configured.""" + model_bytes = b"torch home model" + model_path = tmp_path / "hub" / "checkpoints" / "955717e8-8726e21a.th" + model_path.parent.mkdir(parents=True) + model_path.write_bytes(model_bytes) + monkeypatch.delenv("BANDSCOPE_HTDEMUCS_MODEL_PATH", raising=False) + monkeypatch.setenv("TORCH_HOME", str(tmp_path)) + monkeypatch.setattr(audio_separator_module, "_HTDEMUCS_MODEL_BYTES", len(model_bytes)) + monkeypatch.setattr( + audio_separator_module, + "_HTDEMUCS_MODEL_SHA256", + __import__("hashlib").sha256(model_bytes).hexdigest(), + ) + + assert AudioStemSeparator()._verified_model_artifact_path() == model_path + + +@pytest.mark.parametrize("failure", ["symlink", "wrong-name", "directory", "wrong-size"]) +def test_audio_stem_separator_rejects_untrusted_model_path_shapes( + failure: str, tmp_path: Path +) -> None: + """Reject path indirection, wrong names, non-files, and byte-count drift.""" + target = tmp_path / "955717e8-8726e21a.th" + if failure == "symlink": + real_model = tmp_path / "real-model.th" + real_model.write_bytes(b"model") + target.symlink_to(real_model) + elif failure == "wrong-name": + target = tmp_path / "renamed-model.th" + target.write_bytes(b"model") + elif failure == "directory": + target.mkdir() + else: + target.write_bytes(b"wrong size") + + separator = AudioStemSeparator(AudioSeparationConfig(model_artifact_path=target)) + + with pytest.raises(ValueError, match="model artifact"): + separator._verified_model_artifact_path() + + def test_audio_stem_separator_splits_local_audio_into_canonical_stems( tmp_path, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -319,7 +461,7 @@ def test_audio_stem_separator_caches_model(tmp_path, monkeypatch: pytest.MonkeyP """Ensure the model is loaded once and reused across calls.""" calls = {"n": 0} - def fake_get_model(name: str) -> _FakeModel: + def fake_get_model(name: str, *, repo: Path) -> _FakeModel: calls["n"] += 1 return _FakeModel() @@ -329,6 +471,11 @@ def fake_apply_model( return {name: np.zeros(audio.size, dtype=np.float32) for name in _DEMUCS_SOURCES} _install_fake_demucs(monkeypatch, fake_get_model) + monkeypatch.setattr( + AudioStemSeparator, + "_verified_model_artifact_path", + lambda self: Path("/verified/955717e8-8726e21a.th"), + ) monkeypatch.setattr(AudioStemSeparator, "_apply_model", fake_apply_model) audio_path = tmp_path / "mix.wav" diff --git a/services/analysis-engine/tests/test_supply_chain_policy.py b/services/analysis-engine/tests/test_supply_chain_policy.py index bdc353c4f..24507bf85 100644 --- a/services/analysis-engine/tests/test_supply_chain_policy.py +++ b/services/analysis-engine/tests/test_supply_chain_policy.py @@ -13,6 +13,29 @@ from conftest import load_module, make_symlink_or_skip +def test_node_security_floors_are_locked_to_patched_versions() -> None: + """Keep direct and transitive Node dependencies above current advisory floors.""" + repo_root = Path(__file__).resolve().parents[3] + desktop_package = json.loads( + (repo_root / "apps" / "desktop" / "package.json").read_text(encoding="utf-8") + ) + package_lock = json.loads((repo_root / "package-lock.json").read_text(encoding="utf-8")) + packages = package_lock["packages"] + + assert desktop_package["dependencies"]["pdfjs-dist"] == "6.2.108" + assert packages["node_modules/pdfjs-dist"]["version"] == "6.2.108" + assert tuple(map(int, packages["node_modules/nanoid"]["version"].split("."))) >= ( + 3, + 3, + 17, + ) + assert tuple(map(int, packages["node_modules/undici"]["version"].split("."))) >= ( + 7, + 28, + 1, + ) + + def test_supplemental_inventory_rejects_obsolete_or_missing_runtime_model( tmp_path: Path, ) -> None: @@ -70,6 +93,71 @@ def test_supplemental_inventory_accepts_pinned_htdemucs_runtime_model() -> None: assert violations == [] +@pytest.mark.parametrize( + ("inventory", "expected"), + [ + ([], "supplemental inventory root must be an object"), + ({"modelArtifacts": []}, "supplemental inventory missing runtime model: htdemucs"), + ( + { + "modelArtifacts": [ + { + "name": "Hybrid Transformer Demucs four-source weights", + "runtimeModelName": "htdemucs", + "version": "4.0.1", + "sourceUrl": "https://models.example/htdemucs.th", + "license": "operator-reviewed", + "checksum": f"sha256:{'0' * 64}", + "sizeBytes": True, + "storagePath": "local cache", + "distribution": "runtime-cache", + "releaseUsage": "local separation", + "verification": "full digest before load", + } + ] + }, + "requires positive integer sizeBytes", + ), + ( + { + "modelArtifacts": [ + { + "name": 7, + "runtimeModelName": "htdemucs", + "version": "4.0.1", + "sourceUrl": "https://models.example/htdemucs.th", + "license": "operator-reviewed", + "checksum": f"sha256:{'0' * 64}", + "sizeBytes": 7, + "storagePath": "local cache", + "distribution": "runtime-cache", + "releaseUsage": "local separation", + "verification": "full digest before load", + } + ] + }, + "field name must be a non-empty string", + ), + ], +) +def test_supplemental_inventory_rejects_malformed_schema( + tmp_path: Path, inventory: object, expected: str +) -> None: + """Return stable diagnostics for untrusted inventory shapes and field types.""" + supply_chain = load_module( + "scripts/checks/verify_supply_chain.py", + f"verify_supply_chain_malformed_{abs(hash(expected))}", + ) + inventory_path = tmp_path / "inventory.json" + inventory_path.write_text(json.dumps(inventory), encoding="utf-8") + separator_path = tmp_path / "audio_separator.py" + separator_path.write_text('model_name: str = "htdemucs"\n', encoding="utf-8") + + violations = supply_chain.supplemental_inventory_violations(inventory_path, separator_path) + + assert any(expected in violation for violation in violations) + + def central_required_workflow_policy_text() -> str: """Return the repository policy text that delegates review automation centrally.""" repo_root = Path(__file__).resolve().parents[3] diff --git a/services/analysis-engine/tests/test_youtube.py b/services/analysis-engine/tests/test_youtube.py index 4d1ea9c91..a5c22e48e 100644 --- a/services/analysis-engine/tests/test_youtube.py +++ b/services/analysis-engine/tests/test_youtube.py @@ -1,13 +1,20 @@ """Tests for YouTube import capabilities.""" +import hashlib import importlib import sys +from pathlib import Path from unittest.mock import MagicMock, patch import pytest import yt_dlp # type: ignore -from bandscope_analysis.youtube import MAX_YOUTUBE_URL_LENGTH, download_youtube_audio, validate_url +from bandscope_analysis.youtube import ( + MAX_YOUTUBE_URL_LENGTH, + _verified_ffmpeg_location, + download_youtube_audio, + validate_url, +) def test_validate_url() -> None: @@ -115,7 +122,8 @@ def test_download_youtube_audio_success( assert called_opts["noprogress"] is True assert called_opts["noplaylist"] is True assert called_opts["geo_bypass"] is False - assert called_opts["compat_opts"] == {"no-certifi"} + assert "compat_opts" not in called_opts + assert "nocheckcertificate" not in called_opts assert called_opts["postprocessors"] == [{"key": "FFmpegExtractAudio"}] assert "%(id)s.%(ext)s" in called_opts["outtmpl"] @@ -131,6 +139,109 @@ def test_download_youtube_audio_success( ) +@patch("bandscope_analysis.youtube.os.path.getsize") +@patch("bandscope_analysis.youtube.os.path.exists") +@patch("bandscope_analysis.youtube.yt_dlp.YoutubeDL") +def test_download_uses_verified_absolute_ffmpeg_when_release_identity_is_configured( + mock_ydl_class: MagicMock, + mock_exists: MagicMock, + mock_getsize: MagicMock, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Bind yt-dlp post-processing to exact operator-provisioned executable bytes.""" + ffmpeg_path = tmp_path / "ffmpeg" + ffmpeg_bytes = b"pinned ffmpeg executable" + ffmpeg_path.write_bytes(ffmpeg_bytes) + ffmpeg_path.chmod(0o700) + monkeypatch.setenv("BANDSCOPE_FFMPEG_PATH", str(ffmpeg_path)) + monkeypatch.setenv("BANDSCOPE_FFMPEG_SHA256", hashlib.sha256(ffmpeg_bytes).hexdigest()) + mock_ydl = MagicMock() + mock_ydl_class.return_value.__enter__.return_value = mock_ydl + mock_ydl.extract_info.return_value = { + "id": "abc123DEF45", + "title": "Test Video", + "duration": 60, + } + mock_ydl.prepare_filename.return_value = "/tmp/abc123DEF45.webm" + mock_exists.return_value = True + mock_getsize.return_value = 10 + + result = download_youtube_audio("https://youtube.com/watch?v=abc123DEF45", "/tmp") + + assert result["ok"] is True + called_opts = mock_ydl_class.call_args[0][0] + assert called_opts["ffmpeg_location"] == str(ffmpeg_path.resolve()) + + +@patch("bandscope_analysis.youtube.yt_dlp.YoutubeDL") +def test_download_rejects_changed_ffmpeg_before_provider_access( + mock_ydl_class: MagicMock, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Fail closed when configured release evidence names changed executable bytes.""" + ffmpeg_path = tmp_path / "ffmpeg" + ffmpeg_path.write_bytes(b"changed executable") + ffmpeg_path.chmod(0o700) + monkeypatch.setenv("BANDSCOPE_FFMPEG_PATH", str(ffmpeg_path)) + monkeypatch.setenv("BANDSCOPE_FFMPEG_SHA256", "0" * 64) + + result = download_youtube_audio("https://youtube.com/watch?v=abc123DEF45", "/tmp") + + assert result == { + "ok": False, + "error": { + "code": "download_error", + "message": "YouTube import failed. Please use a local audio file instead.", + }, + } + mock_ydl_class.assert_not_called() + + +@pytest.mark.parametrize( + ("path", "digest", "message"), + [ + (None, "0" * 64, "requires both path and SHA-256"), + ("ffmpeg", None, "requires both path and SHA-256"), + ("ffmpeg", "not-a-digest", "full lowercase SHA-256"), + ("ffmpeg", "0" * 64, "absolute executable path"), + ], +) +def test_verified_ffmpeg_rejects_incomplete_or_relative_identity( + path: str | None, + digest: str | None, + message: str, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Reject incomplete, malformed, or relative release executable identities.""" + if path is None: + monkeypatch.delenv("BANDSCOPE_FFMPEG_PATH", raising=False) + else: + monkeypatch.setenv("BANDSCOPE_FFMPEG_PATH", path) + if digest is None: + monkeypatch.delenv("BANDSCOPE_FFMPEG_SHA256", raising=False) + else: + monkeypatch.setenv("BANDSCOPE_FFMPEG_SHA256", digest) + + with pytest.raises(ValueError, match=message): + _verified_ffmpeg_location() + + +@pytest.mark.parametrize("failure", ["missing", "not-executable"]) +def test_verified_ffmpeg_rejects_unavailable_absolute_executable( + failure: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Reject absent or non-executable absolute ffmpeg candidates.""" + ffmpeg_path = tmp_path / "ffmpeg" + if failure == "not-executable": + ffmpeg_path.write_bytes(b"ffmpeg") + ffmpeg_path.chmod(0o600) + monkeypatch.setenv("BANDSCOPE_FFMPEG_PATH", str(ffmpeg_path)) + monkeypatch.setenv("BANDSCOPE_FFMPEG_SHA256", "0" * 64) + + with pytest.raises(ValueError, match="executable is unavailable"): + _verified_ffmpeg_location() + + @patch("bandscope_analysis.youtube.os.path.getsize") @patch("bandscope_analysis.youtube.os.path.exists") @patch("bandscope_analysis.youtube.yt_dlp.YoutubeDL") @@ -356,6 +467,7 @@ def test_module_execution( # Mock os to ensure runpy uses our mocked filesystem methods mock_os = MagicMock() # Keep some essential attributes + mock_os.environ = {} mock_os.path = MagicMock() mock_os.path.exists.return_value = True mock_os.path.getsize.return_value = 10 * 1024 * 1024 diff --git a/services/analysis-engine/tests/test_youtube_stem_e2e.py b/services/analysis-engine/tests/test_youtube_stem_e2e.py index b1513ab61..27291e5ca 100644 --- a/services/analysis-engine/tests/test_youtube_stem_e2e.py +++ b/services/analysis-engine/tests/test_youtube_stem_e2e.py @@ -22,6 +22,7 @@ MIN_VOCAL_SI_SDR_IMPROVEMENT_DB, KnownStemFixture, _AllowlistedRedirectHandler, + _normalized_correlation, align_active_reference_window, align_known_stem_through_master, download_verified_creator_master, @@ -34,7 +35,7 @@ AudioSeparationConfig, AudioStemSeparator, ) -from bandscope_analysis.youtube import download_youtube_audio +from bandscope_analysis.youtube import _verified_ffmpeg_location, download_youtube_audio class _FakeResponse(io.BytesIO): @@ -146,6 +147,45 @@ def test_align_active_reference_window_recovers_delay_and_loud_section() -> None assert aligned.correlation > 0.99 +def test_identity_correlation_preserves_phase_sign() -> None: + """Do not authenticate a phase-inverted candidate as the same recording.""" + signal = np.array([-2.0, -0.5, 0.5, 2.0], dtype=np.float64) + + assert _normalized_correlation(signal, -signal) == pytest.approx(-1.0) + + +@pytest.mark.parametrize( + ("kwargs", "message"), + [ + ({"sample_rate": 0}, "sample_rate must be positive"), + ({"window_seconds": 0.0}, "alignment durations are invalid"), + ({"max_lag_seconds": -0.1}, "alignment durations are invalid"), + ({"envelope_hop_seconds": 0.0}, "alignment resolution is invalid"), + ({"refinement_seconds": -0.1}, "alignment resolution is invalid"), + ({"window_seconds": 2.0}, "reference is shorter"), + ], +) +def test_align_active_reference_window_rejects_invalid_contract( + kwargs: dict[str, float | int], message: str +) -> None: + """Exercise every caller-controlled alignment validation family.""" + parameters: dict[str, float | int] = { + "sample_rate": 10, + "window_seconds": 0.5, + "max_lag_seconds": 0.2, + "envelope_hop_seconds": 0.1, + "refinement_seconds": 0.1, + } + parameters.update(kwargs) + + with pytest.raises(ValueError, match=message): + align_active_reference_window( + np.arange(10, dtype=np.float64), + np.arange(10, dtype=np.float64), + **parameters, + ) + + def test_download_verified_reference_stem_extracts_only_the_pinned_member( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -321,11 +361,28 @@ def test_required_root_suite_explicitly_excludes_live_youtube_marker() -> None: repo_root = Path(__file__).resolve().parents[3] runner = (repo_root / "scripts/checks/run_root_tests.mjs").read_text(encoding="utf-8") - assert '"-m",\n "not youtube_stem_e2e"' in runner + normalized = " ".join(runner.split()) + assert '"-m", "not youtube_stem_e2e"' in normalized + + +def test_live_benchmark_requires_verified_ffmpeg_before_fixture_access( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Fail closed before network access when executable identity is not configured.""" + monkeypatch.delenv("BANDSCOPE_FFMPEG_PATH", raising=False) + monkeypatch.delenv("BANDSCOPE_FFMPEG_SHA256", raising=False) + + with pytest.raises(AssertionError, match="verified ffmpeg identity"): + _assert_real_youtube_known_stem_separation(tmp_path) def _assert_real_youtube_known_stem_separation(root: Path) -> None: """Run the live benchmark inside an ephemeral, caller-owned media directory.""" + ffmpeg_location = _verified_ffmpeg_location() + assert ffmpeg_location is not None, ( + "Live benchmark requires verified ffmpeg identity via " + "BANDSCOPE_FFMPEG_PATH and BANDSCOPE_FFMPEG_SHA256" + ) fixture = BRAD_SUCKS_FIXTURE reference_path = download_verified_reference_stem(fixture, root) master_path = download_verified_creator_master(fixture, root) diff --git a/supply-chain/supplemental-component-inventory.json b/supply-chain/supplemental-component-inventory.json index 8f4b03e05..e4790fcec 100644 --- a/supply-chain/supplemental-component-inventory.json +++ b/supply-chain/supplemental-component-inventory.json @@ -4,7 +4,7 @@ "packageManagedTools": [ { "name": "yt-dlp", - "version": ">=2026.7.4", + "version": "2026.7.4", "sourceUrl": "https://pypi.org/project/yt-dlp/", "license": "Unlicense", "storagePath": "services/analysis-engine/uv.lock", @@ -18,7 +18,7 @@ "version": "operator-managed supported release", "sourceUrl": "https://ffmpeg.org/download.html", "license": "LGPL-2.1-or-later or GPL-2.0-or-later, depending on build configuration", - "storagePath": "system PATH; not bundled by BandScope", + "storagePath": "operator-provided absolute path; BANDSCOPE_FFMPEG_PATH and BANDSCOPE_FFMPEG_SHA256 bind release evidence; not bundled", "distribution": "operator-provided", "releaseUsage": "Required by yt-dlp audio extraction and media decoding; release preflight records the resolved version." } @@ -35,7 +35,7 @@ "storagePath": "user runtime cache managed by torch.hub/demucs; not committed or bundled", "distribution": "runtime-cache", "releaseUsage": "Loaded locally on supported platforms to separate vocals, bass, drums, and other stems.", - "verification": "Demucs 4.0.1 checks the filename hash prefix 8726e21a before deserialization; the full SHA-256 and byte size were independently recorded on 2026-08-09. Full-hash pre-load enforcement remains a release blocker tracked by ADR-0001." + "verification": "BandScope requires the exact local filename and verifies 84,141,911 bytes plus the full SHA-256 before passing a local-only repository to Demucs; missing, symlinked, or mismatched bytes fail before deserialization." } ], "notes": [ From 59e539667ca278461a5f32ca5bab13b299e97275 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 10 Aug 2026 00:01:37 +0900 Subject: [PATCH 06/34] fix(separation): complete known-stem review hardening --- ARCHITECTURE.md | 11 +- CHANGELOG.md | 16 +- CLAUDE.md | 5 +- docs/PRD.md | 10 +- docs/TRD.md | 38 +- ...e-separation-runtime-and-model-delivery.md | 28 +- .../0002-known-stem-youtube-quality-gate.md | 17 +- docs/adr/README.md | 2 +- docs/architecture/diagrams.md | 74 +-- docs/architecture/overview.md | 13 +- .../real-audio-accuracy-acceptance.md | 4 +- docs/documentation-coverage-matrix.md | 35 +- docs/engineering/acceptance-criteria.md | 9 +- .../youtube-known-stem-validation.md | 71 +-- docs/operations/deploy-runbook.md | 28 +- ...26-03-10-bandscope-cross-platform-build.md | 16 +- docs/plans/2026-03-10-bandscope-harness.md | 16 +- ...026-03-10-bandscope-supply-chain-design.md | 17 +- .../2026-03-10-bandscope-supply-chain.md | 33 +- .../plans/2026-03-28-ml-engine-integration.md | 17 +- docs/release/release-policy.md | 18 +- docs/repository/bootstrap-plan.md | 2 +- docs/security/dependency-policy.md | 6 +- docs/security/sbom-policy.md | 6 +- .../github-bootstrap-execution-policy.md | 9 +- scripts/checks/security_gates.py | 46 +- scripts/checks/verify_docs.py | 10 +- scripts/checks/verify_security_notes.py | 66 +-- scripts/checks/verify_supply_chain.py | 159 ++++++- .../src/bandscope_analysis/api.py | 17 +- .../bandscope_analysis/separation/__init__.py | 3 +- .../separation/audio_separator.py | 179 ++++--- .../src/bandscope_analysis/youtube.py | 177 +++++-- .../tests/known_stem_benchmark.py | 12 +- services/analysis-engine/tests/test_api.py | 8 +- .../tests/test_documentation_policy.py | 35 +- .../analysis-engine/tests/test_separation.py | 444 ++++++++++++------ .../tests/test_supply_chain_policy.py | 245 ++++++++-- .../analysis-engine/tests/test_youtube.py | 362 +++++++++++--- .../tests/test_youtube_stem_e2e.py | 167 +++++-- .../supplemental-component-inventory.json | 24 +- 41 files changed, 1764 insertions(+), 691 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index db2a6e76c..28ee73414 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -93,13 +93,14 @@ Last updated: 2026-08-09 model. - Demucs random temporal shifts are disabled (`shifts=0`) so the same bytes and model produce reproducible local analysis and benchmark evidence. -- Model inference is local after operator provisioning. The runtime requires the exact official - weight artifact in a local cache and never downloads it; missing, symlinked, wrong-sized, or - full-SHA-mismatched bytes fail before Demucs/torch deserialization. +- Model inference is local and fail-closed. A trusted provisioning step must place the exact + official weight artifact in a user-scoped cache before runtime; a missing artifact is never + fetched by the separator. The repository and release artifacts do not bundle the weights. - The exact signature, source URL, full SHA-256, byte size, distribution status, and model-rights uncertainty are tracked in `supply-chain/supplemental-component-inventory.json` and ADR-0001. -- Full-SHA verification before torch deserialization is implemented. A recorded model-rights and - permitted-delivery decision remains a release blocker; the official weights are not bundled. +- The separator verifies a non-symlinked regular file's exact byte size and full SHA-256, then + deserializes those same verified bytes. A model-rights/legal delivery decision remains a release + blocker. - Current dependency markers exclude Demucs on macOS Intel; unsupported platforms must surface the existing safe fallback rather than pretending to separate stems. diff --git a/CHANGELOG.md b/CHANGELOG.md index a7f8aa588..3c5b6e7b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,20 +12,22 @@ - Added canonical PRD, TRD, ADR, architecture/UML/logical-artifact diagrams, traceability, and machine-checked documentation coverage for the known-stem quality boundary. - Replaced the retired FFT-era bandsplit model inventory with the exact htdemucs runtime artifact, - full SHA-256, byte size, delivery status, ffmpeg prerequisite, and release blockers. + full SHA-256, byte size, delivery status, verified ffmpeg/ffprobe prerequisites, and release + blockers. - Display the analyzed song tempo (BPM) as a badge in the rehearsal workspace. - 각 합주 역할(Role)별 개인 연습 진행도를 0~100% 범위로 기록 및 시각화할 수 있는 연습 진척도(`practiceProgress`) 트래커 기능 추가. UI 컨트롤(슬라이더 및 +/- 버튼)과 한/영 다국어 지원 포함. ### Fixed -- Kept YouTube TLS verification enabled while retaining yt-dlp's maintained CA-bundle fallback for - minimal containers without a configured system trust store. +- Kept YouTube TLS verification enabled, using populated OS-managed CA roots when available and + retaining yt-dlp's maintained CA-bundle fallback when the system trust store is empty or fails. - Raised `pdfjs-dist`, `nanoid`, and `undici` to patched versions for the current high-severity advisories and added a mutation-sensitive lockfile floor contract. -- Made htdemucs inference fail closed before deserialization unless the exact local artifact passes - filename, non-symlink, byte-size, and full-SHA-256 checks; the runtime no longer downloads weights. -- Bound release/live benchmark ffmpeg execution to an operator-provided absolute path and full - SHA-256 while preserving fail-closed provider and model evidence. +- Made htdemucs loading offline and fail-closed: the runtime accepts only the inventoried filename, + byte size, and full SHA-256, rejects filesystem identity races, and deserializes the verified + bytes rather than downloading a missing checkpoint. +- Verified exact platform-native sibling ffmpeg/ffprobe executable names and identities before any + live fixture access or yt-dlp invocation. - Reconciled stale CodeRabbit-gate wording with the canonical stable-check and review-equivalent policy; qualifying evidence is now defined against the exact current head, and a rate-limited, status-only, author, or predecessor review is not treated as completed review evidence. diff --git a/CLAUDE.md b/CLAUDE.md index b6ee31220..ed8c8f860 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -60,8 +60,9 @@ Three layers, decoupled through shared contracts: - `services/analysis-engine` — Python package `bandscope_analysis` (librosa/numpy). Entry point `cli.py` reads a JSON job request on stdin and prints a structured job-status JSON envelope on stdout (`--progress-jsonl` streams progress lines). `api.py` orchestrates the pipeline across the `separation`, `sections`, `roles`, `chords`, `ranges`, `temporal`, `transcription`, and `youtube` modules. - Production source separation uses `htdemucs` on supported platforms. The exact runtime model artifact is inventoried but not bundled; operators must provision it locally, and production - verifies its byte size and full SHA-256 before local-only Demucs loading. The active known-stem test crosses the - production YouTube and separator boundaries; see `docs/TRD.md` and the operator guide. + verifies its byte size and full SHA-256 before deserializing those same in-memory bytes. The + active known-stem test crosses the production YouTube and separator boundaries; see `docs/TRD.md` + and the operator guide. Data flow: React UI → Tauri IPC command → Rust validation + Python subprocess over stdin/stdout → job status and progress events emitted back to the UI. diff --git a/docs/PRD.md b/docs/PRD.md index 36430d272..21ee17edf 100644 --- a/docs/PRD.md +++ b/docs/PRD.md @@ -10,9 +10,10 @@ role-specific harmony and range, groove and entry cues, separated stem previews, rehearsal priority. It serves band leaders and players who need actionable preparation without a DAW or notation-grade transcription workflow. -Issue #770 and ADR-0002 establish one essential proof obligation: BandScope must demonstrate that -its production YouTube intake and production separator improve a real, known source rather than -merely returning plausible-looking arrays or synthetic demo output. +The product has one essential proof obligation: BandScope must demonstrate that its production +YouTube intake and production separator improve a real, known source rather than merely returning +plausible-looking arrays or synthetic demo output. GitHub issue #770 and ADR-0002 govern this +requirement. This is a bounded source-separation slice of GitHub issue #770, not completion of its broader harmony, beat/tempo, structure, range, cue, confidence, public-corpus, private-corpus, manifest, @@ -68,7 +69,8 @@ after explicit opt-in. The known-stem lane becomes blocking for a release only after all of the following exist: 1. documented authorization for the chosen live access mode; -2. full-hash pre-load verification of the exact model artifact; +2. full-hash pre-load verification of the exact model artifact and a recorded model-rights/legal + decision for the chosen provisioning or distribution path; 3. at least one recorded passing supported-platform run on the exact release candidate; 4. thresholds calibrated on an authorized YouTube candidate and a drift/flake triage owner; 5. ordinary CI, security, coverage, packaging, SBOM, review, and provenance gates pass. diff --git a/docs/TRD.md b/docs/TRD.md index 5cb917bc4..5dc306a39 100644 --- a/docs/TRD.md +++ b/docs/TRD.md @@ -28,7 +28,7 @@ and data-flow views are in `docs/architecture/diagrams.md`. | TRD-KS-008 | Reject candidate drift when YouTube/master duration differs by > 1.0 s or aligned identity correlation is < 0.90. | Pre-inference live assertions against the pinned finished master. | | TRD-KS-009 | Run deterministic contract/security tests by default and require `BANDSCOPE_RUN_YOUTUBE_STEM_E2E=1` for live network/model execution. | Pytest marker and environment guard. | | TRD-KS-010 | Clean every downloaded/scored artifact on success and failure. | Nested `TemporaryDirectory` plus postcondition. | -| TRD-KS-011 | Bind model identity to inventory and full SHA-256 before release. | `AudioStemSeparator._verified_model_artifact_path` requires the exact local filename, 84,141,911 bytes, and SHA-256 `8726e21a…a8b4` before local-only Demucs loading. | +| TRD-KS-011 | Bind model identity to inventory and full SHA-256 before any torch deserialization. | Inventory records htdemucs signature `955717e8`, 84,141,911 bytes, and SHA-256 `8726e21a…a8b4`; runtime verifies and deserializes the same in-memory bytes with no download fallback. | ## Data and class contracts @@ -75,22 +75,28 @@ global random-seed side effect in the test harness. ## Model delivery and supply chain -`AudioStemSeparator` maps `htdemucs` to signature `955717e8` and requires the exact local -`955717e8-8726e21a.th` artifact. It rejects a missing/symlinked file, wrong byte count, or full -SHA-256 mismatch before passing a local repository to Demucs, so this runtime has no model-download -path. The model remains unbundled; ADR-0001 still requires an explicit rights and permitted-delivery -decision before release readiness. - -`ffmpeg` is operator-provided. Ordinary local imports may use yt-dlp's normal discovery, but release -evidence and the live benchmark require `BANDSCOPE_FFMPEG_PATH` to name an absolute executable and -`BANDSCOPE_FFMPEG_SHA256` to bind its full digest; the production downloader revalidates those bytes -before provider access. yt-dlp is locked at the exact version in `uv.lock`. +`AudioStemSeparator` accepts only Demucs 4.0.1 `htdemucs`, mapped to signature `955717e8` and exact +artifact `955717e8-8726e21a.th`. A trusted provisioning step must place it in the configured +user-scoped cache or provide that exact absolute file through +`BANDSCOPE_HTDEMUCS_MODEL_PATH`. Runtime rejects a missing, symlinked, non-regular, incorrectly +sized, wrongly named, or full-SHA-mismatched artifact before torch deserialization, reads it once, +and deserializes those same verified bytes. It never calls the remote Demucs loader or downloads a +missing checkpoint. The model is not bundled; ADR-0001 keeps the model-rights/legal delivery +decision as a release blocker. + +`ffmpeg` and `ffprobe` are operator-provided siblings and yt-dlp is locked to `2026.7.4`. Ordinary +product use may resolve the media tools from `PATH`, but release/live evidence must pass both +absolute executable paths and both full SHA-256 values as one four-part identity. Preflight records +both paths, hashes, exact platform-native sibling names, version outputs, and shared trusted package +identity before any reference or YouTube access; none may be described as bundled unless packaging +and licensing change. ## Failure taxonomy - `unsupported_url`, `restricted_content`, `duration_exceeded`, `size_exceeded`: production intake policy failures. - `download_failed`, `download_error`, `file_not_found`: live media/provider/tool failures. +- `runtime_dependency_invalid`: configured ffmpeg/ffprobe identity set, layout, or hash failure. - Reference byte/hash/member/redirect error: fixture integrity or SSRF boundary failure. - YouTube/master duration drift above 1.0 s or identity correlation below 0.90: wrong or drifted candidate/transcode. @@ -102,15 +108,17 @@ lane; it does not authorize a bypass or stop unrelated repository work. ## Verification and evidence -Default verification runs the 25 deterministic known-stem contract tests and explicitly excludes -the live marker. A live run uses the exact +Default verification runs every collected deterministic known-stem contract test and explicitly +excludes the live marker. A live run uses the exact command in the operator guide. Evidence must include exact commit and dependency lock, model full hash, fixture archive full hash, public video ID, OS/architecture, result code, correlation, baseline SI-SDR, vocal SI-SDR, improvement, assignment margin, duration, and cleanup result. Raw audio, archive contents, local paths, provider response bodies, cookies, and credentials are forbidden. -On 2026-08-09, commit `5a3648a11d9097b8da48bb4a3ccbd97986aec25b` passed all 13 offline -contract tests. Its explicit live attempt successfully validated the pinned reference archive but +On 2026-08-09, commit `5a3648a11d9097b8da48bb4a3ccbd97986aec25b` passed a 13-test +pre-correction partial suite. It did not yet contain the creator-master authentication, +two-global-offset composition, or explicit root-suite live-marker exclusion cases that raised the +corrected suite to 16. Its explicit live attempt successfully validated the pinned reference archive but failed at the production YouTube download boundary with HTTP 502 and produced no model score. This is failure evidence, not a passing live benchmark. diff --git a/docs/adr/0001-source-separation-runtime-and-model-delivery.md b/docs/adr/0001-source-separation-runtime-and-model-delivery.md index b161341f6..629e85b3e 100644 --- a/docs/adr/0001-source-separation-runtime-and-model-delivery.md +++ b/docs/adr/0001-source-separation-runtime-and-model-delivery.md @@ -1,6 +1,6 @@ # ADR-0001: Source Separation Runtime and Model Delivery -Status: Accepted with release blockers +Status: Proposed on active branch (implementation complete; release blockers remain) Date: 2026-08-09 ## Context and drivers @@ -10,9 +10,8 @@ separation. BandScope now uses Demucs 4.0.1 `htdemucs` to return vocals, bass, d local rehearsal analysis. The production boundary must remain local-first after model provisioning, bounded on CPU, platform-honest, and traceable to an exact model artifact. -The upstream Demucs remote loader downloads weights on first use and verifies only the eight-hex -hash prefix embedded in `955717e8-8726e21a.th`. BandScope does not use that remote loader: its -production boundary requires a pre-provisioned local artifact and verifies 84,141,911 bytes plus SHA-256 +The former Demucs loader could download weights on first use and verified only the eight-hex hash +prefix embedded in `955717e8-8726e21a.th`. The exact artifact is 84,141,911 bytes with SHA-256 `8726e21a993978c7ba086d3872e7608d7d5bfca646ca4aca459ffda844faa8b4`. The Demucs code is MIT licensed, but no separate commercial redistribution grant for the official weights was identified; the upstream licensing discussion characterizes the weights as scientific-use material. @@ -23,14 +22,16 @@ the upstream licensing discussion characterizes the weights as scientific-use ma 2. The old `bandsplit-v1-profile` asset and inventory record are retired and must not reappear. 3. The exact official source URL, signature, full SHA-256, byte size, license uncertainty, cache location, and release usage remain in `supply-chain/supplemental-component-inventory.json`. -4. Runtime retrieval is not equivalent to bundling. Documentation and SBOM evidence must preserve - that distinction. +4. Trusted external provisioning is not equivalent to bundling. Documentation and SBOM evidence + must preserve that distinction. 5. A release claiming source-separation readiness must verify the full SHA-256 before any torch deserialization and must have a recorded legal decision for its chosen download or distribution path. -6. Runtime model download is prohibited. Missing, symlinked, wrong-sized, or full-hash-mismatched - local bytes fail before deserialization. Source separation remains unavailable on macOS Intel - under the current dependency markers. +6. Runtime model retrieval is forbidden. A trusted external provisioning step must populate the + expected user-scoped cache or supply the exact absolute inventoried file through + `BANDSCOPE_HTDEMUCS_MODEL_PATH`; missing, wrongly named, non-regular, symlinked, incorrectly + sized, or full-SHA-mismatched weights fail before deserialization. Source separation remains + unavailable on macOS Intel under the current dependency markers. ## Alternatives considered @@ -38,16 +39,17 @@ the upstream licensing discussion characterizes the weights as scientific-use ma - Bundle official htdemucs weights immediately: rejected because repository/release size and model redistribution rights are unresolved. - Rely on Demucs' eight-hex prefix only: rejected because it is weaker than the repository's - full-integrity policy. + full-integrity policy and still permits deserialization before BandScope verifies exact identity. - Replace with ONNX or another commercially licensed model: viable future work, but it requires parity, quality, platform, performance, and licensing evidence. ## Consequences BandScope obtains real separation quality but inherits torch/Demucs resource cost, platform gaps, -operator provisioning, and an upstream model-rights decision. Release evidence cannot describe the -model as bundled. The supplemental inventory check now fails if the runtime -model is missing, incompletely pinned, or replaced by the retired profile. +an explicit provisioning requirement, and an upstream model-rights decision. The runtime is fully +offline and fails closed when the cache is absent; that does not authorize redistribution or make +the model bundled. The supplemental inventory check fails if the runtime model is missing, +incompletely pinned, or replaced by the retired profile. ## Security and governance implications diff --git a/docs/adr/0002-known-stem-youtube-quality-gate.md b/docs/adr/0002-known-stem-youtube-quality-gate.md index c055f8c6a..861ff687c 100644 --- a/docs/adr/0002-known-stem-youtube-quality-gate.md +++ b/docs/adr/0002-known-stem-youtube-quality-gate.md @@ -48,17 +48,18 @@ threshold review, not threshold weakening. ## Security, privacy, and legal implications -The test crosses public network, archive, decoder, ffmpeg, model, filesystem, and subprocess trust -boundaries. It uses strict HTTPS/host/size/hash/member allowlists, test-owned temporary storage, -bounded media, sanitized diagnostics, and cleanup. It adds no cookies, credentials, account login, -paywall, DRM, geo, or anti-bot bypass. Creator permission for source files does not itself authorize -automated YouTube access; the operator must verify the intended access against current terms and -rights. +The test crosses public network, archive, decoder, the verified sibling `ffmpeg`/`ffprobe` +executable set, model, filesystem, and subprocess trust boundaries. It uses strict +HTTPS/host/size/hash/member allowlists, test-owned temporary storage, bounded media, sanitized +diagnostics, and cleanup. The complete media executable identity is verified before reference +network access. It adds no cookies, credentials, account login, paywall, DRM, geo, or anti-bot +bypass. Creator permission for source files does not itself authorize automated YouTube access; the +operator must verify the intended access against current terms and rights. ## Acceptance, recovery, and rollback -- Sixteen deterministic contract tests pass in ordinary CI; the root runner explicitly excludes the - live marker. +- Every collected deterministic contract test passes in ordinary CI; the root runner explicitly + excludes the live marker. Test count is recorded as evidence, not fixed policy. - A controlled live run on the exact candidate records all required scores and cleanup evidence. - Fixture drift causes a distinct pre-model failure. - Provider/model unavailability remains a failure after explicit opt-in. diff --git a/docs/adr/README.md b/docs/adr/README.md index 8c24568b3..c14c14cdf 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -5,7 +5,7 @@ decision through a new ADR that names the superseded record. | ADR | Status | Decision | |---|---|---| -| `0001-source-separation-runtime-and-model-delivery.md` | Accepted with release blockers | Use real four-source htdemucs locally, require exact full-hash pre-load verification, and retain the model-rights/permitted-delivery decision as a release blocker. | +| `0001-source-separation-runtime-and-model-delivery.md` | Proposed on active branch | Use real four-source htdemucs locally, require trusted external provisioning, verify the exact artifact before deserialization, and retain the model-rights/legal delivery decision as a release blocker. | | `0002-known-stem-youtube-quality-gate.md` | Proposed on active branch | Validate the production YouTube-to-separator path with a creator-published known vocal stem, single alignment, SI-SDR improvement, and assignment margin. | | `0003-ephemeral-benchmark-evidence-model.md` | Proposed on active branch | Keep media and signal arrays ephemeral; retain only bounded evidence when authorized, so a relational ERD is not currently authoritative. | diff --git a/docs/architecture/diagrams.md b/docs/architecture/diagrams.md index 97da3b5e0..01fb1b1a4 100644 --- a/docs/architecture/diagrams.md +++ b/docs/architecture/diagrams.md @@ -24,16 +24,18 @@ sequenceDiagram participant Intake as Production YouTube intake participant Ref as Pinned reference loader participant Align as Global aligner - Operator->>Test: Explicit opt-in - par Untrusted downloads - Test->>Intake: Public YouTube URL - Intake-->>Test: Bounded decoded mix - and - Test->>Ref: Archive + master metadata - Ref-->>Test: Authenticated master + vocals.wav - end + Operator->>Test: Opt-in + runtime identities + Test->>Test: Verify ffmpeg + ffprobe + Test->>Ref: Pinned archive metadata + Ref-->>Test: Authenticated vocals.wav + Test->>Ref: Pinned master metadata + Ref-->>Test: Authenticated master file + Test->>Intake: Public YouTube URL + Intake-->>Test: Bounded audio filepath + Test->>Test: Decode three mono signals Test->>Align: Mix + master + vocal - Align-->>Test: Identity proof + composed 12 s window + Align-->>Test: Correlation + composed 12 s window + Test->>Test: Apply identity threshold ``` ## Known-stem inference and scoring sequence (active branch) @@ -46,7 +48,7 @@ sequenceDiagram Test->>Sep: Scored mix window Sep-->>Test: vocals / bass / drums / other Test->>Score: Stems + mix + reference - Score-->>Test: Identity, SI-SDRi, assignment margin + Score-->>Test: SI-SDRi + assignment margin ``` ## Benchmark state model @@ -94,6 +96,12 @@ classDiagram class AudioStemSeparator { +separate(audio_path) AudioSeparationResult } + class ModelArtifactSpec { + +signature: str + +filename: str + +sha256: str + +size_bytes: int + } class KnownStemBenchmarkWindow { +mixture: ndarray +reference: ndarray @@ -110,6 +118,7 @@ classDiagram KnownStemFixture --> AlignedStemWindow: authenticates assets AlignedStemWindow --> KnownStemBenchmarkWindow: composes two lags KnownStemBenchmarkWindow --> AudioStemSeparator: supplies one mix window + ModelArtifactSpec --> AudioStemSeparator: constrains offline load AudioStemSeparator --> BenchmarkScore: supplies named stems ``` @@ -122,42 +131,51 @@ these values without instantiating a production class. flowchart TB subgraph Desktop["User desktop"] App["BandScope app"] - Benchmark["KnownStemBenchmark
explicit opt-in validation"] - Cache["User-scoped model cache"] + Benchmark["Opt-in known-stem benchmark"] + Engine["Production analysis engine"] + ModelFile["Provisioned model file"] Temp["Ephemeral media root"] - Benchmark --> App - Benchmark --> Cache + App --> Engine + Benchmark -->|"production intake + separator"| Engine Benchmark --> Temp + ModelFile -->|"verified model bytes"| Engine end - YouTube["YouTube media boundary"] --> Benchmark - Source["Pinned creator archive"] --> Benchmark - Master["Pinned creator master"] --> Benchmark - Provisioner["Operator-controlled model provisioning"] --> Cache - Model["Official model host"] --> Provisioner + Operator["Authorized operator"] --> Benchmark + Public["YouTube + pinned creator assets"] --> Benchmark + Model["Official model host"] --> Provisioner["Trusted model provisioner"] + Provisioner -->|"cache or exact path"| ModelFile Benchmark --> Evidence["Bounded numeric evidence"] ``` -The model cache is persistent; media temp is not. The public hosts, cache contents, media, decoders, -and model bytes are untrusted until their respective policy and integrity checks pass. +The benchmark, not the product app, owns public fixture access and bounded evidence. Model +provisioning is a separate trusted operation; runtime loading never downloads a missing checkpoint. +The provisioned model file is persistent; media temp is not. Public hosts, model locations, media, +decoders, and model bytes are untrusted until their respective policy and integrity checks pass. ## Logical artifact relationship model (not a physical ERD) ```mermaid erDiagram KNOWN_STEM_FIXTURE ||--|| REFERENCE_ARCHIVE : pins + KNOWN_STEM_FIXTURE ||--|| REFERENCE_ARCHIVE_MEMBER : selects KNOWN_STEM_FIXTURE ||--|| CREATOR_MASTER : pins KNOWN_STEM_FIXTURE ||--|| YOUTUBE_MIX : identifies - REFERENCE_ARCHIVE ||--|{ REFERENCE_STEM : contains + REFERENCE_ARCHIVE ||--|{ REFERENCE_ARCHIVE_MEMBER : contains + REFERENCE_ARCHIVE_MEMBER ||--|| REFERENCE_STEM : decodes YOUTUBE_MIX ||--|| CREATOR_MASTER : identity-checks CREATOR_MASTER ||--|| ALIGNED_WINDOW : anchors YOUTUBE_MIX ||--|| ALIGNED_WINDOW : yields REFERENCE_STEM ||--|| ALIGNED_WINDOW : aligns ALIGNED_WINDOW ||--|{ SEPARATED_STEM : produces - ALIGNED_WINDOW ||--|| BENCHMARK_EVIDENCE : scores - SEPARATED_STEM }|--|| BENCHMARK_EVIDENCE : contributes + ALIGNED_WINDOW o|--o| BENCHMARK_EVIDENCE : may-score + SEPARATED_STEM }o--o| BENCHMARK_EVIDENCE : may-contribute ``` -Only `KNOWN_STEM_FIXTURE` metadata is version-controlled. `YOUTUBE_MIX`, `CREATOR_MASTER`, -`REFERENCE_STEM`, `ALIGNED_WINDOW`, and `SEPARATED_STEM` bytes are ephemeral. -`BENCHMARK_EVIDENCE` is planned as a bounded artifact, not a database row. ADR-0003 requires a new -physical ERD only if persistence is introduced. +Only `KNOWN_STEM_FIXTURE` metadata is version-controlled. An archive may contain many members, but +the fixture selects and authenticates exactly one `REFERENCE_ARCHIVE_MEMBER` before decoding it as +the reference stem. `YOUTUBE_MIX`, `CREATOR_MASTER`, `REFERENCE_ARCHIVE_MEMBER`, `REFERENCE_STEM`, +`ALIGNED_WINDOW`, and `SEPARATED_STEM` bytes are ephemeral. +`BENCHMARK_EVIDENCE` is planned as a bounded artifact, not a database row. Its aligned-window and +separated-stem relationships are optional because pre-alignment failures (such as the recorded HTTP +502) still produce valid failure evidence. ADR-0003 requires a new physical ERD only if persistence +is introduced. diff --git a/docs/architecture/overview.md b/docs/architecture/overview.md index f4475e749..0c0947a12 100644 --- a/docs/architecture/overview.md +++ b/docs/architecture/overview.md @@ -43,12 +43,13 @@ GitHub is the source of truth for repository governance, PR review, CI/CD, Code - The Python engine uses the real four-source `htdemucs` model on supported platforms; the old FFT profile is retired. -- Inference is local after a trusted cache is provisioned. The runtime never retrieves weights and - verifies the exact inventoried byte size and full SHA-256 before local deserialization. -- The known-stem validation branch defines and exercises the real YouTube intake → creator-master identity → - composed master/vocal alignment → deterministic separator → SI-SDR scoring path. Test-only - reference handling never becomes a general runtime downloader. No passing live run currently - validates that complete path; the recorded YouTube attempts failed before model scoring. +- Inference is local after a trusted cache or exact `BANDSCOPE_HTDEMUCS_MODEL_PATH` is provisioned. + Runtime rejects missing, wrongly named, symlinked, incorrectly sized, or full-SHA-mismatched + weights before deserializing the exact verified bytes; it never retrieves a missing model. +- The known-stem validation branch defines and exercises the real YouTube intake → creator-master + identity → composed master/vocal alignment → deterministic separator → SI-SDR scoring path. + Test-only reference handling never becomes a general runtime downloader. No completed live + production-path pass has yet produced an identity or SI-SDR score. - Media and stem arrays are ephemeral. Only bounded numeric/provenance evidence may be retained after authorization, so no physical benchmark database or ERD exists. diff --git a/docs/doctoring/real-audio-accuracy-acceptance.md b/docs/doctoring/real-audio-accuracy-acceptance.md index c6d96ed80..f800440a8 100644 --- a/docs/doctoring/real-audio-accuracy-acceptance.md +++ b/docs/doctoring/real-audio-accuracy-acceptance.md @@ -99,8 +99,8 @@ The active branch: - provisionally requires duration drift ≤ 1.0 s, master identity correlation ≥ 0.90, vocal SI-SDR improvement ≥ +0.5 dB, and vocal assignment margin ≥ 3.0 dB; - passes `shifts=0` to Demucs for deterministic inference; -- runs 16 metric/alignment/integrity/security/cleanup cases offline and explicitly excludes the live - marker from required CI; +- runs every collected metric/alignment/integrity/security/cleanup case offline and explicitly + excludes the live marker from required CI; - keeps live access explicit opt-in and fail-closed. On 2026-08-09, the offline contract passed at `5a3648a11d9097b8da48bb4a3ccbd97986aec25b`. diff --git a/docs/documentation-coverage-matrix.md b/docs/documentation-coverage-matrix.md index ca6646b90..484ff4b22 100644 --- a/docs/documentation-coverage-matrix.md +++ b/docs/documentation-coverage-matrix.md @@ -11,9 +11,9 @@ canonical PRD, TRD, ADRs, UML, logical data model, traceability, model inventory release/operations criteria. This branch adds those authorities and mechanical presence checks. The documentation graph is now structurally sufficient and explicitly code-current, but the product -is not yet release-ready for source separation. A passing live run, model-rights decision, -threshold calibration, supported-platform evidence, and -bounded evidence artifact remain open. +is not yet release-ready for source separation. A passing live run, model-rights/legal delivery +decision, threshold calibration, supported-platform evidence, and bounded evidence artifact remain +open. Full-hash pre-load verification is now implemented and regression-tested. Issue #770 remains open. This branch must not be described as completing the full real-audio MIR acceptance layer. @@ -24,15 +24,15 @@ acceptance layer. |---|---|---|---| | PRD | `docs/PRD.md` | Adequate for product outcome, scope, users, acceptance, legal boundary, rollout, and non-goals. | Broader multi-fixture/four-stem requirements await evidence. | | TRD | `docs/TRD.md` | Adequate for interfaces, metrics, schema, platform matrix, failure taxonomy, model delivery, evidence, and traceability. | Performance budget and calibrated thresholds are not yet accepted. | -| Architecture | `ARCHITECTURE.md`, `docs/architecture/overview.md` | Updated for htdemucs, local-only full-hash model loading, and known-stem boundaries. | Model-rights and permitted-delivery decision remains open. | -| ADR | `docs/adr/README.md`, ADR-0001..0003 | Captures model, live quality gate, and persistence/ERD decisions with alternatives and supersession. | ADR-0002/0003 remain Proposed until branch merge. | +| Architecture | `ARCHITECTURE.md`, `docs/architecture/overview.md` | Updated for fail-closed htdemucs provisioning and known-stem boundaries. | Model-rights/legal delivery decision remains open. | +| ADR | `docs/adr/README.md`, ADR-0001..0003 | Captures model, live quality gate, and persistence/ERD decisions with alternatives and supersession. | ADR-0001..0003 remain Proposed until branch merge. | | UML | `docs/architecture/diagrams.md` | Component, sequence, state, class, and deployment views included. | No additional UML is needed for the bounded slice. | | ERD/data | `docs/architecture/diagrams.md`, ADR-0003 | Logical artifact relationships and persistence status are explicit. | Physical ERD is intentionally not applicable until persistence exists. | -| Security/privacy | `docs/engineering/youtube-known-stem-validation.md`, `docs/security/app-security.md`, ADRs | Threats, trust boundaries, non-collection, pre-load full-hash verification, cleanup, and legal limits covered. | Model-rights/legal delivery decision remains open. | +| Security/privacy | `docs/engineering/youtube-known-stem-validation.md`, `docs/security/app-security.md`, ADRs | Threats, trust boundaries, non-collection, integrity, cleanup, and legal limits covered; exact model bytes are verified before load. | Rights/platform authorization remains open. | | Test strategy | `docs/TRD.md`, operator guide, acceptance criteria | Offline/live split and metric/failure contracts covered. | No successful live score has been recorded. | | MIR doctoring | `docs/doctoring/real-audio-accuracy-acceptance.md` | Issue #770 metrics, claim boundaries, tiers, and roadmap are separated from the bounded vocal slice. | Accuracy manifest, reports, other MIR families, and corpus tiers remain open. | | Operations/release | runbook and release policy | Preflight, evidence, triage, rollback, and blocking conditions covered. | Platform matrix and live pass are incomplete. | -| Supply chain | supplemental inventory and dependency policy | Retired model removed; exact runtime artifact is verified before load; release ffmpeg is bound to an absolute path/full digest. | Weight retrieval/distribution rights remain unresolved. | +| Supply chain | supplemental inventory and dependency policy | Retired model removed; code/inventory artifact parity, fail-closed pre-load verification, uv.lock-bound yt-dlp, and verified ffmpeg/ffprobe evidence contract recorded. | Model provisioning/distribution rights remain unresolved. | | Automation | active CWL autonomous loop and `docs/workflow/pr-review-merge-scheduler.md` | BandScope continuity and no-status-only termination are covered without creating a competing writer. | Dedicated BandScope loop remains paused due writer topology/active-task capacity. | | Review governance | `docs/security/github-required-checks.md`, governance, gitflow, contributing, bootstrap policy | Stable checks and review are cumulative; qualifying evidence is an exact-head completed CodeRabbit artifact or exact-head independent non-author `APPROVED` review. Status-only, rate-limited, author, or predecessor evidence is excluded. | A provider rate limit can still defer review, blocking only merge. | @@ -44,18 +44,24 @@ acceptance layer. | PRD-KS-002, KS-004 | ADR-0001/0002; Rouard et al. (2023) | `separation/audio_separator.py` | `test_youtube_stem_e2e.py` live case | Exact model identity and supported platform | | PRD-KS-003 | Le Roux et al. (2019) | `tests/known_stem_benchmark.py` | SI-SDR unit tests and live threshold | Calibration plus exact-candidate score | | PRD-KS-005 | ADR-0002 | master identity plus composed global alignment helpers | delayed/composed-window tests; live duration/correlation | Authorized YouTube calibration and drift triage | -| PRD-KS-006, KS-010 | ADR-0002 | pytest marker and failure taxonomy | 25 default offline tests; explicit live failure | Advisory until promotion ADR | +| PRD-KS-006, KS-010 | ADR-0002 | pytest marker and failure taxonomy | Every collected default offline test; explicit live failure | Advisory until promotion ADR | | PRD-KS-008 | ADR-0003 | temporary directory and sanitized errors | cleanup postcondition and archive failure tests | Evidence excludes raw media/paths | | PRD-KS-009 | ADR-0003; NIST AI RMF TEVV | planned bounded evidence schema | No retained score yet | Required before blocking release gate | -| TRD-KS-011 | ADR-0001 | supplemental inventory and local loader | missing/size/hash/local-repository tests | Implemented on active PR; exact-head and protected-main proof pending | +| TRD-KS-011 | ADR-0001 | separator manifest plus supplemental inventory | exact filename/hash/size parity tests | Model-rights/legal delivery blocker | ## Live evidence snapshot | Date | Commit under test | Offline contract | Live result | Classification | |---|---|---|---|---| -| 2026-08-09 | `5a3648a11d9097b8da48bb4a3ccbd97986aec25b` | Historical partial suite: 13 passed via `pytest -m "not youtube_stem_e2e" services/analysis-engine/tests/test_youtube_stem_e2e.py`; it predated separate creator-master authentication, composed two-lag recovery, and deterministic production-separator assertions now present in the 25-case manifest | Reference archive verified; YouTube download failed with HTTP 502 before separation; no score | Exact failure evidence, not a pass | +| 2026-08-09 | `5a3648a11d9097b8da48bb4a3ccbd97986aec25b` | 13 passed | Reference archive verified; YouTube download failed with HTTP 502 before separation; no score | Exact failure evidence, not a pass | | 2026-08-09 | `6e937a34f9036d92e909db3ce8848a5c39dc8e3b` (published byte-identical implementation tree) | Full quickcheck: 680 Python passed, 24 skipped, live marker deselected; 100% source coverage | Archive, extracted vocal, creator master, and pre-provisioned model hash verified; production YouTube download failed with HTTP 502 after 65.49 s; no score | Exact implementation-head failure evidence, not a pass | +The 13-test row is a pre-correction partial suite, not a competing total. It did not contain +`test_download_verified_creator_master_authenticates_exact_file`, +`test_align_known_stem_through_master_composes_two_global_offsets`, or +`test_required_root_suite_explicitly_excludes_live_youtube_marker`; adding those three produced the +later 16-test revision. Current regression additions intentionally make a fixed count non-normative. + Separate creator-master calibration on that environment measured `shifts=0` vocal SI-SDR improvement +1.752 dB and assignment margin +7.631 dB. Dry-vocal/mix correlation was 0.016856, so the branch now uses a separately pinned finished master for identity. This probe did not download @@ -67,9 +73,12 @@ YouTube and is not a live pass. diagram authority, and this matrix; checks cross-links from architecture and the index; and requires contributing, governance, gitflow, bootstrap, and GitHub bootstrap policy to link the canonical required-check authority so review policy cannot silently fork. -`scripts/checks/verify_supply_chain.py` derives the configured separator model name and rejects an -inventory that lacks it, uses the retired bandsplit profile, omits required fields, lacks a full -SHA-256/positive byte size, or uses a non-HTTPS model source. +`scripts/checks/verify_supply_chain.py` derives the configured separator model and exact code-owned +filename/hash/size manifest, then rejects inventory drift. It also binds the yt-dlp record to +`uv.lock`, requires both ffmpeg and ffprobe operator records, rejects the retired bandsplit profile, +and validates every model artifact's schema, types, full SHA-256, positive non-boolean size, and +HTTPS source. `scripts/checks/verify_security_notes.py` recursively requires the exact canonical +`## Security Notes` section in every plan. ## Re-evaluation triggers diff --git a/docs/engineering/acceptance-criteria.md b/docs/engineering/acceptance-criteria.md index 7deee7fd7..8d6061d1d 100644 --- a/docs/engineering/acceptance-criteria.md +++ b/docs/engineering/acceptance-criteria.md @@ -44,7 +44,7 @@ For protected branches, intended checks are documented in `docs/security/github- ## Source-separation quality gates -- Every separator or downloader change must keep the 25 deterministic known-stem metric, alignment, +- Every separator or downloader change must keep all collected deterministic known-stem metric, alignment, archive-integrity, redirect/path, cleanup, and failure-contract cases passing. - A live evidence claim must cross `download_youtube_audio()` and `AudioStemSeparator.separate()` on the same exact candidate, authenticate the separately pinned creator master, compose the two @@ -57,10 +57,9 @@ For protected branches, intended checks are documented in `docs/security/github- provider or model failure does not justify weakening it. - Skipped, disabled, HTTP/provider-failed, model-unavailable, integrity-failed, drifted, non-finite, predecessor-head, or stale-base execution is not passing evidence. -- Before the lane can block a release, ADR-0001/0002 blockers—authorization, a recorded - model-rights/legal delivery decision, exact-candidate pass, calibration, and supported-platform - evidence—must be closed. Full-hash pre-load verification is already implemented and must remain - green. +- Before the lane can block a release, ADR-0001/0002 blockers—content/platform authorization, + full-hash pre-load verification, an explicit model-rights/legal delivery decision, + exact-candidate pass, calibration, and supported-platform evidence—must be closed. ## Evidence policy diff --git a/docs/engineering/youtube-known-stem-validation.md b/docs/engineering/youtube-known-stem-validation.md index 00fc9be83..7892eaba6 100644 --- a/docs/engineering/youtube-known-stem-validation.md +++ b/docs/engineering/youtube-known-stem-validation.md @@ -70,16 +70,18 @@ creator-master probe is not a live pass. ## Running the benchmark -Install the analysis-engine development dependencies. Provision an absolute `ffmpeg` executable and -the exact htdemucs artifact locally; the live lane requires their path/digest identities and never -downloads model weights. +Install the analysis-engine development dependencies. Resolve sibling ffmpeg and ffprobe programs +from one trusted package/build to absolute regular executables and obtain both full SHA-256 values; +`PATH` names alone are not release/live evidence. Provision the exact model file in the user-scoped +torch.hub checkpoints cache or pass its exact absolute path through +`BANDSCOPE_HTDEMUCS_MODEL_PATH` before running. The separator never downloads a missing model. The exact current model artifact is Demucs 4.0.1 htdemucs signature `955717e8`, file `955717e8-8726e21a.th`, 84,141,911 bytes, full SHA-256 -`8726e21a993978c7ba086d3872e7608d7d5bfca646ca4aca459ffda844faa8b4`. It is not bundled. -BandScope requires the exact filename, byte count, and full digest before passing a local repository -to Demucs; missing or changed bytes fail before deserialization. ADR-0001 retains the model-rights -and permitted-delivery decision as a release blocker. +`8726e21a993978c7ba086d3872e7608d7d5bfca646ca4aca459ffda844faa8b4`. It is pre-provisioned and +not bundled. BandScope rejects a missing, symlinked, non-regular, incorrectly sized, or full-SHA +mismatched provisioned file before deserializing the same verified bytes. ADR-0001 keeps the +separate model-rights/legal delivery decision as a release blocker. Before enabling the test, the operator must confirm that the intended use is permitted by the content rightsholder and the applicable YouTube terms. The creator's permission for the reference @@ -87,18 +89,25 @@ source does not by itself grant permission for automated access to YouTube. ```bash UV_CACHE_DIR=/tmp/bandscope-uv-cache \ -BANDSCOPE_FFMPEG_PATH=/absolute/path/to/ffmpeg \ -BANDSCOPE_FFMPEG_SHA256=<64-lowercase-hex-digest> \ -BANDSCOPE_HTDEMUCS_MODEL_PATH=/absolute/path/to/955717e8-8726e21a.th \ BANDSCOPE_RUN_YOUTUBE_STEM_E2E=1 \ +BANDSCOPE_FFMPEG_PATH=/absolute/trusted/path/to/ffmpeg \ +BANDSCOPE_FFMPEG_SHA256=<64-lowercase-hex-digest> \ +BANDSCOPE_FFPROBE_PATH=/absolute/trusted/path/to/ffprobe \ +BANDSCOPE_FFPROBE_SHA256=<64-lowercase-hex-digest> \ +BANDSCOPE_HTDEMUCS_MODEL_PATH=/absolute/trusted/path/to/955717e8-8726e21a.th \ uv run --project services/analysis-engine \ pytest services/analysis-engine/tests/test_youtube_stem_e2e.py \ -m youtube_stem_e2e -vv ``` -If YouTube access, either fixed reference asset, `ffmpeg`, or model weights are unavailable, the -opted-in test fails. It must not silently turn an unavailable or changed fixture into a passing -result. +If YouTube access, either fixed reference asset, the verified `ffmpeg`/`ffprobe` executable set, or +model weights are unavailable, the opted-in test fails. It must not silently turn an unavailable or +changed fixture into a passing result. + +The four media-runtime fields must identify exact platform-native sibling program names. Their +paths, execute permissions, and hashes are verified before the benchmark accesses either reference +asset. The model path must use the exact inventoried filename; the production loader then performs +its independent same-byte size and full-hash verification before deserialization. ## Platform and evidence status @@ -108,16 +117,20 @@ result. - macOS Intel: current dependency markers exclude Demucs; separation must fail safely and offer the product fallback. -On 2026-08-09, exact commit `5a3648a11d9097b8da48bb4a3ccbd97986aec25b` passed all 13 then-current default -offline cases. An explicit live attempt authenticated and extracted the pinned reference archive, +On 2026-08-09, exact commit `5a3648a11d9097b8da48bb4a3ccbd97986aec25b` passed a 13-test +pre-correction partial suite. It lacked +`test_download_verified_creator_master_authenticates_exact_file`, +`test_align_known_stem_through_master_composes_two_global_offsets`, and +`test_required_root_suite_explicitly_excludes_live_youtube_marker`. An explicit live attempt +authenticated and extracted the pinned reference archive, then failed in the production YouTube downloader with HTTP 502 before separation. It produced no correlation or SI-SDR score and is recorded as failure evidence, not a live pass. See `docs/documentation-coverage-matrix.md`. -The corrected branch now has 25 offline known-stem contract cases, including exact extracted-member -hash, creator-master authentication, composed-offset recovery, signed identity correlation, -model/ffmpeg pre-load identity failures, and explicit required-CI exclusion of the live marker. A creator-master-only calibration produced the provisional scores above without -calling YouTube; it is calibration evidence, not exact-candidate success. +The first corrected branch revision raised that suite to 16. The current requirement is to run every +collected offline case—its count may grow with regression coverage—plus explicit required-CI +exclusion of the live marker. A creator-master-only calibration produced the provisional scores +above without calling YouTube; it is calibration evidence, not exact-candidate success. The byte-identical implementation tree published on GitHub as exact commit `6e937a34f9036d92e909db3ce8848a5c39dc8e3b` passed the full quickcheck. A clean live retry @@ -131,8 +144,8 @@ exact implementation-head failure evidence, not a live pass. ### Attack surface The opt-in test crosses three public HTTPS download boundaries, decodes untrusted audio/ZIP data, -writes temporary files, invokes the existing `ffmpeg` yt-dlp postprocessor, and loads the existing -Demucs model. +writes temporary files, invokes yt-dlp with the verified sibling `ffmpeg` and `ffprobe` executables, +and loads the existing Demucs model. ### Trust boundary @@ -148,7 +161,7 @@ the only permitted storage root for downloaded media and extracted references. separator regression. - Login cookies, geo/DRM bypasses, or automated CI execution could expand legal, privacy, and account risk. -- Decoder/model vulnerabilities and operator-provisioned model provenance remain upstream supply-chain risks. +- Decoder/model vulnerabilities and operator provisioning remain upstream supply-chain risks. ### Mitigations @@ -163,8 +176,11 @@ the only permitted storage root for downloaded media and extracted references. master is independently pinned by exact host, byte count, and full SHA-256. - The production YouTube downloader keeps its standard-URL allowlist, duration/size bounds, `noplaylist`, and no-geo-bypass policy. This test adds no cookies, credentials, login, paywall, - DRM, or bot-evasion behavior. TLS validation stays enabled and yt-dlp retains its maintained CA - bundle fallback, so a minimal container does not silently depend on an absent system trust store. + DRM, or bot-evasion behavior. TLS validation stays enabled. yt-dlp uses the operating system's + managed CA trust store when populated and otherwise retains its certifi-backed default. +- Release/live execution supplies sibling ffmpeg and ffprobe absolute regular executables plus both + full SHA-256 values. A partial identity set, unexpected program name/directory, path drift, or + digest mismatch fails before yt-dlp runs. - Alignment is global and bounded. Duration and creator-master identity correlation distinguish fixture drift from model quality failure; the two lags are composed once and model outputs are not optimized after separation. Demucs random shift augmentation is disabled with `shifts=0`. @@ -187,9 +203,10 @@ advice, and the test does not establish platform authorization. Upstream media d weights remain separate trust decisions. The fixture has only one full-length known canonical stem, so the test cannot claim quantitative four-stem accuracy. -The model-weight redistribution license is not established. BandScope now verifies the exact byte -count and full SHA-256 before local-only loading, and no successful exact-candidate live score or supported-platform matrix has yet been -retained. These remain explicit release blockers rather than undocumented assumptions. +The model-weight redistribution/provisioning decision is not established, and no successful +exact-candidate live score or supported-platform matrix has yet been retained. Full-SHA pre-load +verification is implemented, but these remaining items are explicit release blockers rather than +undocumented assumptions. ## References diff --git a/docs/operations/deploy-runbook.md b/docs/operations/deploy-runbook.md index 21491d445..5c97db8fc 100644 --- a/docs/operations/deploy-runbook.md +++ b/docs/operations/deploy-runbook.md @@ -28,21 +28,27 @@ When runtime behavior is touched, verify: For a release candidate that claims YouTube source separation: -1. record exact commit, live base tip, lockfiles, OS, architecture, Python, Demucs, torch, and yt-dlp; - set `BANDSCOPE_FFMPEG_PATH` to an absolute executable and - `BANDSCOPE_FFMPEG_SHA256` to its verified full digest, then record that resolved path, digest, - trusted package/source provenance, and the executable's `-version` output; -2. confirm content/platform authorization and do not provide cookies, credentials, login, paywall, +1. record exact commit, live base tip, lockfiles, OS, architecture, Python, Demucs, torch, and the + exact locked yt-dlp version; +2. resolve sibling ffmpeg and ffprobe programs from one trusted package/build to absolute regular + executables with exact platform-native names (`ffmpeg`/`ffprobe`, or their `.exe` forms), record + both full SHA-256 values, trusted package identity, and version outputs, then pass + `BANDSCOPE_FFMPEG_PATH`, `BANDSCOPE_FFMPEG_SHA256`, `BANDSCOPE_FFPROBE_PATH`, and + `BANDSCOPE_FFPROBE_SHA256`; the benchmark verifies all four before any fixture access, and a + partial set, layout drift, name drift, or mismatch fails preflight; +3. confirm content/platform authorization and do not provide cookies, credentials, login, paywall, DRM, geo, or anti-bot bypasses; -3. verify the htdemucs model's exact source, 84,141,911-byte size, and full SHA-256 from the - supplemental inventory before load; fail closed on cache symlink, mismatch, or missing artifact; -4. authenticate the archive, extracted vocal member, and finished master by exact host, byte count, +4. verify the htdemucs model's exact source, 84,141,911-byte size, and full SHA-256 from the + supplemental inventory, then set `BANDSCOPE_HTDEMUCS_MODEL_PATH` to the exact absolute + `955717e8-8726e21a.th` path; fail closed on a wrong filename, symlink, mismatch, or missing + artifact; +5. authenticate the archive, extracted vocal member, and finished master by exact host, byte count, and full SHA-256; record the master duration and require deterministic Demucs `shifts=0`; -5. run the offline known-stem contract, then the explicit live command from +6. run the offline known-stem contract, then the explicit live command from `docs/engineering/youtube-known-stem-validation.md` on the unchanged candidate; -6. retain bounded numeric/provenance evidence only: duration drift, identity correlation, composed +7. retain bounded numeric/provenance evidence only: duration drift, identity correlation, composed lags, baseline/vocal SI-SDR, improvement, assignment margin, outcome code, and cleanup result; -7. verify the temporary media root is empty and no raw audio, archive content, full path, URL, +8. verify the temporary media root is empty and no raw audio, archive content, full path, URL, cookie, credential, or provider response was retained. The live lane needs a 20-minute operator timeout until calibration establishes a tighter limit. A diff --git a/docs/plans/2026-03-10-bandscope-cross-platform-build.md b/docs/plans/2026-03-10-bandscope-cross-platform-build.md index 3a02aa3a5..f7c0431b6 100644 --- a/docs/plans/2026-03-10-bandscope-cross-platform-build.md +++ b/docs/plans/2026-03-10-bandscope-cross-platform-build.md @@ -8,37 +8,39 @@ **Tech Stack:** GitHub Actions, npm, uv, Rust stable toolchain, Python packaging sanity, zip artifacts, SHA-256 checksums. -**Security Notes:** Cross-platform builds are supply-chain and release-integrity controls. The harness must fail if Windows or macOS coverage, artifact upload, checksum generation, or required-check intent drifts out of policy. +## Security Notes -## Attack surface +Cross-platform builds are supply-chain and release-integrity controls. The harness must fail if Windows or macOS coverage, artifact upload, checksum generation, or required-check intent drifts out of policy. + +### Attack surface - Windows and macOS packaging paths - native dependencies and bundled binaries per OS - release artifact generation and upload -## Trust boundary +### Trust boundary - target-OS build workers in GitHub Actions act as release-path verifiers - branch protections depend on named Windows and macOS build jobs -## Mitigations +### Mitigations - add dedicated Windows and macOS build jobs - upload per-OS artifacts and checksums on PR, push, tag, and release events - document required-check intent in repo docs and verify workflow coverage locally -## Test points +### Test points - local supply-chain verification covers workflow presence and trigger scope - workflow uploads artifact and checksum for both OSes - intended required checks include both OS build jobs -## Realistic threats +### Realistic threats - platform-specific bundle assets can be missing even when the Rust shell compiles locally - release upload credentials can be over-scoped if build and publish concerns share the same job -## Remaining risk +### Remaining risk - notarization and signing remain outside the bootstrap harness until platform credentials exist diff --git a/docs/plans/2026-03-10-bandscope-harness.md b/docs/plans/2026-03-10-bandscope-harness.md index b114c3196..21060df01 100644 --- a/docs/plans/2026-03-10-bandscope-harness.md +++ b/docs/plans/2026-03-10-bandscope-harness.md @@ -8,34 +8,36 @@ **Tech Stack:** npm workspaces, Vite, React, Vitest, Tauri scaffold files, Python 3.12+, uv, pytest, ruff, mypy, Dependabot, CycloneDX JSON SBOM, GitHub Actions SHA pinning. -**Security Notes:** The harness must keep security guidance visible and fail-fast. Future work that touches files, URLs, subprocesses, IPC, WebView, updates, models, or cache/export behavior must include a `Security Notes` section and avoid generic exec/read/write capabilities. +## Security Notes -## Attack surface +The harness must keep security guidance visible and fail-fast. Future work that touches files, URLs, subprocesses, IPC, WebView, updates, models, or cache/export behavior must include a `Security Notes` section and avoid generic exec/read/write capabilities. + +### Attack surface - repo docs and plans that define future file, URL, subprocess, IPC, WebView, model, and update behavior -## Trust boundary +### Trust boundary - future product work crosses user-input, process, IPC, storage, and network boundaries even in a local-first app -## Mitigations +### Mitigations - keep security policy in repo docs, not only in chat - fail plans that omit `Security Notes` - fail obvious dangerous implementation patterns early -## Test points +### Test points - docs presence checks - `Security Notes` structure checks - security pattern checks in quickcheck -## Realistic threats +### Realistic threats - future contributors can copy unsafe bootstrap defaults into production features - local checks can silently miss risky workflow or release-script drift if scope is too narrow -## Remaining risk +### Remaining risk - desktop runtime constraints remain provisional until real IPC and backend flows exist diff --git a/docs/plans/2026-03-10-bandscope-supply-chain-design.md b/docs/plans/2026-03-10-bandscope-supply-chain-design.md index e47ec4847..48b49d4e1 100644 --- a/docs/plans/2026-03-10-bandscope-supply-chain-design.md +++ b/docs/plans/2026-03-10-bandscope-supply-chain-design.md @@ -8,13 +8,16 @@ ## Constraints - lockfiles are mandatory -- dependency review and audit must run in GitHub Actions +- dependency review and audit must run in GitHub Actions; dependency review is supplied by the + organization-level required workflow, while audit remains repository-owned - SBOM generation must produce machine-readable output and survive in GitHub artifacts or releases - bundled binaries and model artifacts must be tracked outside package-manager dependency graphs - dependency review, audit, inventory, and SBOM checks must become required merge gates on both `develop` and `main` - new direct dependencies require written admission rationale covering purpose, dependency class, alternatives, trust, license, security, transitive footprint, and release risk - GitHub Actions references must stay SHA pinned; mutable refs are not an acceptable default -- Repo files define workflows and intended check names; actual required-check enforcement still lives in GitHub branch protection or rulesets. +- Repo files define repository-owned workflows, the organization-level dependency-review + authority, and intended check names; actual required-check enforcement still lives in GitHub + branch protection or rulesets. ## Security Notes @@ -31,13 +34,15 @@ ### Mitigations -- require pinned workflow actions, committed lockfiles, dependency review, audit, SBOM generation, and supplemental inventory +- require pinned repository workflow actions, committed lockfiles, documented organization-level + dependency review, audit, SBOM generation, and supplemental inventory - keep intended required checks visible in repo docs - fail fast when lockfiles, workflows, or inventory files are missing ### Test points -- local harness checks must verify lockfiles, workflow presence, and action pinning +- local harness checks must verify lockfiles, repository workflow presence, organization-level + dependency-review authority, and action pinning - GitHub workflows must run on develop, main, PR, and release-related events - release workflows must retain SBOM artifacts and supplemental inventory - bootstrap reporting must include the exact evidence set for workflow paths, required checks, Dependabot baseline, SBOM retention, and supplemental inventory @@ -71,6 +76,8 @@ ## Decision - Choose the GitHub-first supply-chain baseline. -- Keep package-manager lockfiles, workflow pinning, dependency review, audit, SBOM generation, and supplemental inventory in the repository from bootstrap. +- Keep package-manager lockfiles, repository workflow pinning, the documented organization-level + dependency-review authority, audit, SBOM generation, and supplemental inventory in the + repository from bootstrap. - Treat missing repo state as bootstrap work and treat platform-level branch protection or required checks as `BLOCKED` only when admin permission is unavailable. - Treat missing repo-controlled supply-chain artifacts as `FAILED`, not as deferred follow-up work. diff --git a/docs/plans/2026-03-10-bandscope-supply-chain.md b/docs/plans/2026-03-10-bandscope-supply-chain.md index bd028984a..2f2a0c969 100644 --- a/docs/plans/2026-03-10-bandscope-supply-chain.md +++ b/docs/plans/2026-03-10-bandscope-supply-chain.md @@ -8,39 +8,41 @@ **Tech Stack:** npm workspaces, uv lock, Cargo lock, Dependabot, GitHub Actions, CycloneDX JSON SBOM, supplemental JSON inventory. -**Security Notes:** Supply-chain workflows are part of the public attack surface. The harness must fail if lockfiles, workflow pinning, dependency review, audits, SBOM generation, or supplemental inventory drift out of policy. +## Security Notes -## Attack surface +Supply-chain workflows are part of the public attack surface. The harness must fail if lockfiles, workflow pinning, dependency review, audits, SBOM generation, or supplemental inventory drift out of policy. + +### Attack surface - dependency manifests and lockfiles - GitHub Actions and third-party actions - bundled binaries and model artifacts - release assets and uploaded SBOMs -## Trust boundary +### Trust boundary - package-manager graphs do not fully cover binaries and model artifacts - GitHub workflows and release assets are externally visible supply-chain surfaces -## Mitigations +### Mitigations - commit lockfiles and pin workflow actions by SHA - add dependency review, audit, and SBOM workflows - keep supplemental component inventory in machine-readable form - document intended required checks for develop and main -## Test points +### Test points - local supply-chain verification script - quickcheck path includes supply-chain verification - workflows trigger on develop, main, PR, tag, and release-related events -## Realistic threats +### Realistic threats - over-broad workflow permissions can let PR-modified code affect release surfaces - missing bundled-binary inventory can hide shipped assets outside package-manager graphs -## Remaining risk +### Remaining risk - GitHub-native security signals still depend on repository settings and service availability outside repo control @@ -70,17 +72,21 @@ **Files:** - Create: `.github/dependabot.yml` -- Create: `.github/workflows/dependency-review.yml` - Create: `.github/workflows/security-audit.yml` - Create: `.github/workflows/sbom.yml` - Modify: `.github/workflows/ci.yml` +Dependency review is supplied by the organization-level required workflow recorded in +`docs/workflow/github-bootstrap-execution-policy.md`; this repository intentionally does not +duplicate it as `.github/workflows/dependency-review.yml`. + **Security Notes** - Attack surface: third-party actions, audit tooling, release uploads, and CI permissions. - Trust boundary: GitHub Actions definitions become part of the supply-chain enforcement path. - Mitigations: pin actions by SHA, use least-privilege permissions, and generate machine-readable SBOM artifacts. -- Test points: local checks verify workflow presence, trigger coverage, and action pinning. +- Test points: local checks verify repository workflow presence, organization-level + dependency-review authority, trigger coverage, and action pinning. **Acceptance detail** @@ -101,12 +107,14 @@ - Attack surface: a weak local harness can let unsafe supply-chain drift land before PR review. - Trust boundary: quickcheck is the first enforcement line before GitHub CI. -- Mitigations: fail fast on missing lockfiles, missing workflows, missing inventory, or unpinned actions. +- Mitigations: fail fast on missing lockfiles, missing repository workflows, undocumented + organization-level dependency-review authority, missing inventory, or unpinned actions. - Test points: quickcheck output must include the supply-chain verification step. **Acceptance detail** -- fail on missing lockfiles, missing workflows, missing supplemental inventory, or unpinned actions +- fail on missing lockfiles, missing repository workflows, undocumented organization-level + dependency-review authority, missing supplemental inventory, or unpinned actions - fail if required branch-check names drift from documented policy ### Task 4: Attempt GitHub enforcement and record blockers honestly @@ -126,7 +134,8 @@ **Acceptance detail** -- record the exact workflow paths for dependency review, audit, and SBOM generation +- record the organization-level dependency-review authority and the exact repository workflow + paths for audit and SBOM generation - record the SBOM format and where Actions artifacts and Release assets are retained - record how bundled binaries and model artifacts are tracked - use `FAILED` for missing repo-controlled artifacts and `BLOCKED` only for missing GitHub permission or platform capability diff --git a/docs/plans/2026-03-28-ml-engine-integration.md b/docs/plans/2026-03-28-ml-engine-integration.md index ff92adc4d..a8d1a73e1 100644 --- a/docs/plans/2026-03-28-ml-engine-integration.md +++ b/docs/plans/2026-03-28-ml-engine-integration.md @@ -18,9 +18,9 @@ This document outlines the MECE execution strategy to incrementally substitute m - **Output**: 4 discrete stems (vocals, bass, drums, other). - **Validity**: The active known-stem branch adds production-path vocal SI-SDR improvement and stem assignment checks; see `docs/PRD.md`, `docs/TRD.md`, and ADR-0002. -- **Open release blockers**: full-SHA model verification before deserialization, model-rights - decision, successful exact-candidate live evidence, threshold calibration, and supported-platform - proof. +- **Open release blockers**: model-rights/legal delivery decision, successful exact-candidate live + evidence, threshold calibration, and supported-platform proof. Full-SHA verification before + deserialization is implemented and regression-tested. ### Track 3: Harmonic & Pitch Pipelines (#107) (COMPLETED) @@ -48,15 +48,16 @@ The primary trust boundary is between the user's filesystem (audio files) and th ### Mitigations We restrict audio ingestion through `librosa`/`soundfile` using strict format constraints. Model -inference runs locally and under low privilege where possible. The implemented loader never -retrieves model bytes: it requires trusted local provisioning and verifies the exact filename, byte -count, and full SHA-256 before deserialization. Model rights and permitted delivery remain release -decisions; see ADR-0001. +inference runs locally and under low privilege where possible. A trusted provisioning step must +place the exact inventoried model in the user cache; runtime never downloads a missing model. The +separator rejects symlinks, size drift, and full-SHA mismatch before deserializing the same verified +bytes; see ADR-0001. ### Test Points - Loading truncated or corrupted WAV/MP3 files. - Providing extremely large audio files to test OOM behavior. -- Validating that no external network calls occur after trusted model-cache provisioning. +- Validating that no external network calls occur during model loading, including when the cache is + absent or invalid. ### Realistic Threats - OOM (Out Of Memory) crashing the user's host OS during `demucs` execution. diff --git a/docs/release/release-policy.md b/docs/release/release-policy.md index 18d170a40..26db76452 100644 --- a/docs/release/release-policy.md +++ b/docs/release/release-policy.md @@ -16,7 +16,8 @@ BandScope distributes release artifacts through GitHub Releases. - checksums or equivalent integrity metadata - release notes - the latest SBOM -- supplemental inventory for bundled binaries and model artifacts +- supplemental inventory for lock-managed auxiliary tools, operator-provided or bundled + executables, and model artifacts ## Release rules @@ -31,13 +32,18 @@ BandScope distributes release artifacts through GitHub Releases. - The deterministic known-stem contract is required for every change that touches YouTube intake, decode, separation, alignment, metrics, model delivery, or fixture metadata. - Live known-stem evidence is advisory while ADR-0002 is Proposed. It becomes blocking only through - a superseding/accepted ADR after authorization, full-hash pre-load model verification, calibrated - thresholds, supported-platform evidence, and a stable bounded evidence artifact exist. + a superseding/accepted ADR after authorization, full-hash pre-load model verification, an explicit + model-rights/legal delivery decision, calibrated thresholds, supported-platform evidence, and a + stable bounded evidence artifact exist. - A release must not advertise verified source-separation quality unless the exact integrated release candidate records a passing live production-path run. A skipped, provider-failed, stale, or predecessor-head result does not transfer. -- Release artifacts must identify the exact htdemucs signature/hash and whether weights are bundled, - pre-provisioned, or runtime-fetched. Current policy permits runtime cache retrieval only; it does - not authorize model-weight redistribution. +- Release artifacts must identify the exact htdemucs signature/hash and whether weights are bundled + or pre-provisioned. Runtime fetching is forbidden; current policy requires a verified + pre-provisioned cache or exact `BANDSCOPE_HTDEMUCS_MODEL_PATH` and does not authorize model-weight + redistribution. +- Live evidence must identify sibling ffmpeg/ffprobe executables from one trusted package/build by + exact platform-native name, absolute path, full SHA-256, and version output before fixture access. + Verifying ffmpeg alone is insufficient because yt-dlp may execute ffprobe during postprocessing. - Release rollback must preserve deterministic metric/security coverage and remove any invalid quality claim, scheduled live access, or unverified model artifact. diff --git a/docs/repository/bootstrap-plan.md b/docs/repository/bootstrap-plan.md index 69b586099..069b1cf00 100644 --- a/docs/repository/bootstrap-plan.md +++ b/docs/repository/bootstrap-plan.md @@ -32,8 +32,8 @@ After workflows exist, require these stable checks on `main` and `develop`: - `ci / build-and-test` - `dependency-review` - `security-audit` -- `trivy-fs-scan` - `CodeQL` +- `trivy-fs-scan` - `sbom` - `release-preflight` - `gate / build / windows` diff --git a/docs/security/dependency-policy.md b/docs/security/dependency-policy.md index 662981d3d..24521817c 100644 --- a/docs/security/dependency-policy.md +++ b/docs/security/dependency-policy.md @@ -15,7 +15,8 @@ Because of that, dependency review, security audit, SBOM generation, and supply- - generate machine-readable SBOMs in CI - upload SBOMs as GitHub Actions artifacts - attach release-time SBOMs to GitHub Releases when a release exists -- track bundled binaries and model artifacts outside package-manager graphs +- track lock-managed auxiliary tools, operator-provided executables, bundled binaries, and model + artifacts when ecosystem SBOMs alone do not prove runtime identity - keep dependency review, audit, inventory, and SBOM checks as required protected-branch merge gates - require Windows and macOS build gates for protected-branch changes and release validation @@ -25,7 +26,8 @@ Because of that, dependency review, security audit, SBOM generation, and supply- - Python analysis engine dependencies - Rust and Tauri crate dependencies - GitHub Actions third-party actions -- bundled binaries such as `ffmpeg` and `yt-dlp` +- lock-managed auxiliary tools such as yt-dlp +- operator-provided, non-bundled executables such as ffmpeg and ffprobe - model files, weights, and sidecar assets ## Lockfile and pinning rules diff --git a/docs/security/sbom-policy.md b/docs/security/sbom-policy.md index 06495c14c..241978273 100644 --- a/docs/security/sbom-policy.md +++ b/docs/security/sbom-policy.md @@ -19,8 +19,10 @@ BandScope generates machine-readable SBOMs in GitHub Actions as a bootstrap cont ## Supplemental inventory -Track package-manager-external supply-chain assets in `supply-chain/supplemental-component-inventory.json`, including: +Track runtime supply-chain identities that need evidence beyond generated ecosystem SBOM entries in +`supply-chain/supplemental-component-inventory.json`, including: -- bundled binaries such as `ffmpeg` and `yt-dlp` +- lock-managed auxiliary tools such as yt-dlp, cross-checked to `uv.lock` +- operator-provided, non-bundled executables such as ffmpeg and ffprobe - model files, weights, and sidecar assets - checksums or integrity metadata when available diff --git a/docs/workflow/github-bootstrap-execution-policy.md b/docs/workflow/github-bootstrap-execution-policy.md index 8ff085026..667b36069 100644 --- a/docs/workflow/github-bootstrap-execution-policy.md +++ b/docs/workflow/github-bootstrap-execution-policy.md @@ -38,12 +38,14 @@ The expected sequence is: Bootstrap or setup work is not complete unless GitHub-facing supply-chain controls are both committed and, where permissions allow, enforced: - `.github/dependabot.yml` -- `.github/workflows/dependency-review.yml` +- the organization-level required dependency-review workflow; BandScope intentionally carries no + repo-local duplicate - `.github/workflows/security-audit.yml` - `.github/workflows/codeql.yml` +- `.github/workflows/trivy.yml` - `.github/workflows/sbom.yml` - `.github/workflows/release.yml` -- branch protection or rulesets for `main` and `develop` that require `ci / build-and-test`, `dependency-review`, `security-audit`, `CodeQL`, `sbom`, `release-preflight`, `gate / build / windows`, and `gate / build / macos` +- branch protection or rulesets for `main` and `develop` that require `ci / build-and-test`, `dependency-review`, `security-audit`, `CodeQL`, `trivy-fs-scan`, `sbom`, `release-preflight`, `gate / build / windows`, and `gate / build / macos` - PR workflow that still requests CodeRabbit review and records its result when the provider responds cleanly - release retention for the generated SBOM and supplemental inventory @@ -97,7 +99,8 @@ Do not treat these as TODOs, later hardening, or optional recommendations. ### Phase 6. Bootstrap PR - create `bootstrap/setup` or equivalent from `develop` -- add workflows, security docs, CODEOWNERS, dependency review, `trivy-fs-scan`, SBOM, builds, and required evidence docs +- add repo-owned workflows, security docs, CODEOWNERS, the organization dependency-review binding, + SBOM, builds, and required evidence docs - add or confirm lockfiles, dependency review, audit, SBOM, and supplemental inventory for bundled binaries and model artifacts - merge through PR review, not direct push diff --git a/scripts/checks/security_gates.py b/scripts/checks/security_gates.py index 617d6ce5e..348d87ca8 100644 --- a/scripts/checks/security_gates.py +++ b/scripts/checks/security_gates.py @@ -29,6 +29,23 @@ TARGET_EXTENSIONS = {".py", ".ts", ".tsx", ".js", ".jsx", ".sh", ".yml", ".yaml"} EXCLUDED_PARTS = {"node_modules", ".venv", "dist", "coverage", "target", ".worktrees"} SELF_PATH = Path("scripts/checks/security_gates.py") +VERIFIED_MODEL_LOADER_PATH = Path( + "services/analysis-engine/src/bandscope_analysis/separation/audio_separator.py" +) +VERIFIED_TORCH_LOAD_CALL = re.compile( + r"torch\.load\(\s*(?:#[^\n]*\n\s*)?" + r"io\.BytesIO\(payload\),\s*" + r"map_location=[\"']cpu[\"'],\s*" + r"weights_only=False,?\s*" + r"\)", + re.MULTILINE, +) +VERIFIED_MODEL_LOADER_PREREQUISITES = ( + "payload = _read_verified_model_artifact(", + "hashlib.sha256(payload).hexdigest()", + "artifact.size_bytes", + "stat.S_ISREG", +) def should_scan(path: Path) -> bool: @@ -38,19 +55,38 @@ def should_scan(path: Path) -> bool: ) -def main() -> int: - """Return a failing exit code when a forbidden security pattern is found.""" +def _content_for_pattern_scan(relative_path: Path, content: str) -> str: + """Remove only the one fully constrained checkpoint-deserialization call.""" + if relative_path != VERIFIED_MODEL_LOADER_PATH: + return content + if not all(token in content for token in VERIFIED_MODEL_LOADER_PREREQUISITES): + return content + if len(VERIFIED_TORCH_LOAD_CALL.findall(content)) != 1: + return content + return VERIFIED_TORCH_LOAD_CALL.sub("verified_checkpoint_load()", content, count=1) + + +def security_pattern_violations(repo_root: Path = Path(".")) -> list[str]: + """Return forbidden-pattern violations below ``repo_root``.""" violations: list[str] = [] - for path in Path(".").rglob("*"): + for path in repo_root.rglob("*"): if not path.is_file() or not should_scan(path): continue - if path == SELF_PATH: + relative_path = path.relative_to(repo_root) + if relative_path == SELF_PATH: continue content = path.read_text(encoding="utf-8", errors="ignore") + content = _content_for_pattern_scan(relative_path, content) for pattern, message in RULES: if pattern.search(content): - violations.append(f"{path}: {message}") + violations.append(f"{relative_path}: {message}") + return violations + + +def main() -> int: + """Return a failing exit code when a forbidden security pattern is found.""" + violations = security_pattern_violations() if violations: print("Security gate violations:") diff --git a/scripts/checks/verify_docs.py b/scripts/checks/verify_docs.py index 928e86e2e..d46f83e21 100644 --- a/scripts/checks/verify_docs.py +++ b/scripts/checks/verify_docs.py @@ -110,18 +110,12 @@ def documentation_violations(root: Path = Path(".")) -> list[str]: violations.append(f"{path} missing reference: {required_text}") plans_root = root / "docs" / "plans" if plans_root.exists(): - security_heading = re.compile( - r"^(?:#{1,6}\s+Security Notes\s*$|" - r"\*\*Security Notes(?::)?\*\*)(?:\s|$)", - re.MULTILINE, - ) + security_heading = re.compile(r"^## Security Notes\s*$", re.MULTILINE) for absolute_path in sorted(plans_root.rglob("*.md")): content = absolute_path.read_text(encoding="utf-8") if security_heading.search(content) is None: relative_path = absolute_path.relative_to(root) - violations.append( - f"{relative_path} missing section: Security Notes" - ) + violations.append(f"{relative_path} missing section: ## Security Notes") return violations diff --git a/scripts/checks/verify_security_notes.py b/scripts/checks/verify_security_notes.py index 821a5e940..69c472d9a 100644 --- a/scripts/checks/verify_security_notes.py +++ b/scripts/checks/verify_security_notes.py @@ -1,8 +1,9 @@ -"""Verify that design-plan documents include a complete Security Notes section.""" +"""Verify that every design plan has one complete canonical security section.""" +import re from pathlib import Path -SECURITY_NOTES_TEXT = "Security Notes" +SECURITY_NOTES_HEADING = "## Security Notes" PLAN_DIR = Path("docs/plans") REQUIRED_SUBSECTIONS = [ "attack surface", @@ -15,42 +16,49 @@ def security_notes_section(content: str) -> str: - """Extract the lowercased Security Notes section from a plan document.""" - lowered = content.lower() - marker = SECURITY_NOTES_TEXT.lower() - start = lowered.find(marker) - if start == -1: + """Return the canonical security section, stopping at the next peer heading.""" + lines = content.splitlines() + try: + start = lines.index(SECURITY_NOTES_HEADING) + except ValueError: return "" - end_candidates = [] - for delimiter in ["\n---", "\n## approaches considered", "\n## decision"]: - end = lowered.find(delimiter, start + len(marker)) - if end != -1: - end_candidates.append(end) + section_lines = [lines[start]] + for line in lines[start + 1 :]: + if re.fullmatch(r"#{1,2}\s+.+", line): + break + section_lines.append(line) + return "\n".join(section_lines).lower() - if not end_candidates: - return lowered[start:] - return lowered[start : min(end_candidates)] - - -def main() -> int: - """Return a failing exit code when Security Notes or required subsections are missing.""" - missing: list[str] = [] - for path in sorted(PLAN_DIR.glob("*.md")): +def security_notes_violations(repo_root: Path = Path(".")) -> list[str]: + """Return missing-section and incomplete-section violations below ``repo_root``.""" + violations: list[str] = [] + plan_dir = repo_root / PLAN_DIR + for path in sorted(plan_dir.rglob("*.md")): content = path.read_text(encoding="utf-8") - if SECURITY_NOTES_TEXT not in content: - missing.append(str(path)) + section = security_notes_section(content) + display_path = path.relative_to(repo_root).as_posix() + if not section: + violations.append( + f"{display_path} missing section: {SECURITY_NOTES_HEADING}" + ) continue - lowered = security_notes_section(content) for subsection in REQUIRED_SUBSECTIONS: - if subsection not in lowered: - missing.append(f"{path} missing subsection: {subsection}") + if subsection not in section: + violations.append( + f"{display_path} missing Security Notes subsection: {subsection}" + ) + return violations - if missing: + +def main() -> int: + """Return a failing exit code when Security Notes or required subsections are missing.""" + violations = security_notes_violations() + if violations: print("Missing Security Notes section in:") - for path in missing: - print(f"- {path}") + for violation in violations: + print(f"- {violation}") return 1 print("Security Notes check passed") diff --git a/scripts/checks/verify_supply_chain.py b/scripts/checks/verify_supply_chain.py index 6a55115a6..f72d91970 100644 --- a/scripts/checks/verify_supply_chain.py +++ b/scripts/checks/verify_supply_chain.py @@ -42,8 +42,17 @@ SEPARATOR_IMPLEMENTATION_PATH = Path( "services/analysis-engine/src/bandscope_analysis/separation/audio_separator.py" ) +ANALYSIS_LOCK_PATH = Path("services/analysis-engine/uv.lock") FULL_SHA256_PATTERN = re.compile(r"^sha256:[0-9a-f]{64}$") RUNTIME_MODEL_PATTERN = re.compile(r'model_name:\s*str\s*=\s*"([^"]+)"') +RUNTIME_MODEL_ARTIFACT_PATTERN = re.compile( + r'"(?P[^"]+)"\s*:\s*_ModelArtifactSpec\(\s*' + r'signature="(?P[0-9a-f]+)",\s*' + r'filename="(?P[^"]+)",\s*' + r'sha256="(?P[0-9a-f]{64})",\s*' + r'size_bytes=(?P[0-9_]+),\s*\)', + re.DOTALL, +) REQUIRED_MODEL_ARTIFACT_FIELDS = { "name", "runtimeModelName", @@ -57,12 +66,29 @@ "releaseUsage", "verification", } -MODEL_ARTIFACT_STRING_FIELDS = REQUIRED_MODEL_ARTIFACT_FIELDS - {"sizeBytes"} +REQUIRED_MODEL_STRING_FIELDS = REQUIRED_MODEL_ARTIFACT_FIELDS - {"sizeBytes"} + + +def _separator_model_artifact( + separator_source: str, runtime_model: str +) -> dict[str, str | int] | None: + """Return the exact code-owned artifact manifest for ``runtime_model``.""" + for match in RUNTIME_MODEL_ARTIFACT_PATTERN.finditer(separator_source): + if match.group("runtime_model") != runtime_model: + continue + return { + "signature": match.group("signature"), + "filename": match.group("filename"), + "sha256": match.group("sha256"), + "sizeBytes": int(match.group("size_bytes").replace("_", "")), + } + return None def supplemental_inventory_violations( inventory_path: Path = SUPPLEMENTAL_INVENTORY_PATH, separator_path: Path = SEPARATOR_IMPLEMENTATION_PATH, + analysis_lock_path: Path | None = None, ) -> list[str]: """Return stale, incomplete, or runtime-mismatched model inventory violations.""" violations: list[str] = [] @@ -71,7 +97,7 @@ def supplemental_inventory_violations( except (OSError, json.JSONDecodeError) as error: return [f"supplemental inventory is unreadable: {error.__class__.__name__}"] if not isinstance(inventory, dict): - return ["supplemental inventory root must be an object"] + return ["supplemental inventory must be an object"] try: separator_source = separator_path.read_text(encoding="utf-8") except OSError as error: @@ -84,49 +110,98 @@ def supplemental_inventory_violations( artifacts = inventory.get("modelArtifacts") if not isinstance(artifacts, list): return ["supplemental inventory modelArtifacts must be a list"] + if not artifacts: + return ["supplemental inventory modelArtifacts must not be empty"] + if analysis_lock_path is None: + analysis_lock_path = ( + inventory_path.resolve().parent.parent / ANALYSIS_LOCK_PATH + ) + + package_tools = inventory.get("packageManagedTools") + if not isinstance(package_tools, list): + violations.append("supplemental inventory packageManagedTools must be a list") + else: + yt_dlp_records = [ + tool + for tool in package_tools + if isinstance(tool, dict) and tool.get("name") == "yt-dlp" + ] + if len(yt_dlp_records) != 1: + violations.append( + "supplemental inventory requires exactly one yt-dlp package record" + ) + else: + try: + lock_data = tomllib.loads(analysis_lock_path.read_text(encoding="utf-8")) + locked_packages = lock_data.get("package", []) + locked_versions = [ + package.get("version") + for package in locked_packages + if isinstance(package, dict) and package.get("name") == "yt-dlp" + ] + except (OSError, tomllib.TOMLDecodeError) as error: + violations.append( + f"analysis lock is unreadable: {error.__class__.__name__}" + ) + else: + if len(locked_versions) != 1 or not isinstance(locked_versions[0], str): + violations.append("analysis lock requires exactly one yt-dlp package") + elif yt_dlp_records[0].get("version") != locked_versions[0]: + violations.append( + "supplemental inventory yt-dlp version does not match uv.lock" + ) + + operator_tools = inventory.get("operatorProvidedTools") + if not isinstance(operator_tools, list): + violations.append("supplemental inventory operatorProvidedTools must be a list") + else: + operator_names = { + tool.get("name") for tool in operator_tools if isinstance(tool, dict) + } + for required_tool in ("ffmpeg", "ffprobe"): + if required_tool not in operator_names: + violations.append( + f"supplemental inventory missing operator tool: {required_tool}" + ) matching_runtime_artifacts: list[dict[str, object]] = [] for artifact in artifacts: if not isinstance(artifact, dict): violations.append("supplemental inventory model artifact must be an object") continue - name_value = artifact.get("name") - name = name_value if isinstance(name_value, str) else "" - if name.startswith("bandsplit-"): + artifact_runtime = artifact.get("runtimeModelName") + label = ( + f"runtime model {artifact_runtime}" + if isinstance(artifact_runtime, str) and artifact_runtime.strip() + else "model artifact" + ) + name = artifact.get("name") + if isinstance(name, str) and name.startswith("bandsplit-"): violations.append(f"supplemental inventory contains retired model: {name}") - if artifact.get("runtimeModelName") == runtime_model: + if artifact_runtime == runtime_model: matching_runtime_artifacts.append(artifact) - if not matching_runtime_artifacts: - violations.append( - f"supplemental inventory missing runtime model: {runtime_model}" - ) - return violations - - for artifact in matching_runtime_artifacts: missing_fields = sorted(REQUIRED_MODEL_ARTIFACT_FIELDS - artifact.keys()) if missing_fields: violations.append( - f"supplemental inventory runtime model {runtime_model} missing fields: " + f"supplemental inventory {label} missing fields: " + ", ".join(missing_fields) ) - for field in sorted(MODEL_ARTIFACT_STRING_FIELDS & artifact.keys()): + for field in sorted(REQUIRED_MODEL_STRING_FIELDS & artifact.keys()): value = artifact[field] if not isinstance(value, str) or not value.strip(): violations.append( - f"supplemental inventory runtime model {runtime_model} " - f"field {field} must be a non-empty string" + f"supplemental inventory {label} requires non-empty " + f"string field: {field}" ) checksum = artifact.get("checksum") if not isinstance(checksum, str) or not FULL_SHA256_PATTERN.fullmatch(checksum): violations.append( - f"supplemental inventory runtime model {runtime_model} requires full SHA-256" + f"supplemental inventory {label} requires full SHA-256" ) source_url = artifact.get("sourceUrl") if not isinstance(source_url, str) or not source_url.startswith("https://"): - violations.append( - f"supplemental inventory runtime model {runtime_model} requires HTTPS source" - ) + violations.append(f"supplemental inventory {label} requires HTTPS source") size_bytes = artifact.get("sizeBytes") if ( not isinstance(size_bytes, int) @@ -134,8 +209,46 @@ def supplemental_inventory_violations( or size_bytes <= 0 ): violations.append( - f"supplemental inventory runtime model {runtime_model} " - "requires positive integer sizeBytes" + f"supplemental inventory {label} requires positive sizeBytes" + ) + + if not matching_runtime_artifacts: + violations.append( + f"supplemental inventory missing runtime model: {runtime_model}" + ) + return violations + + separator_artifact = _separator_model_artifact(separator_source, runtime_model) + if separator_artifact is None: + violations.append( + f"separator implementation missing exact artifact manifest: {runtime_model}" + ) + return violations + + for artifact in matching_runtime_artifacts: + if artifact.get("checksum") != f"sha256:{separator_artifact['sha256']}": + violations.append( + f"supplemental inventory runtime model {runtime_model} checksum " + "does not match separator manifest" + ) + if artifact.get("sizeBytes") != separator_artifact["sizeBytes"]: + violations.append( + f"supplemental inventory runtime model {runtime_model} sizeBytes " + "does not match separator manifest" + ) + source_url = artifact.get("sourceUrl") + if not isinstance(source_url, str) or not source_url.endswith( + f"/{separator_artifact['filename']}" + ): + violations.append( + f"supplemental inventory runtime model {runtime_model} filename " + "does not match separator manifest" + ) + version = artifact.get("version") + if not isinstance(version, str) or str(separator_artifact["signature"]) not in version: + violations.append( + f"supplemental inventory runtime model {runtime_model} version " + "does not identify separator signature" ) return violations diff --git a/services/analysis-engine/src/bandscope_analysis/api.py b/services/analysis-engine/src/bandscope_analysis/api.py index 892a00954..a1835b899 100644 --- a/services/analysis-engine/src/bandscope_analysis/api.py +++ b/services/analysis-engine/src/bandscope_analysis/api.py @@ -18,7 +18,7 @@ from bandscope_analysis.roles import RoleExtractor from bandscope_analysis.sections import extract_sections from bandscope_analysis.sections.segmenter import segment_with_boundaries -from bandscope_analysis.separation import AudioStemSeparator +from bandscope_analysis.separation import AudioStemSeparator, ModelArtifactError logger = logging.getLogger(__name__) @@ -898,16 +898,13 @@ def _stem_separation_failure( "Audio source file not found.", "Stem separation failed because the source file was missing.", ) + if isinstance(error, ModelArtifactError): + return ( + "runtime_error", + "Stem separation model is unavailable.", + "Stem separation unavailable because the approved model could not be verified.", + ) if isinstance(error, ValueError): - if "htdemucs model artifact" in error_message.lower(): - return ( - "runtime_error", - "Stem separation model is unavailable.", - ( - "Stem separation unavailable because the verified model artifact " - "is missing or invalid." - ), - ) if "not available on this platform" in error_message or "demucs/torch" in error_message: return ( "runtime_error", diff --git a/services/analysis-engine/src/bandscope_analysis/separation/__init__.py b/services/analysis-engine/src/bandscope_analysis/separation/__init__.py index 5e812203e..a50d01c7a 100644 --- a/services/analysis-engine/src/bandscope_analysis/separation/__init__.py +++ b/services/analysis-engine/src/bandscope_analysis/separation/__init__.py @@ -1,6 +1,6 @@ """Source separation module for audio stems and role stem groups.""" -from .audio_separator import AudioSeparationConfig, AudioStemSeparator +from .audio_separator import AudioSeparationConfig, AudioStemSeparator, ModelArtifactError from .model import ( AudioSeparationResult, AudioStemArray, @@ -21,6 +21,7 @@ "AudioStemPayload", "StemRoleTypeMap", "AudioStemSeparator", + "ModelArtifactError", "StemSeparator", "StemCategory", "StemDescriptor", diff --git a/services/analysis-engine/src/bandscope_analysis/separation/audio_separator.py b/services/analysis-engine/src/bandscope_analysis/separation/audio_separator.py index e13bb9010..df908cdd9 100644 --- a/services/analysis-engine/src/bandscope_analysis/separation/audio_separator.py +++ b/services/analysis-engine/src/bandscope_analysis/separation/audio_separator.py @@ -1,4 +1,4 @@ -"""Local audio source separation using a verified Demucs model. +"""Local audio source separation using an exact verified Demucs model. Replaces the previous FFT band-masking heuristic — which scored around -39 dB SI-SDR on a realistic mix (i.e. not real separation) — with Demucs (htdemucs), a @@ -9,9 +9,12 @@ Security Notes: - Treats the selected audio file as untrusted input: the path is normalized and verified to be a file, and a maximum byte size is enforced before decode. -- Inference runs locally on CPU only after an operator provisions the exact - inventoried model artifact. Missing or changed bytes fail closed before - Demucs can deserialize them; this runtime never downloads model weights. +- Inference runs locally on CPU only after the exact inventoried model has been + provisioned in the trusted user cache. Loading never falls back to a network + retrieval path. +- The cache entry must be a non-symlinked regular file with the exact byte size + and full SHA-256; the verified in-memory bytes are the only bytes passed to + torch checkpoint deserialization. - Does not log or persist raw audio, separated stems, or full source paths. - Fails with bounded, filename-scoped errors so callers can surface a safe failure without leaking local directory structure. @@ -20,8 +23,10 @@ from __future__ import annotations import hashlib +import io import logging import os +import stat import warnings from dataclasses import dataclass from pathlib import Path @@ -44,10 +49,30 @@ # Demucs htdemucs emits these four sources; this is the canonical stem set. _STEM_ORDER: tuple[AudioStemName, ...] = ("vocals", "bass", "drums", "other") _EMPTY_RANGE_EPS = 1e-9 -_HTDEMUCS_MODEL_SIGNATURE = "955717e8" -_HTDEMUCS_MODEL_FILENAME = "955717e8-8726e21a.th" -_HTDEMUCS_MODEL_SHA256 = "8726e21a993978c7ba086d3872e7608d7d5bfca646ca4aca459ffda844faa8b4" -_HTDEMUCS_MODEL_BYTES = 84_141_911 + + +class ModelArtifactError(ValueError): + """Report a missing, untrusted, or unloadable approved model artifact.""" + + +@dataclass(frozen=True) +class _ModelArtifactSpec: + """Exact identity of one approved runtime model checkpoint.""" + + signature: str + filename: str + sha256: str + size_bytes: int + + +_MODEL_ARTIFACTS: dict[str, _ModelArtifactSpec] = { + "htdemucs": _ModelArtifactSpec( + signature="955717e8", + filename="955717e8-8726e21a.th", + sha256="8726e21a993978c7ba086d3872e7608d7d5bfca646ca4aca459ffda844faa8b4", + size_bytes=84_141_911, + ) +} _MODEL_PATH_ENV = "BANDSCOPE_HTDEMUCS_MODEL_PATH" @@ -69,7 +94,7 @@ class AudioSeparationConfig: max_file_bytes: int = MAX_AUDIO_FILE_BYTES max_duration_seconds: float = float(MAX_ANALYSIS_DURATION_SECONDS) model_name: str = "htdemucs" - model_artifact_path: Path | None = None + model_cache_directory: Path | None = None device: str = "cpu" # Disable Demucs' random time-shift augmentation so repeated analysis of # the same bytes is deterministic and benchmark evidence is reproducible. @@ -135,72 +160,66 @@ def _separate_signal( return {name: _as_float_array(sources[name]) for name in _STEM_ORDER} def _load_model(self) -> Any: - """Lazily load and cache the Demucs model. + """Lazily load and cache the exact inventoried Demucs model. Demucs (and torch) are installed only on platforms with current torch wheels (see pyproject platform markers); elsewhere separation fails with a clear error the pipeline already surfaces safely. - The runtime passes a local repository to Demucs, disabling its remote - model path. Full byte size and SHA-256 are checked before Demucs or - torch can deserialize the artifact. + Loading is deliberately offline and fail-closed. The checkpoint must + already exist in the configured cache, and its exact size and full + SHA-256 are verified before the same in-memory bytes are deserialized. """ - if self._model is None: - try: - from demucs.pretrained import ( # type: ignore[import-not-found, unused-ignore] - get_model, - ) - except ImportError as error: - raise ValueError( - "Stem separation is not available on this platform (demucs/torch not installed)" - ) from error - - artifact_path = self._verified_model_artifact_path() - model = get_model(_HTDEMUCS_MODEL_SIGNATURE, repo=artifact_path.parent) - model.eval() - self._model = model - return self._model + if self._model is not None: + return self._model - def _verified_model_artifact_path(self) -> Path: - """Return the exact local htdemucs artifact after full identity checks.""" - configured = self.config.model_artifact_path - if configured is None: - configured_text = os.environ.get(_MODEL_PATH_ENV) - if configured_text: - configured = Path(configured_text) - else: - torch_home = Path( - os.environ.get( - "TORCH_HOME", - Path(os.environ.get("XDG_CACHE_HOME", Path.home() / ".cache")) / "torch", - ) - ) - configured = torch_home / "hub" / "checkpoints" / _HTDEMUCS_MODEL_FILENAME + artifact = _MODEL_ARTIFACTS.get(self.config.model_name) + if artifact is None: + raise ModelArtifactError("Stem separation model is not inventoried") - if configured.is_symlink(): - raise ValueError("The htdemucs model artifact path must not be a symlink") try: - artifact_path = configured.expanduser().resolve(strict=True) - except (FileNotFoundError, OSError) as error: + import torch + from demucs.states import ( # type: ignore[import-not-found, unused-ignore] + load_model, + ) + except ImportError as error: raise ValueError( - "The verified htdemucs model artifact is unavailable; provision the " - f"inventoried file and set {_MODEL_PATH_ENV}" + "Stem separation is not available on this platform (demucs/torch not installed)" ) from error - if not artifact_path.is_file() or artifact_path.name != _HTDEMUCS_MODEL_FILENAME: - raise ValueError( - "The verified htdemucs model artifact is unavailable; the exact " - f"{_HTDEMUCS_MODEL_FILENAME} file is required" - ) - if artifact_path.stat().st_size != _HTDEMUCS_MODEL_BYTES: - raise ValueError("The htdemucs model artifact failed byte-size verification") - digest = hashlib.sha256() - with artifact_path.open("rb") as model_file: - for chunk in iter(lambda: model_file.read(1024 * 1024), b""): - digest.update(chunk) - if digest.hexdigest() != _HTDEMUCS_MODEL_SHA256: - raise ValueError("The htdemucs model artifact failed full SHA-256 verification") - return artifact_path + configured_path = os.environ.get(_MODEL_PATH_ENV) + if self.config.model_cache_directory is not None: + artifact_path = Path(self.config.model_cache_directory) / artifact.filename + elif configured_path: + artifact_path = Path(configured_path) + if (artifact_path.is_absolute(), artifact_path.name) != (True, artifact.filename): + raise ModelArtifactError( + "Stem separation model path must use the absolute inventoried filename" + ) + else: + try: + artifact_path = Path(torch.hub.get_dir()) / "checkpoints" / artifact.filename + except Exception: + raise ModelArtifactError( + "Stem separation model cache location is unavailable" + ) from None + + payload = _read_verified_model_artifact(artifact_path, artifact) + try: + # This exact in-memory payload passed full SHA-256 and size verification above. + package = torch.load( # nosec B614 + io.BytesIO(payload), + map_location="cpu", + weights_only=False, + ) + model = load_model(package) # type: ignore[no-untyped-call] + model.eval() + except Exception: + raise ModelArtifactError( + "Stem separation model failed to load after integrity verification" + ) from None + self._model = model + return self._model def _apply_model(self, model: Any, audio: AudioStemArray) -> dict[str, np.ndarray[Any, Any]]: """Apply Demucs to a mono signal, returning demucs-source-name -> mono array.""" @@ -289,3 +308,37 @@ def _as_float_array(values: object) -> AudioStemArray: array = np.ravel(np.asarray(values, dtype=np.float32)) finite = np.nan_to_num(array, copy=False, nan=0.0, posinf=0.0, neginf=0.0) return cast(AudioStemArray, finite) + + +def _read_verified_model_artifact(path: Path, artifact: _ModelArtifactSpec) -> bytes: + """Read one exact regular cache file and verify its full artifact identity.""" + descriptor: int | None = None + try: + cache_metadata = path.lstat() + if stat.S_ISLNK(cache_metadata.st_mode): + raise ModelArtifactError("Stem separation model cache entry is a symlink") + if not stat.S_ISREG(cache_metadata.st_mode): + raise ModelArtifactError("Stem separation model cache entry is not a regular file") + + flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(path, flags) + opened_metadata = os.fstat(descriptor) + if not stat.S_ISREG(opened_metadata.st_mode): + raise ModelArtifactError("Stem separation model cache entry is not a regular file") + if opened_metadata.st_size != artifact.size_bytes: + raise ModelArtifactError("Stem separation model does not match inventoried byte size") + with os.fdopen(descriptor, "rb", closefd=False) as fileobj: + payload = fileobj.read(artifact.size_bytes + 1) + except FileNotFoundError: + raise ModelArtifactError("Stem separation model is not provisioned") from None + except ModelArtifactError: + raise + except OSError: + raise ModelArtifactError("Stem separation model could not be opened securely") from None + finally: + if descriptor is not None: + os.close(descriptor) + + if hashlib.sha256(payload).hexdigest() != artifact.sha256: + raise ModelArtifactError("Stem separation model does not match inventoried SHA-256") + return payload diff --git a/services/analysis-engine/src/bandscope_analysis/youtube.py b/services/analysis-engine/src/bandscope_analysis/youtube.py index 7db0a2c5b..a14578cc4 100644 --- a/services/analysis-engine/src/bandscope_analysis/youtube.py +++ b/services/analysis-engine/src/bandscope_analysis/youtube.py @@ -5,17 +5,22 @@ Security Notes: - Accepts only bounded, standard HTTPS YouTube watch URLs and disables playlists, geographic bypass, credentials, and interactive authentication. -- Keeps certificate verification enabled and retains yt-dlp's maintained CA - bundle fallback so minimal containers do not depend on an absent system store. +- Keeps certificate verification enabled. It uses the operating-system trust + store when roots are present and otherwise retains yt-dlp's CA fallback. +- Optionally accepts sibling absolute ffmpeg/ffprobe paths only with both full + SHA-256 identities, verifies both regular executables before handoff, and + returns redacted failures. - Rejects metadata over 15 minutes and completed files over 50 MiB, returns sanitized public errors, and never logs the requested URL or downloaded audio. """ import argparse import hashlib +import hmac import json import os import re +import ssl import sys import urllib.parse from pathlib import Path @@ -30,9 +35,8 @@ "Failed to download audio from YouTube. Please use a local audio file instead." ) YOUTUBE_IMPORT_FAILED_MESSAGE = "YouTube import failed. Please use a local audio file instead." -FFMPEG_PATH_ENV = "BANDSCOPE_FFMPEG_PATH" -FFMPEG_SHA256_ENV = "BANDSCOPE_FFMPEG_SHA256" -FULL_SHA256_PATTERN = re.compile(r"^[0-9a-f]{64}$") +RUNTIME_DEPENDENCY_INVALID_MESSAGE = "The configured media runtime failed identity verification." +SHA256_PATTERN = re.compile(r"^[0-9a-f]{64}$") def validate_url(url: str) -> bool: @@ -119,43 +123,117 @@ def _handle_download_error(e: yt_dlp.utils.DownloadError) -> Dict[str, Any]: } -def _verified_ffmpeg_location() -> str | None: - """Return an exact operator-provisioned ffmpeg path when identity is configured.""" - configured_path = os.environ.get(FFMPEG_PATH_ENV) - configured_digest = os.environ.get(FFMPEG_SHA256_ENV) - if configured_path is None and configured_digest is None: +def _system_ca_available() -> bool: + """Return whether the operating-system TLS context contains trusted CA roots. + + Any probe failure is treated as an empty system store so yt-dlp keeps its + default CA behavior. Certificate verification is never disabled. + """ + try: + context = ssl.create_default_context() + return bool(context.get_ca_certs(binary_form=True)) + except Exception: + return False + + +def _verify_executable_artifact( + executable_path: Optional[str], executable_sha256: Optional[str] +) -> Optional[str]: + """Authenticate one executable and return its resolved absolute path. + + The executable must be an absolute, non-symlinked regular file with execute + permission, and the digest must be a canonical full lowercase SHA-256. + """ + if not isinstance(executable_path, str) or not isinstance(executable_sha256, str): return None - if configured_path is None or configured_digest is None: - raise ValueError("ffmpeg release identity requires both path and SHA-256") - if not FULL_SHA256_PATTERN.fullmatch(configured_digest): - raise ValueError("ffmpeg release identity requires a full lowercase SHA-256") - - raw_path = Path(configured_path) - if not raw_path.is_absolute(): - raise ValueError("ffmpeg release identity requires an absolute executable path") + if not SHA256_PATTERN.fullmatch(executable_sha256): + return None + + candidate = Path(executable_path) + if not candidate.is_absolute() or candidate.is_symlink(): + return None + try: - executable_path = raw_path.resolve(strict=True) - except (FileNotFoundError, OSError) as error: - raise ValueError("ffmpeg release executable is unavailable") from error - if not executable_path.is_file() or not os.access(executable_path, os.X_OK): - raise ValueError("ffmpeg release executable is unavailable") - - digest = hashlib.sha256() - with executable_path.open("rb") as executable_file: - for chunk in iter(lambda: executable_file.read(1024 * 1024), b""): - digest.update(chunk) - if digest.hexdigest() != configured_digest: - raise ValueError("ffmpeg release executable failed SHA-256 verification") - return str(executable_path) - - -def download_youtube_audio(url: str, out_dir: str) -> Dict[str, Any]: + resolved = candidate.resolve(strict=True) + if not resolved.is_file() or not os.access(resolved, os.X_OK): + return None + + digest = hashlib.sha256() + with resolved.open("rb") as artifact: + for chunk in iter(lambda: artifact.read(1024 * 1024), b""): + digest.update(chunk) + except (OSError, RuntimeError): + return None + + if not hmac.compare_digest(digest.hexdigest(), executable_sha256): + return None + return str(resolved) + + +def _verify_media_runtime( + ffmpeg_path: Optional[str], + ffmpeg_sha256: Optional[str], + ffprobe_path: Optional[str], + ffprobe_sha256: Optional[str], +) -> tuple[bool, Optional[str]]: + """Authenticate the complete executable set yt-dlp may invoke. + + An omitted four-part identity retains ordinary yt-dlp PATH behavior. Once + any field is configured, all four are mandatory. ffmpeg and ffprobe must be + exact sibling program names because yt-dlp derives its probe path from the + configured ffmpeg location. + """ + identity = (ffmpeg_path, ffmpeg_sha256, ffprobe_path, ffprobe_sha256) + if all(value is None for value in identity): + return True, None + if any(not isinstance(value, str) for value in identity): + return False, None + + verified_ffmpeg = _verify_executable_artifact(ffmpeg_path, ffmpeg_sha256) + verified_ffprobe = _verify_executable_artifact(ffprobe_path, ffprobe_sha256) + if verified_ffmpeg is None or verified_ffprobe is None: + return False, None + + ffmpeg = Path(verified_ffmpeg) + ffprobe = Path(verified_ffprobe) + executable_suffix = {"nt": ".exe"}.get(os.name, "") + if ffmpeg.name != f"ffmpeg{executable_suffix}": + return False, None + if ffprobe.name != f"ffprobe{executable_suffix}" or ffprobe.parent != ffmpeg.parent: + return False, None + return True, verified_ffmpeg + + +def _runtime_dependency_invalid() -> Dict[str, Any]: + """Return the stable redacted response for an untrusted media runtime.""" + return { + "ok": False, + "error": { + "code": "runtime_dependency_invalid", + "message": RUNTIME_DEPENDENCY_INVALID_MESSAGE, + }, + } + + +def download_youtube_audio( + url: str, + out_dir: str, + *, + ffmpeg_path: Optional[str] = None, + ffmpeg_sha256: Optional[str] = None, + ffprobe_path: Optional[str] = None, + ffprobe_sha256: Optional[str] = None, +) -> Dict[str, Any]: """ Download audio from a YouTube URL to the specified directory. Args: url: The YouTube URL to download. out_dir: The directory to save the audio file. + ffmpeg_path: Optional absolute path to a provisioned ffmpeg executable. + ffmpeg_sha256: Full lowercase SHA-256 identity for ``ffmpeg_path``. + ffprobe_path: Optional sibling path to the provisioned ffprobe executable. + ffprobe_sha256: Full lowercase SHA-256 identity for ``ffprobe_path``. Returns: A dictionary containing the result of the download. @@ -169,6 +247,15 @@ def download_youtube_audio(url: str, out_dir: str) -> Dict[str, Any]: }, } + runtime_is_valid, verified_ffmpeg_path = _verify_media_runtime( + ffmpeg_path, + ffmpeg_sha256, + ffprobe_path, + ffprobe_sha256, + ) + if not runtime_is_valid: + return _runtime_dependency_invalid() + ydl_opts: Dict[str, Any] = { "format": "bestaudio/best", "outtmpl": os.path.join(out_dir, "%(id)s.%(ext)s"), @@ -179,11 +266,14 @@ def download_youtube_audio(url: str, out_dir: str) -> Dict[str, Any]: "postprocessors": [{"key": "FFmpegExtractAudio"}], "geo_bypass": False, } + if _system_ca_available(): + # Use managed desktop trust roots only after confirming that the store + # is populated. Otherwise yt-dlp retains its built-in CA fallback. + ydl_opts["compat_opts"] = {"no-certifi"} + if verified_ffmpeg_path is not None: + ydl_opts["ffmpeg_location"] = verified_ffmpeg_path try: - ffmpeg_location = _verified_ffmpeg_location() - if ffmpeg_location is not None: - ydl_opts["ffmpeg_location"] = ffmpeg_location with yt_dlp.YoutubeDL(ydl_opts) as ydl: info = ydl.extract_info(url, download=False) if info is None: @@ -249,9 +339,20 @@ def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("--url", required=True) parser.add_argument("--out-dir", required=True) + parser.add_argument("--ffmpeg-path") + parser.add_argument("--ffmpeg-sha256") + parser.add_argument("--ffprobe-path") + parser.add_argument("--ffprobe-sha256") args = parser.parse_args() - result = download_youtube_audio(args.url, args.out_dir) + result = download_youtube_audio( + args.url, + args.out_dir, + ffmpeg_path=args.ffmpeg_path, + ffmpeg_sha256=args.ffmpeg_sha256, + ffprobe_path=args.ffprobe_path, + ffprobe_sha256=args.ffprobe_sha256, + ) print(json.dumps(result)) sys.exit(0 if result["ok"] else 1) diff --git a/services/analysis-engine/tests/known_stem_benchmark.py b/services/analysis-engine/tests/known_stem_benchmark.py index 9dbf2c9c1..071f17e76 100644 --- a/services/analysis-engine/tests/known_stem_benchmark.py +++ b/services/analysis-engine/tests/known_stem_benchmark.py @@ -204,7 +204,7 @@ def _strongest_window_start(signal: np.ndarray, window_samples: int) -> int: def _normalized_correlation(left: np.ndarray, right: np.ndarray) -> float: - """Return signed zero-mean Pearson correlation for two equal windows.""" + """Return absolute zero-mean Pearson correlation for two equal windows.""" left_centered = left - float(np.mean(left)) right_centered = right - float(np.mean(right)) denominator = math.sqrt( @@ -212,7 +212,7 @@ def _normalized_correlation(left: np.ndarray, right: np.ndarray) -> float: ) if denominator <= _ENERGY_EPSILON: raise ValueError("aligned benchmark window has insufficient audio energy") - return float(np.dot(left_centered, right_centered) / denominator) + return float(abs(np.dot(left_centered, right_centered)) / denominator) def align_active_reference_window( @@ -255,9 +255,7 @@ def align_active_reference_window( ) max_lag_frames = int(math.ceil(max_lag_seconds * sample_rate / hop_samples)) valid_coarse = np.flatnonzero(np.abs(coarse_lags) <= max_lag_frames) - if valid_coarse.size == 0: - raise ValueError("reference fixture has no permitted alignment lag") - best_coarse_index = int(valid_coarse[np.argmax(coarse_correlation[valid_coarse])]) + best_coarse_index = int(valid_coarse[np.argmax(np.abs(coarse_correlation[valid_coarse]))]) coarse_lag_samples = int(coarse_lags[best_coarse_index]) * hop_samples reference_start = _strongest_window_start(reference_signal, window_samples) @@ -282,9 +280,7 @@ def align_active_reference_window( valid_refined = np.flatnonzero( (refined_lags >= 0) & (refined_lags + window_samples <= mixture_search.size) ) - if valid_refined.size == 0: - raise ValueError("reference fixture cannot produce a full scoring window") - best_refined_index = int(valid_refined[np.argmax(refined_correlation[valid_refined])]) + best_refined_index = int(valid_refined[np.argmax(np.abs(refined_correlation[valid_refined]))]) mixture_start = search_start + int(refined_lags[best_refined_index]) mixture_window = mixture_signal[mixture_start : mixture_start + window_samples] correlation = _normalized_correlation(mixture_window, reference_window) diff --git a/services/analysis-engine/tests/test_api.py b/services/analysis-engine/tests/test_api.py index 90f27f6f5..cf458dc50 100644 --- a/services/analysis-engine/tests/test_api.py +++ b/services/analysis-engine/tests/test_api.py @@ -24,6 +24,7 @@ run_analysis_job_updates, validate_analysis_job_request, ) +from bandscope_analysis.separation import ModelArtifactError def test_get_analysis_status_returns_health_payload() -> None: @@ -1008,13 +1009,10 @@ def put(self, item: tuple[str, object]) -> None: "Stem separation unavailable because Demucs or torch is not installed.", ), ( - ValueError("The verified htdemucs model artifact is unavailable"), + ModelArtifactError("Stem separation model is not provisioned"), "runtime_error", "Stem separation model is unavailable.", - ( - "Stem separation unavailable because the verified model artifact " - "is missing or invalid." - ), + "Stem separation unavailable because the approved model could not be verified.", ), ( RuntimeError("oom /secret/audio.wav"), diff --git a/services/analysis-engine/tests/test_documentation_policy.py b/services/analysis-engine/tests/test_documentation_policy.py index 3382670d4..29e3e9876 100644 --- a/services/analysis-engine/tests/test_documentation_policy.py +++ b/services/analysis-engine/tests/test_documentation_policy.py @@ -29,7 +29,7 @@ def test_documentation_contract_accepts_checked_in_authorities() -> None: def test_documentation_contract_checks_every_nested_plan_security_section( tmp_path: Path, ) -> None: - """Reject newly added plan documents that omit their security boundary.""" + """Reject newly added plan documents that omit their canonical security boundary.""" documentation = load_module("scripts/checks/verify_docs.py", "verify_docs_contract_nested_plan") plan = tmp_path / "docs" / "plans" / "future" / "unsafe-plan.md" plan.parent.mkdir(parents=True) @@ -37,4 +37,35 @@ def test_documentation_contract_checks_every_nested_plan_security_section( violations = documentation.documentation_violations(tmp_path) - assert "docs/plans/future/unsafe-plan.md missing section: Security Notes" in violations + assert "docs/plans/future/unsafe-plan.md missing section: ## Security Notes" in violations + + +def test_security_notes_contract_discovers_nested_plan_without_canonical_section( + tmp_path: Path, +) -> None: + """Reject nested plan documents that omit the canonical Security Notes section.""" + security_notes = load_module( + "scripts/checks/verify_security_notes.py", + "verify_security_notes_nested_plan", + ) + plan_path = tmp_path / "docs" / "plans" / "nested" / "new-plan.md" + plan_path.parent.mkdir(parents=True) + plan_path.write_text( + "# New plan\n\nSecurity Notes are considered elsewhere.\n", + encoding="utf-8", + ) + + assert security_notes.security_notes_violations(tmp_path) == [ + "docs/plans/nested/new-plan.md missing section: ## Security Notes" + ] + + +def test_security_notes_contract_accepts_checked_in_plans() -> None: + """Accept every checked-in plan only when its complete canonical section is present.""" + security_notes = load_module( + "scripts/checks/verify_security_notes.py", + "verify_security_notes_repo", + ) + repo_root = Path(__file__).resolve().parents[3] + + assert security_notes.security_notes_violations(repo_root) == [] diff --git a/services/analysis-engine/tests/test_separation.py b/services/analysis-engine/tests/test_separation.py index 53a28366a..be13ce031 100644 --- a/services/analysis-engine/tests/test_separation.py +++ b/services/analysis-engine/tests/test_separation.py @@ -2,14 +2,15 @@ from __future__ import annotations +import hashlib import os import sys -from pathlib import Path -from types import ModuleType +from types import ModuleType, SimpleNamespace import numpy as np import pytest import soundfile as sf +from conftest import make_symlink_or_skip from bandscope_analysis.separation import audio_separator as audio_separator_module from bandscope_analysis.separation.audio_separator import ( @@ -217,181 +218,333 @@ def __exit__(self, *args: object) -> None: return None -def _install_fake_demucs(monkeypatch: pytest.MonkeyPatch, get_model: object) -> None: - """Install a lightweight fake demucs package for import-boundary tests.""" +def _install_fake_verified_model_deserializer( + monkeypatch: pytest.MonkeyPatch, + *, + torch_hub_root: object | None = None, + torch_load: object | None = None, +) -> dict[str, object]: + """Install fake torch/Demucs deserializers and return captured calls.""" + calls: dict[str, object] = {"torch_load_count": 0, "demucs_load_count": 0} + fake_torch = ModuleType("torch") + if torch_hub_root is not None: + fake_torch.hub = SimpleNamespace(get_dir=lambda: torch_hub_root) # type: ignore[attr-defined] + + def default_torch_load( + stream: object, + *, + map_location: str, + weights_only: bool, + ) -> dict[str, object]: + calls["torch_load_count"] = int(calls["torch_load_count"]) + 1 + calls["payload"] = stream.read() # type: ignore[attr-defined] + calls["map_location"] = map_location + calls["weights_only"] = weights_only + return {"verified": True} + + fake_torch.load = torch_load or default_torch_load # type: ignore[attr-defined] demucs_module = ModuleType("demucs") - pretrained_module = ModuleType("demucs.pretrained") - pretrained_module.get_model = get_model # type: ignore[attr-defined] - demucs_module.pretrained = pretrained_module # type: ignore[attr-defined] - monkeypatch.setitem(sys.modules, "demucs", demucs_module) - monkeypatch.setitem(sys.modules, "demucs.pretrained", pretrained_module) + states_module = ModuleType("demucs.states") + def fake_load_model(package: object) -> _FakeModel: + calls["demucs_load_count"] = int(calls["demucs_load_count"]) + 1 + calls["package"] = package + return _FakeModel() -def _patch_demucs(monkeypatch: pytest.MonkeyPatch, per_source: dict | None = None) -> None: - """Patch the Demucs boundary so separation runs without the real model. + states_module.load_model = fake_load_model # type: ignore[attr-defined] + demucs_module.states = states_module # type: ignore[attr-defined] + monkeypatch.setitem(sys.modules, "torch", fake_torch) + monkeypatch.setitem(sys.modules, "demucs", demucs_module) + monkeypatch.setitem(sys.modules, "demucs.states", states_module) + return calls - ``per_source`` optionally maps a demucs source name to a mono numpy array to - return for that stem; unspecified sources return silence. - """ - def fake_get_model(name: str, *, repo: Path) -> _FakeModel: - return _FakeModel() +def _patch_model_spec( + monkeypatch: pytest.MonkeyPatch, + *, + filename: str, + payload: bytes, +) -> None: + """Replace the htdemucs manifest with a small exact test artifact.""" + spec = audio_separator_module._ModelArtifactSpec( + signature="test-signature", + filename=filename, + sha256=hashlib.sha256(payload).hexdigest(), + size_bytes=len(payload), + ) + monkeypatch.setitem(audio_separator_module._MODEL_ARTIFACTS, "htdemucs", spec) - def fake_apply_model( - self: AudioStemSeparator, model: _FakeModel, audio: np.ndarray - ) -> dict[str, np.ndarray]: - samples = int(audio.size) - out = {name: np.zeros(samples, dtype=np.float32) for name in _DEMUCS_SOURCES} - if per_source: - for name in _DEMUCS_SOURCES: - if name in per_source: - row = per_source[name].astype(np.float32) - copy_length = min(samples, int(row.size)) - out[name][:copy_length] = row[:copy_length] - return out - _install_fake_demucs(monkeypatch, fake_get_model) - monkeypatch.setattr( - AudioStemSeparator, - "_verified_model_artifact_path", - lambda self: Path("/verified/955717e8-8726e21a.th"), +def test_audio_stem_separator_verifies_exact_model_bytes_before_deserialization( + tmp_path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Deserialize only the exact inventoried bytes and cache the loaded model.""" + payload = b"verified-model-package" + filename = "test-signature-deadbeef.th" + cache_dir = tmp_path / "torch-hub" / "checkpoints" + cache_dir.mkdir(parents=True) + (cache_dir / filename).write_bytes(payload) + _patch_model_spec(monkeypatch, filename=filename, payload=payload) + calls = _install_fake_verified_model_deserializer( + monkeypatch, + torch_hub_root=tmp_path / "torch-hub", ) - monkeypatch.setattr(AudioStemSeparator, "_apply_model", fake_apply_model) + separator = AudioStemSeparator() + first_model = separator._load_model() + second_model = separator._load_model() -def test_audio_stem_separator_rejects_missing_model_before_demucs_load( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - """Fail closed before Demucs can fetch or deserialize a missing model.""" - calls = {"n": 0} + assert first_model is second_model + assert calls == { + "torch_load_count": 1, + "demucs_load_count": 1, + "payload": payload, + "map_location": "cpu", + "weights_only": False, + "package": {"verified": True}, + } - def fake_get_model(name: str, *, repo: Path) -> _FakeModel: - calls["n"] += 1 - return _FakeModel() - _install_fake_demucs(monkeypatch, fake_get_model) - separator = AudioStemSeparator( - AudioSeparationConfig(model_artifact_path=tmp_path / "missing-model.th") +@pytest.mark.parametrize( + ("payload", "error_pattern"), + [ + (None, "not provisioned"), + (b"short", "byte size"), + (b"tampered-model-package", "SHA-256"), + ], +) +def test_audio_stem_separator_rejects_missing_or_changed_model_before_deserialization( + tmp_path, + monkeypatch: pytest.MonkeyPatch, + payload: bytes | None, + error_pattern: str, +) -> None: + """Fail closed for missing, truncated, or substituted checkpoint bytes.""" + trusted_payload = b"verified-model-package" + filename = "test-signature-deadbeef.th" + if payload is not None and error_pattern == "SHA-256": + payload = payload.ljust(len(trusted_payload), b"!")[: len(trusted_payload)] + _patch_model_spec(monkeypatch, filename=filename, payload=trusted_payload) + cache_dir = tmp_path / "checkpoints" + cache_dir.mkdir() + if payload is not None: + (cache_dir / filename).write_bytes(payload) + + def forbidden_torch_load(*args: object, **kwargs: object) -> object: + raise AssertionError("unverified bytes reached torch.load") + + _install_fake_verified_model_deserializer( + monkeypatch, + torch_load=forbidden_torch_load, ) + separator = AudioStemSeparator(AudioSeparationConfig(model_cache_directory=cache_dir)) - with pytest.raises(ValueError, match="verified htdemucs model artifact is unavailable"): + with pytest.raises(ValueError, match=error_pattern): separator._load_model() - assert calls["n"] == 0 + +def test_audio_stem_separator_rejects_symlinked_model_artifact( + tmp_path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Reject a cache symlink before reading or deserializing its target.""" + payload = b"verified-model-package" + filename = "test-signature-deadbeef.th" + _patch_model_spec(monkeypatch, filename=filename, payload=payload) + cache_dir = tmp_path / "checkpoints" + cache_dir.mkdir() + target = tmp_path / "outside.th" + target.write_bytes(payload) + make_symlink_or_skip(cache_dir / filename, target) + _install_fake_verified_model_deserializer(monkeypatch) + separator = AudioStemSeparator(AudioSeparationConfig(model_cache_directory=cache_dir)) + + with pytest.raises(ValueError, match="symlink"): + separator._load_model() -def test_audio_stem_separator_verifies_full_model_identity_before_local_load( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch +def test_audio_stem_separator_rejects_nonregular_model_artifact( + tmp_path, + monkeypatch: pytest.MonkeyPatch, ) -> None: - """Bind Demucs loading to the exact inventoried bytes in a local repository.""" - model_bytes = b"verified local model" - model_path = tmp_path / "955717e8-8726e21a.th" - model_path.write_bytes(model_bytes) - calls: list[tuple[str, Path]] = [] - - def fake_get_model(name: str, *, repo: Path) -> _FakeModel: - calls.append((name, repo)) - return _FakeModel() + """Reject a directory masquerading as the inventoried checkpoint file.""" + payload = b"verified-model-package" + filename = "test-signature-deadbeef.th" + _patch_model_spec(monkeypatch, filename=filename, payload=payload) + cache_dir = tmp_path / "checkpoints" + cache_dir.mkdir() + (cache_dir / filename).mkdir() + _install_fake_verified_model_deserializer(monkeypatch) + + def forbidden_open(*args: object, **kwargs: object) -> int: + raise AssertionError("nonregular cache entry reached os.open") + + monkeypatch.setattr(audio_separator_module.os, "open", forbidden_open) + separator = AudioStemSeparator(AudioSeparationConfig(model_cache_directory=cache_dir)) + + with pytest.raises(ValueError, match="regular file"): + separator._load_model() + - _install_fake_demucs(monkeypatch, fake_get_model) - monkeypatch.setattr(audio_separator_module, "_HTDEMUCS_MODEL_BYTES", len(model_bytes)) +def test_audio_stem_separator_rejects_opened_file_identity_race( + tmp_path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Re-check the opened descriptor instead of trusting path metadata alone.""" + payload = b"verified-model-package" + filename = "test-signature-deadbeef.th" + _patch_model_spec(monkeypatch, filename=filename, payload=payload) + (tmp_path / filename).write_bytes(payload) + _install_fake_verified_model_deserializer(monkeypatch) monkeypatch.setattr( - audio_separator_module, - "_HTDEMUCS_MODEL_SHA256", - __import__("hashlib").sha256(model_bytes).hexdigest(), + audio_separator_module.os, + "fstat", + lambda _descriptor: SimpleNamespace(st_mode=0, st_size=len(payload)), ) - separator = AudioStemSeparator(AudioSeparationConfig(model_artifact_path=model_path)) + separator = AudioStemSeparator(AudioSeparationConfig(model_cache_directory=tmp_path)) - assert separator._load_model() is separator._load_model() - assert calls == [("955717e8", tmp_path)] + with pytest.raises(ValueError, match="regular file"): + separator._load_model() -def test_audio_stem_separator_rejects_wrong_model_digest_before_local_load( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch +def test_audio_stem_separator_redacts_model_cache_open_errors( + tmp_path, + monkeypatch: pytest.MonkeyPatch, ) -> None: - """Reject same-sized model bytes that fail the full SHA-256 identity check.""" - model_path = tmp_path / "955717e8-8726e21a.th" - model_path.write_bytes(b"untrusted") - called = False - - def fake_get_model(name: str, *, repo: Path) -> _FakeModel: - nonlocal called - called = True - return _FakeModel() + """Redact cache paths when an exact checkpoint cannot be opened safely.""" + payload = b"verified-model-package" + filename = "test-signature-deadbeef.th" + _patch_model_spec(monkeypatch, filename=filename, payload=payload) + (tmp_path / filename).write_bytes(payload) + _install_fake_verified_model_deserializer(monkeypatch) + + def fail_open(*args: object, **kwargs: object) -> int: + raise PermissionError(f"permission denied under {tmp_path}") - _install_fake_demucs(monkeypatch, fake_get_model) - monkeypatch.setattr(audio_separator_module, "_HTDEMUCS_MODEL_BYTES", len(b"untrusted")) - monkeypatch.setattr(audio_separator_module, "_HTDEMUCS_MODEL_SHA256", "0" * 64) - separator = AudioStemSeparator(AudioSeparationConfig(model_artifact_path=model_path)) + monkeypatch.setattr(audio_separator_module.os, "open", fail_open) + separator = AudioStemSeparator(AudioSeparationConfig(model_cache_directory=tmp_path)) - with pytest.raises(ValueError, match="failed full SHA-256 verification"): + with pytest.raises(ValueError, match="could not be opened securely") as error: separator._load_model() + assert str(tmp_path) not in str(error.value) - assert called is False +def test_audio_stem_separator_redacts_default_cache_location_errors( + tmp_path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Redact torch cache details when its default location cannot be resolved.""" + calls = _install_fake_verified_model_deserializer(monkeypatch) + fake_torch = sys.modules["torch"] + + def fail_get_dir() -> object: + raise RuntimeError(f"unsafe cache detail under {tmp_path}") + + fake_torch.hub = SimpleNamespace(get_dir=fail_get_dir) # type: ignore[attr-defined] + separator = AudioStemSeparator() + + with pytest.raises(ValueError, match="cache location is unavailable") as error: + separator._load_model() + assert str(tmp_path) not in str(error.value) + assert calls["torch_load_count"] == 0 + assert calls["demucs_load_count"] == 0 -def test_audio_stem_separator_resolves_verified_model_from_environment( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch + +def test_audio_stem_separator_uses_explicit_model_path_from_environment( + tmp_path, + monkeypatch: pytest.MonkeyPatch, ) -> None: - """Allow explicit environment provisioning without opening a remote model path.""" - model_bytes = b"environment model" - model_path = tmp_path / "955717e8-8726e21a.th" - model_path.write_bytes(model_bytes) - monkeypatch.setenv("BANDSCOPE_HTDEMUCS_MODEL_PATH", str(model_path)) - monkeypatch.setattr(audio_separator_module, "_HTDEMUCS_MODEL_BYTES", len(model_bytes)) - monkeypatch.setattr( - audio_separator_module, - "_HTDEMUCS_MODEL_SHA256", - __import__("hashlib").sha256(model_bytes).hexdigest(), - ) + """Bind operator-provided model paths to the same exact-byte loader.""" + payload = b"verified-model-package" + filename = "test-signature-deadbeef.th" + _patch_model_spec(monkeypatch, filename=filename, payload=payload) + artifact_path = tmp_path / filename + artifact_path.write_bytes(payload) + monkeypatch.setenv("BANDSCOPE_HTDEMUCS_MODEL_PATH", str(artifact_path)) + calls = _install_fake_verified_model_deserializer(monkeypatch) separator = AudioStemSeparator() - assert separator._verified_model_artifact_path() == model_path + assert separator._load_model() is not None + assert calls["payload"] == payload -def test_audio_stem_separator_resolves_verified_model_from_torch_home( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch +def test_audio_stem_separator_rejects_wrong_explicit_model_filename( + tmp_path, + monkeypatch: pytest.MonkeyPatch, ) -> None: - """Use the conventional local torch cache when no explicit path is configured.""" - model_bytes = b"torch home model" - model_path = tmp_path / "hub" / "checkpoints" / "955717e8-8726e21a.th" - model_path.parent.mkdir(parents=True) - model_path.write_bytes(model_bytes) - monkeypatch.delenv("BANDSCOPE_HTDEMUCS_MODEL_PATH", raising=False) - monkeypatch.setenv("TORCH_HOME", str(tmp_path)) - monkeypatch.setattr(audio_separator_module, "_HTDEMUCS_MODEL_BYTES", len(model_bytes)) - monkeypatch.setattr( - audio_separator_module, - "_HTDEMUCS_MODEL_SHA256", - __import__("hashlib").sha256(model_bytes).hexdigest(), + """Keep explicit paths bound to the inventoried checkpoint filename.""" + payload = b"verified-model-package" + _patch_model_spec(monkeypatch, filename="expected-model.th", payload=payload) + artifact_path = tmp_path / "substituted-model.th" + artifact_path.write_bytes(payload) + monkeypatch.setenv("BANDSCOPE_HTDEMUCS_MODEL_PATH", str(artifact_path)) + calls = _install_fake_verified_model_deserializer(monkeypatch) + + separator = AudioStemSeparator() + + with pytest.raises(ValueError, match="inventoried filename"): + separator._load_model() + assert calls["torch_load_count"] == 0 + + +def test_audio_stem_separator_rejects_uninventoried_model(tmp_path) -> None: + """Refuse arbitrary model names that have no exact artifact manifest.""" + separator = AudioStemSeparator( + AudioSeparationConfig( + model_name="untrusted-model", + model_cache_directory=tmp_path, + ) ) - assert AudioStemSeparator()._verified_model_artifact_path() == model_path + with pytest.raises(ValueError, match="not inventoried"): + separator._load_model() -@pytest.mark.parametrize("failure", ["symlink", "wrong-name", "directory", "wrong-size"]) -def test_audio_stem_separator_rejects_untrusted_model_path_shapes( - failure: str, tmp_path: Path +def test_audio_stem_separator_redacts_verified_model_load_errors( + tmp_path, + monkeypatch: pytest.MonkeyPatch, ) -> None: - """Reject path indirection, wrong names, non-files, and byte-count drift.""" - target = tmp_path / "955717e8-8726e21a.th" - if failure == "symlink": - real_model = tmp_path / "real-model.th" - real_model.write_bytes(b"model") - target.symlink_to(real_model) - elif failure == "wrong-name": - target = tmp_path / "renamed-model.th" - target.write_bytes(b"model") - elif failure == "directory": - target.mkdir() - else: - target.write_bytes(b"wrong size") - - separator = AudioStemSeparator(AudioSeparationConfig(model_artifact_path=target)) - - with pytest.raises(ValueError, match="model artifact"): - separator._verified_model_artifact_path() + """Surface a stable error when exact verified bytes still fail to deserialize.""" + payload = b"verified-model-package" + filename = "test-signature-deadbeef.th" + _patch_model_spec(monkeypatch, filename=filename, payload=payload) + (tmp_path / filename).write_bytes(payload) + + def fail_torch_load(*args: object, **kwargs: object) -> object: + raise RuntimeError(f"unsafe detail under {tmp_path}") + + _install_fake_verified_model_deserializer(monkeypatch, torch_load=fail_torch_load) + separator = AudioStemSeparator(AudioSeparationConfig(model_cache_directory=tmp_path)) + + with pytest.raises(ValueError, match="failed to load after integrity verification") as error: + separator._load_model() + assert str(tmp_path) not in str(error.value) + + +def _patch_demucs(monkeypatch: pytest.MonkeyPatch, per_source: dict | None = None) -> None: + """Patch the Demucs boundary so separation runs without the real model. + + ``per_source`` optionally maps a demucs source name to a mono numpy array to + return for that stem; unspecified sources return silence. + """ + + def fake_apply_model( + self: AudioStemSeparator, model: _FakeModel, audio: np.ndarray + ) -> dict[str, np.ndarray]: + samples = int(audio.size) + out = {name: np.zeros(samples, dtype=np.float32) for name in _DEMUCS_SOURCES} + if per_source: + for name in _DEMUCS_SOURCES: + if name in per_source: + row = per_source[name].astype(np.float32) + copy_length = min(samples, int(row.size)) + out[name][:copy_length] = row[:copy_length] + return out + + monkeypatch.setattr(AudioStemSeparator, "_load_model", lambda self: _FakeModel()) + monkeypatch.setattr(AudioStemSeparator, "_apply_model", fake_apply_model) def test_audio_stem_separator_splits_local_audio_into_canonical_stems( @@ -459,33 +612,32 @@ def test_audio_stem_separator_maps_demucs_sources_to_named_stems( def test_audio_stem_separator_caches_model(tmp_path, monkeypatch: pytest.MonkeyPatch) -> None: """Ensure the model is loaded once and reused across calls.""" - calls = {"n": 0} - - def fake_get_model(name: str, *, repo: Path) -> _FakeModel: - calls["n"] += 1 - return _FakeModel() + payload = b"verified-model-package" + filename = "test-signature-deadbeef.th" + _patch_model_spec(monkeypatch, filename=filename, payload=payload) + (tmp_path / filename).write_bytes(payload) + calls = _install_fake_verified_model_deserializer(monkeypatch) def fake_apply_model( self: AudioStemSeparator, model: _FakeModel, audio: np.ndarray ) -> dict[str, np.ndarray]: return {name: np.zeros(audio.size, dtype=np.float32) for name in _DEMUCS_SOURCES} - _install_fake_demucs(monkeypatch, fake_get_model) - monkeypatch.setattr( - AudioStemSeparator, - "_verified_model_artifact_path", - lambda self: Path("/verified/955717e8-8726e21a.th"), - ) monkeypatch.setattr(AudioStemSeparator, "_apply_model", fake_apply_model) audio_path = tmp_path / "mix.wav" sf.write(audio_path, np.zeros(4_000, dtype=np.float32), 8_000) separator = AudioStemSeparator( - AudioSeparationConfig(target_sample_rate=8_000, max_file_bytes=1_000_000) + AudioSeparationConfig( + target_sample_rate=8_000, + max_file_bytes=1_000_000, + model_cache_directory=tmp_path, + ) ) separator.separate(audio_path) separator.separate(audio_path) - assert calls["n"] == 1 + assert calls["torch_load_count"] == 1 + assert calls["demucs_load_count"] == 1 def test_audio_stem_separator_apply_model_uses_demucs_boundary( diff --git a/services/analysis-engine/tests/test_supply_chain_policy.py b/services/analysis-engine/tests/test_supply_chain_policy.py index 24507bf85..da20476a7 100644 --- a/services/analysis-engine/tests/test_supply_chain_policy.py +++ b/services/analysis-engine/tests/test_supply_chain_policy.py @@ -94,68 +94,227 @@ def test_supplemental_inventory_accepts_pinned_htdemucs_runtime_model() -> None: @pytest.mark.parametrize( - ("inventory", "expected"), + ("old", "new", "message"), [ - ([], "supplemental inventory root must be an object"), - ({"modelArtifacts": []}, "supplemental inventory missing runtime model: htdemucs"), ( - { - "modelArtifacts": [ - { - "name": "Hybrid Transformer Demucs four-source weights", - "runtimeModelName": "htdemucs", - "version": "4.0.1", - "sourceUrl": "https://models.example/htdemucs.th", - "license": "operator-reviewed", - "checksum": f"sha256:{'0' * 64}", - "sizeBytes": True, - "storagePath": "local cache", - "distribution": "runtime-cache", - "releaseUsage": "local separation", - "verification": "full digest before load", - } - ] - }, - "requires positive integer sizeBytes", + "955717e8-8726e21a.th", + "955717e8-00000000.th", + "filename does not match separator manifest", ), ( + "8726e21a993978c7ba086d3872e7608d7d5bfca646ca4aca459ffda844faa8b4", + "a" * 64, + "checksum does not match separator manifest", + ), + ("size_bytes=84_141_911", "size_bytes=84_141_912", "sizeBytes does not match"), + ], +) +def test_supplemental_inventory_rejects_separator_manifest_drift( + tmp_path: Path, + old: str, + new: str, + message: str, +) -> None: + """Cross-check code-owned model identity against the supplemental inventory.""" + supply_chain = load_module( + "scripts/checks/verify_supply_chain.py", + f"verify_supply_chain_model_drift_{message.split()[0]}", + ) + repo_root = Path(__file__).resolve().parents[3] + inventory_path = repo_root / "supply-chain" / "supplemental-component-inventory.json" + source_path = ( + repo_root + / "services" + / "analysis-engine" + / "src" + / "bandscope_analysis" + / "separation" + / "audio_separator.py" + ) + drifted_source = source_path.read_text(encoding="utf-8").replace(old, new, 1) + drifted_path = tmp_path / "audio_separator.py" + drifted_path.write_text(drifted_source, encoding="utf-8") + + violations = supply_chain.supplemental_inventory_violations( + inventory_path, + drifted_path, + ) + + assert any(message in violation for violation in violations) + + +def test_supplemental_inventory_rejects_tool_and_nonruntime_model_drift( + tmp_path: Path, +) -> None: + """Bind yt-dlp to uv.lock, require both media tools, and validate every model.""" + supply_chain = load_module( + "scripts/checks/verify_supply_chain.py", + "verify_supply_chain_tool_inventory_drift", + ) + repo_root = Path(__file__).resolve().parents[3] + inventory = json.loads( + (repo_root / "supply-chain" / "supplemental-component-inventory.json").read_text( + encoding="utf-8" + ) + ) + inventory["packageManagedTools"][0]["version"] = "2026.7.3" + inventory["operatorProvidedTools"] = [ + tool for tool in inventory["operatorProvidedTools"] if tool["name"] != "ffprobe" + ] + auxiliary_model = dict(inventory["modelArtifacts"][0]) + auxiliary_model.update( + { + "name": "Auxiliary test model", + "runtimeModelName": "auxiliary-model", + "version": "test-signature", + "sizeBytes": True, + } + ) + inventory["modelArtifacts"].append(auxiliary_model) + inventory_path = tmp_path / "inventory.json" + inventory_path.write_text(json.dumps(inventory), encoding="utf-8") + + violations = supply_chain.supplemental_inventory_violations( + inventory_path, + repo_root + / "services" + / "analysis-engine" + / "src" + / "bandscope_analysis" + / "separation" + / "audio_separator.py", + repo_root / "services" / "analysis-engine" / "uv.lock", + ) + + assert "supplemental inventory yt-dlp version does not match uv.lock" in violations + assert "supplemental inventory missing operator tool: ffprobe" in violations + assert ( + "supplemental inventory runtime model auxiliary-model requires positive sizeBytes" + in violations + ) + + +def test_supplemental_inventory_rejects_non_object_root(tmp_path: Path) -> None: + """Diagnose an array-valued inventory instead of raising an attribute error.""" + supply_chain = load_module( + "scripts/checks/verify_supply_chain.py", + "verify_supply_chain_model_inventory_root_type", + ) + inventory_path = tmp_path / "inventory.json" + inventory_path.write_text("[]", encoding="utf-8") + separator_path = tmp_path / "audio_separator.py" + separator_path.write_text('model_name: str = "htdemucs"\n', encoding="utf-8") + + assert supply_chain.supplemental_inventory_violations( + inventory_path, + separator_path, + ) == ["supplemental inventory must be an object"] + + +def test_supplemental_inventory_rejects_empty_model_artifacts(tmp_path: Path) -> None: + """Reject an object that carries no artifact for the configured runtime model.""" + supply_chain = load_module( + "scripts/checks/verify_supply_chain.py", + "verify_supply_chain_model_inventory_empty", + ) + inventory_path = tmp_path / "inventory.json" + inventory_path.write_text('{"modelArtifacts": []}', encoding="utf-8") + separator_path = tmp_path / "audio_separator.py" + separator_path.write_text('model_name: str = "htdemucs"\n', encoding="utf-8") + + assert supply_chain.supplemental_inventory_violations( + inventory_path, + separator_path, + ) == ["supplemental inventory modelArtifacts must not be empty"] + + +def test_supplemental_inventory_rejects_boolean_size_and_invalid_fields( + tmp_path: Path, +) -> None: + """Require real integer sizes and non-empty typed artifact metadata.""" + supply_chain = load_module( + "scripts/checks/verify_supply_chain.py", + "verify_supply_chain_model_inventory_field_types", + ) + inventory_path = tmp_path / "inventory.json" + inventory_path.write_text( + json.dumps( { "modelArtifacts": [ { - "name": 7, + "name": "Hybrid Transformer Demucs", "runtimeModelName": "htdemucs", - "version": "4.0.1", + "version": "", "sourceUrl": "https://models.example/htdemucs.th", - "license": "operator-reviewed", - "checksum": f"sha256:{'0' * 64}", - "sizeBytes": 7, - "storagePath": "local cache", + "license": [], + "checksum": "sha256:" + ("a" * 64), + "sizeBytes": True, + "storagePath": "cache/checkpoints", "distribution": "runtime-cache", "releaseUsage": "local separation", - "verification": "full digest before load", + "verification": "", } ] - }, - "field name must be a non-empty string", + } ), - ], -) -def test_supplemental_inventory_rejects_malformed_schema( - tmp_path: Path, inventory: object, expected: str -) -> None: - """Return stable diagnostics for untrusted inventory shapes and field types.""" - supply_chain = load_module( - "scripts/checks/verify_supply_chain.py", - f"verify_supply_chain_malformed_{abs(hash(expected))}", + encoding="utf-8", ) - inventory_path = tmp_path / "inventory.json" - inventory_path.write_text(json.dumps(inventory), encoding="utf-8") separator_path = tmp_path / "audio_separator.py" separator_path.write_text('model_name: str = "htdemucs"\n', encoding="utf-8") - violations = supply_chain.supplemental_inventory_violations(inventory_path, separator_path) + violations = supply_chain.supplemental_inventory_violations( + inventory_path, + separator_path, + ) + + assert "supplemental inventory runtime model htdemucs requires positive sizeBytes" in violations + assert ( + "supplemental inventory runtime model htdemucs requires non-empty string field: version" + in violations + ) + assert ( + "supplemental inventory runtime model htdemucs requires non-empty string field: license" + in violations + ) + assert ( + "supplemental inventory runtime model htdemucs requires non-empty string field: " + "verification" in violations + ) + + +def test_security_pattern_gate_accepts_only_verified_model_deserialization() -> None: + """Accept the exact verified checkpoint call while retaining the general pickle ban.""" + security_gates = load_module( + "scripts/checks/security_gates.py", + "security_gates_verified_model_repo", + ) + repo_root = Path(__file__).resolve().parents[3] + + assert security_gates.security_pattern_violations(repo_root) == [] + - assert any(expected in violation for violation in violations) +def test_security_pattern_gate_rejects_second_model_deserialization(tmp_path: Path) -> None: + """Do not let the narrow verified-checkpoint rule hide another torch load site.""" + security_gates = load_module( + "scripts/checks/security_gates.py", + "security_gates_second_model_load", + ) + repo_root = Path(__file__).resolve().parents[3] + source_path = repo_root / security_gates.VERIFIED_MODEL_LOADER_PATH + target_path = tmp_path / security_gates.VERIFIED_MODEL_LOADER_PATH + target_path.parent.mkdir(parents=True) + second_load = "\ntorch." + "load(untrusted_checkpoint)\n" + target_path.write_text( + source_path.read_text(encoding="utf-8") + second_load, + encoding="utf-8", + ) + + violations = security_gates.security_pattern_violations(tmp_path) + + assert violations == [ + f"{security_gates.VERIFIED_MODEL_LOADER_PATH}: " + "Do not load untrusted pickle-style artifacts without a documented trust boundary." + ] def central_required_workflow_policy_text() -> str: diff --git a/services/analysis-engine/tests/test_youtube.py b/services/analysis-engine/tests/test_youtube.py index a5c22e48e..a286b0a34 100644 --- a/services/analysis-engine/tests/test_youtube.py +++ b/services/analysis-engine/tests/test_youtube.py @@ -2,6 +2,8 @@ import hashlib import importlib +import os +import ssl import sys from pathlib import Path from unittest.mock import MagicMock, patch @@ -11,7 +13,7 @@ from bandscope_analysis.youtube import ( MAX_YOUTUBE_URL_LENGTH, - _verified_ffmpeg_location, + _verify_executable_artifact, download_youtube_audio, validate_url, ) @@ -89,8 +91,13 @@ def test_download_youtube_audio_success( mock_ydl_class: MagicMock, mock_exists: MagicMock, mock_getsize: MagicMock, + monkeypatch: pytest.MonkeyPatch, ) -> None: """Test successful download.""" + ssl_context = MagicMock() + ssl_context.get_ca_certs.return_value = [b"managed-ca"] + monkeypatch.setattr(ssl, "create_default_context", lambda: ssl_context) + mock_ydl = MagicMock() mock_ydl_class.return_value.__enter__.return_value = mock_ydl @@ -122,8 +129,7 @@ def test_download_youtube_audio_success( assert called_opts["noprogress"] is True assert called_opts["noplaylist"] is True assert called_opts["geo_bypass"] is False - assert "compat_opts" not in called_opts - assert "nocheckcertificate" not in called_opts + assert called_opts["compat_opts"] == {"no-certifi"} assert called_opts["postprocessors"] == [{"key": "FFmpegExtractAudio"}] assert "%(id)s.%(ext)s" in called_opts["outtmpl"] @@ -139,23 +145,104 @@ def test_download_youtube_audio_success( ) +@patch("bandscope_analysis.youtube.yt_dlp.YoutubeDL") +def test_download_youtube_audio_uses_system_ca_only_when_populated( + mock_ydl_class: MagicMock, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Use OS-managed roots only after confirming the trust store is populated.""" + ssl_context = MagicMock() + ssl_context.get_ca_certs.return_value = [b"managed-ca"] + monkeypatch.setattr(ssl, "create_default_context", lambda: ssl_context) + mock_ydl = MagicMock() + mock_ydl_class.return_value.__enter__.return_value = mock_ydl + mock_ydl.extract_info.return_value = {"id": "abc123DEF45", "duration": 16 * 60} + + result = download_youtube_audio("https://youtube.com/watch?v=abc123DEF45", "/tmp") + + assert result["error"]["code"] == "duration_exceeded" + options = mock_ydl_class.call_args.args[0] + assert options["compat_opts"] == {"no-certifi"} + + +@patch("bandscope_analysis.youtube.yt_dlp.YoutubeDL") +def test_download_youtube_audio_keeps_ytdlp_ca_fallback_for_empty_system_store( + mock_ydl_class: MagicMock, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Retain yt-dlp's certifi fallback when no system roots are available.""" + ssl_context = MagicMock() + ssl_context.get_ca_certs.return_value = [] + monkeypatch.setattr(ssl, "create_default_context", lambda: ssl_context) + mock_ydl = MagicMock() + mock_ydl_class.return_value.__enter__.return_value = mock_ydl + mock_ydl.extract_info.return_value = {"id": "abc123DEF45", "duration": 16 * 60} + + result = download_youtube_audio("https://youtube.com/watch?v=abc123DEF45", "/tmp") + + assert result["error"]["code"] == "duration_exceeded" + options = mock_ydl_class.call_args.args[0] + assert "compat_opts" not in options + + +@patch("bandscope_analysis.youtube.yt_dlp.YoutubeDL") +def test_download_youtube_audio_keeps_ytdlp_ca_fallback_when_store_probe_fails( + mock_ydl_class: MagicMock, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Treat trust-store probe errors as unavailable roots rather than disabling TLS.""" + + def fail_to_create_context() -> ssl.SSLContext: + raise RuntimeError("host trust store unavailable") + + monkeypatch.setattr(ssl, "create_default_context", fail_to_create_context) + mock_ydl = MagicMock() + mock_ydl_class.return_value.__enter__.return_value = mock_ydl + mock_ydl.extract_info.return_value = {"id": "abc123DEF45", "duration": 16 * 60} + + result = download_youtube_audio("https://youtube.com/watch?v=abc123DEF45", "/tmp") + + assert result["error"]["code"] == "duration_exceeded" + options = mock_ydl_class.call_args.args[0] + assert "compat_opts" not in options + + +def _executable_file(path: Path, contents: bytes) -> str: + """Create a regular executable test artifact and return its SHA-256 digest.""" + path.write_bytes(contents) + path.chmod(0o700) + return hashlib.sha256(contents).hexdigest() + + +def test_media_runtime_executable_identity_requires_typed_pair() -> None: + """Reject a missing path/digest before attempting filesystem access.""" + assert _verify_executable_artifact(None, None) is None + + +def _verified_media_runtime(tmp_path: Path, suffix: str = "") -> dict[str, str]: + """Create sibling ffmpeg/ffprobe artifacts and return their exact identities.""" + ffmpeg = tmp_path / f"ffmpeg{suffix}" + ffprobe = tmp_path / f"ffprobe{suffix}" + return { + "ffmpeg_path": str(ffmpeg), + "ffmpeg_sha256": _executable_file(ffmpeg, b"trusted ffmpeg artifact"), + "ffprobe_path": str(ffprobe), + "ffprobe_sha256": _executable_file(ffprobe, b"trusted ffprobe artifact"), + } + + @patch("bandscope_analysis.youtube.os.path.getsize") @patch("bandscope_analysis.youtube.os.path.exists") @patch("bandscope_analysis.youtube.yt_dlp.YoutubeDL") -def test_download_uses_verified_absolute_ffmpeg_when_release_identity_is_configured( +def test_download_youtube_audio_passes_verified_ffmpeg_path_to_ytdlp( mock_ydl_class: MagicMock, mock_exists: MagicMock, mock_getsize: MagicMock, tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, ) -> None: - """Bind yt-dlp post-processing to exact operator-provisioned executable bytes.""" - ffmpeg_path = tmp_path / "ffmpeg" - ffmpeg_bytes = b"pinned ffmpeg executable" - ffmpeg_path.write_bytes(ffmpeg_bytes) - ffmpeg_path.chmod(0o700) - monkeypatch.setenv("BANDSCOPE_FFMPEG_PATH", str(ffmpeg_path)) - monkeypatch.setenv("BANDSCOPE_FFMPEG_SHA256", hashlib.sha256(ffmpeg_bytes).hexdigest()) + """Verify the complete media executable set before handing it to yt-dlp.""" + suffix = ".exe" if os.name == "nt" else "" + runtime = _verified_media_runtime(tmp_path, suffix) mock_ydl = MagicMock() mock_ydl_class.return_value.__enter__.return_value = mock_ydl mock_ydl.extract_info.return_value = { @@ -165,81 +252,215 @@ def test_download_uses_verified_absolute_ffmpeg_when_release_identity_is_configu } mock_ydl.prepare_filename.return_value = "/tmp/abc123DEF45.webm" mock_exists.return_value = True - mock_getsize.return_value = 10 + mock_getsize.return_value = 1024 - result = download_youtube_audio("https://youtube.com/watch?v=abc123DEF45", "/tmp") + result = download_youtube_audio( + "https://youtube.com/watch?v=abc123DEF45", + "/tmp", + **runtime, + ) assert result["ok"] is True - called_opts = mock_ydl_class.call_args[0][0] - assert called_opts["ffmpeg_location"] == str(ffmpeg_path.resolve()) + options = mock_ydl_class.call_args.args[0] + assert options["ffmpeg_location"] == str((tmp_path / f"ffmpeg{suffix}").resolve()) +@pytest.mark.parametrize( + "runtime", + [ + {"ffmpeg_path": "/opt/bandscope/ffmpeg"}, + {"ffmpeg_sha256": "0" * 64}, + { + "ffmpeg_path": "/opt/bandscope/ffmpeg", + "ffmpeg_sha256": "0" * 64, + "ffprobe_path": "/opt/bandscope/ffprobe", + }, + ], +) @patch("bandscope_analysis.youtube.yt_dlp.YoutubeDL") -def test_download_rejects_changed_ffmpeg_before_provider_access( - mock_ydl_class: MagicMock, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +def test_download_youtube_audio_rejects_partial_media_runtime_identity( + mock_ydl_class: MagicMock, + runtime: dict[str, str], ) -> None: - """Fail closed when configured release evidence names changed executable bytes.""" - ffmpeg_path = tmp_path / "ffmpeg" - ffmpeg_path.write_bytes(b"changed executable") - ffmpeg_path.chmod(0o700) - monkeypatch.setenv("BANDSCOPE_FFMPEG_PATH", str(ffmpeg_path)) - monkeypatch.setenv("BANDSCOPE_FFMPEG_SHA256", "0" * 64) - - result = download_youtube_audio("https://youtube.com/watch?v=abc123DEF45", "/tmp") + """Reject a configured runtime unless all four identity fields are present.""" + result = download_youtube_audio( + "https://youtube.com/watch?v=abc123DEF45", + "/tmp", + **runtime, + ) assert result == { "ok": False, "error": { - "code": "download_error", - "message": "YouTube import failed. Please use a local audio file instead.", + "code": "runtime_dependency_invalid", + "message": "The configured media runtime failed identity verification.", }, } mock_ydl_class.assert_not_called() -@pytest.mark.parametrize( - ("path", "digest", "message"), - [ - (None, "0" * 64, "requires both path and SHA-256"), - ("ffmpeg", None, "requires both path and SHA-256"), - ("ffmpeg", "not-a-digest", "full lowercase SHA-256"), - ("ffmpeg", "0" * 64, "absolute executable path"), - ], -) -def test_verified_ffmpeg_rejects_incomplete_or_relative_identity( - path: str | None, - digest: str | None, - message: str, +@pytest.mark.parametrize("invalid_hash", ["0" * 63, "A" * 64, "not-a-sha256"]) +@patch("bandscope_analysis.youtube.yt_dlp.YoutubeDL") +def test_download_youtube_audio_rejects_malformed_ffmpeg_hash( + mock_ydl_class: MagicMock, + invalid_hash: str, + tmp_path: Path, +) -> None: + """Require the canonical full lowercase SHA-256 representation.""" + runtime = _verified_media_runtime(tmp_path) + runtime["ffmpeg_sha256"] = invalid_hash + + result = download_youtube_audio( + "https://youtube.com/watch?v=abc123DEF45", + "/tmp", + **runtime, + ) + + assert result["error"]["code"] == "runtime_dependency_invalid" + mock_ydl_class.assert_not_called() + + +@pytest.mark.parametrize("artifact_kind", ["relative", "missing", "directory", "non_executable"]) +@patch("bandscope_analysis.youtube.yt_dlp.YoutubeDL") +def test_download_youtube_audio_rejects_invalid_ffmpeg_artifact( + mock_ydl_class: MagicMock, + artifact_kind: str, + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: - """Reject incomplete, malformed, or relative release executable identities.""" - if path is None: - monkeypatch.delenv("BANDSCOPE_FFMPEG_PATH", raising=False) + """Reject ffmpeg paths that cannot identify a fixed regular executable file.""" + runtime = _verified_media_runtime(tmp_path) + if artifact_kind == "relative": + ffmpeg = Path("ffmpeg") + elif artifact_kind == "missing": + ffmpeg = tmp_path / "missing-ffmpeg" + elif artifact_kind == "directory": + ffmpeg = tmp_path else: - monkeypatch.setenv("BANDSCOPE_FFMPEG_PATH", path) - if digest is None: - monkeypatch.delenv("BANDSCOPE_FFMPEG_SHA256", raising=False) + ffmpeg = tmp_path / "ffmpeg" + ffmpeg.write_bytes(b"not executable") + monkeypatch.setattr(os, "access", lambda *_args: False) + + runtime["ffmpeg_path"] = str(ffmpeg) + runtime["ffmpeg_sha256"] = "0" * 64 + result = download_youtube_audio( + "https://youtube.com/watch?v=abc123DEF45", + "/tmp", + **runtime, + ) + + assert result["error"]["code"] == "runtime_dependency_invalid" + mock_ydl_class.assert_not_called() + + +@patch("bandscope_analysis.youtube.yt_dlp.YoutubeDL") +def test_download_youtube_audio_rejects_symlinked_ffmpeg( + mock_ydl_class: MagicMock, + tmp_path: Path, +) -> None: + """Reject a replaceable symlink at the configured executable boundary.""" + target = tmp_path / "real-ffmpeg" + expected_hash = _executable_file(target, b"trusted ffmpeg artifact") + ffprobe = tmp_path / "ffprobe" + ffprobe_hash = _executable_file(ffprobe, b"trusted ffprobe artifact") + ffmpeg = tmp_path / "ffmpeg" + try: + ffmpeg.symlink_to(target) + except OSError: + pytest.skip("symlink creation is unavailable on this platform") + + result = download_youtube_audio( + "https://youtube.com/watch?v=abc123DEF45", + "/tmp", + ffmpeg_path=str(ffmpeg), + ffmpeg_sha256=expected_hash, + ffprobe_path=str(ffprobe), + ffprobe_sha256=ffprobe_hash, + ) + + assert result["error"]["code"] == "runtime_dependency_invalid" + mock_ydl_class.assert_not_called() + + +@patch("bandscope_analysis.youtube.yt_dlp.YoutubeDL") +def test_download_youtube_audio_rejects_ffmpeg_hash_mismatch( + mock_ydl_class: MagicMock, + tmp_path: Path, +) -> None: + """Fail closed before yt-dlp when the executable bytes do not match the manifest.""" + runtime = _verified_media_runtime(tmp_path) + runtime["ffmpeg_sha256"] = "0" * 64 + + result = download_youtube_audio( + "https://youtube.com/watch?v=abc123DEF45", + "/tmp", + **runtime, + ) + + assert result["error"]["code"] == "runtime_dependency_invalid" + mock_ydl_class.assert_not_called() + + +@pytest.mark.parametrize("failure", ["probe_hash", "probe_name", "probe_directory", "ffmpeg_name"]) +@patch("bandscope_analysis.youtube.yt_dlp.YoutubeDL") +def test_download_youtube_audio_rejects_unverified_executable_set( + mock_ydl_class: MagicMock, + failure: str, + tmp_path: Path, +) -> None: + """Authenticate every executable yt-dlp may derive from ffmpeg_location.""" + runtime = _verified_media_runtime(tmp_path) + if failure == "probe_hash": + runtime["ffprobe_sha256"] = "0" * 64 + elif failure == "probe_name": + wrong_probe = tmp_path / "media-probe" + runtime["ffprobe_path"] = str(wrong_probe) + runtime["ffprobe_sha256"] = _executable_file(wrong_probe, b"trusted probe") + elif failure == "probe_directory": + probe_directory = tmp_path / "probe-bin" + probe_directory.mkdir() + wrong_probe = probe_directory / "ffprobe" + runtime["ffprobe_path"] = str(wrong_probe) + runtime["ffprobe_sha256"] = _executable_file(wrong_probe, b"trusted probe") else: - monkeypatch.setenv("BANDSCOPE_FFMPEG_SHA256", digest) + wrong_ffmpeg = tmp_path / "media-converter" + runtime["ffmpeg_path"] = str(wrong_ffmpeg) + runtime["ffmpeg_sha256"] = _executable_file(wrong_ffmpeg, b"trusted converter") + + result = download_youtube_audio( + "https://youtube.com/watch?v=abc123DEF45", + "/tmp", + **runtime, + ) - with pytest.raises(ValueError, match=message): - _verified_ffmpeg_location() + assert result["error"]["code"] == "runtime_dependency_invalid" + mock_ydl_class.assert_not_called() -@pytest.mark.parametrize("failure", ["missing", "not-executable"]) -def test_verified_ffmpeg_rejects_unavailable_absolute_executable( - failure: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +@patch("bandscope_analysis.youtube.yt_dlp.YoutubeDL") +def test_download_youtube_audio_rejects_case_mismatched_program_names( + mock_ydl_class: MagicMock, + tmp_path: Path, ) -> None: - """Reject absent or non-executable absolute ffmpeg candidates.""" - ffmpeg_path = tmp_path / "ffmpeg" - if failure == "not-executable": - ffmpeg_path.write_bytes(b"ffmpeg") - ffmpeg_path.chmod(0o600) - monkeypatch.setenv("BANDSCOPE_FFMPEG_PATH", str(ffmpeg_path)) - monkeypatch.setenv("BANDSCOPE_FFMPEG_SHA256", "0" * 64) + """Require the exact sibling names yt-dlp derives on the active platform.""" + suffix = ".EXE" if os.name == "nt" else "" + ffmpeg = tmp_path / f"FFMPEG{suffix}" + ffprobe = tmp_path / f"FFPROBE{suffix}" + runtime = { + "ffmpeg_path": str(ffmpeg), + "ffmpeg_sha256": _executable_file(ffmpeg, b"trusted ffmpeg artifact"), + "ffprobe_path": str(ffprobe), + "ffprobe_sha256": _executable_file(ffprobe, b"trusted ffprobe artifact"), + } + + result = download_youtube_audio( + "https://youtube.com/watch?v=abc123DEF45", + "/tmp", + **runtime, + ) - with pytest.raises(ValueError, match="executable is unavailable"): - _verified_ffmpeg_location() + assert result["error"]["code"] == "runtime_dependency_invalid" + mock_ydl_class.assert_not_called() @patch("bandscope_analysis.youtube.os.path.getsize") @@ -419,6 +640,14 @@ def test_main_block(monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixtu "https://youtube.com/watch?v=abc123DEF45", "--out-dir", "/tmp", + "--ffmpeg-path", + "/opt/bandscope/ffmpeg", + "--ffmpeg-sha256", + "a" * 64, + "--ffprobe-path", + "/opt/bandscope/ffprobe", + "--ffprobe-sha256", + "b" * 64, ] monkeypatch.setattr(sys, "argv", test_args) @@ -431,6 +660,14 @@ def test_main_block(monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixtu with patch.object(sys, "exit") as mock_exit: bandscope_analysis.youtube.main() + mock_download.assert_called_with( + "https://youtube.com/watch?v=abc123DEF45", + "/tmp", + ffmpeg_path="/opt/bandscope/ffmpeg", + ffmpeg_sha256="a" * 64, + ffprobe_path="/opt/bandscope/ffprobe", + ffprobe_sha256="b" * 64, + ) mock_exit.assert_called_with(0) # test failure exit 1 @@ -467,7 +704,6 @@ def test_module_execution( # Mock os to ensure runpy uses our mocked filesystem methods mock_os = MagicMock() # Keep some essential attributes - mock_os.environ = {} mock_os.path = MagicMock() mock_os.path.exists.return_value = True mock_os.path.getsize.return_value = 10 * 1024 * 1024 diff --git a/services/analysis-engine/tests/test_youtube_stem_e2e.py b/services/analysis-engine/tests/test_youtube_stem_e2e.py index 27291e5ca..ce9a291ee 100644 --- a/services/analysis-engine/tests/test_youtube_stem_e2e.py +++ b/services/analysis-engine/tests/test_youtube_stem_e2e.py @@ -5,6 +5,7 @@ import hashlib import io import os +import sys import tempfile import zipfile from dataclasses import replace @@ -22,7 +23,6 @@ MIN_VOCAL_SI_SDR_IMPROVEMENT_DB, KnownStemFixture, _AllowlistedRedirectHandler, - _normalized_correlation, align_active_reference_window, align_known_stem_through_master, download_verified_creator_master, @@ -35,7 +35,7 @@ AudioSeparationConfig, AudioStemSeparator, ) -from bandscope_analysis.youtube import _verified_ffmpeg_location, download_youtube_audio +from bandscope_analysis.youtube import _verify_media_runtime, download_youtube_audio class _FakeResponse(io.BytesIO): @@ -147,42 +147,79 @@ def test_align_active_reference_window_recovers_delay_and_loud_section() -> None assert aligned.correlation > 0.99 -def test_identity_correlation_preserves_phase_sign() -> None: - """Do not authenticate a phase-inverted candidate as the same recording.""" - signal = np.array([-2.0, -0.5, 0.5, 2.0], dtype=np.float64) +def test_align_active_reference_window_is_polarity_invariant() -> None: + """Treat an inverted but otherwise identical waveform as the same audio.""" + rng = np.random.default_rng(20260810) + reference = rng.standard_normal(2_000) + mixture = np.concatenate((np.zeros(73), -reference, np.zeros(27))) - assert _normalized_correlation(signal, -signal) == pytest.approx(-1.0) + aligned = align_active_reference_window( + mixture, + reference, + sample_rate=1_000, + window_seconds=0.8, + max_lag_seconds=0.2, + ) + + assert aligned.lag_samples == 73 + assert aligned.correlation > 0.999 @pytest.mark.parametrize( - ("kwargs", "message"), + ("overrides", "message"), [ - ({"sample_rate": 0}, "sample_rate must be positive"), - ({"window_seconds": 0.0}, "alignment durations are invalid"), - ({"max_lag_seconds": -0.1}, "alignment durations are invalid"), - ({"envelope_hop_seconds": 0.0}, "alignment resolution is invalid"), - ({"refinement_seconds": -0.1}, "alignment resolution is invalid"), - ({"window_seconds": 2.0}, "reference is shorter"), + ({"sample_rate": 0}, "sample_rate"), + ({"window_seconds": 0.0}, "durations"), + ({"max_lag_seconds": -0.1}, "durations"), + ({"envelope_hop_seconds": 0.0}, "resolution"), + ({"refinement_seconds": -0.1}, "resolution"), ], ) -def test_align_active_reference_window_rejects_invalid_contract( - kwargs: dict[str, float | int], message: str +def test_align_active_reference_window_rejects_invalid_configuration( + overrides: dict[str, float | int], + message: str, ) -> None: - """Exercise every caller-controlled alignment validation family.""" - parameters: dict[str, float | int] = { - "sample_rate": 10, + """Reject invalid rate and duration settings before attempting alignment.""" + arguments: dict[str, float | int] = { + "sample_rate": 1_000, "window_seconds": 0.5, "max_lag_seconds": 0.2, - "envelope_hop_seconds": 0.1, - "refinement_seconds": 0.1, } - parameters.update(kwargs) + arguments.update(overrides) with pytest.raises(ValueError, match=message): align_active_reference_window( - np.arange(10, dtype=np.float64), - np.arange(10, dtype=np.float64), - **parameters, + np.ones(1_000), + np.ones(1_000), + **arguments, + ) + + +def test_align_active_reference_window_rejects_short_reference() -> None: + """Require enough reference samples for the complete scored window.""" + with pytest.raises(ValueError, match="reference is shorter"): + align_active_reference_window( + np.ones(1_000), + np.ones(100), + sample_rate=1_000, + window_seconds=0.5, + max_lag_seconds=0.2, + ) + + +def test_align_active_reference_window_rejects_nonoverlapping_mixture() -> None: + """Reject a mixture that cannot supply one complete aligned scoring window.""" + reference = np.zeros(1_000) + reference[500:] = np.linspace(-1.0, 1.0, 500) + + with pytest.raises(ValueError, match="does not overlap"): + align_active_reference_window( + np.ones(100), + reference, + sample_rate=1_000, + window_seconds=0.5, + max_lag_seconds=0.0, + refinement_seconds=0.0, ) @@ -361,35 +398,83 @@ def test_required_root_suite_explicitly_excludes_live_youtube_marker() -> None: repo_root = Path(__file__).resolve().parents[3] runner = (repo_root / "scripts/checks/run_root_tests.mjs").read_text(encoding="utf-8") - normalized = " ".join(runner.split()) - assert '"-m", "not youtube_stem_e2e"' in normalized + normalized_runner = " ".join(runner.split()) + assert '"-m", "not youtube_stem_e2e"' in normalized_runner -def test_live_benchmark_requires_verified_ffmpeg_before_fixture_access( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch +@pytest.mark.parametrize("identity_state", ["missing", "invalid"]) +def test_live_benchmark_verifies_media_runtime_before_fixture_access( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + identity_state: str, ) -> None: - """Fail closed before network access when executable identity is not configured.""" - monkeypatch.delenv("BANDSCOPE_FFMPEG_PATH", raising=False) - monkeypatch.delenv("BANDSCOPE_FFMPEG_SHA256", raising=False) + """Fail closed before reference network access for incomplete or untrusted tools.""" + variable_names = ( + "BANDSCOPE_FFMPEG_PATH", + "BANDSCOPE_FFMPEG_SHA256", + "BANDSCOPE_FFPROBE_PATH", + "BANDSCOPE_FFPROBE_SHA256", + ) + if identity_state == "missing": + for variable_name in variable_names: + monkeypatch.delenv(variable_name, raising=False) + else: + monkeypatch.setenv("BANDSCOPE_FFMPEG_PATH", str(tmp_path / "missing-ffmpeg")) + monkeypatch.setenv("BANDSCOPE_FFMPEG_SHA256", "0" * 64) + monkeypatch.setenv("BANDSCOPE_FFPROBE_PATH", str(tmp_path / "missing-ffprobe")) + monkeypatch.setenv("BANDSCOPE_FFPROBE_SHA256", "0" * 64) + + fixture_accesses: list[str] = [] + + def reject_fixture_access(*_args: object, **_kwargs: object) -> Path: + fixture_accesses.append("reference") + raise AssertionError("fixture access occurred before runtime preflight") + + monkeypatch.setattr( + sys.modules[__name__], + "download_verified_reference_stem", + reject_fixture_access, + ) - with pytest.raises(AssertionError, match="verified ffmpeg identity"): + with pytest.raises(AssertionError, match="ffmpeg and ffprobe"): _assert_real_youtube_known_stem_separation(tmp_path) + assert fixture_accesses == [] + def _assert_real_youtube_known_stem_separation(root: Path) -> None: """Run the live benchmark inside an ephemeral, caller-owned media directory.""" - ffmpeg_location = _verified_ffmpeg_location() - assert ffmpeg_location is not None, ( - "Live benchmark requires verified ffmpeg identity via " - "BANDSCOPE_FFMPEG_PATH and BANDSCOPE_FFMPEG_SHA256" - ) fixture = BRAD_SUCKS_FIXTURE + ffmpeg_path = os.environ.get("BANDSCOPE_FFMPEG_PATH") + ffmpeg_sha256 = os.environ.get("BANDSCOPE_FFMPEG_SHA256") + ffprobe_path = os.environ.get("BANDSCOPE_FFPROBE_PATH") + ffprobe_sha256 = os.environ.get("BANDSCOPE_FFPROBE_SHA256") + assert ffmpeg_path and ffmpeg_sha256 and ffprobe_path and ffprobe_sha256, ( + "Live evidence requires exact ffmpeg and ffprobe path/SHA-256 identities" + ) + runtime_is_valid, verified_ffmpeg_path = _verify_media_runtime( + ffmpeg_path, + ffmpeg_sha256, + ffprobe_path, + ffprobe_sha256, + ) + assert runtime_is_valid and verified_ffmpeg_path is not None, ( + "Live evidence requires verified ffmpeg and ffprobe executable identities" + ) + reference_path = download_verified_reference_stem(fixture, root) master_path = download_verified_creator_master(fixture, root) youtube_dir = root / "youtube" youtube_dir.mkdir() - download = download_youtube_audio(fixture.youtube_url, str(youtube_dir)) + download = download_youtube_audio( + fixture.youtube_url, + str(youtube_dir), + ffmpeg_path=ffmpeg_path, + ffmpeg_sha256=ffmpeg_sha256, + ffprobe_path=ffprobe_path, + ffprobe_sha256=ffprobe_sha256, + ) assert download["ok"], f"YouTube fixture failed: {download.get('error', {}).get('code')}" metadata = download["metadata"] assert metadata["id"] == fixture.video_id @@ -467,8 +552,10 @@ def _assert_real_youtube_known_stem_separation(root: Path) -> None: @pytest.mark.skipif( os.environ.get("BANDSCOPE_RUN_YOUTUBE_STEM_E2E") != "1", reason=( - "live YouTube, the pinned public stem archive, ffmpeg, and Demucs weights are required; " - "set BANDSCOPE_RUN_YOUTUBE_STEM_E2E=1" + "live YouTube, the pinned public stem archive, the verified ffmpeg/ffprobe set, and " + "Demucs weights are required; " + "set BANDSCOPE_RUN_YOUTUBE_STEM_E2E=1, BANDSCOPE_FFMPEG_PATH, and " + "the ffmpeg/ffprobe SHA-256 identity variables" ), ) def test_real_youtube_audio_separates_the_known_vocal_stem(tmp_path: Path) -> None: diff --git a/supply-chain/supplemental-component-inventory.json b/supply-chain/supplemental-component-inventory.json index e4790fcec..57ac1448b 100644 --- a/supply-chain/supplemental-component-inventory.json +++ b/supply-chain/supplemental-component-inventory.json @@ -5,6 +5,7 @@ { "name": "yt-dlp", "version": "2026.7.4", + "minimumVersion": "2026.7.4", "sourceUrl": "https://pypi.org/project/yt-dlp/", "license": "Unlicense", "storagePath": "services/analysis-engine/uv.lock", @@ -18,9 +19,18 @@ "version": "operator-managed supported release", "sourceUrl": "https://ffmpeg.org/download.html", "license": "LGPL-2.1-or-later or GPL-2.0-or-later, depending on build configuration", - "storagePath": "operator-provided absolute path; BANDSCOPE_FFMPEG_PATH and BANDSCOPE_FFMPEG_SHA256 bind release evidence; not bundled", + "storagePath": "operator-configured absolute executable path; not bundled by BandScope", "distribution": "operator-provided", - "releaseUsage": "Required by yt-dlp audio extraction and media decoding; release preflight records the resolved version." + "releaseUsage": "Required by yt-dlp audio extraction and media decoding; release/live preflight verifies the exact absolute path and full executable SHA-256 and records ffmpeg -version plus trusted package provenance." + }, + { + "name": "ffprobe", + "version": "same trusted package/build as ffmpeg", + "sourceUrl": "https://ffmpeg.org/download.html", + "license": "LGPL-2.1-or-later or GPL-2.0-or-later, depending on build configuration", + "storagePath": "operator-configured absolute executable path beside ffmpeg; not bundled by BandScope", + "distribution": "operator-provided", + "releaseUsage": "yt-dlp may invoke ffprobe when extracting audio; release/live preflight verifies its sibling path, full SHA-256, version output, and shared trusted package provenance before ffmpeg_location is passed." } ], "modelArtifacts": [ @@ -29,18 +39,18 @@ "runtimeModelName": "htdemucs", "version": "demucs-4.0.1-signature-955717e8", "sourceUrl": "https://dl.fbaipublicfiles.com/demucs/hybrid_transformer/955717e8-8726e21a.th", - "license": "No separate model-weight redistribution grant identified; runtime retrieval only, not bundled", + "license": "No separate model-weight redistribution grant identified; trusted external provisioning only, not bundled", "checksum": "sha256:8726e21a993978c7ba086d3872e7608d7d5bfca646ca4aca459ffda844faa8b4", "sizeBytes": 84141911, - "storagePath": "user runtime cache managed by torch.hub/demucs; not committed or bundled", - "distribution": "runtime-cache", + "storagePath": "trusted provisioning places exact 955717e8-8726e21a.th in the user-scoped torch.hub checkpoints cache or supplies its absolute BANDSCOPE_HTDEMUCS_MODEL_PATH; not committed or bundled", + "distribution": "pre-provisioned-runtime-cache", "releaseUsage": "Loaded locally on supported platforms to separate vocals, bass, drums, and other stems.", - "verification": "BandScope requires the exact local filename and verifies 84,141,911 bytes plus the full SHA-256 before passing a local-only repository to Demucs; missing, symlinked, or mismatched bytes fail before deserialization." + "verification": "BandScope rejects missing, symlinked, non-regular, incorrectly sized, or full-SHA-mismatched cache entries and deserializes the same 84,141,911 verified bytes only after SHA-256 8726e21a993978c7ba086d3872e7608d7d5bfca646ca4aca459ffda844faa8b4 passes. Runtime network fallback is forbidden." } ], "notes": [ "The retired bandsplit-v1 FFT profile is not a production separator artifact and must not reappear.", "Track source URL, version, full checksum, byte size, license, distribution, storage path, and release usage for every model artifact.", - "Runtime retrieval does not authorize redistribution; release packaging must fail if it attempts to bundle an artifact without an explicit license decision." + "External provisioning does not authorize redistribution; release packaging must fail if it attempts to bundle an artifact without an explicit license decision." ] } From e1cb409bad111508b9fd3869ed4c6dba5d34e812 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 10 Aug 2026 00:43:18 +0900 Subject: [PATCH 07/34] fix(security): restrict checkpoint deserialization --- ARCHITECTURE.md | 8 +- CHANGELOG.md | 5 +- CLAUDE.md | 7 +- docs/TRD.md | 12 +- ...e-separation-runtime-and-model-delivery.md | 23 ++- docs/architecture/overview.md | 4 +- docs/documentation-coverage-matrix.md | 17 +- docs/security/dependency-policy.md | 4 +- scripts/checks/run_analysis_command.py | 16 +- scripts/checks/security_gates.py | 39 +++- .../separation/audio_separator.py | 59 ++++-- .../tests/test_analysis_command.py | 69 +++++++ .../analysis-engine/tests/test_separation.py | 142 ++++++++++++-- .../tests/test_supply_chain_policy.py | 179 +++++++++++++++++- 14 files changed, 522 insertions(+), 62 deletions(-) create mode 100644 services/analysis-engine/tests/test_analysis_command.py diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 28ee73414..72351bc76 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -1,6 +1,6 @@ # ARCHITECTURE.md -Last updated: 2026-08-09 +Last updated: 2026-08-10 ## Documentation authority @@ -99,8 +99,10 @@ Last updated: 2026-08-09 - The exact signature, source URL, full SHA-256, byte size, distribution status, and model-rights uncertainty are tracked in `supply-chain/supplemental-component-inventory.json` and ADR-0001. - The separator verifies a non-symlinked regular file's exact byte size and full SHA-256, then - deserializes those same verified bytes. A model-rights/legal delivery decision remains a release - blocker. + passes those same verified bytes through PyTorch's `weights_only=True` restricted loader with an + exact reviewed global allowlist, strict model construction, and a serialized one-time cache. A + future artifact hash or allowlist change is executable-code review; model-rights/legal delivery + also remains a release blocker. - Current dependency markers exclude Demucs on macOS Intel; unsupported platforms must surface the existing safe fallback rather than pretending to separate stems. diff --git a/CHANGELOG.md b/CHANGELOG.md index 3c5b6e7b0..e6e1feb3f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,9 +25,12 @@ advisories and added a mutation-sensitive lockfile floor contract. - Made htdemucs loading offline and fail-closed: the runtime accepts only the inventoried filename, byte size, and full SHA-256, rejects filesystem identity races, and deserializes the verified - bytes rather than downloading a missing checkpoint. + bytes with PyTorch's restricted `weights_only` loader, an exact reviewed global allowlist, strict + model construction, and serialized one-time caching rather than downloading a missing checkpoint. - Verified exact platform-native sibling ffmpeg/ffprobe executable names and identities before any live fixture access or yt-dlp invocation. +- Isolated Numba's native-code cache for repository analysis commands so a stale or concurrently + compiled virtualenv cache cannot crash deterministic verification. - Reconciled stale CodeRabbit-gate wording with the canonical stable-check and review-equivalent policy; qualifying evidence is now defined against the exact current head, and a rate-limited, status-only, author, or predecessor review is not treated as completed review evidence. diff --git a/CLAUDE.md b/CLAUDE.md index ed8c8f860..93ef03196 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -60,9 +60,10 @@ Three layers, decoupled through shared contracts: - `services/analysis-engine` — Python package `bandscope_analysis` (librosa/numpy). Entry point `cli.py` reads a JSON job request on stdin and prints a structured job-status JSON envelope on stdout (`--progress-jsonl` streams progress lines). `api.py` orchestrates the pipeline across the `separation`, `sections`, `roles`, `chords`, `ranges`, `temporal`, `transcription`, and `youtube` modules. - Production source separation uses `htdemucs` on supported platforms. The exact runtime model artifact is inventoried but not bundled; operators must provision it locally, and production - verifies its byte size and full SHA-256 before deserializing those same in-memory bytes. The - active known-stem test crosses the production YouTube and separator boundaries; see `docs/TRD.md` - and the operator guide. + verifies its byte size and full SHA-256 before passing those same in-memory bytes through a + serialized `weights_only=True` loader with an exact reviewed global allowlist and strict model + construction. The active known-stem test crosses the production YouTube and separator boundaries; + see `docs/TRD.md` and the operator guide. Data flow: React UI → Tauri IPC command → Rust validation + Python subprocess over stdin/stdout → job status and progress events emitted back to the UI. diff --git a/docs/TRD.md b/docs/TRD.md index 5dc306a39..b6f5704b1 100644 --- a/docs/TRD.md +++ b/docs/TRD.md @@ -1,7 +1,7 @@ # BandScope Technical Requirements Document Status: Active authority -Last updated: 2026-08-09 +Last updated: 2026-08-10 ## System contract @@ -28,7 +28,7 @@ and data-flow views are in `docs/architecture/diagrams.md`. | TRD-KS-008 | Reject candidate drift when YouTube/master duration differs by > 1.0 s or aligned identity correlation is < 0.90. | Pre-inference live assertions against the pinned finished master. | | TRD-KS-009 | Run deterministic contract/security tests by default and require `BANDSCOPE_RUN_YOUTUBE_STEM_E2E=1` for live network/model execution. | Pytest marker and environment guard. | | TRD-KS-010 | Clean every downloaded/scored artifact on success and failure. | Nested `TemporaryDirectory` plus postcondition. | -| TRD-KS-011 | Bind model identity to inventory and full SHA-256 before any torch deserialization. | Inventory records htdemucs signature `955717e8`, 84,141,911 bytes, and SHA-256 `8726e21a…a8b4`; runtime verifies and deserializes the same in-memory bytes with no download fallback. | +| TRD-KS-011 | Bind model identity to inventory and full SHA-256 before any restricted torch deserialization. | Inventory records htdemucs signature `955717e8`, 84,141,911 bytes, and SHA-256 `8726e21a…a8b4`; runtime verifies the same in-memory bytes, uses `weights_only=True` with the reviewed minimal global allowlist and strict model construction, serializes concurrent loads, and has no download or unrestricted-loader fallback. | ## Data and class contracts @@ -80,9 +80,11 @@ artifact `955717e8-8726e21a.th`. A trusted provisioning step must place it in th user-scoped cache or provide that exact absolute file through `BANDSCOPE_HTDEMUCS_MODEL_PATH`. Runtime rejects a missing, symlinked, non-regular, incorrectly sized, wrongly named, or full-SHA-mismatched artifact before torch deserialization, reads it once, -and deserializes those same verified bytes. It never calls the remote Demucs loader or downloads a -missing checkpoint. The model is not bundled; ADR-0001 keeps the model-rights/legal delivery -decision as a release blocker. +and passes those same verified bytes to PyTorch's `weights_only=True` restricted loader. The exact +Demucs/NumPy/Fraction allowlist, strict model construction, and serialized one-time cache are guarded +by mutation tests; there is no `weights_only=False` fallback. It never calls the remote Demucs loader +or downloads a missing checkpoint. The model is not bundled; ADR-0001 keeps both the approved-pickle +risk acceptance and model-rights/legal delivery decision as release blockers for a commercial claim. `ffmpeg` and `ffprobe` are operator-provided siblings and yt-dlp is locked to `2026.7.4`. Ordinary product use may resolve the media tools from `PATH`, but release/live evidence must pass both diff --git a/docs/adr/0001-source-separation-runtime-and-model-delivery.md b/docs/adr/0001-source-separation-runtime-and-model-delivery.md index 629e85b3e..1229e6a9a 100644 --- a/docs/adr/0001-source-separation-runtime-and-model-delivery.md +++ b/docs/adr/0001-source-separation-runtime-and-model-delivery.md @@ -25,8 +25,9 @@ the upstream licensing discussion characterizes the weights as scientific-use ma 4. Trusted external provisioning is not equivalent to bundling. Documentation and SBOM evidence must preserve that distinction. 5. A release claiming source-separation readiness must verify the full SHA-256 before any torch - deserialization and must have a recorded legal decision for its chosen download or distribution - path. + deserialization, use PyTorch's `weights_only=True` restricted loader with the reviewed minimal + global allowlist, and have a recorded legal decision for its chosen download or distribution + path. Model construction is strict, and concurrent lazy loads are serialized. 6. Runtime model retrieval is forbidden. A trusted external provisioning step must populate the expected user-scoped cache or supply the exact absolute inventoried file through `BANDSCOPE_HTDEMUCS_MODEL_PATH`; missing, wrongly named, non-regular, symlinked, incorrectly @@ -54,9 +55,15 @@ incompletely pinned, or replaced by the retired profile. ## Security and governance implications Model bytes are untrusted until verified. Full-hash verification must precede pickle/torch checkpoint -deserialization; a post-load hash is insufficient. Cache paths must be user-scoped, non-symlinked, -bounded, and cleaned or quarantined on mismatch. No user-supplied checkpoint is accepted. Model -downloads and errors must not expose tokens, usernames, or full paths. +deserialization; a post-load hash is insufficient. The approved checkpoint still contains pickle +metadata: `weights_only=True` and the exact reviewed Demucs/NumPy/Fraction allowlist reduce but do +not turn it into a non-executable format. Therefore an artifact hash, allowlist, torch, NumPy, or +Demucs compatibility change is reviewed like executable code, never receives a `weights_only=False` +fallback, and must pass the real-artifact load smoke test. The one rule-specific Semgrep/Bandit +suppression is permitted only at this full-hash, same-byte, restricted-loader call; repository gates +reject an unrestricted loader, an expanded allowlist, or another `torch.load` site. Cache paths must +be user-scoped, non-symlinked, bounded, and cleaned or quarantined on mismatch. No user-supplied +checkpoint is accepted. Model downloads and errors must not expose tokens, usernames, or full paths. ## Acceptance, recovery, and rollback @@ -69,9 +76,9 @@ downloads and errors must not expose tokens, usernames, or full paths. ## Supersession triggers -Supersede this ADR when BandScope adopts a differently licensed model, bundles weights, implements an -ONNX/Rust inference path, changes the four-source contract, or makes GPU execution part of the -release baseline. +Supersede this ADR when BandScope adopts a differently licensed model, bundles weights, converts the +approved checkpoint to a non-pickle format such as safetensors, implements an ONNX/Rust inference +path, changes the four-source contract, or makes GPU execution part of the release baseline. ## References diff --git a/docs/architecture/overview.md b/docs/architecture/overview.md index 0c0947a12..8d5f8951b 100644 --- a/docs/architecture/overview.md +++ b/docs/architecture/overview.md @@ -45,7 +45,9 @@ GitHub is the source of truth for repository governance, PR review, CI/CD, Code profile is retired. - Inference is local after a trusted cache or exact `BANDSCOPE_HTDEMUCS_MODEL_PATH` is provisioned. Runtime rejects missing, wrongly named, symlinked, incorrectly sized, or full-SHA-mismatched - weights before deserializing the exact verified bytes; it never retrieves a missing model. + weights before passing the exact verified bytes to the serialized `weights_only=True` loader with + its reviewed minimal global allowlist and strict model construction; it never retrieves a missing + model and never falls back to an unrestricted checkpoint loader. - The known-stem validation branch defines and exercises the real YouTube intake → creator-master identity → composed master/vocal alignment → deterministic separator → SI-SDR scoring path. Test-only reference handling never becomes a general runtime downloader. No completed live diff --git a/docs/documentation-coverage-matrix.md b/docs/documentation-coverage-matrix.md index 484ff4b22..072a528ec 100644 --- a/docs/documentation-coverage-matrix.md +++ b/docs/documentation-coverage-matrix.md @@ -1,6 +1,6 @@ # Documentation Coverage and Traceability Matrix -Last evaluated: 2026-08-09 +Last evaluated: 2026-08-10 Evaluation scope: real known-stem YouTube source-separation validation and the affected BandScope runtime/release boundaries. @@ -14,6 +14,10 @@ The documentation graph is now structurally sufficient and explicitly code-curre is not yet release-ready for source separation. A passing live run, model-rights/legal delivery decision, threshold calibration, supported-platform evidence, and bounded evidence artifact remain open. Full-hash pre-load verification is now implemented and regression-tested. +The same-byte loader now uses `weights_only=True`, one exact reviewed global allowlist, strict Demucs +construction, and a serialized one-time read/load cache. Repository mutation tests reject an +unrestricted fallback, an allowlist expansion or second allowlist API, moved/broad scanner +suppression, and any second `torch.load` site. Issue #770 remains open. This branch must not be described as completing the full real-audio MIR acceptance layer. @@ -28,11 +32,11 @@ acceptance layer. | ADR | `docs/adr/README.md`, ADR-0001..0003 | Captures model, live quality gate, and persistence/ERD decisions with alternatives and supersession. | ADR-0001..0003 remain Proposed until branch merge. | | UML | `docs/architecture/diagrams.md` | Component, sequence, state, class, and deployment views included. | No additional UML is needed for the bounded slice. | | ERD/data | `docs/architecture/diagrams.md`, ADR-0003 | Logical artifact relationships and persistence status are explicit. | Physical ERD is intentionally not applicable until persistence exists. | -| Security/privacy | `docs/engineering/youtube-known-stem-validation.md`, `docs/security/app-security.md`, ADRs | Threats, trust boundaries, non-collection, integrity, cleanup, and legal limits covered; exact model bytes are verified before load. | Rights/platform authorization remains open. | +| Security/privacy | `docs/engineering/youtube-known-stem-validation.md`, `docs/security/app-security.md`, ADRs | Threats, trust boundaries, non-collection, integrity, cleanup, and legal limits covered; exact model bytes use the reviewed restricted loader and serialized one-time cache. | Approved-pickle semantic/provenance risk acceptance plus rights/platform authorization remain open. | | Test strategy | `docs/TRD.md`, operator guide, acceptance criteria | Offline/live split and metric/failure contracts covered. | No successful live score has been recorded. | | MIR doctoring | `docs/doctoring/real-audio-accuracy-acceptance.md` | Issue #770 metrics, claim boundaries, tiers, and roadmap are separated from the bounded vocal slice. | Accuracy manifest, reports, other MIR families, and corpus tiers remain open. | | Operations/release | runbook and release policy | Preflight, evidence, triage, rollback, and blocking conditions covered. | Platform matrix and live pass are incomplete. | -| Supply chain | supplemental inventory and dependency policy | Retired model removed; code/inventory artifact parity, fail-closed pre-load verification, uv.lock-bound yt-dlp, and verified ffmpeg/ffprobe evidence contract recorded. | Model provisioning/distribution rights remain unresolved. | +| Supply chain | supplemental inventory and dependency policy | Retired model removed; code/inventory artifact parity, fail-closed same-byte restricted loading, exact allowlist mutation guards, uv.lock-bound yt-dlp, and verified ffmpeg/ffprobe evidence contract recorded. | Model provisioning/distribution rights and any future non-pickle conversion remain unresolved. | | Automation | active CWL autonomous loop and `docs/workflow/pr-review-merge-scheduler.md` | BandScope continuity and no-status-only termination are covered without creating a competing writer. | Dedicated BandScope loop remains paused due writer topology/active-task capacity. | | Review governance | `docs/security/github-required-checks.md`, governance, gitflow, contributing, bootstrap policy | Stable checks and review are cumulative; qualifying evidence is an exact-head completed CodeRabbit artifact or exact-head independent non-author `APPROVED` review. Status-only, rate-limited, author, or predecessor evidence is excluded. | A provider rate limit can still defer review, blocking only merge. | @@ -47,7 +51,7 @@ acceptance layer. | PRD-KS-006, KS-010 | ADR-0002 | pytest marker and failure taxonomy | Every collected default offline test; explicit live failure | Advisory until promotion ADR | | PRD-KS-008 | ADR-0003 | temporary directory and sanitized errors | cleanup postcondition and archive failure tests | Evidence excludes raw media/paths | | PRD-KS-009 | ADR-0003; NIST AI RMF TEVV | planned bounded evidence schema | No retained score yet | Required before blocking release gate | -| TRD-KS-011 | ADR-0001 | separator manifest plus supplemental inventory | exact filename/hash/size parity tests | Model-rights/legal delivery blocker | +| TRD-KS-011 | ADR-0001 | separator manifest, restricted-loader allowlist, serialized load lock, and supplemental inventory | exact filename/hash/size parity; same-byte `weights_only=True`; strict construction; concurrency/read-once and mutation tests; real-artifact load smoke | Approved-pickle risk acceptance and model-rights/legal delivery blocker; any hash/allowlist/dependency change requires new smoke evidence | ## Live evidence snapshot @@ -78,7 +82,10 @@ filename/hash/size manifest, then rejects inventory drift. It also binds the yt- `uv.lock`, requires both ffmpeg and ffprobe operator records, rejects the retired bandsplit profile, and validates every model artifact's schema, types, full SHA-256, positive non-boolean size, and HTTPS source. `scripts/checks/verify_security_notes.py` recursively requires the exact canonical -`## Security Notes` section in every plan. +`## Security Notes` section in every plan. `scripts/checks/security_gates.py` permits only the one +exact full-hash same-byte `torch.load` call, requires its rule-specific Semgrep and Bandit +suppressions in place, and binds it to `weights_only=True`, the exact global allowlist, and no other +allowlist mutation API. ## Re-evaluation triggers diff --git a/docs/security/dependency-policy.md b/docs/security/dependency-policy.md index 24521817c..843a60572 100644 --- a/docs/security/dependency-policy.md +++ b/docs/security/dependency-policy.md @@ -116,7 +116,9 @@ Retired third-party deprecation and advisory signal: retaining the vulnerable torch build. No repo-local dependency-review allowlist or analysis-engine OSV exception for that advisory is active. Do not restore either stale exception. Separately, ADR-0001 requires full-SHA verification of the exact htdemucs artifact before any torch checkpoint - deserialization can qualify as release-ready. + deserialization, then `weights_only=True`, the exact reviewed global allowlist, strict model + construction, and serialized loading. Any model hash, allowlist, torch, NumPy, or Demucs lock + change requires the exact-artifact smoke test before it can qualify as release-ready. - Yanked `fastrand 2.4.0` was transiently inherited through target-specific `wry`/`dom_query` HTML parsing dependencies and must stay updated to `2.4.1` or newer in `apps/desktop/src-tauri/Cargo.lock`; `scripts/checks/verify_supply_chain.py` guards against reintroducing the yanked version. ## Required checks intent diff --git a/scripts/checks/run_analysis_command.py b/scripts/checks/run_analysis_command.py index 91300ec1d..aa007bb78 100644 --- a/scripts/checks/run_analysis_command.py +++ b/scripts/checks/run_analysis_command.py @@ -2,10 +2,12 @@ from __future__ import annotations +import os import shutil import subprocess import sys from pathlib import Path +from tempfile import TemporaryDirectory REPO_ROOT = Path(__file__).resolve().parents[2] ANALYSIS_ENGINE_DIR = REPO_ROOT / "services" / "analysis-engine" @@ -48,9 +50,19 @@ def main(argv: list[str]) -> int: argv = _normalize_args(argv) command = _analysis_command(argv) - print(f"Running analysis command in {ANALYSIS_ENGINE_DIR}: {subprocess.list2cmdline(command)}") + print( + f"Running analysis command in {ANALYSIS_ENGINE_DIR}: {subprocess.list2cmdline(command)}" + ) try: - completed = subprocess.run(command, cwd=ANALYSIS_ENGINE_DIR, check=False) + with TemporaryDirectory(prefix="bandscope-numba-") as isolated_numba_cache: + command_environment = os.environ.copy() + command_environment.setdefault("NUMBA_CACHE_DIR", isolated_numba_cache) + completed = subprocess.run( + command, + cwd=ANALYSIS_ENGINE_DIR, + check=False, + env=command_environment, + ) except FileNotFoundError as exc: print(f"Unable to start analysis command: {exc}", file=sys.stderr) return 127 diff --git a/scripts/checks/security_gates.py b/scripts/checks/security_gates.py index 348d87ca8..c232e27d9 100644 --- a/scripts/checks/security_gates.py +++ b/scripts/checks/security_gates.py @@ -13,9 +13,19 @@ "Use argument arrays, not string commands, for subprocess calls.", ), ( - re.compile(r"pickle\.load\(|torch\.load\("), + re.compile( + r"\b(?:pickle|torch)\.load\b|" + r"from\s+(?:torch|pickle)\s+import\s+load\b" + ), "Do not load untrusted pickle-style artifacts without a documented trust boundary.", ), + ( + re.compile( + r"\btorch\.serialization\b|" + r"from\s+torch\s+import\s+serialization\b" + ), + "Do not add or mutate PyTorch checkpoint reconstruction globals.", + ), ( re.compile(r"curl\s+[^\n|]*\|\s*(sh|bash)"), "Do not add remote script piping patterns.", @@ -32,11 +42,27 @@ VERIFIED_MODEL_LOADER_PATH = Path( "services/analysis-engine/src/bandscope_analysis/separation/audio_separator.py" ) +VERIFIED_MODEL_SAFE_GLOBALS_DEFINITION = ( + "def _trusted_checkpoint_globals(model_class: type[Any]) -> list[Any]:\n" + ' """Return the minimal globals required by the exact htdemucs checkpoint."""\n' + " return [\n" + " model_class,\n" + ' (_numpy_scalar, "numpy.core.multiarray.scalar"),\n' + ' (np.dtype, "numpy.dtype"),\n' + " type(np.dtype(np.float64)),\n" + " Fraction,\n" + " ]\n" +) VERIFIED_TORCH_LOAD_CALL = re.compile( - r"torch\.load\(\s*(?:#[^\n]*\n\s*)?" + r"with\s+torch\.serialization\.safe_globals\(\s*" + r"_trusted_checkpoint_globals\(HTDemucs\)\s*\):\s*" + r"# Exact full-SHA/size-verified bytes use a minimal restricted allowlist;\s*\n\s*" + r"# ADR-0001 treats any future artifact hash as executable-code review\.\s*\n\s*" + r"# nosemgrep: trailofbits\.python\.pickles-in-pytorch\.pickles-in-pytorch\s*\n\s*" + r"package\s*=\s*torch\.load\(\s*# nosec B614\s*\n\s*" r"io\.BytesIO\(payload\),\s*" r"map_location=[\"']cpu[\"'],\s*" - r"weights_only=False,?\s*" + r"weights_only=True,?\s*" r"\)", re.MULTILINE, ) @@ -45,6 +71,9 @@ "hashlib.sha256(payload).hexdigest()", "artifact.size_bytes", "stat.S_ISREG", + '(_numpy_scalar, "numpy.core.multiarray.scalar")', + '(np.dtype, "numpy.dtype")', + "# nosemgrep: trailofbits.python.pickles-in-pytorch.pickles-in-pytorch", ) @@ -61,6 +90,10 @@ def _content_for_pattern_scan(relative_path: Path, content: str) -> str: return content if not all(token in content for token in VERIFIED_MODEL_LOADER_PREREQUISITES): return content + if content.count("# nosemgrep") != 1 or content.count("# nosec") != 1: + return content + if content.count(VERIFIED_MODEL_SAFE_GLOBALS_DEFINITION) != 1: + return content if len(VERIFIED_TORCH_LOAD_CALL.findall(content)) != 1: return content return VERIFIED_TORCH_LOAD_CALL.sub("verified_checkpoint_load()", content, count=1) diff --git a/services/analysis-engine/src/bandscope_analysis/separation/audio_separator.py b/services/analysis-engine/src/bandscope_analysis/separation/audio_separator.py index df908cdd9..67a62a9b4 100644 --- a/services/analysis-engine/src/bandscope_analysis/separation/audio_separator.py +++ b/services/analysis-engine/src/bandscope_analysis/separation/audio_separator.py @@ -14,7 +14,8 @@ retrieval path. - The cache entry must be a non-symlinked regular file with the exact byte size and full SHA-256; the verified in-memory bytes are the only bytes passed to - torch checkpoint deserialization. + PyTorch's restricted ``weights_only`` checkpoint loader. Reconstruction is + serialized and limited to the minimal globals required by this exact artifact. - Does not log or persist raw audio, separated stems, or full source paths. - Fails with bounded, filename-scoped errors so callers can surface a safe failure without leaking local directory structure. @@ -29,11 +30,14 @@ import stat import warnings from dataclasses import dataclass +from fractions import Fraction from pathlib import Path +from threading import Lock from typing import Any, cast import librosa import numpy as np +from numpy.core.multiarray import scalar as _numpy_scalar from bandscope_analysis.temporal.analyzer import ( KNOWN_LIBROSA_NUMBA_WARNING_FILTERS, @@ -74,6 +78,18 @@ class _ModelArtifactSpec: ) } _MODEL_PATH_ENV = "BANDSCOPE_HTDEMUCS_MODEL_PATH" +_MODEL_LOAD_LOCK = Lock() + + +def _trusted_checkpoint_globals(model_class: type[Any]) -> list[Any]: + """Return the minimal globals required by the exact htdemucs checkpoint.""" + return [ + model_class, + (_numpy_scalar, "numpy.core.multiarray.scalar"), + (np.dtype, "numpy.dtype"), + type(np.dtype(np.float64)), + Fraction, + ] def _contains_parent_path_segment(path: Path) -> bool: @@ -179,6 +195,9 @@ def _load_model(self) -> Any: try: import torch + from demucs.htdemucs import ( # type: ignore[import-not-found, unused-ignore] + HTDemucs, + ) from demucs.states import ( # type: ignore[import-not-found, unused-ignore] load_model, ) @@ -204,22 +223,28 @@ def _load_model(self) -> Any: "Stem separation model cache location is unavailable" ) from None - payload = _read_verified_model_artifact(artifact_path, artifact) - try: - # This exact in-memory payload passed full SHA-256 and size verification above. - package = torch.load( # nosec B614 - io.BytesIO(payload), - map_location="cpu", - weights_only=False, - ) - model = load_model(package) # type: ignore[no-untyped-call] - model.eval() - except Exception: - raise ModelArtifactError( - "Stem separation model failed to load after integrity verification" - ) from None - self._model = model - return self._model + with _MODEL_LOAD_LOCK: + if self._model is not None: + return self._model + payload = _read_verified_model_artifact(artifact_path, artifact) + try: + with torch.serialization.safe_globals(_trusted_checkpoint_globals(HTDemucs)): + # Exact full-SHA/size-verified bytes use a minimal restricted allowlist; + # ADR-0001 treats any future artifact hash as executable-code review. + # nosemgrep: trailofbits.python.pickles-in-pytorch.pickles-in-pytorch + package = torch.load( # nosec B614 + io.BytesIO(payload), + map_location="cpu", + weights_only=True, + ) + model = load_model(package, strict=True) # type: ignore[no-untyped-call] + model.eval() + except Exception: + raise ModelArtifactError( + "Stem separation model failed to load after integrity verification" + ) from None + self._model = model + return self._model def _apply_model(self, model: Any, audio: AudioStemArray) -> dict[str, np.ndarray[Any, Any]]: """Apply Demucs to a mono signal, returning demucs-source-name -> mono array.""" diff --git a/services/analysis-engine/tests/test_analysis_command.py b/services/analysis-engine/tests/test_analysis_command.py new file mode 100644 index 000000000..4970c91fd --- /dev/null +++ b/services/analysis-engine/tests/test_analysis_command.py @@ -0,0 +1,69 @@ +"""Tests for the repository analysis-command launcher.""" + +from __future__ import annotations + +from pathlib import Path +from types import SimpleNamespace + +import pytest +from conftest import load_module + + +def test_analysis_command_isolates_ambient_numba_cache( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Keep native JIT cache files out of a shared or prebuilt virtualenv.""" + runner = load_module( + "scripts/checks/run_analysis_command.py", + "run_analysis_command_isolated_numba_cache", + ) + captured_cache: list[Path] = [] + monkeypatch.delenv("NUMBA_CACHE_DIR", raising=False) + monkeypatch.setattr(runner, "_analysis_command", lambda _argv: ["pytest"]) + + def fake_run( + command: list[str], + *, + cwd: Path, + check: bool, + env: dict[str, str], + ) -> SimpleNamespace: + del command, cwd, check + cache_path = Path(env["NUMBA_CACHE_DIR"]) + assert cache_path.is_dir() + captured_cache.append(cache_path) + return SimpleNamespace(returncode=0) + + monkeypatch.setattr(runner.subprocess, "run", fake_run) + + assert runner.main(["pytest"]) == 0 + assert len(captured_cache) == 1 + assert not captured_cache[0].exists() + + +def test_analysis_command_preserves_explicit_numba_cache( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Honor an operator-provided cache when isolation is intentionally overridden.""" + runner = load_module( + "scripts/checks/run_analysis_command.py", + "run_analysis_command_explicit_numba_cache", + ) + monkeypatch.setenv("NUMBA_CACHE_DIR", str(tmp_path)) + monkeypatch.setattr(runner, "_analysis_command", lambda _argv: ["pytest"]) + + def fake_run( + command: list[str], + *, + cwd: Path, + check: bool, + env: dict[str, str], + ) -> SimpleNamespace: + del command, cwd, check + assert env["NUMBA_CACHE_DIR"] == str(tmp_path) + return SimpleNamespace(returncode=0) + + monkeypatch.setattr(runner.subprocess, "run", fake_run) + + assert runner.main(["pytest"]) == 0 diff --git a/services/analysis-engine/tests/test_separation.py b/services/analysis-engine/tests/test_separation.py index be13ce031..1b4ac7952 100644 --- a/services/analysis-engine/tests/test_separation.py +++ b/services/analysis-engine/tests/test_separation.py @@ -5,6 +5,9 @@ import hashlib import os import sys +from concurrent.futures import ThreadPoolExecutor +from fractions import Fraction +from threading import Event, Lock from types import ModuleType, SimpleNamespace import numpy as np @@ -225,11 +228,31 @@ def _install_fake_verified_model_deserializer( torch_load: object | None = None, ) -> dict[str, object]: """Install fake torch/Demucs deserializers and return captured calls.""" - calls: dict[str, object] = {"torch_load_count": 0, "demucs_load_count": 0} + calls: dict[str, object] = { + "torch_load_count": 0, + "demucs_load_count": 0, + "safe_globals_active": False, + } fake_torch = ModuleType("torch") if torch_hub_root is not None: fake_torch.hub = SimpleNamespace(get_dir=lambda: torch_hub_root) # type: ignore[attr-defined] + class FakeSafeGlobals: + """Capture and model the scoped PyTorch safe-global allowlist.""" + + def __init__(self, globals_to_allow: list[object]) -> None: + calls["safe_globals"] = tuple(globals_to_allow) + + def __enter__(self) -> None: + calls["safe_globals_active"] = True + + def __exit__(self, *args: object) -> None: + calls["safe_globals_active"] = False + + fake_torch.serialization = SimpleNamespace( # type: ignore[attr-defined] + safe_globals=FakeSafeGlobals + ) + def default_torch_load( stream: object, *, @@ -240,21 +263,32 @@ def default_torch_load( calls["payload"] = stream.read() # type: ignore[attr-defined] calls["map_location"] = map_location calls["weights_only"] = weights_only + calls["safe_globals_active_at_load"] = calls["safe_globals_active"] return {"verified": True} fake_torch.load = torch_load or default_torch_load # type: ignore[attr-defined] demucs_module = ModuleType("demucs") + htdemucs_module = ModuleType("demucs.htdemucs") states_module = ModuleType("demucs.states") - def fake_load_model(package: object) -> _FakeModel: + class HTDemucs: + """Stand in for the one model class the checkpoint may reconstruct.""" + + HTDemucs.__module__ = "demucs.htdemucs" + htdemucs_module.HTDemucs = HTDemucs # type: ignore[attr-defined] + + def fake_load_model(package: object, *, strict: bool) -> _FakeModel: calls["demucs_load_count"] = int(calls["demucs_load_count"]) + 1 calls["package"] = package + calls["strict"] = strict return _FakeModel() states_module.load_model = fake_load_model # type: ignore[attr-defined] + demucs_module.htdemucs = htdemucs_module # type: ignore[attr-defined] demucs_module.states = states_module # type: ignore[attr-defined] monkeypatch.setitem(sys.modules, "torch", fake_torch) monkeypatch.setitem(sys.modules, "demucs", demucs_module) + monkeypatch.setitem(sys.modules, "demucs.htdemucs", htdemucs_module) monkeypatch.setitem(sys.modules, "demucs.states", states_module) return calls @@ -296,14 +330,99 @@ def test_audio_stem_separator_verifies_exact_model_bytes_before_deserialization( second_model = separator._load_model() assert first_model is second_model - assert calls == { - "torch_load_count": 1, - "demucs_load_count": 1, - "payload": payload, - "map_location": "cpu", - "weights_only": False, - "package": {"verified": True}, + assert calls["torch_load_count"] == 1 + assert calls["demucs_load_count"] == 1 + assert calls["payload"] == payload + assert calls["map_location"] == "cpu" + assert calls["weights_only"] is True + assert calls["safe_globals_active_at_load"] is True + assert calls["safe_globals_active"] is False + assert calls["package"] == {"verified": True} + assert calls["strict"] is True + + safe_globals = calls["safe_globals"] + assert isinstance(safe_globals, tuple) + explicit_names = { + value[1] + for value in safe_globals + if isinstance(value, tuple) and len(value) == 2 and isinstance(value[1], str) } + assert explicit_names == {"numpy.core.multiarray.scalar", "numpy.dtype"} + assert Fraction in safe_globals + assert type(np.dtype(np.float64)) in safe_globals + assert any( + getattr(value, "__module__", "") == "demucs.htdemucs" + and getattr(value, "__name__", "") == "HTDemucs" + for value in safe_globals + ) + + +def test_audio_stem_separator_serializes_checkpoint_deserialization( + tmp_path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Deserialize once when two callers race the same lazy model instance.""" + payload = b"verified-model-package" + filename = "test-signature-deadbeef.th" + (tmp_path / filename).write_bytes(payload) + _patch_model_spec(monkeypatch, filename=filename, payload=payload) + first_started = Event() + second_started = Event() + release_first = Event() + counter_lock = Lock() + load_count = 0 + read_count = 0 + read_lock = Lock() + verified_read = audio_separator_module._read_verified_model_artifact + + def counted_verified_read(*args: object, **kwargs: object) -> bytes: + nonlocal read_count + with read_lock: + read_count += 1 + return verified_read(*args, **kwargs) # type: ignore[arg-type] + + monkeypatch.setattr( + audio_separator_module, + "_read_verified_model_artifact", + counted_verified_read, + ) + + def blocking_torch_load( + stream: object, + *, + map_location: str, + weights_only: bool, + ) -> dict[str, object]: + nonlocal load_count + del stream, map_location, weights_only + with counter_lock: + load_count += 1 + call_number = load_count + if call_number == 1: + first_started.set() + assert release_first.wait(timeout=5) + else: + second_started.set() + return {"verified": True} + + _install_fake_verified_model_deserializer( + monkeypatch, + torch_load=blocking_torch_load, + ) + separator = AudioStemSeparator(AudioSeparationConfig(model_cache_directory=tmp_path)) + + with ThreadPoolExecutor(max_workers=2) as executor: + first = executor.submit(separator._load_model) + assert first_started.wait(timeout=5) + second = executor.submit(separator._load_model) + assert not second_started.wait(timeout=0.2) + release_first.set() + first_model = first.result(timeout=5) + second_model = second.result(timeout=5) + + assert first_model is second_model + assert load_count == 1 + assert read_count == 1 @pytest.mark.parametrize( @@ -332,7 +451,7 @@ def test_audio_stem_separator_rejects_missing_or_changed_model_before_deserializ (cache_dir / filename).write_bytes(payload) def forbidden_torch_load(*args: object, **kwargs: object) -> object: - raise AssertionError("unverified bytes reached torch.load") + raise AssertionError("unverified bytes reached the checkpoint loader") _install_fake_verified_model_deserializer( monkeypatch, @@ -515,12 +634,13 @@ def test_audio_stem_separator_redacts_verified_model_load_errors( def fail_torch_load(*args: object, **kwargs: object) -> object: raise RuntimeError(f"unsafe detail under {tmp_path}") - _install_fake_verified_model_deserializer(monkeypatch, torch_load=fail_torch_load) + calls = _install_fake_verified_model_deserializer(monkeypatch, torch_load=fail_torch_load) separator = AudioStemSeparator(AudioSeparationConfig(model_cache_directory=tmp_path)) with pytest.raises(ValueError, match="failed to load after integrity verification") as error: separator._load_model() assert str(tmp_path) not in str(error.value) + assert calls["safe_globals_active"] is False def _patch_demucs(monkeypatch: pytest.MonkeyPatch, per_source: dict | None = None) -> None: diff --git a/services/analysis-engine/tests/test_supply_chain_policy.py b/services/analysis-engine/tests/test_supply_chain_policy.py index da20476a7..7ec0dc2cc 100644 --- a/services/analysis-engine/tests/test_supply_chain_policy.py +++ b/services/analysis-engine/tests/test_supply_chain_policy.py @@ -293,7 +293,18 @@ def test_security_pattern_gate_accepts_only_verified_model_deserialization() -> assert security_gates.security_pattern_violations(repo_root) == [] -def test_security_pattern_gate_rejects_second_model_deserialization(tmp_path: Path) -> None: +@pytest.mark.parametrize( + "second_load", + [ + "\ntorch." + "load(untrusted_checkpoint)\n", + "\ntorch." + "load (untrusted_checkpoint)\n", + "\nfrom torch import " + "load as untrusted_load\nuntrusted_load(checkpoint)\n", + ], +) +def test_security_pattern_gate_rejects_second_model_deserialization( + second_load: str, + tmp_path: Path, +) -> None: """Do not let the narrow verified-checkpoint rule hide another torch load site.""" security_gates = load_module( "scripts/checks/security_gates.py", @@ -303,7 +314,6 @@ def test_security_pattern_gate_rejects_second_model_deserialization(tmp_path: Pa source_path = repo_root / security_gates.VERIFIED_MODEL_LOADER_PATH target_path = tmp_path / security_gates.VERIFIED_MODEL_LOADER_PATH target_path.parent.mkdir(parents=True) - second_load = "\ntorch." + "load(untrusted_checkpoint)\n" target_path.write_text( source_path.read_text(encoding="utf-8") + second_load, encoding="utf-8", @@ -317,6 +327,171 @@ def test_security_pattern_gate_rejects_second_model_deserialization(tmp_path: Pa ] +def test_security_pattern_gate_rejects_unrestricted_verified_model_load(tmp_path: Path) -> None: + """Keep the inventoried model exception bound to PyTorch's restricted loader.""" + security_gates = load_module( + "scripts/checks/security_gates.py", + "security_gates_unrestricted_model_load", + ) + repo_root = Path(__file__).resolve().parents[3] + source_path = repo_root / security_gates.VERIFIED_MODEL_LOADER_PATH + target_path = tmp_path / security_gates.VERIFIED_MODEL_LOADER_PATH + target_path.parent.mkdir(parents=True) + unrestricted_source = source_path.read_text(encoding="utf-8").replace( + "weights_only=True", + "weights_only=False", + 1, + ) + target_path.write_text(unrestricted_source, encoding="utf-8") + + violations = security_gates.security_pattern_violations(tmp_path) + + assert violations == [ + f"{security_gates.VERIFIED_MODEL_LOADER_PATH}: " + "Do not load untrusted pickle-style artifacts without a documented trust boundary.", + f"{security_gates.VERIFIED_MODEL_LOADER_PATH}: " + "Do not add or mutate PyTorch checkpoint reconstruction globals.", + ] + + +def test_security_pattern_gate_rejects_expanded_checkpoint_allowlist(tmp_path: Path) -> None: + """Require review when a new reconstructable checkpoint global is introduced.""" + security_gates = load_module( + "scripts/checks/security_gates.py", + "security_gates_expanded_checkpoint_allowlist", + ) + repo_root = Path(__file__).resolve().parents[3] + source_path = repo_root / security_gates.VERIFIED_MODEL_LOADER_PATH + target_path = tmp_path / security_gates.VERIFIED_MODEL_LOADER_PATH + target_path.parent.mkdir(parents=True) + expanded_source = source_path.read_text(encoding="utf-8").replace( + " model_class,\n", + " model_class,\n str,\n", + 1, + ) + target_path.write_text(expanded_source, encoding="utf-8") + + violations = security_gates.security_pattern_violations(tmp_path) + + assert violations == [ + f"{security_gates.VERIFIED_MODEL_LOADER_PATH}: " + "Do not load untrusted pickle-style artifacts without a documented trust boundary.", + f"{security_gates.VERIFIED_MODEL_LOADER_PATH}: " + "Do not add or mutate PyTorch checkpoint reconstruction globals.", + ] + + +@pytest.mark.parametrize( + ("api_name", "spacing"), + [ + ("safe_" + "globals", ""), + ("safe_" + "globals", " "), + ("add_safe_" + "globals", ""), + ("add_safe_" + "globals", "\t"), + ], +) +def test_security_pattern_gate_rejects_additional_checkpoint_global_mutation( + api_name: str, + spacing: str, + tmp_path: Path, +) -> None: + """Reject a second scoped or persistent PyTorch reconstruction allowlist.""" + security_gates = load_module( + "scripts/checks/security_gates.py", + f"security_gates_additional_{api_name}", + ) + repo_root = Path(__file__).resolve().parents[3] + source_path = repo_root / security_gates.VERIFIED_MODEL_LOADER_PATH + target_path = tmp_path / security_gates.VERIFIED_MODEL_LOADER_PATH + target_path.parent.mkdir(parents=True) + extra_allowlist = "\ntorch." + "serialization." + api_name + spacing + "([str])\n" + target_path.write_text( + source_path.read_text(encoding="utf-8") + extra_allowlist, + encoding="utf-8", + ) + + violations = security_gates.security_pattern_violations(tmp_path) + + assert violations == [ + f"{security_gates.VERIFIED_MODEL_LOADER_PATH}: " + "Do not add or mutate PyTorch checkpoint reconstruction globals." + ] + + +@pytest.mark.parametrize( + "alias_import", + [ + "\nfrom torch." + "serialization import safe_globals as extra_safe_globals\n", + "\nfrom torch import " + "serialization as extra_serialization\n", + ], +) +def test_security_pattern_gate_rejects_checkpoint_api_alias_imports( + alias_import: str, + tmp_path: Path, +) -> None: + """Reject standard import aliases that could bypass attribute-call matching.""" + security_gates = load_module( + "scripts/checks/security_gates.py", + "security_gates_checkpoint_alias_import", + ) + repo_root = Path(__file__).resolve().parents[3] + source_path = repo_root / security_gates.VERIFIED_MODEL_LOADER_PATH + target_path = tmp_path / security_gates.VERIFIED_MODEL_LOADER_PATH + target_path.parent.mkdir(parents=True) + target_path.write_text( + source_path.read_text(encoding="utf-8") + alias_import, + encoding="utf-8", + ) + + violations = security_gates.security_pattern_violations(tmp_path) + + assert violations == [ + f"{security_gates.VERIFIED_MODEL_LOADER_PATH}: " + "Do not add or mutate PyTorch checkpoint reconstruction globals." + ] + + +@pytest.mark.parametrize( + ("old", "new"), + [ + ( + "# nosemgrep: trailofbits.python.pickles-in-pytorch.pickles-in-pytorch\n" + " package = torch." + "load( # nosec B614", + "# nosemgrep\n" + " package = torch." + "load( # nosec B614\n" + "# nosemgrep: trailofbits.python.pickles-in-pytorch.pickles-in-pytorch", + ), + ("package = torch." + "load( # nosec B614", "package = torch." + "load("), + ], +) +def test_security_pattern_gate_binds_suppressions_to_exact_model_load( + old: str, + new: str, + tmp_path: Path, +) -> None: + """Keep both scanner exceptions exact, local, and single-purpose.""" + security_gates = load_module( + "scripts/checks/security_gates.py", + "security_gates_moved_model_suppression", + ) + repo_root = Path(__file__).resolve().parents[3] + source_path = repo_root / security_gates.VERIFIED_MODEL_LOADER_PATH + target_path = tmp_path / security_gates.VERIFIED_MODEL_LOADER_PATH + target_path.parent.mkdir(parents=True) + source = source_path.read_text(encoding="utf-8") + assert old in source + target_path.write_text(source.replace(old, new, 1), encoding="utf-8") + + violations = security_gates.security_pattern_violations(tmp_path) + + assert violations == [ + f"{security_gates.VERIFIED_MODEL_LOADER_PATH}: " + "Do not load untrusted pickle-style artifacts without a documented trust boundary.", + f"{security_gates.VERIFIED_MODEL_LOADER_PATH}: " + "Do not add or mutate PyTorch checkpoint reconstruction globals.", + ] + + def central_required_workflow_policy_text() -> str: """Return the repository policy text that delegates review automation centrally.""" repo_root = Path(__file__).resolve().parents[3] From e37b456f55ab3fb117bbaab306ac5846207ad89d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 10 Aug 2026 01:24:59 +0900 Subject: [PATCH 08/34] fix(policy): address exact-head review findings --- CHANGELOG.md | 26 +++++- docs/TRD.md | 3 + ...e-separation-runtime-and-model-delivery.md | 8 +- scripts/checks/security_gates.py | 18 +++- scripts/checks/verify_security_notes.py | 30 ++++++- scripts/checks/verify_supply_chain.py | 24 ++--- .../separation/audio_separator.py | 2 +- .../src/bandscope_analysis/youtube.py | 7 +- .../tests/test_documentation_policy.py | 39 ++++++++ .../tests/test_supply_chain_policy.py | 88 +++++++++++++++++++ .../analysis-engine/tests/test_youtube.py | 5 +- 11 files changed, 226 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e6e1feb3f..4830234b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,8 +21,12 @@ - Kept YouTube TLS verification enabled, using populated OS-managed CA roots when available and retaining yt-dlp's maintained CA-bundle fallback when the system trust store is empty or fails. -- Raised `pdfjs-dist`, `nanoid`, and `undici` to patched versions for the current high-severity - advisories and added a mutation-sensitive lockfile floor contract. +- Raised `pdfjs-dist` 6.1.200 → 6.2.108 (`GHSA-hq66-cqwq-w95j`), `nanoid` 3.3.16 → + 3.3.18 (`GHSA-2v37-7h3g-55p8`), and `undici` 7.28.0 → 7.29.0 + (`GHSA-8xcm-r25x-g524`, `GHSA-4cwx-7wf7-3272`, `GHSA-m8rv-5g2x-5cg5`, + `GHSA-jr45-8vmc-qm54`, `GHSA-v3r7-h72x-cjcm`); `package-lock.json`, the + mutation-sensitive floor test, and an exact-head zero-vulnerability npm audit preserve the + fixed-version evidence. - Made htdemucs loading offline and fail-closed: the runtime accepts only the inventoried filename, byte size, and full SHA-256, rejects filesystem identity races, and deserializes the verified bytes with PyTorch's restricted `weights_only` loader, an exact reviewed global allowlist, strict @@ -35,6 +39,24 @@ policy; qualifying evidence is now defined against the exact current head, and a rate-limited, status-only, author, or predecessor review is not treated as completed review evidence. +### Security Notes + +- Attack surface and trust boundary: YouTube URLs, response metadata, downloaded media, creator + fixtures, ffmpeg/ffprobe executables, and htdemucs checkpoint bytes remain untrusted until their + owning host, shape, size, filesystem identity, and full-hash allowlists pass. +- Mitigations and failure behavior: TLS verification stays enabled; the complete ffmpeg/ffprobe + path-and-hash pair is verified before network fixture access; model loading is offline, + same-byte, restricted to `weights_only=True` plus the exact reviewed globals, and fails closed + without an unrestricted fallback. +- Logging and privacy: raw media, model bytes, separated stems, credentials, and full local paths + are not retained in release evidence or emitted in bounded operator errors. +- Test points: exact-head quickcheck, hosted SAST/Bandit/secret/security scans, mutation tests for + loader and allowlist bypasses, executable-identity rejection tests, supply-chain verification, + and the exact provisioned-model smoke test cover the changed security boundaries. +- Dependency and supply chain: no direct dependency was added; lockfiles retain patched + `pdfjs-dist 6.2.108`, `nanoid 3.3.18`, and `undici 7.29.0`, while the supplemental inventory binds + yt-dlp, ffmpeg/ffprobe, and htdemucs to their declared delivery and integrity contracts. + ## [0.1.3] - 2026-04-29 ### Fixed diff --git a/docs/TRD.md b/docs/TRD.md index b6f5704b1..50ab788e7 100644 --- a/docs/TRD.md +++ b/docs/TRD.md @@ -85,6 +85,9 @@ Demucs/NumPy/Fraction allowlist, strict model construction, and serialized one-t by mutation tests; there is no `weights_only=False` fallback. It never calls the remote Demucs loader or downloads a missing checkpoint. The model is not bundled; ADR-0001 keeps both the approved-pickle risk acceptance and model-rights/legal delivery decision as release blockers for a commercial claim. +The pinned checkpoint's legacy `numpy.core.multiarray.scalar` pickle name remains the sole alias; +the callable is resolved through the locked NumPy 2.x `_core` compatibility path, and NumPy lock +changes require the exact-artifact load smoke because that runtime path is private. `ffmpeg` and `ffprobe` are operator-provided siblings and yt-dlp is locked to `2026.7.4`. Ordinary product use may resolve the media tools from `PATH`, but release/live evidence must pass both diff --git a/docs/adr/0001-source-separation-runtime-and-model-delivery.md b/docs/adr/0001-source-separation-runtime-and-model-delivery.md index 1229e6a9a..9a403436e 100644 --- a/docs/adr/0001-source-separation-runtime-and-model-delivery.md +++ b/docs/adr/0001-source-separation-runtime-and-model-delivery.md @@ -60,8 +60,12 @@ metadata: `weights_only=True` and the exact reviewed Demucs/NumPy/Fraction allow not turn it into a non-executable format. Therefore an artifact hash, allowlist, torch, NumPy, or Demucs compatibility change is reviewed like executable code, never receives a `weights_only=False` fallback, and must pass the real-artifact load smoke test. The one rule-specific Semgrep/Bandit -suppression is permitted only at this full-hash, same-byte, restricted-loader call; repository gates -reject an unrestricted loader, an expanded allowlist, or another `torch.load` site. Cache paths must +suppression is permitted only at this full-hash, same-byte, restricted-loader call. The approved +artifact serializes NumPy's legacy `numpy.core.multiarray.scalar` name; the locked runtime resolves +the identical callable from NumPy 2.x's private `_core` compatibility path while retaining only the +legacy serialized alias, so every NumPy lock change must repeat the exact-artifact smoke test. +Repository gates reject an unrestricted loader, an expanded allowlist, or another `torch.load` site. +Cache paths must be user-scoped, non-symlinked, bounded, and cleaned or quarantined on mismatch. No user-supplied checkpoint is accepted. Model downloads and errors must not expose tokens, usernames, or full paths. diff --git a/scripts/checks/security_gates.py b/scripts/checks/security_gates.py index c232e27d9..54cbd937b 100644 --- a/scripts/checks/security_gates.py +++ b/scripts/checks/security_gates.py @@ -1,5 +1,6 @@ """Scan repository workspace source files for disallowed security patterns.""" +import os import re from pathlib import Path @@ -67,6 +68,7 @@ re.MULTILINE, ) VERIFIED_MODEL_LOADER_PREREQUISITES = ( + "from numpy._core.multiarray import scalar as _numpy_scalar", "payload = _read_verified_model_artifact(", "hashlib.sha256(payload).hexdigest()", "artifact.size_bytes", @@ -99,14 +101,24 @@ def _content_for_pattern_scan(relative_path: Path, content: str) -> str: return VERIFIED_TORCH_LOAD_CALL.sub("verified_checkpoint_load()", content, count=1) +def _workspace_files(repo_root: Path) -> list[Path]: + """Return repository files without descending into excluded dependency trees.""" + files: list[Path] = [] + for directory, dirnames, filenames in os.walk(repo_root): + dirnames[:] = sorted(name for name in dirnames if name not in EXCLUDED_PARTS) + directory_path = Path(directory) + files.extend(directory_path / name for name in sorted(filenames)) + return files + + def security_pattern_violations(repo_root: Path = Path(".")) -> list[str]: """Return forbidden-pattern violations below ``repo_root``.""" violations: list[str] = [] - for path in repo_root.rglob("*"): - if not path.is_file() or not should_scan(path): - continue + for path in _workspace_files(repo_root): relative_path = path.relative_to(repo_root) + if not path.is_file() or not should_scan(relative_path): + continue if relative_path == SELF_PATH: continue content = path.read_text(encoding="utf-8", errors="ignore") diff --git a/scripts/checks/verify_security_notes.py b/scripts/checks/verify_security_notes.py index 69c472d9a..e30c895e9 100644 --- a/scripts/checks/verify_security_notes.py +++ b/scripts/checks/verify_security_notes.py @@ -4,6 +4,9 @@ from pathlib import Path SECURITY_NOTES_HEADING = "## Security Notes" +SECURITY_NOTES_PATTERN = re.compile(r"^## Security Notes\s*$") +PEER_HEADING_PATTERN = re.compile(r"^#{1,2}\s+.+\s*$") +FENCE_PATTERN = re.compile(r"^\s*(?P`{3,}|~{3,})") PLAN_DIR = Path("docs/plans") REQUIRED_SUBSECTIONS = [ "attack surface", @@ -18,14 +21,33 @@ def security_notes_section(content: str) -> str: """Return the canonical security section, stopping at the next peer heading.""" lines = content.splitlines() - try: - start = lines.index(SECURITY_NOTES_HEADING) - except ValueError: + start = next( + ( + index + for index, line in enumerate(lines) + if SECURITY_NOTES_PATTERN.fullmatch(line) + ), + None, + ) + if start is None: return "" section_lines = [lines[start]] + open_fence: tuple[str, int] | None = None for line in lines[start + 1 :]: - if re.fullmatch(r"#{1,2}\s+.+", line): + fence_match = FENCE_PATTERN.match(line) + if fence_match is not None: + marker = fence_match.group("marker") + marker_shape = (marker[0], len(marker)) + if open_fence is None: + open_fence = marker_shape + elif ( + marker_shape[0] == open_fence[0] + and marker_shape[1] >= open_fence[1] + and not line[fence_match.end() :].strip() + ): + open_fence = None + elif open_fence is None and PEER_HEADING_PATTERN.fullmatch(line): break section_lines.append(line) return "\n".join(section_lines).lower() diff --git a/scripts/checks/verify_supply_chain.py b/scripts/checks/verify_supply_chain.py index f72d91970..6aac7c2ac 100644 --- a/scripts/checks/verify_supply_chain.py +++ b/scripts/checks/verify_supply_chain.py @@ -13,6 +13,7 @@ except ModuleNotFoundError: # pragma: no cover - local Python <3.11 fallback. import tomli as tomllib +REPO_ROOT = Path(__file__).resolve().parents[2] REQUIRED_FILES = [ Path("package-lock.json"), Path("services/analysis-engine/uv.lock"), @@ -50,7 +51,7 @@ r'signature="(?P[0-9a-f]+)",\s*' r'filename="(?P[^"]+)",\s*' r'sha256="(?P[0-9a-f]{64})",\s*' - r'size_bytes=(?P[0-9_]+),\s*\)', + r"size_bytes=(?P[0-9_]+),\s*\)", re.DOTALL, ) REQUIRED_MODEL_ARTIFACT_FIELDS = { @@ -113,9 +114,7 @@ def supplemental_inventory_violations( if not artifacts: return ["supplemental inventory modelArtifacts must not be empty"] if analysis_lock_path is None: - analysis_lock_path = ( - inventory_path.resolve().parent.parent / ANALYSIS_LOCK_PATH - ) + analysis_lock_path = REPO_ROOT / ANALYSIS_LOCK_PATH package_tools = inventory.get("packageManagedTools") if not isinstance(package_tools, list): @@ -132,7 +131,9 @@ def supplemental_inventory_violations( ) else: try: - lock_data = tomllib.loads(analysis_lock_path.read_text(encoding="utf-8")) + lock_data = tomllib.loads( + analysis_lock_path.read_text(encoding="utf-8") + ) locked_packages = lock_data.get("package", []) locked_versions = [ package.get("version") @@ -145,7 +146,9 @@ def supplemental_inventory_violations( ) else: if len(locked_versions) != 1 or not isinstance(locked_versions[0], str): - violations.append("analysis lock requires exactly one yt-dlp package") + violations.append( + "analysis lock requires exactly one yt-dlp package" + ) elif yt_dlp_records[0].get("version") != locked_versions[0]: violations.append( "supplemental inventory yt-dlp version does not match uv.lock" @@ -196,9 +199,7 @@ def supplemental_inventory_violations( ) checksum = artifact.get("checksum") if not isinstance(checksum, str) or not FULL_SHA256_PATTERN.fullmatch(checksum): - violations.append( - f"supplemental inventory {label} requires full SHA-256" - ) + violations.append(f"supplemental inventory {label} requires full SHA-256") source_url = artifact.get("sourceUrl") if not isinstance(source_url, str) or not source_url.startswith("https://"): violations.append(f"supplemental inventory {label} requires HTTPS source") @@ -245,7 +246,10 @@ def supplemental_inventory_violations( "does not match separator manifest" ) version = artifact.get("version") - if not isinstance(version, str) or str(separator_artifact["signature"]) not in version: + if ( + not isinstance(version, str) + or str(separator_artifact["signature"]) not in version + ): violations.append( f"supplemental inventory runtime model {runtime_model} version " "does not identify separator signature" diff --git a/services/analysis-engine/src/bandscope_analysis/separation/audio_separator.py b/services/analysis-engine/src/bandscope_analysis/separation/audio_separator.py index 67a62a9b4..a1e708d5b 100644 --- a/services/analysis-engine/src/bandscope_analysis/separation/audio_separator.py +++ b/services/analysis-engine/src/bandscope_analysis/separation/audio_separator.py @@ -37,7 +37,7 @@ import librosa import numpy as np -from numpy.core.multiarray import scalar as _numpy_scalar +from numpy._core.multiarray import scalar as _numpy_scalar from bandscope_analysis.temporal.analyzer import ( KNOWN_LIBROSA_NUMBA_WARNING_FILTERS, diff --git a/services/analysis-engine/src/bandscope_analysis/youtube.py b/services/analysis-engine/src/bandscope_analysis/youtube.py index a14578cc4..691f709f0 100644 --- a/services/analysis-engine/src/bandscope_analysis/youtube.py +++ b/services/analysis-engine/src/bandscope_analysis/youtube.py @@ -136,6 +136,11 @@ def _system_ca_available() -> bool: return False +def _has_execute_permission(path: Path) -> bool: + """Return whether the current process may execute ``path``.""" + return os.access(path, os.X_OK) + + def _verify_executable_artifact( executable_path: Optional[str], executable_sha256: Optional[str] ) -> Optional[str]: @@ -155,7 +160,7 @@ def _verify_executable_artifact( try: resolved = candidate.resolve(strict=True) - if not resolved.is_file() or not os.access(resolved, os.X_OK): + if not resolved.is_file() or not _has_execute_permission(resolved): return None digest = hashlib.sha256() diff --git a/services/analysis-engine/tests/test_documentation_policy.py b/services/analysis-engine/tests/test_documentation_policy.py index 29e3e9876..2734979b7 100644 --- a/services/analysis-engine/tests/test_documentation_policy.py +++ b/services/analysis-engine/tests/test_documentation_policy.py @@ -60,6 +60,45 @@ def test_security_notes_contract_discovers_nested_plan_without_canonical_section ] +def test_security_notes_contract_accepts_trailing_space_and_fenced_headings( + tmp_path: Path, +) -> None: + """Keep fenced headings inside a canonical section with trailing whitespace.""" + security_notes = load_module( + "scripts/checks/verify_security_notes.py", + "verify_security_notes_fenced_headings", + ) + plan_path = tmp_path / "docs" / "plans" / "nested" / "safe-plan.md" + plan_path.parent.mkdir(parents=True) + security_heading = "## Security Notes" + " " + plan_content = f"""# Safe plan + +{security_heading} + +Attack surface: untrusted input. +Trust boundary: validate before use. +Mitigations: fail closed. +Test points: exercise rejection paths. +```text +```python +## This fenced heading is data +``` +~~~text +# This fenced heading is also data +~~~ +Realistic threats: artifact substitution. +Remaining risk: approved artifact provenance. + +## Next section + +This text is outside the security section. +""" + plan_path.write_text(plan_content, encoding="utf-8") + + assert security_notes.security_notes_violations(tmp_path) == [] + assert "outside the security section" not in security_notes.security_notes_section(plan_content) + + def test_security_notes_contract_accepts_checked_in_plans() -> None: """Accept every checked-in plan only when its complete canonical section is present.""" security_notes = load_module( diff --git a/services/analysis-engine/tests/test_supply_chain_policy.py b/services/analysis-engine/tests/test_supply_chain_policy.py index 7ec0dc2cc..30d1cc2d5 100644 --- a/services/analysis-engine/tests/test_supply_chain_policy.py +++ b/services/analysis-engine/tests/test_supply_chain_policy.py @@ -93,6 +93,37 @@ def test_supplemental_inventory_accepts_pinned_htdemucs_runtime_model() -> None: assert violations == [] +def test_supplemental_inventory_uses_repository_lock_for_custom_inventory( + tmp_path: Path, +) -> None: + """Resolve the default analysis lock independently of an inventory fixture path.""" + supply_chain = load_module( + "scripts/checks/verify_supply_chain.py", + "verify_supply_chain_custom_inventory_default_lock", + ) + repo_root = Path(__file__).resolve().parents[3] + inventory_path = tmp_path / "inventory.json" + inventory_path.write_text( + (repo_root / "supply-chain" / "supplemental-component-inventory.json").read_text( + encoding="utf-8" + ), + encoding="utf-8", + ) + + violations = supply_chain.supplemental_inventory_violations( + inventory_path, + repo_root + / "services" + / "analysis-engine" + / "src" + / "bandscope_analysis" + / "separation" + / "audio_separator.py", + ) + + assert violations == [] + + @pytest.mark.parametrize( ("old", "new", "message"), [ @@ -293,6 +324,34 @@ def test_security_pattern_gate_accepts_only_verified_model_deserialization() -> assert security_gates.security_pattern_violations(repo_root) == [] +def test_security_pattern_gate_prunes_excluded_directories( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Do not descend into dependency and build trees during repository scans.""" + security_gates = load_module( + "scripts/checks/security_gates.py", + "security_gates_pruned_directories", + ) + excluded_file = tmp_path / ".venv" / "nested" / "ignored.py" + excluded_file.parent.mkdir(parents=True) + excluded_file.write_text("torch." + "load(untrusted)\n", encoding="utf-8") + source_file = tmp_path / "src" / "safe.py" + source_file.parent.mkdir() + source_file.write_text("value = 1\n", encoding="utf-8") + visited: list[Path] = [] + original_is_file = Path.is_file + + def tracked_is_file(path: Path) -> bool: + visited.append(path) + return original_is_file(path) + + monkeypatch.setattr(Path, "is_file", tracked_is_file) + + assert security_gates.security_pattern_violations(tmp_path) == [] + assert not any(".venv" in path.parts for path in visited) + + @pytest.mark.parametrize( "second_load", [ @@ -381,6 +440,35 @@ def test_security_pattern_gate_rejects_expanded_checkpoint_allowlist(tmp_path: P ] +def test_security_pattern_gate_binds_numpy_scalar_compatibility_import( + tmp_path: Path, +) -> None: + """Keep the legacy pickle name mapped to NumPy's reviewed scalar callable.""" + security_gates = load_module( + "scripts/checks/security_gates.py", + "security_gates_numpy_scalar_import", + ) + repo_root = Path(__file__).resolve().parents[3] + source_path = repo_root / security_gates.VERIFIED_MODEL_LOADER_PATH + target_path = tmp_path / security_gates.VERIFIED_MODEL_LOADER_PATH + target_path.parent.mkdir(parents=True) + mutated_source = source_path.read_text(encoding="utf-8").replace( + "from numpy._core.multiarray import scalar as _numpy_scalar", + "_numpy_scalar = str", + 1, + ) + target_path.write_text(mutated_source, encoding="utf-8") + + violations = security_gates.security_pattern_violations(tmp_path) + + assert violations == [ + f"{security_gates.VERIFIED_MODEL_LOADER_PATH}: " + "Do not load untrusted pickle-style artifacts without a documented trust boundary.", + f"{security_gates.VERIFIED_MODEL_LOADER_PATH}: " + "Do not add or mutate PyTorch checkpoint reconstruction globals.", + ] + + @pytest.mark.parametrize( ("api_name", "spacing"), [ diff --git a/services/analysis-engine/tests/test_youtube.py b/services/analysis-engine/tests/test_youtube.py index a286b0a34..43125afc4 100644 --- a/services/analysis-engine/tests/test_youtube.py +++ b/services/analysis-engine/tests/test_youtube.py @@ -339,7 +339,10 @@ def test_download_youtube_audio_rejects_invalid_ffmpeg_artifact( else: ffmpeg = tmp_path / "ffmpeg" ffmpeg.write_bytes(b"not executable") - monkeypatch.setattr(os, "access", lambda *_args: False) + monkeypatch.setattr( + "bandscope_analysis.youtube._has_execute_permission", + lambda *_args: False, + ) runtime["ffmpeg_path"] = str(ffmpeg) runtime["ffmpeg_sha256"] = "0" * 64 From c4a30b5392b0d5ea416b611ae97c19d5b9338d5a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 10 Aug 2026 01:38:23 +0900 Subject: [PATCH 09/34] fix(policy): enforce GFM security-note boundaries --- scripts/checks/verify_security_notes.py | 4 +- .../tests/test_documentation_policy.py | 42 ++++++++++++++++++- 2 files changed, 43 insertions(+), 3 deletions(-) diff --git a/scripts/checks/verify_security_notes.py b/scripts/checks/verify_security_notes.py index e30c895e9..9f5bc0535 100644 --- a/scripts/checks/verify_security_notes.py +++ b/scripts/checks/verify_security_notes.py @@ -5,8 +5,8 @@ SECURITY_NOTES_HEADING = "## Security Notes" SECURITY_NOTES_PATTERN = re.compile(r"^## Security Notes\s*$") -PEER_HEADING_PATTERN = re.compile(r"^#{1,2}\s+.+\s*$") -FENCE_PATTERN = re.compile(r"^\s*(?P`{3,}|~{3,})") +PEER_HEADING_PATTERN = re.compile(r"^ {0,3}#{1,2}\s+.+\s*$") +FENCE_PATTERN = re.compile(r"^ {0,3}(?P`{3,}|~{3,})") PLAN_DIR = Path("docs/plans") REQUIRED_SUBSECTIONS = [ "attack surface", diff --git a/services/analysis-engine/tests/test_documentation_policy.py b/services/analysis-engine/tests/test_documentation_policy.py index 2734979b7..afee93e06 100644 --- a/services/analysis-engine/tests/test_documentation_policy.py +++ b/services/analysis-engine/tests/test_documentation_policy.py @@ -2,6 +2,7 @@ from pathlib import Path +import pytest from conftest import load_module @@ -79,7 +80,7 @@ def test_security_notes_contract_accepts_trailing_space_and_fenced_headings( Trust boundary: validate before use. Mitigations: fail closed. Test points: exercise rejection paths. -```text + ```text ```python ## This fenced heading is data ``` @@ -99,6 +100,45 @@ def test_security_notes_contract_accepts_trailing_space_and_fenced_headings( assert "outside the security section" not in security_notes.security_notes_section(plan_content) +@pytest.mark.parametrize("indent", [" ", "\t"]) +def test_security_notes_contract_rejects_invalid_fence_indentation( + tmp_path: Path, + indent: str, +) -> None: + """Do not let an indented code block hide the next peer heading.""" + security_notes = load_module( + "scripts/checks/verify_security_notes.py", + f"verify_security_notes_invalid_fence_{indent.encode().hex()}", + ) + plan_path = tmp_path / "docs" / "plans" / "nested" / "unsafe-plan.md" + plan_path.parent.mkdir(parents=True) + plan_content = f"""# Unsafe plan + +## Security Notes + +Attack surface: untrusted input. +Trust boundary: validate before use. +Mitigations: fail closed. +Test points: exercise rejection paths. +{indent}```text + ## Next section + +Realistic threats: this is outside the canonical section. +Remaining risk: this is outside the canonical section. +""" + plan_path.write_text(plan_content, encoding="utf-8") + + violations = security_notes.security_notes_violations(tmp_path) + + assert violations == [ + "docs/plans/nested/unsafe-plan.md missing Security Notes subsection: realistic threats", + "docs/plans/nested/unsafe-plan.md missing Security Notes subsection: remaining risk", + ] + assert "outside the canonical section" not in security_notes.security_notes_section( + plan_content + ) + + def test_security_notes_contract_accepts_checked_in_plans() -> None: """Accept every checked-in plan only when its complete canonical section is present.""" security_notes = load_module( From 189d722c6a1fc6ec8e23363d38eb97da5680f3eb Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 10 Aug 2026 03:17:27 +0900 Subject: [PATCH 10/34] docs: enforce known-stem design authority --- ARCHITECTURE.md | 12 +- CHANGELOG.md | 18 +- docs/PRD.md | 44 +- docs/TRD.md | 136 +- ...e-separation-runtime-and-model-delivery.md | 16 +- .../0002-known-stem-youtube-quality-gate.md | 8 +- ...0003-ephemeral-benchmark-evidence-model.md | 42 +- docs/adr/README.md | 4 +- docs/architecture/diagrams.md | 110 +- docs/architecture/overview.md | 8 +- .../real-audio-accuracy-acceptance.md | 16 +- docs/documentation-coverage-matrix.md | 77 +- docs/engineering/acceptance-criteria.md | 14 +- .../youtube-known-stem-validation.md | 44 +- docs/operations/deploy-runbook.md | 35 +- docs/plans/2026-03-10-bandscope-harness.md | 2 +- ...-issue-32-analysis-orchestration-design.md | 33 +- docs/release/release-policy.md | 24 +- package.json | 4 +- scripts/checks/markdown_sections.py | 252 ++++ scripts/checks/run_analysis_command.py | 6 +- scripts/checks/verify_docs.py | 268 +++- scripts/checks/verify_security_notes.py | 80 +- scripts/harness/quickcheck.sh | 4 +- services/analysis-engine/pyproject.toml | 1 + services/analysis-engine/tests/conftest.py | 11 +- .../tests/test_analysis_command.py | 54 + .../tests/test_documentation_policy.py | 1162 ++++++++++++++++- services/analysis-engine/uv.lock | 2 + 29 files changed, 2270 insertions(+), 217 deletions(-) create mode 100644 scripts/checks/markdown_sections.py diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 72351bc76..d42fba966 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -102,9 +102,13 @@ Last updated: 2026-08-10 passes those same verified bytes through PyTorch's `weights_only=True` restricted loader with an exact reviewed global allowlist, strict model construction, and a serialized one-time cache. A future artifact hash or allowlist change is executable-code review; model-rights/legal delivery - also remains a release blocker. + also remains a release blocker. The repository security owner must separately accept the residual + approved-pickle risk for the exact model hash/dependency lock, with expiry/re-review and rollback, + or approve a non-pickle replacement. - Current dependency markers exclude Demucs on macOS Intel; unsupported platforms must surface the existing safe fallback rather than pretending to separate stems. +- Quality claims are platform-scoped: every advertised OS/architecture needs an unchanged-candidate + pass, while every unproven artifact must exercise and advertise the fallback. ## Known-stem validation boundary @@ -122,6 +126,12 @@ Last updated: 2026-08-10 calibration requirements in ADR-0002 are met. - The capability has no relational persistence. ADR-0003 and the logical artifact model in `docs/architecture/diagrams.md` are authoritative instead of a physical ERD. +- The planned `BenchmarkRun`/`BenchmarkEvidence` aggregate always binds candidate, fixture, model, + and sanitized toolchain provenance; identity and score blocks are stage-dependent. Persistence is + disabled until store/access/TTL/deletion controls are accepted. +- Distinct user-facing import/model/decode/separation recovery states are planned under + PRD-KS-011/TRD-KS-013; the current benchmark failure taxonomy does not claim that product UX is + complete. ## Rehearsal outputs diff --git a/CHANGELOG.md b/CHANGELOG.md index 4830234b8..f3ec75d54 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,8 +25,8 @@ 3.3.18 (`GHSA-2v37-7h3g-55p8`), and `undici` 7.28.0 → 7.29.0 (`GHSA-8xcm-r25x-g524`, `GHSA-4cwx-7wf7-3272`, `GHSA-m8rv-5g2x-5cg5`, `GHSA-jr45-8vmc-qm54`, `GHSA-v3r7-h72x-cjcm`); `package-lock.json`, the - mutation-sensitive floor test, and an exact-head zero-vulnerability npm audit preserve the - fixed-version evidence. + mutation-sensitive floor test, and the required per-candidate zero-vulnerability npm audit + preserve the fixed-version evidence. - Made htdemucs loading offline and fail-closed: the runtime accepts only the inventoried filename, byte size, and full SHA-256, rejects filesystem identity races, and deserializes the verified bytes with PyTorch's restricted `weights_only` loader, an exact reviewed global allowlist, strict @@ -50,12 +50,14 @@ without an unrestricted fallback. - Logging and privacy: raw media, model bytes, separated stems, credentials, and full local paths are not retained in release evidence or emitted in bounded operator errors. -- Test points: exact-head quickcheck, hosted SAST/Bandit/secret/security scans, mutation tests for - loader and allowlist bypasses, executable-identity rejection tests, supply-chain verification, - and the exact provisioned-model smoke test cover the changed security boundaries. -- Dependency and supply chain: no direct dependency was added; lockfiles retain patched - `pdfjs-dist 6.2.108`, `nanoid 3.3.18`, and `undici 7.29.0`, while the supplemental inventory binds - yt-dlp, ffmpeg/ffprobe, and htdemucs to their declared delivery and integrity contracts. +- Test points: each candidate head must pass quickcheck, hosted SAST/Bandit/secret/security scans, + mutation tests for loader and allowlist bypasses, executable-identity rejection tests, + supply-chain verification, and the exact provisioned-model smoke test before merge. +- Dependency and supply chain: no production dependency was added; documentation policy checks now + pin `markdown-it-py 4.0.0` as a direct development dependency so rendered Markdown—not lexical + lookalikes—defines headings and tables. Lockfiles retain patched `pdfjs-dist 6.2.108`, `nanoid` + 3.3.18, and `undici 7.29.0`, while the supplemental inventory binds yt-dlp, ffmpeg/ffprobe, and + htdemucs to their declared delivery and integrity contracts. ## [0.1.3] - 2026-04-29 diff --git a/docs/PRD.md b/docs/PRD.md index 21ee17edf..bd6890e97 100644 --- a/docs/PRD.md +++ b/docs/PRD.md @@ -1,7 +1,7 @@ # BandScope Product Requirements Document Status: Active authority -Last updated: 2026-08-09 +Last updated: 2026-08-10 ## Product outcome @@ -42,8 +42,9 @@ CPU/GPU, and report requirements. | PRD-KS-006 | Keep normal CI deterministic while preserving a real integration proof. | Metric, alignment, integrity, redirect, path, cleanup, and failure tests run offline; live access is explicit opt-in and fail-closed. | `active_branch` | | PRD-KS-007 | Respect content and platform restrictions. | No cookies, account login, paywall, DRM, geo, or anti-bot bypass; operator records authorization before live use. | `active_branch` | | PRD-KS-008 | Keep downloaded media ephemeral and private. | Test-owned directory is removed on success and failure; raw audio, full paths, URLs, tokens, and cookies are not logged or retained. | `active_branch` | -| PRD-KS-009 | Make release quality evidence reviewable. | Exact commit, model identity, fixture hashes, platform, command, outcome, and numeric scores are retained as a bounded CI/operator artifact. | `planned` | +| PRD-KS-009 | Make release quality evidence reviewable without retaining sensitive execution context. | A schema-v1 artifact binds the exact candidate, dependency lock, fixture/model/tool identities, sanitized command template, stage/outcome, applicable numeric blocks, and cleanup result; it contains no raw media, URL, credential, provider body, or local path. | `planned` | | PRD-KS-010 | Fail safely when the live ecosystem is unavailable. | Download/model/integrity/drift failures are distinct, do not become passes, and do not block unrelated development work. | `active_branch` | +| PRD-KS-011 | Give users an honest, recoverable failure experience. | Import, model availability, decode, and separation failures have distinct safe states and tested local-file or fallback guidance without exposing provider bodies or sensitive paths. | `planned` | ## Scope and non-goals @@ -54,26 +55,34 @@ or notation accuracy. The benchmark is a quality sentinel, not a general downloa model-training dataset, or legal opinion. BandScope must not retain user media in hosted telemetry or introduce a relational benchmark -database merely to satisfy documentation conventions. Results remain ephemeral until a separate -audited evidence-retention requirement is accepted. +database merely to satisfy documentation conventions. Automated run artifacts remain disabled and +ephemeral until a separate audited evidence-retention control is accepted. Intentionally reviewed, +non-sensitive historical observations may remain in version-controlled documentation, but they are +not substitutes for schema-v1 exact-candidate evidence. ## Failure experience -The user-facing product must explain whether import, model availability, decode, or separation -failed and offer local-file fallback without exposing raw provider errors or sensitive paths. The -benchmark itself must retain stable diagnostic codes and numeric scores; it must never silently skip -after explicit opt-in. +PRD-KS-011 owns the planned user-facing distinction between import, model availability, decode, and +separation failures. The current bounded benchmark contract is narrower: after explicit opt-in it +must fail closed and must never silently skip. The planned schema-v1 artifact classifies that result +with TRD's stable stage/outcome vocabulary; numeric identity or score fields exist only when the run +reached the corresponding stage. ## Release acceptance The known-stem lane becomes blocking for a release only after all of the following exist: 1. documented authorization for the chosen live access mode; -2. full-hash pre-load verification of the exact model artifact and a recorded model-rights/legal - decision for the chosen provisioning or distribution path; -3. at least one recorded passing supported-platform run on the exact release candidate; +2. full-hash pre-load verification of the exact model artifact, a recorded model-rights/legal + decision for the chosen provisioning or distribution path, and closure of the exact-checkpoint + approved-pickle risk gate defined by ADR-0001; +3. a recorded passing run on the exact release candidate for every OS/architecture on which that + release advertises source separation; every other release artifact must advertise and exercise + the safe fallback instead of inheriting another platform's evidence; 4. thresholds calibrated on an authorized YouTube candidate and a drift/flake triage owner; -5. ordinary CI, security, coverage, packaging, SBOM, review, and provenance gates pass. +5. an accepted evidence-retention control naming the store, access roles, incident owner, TTL + enforcement, and deletion verification, followed by a valid schema-v1 artifact; +6. ordinary CI, security, coverage, packaging, SBOM, review, and provenance gates pass. Until then, the deterministic offline contract is required and live evidence is advisory but must fail closed when deliberately invoked. @@ -81,7 +90,10 @@ fail closed when deliberately invoked. ## Ownership and rollout The analysis-engine owner owns metrics, fixture integrity, alignment, separator integration, and -failure taxonomy. Release engineering owns model/tool inventory and retained evidence. Repository -governance owns rights/platform authorization and the decision to make live execution scheduled or -blocking. Rollout proceeds from local opt-in, to controlled release-candidate evidence, to a -blocking lane only through a superseding ADR. +failure taxonomy. The product/desktop owner owns PRD-KS-011 failure copy and recovery behavior. +Release engineering owns model/tool inventory and, only after authorization, retained evidence. +The repository security owner owns approved-pickle risk review; closure requires an accepted, +time-bounded record scoped to the exact model hash and dependency lock, or migration to an approved +non-pickle artifact. Repository governance owns rights/platform authorization and the decision to +make live execution scheduled or blocking. Rollout proceeds from local opt-in, to controlled +release-candidate evidence, to a blocking lane only through a superseding ADR. diff --git a/docs/TRD.md b/docs/TRD.md index 50ab788e7..185035a87 100644 --- a/docs/TRD.md +++ b/docs/TRD.md @@ -29,6 +29,8 @@ and data-flow views are in `docs/architecture/diagrams.md`. | TRD-KS-009 | Run deterministic contract/security tests by default and require `BANDSCOPE_RUN_YOUTUBE_STEM_E2E=1` for live network/model execution. | Pytest marker and environment guard. | | TRD-KS-010 | Clean every downloaded/scored artifact on success and failure. | Nested `TemporaryDirectory` plus postcondition. | | TRD-KS-011 | Bind model identity to inventory and full SHA-256 before any restricted torch deserialization. | Inventory records htdemucs signature `955717e8`, 84,141,911 bytes, and SHA-256 `8726e21a…a8b4`; runtime verifies the same in-memory bytes, uses `weights_only=True` with the reviewed minimal global allowlist and strict model construction, serializes concurrent loads, and has no download or unrestricted-loader fallback. | +| TRD-KS-012 | Retain only bounded numeric/provenance evidence and never raw media, stems, archives, credentials, provider bodies, or full local paths. | ADR-0003, the exact benchmark-evidence schema below, and `docs/operations/deploy-runbook.md#source-separation-preflight-and-evidence`; persistence remains planned until its retention policy is accepted. | +| TRD-KS-013 | Expose distinct, safe import/model/decode/separation failure states and recovery guidance across the engine/desktop boundary. | `planned`; typed orchestration/desktop contracts and copy tests must cover every PRD-KS-011 state without returning provider bodies or sensitive paths. | ## Data and class contracts @@ -38,11 +40,85 @@ and data-flow views are in `docs/architecture/diagrams.md`. | `AlignedStemWindow` | mixture/reference arrays; single lag; reference start; correlation | Process memory only | | `KnownStemBenchmarkWindow` | YouTube/master lag; master/vocal lag; composed mixture/reference window; identity correlation | Process memory only | | Separation result | canonical stem arrays; sample rate; duration; role types; notes | Process memory and downstream local analysis | -| Benchmark evidence | commit, fixture IDs/hashes, model signature/hash, platform, timestamps, scores, outcome code | `planned`; bounded artifact, never raw audio | +| Benchmark evidence | Schema-v1 `BenchmarkRun` provenance plus stage/outcome and cleanup; optional identity and score blocks governed by the invariants below | `planned`; bounded artifact, never raw audio | No relational database exists for this capability. The logical artifact model in `docs/architecture/diagrams.md` is authoritative; a database ERD would falsely imply persistence. +## Benchmark evidence schema v1 + +Schema v1 is the canonical retained-evidence contract. It is a design contract, not evidence that a +store exists: artifact upload and retention remain disabled until ADR-0003's store, access, TTL, +deletion-verification, and incident-owner controls are accepted. + +### Common run provenance + +Every success or failure record contains the following fields: + +| Field | Contract | +|---|---| +| `schema_version` | Integer literal `1`. | +| `benchmark_id`, `run_id` | Stable public benchmark ID and non-sensitive unique run ID. | +| `candidate` | Exact head commit, tested base commit, and SHA-256 of the dependency lock. | +| `authorization_ref` | Identifier of the recorded content/platform authorization; null only for `authorization_missing`, and never credential or private text. | +| `fixture_identity` | Public video ID plus archive, extracted-member, and creator-master SHA-256/byte-count identities; no full URLs. | +| `model_identity` | Expected inventory name/version, signature, canonical filename, full SHA-256, byte count, `pre-provisioned` delivery mode, and verification status; no cache path. | +| `toolchain_identity` | OS/architecture; locked/observed Python, Demucs, torch, NumPy, and yt-dlp versions; ffmpeg/ffprobe expected basenames/package identity, configured hashes when present, observed versions after verification, per-tool verification status, and `sibling_layout_verified`; no absolute paths. | +| `command_template` | Stable template ID and SHA-256 of the sanitized operator-guide template. Literal environment assignments and invocation paths are forbidden. | +| `started_at`, `finished_at`, `wall_time_seconds` | UTC timestamps and non-negative elapsed wall time. | +| `stage`, `outcome_code` | Last completed or first failing boundary and one stable code from the vocabulary below. | +| `diagnostic_field` | Optional stable schema-field identifier for a malformed/non-finite input; never provider or exception text. | +| `cleanup` | Whether a media root was created, whether cleanup was attempted, and whether the root was empty; never the root path. | + +Absolute executable/model paths are required transient inputs to preflight, not retained identities. +Their canonical basenames, sibling-layout result, hashes, versions, and trusted package identity prove +which tools ran without leaking usernames or local filesystem layout. A preflight failure keeps the +expected/configured non-sensitive identity and a failed verification status; it does not fabricate an +observed version, verified hash, or sibling-layout success. + +### Stable stage and outcome vocabulary + +`stage` is one of `preflight`, `fixture_fetch`, `youtube_download`, `identity`, `separation`, +`scoring`, `cleanup`, or `complete`. `outcome_code` is one of: + +| Stage | Outcome codes | +|---|---| +| `preflight` | `authorization_missing`, `runtime_dependency_invalid`, `model_identity_invalid` | +| `fixture_fetch` | `reference_integrity_invalid` | +| `youtube_download` | `unsupported_url`, `restricted_content`, `duration_exceeded`, `size_exceeded`, `download_failed`, `download_error`, `file_not_found` | +| `identity` | `fixture_duration_drift`, `fixture_identity_mismatch` | +| `separation` | `model_unavailable`, `model_load_failed`, `separator_output_invalid`, `operator_timeout` | +| `scoring` | `score_non_finite`, `quality_threshold_failed` | +| `cleanup` | `cleanup_failed` | +| `complete` | `passed` | +| Any boundary | `internal_error` | + +The record uses the first failing boundary. Provider text and Python exception text are not outcome +codes and are never copied into retained evidence. + +### Optional measured blocks and invariants + +The `identity` block contains downloaded/master durations, duration drift, YouTube-to-master and +master-to-vocal lags, scored-window duration, and identity correlation. The `score` block contains +baseline mixture SI-SDR, vocal SI-SDR, best non-vocal SI-SDR, improvement, and assignment margin. + +- Common provenance, stage/outcome, and cleanup are required for every record. Expected fixture/model + identities bind early failures without claiming that those assets were fetched or verified. +- `authorization_ref` may be null only for `authorization_missing`. A successful preflight requires + non-null authorization plus fully verified model/tool statuses. +- A failure before identity measurement omits `identity`; a failure before scoring omits `score`. +- `fixture_duration_drift` requires the measured durations/drift but may omit correlation and lags. +- `fixture_identity_mismatch` requires the complete `identity` block and omits `score`. +- `score_non_finite` requires the identity block and `diagnostic_field`; its score block contains only + finite values computed before failure and may be partial. `quality_threshold_failed` requires both + complete measured blocks. Non-finite values are never encoded as non-standard JSON numbers. +- `passed` requires `stage=complete`, both measured blocks, every threshold passing, and + `cleanup.media_root_empty=true`, non-null authorization, and verified model/tool identities. +- `cleanup_failed` overrides an otherwise passing outcome. Later-stage fields are never fabricated + for an earlier failure. +- Unknown fields, raw media/stems/archive bytes, full URLs, absolute paths, credentials, cookies, + provider bodies, and literal command environments make the artifact invalid. + ## Metric contract For zero-mean estimate $\hat{s}$ and reference $s$: @@ -61,7 +137,7 @@ are literal thresholds, not values recomputed by production helpers. | Platform | Dependency state | Live lane status | |---|---|---| | Linux x86_64 | Demucs/torch resolved; CPU inference supported | Supported for controlled evidence | -| Windows amd64/arm64 | Demucs dependency marker permits installation; release build must prove wheel/tool compatibility | Unproven | +| Windows amd64/arm64 | Demucs dependency marker permits installation; each architecture must prove wheel/tool compatibility | Unproven | | macOS arm64 | Demucs dependency marker permits installation | Unproven | | macOS Intel | Demucs dependency marker excludes installation | Explicitly unavailable; product must surface safe fallback | @@ -69,6 +145,10 @@ The scored excerpt is 12 seconds, mono PCM at 44.1 kHz, with a 13-second separat and 10 MiB scored-file bound. No release latency ceiling is yet accepted; record wall time and peak memory during calibration rather than inventing a target. +Quality evidence is platform-scoped. A release may advertise source separation only on each exact +OS/architecture with a passing run of the unchanged candidate; an unproven or unavailable artifact +must exercise and advertise the safe fallback. + Production separation passes `shifts=0` to Demucs. This removes its random temporal augmentation so the same audio, model, platform, and precision produce repeatable benchmark inputs and avoids a global random-seed side effect in the test harness. @@ -85,16 +165,20 @@ Demucs/NumPy/Fraction allowlist, strict model construction, and serialized one-t by mutation tests; there is no `weights_only=False` fallback. It never calls the remote Demucs loader or downloads a missing checkpoint. The model is not bundled; ADR-0001 keeps both the approved-pickle risk acceptance and model-rights/legal delivery decision as release blockers for a commercial claim. +The repository security owner closes the pickle gate only with a time-bounded governance record +scoped to the exact model hash, dependency lock, allowlist, exact-artifact smoke/mutation evidence, +and rollback, or by approving a non-pickle replacement. Repository governance separately closes the +rights/delivery gate. The pinned checkpoint's legacy `numpy.core.multiarray.scalar` pickle name remains the sole alias; the callable is resolved through the locked NumPy 2.x `_core` compatibility path, and NumPy lock changes require the exact-artifact load smoke because that runtime path is private. `ffmpeg` and `ffprobe` are operator-provided siblings and yt-dlp is locked to `2026.7.4`. Ordinary -product use may resolve the media tools from `PATH`, but release/live evidence must pass both -absolute executable paths and both full SHA-256 values as one four-part identity. Preflight records -both paths, hashes, exact platform-native sibling names, version outputs, and shared trusted package -identity before any reference or YouTube access; none may be described as bundled unless packaging -and licensing change. +product use may resolve the media tools from `PATH`, but release/live preflight must receive both +absolute executable paths and both full SHA-256 values as one four-part identity. Before any +reference or YouTube access it verifies those paths transiently. Retained evidence records only +canonical platform-native basenames, hashes, version outputs, shared trusted-package identity, and +the sibling-layout result; none may be described as bundled unless packaging and licensing change. ## Failure taxonomy @@ -102,11 +186,12 @@ and licensing change. policy failures. - `download_failed`, `download_error`, `file_not_found`: live media/provider/tool failures. - `runtime_dependency_invalid`: configured ffmpeg/ffprobe identity set, layout, or hash failure. -- Reference byte/hash/member/redirect error: fixture integrity or SSRF boundary failure. -- YouTube/master duration drift above 1.0 s or identity correlation below 0.90: wrong or drifted - candidate/transcode. -- Model import/provisioning/identity/load error: platform or supply-chain failure. -- Non-finite/shape/threshold error: separator correctness failure. +- `reference_integrity_invalid`: reference byte/hash/member/redirect or SSRF-boundary failure. +- `fixture_duration_drift`, `fixture_identity_mismatch`: wrong or drifted candidate/transcode. +- `model_identity_invalid`, `model_unavailable`, `model_load_failed`: platform or supply-chain + failure. +- `separator_output_invalid`, `score_non_finite`, `quality_threshold_failed`: separator correctness + or quality failure. Explicit live invocation converts all of these to a failing test. A failure blocks only the evidence lane; it does not authorize a bypass or stop unrelated repository work. @@ -114,11 +199,11 @@ lane; it does not authorize a bypass or stop unrelated repository work. ## Verification and evidence Default verification runs every collected deterministic known-stem contract test and explicitly -excludes the live marker. A live run uses the exact -command in the operator guide. Evidence must include exact commit and dependency lock, model full -hash, fixture archive full hash, public video ID, OS/architecture, result code, correlation, baseline -SI-SDR, vocal SI-SDR, improvement, assignment margin, duration, and cleanup result. Raw audio, -archive contents, local paths, provider response bodies, cookies, and credentials are forbidden. +excludes the live marker. A live run uses the sanitized command template in the operator guide with +local values supplied only at execution time. Any future retained artifact must validate against +schema v1 above; an earlier-boundary failure omits later measured blocks. Raw audio, archive +contents, local paths, literal environment assignments, provider response bodies, cookies, and +credentials are forbidden. On 2026-08-09, commit `5a3648a11d9097b8da48bb4a3ccbd97986aec25b` passed a 13-test pre-correction partial suite. It did not yet contain the creator-master authentication, @@ -133,17 +218,20 @@ correlation was only 0.016856, proving it was not a valid identity gate. These v the provisional +0.5/+3.0 sentinels and the separate master identity design; they are not an authorized YouTube pass. -After the correction, the byte-identical implementation tree published on GitHub as exact commit -`6e937a34f9036d92e909db3ce8848a5c39dc8e3b` passed the full quickcheck. Its live retry -authenticated all three reference artifacts and the pre-provisioned model full hash, then failed -closed at production YouTube intake with HTTP 502 after 65.49 seconds. No identity or separation -score was emitted. +Historical evidence snapshot: the byte-identical implementation tree published on GitHub as commit +`6e937a34f9036d92e909db3ce8848a5c39dc8e3b` passed the full quickcheck. Its live retry authenticated +all three reference artifacts and the pre-provisioned model full hash, then failed closed at +production YouTube intake with HTTP 502 after 65.49 seconds. No identity or separation score was +emitted. This immutable record applies only to that historical commit, not the current branch head; +current-head offline checks and hosted review evidence belong to PR #828 and must be regenerated +after every commit. ## Traceability `docs/documentation-coverage-matrix.md` maps product requirements and ADRs to modules, tests, and -release controls. Any threshold, fixture, model, persistence, or automation-policy change must -update that matrix and the applicable ADR before merge. +release controls. Any threshold, fixture, model, evidence schema, failure UX, supported-platform, +persistence, or automation-policy change must update that matrix and the applicable ADR before +merge. Issue #770's complete real-audio acceptance program is tracked separately in `docs/doctoring/real-audio-accuracy-acceptance.md`. This TRD implements only its known-vocal-stem diff --git a/docs/adr/0001-source-separation-runtime-and-model-delivery.md b/docs/adr/0001-source-separation-runtime-and-model-delivery.md index 9a403436e..67aefa21f 100644 --- a/docs/adr/0001-source-separation-runtime-and-model-delivery.md +++ b/docs/adr/0001-source-separation-runtime-and-model-delivery.md @@ -33,6 +33,12 @@ the upstream licensing discussion characterizes the weights as scientific-use ma `BANDSCOPE_HTDEMUCS_MODEL_PATH`; missing, wrongly named, non-regular, symlinked, incorrectly sized, or full-SHA-mismatched weights fail before deserialization. Source separation remains unavailable on macOS Intel under the current dependency markers. +7. Restricted loading is a mitigation, not automatic approval of the checkpoint's pickle semantics. + Before a release advertises source separation, the repository security owner must accept that + residual risk in an immutable governance record scoped to the exact model SHA-256, dependency + lock, allowlist, provisioning path, and release line. The record must name an owner, review date, + expiry or re-review trigger, rollback, and the exact-artifact smoke/mutation evidence. Conversion + to an approved non-pickle format closes this gate without a pickle-risk exception. ## Alternatives considered @@ -69,11 +75,19 @@ Cache paths must be user-scoped, non-symlinked, bounded, and cleaned or quarantined on mismatch. No user-supplied checkpoint is accepted. Model downloads and errors must not expose tokens, usernames, or full paths. +The approved-pickle gate is distinct from the model-rights/legal delivery decision. Passing a hash, +restricted-loader, or smoke test does not close either governance question. The security owner may +close the pickle gate only with the scoped record in Decision 7 or an approved non-pickle artifact; +repository governance closes the separate rights/delivery gate. + ## Acceptance, recovery, and rollback - Inventory/model-name consistency check passes. - A corrupt or substituted model fails before deserialization. -- Supported platform tests prove canonical finite stems and known-stem quality. +- Every platform/architecture advertised for source separation proves canonical finite stems and a + passing known-stem run on the exact release candidate; other artifacts prove the safe fallback. +- The repository security owner records the exact-hash/dependency-lock approved-pickle decision and + its expiry/re-review triggers, or approves a non-pickle replacement. - Unsupported platforms return a stable fallback error. - Rollback disables source separation or restores the previous exact approved model artifact; it never restores the FFT profile as a production separator. diff --git a/docs/adr/0002-known-stem-youtube-quality-gate.md b/docs/adr/0002-known-stem-youtube-quality-gate.md index 861ff687c..964db66d0 100644 --- a/docs/adr/0002-known-stem-youtube-quality-gate.md +++ b/docs/adr/0002-known-stem-youtube-quality-gate.md @@ -41,7 +41,8 @@ until rights/platform authorization and an authorized YouTube calibration are re ## Consequences The live lane can fail for provider availability independently of model correctness. That failure is -retained honestly and blocks only that evidence lane. A creator-master probe is calibration +classified honestly and blocks only that evidence lane; automated retention remains disabled until +ADR-0003's store/access/TTL/deletion controls are accepted. A creator-master probe is calibration evidence, not proof that the YouTube candidate passes. A single vocal fixture does not establish four-source or genre-wide validity. Additional fixtures require separate provenance and calibrated threshold review, not threshold weakening. @@ -60,7 +61,10 @@ operator must verify the intended access against current terms and rights. - Every collected deterministic contract test passes in ordinary CI; the root runner explicitly excludes the live marker. Test count is recorded as evidence, not fixed policy. -- A controlled live run on the exact candidate records all required scores and cleanup evidence. +- A controlled live run on the exact candidate emits evidence schema v1 from `docs/TRD.md`; failures + omit stage-dependent identity/score blocks rather than inventing values. +- Each platform/architecture advertised for source separation has its own exact-candidate pass; + evidence from one platform does not transfer to another. - Fixture drift causes a distinct pre-model failure. - Provider/model unavailability remains a failure after explicit opt-in. - Rollback removes the live gate without removing deterministic metric/security tests or weakening diff --git a/docs/adr/0003-ephemeral-benchmark-evidence-model.md b/docs/adr/0003-ephemeral-benchmark-evidence-model.md index 89b476c25..d3a95a035 100644 --- a/docs/adr/0003-ephemeral-benchmark-evidence-model.md +++ b/docs/adr/0003-ephemeral-benchmark-evidence-model.md @@ -14,9 +14,16 @@ product need. Downloaded audio, extracted references, scored windows, and separated stems are ephemeral and live only inside a test-owned temporary directory or process memory. Cleanup occurs on success and -failure. The repository stores fixture metadata and thresholds. When controlled evidence retention -is authorized, retain only exact commit/lock/model/fixture identities, platform, timestamps, numeric -scores, duration, outcome code, and cleanup result as a bounded Actions/operator artifact. +failure. The repository stores fixture metadata and thresholds. Automated evidence persistence is +disabled and remains `planned`. A live run may be inspected transiently, but it may not upload or +retain a new operator/Actions artifact until repository governance accepts a named store, access +roles, TTL enforcement, deletion verification, and incident owner. + +When those controls are accepted, the only permitted retained payload is benchmark evidence schema +v1 in `docs/TRD.md`: common candidate/fixture/model/tool provenance, a sanitized command-template +identity, stable stage/outcome, cleanup, and only the identity or score blocks actually reached. It +never contains literal invocation environment values, absolute paths, full URLs, provider bodies, +credentials, raw media, archives, or stems. No relational database is introduced. `docs/architecture/diagrams.md` contains the authoritative logical artifact relationship model; it is intentionally not a physical ERD. @@ -30,24 +37,33 @@ logical artifact relationship model; it is intentionally not a physical ERD. ## Consequences -Trend analysis is initially manual or artifact-based. Evidence retention must have an explicit TTL -and access policy. Reproduction depends on external fixture availability, so exact identity and -failure codes are essential. A future hosted evidence service is a separate bounded context and may -not access user audio or BandScope's local project files directly. +Trend analysis is initially manual from explicitly documented, non-sensitive observations such as +the historical failure snapshots in this repository. A 30-day TTL is only a proposed default, not an +active retention authorization. Reproduction depends on external fixture availability, so exact +identity and stable failure codes are essential. A future hosted evidence service is a separate +bounded context and may not access user audio or BandScope's local project files directly. ## Security and governance implications Evidence excludes raw audio, source archive content, full URLs, local paths, usernames, cookies, -credentials, and provider response bodies. Actions artifacts must be access-controlled, checksum -bound to the candidate, and expire under repository policy. PII masking is not needed because PII is -not collected; purpose limitation and non-collection are the control. +credentials, literal environment assignments, and provider response bodies. Absolute executable and +model paths are verified transiently; retained identity uses canonical basenames, hashes, versions, +trusted package identity, and a sibling-layout verification flag. Any future Actions artifact must +be access-controlled, checksum-bound to the candidate, and expire under the accepted repository +policy. PII masking is not needed because PII is not collected; purpose limitation and +non-collection are the control. ## Acceptance, recovery, and rollback - Temporary root is empty after the live test exits. -- Logs contain stable public fixture IDs and numeric results only. -- Evidence schema rejects raw media/path fields. -- Rollback deletes retained numeric artifacts according to TTL without affecting local projects. +- Transient logs contain stable public fixture IDs, stage/outcome, and applicable numeric results + only; they do not contain a literal command or local paths. +- Evidence schema v1 rejects raw media/path fields and enforces stage-dependent identity/score + invariants. +- Before persistence is enabled, governance records the exact store, readers/writers, incident owner, + TTL mechanism, deletion verification, and rollback. The proposed initial TTL is 30 days. +- Rollback disables artifact upload and deletes retained numeric artifacts according to the accepted + TTL without affecting local projects. ## Supersession triggers diff --git a/docs/adr/README.md b/docs/adr/README.md index c14c14cdf..ef9b6ed15 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -5,9 +5,9 @@ decision through a new ADR that names the superseded record. | ADR | Status | Decision | |---|---|---| -| `0001-source-separation-runtime-and-model-delivery.md` | Proposed on active branch | Use real four-source htdemucs locally, require trusted external provisioning, verify the exact artifact before deserialization, and retain the model-rights/legal delivery decision as a release blocker. | +| `0001-source-separation-runtime-and-model-delivery.md` | Proposed on active branch | Use real four-source htdemucs locally, require trusted external provisioning, verify the exact artifact before deserialization, and retain both exact-checkpoint approved-pickle acceptance and model-rights/legal delivery as distinct release blockers. | | `0002-known-stem-youtube-quality-gate.md` | Proposed on active branch | Validate the production YouTube-to-separator path with a creator-published known vocal stem, single alignment, SI-SDR improvement, and assignment margin. | -| `0003-ephemeral-benchmark-evidence-model.md` | Proposed on active branch | Keep media and signal arrays ephemeral; retain only bounded evidence when authorized, so a relational ERD is not currently authoritative. | +| `0003-ephemeral-benchmark-evidence-model.md` | Proposed on active branch | Keep media and signal arrays ephemeral; keep automated evidence retention disabled until store/access/TTL/deletion controls are accepted, then permit only schema-v1 bounded evidence, so a relational ERD is not currently authoritative. | Status meanings are `Proposed`, `Accepted`, `Deprecated`, and `Superseded`. An accepted decision may still carry explicit release blockers; acceptance does not assert that every follow-up is shipped. diff --git a/docs/architecture/diagrams.md b/docs/architecture/diagrams.md index 01fb1b1a4..cae9d8fe4 100644 --- a/docs/architecture/diagrams.md +++ b/docs/architecture/diagrams.md @@ -79,18 +79,25 @@ classDiagram class KnownStemFixture { +youtube_url: str +video_id: str + +reference_archive_url: str + +reference_archive_host: str +reference_archive_sha256: str +reference_archive_bytes: int +reference_member: str +reference_member_sha256: str + +reference_member_bytes: int + +creator_master_url: str + +creator_master_host: str +creator_master_sha256: str +creator_master_bytes: int + +creator_master_duration_seconds: float +target_stem: str } class AlignedStemWindow { +mixture: ndarray +reference: ndarray +lag_samples: int + +reference_start: int +correlation: float } class AudioStemSeparator { @@ -107,11 +114,13 @@ classDiagram +reference: ndarray +youtube_to_master_lag_samples: int +master_to_reference_lag_samples: int + +reference_start: int +identity_correlation: float } class BenchmarkScore { +baseline_si_sdr: float +vocal_si_sdr: float + +best_non_vocal_si_sdr: float +improvement_db: float +assignment_margin_db: float } @@ -122,8 +131,74 @@ classDiagram AudioStemSeparator --> BenchmarkScore: supplies named stems ``` -`BenchmarkScore` is a logical contract planned for retained evidence; current test assertions compute -these values without instantiating a production class. +`BenchmarkScore` is a logical value object planned for schema-v1 evidence; current test assertions +compute these values without instantiating a production class. + +## UML evidence aggregate view (planned) + +```mermaid +classDiagram + class BenchmarkRun { + +schema_version: int + +benchmark_id: str + +run_id: str + +stage: str + +outcome_code: str + } + class ReleaseCandidateIdentity { + +head_commit: str + +base_commit: str + +dependency_lock_sha256: str + } + class KnownStemFixture { + +public_video_id: str + +expected_asset_hashes: map + } + class ModelArtifactSpec { + +inventory_identity: str + +expected_sha256: str + +verification_status: str + } + class ToolchainIdentity { + +os_arch: str + +expected_tool_identity: map + +observed_versions: map + +verification_status: map + } + class BenchmarkEvidence { + +started_at: datetime + +finished_at: datetime + +wall_time_seconds: float + +cleanup: CleanupResult + } + class BenchmarkIdentity { + +duration_drift_seconds: float + +youtube_to_master_lag: int + +master_to_vocal_lag: int + +correlation: float + } + class BenchmarkScore { + +baseline_si_sdr: float + +vocal_si_sdr: float + +best_non_vocal_si_sdr: float + +improvement_db: float + +assignment_margin_db: float + } + BenchmarkRun --> ReleaseCandidateIdentity: binds + BenchmarkRun --> KnownStemFixture: configures + BenchmarkRun --> ModelArtifactSpec: verifies + BenchmarkRun --> ToolchainIdentity: executes with + BenchmarkRun *-- BenchmarkEvidence: emits + BenchmarkEvidence "1" *-- "0..1" BenchmarkIdentity: reached identity + BenchmarkEvidence "1" *-- "0..1" BenchmarkScore: reached scoring +``` + +`BenchmarkRun` binds the version-controlled `KnownStemFixture`, exact release candidate, +`ModelArtifactSpec`, and sanitized toolchain identity. Common provenance exists for every outcome; +the identity and score value objects exist only when their stages were reached. Expected identities +remain present when preflight fails, while observed values and successful verification statuses are +never fabricated. The aggregate is a schema contract, not a current production class or an +authorization to persist artifacts. ## Deployment and trust boundaries @@ -144,13 +219,14 @@ flowchart TB Public["YouTube + pinned creator assets"] --> Benchmark Model["Official model host"] --> Provisioner["Trusted model provisioner"] Provisioner -->|"cache or exact path"| ModelFile - Benchmark --> Evidence["Bounded numeric evidence"] + Benchmark -.->|"planned after retention approval"| Evidence["Schema-v1 bounded evidence"] ``` -The benchmark, not the product app, owns public fixture access and bounded evidence. Model -provisioning is a separate trusted operation; runtime loading never downloads a missing checkpoint. -The provisioned model file is persistent; media temp is not. Public hosts, model locations, media, -decoders, and model bytes are untrusted until their respective policy and integrity checks pass. +The benchmark, not the product app, owns public fixture access and any future bounded evidence. +Evidence persistence is currently disabled pending ADR-0003 controls. Model provisioning is a +separate trusted operation; runtime loading never downloads a missing checkpoint. The provisioned +model file is persistent; media temp is not. Public hosts, model locations, media, decoders, and +model bytes are untrusted until their respective policy and integrity checks pass. ## Logical artifact relationship model (not a physical ERD) @@ -167,15 +243,23 @@ erDiagram YOUTUBE_MIX ||--|| ALIGNED_WINDOW : yields REFERENCE_STEM ||--|| ALIGNED_WINDOW : aligns ALIGNED_WINDOW ||--|{ SEPARATED_STEM : produces - ALIGNED_WINDOW o|--o| BENCHMARK_EVIDENCE : may-score - SEPARATED_STEM }o--o| BENCHMARK_EVIDENCE : may-contribute + KNOWN_STEM_FIXTURE ||--o{ BENCHMARK_RUN : configures + RELEASE_CANDIDATE ||--o{ BENCHMARK_RUN : binds + MODEL_ARTIFACT ||--o{ BENCHMARK_RUN : loads + TOOLCHAIN_IDENTITY ||--o{ BENCHMARK_RUN : executes + BENCHMARK_RUN ||--|| BENCHMARK_EVIDENCE : emits + BENCHMARK_EVIDENCE ||--o| IDENTITY_EVIDENCE : may-contain + BENCHMARK_EVIDENCE ||--o| SCORE_EVIDENCE : may-contain + ALIGNED_WINDOW o|--o| IDENTITY_EVIDENCE : may-measure + SEPARATED_STEM }o--o| SCORE_EVIDENCE : may-measure ``` Only `KNOWN_STEM_FIXTURE` metadata is version-controlled. An archive may contain many members, but the fixture selects and authenticates exactly one `REFERENCE_ARCHIVE_MEMBER` before decoding it as the reference stem. `YOUTUBE_MIX`, `CREATOR_MASTER`, `REFERENCE_ARCHIVE_MEMBER`, `REFERENCE_STEM`, `ALIGNED_WINDOW`, and `SEPARATED_STEM` bytes are ephemeral. -`BENCHMARK_EVIDENCE` is planned as a bounded artifact, not a database row. Its aligned-window and -separated-stem relationships are optional because pre-alignment failures (such as the recorded HTTP -502) still produce valid failure evidence. ADR-0003 requires a new physical ERD only if persistence -is introduced. +`BENCHMARK_RUN` always binds the fixture, exact release candidate, model, and sanitized toolchain +identity, so a pre-alignment failure is not orphaned. `BENCHMARK_EVIDENCE` is planned as a bounded +artifact, not a database row; its identity and score blocks are optional because pre-alignment +failures (such as the recorded HTTP 502) have no such measurements. ADR-0003 requires a new physical +ERD only if relational persistence is introduced. diff --git a/docs/architecture/overview.md b/docs/architecture/overview.md index 8d5f8951b..b14b44ac5 100644 --- a/docs/architecture/overview.md +++ b/docs/architecture/overview.md @@ -53,7 +53,13 @@ GitHub is the source of truth for repository governance, PR review, CI/CD, Code Test-only reference handling never becomes a general runtime downloader. No completed live production-path pass has yet produced an identity or SI-SDR score. - Media and stem arrays are ephemeral. Only bounded numeric/provenance evidence may be retained after - authorization, so no physical benchmark database or ERD exists. + content/platform authorization and acceptance of ADR-0003's retention controls. The planned + schema-v1 aggregate binds a benchmark run to its candidate, fixture, + model, and sanitized toolchain identities, with optional identity/score blocks. Retention is + disabled until store/access/TTL/deletion controls are accepted, so no physical benchmark database + or ERD exists. +- Source-separation quality claims are OS/architecture-specific. An exact-candidate pass does not + transfer to another release artifact, and unproven platforms must surface the safe fallback. See `docs/TRD.md`, `docs/adr/README.md`, and `docs/architecture/diagrams.md`. diff --git a/docs/doctoring/real-audio-accuracy-acceptance.md b/docs/doctoring/real-audio-accuracy-acceptance.md index f800440a8..091b9fc8c 100644 --- a/docs/doctoring/real-audio-accuracy-acceptance.md +++ b/docs/doctoring/real-audio-accuracy-acceptance.md @@ -67,6 +67,11 @@ Machine-readable JSON and accessible HTML render the same exact values. Neither private audio, copyrighted excerpts, absolute paths, credentials, cookies, or provider response bodies. +The known-stem slice's narrower schema-v1 `BenchmarkRun`/`BenchmarkEvidence` contract is defined in +`docs/TRD.md`. It is not the complete issue-#770 manifest: it uses sanitized tool/command identities, +stable stage/outcome codes, and optional identity/score blocks so early failures remain valid without +fabricated metrics. + ## Rights, security, and privacy Audio, annotations, metadata, manifests, decoders, model artifacts, and benchmark storage are @@ -87,6 +92,10 @@ does not stop unrelated engineering. Rollback restores the previous exact manife removes unsupported accuracy claims; it does not delete failing evidence, weaken metrics, or replace real audio with mocks. +Automated known-stem evidence retention remains disabled until ADR-0003's store, access, TTL, +deletion-verification, and incident-owner controls are accepted. A passing run is scoped to its exact +release candidate and OS/architecture; it cannot authorize a claim on a different artifact. + ## Current source-separation slice The active branch: @@ -108,9 +117,10 @@ The live attempt failed at YouTube HTTP 502 before model execution, so no passin Creator-master calibration produced deterministic +1.752 dB SI-SDR improvement and +7.631 dB assignment margin, while dry-vocal/mix correlation was only 0.016856. Those results support the provisional sentinel and separate identity check, not a YouTube pass or release-blocking threshold. -The corrected byte-identical implementation tree published on GitHub as exact commit -`6e937a34f9036d92e909db3ce8848a5c39dc8e3b` later passed full quickcheck, but its clean live -retry again failed at YouTube HTTP 502 before separation; live success therefore remains absent. +The historical byte-identical implementation tree published on GitHub as exact commit +`6e937a34f9036d92e909db3ce8848a5c39dc8e3b` later passed full quickcheck, but its clean live retry +again failed at YouTube HTTP 502 before separation. That record applies only to the named commit, +not the current head; live success therefore remains absent. ## References diff --git a/docs/documentation-coverage-matrix.md b/docs/documentation-coverage-matrix.md index 072a528ec..bf7060858 100644 --- a/docs/documentation-coverage-matrix.md +++ b/docs/documentation-coverage-matrix.md @@ -10,10 +10,13 @@ The pre-change repository had a strong benchmark operator note but was insuffici canonical PRD, TRD, ADRs, UML, logical data model, traceability, model inventory consistency, and release/operations criteria. This branch adds those authorities and mechanical presence checks. -The documentation graph is now structurally sufficient and explicitly code-current, but the product -is not yet release-ready for source separation. A passing live run, model-rights/legal delivery -decision, threshold calibration, supported-platform evidence, and bounded evidence artifact remain -open. Full-hash pre-load verification is now implemented and regression-tested. +The documentation graph is now structurally sufficient and explicitly code-current for this bounded +slice: every declared PRD/TRD requirement is mapped to a decision, implementation, test/evidence, +and release control, and that ID coverage is machine-checked. The product is not yet release-ready +for source separation. A passing live run on every advertised platform, model-rights/legal delivery +decision, exact-checkpoint approved-pickle risk acceptance (or non-pickle replacement), threshold +calibration, accepted evidence-retention controls, and a valid schema-v1 artifact remain open. +Full-hash pre-load verification is now implemented and regression-tested. The same-byte loader now uses `weights_only=True`, one exact reviewed global allowlist, strict Demucs construction, and a serialized one-time read/load cache. Repository mutation tests reject an unrestricted fallback, an allowlist expansion or second allowlist API, moved/broad scanner @@ -26,39 +29,44 @@ acceptance layer. | Family | Canonical authority | Assessment | Remaining gap | |---|---|---|---| -| PRD | `docs/PRD.md` | Adequate for product outcome, scope, users, acceptance, legal boundary, rollout, and non-goals. | Broader multi-fixture/four-stem requirements await evidence. | -| TRD | `docs/TRD.md` | Adequate for interfaces, metrics, schema, platform matrix, failure taxonomy, model delivery, evidence, and traceability. | Performance budget and calibrated thresholds are not yet accepted. | -| Architecture | `ARCHITECTURE.md`, `docs/architecture/overview.md` | Updated for fail-closed htdemucs provisioning and known-stem boundaries. | Model-rights/legal delivery decision remains open. | +| PRD | `docs/PRD.md` | Adequate for product outcome, scope, users, platform-scoped acceptance, legal boundary, rollout, and non-goals. | Broader multi-fixture/four-stem requirements and PRD-KS-011 failure UX remain planned. | +| TRD | `docs/TRD.md` | Adequate for interfaces, metrics, versioned stage-aware evidence schema, platform matrix, stable failure taxonomy, model delivery, and traceability. | Performance budget, calibrated thresholds, evidence emitter/store, and TRD-KS-013 remain planned. | +| Architecture | `ARCHITECTURE.md`, `docs/architecture/overview.md` | Updated for fail-closed htdemucs provisioning, platform-scoped claims, evidence aggregate, and known-stem boundaries. | Rights, approved-pickle, retention, and platform evidence gates remain open. | | ADR | `docs/adr/README.md`, ADR-0001..0003 | Captures model, live quality gate, and persistence/ERD decisions with alternatives and supersession. | ADR-0001..0003 remain Proposed until branch merge. | -| UML | `docs/architecture/diagrams.md` | Component, sequence, state, class, and deployment views included. | No additional UML is needed for the bounded slice. | -| ERD/data | `docs/architecture/diagrams.md`, ADR-0003 | Logical artifact relationships and persistence status are explicit. | Physical ERD is intentionally not applicable until persistence exists. | -| Security/privacy | `docs/engineering/youtube-known-stem-validation.md`, `docs/security/app-security.md`, ADRs | Threats, trust boundaries, non-collection, integrity, cleanup, and legal limits covered; exact model bytes use the reviewed restricted loader and serialized one-time cache. | Approved-pickle semantic/provenance risk acceptance plus rights/platform authorization remain open. | +| UML | `docs/architecture/diagrams.md` | Component, sequence, state, implementation-class, evidence-aggregate, and deployment views included. | No additional UML is needed for the bounded slice. | +| ERD/data | `docs/architecture/diagrams.md`, ADR-0003 | Logical run/evidence provenance and optional measured blocks are explicit, including early failure records. | A physical database ERD is intentionally not applicable unless relational persistence is introduced. | +| Security/privacy | `docs/engineering/youtube-known-stem-validation.md`, `docs/security/app-security.md`, ADRs | Threats, trust boundaries, non-collection, sanitized command/tool identity, integrity, cleanup, and legal limits covered; exact model bytes use the reviewed restricted loader and serialized one-time cache. | Exact-checkpoint approved-pickle risk acceptance plus rights/platform authorization remain open. | | Test strategy | `docs/TRD.md`, operator guide, acceptance criteria | Offline/live split and metric/failure contracts covered. | No successful live score has been recorded. | | MIR doctoring | `docs/doctoring/real-audio-accuracy-acceptance.md` | Issue #770 metrics, claim boundaries, tiers, and roadmap are separated from the bounded vocal slice. | Accuracy manifest, reports, other MIR families, and corpus tiers remain open. | -| Operations/release | runbook and release policy | Preflight, evidence, triage, rollback, and blocking conditions covered. | Platform matrix and live pass are incomplete. | -| Supply chain | supplemental inventory and dependency policy | Retired model removed; code/inventory artifact parity, fail-closed same-byte restricted loading, exact allowlist mutation guards, uv.lock-bound yt-dlp, and verified ffmpeg/ffprobe evidence contract recorded. | Model provisioning/distribution rights and any future non-pickle conversion remain unresolved. | +| Operations/release | runbook and release policy | Prospective preflight, schema-v1 evidence, platform scope, triage, rollback, and blocking conditions covered. | Retention stays disabled; platform matrix and live passes are incomplete. | +| Supply chain | supplemental inventory, ADR-0001, and dependency policy | Retired model removed; code/inventory artifact parity, fail-closed same-byte restricted loading, exact allowlist mutation guards, uv.lock-bound yt-dlp, sanitized ffmpeg/ffprobe identity, and explicit pickle-risk closure criteria recorded. | Model rights/delivery and the exact-checkpoint pickle-risk decision remain unresolved. | | Automation | active CWL autonomous loop and `docs/workflow/pr-review-merge-scheduler.md` | BandScope continuity and no-status-only termination are covered without creating a competing writer. | Dedicated BandScope loop remains paused due writer topology/active-task capacity. | | Review governance | `docs/security/github-required-checks.md`, governance, gitflow, contributing, bootstrap policy | Stable checks and review are cumulative; qualifying evidence is an exact-head completed CodeRabbit artifact or exact-head independent non-author `APPROVED` review. Status-only, rate-limited, author, or predecessor evidence is excluded. | A provider rate limit can still defer review, blocking only merge. | ## Requirement-to-evidence traceability -| Requirement | Decision/research | Module or artifact | Test/evidence | Release control | -|---|---|---|---|---| -| PRD-KS-001, KS-007 | ADR-0002; YouTube Terms | `bandscope_analysis.youtube` | `test_youtube.py`; opted-in live test | Authorization preflight | -| PRD-KS-002, KS-004 | ADR-0001/0002; Rouard et al. (2023) | `separation/audio_separator.py` | `test_youtube_stem_e2e.py` live case | Exact model identity and supported platform | -| PRD-KS-003 | Le Roux et al. (2019) | `tests/known_stem_benchmark.py` | SI-SDR unit tests and live threshold | Calibration plus exact-candidate score | -| PRD-KS-005 | ADR-0002 | master identity plus composed global alignment helpers | delayed/composed-window tests; live duration/correlation | Authorized YouTube calibration and drift triage | -| PRD-KS-006, KS-010 | ADR-0002 | pytest marker and failure taxonomy | Every collected default offline test; explicit live failure | Advisory until promotion ADR | -| PRD-KS-008 | ADR-0003 | temporary directory and sanitized errors | cleanup postcondition and archive failure tests | Evidence excludes raw media/paths | -| PRD-KS-009 | ADR-0003; NIST AI RMF TEVV | planned bounded evidence schema | No retained score yet | Required before blocking release gate | -| TRD-KS-011 | ADR-0001 | separator manifest, restricted-loader allowlist, serialized load lock, and supplemental inventory | exact filename/hash/size parity; same-byte `weights_only=True`; strict construction; concurrency/read-once and mutation tests; real-artifact load smoke | Approved-pickle risk acceptance and model-rights/legal delivery blocker; any hash/allowlist/dependency change requires new smoke evidence | +| Product requirement(s) | Technical requirement(s) | Decision/research | Module or artifact | Test/evidence | Release control | +|---|---|---|---|---|---| +| PRD-KS-001, PRD-KS-007, PRD-KS-010 | TRD-KS-001 | ADR-0002; YouTube Terms | `bandscope_analysis.youtube` | production downloader policy tests and opted-in live case | Authorization, duration/size bounds, and four-part media-runtime preflight | +| PRD-KS-005, PRD-KS-007, PRD-KS-010 | TRD-KS-002 | ADR-0002 | `KnownStemFixture` and verified reference/master loaders | exact host, redirect, byte-size, and full-hash tests | Fixture change requires rights, provenance, and integrity review | +| PRD-KS-007, PRD-KS-008, PRD-KS-010 | TRD-KS-003 | ADR-0002/0003 | bounded streaming and one-member archive reader | hostile redirect/archive/member/size tests | No `extractall()`; ephemeral storage only | +| PRD-KS-003, PRD-KS-005 | TRD-KS-004 | ADR-0002 | `align_active_reference_window`, `align_known_stem_through_master` | delayed-window, polarity, composed-lag, and no-prediction-realignment tests | Authorized candidate identity and calibration required | +| PRD-KS-002, PRD-KS-004 | TRD-KS-005 | ADR-0001/0002; Rouard et al. (2023) | `AudioStemSeparator` and canonical separation result | finite/equal-shape/canonical-stem tests and live production-boundary assertion | Exact model identity and a pass for every advertised OS/architecture | +| PRD-KS-003 | TRD-KS-006 | Le Roux et al. (2019) | `zero_mean_si_sdr` | hand-defined metric, silence, shape, finite, and offset-invariance tests | Threshold calibration before blocking promotion | +| PRD-KS-003, PRD-KS-004 | TRD-KS-007 | ADR-0002 | SI-SDR improvement and named-stem assignment assertions | offline score/margin tests and creator-master calibration | Authorized exact-candidate passing scores required | +| PRD-KS-005, PRD-KS-010 | TRD-KS-008 | ADR-0002 | duration and identity-drift gates | duration/correlation negative cases before separator invocation | Drift/flake owner and triage record | +| PRD-KS-006, PRD-KS-007, PRD-KS-010 | TRD-KS-009 | ADR-0002 | pytest live marker, environment guard, and preflight | required-suite marker exclusion and explicit failure cases | Advisory until a superseding promotion ADR | +| PRD-KS-008, PRD-KS-010 | TRD-KS-010 | ADR-0003 | nested temporary roots and cleanup postcondition | success/failure cleanup and path-redaction tests | No raw media/stem retention | +| PRD-KS-002, PRD-KS-010 | TRD-KS-011 | ADR-0001 | separator manifest, restricted-loader allowlist, serialized load lock, and supplemental inventory | filename/hash/size parity; same-byte `weights_only=True`; strict construction; concurrency/read-once and mutation tests; real-artifact load smoke | Security-owner exact-hash/dependency-lock pickle-risk record plus separate rights/legal decision; hash/allowlist/dependency changes trigger re-review | +| PRD-KS-008, PRD-KS-009 | TRD-KS-012 | ADR-0003; NIST AI RMF TEVV | schema-v1 `BenchmarkRun`/`BenchmarkEvidence` aggregate and sanitized operator template | schema/invariant design and historical failure classification exist; emitter/store and retained pass do not | Store/access/TTL/deletion controls and exact-candidate per-platform artifacts required before blocking promotion | +| PRD-KS-011 | TRD-KS-013 | Product failure-experience contract; app-security safe-error rules | planned typed engine/desktop import, model, decode, separation, and recovery states | current downloader/model fallbacks are partial; distinct end-to-end copy/state tests remain planned | Capability cannot claim complete recoverable failure UX until every state and fallback is accepted | ## Live evidence snapshot | Date | Commit under test | Offline contract | Live result | Classification | |---|---|---|---|---| | 2026-08-09 | `5a3648a11d9097b8da48bb4a3ccbd97986aec25b` | 13 passed | Reference archive verified; YouTube download failed with HTTP 502 before separation; no score | Exact failure evidence, not a pass | -| 2026-08-09 | `6e937a34f9036d92e909db3ce8848a5c39dc8e3b` (published byte-identical implementation tree) | Full quickcheck: 680 Python passed, 24 skipped, live marker deselected; 100% source coverage | Archive, extracted vocal, creator master, and pre-provisioned model hash verified; production YouTube download failed with HTTP 502 after 65.49 s; no score | Exact implementation-head failure evidence, not a pass | +| 2026-08-09 | `6e937a34f9036d92e909db3ce8848a5c39dc8e3b` (historical exact commit) | Full quickcheck: 680 Python passed, 24 skipped, live marker deselected; 100% source coverage | Archive, extracted vocal, creator master, and pre-provisioned model hash verified; production YouTube download failed with HTTP 502 after 65.49 s; no score | Historical exact-commit failure evidence; neither a pass nor current-head evidence | The 13-test row is a pre-correction partial suite, not a competing total. It did not contain `test_download_verified_creator_master_authenticates_exact_file`, @@ -74,15 +82,24 @@ YouTube and is not a live pass. ## Machine-checkable contract `scripts/checks/verify_docs.py` requires the canonical index, PRD, TRD, ADR index and records, -diagram authority, and this matrix; checks cross-links from architecture and the index; and requires -contributing, governance, gitflow, bootstrap, and GitHub bootstrap policy to link the canonical -required-check authority so review policy cannot silently fork. +diagram authority, and this matrix; checks cross-links from architecture and the index; requires +every PRD/TRD ID declared in its visible requirements-table row to appear in a visible traceability +table row; rejects undeclared trace IDs; and requires contributing, governance, gitflow, bootstrap, +and GitHub bootstrap policy to link the canonical required-check authority so review policy cannot +silently fork. Both documentation checks share `scripts/checks/markdown_sections.py`, which uses +the directly pinned `markdown-it-py 4.0.0` CommonMark parser with its table rule enabled. Only +rendered top-level headings and rendered canonical outer-pipe tables count: fenced, commented, raw +HTML, or list-nested lookalikes do not. The checker requires exactly one canonical requirements +section/table per PRD and TRD, unique source IDs in the correct family, exactly one six-column +traceability section/table, plain PRD/TRD IDs in their respective columns, and nonempty +decision/module/evidence/release-control cells. `scripts/checks/verify_supply_chain.py` derives the configured separator model and exact code-owned filename/hash/size manifest, then rejects inventory drift. It also binds the yt-dlp record to `uv.lock`, requires both ffmpeg and ffprobe operator records, rejects the retired bandsplit profile, and validates every model artifact's schema, types, full SHA-256, positive non-boolean size, and -HTTPS source. `scripts/checks/verify_security_notes.py` recursively requires the exact canonical -`## Security Notes` section in every plan. `scripts/checks/security_gates.py` permits only the one +HTTPS source. `scripts/checks/verify_security_notes.py` recursively requires the exact visible +canonical `## Security Notes` section and all six visible H3 subsection headings in every plan. +`scripts/checks/security_gates.py` permits only the one exact full-hash same-byte `torch.load` call, requires its rule-specific Semgrep and Bandit suppressions in place, and binds it to `weights_only=True`, the exact global allowlist, and no other allowlist mutation API. @@ -90,5 +107,5 @@ allowlist mutation API. ## Re-evaluation triggers Re-run this matrix whenever the model/signature, fixture, threshold, downloader, separator output -contract, supported platform, persistence policy, evidence retention, workflow scheduling, or -release-blocking status changes. +contract, failure UX, supported platform, evidence schema, persistence policy, evidence retention, +workflow scheduling, or release-blocking status changes. diff --git a/docs/engineering/acceptance-criteria.md b/docs/engineering/acceptance-criteria.md index 8d6061d1d..9eb9c0149 100644 --- a/docs/engineering/acceptance-criteria.md +++ b/docs/engineering/acceptance-criteria.md @@ -49,7 +49,8 @@ For protected branches, intended checks are documented in `docs/security/github- - A live evidence claim must cross `download_youtube_audio()` and `AudioStemSeparator.separate()` on the same exact candidate, authenticate the separately pinned creator master, compose the two global offsets once, and record duration drift, master identity correlation, baseline/vocal - SI-SDR, improvement, assignment margin, model identity, platform, and cleanup. + SI-SDR, improvement, assignment margin, model identity, platform, and cleanup in the stage-aware + schema-v1 contract. Earlier failures omit later measured blocks rather than inventing values. - The provisional live thresholds are YouTube/master duration drift ≤ 1.0 s, identity correlation ≥ 0.90, vocal SI-SDR improvement ≥ +0.5 dB, and vocal assignment margin ≥ 3.0 dB. The quality thresholds are supported by creator-master calibration only; an authorized YouTube baseline is @@ -58,8 +59,15 @@ For protected branches, intended checks are documented in `docs/security/github- - Skipped, disabled, HTTP/provider-failed, model-unavailable, integrity-failed, drifted, non-finite, predecessor-head, or stale-base execution is not passing evidence. - Before the lane can block a release, ADR-0001/0002 blockers—content/platform authorization, - full-hash pre-load verification, an explicit model-rights/legal delivery decision, - exact-candidate pass, calibration, and supported-platform evidence—must be closed. + full-hash pre-load verification, an explicit model-rights/legal delivery decision, the repository + security owner's exact-hash/dependency-lock approved-pickle acceptance (or an approved non-pickle + replacement), exact-candidate pass, calibration, and per-advertised-platform evidence—must be + closed. +- A pass applies only to its exact OS/architecture. Every release artifact that advertises source + separation needs its own exact-candidate pass; every other artifact must prove the safe fallback. +- Evidence upload remains disabled until governance accepts ADR-0003's store, access, TTL, + deletion-verification, and incident-owner controls. A future retained artifact contains a + sanitized command-template identity and never literal environment values or local paths. ## Evidence policy diff --git a/docs/engineering/youtube-known-stem-validation.md b/docs/engineering/youtube-known-stem-validation.md index 7892eaba6..e890e267d 100644 --- a/docs/engineering/youtube-known-stem-validation.md +++ b/docs/engineering/youtube-known-stem-validation.md @@ -72,7 +72,8 @@ creator-master probe is not a live pass. Install the analysis-engine development dependencies. Resolve sibling ffmpeg and ffprobe programs from one trusted package/build to absolute regular executables and obtain both full SHA-256 values; -`PATH` names alone are not release/live evidence. Provision the exact model file in the user-scoped +`PATH` names alone are not sufficient for release/live preflight. The absolute paths are verified +only at execution time and are never retained. Provision the exact model file in the user-scoped torch.hub checkpoints cache or pass its exact absolute path through `BANDSCOPE_HTDEMUCS_MODEL_PATH` before running. The separator never downloads a missing model. @@ -81,7 +82,9 @@ The exact current model artifact is Demucs 4.0.1 htdemucs signature `955717e8`, `8726e21a993978c7ba086d3872e7608d7d5bfca646ca4aca459ffda844faa8b4`. It is pre-provisioned and not bundled. BandScope rejects a missing, symlinked, non-regular, incorrectly sized, or full-SHA mismatched provisioned file before deserializing the same verified bytes. ADR-0001 keeps the -separate model-rights/legal delivery decision as a release blocker. +separate model-rights/legal delivery decision and the repository security owner's exact-hash, +dependency-lock-scoped approved-pickle risk acceptance as release blockers. An approved non-pickle +replacement closes the latter without an exception. Before enabling the test, the operator must confirm that the intended use is permitted by the content rightsholder and the applicable YouTube terms. The creator's permission for the reference @@ -100,6 +103,12 @@ uv run --project services/analysis-engine \ -m youtube_stem_e2e -vv ``` +This block is the sanitized command template `youtube-known-stem-v1`. Local paths and their literal +environment assignments are execution inputs, not evidence fields. A future schema-v1 artifact +retains the template ID/hash plus canonical tool basenames, hashes, versions, trusted-package +identity, and the verified sibling-layout flag. It never retains absolute executable/model paths or +the literal command invocation. + If YouTube access, either fixed reference asset, the verified `ffmpeg`/`ffprobe` executable set, or model weights are unavailable, the opted-in test fails. It must not silently turn an unavailable or changed fixture into a passing result. @@ -107,7 +116,13 @@ changed fixture into a passing result. The four media-runtime fields must identify exact platform-native sibling program names. Their paths, execute permissions, and hashes are verified before the benchmark accesses either reference asset. The model path must use the exact inventoried filename; the production loader then performs -its independent same-byte size and full-hash verification before deserialization. +its independent same-byte size and full-hash verification before deserialization. Only the sanitized +identities described above may enter retained evidence. + +Automated evidence upload/retention is currently disabled. Enabling it requires ADR-0003 governance +to accept the store, access roles, TTL enforcement, deletion verification, and incident owner. Any +artifact must then validate against `docs/TRD.md#benchmark-evidence-schema-v1`; early failures retain +common provenance/stage/cleanup but omit identity or score blocks that were never measured. ## Platform and evidence status @@ -117,6 +132,10 @@ its independent same-byte size and full-hash verification before deserialization - macOS Intel: current dependency markers exclude Demucs; separation must fail safely and offer the product fallback. +A pass is scoped to the exact OS/architecture and unchanged release candidate. Source separation may +be advertised only on each platform/architecture with its own passing record; evidence does not +transfer to another release artifact. + On 2026-08-09, exact commit `5a3648a11d9097b8da48bb4a3ccbd97986aec25b` passed a 13-test pre-correction partial suite. It lacked `test_download_verified_creator_master_authenticates_exact_file`, @@ -132,12 +151,13 @@ collected offline case—its count may grow with regression coverage—plus expl exclusion of the live marker. A creator-master-only calibration produced the provisional scores above without calling YouTube; it is calibration evidence, not exact-candidate success. -The byte-identical implementation tree published on GitHub as exact commit +Historical evidence snapshot: the byte-identical implementation tree published on GitHub as commit `6e937a34f9036d92e909db3ce8848a5c39dc8e3b` passed the full quickcheck. A clean live retry authenticated the archive, extracted vocal, creator master, and pre-provisioned htdemucs full SHA-256. Production YouTube intake again failed closed with `download_failed` after HTTP 502 in -65.49 seconds, before separation. It produced no identity correlation or SI-SDR score and remains -exact implementation-head failure evidence, not a live pass. +65.49 seconds, before separation. It produced no identity correlation or SI-SDR score. This is +historical exact-commit failure evidence—not a live pass or current-head validation. PR #828 owns +current-head offline checks and hosted review evidence, which must be regenerated after each commit. ## Security Notes @@ -180,7 +200,8 @@ the only permitted storage root for downloaded media and extracted references. managed CA trust store when populated and otherwise retains its certifi-backed default. - Release/live execution supplies sibling ffmpeg and ffprobe absolute regular executables plus both full SHA-256 values. A partial identity set, unexpected program name/directory, path drift, or - digest mismatch fails before yt-dlp runs. + digest mismatch fails before yt-dlp runs. Paths are transient verification inputs; future evidence + retains only sanitized identities and never the local paths. - Alignment is global and bounded. Duration and creator-master identity correlation distinguish fixture drift from model quality failure; the two lags are composed once and model outputs are not optimized after separation. Demucs random shift augmentation is disabled with `shifts=0`. @@ -203,10 +224,11 @@ advice, and the test does not establish platform authorization. Upstream media d weights remain separate trust decisions. The fixture has only one full-length known canonical stem, so the test cannot claim quantitative four-stem accuracy. -The model-weight redistribution/provisioning decision is not established, and no successful -exact-candidate live score or supported-platform matrix has yet been retained. Full-SHA pre-load -verification is implemented, but these remaining items are explicit release blockers rather than -undocumented assumptions. +The model-weight redistribution/provisioning decision and exact-checkpoint approved-pickle risk +acceptance are not established, and no successful exact-candidate live score or matrix covering +every advertised platform has yet been retained. Evidence retention itself remains disabled pending +the accepted store/access/TTL/deletion policy. Full-SHA pre-load verification is implemented, but +these remaining items are explicit release blockers rather than undocumented assumptions. ## References diff --git a/docs/operations/deploy-runbook.md b/docs/operations/deploy-runbook.md index 5c97db8fc..33fcd35c0 100644 --- a/docs/operations/deploy-runbook.md +++ b/docs/operations/deploy-runbook.md @@ -26,28 +26,38 @@ When runtime behavior is touched, verify: ## Source-separation preflight and evidence -For a release candidate that claims YouTube source separation: +This is a prospective release procedure. Live execution may be inspected locally, but automated +evidence upload/retention and a release-blocking claim are disabled until ADR-0003's exact store, +access roles, TTL enforcement, deletion verification, and incident owner are accepted. + +After those controls are accepted, repeat this procedure on every OS/architecture where the release +advertises YouTube source separation: 1. record exact commit, live base tip, lockfiles, OS, architecture, Python, Demucs, torch, and the exact locked yt-dlp version; 2. resolve sibling ffmpeg and ffprobe programs from one trusted package/build to absolute regular - executables with exact platform-native names (`ffmpeg`/`ffprobe`, or their `.exe` forms), record + executables with exact platform-native names (`ffmpeg`/`ffprobe`, or their `.exe` forms), verify both full SHA-256 values, trusted package identity, and version outputs, then pass `BANDSCOPE_FFMPEG_PATH`, `BANDSCOPE_FFMPEG_SHA256`, `BANDSCOPE_FFPROBE_PATH`, and `BANDSCOPE_FFPROBE_SHA256`; the benchmark verifies all four before any fixture access, and a - partial set, layout drift, name drift, or mismatch fails preflight; -3. confirm content/platform authorization and do not provide cookies, credentials, login, paywall, - DRM, geo, or anti-bot bypasses; + partial set, layout drift, name drift, or mismatch fails preflight; absolute paths are transient + inputs, while future evidence retains only basenames, hashes, versions, trusted-package identity, + and the sibling-layout result; +3. confirm content/platform authorization, record its non-sensitive governance reference, and do not + provide cookies, credentials, login, paywall, DRM, geo, or anti-bot bypasses; 4. verify the htdemucs model's exact source, 84,141,911-byte size, and full SHA-256 from the supplemental inventory, then set `BANDSCOPE_HTDEMUCS_MODEL_PATH` to the exact absolute `955717e8-8726e21a.th` path; fail closed on a wrong filename, symlink, mismatch, or missing - artifact; + artifact; do not retain that local path, and require both the model-rights/legal record and the + repository security owner's exact-hash approved-pickle risk record from ADR-0001; 5. authenticate the archive, extracted vocal member, and finished master by exact host, byte count, and full SHA-256; record the master duration and require deterministic Demucs `shifts=0`; -6. run the offline known-stem contract, then the explicit live command from - `docs/engineering/youtube-known-stem-validation.md` on the unchanged candidate; -7. retain bounded numeric/provenance evidence only: duration drift, identity correlation, composed - lags, baseline/vocal SI-SDR, improvement, assignment margin, outcome code, and cleanup result; +6. run the offline known-stem contract, then the sanitized live command template from + `docs/engineering/youtube-known-stem-validation.md` on the unchanged candidate; retain the + template ID/hash, never literal environment assignments or local paths; +7. if retention has been authorized, validate the schema-v1 `BenchmarkRun` artifact in + `docs/TRD.md#benchmark-evidence-schema-v1`; every outcome has common provenance/stage/cleanup, + while identity and score blocks exist only when those stages were reached; 8. verify the temporary media root is empty and no raw audio, archive content, full path, URL, cookie, credential, or provider response was retained. @@ -67,8 +77,9 @@ skip/pass. - Rollback removes release-blocking/live scheduling and restores the previous exact approved model; it does not restore the retired FFT profile or weaken intake/security tests. -Evidence artifacts expire after 30 days unless release governance approves a different TTL. Raw -media is never an evidence artifact. +The proposed initial TTL is 30 days, but it is not operative by this document alone. Governance must +accept the store, readers/writers, incident owner, TTL mechanism, and deletion verification before +the first artifact is uploaded. Raw media is never an evidence artifact. ## Incident handling note diff --git a/docs/plans/2026-03-10-bandscope-harness.md b/docs/plans/2026-03-10-bandscope-harness.md index 21060df01..4188a67aa 100644 --- a/docs/plans/2026-03-10-bandscope-harness.md +++ b/docs/plans/2026-03-10-bandscope-harness.md @@ -57,7 +57,7 @@ The harness must keep security guidance visible and fail-fast. Future work that **Step 2: Run docs check and confirm required docs exist** -Run: `python3 scripts/checks/verify_docs.py` +Run: `npm run check:docs` Expected: PASS **Security Notes** diff --git a/docs/plans/2026-03-12-issue-32-analysis-orchestration-design.md b/docs/plans/2026-03-12-issue-32-analysis-orchestration-design.md index b99d33a23..fecf489c8 100644 --- a/docs/plans/2026-03-12-issue-32-analysis-orchestration-design.md +++ b/docs/plans/2026-03-12-issue-32-analysis-orchestration-design.md @@ -88,9 +88,30 @@ The initial result will return the existing demo rehearsal song fixture through ## Security Notes -- Attack surface: React invoke payloads, Rust command handlers, Python subprocess stdin/stdout. -- Trust boundary: frontend -> Tauri IPC -> Python engine subprocess. -- Realistic threats: malformed payload injection, unknown IPC command use, accidental path leakage, raw subprocess error exposure. -- Mitigations: explicit command allowlist, JSON shape validation in all layers, in-memory job store only, redacted error mapping, subprocess argument arrays only. -- Remaining risk: the engine still returns a demo payload, so later audio-backed work must preserve the same validation discipline when real file paths arrive. -- Test points: reject malformed request shapes, reject unknown job ids, verify subprocess errors map to typed safe failures, verify no local HTTP listener is introduced. +### Attack surface + +React invoke payloads, Rust command handlers, and Python subprocess stdin/stdout. + +### Trust boundary + +Frontend -> Tauri IPC -> Python engine subprocess. + +### Realistic threats + +Malformed payload injection, unknown IPC command use, accidental path leakage, and raw subprocess +error exposure. + +### Mitigations + +Explicit command allowlist, JSON shape validation in all layers, in-memory job store only, redacted +error mapping, and subprocess argument arrays only. + +### Remaining risk + +The engine still returns a demo payload, so later audio-backed work must preserve the same +validation discipline when real file paths arrive. + +### Test points + +Reject malformed request shapes, reject unknown job IDs, verify subprocess errors map to typed safe +failures, and verify no local HTTP listener is introduced. diff --git a/docs/release/release-policy.md b/docs/release/release-policy.md index 26db76452..a2366a791 100644 --- a/docs/release/release-policy.md +++ b/docs/release/release-policy.md @@ -33,17 +33,29 @@ BandScope distributes release artifacts through GitHub Releases. decode, separation, alignment, metrics, model delivery, or fixture metadata. - Live known-stem evidence is advisory while ADR-0002 is Proposed. It becomes blocking only through a superseding/accepted ADR after authorization, full-hash pre-load model verification, an explicit - model-rights/legal delivery decision, calibrated thresholds, supported-platform evidence, and a - stable bounded evidence artifact exist. + model-rights/legal delivery decision, repository-security acceptance of the exact-checkpoint + approved-pickle risk (or an approved non-pickle replacement), calibrated thresholds, + platform-scoped evidence, and an authorized schema-v1 bounded evidence artifact exist. +- The approved-pickle record must name its security owner, exact model SHA-256, dependency lock, + allowlist, exact-artifact smoke/mutation evidence, rollback, review date, and expiry/re-review + trigger. It is independent of the model-rights/legal delivery decision. - A release must not advertise verified source-separation quality unless the exact integrated - release candidate records a passing live production-path run. A skipped, provider-failed, stale, - or predecessor-head result does not transfer. + release candidate records a passing live production-path run on every OS/architecture for which + that release advertises the capability. A skipped, provider-failed, stale, predecessor-head, or + different-platform result does not transfer; other artifacts must advertise and exercise the safe + fallback. - Release artifacts must identify the exact htdemucs signature/hash and whether weights are bundled or pre-provisioned. Runtime fetching is forbidden; current policy requires a verified pre-provisioned cache or exact `BANDSCOPE_HTDEMUCS_MODEL_PATH` and does not authorize model-weight redistribution. -- Live evidence must identify sibling ffmpeg/ffprobe executables from one trusted package/build by - exact platform-native name, absolute path, full SHA-256, and version output before fixture access. +- Live preflight must transiently verify sibling ffmpeg/ffprobe executables from one trusted + package/build by exact platform-native name, absolute path, full SHA-256, and version output before + fixture access. Retained evidence contains only their canonical basenames, hashes, version outputs, + shared trusted-package identity, and sibling-layout result; it never contains local paths. Verifying ffmpeg alone is insufficient because yt-dlp may execute ffprobe during postprocessing. +- Evidence upload and retention remain disabled until governance accepts the exact store, access + roles, TTL enforcement, deletion verification, and incident owner required by ADR-0003. Once + enabled, artifacts must validate against `docs/TRD.md#benchmark-evidence-schema-v1`; the literal + command environment and local executable/model paths remain forbidden. - Release rollback must preserve deterministic metric/security coverage and remove any invalid quality claim, scheduled live access, or unverified model artifact. diff --git a/package.json b/package.json index a71236ed0..a6378b5e0 100644 --- a/package.json +++ b/package.json @@ -13,8 +13,8 @@ "scripts": { "ci": "./scripts/harness/quickcheck.sh", "lint:workspaces": "npm run lint --workspaces --if-present", - "check:docs": "python3 scripts/checks/verify_docs.py", - "check:security-notes": "python3 scripts/checks/verify_security_notes.py", + "check:docs": "python3 scripts/checks/run_analysis_command.py python ../../scripts/checks/verify_docs.py", + "check:security-notes": "python3 scripts/checks/run_analysis_command.py python ../../scripts/checks/verify_security_notes.py", "check:security-gates": "python3 scripts/checks/security_gates.py", "check:supply-chain": "python3 scripts/checks/verify_supply_chain.py", "check:github-bootstrap": "python3 scripts/checks/verify_github_bootstrap_policy.py", diff --git a/scripts/checks/markdown_sections.py b/scripts/checks/markdown_sections.py new file mode 100644 index 000000000..d1c76edcc --- /dev/null +++ b/scripts/checks/markdown_sections.py @@ -0,0 +1,252 @@ +"""Parse the bounded GFM block structure used by repository policy checks.""" + +from typing import NamedTuple + +from markdown_it import MarkdownIt +from markdown_it.token import Token + +MARKDOWN = MarkdownIt("commonmark", {"html": True}).enable("table") + + +class MarkdownHeading(NamedTuple): + """Describe one rendered top-level Markdown heading span.""" + + level: int + text: str + start: int + end: int + + +class MarkdownTable(NamedTuple): + """Describe one rendered top-level pipe table.""" + + headers: tuple[str, ...] + rows: tuple[tuple[str, ...], ...] + source_headers: tuple[str, ...] + source_rows: tuple[tuple[str, ...], ...] + start: int + end: int + canonical_outer_pipe: bool + contains_html: bool + + +class MarkdownDocument(NamedTuple): + """Hold normalized source lines and rendered top-level blocks.""" + + lines: list[str] + headings: list[MarkdownHeading] + tables: list[MarkdownTable] + has_unsafe_html: bool + + +def _is_closed_html_comment(content: str) -> bool: + """Return whether HTML source contains only closed comments and whitespace.""" + cursor = 0 + found_comment = False + while cursor < len(content): + while cursor < len(content) and content[cursor] in " \t\r\n": + cursor += 1 + if cursor == len(content): + break + if not content.startswith("", cursor + 4) + if closing < 0: + return False + body = content[cursor + 4 : closing] + if "<" in body or ">" in body: + return False + found_comment = True + cursor = closing + 3 + return found_comment + + +def _token_has_unsafe_html(token: Token) -> bool: + """Return whether a token contains non-comment raw HTML.""" + if token.type == "html_block": + return not _is_closed_html_comment(token.content) + if token.type != "inline": + return False + return any( + child.type == "html_inline" and not _is_closed_html_comment(child.content) + for child in token.children or [] + ) + + +def _visible_inline_text(token: Token) -> str: + """Return rendered semantic text without link targets or HTML attributes.""" + visible: list[str] = [] + for child in token.children or []: + if child.type in {"text", "code_inline", "image"}: + visible.append(child.content) + elif child.type in {"softbreak", "hardbreak"}: + visible.append(" ") + return "".join(visible) + + +def _heading_from_tokens(tokens: list[Token], index: int) -> MarkdownHeading | None: + """Return one top-level rendered heading from a heading-open token.""" + token = tokens[index] + if token.type != "heading_open" or token.level != 0 or token.map is None: + return None + inline = tokens[index + 1] if index + 1 < len(tokens) else None + if inline is None or inline.type != "inline": + return None + return MarkdownHeading( + level=int(token.tag.removeprefix("h")), + text=inline.content.strip(" \t"), + start=token.map[0], + end=token.map[1], + ) + + +def _table_rows( + tokens: list[Token], + start: int, +) -> tuple[list[tuple[str, ...]], list[tuple[str, ...]], int, bool]: + """Return rendered rows and the closing-token index for one table.""" + rows: list[tuple[str, ...]] = [] + source_rows: list[tuple[str, ...]] = [] + row: list[str] | None = None + source_row: list[str] | None = None + cell: list[str] | None = None + source_cell: list[str] | None = None + contains_html = False + index = start + 1 + while index < len(tokens): + token = tokens[index] + if token.type == "table_close": + return rows, source_rows, index, contains_html + if token.type == "tr_open": + row = [] + source_row = [] + elif token.type in {"th_open", "td_open"}: + cell = [] + source_cell = [] + elif token.type == "inline" and cell is not None: + cell.append(_visible_inline_text(token)) + source_cell = source_cell or [] + source_cell.append(token.content) + contains_html = contains_html or any( + child.type == "html_inline" for child in token.children or [] + ) + elif ( + token.type in {"th_close", "td_close"} + and row is not None + and source_row is not None + ): + row.append("".join(cell or []).strip(" \t")) + source_row.append("".join(source_cell or []).strip(" \t")) + cell = None + source_cell = None + elif token.type == "tr_close" and row is not None and source_row is not None: + rows.append(tuple(row)) + source_rows.append(tuple(source_row)) + row = None + source_row = None + index += 1 + return rows, source_rows, len(tokens), contains_html + + +def _table_from_tokens( + tokens: list[Token], + index: int, + lines: list[str], +) -> MarkdownTable | None: + """Return one top-level rendered table from a table-open token.""" + token = tokens[index] + if token.type != "table_open" or token.level != 0 or token.map is None: + return None + rows, source_rows, _, contains_html = _table_rows(tokens, index) + if not rows: + return None + start, end = token.map + source_lines = [line.strip(" \t") for line in lines[start:end] if line.strip(" \t")] + canonical_outer_pipe = bool(source_lines) and all( + line.startswith("|") and line.endswith("|") for line in source_lines + ) + return MarkdownTable( + headers=rows[0], + rows=tuple(rows[1:]), + source_headers=source_rows[0], + source_rows=tuple(source_rows[1:]), + start=start, + end=end, + canonical_outer_pipe=canonical_outer_pipe, + contains_html=contains_html, + ) + + +def _opaque_boundary_from_token(token: Token) -> MarkdownHeading | None: + """Return a fail-closed top-level boundary for rendered raw HTML.""" + if token.map is None or not _token_has_unsafe_html(token): + return None + is_html_block = token.type == "html_block" and token.level == 0 + is_top_level_inline_html = ( + token.type == "inline" + and token.level == 1 + and any(child.type == "html_inline" for child in token.children or []) + ) + if not (is_html_block or is_top_level_inline_html): + return None + return MarkdownHeading(1, "", token.map[0], token.map[1]) + + +def scan_markdown(content: str) -> MarkdownDocument: + """Return rendered top-level headings and tables from normalized Markdown.""" + normalized = content.replace("\r\n", "\n").replace("\r", "\n") + lines = normalized.split("\n") + tokens = MARKDOWN.parse(normalized) + headings: list[MarkdownHeading] = [] + tables: list[MarkdownTable] = [] + has_unsafe_html = False + for index in range(len(tokens)): + has_unsafe_html = has_unsafe_html or _token_has_unsafe_html(tokens[index]) + opaque_boundary = _opaque_boundary_from_token(tokens[index]) + if opaque_boundary is not None: + headings.append(opaque_boundary) + heading = _heading_from_tokens(tokens, index) + if heading is not None: + headings.append(heading) + table = _table_from_tokens(tokens, index, lines) + if table is not None: + tables.append(table) + headings.sort(key=lambda heading: (heading.start, heading.end, heading.level)) + return MarkdownDocument(lines, headings, tables, has_unsafe_html) + + +def section_end( + document: MarkdownDocument, + heading: MarkdownHeading, + *, + maximum_peer_level: int = 2, +) -> int: + """Return the first line of the next rendered top-level peer heading.""" + end = len(document.lines) + for candidate in document.headings: + if candidate.start >= heading.end and candidate.level <= maximum_peer_level: + end = candidate.start + break + return end + + +def section_text( + document: MarkdownDocument, + heading: MarkdownHeading, + *, + maximum_peer_level: int = 2, +) -> str: + """Return raw section source until the next rendered top-level peer heading.""" + end = section_end(document, heading, maximum_peer_level=maximum_peer_level) + return "\n".join(document.lines[heading.end : end]) + + +def section_tables( + document: MarkdownDocument, + heading: MarkdownHeading, + *, + maximum_peer_level: int = 2, +) -> list[MarkdownTable]: + """Return rendered top-level tables inside a canonical section.""" + end = section_end(document, heading, maximum_peer_level=maximum_peer_level) + return [table for table in document.tables if heading.end <= table.start < end] diff --git a/scripts/checks/run_analysis_command.py b/scripts/checks/run_analysis_command.py index aa007bb78..98c4889aa 100644 --- a/scripts/checks/run_analysis_command.py +++ b/scripts/checks/run_analysis_command.py @@ -25,7 +25,11 @@ def _fallback_python() -> str: def _analysis_command(argv: list[str]) -> list[str]: - """Return a uv command, or a local Python module fallback when uv is absent.""" + """Return a local/uv Python script command or Python-module tool command.""" + if argv[0] == "python": + local_python = _fallback_python() + if local_python != sys.executable or not shutil.which("uv"): + return [local_python, *argv[1:]] if shutil.which("uv"): return ["uv", "run", *argv] return [_fallback_python(), "-m", *argv] diff --git a/scripts/checks/verify_docs.py b/scripts/checks/verify_docs.py index d46f83e21..41b423f69 100644 --- a/scripts/checks/verify_docs.py +++ b/scripts/checks/verify_docs.py @@ -3,6 +3,42 @@ import re from pathlib import Path +from markdown_sections import ( + MarkdownDocument, + MarkdownHeading, + MarkdownTable, + scan_markdown, + section_tables, +) + +REPO_ROOT = Path(__file__).resolve().parents[2] +REQUIREMENT_ID_PATTERN = re.compile(r"\b(?:PRD|TRD)-KS-\d{3}\b") +PRODUCT_REQUIREMENT_ID_PATTERN = re.compile(r"PRD-KS-\d{3}") +TECHNICAL_REQUIREMENT_ID_PATTERN = re.compile(r"TRD-KS-\d{3}") +SECURITY_NOTES_HEADING_PATTERN = re.compile(r"^## Security Notes[ \t]*$") +TRACEABILITY_HEADING = "## Requirement-to-evidence traceability" +TRACEABILITY_HEADERS = ( + "Product requirement(s)", + "Technical requirement(s)", + "Decision/research", + "Module or artifact", + "Test/evidence", + "Release control", +) +TRACEABILITY_MATRIX = Path("docs/documentation-coverage-matrix.md") +TRACEABILITY_SOURCES = { + Path("docs/PRD.md"): ( + "Product requirements", + PRODUCT_REQUIREMENT_ID_PATTERN, + ("ID", "Requirement", "Acceptance evidence", "Status"), + ), + Path("docs/TRD.md"): ( + "Technical requirements", + TECHNICAL_REQUIREMENT_ID_PATTERN, + ("ID", "Requirement", "Implementation or proof"), + ), +} + REQUIRED_PATHS = [ Path("README.md"), Path("LICENSE"), @@ -81,7 +117,9 @@ "docs/architecture/diagrams.md", "docs/documentation-coverage-matrix.md", ], - Path("docs/repository/bootstrap-plan.md"): ["docs/security/github-required-checks.md"], + Path("docs/repository/bootstrap-plan.md"): [ + "docs/security/github-required-checks.md" + ], Path("docs/repository/gitflow.md"): ["docs/security/github-required-checks.md"], Path("docs/repository/governance.md"): ["docs/security/github-required-checks.md"], Path("docs/security/github-required-checks.md"): [ @@ -97,9 +135,217 @@ } +def _plain_requirement_ids( + cell: str, + expected_pattern: re.Pattern[str], +) -> tuple[set[str], set[str]]: + """Return expected IDs and plain IDs from the wrong requirement family.""" + requirement_ids: set[str] = set() + wrong_family_ids: set[str] = set() + for token in (item.strip(" \t") for item in cell.split(",")): + if expected_pattern.fullmatch(token): + requirement_ids.add(token) + elif REQUIREMENT_ID_PATTERN.fullmatch(token): + wrong_family_ids.add(token) + return requirement_ids, wrong_family_ids + + +def _canonical_tables( + document: MarkdownDocument, + heading: MarkdownHeading, + expected_headers: tuple[str, ...], +) -> list[MarkdownTable]: + """Return exact-header rendered tables using canonical outer-pipe source.""" + return [ + table + for table in section_tables(document, heading) + if table.headers == expected_headers + and table.source_headers == expected_headers + and table.canonical_outer_pipe + and not table.contains_html + ] + + +def _canonical_h2_headings( + document: MarkdownDocument, + heading_text: str, +) -> list[MarkdownHeading]: + """Return exact column-zero canonical H2 headings from a scanned document.""" + return [ + heading + for heading in document.headings + if heading.level == 2 + and heading.text == heading_text + and document.lines[heading.start].rstrip(" \t") == f"## {heading_text}" + ] + + +def requirement_traceability_violations(root: Path = Path(".")) -> list[str]: + """Return missing and undeclared requirement IDs in the traceability matrix.""" + matrix_path = root / TRACEABILITY_MATRIX + if not matrix_path.exists(): + return [] + matrix_content = matrix_path.read_text(encoding="utf-8") + matrix_document = scan_markdown(matrix_content) + if matrix_document.has_unsafe_html: + return [f"{TRACEABILITY_MATRIX} contains unsupported raw HTML"] + matrix_headings = _canonical_h2_headings( + matrix_document, + "Requirement-to-evidence traceability", + ) + if not matrix_headings: + return [f"{TRACEABILITY_MATRIX} missing section: {TRACEABILITY_HEADING}"] + if len(matrix_headings) != 1: + return [ + f"{TRACEABILITY_MATRIX} has multiple canonical sections: " + f"{TRACEABILITY_HEADING}" + ] + matrix_heading = matrix_headings[0] + traceability_tables = _canonical_tables( + matrix_document, + matrix_heading, + TRACEABILITY_HEADERS, + ) + if not traceability_tables: + return [ + f"{TRACEABILITY_MATRIX} missing canonical requirement traceability table" + ] + if len(traceability_tables) != 1: + return [ + f"{TRACEABILITY_MATRIX} has multiple canonical requirement " + "traceability tables" + ] + traceability_rows = traceability_tables[0].rows + traceability_source_rows = traceability_tables[0].source_rows + if not traceability_rows: + return [ + f"{TRACEABILITY_MATRIX} has empty canonical requirement traceability table" + ] + + declared_by_source: dict[Path, set[str]] = {} + violations: list[str] = [] + source_structure_valid = True + for source, ( + requirement_heading, + requirement_pattern, + requirement_headers, + ) in TRACEABILITY_SOURCES.items(): + source_path = root / source + if source_path.exists(): + source_document = scan_markdown(source_path.read_text(encoding="utf-8")) + if source_document.has_unsafe_html: + violations.append(f"{source} contains unsupported raw HTML") + source_structure_valid = False + continue + source_headings = _canonical_h2_headings( + source_document, + requirement_heading, + ) + if not source_headings: + violations.append(f"{source} missing section: ## {requirement_heading}") + source_structure_valid = False + continue + if len(source_headings) != 1: + violations.append( + f"{source} has multiple canonical sections: ## {requirement_heading}" + ) + source_structure_valid = False + continue + source_heading = source_headings[0] + requirement_tables = _canonical_tables( + source_document, + source_heading, + requirement_headers, + ) + if not requirement_tables: + violations.append(f"{source} missing canonical requirement table") + source_structure_valid = False + continue + if len(requirement_tables) != 1: + violations.append(f"{source} has multiple canonical requirement tables") + source_structure_valid = False + continue + requirement_rows = requirement_tables[0].rows + requirement_source_rows = requirement_tables[0].source_rows + if not requirement_rows: + violations.append(f"{source} has empty canonical requirement table") + source_structure_valid = False + continue + declared_ids: set[str] = set() + for row_number, (row, source_row) in enumerate( + zip(requirement_rows, requirement_source_rows, strict=True), + start=1, + ): + if any(not cell for cell in row): + violations.append( + f"{source} has incomplete canonical requirement row: {row_number}" + ) + requirement_id = source_row[0] + if requirement_pattern.fullmatch(requirement_id): + if requirement_id in declared_ids: + violations.append( + f"{source} declares duplicate requirement: {requirement_id}" + ) + declared_ids.add(requirement_id) + else: + violations.append( + f"{source} has invalid requirement-table ID: {requirement_id}" + ) + declared_by_source[source] = declared_ids + + if not source_structure_valid: + return violations + + declared = ( + set().union(*declared_by_source.values()) if declared_by_source else set() + ) + traced: set[str] = set() + trace_patterns = ( + PRODUCT_REQUIREMENT_ID_PATTERN, + TECHNICAL_REQUIREMENT_ID_PATTERN, + ) + for row_number, (row, source_row) in enumerate( + zip(traceability_rows, traceability_source_rows, strict=True), + start=1, + ): + if any(not cell for cell in row): + violations.append( + f"{TRACEABILITY_MATRIX} has incomplete traceability row: {row_number}" + ) + row_requirement_ids: list[set[str]] = [] + for column, pattern in enumerate(trace_patterns): + requirement_ids, wrong_family_ids = _plain_requirement_ids( + source_row[column], pattern + ) + row_requirement_ids.append(requirement_ids) + traced.update(requirement_ids) + for requirement_id in sorted(wrong_family_ids): + violations.append( + f"{TRACEABILITY_MATRIX} places {requirement_id} in the wrong " + "traceability column" + ) + if any(not requirement_ids for requirement_ids in row_requirement_ids): + violations.append( + f"{TRACEABILITY_MATRIX} row {row_number} must map plain PRD and TRD IDs" + ) + for source, requirement_ids in declared_by_source.items(): + for requirement_id in sorted(requirement_ids - traced): + violations.append( + f"{TRACEABILITY_MATRIX} missing requirement trace: {requirement_id} " + f"(declared in {source})" + ) + for requirement_id in sorted(traced - declared): + violations.append( + f"{TRACEABILITY_MATRIX} references undeclared requirement: {requirement_id}" + ) + return violations + + def documentation_violations(root: Path = Path(".")) -> list[str]: """Return missing canonical files and broken authority-reference violations.""" - violations = [f"missing file: {path}" for path in REQUIRED_PATHS if not (root / path).exists()] + violations = [ + f"missing file: {path}" for path in REQUIRED_PATHS if not (root / path).exists() + ] for path, required_texts in REQUIRED_REFERENCES.items(): absolute_path = root / path if not absolute_path.exists(): @@ -110,18 +356,30 @@ def documentation_violations(root: Path = Path(".")) -> list[str]: violations.append(f"{path} missing reference: {required_text}") plans_root = root / "docs" / "plans" if plans_root.exists(): - security_heading = re.compile(r"^## Security Notes\s*$", re.MULTILINE) for absolute_path in sorted(plans_root.rglob("*.md")): content = absolute_path.read_text(encoding="utf-8") - if security_heading.search(content) is None: + document = scan_markdown(content) + has_security_heading = ( + any( + heading.level == 2 + and heading.text == "Security Notes" + and SECURITY_NOTES_HEADING_PATTERN.fullmatch( + document.lines[heading.start] + ) + for heading in document.headings + ) + and not document.has_unsafe_html + ) + if not has_security_heading: relative_path = absolute_path.relative_to(root) violations.append(f"{relative_path} missing section: ## Security Notes") + violations.extend(requirement_traceability_violations(root)) return violations def main() -> int: """Return a failing exit code when required docs or references are missing.""" - violations = documentation_violations() + violations = documentation_violations(REPO_ROOT) if violations: print("Documentation check failed:") diff --git a/scripts/checks/verify_security_notes.py b/scripts/checks/verify_security_notes.py index 9f5bc0535..869baa75e 100644 --- a/scripts/checks/verify_security_notes.py +++ b/scripts/checks/verify_security_notes.py @@ -3,10 +3,11 @@ import re from pathlib import Path +from markdown_sections import scan_markdown, section_end, section_text + +REPO_ROOT = Path(__file__).resolve().parents[2] SECURITY_NOTES_HEADING = "## Security Notes" -SECURITY_NOTES_PATTERN = re.compile(r"^## Security Notes\s*$") -PEER_HEADING_PATTERN = re.compile(r"^ {0,3}#{1,2}\s+.+\s*$") -FENCE_PATTERN = re.compile(r"^ {0,3}(?P`{3,}|~{3,})") +SECURITY_NOTES_PATTERN = re.compile(r"^## Security Notes[ \t]*$") PLAN_DIR = Path("docs/plans") REQUIRED_SUBSECTIONS = [ "attack surface", @@ -18,39 +19,37 @@ ] -def security_notes_section(content: str) -> str: - """Return the canonical security section, stopping at the next peer heading.""" - lines = content.splitlines() - start = next( - ( - index - for index, line in enumerate(lines) - if SECURITY_NOTES_PATTERN.fullmatch(line) - ), - None, - ) - if start is None: - return "" +def _security_notes_contract(content: str) -> tuple[str, set[str], bool]: + """Return canonical section text, H3 names, and duplicate-section state.""" + document = scan_markdown(content) + if document.has_unsafe_html: + return "", set(), False + headings = [ + candidate + for candidate in document.headings + if candidate.level == 2 + and candidate.text == "Security Notes" + and SECURITY_NOTES_PATTERN.fullmatch(document.lines[candidate.start]) + ] + if len(headings) != 1: + return "", set(), len(headings) > 1 + heading = headings[0] + end = section_end(document, heading) + subsections = { + candidate.text.strip().lower() + for candidate in document.headings + if candidate.level == 3 + and heading.end <= candidate.start < end + and document.lines[candidate.start].rstrip(" \t") == f"### {candidate.text}" + } + section = f"{SECURITY_NOTES_HEADING}\n{section_text(document, heading)}".lower() + return section, subsections, False - section_lines = [lines[start]] - open_fence: tuple[str, int] | None = None - for line in lines[start + 1 :]: - fence_match = FENCE_PATTERN.match(line) - if fence_match is not None: - marker = fence_match.group("marker") - marker_shape = (marker[0], len(marker)) - if open_fence is None: - open_fence = marker_shape - elif ( - marker_shape[0] == open_fence[0] - and marker_shape[1] >= open_fence[1] - and not line[fence_match.end() :].strip() - ): - open_fence = None - elif open_fence is None and PEER_HEADING_PATTERN.fullmatch(line): - break - section_lines.append(line) - return "\n".join(section_lines).lower() + +def security_notes_section(content: str) -> str: + """Return the visible canonical security section up to the next peer heading.""" + section, _, _ = _security_notes_contract(content) + return section def security_notes_violations(repo_root: Path = Path(".")) -> list[str]: @@ -59,15 +58,20 @@ def security_notes_violations(repo_root: Path = Path(".")) -> list[str]: plan_dir = repo_root / PLAN_DIR for path in sorted(plan_dir.rglob("*.md")): content = path.read_text(encoding="utf-8") - section = security_notes_section(content) + section, subsections, duplicate_section = _security_notes_contract(content) display_path = path.relative_to(repo_root).as_posix() + if duplicate_section: + violations.append( + f"{display_path} has multiple canonical sections: {SECURITY_NOTES_HEADING}" + ) + continue if not section: violations.append( f"{display_path} missing section: {SECURITY_NOTES_HEADING}" ) continue for subsection in REQUIRED_SUBSECTIONS: - if subsection not in section: + if subsection not in subsections: violations.append( f"{display_path} missing Security Notes subsection: {subsection}" ) @@ -76,7 +80,7 @@ def security_notes_violations(repo_root: Path = Path(".")) -> list[str]: def main() -> int: """Return a failing exit code when Security Notes or required subsections are missing.""" - violations = security_notes_violations() + violations = security_notes_violations(REPO_ROOT) if violations: print("Missing Security Notes section in:") for violation in violations: diff --git a/scripts/harness/quickcheck.sh b/scripts/harness/quickcheck.sh index f2b87e4e8..1862665b3 100755 --- a/scripts/harness/quickcheck.sh +++ b/scripts/harness/quickcheck.sh @@ -4,8 +4,8 @@ set -euo pipefail REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" cd "$REPO_ROOT" -python3 scripts/checks/verify_docs.py -python3 scripts/checks/verify_security_notes.py +python3 scripts/checks/run_analysis_command.py python ../../scripts/checks/verify_docs.py +python3 scripts/checks/run_analysis_command.py python ../../scripts/checks/verify_security_notes.py python3 scripts/checks/security_gates.py python3 scripts/checks/verify_supply_chain.py python3 scripts/checks/verify_github_bootstrap_policy.py diff --git a/services/analysis-engine/pyproject.toml b/services/analysis-engine/pyproject.toml index c830f63fe..b09ade543 100644 --- a/services/analysis-engine/pyproject.toml +++ b/services/analysis-engine/pyproject.toml @@ -20,6 +20,7 @@ dependencies = [ [dependency-groups] dev = [ "bandit>=1.7.7", + "markdown-it-py==4.0.0", "mypy>=1.15.0", "pytest>=9.0.3", "pytest-cov>=6.0.0", diff --git a/services/analysis-engine/tests/conftest.py b/services/analysis-engine/tests/conftest.py index e926e1e91..5cdd30014 100644 --- a/services/analysis-engine/tests/conftest.py +++ b/services/analysis-engine/tests/conftest.py @@ -2,6 +2,7 @@ from __future__ import annotations +import sys from importlib.util import module_from_spec, spec_from_file_location from pathlib import Path from types import ModuleType @@ -17,7 +18,15 @@ def load_module(relative_path: str, module_name: str) -> ModuleType: assert spec is not None assert spec.loader is not None module = module_from_spec(spec) - spec.loader.exec_module(module) + module_directory = str(module_path.parent) + inserted_module_directory = module_directory not in sys.path + if inserted_module_directory: + sys.path.insert(0, module_directory) + try: + spec.loader.exec_module(module) + finally: + if inserted_module_directory: + sys.path.remove(module_directory) return module diff --git a/services/analysis-engine/tests/test_analysis_command.py b/services/analysis-engine/tests/test_analysis_command.py index 4970c91fd..1e02c01df 100644 --- a/services/analysis-engine/tests/test_analysis_command.py +++ b/services/analysis-engine/tests/test_analysis_command.py @@ -9,6 +9,60 @@ from conftest import load_module +def test_analysis_command_runs_script_with_local_analysis_python( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Use the analysis virtualenv directly for repository Python scripts.""" + runner = load_module( + "scripts/checks/run_analysis_command.py", + "run_analysis_command_local_python_script", + ) + monkeypatch.setattr(runner, "_fallback_python", lambda: "/analysis/python") + monkeypatch.setattr(runner.sys, "executable", "/system/python") + monkeypatch.setattr(runner.shutil, "which", lambda _name: "/usr/bin/uv") + + assert runner._analysis_command(["python", "../../scripts/check.py"]) == [ + "/analysis/python", + "../../scripts/check.py", + ] + + +def test_analysis_command_uses_uv_for_python_script_without_local_venv( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Let uv resolve the project environment when no separate interpreter exists.""" + runner = load_module( + "scripts/checks/run_analysis_command.py", + "run_analysis_command_uv_python_script", + ) + monkeypatch.setattr(runner, "_fallback_python", lambda: runner.sys.executable) + monkeypatch.setattr(runner.shutil, "which", lambda _name: "/usr/bin/uv") + + assert runner._analysis_command(["python", "../../scripts/check.py"]) == [ + "uv", + "run", + "python", + "../../scripts/check.py", + ] + + +def test_analysis_command_runs_python_script_without_uv( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Avoid treating the literal ``python`` launcher as a module name.""" + runner = load_module( + "scripts/checks/run_analysis_command.py", + "run_analysis_command_fallback_python_script", + ) + monkeypatch.setattr(runner, "_fallback_python", lambda: runner.sys.executable) + monkeypatch.setattr(runner.shutil, "which", lambda _name: None) + + assert runner._analysis_command(["python", "../../scripts/check.py"]) == [ + runner.sys.executable, + "../../scripts/check.py", + ] + + def test_analysis_command_isolates_ambient_numba_cache( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/services/analysis-engine/tests/test_documentation_policy.py b/services/analysis-engine/tests/test_documentation_policy.py index afee93e06..4712c8a07 100644 --- a/services/analysis-engine/tests/test_documentation_policy.py +++ b/services/analysis-engine/tests/test_documentation_policy.py @@ -5,6 +5,11 @@ import pytest from conftest import load_module +TRACEABILITY_TABLE_HEADER = ( + "| Product requirement(s) | Technical requirement(s) | Decision/research | " + "Module or artifact | Test/evidence | Release control |" +) + def test_documentation_contract_reports_missing_canonical_authorities(tmp_path: Path) -> None: """Reject a repository that omits the PRD, TRD, ADR index, or diagram authority.""" @@ -41,6 +46,399 @@ def test_documentation_contract_checks_every_nested_plan_security_section( assert "docs/plans/future/unsafe-plan.md missing section: ## Security Notes" in violations +@pytest.mark.parametrize( + "hidden_heading", + ["```markdown\n## Security Notes\n```", ""], +) +def test_documentation_contract_ignores_hidden_plan_security_heading( + tmp_path: Path, + hidden_heading: str, +) -> None: + """Reject a plan whose only canonical-looking security heading is hidden.""" + documentation = load_module( + "scripts/checks/verify_docs.py", + "verify_docs_contract_hidden_plan_security_heading", + ) + plan = tmp_path / "docs" / "plans" / "future" / "unsafe-plan.md" + plan.parent.mkdir(parents=True) + plan.write_text(f"# Plan\n\n{hidden_heading}\n", encoding="utf-8") + + violations = documentation.documentation_violations(tmp_path) + + assert "docs/plans/future/unsafe-plan.md missing section: ## Security Notes" in violations + + +@pytest.mark.parametrize("terminator", ["# Later section", " ## Later section", "Later\n---"]) +def test_documentation_contract_requires_declared_requirement_traceability( + tmp_path: Path, + terminator: str, +) -> None: + """Use real table declarations and stop trace coverage at real peer headings.""" + documentation = load_module( + "scripts/checks/verify_docs.py", + f"verify_docs_contract_requirement_traceability_{terminator.encode().hex()}", + ) + docs = tmp_path / "docs" + docs.mkdir() + (docs / "PRD.md").write_text( + """# PRD + +## Product requirements + +| ID | Requirement | Acceptance evidence | Status | +|---|---|---|---| +| PRD-KS-001 | Product requirement | Evidence | Active | + +| PRD-KS-998 | Bare pipe prose without a delimiter row | +Historical mention PRD-KS-999 is not a declaration row. +""", + encoding="utf-8", + ) + (docs / "TRD.md").write_text( + """# TRD + +## Technical requirements + +| ID | Requirement | Implementation or proof | +|---|---|---| +| TRD-KS-001 | Technical requirement | Proof | +""", + encoding="utf-8", + ) + (docs / "documentation-coverage-matrix.md").write_text( + f"""# Matrix + +```markdown +## Requirement-to-evidence traceability +| PRD-KS-001 | TRD-KS-001 | +``` + +## Requirement-to-evidence traceability + +{TRACEABILITY_TABLE_HEADER} +|---|---|---|---|---|---| +| PRD-KS-001, PRD-KS-999 | none | Decision | Module | Evidence | Control | +| none | [link](https://example.invalid "TRD-KS-001") | Decision | Module | Evidence | Control | + +Not a table. +| none | TRD-KS-001 | +```text +| none | TRD-KS-001 | +``` + +{terminator} + +| none | TRD-KS-001 | +""", + encoding="utf-8", + ) + + assert documentation.requirement_traceability_violations(tmp_path) == [ + "docs/documentation-coverage-matrix.md row 1 must map plain PRD and TRD IDs", + "docs/documentation-coverage-matrix.md row 2 must map plain PRD and TRD IDs", + "docs/documentation-coverage-matrix.md missing requirement trace: TRD-KS-001 " + "(declared in docs/TRD.md)", + "docs/documentation-coverage-matrix.md references undeclared requirement: PRD-KS-999", + ] + + +def test_documentation_contract_requires_traceability_section(tmp_path: Path) -> None: + """Reject a coverage matrix that omits its canonical traceability section.""" + documentation = load_module( + "scripts/checks/verify_docs.py", + "verify_docs_contract_traceability_section", + ) + matrix = tmp_path / "docs" / "documentation-coverage-matrix.md" + matrix.parent.mkdir() + matrix.write_text("# Matrix\n\nNo requirement mapping.\n", encoding="utf-8") + + assert documentation.requirement_traceability_violations(tmp_path) == [ + "docs/documentation-coverage-matrix.md missing section: " + "## Requirement-to-evidence traceability" + ] + + +def test_documentation_contract_rejects_swapped_requirement_families( + tmp_path: Path, +) -> None: + """Bind PRD/TRD declarations and traces to their canonical source and column.""" + documentation = load_module( + "scripts/checks/verify_docs.py", + "verify_docs_contract_swapped_requirement_families", + ) + docs = tmp_path / "docs" + docs.mkdir() + (docs / "PRD.md").write_text( + """# PRD + +## Product requirements + +| ID | Requirement | Acceptance evidence | Status | +|---|---|---|---| +| PRD-KS-001 | Product requirement with an escaped \\| pipe | Evidence | Active | +""", + encoding="utf-8", + ) + (docs / "TRD.md").write_text( + """# TRD + +## Technical requirements + +| ID | Requirement | Implementation or proof | +|---|---|---| +| TRD-KS-001 | Technical requirement | Proof | +""", + encoding="utf-8", + ) + (docs / "documentation-coverage-matrix.md").write_text( + f"""# Matrix + +## Requirement-to-evidence traceability + +{TRACEABILITY_TABLE_HEADER} +|---|---|---|---|---|---| +| TRD-KS-001 | PRD-KS-001 | Decision | Module | Evidence | Control | +""", + encoding="utf-8", + ) + + assert documentation.requirement_traceability_violations(tmp_path) == [ + "docs/documentation-coverage-matrix.md places TRD-KS-001 in the wrong traceability column", + "docs/documentation-coverage-matrix.md places PRD-KS-001 in the wrong traceability column", + "docs/documentation-coverage-matrix.md row 1 must map plain PRD and TRD IDs", + "docs/documentation-coverage-matrix.md missing requirement trace: PRD-KS-001 " + "(declared in docs/PRD.md)", + "docs/documentation-coverage-matrix.md missing requirement trace: TRD-KS-001 " + "(declared in docs/TRD.md)", + ] + + +def test_documentation_contract_rejects_duplicate_canonical_trace_section( + tmp_path: Path, +) -> None: + """Reject an ambiguous matrix instead of checking only its first canonical section.""" + documentation = load_module( + "scripts/checks/verify_docs.py", + "verify_docs_contract_duplicate_trace_section", + ) + matrix = tmp_path / "docs" / "documentation-coverage-matrix.md" + matrix.parent.mkdir() + matrix.write_text( + """# Matrix + +## Requirement-to-evidence traceability + +First section. + +## Requirement-to-evidence traceability + +Second section. +""", + encoding="utf-8", + ) + + assert documentation.requirement_traceability_violations(tmp_path) == [ + "docs/documentation-coverage-matrix.md has multiple canonical sections: " + "## Requirement-to-evidence traceability" + ] + + +def test_documentation_contract_rejects_multiple_canonical_trace_tables( + tmp_path: Path, +) -> None: + """Reject multiple separately rendered mapping tables under one authority heading.""" + documentation = load_module( + "scripts/checks/verify_docs.py", + "verify_docs_contract_multiple_trace_tables", + ) + matrix = tmp_path / "docs" / "documentation-coverage-matrix.md" + matrix.parent.mkdir() + table = f"""{TRACEABILITY_TABLE_HEADER} +|---|---|---|---|---|---| +| PRD-KS-001 | TRD-KS-001 | Decision | Module | Evidence | Control |""" + matrix.write_text( + f"""# Matrix + +## Requirement-to-evidence traceability + +{table} + +{table} +""", + encoding="utf-8", + ) + + assert documentation.requirement_traceability_violations(tmp_path) == [ + "docs/documentation-coverage-matrix.md has multiple canonical requirement " + "traceability tables" + ] + + +def test_documentation_contract_rejects_raw_html_wrapped_trace_authority( + tmp_path: Path, +) -> None: + """Reject an inert or DOM-nested requirements graph wrapped in raw HTML.""" + documentation = load_module( + "scripts/checks/verify_docs.py", + "verify_docs_contract_raw_html_wrapped_trace", + ) + matrix = tmp_path / "docs" / "documentation-coverage-matrix.md" + matrix.parent.mkdir() + matrix.write_text( + f"""# Matrix + + +""", + encoding="utf-8", + ) + + assert documentation.requirement_traceability_violations(tmp_path) == [ + "docs/documentation-coverage-matrix.md contains unsupported raw HTML" + ] + + +def test_documentation_contract_rejects_duplicate_source_id_and_incomplete_trace( + tmp_path: Path, +) -> None: + """Require unique declarations and all six nonempty mapping dimensions.""" + documentation = load_module( + "scripts/checks/verify_docs.py", + "verify_docs_contract_duplicate_id_incomplete_trace", + ) + docs = tmp_path / "docs" + docs.mkdir() + (docs / "PRD.md").write_text( + """# PRD + +## Product requirements + +| ID | Requirement | Acceptance evidence | Status | +|---|---|---|---| +| PRD-KS-001 | Product requirement | Evidence | Active | +| PRD-KS-001 | Duplicate requirement | Evidence | Active | +""", + encoding="utf-8", + ) + (docs / "TRD.md").write_text( + """# TRD + +## Technical requirements + +| ID | Requirement | Implementation or proof | +|---|---|---| +| TRD-KS-001 | Technical requirement | Proof | +""", + encoding="utf-8", + ) + (docs / "documentation-coverage-matrix.md").write_text( + f"""# Matrix + +## Requirement-to-evidence traceability + +{TRACEABILITY_TABLE_HEADER} +|---|---|---|---|---|---| +| PRD-KS-001 | TRD-KS-001 | Decision | Module | Evidence | [](#empty) | +""", + encoding="utf-8", + ) + + assert documentation.requirement_traceability_violations(tmp_path) == [ + "docs/PRD.md declares duplicate requirement: PRD-KS-001", + "docs/documentation-coverage-matrix.md has incomplete traceability row: 1", + ] + + +def test_documentation_contract_does_not_join_hidden_source_table_lines( + tmp_path: Path, +) -> None: + """Keep hidden blocks from synthesizing a requirement table header/delimiter pair.""" + documentation = load_module( + "scripts/checks/verify_docs.py", + "verify_docs_contract_hidden_source_table_separator", + ) + docs = tmp_path / "docs" + docs.mkdir() + (docs / "PRD.md").write_text( + """# PRD + +## Product requirements + +| ID | Requirement | Acceptance evidence | Status | + +|---|---|---|---| +| PRD-KS-001 | Product requirement | Evidence | Active | +""", + encoding="utf-8", + ) + (docs / "TRD.md").write_text( + """# TRD + +## Technical requirements + +| ID | Requirement | Implementation or proof | +|---|---|---| +| TRD-KS-001 | Technical requirement | Proof | +""", + encoding="utf-8", + ) + (docs / "documentation-coverage-matrix.md").write_text( + f"""# Matrix + +## Requirement-to-evidence traceability + +{TRACEABILITY_TABLE_HEADER} +|---|---|---|---|---|---| +| PRD-KS-001 | TRD-KS-001 | Decision | Module | Evidence | Control | +""", + encoding="utf-8", + ) + + assert documentation.requirement_traceability_violations(tmp_path) == [ + "docs/PRD.md missing canonical requirement table" + ] + + +def test_documentation_contract_does_not_join_hidden_trace_table_lines( + tmp_path: Path, +) -> None: + """Keep fenced blocks from attaching a later paragraph to the trace table.""" + documentation = load_module( + "scripts/checks/verify_docs.py", + "verify_docs_contract_hidden_trace_table_separator", + ) + matrix = tmp_path / "docs" / "documentation-coverage-matrix.md" + matrix.parent.mkdir() + matrix.write_text( + f"""# Matrix + +## Requirement-to-evidence traceability + +{TRACEABILITY_TABLE_HEADER} +|---|---|---|---|---|---| +```text +hidden separator +``` +| PRD-KS-001 | TRD-KS-001 | Decision | Module | Evidence | Control | +""", + encoding="utf-8", + ) + + assert documentation.requirement_traceability_violations(tmp_path) == [ + "docs/documentation-coverage-matrix.md has empty canonical requirement traceability table" + ] + + def test_security_notes_contract_discovers_nested_plan_without_canonical_section( tmp_path: Path, ) -> None: @@ -61,10 +459,358 @@ def test_security_notes_contract_discovers_nested_plan_without_canonical_section ] +@pytest.mark.parametrize( + "hidden_section", + [ + """```markdown +## Security Notes +Attack surface Trust boundary Mitigations Test points Realistic threats Remaining risk +```""", + """""", + """
+## Security Notes +Attack surface Trust boundary Mitigations Test points Realistic threats Remaining risk +
""", + ], +) +def test_security_notes_contract_ignores_hidden_canonical_opener( + tmp_path: Path, + hidden_section: str, +) -> None: + """Ignore canonical-looking sections inside code fences and HTML comments.""" + hidden_kind = ( + "fence" + if hidden_section.startswith("`") + else "comment" + if hidden_section.startswith("", + ), + ( + "", + ), + ], +) +def test_security_notes_contract_rejects_raw_html_wrapped_policy_section( + tmp_path: Path, + opening: str, + closing: str, +) -> None: + """Reject canonical-looking evidence made inert or DOM-nested by raw HTML.""" + security_notes = load_module( + "scripts/checks/verify_security_notes.py", + f"verify_security_notes_raw_wrapper_{opening.encode().hex()}", + ) + plan_path = tmp_path / "docs" / "plans" / "nested" / "unsafe-plan.md" + plan_path.parent.mkdir(parents=True) + plan_path.write_text( + f"""# Unsafe plan + +{opening} + +## Security Notes + +### Attack surface +Untrusted input. +### Trust boundary +Validate before use. +### Mitigations +Fail closed. +### Test points +Exercise rejection paths. +### Realistic threats +Artifact substitution. +### Remaining risk +Approved artifact provenance. + +{closing} +""", + encoding="utf-8", + ) + + assert security_notes.security_notes_violations(tmp_path) == [ + "docs/plans/nested/unsafe-plan.md missing section: ## Security Notes" + ] + + +@pytest.mark.parametrize("separator", ["\u2028", "\v", "\f"]) +def test_security_notes_contract_rejects_non_gfm_line_separator( + tmp_path: Path, + separator: str, +) -> None: + """Do not treat Unicode, vertical-tab, or form-feed characters as Markdown lines.""" + security_notes = load_module( + "scripts/checks/verify_security_notes.py", + f"verify_security_notes_non_gfm_separator_{ord(separator):x}", + ) + plan_path = tmp_path / "docs" / "plans" / "nested" / "unsafe-plan.md" + plan_path.parent.mkdir(parents=True) + labels = ( + "Attack surface Trust boundary Mitigations Test points Realistic threats Remaining risk" + ) + plan_path.write_text( + f"# Unsafe plan{separator}## Security Notes{separator}{labels}\n", + encoding="utf-8", + ) + + assert security_notes.security_notes_violations(tmp_path) == [ + "docs/plans/nested/unsafe-plan.md missing section: ## Security Notes" + ] + + +@pytest.mark.parametrize( + "hidden_labels", + [ + """```text +### Attack surface +### Trust boundary +### Mitigations +### Test points +### Realistic threats +### Remaining risk +```""", + """""", + """
+### Attack surface +### Trust boundary +### Mitigations +### Test points +### Realistic threats +### Remaining risk +
""", + ], +) +def test_security_notes_contract_ignores_hidden_required_labels( + tmp_path: Path, + hidden_labels: str, +) -> None: + """Require security labels in visible Markdown rather than code or comments.""" + hidden_kind = ( + "fence" + if hidden_labels.startswith("`") + else "comment" + if hidden_labels.startswith("### {label}" + for label in ( + "Attack surface", + "Trust boundary", + "Mitigations", + "Test points", + "Realistic threats", + "Remaining risk", + ) + ) + plan_path.write_text( + f"# Unsafe plan\n\n## Security Notes\n\n{hidden_labels}\n", + encoding="utf-8", + ) + + assert security_notes.security_notes_violations(tmp_path) == [ + "docs/plans/nested/unsafe-plan.md missing Security Notes subsection: attack surface", + "docs/plans/nested/unsafe-plan.md missing Security Notes subsection: trust boundary", + "docs/plans/nested/unsafe-plan.md missing Security Notes subsection: mitigations", + "docs/plans/nested/unsafe-plan.md missing Security Notes subsection: test points", + "docs/plans/nested/unsafe-plan.md missing Security Notes subsection: realistic threats", + "docs/plans/nested/unsafe-plan.md missing Security Notes subsection: remaining risk", + ] + + +def test_security_notes_contract_fails_closed_when_inline_comment_hides_peer( + tmp_path: Path, +) -> None: + """End policy evidence before an ambiguous multiline inline-comment boundary.""" + security_notes = load_module( + "scripts/checks/verify_security_notes.py", + "verify_security_notes_inline_comment_hides_peer", + ) + plan_path = tmp_path / "docs" / "plans" / "nested" / "unsafe-plan.md" + plan_path.parent.mkdir(parents=True) + plan_path.write_text( + """# Unsafe plan + +## Security Notes + +### Attack surface +Untrusted input. +### Trust boundary +Validate before use. +### Mitigations +Fail closed. +### Test points +Exercise rejection paths. + +text + +### Realistic threats +Outside the canonical section. +### Remaining risk +Outside the canonical section. +""", + encoding="utf-8", + ) + + assert security_notes.security_notes_violations(tmp_path) == [ + "docs/plans/nested/unsafe-plan.md missing Security Notes subsection: realistic threats", + "docs/plans/nested/unsafe-plan.md missing Security Notes subsection: remaining risk", + ] + + +def test_security_notes_contract_rejects_non_gfm_fence_closing_whitespace( + tmp_path: Path, +) -> None: + """Do not close a fence with Unicode whitespace that GFM does not permit.""" + security_notes = load_module( + "scripts/checks/verify_security_notes.py", + "verify_security_notes_non_gfm_fence_close", + ) + plan_path = tmp_path / "docs" / "plans" / "nested" / "unsafe-plan.md" + plan_path.parent.mkdir(parents=True) + plan_path.write_text( + """# Unsafe plan + +## Security Notes + +```text +```  +### Attack surface +### Trust boundary +### Mitigations +### Test points +### Realistic threats +### Remaining risk +""", + encoding="utf-8", + ) + + assert security_notes.security_notes_violations(tmp_path) == [ + "docs/plans/nested/unsafe-plan.md missing Security Notes subsection: attack surface", + "docs/plans/nested/unsafe-plan.md missing Security Notes subsection: trust boundary", + "docs/plans/nested/unsafe-plan.md missing Security Notes subsection: mitigations", + "docs/plans/nested/unsafe-plan.md missing Security Notes subsection: test points", + "docs/plans/nested/unsafe-plan.md missing Security Notes subsection: realistic threats", + "docs/plans/nested/unsafe-plan.md missing Security Notes subsection: remaining risk", + ] + + +@pytest.mark.parametrize("indent", ["", " ", " ", " "]) def test_security_notes_contract_accepts_trailing_space_and_fenced_headings( tmp_path: Path, + indent: str, ) -> None: - """Keep fenced headings inside a canonical section with trailing whitespace.""" + """Keep valid zero-to-three-space GFM fences inside the canonical section.""" security_notes = load_module( "scripts/checks/verify_security_notes.py", "verify_security_notes_fenced_headings", @@ -76,19 +822,25 @@ def test_security_notes_contract_accepts_trailing_space_and_fenced_headings( {security_heading} -Attack surface: untrusted input. -Trust boundary: validate before use. -Mitigations: fail closed. -Test points: exercise rejection paths. - ```text +### Attack surface +Untrusted input. +### Trust boundary +Validate before use. +### Mitigations +Fail closed. +### Test points +Exercise rejection paths. +{indent}```text ```python ## This fenced heading is data ``` ~~~text # This fenced heading is also data ~~~ -Realistic threats: artifact substitution. -Remaining risk: approved artifact provenance. +### Realistic threats +Artifact substitution. +### Remaining risk +Approved artifact provenance. ## Next section @@ -100,12 +852,165 @@ def test_security_notes_contract_accepts_trailing_space_and_fenced_headings( assert "outside the security section" not in security_notes.security_notes_section(plan_content) +@pytest.mark.parametrize( + "peer_heading", + ["#", "##", " #", " ##", "# Next section", " ## Next section"], +) +def test_security_notes_contract_stops_at_valid_atx_peer_heading( + tmp_path: Path, + peer_heading: str, +) -> None: + """Treat empty and named GFM H1/H2 headings as section boundaries.""" + security_notes = load_module( + "scripts/checks/verify_security_notes.py", + f"verify_security_notes_atx_peer_{peer_heading.encode().hex()}", + ) + plan_path = tmp_path / "docs" / "plans" / "nested" / "unsafe-plan.md" + plan_path.parent.mkdir(parents=True) + plan_content = f"""# Unsafe plan + +## Security Notes + +### Attack surface +Untrusted input. +### Trust boundary +Validate before use. +### Mitigations +Fail closed. +### Test points +Exercise rejection paths. +{peer_heading} + +### Realistic threats +This is outside the canonical section. +### Remaining risk +This is outside the canonical section. +""" + plan_path.write_text(plan_content, encoding="utf-8") + + violations = security_notes.security_notes_violations(tmp_path) + + assert violations == [ + "docs/plans/nested/unsafe-plan.md missing Security Notes subsection: realistic threats", + "docs/plans/nested/unsafe-plan.md missing Security Notes subsection: remaining risk", + ] + + +@pytest.mark.parametrize("raw_peer", ["

Actual peer

", "

Actual peer

"]) +def test_security_notes_contract_fails_closed_at_raw_html_peer( + tmp_path: Path, + raw_peer: str, +) -> None: + """Treat rendered top-level raw HTML as an opaque policy-section boundary.""" + security_notes = load_module( + "scripts/checks/verify_security_notes.py", + f"verify_security_notes_raw_html_peer_{raw_peer.encode().hex()}", + ) + plan_path = tmp_path / "docs" / "plans" / "nested" / "unsafe-plan.md" + plan_path.parent.mkdir(parents=True) + plan_path.write_text( + f"""# Unsafe plan + +## Security Notes + +### Attack surface +Untrusted input. +### Trust boundary +Validate before use. +### Mitigations +Fail closed. +### Test points +Exercise rejection paths. + +{raw_peer} + +### Realistic threats +Outside the canonical section. +### Remaining risk +Outside the canonical section. +""", + encoding="utf-8", + ) + + assert security_notes.security_notes_violations(tmp_path) == [ + "docs/plans/nested/unsafe-plan.md missing section: ## Security Notes" + ] + + +def test_security_notes_contract_keeps_h3_subsection_heading(tmp_path: Path) -> None: + """Keep lower-level headings inside the canonical Security Notes section.""" + security_notes = load_module( + "scripts/checks/verify_security_notes.py", + "verify_security_notes_h3_subsection", + ) + plan_path = tmp_path / "docs" / "plans" / "nested" / "safe-plan.md" + plan_path.parent.mkdir(parents=True) + plan_path.write_text( + """# Safe plan + +## Security Notes + +### Attack surface +Untrusted input. +### Trust boundary +Validate before use. +### Mitigations +Fail closed. +### Test points +Exercise rejection paths. +### Realistic threats +Artifact substitution. +### Remaining risk +Approved artifact provenance. +""", + encoding="utf-8", + ) + + assert security_notes.security_notes_violations(tmp_path) == [] + + +def test_security_notes_contract_rejects_list_nested_subsection_headings( + tmp_path: Path, +) -> None: + """Require the six canonical H3 subsections at top-level container depth.""" + security_notes = load_module( + "scripts/checks/verify_security_notes.py", + "verify_security_notes_nested_h3_labels", + ) + plan_path = tmp_path / "docs" / "plans" / "nested" / "unsafe-plan.md" + plan_path.parent.mkdir(parents=True) + labels = "\n".join( + f" ### {label}" + for label in ( + "Attack surface", + "Trust boundary", + "Mitigations", + "Test points", + "Realistic threats", + "Remaining risk", + ) + ) + plan_path.write_text( + f"# Unsafe plan\n\n## Security Notes\n\n- container\n{labels}\n", + encoding="utf-8", + ) + + assert security_notes.security_notes_violations(tmp_path) == [ + "docs/plans/nested/unsafe-plan.md missing Security Notes subsection: attack surface", + "docs/plans/nested/unsafe-plan.md missing Security Notes subsection: trust boundary", + "docs/plans/nested/unsafe-plan.md missing Security Notes subsection: mitigations", + "docs/plans/nested/unsafe-plan.md missing Security Notes subsection: test points", + "docs/plans/nested/unsafe-plan.md missing Security Notes subsection: realistic threats", + "docs/plans/nested/unsafe-plan.md missing Security Notes subsection: remaining risk", + ] + + @pytest.mark.parametrize("indent", [" ", "\t"]) def test_security_notes_contract_rejects_invalid_fence_indentation( tmp_path: Path, indent: str, ) -> None: - """Do not let an indented code block hide the next peer heading.""" + """Fail closed when an indented fence or code block can hide a peer heading.""" security_notes = load_module( "scripts/checks/verify_security_notes.py", f"verify_security_notes_invalid_fence_{indent.encode().hex()}", @@ -116,15 +1021,21 @@ def test_security_notes_contract_rejects_invalid_fence_indentation( ## Security Notes -Attack surface: untrusted input. -Trust boundary: validate before use. -Mitigations: fail closed. -Test points: exercise rejection paths. +### Attack surface +Untrusted input. +### Trust boundary +Validate before use. +### Mitigations +Fail closed. +### Test points +Exercise rejection paths. {indent}```text ## Next section -Realistic threats: this is outside the canonical section. -Remaining risk: this is outside the canonical section. +### Realistic threats +This is outside the canonical section. +### Remaining risk +This is outside the canonical section. """ plan_path.write_text(plan_content, encoding="utf-8") @@ -139,6 +1050,227 @@ def test_security_notes_contract_rejects_invalid_fence_indentation( ) +def test_security_notes_contract_rejects_backtick_in_fence_info(tmp_path: Path) -> None: + """Do not open a GFM backtick fence whose info string contains a backtick.""" + security_notes = load_module( + "scripts/checks/verify_security_notes.py", + "verify_security_notes_invalid_backtick_info", + ) + plan_path = tmp_path / "docs" / "plans" / "nested" / "unsafe-plan.md" + plan_path.parent.mkdir(parents=True) + plan_content = """# Unsafe plan + +## Security Notes + +### Attack surface +Untrusted input. +### Trust boundary +Validate before use. +### Mitigations +Fail closed. +### Test points +Exercise rejection paths. +```bad`info +## Next section + +### Realistic threats +This is outside the canonical section. +### Remaining risk +This is outside the canonical section. +""" + plan_path.write_text(plan_content, encoding="utf-8") + + violations = security_notes.security_notes_violations(tmp_path) + + assert violations == [ + "docs/plans/nested/unsafe-plan.md missing Security Notes subsection: realistic threats", + "docs/plans/nested/unsafe-plan.md missing Security Notes subsection: remaining risk", + ] + + +@pytest.mark.parametrize( + "nested_fence", + [ + "- item\n ```text\n## Actual top-level peer\n```", + "2. item\n ~~~text\n## Actual top-level peer\n~~~", + ], +) +def test_security_notes_contract_respects_list_nested_fence_lifetime( + tmp_path: Path, + nested_fence: str, +) -> None: + """Do not let a list-child fence hide a rendered top-level peer heading.""" + security_notes = load_module( + "scripts/checks/verify_security_notes.py", + f"verify_security_notes_nested_fence_{nested_fence.encode().hex()}", + ) + plan_path = tmp_path / "docs" / "plans" / "nested" / "unsafe-plan.md" + plan_path.parent.mkdir(parents=True) + plan_path.write_text( + f"""# Unsafe plan + +## Security Notes + +### Attack surface +Untrusted input. +### Trust boundary +Validate before use. +### Mitigations +Fail closed. +### Test points +Exercise rejection paths. + +{nested_fence} + +### Realistic threats +Outside the canonical section. +### Remaining risk +Outside the canonical section. +""", + encoding="utf-8", + ) + + assert security_notes.security_notes_violations(tmp_path) == [ + "docs/plans/nested/unsafe-plan.md missing Security Notes subsection: realistic threats", + "docs/plans/nested/unsafe-plan.md missing Security Notes subsection: remaining risk", + ] + + +@pytest.mark.parametrize("underline", ["===", " ---"]) +def test_security_notes_contract_stops_at_setext_peer_heading( + tmp_path: Path, + underline: str, +) -> None: + """Treat GFM Setext H1/H2 headings as canonical section boundaries.""" + security_notes = load_module( + "scripts/checks/verify_security_notes.py", + f"verify_security_notes_setext_peer_{underline.encode().hex()}", + ) + plan_path = tmp_path / "docs" / "plans" / "nested" / "unsafe-plan.md" + plan_path.parent.mkdir(parents=True) + plan_content = f"""# Unsafe plan + +## Security Notes + +### Attack surface +Untrusted input. +### Trust boundary +Validate before use. +### Mitigations +Fail closed. +### Test points +Exercise rejection paths. + +Next section +{underline} + +### Realistic threats +This is outside the canonical section. +### Remaining risk +This is outside the canonical section. +""" + plan_path.write_text(plan_content, encoding="utf-8") + + violations = security_notes.security_notes_violations(tmp_path) + + assert violations == [ + "docs/plans/nested/unsafe-plan.md missing Security Notes subsection: realistic threats", + "docs/plans/nested/unsafe-plan.md missing Security Notes subsection: remaining risk", + ] + assert "next section" not in security_notes.security_notes_section(plan_content) + + +def test_security_notes_contract_excludes_multiline_setext_heading_labels( + tmp_path: Path, +) -> None: + """Exclude every line in a multiline Setext peer heading from the prior section.""" + security_notes = load_module( + "scripts/checks/verify_security_notes.py", + "verify_security_notes_multiline_setext_peer", + ) + plan_path = tmp_path / "docs" / "plans" / "nested" / "unsafe-plan.md" + plan_path.parent.mkdir(parents=True) + plan_path.write_text( + """# Unsafe plan + +## Security Notes + +Attack surface +Trust boundary +Mitigations +Test points +Realistic threats +Remaining risk +Next section +--- +""", + encoding="utf-8", + ) + + assert security_notes.security_notes_violations(tmp_path) == [ + "docs/plans/nested/unsafe-plan.md missing Security Notes subsection: attack surface", + "docs/plans/nested/unsafe-plan.md missing Security Notes subsection: trust boundary", + "docs/plans/nested/unsafe-plan.md missing Security Notes subsection: mitigations", + "docs/plans/nested/unsafe-plan.md missing Security Notes subsection: test points", + "docs/plans/nested/unsafe-plan.md missing Security Notes subsection: realistic threats", + "docs/plans/nested/unsafe-plan.md missing Security Notes subsection: remaining risk", + ] + + +@pytest.mark.parametrize( + "peer_block", + [ + "Next peer\n2. continuation\n---", + "Next peer\n2) continuation\n---", + "Next peer\n continuation\n---", + "Next peer\n\n---", + "Next peer\n\n---", + ], +) +def test_security_notes_contract_fails_closed_at_ambiguous_setext_peer( + tmp_path: Path, + peer_block: str, +) -> None: + """Do not accept H3 evidence after an ambiguous Setext or opaque block boundary.""" + security_notes = load_module( + "scripts/checks/verify_security_notes.py", + f"verify_security_notes_ambiguous_setext_{peer_block.encode().hex()}", + ) + plan_path = tmp_path / "docs" / "plans" / "nested" / "unsafe-plan.md" + plan_path.parent.mkdir(parents=True) + plan_path.write_text( + f"""# Unsafe plan + +## Security Notes + +### Attack surface +Untrusted input. +### Trust boundary +Validate before use. +### Mitigations +Fail closed. +### Test points +Exercise rejection paths. + +{peer_block} + +### Realistic threats +This is outside the canonical section. +### Remaining risk +This is outside the canonical section. +""", + encoding="utf-8", + ) + + expected = [ + "docs/plans/nested/unsafe-plan.md missing Security Notes subsection: realistic threats", + "docs/plans/nested/unsafe-plan.md missing Security Notes subsection: remaining risk", + ] + if "<" in peer_block: + expected = ["docs/plans/nested/unsafe-plan.md missing section: ## Security Notes"] + assert security_notes.security_notes_violations(tmp_path) == expected + + def test_security_notes_contract_accepts_checked_in_plans() -> None: """Accept every checked-in plan only when its complete canonical section is present.""" security_notes = load_module( diff --git a/services/analysis-engine/uv.lock b/services/analysis-engine/uv.lock index 47f7be6ef..626e10212 100644 --- a/services/analysis-engine/uv.lock +++ b/services/analysis-engine/uv.lock @@ -113,6 +113,7 @@ dependencies = [ [package.dev-dependencies] dev = [ { name = "bandit" }, + { name = "markdown-it-py" }, { name = "mypy" }, { name = "pytest" }, { name = "pytest-cov" }, @@ -133,6 +134,7 @@ requires-dist = [ [package.metadata.requires-dev] dev = [ { name = "bandit", specifier = ">=1.7.7" }, + { name = "markdown-it-py", specifier = "==4.0.0" }, { name = "mypy", specifier = ">=1.15.0" }, { name = "pytest", specifier = ">=9.0.3" }, { name = "pytest-cov", specifier = ">=6.0.0" }, From 7b64f0aaa0bd25333574622a0dc5ae43ca73f0fd Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 10 Aug 2026 04:01:41 +0900 Subject: [PATCH 11/34] test(docs): make expected trace messages explicit --- services/analysis-engine/tests/test_documentation_policy.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/services/analysis-engine/tests/test_documentation_policy.py b/services/analysis-engine/tests/test_documentation_policy.py index 4712c8a07..4b918360a 100644 --- a/services/analysis-engine/tests/test_documentation_policy.py +++ b/services/analysis-engine/tests/test_documentation_policy.py @@ -140,7 +140,7 @@ def test_documentation_contract_requires_declared_requirement_traceability( "docs/documentation-coverage-matrix.md row 1 must map plain PRD and TRD IDs", "docs/documentation-coverage-matrix.md row 2 must map plain PRD and TRD IDs", "docs/documentation-coverage-matrix.md missing requirement trace: TRD-KS-001 " - "(declared in docs/TRD.md)", + + "(declared in docs/TRD.md)", "docs/documentation-coverage-matrix.md references undeclared requirement: PRD-KS-999", ] @@ -210,9 +210,9 @@ def test_documentation_contract_rejects_swapped_requirement_families( "docs/documentation-coverage-matrix.md places PRD-KS-001 in the wrong traceability column", "docs/documentation-coverage-matrix.md row 1 must map plain PRD and TRD IDs", "docs/documentation-coverage-matrix.md missing requirement trace: PRD-KS-001 " - "(declared in docs/PRD.md)", + + "(declared in docs/PRD.md)", "docs/documentation-coverage-matrix.md missing requirement trace: TRD-KS-001 " - "(declared in docs/TRD.md)", + + "(declared in docs/TRD.md)", ] From 50f6ddcd16c75fcfcbacb9a04fff355b91fa7339 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 10 Aug 2026 04:02:36 +0900 Subject: [PATCH 12/34] test(youtube): require authorization evidence before live access --- .../tests/test_youtube_stem_e2e.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/services/analysis-engine/tests/test_youtube_stem_e2e.py b/services/analysis-engine/tests/test_youtube_stem_e2e.py index ce9a291ee..fd1caf291 100644 --- a/services/analysis-engine/tests/test_youtube_stem_e2e.py +++ b/services/analysis-engine/tests/test_youtube_stem_e2e.py @@ -548,6 +548,29 @@ def _assert_real_youtube_known_stem_separation(root: Path) -> None: ) +def _require_authorization_ref() -> str: + """Fail closed unless this live run names its governed authorization evidence.""" + authorization_ref = os.environ.get("BANDSCOPE_YOUTUBE_AUTHORIZATION_REF", "").strip() + if not authorization_ref: + pytest.fail( + "authorization_missing: BANDSCOPE_YOUTUBE_AUTHORIZATION_REF is required", + pytrace=False, + ) + return authorization_ref + + +def test_authorization_preflight_requires_a_non_empty_reference( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Reject an opted-in run that cannot identify its authorization evidence.""" + monkeypatch.delenv("BANDSCOPE_YOUTUBE_AUTHORIZATION_REF", raising=False) + with pytest.raises(pytest.fail.Exception, match="authorization_missing"): + _require_authorization_ref() + + monkeypatch.setenv("BANDSCOPE_YOUTUBE_AUTHORIZATION_REF", " governed-record-123 ") + assert _require_authorization_ref() == "governed-record-123" + + @pytest.mark.youtube_stem_e2e @pytest.mark.skipif( os.environ.get("BANDSCOPE_RUN_YOUTUBE_STEM_E2E") != "1", @@ -560,6 +583,7 @@ def _assert_real_youtube_known_stem_separation(root: Path) -> None: ) def test_real_youtube_audio_separates_the_known_vocal_stem(tmp_path: Path) -> None: """Download a real YouTube mix and verify Demucs against its known vocal stem.""" + _require_authorization_ref() with tempfile.TemporaryDirectory(prefix="known-stem-media-", dir=tmp_path) as media_dir: _assert_real_youtube_known_stem_separation(Path(media_dir)) From 0f074084d9ce3474858d8c6046df610bfd3a589a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 10 Aug 2026 04:03:04 +0900 Subject: [PATCH 13/34] docs(youtube): specify authorization preflight contract --- docs/engineering/youtube-known-stem-validation.md | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/docs/engineering/youtube-known-stem-validation.md b/docs/engineering/youtube-known-stem-validation.md index e890e267d..738dd2791 100644 --- a/docs/engineering/youtube-known-stem-validation.md +++ b/docs/engineering/youtube-known-stem-validation.md @@ -90,9 +90,18 @@ Before enabling the test, the operator must confirm that the intended use is per content rightsholder and the applicable YouTube terms. The creator's permission for the reference source does not by itself grant permission for automated access to YouTube. +The live preflight requires a non-empty, opaque `authorization_ref` supplied through +`BANDSCOPE_YOUTUBE_AUTHORIZATION_REF`. It identifies the governed authorization record; it must +not contain credentials or private authorization text. The harness validates this value before it +creates the media workspace or accesses either reference asset, YouTube, or the model. A missing or +blank value terminates at `preflight` with `authorization_missing`, before any network or model +operation. Because evidence emission remains planned, the current harness does not retain or upload +the value. + ```bash UV_CACHE_DIR=/tmp/bandscope-uv-cache \ BANDSCOPE_RUN_YOUTUBE_STEM_E2E=1 \ +BANDSCOPE_YOUTUBE_AUTHORIZATION_REF= \ BANDSCOPE_FFMPEG_PATH=/absolute/trusted/path/to/ffmpeg \ BANDSCOPE_FFMPEG_SHA256=<64-lowercase-hex-digest> \ BANDSCOPE_FFPROBE_PATH=/absolute/trusted/path/to/ffprobe \ @@ -105,9 +114,9 @@ uv run --project services/analysis-engine \ This block is the sanitized command template `youtube-known-stem-v1`. Local paths and their literal environment assignments are execution inputs, not evidence fields. A future schema-v1 artifact -retains the template ID/hash plus canonical tool basenames, hashes, versions, trusted-package -identity, and the verified sibling-layout flag. It never retains absolute executable/model paths or -the literal command invocation. +retains the validated non-secret `authorization_ref`, the template ID/hash, canonical tool +basenames, hashes, versions, trusted-package identity, and the verified sibling-layout flag. It never +retains absolute executable/model paths or the literal command invocation. If YouTube access, either fixed reference asset, the verified `ffmpeg`/`ffprobe` executable set, or model weights are unavailable, the opted-in test fails. It must not silently turn an unavailable or From 1310cbbaa64ca32754229683de580c2e56f5089e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 10 Aug 2026 04:03:20 +0900 Subject: [PATCH 14/34] fix(harness): use cross-platform Python launcher --- package.json | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/package.json b/package.json index a6378b5e0..e6564bb7d 100644 --- a/package.json +++ b/package.json @@ -13,17 +13,17 @@ "scripts": { "ci": "./scripts/harness/quickcheck.sh", "lint:workspaces": "npm run lint --workspaces --if-present", - "check:docs": "python3 scripts/checks/run_analysis_command.py python ../../scripts/checks/verify_docs.py", - "check:security-notes": "python3 scripts/checks/run_analysis_command.py python ../../scripts/checks/verify_security_notes.py", - "check:security-gates": "python3 scripts/checks/security_gates.py", - "check:supply-chain": "python3 scripts/checks/verify_supply_chain.py", - "check:github-bootstrap": "python3 scripts/checks/verify_github_bootstrap_policy.py", - "check:python-docstrings": "python3 scripts/checks/run_analysis_command.py ruff check src tests ../../scripts --select D100,D101,D102,D103,D104,D105,D106,D107", - "ruff:check": "python3 scripts/checks/run_analysis_command.py ruff check src tests", - "ruff:format:check": "python3 scripts/checks/run_analysis_command.py ruff format --check src tests", - "bandit:check": "python3 scripts/checks/run_analysis_command.py bandit -c pyproject.toml -r src", + "check:docs": "python scripts/checks/run_analysis_command.py python ../../scripts/checks/verify_docs.py", + "check:security-notes": "python scripts/checks/run_analysis_command.py python ../../scripts/checks/verify_security_notes.py", + "check:security-gates": "python scripts/checks/security_gates.py", + "check:supply-chain": "python scripts/checks/verify_supply_chain.py", + "check:github-bootstrap": "python scripts/checks/verify_github_bootstrap_policy.py", + "check:python-docstrings": "python scripts/checks/run_analysis_command.py ruff check src tests ../../scripts --select D100,D101,D102,D103,D104,D105,D106,D107", + "ruff:check": "python scripts/checks/run_analysis_command.py ruff check src tests", + "ruff:format:check": "python scripts/checks/run_analysis_command.py ruff format --check src tests", + "bandit:check": "python scripts/checks/run_analysis_command.py bandit -c pyproject.toml -r src", "lint": "npm run lint:workspaces && npm run check:docs && npm run check:security-notes && npm run check:security-gates && npm run check:supply-chain && npm run check:github-bootstrap && npm run check:python-docstrings && npm run ruff:check && npm run ruff:format:check && npm run bandit:check", - "typecheck": "npm run typecheck --workspaces --if-present && python3 scripts/checks/run_analysis_command.py mypy src", + "typecheck": "npm run typecheck --workspaces --if-present && python scripts/checks/run_analysis_command.py mypy src", "test": "node scripts/checks/run_root_tests.mjs", "build": "npm run build --workspaces --if-present", "check:rust": "./scripts/checks/check_rust.sh", From 93fdbe70e605b0546df077b784f5570a2a2859e8 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 10 Aug 2026 04:03:41 +0900 Subject: [PATCH 15/34] fix(harness): run Python checks portably --- scripts/harness/quickcheck.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/harness/quickcheck.sh b/scripts/harness/quickcheck.sh index 1862665b3..e8b066886 100755 --- a/scripts/harness/quickcheck.sh +++ b/scripts/harness/quickcheck.sh @@ -4,11 +4,11 @@ set -euo pipefail REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" cd "$REPO_ROOT" -python3 scripts/checks/run_analysis_command.py python ../../scripts/checks/verify_docs.py -python3 scripts/checks/run_analysis_command.py python ../../scripts/checks/verify_security_notes.py -python3 scripts/checks/security_gates.py -python3 scripts/checks/verify_supply_chain.py -python3 scripts/checks/verify_github_bootstrap_policy.py +python scripts/checks/run_analysis_command.py python ../../scripts/checks/verify_docs.py +python scripts/checks/run_analysis_command.py python ../../scripts/checks/verify_security_notes.py +python scripts/checks/security_gates.py +python scripts/checks/verify_supply_chain.py +python scripts/checks/verify_github_bootstrap_policy.py npm run lint npm run typecheck npm run test From 8e34c0e563fafa08909307109b358cbbebf5537f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 10 Aug 2026 04:16:21 +0900 Subject: [PATCH 16/34] fix(harness): use a cross-platform Python launcher --- CHANGELOG.md | 6 + .../SKILL.md | 4 +- docs/engineering/acceptance-criteria.md | 4 +- docs/engineering/harness-engineering.md | 10 +- package.json | 20 +- scripts/checks/python_launcher.mjs | 41 ++++ scripts/checks/run_python.mjs | 11 + scripts/checks/run_root_tests.mjs | 38 +-- scripts/harness/quickcheck.sh | 10 +- .../tests/test_analysis_command.py | 219 ++++++++++++++++++ 10 files changed, 309 insertions(+), 54 deletions(-) create mode 100644 scripts/checks/python_launcher.mjs create mode 100644 scripts/checks/run_python.mjs diff --git a/CHANGELOG.md b/CHANGELOG.md index f3ec75d54..60e346317 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,9 @@ - Reconciled stale CodeRabbit-gate wording with the canonical stable-check and review-equivalent policy; qualifying evidence is now defined against the exact current head, and a rate-limited, status-only, author, or predecessor review is not treated as completed review evidence. +- Routed root npm/quickcheck Python entry points through a shared Node launcher that selects + `py -3`, `python3`, or `python` in a deterministic platform-specific order without masking + interpreter failures. ### Security Notes @@ -48,6 +51,9 @@ path-and-hash pair is verified before network fixture access; model loading is offline, same-byte, restricted to `weights_only=True` plus the exact reviewed globals, and fails closed without an unrestricted fallback. +- Developer tooling: the cross-platform check launcher is repository-only, invokes only the fixed + `py`, `python3`, or `python` candidates with argument arrays and no shell, and propagates the first + available interpreter's failure instead of retrying past it. - Logging and privacy: raw media, model bytes, separated stems, credentials, and full local paths are not retained in release evidence or emitted in bounded operator errors. - Test points: each candidate head must pass quickcheck, hosted SAST/Bandit/secret/security scans, diff --git a/docs/agents/skills/bandscope-supply-chain-warning-remediation/SKILL.md b/docs/agents/skills/bandscope-supply-chain-warning-remediation/SKILL.md index 6cd9cefc4..a8ce7d33d 100644 --- a/docs/agents/skills/bandscope-supply-chain-warning-remediation/SKILL.md +++ b/docs/agents/skills/bandscope-supply-chain-warning-remediation/SKILL.md @@ -42,8 +42,8 @@ Treat every supply-chain warning as evidence to classify, fix, or track. The goa Run the narrowest command first, then widen as needed: -- `python3 scripts/checks/verify_supply_chain.py` -- `python3 scripts/checks/security_gates.py` +- `node scripts/checks/run_python.mjs scripts/checks/verify_supply_chain.py` +- `node scripts/checks/run_python.mjs scripts/checks/security_gates.py` - `uv run --project services/analysis-engine pytest services/analysis-engine/tests/test_supply_chain_policy.py` - `npm audit --workspaces --audit-level=high` - `BANDSCOPE_ENABLE_RUST_CHECK=1 ./scripts/harness/quickcheck.sh` diff --git a/docs/engineering/acceptance-criteria.md b/docs/engineering/acceptance-criteria.md index 9eb9c0149..d80e0e027 100644 --- a/docs/engineering/acceptance-criteria.md +++ b/docs/engineering/acceptance-criteria.md @@ -27,8 +27,8 @@ Run the narrowest passing set that covers touched areas, and do not claim succes When CI/workflow files, supply-chain controls, or release/security docs are changed, also run: -- `python3 scripts/checks/verify_supply_chain.py` -- `python3 scripts/checks/security_gates.py` +- `node scripts/checks/run_python.mjs scripts/checks/verify_supply_chain.py` +- `node scripts/checks/run_python.mjs scripts/checks/security_gates.py` When runtime-wide confidence is needed, run: diff --git a/docs/engineering/harness-engineering.md b/docs/engineering/harness-engineering.md index 13c0ab37f..f01bbc29d 100644 --- a/docs/engineering/harness-engineering.md +++ b/docs/engineering/harness-engineering.md @@ -19,9 +19,13 @@ Quickcheck aggregates lint/type/test/build and repository policy checks intended ## Supply-chain and workflow policy checks -- `python3 scripts/checks/verify_supply_chain.py` -- `python3 scripts/checks/security_gates.py` -- `python3 scripts/checks/verify_github_bootstrap_policy.py` +- `node scripts/checks/run_python.mjs scripts/checks/verify_supply_chain.py` +- `node scripts/checks/run_python.mjs scripts/checks/security_gates.py` +- `node scripts/checks/run_python.mjs scripts/checks/verify_github_bootstrap_policy.py` + +The Node wrapper selects `py -3`, `python3`, or `python` in a deterministic platform-specific +order. Once a candidate starts, its exit status is authoritative; a failing check never falls +through to another interpreter. ## Python analysis engine notes diff --git a/package.json b/package.json index e6564bb7d..96dbf41aa 100644 --- a/package.json +++ b/package.json @@ -13,17 +13,17 @@ "scripts": { "ci": "./scripts/harness/quickcheck.sh", "lint:workspaces": "npm run lint --workspaces --if-present", - "check:docs": "python scripts/checks/run_analysis_command.py python ../../scripts/checks/verify_docs.py", - "check:security-notes": "python scripts/checks/run_analysis_command.py python ../../scripts/checks/verify_security_notes.py", - "check:security-gates": "python scripts/checks/security_gates.py", - "check:supply-chain": "python scripts/checks/verify_supply_chain.py", - "check:github-bootstrap": "python scripts/checks/verify_github_bootstrap_policy.py", - "check:python-docstrings": "python scripts/checks/run_analysis_command.py ruff check src tests ../../scripts --select D100,D101,D102,D103,D104,D105,D106,D107", - "ruff:check": "python scripts/checks/run_analysis_command.py ruff check src tests", - "ruff:format:check": "python scripts/checks/run_analysis_command.py ruff format --check src tests", - "bandit:check": "python scripts/checks/run_analysis_command.py bandit -c pyproject.toml -r src", + "check:docs": "node scripts/checks/run_python.mjs scripts/checks/run_analysis_command.py python ../../scripts/checks/verify_docs.py", + "check:security-notes": "node scripts/checks/run_python.mjs scripts/checks/run_analysis_command.py python ../../scripts/checks/verify_security_notes.py", + "check:security-gates": "node scripts/checks/run_python.mjs scripts/checks/security_gates.py", + "check:supply-chain": "node scripts/checks/run_python.mjs scripts/checks/verify_supply_chain.py", + "check:github-bootstrap": "node scripts/checks/run_python.mjs scripts/checks/verify_github_bootstrap_policy.py", + "check:python-docstrings": "node scripts/checks/run_python.mjs scripts/checks/run_analysis_command.py ruff check src tests ../../scripts --select D100,D101,D102,D103,D104,D105,D106,D107", + "ruff:check": "node scripts/checks/run_python.mjs scripts/checks/run_analysis_command.py ruff check src tests", + "ruff:format:check": "node scripts/checks/run_python.mjs scripts/checks/run_analysis_command.py ruff format --check src tests", + "bandit:check": "node scripts/checks/run_python.mjs scripts/checks/run_analysis_command.py bandit -c pyproject.toml -r src", "lint": "npm run lint:workspaces && npm run check:docs && npm run check:security-notes && npm run check:security-gates && npm run check:supply-chain && npm run check:github-bootstrap && npm run check:python-docstrings && npm run ruff:check && npm run ruff:format:check && npm run bandit:check", - "typecheck": "npm run typecheck --workspaces --if-present && python scripts/checks/run_analysis_command.py mypy src", + "typecheck": "npm run typecheck --workspaces --if-present && node scripts/checks/run_python.mjs scripts/checks/run_analysis_command.py mypy src", "test": "node scripts/checks/run_root_tests.mjs", "build": "npm run build --workspaces --if-present", "check:rust": "./scripts/checks/check_rust.sh", diff --git a/scripts/checks/python_launcher.mjs b/scripts/checks/python_launcher.mjs new file mode 100644 index 000000000..35cff016a --- /dev/null +++ b/scripts/checks/python_launcher.mjs @@ -0,0 +1,41 @@ +import { spawnSync } from "node:child_process"; +import process from "node:process"; + +export function pythonCandidates(platform = process.platform) { + if (platform === "win32") { + return [ + ["py", ["-3"]], + ["python", []], + ["python3", []], + ]; + } + + return [ + ["python3", []], + ["python", []], + ]; +} + +export function runPython(args, options = {}) { + const { cwd, env, platform = process.platform, stdio = "inherit" } = options; + + for (const [command, prefix] of pythonCandidates(platform)) { + const result = spawnSync(command, [...prefix, ...args], { + cwd, + env, + stdio, + }); + + if (result.error?.code === "ENOENT") { + continue; + } + if (result.error) { + console.error(`Unable to start ${command}: ${result.error.message}`); + return 127; + } + return result.status ?? 1; + } + + console.error("Unable to find a Python interpreter."); + return 127; +} diff --git a/scripts/checks/run_python.mjs b/scripts/checks/run_python.mjs new file mode 100644 index 000000000..35ca83558 --- /dev/null +++ b/scripts/checks/run_python.mjs @@ -0,0 +1,11 @@ +import process from "node:process"; + +import { runPython } from "./python_launcher.mjs"; + +const args = process.argv.slice(2); +if (args.length === 0) { + console.error("Usage: node scripts/checks/run_python.mjs "); + process.exitCode = 2; +} else { + process.exitCode = runPython(args, { cwd: process.cwd() }); +} diff --git a/scripts/checks/run_root_tests.mjs b/scripts/checks/run_root_tests.mjs index 440a40907..48224dbda 100644 --- a/scripts/checks/run_root_tests.mjs +++ b/scripts/checks/run_root_tests.mjs @@ -2,6 +2,8 @@ import { spawnSync } from "node:child_process"; import path from "node:path"; import process from "node:process"; +import { runPython } from "./python_launcher.mjs"; + const workspaceArgs = process.argv.slice(2).filter((arg) => arg !== "--coverage"); function run(command, args) { @@ -25,44 +27,13 @@ function run(command, args) { } } -function runPython(args) { - const candidates = - process.platform === "win32" - ? [ - ["py", ["-3"]], - ["python", []], - ["python3", []], - ] - : [ - ["python3", []], - ["python", []], - ]; - - for (const [command, prefix] of candidates) { - const result = spawnSync(command, [...prefix, ...args], { - stdio: "inherit", - }); - - if (result.error) { - continue; - } - if (result.status !== 0) { - process.exit(result.status ?? 1); - } - return; - } - - console.error("Unable to find a Python interpreter for analysis-engine tests."); - process.exit(127); -} - const npmWorkspaceTestArgs = ["run", "test", "--workspaces", "--if-present"]; if (workspaceArgs.length > 0) { npmWorkspaceTestArgs.push("--", ...workspaceArgs); } run("npm", npmWorkspaceTestArgs); -runPython([ +const pythonStatus = runPython([ "scripts/checks/run_analysis_command.py", "pytest", "tests", @@ -72,3 +43,6 @@ runPython([ "--cov-report=term-missing", "--cov-fail-under=100", ]); +if (pythonStatus !== 0) { + process.exit(pythonStatus); +} diff --git a/scripts/harness/quickcheck.sh b/scripts/harness/quickcheck.sh index e8b066886..57b7dff7c 100755 --- a/scripts/harness/quickcheck.sh +++ b/scripts/harness/quickcheck.sh @@ -4,11 +4,11 @@ set -euo pipefail REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" cd "$REPO_ROOT" -python scripts/checks/run_analysis_command.py python ../../scripts/checks/verify_docs.py -python scripts/checks/run_analysis_command.py python ../../scripts/checks/verify_security_notes.py -python scripts/checks/security_gates.py -python scripts/checks/verify_supply_chain.py -python scripts/checks/verify_github_bootstrap_policy.py +node scripts/checks/run_python.mjs scripts/checks/run_analysis_command.py python ../../scripts/checks/verify_docs.py +node scripts/checks/run_python.mjs scripts/checks/run_analysis_command.py python ../../scripts/checks/verify_security_notes.py +node scripts/checks/run_python.mjs scripts/checks/security_gates.py +node scripts/checks/run_python.mjs scripts/checks/verify_supply_chain.py +node scripts/checks/run_python.mjs scripts/checks/verify_github_bootstrap_policy.py npm run lint npm run typecheck npm run test diff --git a/services/analysis-engine/tests/test_analysis_command.py b/services/analysis-engine/tests/test_analysis_command.py index 1e02c01df..e3bf8bcd6 100644 --- a/services/analysis-engine/tests/test_analysis_command.py +++ b/services/analysis-engine/tests/test_analysis_command.py @@ -2,6 +2,10 @@ from __future__ import annotations +import json +import os +import shutil +import subprocess from pathlib import Path from types import SimpleNamespace @@ -9,6 +13,221 @@ from conftest import load_module +def test_root_check_launchers_use_cross_platform_python() -> None: + """Keep npm and quickcheck entry points reachable across Python installations.""" + repo_root = Path(__file__).resolve().parents[3] + package = json.loads((repo_root / "package.json").read_text(encoding="utf-8")) + python_scripts = ( + "check:docs", + "check:security-notes", + "check:security-gates", + "check:supply-chain", + "check:github-bootstrap", + "check:python-docstrings", + "ruff:check", + "ruff:format:check", + "bandit:check", + "typecheck", + ) + + launcher = "node scripts/checks/run_python.mjs" + assert all(launcher in package["scripts"][name] for name in python_scripts) + quickcheck = (repo_root / "scripts/harness/quickcheck.sh").read_text(encoding="utf-8") + assert quickcheck.count(launcher) == 5 + + +def test_python_launcher_declares_platform_specific_candidate_order() -> None: + """Prefer standard launchers in a deterministic Windows and POSIX order.""" + repo_root = Path(__file__).resolve().parents[3] + launcher_module = (repo_root / "scripts/checks/python_launcher.mjs").as_uri() + node = shutil.which("node") + assert node is not None + expression = ( + f'import {{ pythonCandidates }} from "{launcher_module}"; ' + "console.log(JSON.stringify({" + 'win32: pythonCandidates("win32"), ' + 'linux: pythonCandidates("linux")' + "}));" + ) + + completed = subprocess.run( + [node, "--input-type=module", "--eval", expression], + check=True, + capture_output=True, + text=True, + ) + + assert json.loads(completed.stdout) == { + "win32": [["py", ["-3"]], ["python", []], ["python3", []]], + "linux": [["python3", []], ["python", []]], + } + + +@pytest.mark.skipif(os.name == "nt", reason="POSIX executable fixtures are required") +def test_python_launcher_executes_py_dash_three_for_windows_policy(tmp_path: Path) -> None: + """Exercise the Windows candidate prefix without requiring a Windows host.""" + repo_root = Path(__file__).resolve().parents[3] + launcher_module = (repo_root / "scripts/checks/python_launcher.mjs").as_uri() + node = shutil.which("node") + assert node is not None + py_launcher = tmp_path / "py" + py_launcher.write_text( + '#!/bin/sh\n[ "$1" = "-3" ] || exit 9\nexit 0\n', + encoding="utf-8", + ) + py_launcher.chmod(0o700) + environment = os.environ.copy() + environment["PATH"] = str(tmp_path) + expression = ( + f'import {{ runPython }} from "{launcher_module}"; ' + 'process.exitCode = runPython(["ignored.py"], ' + '{ platform: "win32", env: process.env });' + ) + + completed = subprocess.run( + [node, "--input-type=module", "--eval", expression], + cwd=repo_root, + env=environment, + check=False, + ) + + assert completed.returncode == 0 + + +@pytest.mark.skipif(os.name == "nt", reason="POSIX executable fixtures are required") +def test_python_launcher_uses_python3_only_posix_path(tmp_path: Path) -> None: + """Run successfully where POSIX exposes python3 but no python alias.""" + repo_root = Path(__file__).resolve().parents[3] + node = shutil.which("node") + assert node is not None + python3 = tmp_path / "python3" + python3.symlink_to(Path(os.sys.executable)) + environment = os.environ.copy() + environment["PATH"] = str(tmp_path) + + completed = subprocess.run( + [ + node, + str(repo_root / "scripts/checks/run_python.mjs"), + "-c", + "print('python-launcher-ok')", + ], + cwd=repo_root, + env=environment, + check=False, + capture_output=True, + text=True, + ) + + assert completed.returncode == 0 + assert completed.stdout.strip() == "python-launcher-ok" + + +@pytest.mark.skipif(os.name == "nt", reason="POSIX executable fixtures are required") +def test_python_launcher_falls_back_when_first_candidate_is_missing(tmp_path: Path) -> None: + """Use the next candidate only when the preferred executable is absent.""" + repo_root = Path(__file__).resolve().parents[3] + node = shutil.which("node") + assert node is not None + python = tmp_path / "python" + python.symlink_to(Path(os.sys.executable)) + environment = os.environ.copy() + environment["PATH"] = str(tmp_path) + + completed = subprocess.run( + [ + node, + str(repo_root / "scripts/checks/run_python.mjs"), + "-c", + "print('fallback-ok')", + ], + cwd=repo_root, + env=environment, + check=False, + capture_output=True, + text=True, + ) + + assert completed.returncode == 0 + assert completed.stdout.strip() == "fallback-ok" + + +@pytest.mark.skipif(os.name == "nt", reason="POSIX executable fixtures are required") +def test_python_launcher_does_not_mask_unlaunchable_candidate(tmp_path: Path) -> None: + """Treat a non-ENOENT spawn error as authoritative instead of falling through.""" + repo_root = Path(__file__).resolve().parents[3] + node = shutil.which("node") + assert node is not None + preferred = tmp_path / "python3" + preferred.write_text("not executable\n", encoding="utf-8") + preferred.chmod(0o600) + fallback_marker = tmp_path / "fallback-ran" + fallback = tmp_path / "python" + fallback.write_text('#!/bin/sh\nprintf ran > "$FALLBACK_MARKER"\n', encoding="utf-8") + fallback.chmod(0o700) + environment = os.environ.copy() + environment["PATH"] = str(tmp_path) + environment["FALLBACK_MARKER"] = str(fallback_marker) + + completed = subprocess.run( + [node, str(repo_root / "scripts/checks/run_python.mjs"), "ignored.py"], + cwd=repo_root, + env=environment, + check=False, + capture_output=True, + text=True, + ) + + assert completed.returncode == 127 + assert "Unable to start python3" in completed.stderr + assert not fallback_marker.exists() + + +@pytest.mark.skipif(os.name == "nt", reason="POSIX executable fixtures are required") +def test_python_launcher_does_not_mask_candidate_failure(tmp_path: Path) -> None: + """Return the first available interpreter's failure without trying another.""" + repo_root = Path(__file__).resolve().parents[3] + node = shutil.which("node") + assert node is not None + for name, exit_code in (("python3", 7), ("python", 0)): + candidate = tmp_path / name + candidate.write_text(f"#!/bin/sh\nexit {exit_code}\n", encoding="utf-8") + candidate.chmod(0o700) + environment = os.environ.copy() + environment["PATH"] = str(tmp_path) + + completed = subprocess.run( + [node, str(repo_root / "scripts/checks/run_python.mjs"), "ignored.py"], + cwd=repo_root, + env=environment, + check=False, + ) + + assert completed.returncode == 7 + + +@pytest.mark.skipif(os.name == "nt", reason="POSIX PATH semantics are required") +def test_python_launcher_reports_missing_interpreter(tmp_path: Path) -> None: + """Return 127 instead of silently succeeding when no candidate exists.""" + repo_root = Path(__file__).resolve().parents[3] + node = shutil.which("node") + assert node is not None + environment = os.environ.copy() + environment["PATH"] = str(tmp_path) + + completed = subprocess.run( + [node, str(repo_root / "scripts/checks/run_python.mjs"), "ignored.py"], + cwd=repo_root, + env=environment, + check=False, + capture_output=True, + text=True, + ) + + assert completed.returncode == 127 + assert "Unable to find a Python interpreter" in completed.stderr + + def test_analysis_command_runs_script_with_local_analysis_python( monkeypatch: pytest.MonkeyPatch, ) -> None: From c82d315891c77579bf60d6f1edd38171e865e4e5 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 14 Aug 2026 17:02:56 +0900 Subject: [PATCH 17/34] ci: retrigger exact-head review after stale blocker dismissal From b91f219b77a1d36d6fb14204f8068492324003af Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 14 Aug 2026 19:06:41 +0900 Subject: [PATCH 18/34] fix(security): reject YouTube output path traversal --- .../src/bandscope_analysis/youtube.py | 29 ++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/services/analysis-engine/src/bandscope_analysis/youtube.py b/services/analysis-engine/src/bandscope_analysis/youtube.py index 691f709f0..4b670e303 100644 --- a/services/analysis-engine/src/bandscope_analysis/youtube.py +++ b/services/analysis-engine/src/bandscope_analysis/youtube.py @@ -5,6 +5,8 @@ Security Notes: - Accepts only bounded, standard HTTPS YouTube watch URLs and disables playlists, geographic bypass, credentials, and interactive authentication. +- Rejects parent-directory traversal segments in the local output directory before + the path is passed to yt-dlp. - Keeps certificate verification enabled. It uses the operating-system trust store when roots are present and otherwise retains yt-dlp's CA fallback. - Optionally accepts sibling absolute ffmpeg/ffprobe paths only with both full @@ -36,6 +38,7 @@ ) YOUTUBE_IMPORT_FAILED_MESSAGE = "YouTube import failed. Please use a local audio file instead." RUNTIME_DEPENDENCY_INVALID_MESSAGE = "The configured media runtime failed identity verification." +OUTPUT_DIRECTORY_INVALID_MESSAGE = "The local download directory failed safety validation." SHA256_PATTERN = re.compile(r"^[0-9a-f]{64}$") @@ -81,6 +84,26 @@ def validate_url(url: str) -> bool: return False +def _contains_parent_path_segment(path: str) -> bool: + """Return whether a path contains an explicit parent-directory segment. + + Both POSIX and Windows separators are normalized so a path prepared on one + platform cannot smuggle ``..`` through checks performed on another. + """ + return ".." in path.replace("\\", "/").split("/") + + +def _invalid_output_directory_response() -> Dict[str, Any]: + """Return the stable redacted response for an unsafe output directory.""" + return { + "ok": False, + "error": { + "code": "invalid_output_directory", + "message": OUTPUT_DIRECTORY_INVALID_MESSAGE, + }, + } + + def _find_downloaded_file(actual_filepath: str) -> Optional[str]: """Find the downloaded file, including postprocessor extension changes.""" if not os.path.exists(actual_filepath): @@ -234,7 +257,8 @@ def download_youtube_audio( Args: url: The YouTube URL to download. - out_dir: The directory to save the audio file. + out_dir: The directory to save the audio file. Explicit parent-directory + segments are rejected before the path reaches yt-dlp. ffmpeg_path: Optional absolute path to a provisioned ffmpeg executable. ffmpeg_sha256: Full lowercase SHA-256 identity for ``ffmpeg_path``. ffprobe_path: Optional sibling path to the provisioned ffprobe executable. @@ -252,6 +276,9 @@ def download_youtube_audio( }, } + if _contains_parent_path_segment(out_dir): + return _invalid_output_directory_response() + runtime_is_valid, verified_ffmpeg_path = _verify_media_runtime( ffmpeg_path, ffmpeg_sha256, From da64185a2d5cb372c0b37d6ac39298b801899b72 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 14 Aug 2026 19:07:03 +0900 Subject: [PATCH 19/34] test(security): cover YouTube output directory guard --- .../tests/test_youtube_output_directory.py | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 services/analysis-engine/tests/test_youtube_output_directory.py diff --git a/services/analysis-engine/tests/test_youtube_output_directory.py b/services/analysis-engine/tests/test_youtube_output_directory.py new file mode 100644 index 000000000..c9887783e --- /dev/null +++ b/services/analysis-engine/tests/test_youtube_output_directory.py @@ -0,0 +1,41 @@ +"""Regression tests for the YouTube output-directory guard.""" + +from unittest.mock import MagicMock, patch + +import pytest + +from bandscope_analysis.youtube import ( + OUTPUT_DIRECTORY_INVALID_MESSAGE, + _contains_parent_path_segment, + download_youtube_audio, +) + + +@pytest.mark.parametrize("separator", ["/", "\\"]) +@patch("bandscope_analysis.youtube.yt_dlp.YoutubeDL") +def test_download_rejects_parent_segment( + mock_ydl_class: MagicMock, + separator: str, +) -> None: + """Reject a parent segment regardless of the platform separator.""" + parent = "." * 2 + out_dir = separator.join(("safe", parent, "outside")) + + result = download_youtube_audio( + "https://youtube.com/watch?v=abc123DEF45", + out_dir, + ) + + assert result == { + "ok": False, + "error": { + "code": "invalid_output_directory", + "message": OUTPUT_DIRECTORY_INVALID_MESSAGE, + }, + } + mock_ydl_class.assert_not_called() + + +def test_output_guard_allows_literal_double_dots_inside_name() -> None: + """Keep ordinary names containing two dots when they are not a parent segment.""" + assert _contains_parent_path_segment("safe/my..cache") is False From 5decba5fc57948d30ed25a5944efd04ce9f5f881 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 14 Aug 2026 19:07:34 +0900 Subject: [PATCH 20/34] docs(changelog): record YouTube output directory guard --- CHANGELOG.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 60e346317..10ff0e4cb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,8 @@ ### Fixed +- Rejected POSIX and Windows parent-directory segments at the YouTube download-output boundary + before the path reaches yt-dlp, returning a stable redacted failure without downloader execution. - Kept YouTube TLS verification enabled, using populated OS-managed CA roots when available and retaining yt-dlp's maintained CA-bundle fallback when the system trust store is empty or fails. - Raised `pdfjs-dist` 6.1.200 → 6.2.108 (`GHSA-hq66-cqwq-w95j`), `nanoid` 3.3.16 → @@ -47,10 +49,11 @@ - Attack surface and trust boundary: YouTube URLs, response metadata, downloaded media, creator fixtures, ffmpeg/ffprobe executables, and htdemucs checkpoint bytes remain untrusted until their owning host, shape, size, filesystem identity, and full-hash allowlists pass. -- Mitigations and failure behavior: TLS verification stays enabled; the complete ffmpeg/ffprobe - path-and-hash pair is verified before network fixture access; model loading is offline, - same-byte, restricted to `weights_only=True` plus the exact reviewed globals, and fails closed - without an unrestricted fallback. +- Mitigations and failure behavior: TLS verification stays enabled; parent-directory segments are + rejected before the output template reaches yt-dlp; the complete ffmpeg/ffprobe path-and-hash + pair is verified before network fixture access; model loading is offline, same-byte, restricted + to `weights_only=True` plus the exact reviewed globals, and fails closed without an unrestricted + fallback. - Developer tooling: the cross-platform check launcher is repository-only, invokes only the fixed `py`, `python3`, or `python` candidates with argument arrays and no shell, and propagates the first available interpreter's failure instead of retrying past it. From 901e8e5bc55258beb4304710eeedee6e07d451f4 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 14 Aug 2026 22:17:33 +0900 Subject: [PATCH 21/34] fix(youtube): bind downloads to an allowed output root --- .../src/bandscope_analysis/youtube.py | 94 +++++++++++++++++-- 1 file changed, 85 insertions(+), 9 deletions(-) diff --git a/services/analysis-engine/src/bandscope_analysis/youtube.py b/services/analysis-engine/src/bandscope_analysis/youtube.py index 4b670e303..6167a0e84 100644 --- a/services/analysis-engine/src/bandscope_analysis/youtube.py +++ b/services/analysis-engine/src/bandscope_analysis/youtube.py @@ -5,8 +5,8 @@ Security Notes: - Accepts only bounded, standard HTTPS YouTube watch URLs and disables playlists, geographic bypass, credentials, and interactive authentication. -- Rejects parent-directory traversal segments in the local output directory before - the path is passed to yt-dlp. +- Resolves each local output directory under an explicit caller-owned root, or the + operating-system temporary root by default, before the path reaches yt-dlp. - Keeps certificate verification enabled. It uses the operating-system trust store when roots are present and otherwise retains yt-dlp's CA fallback. - Optionally accepts sibling absolute ffmpeg/ffprobe paths only with both full @@ -24,8 +24,9 @@ import re import ssl import sys +import tempfile import urllib.parse -from pathlib import Path +from pathlib import Path, PureWindowsPath from typing import Any, Dict, Optional import yt_dlp # type: ignore @@ -93,6 +94,71 @@ def _contains_parent_path_segment(path: str) -> bool: return ".." in path.replace("\\", "/").split("/") +def _has_unsafe_windows_path_shape(path: str) -> bool: + """Reject foreign or drive-relative Windows paths before native resolution.""" + windows_path = PureWindowsPath(path) + if windows_path.drive and not windows_path.is_absolute(): + return True + return os.name != "nt" and windows_path.is_absolute() + + +def _resolve_output_directory( + out_dir: str, + allowed_output_root: Optional[str], +) -> Optional[Path]: + """Resolve ``out_dir`` only when it stays inside the allowed output root. + + Relative paths are interpreted below the allowed root. When callers do not + provide a root, BandScope uses the operating-system temporary directory. The + root must already exist; the output directory itself may be created later by + the caller or downloader. Existing direct symlinks are rejected, and parent + symlinks are canonicalized before the containment check. + """ + if not isinstance(out_dir, str) or not out_dir.strip(): + return None + if _contains_parent_path_segment(out_dir) or _has_unsafe_windows_path_shape(out_dir): + return None + + root_value = tempfile.gettempdir() if allowed_output_root is None else allowed_output_root + if not isinstance(root_value, str) or not root_value.strip(): + return None + if _contains_parent_path_segment(root_value) or _has_unsafe_windows_path_shape(root_value): + return None + + root_candidate = Path(root_value).expanduser() + output_candidate = Path(out_dir).expanduser() + if not root_candidate.is_absolute(): + return None + if output_candidate.is_symlink(): + return None + if not output_candidate.is_absolute(): + output_candidate = root_candidate / output_candidate + + try: + resolved_root = root_candidate.resolve(strict=True) + resolved_output = output_candidate.resolve(strict=False) + except (OSError, RuntimeError): + return None + if not resolved_root.is_dir(): + return None + + try: + resolved_output.relative_to(resolved_root) + except ValueError: + return None + return resolved_output + + +def _path_is_within_directory(path: str, directory: Path) -> bool: + """Return whether a downloader-produced path resolves inside ``directory``.""" + try: + candidate = Path(path).resolve(strict=False) + candidate.relative_to(directory) + except (OSError, RuntimeError, ValueError): + return False + return True + + def _invalid_output_directory_response() -> Dict[str, Any]: """Return the stable redacted response for an unsafe output directory.""" return { @@ -247,6 +313,7 @@ def download_youtube_audio( url: str, out_dir: str, *, + allowed_output_root: Optional[str] = None, ffmpeg_path: Optional[str] = None, ffmpeg_sha256: Optional[str] = None, ffprobe_path: Optional[str] = None, @@ -257,8 +324,10 @@ def download_youtube_audio( Args: url: The YouTube URL to download. - out_dir: The directory to save the audio file. Explicit parent-directory - segments are rejected before the path reaches yt-dlp. + out_dir: The directory to save the audio file. The resolved path must + remain within ``allowed_output_root``. + allowed_output_root: Absolute caller-owned output root. When omitted, + the operating-system temporary directory is used. ffmpeg_path: Optional absolute path to a provisioned ffmpeg executable. ffmpeg_sha256: Full lowercase SHA-256 identity for ``ffmpeg_path``. ffprobe_path: Optional sibling path to the provisioned ffprobe executable. @@ -276,7 +345,8 @@ def download_youtube_audio( }, } - if _contains_parent_path_segment(out_dir): + resolved_out_dir = _resolve_output_directory(out_dir, allowed_output_root) + if resolved_out_dir is None: return _invalid_output_directory_response() runtime_is_valid, verified_ffmpeg_path = _verify_media_runtime( @@ -290,7 +360,7 @@ def download_youtube_audio( ydl_opts: Dict[str, Any] = { "format": "bestaudio/best", - "outtmpl": os.path.join(out_dir, "%(id)s.%(ext)s"), + "outtmpl": str(resolved_out_dir / "%(id)s.%(ext)s"), "quiet": True, "no_warnings": True, "noprogress": True, @@ -323,9 +393,11 @@ def download_youtube_audio( info = ydl.extract_info(url, download=True) if info is None: raise Exception("Failed to extract info") - actual_filepath = ydl.prepare_filename(info) + prepared_filepath = ydl.prepare_filename(info) + if not _path_is_within_directory(prepared_filepath, resolved_out_dir): + return _invalid_output_directory_response() - actual_filepath = _find_downloaded_file(actual_filepath) + actual_filepath = _find_downloaded_file(prepared_filepath) if actual_filepath is None: return { @@ -335,6 +407,8 @@ def download_youtube_audio( "message": "Downloaded file could not be found.", }, } + if not _path_is_within_directory(actual_filepath, resolved_out_dir): + return _invalid_output_directory_response() if ( os.path.exists(actual_filepath) @@ -371,6 +445,7 @@ def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("--url", required=True) parser.add_argument("--out-dir", required=True) + parser.add_argument("--allowed-output-root") parser.add_argument("--ffmpeg-path") parser.add_argument("--ffmpeg-sha256") parser.add_argument("--ffprobe-path") @@ -380,6 +455,7 @@ def main() -> None: result = download_youtube_audio( args.url, args.out_dir, + allowed_output_root=args.allowed_output_root, ffmpeg_path=args.ffmpeg_path, ffmpeg_sha256=args.ffmpeg_sha256, ffprobe_path=args.ffprobe_path, From dcbac8edce93493be9eef3d881b2ad00435cef0d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 14 Aug 2026 22:17:54 +0900 Subject: [PATCH 22/34] test(youtube): cover allowed output root containment --- .../tests/test_youtube_output_directory.py | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/services/analysis-engine/tests/test_youtube_output_directory.py b/services/analysis-engine/tests/test_youtube_output_directory.py index c9887783e..6fee9a6f2 100644 --- a/services/analysis-engine/tests/test_youtube_output_directory.py +++ b/services/analysis-engine/tests/test_youtube_output_directory.py @@ -1,5 +1,6 @@ """Regression tests for the YouTube output-directory guard.""" +from pathlib import Path from unittest.mock import MagicMock, patch import pytest @@ -7,6 +8,7 @@ from bandscope_analysis.youtube import ( OUTPUT_DIRECTORY_INVALID_MESSAGE, _contains_parent_path_segment, + _resolve_output_directory, download_youtube_audio, ) @@ -36,6 +38,58 @@ def test_download_rejects_parent_segment( mock_ydl_class.assert_not_called() +@pytest.mark.parametrize("out_dir", ["/bandscope-outside", r"C:\bandscope-outside"]) +@patch("bandscope_analysis.youtube.yt_dlp.YoutubeDL") +def test_download_rejects_absolute_path_outside_allowed_root( + mock_ydl_class: MagicMock, + out_dir: str, + tmp_path: Path, +) -> None: + """Reject POSIX and Windows absolute paths outside the caller-owned root.""" + allowed_root = tmp_path / "allowed-root" + allowed_root.mkdir() + + result = download_youtube_audio( + "https://youtube.com/watch?v=abc123DEF45", + out_dir, + allowed_output_root=str(allowed_root), + ) + + assert result == { + "ok": False, + "error": { + "code": "invalid_output_directory", + "message": OUTPUT_DIRECTORY_INVALID_MESSAGE, + }, + } + mock_ydl_class.assert_not_called() + + +def test_output_directory_resolves_relative_child_under_allowed_root(tmp_path: Path) -> None: + """Resolve a relative child beneath the explicit root without escaping it.""" + allowed_root = tmp_path / "allowed-root" + allowed_root.mkdir() + + resolved = _resolve_output_directory("media", str(allowed_root)) + + assert resolved == allowed_root.resolve() / "media" + + +def test_output_directory_rejects_direct_symlink(tmp_path: Path) -> None: + """Reject an existing direct symlink even when its target stays in the root.""" + allowed_root = tmp_path / "allowed-root" + allowed_root.mkdir() + target = allowed_root / "target" + target.mkdir() + symlink = allowed_root / "media" + try: + symlink.symlink_to(target, target_is_directory=True) + except OSError: + pytest.skip("symlink creation is unavailable on this platform") + + assert _resolve_output_directory(str(symlink), str(allowed_root)) is None + + def test_output_guard_allows_literal_double_dots_inside_name() -> None: """Keep ordinary names containing two dots when they are not a parent segment.""" assert _contains_parent_path_segment("safe/my..cache") is False From 8b2deaa02c56094add903dc7a3dec806dc293d49 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 14 Aug 2026 23:11:18 +0900 Subject: [PATCH 23/34] test(analysis): cover YouTube output guard failures --- .../tests/test_youtube_output_directory.py | 130 +++++++++++++++++- 1 file changed, 125 insertions(+), 5 deletions(-) diff --git a/services/analysis-engine/tests/test_youtube_output_directory.py b/services/analysis-engine/tests/test_youtube_output_directory.py index 6fee9a6f2..53a670b4c 100644 --- a/services/analysis-engine/tests/test_youtube_output_directory.py +++ b/services/analysis-engine/tests/test_youtube_output_directory.py @@ -8,10 +8,13 @@ from bandscope_analysis.youtube import ( OUTPUT_DIRECTORY_INVALID_MESSAGE, _contains_parent_path_segment, + _path_is_within_directory, _resolve_output_directory, download_youtube_audio, ) +YOUTUBE_URL = "https://youtube.com/watch?v=abc123DEF45" + @pytest.mark.parametrize("separator", ["/", "\\"]) @patch("bandscope_analysis.youtube.yt_dlp.YoutubeDL") @@ -23,10 +26,7 @@ def test_download_rejects_parent_segment( parent = "." * 2 out_dir = separator.join(("safe", parent, "outside")) - result = download_youtube_audio( - "https://youtube.com/watch?v=abc123DEF45", - out_dir, - ) + result = download_youtube_audio(YOUTUBE_URL, out_dir) assert result == { "ok": False, @@ -50,7 +50,7 @@ def test_download_rejects_absolute_path_outside_allowed_root( allowed_root.mkdir() result = download_youtube_audio( - "https://youtube.com/watch?v=abc123DEF45", + YOUTUBE_URL, out_dir, allowed_output_root=str(allowed_root), ) @@ -65,6 +65,50 @@ def test_download_rejects_absolute_path_outside_allowed_root( mock_ydl_class.assert_not_called() +@pytest.mark.parametrize( + ("out_dir", "allowed_output_root"), + [ + ("", None), + (r"C:media", None), + ("media", ""), + ("media", "safe/../root"), + ("media", "relative-root"), + ], +) +def test_output_directory_rejects_invalid_path_contracts( + out_dir: str, + allowed_output_root: str | None, +) -> None: + """Reject empty, drive-relative, traversing, and relative-root path contracts.""" + assert _resolve_output_directory(out_dir, allowed_output_root) is None + + +def test_output_directory_rejects_resolution_failure( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Fail closed when canonical path resolution cannot be completed.""" + allowed_root = tmp_path / "allowed-root" + allowed_root.mkdir() + allowed_root_value = str(allowed_root) + + def fail_resolution(_path: Path, *, strict: bool = False) -> Path: + del strict + raise OSError("resolution unavailable") + + monkeypatch.setattr(Path, "resolve", fail_resolution) + + assert _resolve_output_directory("media", allowed_root_value) is None + + +def test_output_directory_rejects_non_directory_root(tmp_path: Path) -> None: + """Require the allowlisted output root to be an existing directory.""" + allowed_root = tmp_path / "allowed-root" + allowed_root.write_text("not a directory", encoding="utf-8") + + assert _resolve_output_directory("media", str(allowed_root)) is None + + def test_output_directory_resolves_relative_child_under_allowed_root(tmp_path: Path) -> None: """Resolve a relative child beneath the explicit root without escaping it.""" allowed_root = tmp_path / "allowed-root" @@ -90,6 +134,82 @@ def test_output_directory_rejects_direct_symlink(tmp_path: Path) -> None: assert _resolve_output_directory(str(symlink), str(allowed_root)) is None +def test_path_guard_rejects_resolved_path_outside_directory(tmp_path: Path) -> None: + """Reject downloader paths that canonicalize outside the resolved directory.""" + allowed_directory = tmp_path / "allowed-root" + allowed_directory.mkdir() + + assert _path_is_within_directory(str(tmp_path / "outside.webm"), allowed_directory) is False + + +@patch("bandscope_analysis.youtube.yt_dlp.YoutubeDL") +def test_download_rejects_prepared_filename_escape( + mock_ydl_class: MagicMock, + tmp_path: Path, +) -> None: + """Reject a downloader-prepared filename outside the resolved output directory.""" + allowed_root = tmp_path / "allowed-root" + allowed_root.mkdir() + mock_ydl = MagicMock() + mock_ydl_class.return_value.__enter__.return_value = mock_ydl + mock_ydl.extract_info.return_value = { + "id": "abc123DEF45", + "title": "Test Video", + "duration": 60, + } + mock_ydl.prepare_filename.return_value = str(tmp_path / "outside.webm") + + result = download_youtube_audio( + YOUTUBE_URL, + "media", + allowed_output_root=str(allowed_root), + ) + + assert result == { + "ok": False, + "error": { + "code": "invalid_output_directory", + "message": OUTPUT_DIRECTORY_INVALID_MESSAGE, + }, + } + + +@patch("bandscope_analysis.youtube._find_downloaded_file") +@patch("bandscope_analysis.youtube.yt_dlp.YoutubeDL") +def test_download_rejects_discovered_file_escape( + mock_ydl_class: MagicMock, + mock_find_downloaded_file: MagicMock, + tmp_path: Path, +) -> None: + """Reject a postprocessed file that resolves outside the output directory.""" + allowed_root = tmp_path / "allowed-root" + allowed_root.mkdir() + output_directory = allowed_root / "media" + mock_ydl = MagicMock() + mock_ydl_class.return_value.__enter__.return_value = mock_ydl + mock_ydl.extract_info.return_value = { + "id": "abc123DEF45", + "title": "Test Video", + "duration": 60, + } + mock_ydl.prepare_filename.return_value = str(output_directory / "abc123DEF45.webm") + mock_find_downloaded_file.return_value = str(tmp_path / "outside.opus") + + result = download_youtube_audio( + YOUTUBE_URL, + "media", + allowed_output_root=str(allowed_root), + ) + + assert result == { + "ok": False, + "error": { + "code": "invalid_output_directory", + "message": OUTPUT_DIRECTORY_INVALID_MESSAGE, + }, + } + + def test_output_guard_allows_literal_double_dots_inside_name() -> None: """Keep ordinary names containing two dots when they are not a parent segment.""" assert _contains_parent_path_segment("safe/my..cache") is False From 7ba3c74c2be869e3abe33ccfb003e6ad3f0ae882 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 14 Aug 2026 23:13:44 +0900 Subject: [PATCH 24/34] test(analysis): align YouTube CLI output-root contract --- services/analysis-engine/tests/test_youtube.py | 1 + 1 file changed, 1 insertion(+) diff --git a/services/analysis-engine/tests/test_youtube.py b/services/analysis-engine/tests/test_youtube.py index 43125afc4..83014c302 100644 --- a/services/analysis-engine/tests/test_youtube.py +++ b/services/analysis-engine/tests/test_youtube.py @@ -666,6 +666,7 @@ def test_main_block(monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixtu mock_download.assert_called_with( "https://youtube.com/watch?v=abc123DEF45", "/tmp", + allowed_output_root=None, ffmpeg_path="/opt/bandscope/ffmpeg", ffmpeg_sha256="a" * 64, ffprobe_path="/opt/bandscope/ffprobe", From 0ddb17d4c6ff98b27901155f3fd12586c6cda366 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 16 Aug 2026 03:42:25 +0900 Subject: [PATCH 25/34] chore(deps): defer shared security baseline to #783 --- apps/desktop/package.json | 2 +- package-lock.json | 20 ++++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/apps/desktop/package.json b/apps/desktop/package.json index e09719b22..e7685d6f0 100644 --- a/apps/desktop/package.json +++ b/apps/desktop/package.json @@ -20,7 +20,7 @@ "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", "lucide-react": "^1.24.0", - "pdfjs-dist": "6.2.108", + "pdfjs-dist": "6.1.200", "react": "^19.2.4", "react-dom": "^19.2.7", "sonner": "^2.0.7", diff --git a/package-lock.json b/package-lock.json index 209617988..cf1c991c1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -32,7 +32,7 @@ "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", "lucide-react": "^1.24.0", - "pdfjs-dist": "6.2.108", + "pdfjs-dist": "6.1.200", "react": "^19.2.4", "react-dom": "^19.2.7", "sonner": "^2.0.7", @@ -6075,9 +6075,9 @@ "license": "MIT" }, "node_modules/nanoid": { - "version": "3.3.18", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.18.tgz", - "integrity": "sha512-DTg4MJbGMWkfi6VZFdNt2/caMbQy4Ou+Op/hJQvGEWcnVfoA1QA+xzRKAzw9jD6+GVOOeYr/mIcuDSdug6F6+w==", + "version": "3.3.16", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.16.tgz", + "integrity": "sha512-bzlKTyNJ7+LdGIIwy8ijFpIqEQIvafahV7eYykJ8Cvh42EdJeODoJ6gUJXpQJvej1BddH8OqTXZNE/KfbWAu8Q==", "dev": true, "funding": [ { @@ -6368,9 +6368,9 @@ } }, "node_modules/pdfjs-dist": { - "version": "6.2.108", - "resolved": "https://registry.npmjs.org/pdfjs-dist/-/pdfjs-dist-6.2.108.tgz", - "integrity": "sha512-YxFb+SQcodN2rnX9Tn3dHYlqfb7NjlzzfONPpJd+AKoKtUjEdevTfbC07d5TcczzOK6261auRkP/M8OBHs9vFQ==", + "version": "6.1.200", + "resolved": "https://registry.npmjs.org/pdfjs-dist/-/pdfjs-dist-6.1.200.tgz", + "integrity": "sha512-o8MolyzirkkLrcdsae/HEOiIcXWI7DS5zGpvqW8xTC2YUsW30rltFw2bDGvw/fskUdEMrQm2br68jzDS5BH2vw==", "license": "Apache-2.0", "engines": { "node": ">=22.13.0 || >=24" @@ -7179,9 +7179,9 @@ } }, "node_modules/undici": { - "version": "7.29.0", - "resolved": "https://registry.npmjs.org/undici/-/undici-7.29.0.tgz", - "integrity": "sha512-IDxfleLmmbSskfWSUATiN1nfn2rDuvnMOqb5CWR92iIfojA0Ud+ulOAAEQ57LPr9rWmsreUyf5lwyao+7GNNVw==", + "version": "7.28.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-7.28.0.tgz", + "integrity": "sha512-cRZYrTDwWznlnRiPjggAGxZXanty6M8RV1ff8Wm4LWXBp7/IG8v5DnOm74DtUBp9OONpK75YlPnIjQqX0dBDtA==", "dev": true, "license": "MIT", "engines": { From 8df645753645740ced74afc6b5b07b8cd3802ad5 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 16 Aug 2026 03:42:56 +0900 Subject: [PATCH 26/34] docs(changelog): defer shared npm remediation to #783 --- CHANGELOG.md | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 10ff0e4cb..7f48ae9fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,12 +23,6 @@ before the path reaches yt-dlp, returning a stable redacted failure without downloader execution. - Kept YouTube TLS verification enabled, using populated OS-managed CA roots when available and retaining yt-dlp's maintained CA-bundle fallback when the system trust store is empty or fails. -- Raised `pdfjs-dist` 6.1.200 → 6.2.108 (`GHSA-hq66-cqwq-w95j`), `nanoid` 3.3.16 → - 3.3.18 (`GHSA-2v37-7h3g-55p8`), and `undici` 7.28.0 → 7.29.0 - (`GHSA-8xcm-r25x-g524`, `GHSA-4cwx-7wf7-3272`, `GHSA-m8rv-5g2x-5cg5`, - `GHSA-jr45-8vmc-qm54`, `GHSA-v3r7-h72x-cjcm`); `package-lock.json`, the - mutation-sensitive floor test, and the required per-candidate zero-vulnerability npm audit - preserve the fixed-version evidence. - Made htdemucs loading offline and fail-closed: the runtime accepts only the inventoried filename, byte size, and full SHA-256, rejects filesystem identity races, and deserializes the verified bytes with PyTorch's restricted `weights_only` loader, an exact reviewed global allowlist, strict @@ -62,11 +56,12 @@ - Test points: each candidate head must pass quickcheck, hosted SAST/Bandit/secret/security scans, mutation tests for loader and allowlist bypasses, executable-identity rejection tests, supply-chain verification, and the exact provisioned-model smoke test before merge. -- Dependency and supply chain: no production dependency was added; documentation policy checks now - pin `markdown-it-py 4.0.0` as a direct development dependency so rendered Markdown—not lexical - lookalikes—defines headings and tables. Lockfiles retain patched `pdfjs-dist 6.2.108`, `nanoid` - 3.3.18, and `undici 7.29.0`, while the supplemental inventory binds yt-dlp, ffmpeg/ffprobe, and - htdemucs to their declared delivery and integrity contracts. +- Dependency and supply chain: no production dependency is added by this benchmark slice; + documentation policy checks pin `markdown-it-py 4.0.0` as a direct development dependency so + rendered Markdown—not lexical lookalikes—defines headings and tables. The shared JavaScript + dependency-security baseline remains owned by canonical #783 and is a prerequisite gate for this + branch; the supplemental inventory separately binds yt-dlp, ffmpeg/ffprobe, and htdemucs to their + declared delivery and integrity contracts. ## [0.1.3] - 2026-04-29 From b7fa59ace39156aaa75d52a0b396c120e9c63362 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 16 Aug 2026 03:46:35 +0900 Subject: [PATCH 27/34] chore(ci): stage one-shot #828 baseline cleanup --- .../repair-828-security-baseline.yml | 70 +++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 .github/workflows/repair-828-security-baseline.yml diff --git a/.github/workflows/repair-828-security-baseline.yml b/.github/workflows/repair-828-security-baseline.yml new file mode 100644 index 000000000..0578c481a --- /dev/null +++ b/.github/workflows/repair-828-security-baseline.yml @@ -0,0 +1,70 @@ +name: repair-828-security-baseline + +on: + push: + branches: + - feature/youtube-known-stem-e2e + +permissions: + contents: write + +jobs: + repair: + runs-on: ubuntu-24.04 + steps: + - name: Checkout contributor branch + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 + with: + ref: feature/youtube-known-stem-e2e + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 + with: + python-version: '3.12' + + - name: Set up uv + uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 + with: + version: '0.8.6' + enable-cache: false + + - name: Remove duplicated canonical npm-baseline test + shell: bash + run: | + python - <<'PY' + import json + from pathlib import Path + + test_path = Path('services/analysis-engine/tests/test_supply_chain_policy.py') + source = test_path.read_text(encoding='utf-8') + start_marker = 'def test_node_security_floors_are_locked_to_patched_versions() -> None:\n' + end_marker = 'def test_supplemental_inventory_rejects_obsolete_or_missing_runtime_model(\n' + if source.count(start_marker) != 1 or source.count(end_marker) != 1: + raise SystemExit('expected exactly one duplicate npm-baseline test block') + start = source.index(start_marker) + end = source.index(end_marker, start) + source = source[:start] + source[end:] + test_path.write_text(source, encoding='utf-8') + + desktop = json.loads(Path('apps/desktop/package.json').read_text(encoding='utf-8')) + if desktop['dependencies']['pdfjs-dist'] != '6.1.200': + raise SystemExit('branch must defer JavaScript baseline remediation to canonical #783') + PY + + - name: Verify bounded supply-chain policy tests + shell: bash + run: | + uv sync --project services/analysis-engine --frozen + uv run --project services/analysis-engine pytest services/analysis-engine/tests/test_supply_chain_policy.py -q + + - name: Self-delete and push bounded repair + shell: bash + run: | + rm .github/workflows/repair-828-security-baseline.yml + git diff --check + git config user.name 'github-actions[bot]' + git config user.email '41898282+github-actions[bot]@users.noreply.github.com' + git add services/analysis-engine/tests/test_supply_chain_policy.py .github/workflows/repair-828-security-baseline.yml + git commit -m 'test(supply-chain): defer shared npm floor to #783' + git push origin HEAD:feature/youtube-known-stem-e2e From db4713f8a297aaaabffdbee37ff1a8df14e5f9b1 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 16 Aug 2026 03:48:55 +0900 Subject: [PATCH 28/34] chore(ci): satisfy checkout default-branch guard --- .github/workflows/repair-828-security-baseline.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/repair-828-security-baseline.yml b/.github/workflows/repair-828-security-baseline.yml index 0578c481a..833381852 100644 --- a/.github/workflows/repair-828-security-baseline.yml +++ b/.github/workflows/repair-828-security-baseline.yml @@ -8,6 +8,11 @@ on: permissions: contents: write +env: + GIT_CONFIG_COUNT: 1 + GIT_CONFIG_KEY_0: init.defaultBranch + GIT_CONFIG_VALUE_0: develop + jobs: repair: runs-on: ubuntu-24.04 From 5584ad7ee25a93d77b26f91db783c5fff1cb7a2d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sat, 15 Aug 2026 18:50:20 +0000 Subject: [PATCH 29/34] test(supply-chain): defer shared npm floor to #783 --- .../repair-828-security-baseline.yml | 75 ------------------- .../tests/test_supply_chain_policy.py | 23 ------ 2 files changed, 98 deletions(-) delete mode 100644 .github/workflows/repair-828-security-baseline.yml diff --git a/.github/workflows/repair-828-security-baseline.yml b/.github/workflows/repair-828-security-baseline.yml deleted file mode 100644 index 833381852..000000000 --- a/.github/workflows/repair-828-security-baseline.yml +++ /dev/null @@ -1,75 +0,0 @@ -name: repair-828-security-baseline - -on: - push: - branches: - - feature/youtube-known-stem-e2e - -permissions: - contents: write - -env: - GIT_CONFIG_COUNT: 1 - GIT_CONFIG_KEY_0: init.defaultBranch - GIT_CONFIG_VALUE_0: develop - -jobs: - repair: - runs-on: ubuntu-24.04 - steps: - - name: Checkout contributor branch - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 - with: - ref: feature/youtube-known-stem-e2e - fetch-depth: 0 - - - name: Set up Python - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 - with: - python-version: '3.12' - - - name: Set up uv - uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 - with: - version: '0.8.6' - enable-cache: false - - - name: Remove duplicated canonical npm-baseline test - shell: bash - run: | - python - <<'PY' - import json - from pathlib import Path - - test_path = Path('services/analysis-engine/tests/test_supply_chain_policy.py') - source = test_path.read_text(encoding='utf-8') - start_marker = 'def test_node_security_floors_are_locked_to_patched_versions() -> None:\n' - end_marker = 'def test_supplemental_inventory_rejects_obsolete_or_missing_runtime_model(\n' - if source.count(start_marker) != 1 or source.count(end_marker) != 1: - raise SystemExit('expected exactly one duplicate npm-baseline test block') - start = source.index(start_marker) - end = source.index(end_marker, start) - source = source[:start] + source[end:] - test_path.write_text(source, encoding='utf-8') - - desktop = json.loads(Path('apps/desktop/package.json').read_text(encoding='utf-8')) - if desktop['dependencies']['pdfjs-dist'] != '6.1.200': - raise SystemExit('branch must defer JavaScript baseline remediation to canonical #783') - PY - - - name: Verify bounded supply-chain policy tests - shell: bash - run: | - uv sync --project services/analysis-engine --frozen - uv run --project services/analysis-engine pytest services/analysis-engine/tests/test_supply_chain_policy.py -q - - - name: Self-delete and push bounded repair - shell: bash - run: | - rm .github/workflows/repair-828-security-baseline.yml - git diff --check - git config user.name 'github-actions[bot]' - git config user.email '41898282+github-actions[bot]@users.noreply.github.com' - git add services/analysis-engine/tests/test_supply_chain_policy.py .github/workflows/repair-828-security-baseline.yml - git commit -m 'test(supply-chain): defer shared npm floor to #783' - git push origin HEAD:feature/youtube-known-stem-e2e diff --git a/services/analysis-engine/tests/test_supply_chain_policy.py b/services/analysis-engine/tests/test_supply_chain_policy.py index 30d1cc2d5..08eb0a307 100644 --- a/services/analysis-engine/tests/test_supply_chain_policy.py +++ b/services/analysis-engine/tests/test_supply_chain_policy.py @@ -13,29 +13,6 @@ from conftest import load_module, make_symlink_or_skip -def test_node_security_floors_are_locked_to_patched_versions() -> None: - """Keep direct and transitive Node dependencies above current advisory floors.""" - repo_root = Path(__file__).resolve().parents[3] - desktop_package = json.loads( - (repo_root / "apps" / "desktop" / "package.json").read_text(encoding="utf-8") - ) - package_lock = json.loads((repo_root / "package-lock.json").read_text(encoding="utf-8")) - packages = package_lock["packages"] - - assert desktop_package["dependencies"]["pdfjs-dist"] == "6.2.108" - assert packages["node_modules/pdfjs-dist"]["version"] == "6.2.108" - assert tuple(map(int, packages["node_modules/nanoid"]["version"].split("."))) >= ( - 3, - 3, - 17, - ) - assert tuple(map(int, packages["node_modules/undici"]["version"].split("."))) >= ( - 7, - 28, - 1, - ) - - def test_supplemental_inventory_rejects_obsolete_or_missing_runtime_model( tmp_path: Path, ) -> None: From 27b9fc0b648368a2a2306f689737d5fe20eb9c3c Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 16 Aug 2026 03:52:31 +0900 Subject: [PATCH 30/34] chore(ci): reestablish exact-head verification From 4695465354af9d6b1a7f7429c91b66284664504f Mon Sep 17 00:00:00 2001 From: seonghobae Date: Mon, 17 Aug 2026 08:21:39 +0000 Subject: [PATCH 31/34] docs(analysis): lock rehearsal metric authority for the known-stem slice MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Schreiber, Urbano, and Müller (2020) own Acc1/Acc2; Acc2 alone cannot accept rehearsal tempo. Raffel 2014 has no Acc1/Acc2. Chiu 2025 keeps beat F-measure at 70 ms. Odekerken/MIREX own WCSR. Le Roux SI-SDR stays primary. This is not a new MIR product; #828 still owns #770. --- AGENTS.md | 1 + ARCHITECTURE.md | 2 +- CHANGELOG.md | 4 ++++ .../real-audio-accuracy-acceptance.md | 22 ++++++++++++++---- .../tests/test_accuracy_metric_contract.py | 23 +++++++++++++++++++ 5 files changed, 47 insertions(+), 5 deletions(-) create mode 100644 services/analysis-engine/tests/test_accuracy_metric_contract.py diff --git a/AGENTS.md b/AGENTS.md index 81070181f..e6add017f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -98,6 +98,7 @@ This section applies to any agent (Claude, Codex, Cursor, opencode, ...) working - Prefer practical, friendly, rehearsal-first wording over academic or authority-heavy language. - Do not reduce the product to a chord analyzer when form, timing, player coordination, simplification, and setup cues are the real rehearsal blockers. - Do not frame usability as a reason to accept weak analysis quality; BandScope should aim for both easy use and high accuracy. +- Do not invent a parallel MIR product. #828 owns the #770 known-stem slice. Tempo Acc2 alone cannot accept rehearsal tempo; cite Schreiber, Urbano, & Müller (2020) for Acc1/Acc2, not Raffel (2014). ## Safety - Do not add network-dependent runtime paths for local analysis. diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index d42fba966..ba16a825e 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -119,7 +119,7 @@ Last updated: 2026-08-10 ephemeral storage. - The finished master proves candidate identity. YouTube-to-master and master-to-vocal global lags are composed once before separation; predicted stems are never realigned. Quality requires - duration/identity checks, zero-mean SI-SDR improvement over the downloaded mixture, and correct + duration/identity checks, zero-mean SI-SDR improvement over the downloaded mixture. SI-SDR remains the primary separation score (Le Roux et al., 2019). This branch does not accept tempo Acc2 alone, and correct vocal-stem assignment margin. - Deterministic metric/integrity/security contracts run in ordinary CI. Live network/model execution is explicit opt-in and cannot be scheduled or made release-blocking until authorization and diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f48ae9fc..bcdd56cd2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## [Unreleased] +### Changed + +- Lock rehearsal metric authority: Le Roux SI-SDR primary, Odekerken/MIREX WCSR, Chiu 2025 ±70 ms beat F-measure, Schreiber/Urbano/Müller Acc1+Acc2 with Acc2-alone forbidden, and Raffel 2014 not cited as an Acc1/Acc2 source. + ### Added - Added an opt-in real-YouTube/Demucs benchmark that verifies vocal separation against a diff --git a/docs/doctoring/real-audio-accuracy-acceptance.md b/docs/doctoring/real-audio-accuracy-acceptance.md index 091b9fc8c..9ebaba674 100644 --- a/docs/doctoring/real-audio-accuracy-acceptance.md +++ b/docs/doctoring/real-audio-accuracy-acceptance.md @@ -33,15 +33,26 @@ substitute for tier 1 or proof that tier 2 redistribution rights exist. | Domain | Required metrics | Interpretation boundary | Current status | |---|---|---|---| -| Source separation | Per-stem SI-SDR/SDR equivalent, improvement over mixture, semantic assignment, mixture consistency, finite output | Energy-ratio metrics do not establish perceptual quality; human listening protocol required for such claims. | Vocal SI-SDRi and assignment implemented on active branch; no passing live score | -| Harmony | Segment chord symbol recall, duration-weighted WCSR, root/major-minor/seventh mappings, no-chord, boundary error | Vocabulary and time alignment must be reported; one opaque aggregate is insufficient. | Planned | -| Beat/tempo | Beat precision/recall/F, continuity-aware metrics, tempo Acc1 and Acc2 | Half/double tempo must remain visible; confidence needs calibration. | Planned | +| Source separation | Le Roux et al. (2019) zero-mean SI-SDR is the primary score; BSSEval-style SDR is supporting only. Report improvement over mixture, semantic assignment, mixture consistency, and finite output. | Energy-ratio metrics do not establish perceptual quality; human listening protocol required for such claims. Acc2-style octave hiding is not a separation metric. | Vocal SI-SDRi and assignment implemented on active branch; no passing live score | +| Harmony | Odekerken et al. (2021) / MIREX duration-weighted WCSR plus segment chord-symbol recall, root/major-minor/seventh mappings, no-chord, and boundary error | Vocabulary and time alignment must be reported; one opaque aggregate is insufficient. | Planned | +| Beat | Precision/recall/F-measure at the Chiu et al. (2025) ±70 ms tolerance, plus continuity-aware metrics when available | Do not widen the 70 ms window after a failure. Raffel et al. (2014) MIR_EVAL supplies beat P/R/F, not tempo Acc1/Acc2. | Planned | +| Tempo | Schreiber, Urbano, & Müller (2020) Acc1 **and** Acc2 | Acc2 alone is forbidden for rehearsal tempo acceptance because octave error hides the count a band will actually play. Raffel et al. (2014) does not define Acc1/Acc2. | Planned | | Structure | Boundary P/R/F at strict/relaxed windows, segment-label agreement, order/repetition/pickup preservation | A correct label with materially wrong boundary remains an error. | Planned | | Range | Note/semitone endpoint error and exact out-of-range classification | Stem/role identity and octave policy must be registered. | Planned | | Rehearsal cues | Entry/dropout/stop/pickup event P/R and timing error | Event tolerance must reflect rehearsal use, not be widened after failure. | Planned | | Role overlap | Activity interval IoU or registered equivalent | Aggregate overlap must not hide severe role-specific misses. | Planned | | Confidence | Reliability/calibration curve and Brier-style score where probabilistic | Confidence text without probabilistic semantics is not scored as calibrated. | Planned | + + +## Metric authority (rehearsal claim rules) + +- Le Roux et al. (2019) SI-SDR is the primary source-separation score for this sentinel. +- Odekerken et al. (2021) and MIREX define duration-weighted WCSR for harmony. +- Chiu et al. (2025) keep beat F-measure at ±70 ms. +- Schreiber, Urbano, & Müller (2020) define tempo Acc1 and Acc2; Acc2 alone is forbidden for rehearsal. +- Raffel et al. (2014) MIR_EVAL does not define Acc1 or Acc2 and must not be cited as their source. + ## Regression and uncertainty policy The first protected baseline is descriptive; thresholds must not be invented as “industry @@ -135,7 +146,10 @@ not the current head; live success therefore remains absent. - Raffel, C., McFee, B., Humphrey, E. J., Salamon, J., Nieto, O., Liang, D., & Ellis, D. P. W. (2014). MIR_EVAL: A transparent implementation of common MIR metrics. In *Proceedings of the 15th International Society for Music Information Retrieval Conference* (pp. 367–372). -- Schreiber, H., & Müller, M. (2020). Music tempo estimation: Are we done yet? +- Chiu, C.-Y., Su, A. W.-Y., & Yang, Y.-H. (2025). Cross-modal approaches to beat tracking: A + case study on Chopin Mazurkas. *Transactions of the International Society for Music + Information Retrieval, 8*(1), 55–69. https://doi.org/10.5334/tismir.238 +- Schreiber, H., Urbano, J., & Müller, M. (2020). Music tempo estimation: Are we done yet? *Transactions of the International Society for Music Information Retrieval, 3*(1), 111–125. https://doi.org/10.5334/tismir.43 - Stöter, F.-R., Liutkus, A., & Ito, N. (2018). The 2018 signal separation evaluation campaign. diff --git a/services/analysis-engine/tests/test_accuracy_metric_contract.py b/services/analysis-engine/tests/test_accuracy_metric_contract.py new file mode 100644 index 000000000..755b318cf --- /dev/null +++ b/services/analysis-engine/tests/test_accuracy_metric_contract.py @@ -0,0 +1,23 @@ +"""Rehearsal metric-authority contract for the known-stem / #770 slice.""" + +from __future__ import annotations + +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[3] +DOCTORING = REPO_ROOT / "docs" / "doctoring" / "real-audio-accuracy-acceptance.md" + + +def test_metric_authority_forbids_acc2_alone_and_names_owners() -> None: + """Doctoring must keep rehearsal metric owners exact and non-substitutable.""" + text = DOCTORING.read_text(encoding="utf-8") + assert "Acc2 alone is forbidden" in text + assert "Schreiber, Urbano, & Müller (2020)" in text + assert "Raffel et al. (2014) MIR_EVAL does not define Acc1 or Acc2" in text + assert "Chiu et al. (2025)" in text + assert "±70 ms" in text + assert "Odekerken et al. (2021)" in text + assert "WCSR" in text + assert "Le Roux et al. (2019)" in text + assert "SI-SDR is the primary" in text + assert "Schreiber, H., & Müller, M. (2020)" not in text From 24a420f920f1e3d2de1720c96dc3acf1b1a312dd Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 17 Aug 2026 17:37:20 +0900 Subject: [PATCH 32/34] test(analysis): lock rehearsal metric admission policy --- .../src/bandscope_analysis/metrics_policy.py | 68 ++++++++++++++++ .../tests/test_metrics_policy.py | 79 +++++++++++++++++++ 2 files changed, 147 insertions(+) create mode 100644 services/analysis-engine/src/bandscope_analysis/metrics_policy.py create mode 100644 services/analysis-engine/tests/test_metrics_policy.py diff --git a/services/analysis-engine/src/bandscope_analysis/metrics_policy.py b/services/analysis-engine/src/bandscope_analysis/metrics_policy.py new file mode 100644 index 000000000..cc057ac4c --- /dev/null +++ b/services/analysis-engine/src/bandscope_analysis/metrics_policy.py @@ -0,0 +1,68 @@ +"""Rehearsal metric admission policy for accuracy and known-stem gates. + +This module does not implement a new MIR estimator. It records the admitted +metric names, forbids rehearsal-unsafe solo scores, and keeps citation +boundaries exact so #828 can own #770 without inventing a parallel product. +""" + +from __future__ import annotations + +from collections.abc import Iterable + +PRIMARY_SEPARATION_METRIC = "si_sdr" +PRIMARY_HARMONY_METRIC = "wcsr" +REHEARSAL_ONSET_TOLERANCE_SECONDS = 0.070 +RAFFEL_MIR_EVAL_TEMPO_METRICS = frozenset({"p_score", "alotc"}) +MIREX_TEMPO_METRICS = frozenset({"acc1", "acc2"}) +FORBIDDEN_SOLO_REHEARSAL_METRICS = frozenset({"acc2"}) + + +def normalize_metric_name(name: str) -> str: + """Return a lowercased, hyphen-stripped metric identifier.""" + return name.strip().lower().replace("-", "_") + + +def is_raffel_tempo_metric(name: str) -> bool: + """Return whether the name exists in Raffel et al. (2014) mir_eval tempo. + + Raffel ``mir_eval.tempo`` exposes P-score and ALOTC. It does not define + Acc1 or Acc2; those names belong to MIREX tempo estimation. + """ + return normalize_metric_name(name) in RAFFEL_MIR_EVAL_TEMPO_METRICS + + +def is_mirex_tempo_accuracy(name: str) -> bool: + """Return whether the name is MIREX tempo Acc1/Acc2, not a Raffel metric.""" + return normalize_metric_name(name) in MIREX_TEMPO_METRICS + + +def rehearsal_onset_tolerance_seconds() -> float: + """Return the Chiu (2025) ±70 ms rehearsal onset/beat window in seconds.""" + return REHEARSAL_ONSET_TOLERANCE_SECONDS + + +def validate_rehearsal_metric_set(metrics: Iterable[str]) -> tuple[str, ...]: + """Admit a rehearsal metric set or raise ``ValueError``. + + Acc2 alone is forbidden: Schreiber, Urbano, and Müller (2020) show that + half/double-tempo credit hides the octave errors that wreck count-ins and + groove lock. Harmony gates use Odekerken/MIREX WCSR. Separation gates use + Le Roux SI-SDR as the primary score. + """ + normalized = tuple(normalize_metric_name(name) for name in metrics if name.strip()) + if not normalized: + raise ValueError("rehearsal metric set must not be empty") + unique = frozenset(normalized) + if unique <= FORBIDDEN_SOLO_REHEARSAL_METRICS: + raise ValueError("Acc2 alone is forbidden for rehearsal acceptance") + return normalized + + +def primary_metric_for_domain(domain: str) -> str: + """Return the primary admitted metric for a registered accuracy domain.""" + key = normalize_metric_name(domain) + if key in {"separation", "source_separation", "stems"}: + return PRIMARY_SEPARATION_METRIC + if key in {"harmony", "chords", "chord"}: + return PRIMARY_HARMONY_METRIC + raise ValueError(f"no primary rehearsal metric is registered for {domain!r}") diff --git a/services/analysis-engine/tests/test_metrics_policy.py b/services/analysis-engine/tests/test_metrics_policy.py new file mode 100644 index 000000000..75aee6b6a --- /dev/null +++ b/services/analysis-engine/tests/test_metrics_policy.py @@ -0,0 +1,79 @@ +"""Tests for rehearsal metric admission policy.""" + +from __future__ import annotations + +import pytest + +from bandscope_analysis.metrics_policy import ( + PRIMARY_HARMONY_METRIC, + PRIMARY_SEPARATION_METRIC, + is_mirex_tempo_accuracy, + is_raffel_tempo_metric, + primary_metric_for_domain, + rehearsal_onset_tolerance_seconds, + validate_rehearsal_metric_set, +) + + +def test_acc2_alone_is_forbidden_for_rehearsal() -> None: + """Acc2-only sets cannot pass rehearsal acceptance.""" + with pytest.raises(ValueError, match="Acc2 alone is forbidden"): + validate_rehearsal_metric_set(["acc2"]) + with pytest.raises(ValueError, match="Acc2 alone is forbidden"): + validate_rehearsal_metric_set(["Acc2", "acc2"]) + + +def test_acc1_and_acc2_together_remain_visible() -> None: + """Half/double-tempo credit may appear only beside Acc1, never alone.""" + assert validate_rehearsal_metric_set(["acc1", "acc2"]) == ("acc1", "acc2") + + +def test_empty_metric_set_is_rejected() -> None: + """An empty rehearsal gate is not a pass.""" + with pytest.raises(ValueError, match="must not be empty"): + validate_rehearsal_metric_set([]) + with pytest.raises(ValueError, match="must not be empty"): + validate_rehearsal_metric_set([" "]) + + +def test_raffel_mir_eval_has_no_acc1_or_acc2() -> None: + """Raffel 2014 tempo metrics are P-score and ALOTC, not Acc1/Acc2.""" + assert is_raffel_tempo_metric("p-score") is True + assert is_raffel_tempo_metric("ALOTC") is True + assert is_raffel_tempo_metric("acc1") is False + assert is_raffel_tempo_metric("acc2") is False + assert is_mirex_tempo_accuracy("acc1") is True + assert is_mirex_tempo_accuracy("Acc2") is True + assert is_mirex_tempo_accuracy("p_score") is False + + +def test_chiu_2025_onset_window_is_70_milliseconds() -> None: + """Rehearsal beat/onset tolerance stays at Chiu (2025) ±70 ms.""" + assert rehearsal_onset_tolerance_seconds() == pytest.approx(0.070) + + +def test_le_roux_si_sdr_is_primary_separation_metric() -> None: + """Source-separation gates use Le Roux SI-SDR as the primary score.""" + assert primary_metric_for_domain("separation") == PRIMARY_SEPARATION_METRIC + assert PRIMARY_SEPARATION_METRIC == "si_sdr" + assert primary_metric_for_domain("stems") == "si_sdr" + assert primary_metric_for_domain("source_separation") == "si_sdr" + + +def test_odekerken_wcsr_is_primary_harmony_metric() -> None: + """Harmony gates use Odekerken/MIREX weighted chord symbol recall.""" + assert primary_metric_for_domain("harmony") == PRIMARY_HARMONY_METRIC + assert PRIMARY_HARMONY_METRIC == "wcsr" + assert primary_metric_for_domain("chords") == "wcsr" + assert primary_metric_for_domain("chord") == "wcsr" + + +def test_unknown_domain_has_no_invented_primary_metric() -> None: + """Unregistered domains fail closed instead of inventing a product metric.""" + with pytest.raises(ValueError, match="no primary rehearsal metric"): + primary_metric_for_domain("genre-embedding") + + +def test_si_sdr_and_wcsr_are_valid_rehearsal_sets() -> None: + """Primary admitted scores form a valid rehearsal metric set.""" + assert validate_rehearsal_metric_set(["SI-SDR", "WCSR"]) == ("si_sdr", "wcsr") From 9331b406e7fbe4d1407f9f7854db1fd9ba12b194 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 18 Aug 2026 07:08:30 +0000 Subject: [PATCH 33/34] test(analysis): require Acc1+Acc2 together and admit Chiu beat F-measure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tempo stays a pair from Schreiber, Urbano, and Müller (2020). Acc2 alone stays forbidden, Raffel 2014 cannot stand in for Acc1/Acc2, beat/onset admits F-measure inside the Chiu 2025 ±70 ms window, and SI-SDR/WCSR remain the primary separation and harmony scores. --- ARCHITECTURE.md | 6 +++- CHANGELOG.md | 2 +- CLAUDE.md | 2 +- .../src/bandscope_analysis/metrics_policy.py | 27 +++++++++++++- .../tests/test_metrics_policy.py | 35 +++++++++++++++++++ 5 files changed, 68 insertions(+), 4 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index ba16a825e..c9f67ca4c 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -119,7 +119,11 @@ Last updated: 2026-08-10 ephemeral storage. - The finished master proves candidate identity. YouTube-to-master and master-to-vocal global lags are composed once before separation; predicted stems are never realigned. Quality requires - duration/identity checks, zero-mean SI-SDR improvement over the downloaded mixture. SI-SDR remains the primary separation score (Le Roux et al., 2019). This branch does not accept tempo Acc2 alone, and correct + duration/identity checks, zero-mean SI-SDR improvement over the downloaded mixture. SI-SDR remains + the primary separation score (Le Roux et al., 2019). Harmony uses Odekerken/MIREX WCSR. Beat/onset + F-measure stays inside Chiu et al. (2025) ±70 ms. Tempo requires Schreiber, Urbano, & Müller + (2020) Acc1 and Acc2 together; Acc2 alone is forbidden, and Raffel (2014) is not an Acc1/Acc2 + source. This branch also requires correct vocal-stem assignment margin. - Deterministic metric/integrity/security contracts run in ordinary CI. Live network/model execution is explicit opt-in and cannot be scheduled or made release-blocking until authorization and diff --git a/CHANGELOG.md b/CHANGELOG.md index bcdd56cd2..bf643eae7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ ### Changed -- Lock rehearsal metric authority: Le Roux SI-SDR primary, Odekerken/MIREX WCSR, Chiu 2025 ±70 ms beat F-measure, Schreiber/Urbano/Müller Acc1+Acc2 with Acc2-alone forbidden, and Raffel 2014 not cited as an Acc1/Acc2 source. +- Lock rehearsal metric authority: Le Roux SI-SDR primary, Odekerken/MIREX WCSR, Chiu 2025 ±70 ms beat F-measure, Schreiber/Urbano/Müller Acc1+Acc2 with Acc2-alone forbidden, and Raffel 2014 not cited as an Acc1/Acc2 source. Tempo has no single primary metric; beat/onset admits F-measure only inside the 70 ms window. ### Added diff --git a/CLAUDE.md b/CLAUDE.md index 93ef03196..b2953de8e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -57,7 +57,7 @@ Three layers, decoupled through shared contracts: - `apps/desktop` — Tauri 2 + Vite + React 19 shell (Tailwind 4, Base UI, Storybook). Feature screens live in `src/features/` (home, workspace, chords, ranges, player, settings). `src/lib/analysis.ts` and `src/lib/job_runner.ts` call typed Tauri IPC commands, with a browser fallback that serves demo data when not running inside Tauri. - `apps/desktop/src-tauri/src/main.rs` — the Rust orchestration boundary. Tauri commands (`start_analysis_job`, `get_analysis_job_status`, `select_local_audio_source`, `import_youtube_url`) validate untrusted input (project IDs, file paths, URLs) and spawn the Python engine as a subprocess. There is no loopback HTTP listener and no network path for local analysis. -- `services/analysis-engine` — Python package `bandscope_analysis` (librosa/numpy). Entry point `cli.py` reads a JSON job request on stdin and prints a structured job-status JSON envelope on stdout (`--progress-jsonl` streams progress lines). `api.py` orchestrates the pipeline across the `separation`, `sections`, `roles`, `chords`, `ranges`, `temporal`, `transcription`, and `youtube` modules. +- `services/analysis-engine` — Python package `bandscope_analysis` (librosa/numpy). Entry point `cli.py` reads a JSON job request on stdin and prints a structured job-status JSON envelope on stdout (`--progress-jsonl` streams progress lines). `api.py` orchestrates the pipeline across the `separation`, `sections`, `roles`, `chords`, `ranges`, `temporal`, `transcription`, and `youtube` modules. Metric admission (#828 owns #770) uses Le Roux SI-SDR, Odekerken/MIREX WCSR, Chiu ±70 ms F-measure, and Schreiber/Urbano/Müller Acc1+Acc2; Acc2 alone is forbidden and Raffel 2014 is not an Acc1/Acc2 source. - Production source separation uses `htdemucs` on supported platforms. The exact runtime model artifact is inventoried but not bundled; operators must provision it locally, and production verifies its byte size and full SHA-256 before passing those same in-memory bytes through a diff --git a/services/analysis-engine/src/bandscope_analysis/metrics_policy.py b/services/analysis-engine/src/bandscope_analysis/metrics_policy.py index cc057ac4c..ee5b4b786 100644 --- a/services/analysis-engine/src/bandscope_analysis/metrics_policy.py +++ b/services/analysis-engine/src/bandscope_analysis/metrics_policy.py @@ -11,9 +11,11 @@ PRIMARY_SEPARATION_METRIC = "si_sdr" PRIMARY_HARMONY_METRIC = "wcsr" +PRIMARY_BEAT_METRIC = "f_measure" +REQUIRED_TEMPO_METRICS = ("acc1", "acc2") REHEARSAL_ONSET_TOLERANCE_SECONDS = 0.070 RAFFEL_MIR_EVAL_TEMPO_METRICS = frozenset({"p_score", "alotc"}) -MIREX_TEMPO_METRICS = frozenset({"acc1", "acc2"}) +MIREX_TEMPO_METRICS = frozenset(REQUIRED_TEMPO_METRICS) FORBIDDEN_SOLO_REHEARSAL_METRICS = frozenset({"acc2"}) @@ -41,6 +43,11 @@ def rehearsal_onset_tolerance_seconds() -> float: return REHEARSAL_ONSET_TOLERANCE_SECONDS +def required_tempo_metrics() -> tuple[str, str]: + """Return the Schreiber, Urbano, and Müller (2020) Acc1+Acc2 pair.""" + return REQUIRED_TEMPO_METRICS + + def validate_rehearsal_metric_set(metrics: Iterable[str]) -> tuple[str, ...]: """Admit a rehearsal metric set or raise ``ValueError``. @@ -58,6 +65,20 @@ def validate_rehearsal_metric_set(metrics: Iterable[str]) -> tuple[str, ...]: return normalized +def validate_tempo_metric_set(metrics: Iterable[str]) -> tuple[str, ...]: + """Admit a tempo set only when Acc1 and Acc2 are both present. + + Raffel et al. (2014) P-score/ALOTC cannot stand in for Acc1/Acc2. + """ + admitted = validate_rehearsal_metric_set(metrics) + unique = frozenset(admitted) + if unique & RAFFEL_MIR_EVAL_TEMPO_METRICS and not unique >= frozenset(REQUIRED_TEMPO_METRICS): + raise ValueError("Raffel 2014 does not define Acc1 or Acc2") + if not unique >= frozenset(REQUIRED_TEMPO_METRICS): + raise ValueError("tempo acceptance requires Acc1 and Acc2") + return admitted + + def primary_metric_for_domain(domain: str) -> str: """Return the primary admitted metric for a registered accuracy domain.""" key = normalize_metric_name(domain) @@ -65,4 +86,8 @@ def primary_metric_for_domain(domain: str) -> str: return PRIMARY_SEPARATION_METRIC if key in {"harmony", "chords", "chord"}: return PRIMARY_HARMONY_METRIC + if key in {"beat", "onset", "onsets"}: + return PRIMARY_BEAT_METRIC + if key == "tempo": + raise ValueError("tempo requires Acc1 and Acc2; Acc2 alone is forbidden") raise ValueError(f"no primary rehearsal metric is registered for {domain!r}") diff --git a/services/analysis-engine/tests/test_metrics_policy.py b/services/analysis-engine/tests/test_metrics_policy.py index 75aee6b6a..d8798e177 100644 --- a/services/analysis-engine/tests/test_metrics_policy.py +++ b/services/analysis-engine/tests/test_metrics_policy.py @@ -5,13 +5,16 @@ import pytest from bandscope_analysis.metrics_policy import ( + PRIMARY_BEAT_METRIC, PRIMARY_HARMONY_METRIC, PRIMARY_SEPARATION_METRIC, is_mirex_tempo_accuracy, is_raffel_tempo_metric, primary_metric_for_domain, rehearsal_onset_tolerance_seconds, + required_tempo_metrics, validate_rehearsal_metric_set, + validate_tempo_metric_set, ) @@ -68,6 +71,38 @@ def test_odekerken_wcsr_is_primary_harmony_metric() -> None: assert primary_metric_for_domain("chord") == "wcsr" +def test_chiu_f_measure_is_primary_beat_metric() -> None: + """Beat/onset gates use F-measure inside the Chiu ±70 ms window.""" + assert primary_metric_for_domain("beat") == PRIMARY_BEAT_METRIC + assert PRIMARY_BEAT_METRIC == "f_measure" + assert primary_metric_for_domain("onset") == "f_measure" + assert primary_metric_for_domain("onsets") == "f_measure" + + +def test_tempo_has_no_single_primary_metric() -> None: + """Tempo cannot collapse to Acc2 or any other single score.""" + with pytest.raises(ValueError, match="tempo requires Acc1 and Acc2"): + primary_metric_for_domain("tempo") + assert required_tempo_metrics() == ("acc1", "acc2") + + +def test_tempo_set_requires_acc1_and_acc2() -> None: + """Schreiber/Urbano/Müller tempo admission is the Acc1+Acc2 pair.""" + assert validate_tempo_metric_set(["acc1", "acc2"]) == ("acc1", "acc2") + with pytest.raises(ValueError, match="tempo acceptance requires Acc1 and Acc2"): + validate_tempo_metric_set(["acc1"]) + with pytest.raises(ValueError, match="Acc2 alone is forbidden"): + validate_tempo_metric_set(["acc2"]) + + +def test_raffel_scores_cannot_replace_acc1_acc2() -> None: + """Raffel P-score/ALOTC are not a rehearsal tempo pair.""" + with pytest.raises(ValueError, match="Raffel 2014 does not define Acc1 or Acc2"): + validate_tempo_metric_set(["p-score"]) + with pytest.raises(ValueError, match="Raffel 2014 does not define Acc1 or Acc2"): + validate_tempo_metric_set(["p_score", "alotc"]) + + def test_unknown_domain_has_no_invented_primary_metric() -> None: """Unregistered domains fail closed instead of inventing a product metric.""" with pytest.raises(ValueError, match="no primary rehearsal metric"): From d50a578739e62156839b8633a7ff118dd70f6fbf Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 27 Aug 2026 17:16:36 +0000 Subject: [PATCH 34/34] test(analysis): close htdemucs cache-descriptor coverage gap Keep #828 as the known-stem vehicle. Fail closed on pre-open lstat and open races, always close an obtained descriptor, and redact cache paths so rehearsal SI-SDR evidence cannot skip the inventoried checkpoint. --- CHANGELOG.md | 2 + .../separation/audio_separator.py | 28 ++++--- .../analysis-engine/tests/test_separation.py | 84 +++++++++++++++++++ 3 files changed, 104 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ddd7cd95f..50832f40b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,8 @@ byte size, and full SHA-256, rejects filesystem identity races, and deserializes the verified bytes with PyTorch's restricted `weights_only` loader, an exact reviewed global allowlist, strict model construction, and serialized one-time caching rather than downloading a missing checkpoint. + Pre-open `lstat` and `open` failures stay redacted and close every obtained descriptor without a + None-check fallthrough, so a raced-away cache entry cannot skip the close or leak a path. - Verified exact platform-native sibling ffmpeg/ffprobe executable names and identities before any live fixture access or yt-dlp invocation. - Isolated Numba's native-code cache for repository analysis commands so a stale or concurrently diff --git a/services/analysis-engine/src/bandscope_analysis/separation/audio_separator.py b/services/analysis-engine/src/bandscope_analysis/separation/audio_separator.py index a1e708d5b..9d1ddc038 100644 --- a/services/analysis-engine/src/bandscope_analysis/separation/audio_separator.py +++ b/services/analysis-engine/src/bandscope_analysis/separation/audio_separator.py @@ -337,16 +337,27 @@ def _as_float_array(values: object) -> AudioStemArray: def _read_verified_model_artifact(path: Path, artifact: _ModelArtifactSpec) -> bytes: """Read one exact regular cache file and verify its full artifact identity.""" - descriptor: int | None = None try: cache_metadata = path.lstat() - if stat.S_ISLNK(cache_metadata.st_mode): - raise ModelArtifactError("Stem separation model cache entry is a symlink") - if not stat.S_ISREG(cache_metadata.st_mode): - raise ModelArtifactError("Stem separation model cache entry is not a regular file") + except FileNotFoundError: + raise ModelArtifactError("Stem separation model is not provisioned") from None + except OSError: + raise ModelArtifactError("Stem separation model could not be opened securely") from None + + if stat.S_ISLNK(cache_metadata.st_mode): + raise ModelArtifactError("Stem separation model cache entry is a symlink") + if not stat.S_ISREG(cache_metadata.st_mode): + raise ModelArtifactError("Stem separation model cache entry is not a regular file") - flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) + flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) + try: descriptor = os.open(path, flags) + except FileNotFoundError: + raise ModelArtifactError("Stem separation model is not provisioned") from None + except OSError: + raise ModelArtifactError("Stem separation model could not be opened securely") from None + + try: opened_metadata = os.fstat(descriptor) if not stat.S_ISREG(opened_metadata.st_mode): raise ModelArtifactError("Stem separation model cache entry is not a regular file") @@ -354,15 +365,12 @@ def _read_verified_model_artifact(path: Path, artifact: _ModelArtifactSpec) -> b raise ModelArtifactError("Stem separation model does not match inventoried byte size") with os.fdopen(descriptor, "rb", closefd=False) as fileobj: payload = fileobj.read(artifact.size_bytes + 1) - except FileNotFoundError: - raise ModelArtifactError("Stem separation model is not provisioned") from None except ModelArtifactError: raise except OSError: raise ModelArtifactError("Stem separation model could not be opened securely") from None finally: - if descriptor is not None: - os.close(descriptor) + os.close(descriptor) if hashlib.sha256(payload).hexdigest() != artifact.sha256: raise ModelArtifactError("Stem separation model does not match inventoried SHA-256") diff --git a/services/analysis-engine/tests/test_separation.py b/services/analysis-engine/tests/test_separation.py index 1b4ac7952..c5f3647fb 100644 --- a/services/analysis-engine/tests/test_separation.py +++ b/services/analysis-engine/tests/test_separation.py @@ -7,6 +7,7 @@ import sys from concurrent.futures import ThreadPoolExecutor from fractions import Fraction +from pathlib import Path from threading import Event, Lock from types import ModuleType, SimpleNamespace @@ -549,6 +550,89 @@ def fail_open(*args: object, **kwargs: object) -> int: assert str(tmp_path) not in str(error.value) +def test_audio_stem_separator_redacts_model_cache_lstat_errors( + tmp_path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Redact cache paths when pre-open metadata lookup fails closed.""" + payload = b"verified-model-package" + filename = "test-signature-deadbeef.th" + artifact_path = tmp_path / filename + artifact_path.write_bytes(payload) + _patch_model_spec(monkeypatch, filename=filename, payload=payload) + _install_fake_verified_model_deserializer(monkeypatch) + original_lstat = Path.lstat + + def fail_lstat(self: Path) -> os.stat_result: + if self == artifact_path: + raise PermissionError(f"permission denied under {tmp_path}") + return original_lstat(self) + + monkeypatch.setattr(Path, "lstat", fail_lstat) + + def forbidden_open(*args: object, **kwargs: object) -> int: + raise AssertionError("failed lstat reached os.open") + + monkeypatch.setattr(audio_separator_module.os, "open", forbidden_open) + separator = AudioStemSeparator(AudioSeparationConfig(model_cache_directory=tmp_path)) + + with pytest.raises(ValueError, match="could not be opened securely") as error: + separator._load_model() + assert str(tmp_path) not in str(error.value) + + +def test_audio_stem_separator_treats_open_toctou_as_unprovisioned( + tmp_path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Keep a raced-away checkpoint fail-closed as not provisioned after lstat.""" + payload = b"verified-model-package" + filename = "test-signature-deadbeef.th" + (tmp_path / filename).write_bytes(payload) + _patch_model_spec(monkeypatch, filename=filename, payload=payload) + _install_fake_verified_model_deserializer(monkeypatch) + + def vanish_on_open(*args: object, **kwargs: object) -> int: + raise FileNotFoundError("checkpoint vanished after lstat") + + monkeypatch.setattr(audio_separator_module.os, "open", vanish_on_open) + separator = AudioStemSeparator(AudioSeparationConfig(model_cache_directory=tmp_path)) + + with pytest.raises(ValueError, match="not provisioned"): + separator._load_model() + + +def test_audio_stem_separator_redacts_model_cache_fstat_errors( + tmp_path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Close the descriptor and redact paths when post-open fstat fails.""" + payload = b"verified-model-package" + filename = "test-signature-deadbeef.th" + (tmp_path / filename).write_bytes(payload) + _patch_model_spec(monkeypatch, filename=filename, payload=payload) + _install_fake_verified_model_deserializer(monkeypatch) + close_count = 0 + original_close = audio_separator_module.os.close + + def counted_close(descriptor: int) -> None: + nonlocal close_count + close_count += 1 + original_close(descriptor) + + def fail_fstat(_descriptor: int) -> os.stat_result: + raise OSError(f"fstat failed under {tmp_path}") + + monkeypatch.setattr(audio_separator_module.os, "fstat", fail_fstat) + monkeypatch.setattr(audio_separator_module.os, "close", counted_close) + separator = AudioStemSeparator(AudioSeparationConfig(model_cache_directory=tmp_path)) + + with pytest.raises(ValueError, match="could not be opened securely") as error: + separator._load_model() + assert str(tmp_path) not in str(error.value) + assert close_count == 1 + + def test_audio_stem_separator_redacts_default_cache_location_errors( tmp_path, monkeypatch: pytest.MonkeyPatch,