From 2c061ad770c85fc58ff2c949551e8706ccd0ec56 Mon Sep 17 00:00:00 2001 From: Alex Rubinsteyn Date: Fri, 2 Oct 2026 00:17:44 -0400 Subject: [PATCH] Ship complete test support in source distributions --- .github/workflows/tests.yml | 18 +++++++--- CHANGELOG.md | 9 +++++ CONTRIBUTING.md | 2 +- MANIFEST.in | 7 ++++ tests/README.md | 35 ++++++++++++++++++ tests/data/README.md | 60 +++++++++++++++++++++++++++++++ tests/data/SHA256SUMS | 39 ++++++++++++++++++++ tests/test_source_distribution.py | 55 ++++++++++++++++++++++++++++ varcode/version.py | 2 +- 9 files changed, 221 insertions(+), 6 deletions(-) create mode 100644 tests/data/README.md create mode 100644 tests/data/SHA256SUMS create mode 100644 tests/test_source_distribution.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f9d497ba..3a484560 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -31,11 +31,17 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - python -m pip install flake8 pytest pytest-cov coveralls ruff + python -m pip install flake8 pytest pytest-cov coveralls ruff build 'setuptools>=64' pip install -r requirements.txt + - name: Verify, build and install the source distribution + run: | + python -m pytest -q tests/test_source_distribution.py + python -m build --sdist --outdir "$RUNNER_TEMP/varcode-dist" + mkdir "$RUNNER_TEMP/varcode-sdist" + tar -xzf "$RUNNER_TEMP"/varcode-dist/varcode-*.tar.gz --strip-components=1 -C "$RUNNER_TEMP/varcode-sdist" # Osteosarc 0.14.x supports every Python version in this matrix; # run native adapter checks against the bundled historical snapshot. - pip install '.[test-data]' + python -m pip install "$RUNNER_TEMP/varcode-sdist[test-data]" - name: Lint with flake8 run: | # stop the build if there are Python syntax errors or undefined names @@ -72,14 +78,16 @@ jobs: pyensembl install --release 95 --species human --custom-mirror https://github.com/openvax/ensembl-data/releases/download/GRCh38.95/ pyensembl install --release 95 --species mouse --custom-mirror https://github.com/openvax/ensembl-data/releases/download/GRCm38.95/ echo "After installing Ensembl releases" && df -h - - name: Run unit tests + - name: Run unit tests from the source distribution + working-directory: ${{ runner.temp }}/varcode-sdist run: | - ./test.sh + ./test.sh --cov-report=xml:coverage.xml # Experimental annotators may decline inputs. Compare protein_diff on # the shared point-edit domain; the full suite above separately checks # SV refusal, partial plugins, and explicit dual-annotator regressions. - name: Cross-annotator parity gate (shared point-edit domain) if: matrix.python-version == '3.11' + working-directory: ${{ runner.temp }}/varcode-sdist env: COVERAGE_FILE: .coverage.parity run: | @@ -88,3 +96,5 @@ jobs: uses: coverallsapp/github-action@v2.3.6 with: parallel: true + file: ${{ runner.temp }}/varcode-sdist/coverage.xml + base-path: ${{ runner.temp }}/varcode-sdist diff --git a/CHANGELOG.md b/CHANGELOG.md index 1314dca3..1d923ca1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Change Log +## [v10.11.4](https://github.com/openvax/varcode/tree/v10.11.4) (2026-10-02) + +- Ship test helpers, bounded fixtures, provenance/checksums and the docs, + examples and scripts required to run tests from the source distribution + (#509). Existing fixture payloads are unchanged; large shared RNA assets + remain external. +- Verify archive contents and fixture integrity, and run the CI test matrix + and parity checks from an installed, unpacked source archive. + ## [v10.11.3](https://github.com/openvax/varcode/tree/v10.11.3) (2026-10-01) - Guard deployment with a clean main/master checkout synchronized with live diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 39e6a878..f57ad499 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -23,7 +23,7 @@ python3 -m venv .venv source .venv/bin/activate python -m pip install -e '.[rna,test-data]' pytest pytest-cov pytest-xdist ruff python -m pip install mkdocs mkdocs-material 'mkdocstrings[python]' -python -m pip install build twine +python -m pip install build twine 'setuptools>=64' ``` `rna` installs the optional BAM reader; `test-data` installs the pinned diff --git a/MANIFEST.in b/MANIFEST.in index 04f196ac..5dfc40f1 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,9 @@ include README.md include LICENSE +include AGENTS.md CHANGELOG.md CONTRIBUTING.md RELEASING.md +include develop.sh lint.sh test.sh deploy.sh mkdocs.yml .coveragerc +recursive-include tests *.py *.md +graft tests/data +recursive-include docs *.md +recursive-include examples *.py +global-exclude *.py[cod] diff --git a/tests/README.md b/tests/README.md index 359f7c11..7752d432 100644 --- a/tests/README.md +++ b/tests/README.md @@ -15,6 +15,41 @@ or modify the reference cache. Install the documented datasets before annotation tests; explicit release choices remain available. The offline guarantees below apply to the bundled Osteosarc snapshot checks. +## Source-distribution contract + +The source archive ships the complete test Python package, the bounded +`tests/data/` tree with its [checksum inventory and provenance](data/README.md), +and the documentation, Python examples, shell scripts and configuration used +by those tests. `MANIFEST.in` declares these inputs; generated bytecode is +excluded. Wheels contain the runtime library, without the development tests. +Large reference datasets and the shared RNA bundle remain external, installed +through the documented PyEnsembl/Osteosarc mechanisms. + +`tests/test_source_distribution.py` checks every bundled fixture hash and +builds an archive to verify that all test support files retain their bytes. +It needs `setuptools>=64`, as installed by the contributor setup. The build +uses the installed backend without downloading build dependencies. + +CI builds and unpacks the source distribution outside the checkout, installs +that source with the `test-data` extra, and runs the full suite and parity +checks from the unpacked directory on Python 3.9/3.10/3.11. To reproduce this +locally, first install the development tools and reference releases from +[CONTRIBUTING.md](../CONTRIBUTING.md), then: + +```sh +python -m build --sdist --outdir /tmp/varcode-dist +mkdir /tmp/varcode-sdist +tar -xzf /tmp/varcode-dist/varcode-X.Y.Z.tar.gz --strip-components=1 -C /tmp/varcode-sdist +cd /tmp/varcode-sdist +python -m pip install '.[test-data]' +./test.sh +``` + +Use a new output/unpack directory and replace `X.Y.Z` with the built version. +The checkout is not needed after the archive is built. The same reference +and optional-dependency requirements apply to checkout and archive tests; +the shared RNA checks can download the pinned bundle on first use. + ## Osteosarc test variants `tests/data/osteosarc_variants.json` contains 179 ready site variants and three diff --git a/tests/data/README.md b/tests/data/README.md new file mode 100644 index 00000000..b1386405 --- /dev/null +++ b/tests/data/README.md @@ -0,0 +1,60 @@ +# Bundled fixture inventory + +`SHA256SUMS` records every file in this directory tree, including the provenance +notes, except the checksum file itself. The fixtures total about 5 MB; the +largest is the 3.6 MiB compressed historical Osteosarc metadata snapshot. +No BAM/CRAM files or shared RNA bundles belong here. See +[the test-data guide](../README.md) for reference installation and external +`openvax-v2` assets. + +## Sources and recipes + +| Files | Recorded source and reproduction information | +|---|---| +| `osteosarc_variants.json`, `osteosarc_snapshot_2026-09-18t.zip` | [Snapshot identity, receipts, archive hash and offline export recipe](../README.md#regenerate-from-the-verified-snapshot). `../collect_osteosarc_variants.py` exports the reviewed snapshot with the pinned Osteosarc version. | +| `osteosarc_observed_junctions.json` | Each record retains its source page, ONT read name, reference identity and source-sequence SHA256. [Shared read verification](../README.md#shared-test-data-openvax-v2) uses the external bundle; only the bounded junction windows are stored here. | +| `osteosarc_esvee_somatic.vcf`, `osteosarc_esvee_shared_svs.vcf` | Published Sid esvee records with FORMAT/sample columns removed. The subset selection and reference releases are documented in [the fusion tests](../test_osteosarc_fusions.py) and [shared SV tests](../test_osteosarc_shared_svs.py). | +| `real_callers/*.vcf` | [Pinned GATK source and synthetic Strelka2/VEP construction notes](real_callers/README.md). | +| `spec_examples/*.vcf` | [VCF specification sections and source](spec_examples/README.md). | +| `documentation/*.vcf` | Small inputs constructed for [executable documentation tests](../test_documentation_examples.py), introduced in [14789b1](https://github.com/openvax/varcode/commit/14789b1) and [8108acd](https://github.com/openvax/varcode/commit/8108acd). | +| `dbnsfp_validation_set.csv`, `somatic_hg19_14muts*.vcf*` | Legacy validation inputs introduced in [5d0d268](https://github.com/openvax/varcode/commit/5d0d268); the compressed VCF was added in [85f232c](https://github.com/openvax/varcode/commit/85f232c). The dbNSFP comparisons use Ensembl 75. | +| `tcga_ov.head*.maf`, `ov.wustle.subset5.maf` | Legacy MAF subsets introduced in [679167a](https://github.com/openvax/varcode/commit/679167a) and [30c9082](https://github.com/openvax/varcode/commit/30c9082); record headers retain their source centers and assemblies. | +| `mouse_vcf_dbsnp_chr1_partial.vcf` | Legacy mouse dbSNP subset introduced in [863167b](https://github.com/openvax/varcode/commit/863167b). | +| `mutect-example*.vcf`, `strelka-example.vcf` | Legacy caller outputs introduced in [302ef49](https://github.com/openvax/varcode/commit/302ef49), retaining available caller/reference header metadata. | +| `simple.*.vcf`, `duplicate-id.*.vcf`, `different-samples.*.vcf`, `same-samples.*.vcf`, `multiallelic.vcf`, `duplicates.vcf`, `duplicates.maf` | Parser/collection regression inputs introduced in [f82edd5](https://github.com/openvax/varcode/commit/f82edd5), [6b53cc7](https://github.com/openvax/varcode/commit/6b53cc7), and [22dcb0f](https://github.com/openvax/varcode/commit/22dcb0f). | + +The original external acquisition recipes for the legacy inputs were not +recorded. Their repository history is the available provenance; no new upstream +identity or scientific validation is inferred here. All fixture payloads in +this release are byte-for-byte those in +[commit 6f29cf8](https://github.com/openvax/varcode/tree/6f29cf8c5201189ae2803ad319aacca8053f7ddf/tests/data). +To recover those exact payloads, use the raw files under that commit's +`tests/data/` tree and verify them against `SHA256SUMS`. The new inventory notes +and checksums are part of the source distribution itself. + +## Verify and update + +From the repository or unpacked source-archive root: + +```sh +python -m pytest -q tests/test_source_distribution.py +``` + +After deliberately changing a fixture or its provenance notes, review the +source, size and test expectations, then regenerate the sorted inventory: + +```python +from hashlib import sha256 +from pathlib import Path + +root = Path("tests/data") +files = sorted(p for p in root.rglob("*") + if p.is_file() and p.name != "SHA256SUMS") +(root / "SHA256SUMS").write_text("".join( + f"{sha256(p.read_bytes()).hexdigest()} {p.relative_to(root).as_posix()}\n" + for p in files)) +``` + +Hashes identify reviewed bytes; they do not establish the scientific correctness +of an expected annotation. Keep independent biological expectations in the +corresponding tests and retain source receipts when adding captured data. diff --git a/tests/data/SHA256SUMS b/tests/data/SHA256SUMS new file mode 100644 index 00000000..1c9b2610 --- /dev/null +++ b/tests/data/SHA256SUMS @@ -0,0 +1,39 @@ +b91532598e8ef0034ac5f9db42ad23b80d4881aac57e278c1c70db57c77c25f1 README.md +e569c37ac74f13111be67d230f0bf03891a419af31afb04c7e3a28212cba8bac dbnsfp_validation_set.csv +a3d5571678f7a4717bec59325fe77131d54d814b27f99c494d944d78a07de501 different-samples.1.vcf +b2cb6090f568cfacdb0df4a9c280161923c255712fa368ac433863a623d126a5 different-samples.2.vcf +236f801922cbf49d2340fce8d8f36dfa43927f7782a06b3719b497e6adcb60e7 documentation/merged_phased.vcf +8ef685edf7465a93f84f25bb8e477c82fc033064fb5361cbc39af1b8a70f3251 documentation/normal.vcf +b0e144d166bfe54f90f7a99c98f536ab2b8019b47d7174bb149fd75abc79cd19 documentation/structural_variants.vcf +83cf46df8895ebaee76e9a986b1b320cca3a44d51ba26cf5ccd74ea144e8f4c1 documentation/variants.vcf +9b1477abc3e7066919df423df17b426417b2f1aa02c718fdece887dd4a735b90 duplicate-id.1.vcf +3683a66ef93e60c4dabb3719ba2642538882667c335534671ef2a76edc8f461e duplicate-id.2.vcf +fb344081f145e465f1afd1d826e7c3cebb6d084d728ccf373adabb9341d02b93 duplicates.maf +f8bf6399170c3acc2db7dd5cde96c0f7da56f986e8817f7055558190985dd013 duplicates.vcf +32a53850ff95ee7b4bfe7d159c93df3b76ba63a61ebc2f61df5e637a7b8ade44 mouse_vcf_dbsnp_chr1_partial.vcf +de0c9703c7b4997dcc323589166d03dd01a9b390a6cdb3cb436201e43f2df73f multiallelic.vcf +ffc371c0d26c1aac8b944a6e3f243e9acc5f72c71d9275f7833d6f4c7d6a2226 mutect-example-headerless.vcf +8d1c9aaf4896556c0f48b4583551af7bd7aba28bf5d03fffcce0fad26448dad6 mutect-example.vcf +fa06fe1f46b701dfc8d2aaad671505afac2ccb5fa32e9ec64aacd8ca4997318d osteosarc_esvee_shared_svs.vcf +26460e9164e6391cdf1faae137a07d64e6f75a204d5f0e447474797926583e3a osteosarc_esvee_somatic.vcf +faf1231c3a1cbc6672998c2b36d0d7dce3f8eb8777ecf1ff8ea1b27c5edb617e osteosarc_observed_junctions.json +cbb688ca5cbe775fc4e6a826124f861d6d605c65e34473582c5d86853a6018fa osteosarc_snapshot_2026-09-18t.zip +04846802f9e7c6e3c45d3634d7931af20c9aac6aa4aec7aa31e80462c3ed8072 osteosarc_variants.json +eaf8f22aec27cce743e7bf32993aa950e127eb5d225d7a159c8ed8e6d5b4ccb3 ov.wustle.subset5.maf +820ed17e47b8bff154554665161e4429fff9f5adfea37e698603b30ef1a221c3 real_callers/README.md +7d5a4f37a0e330129e42ee9eaab3d9abf9101821802feafd223a631058967612 real_callers/mutect2_example.vcf +eabc38358591c380ddb676e2d53a564cb01a33e428b67186a009f5b3cad4ce93 real_callers/strelka2_somatic_snvs.vcf +91c8c7dd7e9915a73fd0e709dde8d5cc74d4f8fa3af14df0dc63dfd400771674 real_callers/vep_annotated_csq.vcf +a3d5571678f7a4717bec59325fe77131d54d814b27f99c494d944d78a07de501 same-samples.1.vcf +a3d5571678f7a4717bec59325fe77131d54d814b27f99c494d944d78a07de501 same-samples.2.vcf +94bf15f0cca60a2d2e14526cc1da7a3933aaedea18f57ba4ada75462ad8943b6 simple.1.vcf +6910e2b188142b0a9702aac57cc88d29987c47e9d4adc1db0bc24b3543d86b47 simple.2.vcf +5d988950e60a08f40fb406d7d40a4e2bf2c4df402c73395753db39933d8409ea somatic_hg19_14muts.space_in_sample_name.vcf +6de8b5931b685b82fcbdd5d009ba65c86ce2eefd41b147dee89511d3588ebeb0 somatic_hg19_14muts.vcf +d5e8e939f887a2b52254a827b1e7a0994791bad6bc319dc8eea2a17c61559440 somatic_hg19_14muts.vcf.gz +749567f5a511672647b113ad3aa79c0258098ac6b27adc1cf55d8112bd110961 spec_examples/README.md +ebce59acc2af48e8cb4258e43f8837073b3a0cac3b85cb0350a965b31d6be23d spec_examples/vcf42_spec_trio.vcf +db4471efa2f67d7da048059cbb83e51fd95c24ce0b2c2cdc9a85154f9c1f73c1 spec_examples/vcf43_spec_sv.vcf +faaed67791d731f81d91bef9b708f933217ca1d03d19f89262cfbff23d55dbd9 strelka-example.vcf +1b78579c3f90e76c60e79bfa55a09ce159860f2fbb37b8c692d05c957edfbf61 tcga_ov.head.maf +9fdf08f2819de3888241cb93e8a303aa6977d03f692bf072042d0737d50dcac2 tcga_ov.head.xychr.maf diff --git a/tests/test_source_distribution.py b/tests/test_source_distribution.py new file mode 100644 index 00000000..0f6bf515 --- /dev/null +++ b/tests/test_source_distribution.py @@ -0,0 +1,55 @@ +"""The source archive must carry the inputs needed to reproduce its tests.""" + +import hashlib +from pathlib import Path +import subprocess +import sys +import tarfile + + +ROOT = Path(__file__).resolve().parents[1] +DATA = ROOT / "tests/data" + + +def test_bundled_fixture_checksums(): + lines = (DATA / "SHA256SUMS").read_text().splitlines() + expected = {name: digest for digest, name in + (line.split(" ", 1) for line in lines)} + assert len(expected) == len(lines), "Duplicate fixture checksum entries" + actual = {path.relative_to(DATA).as_posix(): path for path in DATA.rglob("*") + if path.is_file() and path.name != "SHA256SUMS"} + assert actual.keys() == expected.keys(), "Update the fixture inventory after reviewing changed data" + for name, path in actual.items(): + assert hashlib.sha256(path.read_bytes()).hexdigest() == expected[name], name + + +def test_source_distribution_contains_test_support(tmp_path): + # Run the configured backend without downloads. This also works from an + # extracted sdist, which has no Git metadata or surrounding checkout. + result = subprocess.run( + [sys.executable, "-c", + "import sys; from setuptools.build_meta import build_sdist; build_sdist(sys.argv[1])", + str(tmp_path)], cwd=ROOT, capture_output=True, text=True) + assert result.returncode == 0, result.stdout + result.stderr + archives = list(tmp_path.glob("*.tar.gz")) + assert len(archives) == 1 + + required = {ROOT / name for name in ( + "README.md", "LICENSE", "pyproject.toml", "requirements.txt", "MANIFEST.in", + "AGENTS.md", "CHANGELOG.md", "CONTRIBUTING.md", "RELEASING.md", + "develop.sh", "lint.sh", "test.sh", "deploy.sh", "mkdocs.yml", ".coveragerc", + )} + for pattern in ("tests/**/*.py", "tests/**/*.md", "docs/**/*.md", "examples/**/*.py"): + required.update(ROOT.glob(pattern)) + required.update(path for path in DATA.rglob("*") if path.is_file()) + + with tarfile.open(archives[0]) as archive: + members = {Path(member.name).relative_to(archives[0].name[:-7]).as_posix(): member + for member in archive.getmembers() if member.isfile()} + missing = {path.relative_to(ROOT).as_posix() for path in required} - members.keys() + assert not missing, "Missing source-distribution test support: %s" % sorted(missing) + for path in required: + name = path.relative_to(ROOT).as_posix() + assert archive.extractfile(members[name]).read() == path.read_bytes(), name + assert not any("__pycache__" in Path(name).parts or name.endswith(".pyc") + for name in members) diff --git a/varcode/version.py b/varcode/version.py index 49db6570..d2f4cce5 100644 --- a/varcode/version.py +++ b/varcode/version.py @@ -1 +1 @@ -__version__ = "10.11.3" +__version__ = "10.11.4"