diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 549dc82..3919e31 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,19 +8,18 @@ jobs: strategy: fail-fast: true matrix: - python-version: ["3.8", "3.13"] - poetry-version: ["1.5.1"] + python-version: ["3.9", "3.13"] os: [ubuntu-22.04] #, macos-latest, windows-latest] runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 + - uses: actions/checkout@v5 + - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - - name: Run image - uses: abatilo/actions-poetry@v2 - with: - poetry-version: ${{ matrix.poetry-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install --upgrade poetry - name: Test build run: | poetry config virtualenvs.in-project true @@ -32,22 +31,23 @@ jobs: fail-fast: true runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 + - uses: actions/checkout@v5 + - uses: actions/setup-python@v5 with: - python-version: "3.11" - - name: Run image - uses: abatilo/actions-poetry@v2 - with: - poetry-version: "1.5.1" + python-version: "3.13" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install --upgrade poetry + poetry install --with dev - name: Install the project dependencies run: poetry install - name: Run black formatter run: poetry run black --check --exclude '__pycache__|scripts|sentieon_cli/archive.py' --line-length 79 sentieon_cli - name: Run mypy - run: poetry run mypy sentieon_cli + run: poetry run mypy sentieon_cli --exclude scripts - name: Run flake8 - run: poetry run flake8 . --extend-ignore E231,E221, --exclude .github/scripts/license_message.py,sentieon_cli/scripts/gvcf_combine.py,sentieon_cli/scripts/vcf_mod.py,sentieon_cli/scripts/hybrid_anno.py,sentieon_cli/scripts/hybrid_select.py,sentieon_cli/archive.py,tests # false+ from python 3.12 + run: poetry run flake8 . --extend-ignore E231,E221, --exclude .github/scripts/license_message.py,sentieon_cli/scripts/gvcf_combine.py,sentieon_cli/scripts/vcf_mod.py,sentieon_cli/scripts/hybrid_anno.py,sentieon_cli/scripts/hybrid_select.py,sentieon_cli/archive.py,sentieon_cli/scripts/trimalt.py,playground,tests # false+ from python 3.12 - name: Run the automated tests run: poetry run pytest -v - name: Run doct tests diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4f7682f..a8ca818 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -14,20 +14,19 @@ jobs: fail-fast: true max-parallel: 1 matrix: - python-version: ["3.8", "3.13"] - poetry-version: ["1.5.1"] - sentieon-version: ["202503"] + python-version: ["3.9", "3.13"] + sentieon-version: ["202503.01"] os: [ubuntu-22.04] #, macos-latest, windows-latest] runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 + - uses: actions/checkout@v5 + - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - - name: Run image - uses: abatilo/actions-poetry@v2 - with: - poetry-version: ${{ matrix.poetry-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install --upgrade poetry - name: Install the project dependencies run: | poetry config virtualenvs.in-project true @@ -42,9 +41,16 @@ jobs: pip install multiqc - name: Install sentieon run: | - curl -L https://s3.amazonaws.com/sentieon-release/software/sentieon-genomics-$SENTIEON_VERSION.tar.gz | tar -zxf - + url="https://s3.amazonaws.com/sentieon-release/software/sentieon-genomics-$SENTIEON_VERSION.tar.gz" + bin_dir=$(pwd)/sentieon-genomics-$SENTIEON_VERSION/bin + machine_arch=$(uname -m) + if [[ "$machine_arch" == "arm64" || "$machine_arch" == "aarch64" || "$machine_arch" == "armv7l" ]]; then + url="https://s3.amazonaws.com/sentieon-release/software/arm-sentieon-genomics-$SENTIEON_VERSION.tar.gz" + bin_dir=$(pwd)/arm-sentieon-genomics-$SENTIEON_VERSION/bin + fi + curl -L "$url" | tar -zxf - echo "SENTIEON_VERSION=$SENTIEON_VERSION" >> $GITHUB_ENV - echo "PATH=$(pwd)/sentieon-genomics-$SENTIEON_VERSION/bin:$PATH" >> $GITHUB_ENV + echo "PATH=$bin_dir:$PATH" >> $GITHUB_ENV env: SENTIEON_VERSION: ${{ matrix.sentieon-version }} - name: Download short-read model @@ -91,20 +97,19 @@ jobs: fail-fast: true max-parallel: 1 matrix: - python-version: ["3.8", "3.13"] - poetry-version: ["1.5.1"] - sentieon-version: ["202503"] + python-version: ["3.9", "3.13"] + sentieon-version: ["202503.01"] os: [ubuntu-22.04] #, macos-latest, windows-latest] runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 + - uses: actions/checkout@v5 + - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - - name: Run image - uses: abatilo/actions-poetry@v2 - with: - poetry-version: ${{ matrix.poetry-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install --upgrade poetry - name: Install the project dependencies run: | poetry config virtualenvs.in-project true @@ -112,7 +117,10 @@ jobs: - name: Install bcftools run: | sudo apt-get update - sudo apt-get install -y bcftools + sudo apt-get install libcurl4-gnutls-dev + curl -L https://github.com/samtools/bcftools/releases/download/1.22/bcftools-1.22.tar.bz2 | tar -jxf - + cd bcftools-1.22 + ./configure && sudo make install - name: Install samtools run: | curl -L https://github.com/samtools/samtools/releases/download/1.19.2/samtools-1.19.2.tar.bz2 | tar -jxf - @@ -128,9 +136,16 @@ jobs: sudo chmod ugo+x /usr/local/bin/mosdepth - name: Install sentieon run: | - curl -L https://s3.amazonaws.com/sentieon-release/software/sentieon-genomics-$SENTIEON_VERSION.tar.gz | tar -zxf - + url="https://s3.amazonaws.com/sentieon-release/software/sentieon-genomics-$SENTIEON_VERSION.tar.gz" + bin_dir=$(pwd)/sentieon-genomics-$SENTIEON_VERSION/bin + machine_arch=$(uname -m) + if [[ "$machine_arch" == "arm64" || "$machine_arch" == "aarch64" || "$machine_arch" == "armv7l" ]]; then + url="https://s3.amazonaws.com/sentieon-release/software/arm-sentieon-genomics-$SENTIEON_VERSION.tar.gz" + bin_dir=$(pwd)/arm-sentieon-genomics-$SENTIEON_VERSION/bin + fi + curl -L "$url" | tar -zxf - echo "SENTIEON_VERSION=$SENTIEON_VERSION" >> $GITHUB_ENV - echo "PATH=$(pwd)/sentieon-genomics-$SENTIEON_VERSION/bin:$PATH" >> $GITHUB_ENV + echo "PATH=$bin_dir:$PATH" >> $GITHUB_ENV env: SENTIEON_VERSION: ${{ matrix.sentieon-version }} - name: Download long-read model @@ -204,20 +219,19 @@ jobs: fail-fast: true max-parallel: 1 matrix: - python-version: ["3.8", "3.13"] - poetry-version: ["1.5.1"] - sentieon-version: ["202503.01.rc1"] + python-version: ["3.9", "3.13"] + sentieon-version: ["202503.01"] os: [ubuntu-22.04] #, macos-latest, windows-latest] runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 + - uses: actions/checkout@v5 + - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - - name: Run image - uses: abatilo/actions-poetry@v2 - with: - poetry-version: ${{ matrix.poetry-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install --upgrade poetry - name: Install the project dependencies run: | poetry config virtualenvs.in-project true @@ -225,7 +239,10 @@ jobs: - name: Install bcftools run: | sudo apt-get update - sudo apt-get install -y bcftools + sudo apt-get install libcurl4-gnutls-dev + curl -L https://github.com/samtools/bcftools/releases/download/1.22/bcftools-1.22.tar.bz2 | tar -jxf - + cd bcftools-1.22 + ./configure && sudo make install - name: Install samtools run: | curl -L https://github.com/samtools/samtools/releases/download/1.19.2/samtools-1.19.2.tar.bz2 | tar -jxf - @@ -244,9 +261,16 @@ jobs: pip install multiqc - name: Install sentieon run: | - curl -L https://s3.amazonaws.com/sentieon-release/software/sentieon-genomics-$SENTIEON_VERSION.tar.gz | tar -zxf - + url="https://s3.amazonaws.com/sentieon-release/software/sentieon-genomics-$SENTIEON_VERSION.tar.gz" + bin_dir=$(pwd)/sentieon-genomics-$SENTIEON_VERSION/bin + machine_arch=$(uname -m) + if [[ "$machine_arch" == "arm64" || "$machine_arch" == "aarch64" || "$machine_arch" == "armv7l" ]]; then + url="https://s3.amazonaws.com/sentieon-release/software/arm-sentieon-genomics-$SENTIEON_VERSION.tar.gz" + bin_dir=$(pwd)/arm-sentieon-genomics-$SENTIEON_VERSION/bin + fi + curl -L "$url" | tar -zxf - echo "SENTIEON_VERSION=$SENTIEON_VERSION" >> $GITHUB_ENV - echo "PATH=$(pwd)/sentieon-genomics-$SENTIEON_VERSION/bin:$PATH" >> $GITHUB_ENV + echo "PATH=$bin_dir:$PATH" >> $GITHUB_ENV env: SENTIEON_VERSION: ${{ matrix.sentieon-version }} - name: Download hybrid model @@ -294,3 +318,21 @@ jobs: if [ ! -f "output_hybrid2.vcf.gz" ]; then exit 1 fi + - name: Smoke test - readgroups + run: | + . .venv/bin/activate + samtools addreplacerg -m overwrite_all \ + -o "tests/smoke/illumina_2.bam" \ + -r '@RG\tID:HG002-2\tSM:HG003\tPL:ILLUMINA' -O BAM \ + --reference "tests/smoke/r ef.fa" "tests/smoke/illumina.cram" + samtools index -b "tests/smoke/illumina_2.bam" + sentieon-cli -v dnascope-hybrid -t 1 -r "tests/smoke/r ef.fa" \ + --rgsm "HG001" \ + --sr_aln "tests/smoke/illumina.cram" "tests/smoke/illumina_2.bam" \ + --lr_aln "tests/smoke/sam ple.cram" \ + -m "DNAscope hybrid IlmnPB1.1.bundle" \ + --sr_duplicate_marking none \ + "output_hybrid3.vcf.gz" + if [ ! -f "output_hybrid3.vcf.gz" ]; then + exit 1 + fi diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9eb17ab..d6a9085 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -13,11 +13,11 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 with: - python-version: "3.12" - - name: Run image - uses: abatilo/actions-poetry@v2 - with: - poetry-version: "1.5.1" + python-version: "3.13" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install --upgrade poetry - name: Test build run: | poetry config virtualenvs.in-project true diff --git a/README.md b/README.md index 4240c98..1711756 100644 --- a/README.md +++ b/README.md @@ -8,8 +8,8 @@ A command-line interface for the Sentieon software Download the latest tar.gz file from the GitHub release page, https://github.com/sentieon/sentieon-cli/releases/ and install the package with pip: ```sh -curl -LO https://github.com/Sentieon/sentieon-cli/releases/download/v1.4.0/sentieon_cli-1.4.0.tar.gz -pip install sentieon_cli-1.4.0.tar.gz +curl -LO https://github.com/Sentieon/sentieon-cli/releases/download/v1.5.0/sentieon_cli-1.5.0.tar.gz +pip install sentieon_cli-1.5.0.tar.gz ``` ## Installation with Poetry diff --git a/docs/dnascope-hybrid.md b/docs/dnascope-hybrid.md index 6590166..1168910 100644 --- a/docs/dnascope-hybrid.md +++ b/docs/dnascope-hybrid.md @@ -5,7 +5,7 @@ Sentieon DNAscope Hybrid is a pipeline for germline variant calling using combin The pipeline supports input data in the following formats; both short-read and long-read input are required: * Unaligned short-read data in gzipped FASTQ format. * Aligned short-reads in BAM or CRAM format. -* Unaliged long-read data in the uBAM or uCRAM format. +* Unaligned long-read data in the uBAM or uCRAM format. * Aligned long-read data in BAM or CRAM format. By default, the pipeline will generate the following output files: diff --git a/docs/dnascope.md b/docs/dnascope.md index 6f41c64..8e70936 100644 --- a/docs/dnascope.md +++ b/docs/dnascope.md @@ -94,7 +94,7 @@ sentieon-cli dnascope [-h] \ ``` With uBAM or uCRAM input, the DNAscope pipeline requires the following new arguments: -- `-i SAMPLE_INPUT`: the input input sample file in uBAM or uCRAM format. One or more files can be supplied by passing multiple files after the `-i` argument. +- `-i SAMPLE_INPUT`: the input sample file in uBAM or uCRAM format. One or more files can be supplied by passing multiple files after the `-i` argument. - `--align`: directs the pipeline to align the input reads. The DNAscope pipeline accepts the following new optional arguments: diff --git a/poetry.lock b/poetry.lock index 97dc66a..bcd3313 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.1.3 and should not be changed by hand. [[package]] name = "black" @@ -6,6 +6,7 @@ version = "24.8.0" description = "The uncompromising code formatter." optional = false python-versions = ">=3.8" +groups = ["dev", "test"] files = [ {file = "black-24.8.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:09cdeb74d494ec023ded657f7092ba518e8cf78fa8386155e4a03fdcc44679e6"}, {file = "black-24.8.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:81c6742da39f33b08e791da38410f32e27d632260e599df7245cccee2064afeb"}, @@ -42,7 +43,7 @@ typing-extensions = {version = ">=4.0.1", markers = "python_version < \"3.11\""} [package.extras] colorama = ["colorama (>=0.4.3)"] -d = ["aiohttp (>=3.7.4)", "aiohttp (>=3.7.4,!=3.9.0)"] +d = ["aiohttp (>=3.7.4) ; sys_platform != \"win32\" or implementation_name != \"pypy\"", "aiohttp (>=3.7.4,!=3.9.0) ; sys_platform == \"win32\" and implementation_name == \"pypy\""] jupyter = ["ipython (>=7.8.0)", "tokenize-rt (>=3.2.0)"] uvloop = ["uvloop (>=0.15.2)"] @@ -52,6 +53,7 @@ version = "8.1.8" description = "Composable command line interface toolkit" optional = false python-versions = ">=3.7" +groups = ["dev", "test"] files = [ {file = "click-8.1.8-py3-none-any.whl", hash = "sha256:63c132bbbed01578a06712a2d1f497bb62d9c1c0d329b7903a866228027263b2"}, {file = "click-8.1.8.tar.gz", hash = "sha256:ed53c9d8990d83c2a27deae68e4ee337473f6330c040a31d4225c9574d16096a"}, @@ -66,10 +68,12 @@ version = "0.4.6" description = "Cross-platform colored terminal text." optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +groups = ["main", "dev", "test"] files = [ {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] +markers = {main = "sys_platform == \"win32\"", dev = "sys_platform == \"win32\" or platform_system == \"Windows\"", test = "sys_platform == \"win32\" or platform_system == \"Windows\""} [[package]] name = "colorlog" @@ -77,6 +81,7 @@ version = "6.9.0" description = "Add colours to the output of Python's logging module." optional = false python-versions = ">=3.6" +groups = ["main"] files = [ {file = "colorlog-6.9.0-py3-none-any.whl", hash = "sha256:5906e71acd67cb07a71e779c47c4bcb45fb8c2993eebe9e5adcd6a6f1b283eff"}, {file = "colorlog-6.9.0.tar.gz", hash = "sha256:bfba54a1b93b94f54e1f4fe48395725a3d92fd2a4af702f6bd70946bdc0c6ac2"}, @@ -94,6 +99,8 @@ version = "1.3.0" description = "Backport of PEP 654 (exception groups)" optional = false python-versions = ">=3.7" +groups = ["dev", "test"] +markers = "python_version < \"3.11\"" files = [ {file = "exceptiongroup-1.3.0-py3-none-any.whl", hash = "sha256:4d111e6e0c13d0644cad6ddaa7ed0261a0b36971f6d23e7ec9b4b9097da78a10"}, {file = "exceptiongroup-1.3.0.tar.gz", hash = "sha256:b241f5885f560bc56a59ee63ca4c6a8bfa46ae4ad651af316d4e81817bb9fd88"}, @@ -111,6 +118,7 @@ version = "5.0.4" description = "the modular source code checker: pep8 pyflakes and co" optional = false python-versions = ">=3.6.1" +groups = ["dev"] files = [ {file = "flake8-5.0.4-py2.py3-none-any.whl", hash = "sha256:7a1cf6b73744f5806ab95e526f6f0d8c01c66d7bbe349562d22dfca20610b248"}, {file = "flake8-5.0.4.tar.gz", hash = "sha256:6fbe320aad8d6b95cec8b8e47bc933004678dc63095be98528b7bdd2a9f510db"}, @@ -127,6 +135,7 @@ version = "5.13.0" description = "Read resources from Python packages" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "importlib_resources-5.13.0-py3-none-any.whl", hash = "sha256:9f7bd0c97b79972a6cce36a366356d16d5e13b09679c11a58f1014bfdf8e64b2"}, {file = "importlib_resources-5.13.0.tar.gz", hash = "sha256:82d5c6cca930697dbbd86c93333bb2c2e72861d4789a11c2662b933e5ad2b528"}, @@ -137,7 +146,7 @@ zipp = {version = ">=3.1.0", markers = "python_version < \"3.10\""} [package.extras] docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] -testing = ["pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy (>=0.9.1)", "pytest-ruff"] +testing = ["pytest (>=6)", "pytest-black (>=0.3.7) ; platform_python_implementation != \"PyPy\"", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy (>=0.9.1) ; platform_python_implementation != \"PyPy\"", "pytest-ruff"] [[package]] name = "iniconfig" @@ -145,6 +154,7 @@ version = "2.1.0" description = "brain-dead simple config-ini parsing" optional = false python-versions = ">=3.8" +groups = ["dev", "test"] files = [ {file = "iniconfig-2.1.0-py3-none-any.whl", hash = "sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760"}, {file = "iniconfig-2.1.0.tar.gz", hash = "sha256:3abbd2e30b36733fee78f9c7f7308f2d0050e88f0087fd25c2645f63c773e1c7"}, @@ -156,6 +166,7 @@ version = "0.7.0" description = "McCabe checker, plugin for flake8" optional = false python-versions = ">=3.6" +groups = ["dev"] files = [ {file = "mccabe-0.7.0-py2.py3-none-any.whl", hash = "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e"}, {file = "mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325"}, @@ -167,6 +178,7 @@ version = "1.14.1" description = "Optional static typing for Python" optional = false python-versions = ">=3.8" +groups = ["dev", "test"] files = [ {file = "mypy-1.14.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:52686e37cf13d559f668aa398dd7ddf1f92c5d613e4f8cb262be2fb4fedb0fcb"}, {file = "mypy-1.14.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1fb545ca340537d4b45d3eecdb3def05e913299ca72c290326be19b3804b39c0"}, @@ -226,6 +238,7 @@ version = "1.1.0" description = "Type system extensions for programs checked with the mypy type checker." optional = false python-versions = ">=3.8" +groups = ["dev", "test"] files = [ {file = "mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505"}, {file = "mypy_extensions-1.1.0.tar.gz", hash = "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558"}, @@ -237,6 +250,7 @@ version = "23.2" description = "Core utilities for Python packages" optional = false python-versions = ">=3.7" +groups = ["main", "dev", "test"] files = [ {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, @@ -248,6 +262,7 @@ version = "0.12.1" description = "Utility library for gitignore style pattern matching of file paths." optional = false python-versions = ">=3.8" +groups = ["dev", "test"] files = [ {file = "pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08"}, {file = "pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712"}, @@ -259,6 +274,7 @@ version = "4.3.6" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`." optional = false python-versions = ">=3.8" +groups = ["dev", "test"] files = [ {file = "platformdirs-4.3.6-py3-none-any.whl", hash = "sha256:73e575e1408ab8103900836b97580d5307456908a03e92031bab39e4554cc3fb"}, {file = "platformdirs-4.3.6.tar.gz", hash = "sha256:357fb2acbc885b0419afd3ce3ed34564c13c9b95c89360cd9563f73aa5e2b907"}, @@ -275,6 +291,7 @@ version = "1.5.0" description = "plugin and hook calling mechanisms for python" optional = false python-versions = ">=3.8" +groups = ["dev", "test"] files = [ {file = "pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669"}, {file = "pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1"}, @@ -290,6 +307,7 @@ version = "2.9.1" description = "Python style guide checker" optional = false python-versions = ">=3.6" +groups = ["dev"] files = [ {file = "pycodestyle-2.9.1-py2.py3-none-any.whl", hash = "sha256:d1735fc58b418fd7c5f658d28d943854f8a849b01a5d0a1e6f3f3fdd0166804b"}, {file = "pycodestyle-2.9.1.tar.gz", hash = "sha256:2c9607871d58c76354b697b42f5d57e1ada7d261c261efac224b664affdc5785"}, @@ -301,6 +319,7 @@ version = "2.5.0" description = "passive checker of Python programs" optional = false python-versions = ">=3.6" +groups = ["dev"] files = [ {file = "pyflakes-2.5.0-py2.py3-none-any.whl", hash = "sha256:4579f67d887f804e67edb544428f264b7b24f435b263c4614f384135cea553d2"}, {file = "pyflakes-2.5.0.tar.gz", hash = "sha256:491feb020dca48ccc562a8c0cbe8df07ee13078df59813b83959cbdada312ea3"}, @@ -312,6 +331,7 @@ version = "7.4.4" description = "pytest: simple powerful testing with Python" optional = false python-versions = ">=3.7" +groups = ["dev", "test"] files = [ {file = "pytest-7.4.4-py3-none-any.whl", hash = "sha256:b090cdf5ed60bf4c45261be03239c2c1c22df034fbffe691abe93cd80cea01d8"}, {file = "pytest-7.4.4.tar.gz", hash = "sha256:2cf0005922c6ace4a3e2ec8b4080eb0d9753fdc93107415332f50ce9e7994280"}, @@ -334,6 +354,8 @@ version = "2.2.1" description = "A lil' TOML parser" optional = false python-versions = ">=3.8" +groups = ["dev", "test"] +markers = "python_version < \"3.11\"" files = [ {file = "tomli-2.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:678e4fa69e4575eb77d103de3df8a895e1591b48e740211bd1067378c69e8249"}, {file = "tomli-2.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:023aa114dd824ade0100497eb2318602af309e5a55595f76b626d6d9f3b7b0a6"}, @@ -375,6 +397,7 @@ version = "4.13.2" description = "Backported and Experimental Type Hints for Python 3.8+" optional = false python-versions = ">=3.8" +groups = ["dev", "test"] files = [ {file = "typing_extensions-4.13.2-py3-none-any.whl", hash = "sha256:a439e7c04b49fec3e5d3e2beaa21755cadbbdc391694e28ccdd36ca4a1408f8c"}, {file = "typing_extensions-4.13.2.tar.gz", hash = "sha256:e6c81219bd689f51865d9e372991c540bda33a0379d5573cddb9a3a23f7caaef"}, @@ -386,20 +409,22 @@ version = "3.20.2" description = "Backport of pathlib-compatible object wrapper for zip files" optional = false python-versions = ">=3.8" +groups = ["main"] +markers = "python_version < \"3.10\"" files = [ {file = "zipp-3.20.2-py3-none-any.whl", hash = "sha256:a817ac80d6cf4b23bf7f2828b7cabf326f15a001bea8b1f9b49631780ba28350"}, {file = "zipp-3.20.2.tar.gz", hash = "sha256:bc9eb26f4506fda01b81bcde0ca78103b6e62f991b381fec825435c836edbc29"}, ] [package.extras] -check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1)"] +check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\""] cover = ["pytest-cov"] doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] enabler = ["pytest-enabler (>=2.2)"] -test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more-itertools", "pytest (>=6,!=8.1.*)", "pytest-ignore-flaky"] +test = ["big-O", "importlib-resources ; python_version < \"3.9\"", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more-itertools", "pytest (>=6,!=8.1.*)", "pytest-ignore-flaky"] type = ["pytest-mypy"] [metadata] -lock-version = "2.0" -python-versions = "^3.8" -content-hash = "f500a24ab808858a215de6db45ed6e2a936832c92fac38bf451e50d004bc74dd" +lock-version = "2.1" +python-versions = ">= 3.8" +content-hash = "47eb87b3ebe7ff62dfa41dd69302f3ca080143817f584a5f65dea357fcac28be" diff --git a/pyproject.toml b/pyproject.toml index 8a765c9..daac4e4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,27 +1,29 @@ -#https://stackoverflow.com/questions/75408641/whats-difference-between-tool-poetry-and-project-in-pyproject-toml -[tool.poetry] +[project] name = "sentieon_cli" -version = "1.4.0" +version = "1.5.0" description = "entry point for sentieon command-line tools" -authors = ["Don Freed ", "Brent "] +authors = [ + {name = "Don Freed", email = "don.freed@sentieon.com"}, + {name = "Brent", email = "bpederse@gmail.com"}, +] readme = "README.md" +requires-python = ">= 3.8" classifiers = [ - "Development Status :: 5 — Production/Stable", - 'License :: OSI Approved :: BSD 2-Clause "Simplified License" (BSD-2-Clause)', + "Development Status :: 5 - Production/Stable", + "License :: OSI Approved :: BSD License", "Programming Language :: Python :: 3", ] include = ["LICENSE"] +dependencies = [ + "colorlog (>=6.7.0,<7.0.0)", + "importlib-resources (>=5.0,<6.0)", + "packaging (>=23.2,<26.0)", +] -[tool.poetry.dependencies] -python = "^3.8" -colorlog = "^6.7.0" -importlib-resources = "^5.0" -packaging = "^23.2" - -[tool.poetry.scripts] +[project.scripts] sentieon-cli = "sentieon_cli:main" [tool.mypy] diff --git a/sentieon_cli/__init__.py b/sentieon_cli/__init__.py index db1b05f..5a2e86a 100644 --- a/sentieon_cli/__init__.py +++ b/sentieon_cli/__init__.py @@ -3,6 +3,7 @@ from .dnascope_hybrid import DNAscopeHybridPipeline from .dnascope_longread import DNAscopeLRPipeline from .pangenome import PangenomePipeline +from .sentieon_pangenome import SentieonPangenome def main(): @@ -50,6 +51,12 @@ def main(): pipeline.add_arguments(pangenome_subparser) pangenome_subparser.set_defaults(pipeline=pipeline.main) + # Sentieon pangenome + pipeline = SentieonPangenome() + sentieon_pangenome_subparser = subparsers.add_parser("sentieon-pangenome") + pipeline.add_arguments(sentieon_pangenome_subparser) + sentieon_pangenome_subparser.set_defaults(pipeline=pipeline.main) + args = parser.parse_args() args.pipeline(args) diff --git a/sentieon_cli/base_pangenome.py b/sentieon_cli/base_pangenome.py new file mode 100644 index 0000000..74182c9 --- /dev/null +++ b/sentieon_cli/base_pangenome.py @@ -0,0 +1,170 @@ +""" +A base class for pangenome pipelines +""" + +import copy +from enum import Enum +import json +import pathlib +from typing import List, Optional + +from importlib_resources import files + +from . import command_strings as cmds +from .job import Job +from .pipeline import BasePipeline +from .util import path_arg + + +class SampleSex(Enum): + FEMALE = 1 + MALE = 2 + UNKNOWN = 3 + + +class BasePangenome(BasePipeline): + """A pipeline base class for short reads""" + + params = copy.deepcopy(BasePipeline.params) + params.update( + { + # Required arguments + "gbz": { + "help": "The pangenome graph file in GBZ format.", + "required": True, + "type": path_arg(exists=True, is_file=True), + }, + "hapl": { + "help": "The haplotype file.", + "required": True, + "type": path_arg(exists=True, is_file=True), + }, + "model_bundle": { + "flags": ["-m", "--model_bundle"], + "help": "The model bundle file.", + "required": True, + "type": path_arg(exists=True, is_file=True), + }, + "r1_fastq": { + "nargs": "*", + "help": "Sample R1 fastq files.", + "type": path_arg(exists=True, is_file=True), + }, + "r2_fastq": { + "nargs": "*", + "help": "Sample R2 fastq files.", + "type": path_arg(exists=True, is_file=True), + }, + # Additional arguments + "bam_format": { + "help": ( + "Use the BAM format instead of CRAM for output aligned " + "files." + ), + "action": "store_true", + }, + "dbsnp": { + "flags": ["-d", "--dbsnp"], + "help": ( + "dbSNP vcf file Supplying this file will annotate " + "variants with their dbSNP refSNP ID numbers." + ), + "type": path_arg(exists=True, is_file=True), + }, + "kmer_memory": { + "help": "Memory limit for KMC in GB.", + "default": 30, + "type": int, + }, + "pcr_free": { + "help": "Use arguments for PCR-free data processing", + "action": "store_true", + }, + } + ) + + positionals = BasePipeline.positionals + + def __init__(self) -> None: + super().__init__() + self.gbz: Optional[pathlib.Path] = None + self.hapl: Optional[pathlib.Path] = None + self.model_bundle: Optional[pathlib.Path] = None + self.r1_fastq: List[pathlib.Path] = [] + self.r2_fastq: List[pathlib.Path] = [] + self.bam_format = False + self.dbsnp: Optional[pathlib.Path] = None + self.kmer_memory = 30 + self.pcr_free = False + + def build_kmc_job( + self, kmer_prefix: pathlib.Path, job_threads: int + ) -> Job: + """Build KMC k-mer counting jobs""" + # Create file list for KMC + file_list = pathlib.Path(str(kmer_prefix) + ".paths") + all_fastqs = [] + + # Add R1 files + all_fastqs.extend(self.r1_fastq) + + # Add R2 files if present + if self.r2_fastq: + all_fastqs.extend(self.r2_fastq) + + # Write file list + if not self.dry_run: + with open(file_list, "w") as f: + for fq in all_fastqs: + f.write(f"{fq}\n") + + # Create KMC job + kmc_job = Job( + cmds.cmd_kmc( + kmer_prefix, + file_list, + self.tmp_dir, + memory=self.kmer_memory, + threads=self.cores, + ), + "kmc", + job_threads, + ) + + return kmc_job + + def build_ploidy_job( + self, + ploidy_json: pathlib.Path, + deduped_bam: List[pathlib.Path], + ) -> Job: + """Estimate sample ploidy and sex""" + estimate_ploidy = pathlib.Path( + str(files("sentieon_cli.scripts").joinpath("estimate_ploidy.py")) + ).resolve() + ploidy_job = Job( + cmds.cmd_estimate_ploidy( + ploidy_json, + deduped_bam, + estimate_ploidy, + ), + "estimate-ploidy", + 0, + ) + return ploidy_job + + def get_sex(self, ploidy_json: pathlib.Path) -> None: + """Retrieve the sample sex""" + if self.dry_run: + self.logger.info("Setting sample sex to MALE for dry-run") + self.sample_sex = SampleSex.MALE + return + with open(ploidy_json) as fh: + data = json.load(fh) + sex = data["sex"] + if sex == "female": + self.sample_sex = SampleSex.FEMALE + elif sex == "male": + self.sample_sex = SampleSex.MALE + else: + self.sample_sex = SampleSex.UNKNOWN diff --git a/sentieon_cli/command_strings.py b/sentieon_cli/command_strings.py index c969576..f8debd4 100644 --- a/sentieon_cli/command_strings.py +++ b/sentieon_cli/command_strings.py @@ -4,14 +4,16 @@ """ import io +import os import pathlib import shlex import subprocess as sp import typing -from typing import Any, Optional, List, Dict +from typing import Any, Dict, List, Optional, Union from .driver import BaseDriver from .logging import get_logger +from .shell_pipeline import Command, InputProcSub, Pipeline logger = get_logger(__name__) @@ -19,16 +21,16 @@ def cmd_fai_to_bed( fai: pathlib.Path, bed: pathlib.Path, -) -> str: +) -> Pipeline: cmd = ["awk", "-v", "OFS=\t", "{print $1,0,$2}", str(fai)] - return shlex.join(cmd) + " > " + shlex.quote(str(bed)) + return Pipeline(Command(*cmd), file_output=bed) def cmd_bedtools_subtract( regions_bed: pathlib.Path, phased_bed: pathlib.Path, unphased_bed: pathlib.Path, -) -> str: +) -> Pipeline: cmd = [ "bedtools", "subtract", @@ -37,14 +39,14 @@ def cmd_bedtools_subtract( "-b", str(phased_bed), ] - return shlex.join(cmd) + " > " + shlex.quote(str(unphased_bed)) + return Pipeline(Command(*cmd), file_output=unphased_bed) def cmd_bedtools_merge( in_bed: pathlib.Path, out_bed: pathlib.Path, distance: int = 0, -) -> str: +) -> Pipeline: """Bedtools merge""" cmd = [ "bedtools", @@ -54,7 +56,7 @@ def cmd_bedtools_merge( "-i", str(in_bed), ] - return shlex.join(cmd) + " > " + shlex.quote(str(out_bed)) + return Pipeline(Command(*cmd), file_output=out_bed) def cmd_bedtools_slop( @@ -62,7 +64,7 @@ def cmd_bedtools_slop( out_bed: pathlib.Path, ref_fai: pathlib.Path, slop_size: int = 0, -) -> str: +) -> Pipeline: """Bedtools slop""" cmd = [ "bedtools", @@ -74,35 +76,19 @@ def cmd_bedtools_slop( "-i", str(in_bed), ] - return shlex.join(cmd) + " > " + shlex.quote(str(out_bed)) + return Pipeline(Command(*cmd), file_output=out_bed) def cmd_bedtools_cat_sort_merge( out_bed: pathlib.Path, in_bed: List[pathlib.Path], ref_fai: pathlib.Path, -) -> str: +) -> Pipeline: """Concat, sort and merge multiple BEDs""" - cmds = [] - cmds.append(["cat"] + [shlex.quote(str(x)) for x in in_bed]) - cmds.append( - [ - "bedtools", - "sort", - "-faidx", - str(ref_fai), - "-i", - "-", - ] - ) - cmds.append( - [ - "bedtools", - "merge", - ] - ) - all_cmds = [shlex.join(x) for x in cmds] - return " | ".join(all_cmds) + " > " + shlex.quote(str(out_bed)) + cat_cmd = Command("cat", *[str(x) for x in in_bed]) + sort_cmd = Command("bedtools", "sort", "-faidx", str(ref_fai), "-i", "-") + merge_cmd = Command("bedtools", "merge") + return Pipeline(cat_cmd, sort_cmd, merge_cmd, file_output=out_bed) def name(path: typing.Union[str, io.TextIOWrapper, pathlib.Path]) -> str: @@ -122,17 +108,9 @@ def cmd_pyexec_vcf_mod_haploid_patch( phased_vcf: str, cores: int, kwargs: Dict[str, Any], -) -> str: +) -> Pipeline: """ merge dnascope and dnascope-hp variants - - >>> assert '--hap1 ' in cmd_pyexec_vcf_mod_haploid_patch("h1.vcf.gz", - ... "h2.vcf.gz", "out_hap%d_%stmp.vcf.gz", "HiFi", "ph.vcf", 2, - ... {'vcf_mod_py': 'vcf_mod.py'}) - >>> assert not "--hap1 " in cmd_pyexec_vcf_mod_haploid_patch("h1.vcf.gz", - ... "h2.vcf.gz", "out_hap%d_%stmp.vcf.gz", "ONT", "ph.vcf", 2, - ... {'vcf_mod_py': 'vcf_mod.py'}) - """ assert tech.upper() in ("HIFI", "ONT") @@ -146,7 +124,7 @@ def cmd_pyexec_vcf_mod_haploid_patch( cmd += f" --phased {phased_vcf}" cmd += " --hap1_hp " + hap_patt % (1, "") cmd += " --hap2_hp " + hap_patt % (2, "") - return cmd + return Pipeline(Command(*shlex.split(cmd))) def cmd_pyexec_vcf_mod_patch( @@ -155,7 +133,7 @@ def cmd_pyexec_vcf_mod_patch( vcf_hp: str, cores: int, kwargs: Dict[str, Any], -) -> str: +) -> Pipeline: """Patch DNAscope and DNAscopeHP VCF files""" cmd = [ @@ -171,7 +149,7 @@ def cmd_pyexec_vcf_mod_patch( str(vcf_hp), str(out_vcf), ] - return shlex.join(cmd) + return Pipeline(Command(*cmd)) def cmd_pyexec_gvcf_combine( @@ -180,10 +158,10 @@ def cmd_pyexec_gvcf_combine( out_vcf: str, cores: int, kwargs: Dict[str, Any], -) -> str: +) -> Pipeline: """Combine gVCF files""" - cmd1 = [ + cmd1 = Command( "sentieon", "pyexec", str(kwargs["gvcf_combine_py"]), @@ -193,15 +171,15 @@ def cmd_pyexec_gvcf_combine( gvcf, out_vcf, "-", - ] - cmd2 = [ + ) + cmd2 = Command( "sentieon", "util", "vcfconvert", "-", out_vcf.replace(".vcf.gz", ".g.vcf.gz"), - ] - return shlex.join(cmd1) + "|" + shlex.join(cmd2) + ) + return Pipeline(cmd1, cmd2) def cmd_pyexec_vcf_mod_merge( @@ -213,7 +191,7 @@ def cmd_pyexec_vcf_mod_merge( out_vcf: str, cores: int, kwargs: Dict[str, Any], -) -> str: +) -> Pipeline: """Merge haploid VCF files""" cmd = [ @@ -235,7 +213,7 @@ def cmd_pyexec_vcf_mod_merge( phased_bed, out_vcf, ] - return shlex.join(cmd) + return Pipeline(Command(*cmd)) def cmd_pyexec_vcf_mod_haploid_patch2( @@ -244,7 +222,7 @@ def cmd_pyexec_vcf_mod_haploid_patch2( vcf_hp: str, cores: int, kwargs: Dict[str, Any], -) -> str: +) -> Pipeline: """Patch a single pair of haploid DNAscope and DNAscopeHP VCFs""" cmd = [ @@ -261,7 +239,7 @@ def cmd_pyexec_vcf_mod_haploid_patch2( "--patch_vcf", out_vcf, ] - return shlex.join(cmd) + return Pipeline(Command(*cmd)) def cmd_pyexec_hybrid_select( @@ -271,51 +249,48 @@ def cmd_pyexec_hybrid_select( hybrid_select: pathlib.Path, threads: int, slop_size: int = 1000, -) -> str: - cmds = [] - cmds.append( - [ - "sentieon", - "pyexec", - str(hybrid_select), - "-v", - str(vcf), - "-t", - str(threads), - "-", - ] +) -> Pipeline: + select_cmd = Command( + "sentieon", + "pyexec", + str(hybrid_select), + "-v", + str(vcf), + "-t", + str(threads), + "-", ) - cmds.append( - [ - "bcftools", - "view", - "-f", - "PASS,.", - "-", - ] + view_cmd = Command( + "bcftools", + "view", + "-f", + "PASS,.", + "-", ) - cmds.append( - [ - "bcftools", - "query", - "-f", - "%CHROM\\t%POS0\\t%END\\n", - "-", - ] + query_cmd = Command( + "bcftools", + "query", + "-f", + "%CHROM\\t%POS0\\t%END\\n", + "-", ) - cmds.append( - [ - "bedtools", - "slop", - "-b", - str(slop_size), - "-g", - str(ref_fai), - "-i", - "-", - ] + slop_cmd = Command( + "bedtools", + "slop", + "-b", + str(slop_size), + "-g", + str(ref_fai), + "-i", + "-", + ) + return Pipeline( + select_cmd, + view_cmd, + query_cmd, + slop_cmd, + file_output=out_bed, ) - return " | ".join([shlex.join(x) for x in cmds]) + " > " + str(out_bed) def cmd_pyexec_hybrid_anno( @@ -324,7 +299,7 @@ def cmd_pyexec_hybrid_anno( bed: pathlib.Path, hybrid_anno: pathlib.Path, threads: int, -) -> str: +) -> Pipeline: cmd = [ "sentieon", "pyexec", @@ -337,7 +312,7 @@ def cmd_pyexec_hybrid_anno( str(threads), str(out_vcf), ] - return shlex.join(cmd) + return Pipeline(Command(*cmd)) def hybrid_stage1( @@ -348,49 +323,51 @@ def hybrid_stage1( ins_driver: BaseDriver, stage1_driver: BaseDriver, bwa_model: pathlib.Path, -) -> str: - unset_cmd = ["unset", "bwt_max_mem"] - fq_cmds = [] - fq_cmds.append(shlex.join(stage1_driver.build_cmd())) - fq_cmds.append(shlex.join(ins_driver.build_cmd())) - - aln_cmds = [] - aln_cmds.append( - [ - "sentieon", - "bwa", - "mem", - "-R", - readgroup, - "-t", - str(cores), - "-x", - str(bwa_model), - str(reference), - "-", - ] +) -> Pipeline: + bwa_env = dict(os.environ) + _ = bwa_env.pop("bwt_max_mem", None) + + # Send the input of both fq commands to bwa with cat + fq1_cmd = Command(*stage1_driver.build_cmd()) + fq2_cmd = Command(*ins_driver.build_cmd()) + cat_cmd = Command( + "cat", + InputProcSub(Pipeline(fq1_cmd)), + InputProcSub(Pipeline(fq2_cmd)), ) - aln_cmds.append( - [ - "sentieon", - "util", - "sort", - "-i", - "-", - "-t", - str(cores), - "-o", - str(out_aln), - "--sam2bam", - ] + + bwa_cmd = Command( + "sentieon", + "bwa", + "mem", + "-R", + readgroup, + "-t", + str(cores), + "-x", + str(bwa_model), + str(reference), + "-", + exec_kwargs={"env": bwa_env}, + ) + + sort_cmd = Command( + "sentieon", + "util", + "sort", + "-i", + "-", + "-t", + str(cores), + "-o", + str(out_aln), + "--sam2bam", ) - return ( - shlex.join(unset_cmd) - + "; (" - + " || echo -n 'error'; ".join(fq_cmds) - + " || echo -n 'error') | " - + " | ".join(shlex.join(x) for x in aln_cmds) + return Pipeline( + cat_cmd, + bwa_cmd, + sort_cmd, ) @@ -398,30 +375,26 @@ def hybrid_stage3( out_bam: pathlib.Path, driver: BaseDriver, cores: int, -) -> str: - cmds = [] - cmds.append(driver.build_cmd()) - cmds.append( - [ - "sentieon", - "util", - "sort", - "-i", - "-", - "-t", - str(cores), - "-o", - str(out_bam), - ] +) -> Pipeline: + sort_cmd = Command( + "sentieon", + "util", + "sort", + "-i", + "-", + "-t", + str(cores), + "-o", + str(out_bam), ) - return " | ".join([shlex.join(x) for x in cmds]) + return Pipeline(Command(*driver.build_cmd()), sort_cmd) def bcftools_subset( out_vcf: pathlib.Path, mix_vcf: pathlib.Path, regions_bed: pathlib.Path, -) -> str: +) -> Pipeline: """Subset a vcf with bcftools""" bcftools_subset_cmd = shlex.join( [ @@ -450,7 +423,7 @@ def bcftools_subset( str(out_vcf), ] ) - return ( + shell_cmd = ( f'if [ -s "{regions_bed}" ]; then ' # noqa + bcftools_subset_cmd + " | " @@ -459,32 +432,25 @@ def bcftools_subset( + cp_cmd + "; fi" ) + return Pipeline(Command("sh", "-c", shell_cmd)) def bcftools_concat( out_vcf: pathlib.Path, in_vcfs: List[pathlib.Path], -) -> str: + xargs: List[str] = ["-aD"], +) -> Pipeline: """VCF processing through bcftools concat""" - cmds = [] - cmds.append( - [ - "bcftools", - "concat", - "-aD", - ] - + [str(x) for x in in_vcfs] - ) - cmds.append( - [ - "sentieon", - "util", - "vcfconvert", - "-", - str(out_vcf), - ] + concat_cmd = Command( + "bcftools", + "concat", + "-W=tbi", + "--output", + str(out_vcf), + *xargs, + *[str(x) for x in in_vcfs], ) - return " | ".join([shlex.join(x) for x in cmds]) + return Pipeline(concat_cmd) def filter_norm( @@ -492,41 +458,16 @@ def filter_norm( in_vcf: pathlib.Path, reference: pathlib.Path, exclude_homref: bool = True, -) -> str: +) -> Pipeline: """Trim and normalize""" - cmds = [] - cmd = [ - "bcftools", - "view", - "-a", - ] + view_args = ["-a"] if exclude_homref: - cmd.extend( - [ - "-e", - 'GT="0/0"', - ] - ) - cmd.append(str(in_vcf)) - cmds.append(cmd) - cmds.append( - [ - "bcftools", - "norm", - "-f", - str(reference), - ] - ) - cmds.append( - [ - "sentieon", - "util", - "vcfconvert", - "-", - str(out_vcf), - ] - ) - return " | ".join([shlex.join(x) for x in cmds]) + view_args.extend(["-e", 'GT="0/0"']) + view_args.append(str(in_vcf)) + view_cmd = Command("bcftools", "view", *view_args) + norm_cmd = Command("bcftools", "norm", "-f", str(reference)) + convert_cmd = Command("sentieon", "util", "vcfconvert", "-", str(out_vcf)) + return Pipeline(view_cmd, norm_cmd, convert_cmd) def get_rg_lines( @@ -553,25 +494,6 @@ def get_rg_lines( return rg_lines -def rehead_wgsmetrics( - orig_metrics: pathlib.Path, tmp_metrics: pathlib.Path -) -> str: - """Rehead Sentieon WGS metrics so the file is recognized by MultiQC""" - cmd1 = shlex.join(["mv", str(orig_metrics), str(tmp_metrics)]) - cmd2 = ( - shlex.join(["echo", "'## METRICS CLASS WgsMetrics'"]) - + ">" - + shlex.quote(str(orig_metrics)) - ) - cmd3 = ( - shlex.join(["tail", "-n", "+2", str(tmp_metrics)]) - + ">>" - + shlex.quote(str(orig_metrics)) - ) - cmd4 = shlex.join(["rm", str(tmp_metrics)]) - return "; ".join((cmd1, cmd2, cmd3, cmd4)) - - def cmd_samtools_fastq_minimap2( out_aln: pathlib.Path, input_aln: pathlib.Path, @@ -582,64 +504,49 @@ def cmd_samtools_fastq_minimap2( sample_name: str, input_ref: Optional[pathlib.Path] = None, fastq_taglist: str = "*", - minimap2_args: str = "-Y", + minimap2_args: str = "-YL", util_sort_args: str = "--cram_write_options version=3.0,compressor=rans", -) -> str: +) -> Pipeline: """Re-align an input BAM/CRAM/uBAM/uCRAM file with minimap2""" ref_cmd: List[str] = [] if input_ref: ref_cmd = ["--reference", str(reference)] - cmd1 = ( - [ + cmd_list = [ + Command( "samtools", "fastq", - ] - + ref_cmd - + [ + *ref_cmd, "-@", str(cores), "-T", fastq_taglist, str(input_aln), - ] - ) - cmd2 = [ - "sentieon", - "minimap2", - "-y", - "-t", - str(cores), - "-a", - minimap2_args, - "-x", - f"{model_bundle}/minimap2.model", - str(reference), - "/dev/stdin", + ) ] - cmd3 = [ - "sentieon", - "util", - "sort", - "-i", - "-", - "-t", - str(cores), - "--reference", - str(reference), - "-o", - str(out_aln), - "--sam2bam", - ] + util_sort_args.split() + cmd_list.append( + Command( + "sentieon", + "minimap2", + "-y", + "-t", + str(cores), + "-a", + minimap2_args, + "-x", + f"{model_bundle}/minimap2.model", + str(reference), + "/dev/stdin", + ) + ) # Commands to replace the @RG lines in the header - rg_cmds: List[List[str]] = [] for rg_line in rg_lines: # Add a SM value, if missing if "\tSM:" not in rg_line: rg_line += f"\tSM:{sample_name}" - rg_cmds.append( - [ + cmd_list.append( + Command( "samtools", "addreplacerg", "-r", @@ -647,10 +554,26 @@ def cmd_samtools_fastq_minimap2( "-m", "orphan_only", "-", - ] + ) ) - - return " | ".join([shlex.join(x) for x in (cmd1, cmd2, *rg_cmds, cmd3)]) + cmd_list.append( + Command( + "sentieon", + "util", + "sort", + "-i", + "-", + "-t", + str(cores), + "--reference", + str(reference), + "-o", + str(out_aln), + "--sam2bam", + *util_sort_args.split(), + ) + ) + return Pipeline(*cmd_list) def cmd_samtools_fastq_bwa( @@ -663,33 +586,31 @@ def cmd_samtools_fastq_bwa( input_ref: Optional[pathlib.Path] = None, collate: bool = False, bwa_args: str = "", - bwa_k: str = "100000000", + bwa_k: str = "20000000", fastq_taglist: str = "RG", util_sort_args: str = "--cram_write_options version=3.0,compressor=rans", -) -> str: +) -> Pipeline: """Re-align an input BAM/CRAM/uBAM/uCRAM file with bwa""" ref_cmd: List[str] = [] if input_ref: ref_cmd = ["--reference", str(reference)] - cmd0 = ["perl", "-MFcntl", "-e", "fcntl(STDOUT, 1031, 268435456)"] + pipebuf_cmd = Command( + "perl", "-MFcntl", "-e", "fcntl(STDOUT, 1031, 268435456)" + ) if collate: - cmd1 = ( - [ - "samtools", - "collate", - ] - + ref_cmd - + [ - "-@", - str(cores), - "-O", - "-u", - "-f", - str(input_aln), - ] + collate_cmd = Command( + "samtools", + "collate", + *ref_cmd, + "-@", + str(cores), + "-O", + "-u", + "-f", + str(input_aln), ) - cmd2 = [ + fastq_cmd = Command( "samtools", "fastq", "-@", @@ -697,42 +618,38 @@ def cmd_samtools_fastq_bwa( "-T", fastq_taglist, "-", - ] - else: - cmd1 = [] - cmd2 = ( - [ - "samtools", - "fastq", - ] - + ref_cmd - + [ - "-@", - str(cores), - "-T", - fastq_taglist, - str(input_aln), - ] ) - cmd3 = ( - [ - "sentieon", - "bwa", - "mem", - "-p", - "-C", - "-H", - str(rg_header), - "-t", + else: + collate_cmd = None + fastq_cmd = Command( + "samtools", + "fastq", + *ref_cmd, + "-@", str(cores), - "-x", - f"{model_bundle}/bwa.model", - ] - + bwa_args.split() - + ["-K", bwa_k] - + [str(reference), "/dev/stdin"] + "-T", + fastq_taglist, + str(input_aln), + ) + bwa_cmd = Command( + "sentieon", + "bwa", + "mem", + "-p", + "-C", + "-H", + str(rg_header), + "-t", + str(cores), + "-x", + f"{model_bundle}/bwa.model", + *bwa_args.split(), + "-K", + bwa_k, + str(reference), + "/dev/stdin", ) - cmd4 = [ + sort_cmd = Command( "sentieon", "util", "sort", @@ -745,19 +662,24 @@ def cmd_samtools_fastq_bwa( "-o", str(out_aln), "--sam2bam", - ] + util_sort_args.split() - - if cmd1: - return ( - shlex.join(cmd0) - + ";" - + " | ".join([shlex.join(x) for x in (cmd1, cmd2, cmd3, cmd4)]) + *util_sort_args.split(), + ) + if collate_cmd: + return Pipeline( + pipebuf_cmd, + collate_cmd, + fastq_cmd, + bwa_cmd, + sort_cmd, + skip_pipe=(0,), ) else: - return ( - shlex.join(cmd0) - + ";" - + " | ".join([shlex.join(x) for x in (cmd2, cmd3, cmd4)]) + return Pipeline( + pipebuf_cmd, + fastq_cmd, + bwa_cmd, + sort_cmd, + skip_pipe=(0,), ) @@ -769,17 +691,13 @@ def cmd_fastq_minimap2( model_bundle: pathlib.Path, cores: int, unzip: str = "gzip", - minimap2_args: str = "-Y", + minimap2_args: str = "-YL", util_sort_args: str = "--cram_write_options version=3.0,compressor=rans", -) -> str: +) -> Pipeline: """Align an input fastq file with minimap2""" - cmd1 = [ - unzip, - "-dc", - str(fastq), - ] - cmd2 = [ + cmd1 = Command(unzip, "-dc", str(fastq)) + cmd2 = Command( "sentieon", "minimap2", "-t", @@ -792,8 +710,8 @@ def cmd_fastq_minimap2( readgroup, str(reference), "/dev/stdin", - ] - cmd3 = [ + ) + cmd3 = Command( "sentieon", "util", "sort", @@ -806,8 +724,9 @@ def cmd_fastq_minimap2( "-o", str(out_aln), "--sam2bam", - ] + util_sort_args.split() - return " | ".join([shlex.join(x) for x in (cmd1, cmd2, cmd3)]) + *util_sort_args.split(), + ) + return Pipeline(cmd1, cmd2, cmd3) def cmd_fastq_bwa( @@ -820,111 +739,94 @@ def cmd_fastq_bwa( cores: int, unzip: str = "gzip", bwa_args: str = "", - bwa_k: str = "100000000", + bwa_k: str = "20000000", util_sort_args: str = "--cram_write_options version=3.0,compressor=rans", numa: Optional[str] = None, split: Optional[str] = None, -) -> str: +) -> Pipeline: """Align an input fastq file with bwa""" - pipebuf_cmd = shlex.join( - ["perl", "-MFcntl", "-e", "fcntl(STDOUT, 1031, 268435456)"] + pipebuf_cmd = Command( + "perl", "-MFcntl", "-e", "fcntl(STDOUT, 1031, 268435456)" ) - cmd1 = ( - (["taskset", "-c", numa] if numa else []) - + [ - "sentieon", - "bwa", - "mem", - "-R", - readgroup, - "-t", - str(cores), - "-x", - f"{model_bundle}/bwa.model", - ] - + bwa_args.split() - + ["-K", bwa_k] - + (["-p"] if (r2 and split) else []) - + [ - str(reference), - ] + r1_unzip = Pipeline( + pipebuf_cmd, + Command(str(unzip), "-dc", str(r1)), + skip_pipe=(0,), ) - cmd2 = [ - str(unzip), - "-dc", - str(r1), - ] - cmd3 = [] + r2_unzip = None if r2: - cmd3 = [ - str(unzip), - "-dc", - str(r2), - ] - cmd4 = ( - (["taskset", "-c", numa] if numa else []) - + [ - "sentieon", - "util", - "sort", - "-i", - "-", - "-t", - str(cores), - "--reference", - str(reference), - "-o", - str(out_aln), - "--sam2bam", - ] - + util_sort_args.split() - ) - - cmds = [shlex.join(x) for x in (cmd1, cmd2, cmd3, cmd4)] - cmd_str = "" - if split: - extract_cmd = shlex.join( - ["sentieon", "fqidx", "extract", "-F", str(split), "-K", bwa_k] + r2_unzip = Pipeline( + pipebuf_cmd, + Command(str(unzip), "-dc", str(r2)), + skip_pipe=(0,), ) - cmd_str = ( - pipebuf_cmd - + "; " - + cmds[0] - + " <(" - + pipebuf_cmd - + "; " - + extract_cmd - + " <(" - + pipebuf_cmd - + "; " - + cmds[1] - + ") " + bwa_cmd_args: List[Union[str, InputProcSub]] = [ + *(["taskset", "-c", numa] if numa else []), + "sentieon", + "bwa", + "mem", + "-R", + readgroup, + "-t", + str(cores), + "-x", + f"{model_bundle}/bwa.model", + *bwa_args.split(), + "-K", + bwa_k, + *(["-p"] if (r2 and split) else []), + str(reference), + ] + if split: + extract_cmd = Pipeline( + pipebuf_cmd, + Command( + "sentieon", + "fqidx", + "extract", + "-F", + str(split), + "-K", + bwa_k, + InputProcSub(r1_unzip), + *([InputProcSub(r2_unzip)] if r2_unzip else []), + ), + skip_pipe=(0,), ) - if cmds[2]: - cmd_str += " <(" + pipebuf_cmd + "; " + cmds[2] + ") " - cmd_str += ") " + bwa_cmd_args.append(InputProcSub(extract_cmd)) else: - cmd_str = ( - pipebuf_cmd - + "; " - + cmds[0] - + " <(" - + pipebuf_cmd - + "; " - + cmds[1] - + ") " - ) - if cmds[2]: - cmd_str += " <(" + pipebuf_cmd + "; " + cmds[2] + ") " - cmd_str += " | " + cmds[3] - return cmd_str + bwa_cmd_args.append(InputProcSub(r1_unzip)) + if r2_unzip: + bwa_cmd_args.append(InputProcSub(r2_unzip)) + sort_cmd = Command( + *(["taskset", "-c", numa] if numa else []), + "sentieon", + "util", + "sort", + "-i", + "-", + "-t", + str(cores), + "--reference", + str(reference), + "-o", + str(out_aln), + "--sam2bam", + *util_sort_args.split(), + ) + return Pipeline( + pipebuf_cmd, + Command(str(bwa_cmd_args[0]), *bwa_cmd_args[1:]), + sort_cmd, + skip_pipe=(0,), + ) def cmd_multiqc( input_directory: pathlib.Path, output_directory: pathlib.Path, comment: Optional[str], -) -> str: +) -> Pipeline: """Run MultiQC""" cmd = [ "multiqc", @@ -943,7 +845,7 @@ def cmd_multiqc( str(input_directory), ] ) - return shlex.join(cmd) + return Pipeline(Command(*cmd)) def cmd_mosdepth( @@ -954,7 +856,7 @@ def cmd_mosdepth( xargs: str = ( "--by 500 --no-per-base --use-median -T 1,3,5,10,15,20,30,40,50" ), -) -> str: +) -> Pipeline: cmd = [ "mosdepth", "--fasta", @@ -965,7 +867,7 @@ def cmd_mosdepth( cmd.extend(shlex.split(xargs)) cmd.append(str(output_directory)) cmd.append(str(sample_input)) - return shlex.join(cmd) + return Pipeline(Command(*cmd)) def cmd_kmc( @@ -975,7 +877,7 @@ def cmd_kmc( k=29, memory=128, threads=1, -) -> str: +) -> Pipeline: cmd = [ "kmc", f"-k{k}", @@ -987,7 +889,71 @@ def cmd_kmc( str(output_prefix), str(tmp_dir), ] - return shlex.join(cmd) + return Pipeline(Command(*cmd)) + + +def cmd_extract_kmc( + output_prefix: pathlib.Path, + out_fastq: pathlib.Path, + input_aln: List[pathlib.Path], + reference: pathlib.Path, + extract_model: pathlib.Path, + tmp_dir: pathlib.Path, + threads=1, +) -> Pipeline: + merge_cmd = Command( + "samtools", + "merge", + "--reference", + str(reference), + "-@", + str(threads), + "-u", + "-o", + "/dev/stdout", + *[str(x) for x in input_aln], + ) + view_cmd = Command( + "samtools", + "view", + "-@", + str(threads), + "-ub", + "-F", + "0xf00", + "-", + ) + extract_cmd = Command( + "sentieon", + "pgutil", + "extract", + "-t", + str(threads), + "-T", + "tp,t0", + "-S", + "-PG", + "-m", + str(extract_model), + "-O", + "fastq_compression=1", + "-o", + str(out_fastq), + "-a", + "-", + ) + kmc_cmd = Command( + "kmc", + "-k29", + "-m32", + "-okff", + f"-t{threads}", + "-fa", + "/dev/stdin", + str(output_prefix), + str(tmp_dir), + ) + return Pipeline(merge_cmd, view_cmd, extract_cmd, kmc_cmd) def cmd_vg_haplotypes( @@ -998,7 +964,7 @@ def cmd_vg_haplotypes( threads=1, verbosity=1, xargs: List[str] = ["--include-reference", "--diploid-sampling"], -) -> str: +) -> Pipeline: cmd = [ "vg", "haplotypes", @@ -1015,7 +981,7 @@ def cmd_vg_haplotypes( str(gbz), ] cmd.extend(xargs) - return shlex.join(cmd) + return Pipeline(Command(*cmd)) def cmd_vg_giraffe( @@ -1027,7 +993,7 @@ def cmd_vg_giraffe( sample="sample", threads=1, max_fragment_length=3000, -) -> str: +) -> Pipeline: vg_cmd = [ "vg", "giraffe", @@ -1048,7 +1014,7 @@ def cmd_vg_giraffe( if fastq2: vg_cmd.extend(["-f", str(fastq2)]) - return shlex.join(vg_cmd) + " > " + str(sample_gam) + return Pipeline(Command(*vg_cmd), file_output=sample_gam) def cmd_vg_pack( @@ -1057,10 +1023,9 @@ def cmd_vg_pack( gbz: pathlib.Path, min_mapq=5, threads=1, -) -> str: - cat_cmd = ["cat"] - cat_cmd.extend([str(x) for x in sample_gams]) - vg_cmd = [ +) -> Pipeline: + cat_cmd = Command("cat", *[str(x) for x in sample_gams]) + vg_cmd = Command( "vg", "pack", "-x", @@ -1073,9 +1038,9 @@ def cmd_vg_pack( str(min_mapq), "--threads", str(threads), - ] + ) - return shlex.join(cat_cmd) + " | " + shlex.join(vg_cmd) + return Pipeline(cat_cmd, vg_cmd) def cmd_sv_call( @@ -1087,74 +1052,56 @@ def cmd_sv_call( sample_name="", ref_path="GRCh38", min_indel_size=34, -) -> str: - cmds = [] - cmds.append( - [ - "cat", - str(sv_header), - ] - ) - - cmds.append( - [ - "vg", - "call", - "-r", - str(snarls), - "-k", - str(sample_pack), - "-s", - sample_name, - "-z", - "--ref-sample", - ref_path, - "--progress", - str(gbz), - ] +) -> Pipeline: + # Call SVs with vg call + vg_call_cmd = Command( + "vg", + "call", + "-r", + str(snarls), + "-k", + str(sample_pack), + "-s", + sample_name, + "-z", + "--ref-sample", + ref_path, + "--progress", + str(gbz), ) # Use bcftools to remove smaller variants - cmds.append( - [ - "bcftools", - "view", - "-i", - f"ABS(ILEN) > {min_indel_size}", - "-H", - "-", - ] + view_cmd = Command( + "bcftools", + "view", + "-i", + f"ABS(ILEN) > {min_indel_size}", + "-H", + "-", ) - # Sort the VCF - cmds.append( - [ - "bcftools", - "sort", - "-O", - "v", - ] + # Rehead the vg call output + cat_cmd = Command( + "cat", + str(sv_header), + InputProcSub(Pipeline(vg_call_cmd, view_cmd)), ) - cmds.append( - [ - "sentieon", - "util", - "vcfconvert", - "-", - str(out_svs), - ] + # Sort and compress the output + sort_cmd = Command( + "bcftools", + "sort", + "-O", + "v", ) - - all_cmds = [shlex.join(x) for x in cmds] - return ( - "(" - + all_cmds[0] - + "; " - + " | ".join(all_cmds[1:3]) - + ") | " - + " | ".join(all_cmds[3:]) + convert_cmd = Command( + "sentieon", + "util", + "vcfconvert", + "-", + str(out_svs), ) + return Pipeline(cat_cmd, sort_cmd, convert_cmd) def cmd_vg_surject( @@ -1167,74 +1114,59 @@ def cmd_vg_surject( "--interleaved", "--progress", ], -) -> str: - cmds = [] - cmds.append( - [ - "vg", - "surject", - "--sam-output", - "-t", - str(threads), - "-x", - str(xg), - "--into-paths", - shlex.quote(str(surject_paths_dict)), - ] - ) - cmds[-1].extend(surject_xargs) - cmds[-1].append(str(sample_gam)) +) -> Pipeline: + surject_cmd_list = [ + "vg", + "surject", + "--sam-output", + "-t", + str(threads), + "-x", + str(xg), + "--into-paths", + shlex.quote(str(surject_paths_dict)), + ] + surject_cmd_list.extend(surject_xargs) + surject_cmd_list.append(str(sample_gam)) - cmds.append( - [ - "sentieon", - "util", - "sort", - "--sam2bam", - "-i", - "-", - "-o", - str(out_bam), - ] + sort_cmd = Command( + "sentieon", + "util", + "sort", + "--sam2bam", + "-i", + "-", + "-o", + str(out_bam), ) - all_cmds = [shlex.join(x) for x in cmds] - return " | ".join(all_cmds) + return Pipeline(Command(*surject_cmd_list), sort_cmd) def strip_ctg_prefix( output_header: pathlib.Path, bam: pathlib.Path, prefix: str, -) -> str: +) -> Pipeline: # Also remove the M5 tag - cmds = [] - cmds.append(["samtools", "view", "-H", shlex.quote(str(bam))]) - - cmds.append( - [ - "sed", - "-e", - f"s/\tSN:{prefix}/\tSN:/", - "-e", - "s/\tM5:[0-9a-zA-Z]*\t/\t/", - "-e", - "s/\tM5:[0-9a-zA-Z]*$//", - ] - ) - - return ( - " | ".join([shlex.join(x) for x in cmds]) - + " > " - + shlex.quote(str(output_header)) + samtools_cmd = Command("samtools", "view", "-H", str(bam)) + sed_cmd = Command( + "sed", + "-e", + f"s/\tSN:{prefix}/\tSN:/", + "-e", + "s/\tM5:[0-9a-zA-Z]*\t/\t/", + "-e", + "s/\tM5:[0-9a-zA-Z]*$//", ) + return Pipeline(samtools_cmd, sed_cmd, file_output=output_header) def cmd_estimate_ploidy( output_json: pathlib.Path, aln_files: List[pathlib.Path], ploidy_script: pathlib.Path, -) -> str: +) -> Pipeline: cmd = ( [ "python3", @@ -1247,7 +1179,7 @@ def cmd_estimate_ploidy( str(output_json), ] ) - return shlex.join(cmd) + return Pipeline(Command(*cmd)) def cmd_t1k( @@ -1257,7 +1189,7 @@ def cmd_t1k( gene_coord: pathlib.Path, preset: str, threads=1, -) -> str: +) -> Pipeline: cmd = [ "run-t1k", "--abnormalUnmapFlag", @@ -1274,7 +1206,7 @@ def cmd_t1k( "-b", str(deduped_bam), ] - return shlex.join(cmd) + return Pipeline(Command(*cmd)) def cmd_expansion_hunter( @@ -1284,7 +1216,7 @@ def cmd_expansion_hunter( variant_catalog: pathlib.Path, sex="female", threads=1, -) -> str: +) -> Pipeline: cmd = [ "ExpansionHunter", "--reads", @@ -1300,7 +1232,7 @@ def cmd_expansion_hunter( "--output-prefix", str(out_expansions), ] - return shlex.join(cmd) + return Pipeline(Command(*cmd)) def cmd_segdup_caller( @@ -1309,7 +1241,7 @@ def cmd_segdup_caller( reference: pathlib.Path, sr_bundle: pathlib.Path, genes="CFH,CFHR3,CYP11B1,CYP2D6,GBA,NCF1,PMS2,SMN1,STRC", -) -> str: +) -> Pipeline: cmd = [ "segdup-caller", "--short", @@ -1323,4 +1255,234 @@ def cmd_segdup_caller( "--outdir", str(out_segdup), ] - return shlex.join(cmd) + return Pipeline(Command(*cmd)) + + +def cmd_bwa_extract( + out_bam: pathlib.Path, + out_fastq: pathlib.Path, + reference: pathlib.Path, + fastq1: List[pathlib.Path], + fastq2: List[pathlib.Path], + readgroup: str, + extract_model: pathlib.Path, + bwa_model: pathlib.Path, + threads=1, + unzip: str = "gzip", + bwa_k: str = "2000000", +) -> Pipeline: + """BWA alignment with pgutil extract""" + fq1_cmd = Pipeline( + Command( + unzip, + "-dc", + *[str(x) for x in fastq1], + ) + ) + fq2_cmd = Pipeline( + Command( + unzip, + "-dc", + *[str(x) for x in fastq2], + ) + ) + + bwa_cmd = Command( + "sentieon", + "bwa", + "mem", + "-R", + readgroup, + "-t", + str(threads), + "-x", + str(bwa_model), + "-K", + bwa_k, + str(reference), + InputProcSub(fq1_cmd), + InputProcSub(fq2_cmd), + ) + + extract_cmd = Command( + "sentieon", + "pgutil", + "extract", + "-t", + str(threads), + "-S", + "-PG", + "-m", + str(extract_model), + "-O", + "fastq_compression=1", + "-b", + str(out_bam), + "-o", + str(out_fastq), + ) + + return Pipeline(bwa_cmd, extract_cmd) + + +def cmd_vg_convert_gfa( + output_gfa: pathlib.Path, + input_gbz: pathlib.Path, + reference_name="GRCh38", + threads=1, +) -> Pipeline: + """Convert GBZ to GFA""" + cmd = [ + "vg", + "convert", + "-t", + str(threads), + "-f", + "-Q", + reference_name, + str(input_gbz), + ] + return Pipeline(Command(*cmd), file_output=output_gfa) + + +def cmd_vg_paths_fasta( + output_fasta: pathlib.Path, + input_gbz: pathlib.Path, +) -> Pipeline: + """Extract haplotype sequences as FASTA""" + cmd = [ + "vg", + "paths", + "-x", + str(input_gbz), + "-H", + "-F", + ] + return Pipeline(Command(*cmd), file_output=output_fasta) + + +def cmd_minimap2_lift( + out_bam: pathlib.Path, + sample_fasta: pathlib.Path, + input_fastq: pathlib.Path, + gfa_file: pathlib.Path, + reference_fasta: pathlib.Path, + readgroup: str, + minimap2_model: Union[pathlib.Path, str], + minimap2_i: str = "16G", + threads=1, + mm2_xargs: List[str] = [], +) -> Pipeline: + """Minimap2 alignment with pgutil lift""" + mm2_cmd = Command( + "sentieon", + "minimap2", + "-t", + str(threads), + "-R", + readgroup, + "-a", + "-y", + "-I", + minimap2_i, + "-x", + str(minimap2_model), + *mm2_xargs, + str(sample_fasta), + str(input_fastq), + ) + + lift_cmd = Command( + "sentieon", + "pgutil", + "lift", + "-t", + str(threads), + "-F", + str(reference_fasta) + ".fai", + "-g", + str(gfa_file), + ) + + sort_cmd = Command( + "sentieon", + "util", + "sort", + "-i", + "-", + "-r", + str(reference_fasta), + "-t", + str(threads), + "-o", + str(out_bam), + ) + + return Pipeline(mm2_cmd, lift_cmd, sort_cmd) + + +def cmd_bcftools_merge_trim( + shard_vcf: pathlib.Path, + raw_vcf: pathlib.Path, + pop_vcf: pathlib.Path, + trim_script: pathlib.Path, + subset_bed: pathlib.Path, + merge_rules: str, + merge_xargs: List[str] = [], + view_xargs: List[str] = [], +): + """Merge and trim variants""" + merge_cmd = Command( + "bcftools", + "merge", + "--regions-file", + str(subset_bed), + *merge_xargs, + "-i", + merge_rules, + str(raw_vcf), + str(pop_vcf), + ) + + trim_cmd = Command( + "python3", + str(trim_script), + ) + + view_cmd = Command( + "bcftools", + "view", + *view_xargs, + "-W=tbi", + "-o", + str(shard_vcf), + ) + + return Pipeline(merge_cmd, trim_cmd, view_cmd) + + +def cmd_bcftools_view_regions( + out_vcf: pathlib.Path, + in_vcf: pathlib.Path, + regions: Optional[str] = None, + regions_file: Optional[pathlib.Path] = None, +): + xargs = [] + if regions: + xargs.extend(["--regions", regions]) + if regions_file: + xargs.extend(["--regions-file", str(regions_file)]) + return Pipeline( + Command( + "bcftools", + "view", + "--no-version", + "-W=tbi", + "-O", + "z", + "-o", + str(out_vcf), + *xargs, + str(in_vcf), + ) + ) diff --git a/sentieon_cli/dnascope.py b/sentieon_cli/dnascope.py index 5a62e27..dae9a48 100644 --- a/sentieon_cli/dnascope.py +++ b/sentieon_cli/dnascope.py @@ -8,13 +8,14 @@ import multiprocessing as mp import os import pathlib -import shlex import shutil import sys -from typing import Any, Dict, List, Optional, Set, Tuple +from typing import Any, List, Optional, Set, Tuple import packaging.version +from importlib_resources import files + from . import command_strings as cmds from .dag import DAG from .driver import ( @@ -39,8 +40,8 @@ ) from .job import Job from .pipeline import BasePipeline +from .shell_pipeline import Command, Pipeline from .util import ( - __version__, check_version, library_preloaded, path_arg, @@ -62,10 +63,6 @@ "sentieon driver": packaging.version.Version("202308"), } -MULTIQC_MIN_VERSION = { - "multiqc": packaging.version.Version("1.18"), -} - CNV_MIN_VERSIONS = { "sentieon driver": packaging.version.Version("202308.03"), } @@ -74,187 +71,160 @@ class DNAscopePipeline(BasePipeline): """The DNAscope pipeline""" - params: Dict[str, Dict[str, Any]] = { - "model_bundle": { - "flags": ["-m", "--model_bundle"], - "help": "The model bundle file.", - "required": True, - "type": path_arg(exists=True, is_file=True), - }, - "r1_fastq": { - "nargs": "*", - "help": "Sample R1 fastq files.", - "type": path_arg(exists=True, is_file=True), - }, - "r2_fastq": { - "nargs": "*", - "help": "Sample R2 fastq files.", - "type": path_arg(exists=True, is_file=True), - }, - "readgroups": { - "nargs": "*", - "help": "Readgroup information for the fastq files.", - }, - "reference": { - "flags": ["-r", "--reference"], - "required": True, - "help": "fasta for reference genome.", - "type": path_arg(exists=True, is_file=True), - }, - "sample_input": { - "flags": ["-i", "--sample_input"], - "nargs": "*", - "help": "sample BAM or CRAM file.", - "type": path_arg(exists=True, is_file=True), - }, - "align": { - "help": ( - "Align the reads in the input uBAM/uCRAM file to the " - "reference genome. Assumes paired reads are collated in the " - "input file." - ), - "action": "store_true", - }, - "assay": { - "help": "The type of assay, WGS or WES.", - "choices": ["WGS", "WES"], - "default": "WGS", - }, - "bam_format": { - "help": ( - "Use the BAM format instead of CRAM for output aligned files" - ), - "action": "store_true", - }, - "bed": { - "flags": ["-b", "--bed"], - "help": ( - "Region BED file. Supplying this file will limit variant " - "calling to the intervals inside the BED file." - ), - "type": path_arg(exists=True, is_file=True), - }, - "collate_align": { - "help": ( - "Collate and align the reads in the input BAM/CRAM file to " - "the reference genome. Suitable for coordinate-sorted " - "BAM/CRAM input." - ), - "action": "store_true", - }, - "consensus": { - "help": "Generate consensus reads during dedup", - "action": "store_true", - }, - "cores": { - "flags": ["-t", "--cores"], - "help": ( - "Number of threads/processes to use. Defaults to all " - "available." - ), - "default": mp.cpu_count(), - }, - "dbsnp": { - "flags": ["-d", "--dbsnp"], - "help": ( - "dbSNP vcf file Supplying this file will annotate variants " - "with their dbSNP refSNP ID numbers." - ), - "type": path_arg(exists=True, is_file=True), - }, - "dry_run": { - "help": "Print the commands without running them.", - "action": "store_true", - }, - "duplicate_marking": { - "help": "Options for duplicate marking.", - "choices": ["markdup", "rmdup", "none"], - "default": "markdup", - }, - "gvcf": { - "flags": ["-g", "--gvcf"], - "help": ( - "Generate a gVCF output file along with the VCF." - " (default generates only the VCF)" - ), - "action": "store_true", - }, - "input_ref": { - "help": ( - "Used to decode the input alignment file. Required if the " - "input file is in the CRAM/uCRAM formats." - ), - "type": path_arg(exists=True, is_file=True), - }, - "interval_padding": { - "help": "Amount to pad all intervals.", - "type": int, - }, - "pcr_free": { - "help": "Use arguments for PCR-free data processing", - "action": "store_true", - }, - "skip_metrics": { - "help": "Skip all metrics collection and multiQC", - "action": "store_true", - }, - "skip_multiqc": { - "help": "Skip multiQC report generation", - "action": "store_true", - }, - "skip_small_variants": { - "help": "Skip small variant (SNV/indel) calling", - "action": "store_true", - }, - "skip_svs": { - "help": "Skip SV calling", - "action": "store_true", - }, - "bwa_args": { - # "help": "Extra arguments for sentieon bwa", - "help": argparse.SUPPRESS, - "default": "", - }, - "bwa_k": { - # "help": "The '-K' argument in bwa", - "help": argparse.SUPPRESS, - "default": 100000000, - }, - "bwt_max_mem": { - # Manually set `bwt_max_mem` - "help": argparse.SUPPRESS, - }, - "no_ramdisk": { - # Do not use /dev/shm, even on high memory machines - "help": argparse.SUPPRESS, - "action": "store_true", - }, - "no_split_alignment": { - "help": argparse.SUPPRESS, - "action": "store_true", - }, - "skip_version_check": { - "help": argparse.SUPPRESS, - "action": "store_true", - }, - "util_sort_args": { - # "help": "Extra arguments for sentieon util sort", - "help": argparse.SUPPRESS, - "default": "--cram_write_options version=3.0,compressor=rans", - }, - } - - positionals: Dict[str, Dict[str, Any]] = { - "output_vcf": { - "help": "Output VCF File. The file name must end in .vcf.gz", - "type": path_arg(), - }, - } + params = copy.deepcopy(BasePipeline.params) + params.update( + { + "model_bundle": { + "flags": ["-m", "--model_bundle"], + "help": "The model bundle file.", + "required": True, + "type": path_arg(exists=True, is_file=True), + }, + "r1_fastq": { + "nargs": "*", + "help": "Sample R1 fastq files.", + "type": path_arg(exists=True, is_file=True), + }, + "r2_fastq": { + "nargs": "*", + "help": "Sample R2 fastq files.", + "type": path_arg(exists=True, is_file=True), + }, + "readgroups": { + "nargs": "*", + "help": "Readgroup information for the fastq files.", + }, + "sample_input": { + "flags": ["-i", "--sample_input"], + "nargs": "*", + "help": "sample BAM or CRAM file.", + "type": path_arg(exists=True, is_file=True), + }, + "align": { + "help": ( + "Align the reads in the input uBAM/uCRAM file to the " + "reference genome. Assumes paired reads are collated in " + "the input file." + ), + "action": "store_true", + }, + "assay": { + "help": "The type of assay, WGS or WES.", + "choices": ["WGS", "WES"], + "default": "WGS", + }, + "bam_format": { + "help": ( + "Use the BAM format instead of CRAM for output aligned " + "files." + ), + "action": "store_true", + }, + "bed": { + "flags": ["-b", "--bed"], + "help": ( + "Region BED file. Supplying this file will limit variant " + "calling to the intervals inside the BED file." + ), + "type": path_arg(exists=True, is_file=True), + }, + "collate_align": { + "help": ( + "Collate and align the reads in the input BAM/CRAM file " + "to the reference genome. Suitable for coordinate-sorted " + "BAM/CRAM input." + ), + "action": "store_true", + }, + "consensus": { + "help": "Generate consensus reads during dedup", + "action": "store_true", + }, + "dbsnp": { + "flags": ["-d", "--dbsnp"], + "help": ( + "dbSNP vcf file Supplying this file will annotate " + "variants with their dbSNP refSNP ID numbers." + ), + "type": path_arg(exists=True, is_file=True), + }, + "duplicate_marking": { + "help": "Options for duplicate marking.", + "choices": ["markdup", "rmdup", "none"], + "default": "markdup", + }, + "gvcf": { + "flags": ["-g", "--gvcf"], + "help": ( + "Generate a gVCF output file along with the VCF." + " (default generates only the VCF)" + ), + "action": "store_true", + }, + "input_ref": { + "help": ( + "Used to decode the input alignment file. Required if the " + "input file is in the CRAM/uCRAM formats." + ), + "type": path_arg(exists=True, is_file=True), + }, + "interval_padding": { + "help": "Amount to pad all intervals.", + "type": int, + }, + "pcr_free": { + "help": "Use arguments for PCR-free data processing", + "action": "store_true", + }, + "skip_metrics": { + "help": "Skip all metrics collection and multiQC", + "action": "store_true", + }, + "skip_multiqc": { + "help": "Skip multiQC report generation", + "action": "store_true", + }, + "skip_small_variants": { + "help": "Skip small variant (SNV/indel) calling", + "action": "store_true", + }, + "skip_svs": { + "help": "Skip SV calling", + "action": "store_true", + }, + "bwa_args": { + # "help": "Extra arguments for sentieon bwa", + "help": argparse.SUPPRESS, + "default": "", + }, + "bwa_k": { + # "help": "The '-K' argument in bwa", + "help": argparse.SUPPRESS, + "default": 100000000, + }, + "bwt_max_mem": { + # Manually set `bwt_max_mem` + "help": argparse.SUPPRESS, + }, + "no_ramdisk": { + # Do not use /dev/shm, even on high memory machines + "help": argparse.SUPPRESS, + "action": "store_true", + }, + "no_split_alignment": { + "help": argparse.SUPPRESS, + "action": "store_true", + }, + "util_sort_args": { + # "help": "Extra arguments for sentieon util sort", + "help": argparse.SUPPRESS, + "default": "--cram_write_options version=3.0,compressor=rans", + }, + } + ) def __init__(self) -> None: super().__init__() - self.output_vcf: Optional[pathlib.Path] = None - self.reference: Optional[pathlib.Path] = None self.sample_input: List[pathlib.Path] = [] self.r1_fastq: List[pathlib.Path] = [] self.r2_fastq: List[pathlib.Path] = [] @@ -263,7 +233,6 @@ def __init__(self) -> None: self.dbsnp: Optional[pathlib.Path] = None self.bed: Optional[pathlib.Path] = None self.interval_padding = 0 - self.cores = mp.cpu_count() self.pcr_free = False self.gvcf = False self.duplicate_marking = "markdup" @@ -282,7 +251,6 @@ def __init__(self) -> None: self.util_sort_args = ( "--cram_write_options version=3.0,compressor=rans" ) - self.skip_version_check = False self.bwt_max_mem: Optional[str] = None self.no_ramdisk = False self.no_split_alignment = False @@ -300,7 +268,7 @@ def validate(self) -> None: del self.readgroups del self.duplicate_marking - # vaidate + # validate if not self.sample_input and not ( self.sr_r1_fastq and self.sr_readgroups ): @@ -382,7 +350,7 @@ def check_shm(self, total_input_size: int, n_alignment_jobs) -> bool: if ( shm_free > total_input_size * 2.3 - and total_mem > total_input_size + 40 * 1024**3 * n_alignment_jobs + and total_mem > total_input_size + 60 * 1024**3 * n_alignment_jobs ): self.logger.debug("Using /dev/shm for temporary files") return True @@ -545,10 +513,11 @@ def sr_align_inputs(self) -> Tuple[List[pathlib.Path], Set[Job], Job]: # Create an unscheduled job to remove the aligned inputs rm_job = Job( - shlex.join(["rm"] + [str(x) for x in align_outputs]), + Pipeline( + Command("rm", *[str(x) for x in align_outputs], fail_ok=True) + ), "rm-bam-aln", 0, - True, ) return (res, jobs, rm_job) @@ -636,10 +605,11 @@ def sr_align_fastq( # Create an unscheduled job to remove the aligned inputs rm_job = Job( - shlex.join(["rm"] + [str(x) for x in align_outputs]), + Pipeline( + Command("rm", *[str(x) for x in align_outputs], fail_ok=True) + ), "rm-fq-aln", 0, - True, ) return (res, jobs, rm_job) @@ -689,7 +659,6 @@ def dedup_and_metrics( # WGS metrics wgs_metrics = metrics_dir.joinpath(metric_base + ".wgs.txt") - wgs_metrics_tmp = metrics_dir.joinpath(metric_base + ".wgs.txt.tmp") gc_metrics = metrics_dir.joinpath(metric_base + ".gc_bias.txt") gc_summary = metrics_dir.joinpath(metric_base + ".gc_bias_summary.txt") @@ -724,7 +693,9 @@ def dedup_and_metrics( if not (self.sr_duplicate_marking == "none" and self.skip_metrics): lc_job = Job( - shlex.join(driver.build_cmd()), "locuscollector", self.cores + Pipeline(Command(*driver.build_cmd())), + "locuscollector", + self.cores, ) if self.sr_duplicate_marking == "none": @@ -751,7 +722,9 @@ def dedup_and_metrics( rmdup=(self.sr_duplicate_marking == "rmdup"), ) ) - dedup_job = Job(shlex.join(driver.build_cmd()), "dedup", self.cores) + dedup_job = Job( + Pipeline(Command(*driver.build_cmd())), "dedup", self.cores + ) if self.skip_metrics: return ([deduped], lc_job, dedup_job, None, None) @@ -774,7 +747,7 @@ def dedup_and_metrics( driver.add_algo(HsMetricAlgo(hs_metrics, self.bed, self.bed)) driver.add_algo(InsertSizeMetricAlgo(is_metrics)) metrics_job = Job( - shlex.join(driver.build_cmd()), "metrics", 0 + Pipeline(Command(*driver.build_cmd())), "metrics", 0 ) # Run metrics in the background # Run WgsMetricsAlgo after duplicate marking to account for @@ -786,12 +759,27 @@ def dedup_and_metrics( ) driver.add_algo(CoverageMetrics(coverage_metrics)) metrics_job = Job( - shlex.join(driver.build_cmd()), "metrics", 0 + Pipeline(Command(*driver.build_cmd())), "metrics", 0 ) # Run metrics in the background # Rehead WGS metrics so they are recognized by MultiQC + rehead_script = pathlib.Path( + str( + files("sentieon_cli.scripts").joinpath( + "rehead_wgs_metrics.py" + ) + ) + ) rehead_job = Job( - cmds.rehead_wgsmetrics(wgs_metrics, wgs_metrics_tmp), + Pipeline( + Command( + "sentieon", + "pyexec", + str(rehead_script), + "--metrics_file", + str(wgs_metrics), + ) + ), "Rehead metrics", 0, ) @@ -860,7 +848,9 @@ def sr_call_variants( ) ) call_job = Job( - shlex.join(driver.build_cmd()), "variant-calling", self.cores + Pipeline(Command(*driver.build_cmd())), + "variant-calling", + self.cores, ) # Genotyping and filtering with DNAModelApply @@ -876,12 +866,12 @@ def sr_call_variants( ) ) apply_job = Job( - shlex.join(driver.build_cmd()), "model-apply", self.cores + Pipeline(Command(*driver.build_cmd())), "model-apply", self.cores ) # Remove the tmp_vcf rm_cmd = ["rm", str(tmp_vcf), str(tmp_vcf) + ".tbi"] - rm_job = Job(shlex.join(rm_cmd), "rm-tmp-vcf", 0, True) + rm_job = Job(Pipeline(Command(*rm_cmd, fail_ok=True)), "rm-tmp-vcf", 0) # Genotype gVCFs gvcftyper_job = None @@ -898,7 +888,7 @@ def sr_call_variants( ) ) gvcftyper_job = Job( - shlex.join(driver.build_cmd()), "gvcftyper", self.cores + Pipeline(Command(*driver.build_cmd())), "gvcftyper", self.cores ) # Call SVs @@ -916,14 +906,20 @@ def sr_call_variants( vcf=out_svs_tmp, ) ) - svsolver_job = Job(shlex.join(driver.build_cmd()), "svsolver") + svsolver_job = Job( + Pipeline(Command(*driver.build_cmd())), "svsolver" + ) sv_rm_job = Job( - shlex.join( - ["rm", str(out_svs_tmp), str(out_svs_tmp) + ".tbi"] + Pipeline( + Command( + "rm", + str(out_svs_tmp), + str(out_svs_tmp) + ".tbi", + fail_ok=True, + ) ), "rm-tmp-sv", 0, - True, ) return ( @@ -935,36 +931,6 @@ def sr_call_variants( sv_rm_job, ) - def multiqc(self) -> Optional[Job]: - """Run MultiQC on the metrics files""" - - if not self.skip_version_check: - if not all( - [ - check_version(cmd, min_version) - for (cmd, min_version) in MULTIQC_MIN_VERSION.items() - ] - ): - self.logger.warning( - "Skipping MultiQC. MultiQC version %s or later not found", - MULTIQC_MIN_VERSION["multiqc"], - ) - return None - - metrics_dir = pathlib.Path( - str(self.output_vcf).replace(".vcf.gz", "_metrics") - ) - multiqc_job = Job( - cmds.cmd_multiqc( - metrics_dir, - metrics_dir, - f"Generated by the Sentieon-CLI version {__version__}", - ), - "multiqc", - 0, - ) - return multiqc_job - def call_cnvs( tmp_dir: pathlib.Path, @@ -975,6 +941,7 @@ def call_cnvs( bed: Optional[pathlib.Path] = None, cores: int = mp.cpu_count(), skip_version_check: bool = False, + replace_rg: Optional[List[List[str]]] = None, **_kwargs: Any, ) -> Tuple[Job, Job]: """ @@ -990,6 +957,7 @@ def call_cnvs( driver = Driver( reference=reference, thread_count=cores, + replace_rg=replace_rg, input=sample_input, interval=bed, ) @@ -999,7 +967,9 @@ def call_cnvs( model_bundle.joinpath("cnv.model"), ) ) - cnvscope_job = Job(shlex.join(driver.build_cmd()), "CNVscope", cores) + cnvscope_job = Job( + Pipeline(Command(*driver.build_cmd())), "CNVscope", cores + ) driver = Driver( reference=reference, @@ -1013,7 +983,7 @@ def call_cnvs( ) ) cnvmodelapply_job = Job( - shlex.join(driver.build_cmd()), + Pipeline(Command(*driver.build_cmd())), "CNVModelApply", cores, ) diff --git a/sentieon_cli/dnascope_hybrid.py b/sentieon_cli/dnascope_hybrid.py index 71635b9..d268a6a 100644 --- a/sentieon_cli/dnascope_hybrid.py +++ b/sentieon_cli/dnascope_hybrid.py @@ -4,10 +4,8 @@ import argparse import copy -import multiprocessing as mp import json import pathlib -import shlex import sys from typing import Any, Dict, List, Optional, Set, Tuple @@ -30,6 +28,8 @@ from .dnascope import call_cnvs, DNAscopePipeline from .dnascope_longread import DNAscopeLRPipeline from .job import Job +from .pipeline import BasePipeline +from .shell_pipeline import Command, Pipeline from .util import ( __version__, check_version, @@ -43,7 +43,7 @@ CALLING_MIN_VERSIONS = { "sentieon driver": packaging.version.Version("202503.01"), "bedtools": None, - "bcftools": packaging.version.Version("1.10"), + "bcftools": packaging.version.Version("1.22"), "samtools": packaging.version.Version("1.16"), } @@ -52,194 +52,220 @@ } +class RgInfo: + """A container class for short and long-read readgroups""" + + def __init__( + self, + lr_aln_readgroups: List[List[Dict[str, str]]], + sr_aln_readgroups: List[List[Dict[str, str]]], + lr_read_filter: Optional[str], + sr_read_filter: Optional[str], + hybrid_set_rg: bool, + hybrid_rg_sm: str, + shortread_tech: str, + ): + # readgroup handling for long-reads + self.lr_rg_read_filter: List[str] = [] + self.replace_rg_args: Tuple[List[List[str]], List[List[str]]] = ( + [], + [], + ) + for aln_rgs in lr_aln_readgroups: + self.replace_rg_args[0].append([]) + for rg_line_d in aln_rgs: + id = rg_line_d.get("ID") + new_sm = hybrid_rg_sm if hybrid_set_rg else rg_line_d.get("SM") + self.replace_rg_args[0][-1].append( + f"{id}=ID:{id}\\tSM:{new_sm}\\tLR:1" + ) + if lr_read_filter: + self.lr_rg_read_filter.append( + f"{lr_read_filter},rgid={id}" + ) + + # readgroup handling for short-reads + self.sr_rg_read_filter: List[str] = [] + self.ultima_read_filter: List[str] = [] + for aln_rgs in sr_aln_readgroups: + if hybrid_set_rg: + self.replace_rg_args[1].append([]) + for rg_line_d in aln_rgs: + id = rg_line_d.get("ID") + new_sm = hybrid_rg_sm if hybrid_set_rg else rg_line_d.get("SM") + if hybrid_set_rg: + self.replace_rg_args[1][-1].append( + f"{id}=ID:{id}\\tSM:{new_sm}" + ) + if sr_read_filter: + self.sr_rg_read_filter.append( + f"{sr_read_filter},rgid={id}" + ) + if shortread_tech.upper() == "ULTIMA": + self.ultima_read_filter.append( + f"UltimaReadFilter,rgid={id}" + ) + + class DNAscopeHybridPipeline(DNAscopePipeline, DNAscopeLRPipeline): """The DNAscope Hybrid pipeline""" - params: Dict[str, Dict[str, Any]] = { - "lr_aln": { - "nargs": "*", - "help": "Long-read BAM or CRAM files.", - "type": path_arg(exists=True, is_file=True), - "required": True, - }, - "model_bundle": { - "flags": ["-m", "--model_bundle"], - "help": "The model bundle file.", - "required": True, - "type": path_arg(exists=True, is_file=True), - }, - "reference": { - "flags": ["-r", "--reference"], - "required": True, - "help": "fasta for reference genome.", - "type": path_arg(exists=True, is_file=True), - }, - "sr_aln": { - "nargs": "*", - "help": "Short-read BAM or CRAM files", - "type": path_arg(exists=True, is_file=True), - }, - "sr_r1_fastq": { - "nargs": "*", - "help": "Short-read R1 fastq files", - "type": path_arg(exists=True, is_file=True), - }, - "sr_r2_fastq": { - "nargs": "*", - "help": "Short-read R2 fastq files", - "type": path_arg(exists=True, is_file=True), - }, - "sr_readgroups": { - "nargs": "*", - "help": "Readgroup information for the short-read fastq files", - }, - "bam_format": { - "help": ( - "Use the BAM format instead of CRAM for output aligned files" - ), - "action": "store_true", - }, - "bed": { - "flags": ["-b", "--bed"], - "help": ( - "Region BED file. Supplying this file will limit variant " - "calling to the intervals inside the BED file." - ), - "type": path_arg(exists=True, is_file=True), - }, - "cores": { - "flags": ["-t", "--cores"], - "help": ( - "Number of threads/processes to use. Defaults to all " - "available." - ), - "default": mp.cpu_count(), - }, - "dbsnp": { - "flags": ["-d", "--dbsnp"], - "help": ( - "dbSNP vcf file Supplying this file will annotate variants " - "with their dbSNP refSNP ID numbers." - ), - "type": path_arg(exists=True, is_file=True), - }, - "dry_run": { - "help": "Print the commands without running them.", - "action": "store_true", - }, - "gvcf": { - "flags": ["-g", "--gvcf"], - "help": ( - "Generate a gVCF output file along with the VCF." - " (default generates only the VCF)" - ), - "action": "store_true", - }, - "lr_align_input": { - "help": ( - "Align the input long-read BAM/CRAM/uBAM file to the " - "reference genome" - ), - "action": "store_true", - }, - "lr_input_ref": { - "help": ( - "Used to decode the input long-read alignment file. Required " - "if the input file is in the CRAM/uCRAM formats." - ), - "type": path_arg(exists=True, is_file=True), - }, - "rgsm": { - "help": ( - "Overwrite the SM tag of the input readgroups for " - "compatibility" - ) - }, - "skip_cnv": { - "help": "Skip CNV calling.", - "action": "store_true", - }, - "skip_metrics": { - "help": "Skip all metrics collection and multiQC", - "action": "store_true", - }, - "skip_mosdepth": { - "help": "Skip QC with mosdepth.", - "action": "store_true", - }, - "skip_multiqc": { - "help": "Skip multiQC report generation.", - "action": "store_true", - }, - "skip_svs": { - "help": "Skip SV calling", - "action": "store_true", - }, - "sr_duplicate_marking": { - "help": "Options for duplicate marking.", - "choices": ["markdup", "rmdup", "none"], - "default": "markdup", - }, - "bwa_args": { - # help="Extra arguments for sentieon bwa", - "help": argparse.SUPPRESS, - "default": "", - }, - "bwa_k": { - # help="The '-K' argument in bwa", - "help": argparse.SUPPRESS, - "default": 100000000, - }, - "bwt_max_mem": { - # Manually set `bwt_max_mem` - "help": argparse.SUPPRESS, - }, - "lr_fastq_taglist": { - # help="A comma-separated list of tags to retain. Defaults to " - # "'%(default)s' and the 'RG' tag is required", - "help": argparse.SUPPRESS, - "default": "*", - }, - "lr_read_filter": { - "help": argparse.SUPPRESS, - }, - "minimap2_args": { - # help="Extra arguments for sentieon minimap2", - "help": argparse.SUPPRESS, - "default": "-Y", - }, - "no_split_alignment": { - "help": argparse.SUPPRESS, - "action": "store_true", - }, - "retain_tmpdir": { - "help": argparse.SUPPRESS, - "action": "store_true", - }, - "skip_version_check": { - "help": argparse.SUPPRESS, - "action": "store_true", - }, - "skip_model_apply": { - "help": argparse.SUPPRESS, - "action": "store_true", - }, - "sr_read_filter": { - "help": argparse.SUPPRESS, - }, - "util_sort_args": { - # help="Extra arguments for sentieon util sort", - "help": argparse.SUPPRESS, - "default": "--cram_write_options version=3.0,compressor=rans", - }, - } - - positionals: Dict[str, Dict[str, Any]] = { - "output_vcf": { - "help": "Output VCF File. The file name must end in .vcf.gz", - "type": path_arg(), - }, - } + params = copy.deepcopy(BasePipeline.params) + params.update( + { + "lr_aln": { + "nargs": "*", + "help": "Long-read BAM or CRAM files.", + "type": path_arg(exists=True, is_file=True), + "required": True, + }, + "model_bundle": { + "flags": ["-m", "--model_bundle"], + "help": "The model bundle file.", + "required": True, + "type": path_arg(exists=True, is_file=True), + }, + "sr_aln": { + "nargs": "*", + "help": "Short-read BAM or CRAM files", + "type": path_arg(exists=True, is_file=True), + }, + "sr_r1_fastq": { + "nargs": "*", + "help": "Short-read R1 fastq files", + "type": path_arg(exists=True, is_file=True), + }, + "sr_r2_fastq": { + "nargs": "*", + "help": "Short-read R2 fastq files", + "type": path_arg(exists=True, is_file=True), + }, + "sr_readgroups": { + "nargs": "*", + "help": "Readgroup information for the short-read fastq files", + }, + "bam_format": { + "help": ( + "Use the BAM format instead of CRAM for output aligned " + "files." + ), + "action": "store_true", + }, + "bed": { + "flags": ["-b", "--bed"], + "help": ( + "Region BED file. Supplying this file will limit variant " + "calling to the intervals inside the BED file." + ), + "type": path_arg(exists=True, is_file=True), + }, + "dbsnp": { + "flags": ["-d", "--dbsnp"], + "help": ( + "dbSNP vcf file Supplying this file will annotate " + "variants with their dbSNP refSNP ID numbers." + ), + "type": path_arg(exists=True, is_file=True), + }, + "gvcf": { + "flags": ["-g", "--gvcf"], + "help": ( + "Generate a gVCF output file along with the VCF." + " (default generates only the VCF)" + ), + "action": "store_true", + }, + "lr_align_input": { + "help": ( + "Align the input long-read BAM/CRAM/uBAM file to the " + "reference genome" + ), + "action": "store_true", + }, + "lr_input_ref": { + "help": ( + "Used to decode the input long-read alignment file. " + "Required if the input file is in the CRAM/uCRAM formats." + ), + "type": path_arg(exists=True, is_file=True), + }, + "rgsm": { + "help": ( + "Overwrite the SM tag of the input readgroups for " + "compatibility" + ) + }, + "skip_cnv": { + "help": "Skip CNV calling.", + "action": "store_true", + }, + "skip_metrics": { + "help": "Skip all metrics collection and multiQC", + "action": "store_true", + }, + "skip_mosdepth": { + "help": "Skip QC with mosdepth.", + "action": "store_true", + }, + "skip_multiqc": { + "help": "Skip multiQC report generation.", + "action": "store_true", + }, + "skip_svs": { + "help": "Skip SV calling", + "action": "store_true", + }, + "sr_duplicate_marking": { + "help": "Options for duplicate marking.", + "choices": ["markdup", "rmdup", "none"], + "default": "markdup", + }, + "bwa_args": { + # help="Extra arguments for sentieon bwa", + "help": argparse.SUPPRESS, + "default": "", + }, + "bwa_k": { + # help="The '-K' argument in bwa", + "help": argparse.SUPPRESS, + "default": 100000000, + }, + "bwt_max_mem": { + # Manually set `bwt_max_mem` + "help": argparse.SUPPRESS, + }, + "lr_fastq_taglist": { + # help="A comma-separated list of tags to retain. Defaults to " + # "'%(default)s' and the 'RG' tag is required", + "help": argparse.SUPPRESS, + "default": "*", + }, + "lr_read_filter": { + "help": argparse.SUPPRESS, + }, + "minimap2_args": { + # help="Extra arguments for sentieon minimap2", + "help": argparse.SUPPRESS, + "default": "-Y", + }, + "no_split_alignment": { + "help": argparse.SUPPRESS, + "action": "store_true", + }, + "skip_model_apply": { + "help": argparse.SUPPRESS, + "action": "store_true", + }, + "sr_read_filter": { + "help": argparse.SUPPRESS, + }, + "util_sort_args": { + # help="Extra arguments for sentieon util sort", + "help": argparse.SUPPRESS, + "default": "--cram_write_options version=3.0,compressor=rans", + }, + } + ) def __init__(self) -> None: super().__init__() @@ -444,6 +470,16 @@ def build_dag(self) -> DAG: assert self.reference assert self.model_bundle + rg_info = RgInfo( + self.lr_aln_readgroups, + self.sr_aln_readgroups, + self.lr_read_filter, + self.sr_read_filter, + self.hybrid_set_rg, + self.hybrid_rg_sm, + self.shortread_tech, + ) + # Short-read alignment sr_aln = self.sr_aln ( @@ -486,8 +522,8 @@ def build_dag(self) -> DAG: dag.add_job(job) if not self.skip_mosdepth: - mosdpeth_jobs = self.mosdepth(sample_input=lr_aln) - for job in mosdpeth_jobs: + mosdepth_jobs = self.mosdepth(sample_input=lr_aln) + for job in mosdepth_jobs: dag.add_job(job, realign_jobs) if not self.skip_multiqc: @@ -504,7 +540,9 @@ def build_dag(self) -> DAG: dag.add_job(multiqc_job, multiqc_dependencies) if not self.skip_svs: - longreadsv_job = self.call_svs(lr_aln) + longreadsv_job = self.call_svs( + lr_aln, replace_rg=rg_info.replace_rg_args[0] + ) dag.add_job(longreadsv_job, realign_jobs) sr_preprocessing_jobs: Set[Job] = set() @@ -521,6 +559,11 @@ def build_dag(self) -> DAG: self.bed, self.cores, self.skip_version_check, + replace_rg=( + rg_info.replace_rg_args[1] + if rg_info.replace_rg_args[1] + else None + ), ) dag.add_job(cnvscope_job, sr_preprocessing_jobs) dag.add_job(cnvmodelapply_job, {cnvscope_job}) @@ -545,7 +588,7 @@ def build_dag(self) -> DAG: anno_job, apply_job, norm_job, - ) = self.call_variants(sr_aln, lr_aln) + ) = self.call_variants(sr_aln, lr_aln, rg_info) dag.add_job(call_job, realign_jobs | sr_preprocessing_jobs) dag.add_job(select_job, {call_job}) dag.add_job(mapq0_job, realign_jobs | sr_preprocessing_jobs) @@ -577,6 +620,7 @@ def call_variants( self, sr_aln: List[pathlib.Path], lr_aln: List[pathlib.Path], + rg_info: RgInfo, **_kwargs: Any, ) -> Tuple[ Job, @@ -613,50 +657,6 @@ def call_variants( if not check_version(cmd, min_version): sys.exit(2) - # readgroup handling for long-reads - lr_rg_read_filter: List[str] = [] - replace_rg_args: Tuple[List[List[str]], List[List[str]]] = ([], []) - for aln_rgs in self.lr_aln_readgroups: - replace_rg_args[0].append([]) - for rg_line_d in aln_rgs: - id = rg_line_d.get("ID") - new_sm = ( - self.hybrid_rg_sm - if self.hybrid_set_rg - else rg_line_d.get("SM") - ) - replace_rg_args[0][-1].append( - f"{id}=ID:{id}\\tSM:{new_sm}\\tLR:1" - ) - if self.lr_read_filter: - lr_rg_read_filter.append( - f"{self.lr_read_filter},rgid={id}" - ) - - # readgroup handling for short-reads - sr_rg_read_filter: List[str] = [] - ultima_read_filter: List[str] = [] - for aln_rgs in self.sr_aln_readgroups: - if self.hybrid_set_rg: - replace_rg_args[1].append([]) - for rg_line_d in aln_rgs: - id = rg_line_d.get("ID") - new_sm = ( - self.hybrid_rg_sm - if self.hybrid_set_rg - else rg_line_d.get("SM") - ) - if self.hybrid_set_rg: - replace_rg_args[1][-1].append( - f"{id}=ID:{id}\\tSM:{new_sm}" - ) - if self.sr_read_filter: - sr_rg_read_filter.append( - f"{self.sr_read_filter},rgid={id}" - ) - if self.shortread_tech.upper() == "ULTIMA": - ultima_read_filter.append(f"UltimaReadFilter,rgid={id}") - # First pass - combined variant calling vcf_suffix = ".g.vcf.gz" if self.gvcf else ".vcf.gz" combined_vcf = self.tmp_dir.joinpath("initial" + vcf_suffix) @@ -664,12 +664,12 @@ def call_variants( driver: BaseDriver = Driver( reference=self.reference, thread_count=self.cores, - replace_rg=replace_rg_args[0] + replace_rg_args[1], + replace_rg=rg_info.replace_rg_args[0] + rg_info.replace_rg_args[1], input=lr_aln + sr_aln, interval=self.bed, - read_filter=ultima_read_filter - + lr_rg_read_filter - + sr_rg_read_filter, + read_filter=rg_info.ultima_read_filter + + rg_info.lr_rg_read_filter + + rg_info.sr_rg_read_filter, ) driver.add_algo( DNAscope( @@ -680,7 +680,9 @@ def call_variants( emit_mode=emit_mode, ) ) - call_job = Job(shlex.join(driver.build_cmd()), "calling-1", self.cores) + call_job = Job( + Pipeline(Command(*driver.build_cmd())), "calling-1", self.cores + ) # Region selection hybrid_select = pathlib.Path( @@ -703,9 +705,9 @@ def call_variants( driver = Driver( reference=self.reference, thread_count=self.cores, - replace_rg=replace_rg_args[0] + replace_rg_args[1], + replace_rg=rg_info.replace_rg_args[0] + rg_info.replace_rg_args[1], input=lr_aln + sr_aln, - read_filter=lr_rg_read_filter + sr_rg_read_filter, + read_filter=rg_info.lr_rg_read_filter + rg_info.sr_rg_read_filter, ) driver.add_algo( HybridStage2( @@ -714,7 +716,7 @@ def call_variants( ) ) mapq0_job = Job( - shlex.join(driver.build_cmd()), + Pipeline(Command(*driver.build_cmd())), "mapq0-bed", self.cores, ) @@ -742,16 +744,16 @@ def call_variants( 0, ) rm_cmd = ["rm", str(selected_bed), str(mapq0_slop_bed)] - rm_job1 = Job(shlex.join(rm_cmd), "rm-tmp1", 0, True) + rm_job1 = Job(Pipeline(Command(*rm_cmd, fail_ok=True)), "rm-tmp1", 0) stage1_ins_fa = self.tmp_dir.joinpath("stage1_ins.fa") stage1_ins_bed = self.tmp_dir.joinpath("stage1_ins.bed") ins_driver = Driver( reference=self.reference, thread_count=self.cores, - replace_rg=replace_rg_args[0], + replace_rg=rg_info.replace_rg_args[0], input=lr_aln, - read_filter=lr_rg_read_filter, + read_filter=rg_info.lr_rg_read_filter, ) ins_driver.add_algo( HybridStage1( @@ -768,10 +770,10 @@ def call_variants( stage1_driver = Driver( reference=self.reference, thread_count=self.cores, - replace_rg=replace_rg_args[0], + replace_rg=rg_info.replace_rg_args[0], input=lr_aln, interval=diff_bed, - read_filter=lr_rg_read_filter, + read_filter=rg_info.lr_rg_read_filter, ) stage1_driver.add_algo( HybridStage1( @@ -803,7 +805,7 @@ def call_variants( str(stage1_ins_bed), str(stage1_hap_vcf), ] - rm_job2 = Job(shlex.join(rm_cmd), "rm-tmp2", 0, True) + rm_job2 = Job(Pipeline(Command(*rm_cmd, fail_ok=True)), "rm-tmp2", 0) stage2_bed = self.tmp_dir.joinpath("hybrid_stage2.bed") stage2_unmap_bam = self.tmp_dir.joinpath("hybrid_stage2_unmap.bam") @@ -823,22 +825,22 @@ def call_variants( ) ) second_stage_job = Job( - shlex.join(driver.build_cmd()), + Pipeline(Command(*driver.build_cmd())), "second-stage", self.cores, ) rm_cmd = ["rm", str(stage1_bam), str(stage1_hap_bam)] - rm_job3 = Job(shlex.join(rm_cmd), "rm-tmp3", 0, True) + rm_job3 = Job(Pipeline(Command(*rm_cmd, fail_ok=True)), "rm-tmp3", 0) stage3_bam = self.tmp_dir.joinpath("hybrid_stage3.bam") driver = Driver( reference=self.reference, thread_count=self.cores, - replace_rg=replace_rg_args[0] + replace_rg_args[1], + replace_rg=rg_info.replace_rg_args[0] + rg_info.replace_rg_args[1], input=lr_aln + sr_aln + [stage2_unmap_bam, stage2_alt_bam], interval=stage2_bed, - read_filter=lr_rg_read_filter + sr_rg_read_filter, + read_filter=rg_info.lr_rg_read_filter + rg_info.sr_rg_read_filter, ) driver.add_algo( HybridStage3( @@ -856,17 +858,17 @@ def call_variants( self.cores, ) rm_cmd = ["rm", str(stage2_unmap_bam), str(stage2_alt_bam)] - rm_job4 = Job(shlex.join(rm_cmd), "rm-tmp4", 0, True) + rm_job4 = Job(Pipeline(Command(*rm_cmd, fail_ok=True)), "rm-tmp4", 0) # pass 2 of variant calling pass2_vcf = self.tmp_dir.joinpath("hybrid_pass2.vcf.gz") driver = Driver( reference=self.reference, thread_count=self.cores, - replace_rg=replace_rg_args[0], + replace_rg=rg_info.replace_rg_args[0], input=lr_aln + [stage3_bam], interval=stage2_bed, - read_filter=ultima_read_filter + lr_rg_read_filter, + read_filter=rg_info.ultima_read_filter + rg_info.lr_rg_read_filter, ) driver.add_algo( DNAscope( @@ -877,7 +879,9 @@ def call_variants( emit_mode=emit_mode, ) ) - call2_job = Job(shlex.join(driver.build_cmd()), "call2", self.cores) + call2_job = Job( + Pipeline(Command(*driver.build_cmd())), "call2", self.cores + ) # Merge and normalize the VCFs subset_vcf = self.tmp_dir.joinpath("mix_subset.vcf.gz") @@ -900,7 +904,7 @@ def call_variants( 0, ) rm_cmd = ["rm", str(combined_vcf), str(subset_vcf), str(pass2_vcf)] - rm_job5 = Job(shlex.join(rm_cmd), "rm-tmp5", 0, True) + rm_job5 = Job(Pipeline(Command(*rm_cmd, fail_ok=True)), "rm-tmp5", 0) # Annotate the output VCF hybrid_anno = pathlib.Path( @@ -958,7 +962,7 @@ def call_variants( ) ) apply_job = Job( - shlex.join(driver.build_cmd()), "model-apply", self.cores + Pipeline(Command(*driver.build_cmd())), "model-apply", self.cores ) # Final normalize diff --git a/sentieon_cli/dnascope_longread.py b/sentieon_cli/dnascope_longread.py index 23b7dc4..a0538c6 100644 --- a/sentieon_cli/dnascope_longread.py +++ b/sentieon_cli/dnascope_longread.py @@ -3,12 +3,11 @@ """ import argparse -import multiprocessing as mp +import copy import pathlib -import shlex import shutil import sys -from typing import Any, Dict, List, Optional, Set, Tuple +from typing import Dict, List, Optional, Set, Tuple import packaging.version @@ -28,6 +27,7 @@ ) from .job import Job from .pipeline import BasePipeline +from .shell_pipeline import Command, Pipeline from .util import ( check_version, path_arg, @@ -36,7 +36,7 @@ TOOL_MIN_VERSIONS = { "sentieon driver": packaging.version.Version("202308.01"), - "bcftools": packaging.version.Version("1.10"), + "bcftools": packaging.version.Version("1.22"), "bedtools": None, } @@ -73,168 +73,139 @@ class DNAscopeLRPipeline(BasePipeline): """The DNAscope LongRead pipeline""" - params: Dict[str, Dict[str, Any]] = { - "fastq": { - "nargs": "*", - "help": "Sample fastq files.", - "type": path_arg(exists=True, is_file=True), - }, - "model_bundle": { - "flags": ["-m", "--model_bundle"], - "help": "The model bundle file.", - "required": True, - "type": path_arg(exists=True, is_file=True), - }, - "readgroups": { - "nargs": "*", - "help": "Readgroup information for the fastq files.", - }, - "reference": { - "flags": ["-r", "--reference"], - "required": True, - "help": "fasta for reference genome.", - "type": path_arg(exists=True, is_file=True), - }, - "sample_input": { - "flags": ["-i", "--sample_input"], - "nargs": "*", - "help": "sample BAM or CRAM file.", - "type": path_arg(exists=True, is_file=True), - }, - "align": { - "help": ( - "Align the input BAM/CRAM/uBAM file to the reference genome." - ), - "action": "store_true", - }, - "bam_format": { - "help": ( - "Use the BAM format instead of CRAM for output aligned files." - ), - "action": "store_true", - }, - "bed": { - "flags": ["-b", "--bed"], - "help": ( - "Region BED file. Supplying this file will restrict diploid " - "variant calling to the intervals inside the BED file." - ), - "type": path_arg(exists=True, is_file=True), - }, - "cnv_excluded_regions": { - "help": "Regions to exclude from CNV calling.", - "type": path_arg(exists=True, is_file=True), - }, - "cores": { - "flags": ["-t", "--cores"], - "help": ( - "Number of threads/processes to use. Defaults to all " - "available." - ), - "default": mp.cpu_count(), - }, - "dbsnp": { - "flags": ["-d", "--dbsnp"], - "help": ( - "dbSNP vcf file Supplying this file will annotate variants " - "with their dbSNP refSNP ID numbers." - ), - "type": path_arg(exists=True, is_file=True), - }, - "dry_run": { - "help": "Print the commands without running them.", - "action": "store_true", - }, - "gvcf": { - "flags": ["-g", "--gvcf"], - "help": ( - "Generate a gVCF output file along with the VCF." - " (default generates only the VCF)" - ), - "action": "store_true", - }, - "haploid_bed": { - "help": ( - "A BED file of haploid regions. Supplying this file will " - "perform haploid variant calling across these regions." - ), - "type": path_arg(exists=True, is_file=True), - }, - "input_ref": { - "help": ( - "Used to decode the input alignment file. Required if the " - "input file is in the CRAM/uCRAM formats." - ), - "type": path_arg(exists=True, is_file=True), - }, - "skip_cnv": { - "help": "Skip CNV calling.", - "action": "store_true", - }, - "skip_mosdepth": { - "help": "Skip QC with mosdepth.", - "action": "store_true", - }, - "skip_small_variants": { - "help": "Skip small variant (SNV/indel) calling.", - "action": "store_true", - }, - "skip_svs": { - "help": "Skip SV calling.", - "action": "store_true", - }, - "tech": { - "help": "Sequencing technology used to generate the reads.", - "choices": ["HiFi", "ONT"], - "default": "HiFi", - }, - "fastq_taglist": { - # "help": ( - # "A comma-separated list of tags to retain. Defaults to " - # "'%(default)s' and the 'RG' tag is required." - # ), - "help": argparse.SUPPRESS, - "default": "*", - }, - "minimap2_args": { - # "help": "Extra arguments for sentieon minimap2.", - "help": argparse.SUPPRESS, - "default": "-Y", - }, - "repeat_model": { - "help": argparse.SUPPRESS, - "type": path_arg(exists=True, is_file=True), - }, - "retain_tmpdir": { - "help": argparse.SUPPRESS, - "action": "store_true", - }, - "skip_version_check": { - "help": argparse.SUPPRESS, - "action": "store_true", - }, - "use_pbsv": { - "help": argparse.SUPPRESS, - "action": "store_true", - }, - "util_sort_args": { - # "help": "Extra arguments for sentieon util sort.", - "help": argparse.SUPPRESS, - "default": "--cram_write_options version=3.0,compressor=rans", - }, - } - - positionals: Dict[str, Dict[str, Any]] = { - "output_vcf": { - "help": "Output VCF File. The file name must end in .vcf.gz", - "type": path_arg(), - }, - } + params = copy.deepcopy(BasePipeline.params) + params.update( + { + "fastq": { + "nargs": "*", + "help": "Sample fastq files.", + "type": path_arg(exists=True, is_file=True), + }, + "model_bundle": { + "flags": ["-m", "--model_bundle"], + "help": "The model bundle file.", + "required": True, + "type": path_arg(exists=True, is_file=True), + }, + "readgroups": { + "nargs": "*", + "help": "Readgroup information for the fastq files.", + }, + "sample_input": { + "flags": ["-i", "--sample_input"], + "nargs": "*", + "help": "sample BAM or CRAM file.", + "type": path_arg(exists=True, is_file=True), + }, + "align": { + "help": ( + "Align the input BAM/CRAM/uBAM file to the reference " + "genome." + ), + "action": "store_true", + }, + "bam_format": { + "help": ( + "Use the BAM format instead of CRAM for output aligned " + "files." + ), + "action": "store_true", + }, + "bed": { + "flags": ["-b", "--bed"], + "help": ( + "Region BED file. Supplying this file will restrict " + "diploid variant calling to the intervals inside the BED " + "file." + ), + "type": path_arg(exists=True, is_file=True), + }, + "cnv_excluded_regions": { + "help": "Regions to exclude from CNV calling.", + "type": path_arg(exists=True, is_file=True), + }, + "dbsnp": { + "flags": ["-d", "--dbsnp"], + "help": ( + "dbSNP vcf file Supplying this file will annotate " + "variants with their dbSNP refSNP ID numbers." + ), + "type": path_arg(exists=True, is_file=True), + }, + "gvcf": { + "flags": ["-g", "--gvcf"], + "help": ( + "Generate a gVCF output file along with the VCF." + " (default generates only the VCF)" + ), + "action": "store_true", + }, + "haploid_bed": { + "help": ( + "A BED file of haploid regions. Supplying this file will " + "perform haploid variant calling across these regions." + ), + "type": path_arg(exists=True, is_file=True), + }, + "input_ref": { + "help": ( + "Used to decode the input alignment file. Required if the " + "input file is in the CRAM/uCRAM formats." + ), + "type": path_arg(exists=True, is_file=True), + }, + "skip_cnv": { + "help": "Skip CNV calling.", + "action": "store_true", + }, + "skip_mosdepth": { + "help": "Skip QC with mosdepth.", + "action": "store_true", + }, + "skip_small_variants": { + "help": "Skip small variant (SNV/indel) calling.", + "action": "store_true", + }, + "skip_svs": { + "help": "Skip SV calling.", + "action": "store_true", + }, + "tech": { + "help": "Sequencing technology used to generate the reads.", + "choices": ["HiFi", "ONT"], + "default": "HiFi", + }, + "fastq_taglist": { + # "help": ( + # "A comma-separated list of tags to retain. Defaults to " + # "'%(default)s' and the 'RG' tag is required." + # ), + "help": argparse.SUPPRESS, + "default": "*", + }, + "minimap2_args": { + # "help": "Extra arguments for sentieon minimap2.", + "help": argparse.SUPPRESS, + "default": "-YL", + }, + "repeat_model": { + "help": argparse.SUPPRESS, + "type": path_arg(exists=True, is_file=True), + }, + "use_pbsv": { + "help": argparse.SUPPRESS, + "action": "store_true", + }, + "util_sort_args": { + # "help": "Extra arguments for sentieon util sort.", + "help": argparse.SUPPRESS, + "default": "--cram_write_options version=3.0,compressor=rans", + }, + } + ) def __init__(self) -> None: super().__init__() - self.output_vcf: Optional[pathlib.Path] = None - self.reference: Optional[pathlib.Path] = None self.sample_input: List[pathlib.Path] = [] self.fastq: List[pathlib.Path] = [] self.readgroups: List[str] = [] @@ -245,7 +216,6 @@ def __init__(self) -> None: self.cnv_excluded_regions: Optional[pathlib.Path] = None self.gvcf = False self.tech = "HiFi" - self.dry_run = False self.skip_small_variants = False self.skip_svs = False self.skip_mosdepth = False @@ -254,24 +224,23 @@ def __init__(self) -> None: self.input_ref: Optional[pathlib.Path] = None self.fastq_taglist = "*" self.bam_format = False - self.minimap2_args = "-Y" + self.minimap2_args = "-YL" self.util_sort_args = ( "--cram_write_options version=3.0,compressor=rans" ) self.repeat_model: Optional[pathlib.Path] = None - self.skip_version_check = False - self.retain_tmpdir = False self.use_pbsv = False def validate(self) -> None: # uniquify pipeline attributes self.lr_aln = self.sample_input + self.lr_input_ref = self.input_ref del self.sample_input # validate if not (self.lr_aln or self.fastq): self.logger.error( - "Please suppy either the `--sample_input` or `--fastq` and " + "Please supply either the `--sample_input` or `--fastq` and " "`--readgroups` arguments" ) sys.exit(2) @@ -332,8 +301,8 @@ def build_dag(self) -> DAG: dag.add_job(job) if not self.skip_mosdepth: - mosdpeth_jobs = self.mosdepth(sample_input) - for job in mosdpeth_jobs: + mosdepth_jobs = self.mosdepth(sample_input) + for job in mosdepth_jobs: dag.add_job(job, realign_jobs.union(align_jobs)) if self.use_pbsv or not self.skip_cnv: @@ -347,7 +316,8 @@ def build_dag(self) -> DAG: if not self.skip_cnv: hificnv_job = self.hificnv(merged_bam) - dag.add_job(hificnv_job, {merge_job}) + if hificnv_job: + dag.add_job(hificnv_job, {merge_job}) if not self.skip_small_variants: ( @@ -486,7 +456,7 @@ def lr_align_inputs(self) -> Tuple[List[pathlib.Path], Set[Job]]: self.cores, rg_lines, sample_name, - self.input_ref, + self.lr_input_ref, self.fastq_taglist, self.minimap2_args, self.util_sort_args, @@ -566,12 +536,12 @@ def mosdepth(self, sample_input: List[pathlib.Path]) -> Set[Job]: ) return set() - mosdpeth_jobs = set() + mosdepth_jobs = set() for i, input_file in enumerate(sample_input): mosdepth_dir = pathlib.Path( str(self.output_vcf).replace(".vcf.gz", f"_mosdepth_{i}") ) - mosdpeth_jobs.add( + mosdepth_jobs.add( Job( cmds.cmd_mosdepth( input_file, @@ -583,7 +553,7 @@ def mosdepth(self, sample_input: List[pathlib.Path]) -> Set[Job]: 0, # Run in background ) ) - return mosdpeth_jobs + return mosdepth_jobs def merge_input_files( self, sample_input: List[pathlib.Path] @@ -605,7 +575,7 @@ def merge_input_files( ) driver.add_algo(ReadWriter(merged_bam)) merge_job = Job( - shlex.join(driver.build_cmd()), + Pipeline(Command(*driver.build_cmd())), "merge-bam", 0, ) @@ -629,19 +599,32 @@ def pbsv( ".pbsv_discover.svsig.gz", ) pbsv_discover = Job( - f"pbsv discover --hifi {merged_bam} {pbsv_discover_fn}", + Pipeline( + Command( + "pbsv", + "discover", + "--hifi", + str(merged_bam), + pbsv_discover_fn, + ) + ), "pbsv-discover", 0, ) # pbsv call pbsv_call_fn = str(self.output_vcf).replace(".vcf.gz", ".pbsv.vcf") - cmd = ( - f"pbsv call -j {self.cores} {self.reference} {pbsv_discover_fn} " - f"{pbsv_call_fn}" - ) + call_cmd = [ + "pbsv", + "call", + "-j", + str(self.cores), + str(self.reference), + pbsv_discover_fn, + pbsv_call_fn, + ] pbsv_call = Job( - cmd, + Pipeline(Command(*call_cmd)), "pbsv-call", self.cores, ) @@ -650,24 +633,37 @@ def pbsv( def hificnv( self, merged_bam: pathlib.Path, - ) -> Job: + ) -> Optional[Job]: """ Call CNVs with HiFiCNV """ if not self.skip_version_check: for cmd, min_version in HIFICNV_MIN_VERSIONS.items(): if not check_version(cmd, min_version): - sys.exit(2) + self.logger.warning( + "Skipping hificnv. hificnv version %s or later not " + "found", + HIFICNV_MIN_VERSIONS["hificnv"], + ) + return None hifi_cnv_fn = str(self.output_vcf).replace(".vcf.gz", ".hificnv") - cmd = ( - f"hificnv --bam {merged_bam} --ref {self.reference} " - f"--threads {self.cores} " - f"--output-prefix {hifi_cnv_fn}" - ) + hificnv_cmd = [ + "hificnv", + "--bam", + str(merged_bam), + "--ref", + str(self.reference), + "--threads", + str(self.cores), + "--output-prefix", + hifi_cnv_fn, + ] if self.cnv_excluded_regions: - cmd += f" --exclude {self.cnv_excluded_regions}" - hificnv_job = Job(cmd, "hificnv", self.cores) + hificnv_cmd.extend(["--exclude", str(self.cnv_excluded_regions)]) + hificnv_job = Job( + Pipeline(Command(*hificnv_cmd)), "hificnv", self.cores + ) return hificnv_job def lr_call_variants( @@ -703,8 +699,8 @@ def lr_call_variants( assert self.reference assert self.output_vcf if not self.skip_version_check: - for cmd, min_version in TOOL_MIN_VERSIONS.items(): - if not check_version(cmd, min_version): + for check_cmd, min_version in TOOL_MIN_VERSIONS.items(): + if not check_version(check_cmd, min_version): sys.exit(2) # First pass - diploid calling @@ -733,7 +729,7 @@ def lr_call_variants( ) ) first_calling_job = Job( - shlex.join(driver.build_cmd()), "first-pass", self.cores + Pipeline(Command(*driver.build_cmd())), "first-pass", self.cores ) diploid_vcf = self.tmp_dir.joinpath("out_diploid.vcf.gz") @@ -749,7 +745,9 @@ def lr_call_variants( ) ) first_modelapply_job = Job( - shlex.join(driver.build_cmd()), "first-modelapply", self.cores + Pipeline(Command(*driver.build_cmd())), + "first-modelapply", + self.cores, ) # Phasing and RepeatModel @@ -778,14 +776,28 @@ def lr_call_variants( ) ) phaser_job = Job( - shlex.join(driver.build_cmd()), "variantphaser", self.cores + Pipeline(Command(*driver.build_cmd())), "variantphaser", self.cores ) bcftools_subset_phased_job = None if self.tech.upper() == "ONT": bcftools_subset_phased_job = Job( - f"bcftools view -T {phased_bed} {phased_vcf} \ - | sentieon util vcfconvert - {phased_phased}", + Pipeline( + Command( + "bcftools", + "view", + "-T", + str(phased_bed), + str(phased_vcf), + ), + Command( + "sentieon", + "util", + "vcfconvert", + "-", + str(phased_phased), + ), + ), "bcftools-subset-phased", 0, ) @@ -831,12 +843,24 @@ def lr_call_variants( ) ) repeatmodel_job = Job( - shlex.join(driver.build_cmd()), "repeatmodel", self.cores + Pipeline(Command(*driver.build_cmd())), + "repeatmodel", + self.cores, ) bcftools_subset_unphased_job = Job( - f"bcftools view -T {unphased_bed} {phased_vcf} \ - | sentieon util vcfconvert - {phased_unphased}", + Pipeline( + Command( + "bcftools", + "view", + "-T", + str(unphased_bed), + str(phased_vcf), + ), + Command( + "sentieon", "util", "vcfconvert", "-", str(phased_unphased) + ), + ), "bcftools-subset-unphased", 0, ) @@ -877,7 +901,11 @@ def lr_call_variants( ) ) second_calling_job.add( - Job(shlex.join(driver.build_cmd()), "second-pass", self.cores) + Job( + Pipeline(Command(*driver.build_cmd())), + "second-pass", + self.cores, + ) ) kwargs: Dict[str, str] = dict() @@ -924,7 +952,7 @@ def lr_call_variants( ) second_modelapply_job.add( Job( - shlex.join(driver.build_cmd()), + Pipeline(Command(*driver.build_cmd())), "second-modelapply", self.cores, ) @@ -949,7 +977,9 @@ def lr_call_variants( ) ) calling_unphased_job = Job( - shlex.join(driver.build_cmd()), "calling-unphased", self.cores + Pipeline(Command(*driver.build_cmd())), + "calling-unphased", + self.cores, ) # Patch DNA and DNAHP variants @@ -977,7 +1007,9 @@ def lr_call_variants( ) ) modelapply_unphased_job = Job( - shlex.join(driver.build_cmd()), "modelapply-unphased", self.cores + Pipeline(Command(*driver.build_cmd())), + "modelapply-unphased", + self.cores, ) # merge calls to create the output @@ -1057,7 +1089,9 @@ def lr_call_variants( ) ) haploid_calling_job = Job( - shlex.join(driver.build_cmd()), "haploid-calling", self.cores + Pipeline(Command(*driver.build_cmd())), + "haploid-calling", + self.cores, ) haploid_patch2_job = Job( @@ -1136,6 +1170,7 @@ def lr_call_variants( def call_svs( self, sample_input: List[pathlib.Path], + replace_rg: Optional[List[List[str]]] = None, ) -> Job: """ Call SVs using Sentieon LongReadSV @@ -1152,6 +1187,7 @@ def call_svs( driver = Driver( reference=self.reference, thread_count=self.cores, + replace_rg=replace_rg, input=sample_input, interval=self.bed, ) @@ -1162,6 +1198,6 @@ def call_svs( ) ) longreadsv_job = Job( - shlex.join(driver.build_cmd()), "LongReadSV", self.cores + Pipeline(Command(*driver.build_cmd())), "LongReadSV", self.cores ) return longreadsv_job diff --git a/sentieon_cli/executor.py b/sentieon_cli/executor.py index cd92746..862fb7e 100644 --- a/sentieon_cli/executor.py +++ b/sentieon_cli/executor.py @@ -10,6 +10,7 @@ from .job import Job from .logging import get_logger from .scheduler import ThreadScheduler +from .shell_pipeline import Context logger = get_logger(__name__) @@ -59,9 +60,8 @@ def __init__(self, *args: Any, **kwargs: Any): self.running: List[ Tuple[ Job, - asyncio.subprocess.Process, + Context, asyncio.Task[int], - str, int, ] ] = [] @@ -72,34 +72,41 @@ def exit_gracefully(self, signum, frame) -> None: logger.error("Termination signal detected. Terminating jobs.") self.start_new_jobs = False for running_job in self.running: - _job, proc, _task, _cmd, _start_time = running_job - proc.send_signal(signal=signal.SIGTERM) + for subcommand in running_job[1].commands: + assert subcommand.proc + subcommand.proc.send_signal(signal=signal.SIGTERM) time.sleep(10) for running_job in self.running: - _job, proc, _task, _cmd, _start_time = running_job - if proc.returncode != 0: - proc.kill() + context = running_job[1] + for subcommand in context.commands: + assert subcommand.proc + if subcommand.proc.returncode is None: + subcommand.proc.kill() + asyncio.run(context.cleanup()) async def run_job(self, job: Job) -> None: """Run a job""" cmd = job.shell logger.info("Running: %s", cmd) + context = Context() start_time = time.monotonic_ns() - proc = await asyncio.create_subprocess_shell( - cmd, - stdout=sys.stdout, - stderr=sys.stderr, - executable="/bin/bash", - ) - self.running.append( - ( - job, - proc, - asyncio.create_task(proc.wait()), - cmd, - start_time, + try: + proc = await cmd.run( + context, + stderr=sys.stderr, ) - ) + self.running.append( + ( + job, + context, + asyncio.create_task(proc.wait()), + start_time, + ) + ) + except Exception as e: + logger.error("failed to start command: %s", job.shell) + logger.error("Error: %s", str(e)) + await context.cleanup() def execute(self) -> None: """Execute jobs from the DAG""" @@ -126,18 +133,38 @@ async def _execute(self) -> None: ] # Check job execution - for job, proc, _task, cmd, start_time in finished_jobs: + for job, context, _task, start_time in finished_jobs: end_time = time.monotonic_ns() total_seconds = (end_time - start_time) / 1e9 - if proc.returncode != 0 and not job.fail_ok: - logger.error("Error running command, '%s'", job.shell) + + # Check if the command failed + cmd_failed = False + for subcommand in context.commands: + assert subcommand.proc + ret = ( + await subcommand.proc.wait() + ) # Wait on all sub-commands + if ret != 0 and not subcommand.fail_ok: + logger.error( + f"Return code {ret} for sub-command: {subcommand}" + ) + cmd_failed = True + if ret == -9: + logger.error( + "Sub-command received SIGKILL. " + "Possible out-of-memory?" + ) + + if cmd_failed: + logger.error("Command failure: %s", job.shell) self.jobs_with_errors.append(job) self.start_new_jobs = False else: logger.info( - f"Finished job with status {proc.returncode} in " - f"{total_seconds:.2f}: {cmd}" + f"Finished command in " + f"{total_seconds:.2f}: {job.shell}" ) + await context.cleanup() if not self.start_new_jobs: # Don't start new jobs diff --git a/sentieon_cli/job.py b/sentieon_cli/job.py index 270095b..971a94f 100644 --- a/sentieon_cli/job.py +++ b/sentieon_cli/job.py @@ -2,12 +2,13 @@ Job objects """ -import subprocess as sp +import asyncio import sys import time from typing import Dict, Optional from .logging import get_logger +from .shell_pipeline import Context, Pipeline logger = get_logger(__name__) @@ -17,16 +18,14 @@ class Job: def __init__( self, - shell: str, + pipeline: Pipeline, name: str, threads: int = 1, - fail_ok: bool = False, resources: Optional[Dict[str, int]] = None, ): - self.shell = shell + self.shell = pipeline self.name = name self.threads = threads - self.fail_ok = fail_ok self.resources = {} if resources is None else resources def __hash__(self): @@ -54,12 +53,20 @@ def run(self, dry_run: bool = False): logger.info("running command: %s", self.shell) t0 = time.time() - sp.run( - self.shell, - shell=True, - check=False, - stdout=sys.stdout, - stderr=sys.stderr, - executable="/bin/bash", - ) + context = Context() + try: + asyncio.run( + self.shell.run(context, stdout=sys.stdout, stderr=sys.stderr) + ) + for subcommand in context.commands: + assert subcommand.proc + ret = asyncio.run(subcommand.proc.wait()) + if ret != 0 and not subcommand.fail_ok: + logger.error( + f"subcommand failed with code {ret}: {subcommand}" + ) + except Exception as e: + logger.error(f"Failed to run command: {e}") + finally: + asyncio.run(context.cleanup()) logger.info("finished in: %s seconds", f"{time.time() - t0:.1f}") diff --git a/sentieon_cli/pangenome.py b/sentieon_cli/pangenome.py index f99c039..5cba3f7 100644 --- a/sentieon_cli/pangenome.py +++ b/sentieon_cli/pangenome.py @@ -3,16 +3,14 @@ """ import argparse -from enum import Enum +import copy import itertools import json -import multiprocessing as mp import os import pathlib -import shlex import shutil import sys -from typing import Any, Dict, List, Optional, Tuple +from typing import List, Optional, Tuple import packaging.version @@ -20,6 +18,7 @@ from . import command_strings as cmds from .archive import ar_load +from .base_pangenome import BasePangenome, SampleSex from .dag import DAG from .driver import ( AlignmentStat, @@ -39,7 +38,7 @@ WgsMetricsAlgo, ) from .job import Job -from .pipeline import BasePipeline +from .shell_pipeline import Command, Pipeline from .util import __version__, check_version, parse_rg_line, path_arg, tmp @@ -47,15 +46,20 @@ "kmc": None, "sentieon driver": packaging.version.Version("202503.01"), "vg": None, - "bcftools": packaging.version.Version("1.10"), + "bcftools": packaging.version.Version("1.22"), "samtools": packaging.version.Version("1.16"), +} + +T1K_MIN_VERSION = { "run-t1k": None, +} + +EXPANSION_MIN_VERSION = { "ExpansionHunter": None, - "segdup-caller": None, } -MULTIQC_MIN_VERSION = { - "multiqc": packaging.version.Version("1.18"), +SEGDUP_MIN_VERSION = { + "segdup-caller": None, } @@ -164,184 +168,100 @@ } -class SampleSex(Enum): - FEMALE = 1 - MALE = 2 - UNKNOWN = 3 - - -class PangenomePipeline(BasePipeline): +class PangenomePipeline(BasePangenome): """The Pangenome pipeline""" - params: Dict[str, Dict[str, Any]] = { - # Required arguments - "reference": { - "flags": ["-r", "--reference"], - "required": True, - "help": "fasta for reference genome.", - "type": path_arg(exists=True, is_file=True), - }, - "gbz": { - "help": "The pangenome graph file in GBZ format.", - "required": True, - "type": path_arg(exists=True, is_file=True), - }, - "hapl": { - "help": "The haplotype file.", - "required": True, - "type": path_arg(exists=True, is_file=True), - }, - "snarls": { - "help": ( - "The vg snarls file for the .gbz file. Can be created " - "using vg." - ), - "required": True, - "type": path_arg(exists=True, is_file=True), - }, - "xg": { - "help": "The xg file for the .gbz file. Can be created using vg", - "required": True, - "type": path_arg(exists=True, is_file=True), - }, - "model_bundle": { - "flags": ["-m", "--model_bundle"], - "help": "The model bundle file.", - "required": True, - "type": path_arg(exists=True, is_file=True), - }, - "r1_fastq": { - "nargs": "*", - "help": "Sample R1 fastq files.", - "type": path_arg(exists=True, is_file=True), - }, - "r2_fastq": { - "nargs": "*", - "help": "Sample R2 fastq files.", - "type": path_arg(exists=True, is_file=True), - }, - "readgroups": { - "nargs": "*", - "help": ( - "Readgroup information for the fastq files. Only the ID and " - "SM attributes are used." - ), - }, - # Additional arguments - "bam_format": { - "help": ( - "Use the BAM format instead of CRAM for output aligned files." - ), - "action": "store_true", - }, - "cores": { - "flags": ["-t", "--cores"], - "help": ( - "Number of threads/processes to use. Defaults to all " - "available." - ), - "default": mp.cpu_count(), - }, - "dbsnp": { - "flags": ["-d", "--dbsnp"], - "help": ( - "dbSNP vcf file Supplying this file will annotate variants " - "with their dbSNP refSNP ID numbers." - ), - "type": path_arg(exists=True, is_file=True), - }, - "dry_run": { - "help": "Print the commands without running them.", - "action": "store_true", - }, - "expansion_catalog": { - "help": ( - "An ExpansionHunter variant catalog. Required for expansion " - "calling." - ), - "type": path_arg(exists=True, is_file=True), - }, - "kmer_memory": { - "help": "Memory limit for KMC in GB.", - "default": 58, - "type": int, - }, - "segdup_caller_genes": { - "type": str, - "help": ( - "Genes for SegDup calling. Ex: " - "'CFH,CFHR3,CYP11B1,CYP2D6,GBA,NCF1,PMS2,SMN1,STRC'. Required " - "for SegDup calling." - ), - }, - "t1k_hla_seq": { - "help": ( - "The DNA HLA seq FASTA file for T1K. Required for HLA calling." - ), - "type": path_arg(exists=True, is_file=True), - }, - "t1k_hla_coord": { - "help": ( - "The DNA HLA coord FASTA file for T1K. Required for HLA " - "calling." - ), - "type": path_arg(exists=True, is_file=True), - }, - "t1k_kir_seq": { - "help": ( - "The DNA KIR seq FASTA file for T1K. Required for KIA calling." - ), - "type": path_arg(exists=True, is_file=True), - }, - "t1k_kir_coord": { - "help": ( - "The DNA KIR coord FASTA file for T1K. Required for KIA " - "calling." - ), - "type": path_arg(exists=True, is_file=True), - }, - # Hidden arguments - "retain_tmpdir": { - "help": argparse.SUPPRESS, - "action": "store_true", - }, - "skip_version_check": { - "help": argparse.SUPPRESS, - "action": "store_true", - }, - } + params = copy.deepcopy(BasePangenome.params) + params.update( + { + "readgroups": { + "nargs": "*", + "help": ( + "Readgroup information for the fastq files. Only the ID " + "and SM attributes are used." + ), + }, + # Required arguments + "snarls": { + "help": ( + "The vg snarls file for the .gbz file. Can be created " + "using vg." + ), + "required": True, + "type": path_arg(exists=True, is_file=True), + }, + "xg": { + "help": ( + "The xg file for the .gbz file. Can be created using vg" + ), + "required": True, + "type": path_arg(exists=True, is_file=True), + }, + # Additional arguments + "expansion_catalog": { + "help": ( + "An ExpansionHunter variant catalog. Required for " + "expansion calling." + ), + "type": path_arg(exists=True, is_file=True), + }, + "segdup_caller_genes": { + "type": str, + "help": ( + "Genes for SegDup calling. Ex: " + "'CFH,CFHR3,CYP11B1,CYP2D6,GBA,NCF1,PMS2,SMN1,STRC'. " + "Required for SegDup calling." + ), + }, + "skip_cnv": { + "help": "Skip CNV calling.", + "action": "store_true", + }, + "t1k_hla_seq": { + "help": ( + "The DNA HLA seq FASTA file for T1K. Required for HLA " + "calling." + ), + "type": path_arg(exists=True, is_file=True), + }, + "t1k_hla_coord": { + "help": ( + "The DNA HLA coord FASTA file for T1K. Required for HLA " + "calling." + ), + "type": path_arg(exists=True, is_file=True), + }, + "t1k_kir_seq": { + "help": ( + "The DNA KIR seq FASTA file for T1K. Required for KIA " + "calling." + ), + "type": path_arg(exists=True, is_file=True), + }, + "t1k_kir_coord": { + "help": ( + "The DNA KIR coord FASTA file for T1K. Required for KIA " + "calling." + ), + "type": path_arg(exists=True, is_file=True), + }, + } + ) - positionals: Dict[str, Dict[str, Any]] = { - "output_vcf": { - "help": "Output VCF file. The file name must end in .vcf.gz", - "type": path_arg(), - }, - } + positionals = BasePangenome.positionals def __init__(self) -> None: super().__init__() - self.output_vcf: Optional[pathlib.Path] = None - self.reference: Optional[pathlib.Path] = None - self.gbz: Optional[pathlib.Path] = None - self.hapl: Optional[pathlib.Path] = None + self.readgroups: List[str] = [] self.snarls: Optional[pathlib.Path] = None self.xg: Optional[pathlib.Path] = None - self.r1_fastq: List[pathlib.Path] = [] - self.r2_fastq: List[pathlib.Path] = [] - self.readgroups: List[str] = [] - self.model_bundle: Optional[pathlib.Path] = None - self.bam_format = False - self.dbsnp: Optional[pathlib.Path] = None - self.cores = mp.cpu_count() self.expansion_catalog: Optional[pathlib.Path] = None - self.kmer_memory = 128 self.segdup_caller_genes: Optional[str] = None + self.skip_cnv = False self.t1k_hla_seq: Optional[pathlib.Path] = None self.t1k_hla_coord: Optional[pathlib.Path] = None self.t1k_kir_seq: Optional[pathlib.Path] = None self.t1k_kir_coord: Optional[pathlib.Path] = None - self.retain_tmpdir = False - self.skip_version_check = False def main(self, args: argparse.Namespace) -> None: """Run the pipeline""" @@ -356,8 +276,8 @@ def main(self, args: argparse.Namespace) -> None: executor = self.run(dag) self.check_execution(dag, executor) - self.get_sex() - dag = self.build_second_dag() + self.get_sex(self.ploidy_json) + dag = self.build_second_dag(self.deduped_cram) executor = self.run(dag) self.check_execution(dag, executor) @@ -367,69 +287,7 @@ def main(self, args: argparse.Namespace) -> None: def build_dag(self) -> DAG: return DAG() - def validate(self) -> None: - """Validate pipeline inputs""" - assert self.output_vcf - assert self.reference - - self.validate_bundle() - - if not self.r1_fastq: - self.logger.error("Please supply --r1_fastq arguments") - sys.exit(2) - - if not str(self.output_vcf).endswith(".vcf.gz"): - self.logger.error("The output file should end with '.vcf.gz'") - sys.exit(2) - - if len(self.r1_fastq) != len(self.readgroups): - self.logger.error( - "The number of readgroups does not equal the number of fastq " - "files" - ) - sys.exit(2) - - # Confirm the presence of the reference index file - fai_file = str(self.reference) + ".fai" - if not os.path.isfile(fai_file): - self.logger.error( - "Fasta index file %s does not exist. Please index the " - "reference genome with 'samtools faidx'", - fai_file, - ) - sys.exit(2) - - # Validate readgroups - rg_sample = None - for rg in self.readgroups: - rg_dict = parse_rg_line(rg.replace(r"\t", "\t")) - rg_sm = rg_dict.get("SM") - if not rg_sm: - self.logger.error( - "Found a readgroup without a SM tag: %s", - str(rg), - ) - sys.exit(2) - if rg_sample and rg_sample != rg_sm: - self.logger.error( - "Inconsistent readgroup sample information found in: %s", - str(rg), - ) - sys.exit(2) - rg_sample = rg_sm - if "ID" not in rg_dict: - self.logger.error( - "Found a readgroup without an ID tag: %s", - str(rg), - ) - sys.exit(2) - - if not self.skip_version_check: - for cmd, min_version in PANGENOME_MIN_VERSIONS.items(): - if not check_version(cmd, min_version): - sys.exit(2) - - # Check the T1K files + def validate_t1k(self) -> None: if (self.t1k_hla_seq and not self.t1k_hla_coord) or ( self.t1k_hla_coord and not self.t1k_hla_seq ): @@ -448,6 +306,43 @@ def validate(self) -> None: ) sys.exit(2) + if ( + self.t1k_hla_seq or self.t1k_kir_seq + ) and not self.skip_version_check: + for cmd, min_version in T1K_MIN_VERSION.items(): + if not check_version(cmd, min_version): + sys.exit(2) + + def validate_expansion(self) -> None: + if self.expansion_catalog and not self.skip_version_check: + for cmd, min_version in EXPANSION_MIN_VERSION.items(): + if not check_version(cmd, min_version): + sys.exit(2) + + def validate_segdup(self) -> None: + if self.segdup_caller_genes and not self.skip_version_check: + for cmd, min_version in SEGDUP_MIN_VERSION.items(): + if not check_version(cmd, min_version): + sys.exit(2) + + def validate(self) -> None: + """Validate pipeline inputs""" + assert self.output_vcf + assert self.reference + + self.validate_bundle() + self.validate_fastq_rg(r1_required=True) + self.validate_output_vcf() + self.validate_ref() + self.validate_t1k() + self.validate_expansion() + self.validate_segdup() + + if not self.skip_version_check: + for cmd, min_version in PANGENOME_MIN_VERSIONS.items(): + if not check_version(cmd, min_version): + sys.exit(2) + def validate_bundle(self) -> None: bundle_info_bytes = ar_load( str(self.model_bundle) + "/bundle_info.json" @@ -488,6 +383,43 @@ def validate_bundle(self) -> None: ) sys.exit(2) + def validate_fastq_rg(self, r1_required=False) -> None: + if not self.r1_fastq and r1_required: + self.logger.error("Please supply --r1_fastq arguments") + sys.exit(2) + + if len(self.r1_fastq) != len(self.readgroups): + self.logger.error( + "The number of readgroups does not equal the number of fastq " + "files" + ) + sys.exit(2) + + # Validate readgroups + rg_sample = None + for rg in self.readgroups: + rg_dict = parse_rg_line(rg.replace(r"\t", "\t")) + rg_sm = rg_dict.get("SM") + if not rg_sm: + self.logger.error( + "Found a readgroup without a SM tag: %s", + str(rg), + ) + sys.exit(2) + if rg_sample and rg_sample != rg_sm: + self.logger.error( + "Inconsistent readgroup sample information found in: %s", + str(rg), + ) + sys.exit(2) + rg_sample = rg_sm + if "ID" not in rg_dict: + self.logger.error( + "Found a readgroup without an ID tag: %s", + str(rg), + ) + sys.exit(2) + def configure(self) -> None: """Configure pipeline parameters""" pass @@ -538,7 +470,7 @@ def build_first_dag(self) -> DAG: self.create_sv_header(sv_header, sample_name) # KMC k-mer counting - kmc_job = self.build_kmc_job(kmer_prefix) + kmc_job = self.build_kmc_job(kmer_prefix, self.cores) dag.add_job(kmc_job) # vg haplotypes - create sample-specific pangenome @@ -706,40 +638,6 @@ def create_sv_header(self, sv_header: pathlib.Path, sample_name: str): ] print("\t".join(last_hdr_line), file=fh) - def build_kmc_job(self, kmer_prefix: pathlib.Path) -> Job: - """Build KMC k-mer counting jobs""" - # Create file list for KMC - file_list = pathlib.Path(str(kmer_prefix) + ".paths") - all_fastqs = [] - - # Add R1 files - all_fastqs.extend(self.r1_fastq) - - # Add R2 files if present - if self.r2_fastq: - all_fastqs.extend(self.r2_fastq) - - # Write file list - if not self.dry_run: - with open(file_list, "w") as f: - for fq in all_fastqs: - f.write(f"{fq}\n") - - # Create KMC job - kmc_job = Job( - cmds.cmd_kmc( - kmer_prefix, - file_list, - self.tmp_dir, - memory=self.kmer_memory, - threads=self.cores, - ), - "kmc", - self.cores, - ) - - return kmc_job - def build_haplotypes_job( self, output_gbz: pathlib.Path, kmer_file: pathlib.Path ) -> Job: @@ -929,7 +827,7 @@ def build_dedup_job( driver.add_algo(GCBias(gc_metrics, summary=gc_summary)) lc_job = Job( - shlex.join(driver.build_cmd()), + Pipeline(Command(*driver.build_cmd())), "locuscollector", self.cores, ) @@ -949,7 +847,9 @@ def build_dedup_job( metrics=dedup_metrics, ) ) - dedup_job = Job(shlex.join(driver.build_cmd()), "dedup", self.cores) + dedup_job = Job( + Pipeline(Command(*driver.build_cmd())), "dedup", self.cores + ) return (hdr_jobs, lc_job, dedup_job) @@ -970,7 +870,6 @@ def build_metrics_job( # - Run some metrics after duplicate marking is_metrics = metrics_dir.joinpath(metric_base + ".insert_size.txt") wgs_metrics = metrics_dir.joinpath(metric_base + ".wgs.txt") - wgs_metrics_tmp = metrics_dir.joinpath(metric_base + ".wgs.txt.tmp") driver = Driver( reference=self.reference, @@ -981,48 +880,31 @@ def build_metrics_job( driver.add_algo(InsertSizeMetricAlgo(is_metrics)) driver.add_algo(WgsMetricsAlgo(wgs_metrics, include_unpaired="true")) metrics_job = Job( - shlex.join(driver.build_cmd()), + Pipeline(Command(*driver.build_cmd())), "metrics", self.cores, ) # Rehead WGS metrics so they are recognized by MultiQC - rehead_job = Job( - cmds.rehead_wgsmetrics(wgs_metrics, wgs_metrics_tmp), - "Rehead metrics", - 0, + rehead_script = pathlib.Path( + str( + files("sentieon_cli.scripts").joinpath("rehead_wgs_metrics.py") + ) ) - return (metrics_job, rehead_job) - - def multiqc(self) -> Optional[Job]: - """Run MultiQC on the metrics files""" - assert self.output_vcf - if not self.skip_version_check: - if not all( - [ - check_version(cmd, min_version) - for (cmd, min_version) in MULTIQC_MIN_VERSION.items() - ] - ): - self.logger.warning( - "Skipping MultiQC. MultiQC version %s or later not found", - MULTIQC_MIN_VERSION["multiqc"], + rehead_job = Job( + Pipeline( + Command( + "sentieon", + "pyexec", + str(rehead_script), + "--metrics_file", + str(wgs_metrics), ) - return None - - metrics_dir = pathlib.Path( - str(self.output_vcf).replace(".vcf.gz", "_metrics") - ) - multiqc_job = Job( - cmds.cmd_multiqc( - metrics_dir, - metrics_dir, - f"Generated by the Sentieon-CLI version {__version__}", ), - "multiqc", + "Rehead metrics", 0, ) - return multiqc_job + return (metrics_job, rehead_job) def build_smallvar_job( self, @@ -1038,6 +920,7 @@ def build_smallvar_job( interval_ctgs.extend(["chrX", "chrY"]) interval = ",".join(interval_ctgs) + pcr_indel_model = "NONE" if self.pcr_free else "CONSERVATIVE" driver = Driver( reference=self.reference, thread_count=self.cores, @@ -1048,12 +931,12 @@ def build_smallvar_job( DNAscope( dnascope_tmp, dbsnp=self.dbsnp, - pcr_indel_model="NONE", + pcr_indel_model=pcr_indel_model, model=model, ) ) dnascope_job = Job( - shlex.join(driver.build_cmd()), "dnascope", self.cores + Pipeline(Command(*driver.build_cmd())), "dnascope", self.cores ) driver = Driver( @@ -1068,94 +951,11 @@ def build_smallvar_job( ) ) dnamodelapply_job = Job( - shlex.join(driver.build_cmd()), "model-apply", self.cores + Pipeline(Command(*driver.build_cmd())), "model-apply", self.cores ) - return (dnascope_job, dnamodelapply_job) - def build_ploidy_job( - self, - ploidy_json: pathlib.Path, - deduped_bam: List[pathlib.Path], - ) -> Job: - """Estimate sample ploidy and sex""" - estimate_ploidy = pathlib.Path( - str(files("sentieon_cli.scripts").joinpath("estimate_ploidy.py")) - ).resolve() - ploidy_job = Job( - cmds.cmd_estimate_ploidy( - ploidy_json, - deduped_bam, - estimate_ploidy, - ), - "estimate-ploidy", - 0, - ) - return ploidy_job - - def get_sex(self) -> None: - """Retrieve the sample sex""" - if self.dry_run: - self.logger.info("Setting sample sex to MALE for dry-run") - self.sample_sex = SampleSex.MALE - return - with open(self.ploidy_json) as fh: - data = json.load(fh) - sex = data["sex"] - if sex == "female": - self.sample_sex = SampleSex.FEMALE - elif sex == "male": - self.sample_sex = SampleSex.MALE - else: - self.sample_sex = SampleSex.UNKNOWN - - def build_t1k_job( - self, - out_basename: pathlib.Path, - deduped_bam: pathlib.Path, - gene_seq: pathlib.Path, - gene_coord: pathlib.Path, - preset: str, - ) -> Job: - """Genotype diverse sequences with T1K""" - job = Job( - cmds.cmd_t1k( - out_basename, - deduped_bam, - gene_seq=gene_seq, - gene_coord=gene_coord, - preset=preset, - threads=1, - ), - "T1K-HLA-KIR", - 0, - ) - return job - - def build_segdup_job( - self, - out_segdup: pathlib.Path, - realigned_cram: pathlib.Path, - genes: str, - ) -> Job: - """Call variants in difficult SegDups""" - assert self.reference - assert self.model_bundle - - segdup_job = Job( - cmds.cmd_segdup_caller( - out_segdup, - realigned_cram, - reference=self.reference, - sr_bundle=self.model_bundle, - genes=genes, - ), - "segdup-caller", - self.cores, - ) - return segdup_job - - def build_second_dag(self) -> DAG: + def build_second_dag(self, input_aln: pathlib.Path) -> DAG: """Build the second DAG for the pangenome pipeline""" self.logger.info("Building the second pangenome DAG") dag = DAG() @@ -1181,32 +981,33 @@ def build_second_dag(self) -> DAG: ) # CNV calling - ( - cnvscope_job, - cnvapply_job, - haploid_cnv_job, - haploid_apply_job, - concat_job, - ) = self.build_cnv_jobs( - out_cnv, - cnvscope_tmp, - cnvscope_autosomes, - cnvscope_haploid_tmp, - cnvscope_haploid, - self.deduped_cram, - ) - dag.add_job(cnvscope_job) - dag.add_job(cnvapply_job, {cnvscope_job}) - if haploid_cnv_job and haploid_apply_job and concat_job: - dag.add_job(haploid_cnv_job) - dag.add_job(haploid_apply_job, {haploid_cnv_job}) - dag.add_job(concat_job, {cnvapply_job, haploid_apply_job}) + if not self.skip_cnv: + ( + cnvscope_job, + cnvapply_job, + haploid_cnv_job, + haploid_apply_job, + concat_job, + ) = self.build_cnv_jobs( + out_cnv, + cnvscope_tmp, + cnvscope_autosomes, + cnvscope_haploid_tmp, + cnvscope_haploid, + input_aln, + ) + dag.add_job(cnvscope_job) + dag.add_job(cnvapply_job, {cnvscope_job}) + if haploid_cnv_job and haploid_apply_job and concat_job: + dag.add_job(haploid_cnv_job) + dag.add_job(haploid_apply_job, {haploid_cnv_job}) + dag.add_job(concat_job, {cnvapply_job, haploid_apply_job}) # Repeat expansions if self.expansion_catalog: expansion_job = self.build_expansion_job( out_expansions, - self.deduped_cram, + input_aln, self.expansion_catalog, ) dag.add_job(expansion_job) @@ -1246,7 +1047,9 @@ def build_cnv_jobs( ) ) cnvscope_job = Job( - shlex.join(driver.build_cmd()), "cnvscope-autosomes", self.cores + Pipeline(Command(*driver.build_cmd())), + "cnvscope-autosomes", + self.cores, ) # Autosome model apply @@ -1267,7 +1070,7 @@ def build_cnv_jobs( ) ) cnvapply_job = Job( - shlex.join(driver.build_cmd()), + Pipeline(Command(*driver.build_cmd())), "cnvmodelapply-autosomes", self.cores, ) @@ -1289,7 +1092,9 @@ def build_cnv_jobs( ) ) haploid_cnv_job = Job( - shlex.join(driver.build_cmd()), "cnvscope-haploid", self.cores + Pipeline(Command(*driver.build_cmd())), + "cnvscope-haploid", + self.cores, ) # Sex chrom CNVModelApply @@ -1305,7 +1110,9 @@ def build_cnv_jobs( ) ) haploid_apply_job = Job( - shlex.join(driver.build_cmd()), "cnvmodelapply-haploid", self.cores + Pipeline(Command(*driver.build_cmd())), + "cnvmodelapply-haploid", + self.cores, ) # Concate autosome and sex chromsome VCFs @@ -1347,3 +1154,49 @@ def build_expansion_job( 0, ) return expansion_job + + def build_t1k_job( + self, + out_basename: pathlib.Path, + deduped_bam: pathlib.Path, + gene_seq: pathlib.Path, + gene_coord: pathlib.Path, + preset: str, + ) -> Job: + """Genotype diverse sequences with T1K""" + job = Job( + cmds.cmd_t1k( + out_basename, + deduped_bam, + gene_seq=gene_seq, + gene_coord=gene_coord, + preset=preset, + threads=1, + ), + "T1K-HLA-KIR", + 0, + ) + return job + + def build_segdup_job( + self, + out_segdup: pathlib.Path, + realigned_cram: pathlib.Path, + genes: str, + ) -> Job: + """Call variants in difficult SegDups""" + assert self.reference + assert self.model_bundle + + segdup_job = Job( + cmds.cmd_segdup_caller( + out_segdup, + realigned_cram, + reference=self.reference, + sr_bundle=self.model_bundle, + genes=genes, + ), + "segdup-caller", + self.cores, + ) + return segdup_job diff --git a/sentieon_cli/pipeline.py b/sentieon_cli/pipeline.py index 6522040..649f1c7 100644 --- a/sentieon_cli/pipeline.py +++ b/sentieon_cli/pipeline.py @@ -5,22 +5,67 @@ from abc import ABC, abstractmethod import argparse import multiprocessing as mp +import os import pathlib import shutil -from typing import Any, Dict, List +import sys +from typing import Any, Dict, List, Optional +import packaging.version + +from . import command_strings as cmds from .dag import DAG from .executor import BaseExecutor, DryRunExecutor, LocalExecutor +from .job import Job from .logging import get_logger from .scheduler import ThreadScheduler -from .util import __version__, tmp +from .util import __version__, check_version, path_arg, tmp + +MULTIQC_MIN_VERSION = { + "multiqc": packaging.version.Version("1.18"), +} class BasePipeline(ABC): """A pipeline base class""" - params: Dict[str, Dict[str, Any]] = {} - positionals: Dict[str, Dict[str, Any]] = {} + params: Dict[str, Dict[str, Any]] = { + # Required arguments + "reference": { + "flags": ["-r", "--reference"], + "required": True, + "help": "fasta for reference genome.", + "type": path_arg(exists=True, is_file=True), + }, + # Additional arguments + "cores": { + "flags": ["-t", "--cores"], + "help": ( + "Number of threads/processes to use. Defaults to all " + "available." + ), + "default": mp.cpu_count(), + }, + "dry_run": { + "help": "Print the commands without running them.", + "action": "store_true", + }, + # Hidden arguments + "retain_tmpdir": { + "help": argparse.SUPPRESS, + "action": "store_true", + }, + "skip_version_check": { + "help": argparse.SUPPRESS, + "action": "store_true", + }, + } + positionals: Dict[str, Dict[str, Any]] = { + "output_vcf": { + "help": "Output VCF file. The file name must end in .vcf.gz", + "type": path_arg(), + }, + } @classmethod def add_arguments(cls, parser: argparse.ArgumentParser): @@ -56,10 +101,13 @@ def setup_logging(self, args: argparse.Namespace) -> None: self.logger.info("Starting sentieon-cli version: %s", __version__) def __init__(self) -> None: + self.reference: Optional[pathlib.Path] = None self.cores = mp.cpu_count() self.numa_nodes: List[str] = [] self.dry_run = False self.retain_tmpdir = False + self.skip_version_check = False + self.output_vcf: Optional[pathlib.Path] = None def main(self, args: argparse.Namespace) -> None: """Run the DNAscope pipeline""" @@ -99,6 +147,22 @@ def check_execution( def validate(self) -> None: pass + def validate_output_vcf(self) -> None: + if not str(self.output_vcf).endswith(".vcf.gz"): + self.logger.error("The output file should end with '.vcf.gz'") + sys.exit(2) + + def validate_ref(self) -> None: + # Confirm the presence of the reference index file + fai_file = str(self.reference) + ".fai" + if not os.path.isfile(fai_file): + self.logger.error( + "Fasta index file %s does not exist. Please index the " + "reference genome with 'samtools faidx'", + fai_file, + ) + sys.exit(2) + @abstractmethod def configure(self) -> None: pass @@ -107,6 +171,36 @@ def configure(self) -> None: def build_dag(self) -> DAG: pass + def multiqc(self) -> Optional[Job]: + """Run MultiQC on the metrics files""" + + if not self.skip_version_check: + if not all( + [ + check_version(cmd, min_version) + for (cmd, min_version) in MULTIQC_MIN_VERSION.items() + ] + ): + self.logger.warning( + "Skipping MultiQC. MultiQC version %s or later not found", + MULTIQC_MIN_VERSION["multiqc"], + ) + return None + + metrics_dir = pathlib.Path( + str(self.output_vcf).replace(".vcf.gz", "_metrics") + ) + multiqc_job = Job( + cmds.cmd_multiqc( + metrics_dir, + metrics_dir, + f"Generated by the Sentieon-CLI version {__version__}", + ), + "multiqc", + 0, + ) + return multiqc_job + def run(self, dag: DAG) -> BaseExecutor: """Execute the DAG""" self.logger.debug("Creating the scheduler") diff --git a/sentieon_cli/runner.py b/sentieon_cli/runner.py deleted file mode 100644 index bba8dfb..0000000 --- a/sentieon_cli/runner.py +++ /dev/null @@ -1,22 +0,0 @@ -import subprocess as sp -import sys -import time - -from .logging import get_logger - -logger = get_logger(__name__) - - -def run(cmd: str): - """Run a command.""" - logger.info("running command: %s", cmd) - t0 = time.time() - sp.run( - cmd, - shell=True, - check=True, - stdout=sys.stdout, - stderr=sys.stderr, - executable="/bin/bash", - ) - logger.info("finished in: %s seconds", f"{time.time() - t0:.1f}") diff --git a/sentieon_cli/scripts/rehead_wgs_metrics.py b/sentieon_cli/scripts/rehead_wgs_metrics.py new file mode 100644 index 0000000..d00791d --- /dev/null +++ b/sentieon_cli/scripts/rehead_wgs_metrics.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 + +""" +Rehead the WGSMetricsAlgo output for MultiQC +""" + +# Copyright (c) 2025 Sentieon Inc. All rights reserved + +import argparse +import os.path +import shutil +import tempfile + + +def process_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--metrics_file", + required=True, + help="The input metrics file", + ) + return parser.parse_args() + + +def main(args: argparse.Namespace) -> None: + metrics_file = args.metrics_file + with tempfile.TemporaryDirectory() as tmp_dir: + temp_metrics = os.path.join(tmp_dir, os.path.basename(metrics_file)) + shutil.move(metrics_file, temp_metrics) + + with ( + open(temp_metrics, 'r') as in_fh, + open(metrics_file, 'w') as out_fh + ): + print("'## METRICS CLASS WgsMetrics'", file=out_fh) + line = in_fh.readline() + for line in in_fh: + print(line.rstrip(), file=out_fh) + + +if __name__ == "__main__": + args = process_args() + main(args) diff --git a/sentieon_cli/scripts/trimalt.py b/sentieon_cli/scripts/trimalt.py new file mode 100644 index 0000000..3933d6f --- /dev/null +++ b/sentieon_cli/scripts/trimalt.py @@ -0,0 +1,97 @@ +import re +import os +import sys + +kvpat = re.compile(r'(.*?)=(".*?"|.*?)(?:,|$)') +def parse_line(line): + s = line.index('<') + e = line.index('>') + return dict(kvpat.findall(line[s+1:e])) + +def parse_kv(kv): + kv = kv.split('=',1) + return len(kv) == 2 and kv or [kv[0], True] + +infos, fmts = {}, {} +for line in sys.stdin: + if line[0] == '#': + if line.startswith('##INFO'): + d = parse_line(line) + if d['Number'] in 'ARG': + infos[d['ID']] = d['Number'] + if line.startswith('##FORMAT'): + d = parse_line(line) + if d['Number'] in 'ARG': + fmts[d['ID']] = d['Number'] + sys.stdout.write(line) + continue + + vals = line.rstrip().split('\t') + if len(vals) < 9 or vals[8] == '.': + # pop vcf only site, skip + continue + + ka, kr, kg, info = None, None, None, None + if len(vals) >= 8 and vals[7] != '.': + info = list(map(parse_kv, vals[7].split(';'))) + for k,v in info: + if k == 'AF': + ka = [x != '.' for x in v.split(',')] + kr = [True] + ka + break + + if ka is None: + # no-call, drop + continue + + if all(ka): + sys.stdout.write(line) + continue + + if vals[4] != '.': + v = vals[4].split(',') + v = [x for x,y in zip(v, ka) if y] + r = vals[3] + #n = len(os.path.commonprefix([r[:0:-1]] + [a[:0:-1] for a in v])) + n = min(len(r), min(map(len, v))) - 1 + if n > 0: + r, v = r[:-n], [a[:-n] for a in v] + vals[3] = r + vals[4] = ','.join(v) + + if info: + for i,(k,v) in enumerate(info): + n = infos.get(k) if v != '.' else None + if n == 'A': + v = v.split(',') + info[i] = k + '=' + ','.join(x for x,y in zip(v, ka) if y) + elif n == 'R': + v = v.split(',') + info[i] = k + '=' + ','.join(x for x,y in zip(v, kr) if y) + elif v is True: + info[i] = k + else: + info[i] = k + '=' + v + vals[7] = ';'.join(info) + + if len(vals) >= 10 and vals[9] != '.': + fmt = vals[8].split(':') + kg = [x and y for i,x in enumerate(kr) for y in kr[:i+1]] + for j,val in enumerate(vals[9:]): + val = val.split(':') + for i,v in enumerate(val): + n = fmts.get(fmt[i]) if v != '.' else None + if n == 'A': + v = v.split(',') + val[i] = ','.join(x for x,y in zip(v, ka) if y) + elif n == 'R': + v = v.split(',') + val[i] = ','.join(x for x,y in zip(v, kr) if y) + elif n == 'G': + v = v.split(',') + val[i] = ','.join(x for x,y in zip(v, kg) if y) + # GT not affected, new alt alleles are added at the end + vals[9+j] = ':'.join(val) + + line = '\t'.join(vals) + '\n' + sys.stdout.write(line) diff --git a/sentieon_cli/scripts/vcf_mod.py b/sentieon_cli/scripts/vcf_mod.py index 2ecc89c..148a99a 100644 --- a/sentieon_cli/scripts/vcf_mod.py +++ b/sentieon_cli/scripts/vcf_mod.py @@ -623,30 +623,31 @@ def join2(f0, f1, f2, v0, v1, v2, pos, bed): continue elif t == 'R': if k == 'RPA': - d0 = d1 = d2 = [-1] * (len(v.alt)+1) - if v0 and v0.info.get(k): - s = v0.info.get(k) - d0 = [i >= 0 and s[i] or -1 for i in m0] - if v1 and v1.info.get(k): + d1 = d2 = [-1] * (len(v.alt)+1) + if i1 > 0 and v1.info.get(k): s = v1.info.get(k) d1 = [i >= 0 and s[i] or -1 for i in m1] - if v2 and v2.info.get(k): + if i2 > 0 and v2.info.get(k): s = v2.info.get(k) d2 = [i >= 0 and s[i] or -1 for i in m2] - d = [max(e) for e in zip(d0,d1,d2)] + d = [max(e) for e in zip(d1,d2)] if min(d) > 0: v.info[k] = d - continue + continue else: if k in ('STR', 'RU'): - d = None - d = d and d or v1 and v1.info.get(k) - d = d and d or v2 and v2.info.get(k) - d = d and d or v0 and v0.info.get(k) - if d: + if i1 > 0 and i2 > 0 and v1.info.get(k) == v2.info.get(k): + d = v1.info.get(k) + elif i1 > 0: + d = v1.info.get(k) + elif i2 > 0: + d = v2.info.get(k) + else: + d = None + if d is not None: v.info[k] = d - continue - if k == 'DP': + continue + elif k == 'DP': d = -1 if d < 0 and v0: d = v0.info.get(k, -1) @@ -656,15 +657,15 @@ def join2(f0, f1, f2, v0, v1, v2, pos, bed): d += v2 and v2.info.get(k, 0) or 0 if d >= 0: v.info[k] = d - continue - if k == 'ML_PROB': + continue + elif k == 'ML_PROB': d = (u.info.get(k) for u in (v1,v2) if u) d = [e for e in d if e is not None] if d: d = functools.reduce(operator.mul, d, 1) v.info[k] = round(d, 2) - continue - if k not in ('QD',): + continue + elif k not in ('QD',): continue v.info.pop(k, None) diff --git a/sentieon_cli/sentieon_pangenome.py b/sentieon_cli/sentieon_pangenome.py new file mode 100644 index 0000000..862d481 --- /dev/null +++ b/sentieon_cli/sentieon_pangenome.py @@ -0,0 +1,1027 @@ +""" +Sentieon's pangenome alignment and variant calling pipeline +""" + +import argparse +import copy +import json +import pathlib +import re +import shutil +import subprocess as sp +import sys +from typing import Dict, List, NamedTuple, Optional, Set, Tuple, Union + +import packaging.version + +from importlib_resources import files + +from . import command_strings as cmds +from .archive import ar_load +from .base_pangenome import BasePangenome +from .dag import DAG +from .driver import ( + AlignmentStat, + BaseDistributionByCycle, + CoverageMetrics, + Dedup, + DNAModelApply, + DNAscope, + Driver, + GCBias, + InsertSizeMetricAlgo, + LocusCollector, + MeanQualityByCycle, + QualDistribution, + WgsMetricsAlgo, +) +from .job import Job +from .logging import get_logger +from .shell_pipeline import Command, Pipeline +from .util import ( + __version__, + check_version, + parse_rg_line, + path_arg, + tmp, + total_memory, +) + +SENT_PANGENOME_MIN_VERSIONS = { + "kmc": None, + "sentieon driver": packaging.version.Version("202503.02"), + "vg": None, + "bcftools": packaging.version.Version("1.22"), + "samtools": packaging.version.Version("1.16"), +} + +GRCH38_CONTIGS: Dict[str, int] = { + "chr1": 248956422, + "chr2": 242193529, + "chr3": 198295559, + "chr4": 190214555, + "chr5": 181538259, + "chr6": 170805979, + "chr7": 159345973, + "chr8": 145138636, + "chr9": 138394717, + "chr10": 133797422, + "chr11": 135086622, + "chr12": 133275309, + "chr13": 114364328, + "chr14": 107043718, + "chr15": 101991189, + "chr16": 90338345, + "chr17": 83257441, + "chr18": 80373285, + "chr19": 58617616, + "chr20": 64444167, + "chr21": 46709983, + "chr22": 50818468, + "chrX": 156040895, + "chrY": 57227415, + "chrM": 16569, +} + +logger = get_logger(__name__) + + +class Shard(NamedTuple): + contig: str + start: int + stop: int + + def __str__(self) -> str: + return f"{self.contig}:{self.start}-{self.stop}" + + def bcftools_str(self) -> str: + return f"{{{self.contig}}}:{self.start}-{self.stop}" + + +def parse_fai(ref_fai: pathlib.Path) -> Dict[str, Dict[str, int]]: + """Parse a faidx index""" + contigs: Dict[str, Dict[str, int]] = {} + with open(ref_fai) as fh: + for line in fh: + try: + chrom, length, offset, lb, lw = line.rstrip().split() + except ValueError as err: + logger.error( + "Reference fasta index (.fai) does not have the expected " + "format" + ) + raise err + contigs[chrom] = { + "length": int(length), + "offset": int(offset), + "linebases": int(lb), + "linewidth": int(lw), + } + return contigs + + +def determine_shards_from_fai( + fai_data: Dict[str, Dict[str, int]], step: int +) -> List[Shard]: + """Generate shards of the genome from the fasta index""" + shards: List[Shard] = [] + for ctg, d in fai_data.items(): + pos = 1 + length = d["length"] + while pos <= length: + end = pos + step - 1 + end = end if end < length else length + shards.append(Shard(ctg, pos, end)) + pos = end + 1 + return shards + + +def vcf_contigs( + in_vcf: pathlib.Path, dry_run=False +) -> Dict[str, Optional[int]]: + """Report the contigs in the input VCF""" + if dry_run: + return { + "chr1": 100, + "chr2": 200, + "chr3": 300, + } + kvpat = re.compile(r'(.*?)=(".*?"|.*?)(?:,|$)') + cmd = ["bcftools", "view", "-h", str(in_vcf)] + p = sp.run(cmd, capture_output=True, text=True) + contigs: Dict[str, Optional[int]] = {} + for line in p.stdout.split("\n"): + if not line.startswith("##contig"): + continue + s = line.index("<") + e = line.index(">") + d = dict(kvpat.findall(line[s + 1 : e])) # noqa: E203 + ctg: str = d["ID"] + length: Optional[str] = d.get("length", None) + contigs[ctg] = int(length) if length else None + return contigs + + +def vcf_id(in_vcf: pathlib.Path) -> Optional[str]: + """Collect the SentieonVcfID header""" + cmd = ["bcftools", "view", "-h", str(in_vcf)] + p = sp.run(cmd, capture_output=True, text=True) + for line in p.stdout.split("\n"): + if line.startswith("##SentieonVcfID="): + i = line.index("=") + return line[i + 1 :] # noqa: E203 + return None + + +class SentieonPangenome(BasePangenome): + """The Sentieon pangenome pipeline""" + + params = copy.deepcopy(BasePangenome.params) + params.update( + { + # Required arguments + "readgroup": { + "help": "Readgroup information for the fastq files.", + }, + "sample_input": { + "flags": ["-i", "--sample_input"], + "nargs": "*", + "help": "sample BAM or CRAM file.", + "type": path_arg(exists=True, is_file=True), + }, + "pop_vcf": { + "flags": ["--pop_vcf"], + "help": ( + "A VCF containing annotations for use with DNAModelApply." + ), + "type": path_arg(exists=True, is_file=True), + "required": True, + }, + # Additional arguments + "bed": { + "flags": ["-b", "--bed"], + "help": ( + "Region BED file. Supplying this file will limit variant " + "calling to the intervals inside the BED file." + ), + "type": path_arg(exists=True, is_file=True), + }, + "skip_metrics": { + "help": "Skip metrics collection and multiQC", + "action": "store_true", + }, + "skip_multiqc": { + "help": "Skip multiQC report generation", + "action": "store_true", + }, + # Hidden arguments + "skip_contig_checks": { + "help": argparse.SUPPRESS, + "action": "store_true", + }, + "skip_pangenome_name_checks": { + "help": argparse.SUPPRESS, + "action": "store_true", + }, + "skip_pop_vcf_id_check": { + "help": argparse.SUPPRESS, + "action": "store_true", + }, + } + ) + + positionals = BasePangenome.positionals + + def __init__(self) -> None: + super().__init__() + self.readgroup: Optional[str] = None + self.sample_input: List[pathlib.Path] = [] + self.bed: Optional[pathlib.Path] = None + self.pop_vcf: Optional[pathlib.Path] = None + self.skip_metrics = False + self.skip_multiqc = False + self.skip_contig_checks: bool = False + self.skip_pangenome_name_checks: bool = False + self.skip_pop_vcf_id_check: bool = False + + def main(self, args: argparse.Namespace) -> None: + """Run the pipeline""" + self.handle_arguments(args) + self.setup_logging(args) + self.validate_ref() + + self.fai_data = parse_fai(pathlib.Path(str(self.reference) + ".fai")) + self.pop_vcf_contigs: Dict[str, Optional[int]] = {} + if self.pop_vcf: + self.pop_vcf_contigs = vcf_contigs(self.pop_vcf, self.dry_run) + self.logger.debug("VCF contigs are: %s", self.pop_vcf_contigs) + + self.validate() + self.shards = determine_shards_from_fai( + self.fai_data, 10 * 1000 * 1000 + ) + + tmp_dir_str = tmp() + self.tmp_dir = pathlib.Path(tmp_dir_str) + + dag = self.build_first_dag() + executor = self.run(dag) + self.check_execution(dag, executor) + + if not self.retain_tmpdir: + shutil.rmtree(tmp_dir_str) + + def validate(self) -> None: + """Validate pipeline inputs""" + self.validate_bundle() + self.validate_fastq_rg() + self.validate_output_vcf() + self.validate_ref() + self.collect_readgroups() + + if not self.sample_input and not self.r1_fastq: + self.logger.error( + "Please supply either the `--sample_input` or `--r1_fastq` " + "and `--readgroups` arguments" + ) + sys.exit(2) + + if self.sample_input and self.r1_fastq: + self.logger.error( + "Supplying both `--r1_fastq` and `--sample_input` is not " + "supported" + ) + sys.exit(2) + + if not self.skip_version_check: + for cmd, min_version in SENT_PANGENOME_MIN_VERSIONS.items(): + if not check_version(cmd, min_version): + sys.exit(2) + + if self.bed is None: + self.logger.info( + "A BED file is recommended to avoid variant calling " + "across decoy and unplaced contigs." + ) + + if not self.skip_pangenome_name_checks: + if not str(self.gbz).endswith("grch38.gbz"): + self.logger.error( + "The `--gbz` file does not have the expected suffix. " + "Check that you are using a GRCh38 pangenome." + ) + sys.exit(2) + + if not str(self.hapl).endswith("grch38.hapl"): + self.logger.error( + "The `--hapl` file does not have the expected suffix. " + "Check that you re using a GRCh38 pangenome." + ) + sys.exit(2) + + if not self.skip_contig_checks: + # Check the fai file contigs + mismatch_contigs: Set[str] = set() + for ctg, length in GRCH38_CONTIGS.items(): + d = self.fai_data.get(ctg, {}) + fai_length = d.get("length", -1) + if length != fai_length: + mismatch_contigs.add(ctg) + if mismatch_contigs: + mismatch_contigs_s = ", ".join(mismatch_contigs) + self.logger.error( + "Reference contigs with unexpected lengths: %s", + mismatch_contigs_s, + ) + sys.exit(2) + + # Check the pop VCF file contigs + if not self.dry_run: + mismatch_contigs = set() + for ctg, length in GRCH38_CONTIGS.items(): + vcf_length = self.pop_vcf_contigs.get(ctg, -1) + if length != vcf_length: + mismatch_contigs.add(ctg) + if mismatch_contigs: + mismatch_contigs_s = ", ".join(mismatch_contigs) + self.logger.error( + "Pop VCF contigs with unexpected lengths: %s", + mismatch_contigs_s, + ) + sys.exit(2) + + def validate_bundle(self) -> None: + assert self.pop_vcf + assert self.gbz + bundle_info_bytes = ar_load( + str(self.model_bundle) + "/bundle_info.json" + ) + if isinstance(bundle_info_bytes, list): + bundle_info_bytes = b"{}" + + bundle_info = json.loads(bundle_info_bytes.decode()) + try: + req_version = packaging.version.Version( + bundle_info["minScriptVersion"] + ) + bundle_pipeline = bundle_info["pipeline"] + self.tech: str = bundle_info["platform"].upper() + bundle_vcf_id = bundle_info["SentieonVcfID"] + bundle_pangenome = bundle_info.get( + "pangenome", "hprc-v2.0-mc-grch38.gbz" + ) + except KeyError: + self.logger.error( + "The model bundle does not have the expected attributes" + ) + sys.exit(2) + if req_version > packaging.version.Version(__version__): + self.logger.error( + "The model bundle requires version %s or later of the " + "sentieon-cli.", + req_version, + ) + sys.exit(2) + if bundle_pipeline != "Sentieon pangenome": + self.logger.error("The model bundle is for a different pipeline.") + sys.exit(2) + if self.gbz.name != bundle_pangenome: + self.logger.warning( + "The `--gbz` file name is not '%s'. " + "This model is optimized for the %s pangenome.", + bundle_pangenome, + bundle_pangenome, + ) + + bundle_members = set(ar_load(str(self.model_bundle))) + if ( + "dnascope.model" not in bundle_members + or "extract.model" not in bundle_members + or "minimap2.model" not in bundle_members + ): + self.logger.error( + "Expected model files not found in the model bundle file" + ) + sys.exit(2) + + if not self.skip_pop_vcf_id_check and not self.dry_run: + pop_vcf_id = vcf_id(self.pop_vcf) + if bundle_vcf_id != pop_vcf_id: + self.logger.error( + "The ID of the `--pop_vcf` does not match the model bundle" + ) + sys.exit(2) + + def validate_fastq_rg(self) -> None: + if len(self.r1_fastq) != len(self.r2_fastq): + self.logger.error( + "The number of input `--r1_fastq` files does not equal the " + "number of `--r2_fastq` files" + ) + sys.exit(2) + + if (len(self.r1_fastq) > 0 and not self.readgroup) or ( + self.readgroup and len(self.r1_fastq) < 1 + ): + self.logger.error( + "`--r1_fastq`, `--r2_fastq`, and `--readgroup` are required " + "with fastq input. `--readgroup` cannot be used with bam/cram " + "input." + ) + sys.exit(2) + + def collect_readgroups(self) -> None: + """Collect readgroup tags""" + self.bam_readgroups: List[Dict[str, str]] = [] + for aln in self.sample_input: + aln_rgs = cmds.get_rg_lines(aln, self.dry_run) + for rg_line in aln_rgs: + self.bam_readgroups.append(parse_rg_line(rg_line)) + break # currently just the first RG line for each input + + self.fastq_readgroup: Dict[str, str] = {} + if not self.readgroup: + return + + parsed_rg = parse_rg_line(self.readgroup.replace(r"\t", "\t")) + if not parsed_rg.get("ID"): + self.logger.error( + "Readgroup '%s' does not have a RGID tag", + self.readgroup, + ) + sys.exit(2) + if parsed_rg.get("SM", None) is None: + self.logger.error( + "Readgroup '%s' does not have a RGSM tag", + self.readgroup, + ) + self.fastq_readgroup = parsed_rg + + def configure(self) -> None: + """Configure pipeline parameters""" + pass + + def build_dag(self) -> DAG: + return DAG() + + def build_first_dag(self) -> DAG: + """Build the main DAG for the Sentieon pangenome pipeline""" + assert self.reference + assert self.model_bundle + + self.logger.info("Building the Sentieon pangenome DAG") + dag = DAG() + + # Output files + suffix = "bam" if self.bam_format else "cram" + out_bwa_aln = pathlib.Path( + str(self.output_vcf).replace(".vcf.gz", f"_bwa_deduped.{suffix}") + ) + out_mm2_aln = pathlib.Path( + str(self.output_vcf).replace(".vcf.gz", f"_mm2_deduped.{suffix}") + ) + self.ploidy_json = pathlib.Path( + str(self.output_vcf).replace(".vcf.gz", "_ploidy.json") + ) + + # Intermediate file paths + bwa_bam = self.tmp_dir.joinpath("sample-bwa.bam") + ext_fastq = self.tmp_dir.joinpath("sample-bwa.fq.gz") + kmer_prefix = self.tmp_dir.joinpath("sample.fq") + kmer_file = pathlib.Path(str(kmer_prefix) + ".kff") + sample_pangenome = self.tmp_dir.joinpath("sample_pangenome.gbz") + sample_gfa = self.tmp_dir.joinpath("sample-hap.gfa") + sample_fasta = self.tmp_dir.joinpath("sample-hap.fa") + mm2_bam = self.tmp_dir.joinpath("sample-mm2.bam") + raw_vcf = self.tmp_dir.joinpath("sample-dnascope.vcf.gz") + transfer_vcf = self.tmp_dir.joinpath("sample-dnascope_transfer.vcf.gz") + # - Ensure we have a bam file for t1k + self.rw_bam = self.tmp_dir.joinpath("sample_deduped.bam") + + bwa_lc_dependencies: Set[Job] = set() + haplotype_dependencies: Set[Job] = set() + mm2_dependencies: Set[Job] = set() + dnascope_bams: List[pathlib.Path] = [] + + total_mem_gb = total_memory() / (1024.0**3) + + if self.r1_fastq: + # KMC k-mer counting + kmc_job = self.build_kmc_job(kmer_prefix, 0) # run in background + dag.add_job(kmc_job) + haplotype_dependencies.add(kmc_job) + + # BWA alignment and extraction + bwa_job = self.build_alignment_job(bwa_bam, ext_fastq) + dag.add_job(bwa_job) + mm2_dependencies.add(bwa_job) + bwa_lc_dependencies.add(bwa_job) + # Do not run vg-haplotypes with bwa in low-mem environments + if total_mem_gb < 70: + haplotype_dependencies.add(bwa_job) + else: + dnascope_bams = copy.deepcopy(self.sample_input) + extract_kmc_job = Job( + cmds.cmd_extract_kmc( + kmer_prefix, + ext_fastq, + self.sample_input, + self.reference, + self.model_bundle.joinpath("extract.model"), + self.tmp_dir, + threads=self.cores, + ), + "extract-kmc", + self.cores, + ) + dag.add_job(extract_kmc_job) + haplotype_dependencies.add(extract_kmc_job) + + # vg haplotypes - create a sample-specific pangenome + haplotypes_job = self.build_haplotypes_job(sample_pangenome, kmer_file) + dag.add_job(haplotypes_job, haplotype_dependencies) + + # convert the sample pangenome + gfa_job = self.build_gfa_job(sample_gfa, sample_pangenome) + fasta_job = self.build_fasta_job(sample_fasta, sample_pangenome) + dag.add_job(gfa_job, {haplotypes_job}) + dag.add_job(fasta_job, {haplotypes_job}) + + # minimap2 alignment of the extracted fastq + dnascope_dependencies = set() + mm2_job = self.build_minimap2_lift_job( + mm2_bam, + ext_fastq, + sample_fasta, + sample_gfa, + ) + dag.add_job(mm2_job, mm2_dependencies | {gfa_job, fasta_job}) + dnascope_dependencies.add(mm2_job) + + # With fastq input, perform dedup and metrics + if self.r1_fastq: + dnascope_bams.append(out_bwa_aln) + dnascope_bams.append(out_mm2_aln) + bwa_lc_job, bwa_dedup_job = self.build_dedup_job( + out_bwa_aln, [bwa_bam], "bwa" + ) + mm2_lc_job, mm2_dedup_job = self.build_dedup_job( + out_mm2_aln, [mm2_bam], "mm2", left_align=True + ) + dag.add_job(bwa_lc_job, bwa_lc_dependencies) + dag.add_job(bwa_dedup_job, {bwa_lc_job}) + dnascope_dependencies.add(bwa_dedup_job) + dag.add_job(mm2_lc_job, {mm2_job}) + dag.add_job(mm2_dedup_job, {mm2_lc_job}) + dnascope_dependencies.add(mm2_dedup_job) + + if not self.skip_metrics: + metrics_job, rehead_job = self.build_metrics_job( + [out_bwa_aln, out_mm2_aln], + ) + dag.add_job(metrics_job, {bwa_dedup_job, mm2_dedup_job}) + dag.add_job(rehead_job, {metrics_job}) + if not self.skip_multiqc: + multiqc_job = self.multiqc() + if multiqc_job: + dag.add_job(multiqc_job, {rehead_job}) + else: + dnascope_bams.append(mm2_bam) + + # DNAscope calling with bwa and mm2 input + apply_dependencies = set() + dnascope_job = self.build_dnascope_job(raw_vcf, dnascope_bams) + dag.add_job(dnascope_job, dnascope_dependencies) + apply_dependencies.add(dnascope_job) + + # transfer annotations from the pop_vcf + if self.pop_vcf: + transfer_jobs, concat_job = self.build_transfer_jobs( + transfer_vcf, raw_vcf + ) + for job in transfer_jobs: + dag.add_job(job, {dnascope_job}) + dag.add_job(concat_job, set(transfer_jobs)) + apply_dependencies.add(concat_job) + + # DNAModelApply + apply_job = self.build_dnamodelapply_job(transfer_vcf) + dag.add_job(apply_job, apply_dependencies) + + return dag + + def build_alignment_job( + self, + sample_bam: pathlib.Path, + sample_fastq: pathlib.Path, + ) -> Job: + """Build the alignment and extract jobs""" + assert self.reference + assert self.model_bundle + + unzip = "igzip" + if not shutil.which(unzip): + self.logger.info( + "igzip is recommended for decompression, but is not " + "available. Falling back to gzip." + ) + unzip = "gzip" + + rg = copy.deepcopy(self.fastq_readgroup) + rg["ID"] = rg["ID"] + "-bwa" + bwa_job = Job( + cmds.cmd_bwa_extract( + sample_bam, + sample_fastq, + self.reference, + self.r1_fastq, + self.r2_fastq, + "@RG\\t" + "\\t".join([f"{x[0]}:{x[1]}" for x in rg.items()]), + self.model_bundle.joinpath("extract.model"), + self.model_bundle.joinpath("bwa.model"), + self.cores, + unzip=unzip, + ), + "bwa-extract", + self.cores, + ) + return bwa_job + + def build_haplotypes_job( + self, output_gbz: pathlib.Path, kmer_file: pathlib.Path + ) -> Job: + """Build vg haplotypes job""" + assert self.hapl + assert self.gbz + + haplotypes_job = Job( + cmds.cmd_vg_haplotypes( + output_gbz, + kmer_file, + self.hapl, + self.gbz, + threads=self.cores, + xargs=[ + "--include-reference", + "--diploid-sampling", + "--set-reference", + "GRCh38", + ], + ), + "vg-haplotypes", + self.cores, + ) + return haplotypes_job + + def build_gfa_job( + self, output_gfa: pathlib.Path, input_gbz: pathlib.Path + ) -> Job: + """Build vg convert to GFA job""" + gfa_job = Job( + cmds.cmd_vg_convert_gfa( + output_gfa, + input_gbz, + threads=self.cores, + ), + "vg-convert-gfa", + 0, + ) + return gfa_job + + def build_fasta_job( + self, output_fasta: pathlib.Path, input_gbz: pathlib.Path + ) -> Job: + """Build vg paths to FASTA job""" + fasta_job = Job( + cmds.cmd_vg_paths_fasta( + output_fasta, + input_gbz, + ), + "vg-paths-fasta", + 0, + ) + return fasta_job + + def build_minimap2_lift_job( + self, + mm2_bam: pathlib.Path, + ext_fastq: pathlib.Path, + sample_fasta: pathlib.Path, + sample_gfa: pathlib.Path, + ) -> Job: + """Build minimap2 alignment with pgutil lift job""" + assert self.model_bundle + assert self.reference + + rg = ( + self.fastq_readgroup + if self.fastq_readgroup + else self.bam_readgroups[0] + ) + rg2 = copy.deepcopy(rg) + rg2["ID"] = rg2["ID"] + "-mm2" + rg2["LR"] = "1" + + mm2_model: Union[str, pathlib.Path] = self.model_bundle.joinpath( + "minimap2.model" + ) + mm2_job = Job( + cmds.cmd_minimap2_lift( + mm2_bam, + sample_fasta, + ext_fastq, + sample_gfa, + self.reference, + "@RG\\t" + "\\t".join([f"{x[0]}:{x[1]}" for x in rg2.items()]), + mm2_model, + threads=self.cores, + ), + "mm2-lift", + self.cores, + ) + return mm2_job + + def build_dedup_job( + self, + output_bam, + input_bam: List[pathlib.Path], + tag: str, + left_align=False, + ) -> Tuple[Job, Job]: + """Build deduplication job""" + score_file = self.tmp_dir.joinpath(f"sample-{tag}-score.txt.gz") + + read_filters = [] + if left_align: + read_filters.append( + "IndelLeftAlignReadTransform," + f"rgid={self.fastq_readgroup['ID']}-mm2" + ) + + # LocusCollector + Dedup + driver = Driver( + reference=self.reference, + thread_count=self.cores, + input=input_bam, + read_filter=read_filters, + ) + driver.add_algo(LocusCollector(score_file)) + + lc_job = Job( + Pipeline(Command(*driver.build_cmd())), + f"locuscollector-{tag}", + self.cores, + ) + + driver2 = Driver( + reference=self.reference, + thread_count=self.cores, + input=input_bam, + read_filter=read_filters, + ) + driver2.add_algo(Dedup(output_bam, score_file)) + + dedup_job = Job( + Pipeline(Command(*driver2.build_cmd())), + f"dedup-{tag}", + self.cores, + ) + + return lc_job, dedup_job + + def build_metrics_job( + self, + sample_input: List[pathlib.Path], + ) -> Tuple[Job, Job]: + """Build a metrics job""" + assert self.output_vcf + + # Create the metrics directory + sample_name = self.output_vcf.name.replace(".vcf.gz", "") + metric_base = sample_name + ".txt" + metrics_dir = pathlib.Path( + str(self.output_vcf).replace(".vcf.gz", "_metrics") + ) + if not self.dry_run: + metrics_dir.mkdir(exist_ok=True) + + is_metrics = metrics_dir.joinpath(metric_base + ".insert_size.txt") + mqbc_metrics = metrics_dir.joinpath( + metric_base + ".mean_qual_by_cycle.txt" + ) + bdbc_metrics = metrics_dir.joinpath( + metric_base + ".base_distribution_by_cycle.txt" + ) + qualdist_metrics = metrics_dir.joinpath( + metric_base + ".qual_distribution.txt" + ) + as_metrics = metrics_dir.joinpath(metric_base + ".alignment_stat.txt") + coverage_metrics = metrics_dir.joinpath("coverage") + + # WGS metrics + wgs_metrics = metrics_dir.joinpath(metric_base + ".wgs.txt") + gc_metrics = metrics_dir.joinpath(metric_base + ".gc_bias.txt") + gc_summary = metrics_dir.joinpath(metric_base + ".gc_bias_summary.txt") + + driver = Driver( + reference=self.reference, + thread_count=self.cores, + input=sample_input, + ) + + driver.add_algo(InsertSizeMetricAlgo(is_metrics)) + driver.add_algo(MeanQualityByCycle(mqbc_metrics)) + driver.add_algo(BaseDistributionByCycle(bdbc_metrics)) + driver.add_algo(QualDistribution(qualdist_metrics)) + driver.add_algo(AlignmentStat(as_metrics)) + driver.add_algo(GCBias(gc_metrics, summary=gc_summary)) + driver.add_algo(WgsMetricsAlgo(wgs_metrics, include_unpaired="true")) + driver.add_algo(CoverageMetrics(coverage_metrics)) + + metrics_job = Job(Pipeline(Command(*driver.build_cmd())), "metrics", 0) + + rehead_script = pathlib.Path( + str( + files("sentieon_cli.scripts").joinpath("rehead_wgs_metrics.py") + ) + ) + rehead_job = Job( + Pipeline( + Command( + "sentieon", + "pyexec", + str(rehead_script), + "--metrics_file", + str(wgs_metrics), + ) + ), + "Rehead metrics", + 0, + ) + return (metrics_job, rehead_job) + + def build_dnascope_job( + self, + out_vcf: pathlib.Path, + input_bams: List[pathlib.Path], + ) -> Job: + assert self.model_bundle + + read_filters = [] + if self.tech.upper() == "ULTIMA": + read_filters.append("UltimaReadFilter") + + pcr_indel_model = "NONE" if self.pcr_free else "CONSERVATIVE" + driver = Driver( + reference=self.reference, + thread_count=self.cores, + input=input_bams, + interval=self.bed, + read_filter=read_filters, + ) + driver.add_algo( + DNAscope( + out_vcf, + model=self.model_bundle.joinpath("dnascope.model"), + pcr_indel_model=pcr_indel_model, + dbsnp=self.dbsnp, + ) + ) + return Job( + Pipeline(Command(*driver.build_cmd())), + "dnascope-raw", + self.cores, + ) + + def build_transfer_jobs( + self, out_vcf: pathlib.Path, raw_vcf: pathlib.Path + ) -> Tuple[List[Job], Job]: + """Transfer annotations from the pop_vcf to the raw_vcf""" + assert self.pop_vcf + + # Generate merge rules from the population VCF + merge_rules = "AC_v20:sum,AF_v20:sum,AC_genomes:sum,AF_genomes:sum" + if not self.dry_run: + kvpat = re.compile(r'(.*?)=(".*?"|.*?)(?:,|$)') + cmd = ["bcftools", "view", "-h", str(self.pop_vcf)] + p = sp.run(cmd, capture_output=True, text=True) + id_fields: List[str] = [] + for line in p.stdout.split("\n"): + if not line.startswith("##INFO"): + continue + if ",Number=A" not in line: + continue + s = line.index("<") + e = line.index(">") + d = dict(kvpat.findall(line[s + 1 : e])) # noqa: E203 + id_fields.append(d["ID"]) + merge_rules = ",".join([x + ":sum" for x in id_fields]) + + # Merge VCFs by shards + sharded_vcfs: List[pathlib.Path] = [] + sharded_merge_jobs: List[Job] = [] + trim_script = pathlib.Path( + str(files("sentieon_cli.scripts").joinpath("trimalt.py")) + ).resolve() + seen_contigs: Set[str] = set() + for i, shard in enumerate(self.shards): + # Use a BED file for unusual contig names + subset_bed = self.tmp_dir.joinpath( + f"sample-dnascope_transfer-subset{i}.bed" + ) + + # Extract contigs not in the pop vcf as merge will fail + if shard.contig not in self.pop_vcf_contigs: + if shard.contig in seen_contigs: + continue + self.logger.info( + "Skipping transfer for contig: %s", shard.contig + ) + seen_contigs.add(shard.contig) + subset_vcf = self.tmp_dir.joinpath( + f"sample-dnascope_transfer-subset{i}.vcf.gz" + ) + + ctg_len = self.fai_data[shard.contig]["length"] + if not self.dry_run: + with open(subset_bed, "w") as fh: + print(f"{shard.contig}\t0\t{ctg_len}", file=fh) + + view_job = Job( + cmds.cmd_bcftools_view_regions( + subset_vcf, + raw_vcf, + regions_file=subset_bed, + ), + "merge-trim-extra", + 1, + ) + sharded_merge_jobs.append(view_job) + sharded_vcfs.append(subset_vcf) + else: + if not self.dry_run: + with open(subset_bed, "w") as fh: + print( + f"{shard.contig}\t{shard.start}\t{shard.stop}", + file=fh, + ) + + self.logger.debug("Transferring shard: %s", shard) + shard_vcf = self.tmp_dir.joinpath( + f"sample-dnascope_transfer-shard{i}.vcf.gz" + ) + merge_job = Job( + cmds.cmd_bcftools_merge_trim( + shard_vcf, + raw_vcf, + self.pop_vcf, + trim_script, + subset_bed, + merge_rules=merge_rules, + merge_xargs=[ + "--no-version", + "--regions-overlap", + "pos", + "-m", + "all", + ], + view_xargs=["--no-version"], + ), + f"merge-trim-{i}", + 1, + ) + sharded_merge_jobs.append(merge_job) + sharded_vcfs.append(shard_vcf) + + # Concat all shards + concat_job = Job( + cmds.bcftools_concat( + out_vcf, + sharded_vcfs, + xargs=["--no-version", "--threads", str(self.cores)], + ), + "merge-trim-concat", + self.cores, + ) + return (sharded_merge_jobs, concat_job) + + def build_dnamodelapply_job(self, in_vcf) -> Job: + assert self.output_vcf + assert self.model_bundle + + driver = Driver( + reference=self.reference, + thread_count=self.cores, + ) + driver.add_algo( + DNAModelApply( + model=self.model_bundle.joinpath("dnascope.model"), + vcf=in_vcf, + output=self.output_vcf, + ) + ) + return Job( + Pipeline(Command(*driver.build_cmd())), + "model-apply", + self.cores, + ) diff --git a/sentieon_cli/shell_pipeline.py b/sentieon_cli/shell_pipeline.py new file mode 100644 index 0000000..db2413a --- /dev/null +++ b/sentieon_cli/shell_pipeline.py @@ -0,0 +1,360 @@ +"""Complex shell pipelines with process substitution""" + +from __future__ import annotations +import asyncio +import io +import os +import pathlib +import tempfile +import shlex +import sys +from abc import ABC, abstractmethod +from typing import Any, Dict, IO, Iterable, List, Optional, Union + +from .logging import get_logger + +logger = get_logger(__name__) + + +class Context: + """Holds shared resources.""" + + def __init__(self) -> None: + # This directory is already unique per pipeline run + self.temp_dir = tempfile.TemporaryDirectory() + # Background tasks to wait for (e.g., proc sub processes) + self.tasks: List[asyncio.Task] = [] + # Hold the commands run in this context + self.commands: List[Command] = [] + self._counter = 0 # Monotonic counter + self.file_handles: List[io.IOBase] = [] + + def get_new_fifo(self) -> str: + """Creates a new unique FIFO and returns its path.""" + self._counter += 1 + # Simple, readable names like fifo_1, fifo_2, etc. + path = os.path.join(self.temp_dir.name, f"fifo_{self._counter}") + os.mkfifo(path) + return path + + async def cleanup(self) -> None: + if self.tasks: + await asyncio.gather(*self.tasks) + for fh in self.file_handles: + fh.close() + self.temp_dir.cleanup() + + +class ShellNode(ABC): + """Abstract base class for any node in the shell syntax tree.""" + + +class Command(ShellNode): + """Represents a single command (e.g., 'grep', 'cat').""" + + def __init__( + self, + executable: str, + *args: str | ProcSub, + fail_ok=False, + exec_kwargs: Optional[Dict] = None, + ) -> None: + self.executable = executable + self.args = list(args) + self.fail_ok = fail_ok + self.proc: Union[asyncio.subprocess.Process, None] = None + self.exec_kwargs: Dict[str, Any] = exec_kwargs if exec_kwargs else {} + + async def run( + self, + context: Context, + stdin: Union[IO, int, None] = None, + stdout: Union[IO, int, None] = None, + stderr: Union[IO, int, None] = None, + ) -> asyncio.subprocess.Process: + # 1. Resolve Arguments (Handle Process Substitutions) + final_args = [self.executable] + + for arg in self.args: + if isinstance(arg, ProcSub): + # Recursively setup the process substitution + fifo_path = await arg.setup(context) + final_args.append(fifo_path) + else: + final_args.append(str(arg)) + + # 2. Run the process + # We allow the caller to wait for this process + self.proc = await asyncio.create_subprocess_exec( + *final_args, + stdin=stdin, + stdout=stdout, + stderr=stderr, + **self.exec_kwargs, + ) + context.commands.append(self) + return self.proc + + def __str__(self) -> str: + return shlex.join([self.executable] + [str(x) for x in self.args]) + + def __repr__(self) -> str: + return ( + f"{self.__class__.__name__}({self.executable}, " + + ", ".join([repr(x) for x in self.args]) + + f", fail_ok={self.fail_ok})" + ) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, Command): + return NotImplemented + return ( + self.executable == other.executable + and self.args == other.args + and self.fail_ok == other.fail_ok + and self.proc == other.proc + ) + + def __hash__(self) -> int: + return hash(tuple([self.executable, self.fail_ok] + self.args)) + + +class Pipeline(ShellNode): + """Represents a sequence of commands connected by pipes (|).""" + + def __init__( + self, + *nodes: Command, + skip_pipe: Optional[Iterable[int]] = None, + file_input: Optional[pathlib.Path] = None, + file_output: Optional[pathlib.Path] = None, + ): + self.nodes = list(nodes) + assert self.nodes # Nodes cannot be empty + self.skip_pipe = set(skip_pipe) if skip_pipe else set() + self.file_input = file_input + self.file_output = file_output + + async def run( + self, + context: Context, + stdin: Union[IO, int, None] = None, + stdout: Union[IO, int, None] = None, + stderr: Union[IO, int, None] = None, + ) -> asyncio.subprocess.Process: + # We cannot have handles from both files and to run() + if self.file_input and stdin: + logger.error( + "Pipeline %s with both file and stdin input. %s", + self, + str(stdin), + ) + sys.exit(1) + if self.file_output and stdout: + logger.error( + "Pipeline %s with both file and stdout output. %s", + self, + str(stdout), + ) + sys.exit(1) + + processes = [] + # We need to close these FDs in the parent after passing them to + # children + fds_to_close = [] + + # pass the file handle as input to the pipeline + current_stdin = stdin + if self.file_input: + current_stdin = open(self.file_input, "rb") + context.file_handles.append(current_stdin) + + try: + # Chain all commands except the last one + for i in range(len(self.nodes) - 1): + read_fd = None + write_fd = None + if i not in self.skip_pipe: + # Create OS pipe + read_fd, write_fd = os.pipe() + fds_to_close.extend([read_fd, write_fd]) + + # Run the node (Command or sub-Pipeline) + # stdout goes to the write-end of the pipe + proc = await self.nodes[i].run( + context, + stdin=current_stdin, + stdout=write_fd, + stderr=stderr, + ) + processes.append(proc) + + if write_fd: + # Close write_fd in parent (child has it now) + os.close(write_fd) + fds_to_close.remove(write_fd) + + # Next command reads from the read-end + current_stdin = read_fd + + # Send the pipeline output to a file + if self.file_output: + stdout = open(self.file_output, "wb") + context.file_handles.append(stdout) + + # Run the final command + last_proc = await self.nodes[-1].run( + context, + stdin=current_stdin, + stdout=stdout, + stderr=stderr, + ) + processes.append(last_proc) + + finally: + # Cleanup FDs + for fd in fds_to_close: + try: + os.close(fd) + except OSError: + pass + + return processes[-1] # Return the last process handle + + def __str__(self) -> str: + res = [] + if self.file_input: + res.append(f"<'{self.file_input}' ") + res.append(str(self.nodes[0])) + for i, node in enumerate(self.nodes[1:]): + if i in self.skip_pipe: + res.append("; ") + else: + res.append(" | ") + res.append(str(node)) + if self.file_output: + res.append(f" >'{self.file_output}'") + return "".join(res) + + def __repr__(self) -> str: + return ( + f"{self.__class__.__name__}(" + + ", ".join(repr(x) for x in self.nodes) + + f"skip_pipe={self.skip_pipe},file_input={self.file_input}," + + f"file_output={self.file_output})" + ) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, Pipeline): + return NotImplemented + return ( + self.nodes == other.nodes + and self.skip_pipe == other.skip_pipe + and self.file_input == other.file_input + and self.file_output == other.file_output + ) + + def __hash__(self) -> int: + attrs: List[Any] = self.nodes + [self.file_input, self.file_output] + if self.skip_pipe is None: + attrs.append(None) + else: + attrs.append(tuple(self.skip_pipe)) + return hash(tuple(attrs)) + + +class ProcSub(ABC): + """Abstract base class for Process Substitutions (<(...) and >(...)).""" + + def __init__(self, node: Pipeline) -> None: + self.node = node + self.fifo_path: str | None = None + + @abstractmethod + async def setup(self, context: Context) -> str: + """Prepares the FIFO and starts the background process.""" + pass + + def __repr__(self) -> str: + return self.__class__.__name__ + "(" + repr(self.node) + ")" + + def __eq__(self, other: object) -> bool: + if not isinstance(other, ProcSub): + return NotImplemented + return self.node == other.node + + def __hash__(self) -> int: + return hash((self.__class__.__name__, self.node)) + + +class InputProcSub(ProcSub): + """ + Represents <(cmd). + The inner command WRITES to a FIFO. + The outer command receives the filename. + """ + + async def setup(self, context: Context) -> str: + self.fifo_path = context.get_new_fifo() + + # We must open the FIFO for writing. + # WARNING: opening a FIFO blocks until the other end is opened. + # We run the open() in a thread to prevent blocking the async event + # loop. + def open_fifo_write(): + assert self.fifo_path + return open(self.fifo_path, "w") + + # Start the task that waits for the open, then runs the process + async def run_inner(): + # This awaits until the outer command opens the file for reading! + write_handle = await asyncio.to_thread(open_fifo_write) + try: + proc = await self.node.run( + context, + stdout=write_handle, + ) + await proc.wait() + finally: + write_handle.close() + + # Schedule this task in the background + task = asyncio.create_task(run_inner()) + context.tasks.append(task) + + return self.fifo_path + + def __str__(self) -> str: + return "<(" + str(self.node) + ")" + + +class OutputProcSub(ProcSub): + """ + Represents >(cmd). + The inner command READS from a FIFO. + The outer command receives the filename. + """ + + async def setup(self, context: Context) -> str: + self.fifo_path = context.get_new_fifo() + + def open_fifo_read(): + assert self.fifo_path + return open(self.fifo_path, "r") + + async def run_inner(): + # This awaits until the outer command opens the file for writing! + read_handle = await asyncio.to_thread(open_fifo_read) + try: + proc = await self.node.run(context, stdin=read_handle) + await proc.wait() + finally: + read_handle.close() + + task = asyncio.create_task(run_inner()) + context.tasks.append(task) + + return self.fifo_path + + def __str__(self) -> str: + return ">(" + str(self.node) + ")" diff --git a/sentieon_cli/util.py b/sentieon_cli/util.py index a7fb5db..32395e1 100644 --- a/sentieon_cli/util.py +++ b/sentieon_cli/util.py @@ -17,7 +17,7 @@ from .logging import get_logger -__version__ = "1.4.0" +__version__ = "1.5.0" logger = get_logger(__name__) @@ -174,7 +174,7 @@ def split_numa_nodes(numa_nodes: List[str]) -> List[str]: def split_alignment(cores: int) -> List[str]: """split large alignment tasks into smaller parts on large machines""" numa_nodes = find_numa_nodes() - while len(numa_nodes) > 0 and cores / len(numa_nodes) >= 48: + while len(numa_nodes) > 0 and cores / len(numa_nodes) > 64: numa_nodes = split_numa_nodes(numa_nodes) if cores > 32 and numa_nodes: return numa_nodes diff --git a/tests/integration/test_executor.py b/tests/integration/test_executor.py new file mode 100644 index 0000000..7199ecc --- /dev/null +++ b/tests/integration/test_executor.py @@ -0,0 +1,117 @@ +""" +Integration tests for the Executor +""" + +import io +import os +import pathlib +import sys +import tempfile +from unittest.mock import patch + +import pytest + +# Add the parent directory to the path +sys.path.insert( + 0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +) + +from sentieon_cli.dag import DAG # noqa: E402 +from sentieon_cli.executor import LocalExecutor # noqa: E402 +from sentieon_cli.job import Job # noqa: E402 +from sentieon_cli.scheduler import ThreadScheduler # noqa: E402 +from sentieon_cli.shell_pipeline import ( # noqa: E402 + Command, + InputProcSub, + Pipeline, +) + + +def test_local_executor_simple_job(): + """Test LocalExecutor with a single, simple job""" + with tempfile.TemporaryDirectory() as tmp_dir_str: + tmp_dir = pathlib.Path(tmp_dir_str) + cmd_out = tmp_dir / "test_out.txt" + dag = DAG() + job = Job(Pipeline(Command("echo", "hello executor"), file_output=cmd_out), "echo-job") + dag.add_job(job) + + scheduler = ThreadScheduler(dag, 1) + executor = LocalExecutor(scheduler) + + executor.execute() + + cmd_out_str = open(cmd_out).read() + assert "hello executor" in cmd_out_str + assert len(executor.jobs_with_errors) == 0 + + +def test_local_executor_pipeline_job(): + """Test LocalExecutor with a job containing a pipeline""" + with tempfile.TemporaryDirectory() as tmp_dir_str: + tmp_dir = pathlib.Path(tmp_dir_str) + cmd_out = tmp_dir / "test_out.txt" + dag = DAG() + pipeline = Pipeline(Command("echo", "hello pipeline"), Command("cat"), file_output=cmd_out) + job = Job(pipeline, "pipeline-job") + dag.add_job(job) + + scheduler = ThreadScheduler(dag, 1) + executor = LocalExecutor(scheduler) + + executor.execute() + + cmd_out_str = open(cmd_out).read() + assert "hello pipeline" in cmd_out_str + assert len(executor.jobs_with_errors) == 0 + +def test_local_executor_failing_job(): + """Test LocalExecutor with a job that fails""" + with tempfile.TemporaryDirectory() as tmp_dir_str: + tmp_dir = pathlib.Path(tmp_dir_str) + cmd_in = tmp_dir / "test_in.txt" + dag = DAG() + # This command will fail + job = Job(Pipeline(Command("cat", str(cmd_in))), "failing-job") + dag.add_job(job) + + scheduler = ThreadScheduler(dag, 1) + executor = LocalExecutor(scheduler) + + executor.execute() + + assert len(executor.jobs_with_errors) == 1 + assert executor.jobs_with_errors[0] == job + +def test_local_executor_proc_sub_job(): + """Test LocalExecutor with a job using process substitution""" + with tempfile.TemporaryDirectory() as tmp_dir_str: + tmp_dir = pathlib.Path(tmp_dir_str) + cmd_out = tmp_dir / "test_out.txt" + dag = DAG() + command = Command( + "diff", + InputProcSub(Pipeline(Command("echo", "a"))), + InputProcSub(Pipeline(Command("echo", "b"))), + ) + job = Job(Pipeline(command, file_output=cmd_out), "proc-sub-job") + dag.add_job(job) + + # Need at least 2 threads for the two proc subs + scheduler = ThreadScheduler(dag, 2) + executor = LocalExecutor(scheduler) + + executor.execute() + + # diff will write to stdout and fail + output = open(cmd_out).read() + assert "1c1" in output + assert "< a" in output + assert "> b" in output + + # The job should be marked as an error because the diff command fails + assert len(executor.jobs_with_errors) == 1 + + +if __name__ == "__main__": + pytest.main([__file__]) \ No newline at end of file diff --git a/tests/integration/test_pipeline_execution.py b/tests/integration/test_pipeline_execution.py index 774be27..1f0ab4a 100644 --- a/tests/integration/test_pipeline_execution.py +++ b/tests/integration/test_pipeline_execution.py @@ -11,7 +11,6 @@ sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))) from tests.utils.test_helpers import ( - CommandValidator, DAGAnalyzer, setup_basic_test_environment, teardown_test_environment @@ -42,12 +41,9 @@ def test_dnascope_dry_run_execution(self): commands = [job.shell for job in all_jobs] # Validate commands - sentieon_commands = [cmd for cmd in commands if cmd.startswith("sentieon")] + sentieon_commands = [cmd for subcommands in commands for cmd in subcommands.nodes if cmd.executable.startswith("sentieon")] assert len(sentieon_commands) > 0, "Should have sentieon commands" - for cmd in sentieon_commands: - assert CommandValidator.validate_sentieon_command(cmd), f"Invalid command: {cmd}" - def test_longread_dry_run_execution(self): """Test long-read pipeline dry-run execution""" pipeline = self.helper.create_longread_pipeline() @@ -61,7 +57,7 @@ def test_longread_dry_run_execution(self): commands = [job.shell for job in all_jobs] # Should have complex phased calling commands - variant_commands = [cmd for cmd in commands if "DNAscope" in cmd or "VariantPhaser" in cmd] + variant_commands = [cmd for subcommands in commands for cmd in subcommands.nodes if "DNAscope" in cmd.args or "VariantPhaser" in cmd.args] assert len(variant_commands) > 1, "Should have multiple variant calling steps" def test_hybrid_dry_run_execution(self): @@ -100,6 +96,48 @@ def test_hybrid_dry_run_execution(self): hybrid_stages = [name for name in job_names if "stage" in name.lower()] assert len(hybrid_stages) > 0, "Should have hybrid staging jobs" + def test_hybrid_dry_run_execution_rgsm(self): + """Test hybrid pipeline dry-run execution with --rgsm""" + pipeline = self.helper.create_hybrid_pipeline() + pipeline.rgsm = "HG001" + + # Mock the archive loading and readgroup functions + with patch('sentieon_cli.dnascope_hybrid.ar_load') as mock_ar_load, \ + patch('sentieon_cli.command_strings.get_rg_lines') as mock_get_rg: + + # Mock bundle info + import json + bundle_info = { + "longReadPlatform": "HiFi", + "shortReadPlatform": "Illumina", + "minScriptVersion": "1.0.0", + "pipeline": "DNAscope Hybrid" + } + mock_ar_load.side_effect = [ + json.dumps(bundle_info).encode(), # First call for bundle_info.json + ["longreadsv.model", "cnv.model", "bwa.model"] # Second call for bundle members + ] + + # Mock readgroup lines + mock_get_rg.return_value = ["@RG\tID:test\tSM:sample"] + + pipeline.validate() + pipeline.configure() + + dag = pipeline.build_dag() + all_jobs = [job for job in list(dag.waiting_jobs.keys()) + list(dag.ready_jobs.keys())] + + # with --rgsm should use replace_rg during first pass + first_pass_job = [job for job in all_jobs if job.name == "calling-1"][0] + assert "--replace_rg" in first_pass_job.shell.nodes[0].args + + # should use replace_rg in CNV calling + cnv_job = [job for job in all_jobs if job.name == "CNVscope"][0] + assert "--replace_rg" in cnv_job.shell.nodes[0].args + + # should use replace_rg in SV calling + sv_job = [job for job in all_jobs if job.name == "LongReadSV"][0] + assert "--replace_rg" in sv_job.shell.nodes[0].args class TestPipelineCommandGeneration: """Test command generation for different pipeline configurations""" @@ -130,10 +168,14 @@ def test_dnascope_command_parameters(self): assert variant_job is not None, "Should have variant calling job" # Check that command includes expected parameters - command = variant_job.shell - assert "--algo DNAscope" in command - assert "dbsnp.vcf.gz" in command, "Should include dbSNP file" - assert "emit_mode" in command, "Should have emit mode for gVCF" + commands = variant_job.shell.nodes + dnascope_cmds = [cmd for cmd in commands if "DNAscope" in cmd.args] + assert len(dnascope_cmds) > 0 + for cmd in dnascope_cmds: + dnascope_idx = cmd.args.index("DNAscope") + assert cmd.args[dnascope_idx - 1] == "--algo" + assert "--dbsnp" in cmd.args, "Should include dbSNP file" + assert "--emit_mode" in cmd.args, "Should have emit mode for gVCF" def test_resource_allocation_in_commands(self): """Test that resource allocation is reflected in commands""" @@ -176,11 +218,11 @@ def test_conditional_algorithm_inclusion(self): commands2 = [job.shell for job in list(dag2.waiting_jobs.keys()) + list(dag2.ready_jobs.keys())] # Metrics pipeline should have more algorithms - metrics_algos1 = [cmd for cmd in commands1 if "--algo" in cmd and any( - algo in cmd for algo in ["InsertSizeMetricAlgo", "MeanQualityByCycle", "GCBias"] + metrics_algos1 = [cmd for subcommand in commands1 for cmd in subcommand.nodes if any( + algo in cmd.args for algo in ["InsertSizeMetricAlgo", "MeanQualityByCycle", "GCBias"] )] - metrics_algos2 = [cmd for cmd in commands2 if "--algo" in cmd and any( - algo in cmd for algo in ["InsertSizeMetricAlgo", "MeanQualityByCycle", "GCBias"] + metrics_algos2 = [cmd for subcommand in commands2 for cmd in subcommand.nodes if any( + algo in cmd.args for algo in ["InsertSizeMetricAlgo", "MeanQualityByCycle", "GCBias"] )] assert len(metrics_algos1) > len(metrics_algos2), "Metrics enabled should have more metric algorithms" @@ -303,25 +345,6 @@ def test_variant_calling_dependencies(self): # Should have some dependencies assert len(dag.waiting_jobs[variant_job]) > 0, "Variant calling should have dependencies" - def test_cleanup_jobs_at_end(self): - """Test that cleanup jobs are scheduled after main processing""" - pipeline = self.helper.create_dnascope_pipeline(use_bam=True) - pipeline.validate() - pipeline.configure() - - dag = pipeline.build_dag() - - # Find cleanup jobs (typically have fail_ok=True) - cleanup_jobs = [job for job in list(dag.waiting_jobs.keys()) + list(dag.ready_jobs.keys()) - if job.fail_ok] - - if cleanup_jobs: - # Cleanup jobs should depend on main processing jobs - for cleanup_job in cleanup_jobs: - if cleanup_job in dag.waiting_jobs: - deps = dag.waiting_jobs[cleanup_job] - assert len(deps) > 0, "Cleanup jobs should have dependencies" - if __name__ == "__main__": pytest.main([__file__]) diff --git a/tests/unit/test_dag_construction.py b/tests/unit/test_dag_construction.py index ce3c000..00be2c9 100644 --- a/tests/unit/test_dag_construction.py +++ b/tests/unit/test_dag_construction.py @@ -5,6 +5,7 @@ import pathlib import pytest import tempfile +import shlex import sys import os from unittest.mock import patch @@ -16,6 +17,7 @@ from sentieon_cli.dnascope_longread import DNAscopeLRPipeline from sentieon_cli.dag import DAG from sentieon_cli.job import Job +from sentieon_cli.shell_pipeline import Pipeline, Command class TestDAGConstruction: @@ -191,32 +193,57 @@ class TestDAGJobProperties: def test_job_thread_allocation(self): """Test that jobs have appropriate thread allocation""" # Test high-CPU jobs get multiple threads - job = Job("sentieon driver --algo DNAscope", "variant-calling", 8) + job = Job( + Pipeline(Command(*shlex.split("sentieon driver --algo DNAscope"))), + "variant-calling", + 8, + ) assert job.threads == 8 # Test lightweight jobs get fewer threads - job = Job("rm temp_file.vcf", "cleanup", 0) + job = Job( + Pipeline(Command("rm", "temp_file.vcf")), + "cleanup", + 0, + ) assert job.threads == 0 def test_job_resource_requirements(self): """Test that jobs specify resource requirements correctly""" # Test NUMA-aware job - job = Job("align-command", "alignment", 4, resources={"node0": 1}) + job = Job( + Pipeline(Command("align-command")), + "alignment", + 4, + resources={"node0": 1}, + ) assert job.resources == {"node0": 1} # Test job without special resources - job = Job("simple-command", "simple", 2) + job = Job( + Pipeline(Command("simple-command")), + "simple", + 2, + ) assert job.resources == {} def test_job_failure_tolerance(self): """Test that cleanup jobs are marked as failure-tolerant""" # Cleanup jobs should be failure tolerant - job = Job("rm temp_files", "cleanup", 0, fail_ok=True) - assert job.fail_ok is True + job = Job( + Pipeline(Command("rm", "temp_files", fail_ok=True)), + "cleanup", + 0, + ) + assert job.shell.nodes[0].fail_ok is True # Critical jobs should not be failure tolerant - job = Job("sentieon driver", "variant-calling", 4, fail_ok=False) - assert job.fail_ok is False + job = Job( + Pipeline(Command("sentieon", "driver")), + "variant-calling", + 4, + ) + assert job.shell.nodes[0].fail_ok is False class TestLongReadDAGConstruction: @@ -312,9 +339,9 @@ def test_dag_has_no_cycles(self): dag = DAG() # Create test jobs - job1 = Job("command1", "job1") - job2 = Job("command2", "job2") - job3 = Job("command3", "job3") + job1 = Job(Pipeline(Command("command1")), "job1") + job2 = Job(Pipeline(Command("command2")), "job2") + job3 = Job(Pipeline(Command("command3")), "job3") # Add jobs with dependencies: job1 -> job2 -> job3 dag.add_job(job1) @@ -335,9 +362,9 @@ def test_dag_execution_simulation(self): dag = DAG() # Create test jobs - job1 = Job("command1", "job1") - job2 = Job("command2", "job2") - job3 = Job("command3", "job3") + job1 = Job(Pipeline(Command("command1")), "job1") + job2 = Job(Pipeline(Command("command2")), "job2") + job3 = Job(Pipeline(Command("command3")), "job3") # Add jobs: job1 and job2 can run in parallel, then job3 dag.add_job(job1) diff --git a/tests/unit/test_shell_pipeline.py b/tests/unit/test_shell_pipeline.py new file mode 100644 index 0000000..943d5b9 --- /dev/null +++ b/tests/unit/test_shell_pipeline.py @@ -0,0 +1,173 @@ +""" +Unit tests for shell_pipeline.py +""" + +import asyncio +import os +import pathlib +import sys +import tempfile + +import pytest + +# Add the parent directory to the path so we can import sentieon_cli +sys.path.insert( + 0, + os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")), +) + +from sentieon_cli.shell_pipeline import ( # noqa: E402 + Command, + Context, + InputProcSub, + OutputProcSub, + Pipeline, +) + + +@pytest.mark.asyncio +async def test_simple_command(): + """Test running a simple command""" + cmd = Command("echo", "hello world") + context = Context() + + # Use a file for stdout + with tempfile.NamedTemporaryFile(mode="w+", delete=False) as stdout_file: + proc = await cmd.run(context, stdout=stdout_file) + await proc.wait() + + stdout_file.seek(0) + output = stdout_file.read().strip() + + await context.cleanup() + os.remove(stdout_file.name) + + assert output == "hello world" + assert proc.returncode == 0 + + +@pytest.mark.asyncio +async def test_simple_pipeline(): + """Test a simple pipeline: echo hello | cat""" + cmd1 = Command("echo", "hello pipeline") + cmd2 = Command("cat") + pipeline = Pipeline(cmd1, cmd2) + context = Context() + + with tempfile.NamedTemporaryFile(mode="w+", delete=False) as stdout_file: + proc = await pipeline.run(context, stdout=stdout_file) + await proc.wait() + + stdout_file.seek(0) + output = stdout_file.read().strip() + + await context.cleanup() + os.remove(stdout_file.name) + + assert output == "hello pipeline" + assert proc.returncode == 0 + + +@pytest.mark.asyncio +async def test_pipeline_with_file_io(): + """Test a pipeline with file input and output""" + context = Context() + + # Create input and output files + with tempfile.NamedTemporaryFile(mode="w", delete=False) as infile: + infile.write("hello from file") + infile_path = pathlib.Path(infile.name) + + outfile_path = pathlib.Path(context.temp_dir.name) / "output.txt" + + cmd1 = Command("cat") + cmd2 = Command("wc", "-c") + pipeline = Pipeline(cmd1, cmd2, file_input=infile_path, file_output=outfile_path) + + proc = await pipeline.run(context) + await proc.wait() + + # Verify output file content + with open(outfile_path, "r") as f: + output = f.read().strip() + + await context.cleanup() + os.remove(infile_path) + + assert output == str(len("hello from file")) + assert proc.returncode == 0 + + +@pytest.mark.asyncio +async def test_input_process_substitution(): + """Test input process substitution: diff <(echo 1) <(echo 2)""" + context = Context() + + cmd = Command( + "diff", + InputProcSub(Pipeline(Command("echo", "1"))), + InputProcSub(Pipeline(Command("echo", "2"))), + ) + + with tempfile.NamedTemporaryFile(mode="w+", delete=False) as stdout_file: + proc = await cmd.run(context, stdout=stdout_file) + await proc.wait() + + stdout_file.seek(0) + output = stdout_file.read().strip() + + await context.cleanup() + os.remove(stdout_file.name) + + # The diff output should indicate a change from 1 to 2 + assert "1c1" in output + assert "< 1" in output + assert "---" in output + assert "> 2" in output + assert proc.returncode != 0 # diff exits with 1 if inputs differ + + +@pytest.mark.asyncio +async def test_output_process_substitution(): + """Test output process substitution: tee >(wc -c)""" + context = Context() + + # We will write the output of wc -c to a file to check it + outfile_path = pathlib.Path(context.temp_dir.name) / "wc_output.txt" + + # The pipeline will be: echo "hello" | tee >(wc -c > outfile.txt) + # The main output of the pipeline should be "hello" + # The output of the process substitution should be the character count "6" + + # Inner pipeline for process substitution + wc_cmd = Command("wc", "-c") + wc_pipeline = Pipeline(wc_cmd, file_output=outfile_path) + + # Main pipeline + echo_cmd = Command("echo", "hello") + tee_cmd = Command("tee", OutputProcSub(wc_pipeline)) + main_pipeline = Pipeline(echo_cmd, tee_cmd) + + with tempfile.NamedTemporaryFile(mode="w+", delete=False) as stdout_file: + proc = await main_pipeline.run(context, stdout=stdout_file) + await proc.wait() + + # Wait for background tasks from process substitution to finish + await asyncio.gather(*context.tasks) + + stdout_file.seek(0) + main_output = stdout_file.read().strip() + + # Verify main output + assert main_output == "hello" + + # Verify process substitution output + with open(outfile_path, "r") as f: + wc_output = f.read().strip() + + # 'echo' adds a newline, so it should be 6 characters for `wc -c` + assert wc_output == str(len("hello\n")) + + await context.cleanup() + os.remove(stdout_file.name) + assert proc.returncode == 0 diff --git a/tests/utils/test_helpers.py b/tests/utils/test_helpers.py index 14e47a2..8c214bc 100644 --- a/tests/utils/test_helpers.py +++ b/tests/utils/test_helpers.py @@ -224,26 +224,6 @@ def cleanup(self): self.fs.cleanup() -class CommandValidator: - """Helper for validating generated commands""" - - @staticmethod - def validate_sentieon_command(command: str) -> bool: - """Validate that a sentieon command is well-formed""" - if not command.startswith("sentieon"): - return False - - parts = command.split() - if len(parts) < 2: - return False - - # Should have driver or util as second argument - if parts[1] not in ["driver", "util"]: - return False - - return True - - class DAGAnalyzer: """Helper for analyzing DAG structure in tests"""